From 1f284007b5b27c1b3bf37be509d96e87825c2124 Mon Sep 17 00:00:00 2001 From: Suresh N S Date: Fri, 8 May 2026 16:44:28 +0530 Subject: [PATCH 1/8] Adding support for Relevance Functions Signed-off-by: Suresh N S --- .../analytics/spi/ScalarFunction.java | 5 + .../opensearch/be/lucene/ConversionUtils.java | 57 ++++ .../lucene/LuceneAnalyticsBackendPlugin.java | 5 + .../be/lucene/QuerySerializerRegistry.java | 70 ++++- .../be/lucene/ConversionUtilsTests.java | 207 +++++++++++++++ ...ultiFieldExtractionOrderPropertyTests.java | 194 ++++++++++++++ .../MultiFieldSerializationPropertyTests.java | 243 ++++++++++++++++++ .../lucene/QuerySerializerRegistryTests.java | 61 +++++ ...SingleFieldSerializationPropertyTests.java | 214 +++++++++++++++ .../planner/rules/OpenSearchFilterRule.java | 20 +- 10 files changed, 1065 insertions(+), 11 deletions(-) create mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java create mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldExtractionOrderPropertyTests.java create mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldSerializationPropertyTests.java create mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java create mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/SingleFieldSerializationPropertyTests.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java index 42bfc7ec1ce92..0794dfa456148 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java @@ -47,6 +47,11 @@ public enum ScalarFunction { // ── Full-text search ───────────────────────────────────────────── MATCH(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), MATCH_PHRASE(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + MATCH_BOOL_PREFIX(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + MATCH_PHRASE_PREFIX(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + MULTI_MATCH(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + QUERY_STRING(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + SIMPLE_QUERY_STRING(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), FUZZY(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), WILDCARD(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), REGEXP(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java index fcd4edf7f311c..4bc125adc8409 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java @@ -18,6 +18,7 @@ import org.opensearch.index.query.QueryBuilder; import java.io.IOException; +import java.util.ArrayList; import java.util.List; /** @@ -77,4 +78,60 @@ static byte[] serializeQueryBuilder(QueryBuilder queryBuilder) { throw new IllegalStateException("Failed to serialize delegated query: " + queryBuilder, exception); } } + + /** + * Extracts multiple field names from a MAP_VALUE_CONSTRUCTOR operand + * for multi-field full-text functions (multi_match, query_string, simple_query_string). + * + *

The operand structure for multi-field functions: + * {@code MAP('fields', MAP('field1':VARCHAR, boost1:DOUBLE, 'field2':VARCHAR, boost2:DOUBLE, ...))} + * The outer MAP has key='fields' at index 0 and a nested MAP at index 1. + * The nested MAP is a Calcite MAP_VALUE_CONSTRUCTOR with strict alternating key-value pairs: + * field name (VARCHAR) at even indices, boost value (DOUBLE) at odd indices. + * + *

Also supports the RexInputRef-based structure for single-field fallback: + * {@code MAP('field', $ref1, 'field', $ref2, ...)} + * + *

TODO: extract per-field boost values and return them alongside field names. + */ + static List extractFieldsFromRelevanceMap(RexCall call, int operandIndex, List fieldStorage) { + RexNode operand = call.getOperands().get(operandIndex); + List fields = new ArrayList<>(); + if (operand instanceof RexCall outerMapCall) { + // Check if the value (index 1) is a nested MAP containing field name/boost pairs + if (outerMapCall.getOperands().size() >= 2) { + RexNode value = outerMapCall.getOperands().get(1); + if (value instanceof RexCall nestedMapCall) { + // Nested MAP: strict alternating key-value pairs from MAP_VALUE_CONSTRUCTOR. + // Even indices (0, 2, 4...) are field name VARCHAR literals. + // Odd indices (1, 3, 5...) are boost DOUBLE literals (ignored for now). + List nestedOperands = nestedMapCall.getOperands(); + for (int i = 0; i < nestedOperands.size(); i += 2) { + RexNode fieldNode = nestedOperands.get(i); + if (fieldNode instanceof RexLiteral fieldLiteral) { + fields.add(fieldLiteral.getValueAs(String.class)); + } + } + if (fields.isEmpty() == false) { + return fields; + } + } + } + // Fallback: RexInputRef-based structure MAP('field', $ref1, 'field', $ref2, ...) + List mapOperands = outerMapCall.getOperands(); + for (int i = 1; i < mapOperands.size(); i += 2) { + RexNode val = mapOperands.get(i); + if (val instanceof RexInputRef inputRef) { + fields.add(FieldStorageInfo.resolve(fieldStorage, inputRef.getIndex()).getFieldName()); + } + } + } + if (operand instanceof RexInputRef inputRef) { + fields.add(FieldStorageInfo.resolve(fieldStorage, inputRef.getIndex()).getFieldName()); + } + if (fields.isEmpty()) { + throw new IllegalArgumentException("Cannot extract field names from operand " + operandIndex + ": " + operand); + } + return fields; + } } diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java index 5a59dda788db0..82f628f748e27 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java @@ -64,6 +64,11 @@ public class LuceneAnalyticsBackendPlugin implements AnalyticsSearchBackendPlugi private static final Set FULL_TEXT_OPS = Set.of( ScalarFunction.MATCH, ScalarFunction.MATCH_PHRASE, + ScalarFunction.MATCH_BOOL_PREFIX, + ScalarFunction.MATCH_PHRASE_PREFIX, + ScalarFunction.MULTI_MATCH, + ScalarFunction.QUERY_STRING, + ScalarFunction.SIMPLE_QUERY_STRING, ScalarFunction.FUZZY, ScalarFunction.WILDCARD, ScalarFunction.REGEXP diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java index 13bda07674b22..72038f646341b 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java @@ -12,7 +12,13 @@ import org.opensearch.analytics.spi.DelegatedPredicateSerializer; import org.opensearch.analytics.spi.FieldStorageInfo; import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.index.query.MatchBoolPrefixQueryBuilder; +import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; +import org.opensearch.index.query.MatchPhraseQueryBuilder; import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.MultiMatchQueryBuilder; +import org.opensearch.index.query.QueryStringQueryBuilder; +import org.opensearch.index.query.SimpleQueryStringBuilder; import java.util.List; import java.util.Map; @@ -27,9 +33,14 @@ */ final class QuerySerializerRegistry { - private static final Map SERIALIZERS = Map.of( - ScalarFunction.MATCH, - QuerySerializerRegistry::serializeMatch + private static final Map SERIALIZERS = Map.ofEntries( + Map.entry(ScalarFunction.MATCH, QuerySerializerRegistry::serializeMatch), + Map.entry(ScalarFunction.MATCH_PHRASE, QuerySerializerRegistry::serializeMatchPhrase), + Map.entry(ScalarFunction.MATCH_BOOL_PREFIX, QuerySerializerRegistry::serializeMatchBoolPrefix), + Map.entry(ScalarFunction.MATCH_PHRASE_PREFIX, QuerySerializerRegistry::serializeMatchPhrasePrefix), + Map.entry(ScalarFunction.MULTI_MATCH, QuerySerializerRegistry::serializeMultiMatch), + Map.entry(ScalarFunction.QUERY_STRING, QuerySerializerRegistry::serializeQueryString), + Map.entry(ScalarFunction.SIMPLE_QUERY_STRING, QuerySerializerRegistry::serializeSimpleQueryString) ); private QuerySerializerRegistry() {} @@ -41,8 +52,59 @@ static Map getSerializers() { private static byte[] serializeMatch(RexCall call, List fieldStorage) { String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: extract optional params (operator, analyzer, fuzziness) from operands 2+ + // TODO: extract optional params (operator, analyzer, fuzziness, boost) from operands 2+ MatchQueryBuilder queryBuilder = new MatchQueryBuilder(fieldName, queryText); return ConversionUtils.serializeQueryBuilder(queryBuilder); } + + private static byte[] serializeMatchPhrase(RexCall call, List fieldStorage) { + String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); + String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + MatchPhraseQueryBuilder queryBuilder = new MatchPhraseQueryBuilder(fieldName, queryText); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeMatchBoolPrefix(RexCall call, List fieldStorage) { + String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); + String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + MatchBoolPrefixQueryBuilder queryBuilder = new MatchBoolPrefixQueryBuilder(fieldName, queryText); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeMatchPhrasePrefix(RexCall call, List fieldStorage) { + String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); + String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + MatchPhrasePrefixQueryBuilder queryBuilder = new MatchPhrasePrefixQueryBuilder(fieldName, queryText); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeMultiMatch(RexCall call, List fieldStorage) { + List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); + String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder + MultiMatchQueryBuilder queryBuilder = new MultiMatchQueryBuilder(queryText, fields.toArray(String[]::new)); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeQueryString(RexCall call, List fieldStorage) { + List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); + String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder + QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder(queryText); + for (String field : fields) { + queryBuilder.field(field); + } + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeSimpleQueryString(RexCall call, List fieldStorage) { + List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); + String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder + SimpleQueryStringBuilder queryBuilder = new SimpleQueryStringBuilder(queryText); + for (String field : fields) { + queryBuilder.field(field); + } + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } } diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java new file mode 100644 index 0000000000000..3a982034b0b25 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java @@ -0,0 +1,207 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.FieldType; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +/** + * Unit tests for {@link ConversionUtils#extractFieldsFromRelevanceMap}. + */ +public class ConversionUtilsTests extends OpenSearchTestCase { + + private static final SqlFunction MULTI_MATCH_FUNCTION = new SqlFunction( + "MULTI_MATCH", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + } + + /** + * Tests that extractFieldsFromRelevanceMap returns a single-element list when the + * nested MAP operand contains exactly one field name/boost pair. + * Validates Requirement 9.2. + */ + public void testSingleFieldExtractionReturnsOneElementList() { + // Structure: MULTI_MATCH(MAP('fields', MAP('title', 1.0)), MAP('query', 'hello')) + RelDataType doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE); + + // Inner nested MAP: field name literal + boost literal (boost ignored, only field name extracted) + RexNode fieldNameLiteral = rexBuilder.makeLiteral("title"); + RexNode boostLiteral = rexBuilder.makeExactLiteral(new java.math.BigDecimal("1.0"), doubleType); + RexNode nestedMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, fieldNameLiteral, boostLiteral); + + // Outer MAP: MAP('fields', nestedMap) + RexNode fieldsKeyLiteral = rexBuilder.makeLiteral("fields"); + RexNode outerMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, fieldsKeyLiteral, nestedMap); + + // Query MAP: MAP('query', 'hello') + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("hello") + ); + + // Top-level call: MULTI_MATCH(outerMap, queryMap) + RexCall topCall = (RexCall) rexBuilder.makeCall(MULTI_MATCH_FUNCTION, outerMap, queryMap); + + // FieldStorageInfo list is not used in the nested MAP path (literal field names) + List fieldStorage = List.of(); + + List result = ConversionUtils.extractFieldsFromRelevanceMap(topCall, 0, fieldStorage); + + assertEquals("Should return exactly one field", 1, result.size()); + assertEquals("title", result.get(0)); + } + + /** + * Tests that extractFieldsFromRelevanceMap returns all fields in order when the + * nested MAP operand contains multiple field name/boost pairs. + * Validates Requirement 9.1. + */ + public void testMultiFieldExtractionReturnsAllFieldsInOrder() { + // Structure: MAP('fields', MAP('title', 2.0, 'body', 1.0, 'tags', 0.5)) + RelDataType doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE); + + RexNode titleLiteral = rexBuilder.makeLiteral("title"); + RexNode titleBoost = rexBuilder.makeExactLiteral(new java.math.BigDecimal("2.0"), doubleType); + RexNode bodyLiteral = rexBuilder.makeLiteral("body"); + RexNode bodyBoost = rexBuilder.makeExactLiteral(new java.math.BigDecimal("1.0"), doubleType); + RexNode tagsLiteral = rexBuilder.makeLiteral("tags"); + RexNode tagsBoost = rexBuilder.makeExactLiteral(new java.math.BigDecimal("0.5"), doubleType); + + // Nested MAP with 3 field/boost pairs + RexNode nestedMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + titleLiteral, + titleBoost, + bodyLiteral, + bodyBoost, + tagsLiteral, + tagsBoost + ); + + // Outer MAP: MAP('fields', nestedMap) + RexNode outerMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, rexBuilder.makeLiteral("fields"), nestedMap); + + // Query MAP + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("search text") + ); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MULTI_MATCH_FUNCTION, outerMap, queryMap); + List fieldStorage = List.of(); + + List result = ConversionUtils.extractFieldsFromRelevanceMap(topCall, 0, fieldStorage); + + assertEquals("Should return exactly three fields", 3, result.size()); + assertEquals("title", result.get(0)); + assertEquals("body", result.get(1)); + assertEquals("tags", result.get(2)); + } + + /** + * Tests that extractFieldsFromRelevanceMap works with the RexInputRef fallback path + * when the operand uses the MAP('field', $ref1, 'field', $ref2, ...) structure. + * Validates Requirement 9.1 (fallback path). + */ + public void testMultiFieldExtractionWithRexInputRefFallback() { + // Structure: MAP('field', $0, 'field', $1) — RexInputRef-based multi-field + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + RexNode key1 = rexBuilder.makeLiteral("field"); + RexNode ref0 = rexBuilder.makeInputRef(varcharType, 0); + RexNode key2 = rexBuilder.makeLiteral("field"); + RexNode ref1 = rexBuilder.makeInputRef(varcharType, 1); + + RexNode fieldMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, key1, ref0, key2, ref1); + + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("test query") + ); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MULTI_MATCH_FUNCTION, fieldMap, queryMap); + + // FieldStorageInfo list maps index 0 → "title", index 1 → "body" + List fieldStorage = List.of( + new FieldStorageInfo("title", "keyword", FieldType.KEYWORD, List.of(), List.of("lucene"), List.of(), false), + new FieldStorageInfo("body", "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false) + ); + + List result = ConversionUtils.extractFieldsFromRelevanceMap(topCall, 0, fieldStorage); + + assertEquals("Should return exactly two fields", 2, result.size()); + assertEquals("title", result.get(0)); + assertEquals("body", result.get(1)); + } + + /** + * Tests that extractFieldsFromRelevanceMap throws IllegalArgumentException when the + * operand contains no resolvable field references. + * Validates Requirement 9.3. + */ + public void testNoFieldsThrowsIllegalArgumentException() { + // Structure: MAP('fields', 'not_a_map') — value is a literal, not a nested RexCall + RexNode outerMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("fields"), + rexBuilder.makeLiteral("not_a_map") + ); + + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("test") + ); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MULTI_MATCH_FUNCTION, outerMap, queryMap); + List fieldStorage = List.of(); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> ConversionUtils.extractFieldsFromRelevanceMap(topCall, 0, fieldStorage) + ); + assertTrue( + "Exception message should mention operand index", + exception.getMessage().contains("Cannot extract field names from operand 0") + ); + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldExtractionOrderPropertyTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldExtractionOrderPropertyTests.java new file mode 100644 index 0000000000000..8f9b258d791d8 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldExtractionOrderPropertyTests.java @@ -0,0 +1,194 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.FieldType; +import org.opensearch.test.OpenSearchTestCase; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; + +/** + * Property-based test for multi-field extraction order preservation. + * + *

Feature: lucene-fulltext-serializers, Property 3: Multi-field extraction preserves all fields in order + * + *

For any MAP_VALUE_CONSTRUCTOR RexCall operand containing N >= 1 RexInputRef entries, + * {@code extractFieldsFromRelevanceMap} shall return a list of exactly N field names, each matching + * the field name resolved from the corresponding FieldStorageInfo entry, in the same order as they + * appear in the operand. + * + *

Validates: Requirements 9.1, 9.2 + */ +public class MultiFieldExtractionOrderPropertyTests extends OpenSearchTestCase { + + private static final int ITERATIONS = 100; + + private static final SqlFunction MULTI_MATCH_FUNCTION = new SqlFunction( + "MULTI_MATCH", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + } + + /** + * Property 3: Multi-field extraction preserves all fields in order (nested MAP literal path). + * + * Generate a random list of 1-10 field names, construct a mock MAP_VALUE_CONSTRUCTOR RexCall + * with the nested MAP structure (field name literals at even indices, boost literals at odd indices), + * invoke extractFieldsFromRelevanceMap, and assert the returned list matches expected field names + * in exact order. + */ + public void testExtractionPreservesOrderNestedMapLiteralPath() { + for (int iter = 0; iter < ITERATIONS; iter++) { + int fieldCount = randomIntBetween(1, 10); + List expectedFields = generateUniqueFieldNames(fieldCount); + + RexCall call = buildNestedMapLiteralCall(expectedFields); + List fieldStorage = List.of(); // not used in nested MAP literal path + + List result = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); + + assertEquals("Iteration " + iter + ": field count mismatch", expectedFields.size(), result.size()); + for (int i = 0; i < expectedFields.size(); i++) { + assertEquals("Iteration " + iter + ": field at index " + i + " mismatch", expectedFields.get(i), result.get(i)); + } + } + } + + /** + * Property 3: Multi-field extraction preserves all fields in order (RexInputRef fallback path). + * + * Generate a random list of 1-10 field names, construct a mock MAP_VALUE_CONSTRUCTOR RexCall + * with RexInputRef entries at odd indices and a corresponding FieldStorageInfo list, + * invoke extractFieldsFromRelevanceMap, and assert the returned list matches expected field names + * in exact order. + */ + public void testExtractionPreservesOrderRexInputRefPath() { + for (int iter = 0; iter < ITERATIONS; iter++) { + int fieldCount = randomIntBetween(1, 10); + List expectedFields = generateUniqueFieldNames(fieldCount); + + List fieldStorage = new ArrayList<>(); + for (String fieldName : expectedFields) { + fieldStorage.add(new FieldStorageInfo(fieldName, "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false)); + } + + RexCall call = buildRexInputRefCall(fieldCount); + + List result = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); + + assertEquals("Iteration " + iter + ": field count mismatch", expectedFields.size(), result.size()); + for (int i = 0; i < expectedFields.size(); i++) { + assertEquals("Iteration " + iter + ": field at index " + i + " mismatch", expectedFields.get(i), result.get(i)); + } + } + } + + /** + * Generates a list of unique random field names. + */ + private List generateUniqueFieldNames(int count) { + List names = new ArrayList<>(); + while (names.size() < count) { + String candidate = randomAlphaOfLengthBetween(1, 50); + if (names.contains(candidate) == false) { + names.add(candidate); + } + } + return names; + } + + /** + * Builds a mock RexCall using the nested MAP literal structure: + * MULTI_MATCH(MAP('fields', MAP('field1', 1.0, 'field2', 1.0, ...)), MAP('query', 'text')) + * + * The nested MAP has alternating field name literals (even indices) and boost literals (odd indices). + */ + private RexCall buildNestedMapLiteralCall(List fieldNames) { + RelDataType doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE); + + // Build nested MAP operands: field1, boost1, field2, boost2, ... + List nestedMapOperands = new ArrayList<>(); + for (String field : fieldNames) { + nestedMapOperands.add(rexBuilder.makeLiteral(field)); + nestedMapOperands.add(rexBuilder.makeExactLiteral(new BigDecimal("1.0"), doubleType)); + } + + // Nested MAP: MAP_VALUE_CONSTRUCTOR(field1, 1.0, field2, 1.0, ...) + RexNode nestedMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, nestedMapOperands.toArray(new RexNode[0])); + + // Outer MAP: MAP('fields', nestedMap) + RexNode outerMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, rexBuilder.makeLiteral("fields"), nestedMap); + + // Query MAP: MAP('query', 'text') + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("test query") + ); + + return (RexCall) rexBuilder.makeCall(MULTI_MATCH_FUNCTION, outerMap, queryMap); + } + + /** + * Builds a mock RexCall using the RexInputRef fallback structure: + * MULTI_MATCH(MAP('field', $0, 'field', $1, ...), MAP('query', 'text')) + * + * The MAP has alternating key literals ('field') at even indices and RexInputRef at odd indices. + */ + private RexCall buildRexInputRefCall(int fieldCount) { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + // Build MAP operands: 'field', $0, 'field', $1, ... + List mapOperands = new ArrayList<>(); + for (int i = 0; i < fieldCount; i++) { + mapOperands.add(rexBuilder.makeLiteral("field")); + mapOperands.add(rexBuilder.makeInputRef(varcharType, i)); + } + + RexNode fieldMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, mapOperands.toArray(new RexNode[0])); + + // Query MAP: MAP('query', 'text') + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("test query") + ); + + return (RexCall) rexBuilder.makeCall(MULTI_MATCH_FUNCTION, fieldMap, queryMap); + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldSerializationPropertyTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldSerializationPropertyTests.java new file mode 100644 index 0000000000000..f467b284154f7 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldSerializationPropertyTests.java @@ -0,0 +1,243 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.DelegatedPredicateSerializer; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.index.query.MultiMatchQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryStringQueryBuilder; +import org.opensearch.index.query.SimpleQueryStringBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Property-based test for multi-field serialization round-trip. + * + * + *

For any non-empty list of valid field names and non-empty query string, serializing a + * multi-field full-text predicate (multi_match, query_string, or simple_query_string) via its + * registered serializer and then deserializing the resulting bytes with + * {@code readNamedWriteable(QueryBuilder.class)} shall produce a QueryBuilder of the correct + * subclass containing all the specified fields and the same query text. + * + */ +public class MultiFieldSerializationPropertyTests extends OpenSearchTestCase { + + private static final NamedWriteableRegistry WRITEABLE_REGISTRY = new NamedWriteableRegistry( + List.of( + new NamedWriteableRegistry.Entry(QueryBuilder.class, MultiMatchQueryBuilder.NAME, MultiMatchQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, QueryStringQueryBuilder.NAME, QueryStringQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, SimpleQueryStringBuilder.NAME, SimpleQueryStringBuilder::new) + ) + ); + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private Map serializers; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + serializers = QuerySerializerRegistry.getSerializers(); + } + + /** + * Multi-field serialization round-trip for MULTI_MATCH. + * + * For a random list of 1-5 field names and a random query string, + * serializing via the MULTI_MATCH serializer and deserializing produces a + * MultiMatchQueryBuilder with all fields present and the correct query text. + */ + public void testMultiMatchRoundTrip() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.MULTI_MATCH); + assertNotNull("MULTI_MATCH serializer must be registered", serializer); + + int fieldCount = randomIntBetween(1, 5); + List fieldNames = generateUniqueFieldNames(fieldCount); + String queryText = randomAlphaOfLengthBetween(1, 200); + + RexCall call = buildMultiFieldRexCall(fieldNames, queryText, "MULTI_MATCH"); + List fieldStorage = List.of(); // not used in nested MAP literal path + + byte[] serialized = serializer.serialize(call, fieldStorage); + assertNotNull("Serialized bytes should not be null", serialized); + assertTrue("Serialized bytes should not be empty", serialized.length > 0); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue( + "Deserialized should be MultiMatchQueryBuilder but was " + deserialized.getClass().getSimpleName(), + deserialized instanceof MultiMatchQueryBuilder + ); + MultiMatchQueryBuilder multiMatch = (MultiMatchQueryBuilder) deserialized; + assertEquals("Query text mismatch", queryText, multiMatch.value()); + assertEquals("Field count mismatch", fieldCount, multiMatch.fields().size()); + for (String expectedField : fieldNames) { + assertTrue("Missing field: " + expectedField, multiMatch.fields().containsKey(expectedField)); + } + } + } + + /** + * Multi-field serialization round-trip for QUERY_STRING. + * + * For a random list of 1-5 field names and a random query string, + * serializing via the QUERY_STRING serializer and deserializing produces a + * QueryStringQueryBuilder with all fields present and the correct query text. + */ + public void testQueryStringRoundTrip() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.QUERY_STRING); + assertNotNull("QUERY_STRING serializer must be registered", serializer); + + int fieldCount = randomIntBetween(1, 5); + List fieldNames = generateUniqueFieldNames(fieldCount); + String queryText = randomAlphaOfLengthBetween(1, 200); + + RexCall call = buildMultiFieldRexCall(fieldNames, queryText, "QUERY_STRING"); + List fieldStorage = List.of(); + + byte[] serialized = serializer.serialize(call, fieldStorage); + assertNotNull("Serialized bytes should not be null", serialized); + assertTrue("Serialized bytes should not be empty", serialized.length > 0); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue( + "Deserialized should be QueryStringQueryBuilder but was " + deserialized.getClass().getSimpleName(), + deserialized instanceof QueryStringQueryBuilder + ); + QueryStringQueryBuilder queryString = (QueryStringQueryBuilder) deserialized; + assertEquals("Query text mismatch", queryText, queryString.queryString()); + assertEquals("Field count mismatch", fieldCount, queryString.fields().size()); + for (String expectedField : fieldNames) { + assertTrue("Missing field: " + expectedField, queryString.fields().containsKey(expectedField)); + } + } + } + + /** + * Multi-field serialization round-trip for SIMPLE_QUERY_STRING. + * + * For a random list of 1-5 field names and a random query string, + * serializing via the SIMPLE_QUERY_STRING serializer and deserializing produces a + * SimpleQueryStringBuilder with all fields present and the correct query text. + */ + public void testSimpleQueryStringRoundTrip() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.SIMPLE_QUERY_STRING); + assertNotNull("SIMPLE_QUERY_STRING serializer must be registered", serializer); + + int fieldCount = randomIntBetween(1, 5); + List fieldNames = generateUniqueFieldNames(fieldCount); + String queryText = randomAlphaOfLengthBetween(1, 200); + + RexCall call = buildMultiFieldRexCall(fieldNames, queryText, "SIMPLE_QUERY_STRING"); + List fieldStorage = List.of(); + + byte[] serialized = serializer.serialize(call, fieldStorage); + assertNotNull("Serialized bytes should not be null", serialized); + assertTrue("Serialized bytes should not be empty", serialized.length > 0); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue( + "Deserialized should be SimpleQueryStringBuilder but was " + deserialized.getClass().getSimpleName(), + deserialized instanceof SimpleQueryStringBuilder + ); + SimpleQueryStringBuilder simpleQueryString = (SimpleQueryStringBuilder) deserialized; + assertEquals("Query text mismatch", queryText, simpleQueryString.value()); + assertEquals("Field count mismatch", fieldCount, simpleQueryString.fields().size()); + for (String expectedField : fieldNames) { + assertTrue("Missing field: " + expectedField, simpleQueryString.fields().containsKey(expectedField)); + } + } + } + + /** + * Generates a list of unique random field names. + */ + private List generateUniqueFieldNames(int count) { + List names = new ArrayList<>(); + while (names.size() < count) { + String candidate = randomAlphaOfLengthBetween(1, 50); + if (names.contains(candidate) == false) { + names.add(candidate); + } + } + return names; + } + + /** + * Builds a mock RexCall simulating the multi-field relevance function structure: + * FUNC(MAP('fields', MAP('field1', 1.0, 'field2', 1.0, ...)), MAP('query', 'queryText')) + * + * The serializer extracts fields via extractFieldsFromRelevanceMap(call, 0, fieldStorage) + * which expects operand 0 to be an outer MAP with a nested MAP at child index 1. + * The nested MAP has alternating field name literals (even indices) and boost literals (odd indices). + */ + private RexCall buildMultiFieldRexCall(List fieldNames, String queryText, String functionName) { + RelDataType doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE); + + // Build nested MAP operands: field1, boost1, field2, boost2, ... + List nestedMapOperands = new ArrayList<>(); + for (String field : fieldNames) { + nestedMapOperands.add(rexBuilder.makeLiteral(field)); + nestedMapOperands.add(rexBuilder.makeExactLiteral(new BigDecimal("1.0"), doubleType)); + } + + // Nested MAP: MAP_VALUE_CONSTRUCTOR(field1, 1.0, field2, 1.0, ...) + RexNode nestedMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, nestedMapOperands.toArray(new RexNode[0])); + + // Outer MAP: MAP('fields', nestedMap) + RexNode outerMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, rexBuilder.makeLiteral("fields"), nestedMap); + + // Query MAP: MAP('query', 'queryText') + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral(queryText) + ); + + // Top-level function call + SqlFunction sqlFunction = new SqlFunction( + functionName, + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + return (RexCall) rexBuilder.makeCall(sqlFunction, outerMap, queryMap); + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java new file mode 100644 index 0000000000000..5b8e97bd92a0f --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.opensearch.analytics.spi.DelegatedPredicateSerializer; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Map; +import java.util.Set; + +/** + * Unit tests for {@link QuerySerializerRegistry} completeness. + * Validates Requirement 10.1. + */ +public class QuerySerializerRegistryTests extends OpenSearchTestCase { + + /** + * Verifies that getSerializers() returns exactly 7 entries. + */ + public void testRegistryContainsExactlySevenEntries() { + Map serializers = QuerySerializerRegistry.getSerializers(); + assertEquals("Registry should contain exactly 7 serializer entries", 7, serializers.size()); + } + + /** + * Verifies that the key set matches the expected ScalarFunction constants. + */ + public void testRegistryKeySetMatchesExpectedFunctions() { + Map serializers = QuerySerializerRegistry.getSerializers(); + + Set expectedKeys = Set.of( + ScalarFunction.MATCH, + ScalarFunction.MATCH_PHRASE, + ScalarFunction.MATCH_BOOL_PREFIX, + ScalarFunction.MATCH_PHRASE_PREFIX, + ScalarFunction.MULTI_MATCH, + ScalarFunction.QUERY_STRING, + ScalarFunction.SIMPLE_QUERY_STRING + ); + + assertEquals("Registry keys should match expected full-text functions", expectedKeys, serializers.keySet()); + } + + /** + * Verifies that all serializer values are non-null (each entry has a valid lambda). + */ + public void testAllSerializerValuesAreNonNull() { + Map serializers = QuerySerializerRegistry.getSerializers(); + + for (Map.Entry entry : serializers.entrySet()) { + assertNotNull("Serializer for " + entry.getKey() + " should not be null", entry.getValue()); + } + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/SingleFieldSerializationPropertyTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/SingleFieldSerializationPropertyTests.java new file mode 100644 index 0000000000000..d2493bda9f69c --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/SingleFieldSerializationPropertyTests.java @@ -0,0 +1,214 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.DelegatedPredicateSerializer; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.FieldType; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.index.query.MatchBoolPrefixQueryBuilder; +import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; +import org.opensearch.index.query.MatchPhraseQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Property-based test for single-field serialization round-trip. + * + *

Feature: lucene-fulltext-serializers, Property: Single-field serialization round-trip + * + *

For any valid field name and non-empty query string, serializing a single-field full-text + * predicate (match_phrase, match_bool_prefix, or match_phrase_prefix) via its registered serializer + * and then deserializing the resulting bytes with {@code readNamedWriteable(QueryBuilder.class)} + * shall produce a QueryBuilder of the correct subclass containing the same field name and query text. + * + *

Validates: Requirements 3.5, 4.5, 5.5 + */ +public class SingleFieldSerializationPropertyTests extends OpenSearchTestCase { + + private static final NamedWriteableRegistry WRITEABLE_REGISTRY = new NamedWriteableRegistry( + List.of( + new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchPhraseQueryBuilder.NAME, MatchPhraseQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchBoolPrefixQueryBuilder.NAME, MatchBoolPrefixQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchPhrasePrefixQueryBuilder.NAME, MatchPhrasePrefixQueryBuilder::new) + ) + ); + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private Map serializers; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + serializers = QuerySerializerRegistry.getSerializers(); + } + + /** + * Single-field serialization round-trip for MATCH_PHRASE. + * + * For any random field name (1-50 alphanumeric chars) and random query string (1-200 chars), + * serializing via the MATCH_PHRASE serializer and deserializing produces a MatchPhraseQueryBuilder + * with the same field and query. + */ + public void testMatchPhraseRoundTrip() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.MATCH_PHRASE); + assertNotNull("MATCH_PHRASE serializer must be registered", serializer); + + String fieldName = randomAlphaOfLengthBetween(1, 50); + String queryText = randomAlphaOfLengthBetween(1, 200); + + RexCall call = buildSingleFieldRexCall(fieldName, queryText, "MATCH_PHRASE"); + List fieldStorage = List.of( + new FieldStorageInfo(fieldName, "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false) + ); + + byte[] serialized = serializer.serialize(call, fieldStorage); + assertNotNull("Serialized bytes should not be null", serialized); + assertTrue("Serialized bytes should not be empty", serialized.length > 0); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue( + "Deserialized should be MatchPhraseQueryBuilder but was " + deserialized.getClass().getSimpleName(), + deserialized instanceof MatchPhraseQueryBuilder + ); + MatchPhraseQueryBuilder matchPhrase = (MatchPhraseQueryBuilder) deserialized; + assertEquals("Field name mismatch", fieldName, matchPhrase.fieldName()); + assertEquals("Query text mismatch", queryText, matchPhrase.value()); + } + } + + /** + * Single-field serialization round-trip for MATCH_BOOL_PREFIX. + * + * For any random field name (1-50 alphanumeric chars) and random query string (1-200 chars), + * serializing via the MATCH_BOOL_PREFIX serializer and deserializing produces a + * MatchBoolPrefixQueryBuilder with the same field and query. + */ + public void testMatchBoolPrefixRoundTrip() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.MATCH_BOOL_PREFIX); + assertNotNull("MATCH_BOOL_PREFIX serializer must be registered", serializer); + + String fieldName = randomAlphaOfLengthBetween(1, 50); + String queryText = randomAlphaOfLengthBetween(1, 200); + + RexCall call = buildSingleFieldRexCall(fieldName, queryText, "MATCH_BOOL_PREFIX"); + List fieldStorage = List.of( + new FieldStorageInfo(fieldName, "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false) + ); + + byte[] serialized = serializer.serialize(call, fieldStorage); + assertNotNull("Serialized bytes should not be null", serialized); + assertTrue("Serialized bytes should not be empty", serialized.length > 0); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue( + "Deserialized should be MatchBoolPrefixQueryBuilder but was " + deserialized.getClass().getSimpleName(), + deserialized instanceof MatchBoolPrefixQueryBuilder + ); + MatchBoolPrefixQueryBuilder matchBoolPrefix = (MatchBoolPrefixQueryBuilder) deserialized; + assertEquals("Field name mismatch", fieldName, matchBoolPrefix.fieldName()); + assertEquals("Query text mismatch", queryText, matchBoolPrefix.value()); + } + } + + /** + * Single-field serialization round-trip for MATCH_PHRASE_PREFIX. + * + * For any random field name (1-50 alphanumeric chars) and random query string (1-200 chars), + * serializing via the MATCH_PHRASE_PREFIX serializer and deserializing produces a + * MatchPhrasePrefixQueryBuilder with the same field and query. + */ + public void testMatchPhrasePrefixRoundTrip() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.MATCH_PHRASE_PREFIX); + assertNotNull("MATCH_PHRASE_PREFIX serializer must be registered", serializer); + + String fieldName = randomAlphaOfLengthBetween(1, 50); + String queryText = randomAlphaOfLengthBetween(1, 200); + + RexCall call = buildSingleFieldRexCall(fieldName, queryText, "MATCH_PHRASE_PREFIX"); + List fieldStorage = List.of( + new FieldStorageInfo(fieldName, "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false) + ); + + byte[] serialized = serializer.serialize(call, fieldStorage); + assertNotNull("Serialized bytes should not be null", serialized); + assertTrue("Serialized bytes should not be empty", serialized.length > 0); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue( + "Deserialized should be MatchPhrasePrefixQueryBuilder but was " + deserialized.getClass().getSimpleName(), + deserialized instanceof MatchPhrasePrefixQueryBuilder + ); + MatchPhrasePrefixQueryBuilder matchPhrasePrefix = (MatchPhrasePrefixQueryBuilder) deserialized; + assertEquals("Field name mismatch", fieldName, matchPhrasePrefix.fieldName()); + assertEquals("Query text mismatch", queryText, matchPhrasePrefix.value()); + } + } + + /** + * Builds a mock RexCall simulating the single-field relevance function structure: + * FUNC(MAP('field', $0), MAP('query', 'queryText')) + * + * The serializer extracts the field via extractFieldFromRelevanceMap(call, 0, fieldStorage) + * which expects operand 0 to be a RexCall (MAP) with a RexInputRef at child index 1. + * It extracts the query via extractStringFromRelevanceMap(call, 1) which expects operand 1 + * to be a RexCall (MAP) with a RexLiteral at child index 1. + */ + private RexCall buildSingleFieldRexCall(String fieldName, String queryText, String functionName) { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + // Operand 0: MAP('field', $0) — field reference + RexNode fieldKey = rexBuilder.makeLiteral("field"); + RexNode fieldRef = rexBuilder.makeInputRef(varcharType, 0); + RexNode fieldMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, fieldKey, fieldRef); + + // Operand 1: MAP('query', 'queryText') — query text literal + RexNode queryKey = rexBuilder.makeLiteral("query"); + RexNode queryLiteral = rexBuilder.makeLiteral(queryText); + RexNode queryMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, queryKey, queryLiteral); + + // Top-level function call + SqlFunction sqlFunction = new SqlFunction( + functionName, + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + return (RexCall) rexBuilder.makeCall(sqlFunction, fieldMap, queryMap); + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java index 379240c44ee81..a032c8cf2f255 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java @@ -144,7 +144,20 @@ private List resolveViableBackends( CapabilityRegistry registry = context.getCapabilityRegistry(); + ScalarFunction function = ScalarFunction.fromSqlOperatorWithFallback(predicate.getOperator()); + if (function == null) { + throw new IllegalStateException( + "Unrecognized filter operator [" + predicate.getOperator().getName() + " / " + predicate.getKind() + "]" + ); + } + if (fieldIndices.isEmpty()) { + // Multi-field full-text functions (multi_match, query_string, simple_query_string) + // encode field names as string literals in nested MAPs rather than RexInputRef. + // Resolve viability against any backend that supports the function on text fields. + if (function.getCategory() == ScalarFunction.Category.FULL_TEXT) { + return new ArrayList<>(registry.filterBackendsAnyFormat(function, FieldType.TEXT)); + } throw new UnsupportedOperationException( "Constant predicate with no field references reached the filter rule: [" + predicate @@ -152,13 +165,6 @@ private List resolveViableBackends( ); } - ScalarFunction function = ScalarFunction.fromSqlOperatorWithFallback(predicate.getOperator()); - if (function == null) { - throw new IllegalStateException( - "Unrecognized filter operator [" + predicate.getOperator().getName() + " / " + predicate.getKind() + "]" - ); - } - Set viableSet = new HashSet<>(registry.filterCapableBackends()); for (int fieldIndex : fieldIndices) { From c7590aee7dbad206740cfebb4097c62aee9754b4 Mon Sep 17 00:00:00 2001 From: Suresh N S Date: Mon, 11 May 2026 13:01:46 +0530 Subject: [PATCH 2/8] Addressing comments Signed-off-by: Suresh N S --- .../opensearch/be/lucene/ConversionUtils.java | 5 + .../be/lucene/QuerySerializerRegistry.java | 51 +-- .../be/lucene/ConversionUtilsTests.java | 9 +- .../lucene/QuerySerializerRegistryTests.java | 61 ---- ...SingleFieldSerializationPropertyTests.java | 27 +- .../DSL_QUERY_EXECUTOR_ARCHITECTURE.md | 313 ++++++++++++++++++ 6 files changed, 367 insertions(+), 99 deletions(-) delete mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java create mode 100644 sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java index 4bc125adc8409..1de8c762aa1bd 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java @@ -92,6 +92,11 @@ static byte[] serializeQueryBuilder(QueryBuilder queryBuilder) { *

Also supports the RexInputRef-based structure for single-field fallback: * {@code MAP('field', $ref1, 'field', $ref2, ...)} * + *

Note: This method is intentionally not recursive. The MAP nesting depth is bounded + * to at most 2 levels by Calcite's MAP_VALUE_CONSTRUCTOR design: an outer MAP holding + * the 'fields' key and a nested MAP holding field-name/boost pairs. Deeper nesting does + * not occur in the PPL relevance function encoding. + * *

TODO: extract per-field boost values and return them alongside field names. */ static List extractFieldsFromRelevanceMap(RexCall call, int operandIndex, List fieldStorage) { diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java index 72038f646341b..b165e08e056d9 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java @@ -27,12 +27,9 @@ * Registry of per-function query serializers for delegated predicates. * Each serializer converts a Calcite RexCall into serialized QueryBuilder bytes * that the Lucene backend can deserialize at the data node. - * - *

TODO: add serializers for match_phrase, match_bool_prefix, match_phrase_prefix. - * TODO: add multi-field relevance serializers for multi_match, query_string, simple_query_string. */ final class QuerySerializerRegistry { - + private static final Map SERIALIZERS = Map.ofEntries( Map.entry(ScalarFunction.MATCH, QuerySerializerRegistry::serializeMatch), Map.entry(ScalarFunction.MATCH_PHRASE, QuerySerializerRegistry::serializeMatchPhrase), @@ -49,48 +46,62 @@ static Map getSerializers() { return SERIALIZERS; } + // TODO: Extract each serialize* method into its own dedicated class once we handle more parameters. + // These methods are expected to grow significantly as optional parameters are added. + private static byte[] serializeMatch(RexCall call, List fieldStorage) { String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: extract optional params (operator, analyzer, fuzziness, boost) from operands 2+ - MatchQueryBuilder queryBuilder = new MatchQueryBuilder(fieldName, queryText); + String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); + // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params + // (operator, analyzer, fuzziness, boost) and apply them to the QueryBuilder. + MatchQueryBuilder queryBuilder = new MatchQueryBuilder(fieldName, fieldValue); return ConversionUtils.serializeQueryBuilder(queryBuilder); } private static byte[] serializeMatchPhrase(RexCall call, List fieldStorage) { String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); - MatchPhraseQueryBuilder queryBuilder = new MatchPhraseQueryBuilder(fieldName, queryText); + String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); + // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params + // (slop, analyzer, zero_terms_query) and apply them to the QueryBuilder. + MatchPhraseQueryBuilder queryBuilder = new MatchPhraseQueryBuilder(fieldName, fieldValue); return ConversionUtils.serializeQueryBuilder(queryBuilder); } private static byte[] serializeMatchBoolPrefix(RexCall call, List fieldStorage) { String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); - MatchBoolPrefixQueryBuilder queryBuilder = new MatchBoolPrefixQueryBuilder(fieldName, queryText); + String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); + // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params + // (analyzer, fuzziness, operator, minimum_should_match) and apply them to the QueryBuilder. + MatchBoolPrefixQueryBuilder queryBuilder = new MatchBoolPrefixQueryBuilder(fieldName, fieldValue); return ConversionUtils.serializeQueryBuilder(queryBuilder); } private static byte[] serializeMatchPhrasePrefix(RexCall call, List fieldStorage) { String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); - MatchPhrasePrefixQueryBuilder queryBuilder = new MatchPhrasePrefixQueryBuilder(fieldName, queryText); + String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); + // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params + // (slop, analyzer, max_expansions, zero_terms_query) and apply them to the QueryBuilder. + MatchPhrasePrefixQueryBuilder queryBuilder = new MatchPhrasePrefixQueryBuilder(fieldName, fieldValue); return ConversionUtils.serializeQueryBuilder(queryBuilder); } private static byte[] serializeMultiMatch(RexCall call, List fieldStorage) { List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder - MultiMatchQueryBuilder queryBuilder = new MultiMatchQueryBuilder(queryText, fields.toArray(String[]::new)); + // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params + // (type, operator, analyzer, fuzziness, minimum_should_match) and apply them to the QueryBuilder. + MultiMatchQueryBuilder queryBuilder = new MultiMatchQueryBuilder(fieldValue, fields.toArray(String[]::new)); return ConversionUtils.serializeQueryBuilder(queryBuilder); } private static byte[] serializeQueryString(RexCall call, List fieldStorage) { List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder - QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder(queryText); + // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params + // (default_operator, analyzer, allow_leading_wildcard) and apply them to the QueryBuilder. + QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder(fieldValue); for (String field : fields) { queryBuilder.field(field); } @@ -99,9 +110,11 @@ private static byte[] serializeQueryString(RexCall call, List private static byte[] serializeSimpleQueryString(RexCall call, List fieldStorage) { List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder - SimpleQueryStringBuilder queryBuilder = new SimpleQueryStringBuilder(queryText); + // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params + // (default_operator, analyzer, flags, minimum_should_match) and apply them to the QueryBuilder. + SimpleQueryStringBuilder queryBuilder = new SimpleQueryStringBuilder(fieldValue); for (String field : fields) { queryBuilder.field(field); } diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java index 3a982034b0b25..7d56775d31b33 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java @@ -25,6 +25,7 @@ import org.opensearch.analytics.spi.FieldType; import org.opensearch.test.OpenSearchTestCase; +import java.math.BigDecimal; import java.util.List; /** @@ -62,7 +63,7 @@ public void testSingleFieldExtractionReturnsOneElementList() { // Inner nested MAP: field name literal + boost literal (boost ignored, only field name extracted) RexNode fieldNameLiteral = rexBuilder.makeLiteral("title"); - RexNode boostLiteral = rexBuilder.makeExactLiteral(new java.math.BigDecimal("1.0"), doubleType); + RexNode boostLiteral = rexBuilder.makeExactLiteral(new BigDecimal("1.0"), doubleType); RexNode nestedMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, fieldNameLiteral, boostLiteral); // Outer MAP: MAP('fields', nestedMap) @@ -98,11 +99,11 @@ public void testMultiFieldExtractionReturnsAllFieldsInOrder() { RelDataType doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE); RexNode titleLiteral = rexBuilder.makeLiteral("title"); - RexNode titleBoost = rexBuilder.makeExactLiteral(new java.math.BigDecimal("2.0"), doubleType); + RexNode titleBoost = rexBuilder.makeExactLiteral(new BigDecimal("2.0"), doubleType); RexNode bodyLiteral = rexBuilder.makeLiteral("body"); - RexNode bodyBoost = rexBuilder.makeExactLiteral(new java.math.BigDecimal("1.0"), doubleType); + RexNode bodyBoost = rexBuilder.makeExactLiteral(new BigDecimal("1.0"), doubleType); RexNode tagsLiteral = rexBuilder.makeLiteral("tags"); - RexNode tagsBoost = rexBuilder.makeExactLiteral(new java.math.BigDecimal("0.5"), doubleType); + RexNode tagsBoost = rexBuilder.makeExactLiteral(new BigDecimal("0.5"), doubleType); // Nested MAP with 3 field/boost pairs RexNode nestedMap = rexBuilder.makeCall( diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java deleted file mode 100644 index 5b8e97bd92a0f..0000000000000 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.be.lucene; - -import org.opensearch.analytics.spi.DelegatedPredicateSerializer; -import org.opensearch.analytics.spi.ScalarFunction; -import org.opensearch.test.OpenSearchTestCase; - -import java.util.Map; -import java.util.Set; - -/** - * Unit tests for {@link QuerySerializerRegistry} completeness. - * Validates Requirement 10.1. - */ -public class QuerySerializerRegistryTests extends OpenSearchTestCase { - - /** - * Verifies that getSerializers() returns exactly 7 entries. - */ - public void testRegistryContainsExactlySevenEntries() { - Map serializers = QuerySerializerRegistry.getSerializers(); - assertEquals("Registry should contain exactly 7 serializer entries", 7, serializers.size()); - } - - /** - * Verifies that the key set matches the expected ScalarFunction constants. - */ - public void testRegistryKeySetMatchesExpectedFunctions() { - Map serializers = QuerySerializerRegistry.getSerializers(); - - Set expectedKeys = Set.of( - ScalarFunction.MATCH, - ScalarFunction.MATCH_PHRASE, - ScalarFunction.MATCH_BOOL_PREFIX, - ScalarFunction.MATCH_PHRASE_PREFIX, - ScalarFunction.MULTI_MATCH, - ScalarFunction.QUERY_STRING, - ScalarFunction.SIMPLE_QUERY_STRING - ); - - assertEquals("Registry keys should match expected full-text functions", expectedKeys, serializers.keySet()); - } - - /** - * Verifies that all serializer values are non-null (each entry has a valid lambda). - */ - public void testAllSerializerValuesAreNonNull() { - Map serializers = QuerySerializerRegistry.getSerializers(); - - for (Map.Entry entry : serializers.entrySet()) { - assertNotNull("Serializer for " + entry.getKey() + " should not be null", entry.getValue()); - } - } -} diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/SingleFieldSerializationPropertyTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/SingleFieldSerializationPropertyTests.java index d2493bda9f69c..2d8b6e5229486 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/SingleFieldSerializationPropertyTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/SingleFieldSerializationPropertyTests.java @@ -39,11 +39,11 @@ import java.util.Map; /** - * Property-based test for single-field serialization round-trip. + * Serialization round-trip tests for single-field full-text serializers. * - *

Feature: lucene-fulltext-serializers, Property: Single-field serialization round-trip + *

Feature: lucene-fulltext-serializers * - *

For any valid field name and non-empty query string, serializing a single-field full-text + *

For a given field name and query string, serializing a single-field full-text * predicate (match_phrase, match_bool_prefix, or match_phrase_prefix) via its registered serializer * and then deserializing the resulting bytes with {@code readNamedWriteable(QueryBuilder.class)} * shall produce a QueryBuilder of the correct subclass containing the same field name and query text. @@ -75,16 +75,15 @@ public void setUp() throws Exception { /** * Single-field serialization round-trip for MATCH_PHRASE. * - * For any random field name (1-50 alphanumeric chars) and random query string (1-200 chars), - * serializing via the MATCH_PHRASE serializer and deserializing produces a MatchPhraseQueryBuilder + * Serializing via the MATCH_PHRASE serializer and deserializing produces a MatchPhraseQueryBuilder * with the same field and query. */ public void testMatchPhraseRoundTrip() throws IOException { DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.MATCH_PHRASE); assertNotNull("MATCH_PHRASE serializer must be registered", serializer); - String fieldName = randomAlphaOfLengthBetween(1, 50); - String queryText = randomAlphaOfLengthBetween(1, 200); + String fieldName = "title"; + String queryText = "quick brown fox"; RexCall call = buildSingleFieldRexCall(fieldName, queryText, "MATCH_PHRASE"); List fieldStorage = List.of( @@ -110,16 +109,15 @@ public void testMatchPhraseRoundTrip() throws IOException { /** * Single-field serialization round-trip for MATCH_BOOL_PREFIX. * - * For any random field name (1-50 alphanumeric chars) and random query string (1-200 chars), - * serializing via the MATCH_BOOL_PREFIX serializer and deserializing produces a + * Serializing via the MATCH_BOOL_PREFIX serializer and deserializing produces a * MatchBoolPrefixQueryBuilder with the same field and query. */ public void testMatchBoolPrefixRoundTrip() throws IOException { DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.MATCH_BOOL_PREFIX); assertNotNull("MATCH_BOOL_PREFIX serializer must be registered", serializer); - String fieldName = randomAlphaOfLengthBetween(1, 50); - String queryText = randomAlphaOfLengthBetween(1, 200); + String fieldName = "description"; + String queryText = "opensearch full text"; RexCall call = buildSingleFieldRexCall(fieldName, queryText, "MATCH_BOOL_PREFIX"); List fieldStorage = List.of( @@ -145,16 +143,15 @@ public void testMatchBoolPrefixRoundTrip() throws IOException { /** * Single-field serialization round-trip for MATCH_PHRASE_PREFIX. * - * For any random field name (1-50 alphanumeric chars) and random query string (1-200 chars), - * serializing via the MATCH_PHRASE_PREFIX serializer and deserializing produces a + * Serializing via the MATCH_PHRASE_PREFIX serializer and deserializing produces a * MatchPhrasePrefixQueryBuilder with the same field and query. */ public void testMatchPhrasePrefixRoundTrip() throws IOException { DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.MATCH_PHRASE_PREFIX); assertNotNull("MATCH_PHRASE_PREFIX serializer must be registered", serializer); - String fieldName = randomAlphaOfLengthBetween(1, 50); - String queryText = randomAlphaOfLengthBetween(1, 200); + String fieldName = "content"; + String queryText = "search engine optim"; RexCall call = buildSingleFieldRexCall(fieldName, queryText, "MATCH_PHRASE_PREFIX"); List fieldStorage = List.of( diff --git a/sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md b/sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md new file mode 100644 index 0000000000000..6eee9a29d271e --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md @@ -0,0 +1,313 @@ +# DSL Query Executor Plugin — Architecture & Flow Diagram + +> Architecture for the `dsl-query-executor` plugin on the `feature/datafusion` branch. +> This replaces the `DslLogicalPlanPlugin` / `DslConverterPlugin` approach with an +> action-filter intercept pattern and direct converter invocation. + +## Overall Architecture + +``` +┌──────────────────────────────────────────────────────────────────────────┐ +│ OpenSearch Server │ +│ │ +│ _search request (SearchAction) │ +│ │ │ +│ ▼ │ +│ SearchActionFilter (ActionFilter, order=MIN_VALUE) │ +│ │ intercepts SearchAction.NAME │ +│ │ dispatches via NodeClient │ +│ ▼ │ +│ TransportDslExecuteAction.doExecute() ← DslExecuteAction.INSTANCE │ +│ │ │ +│ ├─► resolveToSingleIndex(request) │ +│ │ └─► aliases/wildcards → single concrete index │ +│ │ │ +│ ├─► SearchSourceConverter(engineContext.getSchema()) │ +│ │ │ │ +│ │ ├─► catalogReader.getTable(indexName) │ +│ │ │ │ +│ │ ├─► Shared base: TableScan → Filter │ +│ │ │ │ +│ │ ├─► HITS path: │ +│ │ │ Project → Sort │ +│ │ │ │ +│ │ └─► AGGS path: │ +│ │ AggregationTreeWalker.walk() │ +│ │ → List │ +│ │ AggregateConverter.convert() (per metadata) │ +│ │ │ +│ ├─► QueryPlanExecutor.execute(plans) ← RelNode → rows │ +│ │ └─► analytics-engine execution │ +│ │ │ +│ └─► SearchResponseBuilder.build(results, tookInMillis) │ +│ └─► results → SearchResponse │ +│ │ +│ Dependencies: │ +│ analytics-engine — QueryPlanExecutor, EngineContext (via Guice) │ +│ analytics-framework — Calcite, shared SPI interfaces │ +│ │ +└──────────────────────────────────────────────────────────────────────────┘ +``` + +## Class Hierarchy + +### Plugin Layer +``` +DslQueryExecutorPlugin (extends Plugin, implements ActionPlugin) + ├── SearchActionFilter — intercepts _search, dispatches to DslExecuteAction + ├── TransportDslExecuteAction — orchestrates DSL → QueryPlans → SearchResponse + │ (Guice-injected: EngineContext, QueryPlanExecutor, ClusterService, + │ IndexNameExpressionResolver) + └── DslExecuteAction — action registration (INSTANCE singleton) +``` + +### Converter Layer +``` +SearchSourceConverter (replaces DslLogicalPlanService) + ├── SchemaPlus — from EngineContext (analytics engine), NOT IndexMappingClient + ├── RelDataTypeFactory — Calcite type factory, created internally + ├── HepPlanner — Calcite heuristic planner + ├── RelOptCluster — Calcite planning cluster + ├── CalciteCatalogReader — resolves table metadata from schema + ├── FilterConverter — DSL query → LogicalFilter + ├── ProjectConverter — _source fields → LogicalProject + ├── SortConverter — sort + pagination → LogicalSort + ├── AggregateConverter — AggregationMetadata → LogicalAggregate (standalone) + └── AggregationTreeWalker — walks DSL agg tree → List +``` + +### Shared Context +``` +ConversionContext (simplified) + ├── SearchSourceBuilder — original DSL source + ├── RelOptCluster — Calcite planning cluster + └── RelOptTable — resolved table reference + Provides: getSearchSource(), getCluster(), getTable(), getRowType(), getRexBuilder() + Removed: indexName, indexSchema, capabilities, mutable AggregationMetadata +``` + +### Converter Classes +``` +AbstractDslConverter (abstract) + convert(RelNode input, ConversionContext ctx) + └── isApplicable(ctx) → doConvert(input, ctx) + No PipelinePhase or ordering — converters called directly in sequence + +FilterConverter (extends AbstractDslConverter) + └── skips MatchAllQueryBuilder; uses QueryRegistry → RexNode + +ProjectConverter (extends AbstractDslConverter) + └── _source field selection → LogicalProject + +SortConverter (extends AbstractDslConverter) + └── sort clauses + offset/fetch → LogicalSort + +AggregateConverter (standalone — NOT AbstractDslConverter) + convert(RelNode input, AggregationMetadata metadata) + └── LogicalAggregate.create(input, groupByBitSet, null, aggregateCalls) +``` + +### Query Translation Layer +``` +QueryRegistry (extends HandlerRegistry) + └── Map, QueryTranslator> + +QueryRegistryFactory.create(): + ├── TermQueryTranslator → EQUALS($field, literal) + └── MatchAllQueryTranslator → boolean TRUE +``` + +### Aggregation Translation Layer +``` +AggregationRegistry (extends HandlerRegistry) + └── Map, AggregationType> + +AggregationRegistryFactory.create(): + ├── AvgMetricTranslator → AVG() + ├── SumMetricTranslator → SUM() + ├── MinMetricTranslator → MIN() + ├── MaxMetricTranslator → MAX() + └── TermsBucketTranslator → GROUP BY field +``` + +### Aggregation Tree Walker +``` +AggregationTreeWalker.walk(aggs, rowType, typeFactory) + └── walkRecursive(aggs, currentGroupings, granularities) + ├── BucketShape → accumulate groupings, recurse into sub-aggs + └── MetricTranslator → add AggregateCall to builder + +Output: List + Each contains: ImmutableBitSet (GROUP BY), List +``` + +### Execution & Response Layer +``` +DslQueryPlanExecutor + └── wraps analytics-engine QueryPlanExecutor> + iterates QueryPlans, executes each RelNode, collects ExecutionResults + logs plan via relNode.explain() at INFO level + +SearchResponseBuilder (simplified) + └── static build(List results, long tookInMillis) + currently returns empty SearchHits (TODO: populate from results) +``` + +## Detailed Flow: TransportDslExecuteAction.doExecute() + +``` +1. SearchActionFilter.apply(action, request, listener, chain) + │ action == SearchAction.NAME? + │ yes → dispatch to DslExecuteAction via NodeClient + │ no → chain.proceed(action, request, listener) + │ + 2. TransportDslExecuteAction.doExecute(task, request, listener) + │ │ + │ 2a. resolveToSingleIndex(request) + │ │ → uses IndexNameExpressionResolver + │ │ → aliases/wildcards → single concrete index name + │ │ + │ 2b. new SearchSourceConverter(engineContext.getSchema()) + │ │ → receives SchemaPlus from analytics engine (via Guice) + │ │ → creates Calcite infrastructure internally: + │ │ RelDataTypeFactory, HepPlanner, RelOptCluster, CalciteCatalogReader + │ │ → creates converters: FilterConverter, ProjectConverter, + │ │ SortConverter, AggregateConverter + │ │ → creates AggregationTreeWalker with AggregationRegistryFactory.create() + │ │ + │ 2c. converter.convert(request.source(), indexName) + │ │ │ + │ │ 2c-i. catalogReader.getTable(List.of(indexName)) + │ │ │ → resolve table from analytics engine schema + │ │ │ + │ │ 2c-ii. Create ConversionContext(searchSource, cluster, table) + │ │ │ + │ │ 2c-iii. Shared base: + │ │ │ LogicalTableScan.create(cluster, table) + │ │ │ → filterConverter.convert(base, ctx) + │ │ │ Result: scan → filter (shared by both paths) + │ │ │ + │ │ 2c-iv. [HITS path] + │ │ │ projectConverter.convert(filtered, ctx) + │ │ │ → sortConverter.convert(projected, ctx) + │ │ │ Result: scan → filter → project → sort + │ │ │ + │ │ 2c-v. [AGGS path] + │ │ treeWalker.walk(aggs, rowType, typeFactory) + │ │ → List + │ │ For each metadata: + │ │ aggregateConverter.convert(filtered, metadata) + │ │ Result: scan → filter → aggregate (per granularity) + │ │ │ + │ │ 2c-vi. Return QueryPlans (HITS plan + N AGGREGATION plans) + │ │ + │ 2d. planExecutor.execute(plans) + │ │ → analytics-engine QueryPlanExecutor> + │ │ → iterates plans, executes each RelNode + │ │ → collects List + │ │ + │ 2e. SearchResponseBuilder.build(results, tookInMillis) + │ → assembles SearchResponse from execution results + │ → currently returns empty SearchHits (TODO) +``` + +## Key Differences from DslLogicalPlanPlugin Architecture + +| Aspect | Old (DslLogicalPlanPlugin) | New (DslQueryExecutorPlugin) | +|--------|---------------------------|------------------------------| +| Entry point | `DslConverterPlugin.convertDsl()` called by server | `SearchActionFilter` intercepts `_search` at action layer | +| Schema source | `IndexMappingClient` fetches mappings from cluster | `EngineContext.getSchema()` from analytics engine (Guice) | +| Converter orchestration | `ConversionPipeline` auto-sorts by `PipelinePhase` order | Converters called directly in sequence — no pipeline | +| Scan + Filter | Each pipeline starts with its own Scan → Filter | Shared base: single Scan → Filter, then branch for hits vs aggs | +| AggregateConverter | Extends `AbstractDslConverter`, uses `ConversionContext` | Standalone class, takes `(RelNode, AggregationMetadata)` directly | +| ConversionContext | Holds indexName, indexSchema, capabilities, mutable agg metadata | Simplified: only SearchSourceBuilder, cluster, table | +| Registry creation | Inline in `DslLogicalPlanService` constructor | Factory classes: `QueryRegistryFactory`, `AggregationRegistryFactory` | +| Execution | `DefaultQueryPlanExecutor` → `RelNodeExecutor` (placeholder) | `DslQueryPlanExecutor` wraps analytics-engine `QueryPlanExecutor` | +| Dependencies | Self-contained (uses OpenSearch Client for schema) | Depends on `analytics-engine` and `analytics-framework` (extendedPlugins) | + +## How to Add New Query Translators + +### Adding a new term-level query (e.g., `terms` → IN operator) + +1. Create `TermsQueryTranslator.java` in `org.opensearch.dsl.query`: +```java +public class TermsQueryTranslator implements QueryTranslator { + public Class getQueryType() { + return TermsQueryBuilder.class; + } + public RexNode convert(QueryBuilder query, ConversionContext ctx) { + // Build IN($field, val1, val2, ...) using SqlStdOperatorTable.IN + } +} +``` + +2. Register in `QueryRegistryFactory.create()`: +```java +public static QueryRegistry create() { + QueryRegistry registry = new QueryRegistry(); + registry.register(new TermQueryTranslator()); + registry.register(new MatchAllQueryTranslator()); + registry.register(new TermsQueryTranslator()); // ← add here + return registry; +} +``` + +### Adding a new single-value metric aggregation (e.g., `value_count`) + +1. Create `ValueCountMetricTranslator.java` in `org.opensearch.dsl.aggregation.metric`: +```java +public class ValueCountMetricTranslator extends AbstractMetricTranslator { + public Class getAggregationType() { ... } + protected SqlAggFunction getAggFunction() { return SqlStdOperatorTable.COUNT; } + protected String getFieldName(ValueCountAggregationBuilder agg) { return agg.field(); } + public InternalAggregation toInternalAggregation(String name, Object value) { + return new InternalValueCount(name, value == null ? 0 : ((Number)value).longValue(), Map.of()); + } +} +``` + +2. Register in `AggregationRegistryFactory.create()`: +```java +public static AggregationRegistry create() { + AggregationRegistry registry = new AggregationRegistry(); + registry.register(new AvgMetricTranslator()); + registry.register(new SumMetricTranslator()); + // ...existing registrations... + registry.register(new ValueCountMetricTranslator()); // ← add here + return registry; +} +``` + +### Adding a new bucket aggregation (e.g., `histogram`) + +1. Create `HistogramBucketShape.java` implementing `BucketShape`: +```java +public class HistogramBucketShape implements BucketShape { + public GroupingInfo getGrouping(HistogramAggregationBuilder agg) { + // Return ExpressionGrouping with FLOOR(field/interval)*interval + } + public BucketOrder getOrder(...) { ... } + public Collection getSubAggregations(...) { ... } + public InternalAggregation toBucketAggregation(..., List buckets) { + // Build InternalHistogram from bucket entries + } +} +``` + +2. Register in `AggregationRegistryFactory.create()`. + +## Key Design Patterns + +1. **Action Filter intercept** — `SearchActionFilter` sits at `Integer.MIN_VALUE` order, intercepting all `_search` requests before any other filter. Non-search actions pass through unchanged. This replaces the `DslConverterPlugin` interface that required server-side integration. + +2. **Direct converter sequencing** — Instead of a `ConversionPipeline` that auto-sorts by phase order, converters are called explicitly in a fixed sequence within `SearchSourceConverter.convert()`. This makes the flow easier to follow and removes the indirection of phase ordering. + +3. **Shared Scan→Filter base** — Both hits and aggregation paths share a single `LogicalTableScan → LogicalFilter` base. The old architecture created separate scan/filter nodes per pipeline. This avoids redundant work and makes the branching point explicit. + +4. **Schema from analytics engine** — The schema comes from `EngineContext.getSchema()` (injected via Guice from the `analytics-engine` plugin) rather than being fetched from cluster mappings via `IndexMappingClient`. This decouples the DSL converter from cluster state and leverages the engine's type system. + +5. **Registry factory pattern** — `QueryRegistryFactory.create()` and `AggregationRegistryFactory.create()` encapsulate registry construction. Adding a new translator is: implement the interface + add one line to the factory method. + +6. **Standalone AggregateConverter** — Unlike other converters that extend `AbstractDslConverter` and receive `ConversionContext`, `AggregateConverter` takes `(RelNode, AggregationMetadata)` directly. This reflects that aggregation conversion depends on walker output, not on the shared conversion context. + +7. **Granularity decomposition** — `AggregationTreeWalker` still produces one `AggregationMetadata` per unique GROUP BY key set. The converter loops over them, creating one `LogicalAggregate` per granularity from the shared filtered base. From e384326f60cb08a81c3b55278d06d6e976440e85 Mon Sep 17 00:00:00 2001 From: Suresh N S Date: Mon, 11 May 2026 15:38:00 +0530 Subject: [PATCH 3/8] Removing the unintentional checkin of md file Signed-off-by: Suresh N S --- .../DSL_QUERY_EXECUTOR_ARCHITECTURE.md | 313 ------------------ 1 file changed, 313 deletions(-) delete mode 100644 sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md diff --git a/sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md b/sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md deleted file mode 100644 index 6eee9a29d271e..0000000000000 --- a/sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md +++ /dev/null @@ -1,313 +0,0 @@ -# DSL Query Executor Plugin — Architecture & Flow Diagram - -> Architecture for the `dsl-query-executor` plugin on the `feature/datafusion` branch. -> This replaces the `DslLogicalPlanPlugin` / `DslConverterPlugin` approach with an -> action-filter intercept pattern and direct converter invocation. - -## Overall Architecture - -``` -┌──────────────────────────────────────────────────────────────────────────┐ -│ OpenSearch Server │ -│ │ -│ _search request (SearchAction) │ -│ │ │ -│ ▼ │ -│ SearchActionFilter (ActionFilter, order=MIN_VALUE) │ -│ │ intercepts SearchAction.NAME │ -│ │ dispatches via NodeClient │ -│ ▼ │ -│ TransportDslExecuteAction.doExecute() ← DslExecuteAction.INSTANCE │ -│ │ │ -│ ├─► resolveToSingleIndex(request) │ -│ │ └─► aliases/wildcards → single concrete index │ -│ │ │ -│ ├─► SearchSourceConverter(engineContext.getSchema()) │ -│ │ │ │ -│ │ ├─► catalogReader.getTable(indexName) │ -│ │ │ │ -│ │ ├─► Shared base: TableScan → Filter │ -│ │ │ │ -│ │ ├─► HITS path: │ -│ │ │ Project → Sort │ -│ │ │ │ -│ │ └─► AGGS path: │ -│ │ AggregationTreeWalker.walk() │ -│ │ → List │ -│ │ AggregateConverter.convert() (per metadata) │ -│ │ │ -│ ├─► QueryPlanExecutor.execute(plans) ← RelNode → rows │ -│ │ └─► analytics-engine execution │ -│ │ │ -│ └─► SearchResponseBuilder.build(results, tookInMillis) │ -│ └─► results → SearchResponse │ -│ │ -│ Dependencies: │ -│ analytics-engine — QueryPlanExecutor, EngineContext (via Guice) │ -│ analytics-framework — Calcite, shared SPI interfaces │ -│ │ -└──────────────────────────────────────────────────────────────────────────┘ -``` - -## Class Hierarchy - -### Plugin Layer -``` -DslQueryExecutorPlugin (extends Plugin, implements ActionPlugin) - ├── SearchActionFilter — intercepts _search, dispatches to DslExecuteAction - ├── TransportDslExecuteAction — orchestrates DSL → QueryPlans → SearchResponse - │ (Guice-injected: EngineContext, QueryPlanExecutor, ClusterService, - │ IndexNameExpressionResolver) - └── DslExecuteAction — action registration (INSTANCE singleton) -``` - -### Converter Layer -``` -SearchSourceConverter (replaces DslLogicalPlanService) - ├── SchemaPlus — from EngineContext (analytics engine), NOT IndexMappingClient - ├── RelDataTypeFactory — Calcite type factory, created internally - ├── HepPlanner — Calcite heuristic planner - ├── RelOptCluster — Calcite planning cluster - ├── CalciteCatalogReader — resolves table metadata from schema - ├── FilterConverter — DSL query → LogicalFilter - ├── ProjectConverter — _source fields → LogicalProject - ├── SortConverter — sort + pagination → LogicalSort - ├── AggregateConverter — AggregationMetadata → LogicalAggregate (standalone) - └── AggregationTreeWalker — walks DSL agg tree → List -``` - -### Shared Context -``` -ConversionContext (simplified) - ├── SearchSourceBuilder — original DSL source - ├── RelOptCluster — Calcite planning cluster - └── RelOptTable — resolved table reference - Provides: getSearchSource(), getCluster(), getTable(), getRowType(), getRexBuilder() - Removed: indexName, indexSchema, capabilities, mutable AggregationMetadata -``` - -### Converter Classes -``` -AbstractDslConverter (abstract) - convert(RelNode input, ConversionContext ctx) - └── isApplicable(ctx) → doConvert(input, ctx) - No PipelinePhase or ordering — converters called directly in sequence - -FilterConverter (extends AbstractDslConverter) - └── skips MatchAllQueryBuilder; uses QueryRegistry → RexNode - -ProjectConverter (extends AbstractDslConverter) - └── _source field selection → LogicalProject - -SortConverter (extends AbstractDslConverter) - └── sort clauses + offset/fetch → LogicalSort - -AggregateConverter (standalone — NOT AbstractDslConverter) - convert(RelNode input, AggregationMetadata metadata) - └── LogicalAggregate.create(input, groupByBitSet, null, aggregateCalls) -``` - -### Query Translation Layer -``` -QueryRegistry (extends HandlerRegistry) - └── Map, QueryTranslator> - -QueryRegistryFactory.create(): - ├── TermQueryTranslator → EQUALS($field, literal) - └── MatchAllQueryTranslator → boolean TRUE -``` - -### Aggregation Translation Layer -``` -AggregationRegistry (extends HandlerRegistry) - └── Map, AggregationType> - -AggregationRegistryFactory.create(): - ├── AvgMetricTranslator → AVG() - ├── SumMetricTranslator → SUM() - ├── MinMetricTranslator → MIN() - ├── MaxMetricTranslator → MAX() - └── TermsBucketTranslator → GROUP BY field -``` - -### Aggregation Tree Walker -``` -AggregationTreeWalker.walk(aggs, rowType, typeFactory) - └── walkRecursive(aggs, currentGroupings, granularities) - ├── BucketShape → accumulate groupings, recurse into sub-aggs - └── MetricTranslator → add AggregateCall to builder - -Output: List - Each contains: ImmutableBitSet (GROUP BY), List -``` - -### Execution & Response Layer -``` -DslQueryPlanExecutor - └── wraps analytics-engine QueryPlanExecutor> - iterates QueryPlans, executes each RelNode, collects ExecutionResults - logs plan via relNode.explain() at INFO level - -SearchResponseBuilder (simplified) - └── static build(List results, long tookInMillis) - currently returns empty SearchHits (TODO: populate from results) -``` - -## Detailed Flow: TransportDslExecuteAction.doExecute() - -``` -1. SearchActionFilter.apply(action, request, listener, chain) - │ action == SearchAction.NAME? - │ yes → dispatch to DslExecuteAction via NodeClient - │ no → chain.proceed(action, request, listener) - │ - 2. TransportDslExecuteAction.doExecute(task, request, listener) - │ │ - │ 2a. resolveToSingleIndex(request) - │ │ → uses IndexNameExpressionResolver - │ │ → aliases/wildcards → single concrete index name - │ │ - │ 2b. new SearchSourceConverter(engineContext.getSchema()) - │ │ → receives SchemaPlus from analytics engine (via Guice) - │ │ → creates Calcite infrastructure internally: - │ │ RelDataTypeFactory, HepPlanner, RelOptCluster, CalciteCatalogReader - │ │ → creates converters: FilterConverter, ProjectConverter, - │ │ SortConverter, AggregateConverter - │ │ → creates AggregationTreeWalker with AggregationRegistryFactory.create() - │ │ - │ 2c. converter.convert(request.source(), indexName) - │ │ │ - │ │ 2c-i. catalogReader.getTable(List.of(indexName)) - │ │ │ → resolve table from analytics engine schema - │ │ │ - │ │ 2c-ii. Create ConversionContext(searchSource, cluster, table) - │ │ │ - │ │ 2c-iii. Shared base: - │ │ │ LogicalTableScan.create(cluster, table) - │ │ │ → filterConverter.convert(base, ctx) - │ │ │ Result: scan → filter (shared by both paths) - │ │ │ - │ │ 2c-iv. [HITS path] - │ │ │ projectConverter.convert(filtered, ctx) - │ │ │ → sortConverter.convert(projected, ctx) - │ │ │ Result: scan → filter → project → sort - │ │ │ - │ │ 2c-v. [AGGS path] - │ │ treeWalker.walk(aggs, rowType, typeFactory) - │ │ → List - │ │ For each metadata: - │ │ aggregateConverter.convert(filtered, metadata) - │ │ Result: scan → filter → aggregate (per granularity) - │ │ │ - │ │ 2c-vi. Return QueryPlans (HITS plan + N AGGREGATION plans) - │ │ - │ 2d. planExecutor.execute(plans) - │ │ → analytics-engine QueryPlanExecutor> - │ │ → iterates plans, executes each RelNode - │ │ → collects List - │ │ - │ 2e. SearchResponseBuilder.build(results, tookInMillis) - │ → assembles SearchResponse from execution results - │ → currently returns empty SearchHits (TODO) -``` - -## Key Differences from DslLogicalPlanPlugin Architecture - -| Aspect | Old (DslLogicalPlanPlugin) | New (DslQueryExecutorPlugin) | -|--------|---------------------------|------------------------------| -| Entry point | `DslConverterPlugin.convertDsl()` called by server | `SearchActionFilter` intercepts `_search` at action layer | -| Schema source | `IndexMappingClient` fetches mappings from cluster | `EngineContext.getSchema()` from analytics engine (Guice) | -| Converter orchestration | `ConversionPipeline` auto-sorts by `PipelinePhase` order | Converters called directly in sequence — no pipeline | -| Scan + Filter | Each pipeline starts with its own Scan → Filter | Shared base: single Scan → Filter, then branch for hits vs aggs | -| AggregateConverter | Extends `AbstractDslConverter`, uses `ConversionContext` | Standalone class, takes `(RelNode, AggregationMetadata)` directly | -| ConversionContext | Holds indexName, indexSchema, capabilities, mutable agg metadata | Simplified: only SearchSourceBuilder, cluster, table | -| Registry creation | Inline in `DslLogicalPlanService` constructor | Factory classes: `QueryRegistryFactory`, `AggregationRegistryFactory` | -| Execution | `DefaultQueryPlanExecutor` → `RelNodeExecutor` (placeholder) | `DslQueryPlanExecutor` wraps analytics-engine `QueryPlanExecutor` | -| Dependencies | Self-contained (uses OpenSearch Client for schema) | Depends on `analytics-engine` and `analytics-framework` (extendedPlugins) | - -## How to Add New Query Translators - -### Adding a new term-level query (e.g., `terms` → IN operator) - -1. Create `TermsQueryTranslator.java` in `org.opensearch.dsl.query`: -```java -public class TermsQueryTranslator implements QueryTranslator { - public Class getQueryType() { - return TermsQueryBuilder.class; - } - public RexNode convert(QueryBuilder query, ConversionContext ctx) { - // Build IN($field, val1, val2, ...) using SqlStdOperatorTable.IN - } -} -``` - -2. Register in `QueryRegistryFactory.create()`: -```java -public static QueryRegistry create() { - QueryRegistry registry = new QueryRegistry(); - registry.register(new TermQueryTranslator()); - registry.register(new MatchAllQueryTranslator()); - registry.register(new TermsQueryTranslator()); // ← add here - return registry; -} -``` - -### Adding a new single-value metric aggregation (e.g., `value_count`) - -1. Create `ValueCountMetricTranslator.java` in `org.opensearch.dsl.aggregation.metric`: -```java -public class ValueCountMetricTranslator extends AbstractMetricTranslator { - public Class getAggregationType() { ... } - protected SqlAggFunction getAggFunction() { return SqlStdOperatorTable.COUNT; } - protected String getFieldName(ValueCountAggregationBuilder agg) { return agg.field(); } - public InternalAggregation toInternalAggregation(String name, Object value) { - return new InternalValueCount(name, value == null ? 0 : ((Number)value).longValue(), Map.of()); - } -} -``` - -2. Register in `AggregationRegistryFactory.create()`: -```java -public static AggregationRegistry create() { - AggregationRegistry registry = new AggregationRegistry(); - registry.register(new AvgMetricTranslator()); - registry.register(new SumMetricTranslator()); - // ...existing registrations... - registry.register(new ValueCountMetricTranslator()); // ← add here - return registry; -} -``` - -### Adding a new bucket aggregation (e.g., `histogram`) - -1. Create `HistogramBucketShape.java` implementing `BucketShape`: -```java -public class HistogramBucketShape implements BucketShape { - public GroupingInfo getGrouping(HistogramAggregationBuilder agg) { - // Return ExpressionGrouping with FLOOR(field/interval)*interval - } - public BucketOrder getOrder(...) { ... } - public Collection getSubAggregations(...) { ... } - public InternalAggregation toBucketAggregation(..., List buckets) { - // Build InternalHistogram from bucket entries - } -} -``` - -2. Register in `AggregationRegistryFactory.create()`. - -## Key Design Patterns - -1. **Action Filter intercept** — `SearchActionFilter` sits at `Integer.MIN_VALUE` order, intercepting all `_search` requests before any other filter. Non-search actions pass through unchanged. This replaces the `DslConverterPlugin` interface that required server-side integration. - -2. **Direct converter sequencing** — Instead of a `ConversionPipeline` that auto-sorts by phase order, converters are called explicitly in a fixed sequence within `SearchSourceConverter.convert()`. This makes the flow easier to follow and removes the indirection of phase ordering. - -3. **Shared Scan→Filter base** — Both hits and aggregation paths share a single `LogicalTableScan → LogicalFilter` base. The old architecture created separate scan/filter nodes per pipeline. This avoids redundant work and makes the branching point explicit. - -4. **Schema from analytics engine** — The schema comes from `EngineContext.getSchema()` (injected via Guice from the `analytics-engine` plugin) rather than being fetched from cluster mappings via `IndexMappingClient`. This decouples the DSL converter from cluster state and leverages the engine's type system. - -5. **Registry factory pattern** — `QueryRegistryFactory.create()` and `AggregationRegistryFactory.create()` encapsulate registry construction. Adding a new translator is: implement the interface + add one line to the factory method. - -6. **Standalone AggregateConverter** — Unlike other converters that extend `AbstractDslConverter` and receive `ConversionContext`, `AggregateConverter` takes `(RelNode, AggregationMetadata)` directly. This reflects that aggregation conversion depends on walker output, not on the shared conversion context. - -7. **Granularity decomposition** — `AggregationTreeWalker` still produces one `AggregationMetadata` per unique GROUP BY key set. The converter loops over them, creating one `LogicalAggregate` per granularity from the shared filtered base. From 1c58b1e414a6206b303afe2735ba3a9a1d8e8761 Mon Sep 17 00:00:00 2001 From: Suresh N S Date: Fri, 8 May 2026 16:44:28 +0530 Subject: [PATCH 4/8] Adding support for Relevance Functions Signed-off-by: Suresh N S --- .../be/lucene/QuerySerializerRegistry.java | 53 +++++++++++++++- .../lucene/QuerySerializerRegistryTests.java | 61 +++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java index b165e08e056d9..64aaa34971a46 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java @@ -29,7 +29,7 @@ * that the Lucene backend can deserialize at the data node. */ final class QuerySerializerRegistry { - + private static final Map SERIALIZERS = Map.ofEntries( Map.entry(ScalarFunction.MATCH, QuerySerializerRegistry::serializeMatch), Map.entry(ScalarFunction.MATCH_PHRASE, QuerySerializerRegistry::serializeMatchPhrase), @@ -120,4 +120,55 @@ private static byte[] serializeSimpleQueryString(RexCall call, List fieldStorage) { + String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); + String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + MatchPhraseQueryBuilder queryBuilder = new MatchPhraseQueryBuilder(fieldName, queryText); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeMatchBoolPrefix(RexCall call, List fieldStorage) { + String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); + String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + MatchBoolPrefixQueryBuilder queryBuilder = new MatchBoolPrefixQueryBuilder(fieldName, queryText); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeMatchPhrasePrefix(RexCall call, List fieldStorage) { + String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); + String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + MatchPhrasePrefixQueryBuilder queryBuilder = new MatchPhrasePrefixQueryBuilder(fieldName, queryText); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeMultiMatch(RexCall call, List fieldStorage) { + List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); + String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder + MultiMatchQueryBuilder queryBuilder = new MultiMatchQueryBuilder(queryText, fields.toArray(String[]::new)); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeQueryString(RexCall call, List fieldStorage) { + List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); + String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder + QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder(queryText); + for (String field : fields) { + queryBuilder.field(field); + } + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeSimpleQueryString(RexCall call, List fieldStorage) { + List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); + String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); + // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder + SimpleQueryStringBuilder queryBuilder = new SimpleQueryStringBuilder(queryText); + for (String field : fields) { + queryBuilder.field(field); + } + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } } diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java new file mode 100644 index 0000000000000..5b8e97bd92a0f --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.opensearch.analytics.spi.DelegatedPredicateSerializer; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Map; +import java.util.Set; + +/** + * Unit tests for {@link QuerySerializerRegistry} completeness. + * Validates Requirement 10.1. + */ +public class QuerySerializerRegistryTests extends OpenSearchTestCase { + + /** + * Verifies that getSerializers() returns exactly 7 entries. + */ + public void testRegistryContainsExactlySevenEntries() { + Map serializers = QuerySerializerRegistry.getSerializers(); + assertEquals("Registry should contain exactly 7 serializer entries", 7, serializers.size()); + } + + /** + * Verifies that the key set matches the expected ScalarFunction constants. + */ + public void testRegistryKeySetMatchesExpectedFunctions() { + Map serializers = QuerySerializerRegistry.getSerializers(); + + Set expectedKeys = Set.of( + ScalarFunction.MATCH, + ScalarFunction.MATCH_PHRASE, + ScalarFunction.MATCH_BOOL_PREFIX, + ScalarFunction.MATCH_PHRASE_PREFIX, + ScalarFunction.MULTI_MATCH, + ScalarFunction.QUERY_STRING, + ScalarFunction.SIMPLE_QUERY_STRING + ); + + assertEquals("Registry keys should match expected full-text functions", expectedKeys, serializers.keySet()); + } + + /** + * Verifies that all serializer values are non-null (each entry has a valid lambda). + */ + public void testAllSerializerValuesAreNonNull() { + Map serializers = QuerySerializerRegistry.getSerializers(); + + for (Map.Entry entry : serializers.entrySet()) { + assertNotNull("Serializer for " + entry.getKey() + " should not be null", entry.getValue()); + } + } +} From 67fbccc9d4dd5b0e3523ff734c6fd6d5e6e41fa7 Mon Sep 17 00:00:00 2001 From: Suresh N S Date: Mon, 11 May 2026 13:01:46 +0530 Subject: [PATCH 5/8] Addressing comments Signed-off-by: Suresh N S --- .../be/lucene/QuerySerializerRegistry.java | 53 +-- .../lucene/QuerySerializerRegistryTests.java | 61 ---- .../DSL_QUERY_EXECUTOR_ARCHITECTURE.md | 313 ++++++++++++++++++ 3 files changed, 314 insertions(+), 113 deletions(-) delete mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java create mode 100644 sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java index 64aaa34971a46..b165e08e056d9 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java @@ -29,7 +29,7 @@ * that the Lucene backend can deserialize at the data node. */ final class QuerySerializerRegistry { - + private static final Map SERIALIZERS = Map.ofEntries( Map.entry(ScalarFunction.MATCH, QuerySerializerRegistry::serializeMatch), Map.entry(ScalarFunction.MATCH_PHRASE, QuerySerializerRegistry::serializeMatchPhrase), @@ -120,55 +120,4 @@ private static byte[] serializeSimpleQueryString(RexCall call, List fieldStorage) { - String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); - MatchPhraseQueryBuilder queryBuilder = new MatchPhraseQueryBuilder(fieldName, queryText); - return ConversionUtils.serializeQueryBuilder(queryBuilder); - } - - private static byte[] serializeMatchBoolPrefix(RexCall call, List fieldStorage) { - String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); - MatchBoolPrefixQueryBuilder queryBuilder = new MatchBoolPrefixQueryBuilder(fieldName, queryText); - return ConversionUtils.serializeQueryBuilder(queryBuilder); - } - - private static byte[] serializeMatchPhrasePrefix(RexCall call, List fieldStorage) { - String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); - MatchPhrasePrefixQueryBuilder queryBuilder = new MatchPhrasePrefixQueryBuilder(fieldName, queryText); - return ConversionUtils.serializeQueryBuilder(queryBuilder); - } - - private static byte[] serializeMultiMatch(RexCall call, List fieldStorage) { - List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder - MultiMatchQueryBuilder queryBuilder = new MultiMatchQueryBuilder(queryText, fields.toArray(String[]::new)); - return ConversionUtils.serializeQueryBuilder(queryBuilder); - } - - private static byte[] serializeQueryString(RexCall call, List fieldStorage) { - List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder - QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder(queryText); - for (String field : fields) { - queryBuilder.field(field); - } - return ConversionUtils.serializeQueryBuilder(queryBuilder); - } - - private static byte[] serializeSimpleQueryString(RexCall call, List fieldStorage) { - List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder - SimpleQueryStringBuilder queryBuilder = new SimpleQueryStringBuilder(queryText); - for (String field : fields) { - queryBuilder.field(field); - } - return ConversionUtils.serializeQueryBuilder(queryBuilder); - } } diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java deleted file mode 100644 index 5b8e97bd92a0f..0000000000000 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/QuerySerializerRegistryTests.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.be.lucene; - -import org.opensearch.analytics.spi.DelegatedPredicateSerializer; -import org.opensearch.analytics.spi.ScalarFunction; -import org.opensearch.test.OpenSearchTestCase; - -import java.util.Map; -import java.util.Set; - -/** - * Unit tests for {@link QuerySerializerRegistry} completeness. - * Validates Requirement 10.1. - */ -public class QuerySerializerRegistryTests extends OpenSearchTestCase { - - /** - * Verifies that getSerializers() returns exactly 7 entries. - */ - public void testRegistryContainsExactlySevenEntries() { - Map serializers = QuerySerializerRegistry.getSerializers(); - assertEquals("Registry should contain exactly 7 serializer entries", 7, serializers.size()); - } - - /** - * Verifies that the key set matches the expected ScalarFunction constants. - */ - public void testRegistryKeySetMatchesExpectedFunctions() { - Map serializers = QuerySerializerRegistry.getSerializers(); - - Set expectedKeys = Set.of( - ScalarFunction.MATCH, - ScalarFunction.MATCH_PHRASE, - ScalarFunction.MATCH_BOOL_PREFIX, - ScalarFunction.MATCH_PHRASE_PREFIX, - ScalarFunction.MULTI_MATCH, - ScalarFunction.QUERY_STRING, - ScalarFunction.SIMPLE_QUERY_STRING - ); - - assertEquals("Registry keys should match expected full-text functions", expectedKeys, serializers.keySet()); - } - - /** - * Verifies that all serializer values are non-null (each entry has a valid lambda). - */ - public void testAllSerializerValuesAreNonNull() { - Map serializers = QuerySerializerRegistry.getSerializers(); - - for (Map.Entry entry : serializers.entrySet()) { - assertNotNull("Serializer for " + entry.getKey() + " should not be null", entry.getValue()); - } - } -} diff --git a/sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md b/sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md new file mode 100644 index 0000000000000..6eee9a29d271e --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md @@ -0,0 +1,313 @@ +# DSL Query Executor Plugin — Architecture & Flow Diagram + +> Architecture for the `dsl-query-executor` plugin on the `feature/datafusion` branch. +> This replaces the `DslLogicalPlanPlugin` / `DslConverterPlugin` approach with an +> action-filter intercept pattern and direct converter invocation. + +## Overall Architecture + +``` +┌──────────────────────────────────────────────────────────────────────────┐ +│ OpenSearch Server │ +│ │ +│ _search request (SearchAction) │ +│ │ │ +│ ▼ │ +│ SearchActionFilter (ActionFilter, order=MIN_VALUE) │ +│ │ intercepts SearchAction.NAME │ +│ │ dispatches via NodeClient │ +│ ▼ │ +│ TransportDslExecuteAction.doExecute() ← DslExecuteAction.INSTANCE │ +│ │ │ +│ ├─► resolveToSingleIndex(request) │ +│ │ └─► aliases/wildcards → single concrete index │ +│ │ │ +│ ├─► SearchSourceConverter(engineContext.getSchema()) │ +│ │ │ │ +│ │ ├─► catalogReader.getTable(indexName) │ +│ │ │ │ +│ │ ├─► Shared base: TableScan → Filter │ +│ │ │ │ +│ │ ├─► HITS path: │ +│ │ │ Project → Sort │ +│ │ │ │ +│ │ └─► AGGS path: │ +│ │ AggregationTreeWalker.walk() │ +│ │ → List │ +│ │ AggregateConverter.convert() (per metadata) │ +│ │ │ +│ ├─► QueryPlanExecutor.execute(plans) ← RelNode → rows │ +│ │ └─► analytics-engine execution │ +│ │ │ +│ └─► SearchResponseBuilder.build(results, tookInMillis) │ +│ └─► results → SearchResponse │ +│ │ +│ Dependencies: │ +│ analytics-engine — QueryPlanExecutor, EngineContext (via Guice) │ +│ analytics-framework — Calcite, shared SPI interfaces │ +│ │ +└──────────────────────────────────────────────────────────────────────────┘ +``` + +## Class Hierarchy + +### Plugin Layer +``` +DslQueryExecutorPlugin (extends Plugin, implements ActionPlugin) + ├── SearchActionFilter — intercepts _search, dispatches to DslExecuteAction + ├── TransportDslExecuteAction — orchestrates DSL → QueryPlans → SearchResponse + │ (Guice-injected: EngineContext, QueryPlanExecutor, ClusterService, + │ IndexNameExpressionResolver) + └── DslExecuteAction — action registration (INSTANCE singleton) +``` + +### Converter Layer +``` +SearchSourceConverter (replaces DslLogicalPlanService) + ├── SchemaPlus — from EngineContext (analytics engine), NOT IndexMappingClient + ├── RelDataTypeFactory — Calcite type factory, created internally + ├── HepPlanner — Calcite heuristic planner + ├── RelOptCluster — Calcite planning cluster + ├── CalciteCatalogReader — resolves table metadata from schema + ├── FilterConverter — DSL query → LogicalFilter + ├── ProjectConverter — _source fields → LogicalProject + ├── SortConverter — sort + pagination → LogicalSort + ├── AggregateConverter — AggregationMetadata → LogicalAggregate (standalone) + └── AggregationTreeWalker — walks DSL agg tree → List +``` + +### Shared Context +``` +ConversionContext (simplified) + ├── SearchSourceBuilder — original DSL source + ├── RelOptCluster — Calcite planning cluster + └── RelOptTable — resolved table reference + Provides: getSearchSource(), getCluster(), getTable(), getRowType(), getRexBuilder() + Removed: indexName, indexSchema, capabilities, mutable AggregationMetadata +``` + +### Converter Classes +``` +AbstractDslConverter (abstract) + convert(RelNode input, ConversionContext ctx) + └── isApplicable(ctx) → doConvert(input, ctx) + No PipelinePhase or ordering — converters called directly in sequence + +FilterConverter (extends AbstractDslConverter) + └── skips MatchAllQueryBuilder; uses QueryRegistry → RexNode + +ProjectConverter (extends AbstractDslConverter) + └── _source field selection → LogicalProject + +SortConverter (extends AbstractDslConverter) + └── sort clauses + offset/fetch → LogicalSort + +AggregateConverter (standalone — NOT AbstractDslConverter) + convert(RelNode input, AggregationMetadata metadata) + └── LogicalAggregate.create(input, groupByBitSet, null, aggregateCalls) +``` + +### Query Translation Layer +``` +QueryRegistry (extends HandlerRegistry) + └── Map, QueryTranslator> + +QueryRegistryFactory.create(): + ├── TermQueryTranslator → EQUALS($field, literal) + └── MatchAllQueryTranslator → boolean TRUE +``` + +### Aggregation Translation Layer +``` +AggregationRegistry (extends HandlerRegistry) + └── Map, AggregationType> + +AggregationRegistryFactory.create(): + ├── AvgMetricTranslator → AVG() + ├── SumMetricTranslator → SUM() + ├── MinMetricTranslator → MIN() + ├── MaxMetricTranslator → MAX() + └── TermsBucketTranslator → GROUP BY field +``` + +### Aggregation Tree Walker +``` +AggregationTreeWalker.walk(aggs, rowType, typeFactory) + └── walkRecursive(aggs, currentGroupings, granularities) + ├── BucketShape → accumulate groupings, recurse into sub-aggs + └── MetricTranslator → add AggregateCall to builder + +Output: List + Each contains: ImmutableBitSet (GROUP BY), List +``` + +### Execution & Response Layer +``` +DslQueryPlanExecutor + └── wraps analytics-engine QueryPlanExecutor> + iterates QueryPlans, executes each RelNode, collects ExecutionResults + logs plan via relNode.explain() at INFO level + +SearchResponseBuilder (simplified) + └── static build(List results, long tookInMillis) + currently returns empty SearchHits (TODO: populate from results) +``` + +## Detailed Flow: TransportDslExecuteAction.doExecute() + +``` +1. SearchActionFilter.apply(action, request, listener, chain) + │ action == SearchAction.NAME? + │ yes → dispatch to DslExecuteAction via NodeClient + │ no → chain.proceed(action, request, listener) + │ + 2. TransportDslExecuteAction.doExecute(task, request, listener) + │ │ + │ 2a. resolveToSingleIndex(request) + │ │ → uses IndexNameExpressionResolver + │ │ → aliases/wildcards → single concrete index name + │ │ + │ 2b. new SearchSourceConverter(engineContext.getSchema()) + │ │ → receives SchemaPlus from analytics engine (via Guice) + │ │ → creates Calcite infrastructure internally: + │ │ RelDataTypeFactory, HepPlanner, RelOptCluster, CalciteCatalogReader + │ │ → creates converters: FilterConverter, ProjectConverter, + │ │ SortConverter, AggregateConverter + │ │ → creates AggregationTreeWalker with AggregationRegistryFactory.create() + │ │ + │ 2c. converter.convert(request.source(), indexName) + │ │ │ + │ │ 2c-i. catalogReader.getTable(List.of(indexName)) + │ │ │ → resolve table from analytics engine schema + │ │ │ + │ │ 2c-ii. Create ConversionContext(searchSource, cluster, table) + │ │ │ + │ │ 2c-iii. Shared base: + │ │ │ LogicalTableScan.create(cluster, table) + │ │ │ → filterConverter.convert(base, ctx) + │ │ │ Result: scan → filter (shared by both paths) + │ │ │ + │ │ 2c-iv. [HITS path] + │ │ │ projectConverter.convert(filtered, ctx) + │ │ │ → sortConverter.convert(projected, ctx) + │ │ │ Result: scan → filter → project → sort + │ │ │ + │ │ 2c-v. [AGGS path] + │ │ treeWalker.walk(aggs, rowType, typeFactory) + │ │ → List + │ │ For each metadata: + │ │ aggregateConverter.convert(filtered, metadata) + │ │ Result: scan → filter → aggregate (per granularity) + │ │ │ + │ │ 2c-vi. Return QueryPlans (HITS plan + N AGGREGATION plans) + │ │ + │ 2d. planExecutor.execute(plans) + │ │ → analytics-engine QueryPlanExecutor> + │ │ → iterates plans, executes each RelNode + │ │ → collects List + │ │ + │ 2e. SearchResponseBuilder.build(results, tookInMillis) + │ → assembles SearchResponse from execution results + │ → currently returns empty SearchHits (TODO) +``` + +## Key Differences from DslLogicalPlanPlugin Architecture + +| Aspect | Old (DslLogicalPlanPlugin) | New (DslQueryExecutorPlugin) | +|--------|---------------------------|------------------------------| +| Entry point | `DslConverterPlugin.convertDsl()` called by server | `SearchActionFilter` intercepts `_search` at action layer | +| Schema source | `IndexMappingClient` fetches mappings from cluster | `EngineContext.getSchema()` from analytics engine (Guice) | +| Converter orchestration | `ConversionPipeline` auto-sorts by `PipelinePhase` order | Converters called directly in sequence — no pipeline | +| Scan + Filter | Each pipeline starts with its own Scan → Filter | Shared base: single Scan → Filter, then branch for hits vs aggs | +| AggregateConverter | Extends `AbstractDslConverter`, uses `ConversionContext` | Standalone class, takes `(RelNode, AggregationMetadata)` directly | +| ConversionContext | Holds indexName, indexSchema, capabilities, mutable agg metadata | Simplified: only SearchSourceBuilder, cluster, table | +| Registry creation | Inline in `DslLogicalPlanService` constructor | Factory classes: `QueryRegistryFactory`, `AggregationRegistryFactory` | +| Execution | `DefaultQueryPlanExecutor` → `RelNodeExecutor` (placeholder) | `DslQueryPlanExecutor` wraps analytics-engine `QueryPlanExecutor` | +| Dependencies | Self-contained (uses OpenSearch Client for schema) | Depends on `analytics-engine` and `analytics-framework` (extendedPlugins) | + +## How to Add New Query Translators + +### Adding a new term-level query (e.g., `terms` → IN operator) + +1. Create `TermsQueryTranslator.java` in `org.opensearch.dsl.query`: +```java +public class TermsQueryTranslator implements QueryTranslator { + public Class getQueryType() { + return TermsQueryBuilder.class; + } + public RexNode convert(QueryBuilder query, ConversionContext ctx) { + // Build IN($field, val1, val2, ...) using SqlStdOperatorTable.IN + } +} +``` + +2. Register in `QueryRegistryFactory.create()`: +```java +public static QueryRegistry create() { + QueryRegistry registry = new QueryRegistry(); + registry.register(new TermQueryTranslator()); + registry.register(new MatchAllQueryTranslator()); + registry.register(new TermsQueryTranslator()); // ← add here + return registry; +} +``` + +### Adding a new single-value metric aggregation (e.g., `value_count`) + +1. Create `ValueCountMetricTranslator.java` in `org.opensearch.dsl.aggregation.metric`: +```java +public class ValueCountMetricTranslator extends AbstractMetricTranslator { + public Class getAggregationType() { ... } + protected SqlAggFunction getAggFunction() { return SqlStdOperatorTable.COUNT; } + protected String getFieldName(ValueCountAggregationBuilder agg) { return agg.field(); } + public InternalAggregation toInternalAggregation(String name, Object value) { + return new InternalValueCount(name, value == null ? 0 : ((Number)value).longValue(), Map.of()); + } +} +``` + +2. Register in `AggregationRegistryFactory.create()`: +```java +public static AggregationRegistry create() { + AggregationRegistry registry = new AggregationRegistry(); + registry.register(new AvgMetricTranslator()); + registry.register(new SumMetricTranslator()); + // ...existing registrations... + registry.register(new ValueCountMetricTranslator()); // ← add here + return registry; +} +``` + +### Adding a new bucket aggregation (e.g., `histogram`) + +1. Create `HistogramBucketShape.java` implementing `BucketShape`: +```java +public class HistogramBucketShape implements BucketShape { + public GroupingInfo getGrouping(HistogramAggregationBuilder agg) { + // Return ExpressionGrouping with FLOOR(field/interval)*interval + } + public BucketOrder getOrder(...) { ... } + public Collection getSubAggregations(...) { ... } + public InternalAggregation toBucketAggregation(..., List buckets) { + // Build InternalHistogram from bucket entries + } +} +``` + +2. Register in `AggregationRegistryFactory.create()`. + +## Key Design Patterns + +1. **Action Filter intercept** — `SearchActionFilter` sits at `Integer.MIN_VALUE` order, intercepting all `_search` requests before any other filter. Non-search actions pass through unchanged. This replaces the `DslConverterPlugin` interface that required server-side integration. + +2. **Direct converter sequencing** — Instead of a `ConversionPipeline` that auto-sorts by phase order, converters are called explicitly in a fixed sequence within `SearchSourceConverter.convert()`. This makes the flow easier to follow and removes the indirection of phase ordering. + +3. **Shared Scan→Filter base** — Both hits and aggregation paths share a single `LogicalTableScan → LogicalFilter` base. The old architecture created separate scan/filter nodes per pipeline. This avoids redundant work and makes the branching point explicit. + +4. **Schema from analytics engine** — The schema comes from `EngineContext.getSchema()` (injected via Guice from the `analytics-engine` plugin) rather than being fetched from cluster mappings via `IndexMappingClient`. This decouples the DSL converter from cluster state and leverages the engine's type system. + +5. **Registry factory pattern** — `QueryRegistryFactory.create()` and `AggregationRegistryFactory.create()` encapsulate registry construction. Adding a new translator is: implement the interface + add one line to the factory method. + +6. **Standalone AggregateConverter** — Unlike other converters that extend `AbstractDslConverter` and receive `ConversionContext`, `AggregateConverter` takes `(RelNode, AggregationMetadata)` directly. This reflects that aggregation conversion depends on walker output, not on the shared conversion context. + +7. **Granularity decomposition** — `AggregationTreeWalker` still produces one `AggregationMetadata` per unique GROUP BY key set. The converter loops over them, creating one `LogicalAggregate` per granularity from the shared filtered base. From 951c57160e3121ee8d13a9fa43d05f14fe3fbc61 Mon Sep 17 00:00:00 2001 From: Suresh N S Date: Mon, 11 May 2026 15:38:00 +0530 Subject: [PATCH 6/8] Removing the unintentional checkin of md file Signed-off-by: Suresh N S --- .../DSL_QUERY_EXECUTOR_ARCHITECTURE.md | 313 ------------------ 1 file changed, 313 deletions(-) delete mode 100644 sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md diff --git a/sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md b/sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md deleted file mode 100644 index 6eee9a29d271e..0000000000000 --- a/sandbox/plugins/dsl-query-executor/DSL_QUERY_EXECUTOR_ARCHITECTURE.md +++ /dev/null @@ -1,313 +0,0 @@ -# DSL Query Executor Plugin — Architecture & Flow Diagram - -> Architecture for the `dsl-query-executor` plugin on the `feature/datafusion` branch. -> This replaces the `DslLogicalPlanPlugin` / `DslConverterPlugin` approach with an -> action-filter intercept pattern and direct converter invocation. - -## Overall Architecture - -``` -┌──────────────────────────────────────────────────────────────────────────┐ -│ OpenSearch Server │ -│ │ -│ _search request (SearchAction) │ -│ │ │ -│ ▼ │ -│ SearchActionFilter (ActionFilter, order=MIN_VALUE) │ -│ │ intercepts SearchAction.NAME │ -│ │ dispatches via NodeClient │ -│ ▼ │ -│ TransportDslExecuteAction.doExecute() ← DslExecuteAction.INSTANCE │ -│ │ │ -│ ├─► resolveToSingleIndex(request) │ -│ │ └─► aliases/wildcards → single concrete index │ -│ │ │ -│ ├─► SearchSourceConverter(engineContext.getSchema()) │ -│ │ │ │ -│ │ ├─► catalogReader.getTable(indexName) │ -│ │ │ │ -│ │ ├─► Shared base: TableScan → Filter │ -│ │ │ │ -│ │ ├─► HITS path: │ -│ │ │ Project → Sort │ -│ │ │ │ -│ │ └─► AGGS path: │ -│ │ AggregationTreeWalker.walk() │ -│ │ → List │ -│ │ AggregateConverter.convert() (per metadata) │ -│ │ │ -│ ├─► QueryPlanExecutor.execute(plans) ← RelNode → rows │ -│ │ └─► analytics-engine execution │ -│ │ │ -│ └─► SearchResponseBuilder.build(results, tookInMillis) │ -│ └─► results → SearchResponse │ -│ │ -│ Dependencies: │ -│ analytics-engine — QueryPlanExecutor, EngineContext (via Guice) │ -│ analytics-framework — Calcite, shared SPI interfaces │ -│ │ -└──────────────────────────────────────────────────────────────────────────┘ -``` - -## Class Hierarchy - -### Plugin Layer -``` -DslQueryExecutorPlugin (extends Plugin, implements ActionPlugin) - ├── SearchActionFilter — intercepts _search, dispatches to DslExecuteAction - ├── TransportDslExecuteAction — orchestrates DSL → QueryPlans → SearchResponse - │ (Guice-injected: EngineContext, QueryPlanExecutor, ClusterService, - │ IndexNameExpressionResolver) - └── DslExecuteAction — action registration (INSTANCE singleton) -``` - -### Converter Layer -``` -SearchSourceConverter (replaces DslLogicalPlanService) - ├── SchemaPlus — from EngineContext (analytics engine), NOT IndexMappingClient - ├── RelDataTypeFactory — Calcite type factory, created internally - ├── HepPlanner — Calcite heuristic planner - ├── RelOptCluster — Calcite planning cluster - ├── CalciteCatalogReader — resolves table metadata from schema - ├── FilterConverter — DSL query → LogicalFilter - ├── ProjectConverter — _source fields → LogicalProject - ├── SortConverter — sort + pagination → LogicalSort - ├── AggregateConverter — AggregationMetadata → LogicalAggregate (standalone) - └── AggregationTreeWalker — walks DSL agg tree → List -``` - -### Shared Context -``` -ConversionContext (simplified) - ├── SearchSourceBuilder — original DSL source - ├── RelOptCluster — Calcite planning cluster - └── RelOptTable — resolved table reference - Provides: getSearchSource(), getCluster(), getTable(), getRowType(), getRexBuilder() - Removed: indexName, indexSchema, capabilities, mutable AggregationMetadata -``` - -### Converter Classes -``` -AbstractDslConverter (abstract) - convert(RelNode input, ConversionContext ctx) - └── isApplicable(ctx) → doConvert(input, ctx) - No PipelinePhase or ordering — converters called directly in sequence - -FilterConverter (extends AbstractDslConverter) - └── skips MatchAllQueryBuilder; uses QueryRegistry → RexNode - -ProjectConverter (extends AbstractDslConverter) - └── _source field selection → LogicalProject - -SortConverter (extends AbstractDslConverter) - └── sort clauses + offset/fetch → LogicalSort - -AggregateConverter (standalone — NOT AbstractDslConverter) - convert(RelNode input, AggregationMetadata metadata) - └── LogicalAggregate.create(input, groupByBitSet, null, aggregateCalls) -``` - -### Query Translation Layer -``` -QueryRegistry (extends HandlerRegistry) - └── Map, QueryTranslator> - -QueryRegistryFactory.create(): - ├── TermQueryTranslator → EQUALS($field, literal) - └── MatchAllQueryTranslator → boolean TRUE -``` - -### Aggregation Translation Layer -``` -AggregationRegistry (extends HandlerRegistry) - └── Map, AggregationType> - -AggregationRegistryFactory.create(): - ├── AvgMetricTranslator → AVG() - ├── SumMetricTranslator → SUM() - ├── MinMetricTranslator → MIN() - ├── MaxMetricTranslator → MAX() - └── TermsBucketTranslator → GROUP BY field -``` - -### Aggregation Tree Walker -``` -AggregationTreeWalker.walk(aggs, rowType, typeFactory) - └── walkRecursive(aggs, currentGroupings, granularities) - ├── BucketShape → accumulate groupings, recurse into sub-aggs - └── MetricTranslator → add AggregateCall to builder - -Output: List - Each contains: ImmutableBitSet (GROUP BY), List -``` - -### Execution & Response Layer -``` -DslQueryPlanExecutor - └── wraps analytics-engine QueryPlanExecutor> - iterates QueryPlans, executes each RelNode, collects ExecutionResults - logs plan via relNode.explain() at INFO level - -SearchResponseBuilder (simplified) - └── static build(List results, long tookInMillis) - currently returns empty SearchHits (TODO: populate from results) -``` - -## Detailed Flow: TransportDslExecuteAction.doExecute() - -``` -1. SearchActionFilter.apply(action, request, listener, chain) - │ action == SearchAction.NAME? - │ yes → dispatch to DslExecuteAction via NodeClient - │ no → chain.proceed(action, request, listener) - │ - 2. TransportDslExecuteAction.doExecute(task, request, listener) - │ │ - │ 2a. resolveToSingleIndex(request) - │ │ → uses IndexNameExpressionResolver - │ │ → aliases/wildcards → single concrete index name - │ │ - │ 2b. new SearchSourceConverter(engineContext.getSchema()) - │ │ → receives SchemaPlus from analytics engine (via Guice) - │ │ → creates Calcite infrastructure internally: - │ │ RelDataTypeFactory, HepPlanner, RelOptCluster, CalciteCatalogReader - │ │ → creates converters: FilterConverter, ProjectConverter, - │ │ SortConverter, AggregateConverter - │ │ → creates AggregationTreeWalker with AggregationRegistryFactory.create() - │ │ - │ 2c. converter.convert(request.source(), indexName) - │ │ │ - │ │ 2c-i. catalogReader.getTable(List.of(indexName)) - │ │ │ → resolve table from analytics engine schema - │ │ │ - │ │ 2c-ii. Create ConversionContext(searchSource, cluster, table) - │ │ │ - │ │ 2c-iii. Shared base: - │ │ │ LogicalTableScan.create(cluster, table) - │ │ │ → filterConverter.convert(base, ctx) - │ │ │ Result: scan → filter (shared by both paths) - │ │ │ - │ │ 2c-iv. [HITS path] - │ │ │ projectConverter.convert(filtered, ctx) - │ │ │ → sortConverter.convert(projected, ctx) - │ │ │ Result: scan → filter → project → sort - │ │ │ - │ │ 2c-v. [AGGS path] - │ │ treeWalker.walk(aggs, rowType, typeFactory) - │ │ → List - │ │ For each metadata: - │ │ aggregateConverter.convert(filtered, metadata) - │ │ Result: scan → filter → aggregate (per granularity) - │ │ │ - │ │ 2c-vi. Return QueryPlans (HITS plan + N AGGREGATION plans) - │ │ - │ 2d. planExecutor.execute(plans) - │ │ → analytics-engine QueryPlanExecutor> - │ │ → iterates plans, executes each RelNode - │ │ → collects List - │ │ - │ 2e. SearchResponseBuilder.build(results, tookInMillis) - │ → assembles SearchResponse from execution results - │ → currently returns empty SearchHits (TODO) -``` - -## Key Differences from DslLogicalPlanPlugin Architecture - -| Aspect | Old (DslLogicalPlanPlugin) | New (DslQueryExecutorPlugin) | -|--------|---------------------------|------------------------------| -| Entry point | `DslConverterPlugin.convertDsl()` called by server | `SearchActionFilter` intercepts `_search` at action layer | -| Schema source | `IndexMappingClient` fetches mappings from cluster | `EngineContext.getSchema()` from analytics engine (Guice) | -| Converter orchestration | `ConversionPipeline` auto-sorts by `PipelinePhase` order | Converters called directly in sequence — no pipeline | -| Scan + Filter | Each pipeline starts with its own Scan → Filter | Shared base: single Scan → Filter, then branch for hits vs aggs | -| AggregateConverter | Extends `AbstractDslConverter`, uses `ConversionContext` | Standalone class, takes `(RelNode, AggregationMetadata)` directly | -| ConversionContext | Holds indexName, indexSchema, capabilities, mutable agg metadata | Simplified: only SearchSourceBuilder, cluster, table | -| Registry creation | Inline in `DslLogicalPlanService` constructor | Factory classes: `QueryRegistryFactory`, `AggregationRegistryFactory` | -| Execution | `DefaultQueryPlanExecutor` → `RelNodeExecutor` (placeholder) | `DslQueryPlanExecutor` wraps analytics-engine `QueryPlanExecutor` | -| Dependencies | Self-contained (uses OpenSearch Client for schema) | Depends on `analytics-engine` and `analytics-framework` (extendedPlugins) | - -## How to Add New Query Translators - -### Adding a new term-level query (e.g., `terms` → IN operator) - -1. Create `TermsQueryTranslator.java` in `org.opensearch.dsl.query`: -```java -public class TermsQueryTranslator implements QueryTranslator { - public Class getQueryType() { - return TermsQueryBuilder.class; - } - public RexNode convert(QueryBuilder query, ConversionContext ctx) { - // Build IN($field, val1, val2, ...) using SqlStdOperatorTable.IN - } -} -``` - -2. Register in `QueryRegistryFactory.create()`: -```java -public static QueryRegistry create() { - QueryRegistry registry = new QueryRegistry(); - registry.register(new TermQueryTranslator()); - registry.register(new MatchAllQueryTranslator()); - registry.register(new TermsQueryTranslator()); // ← add here - return registry; -} -``` - -### Adding a new single-value metric aggregation (e.g., `value_count`) - -1. Create `ValueCountMetricTranslator.java` in `org.opensearch.dsl.aggregation.metric`: -```java -public class ValueCountMetricTranslator extends AbstractMetricTranslator { - public Class getAggregationType() { ... } - protected SqlAggFunction getAggFunction() { return SqlStdOperatorTable.COUNT; } - protected String getFieldName(ValueCountAggregationBuilder agg) { return agg.field(); } - public InternalAggregation toInternalAggregation(String name, Object value) { - return new InternalValueCount(name, value == null ? 0 : ((Number)value).longValue(), Map.of()); - } -} -``` - -2. Register in `AggregationRegistryFactory.create()`: -```java -public static AggregationRegistry create() { - AggregationRegistry registry = new AggregationRegistry(); - registry.register(new AvgMetricTranslator()); - registry.register(new SumMetricTranslator()); - // ...existing registrations... - registry.register(new ValueCountMetricTranslator()); // ← add here - return registry; -} -``` - -### Adding a new bucket aggregation (e.g., `histogram`) - -1. Create `HistogramBucketShape.java` implementing `BucketShape`: -```java -public class HistogramBucketShape implements BucketShape { - public GroupingInfo getGrouping(HistogramAggregationBuilder agg) { - // Return ExpressionGrouping with FLOOR(field/interval)*interval - } - public BucketOrder getOrder(...) { ... } - public Collection getSubAggregations(...) { ... } - public InternalAggregation toBucketAggregation(..., List buckets) { - // Build InternalHistogram from bucket entries - } -} -``` - -2. Register in `AggregationRegistryFactory.create()`. - -## Key Design Patterns - -1. **Action Filter intercept** — `SearchActionFilter` sits at `Integer.MIN_VALUE` order, intercepting all `_search` requests before any other filter. Non-search actions pass through unchanged. This replaces the `DslConverterPlugin` interface that required server-side integration. - -2. **Direct converter sequencing** — Instead of a `ConversionPipeline` that auto-sorts by phase order, converters are called explicitly in a fixed sequence within `SearchSourceConverter.convert()`. This makes the flow easier to follow and removes the indirection of phase ordering. - -3. **Shared Scan→Filter base** — Both hits and aggregation paths share a single `LogicalTableScan → LogicalFilter` base. The old architecture created separate scan/filter nodes per pipeline. This avoids redundant work and makes the branching point explicit. - -4. **Schema from analytics engine** — The schema comes from `EngineContext.getSchema()` (injected via Guice from the `analytics-engine` plugin) rather than being fetched from cluster mappings via `IndexMappingClient`. This decouples the DSL converter from cluster state and leverages the engine's type system. - -5. **Registry factory pattern** — `QueryRegistryFactory.create()` and `AggregationRegistryFactory.create()` encapsulate registry construction. Adding a new translator is: implement the interface + add one line to the factory method. - -6. **Standalone AggregateConverter** — Unlike other converters that extend `AbstractDslConverter` and receive `ConversionContext`, `AggregateConverter` takes `(RelNode, AggregationMetadata)` directly. This reflects that aggregation conversion depends on walker output, not on the shared conversion context. - -7. **Granularity decomposition** — `AggregationTreeWalker` still produces one `AggregationMetadata` per unique GROUP BY key set. The converter loops over them, creating one `LogicalAggregate` per granularity from the shared filtered base. From 53d5bf631d0d7377db0496695502f281a45d436b Mon Sep 17 00:00:00 2001 From: Suresh N S Date: Wed, 13 May 2026 14:16:56 +0530 Subject: [PATCH 7/8] Addressing comments from 2nd iteration Signed-off-by: Suresh N S --- .../opensearch/be/lucene/ConversionUtils.java | 3 +- ...ultiFieldExtractionOrderPropertyTests.java | 44 ++++++++----------- 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java index 1de8c762aa1bd..b5d6b435b2239 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java @@ -130,8 +130,7 @@ static List extractFieldsFromRelevanceMap(RexCall call, int operandIndex fields.add(FieldStorageInfo.resolve(fieldStorage, inputRef.getIndex()).getFieldName()); } } - } - if (operand instanceof RexInputRef inputRef) { + } else if (operand instanceof RexInputRef inputRef) { fields.add(FieldStorageInfo.resolve(fieldStorage, inputRef.getIndex()).getFieldName()); } if (fields.isEmpty()) { diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldExtractionOrderPropertyTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldExtractionOrderPropertyTests.java index 8f9b258d791d8..e3d891eadc590 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldExtractionOrderPropertyTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldExtractionOrderPropertyTests.java @@ -43,8 +43,6 @@ */ public class MultiFieldExtractionOrderPropertyTests extends OpenSearchTestCase { - private static final int ITERATIONS = 100; - private static final SqlFunction MULTI_MATCH_FUNCTION = new SqlFunction( "MULTI_MATCH", SqlKind.OTHER_FUNCTION, @@ -73,19 +71,17 @@ public void setUp() throws Exception { * in exact order. */ public void testExtractionPreservesOrderNestedMapLiteralPath() { - for (int iter = 0; iter < ITERATIONS; iter++) { - int fieldCount = randomIntBetween(1, 10); - List expectedFields = generateUniqueFieldNames(fieldCount); + int fieldCount = randomIntBetween(1, 10); + List expectedFields = generateUniqueFieldNames(fieldCount); - RexCall call = buildNestedMapLiteralCall(expectedFields); - List fieldStorage = List.of(); // not used in nested MAP literal path + RexCall call = buildNestedMapLiteralCall(expectedFields); + List fieldStorage = List.of(); // not used in nested MAP literal path - List result = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); + List result = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); - assertEquals("Iteration " + iter + ": field count mismatch", expectedFields.size(), result.size()); - for (int i = 0; i < expectedFields.size(); i++) { - assertEquals("Iteration " + iter + ": field at index " + i + " mismatch", expectedFields.get(i), result.get(i)); - } + assertEquals("field count mismatch", expectedFields.size(), result.size()); + for (int i = 0; i < expectedFields.size(); i++) { + assertEquals("field at index " + i + " mismatch", expectedFields.get(i), result.get(i)); } } @@ -98,23 +94,21 @@ public void testExtractionPreservesOrderNestedMapLiteralPath() { * in exact order. */ public void testExtractionPreservesOrderRexInputRefPath() { - for (int iter = 0; iter < ITERATIONS; iter++) { - int fieldCount = randomIntBetween(1, 10); - List expectedFields = generateUniqueFieldNames(fieldCount); + int fieldCount = randomIntBetween(1, 10); + List expectedFields = generateUniqueFieldNames(fieldCount); - List fieldStorage = new ArrayList<>(); - for (String fieldName : expectedFields) { - fieldStorage.add(new FieldStorageInfo(fieldName, "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false)); - } + List fieldStorage = new ArrayList<>(); + for (String fieldName : expectedFields) { + fieldStorage.add(new FieldStorageInfo(fieldName, "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false)); + } - RexCall call = buildRexInputRefCall(fieldCount); + RexCall call = buildRexInputRefCall(fieldCount); - List result = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); + List result = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); - assertEquals("Iteration " + iter + ": field count mismatch", expectedFields.size(), result.size()); - for (int i = 0; i < expectedFields.size(); i++) { - assertEquals("Iteration " + iter + ": field at index " + i + " mismatch", expectedFields.get(i), result.get(i)); - } + assertEquals("field count mismatch", expectedFields.size(), result.size()); + for (int i = 0; i < expectedFields.size(); i++) { + assertEquals("field at index " + i + " mismatch", expectedFields.get(i), result.get(i)); } } From 6ab7900e60d88f2caafd2343b7e987ed02e0d27f Mon Sep 17 00:00:00 2001 From: Suresh N S Date: Thu, 14 May 2026 09:03:35 +0530 Subject: [PATCH 8/8] Fixing a bug where MAP ordering can be random Updated code to remove the assumption on the ordering of the elements within the MAP structure Signed-off-by: Suresh N S --- .../opensearch/be/lucene/ConversionUtils.java | 64 ++++++++ .../be/lucene/QuerySerializerRegistry.java | 97 +++++++----- .../be/lucene/ConversionUtilsTests.java | 148 +++++++++++++++++- 3 files changed, 265 insertions(+), 44 deletions(-) diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java index b5d6b435b2239..bc3b2ea802588 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java @@ -31,6 +31,13 @@ */ final class ConversionUtils { + /** MAP key for single-field relevance operands. */ + static final String KEY_FIELD = "field"; + /** MAP key for multi-field relevance operands. */ + static final String KEY_FIELDS = "fields"; + /** MAP key for the query text operand. */ + static final String KEY_QUERY = "query"; + private ConversionUtils() {} /** @@ -79,6 +86,63 @@ static byte[] serializeQueryBuilder(QueryBuilder queryBuilder) { } } + /** + * Extracts the key string from a MAP_VALUE_CONSTRUCTOR operand: MAP('key', value). + * Returns null if the operand is not a MAP or the key is not a string literal. + */ + static String extractMapKey(RexCall call, int operandIndex) { + RexNode operand = call.getOperands().get(operandIndex); + if (operand instanceof RexCall mapCall && mapCall.getOperands().size() >= 2) { + RexNode key = mapCall.getOperands().get(0); + if (key instanceof RexLiteral literal) { + return literal.getValueAs(String.class); + } + } + return null; + } + + /** + * Extracted operands from a relevance function RexCall. + * @param fieldName single field name (null if not present or multi-field) + * @param fields multiple field names (null if not present) + * @param query the query string (null if not found) + */ + record RelevanceOperands(String fieldName, List fields, String query) { + } + + /** + * Extracts field/fields and query from a relevance function RexCall by MAP key lookup, + * with positional fallback for non-MAP operand structures (e.g. MATCH($ref, literal)). + * + * @param call the relevance function RexCall + * @param fieldStorage per-column storage metadata for resolving field names + * @return extracted operands + */ + static RelevanceOperands extractRelevanceOperands(RexCall call, List fieldStorage) { + String fieldName = null; + List fields = null; + String query = null; + + for (int i = 0; i < call.getOperands().size(); i++) { + String key = extractMapKey(call, i); + if (KEY_FIELD.equals(key)) { + fieldName = extractFieldFromRelevanceMap(call, i, fieldStorage); + } else if (KEY_FIELDS.equals(key)) { + fields = extractFieldsFromRelevanceMap(call, i, fieldStorage); + } else if (KEY_QUERY.equals(key)) { + query = extractStringFromRelevanceMap(call, i); + } + } + + // Fallback: positional extraction for non-MAP operand structures (e.g. MATCH($ref, literal)) + if (fieldName == null && fields == null && query == null && call.getOperands().size() >= 2) { + fieldName = extractFieldFromRelevanceMap(call, 0, fieldStorage); + query = extractStringFromRelevanceMap(call, 1); + } + + return new RelevanceOperands(fieldName, fields, query); + } + /** * Extracts multiple field names from a MAP_VALUE_CONSTRUCTOR operand * for multi-field full-text functions (multi_match, query_string, simple_query_string). diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java index b165e08e056d9..8d5b101635de0 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java @@ -29,7 +29,7 @@ * that the Lucene backend can deserialize at the data node. */ final class QuerySerializerRegistry { - + private static final Map SERIALIZERS = Map.ofEntries( Map.entry(ScalarFunction.MATCH, QuerySerializerRegistry::serializeMatch), Map.entry(ScalarFunction.MATCH_PHRASE, QuerySerializerRegistry::serializeMatchPhrase), @@ -50,73 +50,84 @@ static Map getSerializers() { // These methods are expected to grow significantly as optional parameters are added. private static byte[] serializeMatch(RexCall call, List fieldStorage) { - String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); - String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params - // (operator, analyzer, fuzziness, boost) and apply them to the QueryBuilder. - MatchQueryBuilder queryBuilder = new MatchQueryBuilder(fieldName, fieldValue); + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.fieldName() == null || operands.query() == null) { + throw new IllegalArgumentException("match requires 'field' and 'query' parameters, got: " + call); + } + // TODO: extract optional params (operator, analyzer, fuzziness, boost) + MatchQueryBuilder queryBuilder = new MatchQueryBuilder(operands.fieldName(), operands.query()); return ConversionUtils.serializeQueryBuilder(queryBuilder); } private static byte[] serializeMatchPhrase(RexCall call, List fieldStorage) { - String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); - String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params - // (slop, analyzer, zero_terms_query) and apply them to the QueryBuilder. - MatchPhraseQueryBuilder queryBuilder = new MatchPhraseQueryBuilder(fieldName, fieldValue); + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.fieldName() == null || operands.query() == null) { + throw new IllegalArgumentException("match_phrase requires 'field' and 'query' parameters, got: " + call); + } + // TODO: extract optional params (slop, analyzer, zero_terms_query) + MatchPhraseQueryBuilder queryBuilder = new MatchPhraseQueryBuilder(operands.fieldName(), operands.query()); return ConversionUtils.serializeQueryBuilder(queryBuilder); } private static byte[] serializeMatchBoolPrefix(RexCall call, List fieldStorage) { - String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); - String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params - // (analyzer, fuzziness, operator, minimum_should_match) and apply them to the QueryBuilder. - MatchBoolPrefixQueryBuilder queryBuilder = new MatchBoolPrefixQueryBuilder(fieldName, fieldValue); + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.fieldName() == null || operands.query() == null) { + throw new IllegalArgumentException("match_bool_prefix requires 'field' and 'query' parameters, got: " + call); + } + // TODO: extract optional params (analyzer, fuzziness, operator, minimum_should_match) + MatchBoolPrefixQueryBuilder queryBuilder = new MatchBoolPrefixQueryBuilder(operands.fieldName(), operands.query()); return ConversionUtils.serializeQueryBuilder(queryBuilder); } private static byte[] serializeMatchPhrasePrefix(RexCall call, List fieldStorage) { - String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); - String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params - // (slop, analyzer, max_expansions, zero_terms_query) and apply them to the QueryBuilder. - MatchPhrasePrefixQueryBuilder queryBuilder = new MatchPhrasePrefixQueryBuilder(fieldName, fieldValue); + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.fieldName() == null || operands.query() == null) { + throw new IllegalArgumentException("match_phrase_prefix requires 'field' and 'query' parameters, got: " + call); + } + // TODO: extract optional params (slop, analyzer, max_expansions, zero_terms_query) + MatchPhrasePrefixQueryBuilder queryBuilder = new MatchPhrasePrefixQueryBuilder(operands.fieldName(), operands.query()); return ConversionUtils.serializeQueryBuilder(queryBuilder); } private static byte[] serializeMultiMatch(RexCall call, List fieldStorage) { - List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); - String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder - // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params - // (type, operator, analyzer, fuzziness, minimum_should_match) and apply them to the QueryBuilder. - MultiMatchQueryBuilder queryBuilder = new MultiMatchQueryBuilder(fieldValue, fields.toArray(String[]::new)); + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.query() == null) { + throw new IllegalArgumentException("multi_match requires a 'query' parameter, got: " + call); + } + // TODO: extract per-field boost values and optional params (type, operator, analyzer, fuzziness) + List fields = operands.fields(); + MultiMatchQueryBuilder queryBuilder = fields != null + ? new MultiMatchQueryBuilder(operands.query(), fields.toArray(String[]::new)) + : new MultiMatchQueryBuilder(operands.query()); return ConversionUtils.serializeQueryBuilder(queryBuilder); } private static byte[] serializeQueryString(RexCall call, List fieldStorage) { - List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); - String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder - // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params - // (default_operator, analyzer, allow_leading_wildcard) and apply them to the QueryBuilder. - QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder(fieldValue); - for (String field : fields) { - queryBuilder.field(field); + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.query() == null) { + throw new IllegalArgumentException("query_string requires a 'query' parameter, got: " + call); + } + // TODO: extract optional params (default_operator, analyzer, allow_leading_wildcard) + QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder(operands.query()); + if (operands.fields() != null) { + for (String field : operands.fields()) { + queryBuilder.field(field); + } } return ConversionUtils.serializeQueryBuilder(queryBuilder); } private static byte[] serializeSimpleQueryString(RexCall call, List fieldStorage) { - List fields = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); - String fieldValue = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: extract per-field boost values from operand 0 and pass to QueryBuilder - // TODO: Use ConversionUtils.extractOptionalParams(call, 2) to extract optional params - // (default_operator, analyzer, flags, minimum_should_match) and apply them to the QueryBuilder. - SimpleQueryStringBuilder queryBuilder = new SimpleQueryStringBuilder(fieldValue); - for (String field : fields) { - queryBuilder.field(field); + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.query() == null) { + throw new IllegalArgumentException("simple_query_string requires a 'query' parameter, got: " + call); + } + // TODO: extract optional params (default_operator, analyzer, flags, minimum_should_match) + SimpleQueryStringBuilder queryBuilder = new SimpleQueryStringBuilder(operands.query()); + if (operands.fields() != null) { + for (String field : operands.fields()) { + queryBuilder.field(field); + } } return ConversionUtils.serializeQueryBuilder(queryBuilder); } diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java index 7d56775d31b33..073cd4f5d4b93 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java @@ -29,7 +29,7 @@ import java.util.List; /** - * Unit tests for {@link ConversionUtils#extractFieldsFromRelevanceMap}. + * Unit tests for {@link ConversionUtils}. */ public class ConversionUtilsTests extends OpenSearchTestCase { @@ -42,6 +42,24 @@ public class ConversionUtilsTests extends OpenSearchTestCase { SqlFunctionCategory.USER_DEFINED_FUNCTION ); + private static final SqlFunction MATCH_FUNCTION = new SqlFunction( + "MATCH", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private static final SqlFunction QUERY_STRING_FUNCTION = new SqlFunction( + "QUERY_STRING", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + private RelDataTypeFactory typeFactory; private RexBuilder rexBuilder; @@ -205,4 +223,132 @@ public void testNoFieldsThrowsIllegalArgumentException() { exception.getMessage().contains("Cannot extract field names from operand 0") ); } + + // --- Tests for extractMapKey --- + + /** + * Tests that extractMapKey returns the key string from a MAP_VALUE_CONSTRUCTOR operand. + * Structure: MATCH(MAP('field', $0), MAP('query', 'hello')) + * extractMapKey(call, 0) should return "field". + */ + public void testExtractMapKey_returnsKeyFromMapCall() { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + RexNode fieldMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("field"), + rexBuilder.makeInputRef(varcharType, 0) + ); + + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("hello") + ); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MATCH_FUNCTION, fieldMap, queryMap); + + String key = ConversionUtils.extractMapKey(topCall, 0); + assertEquals("field", key); + } + + /** + * Tests that extractMapKey returns null when the operand is a plain RexInputRef (not a MAP). + */ + public void testExtractMapKey_returnsNullForNonMapOperand() { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + // Direct RexInputRef operand (no MAP wrapper) + RexNode inputRef = rexBuilder.makeInputRef(varcharType, 0); + RexNode literal = rexBuilder.makeLiteral("hello"); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MATCH_FUNCTION, inputRef, literal); + + // RexInputRef is not a RexCall, so extractMapKey should return null + assertNull(ConversionUtils.extractMapKey(topCall, 0)); + // RexLiteral is not a RexCall, so extractMapKey should return null + assertNull(ConversionUtils.extractMapKey(topCall, 1)); + } + + // --- Tests for extractRelevanceOperands --- + + /** + * Tests that extractRelevanceOperands correctly extracts field and query from MAP-wrapped operands. + * Structure: MATCH(MAP('field', $0), MAP('query', 'hello')) + * Should resolve fieldName from fieldStorage and query from the literal. + */ + public void testExtractRelevanceOperands_mapWrappedFieldAndQuery() { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + RexNode fieldMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("field"), + rexBuilder.makeInputRef(varcharType, 0) + ); + + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("hello") + ); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MATCH_FUNCTION, fieldMap, queryMap); + + List fieldStorage = List.of( + new FieldStorageInfo("status", "keyword", FieldType.KEYWORD, List.of(), List.of("lucene"), List.of(), false) + ); + + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(topCall, fieldStorage); + + assertEquals("status", operands.fieldName()); + assertNull(operands.fields()); + assertEquals("hello", operands.query()); + } + + /** + * Tests that extractRelevanceOperands handles a query-only MAP structure (no field MAP). + * Structure: query_string(MAP('query', 'brewing')) + * Should return fieldName=null, fields=null, query="brewing". + */ + public void testExtractRelevanceOperands_queryOnlyNoField() { + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("brewing") + ); + + RexCall topCall = (RexCall) rexBuilder.makeCall(QUERY_STRING_FUNCTION, queryMap); + + List fieldStorage = List.of(); + + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(topCall, fieldStorage); + + assertNull(operands.fieldName()); + assertNull(operands.fields()); + assertEquals("brewing", operands.query()); + } + + /** + * Tests that extractRelevanceOperands falls back to positional extraction when operands + * are not MAP-wrapped (e.g. MATCH($0, 'hello') with direct RexInputRef and RexLiteral). + */ + public void testExtractRelevanceOperands_positionalFallback() { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + // Direct operands without MAP wrappers + RexNode inputRef = rexBuilder.makeInputRef(varcharType, 0); + RexNode literal = rexBuilder.makeLiteral("hello"); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MATCH_FUNCTION, inputRef, literal); + + List fieldStorage = List.of( + new FieldStorageInfo("status", "keyword", FieldType.KEYWORD, List.of(), List.of("lucene"), List.of(), false) + ); + + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(topCall, fieldStorage); + + assertEquals("status", operands.fieldName()); + assertNull(operands.fields()); + assertEquals("hello", operands.query()); + } }