The bridge has two responsibilities: + *
Arrow data never crosses the JNI boundary into the JVM heap.
+ * Consumers read from the native stream via Arrow Flight or
+ * direct native-memory access using the returned handle.
+ *
+ * @param Called once per shard before any {@link #execute} calls. Back-end
+ * plugins that need shard-level resources (Lucene readers, query
+ * contexts, etc.) should acquire and cache them here for reuse
+ * across multiple {@code execute} calls within the same shard.
+ *
+ * The default implementation is a no-op, suitable for engines
+ * that do not need shard-level initialization (e.g., native engines
+ * with their own resource management).
+ *
+ * @param context shard execution context provided by the analytics engine
+ */
+ default void initialize(ShardExecutionContext context) {}
+
+ /**
+ * Converts a logical plan fragment into the native engine's serialised
+ * format.
+ *
+ * @param fragment the logical plan subtree to serialise
+ * @return the serialised plan in the engine's wire format
+ */
+ Fragment convertFragment(LogicalPlan fragment);
+
+ /**
+ * Submits the serialised plan to the native engine for execution and
+ * returns an opaque handle to the result stream.
+ *
+ * The returned handle is a pointer into native memory (e.g., a
+ * {@code long} address of a Rust {@code RecordBatchStream}). The
+ * caller must eventually close the stream through a corresponding
+ * native call to avoid leaking resources.
+ *
+ * @param fragment the serialised plan produced by {@link #convertFragment}
+ * @return an opaque handle to the native result stream
+ */
+ Stream execute(Fragment fragment);
+
+ /**
+ * Releases shard-level resources acquired during {@link #initialize}.
+ *
+ * Called once after all {@code execute} calls for a shard are
+ * complete. Back-end plugins should release any cached readers,
+ * searchers, or contexts here.
+ *
+ * The default implementation is a no-op.
+ */
+ default void close() {}
+}
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ShardExecutionContext.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ShardExecutionContext.java
new file mode 100644
index 0000000000000..1b00daf881f8f
--- /dev/null
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ShardExecutionContext.java
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.backend;
+
+/**
+ * Opaque context representing a shard-level execution environment.
+ *
+ * The analytics engine creates an instance of this context for each
+ * shard being queried and passes it to back-end plugins via
+ * {@link EngineBridge#initialize(ShardExecutionContext)}. Back-end
+ * plugins that need shard-level resources (e.g., Lucene readers,
+ * query contexts) downcast to the concrete implementation provided
+ * by the engine.
+ *
+ * This interface lives in {@code analytics-framework} (a {@code libs}
+ * module) and intentionally does not reference server types like
+ * {@code IndexShard} or {@code QueryShardContext}. Concrete
+ * implementations that wrap those types live in the engine plugin.
+ *
+ * @opensearch.internal
+ */
+public interface ShardExecutionContext {
+
+ /**
+ * Returns the shard identifier (index name + shard number) for
+ * logging and diagnostics.
+ */
+ String shardId();
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/DESIGN.md b/sandbox/plugins/analytics-backend-lucene/DESIGN.md
new file mode 100644
index 0000000000000..9810c43072a6b
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/DESIGN.md
@@ -0,0 +1,493 @@
+# Lucene Backend Plugin — Design
+
+## Scope
+
+This document covers the design of the Lucene Backend Plugin (`analytics-backend-lucene`) for the analytics query engine. In scope:
+
+- Plugin registration and SPI integration with the analytics engine
+- Planner-side predicate identification, validation, and conversion (RexNode → QueryBuilder → byte[])
+- Executor-side shard initialization, Lucene query execution, and BitSet result packaging
+- Segment-level and batch-level execution aligned with DataFusion's Parquet processing
+- Extensible `PredicateHandler` registry for adding new predicate types
+- Weight caching, resource lifecycle management, and fail-fast validation
+- Custom `LUCENE_DELEGATED` operator for in-place plan annotation (design only, not yet implemented)
+
+Out of scope (upstream dependencies):
+
+- `DefaultPlanExecutor` predicate splitting and lifecycle orchestration
+- `LUCENE_DELEGATED` custom SqlOperator implementation
+- BitSet handoff mechanism to DataFusion
+- Multi-shard coordination and fan-out from coordinator
+- Wiring `ClusterService`/`IndicesService` into the executor
+
+## Overview
+
+DataFusion handles columnar scans, numeric filtering, and aggregations over Parquet DocValues. Lucene handles text-search predicates (EQUALS, LIKE on keyword fields) against inverted indices. The coordinator identifies Lucene-compatible predicates and serializes them. The data node executes them against local shards and returns a BitSet for DataFusion to intersect with its scan results.
+
+---
+
+## Planner vs Executor
+
+The Lucene Backend Plugin participates in two distinct phases with different roles:
+
+- **Planner phase** (coordinator): identifies Lucene-eligible predicates, converts RexNode → QueryBuilder → serialized bytes. No shard context. Pure logical transformation. This is `convertFragment`.
+- **Executor phase** (data node): receives serialized bytes, initializes shard resources, executes Lucene queries, returns BitSet. Requires IndexShard, Engine.Searcher, QueryShardContext. This is `initialize` / `execute` / `close`.
+
+Both phases use `LuceneEngineBridge` but through different methods:
+
+| Phase | Role | Bridge Method | Runs On | Needs Shard Context |
+|----------|-----------------------------------------------|---------------------|-------------|---------------------|
+| Planner | Identify eligible predicates | `canHandle()` | Coordinator | No (uses cluster state + Calcite types) |
+| Planner | Convert RexNode → QueryBuilder → byte[] | `convertFragment()` | Coordinator | No |
+| Executor | Acquire shard resources (searcher, QSC) | `initialize()` | Data Node | Yes |
+| Executor | Deserialize → Lucene Query → search → BitSet | `execute()` | Data Node | Yes (cached) |
+| Executor | Release searcher, allocator | `close()` | Data Node | Yes (cleanup) |
+
+### Planner Validation: Fail Fast
+
+The planner runs on the coordinator which has access to `ClusterState` → `IndexMetadata` → `MappingMetadata`. This means the planner can validate at planning time — before any bytes are sent to data nodes:
+
+- **Field existence**: does the field exist in the index mapping?
+- **Field type**: is it `keyword` or `text` (has Lucene inverted index) vs `integer`/`long`/`date` (no inverted index for text search)?
+- **Index existence**: does the target index exist at all?
+
+Currently `canHandle` checks the Calcite `SqlTypeName` (VARCHAR vs INTEGER) as a proxy. This catches obvious mismatches (numeric field routed to Lucene) but can't distinguish `keyword` from `text` from `ip` since all map to VARCHAR.
+
+For full fail-fast validation, the planner should check the actual OpenSearch field type from cluster state before routing to LBP. The recommended approach:
+
+```
+Planner validation chain (coordinator):
+ 1. Planner checks ClusterState → MappingMetadata → field type
+ → "verb" is "keyword" → has inverted index → Lucene-eligible
+ → "status" is "integer" → no inverted index → DataFusion only
+ 2. PredicateHandlerRegistry.canHandle(call, rowType)
+ → secondary check: operand shape, Calcite type
+ 3. luceneBridge.convertFragment(...)
+ → conversion only happens after both checks pass
+```
+
+This ensures errors like "field doesn't exist" or "field is numeric, not text" are caught at planning time on the coordinator, not at execution time on the data node.
+
+**Current state of upstream access:** `AnalyticsPlugin` has `ClusterService` and passes it to `DefaultEngineContext`. However, `DefaultPlanExecutor` does not currently receive `ClusterService` or `EngineContext` — it only gets `List Registers as an {@link AnalyticsSearchBackendPlugin} providing Lucene-based
+ * predicate execution. Supported operators are derived from {@link PredicateHandlerRegistry}.
+ */
+public class LuceneBackendPlugin extends Plugin implements AnalyticsSearchBackendPlugin {
+
+ @Override
+ public String name() {
+ return "lucene";
+ }
+
+ @Override
+ public SearchExecEngine
- * This class is stateless with respect to active queries
- *
- * @opensearch.experimental
- */
-@ExperimentalApi
-public class LuceneEngineSearcher implements EngineSearcher
+ * Planner side ({@link #convertFragment}): RexNode → QueryBuilder → byte[].
+ * Executor side: delegates Lucene query execution to {@link LuceneIndexFilterProvider}
+ * for Weight creation, per-segment scoring, and doc range collection.
+ * Packages results as Arrow BitVector.
+ */
+public class LuceneFilterExecutor implements EngineBridge Bridges the upstream execution interface to the Lucene search path:
+ * This is the single search execution path for Lucene in the analytics engine.
+ * {@link LuceneIndexFilterProvider} handles per-segment collection;
+ * this class orchestrates them through {@link LuceneFilterExecutor}.
+ */
+public class LuceneSearchExecEngine implements SearchExecEngine Each handler bundles three concerns that must stay in sync:
+ * To add a new Lucene predicate type (e.g., range, regex, fuzzy):
+ * implement this interface and register it in {@link PredicateHandlerRegistry}.
+ */
+public interface PredicateHandler {
+
+ /** The Calcite {@link SqlKind} this handler processes (e.g., EQUALS, LIKE). */
+ SqlKind sqlKind();
+
+ /** The Calcite {@link SqlOperator} to advertise in the plugin's operator table. */
+ SqlOperator sqlOperator();
+
+ /**
+ * Converts a {@link RexCall} of the matching {@link #sqlKind()} into a {@link QueryBuilder}.
+ *
+ * @param call the Calcite function call to convert
+ * @param inputRowType the row type of the input relation (for resolving field names)
+ * @param mapperService the index mapper service for field type validation, or null if unavailable
+ * @return the corresponding QueryBuilder
+ * @throws IllegalArgumentException if the call has unsupported operand types
+ */
+ QueryBuilder convert(RexCall call, RelDataType inputRowType, MapperService mapperService);
+
+ /**
+ * Returns true if this handler can process the given {@link RexCall}.
+ * Used by the coordinator to validate predicates before conversion.
+ *
+ * The default implementation returns true for any call matching {@link #sqlKind()}.
+ * Handlers that need field-type-aware validation (e.g., only keyword fields)
+ * should override this.
+ *
+ * @param call the Calcite function call to check
+ * @param inputRowType the row type of the input relation
+ * @param mapperService the index mapper service for field type validation, or null if unavailable
+ * @return true if this handler can convert the call
+ */
+ default boolean canHandle(RexCall call, RelDataType inputRowType, MapperService mapperService) {
+ return call.getKind() == sqlKind();
+ }
+
+ /**
+ * Returns the {@link NamedWriteableRegistry} entries needed to serialize/deserialize
+ * the {@link QueryBuilder} types this handler produces.
+ */
+ List This is the single place to register new Lucene predicate types.
+ * All downstream components ({@link RexToQueryBuilderConverter},
+ * {@link QueryBuilderSerializer}, {@link LuceneBackendPlugin}) derive
+ * their behavior from this registry.
+ */
+public final class PredicateHandlerRegistry {
+
+ private static final List The registry entries are derived from {@link PredicateHandlerRegistry},
+ * so adding a new predicate handler automatically makes its QueryBuilder
+ * types serializable — no changes needed here.
+ */
+public class QueryBuilderSerializer {
+
+ private static final NamedWriteableRegistry REGISTRY = new NamedWriteableRegistry(
+ PredicateHandlerRegistry.allNamedWriteableEntries()
+ );
+
+ /**
+ * Serializes a {@link QueryBuilder} into a byte array.
+ *
+ * @param queryBuilder the query builder to serialize
+ * @return the serialized byte array
+ * @throws IllegalArgumentException if serialization fails
+ */
+ public static byte[] serialize(QueryBuilder queryBuilder) {
+ try (BytesStreamOutput out = new BytesStreamOutput()) {
+ out.writeNamedWriteable(queryBuilder);
+ return BytesReference.toBytes(out.bytes());
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Failed to serialize QueryBuilder: " + e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Deserializes a byte array back into a {@link QueryBuilder}.
+ *
+ * @param data the byte array to deserialize
+ * @return the deserialized QueryBuilder
+ * @throws IllegalArgumentException if the byte array is null, empty, or corrupted
+ */
+ public static QueryBuilder deserialize(byte[] data) {
+ if (data == null || data.length == 0) {
+ throw new IllegalArgumentException("Fragment byte array must not be null or empty");
+ }
+ try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(data), REGISTRY)) {
+ return in.readNamedWriteable(QueryBuilder.class);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Failed to deserialize QueryBuilder: " + e.getMessage(), e);
+ }
+ }
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/predicate/RexToQueryBuilderConverter.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/predicate/RexToQueryBuilderConverter.java
new file mode 100644
index 0000000000000..4e41091f12fa5
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/predicate/RexToQueryBuilderConverter.java
@@ -0,0 +1,73 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene.predicate;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexVisitorImpl;
+import org.apache.calcite.sql.SqlKind;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.query.QueryBuilder;
+
+/**
+ * Converts Calcite {@link RexNode} expressions into OpenSearch {@link QueryBuilder} instances.
+ *
+ * Delegates to {@link PredicateHandler} implementations registered in
+ * {@link PredicateHandlerRegistry}. When a {@link MapperService} is available
+ * (data node, post-initialize), field type validation uses actual OpenSearch
+ * mappings. When null (coordinator), falls back to Calcite type checks.
+ */
+public class RexToQueryBuilderConverter extends RexVisitorImpl Back-end plugins that need shard-level resources (e.g., the Lucene
+ * backend) downcast the opaque {@link ShardExecutionContext} to this type
+ * to access {@link IndexShard} for acquiring searchers and
+ * {@link IndexService} for creating {@link QueryShardContext} instances.
+ *
+ * @opensearch.internal
+ */
+public class DefaultShardExecutionContext implements ShardExecutionContext {
+
+ private final IndexShard indexShard;
+ private final IndexService indexService;
+
+ public DefaultShardExecutionContext(IndexShard indexShard, IndexService indexService) {
+ this.indexShard = indexShard;
+ this.indexService = indexService;
+ }
+
+ @Override
+ public String shardId() {
+ return indexShard.shardId().toString();
+ }
+
+ /** Returns the {@link IndexShard} for acquiring searchers. */
+ public IndexShard indexShard() {
+ return indexShard;
+ }
+
+ /** Returns the {@link IndexService} for creating query contexts. */
+ public IndexService indexService() {
+ return indexService;
+ }
+
+ /**
+ * Creates a {@link QueryShardContext} bound to the given searcher.
+ *
+ * @param searcher the index searcher to bind
+ * @return a new QueryShardContext
+ */
+ public QueryShardContext createQueryShardContext(IndexSearcher searcher) {
+ return indexService.newQueryShardContext(
+ indexShard.shardId().id(),
+ searcher,
+ System::currentTimeMillis,
+ null
+ );
+ }
+}
+ *
+ *
+ *
+ *
+ *
+ *