diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java index c79c63a4d63c5..3bd5ca316f6cb 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -142,6 +142,10 @@ public class DataFusionFragmentConvertor implements FragmentConvertor { FunctionMappings.s(SqlLibraryOperators.CONCAT_WS, "concat_ws"), FunctionMappings.s(SqlLibraryOperators.ILIKE, "ilike"), FunctionMappings.s(SqlLibraryOperators.DATE_PART, "date_part"), + // Engine-output cast rewrite target — see DatetimeOutputCastRewriter (issue #5420). + // Routes Calcite's TO_CHAR call to DataFusion's native `to_char` so PPL's + // documented space-separator timestamp output is preserved on the AE path. + FunctionMappings.s(SqlLibraryOperators.TO_CHAR, "to_char"), FunctionMappings.s(ConvertTzAdapter.LOCAL_CONVERT_TZ_OP, "convert_tz"), FunctionMappings.s(UnixTimestampAdapter.LOCAL_TO_UNIXTIME_OP, "to_unixtime"), // Niladic ops from DateTimeAdapters — each maps 1:1 to a DF builtin. @@ -323,6 +327,10 @@ private byte[] convertToSubstrait(RelNode fragment) { // with "Unable to convert the type NULL". The widening only changes literal type // tags; semantics and field names (used by Plan.Root.names) are unchanged. RelNode preprocessed = UntypedNullPreprocessor.rewrite(fragment); + // Rewrite DatetimeOutputCastRule's CAST( AS VARCHAR) to to_char(...) so + // DataFusion emits PPL's space-separator timestamp format instead of Arrow's ISO-T. + // See issue #5420. + preprocessed = DatetimeOutputCastRewriter.rewrite(preprocessed); RelRoot root = RelRoot.of(preprocessed, SqlKind.SELECT); SubstraitRelVisitor visitor = createVisitor(preprocessed); Rel substraitRel; @@ -364,6 +372,8 @@ private Rel convertStandalone(RelNode operator) { // wrapper conversion is just as susceptible to a SqlTypeName.NULL literal lurking in // a CASE call attached on top of an inner plan. RelNode preprocessed = UntypedNullPreprocessor.rewrite(operator); + // Same rationale as convertToSubstrait — issue #5420. + preprocessed = DatetimeOutputCastRewriter.rewrite(preprocessed); SubstraitRelVisitor visitor = createVisitor(preprocessed); return visitor.apply(preprocessed); } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatetimeOutputCastRewriter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatetimeOutputCastRewriter.java new file mode 100644 index 0000000000000..d04b92a1b12e9 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatetimeOutputCastRewriter.java @@ -0,0 +1,183 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.type.SqlTypeName; + +import java.util.ArrayList; +import java.util.List; + +/** + * Pre-isthmus pass that rewrites the engine-output cast emitted by + * {@code DatetimeOutputCastRule} from {@code CAST( AS VARCHAR)} to + * {@code to_char(, '%Y-%m-%d %H:%M:%S')} so the DataFusion runtime + * emits PPL's documented space-separator format instead of Arrow's ISO-8601 + * {@code T}-separator. + * + *

Issue: opensearch-project/sql#5420. + * + *

Background: {@code DatetimeOutputCastRule} (api/spec/datetime) wraps every + * datetime field at the OUTERMOST {@link LogicalProject} in + * {@code CAST(... AS VARCHAR)} so the unified planner never has to know which + * backend serializes datetimes. Calcite's reference planner emits ANSI + * {@code "2024-01-15 12:00:00"}; DataFusion's Arrow CAST kernel emits + * {@code "2024-01-15T12:00:00"}. The session config + * {@code datafusion.format.timestamp_format} only affects the CLI display + * pipeline, not the Arrow cast kernel — verified by the issue's reporter. + * + *

Scope is intentionally narrow: + *

    + *
  • Only the output {@link Project} is inspected. {@code DatetimeOutputCastRule} + * wraps the input in exactly one final {@link Project}; the unified planner + * may then wrap that Project in a single {@link Sort} (system query-size + * limit). The rewriter therefore inspects either the root {@link Project} + * or, when the root is a {@link Sort}, the {@link Project} sitting + * directly beneath it. Any deeper {@link Project} (whether user-authored + * or optimizer-generated) carries expressions that must round-trip + * verbatim.
    • + *
  • Only direct project slots — {@code project.getProjects().get(i)} — are + * inspected. Nested casts inside {@code CASE}/{@code COALESCE}/UDF args + * were authored by the user query and must round-trip verbatim.
    • + *
  • Only {@code CAST(... AS VARCHAR)} matches the rule's output shape; + * {@code CAST(... AS CHAR(n))} is user-authored and has different + * length/padding semantics that {@code to_char} does not preserve.
    • + *
  • Only {@link SqlTypeName#TIMESTAMP} sources are rewritten. PPL's + * {@code DATE} (no clock) and {@code TIME} (no calendar) cast cleanly + * through Arrow already, and {@link SqlTypeName#TIMESTAMP_WITH_LOCAL_TIME_ZONE} + * depends on the DataFusion session timezone — emitting a literal space + * format there could silently lie about the instant. Defer until a + * concrete failing case lands.
    • + *
  • The rewriter assumes {@code DatetimeUdtNormalizeRule} (also a + * postAnalysisRule, ordered before {@code DatetimeOutputCastRule}) has + * already normalized {@code ExprUDT.EXPR_TIMESTAMP} → standard + * {@code SqlTypeName.TIMESTAMP}, so we only need to match the standard + * SqlTypeName here.
    • + *
+ * + *

The Substrait emit path is wired in {@code DataFusionFragmentConvertor}: + * {@code SqlLibraryOperators.TO_CHAR} is mapped to the Substrait extension + * name {@code to_char} declared in {@code opensearch_scalar_functions.yaml}, + * which DataFusion resolves to its native {@code to_char} scalar function. + * + * @opensearch.internal + */ +final class DatetimeOutputCastRewriter { + + /** + * PPL's documented timestamp output format (space separator). Mirrors the + * format used by Calcite's reference planner so the analytics-engine path + * matches per-row output exactly. + */ + static final String PPL_TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S"; + + private DatetimeOutputCastRewriter() {} + + /** + * Rewrite the engine-output {@code CAST( AS VARCHAR)} slots in the + * output {@link Project}. Returns {@code root} unchanged when the output + * Project cannot be located (e.g. raw scan / aggregate fragment) or when no + * slot matches. + * + *

The output Project is located in one of two shapes: + *

    + *
  1. {@code root} is itself a {@link Project} — the rule's output sits at + * the root.
    2. + *
  2. {@code root} is a {@link Sort} (the unified planner's + * {@code LogicalSystemLimit} system query-size cap) and its input is a + * {@link Project} — rewrite that Project's slots and rebuild the Sort + * on top of the rewritten Project.
    3. + *
+ * + *

Matches any {@link Project} subclass — {@code DatetimeOutputCastRule} + * emits a {@link LogicalProject}, but engine-side optimizer rules + * (e.g. {@code OpenSearchProjectRule}) may have already converted the + * matched Project to a custom {@link Project} subclass (e.g. + * {@code OpenSearchProject}). {@link Project#copy} on the matched subclass + * round-trips back to the same subclass, so any subclass-specific state + * (viable backends, traits) is preserved. + * + *

The traversal is intentionally NOT recursive: only the output Project + * is rewritten — any deeper {@link Project} carries expressions that must + * round-trip verbatim. + */ + static RelNode rewrite(RelNode root) { + if (root instanceof Project project) { + Project rewritten = rewriteOutputProject(project); + return rewritten == project ? root : rewritten; + } + if (root instanceof Sort sort && sort.getInput() instanceof Project project) { + Project rewritten = rewriteOutputProject(project); + if (rewritten == project) { + return root; + } + return sort.copy(sort.getTraitSet(), rewritten, sort.getCollation(), sort.offset, sort.fetch); + } + return root; + } + + /** + * Returns a new {@link Project} (same subclass as {@code project}) with + * engine-output cast slots rewritten, or returns {@code project} unchanged + * when no slot matched. + */ + private static Project rewriteOutputProject(Project project) { + List oldProjects = project.getProjects(); + List newProjects = new ArrayList<>(oldProjects.size()); + boolean changed = false; + RexBuilder rexBuilder = project.getCluster().getRexBuilder(); + for (RexNode expr : oldProjects) { + RexNode rewritten = rewriteDirectOutputCast(expr, rexBuilder); + if (rewritten != expr) { + changed = true; + } + newProjects.add(rewritten); + } + if (!changed) { + return project; + } + return project.copy(project.getTraitSet(), project.getInput(), newProjects, project.getRowType()); + } + + /** + * Returns a {@code to_char(, format)} call when {@code expr} is the + * exact shape {@code CAST( AS VARCHAR)} produced by + * {@code DatetimeOutputCastRule}; otherwise returns {@code expr} unchanged. + * + *

Note: deliberately not recursive — see class-level scope notes. + */ + private static RexNode rewriteDirectOutputCast(RexNode expr, RexBuilder rexBuilder) { + if (!(expr instanceof RexCall call) || call.getKind() != SqlKind.CAST) { + return expr; + } + RexNode source = call.getOperands().get(0); + SqlTypeName sourceType = source.getType().getSqlTypeName(); + SqlTypeName targetType = call.getType().getSqlTypeName(); + if (sourceType != SqlTypeName.TIMESTAMP) { + return expr; + } + // VARCHAR-only: DatetimeOutputCastRule emits CAST(... AS VARCHAR) (length-unspecified). + // CHAR(n) is user-authored and has length/padding semantics that to_char does not preserve. + if (targetType != SqlTypeName.VARCHAR) { + return expr; + } + RelDataType formatType = rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); + RexNode formatLiteral = rexBuilder.makeLiteral(PPL_TIMESTAMP_FORMAT, formatType, true); + return rexBuilder.makeCall(call.getType(), SqlLibraryOperators.TO_CHAR, List.of(source, formatLiteral)); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml index a5e5b74c515b1..3f3f8b0e02c14 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml @@ -525,6 +525,21 @@ scalar_functions: nullability: DECLARED_OUTPUT return: string + # to_char(value, format) — render a timestamp using a strftime-style format string. + # Bound here so that DatetimeOutputCastRewriter can rewrite the engine-output cast + # `CAST( AS VARCHAR)` to `to_char(, '%Y-%m-%d %H:%M:%S')` and + # have the call serialize through this extension instead of Calcite's stdlib URN. + # DataFusion exposes a native `to_char` scalar that consumes strftime tokens. See + # issue #5420 (engine-output format divergence between Calcite and DataFusion). + - name: "to_char" + description: "Render a timestamp using a strftime-style format string (engine-output cast target)." + impls: + - args: + - { name: value, value: "precision_timestamp

" } + - { name: format, value: string } + nullability: DECLARED_OUTPUT + return: string + # tonumber(string, base) — parse `string` as a base-N integer - name: "tonumber" description: "Parse a string to a number in the given radix (2-36). Returns NULL on parse failure." diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java index 3b23c7adbeccd..e1140aa86761d 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java @@ -75,7 +75,10 @@ public void setUp() throws Exception { t.setContextClassLoader(DataFusionFragmentConvertorTests.class.getClassLoader()); SimpleExtension.ExtensionCollection delegationExtensions = SimpleExtension.load(List.of("/delegation_functions.yaml")); SimpleExtension.ExtensionCollection aggregateExtensions = SimpleExtension.load(List.of("/opensearch_aggregate_functions.yaml")); - extensions = DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions).merge(aggregateExtensions); + SimpleExtension.ExtensionCollection scalarExtensions = SimpleExtension.load(List.of("/opensearch_scalar_functions.yaml")); + extensions = DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions) + .merge(aggregateExtensions) + .merge(scalarExtensions); } finally { t.setContextClassLoader(prev); } @@ -576,6 +579,47 @@ public void testApproxCountDistinctRenamed() throws Exception { * SUM aggregate is not affected by the rename map — its extension function * name remains unchanged. */ + /** + * End-to-end: a {@code Project[CAST(ts AS VARCHAR)]} fragment must serialize + * with a {@code to_char} extension function, not a raw Substrait {@code cast}, + * proving {@link DatetimeOutputCastRewriter} fires inside the convertor and + * the {@code to_char} declaration in {@code opensearch_scalar_functions.yaml} + * is reachable through {@code FunctionMappings}. See issue + * sql#5420. + */ + public void testProjectTimestampOutputCastEmitsToCharExtension() throws Exception { + RelDataTypeFactory.Builder b = typeFactory.builder(); + b.add("ts", typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 6), true)); + RelNode scan = new DataFusionFragmentConvertor.StageInputTableScan(cluster, cluster.traitSet(), "test_index", b.build()); + + RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + RexNode tsField = rexBuilder.makeInputRef(scan, 0); + RexNode castExpr = rexBuilder.makeCast(varcharType, tsField); + RelNode project = org.apache.calcite.rel.logical.LogicalProject.create( + scan, + List.of(), + List.of(castExpr), + List.of("ts_str"), + java.util.Set.of() + ); + + byte[] bytes = newConvertor().convertShardScanFragment("test_index", project); + Plan plan = decodeSubstrait(bytes); + + boolean foundToChar = false; + for (SimpleExtensionDeclaration decl : plan.getExtensionsList()) { + if (decl.hasExtensionFunction()) { + String name = decl.getExtensionFunction().getName(); + String baseName = name.contains(":") ? name.substring(0, name.indexOf(':')) : name; + if (baseName.equals("to_char")) { + foundToChar = true; + break; + } + } + } + assertTrue("CAST( AS VARCHAR) must serialize as the to_char extension, not a raw cast", foundToChar); + } + public void testOtherFunctionsNotRenamed() throws Exception { RelNode scan = buildTableScan("test_index", "A"); LogicalAggregate agg = buildSumAggregate(scan, 0); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatetimeOutputCastRewriterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatetimeOutputCastRewriterTests.java new file mode 100644 index 0000000000000..b710c70c80d38 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatetimeOutputCastRewriterTests.java @@ -0,0 +1,320 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.logical.LogicalSort; +import org.apache.calcite.rel.logical.LogicalValues; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; +import java.util.Set; + +/** + * Tests for {@link DatetimeOutputCastRewriter}. Builds Calcite RelNode trees that match + * (and don't match) the engine-output {@code CAST( AS VARCHAR)} shape that + * {@code DatetimeOutputCastRule} produces, and asserts the rewriter narrows precisely + * to direct project slots over standard {@code TIMESTAMP}. + */ +public class DatetimeOutputCastRewriterTests extends OpenSearchTestCase { + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private RelOptCluster cluster; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + cluster = RelOptCluster.create(planner, rexBuilder); + } + + /** + * Motivating shape: outer Project slot is exactly {@code CAST( AS VARCHAR)}. + * Rewriter must replace it with a {@code TO_CHAR(, '%Y-%m-%d %H:%M:%S')} + * call whose result type matches the original cast's VARCHAR type. + */ + public void testDirectTimestampOutputCastIsRewrittenToToChar() { + RelDataType timestampType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 6), true); + RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + + RelNode values = singleRowWithTimestampField(timestampType); + RexNode tsField = rexBuilder.makeInputRef(values, 0); + RexNode castExpr = rexBuilder.makeCast(varcharType, tsField); + RelNode project = LogicalProject.create(values, List.of(), List.of(castExpr), List.of("ts_str"), Set.of()); + + RelNode rewritten = DatetimeOutputCastRewriter.rewrite(project); + LogicalProject rewrittenProj = (LogicalProject) rewritten; + RexNode rewrittenSlot = rewrittenProj.getProjects().get(0); + + assertTrue("Expected the slot to be rewritten into a function call", rewrittenSlot instanceof RexCall); + RexCall call = (RexCall) rewrittenSlot; + assertEquals("Slot operator must be Calcite's TO_CHAR", SqlLibraryOperators.TO_CHAR, call.getOperator()); + assertEquals("TO_CHAR result type must match the original CAST's VARCHAR type", varcharType, call.getType()); + assertEquals( + "First operand must remain the original TIMESTAMP source ref", + tsField.toString(), + call.getOperands().get(0).toString() + ); + + RexNode formatOperand = call.getOperands().get(1); + assertTrue("Second operand must be a literal format string", formatOperand instanceof RexLiteral); + assertEquals( + "Format must be PPL's space-separator timestamp pattern", + DatetimeOutputCastRewriter.PPL_TIMESTAMP_FORMAT, + ((RexLiteral) formatOperand).getValueAs(String.class) + ); + } + + /** + * When the root rel is not a {@link org.apache.calcite.rel.logical.LogicalProject} + * (e.g. a {@code LogicalFilter} fragment), the tree must round-trip verbatim. + * The rewriter only inspects the root project introduced by + * {@code DatetimeOutputCastRule}; non-project roots are out of scope. + */ + public void testCastInsideFilterPredicateIsUntouched() { + RelDataType timestampType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 6), true); + RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + + RelNode values = singleRowWithTimestampField(timestampType); + RexNode tsField = rexBuilder.makeInputRef(values, 0); + RexNode castInPredicate = rexBuilder.makeCast(varcharType, tsField); + RexNode literal = rexBuilder.makeLiteral("2024-01-15 12:00:00"); + RexNode predicate = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, castInPredicate, literal); + RelNode filter = LogicalFilter.create(values, predicate); + + RelNode rewritten = DatetimeOutputCastRewriter.rewrite(filter); + + assertSame("Filter tree must round-trip identical when no Project slot matches", filter, rewritten); + } + + /** + * A {@code CAST( AS VARCHAR)} buried inside a CASE branch is also + * user-authored (e.g. {@code SELECT CASE WHEN ... THEN CAST(ts AS VARCHAR) END}); + * the rewriter must leave nested casts alone. + */ + public void testNestedCastInsideCaseIsUntouched() { + RelDataType timestampType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 6), true); + RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + RelDataType boolType = typeFactory.createSqlType(SqlTypeName.BOOLEAN); + + RelNode values = singleRowWithTimestampField(timestampType); + RexNode tsField = rexBuilder.makeInputRef(values, 0); + RexNode condition = rexBuilder.makeLiteral(true, boolType); + RexNode innerCast = rexBuilder.makeCast(varcharType, tsField); + RexNode elseLit = rexBuilder.makeNullLiteral(varcharType); + RexNode caseExpr = rexBuilder.makeCall(SqlStdOperatorTable.CASE, condition, innerCast, elseLit); + RelNode project = LogicalProject.create(values, List.of(), List.of(caseExpr), List.of("guarded_ts"), Set.of()); + + RelNode rewritten = DatetimeOutputCastRewriter.rewrite(project); + LogicalProject rewrittenProj = (LogicalProject) rewritten; + RexNode rewrittenSlot = rewrittenProj.getProjects().get(0); + + assertTrue("Outer slot must remain a CASE call", rewrittenSlot instanceof RexCall); + assertEquals("Outer CASE operator must be unchanged", SqlStdOperatorTable.CASE, ((RexCall) rewrittenSlot).getOperator()); + // The inner CAST must round-trip verbatim — no TO_CHAR substitution inside the CASE branch. + RexNode innerThen = ((RexCall) rewrittenSlot).getOperands().get(1); + assertEquals(innerCast.toString(), innerThen.toString()); + } + + /** + * DATE and TIME sources are not rewritten — Arrow's CAST kernel already produces + * PPL's expected format for these (no calendar/no clock), and the issue scope is + * TIMESTAMP-only. A standard non-datetime cast is also untouched as a sanity check. + */ + public void testNonTimestampCastsAreUntouched() { + RelDataType dateType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.DATE), true); + RelDataType timeType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIME), true); + RelDataType intType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.INTEGER), true); + RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + + RelDataType rowType = typeFactory.builder().add("d", dateType).add("t", timeType).add("n", intType).build(); + RelNode values = LogicalValues.createEmpty(cluster, rowType); + + RexNode dateCast = rexBuilder.makeCast(varcharType, rexBuilder.makeInputRef(values, 0)); + RexNode timeCast = rexBuilder.makeCast(varcharType, rexBuilder.makeInputRef(values, 1)); + RexNode intCast = rexBuilder.makeCast(varcharType, rexBuilder.makeInputRef(values, 2)); + RelNode project = LogicalProject.create( + values, + List.of(), + List.of(dateCast, timeCast, intCast), + List.of("d_str", "t_str", "n_str"), + Set.of() + ); + + RelNode rewritten = DatetimeOutputCastRewriter.rewrite(project); + // No matching slots → tree must be returned identical. + assertSame("Non-TIMESTAMP source casts must round-trip identical", project, rewritten); + } + + /** + * A direct slot that is NOT a CAST (e.g. a plain field reference) must round-trip + * identical — the rewriter only matches the precise CAST shape. + */ + public void testNonCastSlotIsUntouched() { + RelDataType timestampType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 6), true); + RelNode values = singleRowWithTimestampField(timestampType); + RexNode tsField = rexBuilder.makeInputRef(values, 0); + RelNode project = LogicalProject.create(values, List.of(), List.of(tsField), List.of("ts"), Set.of()); + + RelNode rewritten = DatetimeOutputCastRewriter.rewrite(project); + assertSame(project, rewritten); + } + + /** + * A direct {@code CAST( AS VARCHAR)} living inside an INNER + * {@link LogicalProject} (i.e. not the root) must round-trip verbatim. + * {@code DatetimeOutputCastRule} only adds a single root-level project; any + * inner project carries user/optimizer-authored expressions whose CAST + * shape happens to match the engine-output rule but is semantically + * different and must be left alone. + */ + public void testInnerProjectDirectTimestampCastIsUntouched() { + RelDataType timestampType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 6), true); + RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + + RelNode values = singleRowWithTimestampField(timestampType); + RexNode tsField = rexBuilder.makeInputRef(values, 0); + RexNode innerCast = rexBuilder.makeCast(varcharType, tsField); + RelNode innerProject = LogicalProject.create(values, List.of(), List.of(innerCast), List.of("s"), Set.of()); + + RexNode outerRef = rexBuilder.makeInputRef(innerProject, 0); + RelNode outerProject = LogicalProject.create(innerProject, List.of(), List.of(outerRef), List.of("s"), Set.of()); + + RelNode rewritten = DatetimeOutputCastRewriter.rewrite(outerProject); + LogicalProject rewrittenOuter = (LogicalProject) rewritten; + LogicalProject rewrittenInner = (LogicalProject) rewrittenOuter.getInput(); + + assertEquals( + "Inner project's direct CAST slot must round-trip verbatim", + innerCast.toString(), + rewrittenInner.getProjects().get(0).toString() + ); + } + + /** + * {@code CAST( AS CHAR(n))} is user-authored — CHAR has fixed + * length and padding semantics that {@code to_char} does not preserve. + * {@code DatetimeOutputCastRule} only ever emits {@code AS VARCHAR}, so + * CHAR targets must round-trip verbatim. + */ + public void testTimestampCastToCharIsUntouched() { + RelDataType timestampType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 6), true); + RelDataType charType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.CHAR, 32), true); + + RelNode values = singleRowWithTimestampField(timestampType); + RexNode tsField = rexBuilder.makeInputRef(values, 0); + RexNode charCast = rexBuilder.makeCast(charType, tsField); + RelNode project = LogicalProject.create(values, List.of(), List.of(charCast), List.of("ts_char"), Set.of()); + + RelNode rewritten = DatetimeOutputCastRewriter.rewrite(project); + assertSame("CAST(... AS CHAR(n)) must round-trip verbatim — out of rule scope", project, rewritten); + } + + /** + * The format string is {@code "%Y-%m-%d %H:%M:%S"} — seconds-only — to match + * Calcite's reference output for {@code CAST(TIMESTAMP AS VARCHAR)}, which + * truncates fractional seconds. This pins that contract: a TIMESTAMP(9) + * source still resolves to the seconds-only format literal in the + * rewritten {@code TO_CHAR} call. + */ + public void testTimestampPrecisionDoesNotChangeFormat() { + RelDataType nanoTimestampType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 9), true); + RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + + RelNode values = singleRowWithTimestampField(nanoTimestampType); + RexNode tsField = rexBuilder.makeInputRef(values, 0); + RexNode castExpr = rexBuilder.makeCast(varcharType, tsField); + RelNode project = LogicalProject.create(values, List.of(), List.of(castExpr), List.of("ts_str"), Set.of()); + + RelNode rewritten = DatetimeOutputCastRewriter.rewrite(project); + RexCall call = (RexCall) ((LogicalProject) rewritten).getProjects().get(0); + RexLiteral formatLit = (RexLiteral) call.getOperands().get(1); + assertEquals( + "TIMESTAMP(9) source must still resolve to the seconds-only format — fractional seconds are dropped", + DatetimeOutputCastRewriter.PPL_TIMESTAMP_FORMAT, + formatLit.getValueAs(String.class) + ); + } + + /** + * Production shape from the unified planner: the output Project sits directly under a + * {@code LogicalSystemLimit} (a {@link LogicalSort} with no collation and a fixed fetch). + * The rewriter must descend through that single Sort wrapper, rewrite the Project's + * cast slots, and reattach the Sort on top of the rewritten Project. + */ + public void testSortOverProjectDirectTimestampOutputCastIsRewritten() { + RelDataType timestampType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 6), true); + RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); + + RelNode values = singleRowWithTimestampField(timestampType); + RexNode tsField = rexBuilder.makeInputRef(values, 0); + RexNode castExpr = rexBuilder.makeCast(varcharType, tsField); + RelNode project = LogicalProject.create(values, List.of(), List.of(castExpr), List.of("ts_str"), Set.of()); + RexNode fetch = rexBuilder.makeLiteral(10000, intType); + RelNode sort = LogicalSort.create(project, RelCollations.EMPTY, null, fetch); + + RelNode rewritten = DatetimeOutputCastRewriter.rewrite(sort); + + assertTrue("Rewritten root must remain a Sort", rewritten instanceof LogicalSort); + LogicalSort rewrittenSort = (LogicalSort) rewritten; + assertSame("Sort fetch must round-trip identical", fetch, rewrittenSort.fetch); + assertTrue("Sort input must remain a Project", rewrittenSort.getInput() instanceof LogicalProject); + LogicalProject rewrittenProj = (LogicalProject) rewrittenSort.getInput(); + RexNode rewrittenSlot = rewrittenProj.getProjects().get(0); + assertTrue("Inner project slot must be rewritten to a TO_CHAR call", rewrittenSlot instanceof RexCall); + RexCall call = (RexCall) rewrittenSlot; + assertEquals("Slot operator must be Calcite's TO_CHAR", SqlLibraryOperators.TO_CHAR, call.getOperator()); + assertEquals( + "First operand must remain the original TIMESTAMP source ref", + tsField.toString(), + call.getOperands().get(0).toString() + ); + } + + /** + * If the root is a {@link LogicalSort} whose input is NOT a {@link LogicalProject} + * (e.g. Sort directly over a scan/values), the tree must round-trip identical. + * The rewriter only descends through the Sort when its input is a Project. + */ + public void testSortOverNonProjectIsUntouched() { + RelDataType timestampType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 6), true); + RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); + RelNode values = singleRowWithTimestampField(timestampType); + RexNode fetch = rexBuilder.makeLiteral(10000, intType); + RelNode sort = LogicalSort.create(values, RelCollations.EMPTY, null, fetch); + + RelNode rewritten = DatetimeOutputCastRewriter.rewrite(sort); + assertSame("Sort over non-Project must round-trip identical", sort, rewritten); + } + + private RelNode singleRowWithTimestampField(RelDataType timestampType) { + RelDataType rowType = typeFactory.builder().add("ts", timestampType).build(); + return LogicalValues.createEmpty(cluster, rowType); + } +}