diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java index 5baa74d73fa4d..ee32b35be4b9e 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java @@ -56,6 +56,7 @@ public enum ScalarFunction { UPPER(Category.STRING, SqlKind.OTHER_FUNCTION), LOWER(Category.STRING, SqlKind.OTHER_FUNCTION), TRIM(Category.STRING, SqlKind.TRIM), + SUBSTR(Category.STRING, SqlKind.OTHER_FUNCTION), SUBSTRING(Category.STRING, SqlKind.OTHER_FUNCTION), /** * String concatenation. Calcite's {@code SqlStdOperatorTable.CONCAT} is a @@ -66,9 +67,23 @@ public enum ScalarFunction { * rename surfaces as a compile error rather than as a silent string mismatch at runtime. */ CONCAT(Category.STRING, SqlKind.OTHER_FUNCTION, SqlStdOperatorTable.CONCAT), + CONCAT_WS(Category.STRING, SqlKind.OTHER_FUNCTION), CHAR_LENGTH(Category.STRING, SqlKind.OTHER_FUNCTION), REPLACE(Category.STRING, SqlKind.OTHER_FUNCTION), REGEXP_REPLACE(Category.STRING, SqlKind.OTHER_FUNCTION), + ASCII(Category.STRING, SqlKind.OTHER_FUNCTION), + LEFT(Category.STRING, SqlKind.OTHER_FUNCTION), + LENGTH(Category.STRING, SqlKind.OTHER_FUNCTION), + LOCATE(Category.STRING, SqlKind.OTHER_FUNCTION), + POSITION(Category.STRING, SqlKind.POSITION), + LTRIM(Category.STRING, SqlKind.OTHER_FUNCTION), + RTRIM(Category.STRING, SqlKind.OTHER_FUNCTION), + REVERSE(Category.STRING, SqlKind.OTHER_FUNCTION), + RIGHT(Category.STRING, SqlKind.OTHER_FUNCTION), + TOSTRING(Category.STRING, SqlKind.OTHER_FUNCTION), + NUMBER_TO_STRING(Category.STRING, SqlKind.OTHER_FUNCTION), // Alias for TOSTRING + TONUMBER(Category.STRING, SqlKind.OTHER_FUNCTION), + STRCMP(Category.STRING, SqlKind.OTHER_FUNCTION), // ── Math ───────────────────────────────────────────────────────── PLUS(Category.MATH, SqlKind.PLUS), diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs index 52c74a888e650..c226deabb2fbe 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs @@ -111,6 +111,7 @@ pub async unsafe fn create_session_context( .build(); let ctx = SessionContext::new_with_state(state); + crate::udf::register_all(&ctx); // Register default ListingTable for parquet scans let listing_options = ListingOptions::new(Arc::new(ParquetFormat::new())) diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs index 3d13de51967b6..2053625b544c0 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs @@ -113,10 +113,14 @@ pub(crate) fn coerce_args( } pub mod convert_tz; +pub mod tonumber; +pub mod tostring; pub fn register_all(ctx: &SessionContext) { convert_tz::register_all(ctx); - log::info!("OpenSearch UDF register_all: convert_tz registered"); + tonumber::register_all(ctx); + tostring::register_all(ctx); + log::info!("OpenSearch UDF register_all: convert_tz, tonumber, tostring registered"); } #[cfg(test)] diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tonumber.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tonumber.rs new file mode 100644 index 0000000000000..765056a176774 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tonumber.rs @@ -0,0 +1,396 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! [`tonumber(string, base)`](https://docs.opensearch.org/latest/sql-and-ppl/ppl/functions/conversion/#tonumber) +//! base-N integer parse. +//! +//! # Semantics +//! * `base` must be in the inclusive range `[2, 36]` +//! * Output type is `Float64` + +use std::any::Any; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +use datafusion::arrow::array::{Array, ArrayRef, Float64Array, Float64Builder, StringArray}; +use datafusion::arrow::datatypes::DataType; +use datafusion::common::{exec_err, Result, ScalarValue}; +use datafusion::error::DataFusionError; +use datafusion::execution::context::SessionContext; +use datafusion::logical_expr::{ + ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature, + Volatility, +}; + +pub fn register_all(ctx: &SessionContext) { + ctx.register_udf(ScalarUDF::from(ToNumberUdf::new())); +} + +/// `tonumber(varchar, int)` → fp64. Base-N string-to-integer parse widened to double +#[derive(Debug)] +pub struct ToNumberUdf { + signature: Signature, +} + +impl ToNumberUdf { + pub fn new() -> Self { + Self { + signature: Signature::one_of( + vec![TypeSignature::Exact(vec![DataType::Utf8, DataType::Int32])], + Volatility::Immutable, + ), + } + } +} + +impl Default for ToNumberUdf { + fn default() -> Self { + Self::new() + } +} + +// `ScalarUDFImpl` requires `DynEq` + `DynHash`. All instances are functionally +// identical (no parameterization), so equality is trivial. +impl PartialEq for ToNumberUdf { + fn eq(&self, _: &Self) -> bool { + true + } +} +impl Eq for ToNumberUdf {} +impl Hash for ToNumberUdf { + fn hash(&self, state: &mut H) { + "tonumber".hash(state); + } +} + +/// How the resolved (constant) `base` argument behaves across the batch. +/// Resolved once up front so the hot loop doesn't repeat the scalar dispatch +/// or the range check on every row. +enum BaseMode { + /// Base is `NULL` or outside `[2, 36]`. Every output row is NULL regardless + /// of the value column — we skip reading it. + AllNull, + /// Base is valid. Carries the validated radix as `u32` (the type + /// {@code i64::from_str_radix} wants), so the per-row code skips both the + /// scalar dispatch and the range check. + Valid(u32), +} + +/// How the `value` argument is supplied +enum ValueSource<'a> { + Scalar(Option<&'a str>), + Array(&'a StringArray), +} + +impl<'a> ValueSource<'a> { + /// Returns the string at row `i`, or `None` + fn at(&self, i: usize) -> Option<&str> { + match self { + ValueSource::Scalar(s) => *s, + ValueSource::Array(arr) if arr.is_null(i) => None, + ValueSource::Array(arr) => Some(arr.value(i)), + } + } +} + +impl ScalarUDFImpl for ToNumberUdf { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "tonumber" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + if args.args.len() != 2 { + return exec_err!( + "tonumber expects exactly 2 arguments (string, base), got {}", + args.args.len() + ); + } + + let value_col = &args.args[0]; + let base_col = &args.args[1]; + + // Full-scalar fast path — a plain literal `tonumber('FA34', 16)` plan. + if let ( + ColumnarValue::Scalar(ScalarValue::Utf8(s)), + ColumnarValue::Scalar(ScalarValue::Int32(b)), + ) = (value_col, base_col) + { + return Ok(ColumnarValue::Scalar(ScalarValue::Float64( + parse_with_base(s.as_deref(), *b), + ))); + } + + let n = args.number_rows; + let BaseMode::Valid(radix) = resolve_base(base_col)? else { + let mut builder = Float64Builder::with_capacity(n); + builder.append_nulls(n); + return Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef)); + }; + + // Materialize the value column. For scalars we keep the raw &str to + // avoid a pointless n-wide StringArray allocation. + let values_arr_ref: Option = match value_col { + ColumnarValue::Array(_) => Some(value_col.clone().into_array(n)?), + _ => None, + }; + let values: ValueSource = match (&values_arr_ref, value_col) { + (Some(arr), _) => { + let sa = arr.as_any().downcast_ref::().ok_or_else(|| { + DataFusionError::Internal(format!( + "tonumber: value expected Utf8, got {:?}", + arr.data_type() + )) + })?; + ValueSource::Array(sa) + } + (None, ColumnarValue::Scalar(ScalarValue::Utf8(opt))) => { + ValueSource::Scalar(opt.as_deref()) + } + (None, other) => { + return exec_err!("tonumber: value expected Utf8, got {other:?}"); + } + }; + + let mut builder = Float64Builder::with_capacity(n); + for i in 0..n { + match values.at(i).and_then(|s| i64::from_str_radix(s, radix).ok()) { + Some(v) => builder.append_value(v as f64), + None => builder.append_null(), + } + } + let out: Float64Array = builder.finish(); + Ok(ColumnarValue::Array(Arc::new(out) as ArrayRef)) + } +} + +/// Resolve the `base` argument +fn resolve_base(base_col: &ColumnarValue) -> Result { + match base_col { + ColumnarValue::Scalar(ScalarValue::Int32(None)) => Ok(BaseMode::AllNull), + ColumnarValue::Scalar(ScalarValue::Int32(Some(b))) => Ok(match validate_base(*b) { + Some(r) => BaseMode::Valid(r), + None => BaseMode::AllNull, + }), + ColumnarValue::Scalar(other) => { + exec_err!("tonumber: base expected Int32 literal, got {other:?}") + } + ColumnarValue::Array(arr) => { + exec_err!( + "tonumber: base must be a literal integer, got column of type {:?}", + arr.data_type() + ) + } + } +} + +/// Convert `base` to the radix expected by [`i64::from_str_radix`], or `None` +/// when the base is outside `[2, 36]` range. +fn validate_base(base: i32) -> Option { + if (2..=36).contains(&base) { + Some(base as u32) + } else { + None + } +} + +fn parse_with_base(s: Option<&str>, base: Option) -> Option { + let s = s?; + let radix = validate_base(base?)?; + i64::from_str_radix(s, radix).ok().map(|v| v as f64) +} + +// ─── tests ────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::arrow::array::{AsArray, Int32Array}; + use datafusion::arrow::datatypes::Field; + + fn invoke_scalar(value: Option<&str>, base: Option) -> Option { + let u = ToNumberUdf::new(); + let args = ScalarFunctionArgs { + args: vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(value.map(|s| s.to_string()))), + ColumnarValue::Scalar(ScalarValue::Int32(base)), + ], + arg_fields: vec![ + Arc::new(Field::new("v", DataType::Utf8, true)), + Arc::new(Field::new("b", DataType::Int32, true)), + ], + number_rows: 1, + return_field: Arc::new(Field::new(u.name(), DataType::Float64, true)), + config_options: Arc::new(Default::default()), + }; + match u.invoke_with_args(args).unwrap() { + ColumnarValue::Scalar(ScalarValue::Float64(opt)) => opt, + other => panic!("expected Float64 scalar, got {other:?}"), + } + } + + #[test] + fn base10_integer_doc_example() { + assert_eq!(invoke_scalar(Some("4598"), Some(10)), Some(4598.0)); + } + + #[test] + fn binary_doc_example() { + assert_eq!(invoke_scalar(Some("010101"), Some(2)), Some(21.0)); + } + + #[test] + fn hex_doc_example() { + assert_eq!(invoke_scalar(Some("FA34"), Some(16)), Some(64052.0)); + assert_eq!(invoke_scalar(Some("fa34"), Some(16)), Some(64052.0)); + } + + #[test] + fn signed_inputs_parse() { + assert_eq!(invoke_scalar(Some("-21"), Some(10)), Some(-21.0)); + assert_eq!(invoke_scalar(Some("+FA"), Some(16)), Some(250.0)); + assert_eq!(invoke_scalar(Some("-10"), Some(2)), Some(-2.0)); + } + + #[test] + fn base_boundary_values() { + assert_eq!(invoke_scalar(Some("1"), Some(2)), Some(1.0)); + assert_eq!(invoke_scalar(Some("Z"), Some(36)), Some(35.0)); + } + + #[test] + fn out_of_range_base_returns_null() { + assert!(invoke_scalar(Some("10"), Some(0)).is_none()); + assert!(invoke_scalar(Some("10"), Some(1)).is_none()); + assert!(invoke_scalar(Some("10"), Some(37)).is_none()); + assert!(invoke_scalar(Some("10"), Some(-5)).is_none()); + } + + #[test] + fn unparseable_string_returns_null() { + assert!(invoke_scalar(Some("FA34"), Some(10)).is_none()); + assert!(invoke_scalar(Some("12"), Some(2)).is_none()); + assert!(invoke_scalar(Some(""), Some(10)).is_none()); + assert!(invoke_scalar(Some("1 2"), Some(10)).is_none()); + assert!(invoke_scalar(Some("3.14"), Some(10)).is_none()); + } + + #[test] + fn overflow_returns_null() { + assert!(invoke_scalar(Some("9223372036854775808"), Some(10)).is_none()); + } + + #[test] + fn null_inputs_return_null() { + assert!(invoke_scalar(None, Some(10)).is_none()); + assert!(invoke_scalar(Some("10"), None).is_none()); + assert!(invoke_scalar(None, None).is_none()); + } + + #[test] + fn array_values_with_scalar_base_takes_fast_path() { + let u = ToNumberUdf::new(); + let values: ArrayRef = Arc::new(StringArray::from(vec![ + Some("FA34"), + Some("nope"), + None, + Some("ff"), + ])); + let args = ScalarFunctionArgs { + args: vec![ + ColumnarValue::Array(values), + ColumnarValue::Scalar(ScalarValue::Int32(Some(16))), + ], + arg_fields: vec![ + Arc::new(Field::new("v", DataType::Utf8, true)), + Arc::new(Field::new("b", DataType::Int32, true)), + ], + number_rows: 4, + return_field: Arc::new(Field::new("out", DataType::Float64, true)), + config_options: Arc::new(Default::default()), + }; + let arr = match u.invoke_with_args(args).unwrap() { + ColumnarValue::Array(a) => a, + other => panic!("expected array, got {other:?}"), + }; + let f = arr.as_primitive::(); + assert_eq!(f.value(0), 64052.0); + assert!(f.is_null(1), "unparseable → NULL"); + assert!(f.is_null(2), "null input → NULL"); + assert_eq!(f.value(3), 255.0); + } + + #[test] + fn scalar_null_base_produces_all_null_output() { + let u = ToNumberUdf::new(); + let values: ArrayRef = Arc::new(StringArray::from(vec![ + Some("FA34"), + Some("nope"), + Some("10"), + ])); + let args = ScalarFunctionArgs { + args: vec![ + ColumnarValue::Array(values), + ColumnarValue::Scalar(ScalarValue::Int32(None)), + ], + arg_fields: vec![ + Arc::new(Field::new("v", DataType::Utf8, true)), + Arc::new(Field::new("b", DataType::Int32, true)), + ], + number_rows: 3, + return_field: Arc::new(Field::new("out", DataType::Float64, true)), + config_options: Arc::new(Default::default()), + }; + let arr = match u.invoke_with_args(args).unwrap() { + ColumnarValue::Array(a) => a, + other => panic!("expected array, got {other:?}"), + }; + let f = arr.as_primitive::(); + assert_eq!(f.len(), 3); + for i in 0..3 { + assert!(f.is_null(i), "row {i} must be NULL"); + } + } + + #[test] + fn scalar_out_of_range_base_produces_all_null_output() { + let u = ToNumberUdf::new(); + let values: ArrayRef = Arc::new(StringArray::from(vec![Some("FA34"), Some("10")])); + let args = ScalarFunctionArgs { + args: vec![ + ColumnarValue::Array(values), + ColumnarValue::Scalar(ScalarValue::Int32(Some(42))), + ], + arg_fields: vec![ + Arc::new(Field::new("v", DataType::Utf8, true)), + Arc::new(Field::new("b", DataType::Int32, true)), + ], + number_rows: 2, + return_field: Arc::new(Field::new("out", DataType::Float64, true)), + config_options: Arc::new(Default::default()), + }; + let arr = match u.invoke_with_args(args).unwrap() { + ColumnarValue::Array(a) => a, + other => panic!("expected array, got {other:?}"), + }; + let f = arr.as_primitive::(); + assert!(f.is_null(0)); + assert!(f.is_null(1)); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tostring.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tostring.rs new file mode 100644 index 0000000000000..e9593e1a0d79f --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tostring.rs @@ -0,0 +1,595 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! [`tostring(value, format)`](https://docs.opensearch.org/latest/sql-and-ppl/ppl/functions/conversion/#tostring) +//! format modes in one UDF. +//! +//! Mirror's PPL's +//! [`ToStringFunction`](https://github.com/opensearch-project/sql/blob/main/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java) +//! +//! Supported format values: +//! * `"binary"` — number → base-2 string of its integer part. Negative values use the +//! signed-magnitude representation `-1xxxx` (matches `BigInteger.toString(2)`). +//! * `"hex"` — number → lowercase hexadecimal of its integer part. Negative values use +//! the signed-magnitude representation `-xx` (matches `BigInteger.toString(16)`). +//! * `"commas"` — number with comma grouping; rounded to 2 decimals when the value has +//! a fractional component, otherwise no decimals. +//! * `"duration"` — integer seconds → `HH:MM:SS` **wall-clock** rendering. +//! * `"duration_millis"` — integer milliseconds → `HH:MM:SS` wall-clock rendering. + +use std::any::Any; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +use datafusion::arrow::array::{Array, ArrayRef, AsArray, StringArray}; +use datafusion::arrow::datatypes::{DataType, Float64Type, Int64Type}; +use datafusion::common::{exec_err, Result, ScalarValue}; +use datafusion::execution::context::SessionContext; +use datafusion::logical_expr::{ + ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature, Volatility, +}; + +pub fn register_all(ctx: &SessionContext) { + ctx.register_udf(ScalarUDF::from(ToStringUdf::new())); +} + +/// `tostring(bigint, varchar)` / `tostring(fp64, varchar)` → varchar. +#[derive(Debug)] +pub struct ToStringUdf { + signature: Signature, +} + +impl ToStringUdf { + pub fn new() -> Self { + Self { + signature: Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Int64, DataType::Utf8]), + TypeSignature::Exact(vec![DataType::Float64, DataType::Utf8]), + ], + Volatility::Immutable, + ), + } + } +} + +impl Default for ToStringUdf { + fn default() -> Self { + Self::new() + } +} + +// `ScalarUDFImpl` requires DynEq+DynHash. All `ToStringUdf` instances are functionally +// identical — there's no meaningful "parameterization" — so they compare equal and hash +// identically. +impl PartialEq for ToStringUdf { + fn eq(&self, _: &Self) -> bool { + true + } +} +impl Eq for ToStringUdf {} +impl Hash for ToStringUdf { + fn hash(&self, state: &mut H) { + "tostring".hash(state); + } +} + +impl ScalarUDFImpl for ToStringUdf { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "tostring" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Utf8) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + if args.args.len() != 2 { + return exec_err!( + "tostring expects exactly 2 arguments (value, format), got {}", + args.args.len() + ); + } + + let format_col = &args.args[1]; + + match &args.args[0] { + ColumnarValue::Scalar(ScalarValue::Int64(value)) => Ok(ColumnarValue::Scalar( + ScalarValue::Utf8(scalar_to_str(*value, format_col, 0, format_i64_as)?), + )), + ColumnarValue::Scalar(ScalarValue::Float64(value)) => Ok(ColumnarValue::Scalar( + ScalarValue::Utf8(scalar_to_str(*value, format_col, 0, format_f64_as)?), + )), + ColumnarValue::Scalar(other) => { + exec_err!("tostring: expected BIGINT or DOUBLE value, got {other:?}") + } + + ColumnarValue::Array(arr) => match arr.data_type() { + DataType::Int64 => { + let typed = arr.as_primitive::(); + let out: StringArray = (0..typed.len()) + .map(|i| { + if typed.is_null(i) { + Ok(None) + } else { + scalar_to_str(Some(typed.value(i)), format_col, i, format_i64_as) + } + }) + .collect::>>()? + .into_iter() + .collect(); + Ok(ColumnarValue::Array(Arc::new(out) as ArrayRef)) + } + DataType::Float64 => { + let typed = arr.as_primitive::(); + let out: StringArray = (0..typed.len()) + .map(|i| { + if typed.is_null(i) { + Ok(None) + } else { + scalar_to_str(Some(typed.value(i)), format_col, i, format_f64_as) + } + }) + .collect::>>()? + .into_iter() + .collect(); + Ok(ColumnarValue::Array(Arc::new(out) as ArrayRef)) + } + other => exec_err!("tostring: expected Int64 or Float64 value array, got {other:?}"), + }, + } + } +} + +/// Per-row dispatcher for one concrete value type. +/// +/// * `value` — the already-unwrapped optional; {@code None} short-circuits to `Ok(None)` +/// (null propagation). +/// * `format_col` — the format `ColumnarValue`. Resolved once per row via +/// [`format_at`]; the row index is only consulted when the caller passed an array. +/// * `row` — only used when `format_col` is an array; ignored for the scalar path. +/// * `formatter` — per-value-type rendering function (one for `i64`, one for `f64`). +fn scalar_to_str( + value: Option, + format_col: &ColumnarValue, + row: usize, + formatter: fn(T, &str) -> String, +) -> Result> { + let Some(v) = value else { + return Ok(None); + }; + let format = format_at(format_col, row)?; + match format { + Some(f) => Ok(Some(formatter(v, f.as_str()))), + None => Ok(None), + } +} + +/// Pulls the format string for `row` out of `format_col`. Returns `Ok(None)` for null +/// format cells. +fn format_at(format_col: &ColumnarValue, row: usize) -> Result> { + match format_col { + ColumnarValue::Scalar(ScalarValue::Utf8(opt)) | ColumnarValue::Scalar(ScalarValue::LargeUtf8(opt)) => { + Ok(opt.clone()) + } + ColumnarValue::Scalar(other) => { + exec_err!("tostring: format must be VARCHAR, got {other:?}") + } + ColumnarValue::Array(arr) => match arr.data_type() { + DataType::Utf8 => { + let strs = arr.as_string::(); + if strs.is_null(row) { + Ok(None) + } else { + Ok(Some(strs.value(row).to_string())) + } + } + DataType::LargeUtf8 => { + let strs = arr.as_string::(); + if strs.is_null(row) { + Ok(None) + } else { + Ok(Some(strs.value(row).to_string())) + } + } + other => exec_err!("tostring: expected Utf8 format array, got {other:?}"), + }, + } +} + +/// Format modes, case-sensitive to match the SQL plugin's Java reference +/// (`ToStringFunction.DURATION_FORMAT`, etc.). +mod mode { + pub const BINARY: &str = "binary"; + pub const HEX: &str = "hex"; + pub const COMMAS: &str = "commas"; + pub const DURATION: &str = "duration"; + pub const DURATION_MILLIS: &str = "duration_millis"; +} + +/// Render an `i64` value per the requested format. Unknown modes fall through to plain +/// decimal rendering. +fn format_i64_as(value: i64, format: &str) -> String { + match format { + mode::BINARY => format_binary_i64(value), + mode::HEX => format_hex_i64(value), + mode::COMMAS => format_commas_i64(value), + mode::DURATION => format_duration_seconds(value), + mode::DURATION_MILLIS => format_duration_seconds(value.div_euclid(1_000)), + _ => value.to_string(), + } +} + +/// Render an `f64` value per the requested format. Mirrors the Java reference's strategy +/// of routing `binary` / `hex` / `duration*` through `BigDecimal.toBigInteger()` — i.e. +/// truncate toward zero, then apply the integer formatter. +fn format_f64_as(value: f64, format: &str) -> String { + match format { + mode::BINARY => format_binary_i64(truncate_to_i64(value)), + mode::HEX => format_hex_i64(truncate_to_i64(value)), + mode::COMMAS => format_commas_f64(value), + mode::DURATION => format_duration_seconds(truncate_to_i64(value)), + mode::DURATION_MILLIS => format_duration_seconds(truncate_to_i64(value).div_euclid(1_000)), + _ => { + if !value.is_finite() { + return value.to_string(); + } + // Drop a trailing `.0` so `tostring(42.0)` reads as `"42"`, matching + // `BigDecimal.valueOf(42.0).toString() == "42.0"` → passed to `NumberFormat` with no + // decimals would print `"42"`. + let rendered = format!("{value}"); + rendered.strip_suffix(".0").map_or(rendered.clone(), |s| s.to_string()) + } + } +} + +fn truncate_to_i64(value: f64) -> i64 { + if !value.is_finite() { + return 0; + } + value as i64 +} + +fn format_binary_i64(v: i64) -> String { + // Mirrors `BigInteger.toString(2)` for positive values; for negatives the Java + // reference emits a leading '-' via BigInteger's signed radix representation. We match + // that by formatting the absolute value and re-prepending the sign. + if v < 0 { + // Work in i128 so i64::MIN doesn't overflow on abs. + format!("-{:b}", (v as i128).unsigned_abs()) + } else { + format!("{:b}", v as u64) + } +} + +fn format_hex_i64(v: i64) -> String { + // Same rationale as `format_binary_i64` — BigInteger.toString(16) on negatives prints + // a leading '-'. Lowercase to match the Java reference (BigInteger uses lowercase). + if v < 0 { + format!("-{:x}", (v as i128).unsigned_abs()) + } else { + format!("{:x}", v as u64) + } +} + +fn format_commas_i64(v: i64) -> String { + let is_negative = v < 0; + let abs_str = if is_negative { + format!("{}", (v as i128).unsigned_abs()) + } else { + v.to_string() + }; + let mut out = String::with_capacity(abs_str.len() + abs_str.len() / 3 + 1); + if is_negative { + out.push('-'); + } + insert_thousands_separators(&abs_str, &mut out); + out +} + +fn format_commas_f64(v: f64) -> String { + // Non-finite fall through to native rendering (matches Double.toString for Infinity/NaN). + if !v.is_finite() { + return v.to_string(); + } + let is_negative = v.is_sign_negative(); + // rounds the number to the nearest two decimal places. + let rounded = format!("{:.2}", v.abs()); + let (whole, frac) = rounded + .split_once('.') + .expect("{:.2} always produces a decimal point"); + let mut out = String::with_capacity(rounded.len() + whole.len() / 3 + 2); + if is_negative { + out.push('-'); + } + insert_thousands_separators(whole, &mut out); + // Drop `.00` so integral values render without a decimal tail (`39225 → "39,225"`), + // but keep a single-digit fractional (`.50`) when present + if frac != "00" { + out.push('.'); + // Trim a trailing '0' when exactly one digit would be meaningful, e.g. `.50 → .5`. + if frac.ends_with('0') { + out.push_str(&frac[..frac.len() - 1]); + } else { + out.push_str(frac); + } + } + out +} + +/// Allocation-free thousands-separator insertion. Appends `digits` to `out` with a `,` +/// after every 3 digits counted from the right. +fn insert_thousands_separators(digits: &str, out: &mut String) { + let bytes = digits.as_bytes(); + let len = bytes.len(); + for (i, b) in bytes.iter().enumerate() { + let from_right = len - i; + out.push(*b as char); + if from_right > 1 && (from_right - 1) % 3 == 0 { + out.push(','); + } + } +} + +/// Format a signed number of seconds to wall-clock `HH:MM:SS` format +fn format_duration_seconds(total_seconds: i64) -> String { + // Use `rem_euclid` to get the non-negative second-of-day. + let second_of_day = total_seconds.rem_euclid(86_400); + let h = second_of_day / 3600; + let m = (second_of_day / 60) % 60; + let s = second_of_day % 60; + format!("{:02}:{:02}:{:02}", h, m, s) +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::arrow::array::{Float64Array, Int64Array}; + use datafusion::arrow::datatypes::Field; + + fn udf() -> ToStringUdf { + ToStringUdf::new() + } + + fn invoke_scalar( + value: ScalarValue, + value_type: DataType, + format: &str, + ) -> Result { + let u = udf(); + let return_field = Arc::new(Field::new(u.name(), DataType::Utf8, true)); + let args = ScalarFunctionArgs { + args: vec![ + ColumnarValue::Scalar(value), + ColumnarValue::Scalar(ScalarValue::Utf8(Some(format.to_string()))), + ], + arg_fields: vec![ + Arc::new(Field::new("v", value_type, true)), + Arc::new(Field::new("f", DataType::Utf8, true)), + ], + number_rows: 1, + return_field, + config_options: Arc::new(Default::default()), + }; + u.invoke_with_args(args) + } + + fn invoke_array(value: ArrayRef, format: &str) -> Result { + let u = udf(); + let return_field = Arc::new(Field::new(u.name(), DataType::Utf8, true)); + let value_type = value.data_type().clone(); + let args = ScalarFunctionArgs { + args: vec![ + ColumnarValue::Array(value), + ColumnarValue::Scalar(ScalarValue::Utf8(Some(format.to_string()))), + ], + arg_fields: vec![ + Arc::new(Field::new("v", value_type, true)), + Arc::new(Field::new("f", DataType::Utf8, true)), + ], + number_rows: 3, + return_field, + config_options: Arc::new(Default::default()), + }; + u.invoke_with_args(args) + } + + fn utf8(v: ColumnarValue) -> String { + match v { + ColumnarValue::Scalar(ScalarValue::Utf8(Some(s))) => s, + other => panic!("expected Utf8 scalar, got {other:?}"), + } + } + + #[test] + fn hex_matches_bigint_tohex() { + let out = invoke_scalar(ScalarValue::Int64(Some(39225)), DataType::Int64, "hex").unwrap(); + assert_eq!(utf8(out), "9939"); + } + + #[test] + fn binary_matches_biginteger_tostring_2() { + let out = invoke_scalar(ScalarValue::Int64(Some(39225)), DataType::Int64, "binary").unwrap(); + assert_eq!(utf8(out), "1001100100111001"); + } + + #[test] + fn binary_negative_uses_signed_biginteger_repr() { + let out = invoke_scalar(ScalarValue::Int64(Some(-5)), DataType::Int64, "binary").unwrap(); + assert_eq!(utf8(out), "-101"); + } + + #[test] + fn commas_integer_doc_example() { + let out = invoke_scalar(ScalarValue::Int64(Some(39225)), DataType::Int64, "commas").unwrap(); + assert_eq!(utf8(out), "39,225"); + } + + #[test] + fn commas_float_rounds_to_two_decimals() { + let out = invoke_scalar( + ScalarValue::Float64(Some(1234.5678)), + DataType::Float64, + "commas", + ) + .unwrap(); + assert_eq!(utf8(out), "1,234.57"); + } + + #[test] + fn commas_float_drops_trailing_double_zero() { + let out = invoke_scalar( + ScalarValue::Float64(Some(39225.0)), + DataType::Float64, + "commas", + ) + .unwrap(); + assert_eq!(utf8(out), "39,225"); + } + + #[test] + fn duration_seconds_doc_example() { + let out = invoke_scalar(ScalarValue::Int64(Some(6500)), DataType::Int64, "duration").unwrap(); + assert_eq!(utf8(out), "01:48:20"); + } + + #[test] + fn duration_bigdecimal_positive_example() { + let out = invoke_scalar(ScalarValue::Int64(Some(3661)), DataType::Int64, "duration").unwrap(); + assert_eq!(utf8(out), "01:01:01"); + } + + #[test] + fn duration_negative_wraps_to_pre_epoch_clock_time() { + let out = invoke_scalar( + ScalarValue::Float64(Some(-3661.4)), + DataType::Float64, + "duration", + ) + .unwrap(); + assert_eq!(utf8(out), "22:58:59"); + } + + #[test] + fn duration_wraps_modulo_24h() { + let day = invoke_scalar( + ScalarValue::Int64(Some(86_400)), + DataType::Int64, + "duration", + ) + .unwrap(); + assert_eq!(utf8(day), "00:00:00"); + + let day_plus_hour = invoke_scalar( + ScalarValue::Int64(Some(86_400 + 3_600)), + DataType::Int64, + "duration", + ) + .unwrap(); + assert_eq!(utf8(day_plus_hour), "01:00:00"); + } + + #[test] + fn duration_millis_truncates_subseconds() { + let out = invoke_scalar( + ScalarValue::Int64(Some(6_500_999)), + DataType::Int64, + "duration_millis", + ) + .unwrap(); + assert_eq!(utf8(out), "01:48:20"); + } + + #[test] + fn duration_millis_negative_wraps_via_floor_div() { + let out = invoke_scalar( + ScalarValue::Int64(Some(-3_661_000)), + DataType::Int64, + "duration_millis", + ) + .unwrap(); + assert_eq!(utf8(out), "22:58:59"); + } + + #[test] + fn unknown_format_falls_through_to_plain_decimal() { + let out = invoke_scalar(ScalarValue::Int64(Some(42)), DataType::Int64, "xyzzy").unwrap(); + assert_eq!(utf8(out), "42"); + } + + #[test] + fn null_value_yields_null() { + let out = invoke_scalar(ScalarValue::Int64(None), DataType::Int64, "hex").unwrap(); + match out { + ColumnarValue::Scalar(ScalarValue::Utf8(None)) => {} + other => panic!("expected Utf8(None), got {other:?}"), + } + } + + #[test] + fn null_format_yields_null() { + let u = udf(); + let return_field = Arc::new(Field::new(u.name(), DataType::Utf8, true)); + let args = ScalarFunctionArgs { + args: vec![ + ColumnarValue::Scalar(ScalarValue::Int64(Some(42))), + ColumnarValue::Scalar(ScalarValue::Utf8(None)), + ], + arg_fields: vec![ + Arc::new(Field::new("v", DataType::Int64, true)), + Arc::new(Field::new("f", DataType::Utf8, true)), + ], + number_rows: 1, + return_field, + config_options: Arc::new(Default::default()), + }; + let out = u.invoke_with_args(args).unwrap(); + match out { + ColumnarValue::Scalar(ScalarValue::Utf8(None)) => {} + other => panic!("expected Utf8(None), got {other:?}"), + } + } + + #[test] + fn array_int_hex_with_scalar_format() { + let array: ArrayRef = Arc::new(Int64Array::from(vec![Some(15), None, Some(255)])); + let out = invoke_array(array, "hex").unwrap(); + match out { + ColumnarValue::Array(arr) => { + let s = arr.as_string::(); + assert_eq!(s.value(0), "f"); + assert!(s.is_null(1)); + assert_eq!(s.value(2), "ff"); + } + other => panic!("expected array, got {other:?}"), + } + } + + #[test] + fn array_float_commas_with_scalar_format() { + let array: ArrayRef = Arc::new(Float64Array::from(vec![Some(1234.5), None, Some(0.0)])); + let out = invoke_array(array, "commas").unwrap(); + match out { + ColumnarValue::Array(arr) => { + let s = arr.as_string::(); + assert_eq!(s.value(0), "1,234.5"); + assert!(s.is_null(1)); + assert_eq!(s.value(2), "0"); + } + other => panic!("expected array, got {other:?}"), + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index e883dac590ec9..a7414eac65639 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -190,7 +190,7 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP ScalarFunction.CURRENT_TIME, ScalarFunction.CURTIME, ScalarFunction.CONVERT_TZ, - ScalarFunction.UNIX_TIMESTAMP + ScalarFunction.UNIX_TIMESTAMP, // DATE(expr) / TIME(expr) / MAKETIME(h,m,s) are intentionally not advertised: // PPL's Calcite binding for these returns VARCHAR rather than DATE/TIME, so // downstream `year(date(ts))` / `hour(maketime(...))` lowers to @@ -201,6 +201,25 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP // and we'd need a dedicated adapter + yaml entry to route it to DataFusion's // date_part. Left on legacy engine until that adapter lands; PPL date-part // functions cover the same semantics. + ScalarFunction.ASCII, + ScalarFunction.CONCAT_WS, + ScalarFunction.LEFT, + ScalarFunction.LENGTH, + ScalarFunction.CHAR_LENGTH, + ScalarFunction.LOCATE, + ScalarFunction.POSITION, + ScalarFunction.LOWER, + ScalarFunction.LTRIM, + ScalarFunction.REVERSE, + ScalarFunction.RIGHT, + ScalarFunction.RTRIM, + ScalarFunction.TRIM, + ScalarFunction.SUBSTR, + ScalarFunction.UPPER, + ScalarFunction.STRCMP, + ScalarFunction.TOSTRING, + ScalarFunction.NUMBER_TO_STRING, + ScalarFunction.TONUMBER ); private static final Set AGG_FUNCTIONS = Set.of( @@ -309,13 +328,16 @@ public Map scalarFunctionAdapters() { Map.entry(ScalarFunction.HOUR, hour), Map.entry(ScalarFunction.HOUR_OF_DAY, hour), Map.entry(ScalarFunction.LIKE, new LikeAdapter()), + Map.entry(ScalarFunction.LOCATE, new PositionAdapter()), Map.entry(ScalarFunction.MICROSECOND, DatePartAdapters.microsecond()), Map.entry(ScalarFunction.MINUTE, minute), Map.entry(ScalarFunction.MINUTE_OF_HOUR, minute), Map.entry(ScalarFunction.MOD, new StdOperatorRewriteAdapter("MOD", SqlStdOperatorTable.MOD)), Map.entry(ScalarFunction.MONTH, month), Map.entry(ScalarFunction.MONTH_OF_YEAR, month), + Map.entry(ScalarFunction.NUMBER_TO_STRING, new ToStringFunctionAdapter()), Map.entry(ScalarFunction.NOW, now), + Map.entry(ScalarFunction.POSITION, new PositionAdapter()), Map.entry(ScalarFunction.QUARTER, DatePartAdapters.quarter()), Map.entry(ScalarFunction.REGEXP_REPLACE, new RegexpReplaceAdapter()), Map.entry(ScalarFunction.SARG_PREDICATE, new SargAdapter()), @@ -323,7 +345,12 @@ public Map scalarFunctionAdapters() { Map.entry(ScalarFunction.SCALAR_MIN, nameMapping(SqlLibraryOperators.LEAST)), Map.entry(ScalarFunction.SIGN, nameMapping(SignumFunction.FUNCTION)), Map.entry(ScalarFunction.SINH, new HyperbolicOperatorAdapter(SqlLibraryOperators.SINH)), + Map.entry(ScalarFunction.STRCMP, new StrcmpFunctionAdapter()), + Map.entry(ScalarFunction.SUBSTR, nameMapping(SqlStdOperatorTable.SUBSTRING)), + Map.entry(ScalarFunction.SUBSTRING, nameMapping(SqlStdOperatorTable.SUBSTRING)), Map.entry(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter()), + Map.entry(ScalarFunction.TONUMBER, new ToNumberFunctionAdapter()), + Map.entry(ScalarFunction.TOSTRING, new ToStringFunctionAdapter()), Map.entry(ScalarFunction.UNIX_TIMESTAMP, new UnixTimestampAdapter()), Map.entry(ScalarFunction.WEEK, week), Map.entry(ScalarFunction.WEEK_OF_YEAR, week), diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java index 2614d3c3f9ebb..23f7012bb8208 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -99,6 +99,10 @@ public class DataFusionFragmentConvertor implements FragmentConvertor { */ private static final List ADDITIONAL_SCALAR_SIGS = List.of( FunctionMappings.s(DelegatedPredicateFunction.FUNCTION, DelegatedPredicateFunction.NAME), + FunctionMappings.s(SqlStdOperatorTable.ASCII, "ascii"), + FunctionMappings.s(SqlStdOperatorTable.CHAR_LENGTH, "length"), + FunctionMappings.s(SqlLibraryOperators.CONCAT_FUNCTION, "concat"), + FunctionMappings.s(SqlLibraryOperators.CONCAT_WS, "concat_ws"), FunctionMappings.s(SqlLibraryOperators.ILIKE, "ilike"), FunctionMappings.s(SqlLibraryOperators.DATE_PART, "date_part"), FunctionMappings.s(ConvertTzAdapter.LOCAL_CONVERT_TZ_OP, "convert_tz"), @@ -110,6 +114,11 @@ public class DataFusionFragmentConvertor implements FragmentConvertor { FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"), FunctionMappings.s(SqlStdOperatorTable.REPLACE, "replace"), FunctionMappings.s(SqlLibraryOperators.REGEXP_REPLACE_3, "regexp_replace"), + FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"), + FunctionMappings.s(SqlLibraryOperators.REVERSE, "reverse"), + FunctionMappings.s(PositionAdapter.STRPOS, "strpos"), + FunctionMappings.s(ToNumberFunctionAdapter.TONUMBER, "tonumber"), + FunctionMappings.s(ToStringFunctionAdapter.TOSTRING, "tostring"), FunctionMappings.s(SqlStdOperatorTable.TRUNCATE, "trunc"), FunctionMappings.s(SqlStdOperatorTable.CBRT, "cbrt"), FunctionMappings.s(SqlStdOperatorTable.COT, "cot"), diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java index 01449e41a20dc..a866e09db23fb 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java @@ -20,6 +20,7 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.opensearch.analytics.backend.EngineResultBatch; import org.opensearch.analytics.backend.EngineResultStream; +import org.opensearch.analytics.exec.ArrowValues; import org.opensearch.be.datafusion.nativelib.NativeBridge; import org.opensearch.be.datafusion.nativelib.StreamHandle; import org.opensearch.common.annotation.ExperimentalApi; @@ -200,7 +201,7 @@ public Object getFieldValue(String fieldName, int rowIndex) { if (vector == null) { throw new IllegalArgumentException("Unknown field: " + fieldName); } - return vector.getObject(rowIndex); + return ArrowValues.toJavaValue(vector, rowIndex); } } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PositionAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PositionAdapter.java new file mode 100644 index 0000000000000..53016105ebc92 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PositionAdapter.java @@ -0,0 +1,104 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; + +import java.math.BigDecimal; +import java.util.List; + +/** + * Adapts PPL {@code POSITION(substr IN str[, start])} to DataFusion's {@code strpos(str, substr)}. + * + *

PPL emits a 2-arg {@code POSITION(substr, str)} for {@code locate(substr, str)} / + * {@code position(substr IN str)}, and a 3-arg {@code POSITION(substr, str, start)} for + * PPL's 3-arg {@code locate(substr, str, start)} (PPL's frontend maps both surface spellings + * into {@link SqlKind#POSITION}). DataFusion's {@code strpos} is + * {@code (str, substr)} with no {@code start} parameter, so: + * + *

    + *
  • 2-arg form: swap operands → {@code strpos(str, substr)}.
    • + *
  • 3-arg form: decompose as + * {@code CASE WHEN strpos(substring(str, start), substr) = 0 + * THEN 0 + * ELSE strpos(substring(str, start), substr) + start - 1 + * END}. + * Preserves 1-indexed semantics and returns 0 when the substring isn't found.
    • + *
+ * + * @opensearch.internal + */ +class PositionAdapter implements ScalarFunctionAdapter { + + /** Locally-declared {@code strpos} operator. The + * {@link io.substrait.isthmus.expression.FunctionMappings.Sig} entry in + * {@link DataFusionFragmentConvertor#ADDITIONAL_SCALAR_SIGS} pairs it with the + * {@code strpos} extension name declared in {@code opensearch_scalar_functions.yaml}. */ + static final SqlFunction STRPOS = new SqlFunction( + "strpos", + SqlKind.OTHER_FUNCTION, + ReturnTypes.INTEGER, + null, + OperandTypes.ANY_ANY, + SqlFunctionCategory.STRING + ); + + @Override + public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) { + List operands = original.getOperands(); + if (operands.size() < 2 || operands.size() > 3) { + return original; + } + + RexBuilder rexBuilder = cluster.getRexBuilder(); + RexNode substr = operands.get(0); + RexNode str = operands.get(1); + + if (operands.size() == 2) { + // Simple swap: POSITION(substr, str) → strpos(str, substr) + return rexBuilder.makeCall(original.getType(), STRPOS, List.of(str, substr)); + } + + // 3-arg: POSITION(substr, str, start) → decompose via substring. + RexNode start = operands.get(2); + RelDataTypeFactory typeFactory = cluster.getTypeFactory(); + RelDataType intType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.INTEGER), true); + + // tail = substring(str, start) + RexNode tail = rexBuilder.makeCall(SqlStdOperatorTable.SUBSTRING, str, start); + // posInTail = strpos(tail, substr) — 1-indexed, 0 when not found. + RexNode posInTail = rexBuilder.makeCall(STRPOS, tail, substr); + + RexNode zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO, intType); + RexNode one = rexBuilder.makeExactLiteral(BigDecimal.ONE, intType); + RexNode isZero = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, posInTail, zero); + RexNode adjusted = rexBuilder.makeCall( + SqlStdOperatorTable.MINUS, + rexBuilder.makeCall(SqlStdOperatorTable.PLUS, posInTail, start), + one + ); + + // CASE WHEN posInTail = 0 THEN 0 ELSE posInTail + start - 1 END + return rexBuilder.makeCall(intType, SqlStdOperatorTable.CASE, List.of(isZero, zero, adjusted)); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/StrcmpFunctionAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/StrcmpFunctionAdapter.java new file mode 100644 index 0000000000000..b59be4bf17008 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/StrcmpFunctionAdapter.java @@ -0,0 +1,93 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; + +import java.math.BigDecimal; +import java.util.List; + +/** + * Adapts PPL {@code strcmp(a, b)} into a pure Substrait/DataFusion CASE expression. + *
    + *
  • {@code -1} when {@code a < b}
    • + *
  • {@code 0} when {@code a = b}
    • + *
  • {@code 1} when {@code a > b}
    • + *
  • {@code NULL} when either operand is {@code NULL}
    • + *
+ * + *

Rewrite: + * 

{@code
+ *   strcmp(a, b)
+ *     →
+ *   CASE
+ *     WHEN a IS NULL OR b IS NULL THEN NULL
+ *     WHEN a < b THEN -1
+ *     WHEN a = b THEN  0
+ *     ELSE              1
+ *   END
+ * }
+ * + *

Why the adapter beats a row-by-row Rust UDF: the {@code <} and {@code =} + * comparisons between {@code StringArray} operands lower to arrow-rs compute + * kernels ({@code arrow::compute::lt}, {@code arrow::compute::eq}) which are + * SIMD-vectorized on x86_64 (AVX2) and arm64 (NEON). The CASE ({@code ifelse}) + * is also an arrow vectorized kernel. A UDF that loops + * {@code for i in 0..n { str::cmp(...) }} per row is strictly slower — it + * amortizes FFI over the batch but the inner compare is scalar. + * + *

PPL's frontend reverses {@code strcmp}'s args vs. user order. This adapter + * swaps them back — operands are consumed as {@code (arg1, arg0)} from the + * original call so the resulting {@code a < b} / {@code a = b} maps 1:1 to the + * user-intended {@code -1 / 0 / 1} convention. + * + * @opensearch.internal + */ +class StrcmpFunctionAdapter implements ScalarFunctionAdapter { + + @Override + public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) { + List operands = original.getOperands(); + if (operands.size() != 2) { + return original; + } + RexBuilder rexBuilder = cluster.getRexBuilder(); + // Swap to undo the PPL frontend's argument reversal. + RexNode a = operands.get(1); + RexNode b = operands.get(0); + + RelDataType intType = cluster.getTypeFactory() + .createTypeWithNullability(cluster.getTypeFactory().createSqlType(SqlTypeName.INTEGER), true); + RexNode neg1 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(-1), intType); + RexNode zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO, intType); + RexNode one = rexBuilder.makeExactLiteral(BigDecimal.ONE, intType); + RexNode nullLit = rexBuilder.makeNullLiteral(intType); + + // NULL propagation must be explicit — SQL comparators on NULL return NULL, but + // the CASE below needs to short-circuit them so we don't fall through to the + // `ELSE 1` branch when either operand is NULL. + RexNode aIsNull = rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, a); + RexNode bIsNull = rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, b); + RexNode anyNull = rexBuilder.makeCall(SqlStdOperatorTable.OR, aIsNull, bIsNull); + + RexNode lessThan = rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, a, b); + RexNode equalTo = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, a, b); + + // CASE WHEN anyNull THEN NULL WHEN aPer the + * + * PPL {@code tonumber} docs: + * + *

+ * {@code tonumber(string[, base])} — converts the string value to a number. If the + * {@code base} parameter is omitted, base 10 is assumed. Returns NULL when the string + * cannot be parsed. + *
+ * + * @opensearch.internal + */ +class ToNumberFunctionAdapter implements ScalarFunctionAdapter { + + static final SqlFunction TONUMBER = new SqlFunction( + "tonumber", + SqlKind.OTHER_FUNCTION, + ReturnTypes.DOUBLE, + null, + OperandTypes.family(), + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + @Override + public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) { + List operands = original.getOperands(); + if (operands.isEmpty()) { + return original; + } + RexNode value = operands.get(0); + + // 1-arg — implied base 10. DataFusion's built-in CAST(str AS DOUBLE) returns NULL on + // parse failure. + if (operands.size() == 1) { + return makeSafeDoubleCast(value, cluster); + } + + // 2-arg — rebuild as tonumber(CAST(value AS VARCHAR), CAST(base AS INTEGER)) + if (operands.size() == 2) { + RexNode base = operands.get(1); + RexNode normalizedValue = castTo(value, SqlTypeName.VARCHAR, cluster); + RexNode normalizedBase = castTo(base, SqlTypeName.INTEGER, cluster); + return cluster.getRexBuilder().makeCall(original.getType(), TONUMBER, List.of(normalizedValue, normalizedBase)); + } + + return original; + } + + /** + * Casts {@code operand} to {@code target} while preserving its nullability. Returns the + * operand unchanged when it's already the target type so we don't layer redundant CASTs. + */ + private static RexNode castTo(RexNode operand, SqlTypeName target, RelOptCluster cluster) { + if (operand.getType().getSqlTypeName() == target) { + return operand; + } + RelDataTypeFactory factory = cluster.getTypeFactory(); + RelDataType targetType = factory.createTypeWithNullability(factory.createSqlType(target), operand.getType().isNullable()); + return cluster.getRexBuilder().makeCast(targetType, operand); + } + + /** + * Wraps the single operand in a SAFE_CAST to DOUBLE. SAFE_CAST serialises as a substrait + * cast with {@code FAILURE_BEHAVIOR_RETURN_NULL}, which DataFusion maps to + * {@code try_cast} — so parse failures yield NULL instead of raising. + */ + private static RexNode makeSafeDoubleCast(RexNode value, RelOptCluster cluster) { + RelDataTypeFactory factory = cluster.getTypeFactory(); + RelDataType doubleType = factory.createTypeWithNullability(factory.createSqlType(SqlTypeName.DOUBLE), true); + // RexBuilder.makeCast(type, exp, matchNullability, safe) — the `safe` flag produces a + // SqlKind.SAFE_CAST call instead of a plain CAST. + return cluster.getRexBuilder().makeCast(doubleType, value, true, true); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ToStringFunctionAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ToStringFunctionAdapter.java new file mode 100644 index 0000000000000..583b5975383eb --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ToStringFunctionAdapter.java @@ -0,0 +1,225 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; + +import java.util.List; +import java.util.Locale; + +/** + * Rewrites PPL {@code tostring(value[, format])} into a DataFusion-compatible expression. + * + *

Per the + * + * PPL {@code tostring} docs: + * + *

+ * {@code tostring(value[, format])} — converts the value to a string representation. + * If a format is provided, converts numbers to the specified format type. For Boolean + * values, converts to {@code TRUE} or {@code FALSE}. The {@code format} parameter is + * only used when {@code value} is a number and is ignored for Booleans. + *
+ * + *

Handles two arrival shapes: + *

    + *
  1. Native {@code tostring(value[, format])} — dispatched as-is.
    2. + *
  2. {@code NUMBER_TO_STRING(num)} — PPL's {@code ExtendedRexBuilder.makeCast} override + * intercepts {@code CAST(num AS VARCHAR)} for approximate-numeric / decimal source types + * and rewrites it into a call to {@code PPLBuiltinOperators.NUMBER_TO_STRING}. That + * PPL-plugin-defined UDF isn't in any Substrait catalog, so isthmus cannot resolve it. + * We treat it as the single-arg {@code tostring} shape and lower it to a plain VARCHAR + * CAST. + *
    3. + *
+ * + * @opensearch.internal + */ +class ToStringFunctionAdapter implements ScalarFunctionAdapter { + + /** + * Target numeric type for a given PPL format mode. {@link #COMMAS} preserves fractional + * precision because it renders rounded to 2 decimals. All other modes fold to BIGINT because + * their output is defined on the integer part of the value (cf. PPL docs: binary/hex/duration + * are integer conversions). + */ + private enum Format { + HEX("hex", SqlTypeName.BIGINT), + BINARY("binary", SqlTypeName.BIGINT), + COMMAS("commas", /* preserveFractional */ null), + DURATION("duration", SqlTypeName.BIGINT), + DURATION_MILLIS("duration_millis", SqlTypeName.BIGINT); + + final String literal; + /** + * Target type for the numeric argument, or {@code null} for {@link #COMMAS} which + * picks BIGINT vs DOUBLE based on the source type. + */ + final SqlTypeName fixedTarget; + + Format(String literal, SqlTypeName fixedTarget) { + this.literal = literal; + this.fixedTarget = fixedTarget; + } + + /** Case-insensitive lookup matching the PPL spec. Returns {@code null} when unknown. */ + static Format from(String modeLiteral) { + if (modeLiteral == null) return null; + String lower = modeLiteral.toLowerCase(Locale.ROOT); + for (Format f : values()) { + if (f.literal.equals(lower)) return f; + } + return null; + } + + /** + * Choose the target type for the numeric argument given the source RexNode type. + * For every mode except {@link #COMMAS} this is a fixed BIGINT; COMMAS preserves + * fractional types by routing through DOUBLE and widens integers to BIGINT. + */ + SqlTypeName targetFor(SqlTypeName source) { + if (fixedTarget != null) { + return fixedTarget; + } + return isFractional(source) ? SqlTypeName.DOUBLE : SqlTypeName.BIGINT; + } + } + + /** + * Synthetic {@code tostring} operator used when we rebuild the 2-arg call. It mirrors the + * shape of the PPL operator but is keyed on the literal name {@code "tostring"} — which is + * the name the Rust UDF registers under and the YAML extension declares. A dedicated operator + * gives the isthmus name-based resolver a deterministic hook; we don't have to rely on the + * incoming RexCall's operator being correctly named. + */ + static final SqlFunction TOSTRING = new SqlFunction( + "tostring", + SqlKind.OTHER_FUNCTION, + ReturnTypes.VARCHAR, + null, + OperandTypes.family(), + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + @Override + public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) { + List operands = original.getOperands(); + if (operands.isEmpty()) { + return original; + } + RexNode value = operands.get(0); + + // NUMBER_TO_STRING is PPL's intercepted numeric-to-varchar cast. Treat it identically to + // the 1-arg tostring shape: lower to a plain CAST(value AS VARCHAR) that isthmus / + // DataFusion can serialise. + if (ScalarFunction.NUMBER_TO_STRING.name().equalsIgnoreCase(original.getOperator().getName())) { + return makeVarcharCast(original, value, cluster); + } + + // tostring renders booleans as the uppercase literals TRUE / FALSE (format arg is ignored for booleans). + if (value.getType().getSqlTypeName() == SqlTypeName.BOOLEAN) { + return makeBooleanToString(original, value, cluster); + } + + // 1-arg: tostring(x) → CAST(x AS VARCHAR) + if (operands.size() == 1) { + return makeVarcharCast(original, value, cluster); + } + + // 2-arg: tostring(x, format). Only rewrite when the format arg is a string literal with + // a known mode; otherwise pass the call through so the downstream planner fails loudly. + if (operands.size() == 2 && operands.get(1) instanceof RexLiteral formatLit && isStringLiteral(formatLit)) { + Format mode = Format.from(formatLit.getValueAs(String.class)); + if (mode != null) { + return rebuildCall(original, value, formatLit, mode, cluster); + } + } + + return original; + } + + /** + * Lower a BOOLEAN-valued {@code tostring} call to + * {@code CASE WHEN value THEN 'TRUE' WHEN NOT value THEN 'FALSE' END}. + */ + private static RexNode makeBooleanToString(RexCall original, RexNode value, RelOptCluster cluster) { + RelDataTypeFactory factory = cluster.getTypeFactory(); + RelDataType varcharType = factory.createTypeWithNullability( + factory.createSqlType(SqlTypeName.VARCHAR), + original.getType().isNullable() + ); + RexNode trueLit = cluster.getRexBuilder().makeLiteral("TRUE"); + RexNode falseLit = cluster.getRexBuilder().makeLiteral("FALSE"); + RexNode notValue = cluster.getRexBuilder().makeCall(SqlStdOperatorTable.NOT, value); + return cluster.getRexBuilder() + .makeCall( + varcharType, + SqlStdOperatorTable.CASE, + List.of(value, trueLit, notValue, falseLit, cluster.getRexBuilder().makeNullLiteral(varcharType)) + ); + } + + /** + * Rebuild the 2-arg call as {@code tostring(CAST(value AS ), formatLit)}. The CAST + * ensures the numeric argument matches the Rust UDF's declared BIGINT/FLOAT64 signatures; + * the format literal is forwarded verbatim so the UDF's per-row dispatch sees the exact + * mode string the caller supplied. + */ + private static RexNode rebuildCall(RexCall original, RexNode value, RexLiteral formatLit, Format mode, RelOptCluster cluster) { + SqlTypeName target = mode.targetFor(value.getType().getSqlTypeName()); + RexNode normalized = castTo(value, target, cluster); + return cluster.getRexBuilder().makeCall(original.getType(), TOSTRING, List.of(normalized, formatLit)); + } + + private static boolean isStringLiteral(RexLiteral literal) { + SqlTypeName sqlType = literal.getType().getSqlTypeName(); + return sqlType == SqlTypeName.CHAR || sqlType == SqlTypeName.VARCHAR; + } + + private static boolean isFractional(SqlTypeName type) { + return type == SqlTypeName.FLOAT || type == SqlTypeName.DOUBLE || type == SqlTypeName.REAL || type == SqlTypeName.DECIMAL; + } + + /** + * Casts {@code operand} to {@code target} while preserving its nullability. Returns the + * operand unchanged when it's already the target type so we don't layer redundant CASTs. + */ + private static RexNode castTo(RexNode operand, SqlTypeName target, RelOptCluster cluster) { + if (operand.getType().getSqlTypeName() == target) { + return operand; + } + RelDataTypeFactory factory = cluster.getTypeFactory(); + RelDataType targetType = factory.createTypeWithNullability(factory.createSqlType(target), operand.getType().isNullable()); + return cluster.getRexBuilder().makeCast(targetType, operand); + } + + private static RexNode makeVarcharCast(RexCall original, RexNode value, RelOptCluster cluster) { + RelDataTypeFactory factory = cluster.getTypeFactory(); + RelDataType varcharType = factory.createTypeWithNullability( + factory.createSqlType(SqlTypeName.VARCHAR), + original.getType().isNullable() + ); + return cluster.getRexBuilder().makeCast(varcharType, value); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml index 34e70f5c3bdc5..4caf7c185dc96 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml @@ -253,3 +253,67 @@ scalar_functions: - value: "string" name: "replacement" return: string + + # ascii(str) — Unicode code point of the first character. + - name: "ascii" + description: "Return the unicode code point of the first character of the input string." + impls: + - args: + - { name: str, value: "varchar" } + nullability: DECLARED_OUTPUT + return: i32 + - args: + - { name: str, value: "string" } + nullability: DECLARED_OUTPUT + return: i32 + + # strpos(str, substr) — 1-based position of substr in str, 0 if not found. + # Target of PPL's `locate` and `position` adapters. + - name: "strpos" + description: "Return the 1-based position of `substr` within `str`, or 0 when absent." + impls: + - args: + - { name: str, value: "string" } + - { name: substr, value: "string" } + nullability: DECLARED_OUTPUT + return: i32 + - args: + - { name: str, value: "varchar" } + - { name: substr, value: "varchar" } + nullability: DECLARED_OUTPUT + return: i32 + - args: + - { name: str, value: "string" } + - { name: substr, value: "varchar" } + nullability: DECLARED_OUTPUT + return: i32 + - args: + - { name: str, value: "varchar" } + - { name: substr, value: "string" } + nullability: DECLARED_OUTPUT + return: i32 + + # tostring(x, format) — (hex / binary / commas / duration / duration_millis). + - name: "tostring" + description: "Convert a number to a string using the requested format (hex/binary/commas/duration/duration_millis)." + impls: + - args: + - { name: value, value: i64 } + - { name: format, value: string } + nullability: DECLARED_OUTPUT + return: string + - args: + - { name: value, value: fp64 } + - { name: format, value: string } + nullability: DECLARED_OUTPUT + return: string + + # tonumber(string, base) — parse `string` as a base-N integer + - name: "tonumber" + description: "Parse a string to a number in the given radix (2-36). Returns NULL on parse failure." + impls: + - args: + - { name: value, value: string } + - { name: base, value: i32 } + nullability: DECLARED_OUTPUT + return: fp64 diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/PositionAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/PositionAdapterTests.java new file mode 100644 index 0000000000000..22a9ea44420cf --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/PositionAdapterTests.java @@ -0,0 +1,236 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.test.OpenSearchTestCase; + +import java.math.BigDecimal; +import java.util.List; + +/** + * Unit tests for {@link PositionAdapter}. + * + *

Coverage: + *

    + *
  • 2-arg form: {@code POSITION(substr, str)} swaps operands to + * {@code strpos(str, substr)}.
    • + *
  • 3-arg form: {@code POSITION(substr, str, start)} decomposes into a CASE + * expression around {@code substring(str, start)} + {@code strpos} + offset + * arithmetic so the 1-indexed {@code start} parameter and the + * "{@code 0} on not found" contract both hold.
    • + *
  • Malformed arity passes through unchanged (no 0, 1, or 4-arg rewrite).
    • + *
+ */ +public class PositionAdapterTests extends OpenSearchTestCase { + + private static final SqlFunction POSITION = new SqlFunction( + "POSITION", + SqlKind.POSITION, + ReturnTypes.INTEGER, + null, + OperandTypes.family(), + SqlFunctionCategory.STRING + ); + + private final PositionAdapter adapter = new PositionAdapter(); + + /** {@code POSITION('U', 'FURNITURE')} → {@code strpos('FURNITURE', 'U')}. */ + public void testTwoArgSwapsOperands() { + Cluster cluster = newCluster(); + RexNode substr = cluster.stringLiteral("U"); + RexNode str = cluster.stringLiteral("FURNITURE"); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(POSITION, substr, str); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + RexCall outCall = assertStrposCall(out); + assertEquals("strpos must be (str, substr) — 2 operands", 2, outCall.getOperands().size()); + assertSame("first operand is str (was the second POSITION arg)", str, outCall.getOperands().get(0)); + assertSame("second operand is substr (was the first POSITION arg)", substr, outCall.getOperands().get(1)); + } + + /** + * {@code POSITION('U', 'FURNITURE', 3)} decomposes to + * {@code CASE WHEN strpos(substring(str, start), substr) = 0 THEN 0 ELSE strpos(...) + start - 1 END}. + * This test asserts the outer CASE shape; the inner sub-calls are validated separately. + */ + public void testThreeArgDecomposesToCaseOfSubstringStrpos() { + Cluster cluster = newCluster(); + RexNode substr = cluster.stringLiteral("U"); + RexNode str = cluster.stringLiteral("FURNITURE"); + RexNode start = cluster.intLiteral(3); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(POSITION, substr, str, start); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + assertEquals("3-arg POSITION lowers to CASE", SqlKind.CASE, out.getKind()); + RexCall caseCall = (RexCall) out; + assertEquals("CASE shape — WHEN cond THEN 0 ELSE adjusted", 3, caseCall.getOperands().size()); + + // operand[0]: strpos(substring(str, start), substr) = 0 + RexCall whenCond = (RexCall) caseCall.getOperands().get(0); + assertEquals("WHEN is an equality test", SqlKind.EQUALS, whenCond.getKind()); + + // operand[1]: the THEN value is the literal 0. + assertEquals( + "THEN returns 0 when substring didn't contain substr", + 0, + ((org.apache.calcite.rex.RexLiteral) caseCall.getOperands().get(1)).getValueAs(Integer.class).intValue() + ); + + // operand[2]: the ELSE arm is strpos(...) + start - 1. + RexCall elseArm = (RexCall) caseCall.getOperands().get(2); + assertEquals("ELSE performs the final offset subtraction", SqlKind.MINUS, elseArm.getKind()); + } + + public void testThreeArgElseArmBuildsSubstringAndStrpos() { + Cluster cluster = newCluster(); + RexNode substr = cluster.stringLiteral("U"); + RexNode str = cluster.stringLiteral("FURNITURE"); + RexNode start = cluster.intLiteral(3); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(POSITION, substr, str, start); + + RexCall caseCall = (RexCall) adapter.adapt(call, List.of(), cluster.cluster); + + // ELSE shape: MINUS(PLUS(strpos(substring(str, start), substr), start), 1) + RexCall minusCall = (RexCall) caseCall.getOperands().get(2); + RexCall plusCall = (RexCall) minusCall.getOperands().get(0); + assertEquals(SqlKind.PLUS, plusCall.getKind()); + RexCall strposInElse = (RexCall) plusCall.getOperands().get(0); + assertSame("ELSE arm's strpos reuses the shared operator", PositionAdapter.STRPOS, strposInElse.getOperator()); + + RexCall substringCall = (RexCall) strposInElse.getOperands().get(0); + assertSame( + "substring call uses the standard SqlStdOperatorTable.SUBSTRING", + SqlStdOperatorTable.SUBSTRING, + substringCall.getOperator() + ); + assertSame("substring(str, start) — str is the original second POSITION operand", str, substringCall.getOperands().get(0)); + assertSame("substring(str, start) — start is the original third POSITION operand", start, substringCall.getOperands().get(1)); + assertSame("strpos substr is the original first POSITION operand", substr, strposInElse.getOperands().get(1)); + } + + public void testThreeArgWhenConditionMirrorsElseStrpos() { + Cluster cluster = newCluster(); + RexNode substr = cluster.stringLiteral("U"); + RexNode str = cluster.stringLiteral("FURNITURE"); + RexNode start = cluster.intLiteral(3); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(POSITION, substr, str, start); + + RexCall caseCall = (RexCall) adapter.adapt(call, List.of(), cluster.cluster); + + RexCall whenCond = (RexCall) caseCall.getOperands().get(0); + // WHEN: strpos(substring(str, start), substr) = 0 + RexCall strposInWhen = (RexCall) whenCond.getOperands().get(0); + assertSame("WHEN condition's strpos is the shared operator", PositionAdapter.STRPOS, strposInWhen.getOperator()); + RexCall substringInWhen = (RexCall) strposInWhen.getOperands().get(0); + assertSame(SqlStdOperatorTable.SUBSTRING, substringInWhen.getOperator()); + assertSame(str, substringInWhen.getOperands().get(0)); + assertSame(start, substringInWhen.getOperands().get(1)); + } + + public void testAdaptedStrposIsTheSharedOperatorInstance() { + Cluster cluster = newCluster(); + RexNode substr = cluster.stringLiteral("a"); + RexNode str = cluster.stringLiteral("abc"); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(POSITION, substr, str); + + RexCall outCall = assertStrposCall(adapter.adapt(call, List.of(), cluster.cluster)); + + assertSame( + "adapter must emit the shared PositionAdapter.STRPOS instance, not a clone", + PositionAdapter.STRPOS, + outCall.getOperator() + ); + assertEquals( + "operator name is 'strpos' — what DataFusion's substrait consumer expects", + "strpos", + PositionAdapter.STRPOS.getName() + ); + } + + public void testOneArgPassesThrough() { + Cluster cluster = newCluster(); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(POSITION, cluster.stringLiteral("a")); + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + assertSame("1-arg POSITION is malformed and must pass through", call, out); + } + + public void testFourArgPassesThrough() { + Cluster cluster = newCluster(); + RexCall call = (RexCall) cluster.rexBuilder.makeCall( + POSITION, + cluster.stringLiteral("a"), + cluster.stringLiteral("abc"), + cluster.intLiteral(1), + cluster.intLiteral(1) + ); + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + assertSame("4-arg POSITION is malformed and must pass through", call, out); + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + /** Assert the adapted call is a 2-arg {@code strpos} call routed through the shared operator. */ + private static RexCall assertStrposCall(RexNode out) { + assertTrue("expected a RexCall, got " + out.getClass(), out instanceof RexCall); + RexCall outCall = (RexCall) out; + assertSame( + "operator is the shared strpos registered against the FunctionMappings.Sig", + PositionAdapter.STRPOS, + outCall.getOperator() + ); + return outCall; + } + + private static Cluster newCluster() { + RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + RexBuilder rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); + return new Cluster(cluster, typeFactory, rexBuilder); + } + + private static final class Cluster { + final RelOptCluster cluster; + final RelDataTypeFactory typeFactory; + final RexBuilder rexBuilder; + + Cluster(RelOptCluster cluster, RelDataTypeFactory typeFactory, RexBuilder rexBuilder) { + this.cluster = cluster; + this.typeFactory = typeFactory; + this.rexBuilder = rexBuilder; + } + + RexNode intLiteral(int value) { + RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); + return rexBuilder.makeExactLiteral(BigDecimal.valueOf(value), intType); + } + + RexNode stringLiteral(String value) { + return rexBuilder.makeLiteral(value); + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/StrcmpFunctionAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/StrcmpFunctionAdapterTests.java new file mode 100644 index 0000000000000..130412f24c1fd --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/StrcmpFunctionAdapterTests.java @@ -0,0 +1,110 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +/** + * Unit tests for {@link StrcmpFunctionAdapter}. + * + *

The adapter decomposes {@code strcmp(a, b)} into a CASE expression using built-in + * comparison operators ({@code <}, {@code =}) and swaps the arguments to undo the PPL + * frontend's reversal. These tests verify the CASE shape and argument swap. + */ +public class StrcmpFunctionAdapterTests extends OpenSearchTestCase { + + private static final SqlFunction STRCMP = new SqlFunction( + "STRCMP", + SqlKind.OTHER_FUNCTION, + ReturnTypes.INTEGER, + null, + OperandTypes.family(), + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private final StrcmpFunctionAdapter adapter = new StrcmpFunctionAdapter(); + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private RelOptCluster cluster; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + cluster = RelOptCluster.create(planner, rexBuilder); + } + + private RexNode varcharInputRef(int index) { + RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + return rexBuilder.makeInputRef(varcharType, index); + } + + /** The adapter produces a CASE expression with INTEGER return type. */ + public void testTwoArgProducesCaseExpression() { + RexNode arg0 = rexBuilder.makeLiteral("Amber"); + RexNode arg1 = varcharInputRef(0); + RexCall call = (RexCall) rexBuilder.makeCall(STRCMP, arg0, arg1); + + RexNode out = adapter.adapt(call, List.of(), cluster); + + assertTrue("result must be a RexCall", out instanceof RexCall); + RexCall outCall = (RexCall) out; + assertEquals("decomposed to CASE", SqlKind.CASE, outCall.getKind()); + assertEquals("return type is INTEGER", SqlTypeName.INTEGER, outCall.getType().getSqlTypeName()); + // CASE has 7 operands: (anyNull, nullLit, lessThan, neg1, equalTo, zero, one) + assertEquals("CASE has 7 operands (3 WHEN/THEN pairs + ELSE)", 7, outCall.getOperands().size()); + } + + /** Arguments are swapped — arg1 becomes 'a' (lhs) and arg0 becomes 'b' (rhs) in the comparisons. */ + public void testArgumentsAreSwapped() { + RexNode arg0 = rexBuilder.makeLiteral("literal_rhs"); + RexNode arg1 = varcharInputRef(0); // column — should become lhs after swap + RexCall call = (RexCall) rexBuilder.makeCall(STRCMP, arg0, arg1); + + RexNode out = adapter.adapt(call, List.of(), cluster); + + RexCall caseCall = (RexCall) out; + // The LESS_THAN comparison is at operand index 2: WHEN a < b THEN -1 + // After swap: a = arg1 (inputRef), b = arg0 (literal) + RexCall lessThan = (RexCall) caseCall.getOperands().get(2); + assertEquals(SqlKind.LESS_THAN, lessThan.getKind()); + // lhs of < should be the column (arg1), rhs should be the literal (arg0) + assertSame("lhs of < is the column (original arg1)", arg1, lessThan.getOperands().get(0)); + assertSame("rhs of < is the literal (original arg0)", arg0, lessThan.getOperands().get(1)); + } + + /** Non-standard arity (e.g. 1 arg) passes through unchanged. */ + public void testSingleArgPassesThrough() { + RexCall call = (RexCall) rexBuilder.makeCall(STRCMP, varcharInputRef(0)); + + RexNode out = adapter.adapt(call, List.of(), cluster); + + assertSame("non-2-arg call passes through unchanged", call, out); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToNumberFunctionAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToNumberFunctionAdapterTests.java new file mode 100644 index 0000000000000..ac6337e998485 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToNumberFunctionAdapterTests.java @@ -0,0 +1,164 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.test.OpenSearchTestCase; + +import java.math.BigDecimal; +import java.util.List; + +public class ToNumberFunctionAdapterTests extends OpenSearchTestCase { + + /** Synthetic tonumber operator used to build input RexCalls */ + private static final SqlFunction TONUMBER = new SqlFunction( + "tonumber", + SqlKind.OTHER_FUNCTION, + ReturnTypes.DOUBLE, + null, + OperandTypes.ANY_ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private final ToNumberFunctionAdapter adapter = new ToNumberFunctionAdapter(); + + /** {@code tonumber(x)} rewrites to {@code CAST(x AS DOUBLE)}. */ + public void testSingleArgRewritesToDoubleCast() { + Cluster cluster = newCluster(); + RexNode input = cluster.stringLiteral("4598.678"); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(TONUMBER, input); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + assertEquals("kind=SAFE_CAST", SqlKind.SAFE_CAST, out.getKind()); + assertEquals("result type is DOUBLE", SqlTypeName.DOUBLE, out.getType().getSqlTypeName()); + RexCall castCall = (RexCall) out; + assertEquals("single operand", 1, castCall.getOperands().size()); + assertSame("operand preserved by identity", input, castCall.getOperands().get(0)); + } + + /** + * {@code tonumber(x, base)} stays a {@code tonumber} + */ + public void testTwoArgKeepsTonumberCallAndNormalizesOperands() { + Cluster cluster = newCluster(); + RexNode input = cluster.stringLiteral("FA34"); + RexNode base = cluster.intLiteral(16); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(TONUMBER, input, base); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + RexCall outCall = assertTonumberCall(out); + assertEquals("two operands — value + base", 2, outCall.getOperands().size()); + RexNode valueArg = outCall.getOperands().get(0); + RexNode baseArg = outCall.getOperands().get(1); + assertEquals("value arg normalized to VARCHAR", SqlTypeName.VARCHAR, valueArg.getType().getSqlTypeName()); + assertEquals("base arg normalized to INTEGER", SqlTypeName.INTEGER, baseArg.getType().getSqlTypeName()); + } + + /** {@code tonumber(VARCHAR, INTEGER)} — already-normalized operands don't get redundant CASTs. */ + public void testTwoArgOnMatchingTypesSkipsRedundantCast() { + Cluster cluster = newCluster(); + RexNode input = cluster.varcharInputRef(0); + RexNode base = cluster.intLiteral(2); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(TONUMBER, input, base); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + RexCall outCall = assertTonumberCall(out); + assertSame("VARCHAR operand kept as-is", input, outCall.getOperands().get(0)); + assertSame("INTEGER literal kept as-is", base, outCall.getOperands().get(1)); + } + + /** Zero-operand {@code tonumber} is degenerate; adapter should pass it through unchanged. */ + public void testZeroArgPassesThrough() { + Cluster cluster = newCluster(); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(TONUMBER); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + assertSame(call, out); + } + + /** Arities above 2 aren't declared in the PPL spec — pass through so planning fails loudly. */ + public void testThreeArgPassesThrough() { + Cluster cluster = newCluster(); + RexCall call = (RexCall) cluster.rexBuilder.makeCall( + TONUMBER, + cluster.stringLiteral("10"), + cluster.intLiteral(10), + cluster.intLiteral(0) + ); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + assertSame(call, out); + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + private static RexCall assertTonumberCall(RexNode out) { + assertTrue("expected a RexCall, got " + out.getClass(), out instanceof RexCall); + RexCall outCall = (RexCall) out; + assertSame( + "operator is the synthetic `tonumber` that resolves to the Rust UDF", + ToNumberFunctionAdapter.TONUMBER, + outCall.getOperator() + ); + return outCall; + } + + private static Cluster newCluster() { + RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + RexBuilder rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); + return new Cluster(cluster, typeFactory, rexBuilder); + } + + private static final class Cluster { + final RelOptCluster cluster; + final RelDataTypeFactory typeFactory; + final RexBuilder rexBuilder; + + Cluster(RelOptCluster cluster, RelDataTypeFactory typeFactory, RexBuilder rexBuilder) { + this.cluster = cluster; + this.typeFactory = typeFactory; + this.rexBuilder = rexBuilder; + } + + RexNode intLiteral(int value) { + RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); + return rexBuilder.makeExactLiteral(BigDecimal.valueOf(value), intType); + } + + RexNode stringLiteral(String value) { + return rexBuilder.makeLiteral(value); + } + + RexNode varcharInputRef(int index) { + RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + return rexBuilder.makeInputRef(varcharType, index); + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToStringFunctionAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToStringFunctionAdapterTests.java new file mode 100644 index 0000000000000..beafd6e34f5e7 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToStringFunctionAdapterTests.java @@ -0,0 +1,286 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.test.OpenSearchTestCase; + +import java.math.BigDecimal; +import java.util.List; + +public class ToStringFunctionAdapterTests extends OpenSearchTestCase { + + private final ToStringFunctionAdapter adapter = new ToStringFunctionAdapter(); + + /** Synthetic tostring operator used to build input RexCalls. */ + private static final SqlFunction TOSTRING = new SqlFunction( + "tostring", + SqlKind.OTHER_FUNCTION, + ReturnTypes.VARCHAR, + null, + OperandTypes.ANY_ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + /** {@code tostring(x)} rewrites to {@code CAST(x AS VARCHAR)}. */ + public void testSingleArgRewritesToVarcharCast() { + Cluster cluster = newCluster(); + RexNode input = cluster.intLiteral(39225); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, input); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + assertEquals("kind=CAST", SqlKind.CAST, out.getKind()); + assertEquals("result type is VARCHAR", SqlTypeName.VARCHAR, out.getType().getSqlTypeName()); + RexCall castCall = (RexCall) out; + assertEquals("single operand", 1, castCall.getOperands().size()); + assertSame("operand preserved by identity", input, castCall.getOperands().get(0)); + } + + /** + * {@code tostring(x, 'hex')} stays a {@code tostring} call (operator rebound to the + * name the Rust UDF registers under) with the numeric argument widened to BIGINT. + */ + public void testHexFormatKeepsTostringCallAndWidensToBigint() { + Cluster cluster = newCluster(); + RexNode intInput = cluster.intLiteral(255); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, intInput, cluster.stringLiteral("hex")); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + RexCall outCall = assertTostringCall(out); + assertEquals("two operands — value + format literal", 2, outCall.getOperands().size()); + RexNode operand = outCall.getOperands().get(0); + assertEquals("integer widened to BIGINT to match the UDF signature", SqlTypeName.BIGINT, operand.getType().getSqlTypeName()); + } + + /** {@code tostring(bigint, 'binary')} — no CAST needed because the operand is already BIGINT. */ + public void testBinaryFormatOnBigintDoesNotReinsertCast() { + Cluster cluster = newCluster(); + RexNode bigintInput = cluster.rexBuilder.makeExactLiteral( + BigDecimal.valueOf(100L), + cluster.typeFactory.createSqlType(SqlTypeName.BIGINT) + ); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, bigintInput, cluster.stringLiteral("binary")); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + RexCall outCall = assertTostringCall(out); + assertSame("bigint operand is used directly — no redundant CAST", bigintInput, outCall.getOperands().get(0)); + } + + /** {@code tostring(double, 'commas')} preserves fractional precision by routing through DOUBLE. */ + public void testCommasFormatOnDoublePreservesFractionalPrecision() { + Cluster cluster = newCluster(); + RexNode doubleInput = cluster.rexBuilder.makeApproxLiteral( + BigDecimal.valueOf(12.5), + cluster.typeFactory.createSqlType(SqlTypeName.DOUBLE) + ); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, doubleInput, cluster.stringLiteral("commas")); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + RexCall outCall = assertTostringCall(out); + RexNode operand = outCall.getOperands().get(0); + assertEquals( + "double kept as DOUBLE — 2-decimal rounding happens inside the UDF", + SqlTypeName.DOUBLE, + operand.getType().getSqlTypeName() + ); + } + + /** {@code tostring(int, 'commas')} widens integer sources to BIGINT, same as every other mode. */ + public void testCommasFormatOnIntegerWidensToBigint() { + Cluster cluster = newCluster(); + RexNode intInput = cluster.intLiteral(12345); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, intInput, cluster.stringLiteral("commas")); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + RexCall outCall = assertTostringCall(out); + assertEquals(SqlTypeName.BIGINT, outCall.getOperands().get(0).getType().getSqlTypeName()); + } + + /** {@code tostring(x, 'xyzzy')} is an unsupported format; the call is returned unchanged. */ + public void testUnsupportedFormatPassesThrough() { + Cluster cluster = newCluster(); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, cluster.intLiteral(42), cluster.stringLiteral("xyzzy")); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + assertSame("unknown format mode should leave the RexCall untouched so downstream planning fails loudly", call, out); + } + + /** + * {@code tostring(BOOLEAN)} lowers to a {@code CASE} that emits the uppercase + * {@code 'TRUE'} / {@code 'FALSE'} + */ + public void testBooleanOneArgLowersToCase() { + Cluster cluster = newCluster(); + RexNode boolInput = cluster.booleanLiteral(true); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, boolInput); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + assertEquals("boolean tostring lowers to CASE", SqlKind.CASE, out.getKind()); + assertEquals("CASE returns VARCHAR", SqlTypeName.VARCHAR, out.getType().getSqlTypeName()); + RexCall caseCall = (RexCall) out; + // CASE shape: WHEN value THEN 'TRUE' WHEN NOT value THEN 'FALSE' ELSE NULL. + assertEquals("CASE has two WHEN branches plus ELSE — 5 operands total", 5, caseCall.getOperands().size()); + assertEquals("first THEN literal is uppercase TRUE", "TRUE", ((RexLiteral) caseCall.getOperands().get(1)).getValueAs(String.class)); + assertEquals( + "second THEN literal is uppercase FALSE", + "FALSE", + ((RexLiteral) caseCall.getOperands().get(3)).getValueAs(String.class) + ); + } + + /** + * {@code tostring(BOOLEAN, '')} ignores the format + */ + public void testBooleanTwoArgIgnoresFormat() { + Cluster cluster = newCluster(); + RexNode boolInput = cluster.nullableBooleanInputRef(0); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, boolInput, cluster.stringLiteral("hex")); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + assertEquals("boolean tostring(x, fmt) lowers to CASE regardless of format", SqlKind.CASE, out.getKind()); + RexCall caseCall = (RexCall) out; + assertEquals("TRUE", ((RexLiteral) caseCall.getOperands().get(1)).getValueAs(String.class)); + assertEquals("FALSE", ((RexLiteral) caseCall.getOperands().get(3)).getValueAs(String.class)); + } + + // ── NUMBER_TO_STRING: PPL's intercepted numeric-to-varchar cast ─────────── + + /** + * PPL's {@code ExtendedRexBuilder.makeCast} rewrites {@code CAST(num AS VARCHAR)} into a + * {@code NUMBER_TO_STRING(num)} call. That PPL-plugin UDF isn't in any Substrait catalog, + * so the adapter must lower it back to a plain VARCHAR cast for DataFusion — DataFusion's + * native numeric-to-string formatting is used in place of Java's {@code Number.toString}. + */ + public void testNumberToStringLowersToVarcharCast() { + Cluster cluster = newCluster(); + RexNode doubleInput = cluster.rexBuilder.makeApproxLiteral( + BigDecimal.valueOf(12.3), + cluster.typeFactory.createSqlType(SqlTypeName.DOUBLE) + ); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(NUMBER_TO_STRING, doubleInput); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + assertEquals("NUMBER_TO_STRING lowers to CAST", SqlKind.CAST, out.getKind()); + assertEquals("result type is VARCHAR", SqlTypeName.VARCHAR, out.getType().getSqlTypeName()); + RexCall castCall = (RexCall) out; + assertEquals("single operand", 1, castCall.getOperands().size()); + assertSame("numeric operand preserved by identity", doubleInput, castCall.getOperands().get(0)); + } + + /** + * {@code NUMBER_TO_STRING} over a DECIMAL source — still lowers to a VARCHAR cast. The + * adapter branches on operator name, not operand type, so decimal and approximate-numeric + * paths both route identically. + */ + public void testNumberToStringOnDecimalLowersToVarcharCast() { + Cluster cluster = newCluster(); + RelDataType decimalType = cluster.typeFactory.createSqlType(SqlTypeName.DECIMAL, 10, 2); + RexNode decimalInput = cluster.rexBuilder.makeExactLiteral(BigDecimal.valueOf(12.3), decimalType); + RexCall call = (RexCall) cluster.rexBuilder.makeCall(NUMBER_TO_STRING, decimalInput); + + RexNode out = adapter.adapt(call, List.of(), cluster.cluster); + + assertEquals("decimal NUMBER_TO_STRING also lowers to CAST", SqlKind.CAST, out.getKind()); + assertEquals(SqlTypeName.VARCHAR, out.getType().getSqlTypeName()); + RexCall castCall = (RexCall) out; + assertSame(decimalInput, castCall.getOperands().get(0)); + } + + /** Synthetic {@code NUMBER_TO_STRING} operator — the PPL plugin's + * {@code PPLBuiltinOperators.NUMBER_TO_STRING} isn't reachable from this module, so we + * declare a same-named clone that the adapter will match by + * {@link org.apache.calcite.sql.SqlOperator#getName()}. */ + private static final SqlFunction NUMBER_TO_STRING = new SqlFunction( + "NUMBER_TO_STRING", + SqlKind.OTHER_FUNCTION, + ReturnTypes.VARCHAR, + null, + OperandTypes.NUMERIC, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + // ── Helpers ─────────────────────────────────────────────────────────────── + + /** + * Assert that the rewrite produced a {@code tostring(...)} call routed through + * {@link ToStringFunctionAdapter#TOSTRING}. Returns the RexCall for further assertions. + */ + private static RexCall assertTostringCall(RexNode out) { + assertTrue("expected a RexCall, got " + out.getClass(), out instanceof RexCall); + RexCall outCall = (RexCall) out; + assertSame( + "operator is the synthetic `tostring` that resolves to the Rust UDF", + ToStringFunctionAdapter.TOSTRING, + outCall.getOperator() + ); + return outCall; + } + + private static Cluster newCluster() { + RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + RexBuilder rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); + return new Cluster(cluster, typeFactory, rexBuilder); + } + + private static final class Cluster { + final RelOptCluster cluster; + final RelDataTypeFactory typeFactory; + final RexBuilder rexBuilder; + + Cluster(RelOptCluster cluster, RelDataTypeFactory typeFactory, RexBuilder rexBuilder) { + this.cluster = cluster; + this.typeFactory = typeFactory; + this.rexBuilder = rexBuilder; + } + + RexNode intLiteral(int value) { + RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); + return rexBuilder.makeExactLiteral(BigDecimal.valueOf(value), intType); + } + + RexNode stringLiteral(String value) { + return rexBuilder.makeLiteral(value); + } + + RexNode booleanLiteral(boolean value) { + return rexBuilder.makeLiteral(value); + } + + RexNode nullableBooleanInputRef(int index) { + RelDataType boolType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BOOLEAN), true); + return rexBuilder.makeInputRef(boolType, index); + } + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/ArrowValues.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/ArrowValues.java index f1da7261ce75d..6b0ee31226f39 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/ArrowValues.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/ArrowValues.java @@ -10,6 +10,7 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.util.Text; import java.nio.charset.StandardCharsets; @@ -17,7 +18,7 @@ * Helpers for reading Arrow vector cells as plain Java values at the * external query API edge. */ -final class ArrowValues { +public final class ArrowValues { private ArrowValues() {} @@ -25,14 +26,19 @@ private ArrowValues() {} * Returns the cell at {@code index} in {@code vector} as a Java value: * {@code null} when the cell is null, a UTF-8 {@link String} for * {@link VarCharVector} cells (rather than the raw {@code Text} that - * {@code getObject} returns), and {@link FieldVector#getObject} for - * every other vector type. + * {@code getObject} returns), {@link Text#toString()} for any other vector + * type whose {@code getObject} returns a {@link Text} and + * {@link FieldVector#getObject} for every other vector type. */ - static Object toJavaValue(FieldVector vector, int index) { + public static Object toJavaValue(FieldVector vector, int index) { if (vector.isNull(index)) return null; if (vector instanceof VarCharVector v) { return new String(v.get(index), StandardCharsets.UTF_8); } - return vector.getObject(index); + Object obj = vector.getObject(index); + if (obj instanceof Text t) { + return t.toString(); + } + return obj; } } diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StringScalarFunctionsIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StringScalarFunctionsIT.java new file mode 100644 index 0000000000000..e44ebbad0e422 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StringScalarFunctionsIT.java @@ -0,0 +1,400 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.text.NumberFormat; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * End-to-end coverage for PPL string scalar functions + * + *

Covers three categories of routing: + *

    + *
  • Direct-match Substrait signatures: {@code ascii}, {@code concat}, + * {@code concat_ws}, {@code left}, {@code lower}, {@code ltrim}, + * {@code reverse}, {@code right}, {@code rtrim}, {@code substring}, + * {@code upper}.
    • + *
  • Name-mapping adapter rewrites (PPL name ≠ DataFusion name) registered in + * {@code DataFusionAnalyticsBackendPlugin.scalarFunctionAdapters()}: + * {@code length → char_length}, {@code locate → strpos} (with arg swap + * and optional 3-arg decomposition), {@code position → strpos} (arg swap), + * {@code substr → substring}, {@code trim → btrim}.
    • + *
  • Full {@link org.opensearch.analytics.spi.ScalarFunctionAdapter} plans: + * {@code strcmp} (decomposed to a SIMD-vectorized {@code CASE} expression) + * and {@code tostring} / {@code tonumber}.
    • + *
+ * + *

Each test pins a single row of the {@code calcs} dataset via + * {@code where key='keyNN'} — field references prevent Calcite's + * {@code ReduceExpressionsRule} from constant-folding the expression on the + * coordinator, forcing the call to travel through Substrait into DataFusion + * where the function wiring is actually exercised. + * + *

Where inputs must be literals (e.g. to exercise a specific parse path), + * tests are constructed so the expected output is only producible by the + * function under test — not by Calcite's constant-folder short-circuiting. For + * example, {@code tostring(int0 * 12345, 'commas')} on {@code int0=1} yields + * {@code "12,345"} which proves the commas format path was evaluated; a + * passthrough would produce {@code "12345"}. + * + *

Fixture row values used (from {@code calcs/bulk.json}): + *

    + *
  • {@code key00}: str0="FURNITURE", str2="one", num0=12.3, int0=1, int3=8
    • + *
  • {@code key04}: str0="OFFICE SUPPLIES", str2="five", num0=3.5, int0=7
    • + *
+ */ +public class StringScalarFunctionsIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + /** Base query template: filter to exactly one row (cardinality 1) keyed by {@code key}. */ + private String oneRow(String key) { + return "source=" + DATASET.indexName + " | where key='" + key + "' | head 1 "; + } + + // ── ascii ─────────────────────────────────────────────────────────────── + + /** {@code ascii(str0)} on {@code str0="FURNITURE"} → 70 (ASCII code of 'F') */ + public void testAscii() throws IOException { + assertFirstRowLong(oneRow("key00") + "| eval v = ascii(str0) | fields v", (long) 'F'); + } + + /** {@code ascii(str0)} on {@code key04} (str0="OFFICE SUPPLIES") → 79 (ASCII code of 'O')*/ + public void testAsciiDifferentRow() throws IOException { + assertFirstRowLong(oneRow("key04") + "| eval v = ascii(str0) | fields v", (long) 'O'); + } + + // ── concat / concat_ws ────────────────────────────────────────────────── + + /** Two-field {@code concat(str0, str2)} on row 0 → "FURNITUREone". Both operands are field refs */ + public void testConcat() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = concat(str0, str2) | fields v", "FURNITUREone"); + } + + /** {@code concat_ws(':', str0, str2)} on row 0 → "FURNITURE:one" */ + public void testConcatWs() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = concat_ws(':', str0, str2) | fields v", "FURNITURE:one"); + } + + // ── left / right ───────────────────────────────────────────────────────── + + /** {@code left('FURNITURE', 3)} → "FUR". Verifies length-1 prefix extraction */ + public void testLeft() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = left(str0, 3) | fields v", "FUR"); + } + + /** {@code left(str0, length(str0))} on row 0 → "FURNITURE" (full string). */ + public void testLeftWithComputedLength() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = left(str0, length(str0)) | fields v", "FURNITURE"); + } + + /** {@code right('FURNITURE', 3)} → "URE". Verifies suffix extraction; a left() misroute would + * return "FUR". */ + public void testRight() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = right(str0, 3) | fields v", "URE"); + } + + // ── lower / upper ──────────────────────────────────────────────────────── + + /** {@code lower('FURNITURE')} → "furniture". */ + public void testLower() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = lower(str0) | fields v", "furniture"); + } + + /** {@code upper('one')} → "ONE". Complements testLower. */ + public void testUpper() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = upper(str2) | fields v", "ONE"); + } + + // ── ltrim / rtrim / trim ──────────────────────────────────────────────── + + /** {@code ltrim(concat(' ', str2))} on row 0 → "one". The {@code concat} forces runtime + * evaluation (Calcite can't fold the call because {@code str2} is a column ref), and the + * leading spaces guarantee only ltrim could produce "one" from the 6-character input. */ + public void testLtrim() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = ltrim(concat(' ', str2)) | fields v", "one"); + } + + /** {@code rtrim(concat(str2, ' '))} on row 0 → "one". Trailing-spaces counterpart to ltrim; + * verifies the right-side whitespace removal. */ + public void testRtrim() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = rtrim(concat(str2, ' ')) | fields v", "one"); + } + + /** {@code trim(concat(' ', str2, ' '))} on row 0 → "one". */ + public void testTrim() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = trim(concat(' ', str2, ' ')) | fields v", "one"); + } + + // ── reverse ────────────────────────────────────────────────────────────── + + /** {@code reverse('FURNITURE')} on a field → "ERUTINRUF". */ + public void testReverse() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = reverse(str0) | fields v", "ERUTINRUF"); + } + + /** {@code reverse(concat(str2, str0))} → "ERUTINRUFeno". Composed with concat so the input is + * computed at runtime ({@code "one" + "FURNITURE" = "oneFURNITURE"}) and its reverse is a + * 12-char string that could only come from an actual character-by-character reversal. */ + public void testReverseOfConcat() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = reverse(concat(str2, str0)) | fields v", "ERUTINRUFeno"); + } + + // ── substring ──────────────────────────────────────────────────────────── + + /** {@code substring('FURNITURE', 2)} → "URNITURE" (8 chars, from index 2 to end). */ + public void testSubstringTwoArg() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = substring(str0, 2) | fields v", "URNITURE"); + } + + /** {@code substring('FURNITURE', 2, 3)} → "URN". Length-bounded 3-arg form; verifies both + * start-position and length semantics simultaneously. */ + public void testSubstringThreeArg() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = substring(str0, 2, 3) | fields v", "URN"); + } + + // ── length ─────────────────────────────────────────────────────────────── + + /** {@code length('FURNITURE')} → 9.*/ + public void testLength() throws IOException { + assertFirstRowLong(oneRow("key00") + "| eval v = length(str0) | fields v", 9); + } + + /** {@code length('OFFICE SUPPLIES')} on key04 → 15. */ + public void testLengthDifferentRow() throws IOException { + assertFirstRowLong(oneRow("key04") + "| eval v = length(str0) | fields v", 15); + } + + // ── locate / position ─────────────────────────────────────────────────── + + /** {@code locate('U', 'FURNITURE')} → 2 (1-based position of first 'U'). */ + public void testLocate() throws IOException { + assertFirstRowLong(oneRow("key00") + "| eval v = locate('U', str0) | fields v", 2); + } + + /** {@code locate('U', 'FURNITURE', 3)} → 7. Start-index=3 skips the first 'U' at position 2 + * and finds the second 'U' at position 7. */ + public void testLocateWithStart() throws IOException { + assertFirstRowLong(oneRow("key00") + "| eval v = locate('U', str0, 3) | fields v", 7); + } + + /** {@code locate('XYZ', str0)} → 0 (not found). */ + public void testLocateNotFound() throws IOException { + assertFirstRowLong(oneRow("key00") + "| eval v = locate('XYZ', str0) | fields v", 0); + } + + /** {@code position('RNI' IN 'FURNITURE')} → 3. */ + public void testPosition() throws IOException { + assertFirstRowLong(oneRow("key00") + "| eval v = position(\"RNI\" IN str0) | fields v", 3); + } + + // ── strcmp ─────────────────────────────────────────────────────────────── + + /** {@code strcmp('hello', 'hello world')} → -1 (lhs < rhs). */ + public void testStrcmpLess() throws IOException { + assertFirstRowLong(oneRow("key00") + "| eval v = strcmp('hello', 'hello world') | fields v", -1); + } + + /** {@code strcmp('foo', 'foo')} → 0. */ + public void testStrcmpEqual() throws IOException { + assertFirstRowLong(oneRow("key00") + "| eval v = strcmp('foo', 'foo') | fields v", 0); + } + + /** {@code strcmp('banana', 'apple')} → 1 (lhs > rhs). */ + public void testStrcmpGreater() throws IOException { + assertFirstRowLong(oneRow("key00") + "| eval v = strcmp('banana', 'apple') | fields v", 1); + } + + /** {@code strcmp(str0, 'FURNITURE')} on row 0 (str0='FURNITURE') → 0. Verifies the adapter + * handles column references correctly: PPL frontend reverses args internally, and the + * adapter must swap back for the user-intended semantics. */ + public void testStrcmpColumnEqual() throws IOException { + assertFirstRowLong(oneRow("key00") + "| eval v = strcmp(str0, 'FURNITURE') | fields v", 0); + } + + /** {@code strcmp(str0, 'AAA')} */ + public void testStrcmpColumnGreater() throws IOException { + assertFirstRowLong(oneRow("key00") + "| eval v = strcmp(str0, 'AAA') | fields v", 1); + } + + /** {@code strcmp(str0, 'ZZZ')} */ + public void testStrcmpColumnLess() throws IOException { + assertFirstRowLong(oneRow("key00") + "| eval v = strcmp(str0, 'ZZZ') | fields v", -1); + } + + // ── tostring — basic ──────────────────────────────────────────────────── + + /** {@code tostring(num0)} on row 0 (num0=12.3) → "12.3". */ + public void testToStringOnDouble() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = tostring(num0) | fields v", "12.3"); + } + + /** {@code tostring(int0)} on row 0 (int0=1) → "1". */ + public void testToStringOnInteger() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = tostring(int0) | fields v", "1"); + } + + /** {@code tostring(1=1)} → "TRUE". Boolean literal routes through the adapter's CASE + * WHEN x THEN 'TRUE' WHEN NOT x THEN 'FALSE' END rewrite. */ + public void testToStringOnBooleanTrue() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = tostring(1=1) | fields v", "TRUE"); + } + + /** {@code tostring(1=0)} → "FALSE" */ + public void testToStringOnBooleanFalse() throws IOException { + assertFirstRowString(oneRow("key00") + "| eval v = tostring(1=0) | fields v", "FALSE"); + } + + // ── tostring — format modes ───────────────────────────────────────────── + + /** + * {@code tostring(int0 * 255, 'hex')} on row 0 (int0=1) → "ff". + */ + public void testToStringHexFormat() throws IOException { + Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tostring(int0 * 255, 'hex') | fields v"); + assertNotNull("hex cell must not be null", cell); + assertTrue("hex cell must be String but was " + cell.getClass(), cell instanceof String); + assertEquals("tostring(255, 'hex')", "ff", ((String) cell).toLowerCase(Locale.US)); + } + + /** + * {@code tostring(int0 * 21, 'binary')} on row 0 (int0=1) → "10101". + */ + public void testToStringBinaryFormat() throws IOException { + Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tostring(int0 * 21, 'binary') | fields v"); + assertNotNull("binary cell must not be null", cell); + assertTrue("binary cell must be String but was " + cell.getClass(), cell instanceof String); + assertEquals("tostring(21, 'binary')", "10101", cell); + } + + /** + * {@code tostring(int0 * 12345, 'commas')} on row 0 (int0=1) → "12,345". + */ + public void testToStringCommasFormat() throws IOException { + Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tostring(int0 * 12345, 'commas') | fields v"); + assertNotNull("commas cell must not be null", cell); + assertTrue("commas cell must be String but was " + cell.getClass(), cell instanceof String); + NumberFormat nf = NumberFormat.getNumberInstance(Locale.US); + nf.setMinimumFractionDigits(0); + nf.setMaximumFractionDigits(2); + assertEquals("tostring(12345, 'commas')", nf.format(12345L), cell); + } + + /** + * {@code tostring(int0 * 3661, 'duration')} on row 0 (int0=1) → "01:01:01". + * one. + */ + public void testToStringDurationFormat() throws IOException { + Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tostring(int0 * 3661, 'duration') | fields v"); + assertNotNull("duration cell must not be null", cell); + assertTrue("duration cell must be String but was " + cell.getClass(), cell instanceof String); + assertEquals("tostring(3661, 'duration')", "01:01:01", cell); + } + + /** + * {@code tostring(int0 * 3_661_000, 'duration_millis')} on row 0 (int0=1) → "01:01:01". + */ + public void testToStringDurationMillisFormat() throws IOException { + Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tostring(int0 * 3661000, 'duration_millis') | fields v"); + assertNotNull("duration_millis cell must not be null", cell); + assertTrue("duration_millis cell must be String but was " + cell.getClass(), cell instanceof String); + assertEquals("tostring(3661000, 'duration_millis')", "01:01:01", cell); + } + + // ── tonumber ──────────────────────────────────────────────────────────── + + /** {@code tonumber('4598')} → 4598.0 */ + public void testToNumberDecimalInteger() throws IOException { + assertFirstRowDouble(oneRow("key00") + "| eval v = tonumber('4598') | fields v", 4598.0, 0.0); + } + + /** {@code tonumber('4598.678')} → 4598.678 */ + public void testToNumberDecimalFractional() throws IOException { + assertFirstRowDouble(oneRow("key00") + "| eval v = tonumber('4598.678') | fields v", 4598.678, 1e-9); + } + + /** {@code tonumber('010101', 2)} → 21. Base-2 parse */ + public void testToNumberBinary() throws IOException { + assertFirstRowDouble(oneRow("key00") + "| eval v = tonumber('010101', 2) | fields v", 21.0, 0.0); + } + + /** {@code tonumber('FA34', 16)} → 64052. Base-16 parse with uppercase hex digits */ + public void testToNumberHex() throws IOException { + assertFirstRowDouble(oneRow("key00") + "| eval v = tonumber('FA34', 16) | fields v", 64052.0, 0.0); + } + + /** {@code tonumber('101', 8)} → 65 (octal 101 = 64 + 1) */ + public void testToNumberOctal() throws IOException { + assertFirstRowDouble(oneRow("key00") + "| eval v = tonumber('101', 8) | fields v", 65.0, 0.0); + } + + /** {@code tonumber('abc')} → NULL. Unparseable input */ + public void testToNumberReturnsNullOnParseFailure() throws IOException { + Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tonumber('abc') | fields v"); + assertNull("tonumber('abc') should be NULL but was " + cell, cell); + } + + /** {@code tonumber('FA34', 10)} → NULL */ + public void testToNumberBaseMismatchReturnsNull() throws IOException { + Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tonumber('FA34', 10) | fields v"); + assertNull("tonumber('FA34', 10) should be NULL but was " + cell, cell); + } + + // ── helpers ───────────────────────────────────────────────────────────── + + private void assertFirstRowString(String ppl, String expected) throws IOException { + Object cell = firstRowFirstCell(ppl); + assertNotNull("Expected non-null result for query [" + ppl + "]", cell); + assertEquals("Value mismatch for query: " + ppl, expected, cell); + } + + private void assertFirstRowLong(String ppl, long expected) throws IOException { + Object cell = firstRowFirstCell(ppl); + assertTrue("Expected numeric result for query [" + ppl + "] but got: " + cell, cell instanceof Number); + assertEquals("Value mismatch for query: " + ppl, expected, ((Number) cell).longValue()); + } + + private void assertFirstRowDouble(String ppl, double expected, double delta) throws IOException { + Object cell = firstRowFirstCell(ppl); + assertTrue("Expected numeric result for query [" + ppl + "] but got: " + cell, cell instanceof Number); + assertEquals("Value mismatch for query: " + ppl, expected, ((Number) cell).doubleValue(), delta); + } + + private Object firstRowFirstCell(String ppl) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> rows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' for query: " + ppl, rows); + assertTrue("Expected at least one row for query: " + ppl, rows.size() >= 1); + return rows.get(0).get(0); + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } +}