diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index c8600bb9809..416f0d382d4 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -478,6 +478,37 @@ public static Span span(UnresolvedExpression field, UnresolvedExpression value, return new Span(field, value, unit); } + /** + * Creates a Span expression from a field and a span length literal. Parses string literals to + * extract numeric value and time unit (e.g., "1h" -> value=1, unit=h). + * + * @param field The field expression to apply the span to + * @param spanLengthLiteral The literal value containing either a string with embedded unit (e.g., + * "1h", "30m") or a plain number + * @return A Span expression with parsed value and unit + */ + public static Span spanFromSpanLengthLiteral( + UnresolvedExpression field, Literal spanLengthLiteral) { + if (spanLengthLiteral.getType() == DataType.STRING) { + String spanText = spanLengthLiteral.getValue().toString(); + String valueStr = spanText.replaceAll("[^0-9]", ""); + String unitStr = spanText.replaceAll("[0-9]", ""); + + if (valueStr.isEmpty()) { + // No numeric value found, use the literal as-is + return new Span(field, spanLengthLiteral, SpanUnit.NONE); + } else { + // Parse numeric value and unit + Integer value = Integer.parseInt(valueStr); + SpanUnit unit = unitStr.isEmpty() ? SpanUnit.NONE : SpanUnit.of(unitStr); + return span(field, intLiteral(value), unit); + } + } else { + // Non-string literal (e.g., integer) + return span(field, spanLengthLiteral, SpanUnit.NONE); + } + } + public static Sort sort(UnresolvedPlan input, Field... sorts) { return new Sort(Arrays.asList(sorts)).attach(input); } diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java b/core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java index 438c46e7e53..aadc94d0b8b 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java @@ -17,20 +17,37 @@ public enum SpanUnit { NONE(""), MILLISECOND("ms"), MS("ms"), + SECONDS("s"), SECOND("s"), + SECS("s"), + SEC("s"), S("s"), + MINUTES("m"), MINUTE("m"), + MINS("m"), + MIN("m"), m("m"), + HOURS("h"), HOUR("h"), + HRS("h"), + HR("h"), H("h"), + DAYS("d"), DAY("d"), D("d"), + WEEKS("w"), WEEK("w"), W("w"), MONTH("M"), + MONTHS("M"), + MON("M"), M("M"), + QUARTERS("q"), QUARTER("q"), + QTRS("q"), + QTR("q"), Q("q"), + YEARS("y"), YEAR("y"), Y("y"); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java index b355e5f9b2b..6c6929c752e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java @@ -506,33 +506,6 @@ public void testBinSpanWithStartEndNeverShrinkRange() throws IOException { rows("39-40", 39)); } - @Test - public void testBinFloatingPointSpanBasicFunctionality() throws IOException { - JSONObject result = - executeQuery( - String.format( - "source=%s | bin age span=2.5 | fields age | head 3", TEST_INDEX_ACCOUNT)); - verifySchema(result, schema("age", null, "string")); - - // Test that floating point spans work with proper range formatting - verifyDataRows(result, rows("27.5-30.0"), rows("30.0-32.5"), rows("35.0-37.5")); - } - - @Test - public void testBinFloatingPointSpanWithStats() throws IOException { - JSONObject result = - executeQuery( - String.format( - "source=%s | bin balance span=15000.5 | fields balance | sort balance |" - + " head 2", - TEST_INDEX_ACCOUNT)); - - verifySchema(result, schema("balance", null, "string")); - - // Test floating point spans without aggregation - verify proper decimal formatting - verifyDataRows(result, rows("0.0-15000.5"), rows("0.0-15000.5")); - } - @Test @Ignore public void testBinWithNumericSpanStatsCount() throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index 2e9d62f0ace..69b1d69cc66 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -456,7 +456,7 @@ public void testAvgByTimeSpanAndFields() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | stats avg(balance) by span(birthdate, 1 month) as age_balance", + "source=%s | stats avg(balance) by span(birthdate, 1month) as age_balance", TEST_INDEX_BANK)); verifySchema(actual, schema("age_balance", "timestamp"), schema("avg(balance)", "double")); verifyDataRows( @@ -473,7 +473,7 @@ public void testCountByCustomTimeSpanWithDifferentUnits() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | head 5 | stats count(datetime0) by span(datetime0, 15 minute) as" + "source=%s | head 5 | stats count(datetime0) by span(datetime0, 15minute) as" + " datetime_span", TEST_INDEX_CALCS)); verifySchema( @@ -489,7 +489,7 @@ public void testCountByCustomTimeSpanWithDifferentUnits() throws IOException { actual = executeQuery( String.format( - "source=%s | head 5 | stats count(datetime0) by span(datetime0, 5 second) as" + "source=%s | head 5 | stats count(datetime0) by span(datetime0, 5second) as" + " datetime_span", TEST_INDEX_CALCS)); verifySchema( @@ -505,7 +505,7 @@ public void testCountByCustomTimeSpanWithDifferentUnits() throws IOException { actual = executeQuery( String.format( - "source=%s | head 5 | stats count(datetime0) by span(datetime0, 3 month) as" + "source=%s | head 5 | stats count(datetime0) by span(datetime0, 3month) as" + " datetime_span", TEST_INDEX_CALCS)); verifySchema( @@ -519,7 +519,7 @@ public void testCountByNullableTimeSpan() throws IOException { executeQuery( String.format( "source=%s | head 5 | stats count(datetime0), count(datetime1) by span(time1," - + " 15 minute) as time_span", + + " 15minute) as time_span", TEST_INDEX_CALCS)); verifySchema( actual, @@ -539,8 +539,7 @@ public void testCountByDateTypeSpanWithDifferentUnits() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | stats count(strict_date) by span(strict_date, 1 day) as" - + " date_span", + "source=%s | stats count(strict_date) by span(strict_date, 1day) as" + " date_span", TEST_INDEX_DATE_FORMATS)); verifySchema(actual, schema("date_span", "date"), schema("count(strict_date)", "bigint")); verifyDataRows(actual, rows(2, "1984-04-12")); @@ -548,7 +547,7 @@ public void testCountByDateTypeSpanWithDifferentUnits() throws IOException { actual = executeQuery( String.format( - "source=%s | stats count(basic_date) by span(basic_date, 1 year) as" + " date_span", + "source=%s | stats count(basic_date) by span(basic_date, 1year) as" + " date_span", TEST_INDEX_DATE_FORMATS)); verifySchema(actual, schema("date_span", "date"), schema("count(basic_date)", "bigint")); verifyDataRows(actual, rows(2, "1984-01-01")); @@ -556,7 +555,7 @@ public void testCountByDateTypeSpanWithDifferentUnits() throws IOException { actual = executeQuery( String.format( - "source=%s | stats count(year_month_day) by span(year_month_day, 1 month)" + "source=%s | stats count(year_month_day) by span(year_month_day, 1month)" + " as date_span", TEST_INDEX_DATE_FORMATS)); verifySchema(actual, schema("date_span", "date"), schema("count(year_month_day)", "bigint")); @@ -569,7 +568,7 @@ public void testCountByTimeTypeSpanWithDifferentUnits() throws IOException { executeQuery( String.format( "source=%s | stats count(hour_minute_second) by span(hour_minute_second, 1" - + " minute) as time_span", + + "minute) as time_span", TEST_INDEX_DATE_FORMATS)); verifySchema( actual, schema("time_span", "time"), schema("count(hour_minute_second)", "bigint")); @@ -578,7 +577,7 @@ public void testCountByTimeTypeSpanWithDifferentUnits() throws IOException { actual = executeQuery( String.format( - "source=%s | stats count(custom_time) by span(custom_time, 1 second) as" + "source=%s | stats count(custom_time) by span(custom_time, 1second) as" + " time_span", TEST_INDEX_DATE_FORMATS)); verifySchema(actual, schema("time_span", "time"), schema("count(custom_time)", "bigint")); @@ -587,7 +586,7 @@ public void testCountByTimeTypeSpanWithDifferentUnits() throws IOException { actual = executeQuery( String.format( - "source=%s | stats count(hour) by span(hour, 6 hour) as time_span", + "source=%s | stats count(hour) by span(hour, 6hour) as time_span", TEST_INDEX_DATE_FORMATS)); verifySchema(actual, schema("time_span", "time"), schema("count(hour)", "bigint")); verifyDataRows(actual, rows(2, "06:00:00")); @@ -687,7 +686,7 @@ public void testCountBySpanForCustomFormats() throws IOException { executeQuery( String.format( "source=%s | stats count(custom_date_or_date) by span(custom_date_or_date, 1" - + " month) as date_span", + + "month) as date_span", TEST_INDEX_DATE_FORMATS)); verifySchema( actual, schema("date_span", "date"), schema("count(custom_date_or_date)", "bigint")); @@ -697,7 +696,7 @@ public void testCountBySpanForCustomFormats() throws IOException { executeQuery( String.format( "source=%s | stats count(custom_date_or_custom_time) by" - + " span(custom_date_or_custom_time, 1 hour) as timestamp_span", + + " span(custom_date_or_custom_time, 1hour) as timestamp_span", TEST_INDEX_DATE_FORMATS)); verifySchema( actual, @@ -709,7 +708,7 @@ public void testCountBySpanForCustomFormats() throws IOException { executeQuery( String.format( "source=%s | stats count(custom_no_delimiter_ts) by span(custom_no_delimiter_ts, 1" - + " hour) as timestamp_span", + + "hour) as timestamp_span", TEST_INDEX_DATE_FORMATS)); verifySchema( actual, @@ -721,7 +720,7 @@ public void testCountBySpanForCustomFormats() throws IOException { executeQuery( String.format( "source=%s | stats count(incomplete_custom_time) by span(incomplete_custom_time, 12" - + " hour) as time_span", + + "hour) as time_span", TEST_INDEX_DATE_FORMATS)); verifySchema( actual, schema("time_span", "time"), schema("count(incomplete_custom_time)", "bigint")); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java index da32d3a79b9..80d34c2ff91 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java @@ -917,4 +917,77 @@ public void testSearchWithDateINOperator() throws IOException { rows("2024-01-15 10:30:00.123456789", "INFO"), rows("2024-01-15 10:30:01.23456789", "ERROR")); } + + @Test + public void testSearchWithTraceId() throws IOException { + // Test 1: Search for specific traceId + JSONObject specificTraceId = + executeQuery( + String.format( + "search source=%s b3cb01a03c846973fd496b973f49be85 | fields" + " traceId, body", + TEST_INDEX_OTEL_LOGS)); + verifyDataRows( + specificTraceId, + rows( + "b3cb01a03c846973fd496b973f49be85", + "User e1ce63e6-8501-11f0-930d-c2fcbdc05f14 adding 4 of product HQTGWGPNH4 to cart")); + } + + @Test + public void testSearchWithSpanLength() throws IOException { + // Test searching for SPANLENGTH keyword in free text search + // This tests that SPANLENGTH tokens like "3month" are searchable + JSONObject result = + executeQuery( + String.format( + "search source=%s 3month | fields body, `attributes.span.duration`", + TEST_INDEX_OTEL_LOGS)); + verifyDataRows(result, rows("Processing data for 3month period", "3month")); + } + + @Test + public void testSearchWithSpanLengthInField() throws IOException { + // Test searching for SPANLENGTH value in a specific field + JSONObject result = + executeQuery( + String.format( + "search source=%s `attributes.span.duration`=\\\"3month\\\" | fields body," + + " `attributes.span.duration`", + TEST_INDEX_OTEL_LOGS)); + verifyDataRows(result, rows("Processing data for 3month period", "3month")); + } + + @Test + public void testSearchWithNumericIdVsSpanLength() throws IOException { + // Test that NUMERIC_ID tokens like "1s4f7" (which start with what could be a SPANLENGTH like + // "1s") + // are properly searchable as complete tokens + // This verifies that NUMERIC_ID takes precedence over SPANLENGTH when applicable + + // Test 1: Search for the NUMERIC_ID token in free text + JSONObject numericIdResult = + executeQuery( + String.format( + "search source=%s 1s4f7 | fields body, `attributes.transaction.id`", + TEST_INDEX_OTEL_LOGS)); + verifyDataRows(numericIdResult, rows("Transaction ID 1s4f7 processed successfully", "1s4f7")); + + // Test 2: Search for NUMERIC_ID in specific field + JSONObject fieldSearchResult = + executeQuery( + String.format( + "search source=%s `attributes.transaction.id`=1s4f7 | fields body," + + " `attributes.transaction.id`", + TEST_INDEX_OTEL_LOGS)); + verifyDataRows(fieldSearchResult, rows("Transaction ID 1s4f7 processed successfully", "1s4f7")); + + // Test 3: Verify that searching for just "1s" (which would be a SPANLENGTH) + // does NOT match the "1s4f7" token + JSONObject spanLengthSearchResult = + executeQuery( + String.format( + "search source=%s `attributes.transaction.id`=1s | fields body", + TEST_INDEX_OTEL_LOGS)); + verifyDataRows(spanLengthSearchResult); // Should return no results + } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_basic_text.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_basic_text.json index 0c69b0f25e2..eef64285c9c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_basic_text.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_basic_text.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$153], severityNumber=[$154], time=[$155], body=[$156])\n LogicalFilter(condition=[query_string(MAP('query', 'ERROR':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'ERROR':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body], FILTER->query_string(MAP('query', 'ERROR':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"ERROR\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\",\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\",\"time\",\"body\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_numeric_comparison.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_numeric_comparison.json index 47ef52587fb..78546e3211f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_numeric_comparison.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_numeric_comparison.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$153], severityNumber=[$154], time=[$155], body=[$156])\n LogicalFilter(condition=[query_string(MAP('query', 'severityNumber:>15':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'severityNumber:>15':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body], FILTER->query_string(MAP('query', 'severityNumber:>15':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityNumber:>15\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\",\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\",\"time\",\"body\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_wildcard_star.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_wildcard_star.json index 18d5ca27c78..d89d0ec8856 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_wildcard_star.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_wildcard_star.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$153], severityNumber=[$154], time=[$155], body=[$156])\n LogicalFilter(condition=[query_string(MAP('query', 'severityText:ERR*':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'severityText:ERR*':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body], FILTER->query_string(MAP('query', 'severityText:ERR*':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityText:ERR*\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\",\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\",\"time\",\"body\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_basic_text.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_basic_text.json index 1cdab05e428..d02842229d8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_basic_text.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_basic_text.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$153], severityNumber=[$154], time=[$155], body=[$156])\n LogicalFilter(condition=[query_string(MAP('query', 'ERROR':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..162=[{inputs}], proj#0..3=[{exprs}], severityText=[$t7], resource=[$t8], flags=[$t23], attributes=[$t24], droppedAttributesCount=[$t153], severityNumber=[$t154], time=[$t155], body=[$t156])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->query_string(MAP('query', 'ERROR':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"ERROR\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'ERROR':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..171=[{inputs}], proj#0..3=[{exprs}], severityText=[$t7], resource=[$t8], flags=[$t23], attributes=[$t24], droppedAttributesCount=[$t162], severityNumber=[$t163], time=[$t164], body=[$t165])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->query_string(MAP('query', 'ERROR':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"ERROR\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_numeric_comparison.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_numeric_comparison.json index 4c33e3d38a7..b00e5090c09 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_numeric_comparison.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_numeric_comparison.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$153], severityNumber=[$154], time=[$155], body=[$156])\n LogicalFilter(condition=[query_string(MAP('query', 'severityNumber:>15':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..162=[{inputs}], proj#0..3=[{exprs}], severityText=[$t7], resource=[$t8], flags=[$t23], attributes=[$t24], droppedAttributesCount=[$t153], severityNumber=[$t154], time=[$t155], body=[$t156])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->query_string(MAP('query', 'severityNumber:>15':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityNumber:>15\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'severityNumber:>15':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..171=[{inputs}], proj#0..3=[{exprs}], severityText=[$t7], resource=[$t8], flags=[$t23], attributes=[$t24], droppedAttributesCount=[$t162], severityNumber=[$t163], time=[$t164], body=[$t165])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->query_string(MAP('query', 'severityNumber:>15':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityNumber:>15\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_wildcard_star.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_wildcard_star.json index 9c3a54491fa..7a2324fbcb5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_wildcard_star.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_search_wildcard_star.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$153], severityNumber=[$154], time=[$155], body=[$156])\n LogicalFilter(condition=[query_string(MAP('query', 'severityText:ERR*':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..162=[{inputs}], proj#0..3=[{exprs}], severityText=[$t7], resource=[$t8], flags=[$t23], attributes=[$t24], droppedAttributesCount=[$t153], severityNumber=[$t154], time=[$t155], body=[$t156])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->query_string(MAP('query', 'severityText:ERR*':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityText:ERR*\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'severityText:ERR*':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..171=[{inputs}], proj#0..3=[{exprs}], severityText=[$t7], resource=[$t8], flags=[$t23], attributes=[$t24], droppedAttributesCount=[$t162], severityNumber=[$t163], time=[$t164], body=[$t165])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->query_string(MAP('query', 'severityText:ERR*':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityText:ERR*\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/otellogs.json b/integ-test/src/test/resources/otellogs.json index 61576f423c6..7fee3b08cc0 100644 --- a/integ-test/src/test/resources/otellogs.json +++ b/integ-test/src/test/resources/otellogs.json @@ -58,3 +58,7 @@ {"@timestamp": "2024-01-15T10:30:28.901234567Z", "time": "2024-01-15T10:30:28.901234567Z", "severityNumber": 9, "severityText": "INFO", "body": "Health check passed for all services including email-service@health.monitor", "traceId": "", "spanId": "", "flags": 0, "resource": {"attributes": {"service.name": "health-checker"}}, "attributes": {"health.status": "healthy", "health.services_checked": 15, "monitor.email": "email-service@health.monitor"}} {"index": {"_id": "30"}} {"@timestamp": "2024-01-15T10:30:29.012345678Z", "time": "2024-01-15T10:30:29.012345678Z", "severityNumber": 9, "severityText": "INFO", "body": "CORS request from origin https://app.example.com to access /api/users/email containing sensitive@data.secure", "traceId": "", "spanId": "", "flags": 0, "resource": {"attributes": {"service.name": "api-gateway"}}, "attributes": {"cors.origin": "https://app.example.com", "cors.method": "POST", "endpoint": "/api/users/email", "data.classification": "sensitive@data.secure"}} +{"index": {"_id": "31"}} +{"@timestamp": "2024-01-15T10:30:30.123456789Z", "time": "2024-01-15T10:30:30.123456789Z", "severityNumber": 9, "severityText": "INFO", "body": "Processing data for 3month period", "traceId": "", "spanId": "", "flags": 0, "resource": {"attributes": {"service.name": "analytics-service"}}, "attributes": {"span.duration": "3month", "span.type": "analytics", "processing.status": "started"}} +{"index": {"_id": "32"}} +{"@timestamp": "2024-01-15T10:30:31.123456789Z", "time": "2024-01-15T10:30:31.123456789Z", "severityNumber": 9, "severityText": "INFO", "body": "Transaction ID 1s4f7 processed successfully", "traceId": "", "spanId": "", "flags": 0, "resource": {"attributes": {"service.name": "transaction-service"}}, "attributes": {"transaction.id": "1s4f7", "transaction.status": "completed", "transaction.amount": 250.50}} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 48b051e456b..b817dc49632 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -496,35 +496,25 @@ MINSPAN: 'MINSPAN'; START: 'START'; END: 'END'; ALIGNTIME: 'ALIGNTIME'; -MS: 'MS'; -S: 'S'; -M: 'M'; -H: 'H'; -W: 'W'; -Q: 'Q'; -Y: 'Y'; - -// Extended timescale units -SEC: 'SEC'; -SECS: 'SECS'; -SECONDS: 'SECONDS'; -MINS: 'MINS'; -MINUTES: 'MINUTES'; -HR: 'HR'; -HRS: 'HRS'; -HOURS: 'HOURS'; -DAYS: 'DAYS'; -MON: 'MON'; -MONTHS: 'MONTHS'; -US: 'US'; -CS: 'CS'; -DS: 'DS'; - - // PERCENTILE SHORTCUT FUNCTIONS // Must precede ID to avoid conflicts with identifier matching PERCENTILE_SHORTCUT: PERC(INTEGER_LITERAL | DECIMAL_LITERAL) | 'P'(INTEGER_LITERAL | DECIMAL_LITERAL); +SPANLENGTH: [0-9]+ ( + 'US'|'MS'|'CS'|'DS' + |'S'|'SEC'|'SECS'|'SECOND'|'SECONDS' + |'MIN'|'MINS'|'MINUTE'|'MINUTES' + |'H'|'HR'|'HRS'|'HOUR'|'HOURS' + |'H'|'HR'|'HRS'|'HOUR'|'HOURS' + |'D'|'DAY'|'DAYS' + |'W'|'WEEK'|'WEEKS' + |'M'|'MON'|'MONTH'|'MONTHS' + |'Q'|'QTR'|'QTRS'|'QUARTER'|'QUARTERS' + |'Y'|'YR'|'YRS'|'YEAR'|'YEARS' +); + +NUMERIC_ID : DEC_DIGIT+ ID_LITERAL; + // LITERALS AND VALUES //STRING_LITERAL: DQUOTA_STRING | SQUOTA_STRING | BQUOTA_STRING; ID: ID_LITERAL; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 0bc7b784338..39165001bc9 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -140,6 +140,7 @@ searchLiteral : numericLiteral | booleanLiteral | ID + | NUMERIC_ID | stringLiteral | searchableKeyWord ; @@ -259,8 +260,8 @@ timechartArg ; spanLiteral - : integerLiteral timespanUnit - | stringLiteral + : SPANLENGTH + | INTEGER_LITERAL ; evalCommand @@ -276,9 +277,9 @@ binCommand ; binOption - : SPAN EQUAL span = spanValue + : SPAN EQUAL span = binSpanValue | BINS EQUAL bins = integerLiteral - | MINSPAN EQUAL minspan = literalValue (minspanUnit = timespanUnit)? + | MINSPAN EQUAL minspan = spanLiteral | ALIGNTIME EQUAL aligntime = aligntimeValue | START EQUAL start = numericLiteral | END EQUAL end = numericLiteral @@ -290,11 +291,9 @@ aligntimeValue | literalValue ; -spanValue - : literalValue (timespanUnit)? # numericSpanValue +binSpanValue + : spanLiteral # numericSpanValue | logSpanValue # logBasedSpanValue - | ident timespanUnit # extendedTimeSpanValue - | ident # identifierSpanValue ; logSpanValue @@ -598,7 +597,7 @@ bySpanClause ; spanClause - : SPAN LT_PRTHS fieldExpression COMMA value = literalValue (unit = timespanUnit)? RT_PRTHS + : SPAN LT_PRTHS fieldExpression COMMA value = spanLiteral RT_PRTHS ; sortbyClause @@ -1308,40 +1307,6 @@ intervalUnit | YEAR_MONTH ; -timespanUnit - : MS - | S - | M - | H - | D - | W - | Q - | Y - | MILLISECOND - | SECOND - | MINUTE - | HOUR - | DAY - | WEEK - | MONTH - | QUARTER - | YEAR - | SEC - | SECS - | SECONDS - | MINS - | MINUTES - | HR - | HRS - | HOURS - | DAYS - | MON - | MONTHS - | US - | CS - | DS - ; - valueList : LT_PRTHS literalValue (COMMA literalValue)* RT_PRTHS ; @@ -1388,8 +1353,8 @@ keywordsCanBeId searchableKeyWord : D // OD SQL and ODBC special - | timespanUnit | SPAN + | SPANLENGTH | evalFunctionName | jsonFunctionName | relevanceArgName @@ -1505,4 +1470,4 @@ searchableKeyWord | LEFT_HINT | RIGHT_HINT | PERCENTILE_SHORTCUT - ; + ; \ No newline at end of file diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 774eb73dff6..de31abbf3bd 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -46,26 +46,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.EmptySourcePropagateVisitor; import org.opensearch.sql.ast.dsl.AstDSL; -import org.opensearch.sql.ast.expression.Alias; -import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta; -import org.opensearch.sql.ast.expression.Argument; -import org.opensearch.sql.ast.expression.DataType; -import org.opensearch.sql.ast.expression.EqualTo; -import org.opensearch.sql.ast.expression.Field; -import org.opensearch.sql.ast.expression.Let; -import org.opensearch.sql.ast.expression.Literal; -import org.opensearch.sql.ast.expression.Map; -import org.opensearch.sql.ast.expression.ParseMethod; -import org.opensearch.sql.ast.expression.PatternMethod; -import org.opensearch.sql.ast.expression.PatternMode; -import org.opensearch.sql.ast.expression.QualifiedName; -import org.opensearch.sql.ast.expression.SearchAnd; -import org.opensearch.sql.ast.expression.SearchExpression; -import org.opensearch.sql.ast.expression.SearchGroup; -import org.opensearch.sql.ast.expression.SpanUnit; -import org.opensearch.sql.ast.expression.UnresolvedArgument; -import org.opensearch.sql.ast.expression.UnresolvedExpression; -import org.opensearch.sql.ast.expression.WindowFunction; +import org.opensearch.sql.ast.expression.*; import org.opensearch.sql.ast.tree.AD; import org.opensearch.sql.ast.tree.Aggregation; import org.opensearch.sql.ast.tree.Append; @@ -520,12 +501,7 @@ public UnresolvedPlan visitBinCommand(BinCommandContext ctx) { if (!seenParams.add("MINSPAN")) { throw new IllegalArgumentException("Duplicate MINSPAN parameter in bin command"); } - String minspanValue = option.minspan.getText(); - String minspanUnit = option.minspanUnit != null ? option.minspanUnit.getText() : null; - minspan = - minspanUnit != null - ? org.opensearch.sql.ast.dsl.AstDSL.stringLiteral(minspanValue + minspanUnit) - : internalVisitExpression(option.minspan); + minspan = internalVisitExpression(option.minspan); } // ALIGNTIME parameter @@ -625,14 +601,13 @@ public UnresolvedPlan visitTimechartCommand(OpenSearchPPLParser.TimechartCommand AstDSL.span(AstDSL.field("@timestamp"), AstDSL.intLiteral(1), SpanUnit.of("m")); Integer limit = 10; Boolean useOther = true; - // Process timechart parameters for (OpenSearchPPLParser.TimechartParameterContext paramCtx : ctx.timechartParameter()) { if (paramCtx.spanClause() != null) { binExpression = internalVisitExpression(paramCtx.spanClause()); } else if (paramCtx.spanLiteral() != null) { - // Convert span=1h to span(@timestamp, 1h) - binExpression = internalVisitExpression(paramCtx.spanLiteral()); + Literal literal = (Literal) internalVisitExpression(paramCtx.spanLiteral()); + binExpression = AstDSL.spanFromSpanLengthLiteral(AstDSL.field("@timestamp"), literal); } else if (paramCtx.timechartArg() != null) { OpenSearchPPLParser.TimechartArgContext argCtx = paramCtx.timechartArg(); if (argCtx.LIMIT() != null && argCtx.integerLiteral() != null) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 86d76f09890..b4de0298e38 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -612,29 +612,14 @@ public UnresolvedExpression visitBySpanClause(BySpanClauseContext ctx) { @Override public UnresolvedExpression visitSpanClause(SpanClauseContext ctx) { - String unit = ctx.unit != null ? ctx.unit.getText() : ""; - return new Span(visit(ctx.fieldExpression()), visit(ctx.value), SpanUnit.of(unit)); - } - - // Handle new syntax: span=1h - @Override - public UnresolvedExpression visitSpanLiteral(OpenSearchPPLParser.SpanLiteralContext ctx) { - if (ctx.integerLiteral() != null && ctx.timespanUnit() != null) { - return new Span( - AstDSL.field("@timestamp"), - new Literal(Integer.parseInt(ctx.integerLiteral().getText()), DataType.INTEGER), - SpanUnit.of(ctx.timespanUnit().getText())); - } - - if (ctx.integerLiteral() != null) { - return new Span( - AstDSL.field("@timestamp"), - new Literal(Integer.parseInt(ctx.integerLiteral().getText()), DataType.INTEGER), - SpanUnit.of("")); + UnresolvedExpression fieldExpression; + if (ctx.fieldExpression() != null) { + fieldExpression = visit(ctx.fieldExpression()); + } else { + fieldExpression = AstDSL.field("@timestamp"); } - - return new Span( - AstDSL.field("@timestamp"), new Literal(ctx.getText(), DataType.STRING), SpanUnit.of("")); + Literal literal = (Literal) visit(ctx.value); + return AstDSL.spanFromSpanLengthLiteral(fieldExpression, literal); } @Override @@ -772,19 +757,21 @@ private List multiFieldRelevanceArguments( // New visitor methods for spanValue grammar rules @Override - public UnresolvedExpression visitNumericSpanValue( - OpenSearchPPLParser.NumericSpanValueContext ctx) { - String spanValue = ctx.literalValue().getText(); - String spanUnit = ctx.timespanUnit() != null ? ctx.timespanUnit().getText() : null; - - if (spanUnit != null) { - // Create combined span like "1h", "30m", etc. - return org.opensearch.sql.ast.dsl.AstDSL.stringLiteral(spanValue + spanUnit); + public UnresolvedExpression visitSpanLiteral(OpenSearchPPLParser.SpanLiteralContext ctx) { + if (ctx.INTEGER_LITERAL() != null) { + return AstDSL.intLiteral(Integer.parseInt(ctx.INTEGER_LITERAL().getText())); } else { - return visit(ctx.literalValue()); + return AstDSL.stringLiteral(ctx.getText()); } } + @Override + public UnresolvedExpression visitNumericSpanValue( + OpenSearchPPLParser.NumericSpanValueContext ctx) { + // This handles span values that come from spanLiteral rule + return visit(ctx.spanLiteral()); + } + @Override public UnresolvedExpression visitLogWithBaseSpan(OpenSearchPPLParser.LogWithBaseSpanContext ctx) { return org.opensearch.sql.ast.dsl.AstDSL.stringLiteral(ctx.getText()); @@ -892,11 +879,6 @@ public SearchLiteral visitSearchLiteral(OpenSearchPPLParser.SearchLiteralContext // Boolean literal Literal booleanLiteral = (Literal) visit(ctx.booleanLiteral()); return new SearchLiteral(booleanLiteral, false); - } else if (ctx.ID() != null) { - return new SearchLiteral(new Literal(ctx.ID().getText(), DataType.STRING), false); - } else if (ctx.searchableKeyWord() != null) { - return new SearchLiteral( - new Literal(ctx.searchableKeyWord().getText(), DataType.STRING), false); } // Default return new SearchLiteral(new Literal(ctx.getText(), DataType.STRING), false); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java index b5d154273cc..0e0068b5169 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java @@ -457,7 +457,7 @@ public void testAvgBySpanAndFields() { @Test public void testAvgByTimeSpanAndFields() { String ppl = - "source=EMP | stats avg(SAL) by span(HIREDATE, 1 year) as hiredate_span, DEPTNO | sort" + "source=EMP | stats avg(SAL) by span(HIREDATE, 1year) as hiredate_span, DEPTNO | sort" + " DEPTNO, hiredate_span"; RelNode root = getRelNode(ppl); String expectedLogical =