Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
82 commits
Select commit Hold shift + click to select a range
a141c22
Support bin command with Calcite
ahkcs Jul 14, 2025
36d3d2c
add .gitignore
ahkcs Jul 14, 2025
366ff5f
Remove settings file
ahkcs Jul 14, 2025
0426092
add performance testing
ahkcs Jul 15, 2025
bfce28c
Fix CI
ahkcs Jul 15, 2025
9a79063
fix style
ahkcs Jul 15, 2025
6ef78c4
support minspan
ahkcs Jul 15, 2025
7d0e75b
support aligntime
ahkcs Jul 15, 2025
47b6764
fix
ahkcs Jul 15, 2025
52f55dd
fix CI
ahkcs Jul 16, 2025
410c088
refactor
ahkcs Jul 18, 2025
39525bd
refactor
ahkcs Jul 18, 2025
a664bd2
formatting
ahkcs Jul 18, 2025
ead4169
fix CI
ahkcs Jul 18, 2025
6e2d5ec
Big Bin commands Modification
ahkcs Jul 29, 2025
f5d60a3
fix CI
ahkcs Jul 29, 2025
727c9b2
fix syntax
ahkcs Jul 29, 2025
be3b123
Major change in Bin command
ahkcs Aug 5, 2025
bcf5408
update CalciteExplainIT
ahkcs Aug 6, 2025
3c79545
fix range string parsing
ahkcs Aug 7, 2025
7145101
fix aligntime
ahkcs Aug 10, 2025
2fae14c
fix subseconds
ahkcs Aug 10, 2025
367031a
IT change
ahkcs Aug 11, 2025
1e00bc4
fix IT
ahkcs Aug 11, 2025
4e731b8
update explain IT
ahkcs Aug 11, 2025
2cc7d36
update bin.rst
ahkcs Aug 11, 2025
6370d0c
fix IT
ahkcs Aug 11, 2025
a86cd6d
delete unnecessary code
ahkcs Aug 11, 2025
a55e93e
fix CI
ahkcs Aug 11, 2025
edfaf6f
fix bin aggregation
ahkcs Aug 12, 2025
7f2a874
fix IT
ahkcs Aug 12, 2025
71873b4
fixes
ahkcs Aug 12, 2025
8478df3
fix
ahkcs Aug 12, 2025
e1824de
add thorough IT tests
ahkcs Aug 12, 2025
ff7447d
add support for log
ahkcs Aug 13, 2025
04bffcd
remove
ahkcs Aug 13, 2025
66e33a0
fix
ahkcs Aug 13, 2025
acc9d94
wraps the bins parameter logic
ahkcs Aug 14, 2025
47bd4ad
TIMESTAMP and flexible order of command options
ahkcs Aug 14, 2025
fd8d488
fix CI
ahkcs Aug 14, 2025
4eced40
optimize plan
ahkcs Aug 15, 2025
dd711e4
renaming
ahkcs Aug 15, 2025
70ce83b
use mathematic approach for nice_width
ahkcs Aug 15, 2025
230696f
fix doc and log
ahkcs Aug 15, 2025
5a988cc
refactored the Bin class into an abstract class
ahkcs Aug 18, 2025
9c8f749
Refactoring
ahkcs Aug 18, 2025
dba465b
fix CI
ahkcs Aug 18, 2025
b749400
rebase and fix CI
ahkcs Aug 18, 2025
44bd865
fixes
ahkcs Aug 18, 2025
a2c040d
fix aggregation issue
ahkcs Aug 19, 2025
9d56ada
style fixes
ahkcs Aug 19, 2025
aebfb2d
remove changes on datetime functions
ahkcs Aug 19, 2025
b26379f
revert changes
ahkcs Aug 19, 2025
d6d036f
put files back
ahkcs Aug 19, 2025
3815fc0
fixes
ahkcs Aug 19, 2025
fc11a62
fix
ahkcs Aug 19, 2025
a936353
bin.rst update
ahkcs Aug 20, 2025
4db6506
remove
ahkcs Aug 20, 2025
f988430
Add to CrossClusterSearchIT
ahkcs Aug 20, 2025
26debf1
Add CrossClusterBinCommandIT
ahkcs Aug 20, 2025
c0a1bba
enable calcite
ahkcs Aug 20, 2025
f2c3401
Fix
ahkcs Aug 20, 2025
0b2d843
Fix CI
ahkcs Aug 20, 2025
c7bd21a
fixes
ahkcs Aug 21, 2025
4d81ccd
fix explain
ahkcs Aug 22, 2025
350eaf4
Fixes
ahkcs Aug 22, 2025
c844af2
remove
ahkcs Aug 22, 2025
e3a765e
update
ahkcs Aug 22, 2025
6f7ae56
Add UT
ahkcs Aug 22, 2025
598e84a
fix CI
ahkcs Aug 22, 2025
7baf3f1
modify bin.rst
ahkcs Aug 22, 2025
f2232fb
modify path
ahkcs Aug 22, 2025
cd3b499
update doc and redundancy
ahkcs Aug 22, 2025
679368c
move cross cluster IT
ahkcs Aug 22, 2025
183dbdc
Merge branch 'main' into feat/bin_command
ahkcs Aug 22, 2025
4079fdf
fix CI
ahkcs Aug 22, 2025
5731736
separate aggregation
ahkcs Aug 22, 2025
b2fbe70
remove unused test
ahkcs Aug 22, 2025
6f4c801
fixes
ahkcs Aug 25, 2025
2291506
fixes
ahkcs Aug 25, 2025
10dd000
update bin.rst
ahkcs Aug 25, 2025
71ee3d3
fixes
ahkcs Aug 25, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import org.opensearch.sql.ast.tree.AD;
import org.opensearch.sql.ast.tree.Aggregation;
import org.opensearch.sql.ast.tree.AppendCol;
import org.opensearch.sql.ast.tree.Bin;
import org.opensearch.sql.ast.tree.CloseCursor;
import org.opensearch.sql.ast.tree.Dedupe;
import org.opensearch.sql.ast.tree.Eval;
Expand Down Expand Up @@ -669,6 +670,12 @@ public LogicalPlan visitML(ML node, AnalysisContext context) {
return new LogicalML(child, node.getArguments());
}

@Override
public LogicalPlan visitBin(Bin node, AnalysisContext context) {
throw new UnsupportedOperationException(
"Bin command is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
}

@Override
public LogicalPlan visitExpand(Expand expand, AnalysisContext context) {
throw new UnsupportedOperationException(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.opensearch.sql.ast.tree.AD;
import org.opensearch.sql.ast.tree.Aggregation;
import org.opensearch.sql.ast.tree.AppendCol;
import org.opensearch.sql.ast.tree.Bin;
import org.opensearch.sql.ast.tree.CloseCursor;
import org.opensearch.sql.ast.tree.Dedupe;
import org.opensearch.sql.ast.tree.Eval;
Expand Down Expand Up @@ -213,6 +214,10 @@ public T visitBetween(Between node, C context) {
return visitChildren(node, context);
}

public T visitBin(Bin node, C context) {
return visitChildren(node, context);
}

public T visitArgument(Argument node, C context) {
return visitChildren(node, context);
}
Expand Down
94 changes: 94 additions & 0 deletions core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java
Original file line number Diff line number Diff line change
Expand Up @@ -49,24 +49,30 @@
import org.opensearch.sql.ast.expression.WindowFunction;
import org.opensearch.sql.ast.expression.Xor;
import org.opensearch.sql.ast.tree.Aggregation;
import org.opensearch.sql.ast.tree.Bin;
import org.opensearch.sql.ast.tree.CountBin;
import org.opensearch.sql.ast.tree.Dedupe;
import org.opensearch.sql.ast.tree.DefaultBin;
import org.opensearch.sql.ast.tree.DescribeRelation;
import org.opensearch.sql.ast.tree.Eval;
import org.opensearch.sql.ast.tree.Expand;
import org.opensearch.sql.ast.tree.FillNull;
import org.opensearch.sql.ast.tree.Filter;
import org.opensearch.sql.ast.tree.Head;
import org.opensearch.sql.ast.tree.Limit;
import org.opensearch.sql.ast.tree.MinSpanBin;
import org.opensearch.sql.ast.tree.Parse;
import org.opensearch.sql.ast.tree.Patterns;
import org.opensearch.sql.ast.tree.Project;
import org.opensearch.sql.ast.tree.RangeBin;
import org.opensearch.sql.ast.tree.RareTopN;
import org.opensearch.sql.ast.tree.RareTopN.CommandType;
import org.opensearch.sql.ast.tree.Relation;
import org.opensearch.sql.ast.tree.RelationSubquery;
import org.opensearch.sql.ast.tree.Rename;
import org.opensearch.sql.ast.tree.Sort;
import org.opensearch.sql.ast.tree.Sort.SortOption;
import org.opensearch.sql.ast.tree.SpanBin;
import org.opensearch.sql.ast.tree.SubqueryAlias;
import org.opensearch.sql.ast.tree.TableFunction;
import org.opensearch.sql.ast.tree.Trendline;
Expand Down Expand Up @@ -561,4 +567,92 @@ public static FillNull fillNull(
}
return FillNull.ofVariousValue(replacementsBuilder.build()).attach(input);
}

/**
* Creates a Bin node with an input plan for binning field values into discrete buckets.
*
* @param input the input plan
* @param field the field expression to bin
* @param arguments optional arguments for bin configuration (span, bins, minspan, aligntime,
* start, end, alias)
* @return Bin node attached to the input plan
*/
public static Bin bin(UnresolvedPlan input, UnresolvedExpression field, Argument... arguments) {
Bin binNode = bin(field, arguments);
binNode.attach(input);
return binNode;
}
Comment on lines +570 to +584
Copy link
Collaborator

@yuancu yuancu Aug 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I doubt whether changes to AstDSL.java is necessary since you have marked bin command as only available when Calcite is enabled. AstDSL.java is mainly used to build expression AST for v2. (Please correct me if I am wrong)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is AST layer. We can keep this API.


/**
* Creates a Bin node for binning field values into discrete buckets. Returns the appropriate Bin
* subclass based on parameter priority: 1. SPAN (highest) -> SpanBin 2. MINSPAN -> MinSpanBin 3.
* BINS -> CountBin 4. START/END only -> RangeBin 5. No params -> DefaultBin
*
* @param field the field expression to bin
* @param arguments optional arguments for bin configuration (span, bins, minspan, aligntime,
* start, end, alias)
* @return Bin node with the specified field and configuration
*/
public static Bin bin(UnresolvedExpression field, Argument... arguments) {
UnresolvedExpression span = null;
Integer bins = null;
UnresolvedExpression minspan = null;
UnresolvedExpression aligntime = null;
UnresolvedExpression start = null;
UnresolvedExpression end = null;
String alias = null;

for (Argument arg : arguments) {
switch (arg.getArgName()) {
case "span":
span = arg.getValue();
break;
case "bins":
bins =
(arg.getValue()).getValue() instanceof Integer
? (Integer) (arg.getValue()).getValue()
: null;
break;
case "minspan":
minspan = arg.getValue();
break;
case "aligntime":
aligntime = arg.getValue();
break;
case "start":
start = arg.getValue();
break;
case "end":
end = arg.getValue();
break;
case "alias":
alias = arg.getValue().toString();
break;
}
}

// Create appropriate Bin subclass based on priority order
if (span != null) {
// 1. SPAN (highest priority) -> SpanBin
return SpanBin.builder().field(field).span(span).aligntime(aligntime).alias(alias).build();
} else if (minspan != null) {
// 2. MINSPAN (second priority) -> MinSpanBin
return MinSpanBin.builder()
.field(field)
.minspan(minspan)
.start(start)
.end(end)
.alias(alias)
.build();
} else if (bins != null) {
// 3. BINS (third priority) -> CountBin
return CountBin.builder().field(field).bins(bins).start(start).end(end).alias(alias).build();
} else if (start != null || end != null) {
// 4. START/END only (fourth priority) -> RangeBin
return RangeBin.builder().field(field).start(start).end(end).alias(alias).build();
} else {
// 5. No parameters (default) -> DefaultBin
return DefaultBin.builder().field(field).alias(alias).build();
}
}
}
57 changes: 57 additions & 0 deletions core/src/main/java/org/opensearch/sql/ast/tree/Bin.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.ast.tree;

import com.google.common.collect.ImmutableList;
import java.util.List;
import javax.annotation.Nullable;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.opensearch.sql.ast.AbstractNodeVisitor;
import org.opensearch.sql.ast.expression.UnresolvedExpression;

/** Abstract AST node representing Bin operations with type-safe derived classes. */
@Getter
@Setter
@ToString
@EqualsAndHashCode(callSuper = false)
public abstract class Bin extends UnresolvedPlan {

private UnresolvedPlan child;

protected final UnresolvedExpression field;

@Nullable protected final String alias;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's okey but I don't suggest to use @Nullable annotation. Instead, I suggest to use Optional.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated to use Optional

Copy link
Collaborator

@yuancu yuancu Aug 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@LantaoJin Using optional as a field type will cause a warning, e.g. 'Optional<String>' used as type for field 'name' . It seems that using Optional type as field / parameter types is discouraged? Java
'Optional' used as field or parameter type - JetBrains

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @yuancu. For now, there is no harmful to use Optional as type of field 'name' because the UnresolvedPlan itself is not serializable.


protected Bin(UnresolvedExpression field, @Nullable String alias) {
this.field = field;
this.alias = alias;
}

/**
* Validates the parameters specific to this bin type. Each subclass implements its own validation
* logic.
*/
public abstract void validate();

@Override
public Bin attach(UnresolvedPlan child) {
this.child = child;
return this;
}

@Override
public List<UnresolvedPlan> getChild() {
return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child);
}

@Override
public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
return nodeVisitor.visitBin(this, context);
}
}
55 changes: 55 additions & 0 deletions core/src/main/java/org/opensearch/sql/ast/tree/CountBin.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.ast.tree;

import javax.annotation.Nullable;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.ToString;
import org.opensearch.sql.ast.expression.UnresolvedExpression;
import org.opensearch.sql.calcite.utils.binning.BinConstants;

/**
* AST node representing count-based bin operation. This is the third priority bin type that uses
* "nice number" algorithm to create a specific number of bins. Supports start/end range parameters.
*/
@Getter
@ToString(callSuper = true)
@EqualsAndHashCode(callSuper = true)
public class CountBin extends Bin {

private final Integer bins;

@Nullable private final UnresolvedExpression start;

@Nullable private final UnresolvedExpression end;

@Builder
public CountBin(
UnresolvedExpression field,
@Nullable String alias,
Integer bins,
@Nullable UnresolvedExpression start,
@Nullable UnresolvedExpression end) {
super(field, alias);
this.bins = bins;
this.start = start;
this.end = end;
validate();
}

@Override
public void validate() {
// Bins count validation based on documentation
if (bins < BinConstants.MIN_BINS || bins > BinConstants.MAX_BINS) {
throw new IllegalArgumentException(
String.format(
"The bins parameter must be between %d and %d, got: %d",
BinConstants.MIN_BINS, BinConstants.MAX_BINS, bins));
}
}
}
35 changes: 35 additions & 0 deletions core/src/main/java/org/opensearch/sql/ast/tree/DefaultBin.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.ast.tree;

import javax.annotation.Nullable;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.ToString;
import org.opensearch.sql.ast.expression.UnresolvedExpression;

/**
* AST node representing default magnitude-based bin operation. This is the lowest priority bin type
* that uses automatic magnitude-based algorithm when no explicit binning parameters are specified.
*/
@Getter
@ToString(callSuper = true)
@EqualsAndHashCode(callSuper = true)
public class DefaultBin extends Bin {

@Builder
public DefaultBin(UnresolvedExpression field, @Nullable String alias) {
super(field, alias);
validate();
}

@Override
public void validate() {
// Default bin has no additional parameters to validate
// Field validation is already handled in the base class
}
}
46 changes: 46 additions & 0 deletions core/src/main/java/org/opensearch/sql/ast/tree/MinSpanBin.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.ast.tree;

import javax.annotation.Nullable;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.ToString;
import org.opensearch.sql.ast.expression.UnresolvedExpression;

/**
* AST node representing minimum span-based bin operation. This is the second priority bin type that
* uses magnitude-based algorithm with minimum span constraint. Supports start/end range parameters.
*/
@Getter
@ToString(callSuper = true)
@EqualsAndHashCode(callSuper = true)
public class MinSpanBin extends Bin {

private final UnresolvedExpression minspan;

@Nullable private final UnresolvedExpression start;

@Nullable private final UnresolvedExpression end;

@Builder
public MinSpanBin(
UnresolvedExpression field,
@Nullable String alias,
UnresolvedExpression minspan,
@Nullable UnresolvedExpression start,
@Nullable UnresolvedExpression end) {
super(field, alias);
this.minspan = minspan;
this.start = start;
this.end = end;
validate();
}

@Override
public void validate() {}
}
Loading
Loading