Merge branch 'main' of https://github.com/datafuselabs/databend into …

…distributed-sort

.github/workflows/release.yml

@@ -106,6 +106,7 @@ jobs:
           gh release view --repo datafuselabs/databend ${{ needs.create_release.outputs.version }} >> $df
           sed -i -E 's/^--$/---/g' $df
           sed -i -E '/^asset:/d' $df
+          sed -i -E 's_https://github.com/datafuselabs/databend/pull/([0-9]+)_[#\1](https://github.com/datafuselabs/databend/pull/\1)_g' $df
           git add docs/release-nightly
           git status
       - uses: peter-evans/create-pull-request@v4

README.md

@@ -62,7 +62,7 @@
 
 ### 1. Databend Serverless Cloud
 
-The fastest way to try Databend, [Databend Cloud](https://databend.rs/doc/cloud/)
+The fastest way to try Databend, [Databend Cloud](https://databend.com)
 
 ### 2. Install Databend from Docker
 

src/query/ast/src/ast/format/ast_format.rs

@@ -2777,6 +2777,16 @@ impl<'ast> Visitor<'ast> for AstFormatVisitor {
             children.push(window_list_node);
         }
 
+        if let Some(qualify) = &stmt.qualify {
+            self.visit_expr(qualify);
+            let qualify_child = self.children.pop().unwrap();
+            let qualify_name = "Qualify".to_string();
+            let qualify_format_ctx = AstFormatContext::with_children(qualify_name, 1);
+            let qualify_node =
+                FormatTreeNode::with_children(qualify_format_ctx, vec![qualify_child]);
+            children.push(qualify_node);
+        }
+
         let name = "SelectQuery".to_string();
         let format_ctx = AstFormatContext::with_children(name, children.len());
         let node = FormatTreeNode::with_children(format_ctx, children);

src/query/ast/src/ast/query.rs

@@ -92,6 +92,8 @@ pub struct SelectStmt {
     pub having: Option<Expr>,
     // `WINDOW` clause
     pub window_list: Option<Vec<WindowDefinition>>,
+    // `QUALIFY` clause
+    pub qualify: Option<Expr>,
 }
 
 /// Group by Clause.

src/query/ast/src/parser/expr.rs

@@ -296,6 +296,13 @@ pub enum ExprElement {
         args: Vec<Expr>,
         lambda: Option<Lambda>,
     },
+    /// python/rust list comprehension
+    ListComprehension {
+        source: Expr,
+        param: Identifier,
+        filter: Option<Expr>,
+        result: Expr,
+    },
     /// An expression between parentheses
     Group(Expr),
     /// `[1, 2, 3]`
@@ -518,6 +525,44 @@ impl<'a, I: Iterator<Item = WithSpan<'a, ExprElement>>> PrattParser<I> for ExprP
                 span: transform_span(elem.span.0),
                 exprs,
             },
+            ExprElement::ListComprehension {
+                source,
+                param,
+                filter,
+                result,
+            } => {
+                let span = transform_span(elem.span.0);
+                let mut source = source;
+
+                // array_filter(source, filter)
+                if let Some(filter) = filter {
+                    source = Expr::FunctionCall {
+                        span,
+                        distinct: false,
+                        name: Identifier::from_name("array_filter"),
+                        args: vec![source],
+                        params: vec![],
+                        window: None,
+                        lambda: Some(Lambda {
+                            params: vec![param.clone()],
+                            expr: Box::new(filter),
+                        }),
+                    };
+                }
+                // array_map(source, result)
+                Expr::FunctionCall {
+                    span,
+                    distinct: false,
+                    name: Identifier::from_name("array_map"),
+                    args: vec![source],
+                    params: vec![],
+                    window: None,
+                    lambda: Some(Lambda {
+                        params: vec![param.clone()],
+                        expr: Box::new(result),
+                    }),
+                }
+            }
             ExprElement::Map { kvs } => Expr::Map {
                 span: transform_span(elem.span.0),
                 kvs,
@@ -1022,6 +1067,28 @@ pub fn expr_element(i: Input) -> IResult<WithSpan<ExprElement>> {
         )),
     );
 
+    // python style list comprehensions
+    // python: [i for i in range(10) if i%2==0 ]
+    // sql: [i for i in range(10) if i%2 = 0 ]
+    let list_comprehensions = check_experimental_chain_function(
+        true,
+        map(
+            rule! {
+                "[" ~ #subexpr(0) ~ FOR ~ #ident ~ IN
+                ~ #subexpr(0) ~ (IF ~ #subexpr(2))? ~ "]"
+            },
+            |(_, result, _, param, _, source, opt_filter, _)| {
+                let filter = opt_filter.map(|(_, filter)| filter);
+                ExprElement::ListComprehension {
+                    source,
+                    param,
+                    filter,
+                    result,
+                }
+            },
+        ),
+    );
+
     // Floating point literal with leading dot will be parsed as a period map access,
     // and then will be converted back to a floating point literal if the map access
     // is not following a primary element nor a postfix element.
@@ -1154,6 +1221,7 @@ pub fn expr_element(i: Input) -> IResult<WithSpan<ExprElement>> {
             | #trim_from : "`TRIM([(BOTH | LEADEING | TRAILING) ... FROM ...)`"
             | #is_distinct_from: "`... IS [NOT] DISTINCT FROM ...`"
             | #chain_function_call : "x.func(...)"
+            | #list_comprehensions: "[expr for x in ... [if ...]]"
             | #count_all_with_window : "`COUNT(*) OVER ...`"
             | #function_call : "<function>"
             | #case : "`CASE ... END`"
@@ -1684,6 +1752,11 @@ pub fn parse_float(text: &str) -> Result<Literal, ErrorKind> {
 
 pub fn parse_uint(text: &str, radix: u32) -> Result<Literal, ErrorKind> {
     let text = text.trim_start_matches('0');
+    let contains_underscore = text.contains('_');
+    if contains_underscore {
+        let text = text.replace(|p| p == '_', "");
+        return parse_uint(&text, radix);
+    }
 
     if text.is_empty() {
         return Ok(Literal::UInt64(0));

src/query/ast/src/parser/query.rs

@@ -58,6 +58,7 @@ pub enum SetOperationElement {
         group_by: Option<GroupBy>,
         having: Box<Option<Expr>>,
         window_list: Option<Vec<WindowDefinition>>,
+        qualify: Box<Option<Expr>>,
     },
     SetOperation {
         op: SetOperator,
@@ -104,6 +105,7 @@ pub fn set_operation_element(i: Input) -> IResult<WithSpan<SetOperationElement>>
                 ~ ( GROUP ~ ^BY ~ ^#group_by_items )?
                 ~ ( HAVING ~ ^#expr )?
                 ~ ( WINDOW ~ ^#comma_separated_list1(window_clause) )?
+                ~ ( QUALIFY ~ ^#expr )?
         },
         |(
             _select,
@@ -115,6 +117,7 @@ pub fn set_operation_element(i: Input) -> IResult<WithSpan<SetOperationElement>>
             opt_group_by_block,
             opt_having_block,
             opt_window_block,
+            opt_qualify_block,
         )| {
             SetOperationElement::SelectStmt {
                 hints: opt_hints,
@@ -129,6 +132,7 @@ pub fn set_operation_element(i: Input) -> IResult<WithSpan<SetOperationElement>>
                 group_by: opt_group_by_block.map(|(_, _, group_by)| group_by),
                 having: Box::new(opt_having_block.map(|(_, having)| having)),
                 window_list: opt_window_block.map(|(_, windows)| windows),
+                qualify: Box::new(opt_qualify_block.map(|(_, qualify)| qualify)),
             }
         },
     );
@@ -142,6 +146,7 @@ pub fn set_operation_element(i: Input) -> IResult<WithSpan<SetOperationElement>>
                 ~ ( GROUP ~ ^BY ~ ^#group_by_items )?
                 ~ ( HAVING ~ ^#expr )?
                 ~ ( WINDOW ~ ^#comma_separated_list1(window_clause) )?
+                ~ ( QUALIFY ~ ^#expr )?
         },
         |(
             opt_from_block,
@@ -153,6 +158,7 @@ pub fn set_operation_element(i: Input) -> IResult<WithSpan<SetOperationElement>>
             opt_group_by_block,
             opt_having_block,
             opt_window_block,
+            opt_qualify_block,
         )| {
             SetOperationElement::SelectStmt {
                 hints: opt_hints,
@@ -167,6 +173,7 @@ pub fn set_operation_element(i: Input) -> IResult<WithSpan<SetOperationElement>>
                 group_by: opt_group_by_block.map(|(_, _, group_by)| group_by),
                 having: Box::new(opt_having_block.map(|(_, having)| having)),
                 window_list: opt_window_block.map(|(_, windows)| windows),
+                qualify: Box::new(opt_qualify_block.map(|(_, qualify)| qualify)),
             }
         },
     );
@@ -262,6 +269,7 @@ impl<'a, I: Iterator<Item = WithSpan<'a, SetOperationElement>>> PrattParser<I>
                 group_by,
                 having,
                 window_list,
+                qualify,
             } => SetExpr::Select(Box::new(SelectStmt {
                 span: transform_span(input.span.0),
                 hints,
@@ -272,6 +280,7 @@ impl<'a, I: Iterator<Item = WithSpan<'a, SetOperationElement>>> PrattParser<I>
                 group_by,
                 having: *having,
                 window_list,
+                qualify: *qualify,
             })),
             SetOperationElement::Values(values) => SetExpr::Values {
                 span: transform_span(input.span.0),

src/query/ast/src/parser/token.rs

@@ -159,7 +159,7 @@ pub enum TokenKind {
     #[regex(r"0[xX][a-fA-F0-9]+")]
     MySQLLiteralHex,
 
-    #[regex(r"[0-9]+")]
+    #[regex(r"[0-9]+(_|[0-9])*")]
     LiteralInteger,
 
     #[regex(r"[0-9]+[eE][+-]?[0-9]+")]
@@ -807,6 +807,8 @@ pub enum TokenKind {
     PRESIGN,
     #[token("PRIVILEGES", ignore(ascii_case))]
     PRIVILEGES,
+    #[token("QUALIFY", ignore(ascii_case))]
+    QUALIFY,
     #[token("REMOVE", ignore(ascii_case))]
     REMOVE,
     #[token("RETAIN", ignore(ascii_case))]
@@ -1301,6 +1303,7 @@ impl TokenKind {
             | TokenKind::OF
             | TokenKind::ORDER
             | TokenKind::OVER
+            | TokenKind::QUALIFY
             | TokenKind::ROWS
             // | TokenKind::PRECISION
             // | TokenKind::RETURNING
@@ -1355,6 +1358,7 @@ impl TokenKind {
             | TokenKind::FALSE
             // | TokenKind::FOREIGN
             // | TokenKind::FREEZE
+            | TokenKind::FOR
             | TokenKind::FULL
             // | TokenKind::ILIKE
             | TokenKind::IN
@@ -1404,7 +1408,6 @@ impl TokenKind {
             | TokenKind::ATTACH
             | TokenKind::EXCEPT
             // | TokenKind::FETCH
-            | TokenKind::FOR
             | TokenKind::FROM
             // | TokenKind::GRANT
             | TokenKind::GROUP
@@ -1421,6 +1424,7 @@ impl TokenKind {
             | TokenKind::ORDER
             | TokenKind::OVER
             | TokenKind::PARTITION
+            | TokenKind::QUALIFY
             | TokenKind::ROWS
             | TokenKind::RANGE
             // | TokenKind::OVERLAPS