Feather/linesplit and line continue

.gitignore

@@ -1,3 +1,4 @@
 /target
 history.txt
 *.cast
+/target*

CHANGELOG.md

@@ -1,8 +1,7 @@
 # Changelog
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+### [0.2.0-lineclip] - 2025-3-29
+- use `\n` or `;` to split statement. which means you don't have to type `;` to every lineend.
+- use `\\n` to continue a line.
 
 ## [Unreleased]
 

src/bin.rs

@@ -273,6 +273,12 @@ fn syntax_highlight(line: &str) -> String {
             (TokenKind::Whitespace, w) => {
                 result.push_str(w);
             }
+            (TokenKind::LineBreak, w) => {
+                result.push_str(w);
+            }
+            // (TokenKind::LineContinuation, w) => {
+            //     result.push_str(w);
+            // }
             (TokenKind::Comment, w) => {
                 result.push_str("\x1b[38;5;247m");
                 is_colored = true;
@@ -546,14 +552,14 @@ fn run_file(path: PathBuf, env: &mut Environment) -> Result<Expression, Error> {
 fn main() -> Result<(), Error> {
     let matches = App::new(
         r#"
-        888                            
-        888                            
-        888                            
-    .d88888 888  888 88888b.   .d88b.  
-   d88" 888 888  888 888 "88b d8P  Y8b 
-   888  888 888  888 888  888 88888888 
-   Y88b 888 Y88b 888 888  888 Y8b.     
-    "Y88888  "Y88888 888  888  "Y8888  
+        888
+        888
+        888
+    .d88888 888  888 88888b.   .d88b.
+   d88" 888 888  888 888 "88b d8P  Y8b
+   888  888 888  888 888  888 88888888
+   Y88b 888 Y88b 888 888  888 Y8b.
+    "Y88888  "Y88888 888  888  "Y8888
    "#,
     )
     .author(crate_authors!())

src/parser.rs

@@ -183,38 +183,62 @@ fn is_symbol_like(kind: TokenKind) -> bool {
 }
 
 fn parse_statement(input: Tokens<'_>) -> IResult<Tokens<'_>, Expression, SyntaxError> {
+    // let (input, _) = opt(kind(TokenKind::LineBreak))(input)?;
     let (input, expr) = parse_expression(input)?;
-    match (&expr, text(";")(input)) {
-        (Expression::For(_, _, _), Ok((input, _))) => Ok((input, expr)),
-        (Expression::For(_, _, _), Err(_)) => Ok((input, expr)),
-        (Expression::If(_, _, _), Ok((input, _))) => Ok((input, expr)),
-        (Expression::If(_, _, _), Err(_)) => Ok((input, expr)),
-
-        (_, Ok((input, _))) => Ok((input, expr)),
-        (_, Err(_)) => Err(SyntaxError::expected(
-            input.get_str_slice(),
-            ";",
-            None,
-            Some("try adding a semicolon"),
-        )),
-    }
+    // dbg!("---[got expression]---", &expr, input.get_str_slice());
+
+    // 尝试消费终止符（分号或换行符）
+    match &expr {
+        // 控制结构不需要显式终止符
+        Expression::For(_, _, _) | Expression::If(_, _, _) => {
+            // opt(kind(TokenKind::NewLine))(input)?; //消费换行符
+            return Ok((input, expr));
+        }
+        // 普通表达式需要终止符
+        _ => {
+            // 尝试匹配分号或换行符
+            // (input, _) = lineterminator(input)?;
+            // 检查是否存在其他终止符（如分号）
+            // 允许行继续符出现在语句结尾
+            // let (input, _) = alt((
+            //     map(kind(TokenKind::LineContinuation), |_| ()),
+            //     map(kind(TokenKind::LineBreak), |_| ()),
+            // ))(input)?;
+            let (input, _) = kind(TokenKind::LineBreak)(input)?;
+            // let (input, _) = opt(kind(TokenKind::LineBreak))(input)?;
+            // let (input, _) = opt(text(";"))(input)?;
+
+            // dbg!("---[got expression lineEnd]---");
+            return Ok((input, expr));
+        }
+    };
 }
 
 fn parse_script_tokens(
     input: Tokens<'_>,
     require_eof: bool,
 ) -> IResult<Tokens<'_>, Expression, SyntaxError> {
-    // println!("hmm {}", input);
+    // print!("passing scripot tokens")
+    // dbg!("parse script tokens ------>", input);
     let (input, mut exprs) = many0(parse_statement)(input)?;
+    // dbg!("parse_statement-->", input.get_str_slice(), &exprs);
 
-    let (mut input, last) = opt(terminated(parse_expression, opt(text(";"))))(input)?;
+    // 解析最后一行，可选的;作为终止
+    let (input, last) = opt(terminated(
+        parse_expression,
+        opt(kind(TokenKind::LineBreak)),
+    ))(input)?;
+    // dbg!("after terminated-->", input.get_str_slice(), &last);
 
     if let Some(expr) = last {
         exprs.push(expr);
     }
+    // 新增：清理所有末尾换行符
+    let (input, _) = many0(kind(TokenKind::LineBreak))(input)?;
 
     if require_eof {
-        input = eof(input)
+        // input.is_empty()
+        eof(input)
             .map_err(|_: nom::Err<SyntaxError>| {
                 SyntaxError::expected(input.get_str_slice(), "end of input", None, None)
             })?
@@ -451,7 +475,8 @@ fn parse_for_loop(input: Tokens<'_>) -> IResult<Tokens<'_>, Expression, SyntaxEr
 
 fn parse_if(input: Tokens<'_>) -> IResult<Tokens<'_>, Expression, SyntaxError> {
     let (input, _) = text("if")(input)?;
-
+    //
+    // dbg!(input);
     let (input, cond) = parse_expression_prec_six(input).map_err(|_| {
         SyntaxError::unrecoverable(
             input.get_str_slice(),
@@ -543,10 +568,19 @@ fn parse_apply_operator(input: Tokens<'_>) -> IResult<Tokens<'_>, Expression, Sy
 
 fn parse_expression(input: Tokens<'_>) -> IResult<Tokens<'_>, Expression, SyntaxError> {
     no_terminating_punctuation(input)?;
+    // 过滤行继续符和后续换行符
+    let (input, _) = opt(kind(TokenKind::LineBreak))(input)?; // 消费行继续符
+
+    // let (input, _) = many0(alt((
+    //     // 匹配0次或多次
+    //     kind(TokenKind::LineContinuation), // 消费行继续符
+    //     kind(TokenKind::LineBreak), // 消费换行符
+    // )))(input)?;
 
     let expr_parser = parse_expression_prec_seven;
 
     let (input, head) = expr_parser(input)?;
+    // let (input, _) = opt(kind(TokenKind::LineContinuation))(input)?; // 消费行继续符
 
     let (input, list) = many0(pair(
         alt((text("|"), text(">>>"), text(">>"), text("<<"))),

src/tokenizer.rs

@@ -42,7 +42,13 @@ fn parse_token(input: Input) -> TokenizationResult<'_, (Token, Diagnostic)> {
     if input.is_empty() {
         Err(NOT_FOUND)
     } else {
+        // dbg!("------>", input);
+
         Ok(alt((
+            // 优先处理续航、换行符（新增）
+            map_valid_token(line_continuation, TokenKind::Whitespace),
+            // triple_quote_string,
+            map_valid_token(linebreak, TokenKind::LineBreak),
             map_valid_token(long_operator, TokenKind::Operator),
             map_valid_token(any_punctuation, TokenKind::Punctuation),
             map_valid_token(any_keyword, TokenKind::Keyword),
@@ -141,6 +147,59 @@ fn string_literal(input: Input<'_>) -> TokenizationResult<'_, (Token, Diagnostic
     let token = Token::new(TokenKind::StringLiteral, range);
     Ok((rest, (token, diagnostics)))
 }
+// 新增函数：专门处理三重引号字符串
+// fn triple_quote_string(input: Input<'_>) -> TokenizationResult<'_, (Token, Diagnostic)> {
+//     // 1. 匹配起始 """
+//     let (rest, _) = input.strip_prefix("\"\"\"").ok_or(NOT_FOUND)?;
+
+//     let mut content = String::new();
+//     let mut errors = Vec::new();
+//     let mut current = rest;
+//     let start_offset = input.get(offset);
+
+//     // 2. 遍历直到找到结束 """ 或输入结束
+//     loop {
+//         // 检测结束标记 """
+//         if let Some(new_rest) = current.strip_prefix("\"\"\"") {
+//             current = new_rest;
+//             break;
+//         }
+
+//         // 处理转义字符（可选，根据需求）
+//         if let Some('\\') = current.chars().next() {
+//             let (r, escaped_char) = parse_escape(current)?;
+//             content.push(escaped_char);
+//             current = r;
+//             continue;
+//         }
+
+//         // 消费普通字符
+//         let next_special = current.find(|c| c == '\\' || c == '"');
+//         let (text_part, remaining) = match next_special {
+//             Some(pos) => current.split_at(pos),
+//             None => current.split_at(current.len()),
+//         };
+
+//         content.push_str(text_part.to_str(current.get(str)));
+//         current = remaining;
+
+//         // 输入耗尽但未找到结束符
+//         if current.is_empty() {
+//             errors.push(input.get(str).get(start_offset..input.len()));
+//             break;
+//         }
+//     }
+
+//     // 3. 生成Token和诊断信息
+//     let (rest, range) = input.split_until(current);
+//     let token = Token::new(TokenKind::StringLiteral, range);
+//     let diag = if errors.is_empty() {
+//         Diagnostic::Valid
+//     } else {
+//         Diagnostic::InvalidStringEscapes(errors.into_boxed_slice())
+//     };
+//     Ok((rest, (token, diag)))
+// }
 
 fn number_literal(input: Input<'_>) -> TokenizationResult<'_, (Token, Diagnostic)> {
     // skip sign
@@ -211,6 +270,86 @@ fn whitespace(input: Input<'_>) -> TokenizationResult<'_> {
     Ok(input.split_at(ws_chars))
 }
 
+fn find_prev_char(original_str: &str, current_offset: usize) -> Option<char> {
+    // let current_offset = original_str.len() - rest.len();
+    let first_c = original_str.get(..current_offset);
+    match first_c {
+        Some(c) => {
+            if !c.is_empty() {
+                return c.chars().last();
+            }
+            // 2. 反向计算前导空白字节长度
+            let ws_len = c
+                .chars()
+                .rev()
+                .take_while(|c| c.is_whitespace() && *c != '\n')
+                .map(|c| c.len_utf8())
+                .sum();
+
+            // 3. 安全切割空白部分
+            let ws_start = current_offset.saturating_sub(ws_len);
+            let before_nl = original_str.get(..ws_start).unwrap_or("");
+
+            // 4. 获取最后一个非空白字符
+            return before_nl.chars().last();
+        }
+        None => return None,
+    }
+}
+
+fn linebreak(input: Input<'_>) -> TokenizationResult<'_> {
+    // dbg!("--->", input.as_str_slice());
+
+    if let Some((rest, nl_slice)) = input.strip_prefix("\n") {
+        // dbg!(nl_slice);
+        let original_str = input.as_original_str();
+
+        // 1. 计算换行符的字节位置
+        let current_offset = original_str.len().saturating_sub(rest.len() + 1);
+
+        match find_prev_char(original_str, current_offset) {
+            Some(c) => {
+                // dbg!(c);
+                if matches!(c, '{' | '(' | '[' | ',' | '>' | '=' | ';' | '\n' | '\\') {
+                    // skip ; and \n because there's already a linebreak parsed.
+                    // > is for ->
+                    // dbg!("=== skip ");
+                    return Err(NOT_FOUND);
+                }
+            }
+            // 读取前面字符失败，跳过
+            None => return Err(NOT_FOUND),
+        }
+        // dbg!("---> LineBreak ");
+
+        Ok((rest, nl_slice))
+    } else if let Some((rest, matched)) = input.strip_prefix(";") {
+        Ok((rest, matched))
+    } else {
+        Err(NOT_FOUND)
+    }
+}
+// 新增续行符解析函数
+fn line_continuation(input: Input<'_>) -> TokenizationResult<'_> {
+    if let Some((rest, matched)) = input.strip_prefix("\\\n") {
+        // println!("rest={},matched=", rest, matched);
+        // // dbg!(rest, matched);
+        Ok((rest, matched))
+    } else {
+        Err(NOT_FOUND)
+    }
+}
+// 新增行继续符识别逻辑
+// fn line_continuation(input: Input<'_>) -> TokenizationResult<'_> {
+//     if let Some((rest, _)) = input.strip_prefix("\\") {
+//         // 消费后续所有空白（包括换行符）
+//         let ws = rest.chars().take_while(char::is_ascii_digit).count();
+//         let (rest, _) = rest.split_at(ws);
+//         Ok((rest, input.split_at(1).1))
+//     } else {
+//         Err(NOT_FOUND)
+//     }
+// }
 fn comment(input: Input<'_>) -> TokenizationResult<'_> {
     if input.starts_with('#') {
         let len = input

src/tokens.rs

@@ -105,6 +105,8 @@ pub enum TokenKind {
     BooleanLiteral,
     Symbol,
     Whitespace,
+    LineBreak, //add newline
+    // LineContinuation, //add linecontinue
     Comment,
 }