Reduce tokenizer/parser overhead

Cargo.toml

@@ -97,6 +97,9 @@ required-features = ["derive"]
 [[example]]
 name = "sqlite_history"
 required-features = ["with-sqlite-history"]
+[[example]]
+name = "continuation_prompt"
+required-features = ["custom-bindings", "derive"]
 
 [package.metadata.docs.rs]
 features = [

examples/continuation_prompt.rs

@@ -0,0 +1,73 @@
+use std::borrow::Cow;
+
+use rustyline::highlight::Highlighter;
+use rustyline::validate::{ValidationContext, ValidationResult, Validator};
+use rustyline::{Cmd, Editor, EventHandler, Helper, KeyCode, KeyEvent, Modifiers, Result};
+use rustyline::{Completer, Hinter};
+
+#[derive(Completer, Hinter)]
+struct InputValidator {
+    bracket_level: i32,
+    /// re-render only when input just changed
+    /// not render after cursor moving
+    need_render: bool,
+}
+
+impl Helper for InputValidator {
+    fn update_after_edit(&mut self, line: &str, _pos: usize, _forced_refresh: bool) {
+        self.bracket_level = line.chars().fold(0, |level, c| {
+            if c == '(' {
+                level + 1
+            } else if c == ')' {
+                level - 1
+            } else {
+                level
+            }
+        });
+        self.need_render = true;
+    }
+}
+
+impl Validator for InputValidator {
+    fn validate(&mut self, _ctx: &mut ValidationContext) -> Result<ValidationResult> {
+        if self.bracket_level > 0 {
+            Ok(ValidationResult::Incomplete)
+            // Ok(ValidationResult::Incomplete(2))
+        } else if self.bracket_level < 0 {
+            Ok(ValidationResult::Invalid(Some(format!(
+                " - excess {} close bracket",
+                -self.bracket_level
+            ))))
+        } else {
+            Ok(ValidationResult::Valid(None))
+        }
+    }
+}
+
+impl Highlighter for InputValidator {
+    fn highlight_char(&mut self, _line: &str, _pos: usize, _forced: bool) -> bool {
+        self.need_render
+    }
+    fn highlight<'l>(&mut self, line: &'l str, pos: usize) -> Cow<'l, str> {
+        self.need_render = false;
+        Cow::Borrowed(&line)
+    }
+}
+
+fn main() -> Result<()> {
+    let h = InputValidator {
+        bracket_level: 0,
+        need_render: true,
+    };
+    let mut rl = Editor::new()?;
+    rl.set_helper(Some(h));
+    rl.bind_sequence(
+        KeyEvent(KeyCode::Char('s'), Modifiers::CTRL),
+        EventHandler::Simple(Cmd::Newline),
+    );
+
+    let input = rl.readline(">> ")?;
+    println!("Input: {input}");
+
+    Ok(())
+}

examples/custom_key_bindings.rs

@@ -14,7 +14,7 @@ struct MyHelper(#[rustyline(Hinter)] HistoryHinter);
 
 impl Highlighter for MyHelper {
     fn highlight_prompt<'b, 's: 'b, 'p: 'b>(
-        &'s self,
+        &'s mut self,
         prompt: &'p str,
         default: bool,
     ) -> Cow<'b, str> {
@@ -25,7 +25,7 @@ impl Highlighter for MyHelper {
         }
     }
 
-    fn highlight_hint<'h>(&self, hint: &'h str) -> Cow<'h, str> {
+    fn highlight_hint<'h>(&mut self, hint: &'h str) -> Cow<'h, str> {
         Owned(format!("\x1b[1m{hint}\x1b[m"))
     }
 }

examples/diy_hints.rs

@@ -52,7 +52,7 @@ impl CommandHint {
 impl Hinter for DIYHinter {
     type Hint = CommandHint;
 
-    fn hint(&self, line: &str, pos: usize, _ctx: &Context<'_>) -> Option<CommandHint> {
+    fn hint(&mut self, line: &str, pos: usize, _ctx: &Context<'_>) -> Option<CommandHint> {
         if line.is_empty() || pos < line.len() {
             return None;
         }

examples/example.rs

@@ -22,7 +22,7 @@ struct MyHelper {
 
 impl Highlighter for MyHelper {
     fn highlight_prompt<'b, 's: 'b, 'p: 'b>(
-        &'s self,
+        &'s mut self,
         prompt: &'p str,
         default: bool,
     ) -> Cow<'b, str> {
@@ -33,25 +33,25 @@ impl Highlighter for MyHelper {
         }
     }
 
-    fn highlight_hint<'h>(&self, hint: &'h str) -> Cow<'h, str> {
+    fn highlight_hint<'h>(&mut self, hint: &'h str) -> Cow<'h, str> {
         Owned("\x1b[1m".to_owned() + hint + "\x1b[m")
     }
 
     #[cfg(any(not(feature = "split-highlight"), feature = "ansi-str"))]
-    fn highlight<'l>(&self, line: &'l str, pos: usize) -> Cow<'l, str> {
+    fn highlight<'l>(&mut self, line: &'l str, pos: usize) -> Cow<'l, str> {
         self.highlighter.highlight(line, pos)
     }
 
     #[cfg(all(feature = "split-highlight", not(feature = "ansi-str")))]
     fn highlight_line<'l>(
-        &self,
+        &mut self,
         line: &'l str,
         pos: usize,
     ) -> impl Iterator<Item = impl 'l + rustyline::highlight::StyledBlock> {
         self.highlighter.highlight_line(line, pos)
     }
 
-    fn highlight_char(&self, line: &str, pos: usize, forced: bool) -> bool {
+    fn highlight_char(&mut self, line: &str, pos: usize, forced: bool) -> bool {
         self.highlighter.highlight_char(line, pos, forced)
     }
 }

examples/input_validation.rs

@@ -6,7 +6,7 @@ use rustyline::{Editor, Result};
 struct InputValidator {}
 
 impl Validator for InputValidator {
-    fn validate(&self, ctx: &mut ValidationContext) -> Result<ValidationResult> {
+    fn validate(&mut self, ctx: &mut ValidationContext) -> Result<ValidationResult> {
         use ValidationResult::{Incomplete, Invalid, Valid};
         let input = ctx.input();
         let result = if !input.starts_with("SELECT") {

examples/read_password.rs

@@ -10,7 +10,7 @@ struct MaskingHighlighter {
 
 impl Highlighter for MaskingHighlighter {
     #[cfg(any(not(feature = "split-highlight"), feature = "ansi-str"))]
-    fn highlight<'l>(&self, line: &'l str, _pos: usize) -> std::borrow::Cow<'l, str> {
+    fn highlight<'l>(&mut self, line: &'l str, _pos: usize) -> std::borrow::Cow<'l, str> {
         use unicode_width::UnicodeWidthStr;
         if self.masking {
             std::borrow::Cow::Owned(" ".repeat(line.width()))
@@ -21,7 +21,7 @@ impl Highlighter for MaskingHighlighter {
 
     #[cfg(all(feature = "split-highlight", not(feature = "ansi-str")))]
     fn highlight_line<'l>(
-        &self,
+        &mut self,
         line: &'l str,
         _pos: usize,
     ) -> impl Iterator<Item = impl 'l + rustyline::highlight::StyledBlock> {
@@ -37,7 +37,7 @@ impl Highlighter for MaskingHighlighter {
         }
     }
 
-    fn highlight_char(&self, _line: &str, _pos: usize, _forced: bool) -> bool {
+    fn highlight_char(&mut self, _line: &str, _pos: usize, _forced: bool) -> bool {
         self.masking
     }
 }

rustyline-derive/src/lib.rs

@@ -51,16 +51,16 @@ pub fn completer_macro_derive(input: TokenStream) -> TokenStream {
                 type Candidate = <#field_type as ::rustyline::completion::Completer>::Candidate;
 
                 fn complete(
-                    &self,
+                    &mut self,
                     line: &str,
                     pos: usize,
                     ctx: &::rustyline::Context<'_>,
                 ) -> ::rustyline::Result<(usize, ::std::vec::Vec<Self::Candidate>)> {
-                    ::rustyline::completion::Completer::complete(&self.#field_name_or_index, line, pos, ctx)
+                    ::rustyline::completion::Completer::complete(&mut self.#field_name_or_index, line, pos, ctx)
                 }
 
-                fn update(&self, line: &mut ::rustyline::line_buffer::LineBuffer, start: usize, elected: &str, cl: &mut ::rustyline::Changeset) {
-                    ::rustyline::completion::Completer::update(&self.#field_name_or_index, line, start, elected, cl)
+                fn update(&mut self, line: &mut ::rustyline::line_buffer::LineBuffer, start: usize, elected: &str, cl: &mut ::rustyline::Changeset) {
+                    ::rustyline::completion::Completer::update(&mut self.#field_name_or_index, line, start, elected, cl)
                 }
             }
         }
@@ -103,41 +103,41 @@ pub fn highlighter_macro_derive(input: TokenStream) -> TokenStream {
             #[automatically_derived]
             impl #impl_generics ::rustyline::highlight::Highlighter for #name #ty_generics #where_clause {
                 #[cfg(any(not(feature = "split-highlight"), feature = "ansi-str"))]
-                fn highlight<'l>(&self, line: &'l str, pos: usize) -> ::std::borrow::Cow<'l, str> {
-                    ::rustyline::highlight::Highlighter::highlight(&self.#field_name_or_index, line, pos)
+                fn highlight<'l>(&mut self, line: &'l str, pos: usize) -> ::std::borrow::Cow<'l, str> {
+                    ::rustyline::highlight::Highlighter::highlight(&mut self.#field_name_or_index, line, pos)
                 }
 
                 #[cfg(all(feature = "split-highlight", not(feature = "ansi-str")))]
                 fn highlight_line<'l>(
-                    &self,
+                    &mut self,
                     line: &'l str,
                     pos: usize,
                 ) -> impl Iterator<Item = impl 'l + ::rustyline::highlight::StyledBlock> {
-                    ::rustyline::highlight::Highlighter::highlight_line(&self.#field_name_or_index, line, pos)
+                    ::rustyline::highlight::Highlighter::highlight_line(&mut self.#field_name_or_index, line, pos)
                 }
 
                 fn highlight_prompt<'b, 's: 'b, 'p: 'b>(
-                    &'s self,
+                    &'s mut self,
                     prompt: &'p str,
                     default: bool,
                 ) -> ::std::borrow::Cow<'b, str> {
-                    ::rustyline::highlight::Highlighter::highlight_prompt(&self.#field_name_or_index, prompt, default)
+                    ::rustyline::highlight::Highlighter::highlight_prompt(&mut self.#field_name_or_index, prompt, default)
                 }
 
-                fn highlight_hint<'h>(&self, hint: &'h str) -> ::std::borrow::Cow<'h, str> {
-                    ::rustyline::highlight::Highlighter::highlight_hint(&self.#field_name_or_index, hint)
+                fn highlight_hint<'h>(&mut self, hint: &'h str) -> ::std::borrow::Cow<'h, str> {
+                    ::rustyline::highlight::Highlighter::highlight_hint(&mut self.#field_name_or_index, hint)
                 }
 
                 fn highlight_candidate<'c>(
-                    &self,
+                    &mut self,
                     candidate: &'c str,
                     completion: ::rustyline::config::CompletionType,
                 ) -> ::std::borrow::Cow<'c, str> {
-                    ::rustyline::highlight::Highlighter::highlight_candidate(&self.#field_name_or_index, candidate, completion)
+                    ::rustyline::highlight::Highlighter::highlight_candidate(&mut self.#field_name_or_index, candidate, completion)
                 }
 
-                fn highlight_char(&self, line: &str, pos: usize, forced: bool) -> bool {
-                    ::rustyline::highlight::Highlighter::highlight_char(&self.#field_name_or_index, line, pos, forced)
+                fn highlight_char(&mut self, line: &str, pos: usize, forced: bool) -> bool {
+                    ::rustyline::highlight::Highlighter::highlight_char(&mut self.#field_name_or_index, line, pos, forced)
                 }
             }
         }
@@ -166,8 +166,8 @@ pub fn hinter_macro_derive(input: TokenStream) -> TokenStream {
             impl #impl_generics ::rustyline::hint::Hinter for #name #ty_generics #where_clause {
                 type Hint = <#field_type as ::rustyline::hint::Hinter>::Hint;
 
-                fn hint(&self, line: &str, pos: usize, ctx: &::rustyline::Context<'_>) -> ::std::option::Option<Self::Hint> {
-                    ::rustyline::hint::Hinter::hint(&self.#field_name_or_index, line, pos, ctx)
+                fn hint(&mut self, line: &str, pos: usize, ctx: &::rustyline::Context<'_>) -> ::std::option::Option<Self::Hint> {
+                    ::rustyline::hint::Hinter::hint(&mut self.#field_name_or_index, line, pos, ctx)
                 }
             }
         }
@@ -195,14 +195,14 @@ pub fn validator_macro_derive(input: TokenStream) -> TokenStream {
             #[automatically_derived]
             impl #impl_generics ::rustyline::validate::Validator for #name #ty_generics #where_clause {
                 fn validate(
-                    &self,
+                    &mut self,
                     ctx: &mut ::rustyline::validate::ValidationContext,
                 ) -> ::rustyline::Result<::rustyline::validate::ValidationResult> {
-                    ::rustyline::validate::Validator::validate(&self.#field_name_or_index, ctx)
+                    ::rustyline::validate::Validator::validate(&mut self.#field_name_or_index, ctx)
                 }
 
-                fn validate_while_typing(&self) -> bool {
-                    ::rustyline::validate::Validator::validate_while_typing(&self.#field_name_or_index)
+                fn validate_while_typing(&mut self) -> bool {
+                    ::rustyline::validate::Validator::validate_while_typing(&mut self.#field_name_or_index)
                 }
             }
         }

src/completion.rs

@@ -90,7 +90,7 @@ pub trait Completer {
     ///
     /// ("ls /usr/loc", 11) => Ok((3, vec!["/usr/local/"]))
     fn complete(
-        &self, // FIXME should be `&mut self`
+        &mut self, // FIXME should be `&mut self`
         line: &str,
         pos: usize,
         ctx: &Context<'_>,
@@ -99,7 +99,7 @@ pub trait Completer {
         Ok((0, Vec::with_capacity(0)))
     }
     /// Updates the edited `line` with the `elected` candidate.
-    fn update(&self, line: &mut LineBuffer, start: usize, elected: &str, cl: &mut Changeset) {
+    fn update(&mut self, line: &mut LineBuffer, start: usize, elected: &str, cl: &mut Changeset) {
         let end = line.pos();
         line.replace(start..end, elected, cl);
     }
@@ -108,48 +108,48 @@ pub trait Completer {
 impl Completer for () {
     type Candidate = String;
 
-    fn update(&self, _line: &mut LineBuffer, _start: usize, _elected: &str, _cl: &mut Changeset) {
+    fn update(&mut self, _line: &mut LineBuffer, _start: usize, _elected: &str, _cl: &mut Changeset) {
         unreachable!();
     }
 }
 
-impl<'c, C: ?Sized + Completer> Completer for &'c C {
+impl<'c, C: ?Sized + Completer> Completer for &'c mut C {
     type Candidate = C::Candidate;
 
     fn complete(
-        &self,
+        &mut self,
         line: &str,
         pos: usize,
         ctx: &Context<'_>,
     ) -> Result<(usize, Vec<Self::Candidate>)> {
         (**self).complete(line, pos, ctx)
     }
 
-    fn update(&self, line: &mut LineBuffer, start: usize, elected: &str, cl: &mut Changeset) {
+    fn update(&mut self, line: &mut LineBuffer, start: usize, elected: &str, cl: &mut Changeset) {
         (**self).update(line, start, elected, cl);
     }
 }
-macro_rules! box_completer {
-    ($($id: ident)*) => {
-        $(
-            impl<C: ?Sized + Completer> Completer for $id<C> {
-                type Candidate = C::Candidate;
-
-                fn complete(&self, line: &str, pos: usize, ctx: &Context<'_>) -> Result<(usize, Vec<Self::Candidate>)> {
-                    (**self).complete(line, pos, ctx)
-                }
-                fn update(&self, line: &mut LineBuffer, start: usize, elected: &str, cl: &mut Changeset) {
-                    (**self).update(line, start, elected, cl)
-                }
-            }
-        )*
-    }
-}
+// macro_rules! box_completer {
+//     ($($id: ident)*) => {
+//         $(
+//             impl<C: ?Sized + Completer> Completer for $id<C> {
+//                 type Candidate = C::Candidate;
+
+//                 fn complete(&self, line: &str, pos: usize, ctx: &Context<'_>) -> Result<(usize, Vec<Self::Candidate>)> {
+//                     (**self).complete(line, pos, ctx)
+//                 }
+//                 fn update(&self, line: &mut LineBuffer, start: usize, elected: &str, cl: &mut Changeset) {
+//                     (**self).update(line, start, elected, cl)
+//                 }
+//             }
+//         )*
+//     }
+// }
 
 use crate::undo::Changeset;
 use std::rc::Rc;
-use std::sync::Arc;
-box_completer! { Box Rc Arc }
+// use std::sync::Arc;
+// box_completer! { Box Rc Arc }
 
 /// A `Completer` for file and folder names.
 pub struct FilenameCompleter {
@@ -257,7 +257,7 @@ impl Default for FilenameCompleter {
 impl Completer for FilenameCompleter {
     type Candidate = Pair;
 
-    fn complete(&self, line: &str, pos: usize, _ctx: &Context<'_>) -> Result<(usize, Vec<Pair>)> {
+    fn complete(&mut self, line: &str, pos: usize, _ctx: &Context<'_>) -> Result<(usize, Vec<Pair>)> {
         self.complete_path(line, pos)
     }
 }