Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions src/core/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,30 @@ Three-tier filter lookup (first match wins):
2. `~/.config/rtk/filters.toml` (user-global)
3. Built-in filters concatenated by `build.rs` at compile time

## Source File Comment Stripping

`src/core/filter.rs` is a separate engine from the TOML DSL: it filters *source
files* (used by `rtk read`) rather than command output. At `-l minimal` it
strips comments using the per-language delimiters in
`Language::comment_patterns()`.

Python does not use that walk. It has no block comments — `"""` opens a
*string*, which may be a docstring or an ordinary value — so it gets a
string-aware path that removes `#` comments and leaves string contents alone.
Matching `"""` as a block delimiter misread both of these:

```python
QUERY = """ # contains """ without starting with it
SELECT 1
"""

"""Module doc.""" # opens and closes on one line
```

Docstrings are kept at `minimal`. `aggressive` has no string awareness: it
keeps a line inside a string when that line looks like an import or a
signature.

## Tracking Database Schema

```sql
Expand Down
303 changes: 303 additions & 0 deletions src/core/filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,132 @@ pub struct MinimalFilter;

static MULTIPLE_BLANK_LINES: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\n{3,}").unwrap());

/// Advances triple-quoted string state across one line, returning the delimiter
/// still open at end of line. The two quote kinds are tracked separately so a
/// `'''` inside a `"""` string is text rather than a close. Outside a string, a
/// one-line string literal is skipped whole and a `#` ends the scan, so a `"""`
/// written inside `'"""'` or after a trailing comment opens nothing. A backslash
/// escapes the next byte in every kind of string, raw ones included.
fn advance_triple_quote(line: &str, open: Option<&'static str>) -> Option<&'static str> {
let bytes = line.as_bytes();
let mut state = open;
let mut i = 0;

while i < bytes.len() {
let rest = &bytes[i..];
if let Some(current) = state {
if rest[0] == b'\\' {
i += 2;
} else if rest.starts_with(current.as_bytes()) {
state = None;
i += 3;
} else {
i += 1;
}
continue;
}

match rest[0] {
b'#' => break,
b'"' if rest.starts_with(b"\"\"\"") => {
state = Some("\"\"\"");
i += 3;
}
b'\'' if rest.starts_with(b"'''") => {
state = Some("'''");
i += 3;
}
quote @ (b'"' | b'\'') => {
let interpolated = has_interpolation_prefix(&bytes[..i]);
let mut depth = 0usize;
i += 1;
while i < bytes.len() {
match bytes[i] {
b'\\' => i += 1,
b'{' if interpolated => {
if depth == 0 && bytes.get(i + 1) == Some(&b'{') {
i += 1;
} else {
depth += 1;
}
}
b'}' if interpolated && depth > 0 => depth -= 1,
b if b == quote && depth == 0 => break,
_ => {}
}
i += 1;
}
i += 1;
}
_ => i += 1,
}
}
state
}

/// True when the identifier ending at `before` is an f-string or t-string prefix.
/// A replacement field in those may reuse the outer quote (PEP 701), so the
/// quote only ends the string outside `{...}`.
fn has_interpolation_prefix(before: &[u8]) -> bool {
let start = before
.iter()
.rposition(|b| !(b.is_ascii_alphanumeric() || *b == b'_'))
.map_or(0, |p| p + 1);
matches!(
before[start..].to_ascii_lowercase().as_slice(),
b"f" | b"fr" | b"rf" | b"t" | b"tr" | b"rt"
)
}

/// Python has no block comments. `"""` opens a *string*, which may be a
/// docstring or an ordinary value, so it cannot be matched with the
/// line-oriented block-comment rules the other languages use: a line such as
/// `QUERY = """` both contains and "closes" the delimiter, and a single-line
/// docstring toggles the state once and never back.
///
/// Minimal keeps docstrings, so the only thing to remove here is `#` comments,
/// and the only state needed is whether we are inside a triple-quoted string.
fn filter_python_minimal(content: &str) -> String {
let mut result = String::with_capacity(content.len());
let mut open_string: Option<&'static str> = None;

for line in content.lines() {
let trimmed = line.trim();

// Inside a string every line is literal text, including one that starts
// with `#`.
if open_string.is_some() {
result.push_str(line);
result.push('\n');
open_string = advance_triple_quote(line, open_string);
continue;
}

// A comment's contents are not code, so any delimiter in it is not real.
if trimmed.starts_with('#') {
continue;
}

if trimmed.is_empty() {
result.push('\n');
continue;
}

result.push_str(line);
result.push('\n');
open_string = advance_triple_quote(line, None);
}

let result = MULTIPLE_BLANK_LINES.replace_all(&result, "\n\n");
result.trim().to_string()
}

impl FilterStrategy for MinimalFilter {
fn filter(&self, content: &str, lang: &Language) -> String {
if *lang == Language::Python {
return filter_python_minimal(content);
}

let patterns = lang.comment_patterns();
let mut result = String::with_capacity(content.len());
let mut in_block_comment = false;
Expand Down Expand Up @@ -461,6 +585,185 @@ mod tests {
);
}

// --- Python triple-quoted strings in minimal mode ---

#[test]
fn test_minimal_python_keeps_multiline_string_assignment() {
let code = r#"QUERY = """
SELECT id
FROM users
"""
import os
"#;
let result = MinimalFilter.filter(code, &Language::Python);
assert!(
result.contains(r#"QUERY = """#),
"the line opening the string was dropped, leaving its body as loose text:\n{}",
result
);
assert!(
result.contains("SELECT id"),
"string body lost:\n{}",
result
);
assert!(
result.contains("import os"),
"code after string lost:\n{}",
result
);
}

#[test]
fn test_minimal_python_strips_comments_after_oneline_docstring() {
let code = r#""""Module doc."""
import os
# strip me
def go():
return 1
"#;
let result = MinimalFilter.filter(code, &Language::Python);
assert!(
!result.contains("# strip me"),
"a one-line docstring left the filter stuck in docstring mode, so no \
later comment was stripped:\n{}",
result
);
assert!(
result.contains(r#""""Module doc.""""#),
"docstring lost:\n{}",
result
);
assert!(result.contains("def go():"), "code lost:\n{}", result);
}

#[test]
fn test_minimal_python_keeps_hash_inside_docstring() {
let code = r#""""
# not a comment, it is prose
"""
x = 1
"#;
let result = MinimalFilter.filter(code, &Language::Python);
assert!(
result.contains("# not a comment, it is prose"),
"text inside a docstring was stripped as a comment:\n{}",
result
);
assert!(result.contains("x = 1"));
}

#[test]
fn test_minimal_python_single_quoted_docstring_does_not_close_double() {
let code = r#""""Doc mentioning ''' inline."""
# strip me
x = 1
"#;
let result = MinimalFilter.filter(code, &Language::Python);
assert!(
!result.contains("# strip me"),
"a ''' inside a \"\"\" string confused the tracker:\n{}",
result
);
assert!(result.contains("x = 1"));
}

#[test]
fn test_minimal_python_still_strips_plain_comments() {
let code = r#"# leading comment
import os
x = 1 # trailing comment kept, matching prior behavior
"#;
let result = MinimalFilter.filter(code, &Language::Python);
assert!(!result.contains("# leading comment"));
assert!(result.contains("import os"));
assert!(result.contains("x = 1"));
}

#[test]
fn test_minimal_python_triple_quote_in_one_line_string_opens_nothing() {
let code = r#"marker = '"""'
# strip me
def f():
"""
# prose inside the docstring
"""
return 1
"#;
let result = MinimalFilter.filter(code, &Language::Python);
assert!(result.contains(r#"marker = '"""'"#));
assert!(
!result.contains("# strip me"),
"a \"\"\" inside a one-line string opened a string:\n{}",
result
);
assert!(
result.contains("# prose inside the docstring"),
"string state inverted, so docstring text was stripped:\n{}",
result
);
}

#[test]
fn test_minimal_python_triple_quote_in_trailing_comment_opens_nothing() {
let code = r#"x = 1 # see """
# strip me
SCRIPT = """
# shell comment inside the string
"""
"#;
let result = MinimalFilter.filter(code, &Language::Python);
assert!(
!result.contains("# strip me"),
"a \"\"\" inside a trailing comment opened a string:\n{}",
result
);
assert!(
result.contains("# shell comment inside the string"),
"string state inverted, so string text was stripped:\n{}",
result
);
}

#[test]
fn test_minimal_python_fstring_field_reusing_quote_opens_nothing() {
let code = r#"v = f"{"""x"""}"
# strip me
w = f"{'"'}" + """
# inside the string
"""
if"{" == v:
pass
# strip me too
"#;
let result = MinimalFilter.filter(code, &Language::Python);
assert!(
!result.contains("# strip me"),
"a quote inside an f-string field ended the string:\n{}",
result
);
assert!(
result.contains("# inside the string"),
"string state inverted, so string text was stripped:\n{}",
result
);
}

#[test]
fn test_minimal_python_escaped_quote_does_not_close_string() {
let code = r#"s = r"""a\"""
# still inside the string
"""
# strip me
"#;
let result = MinimalFilter.filter(code, &Language::Python);
assert!(
result.contains("# still inside the string"),
"an escaped quote closed the string early:\n{}",
result
);
assert!(!result.contains("# strip me"));
}

#[test]
fn test_minimal_filter_removes_comments() {
let code = r#"
Expand Down
Loading