[ruff] Detect syntax errors in individual notebook cells

crates/ruff_db/src/parsed.rs

@@ -8,7 +8,8 @@ use ruff_python_ast::{
     StringLiteral,
 };
 use ruff_python_parser::{
-    ParseError, ParseErrorType, ParseOptions, Parsed, parse_string_annotation, parse_unchecked,
+    ParseError, ParseErrorType, ParseOptions, Parsed, parse_cells_unchecked,
+    parse_string_annotation, parse_unchecked,
 };
 
 use crate::Db;
@@ -49,9 +50,17 @@ pub fn parsed_module_impl(db: &dyn Db, file: File) -> Parsed<ModModule> {
 
     let target_version = db.python_version();
     let options = ParseOptions::from(ty).with_target_version(target_version);
-    parse_unchecked(&source, options)
-        .try_into_module()
-        .expect("PySourceType always parses into a module")
+
+    // Notebooks parse each cell as an independent module so a syntax error confined to one cell is
+    // surfaced instead of being masked by a later cell's content. Regular files take the existing
+    // single-parse path.
+    if let Some(notebook) = source.as_notebook() {
+        parse_cells_unchecked(&source, notebook.cell_offsets().content_ranges(), &options)
+    } else {
+        parse_unchecked(&source, options)
+            .try_into_module()
+            .expect("PySourceType always parses into a module")
+    }
 }
 
 pub fn parsed_string_annotation(

crates/ruff_dev/src/print_tokens.rs

@@ -4,9 +4,9 @@ use std::path::PathBuf;
 
 use anyhow::Result;
 
+use ruff_linter::linter::parse_unchecked_source;
 use ruff_linter::source_kind::SourceKind;
-use ruff_python_ast::{PySourceType, SourceType};
-use ruff_python_parser::parse_unchecked_source;
+use ruff_python_ast::{PySourceType, PythonVersion, SourceType};
 
 #[derive(clap::Args)]
 pub(crate) struct Args {
@@ -24,7 +24,7 @@ pub(crate) fn main(args: &Args) -> Result<()> {
                 args.file.display()
             )
         })?;
-    let parsed = parse_unchecked_source(source_kind.source_code(), source_type);
+    let parsed = parse_unchecked_source(&source_kind, source_type, PythonVersion::default());
     for token in parsed.tokens() {
         println!("{token:#?}");
     }

crates/ruff_linter/resources/mdtest/notebook/cell-boundaries.md

@@ -0,0 +1,122 @@
+# Notebook cell parsing
+
+Ruff parses every notebook cell as its own module while retaining a single combined module for
+linting.
+
+## Syntax errors stop at cell boundaries
+
+A decorator at the end of a cell would apply to a definition in the next cell if the sources were
+concatenated. Parsing the cells independently must instead report the syntax error in the decorator's
+cell.
+
+`syntax-error.ipynb`:
+
+```ipynb
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": ["# snapshot\n", "@deco"]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": ["def f(): pass"]
+    }
+  ],
+  "metadata": {},
+  "nbformat": 4,
+  "nbformat_minor": 4
+}
+```
+
+```snapshot
+error[invalid-syntax]: Expected class, function definition or async function definition after decorator
+ --> src/syntax-error.ipynb:cell 1:2:6
+  |
+2 | @deco
+  |      ^
+  |
+```
+
+## Notebooks without Python cells
+
+A notebook containing no Python code cells still parses successfully.
+
+`no-code-cells.ipynb`:
+
+```ipynb
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": ["# Nothing to check"]
+    }
+  ],
+  "metadata": {},
+  "nbformat": 4,
+  "nbformat_minor": 4
+}
+```
+
+## Cell boundary tokens
+
+Per-cell parsing inserts tokens at each boundary before merging the cells into one module. This
+notebook exercises a trailing `Dedent`, a definition referenced across cells, and a range suppression
+whose directive and import are in separate cells. None of the selected rules should report a
+diagnostic.
+
+```toml
+[lint]
+select = [
+  "E301",
+  "E302",
+  "E303",
+  "E305",
+  "E306",
+  "F401",
+  "F821",
+  "W291",
+  "W293",
+  "W391",
+]
+```
+
+`boundary-tokens.ipynb`:
+
+```ipynb
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": ["def compute():\n", "    return 1"]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": ["# ruff: disable[F401]"]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": ["import json\n", "print(compute())"]
+    }
+  ],
+  "metadata": {},
+  "nbformat": 4,
+  "nbformat_minor": 4
+}
+```

crates/ruff_linter/src/linter.rs

@@ -767,9 +767,11 @@ impl ParseSource {
     }
 }
 
-/// Like [`ruff_python_parser::parse_unchecked_source`] but with an additional [`PythonVersion`]
-/// argument.
-fn parse_unchecked_source(
+/// Like [`ruff_python_parser::parse_unchecked_source`], but with an explicit [`PythonVersion`] and
+/// per-cell notebook parsing.
+///
+/// Per-cell modules are merged so definitions remain visible across cells.
+pub fn parse_unchecked_source(
     source_kind: &SourceKind,
     source_type: PySourceType,
     target_version: PythonVersion,
@@ -778,9 +780,16 @@ fn parse_unchecked_source(
     // SAFETY: Safe because `PySourceType` always parses to a `ModModule`. See
     // `ruff_python_parser::parse_unchecked_source`. We use `parse_unchecked` (and thus
     // have to unwrap) in order to pass the `PythonVersion` via `ParseOptions`.
-    ruff_python_parser::parse_unchecked(source_kind.source_code(), options)
-        .try_into_module()
-        .expect("PySourceType always parses into a module")
+    match source_kind.as_ipy_notebook() {
+        Some(notebook) => ruff_python_parser::parse_cells_unchecked(
+            source_kind.source_code(),
+            notebook.cell_offsets().content_ranges(),
+            &options,
+        ),
+        None => ruff_python_parser::parse_unchecked(source_kind.source_code(), options)
+            .try_into_module()
+            .expect("PySourceType always parses into a module"),
+    }
 }
 
 #[cfg(test)]
@@ -791,14 +800,13 @@ mod tests {
     use ruff_python_ast::{PySourceType, PythonVersion};
     use ruff_python_codegen::Stylist;
     use ruff_python_index::Indexer;
-    use ruff_python_parser::ParseOptions;
     use ruff_python_trivia::textwrap::dedent;
     use test_case::test_case;
 
     use ruff_db::diagnostic::Diagnostic;
     use ruff_notebook::{Notebook, NotebookError};
 
-    use crate::linter::check_path;
+    use crate::linter::{check_path, parse_unchecked_source};
     use crate::registry::Rule;
     use crate::settings::LinterSettings;
     use crate::source_kind::SourceKind;
@@ -964,11 +972,9 @@ mod tests {
     ) -> Vec<Diagnostic> {
         let source_type = PySourceType::from(path);
         let target_version = settings.resolve_target_version(path);
-        let options =
-            ParseOptions::from(source_type).with_target_version(target_version.parser_version());
-        let parsed = ruff_python_parser::parse_unchecked(source_kind.source_code(), options)
-            .try_into_module()
-            .expect("PySourceType always parses into a module");
+        // Mirror the production parse path so notebooks are validated cell by cell.
+        let parsed =
+            parse_unchecked_source(source_kind, source_type, target_version.parser_version());
         let locator = Locator::new(source_kind.source_code());
         let stylist = Stylist::from_tokens(parsed.tokens(), locator.contents());
         let indexer = Indexer::from_tokens(parsed.tokens(), locator.contents());

crates/ruff_linter/src/test.rs

@@ -18,7 +18,7 @@ use ruff_notebook::NotebookError;
 use ruff_python_ast::PySourceType;
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
-use ruff_python_parser::{ParseError, ParseOptions};
+use ruff_python_parser::ParseError;
 use ruff_python_trivia::textwrap::dedent;
 use ruff_source_file::SourceFileBuilder;
 
@@ -231,11 +231,11 @@ pub fn test_contents<'a>(
 ) -> (Vec<Diagnostic>, Cow<'a, SourceKind>) {
     let source_type = PySourceType::from(path);
     let target_version = settings.resolve_target_version(path);
-    let options =
-        ParseOptions::from(source_type).with_target_version(target_version.parser_version());
-    let parsed = ruff_python_parser::parse_unchecked(source_kind.source_code(), options.clone())
-        .try_into_module()
-        .expect("PySourceType always parses into a module");
+    let parsed = crate::linter::parse_unchecked_source(
+        source_kind,
+        source_type,
+        target_version.parser_version(),
+    );
     let locator = Locator::new(source_kind.source_code());
     let stylist = Stylist::from_tokens(parsed.tokens(), locator.contents());
     let indexer = Indexer::from_tokens(parsed.tokens(), locator.contents());
@@ -299,10 +299,11 @@ pub fn test_contents<'a>(
 
             transformed = Cow::Owned(transformed.updated(fixed_contents, &source_map));
 
-            let parsed =
-                ruff_python_parser::parse_unchecked(transformed.source_code(), options.clone())
-                    .try_into_module()
-                    .expect("PySourceType always parses into a module");
+            let parsed = crate::linter::parse_unchecked_source(
+                &transformed,
+                source_type,
+                target_version.parser_version(),
+            );
             let locator = Locator::new(transformed.source_code());
             let stylist = Stylist::from_tokens(parsed.tokens(), locator.contents());
             let indexer = Indexer::from_tokens(parsed.tokens(), locator.contents());

crates/ruff_mdtest/src/lib.rs

@@ -68,8 +68,8 @@ fn run_test(
             }
 
             assert!(
-                matches!(embedded.lang, "py" | "pyi" | "python"),
-                "Supported file types are: py (or python), pyi, and ignore"
+                matches!(embedded.lang, "py" | "pyi" | "python" | "ipynb"),
+                "Supported file types are: py (or python), pyi, ipynb, and ignore"
             );
 
             let full_path = embedded.full_path(&project_root);
@@ -106,9 +106,14 @@ fn run_test(
         .filter_map(|test_file| {
             let mdtest_result = attempt_test(
                 |file| {
-                    let source_kind = SourceKind::Python {
-                        code: source_text(db, file).as_str().to_string(),
-                        is_stub: file.is_stub(db),
+                    let source = source_text(db, file);
+                    let source_kind = if let Some(notebook) = source.as_notebook() {
+                        SourceKind::ipy_notebook(notebook.clone())
+                    } else {
+                        SourceKind::Python {
+                            code: source.as_str().to_string(),
+                            is_stub: file.is_stub(db),
+                        }
                     };
                     let path = file
                         .path(db)

crates/ruff_python_ast/src/token/tokens.rs

@@ -225,6 +225,15 @@ impl Tokens {
     }
 }
 
+impl IntoIterator for Tokens {
+    type Item = Token;
+    type IntoIter = std::vec::IntoIter<Token>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.raw.into_iter()
+    }
+}
+
 impl<'a> IntoIterator for &'a Tokens {
     type Item = &'a Token;
     type IntoIter = std::slice::Iter<'a, Token>;