Red knot: Add file watching and cancellation (#11127)

Author: MichaReiser

Cargo.lock

@@ -30,7 +30,7 @@ console_error_panic_hook = { version = "0.1.7" }
 console_log = { version = "1.0.0" }
 countme = { version = "3.0.1" }
 criterion = { version = "0.5.1", default-features = false }
-crossbeam = { version = "0.8.4" }
+crossbeam-channel = { version = "0.5.12" }
 dashmap = { version = "5.5.3" }
 dirs = { version = "5.0.0" }
 drop_bomb = { version = "0.1.5" }

Cargo.toml

@@ -17,12 +17,18 @@ ruff_python_ast = { path = "../ruff_python_ast" }
 ruff_python_trivia = { path = "../ruff_python_trivia" }
 ruff_text_size = { path = "../ruff_text_size" }
 ruff_index = { path = "../ruff_index" }
+ruff_notebook = { path = "../ruff_notebook" }
 
 anyhow = { workspace = true }
+bitflags = { workspace = true }
+ctrlc = "3.4.4"
+crossbeam-channel = { workspace = true }
 dashmap = { workspace = true }
 hashbrown = { workspace = true }
 log = { workspace = true }
+notify = { workspace = true }
 parking_lot = { workspace = true }
+rayon = { workspace = true }
 rustc-hash = { workspace = true }
 smallvec = { workspace = true }
 smol_str = "0.2.1"

crates/red_knot/Cargo.toml

@@ -0,0 +1,65 @@
+use std::sync::{Arc, Condvar, Mutex};
+
+#[derive(Debug, Default)]
+pub struct CancellationSource {
+    signal: Arc<(Mutex<bool>, Condvar)>,
+}
+
+impl CancellationSource {
+    pub fn new() -> Self {
+        Self {
+            signal: Arc::new((Mutex::new(false), Condvar::default())),
+        }
+    }
+
+    pub fn cancel(&self) {
+        let (cancelled, condvar) = &*self.signal;
+
+        let mut cancelled = cancelled.lock().unwrap();
+
+        if *cancelled {
+            return;
+        }
+
+        *cancelled = true;
+        condvar.notify_all();
+    }
+
+    pub fn is_cancelled(&self) -> bool {
+        let (cancelled, _) = &*self.signal;
+
+        *cancelled.lock().unwrap()
+    }
+
+    pub fn token(&self) -> CancellationToken {
+        CancellationToken {
+            signal: self.signal.clone(),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct CancellationToken {
+    signal: Arc<(Mutex<bool>, Condvar)>,
+}
+
+impl CancellationToken {
+    /// Returns `true` if cancellation has been requested.
+    pub fn is_cancelled(&self) -> bool {
+        let (cancelled, _) = &*self.signal;
+
+        *cancelled.lock().unwrap()
+    }
+
+    pub fn wait(&self) {
+        let (bool, condvar) = &*self.signal;
+
+        let lock = condvar
+            .wait_while(bool.lock().unwrap(), |bool| !*bool)
+            .unwrap();
+
+        debug_assert!(*lock);
+
+        drop(lock);
+    }
+}

crates/red_knot/src/cancellation.rs

@@ -2,6 +2,7 @@ use std::path::Path;
 use std::sync::Arc;
 
 use crate::files::FileId;
+use crate::lint::{Diagnostics, LintSyntaxStorage};
 use crate::module::{Module, ModuleData, ModuleName, ModuleResolver, ModuleSearchPath};
 use crate::parse::{Parsed, ParsedStorage};
 use crate::source::{Source, SourceStorage};
@@ -16,6 +17,8 @@ pub trait SourceDb {
     fn source(&self, file_id: FileId) -> Source;
 
     fn parse(&self, file_id: FileId) -> Parsed;
+
+    fn lint_syntax(&self, file_id: FileId) -> Diagnostics;
 }
 
 pub trait SemanticDb: SourceDb {
@@ -38,6 +41,7 @@ pub trait Db: SemanticDb {}
 pub struct SourceJar {
     pub sources: SourceStorage,
     pub parsed: ParsedStorage,
+    pub lint_syntax: LintSyntaxStorage,
 }
 
 #[derive(Debug, Default)]
@@ -70,6 +74,7 @@ pub trait HasJar<T> {
 pub(crate) mod tests {
     use crate::db::{HasJar, SourceDb, SourceJar};
     use crate::files::{FileId, Files};
+    use crate::lint::{lint_syntax, Diagnostics};
     use crate::module::{
         add_module, path_to_module, resolve_module, set_module_search_paths, Module, ModuleData,
         ModuleName, ModuleSearchPath,
@@ -127,6 +132,10 @@ pub(crate) mod tests {
         fn parse(&self, file_id: FileId) -> Parsed {
             parse(self, file_id)
         }
+
+        fn lint_syntax(&self, file_id: FileId) -> Diagnostics {
+            lint_syntax(self, file_id)
+        }
     }
 
     impl SemanticDb for TestDb {

crates/red_knot/src/db.rs

@@ -9,15 +9,18 @@ use crate::files::FileId;
 
 pub mod ast_ids;
 pub mod cache;
+pub mod cancellation;
 pub mod db;
 pub mod files;
 pub mod hir;
+pub mod lint;
 pub mod module;
 mod parse;
 pub mod program;
 pub mod source;
 mod symbols;
 mod types;
+pub mod watch;
 
 pub(crate) type FxDashMap<K, V> = dashmap::DashMap<K, V, BuildHasherDefault<FxHasher>>;
 #[allow(unused)]

crates/red_knot/src/lib.rs

@@ -0,0 +1,124 @@
+use std::ops::{Deref, DerefMut};
+use std::sync::Arc;
+
+use ruff_python_ast::visitor::Visitor;
+use ruff_python_ast::StringLiteral;
+
+use crate::cache::KeyValueCache;
+use crate::db::{HasJar, SourceDb, SourceJar};
+use crate::files::FileId;
+
+pub(crate) fn lint_syntax<Db>(db: &Db, file_id: FileId) -> Diagnostics
+where
+    Db: SourceDb + HasJar<SourceJar>,
+{
+    let storage = &db.jar().lint_syntax;
+
+    storage.get(&file_id, |file_id| {
+        let mut diagnostics = Vec::new();
+
+        let source = db.source(*file_id);
+        lint_lines(source.text(), &mut diagnostics);
+
+        let parsed = db.parse(*file_id);
+
+        if parsed.errors().is_empty() {
+            let ast = parsed.ast();
+
+            let mut visitor = SyntaxLintVisitor {
+                diagnostics,
+                source: source.text(),
+            };
+            visitor.visit_body(&ast.body);
+            diagnostics = visitor.diagnostics
+        } else {
+            diagnostics.extend(parsed.errors().iter().map(|err| err.to_string()));
+        }
+
+        Diagnostics::from(diagnostics)
+    })
+}
+
+pub(crate) fn lint_lines(source: &str, diagnostics: &mut Vec<String>) {
+    for (line_number, line) in source.lines().enumerate() {
+        if line.len() < 88 {
+            continue;
+        }
+
+        let char_count = line.chars().count();
+        if char_count > 88 {
+            diagnostics.push(format!(
+                "Line {} is too long ({} characters)",
+                line_number + 1,
+                char_count
+            ));
+        }
+    }
+}
+
+#[derive(Debug)]
+struct SyntaxLintVisitor<'a> {
+    diagnostics: Vec<String>,
+    source: &'a str,
+}
+
+impl Visitor<'_> for SyntaxLintVisitor<'_> {
+    fn visit_string_literal(&mut self, string_literal: &'_ StringLiteral) {
+        // A very naive implementation of use double quotes
+        let text = &self.source[string_literal.range];
+
+        if text.starts_with('\'') {
+            self.diagnostics
+                .push("Use double quotes for strings".to_string());
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum Diagnostics {
+    Empty,
+    List(Arc<Vec<String>>),
+}
+
+impl Diagnostics {
+    pub fn as_slice(&self) -> &[String] {
+        match self {
+            Diagnostics::Empty => &[],
+            Diagnostics::List(list) => list.as_slice(),
+        }
+    }
+}
+
+impl Deref for Diagnostics {
+    type Target = [String];
+    fn deref(&self) -> &Self::Target {
+        self.as_slice()
+    }
+}
+
+impl From<Vec<String>> for Diagnostics {
+    fn from(value: Vec<String>) -> Self {
+        if value.is_empty() {
+            Diagnostics::Empty
+        } else {
+            Diagnostics::List(Arc::new(value))
+        }
+    }
+}
+
+#[derive(Default, Debug)]
+pub struct LintSyntaxStorage(KeyValueCache<FileId, Diagnostics>);
+
+impl Deref for LintSyntaxStorage {
+    type Target = KeyValueCache<FileId, Diagnostics>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl DerefMut for LintSyntaxStorage {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}