diff --git a/Cargo.lock b/Cargo.lock index 2303a839d8f4f0..219800edbcbb3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2175,6 +2175,15 @@ dependencies = [ "zed_actions", ] +[[package]] +name = "betlang" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a5811a1a59386e19785588e5bb2384b7d543b4870acedd1c7a1cb177b13b7d" +dependencies = [ + "fearless_simd", +] + [[package]] name = "bigdecimal" version = "0.4.8" @@ -5765,6 +5774,7 @@ dependencies = [ "indoc", "itertools 0.14.0", "language", + "language_detection", "languages", "linkify", "log", @@ -6554,6 +6564,12 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "fearless_simd" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" + [[package]] name = "feature_flags" version = "0.1.0" @@ -9766,6 +9782,15 @@ dependencies = [ "tree-sitter", ] +[[package]] +name = "language_detection" +version = "0.1.0" +dependencies = [ + "betlang", + "gpui", + "language", +] + [[package]] name = "language_extension" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 3be6ec0967f83b..617a5d51718c89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -114,6 +114,7 @@ members = [ "crates/keymap_editor", "crates/language", "crates/language_core", + "crates/language_detection", "crates/language_extension", "crates/language_model", "crates/language_model_core", @@ -376,6 +377,7 @@ json_schema_store = { path = "crates/json_schema_store" } keymap_editor = { path = "crates/keymap_editor" } language = { path = "crates/language" } language_core = { path = "crates/language_core" } +language_detection = { path = "crates/language_detection" } language_extension = { path = "crates/language_extension" } language_model = { path = "crates/language_model" } language_model_core = { path = "crates/language_model_core" } @@ -554,6 +556,7 @@ aws-smithy-runtime-api = { version = "1.9.2", features = ["http-1x", "client"] } aws-smithy-types = { version = "1.3.4", features = ["http-body-1-x"] } backtrace = "0.3" base64 = "0.22" +betlang = "0.1.0" bitflags = "2.6.0" brotli = "8.0.2" bytes = "1.0" diff --git a/crates/editor/Cargo.toml b/crates/editor/Cargo.toml index b6df4a370fc87a..2f884ae0426fc1 100644 --- a/crates/editor/Cargo.toml +++ b/crates/editor/Cargo.toml @@ -56,6 +56,7 @@ indoc.workspace = true edit_prediction_types.workspace = true itertools.workspace = true language.workspace = true +language_detection.workspace = true linkify.workspace = true log.workspace = true lsp.workspace = true diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 8cbbba449e8064..07bb4f9cc0aa4d 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -190,6 +190,7 @@ use language::{ }, point_from_lsp, point_to_lsp, text_diff_with_options, }; +use language_detection::detect_language; use linked_editing_ranges::refresh_linked_ranges; use lsp::{ CodeActionKind, CompletionItemKind, CompletionTriggerKind, InsertTextFormat, InsertTextMode, @@ -296,6 +297,8 @@ const CURSOR_BLINK_INTERVAL: Duration = Duration::from_millis(500); const MAX_LINE_LEN: usize = 1024; const MIN_NAVIGATION_HISTORY_ROW_DELTA: i64 = 10; const MAX_SELECTION_HISTORY_LEN: usize = 1024; +const MIN_LANGUAGE_DETECTION_LEN: usize = 20; +const MIN_LANGUAGE_DETECTION_CONFIDENCE: f32 = 0.5; pub(crate) const CURSORS_VISIBLE_FOR: Duration = Duration::from_millis(2000); #[doc(hidden)] pub const CODE_ACTIONS_DEBOUNCE_TIMEOUT: Duration = Duration::from_millis(250); @@ -9542,8 +9545,9 @@ impl Editor { cx.emit(EditorEvent::TitleChanged); } + let buffer_id = buffer.read(cx).remote_id(); + if self.project.is_some() { - let buffer_id = buffer.read(cx).remote_id(); self.register_buffer(buffer_id, cx); self.update_lsp_data(Some(buffer_id), window, cx); self.refresh_inlay_hints( @@ -9551,6 +9555,8 @@ impl Editor { cx, ); } + + self.detect_buffer_language(buffer_id, cx); } cx.emit(EditorEvent::BufferEdited); @@ -10804,6 +10810,46 @@ impl Editor { self.refresh_document_symbols(for_buffer, cx); } + fn detect_buffer_language(&self, buffer_id: BufferId, cx: &mut Context) { + if DisableAiSettings::get_global(cx).disable_ai { + return; + } + + let Some(buffer_entity) = self.buffer().read(cx).buffer(buffer_id) else { + return; + }; + + let buffer = buffer_entity.read(cx); + if buffer.file().is_some() { + return; + } + + let buffer_snapshot = buffer.snapshot(); + if buffer_snapshot.len() < MIN_LANGUAGE_DETECTION_LEN { + return; + } + + let Some(language_registry) = buffer.language_registry() else { + return; + }; + let buffer_version = buffer_snapshot.version().clone(); + let detected_language = detect_language(buffer_snapshot, language_registry, cx); + + cx.spawn(async move |_, cx| { + if let Some((detected_language, confidence)) = detected_language.await { + if confidence < MIN_LANGUAGE_DETECTION_CONFIDENCE { + return; + } + buffer_entity.update(cx, |buffer, cx| { + if buffer.file().is_none() && !buffer.version().changed_since(&buffer_version) { + buffer.set_language(Some(detected_language), cx); + } + }); + } + }) + .detach(); + } + fn register_visible_buffers(&mut self, cx: &mut Context) { if !self.lsp_data_enabled() { return; diff --git a/crates/editor/src/editor_tests.rs b/crates/editor/src/editor_tests.rs index 516797a740ddda..e982aa48ef4728 100644 --- a/crates/editor/src/editor_tests.rs +++ b/crates/editor/src/editor_tests.rs @@ -35,8 +35,7 @@ use language::{ tree_sitter_python, }; use language_settings::Formatter; -use languages::markdown_lang; -use languages::rust_lang; +use languages::{language, markdown_lang, rust_lang}; use lsp::{CompletionParams, DEFAULT_LSP_REQUEST_TIMEOUT}; use multi_buffer::{IndentGuide, MultiBuffer, MultiBufferOffset, MultiBufferOffsetUtf16, PathKey}; use parking_lot::Mutex; @@ -9484,6 +9483,80 @@ async fn test_kill_ring_yank_pastes_accumulated_kill_at_each_cursor(cx: &mut Tes cx.assert_editor_state("aone\nˇ bone\nˇ"); } +#[gpui::test] +async fn test_editing_untitled_buffer_redetects_language(cx: &mut TestAppContext) { + init_test(cx, |_| {}); + + let fs = FakeFs::new(cx.executor()); + let project = Project::test(fs, [], cx).await; + let go_language = language("go", tree_sitter_go::LANGUAGE.into()); + project.read_with(cx, |project, _| { + project.languages().add(rust_lang()); + project.languages().add(go_language.clone()); + }); + let buffer = project + .update(cx, |project, cx| project.create_buffer(None, true, cx)) + .await + .unwrap(); + let window = cx.add_window(|window, cx| { + let editor = build_editor_with_project( + project, + MultiBuffer::build_from_buffer(buffer.clone(), cx), + window, + cx, + ); + window.focus(&editor.focus_handle(cx), cx); + editor + }); + let editor = window.root(cx).unwrap(); + let cx = &mut VisualTestContext::from_window(*window, cx); + + assert_eq!( + buffer.read_with(cx, |buffer, _| buffer.language().unwrap().name()), + PLAIN_TEXT.name() + ); + + editor.update_in(cx, |editor, window, cx| { + editor.insert("fn main() {}", window, cx); + }); + cx.run_until_parked(); + + assert_eq!( + buffer.read_with(cx, |buffer, _| buffer.language().unwrap().name()), + PLAIN_TEXT.name() + ); + + editor.update_in(cx, |editor, window, cx| { + editor.select_all(&SelectAll, window, cx); + editor.insert("fn main() { println!(\"hello\"); }", window, cx); + }); + cx.run_until_parked(); + + assert_eq!( + buffer.read_with(cx, |buffer, _| buffer.language().unwrap().name()), + rust_lang().name() + ); + + editor.update_in(cx, |editor, window, cx| { + editor.select_all(&SelectAll, window, cx); + editor.backspace(&Backspace, window, cx); + }); + cx.run_until_parked(); + + editor.update_in(cx, |editor, window, cx| { + cx.write_to_clipboard(ClipboardItem::new_string( + "package main\n\nimport \"fmt\"\n\nfunc main() { fmt.Println(\"hello\") }".to_string(), + )); + editor.paste(&Paste, window, cx); + }); + cx.run_until_parked(); + + assert_eq!( + buffer.read_with(cx, |buffer, _| buffer.language().unwrap().name()), + go_language.name() + ); +} + #[gpui::test] async fn test_clipboard(cx: &mut TestAppContext) { init_test(cx, |_| {}); diff --git a/crates/language_detection/Cargo.toml b/crates/language_detection/Cargo.toml new file mode 100644 index 00000000000000..b2d80c6f162ecd --- /dev/null +++ b/crates/language_detection/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "language_detection" +version = "0.1.0" +edition.workspace = true +publish.workspace = true +license = "GPL-3.0-or-later" + +[lints] +workspace = true + +[lib] +path = "src/language_detection.rs" + +[dependencies] +betlang.workspace = true +gpui.workspace = true +language.workspace = true diff --git a/crates/language_detection/LICENSE-GPL b/crates/language_detection/LICENSE-GPL new file mode 120000 index 00000000000000..89e542f750cd38 --- /dev/null +++ b/crates/language_detection/LICENSE-GPL @@ -0,0 +1 @@ +../../LICENSE-GPL \ No newline at end of file diff --git a/crates/language_detection/src/language_detection.rs b/crates/language_detection/src/language_detection.rs new file mode 100644 index 00000000000000..3c41bf34a88dbc --- /dev/null +++ b/crates/language_detection/src/language_detection.rs @@ -0,0 +1,58 @@ +use gpui::{App, AppContext, Task}; +use language::{BufferSnapshot, Language, LanguageRegistry}; +use std::sync::Arc; + +const SAMPLE_BLOCK_SIZE: usize = 4096; + +fn language_registry_key(language: betlang::Language) -> &'static str { + match language { + betlang::Language::ObjectiveC => "Objective-C", + betlang::Language::Shell => "Shell Script", + _ => language.slug(), + } +} + +pub fn detect_language( + buffer: BufferSnapshot, + language_registry: Arc, + cx: &mut App, +) -> Task, f32)>> { + let source = extract_sample(&buffer); + cx.background_spawn(async move { + let detection = betlang::detect(source); + let (confidence, language) = detection.top_languages().next()?; + let language = language_registry + .language_for_name_or_extension(language_registry_key(language)) + .await + .ok()?; + + Some((language, confidence)) + }) +} + +fn extract_sample(buffer: &BufferSnapshot) -> Vec { + let source_length = buffer.len(); + let ranges = if source_length <= SAMPLE_BLOCK_SIZE * 2 { + vec![0..source_length] + } else { + vec![ + 0..SAMPLE_BLOCK_SIZE, + source_length - SAMPLE_BLOCK_SIZE..source_length, + ] + }; + + ranges + .into_iter() + .flat_map(|range| buffer.bytes_in_range(range)) + .flat_map(|chunk| chunk.iter().copied()) + .collect() +} + +#[cfg(test)] +mod tests { + #[test] + fn detects_rust_source() { + let detection = betlang::detect("fn main() { println!(\"hello\"); }"); + assert_eq!(detection.language(), Some(betlang::Language::Rust)); + } +}