Edit prediction: add some typescript evals, fix a teacher model failure mode

crates/edit_prediction_cli/evals/.zed/settings.json

@@ -1,3 +1,4 @@
 {
   "remove_trailing_whitespace_on_save": false,
+  "soft_wrap": "none",
 }

crates/edit_prediction_cli/evals/vscode--add-async-and-await.md

@@ -0,0 +1,88 @@
++++
+repository_url = "https://github.com/microsoft/vscode"
+revision = "29e6da6efa2287aaa981635a475d425ff4fd5d5c"
++++
+
+## Edit History
+
+```diff
+--- a/src/vs/workbench/contrib/debug/browser/debugCommands.ts
++++ b/src/vs/workbench/contrib/debug/browser/debugCommands.ts
+@@ -304,8 +304,8 @@ CommandsRegistry.registerCommand({
+
+ CommandsRegistry.registerCommand({
+ 	id: REVERSE_CONTINUE_ID,
+-	handler: (accessor: ServicesAccessor, _: string, context: CallStackContext | unknown) => {
+-		getThreadAndRun(accessor, context, thread => thread.reverseContinue());
++	handler: async (accessor: ServicesAccessor, _: string, context: CallStackContext | unknown) => {
++		await getThreadAndRun(accessor, context, thread => thread.reverseContinue());
+ 	}
+ });
+--- a/src/vs/workbench/contrib/debug/browser/debugCommands.ts
++++ b/src/vs/workbench/contrib/debug/browser/debugCommands.ts
+@@ -311,11 +311,11 @@ CommandsRegistry.registerCommand({
+
+ CommandsRegistry.registerCommand({
+ 	id: STEP_BACK_ID,
+-	handler: (accessor: ServicesAccessor, _: string, context: CallStackContext | unknown) => {
++	handler: async (accessor: ServicesAccessor, _: string, context: CallStackContext | unknown) => {
+ 		const contextKeyService = accessor.get(IContextKeyService);
+ 		if (CONTEXT_DISASSEMBLY_VIEW_FOCUS.getValue(contextKeyService)) {
+-			getThreadAndRun(accessor, context, (thread: IThread) => thread.stepBack('instruction'));
++			await getThreadAndRun(accessor, context, (thread: IThread) => thread.stepBack('instruction'));
+ 		} else {
+-			getThreadAndRun(accessor, context, (thread: IThread) => thread.stepBack());
++			await getThreadAndRun(accessor, context, (thread: IThread) => thread.stepBack());
+ 		}
+ 	}
+ });
+--- a/src/vs/workbench/contrib/debug/browser/debugCommands.ts
++++ b/src/vs/workbench/contrib/debug/browser/debugCommands.ts
+@@ -323,8 +323,8 @@ CommandsRegistry.registerCommand({
+
+ CommandsRegistry.registerCommand({
+ 	id: TERMINATE_THREAD_ID,
+-	handler: (accessor: ServicesAccessor, _: string, context: CallStackContext | unknown) => {
+-		getThreadAndRun(accessor, context, thread => thread.terminate());
++	handler: async (accessor: ServicesAccessor, _: string, context: CallStackContext | unknown) => {
++		await getThreadAndRun(accessor, context, thread => thread.terminate());
+ 	}
+ });
+```
+
+## Cursor Position
+
+```src/vs/workbench/contrib/debug/browser/debugCommands.ts
+	weight: KeybindingWeight.WorkbenchContrib,
+	primary: isWeb ? (KeyMod.Alt | KeyCode.F10) : KeyCode.F10, // Browsers do not allow F10 to be binded so we have to bind an alternative
+	when: CONTEXT_DEBUG_STATE.isEqualTo('stopped'),
+	handler: (accessor: ServicesAccessor, _: string, context: CallStackContext | unknown) => {
+	//       ^[CURSOR_POSITION]
+		const contextKeyService = accessor.get(IContextKeyService);
+		if (CONTEXT_DISASSEMBLY_VIEW_FOCUS.getValue(contextKeyService)) {
+			getThreadAndRun(accessor, context, (thread: IThread) => thread.next('instruction'));
+		} else {
+```
+
+## Expected Patch
+
+```diff
+--- a/src/vs/workbench/contrib/debug/browser/debugCommands.ts
++++ b/src/vs/workbench/contrib/debug/browser/debugCommands.ts
+@@ -467,10 +467,10 @@ KeybindingsRegistry.registerCommandAndKeybindingRule({
+ 	weight: KeybindingWeight.WorkbenchContrib,
+ 	primary: isWeb ? (KeyMod.Alt | KeyCode.F10) : KeyCode.F10, // Browsers do not allow F10 to be binded so we have to bind an alternative
+ 	when: CONTEXT_DEBUG_STATE.isEqualTo('stopped'),
+-	handler: (accessor: ServicesAccessor, _: string, context: CallStackContext | unknown) => {
++	handler: async (accessor: ServicesAccessor, _: string, context: CallStackContext | unknown) => {
+ 		const contextKeyService = accessor.get(IContextKeyService);
+ 		if (CONTEXT_DISASSEMBLY_VIEW_FOCUS.getValue(contextKeyService)) {
+-			getThreadAndRun(accessor, context, (thread: IThread) => thread.next('instruction'));
++			await getThreadAndRun(accessor, context, (thread: IThread) => thread.next('instruction'));
+ 		} else {
+-			getThreadAndRun(accessor, context, (thread: IThread) => thread.next());
++			await getThreadAndRun(accessor, context, (thread: IThread) => thread.next());
+ 		}
+ 	}
+ });
+```

crates/edit_prediction_cli/evals/vscode--add-class-decorator.md

@@ -0,0 +1,74 @@
++++
+repository_url = "https://github.com/microsoft/vscode"
+revision = "6f6e26fcdf0a7ca5084e0da284cd7a5b2d41ae4d"
++++
+
+## Edit History
+
+```diff
+--- a/src/vs/workbench/api/common/extHostTypes.ts
++++ b/src/vs/workbench/api/common/extHostTypes.ts
+@@ -18,6 +18,14 @@ import { FileSystemProviderErrorCode, markAsFileSystemProviderError } from 'vs/
+ import type * as vscode from 'vscode';
+
++function es5ClassCompat(target: Function): any {
++	///@ts-expect-error
++	function _() { return Reflect.construct(target, arguments, this.constructor); }
++	Object.defineProperty(_, 'name', Object.getOwnPropertyDescriptor(target, 'name')!);
++	Object.setPrototypeOf(_, target);
++	Object.setPrototypeOf(_.prototype, target.prototype);
++	return _;
++}
++
++@es5ClassCompat
+ export class Disposable {
+--- a/src/vs/workbench/api/common/extHostTypes.ts
++++ b/src/vs/workbench/api/common/extHostTypes.ts
+@@ -50,6 +58,7 @@ export class Disposable {
+ 	}
+ }
+
++@es5ClassCompat
+ export class Position {
+
+ 	static Min(...positions: Position[]): Position {
+--- a/src/vs/workbench/api/common/extHostTypes.ts
++++ b/src/vs/workbench/api/common/extHostTypes.ts
+@@ -220,6 +229,7 @@ export class Position {
+ 	}
+ }
+
++@es5ClassCompat
+ export class Range {
+
+ 	static isRange(thing: any): thing is vscode.Range {
+```
+
+## Cursor Position
+
+```src/vs/workbench/api/common/extHostTypes.ts
+	Prepend = 3
+}
+
+export class TextEdit {
+// <[CURSOR_POSITION]
+
+	static isTextEdit(thing: any): thing is TextEdit {
+		if (thing instanceof TextEdit) {
+			return true;
+```
+
+## Expected Patch
+
+```diff
+--- a/src/vs/workbench/api/common/extHostTypes.ts
++++ b/src/vs/workbench/api/common/extHostTypes.ts
+@@ -475,6 +485,7 @@ export enum EnvironmentVariableMutatorType {
+ 	Prepend = 3
+ }
+
++@es5ClassCompat
+ export class TextEdit {
+
+ 	static isTextEdit(thing: any): thing is TextEdit {
+```

crates/edit_prediction_cli/evals/vscode--add-interface-method.md

@@ -0,0 +1,113 @@
++++
+repository_url = "https://github.com/microsoft/vscode"
+revision = "b64eaf598008e2d600a81d846108f72cb37b48e2"
++++
+
+## Edit History
+
+```diff
+--- a/src/vs/platform/window/electron-main/window.ts
++++ b/src/vs/platform/window/electron-main/window.ts
+@@ -1,49 +1,50 @@
+ export interface ICodeWindow extends IDisposable {
+
+ 	readonly onWillLoad: Event<ILoadEvent>;
+ 	readonly onDidSignalReady: Event<void>;
++	readonly onDidTriggerSystemContextMenu: Event<{ x: number; y: number }>;
+ 	readonly onDidClose: Event<void>;
+ 	readonly onDidDestroy: Event<void>;
+
+ 	readonly whenClosedOrLoaded: Promise<void>;
+--- a/src/vs/platform/windows/electron-main/window.ts
++++ b/src/vs/platform/windows/electron-main/window.ts
+@@ -63,60 +63,63 @@ const enum ReadyState {
+ export class CodeWindow extends Disposable implements ICodeWindow {
+
+ 	//#region Events
+
+ 	private readonly _onWillLoad = this._register(new Emitter<ILoadEvent>());
+ 	readonly onWillLoad = this._onWillLoad.event;
+
+ 	private readonly _onDidSignalReady = this._register(new Emitter<void>());
+ 	readonly onDidSignalReady = this._onDidSignalReady.event;
+
++	private readonly _onDidTriggerSystemContextMenu = this._register(new Emitter<{ x: number; y: number }>());
++	readonly onDidTriggerSystemContextMenu = this._onDidTriggerSystemContextMenu.event;
++
+ 	private readonly _onDidClose = this._register(new Emitter<void>());
+ 	readonly onDidClose = this._onDidClose.event;
+
+ 	private readonly _onDidDestroy = this._register(new Emitter<void>());
+ 	readonly onDidDestroy = this._onDidDestroy.event;
+
+ 	//#endregion
+--- a/src/vs/platform/windows/electron-main/windows.ts
++++ b/src/vs/platform/windows/electron-main/windows.ts
+@@ -1,54 +1,55 @@
+ export interface IWindowsMainService {
+
+ 	readonly _serviceBrand: undefined;
+
+ 	readonly onDidChangeWindowsCount: Event<IWindowsCountChangedEvent>;
+
+ 	readonly onDidOpenWindow: Event<ICodeWindow>;
+ 	readonly onDidSignalReadyWindow: Event<ICodeWindow>;
++	readonly onDidTriggerSystemContextMenu: Event<{ window: ICodeWindow; x: number; y: number }>;
+ 	readonly onDidDestroyWindow: Event<ICodeWindow>;
+--- a/src/vs/platform/windows/electron-main/windowsMainService.ts
++++ b/src/vs/platform/windows/electron-main/windowsMainService.ts
+@@ -160,60 +160,63 @@ interface ISingleFolderWorkspacePathToOpen extends IPathToOpen {
+ export class WindowsMainService extends Disposable implements IWindowsMainService {
+
+ 	declare readonly _serviceBrand: undefined;
+
+ 	private static readonly WINDOWS: ICodeWindow[] = [];
+
+ 	private readonly _onDidOpenWindow = this._register(new Emitter<ICodeWindow>());
+ 	readonly onDidOpenWindow = this._onDidOpenWindow.event;
+
+ 	private readonly _onDidSignalReadyWindow = this._register(new Emitter<ICodeWindow>());
+ 	readonly onDidSignalReadyWindow = this._onDidSignalReadyWindow.event;
+
+ 	private readonly _onDidDestroyWindow = this._register(new Emitter<ICodeWindow>());
+ 	readonly onDidDestroyWindow = this._onDidDestroyWindow.event;
+
+ 	private readonly _onDidChangeWindowsCount = this._register(new Emitter<IWindowsCountChangedEvent>());
+ 	readonly onDidChangeWindowsCount = this._onDidChangeWindowsCount.event;
+
++	private readonly _onDidTriggerSystemContextMenu = this._register(new Emitter<{ window: ICodeWindow; x: number; y: number }>());
++	readonly onDidTriggerSystemContextMenu = this._onDidTriggerSystemContextMenu.event;
++
+ 	private readonly windowsStateHandler = this._register(new WindowsStateHandler(this, this.stateMainService, this.lifecycleMainService, this.logService, this.configurationService));
+```
+
+## Cursor Position
+
+```src/vs/platform/windows/test/electron-main/windowsFinder.test.ts
+	function createTestCodeWindow(options: { lastFocusTime: number; openedFolderUri?: URI; openedWorkspace?: IWorkspaceIdentifier }): ICodeWindow {
+		return new class implements ICodeWindow {
+			onWillLoad: Event<ILoadEvent> = Event.None;
+			onDidSignalReady: Event<void> = Event.None;
+			// <[CURSOR_POSITION]
+			onDidClose: Event<void> = Event.None;
+			onDidDestroy: Event<void> = Event.None;
+			whenClosedOrLoaded: Promise<void> = Promise.resolve();
+			id: number = -1;
+```
+
+## Expected Patch
+
+```diff
+--- a/src/vs/platform/windows/test/electron-main/windowsFinder.test.ts
++++ b/src/vs/platform/windows/test/electron-main/windowsFinder.test.ts
+@@ -7,60 +7,61 @@ import * as assert from 'assert';
+ 	function createTestCodeWindow(options: { lastFocusTime: number; openedFolderUri?: URI; openedWorkspace?: IWorkspaceIdentifier }): ICodeWindow {
+ 		return new class implements ICodeWindow {
+ 			onWillLoad: Event<ILoadEvent> = Event.None;
++			onDidTriggerSystemContextMenu: Event<{ x: number; y: number }> = Event.None;
+ 			onDidSignalReady: Event<void> = Event.None;
+ 			onDidClose: Event<void> = Event.None;
+ 			onDidDestroy: Event<void> = Event.None;
+ 			whenClosedOrLoaded: Promise<void> = Promise.resolve();
+ 			id: number = -1;
+```

crates/edit_prediction_cli/src/format_prompt.rs

@@ -9,8 +9,8 @@ use anyhow::{Context as _, Result, anyhow};
 use edit_prediction::udiff;
 use gpui::AsyncApp;
 use similar::DiffableStr;
+use std::ops::Range;
 use std::sync::Arc;
-use std::{fmt::Write as _, ops::Range};
 use zeta_prompt::{
     ZetaFormat, excerpt_range_for_format, format_zeta_prompt, resolve_cursor_region,
 };
@@ -258,7 +258,6 @@ impl TeacherPrompt {
 
     pub fn format_context(example: &Example) -> String {
         let related_files = example.prompt_inputs.as_ref().map(|pi| &pi.related_files);
-
         let Some(related_files) = related_files else {
             return "(No context)".to_string();
         };
@@ -267,27 +266,10 @@ impl TeacherPrompt {
             return "(No context)".to_string();
         }
 
-        let mut prompt = String::new();
-        for file in related_files {
-            let path_str = file.path.to_string_lossy();
-            writeln!(&mut prompt, "`````{path_str}").ok();
-
-            let mut prev_row = 0;
-            for excerpt in &file.excerpts {
-                if excerpt.row_range.start > prev_row {
-                    prompt.push_str("…\n");
-                }
-                prompt.push_str(&excerpt.text);
-                prompt.push('\n');
-                prev_row = excerpt.row_range.end;
-            }
-            if prev_row < file.max_row {
-                prompt.push_str("…\n");
-            }
-            prompt.push_str("\n`````\n");
-        }
-
-        prompt
+        let prefix = "`````";
+        let suffix = "`````\n\n";
+        let max_tokens = 1024;
+        zeta_prompt::format_related_files_within_budget(related_files, &prefix, &suffix, max_tokens)
     }
 
     fn format_cursor_excerpt(

crates/edit_prediction_cli/src/git.rs

@@ -91,7 +91,7 @@ pub async fn ensure_repo_cloned(repo_url: &str) -> Result<PathBuf> {
     }
 
     // Always fetch to get latest commits
-    run_git(&repo_path, &["fetch", "origin"]).await?;
+    run_git(&repo_path, &["fetch", "--depth", "1000", "origin"]).await?;
 
     // Check if we have a valid HEAD, if not checkout FETCH_HEAD
     let has_head = run_git(&repo_path, &["rev-parse", "HEAD"]).await.is_ok();

crates/edit_prediction_cli/src/main.rs

@@ -39,6 +39,7 @@ use zeta_prompt::ZetaFormat;
 
 use reqwest_client::ReqwestClient;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use std::env;
 use std::fmt::Display;
 use std::fs::{File, OpenOptions};
 use std::hash::{Hash, Hasher};
@@ -897,8 +898,18 @@ fn main() {
         }
 
         Command::Synthesize(synth_args) => {
-            let Some(output_dir) = args.output else {
-                panic!("output dir is required");
+            let output_dir = if let Some(output_dir) = args.output {
+                output_dir
+            } else {
+                let default_output_dir = env::current_dir()
+                    .unwrap()
+                    .join("crates/edit_prediction_cli/evals-generated");
+                if default_output_dir.parent().unwrap().exists() {
+                    std::fs::create_dir(&default_output_dir).ok();
+                    default_output_dir
+                } else {
+                    panic!("output dir is required");
+                }
             };
             let config = SynthesizeConfig {
                 repo_urls: synth_args.repos.clone(),