Merge pull request #222 from rjmacarthy/improvement/multiline-parsing

Better multiline completions

package.json

@@ -2,7 +2,7 @@
   "name": "twinny",
   "displayName": "twinny - AI Code Completion and Chat",
   "description": "Locally hosted AI code completion plugin for vscode",
-  "version": "3.11.20",
+  "version": "3.11.23",
   "icon": "assets/icon.png",
   "keywords": [
     "code-inference",
@@ -255,11 +255,11 @@
           "default": true,
           "description": "Activates or deactivates the Twinny extension."
         },
-        "twinny.disableAutoSuggest": {
+        "twinny.autoSuggestEnabled": {
           "order": 1,
           "type": "boolean",
-          "default": false,
-          "description": "Disables automatic suggestions, manual trigger (default shortcut Alt+\\)."
+          "default": true,
+          "description": "Enable automatic completion suggestions, manual trigger (default shortcut Alt+\\)."
         },
         "twinny.contextLength": {
           "order": 2,
@@ -282,28 +282,28 @@
           "description": "Sets the model's creativity level (temperature) for generating completions.",
           "required": true
         },
-        "twinny.useMultiLineCompletions": {
+        "twinny.multilineCompletionsEnabled": {
           "order": 5,
           "type": "boolean",
-          "default": false,
-          "description": "Use multiline completions (Can be inaccurate)."
+          "default": true,
+          "description": "Experimental feature: enables the generation of multi-line completions."
         },
         "twinny.maxLines": {
           "dependencies": {
-            "twinny.useMultiLineCompletions": true
+            "twinny.multilineCompletionsEnabled": true
           },
           "order": 6,
           "type": "number",
-          "default": 7,
-          "description": "Maximum number of lines to use for multi line completions. Applicable only when useMultiLineCompletions is enabled."
+          "default": 30,
+          "description": "Maximum number of lines to use for multi line completions. Applicable only when multilineCompletionsEnabled is enabled."
         },
         "twinny.useFileContext": {
           "order": 8,
           "type": "boolean",
           "default": false,
           "description": "Enables scanning of neighbouring documents to enhance completion prompts. (Experimental)"
         },
-        "twinny.enableCompletionCache": {
+        "twinny.completionCacheEnabled": {
           "order": 9,
           "type": "boolean",
           "default": false,

src/common/constants.ts

@@ -21,6 +21,7 @@ export const MAX_CONTEXT_LINE_COUNT = 200
 export const SKIP_DECLARATION_SYMBOLS = ['=']
 export const IMPORT_SEPARATOR = [',', '{']
 export const SKIP_IMPORT_KEYWORDS_AFTER = ['from', 'as', 'import']
+export const MIN_COMPLETION_CHUNKS = 2
 
 export const EVENT_NAME = {
   twinngAddMessage: 'twinny-add-message',
@@ -53,7 +54,7 @@ export const EVENT_NAME = {
   twinnySetWorkspaceContext: 'twinny-set-workspace-context',
   twinnyStopGeneration: 'twinny-stop-generation',
   twinnyTextSelection: 'twinny-text-selection',
-  twinnyWorkspaceContext: 'twinny-workspace-context',
+  twinnyWorkspaceContext: 'twinny-workspace-context'
 }
 
 export const TWINNY_COMMAND_NAME = {
@@ -76,16 +77,16 @@ export const TWINNY_COMMAND_NAME = {
   sendTerminalText: 'twinny.sendTerminalText',
   getGitCommitMessage: 'twinny.getGitCommitMessage',
   newChat: 'twinny.newChat',
-  focusSidebar: 'twinny.sidebar.focus',
-};
+  focusSidebar: 'twinny.sidebar.focus'
+}
 
 export const CONVERSATION_EVENT_NAME = {
   saveConversation: 'twinny.save-conversation',
   getConversations: 'twinny.get-conversations',
   setActiveConversation: 'twinny.set-active-conversation',
   getActiveConversation: 'twinny.get-active-conversation',
   saveLastConversation: 'twinny.save-last-conversation',
-  removeConversation: 'twinny.remove-conversation',
+  removeConversation: 'twinny.remove-conversation'
 }
 
 export const PROVIDER_EVENT_NAME = {
@@ -251,3 +252,38 @@ export const WASM_LANGAUAGES: { [key: string]: string } = {
   rdl: 'systemrdl',
   toml: 'toml'
 }
+
+export const MULTILINE_OUTSIDE = [
+  'class_body',
+  'interface_body',
+  'interface',
+  'class',
+  'program',
+  'identifier',
+  'export'
+]
+
+export const MULTILINE_INSIDE = [
+  'body',
+  'export_statement',
+  'formal_parameters',
+  'function_definition',
+  'named_imports',
+  'object_pattern',
+  'object_type',
+  'object',
+  'parenthesized_expression',
+  'statement_block'
+]
+
+export const MULTILINE_TYPES = [...MULTILINE_OUTSIDE, ...MULTILINE_INSIDE]
+
+export const MULTI_LINE_DELIMITERS = ['\n\n', '\r\n\r\n']
+
+export const MULTI_LINE_REACT = [
+  'jsx_closing_element',
+  'jsx_element',
+  'jsx_element',
+  'jsx_opening_element',
+  'jsx_self_closing_element',
+]

src/extension/completion-formatter.ts

@@ -4,6 +4,7 @@ import { CLOSING_BRACKETS, OPENING_BRACKETS, QUOTES } from '../common/constants'
 import { Bracket } from '../common/types'
 import { getLanguage } from './utils'
 import { supportedLanguages } from '../common/languages'
+import { getLineBreakCount } from '../webview/utils'
 
 export class CompletionFormatter {
   private _characterAfterCursor: string
@@ -235,7 +236,7 @@ export class CompletionFormatter {
     const languageId =
       supportedLanguages[language.languageId as keyof typeof supportedLanguages]
 
-    if (this._normalisedCompletion.startsWith('// File:')) {
+    if (this._normalisedCompletion.startsWith('// File:') || this._normalisedCompletion === '//') {
       this._completion = ''
       return this
     }
@@ -248,17 +249,12 @@ export class CompletionFormatter {
       return this
     }
 
-    const comments = `${languageId.syntaxComments.start}${
-      languageId.syntaxComments.end ?? ''
-    }`
-    const score = this._normalisedCompletion.score(comments, 0.1)
-    if (this._normalisedCompletion.startsWith(comments) || score > 0.3) {
-      this._completion = ''
-      return this
-    }
-
     if (!languageId || !languageId.syntaxComments) return this
 
+    const lineCount = getLineBreakCount(this._completion)
+
+    if (lineCount > 1) return this
+
     const completionLines = this._completion.split('\n').filter((line) => {
       const startsWithComment = line.startsWith(languageId.syntaxComments.start)
       const includesCommentReference = /\b(Language|File|End):\s*(.*)\b/.test(

src/extension/parser-utils.ts

@@ -1,12 +1,13 @@
-import Parser from 'web-tree-sitter'
+import Parser, { SyntaxNode } from 'web-tree-sitter'
 import {
   WASM_LANGAUAGES
 } from '../common/constants'
 import path from 'path'
 import { Logger } from '../common/logger'
+import { Position } from 'vscode'
 const logger = new Logger()
 
-export const getParserForFile = async (
+export const getParser = async (
   filePath: string
 ): Promise<Parser | undefined> => {
   await Parser.init()
@@ -28,3 +29,33 @@ export const getParserForFile = async (
 
   return parser
 }
+
+export function getNodeAtPosition(
+  tree: Parser.Tree | undefined,
+  position: Position
+): SyntaxNode | null {
+  let foundNode: SyntaxNode | null = null
+  const visitedNodes: SyntaxNode[] = []
+  if (!tree || !position) {
+    return null
+  }
+
+  function searchNode(node: SyntaxNode): boolean {
+    if (
+      position.line >= node.startPosition.row &&
+      position.line <= node.endPosition.row
+    ) {
+      foundNode = node
+      for (const child of node.children) {
+        visitedNodes.push(child)
+        if (searchNode(child)) break
+      }
+      return true
+    }
+    return false
+  }
+
+  searchNode(tree.rootNode)
+
+  return foundNode
+}