From 7ee8219a4d6f1faaeb703834cb90fac6dd0ccc63 Mon Sep 17 00:00:00 2001 From: "J.C. Zhong" Date: Tue, 10 Dec 2024 13:48:58 -0800 Subject: [PATCH] feat: add column hover tooltip and column value auto completion (#1526) * feat: add column hover tooltip and column value auto completion * comments * address comments --- package.json | 2 +- querybook/server/models/metastore.py | 1 + .../FunctionDocumentationTooltip.tsx | 2 +- .../CodeMirrorTooltip/TableColumnTooltip.tsx | 83 +++++ .../CodeMirrorTooltip/TableTooltip.tsx | 8 +- .../components/QueryEditor/QueryEditor.scss | 4 +- .../components/QueryEditor/QueryEditor.tsx | 20 +- querybook/webapp/const/metastore.ts | 2 +- .../extensions/useAutoCompleteExtension.ts | 148 ++++++-- .../extensions/useHoverTooltipExtension.tsx | 86 ++--- .../hooks/queryEditor/useAutoComplete.ts | 31 -- .../webapp/hooks/queryEditor/useSqlParser.ts | 22 ++ .../hooks/redux/useUserQueryEditorConfig.ts | 2 +- querybook/webapp/lib/codemirror/utils.ts | 30 +- .../lib/sql-helper/sql-autocompleter.ts | 300 ---------------- querybook/webapp/lib/sql-helper/sql-parser.ts | 323 ++++++++++++++++++ 16 files changed, 634 insertions(+), 430 deletions(-) create mode 100644 querybook/webapp/components/CodeMirrorTooltip/TableColumnTooltip.tsx delete mode 100644 querybook/webapp/hooks/queryEditor/useAutoComplete.ts create mode 100644 querybook/webapp/hooks/queryEditor/useSqlParser.ts delete mode 100644 querybook/webapp/lib/sql-helper/sql-autocompleter.ts create mode 100644 querybook/webapp/lib/sql-helper/sql-parser.ts diff --git a/package.json b/package.json index f41744c41..b4f90f53e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "querybook", - "version": "3.37.1", + "version": "3.38.0", "description": "A Big Data Webapp", "private": true, "scripts": { diff --git a/querybook/server/models/metastore.py b/querybook/server/models/metastore.py index 90e1ad8a3..c4fb20c3f 100644 --- a/querybook/server/models/metastore.py +++ b/querybook/server/models/metastore.py @@ -316,6 +316,7 @@ def to_dict(self, include_table=False): "comment": self.comment, "description": self.description, "table_id": self.table_id, + "stats": self.statistics, } if include_table: diff --git a/querybook/webapp/components/CodeMirrorTooltip/FunctionDocumentationTooltip.tsx b/querybook/webapp/components/CodeMirrorTooltip/FunctionDocumentationTooltip.tsx index fffe9002c..c6a2246f2 100644 --- a/querybook/webapp/components/CodeMirrorTooltip/FunctionDocumentationTooltip.tsx +++ b/querybook/webapp/components/CodeMirrorTooltip/FunctionDocumentationTooltip.tsx @@ -27,7 +27,7 @@ export const FunctionDocumentationTooltip: React.FunctionComponent = ({ return (
-
{signature}
+
{signature}
Returns
{returnType}
diff --git a/querybook/webapp/components/CodeMirrorTooltip/TableColumnTooltip.tsx b/querybook/webapp/components/CodeMirrorTooltip/TableColumnTooltip.tsx new file mode 100644 index 000000000..1d336dbcc --- /dev/null +++ b/querybook/webapp/components/CodeMirrorTooltip/TableColumnTooltip.tsx @@ -0,0 +1,83 @@ +import { ContentState } from 'draft-js'; +import React from 'react'; + +import { TableTag } from 'components/DataTableTags/DataTableTags'; +import { IDataColumn } from 'const/metastore'; +import { useResource } from 'hooks/useResource'; +import { TableColumnResource } from 'resource/table'; +import { Tag, TagGroup } from 'ui/Tag/Tag'; + +interface IProps { + column: IDataColumn; +} + +export const TableColumnTooltip: React.FunctionComponent = ({ + column, +}) => { + const { data: tags } = useResource( + React.useCallback( + () => TableColumnResource.getTags(column.id), + [column.id] + ) + ); + + const tagsDOM = (tags || []).map((tag) => ( + + )); + + const description = + typeof column.description === 'string' + ? column.description + : (column.description as ContentState).getPlainText(); + + const statsDOM = (column.stats || []).map((stat, i) => { + const formattedValue = Array.isArray(stat.value) + ? stat.value.join(', ') + : stat.value; + return ( + + {stat.key} + + {formattedValue} + + + ); + }); + + const contentDOM = ( + <> +
+
{column.name}
+
+ {column.type && ( +
+
Type:
+
{column.type}
+
+ )} + {tagsDOM.length > 0 && ( +
{tagsDOM}
+ )} + {column.comment && ( +
+
Definition
+
{column.comment}
+
+ )} + {description && ( +
+
Description
+
{description}
+
+ )} + {statsDOM.length && ( +
+
Stats
+
{statsDOM}
+
+ )} + + ); + + return
{contentDOM}
; +}; diff --git a/querybook/webapp/components/CodeMirrorTooltip/TableTooltip.tsx b/querybook/webapp/components/CodeMirrorTooltip/TableTooltip.tsx index b062b079a..bc1db3888 100644 --- a/querybook/webapp/components/CodeMirrorTooltip/TableTooltip.tsx +++ b/querybook/webapp/components/CodeMirrorTooltip/TableTooltip.tsx @@ -95,7 +95,7 @@ export const TableTooltip: React.FunctionComponent = ({ const contentDOM = ( <> -
+
{tableName}
{pinToSidebarButton} @@ -110,7 +110,7 @@ export const TableTooltip: React.FunctionComponent = ({ ); - return
{contentDOM}
; + return
{contentDOM}
; }; export const TableTooltipByName: React.FunctionComponent<{ @@ -144,7 +144,9 @@ export const TableTooltipByName: React.FunctionComponent<{ metastoreId ) ); - setTableId(table.id); + if (table) { + setTableId(table.id); + } } catch (error) { console.error('Error fetching table:', error); } diff --git a/querybook/webapp/components/QueryEditor/QueryEditor.scss b/querybook/webapp/components/QueryEditor/QueryEditor.scss index 3107a7f8b..a10773833 100644 --- a/querybook/webapp/components/QueryEditor/QueryEditor.scss +++ b/querybook/webapp/components/QueryEditor/QueryEditor.scss @@ -74,9 +74,9 @@ word-break: break-word; font-size: var(--text-size); - .table-tooltip-header { + .tooltip-header { font-size: var(--text-size); - color: var(--text-dark); + color: var(--color-accent-dark); font-weight: var(--bold-font); justify-content: space-between; align-items: start; diff --git a/querybook/webapp/components/QueryEditor/QueryEditor.tsx b/querybook/webapp/components/QueryEditor/QueryEditor.tsx index 1a72a1c01..9730d8a37 100644 --- a/querybook/webapp/components/QueryEditor/QueryEditor.tsx +++ b/querybook/webapp/components/QueryEditor/QueryEditor.tsx @@ -17,7 +17,10 @@ import toast from 'react-hot-toast'; import { TDataDocMetaVariables } from 'const/datadoc'; import KeyMap from 'const/keyMap'; import { IDataTable } from 'const/metastore'; -import { useAutoCompleteExtension } from 'hooks/queryEditor/extensions/useAutoCompleteExtension'; +import { + AutoCompleteType, + useAutoCompleteExtension, +} from 'hooks/queryEditor/extensions/useAutoCompleteExtension'; import { useEventsExtension } from 'hooks/queryEditor/extensions/useEventsExtension'; import { useHoverTooltipExtension } from 'hooks/queryEditor/extensions/useHoverTooltipExtension'; import { useKeyMapExtension } from 'hooks/queryEditor/extensions/useKeyMapExtension'; @@ -26,13 +29,12 @@ import { useOptionsExtension } from 'hooks/queryEditor/extensions/useOptionsExte import { useSearchExtension } from 'hooks/queryEditor/extensions/useSearchExtension'; import { useSqlCompleteExtension } from 'hooks/queryEditor/extensions/useSqlCompleteExtension'; import { useStatusBarExtension } from 'hooks/queryEditor/extensions/useStatusBarExtension'; -import { useAutoComplete } from 'hooks/queryEditor/useAutoComplete'; import { useCodeAnalysis } from 'hooks/queryEditor/useCodeAnalysis'; import { useLint } from 'hooks/queryEditor/useLint'; +import { useSqlParser } from 'hooks/queryEditor/useSqlParser'; import useDeepCompareEffect from 'hooks/useDeepCompareEffect'; import { CodeMirrorKeyMap } from 'lib/codemirror'; import { mixedSQL } from 'lib/codemirror/codemirror-mixed'; -import { AutoCompleteType } from 'lib/sql-helper/sql-autocompleter'; import { format, ISQLFormatOptions } from 'lib/sql-helper/sql-formatter'; import { TableToken } from 'lib/sql-helper/sql-lexer'; import { navigateWithinEnv } from 'lib/utils/query-string'; @@ -204,12 +206,7 @@ export const QueryEditor: React.FC< language, query: value, }); - const autoCompleterRef = useAutoComplete( - metastoreId, - autoCompleteType, - language, - codeAnalysis - ); + const sqlParserRef = useSqlParser(metastoreId, language, codeAnalysis); const tableReferences: TableToken[] = useMemo( () => @@ -287,7 +284,8 @@ export const QueryEditor: React.FC< const autoCompleteExtension = useAutoCompleteExtension({ view: editorRef.current?.view, - autoCompleterRef, + sqlParserRef, + type: autoCompleteType, }); const lintExtension = useLintExtension({ @@ -296,7 +294,7 @@ export const QueryEditor: React.FC< const { extension: hoverTooltipExtension, getTableAtCursor } = useHoverTooltipExtension({ - codeAnalysisRef, + sqlParserRef, metastoreId, language, }); diff --git a/querybook/webapp/const/metastore.ts b/querybook/webapp/const/metastore.ts index f041db549..927e46452 100644 --- a/querybook/webapp/const/metastore.ts +++ b/querybook/webapp/const/metastore.ts @@ -127,9 +127,9 @@ export interface IDataColumn { name: string; table_id: number; type: string; + stats?: ITableColumnStats[]; } export interface IDetailedDataColumn extends IDataColumn { - stats?: ITableColumnStats[]; tags?: ITag[]; data_element_association?: IDataElementAssociation; } diff --git a/querybook/webapp/hooks/queryEditor/extensions/useAutoCompleteExtension.ts b/querybook/webapp/hooks/queryEditor/extensions/useAutoCompleteExtension.ts index 8122f4f67..652467893 100644 --- a/querybook/webapp/hooks/queryEditor/extensions/useAutoCompleteExtension.ts +++ b/querybook/webapp/hooks/queryEditor/extensions/useAutoCompleteExtension.ts @@ -1,41 +1,151 @@ import { autocompletion, + Completion, CompletionContext, + CompletionResult, startCompletion, } from '@codemirror/autocomplete'; import { EditorView } from '@uiw/react-codemirror'; import { useCallback, useEffect, useMemo, useState } from 'react'; -import { SqlAutoCompleter } from 'lib/sql-helper/sql-autocompleter'; +import { CodeMirrorToken } from 'lib/codemirror/utils'; +import { IPosition } from 'lib/sql-helper/sql-lexer'; +import { SqlParser } from 'lib/sql-helper/sql-parser'; + +export type AutoCompleteType = 'none' | 'schema' | 'all'; // STATIC const RESULT_MAX_LENGTH = 10; export const useAutoCompleteExtension = ({ view, - autoCompleterRef, + sqlParserRef, + type = 'all', }: { view: EditorView; - autoCompleterRef: React.MutableRefObject; + sqlParserRef: React.MutableRefObject; + type: AutoCompleteType; }) => { const [typing, setTyping] = useState(false); - const getCompletions = useCallback(async (context: CompletionContext) => { - // Get the token before the cursor, token could be schema.table.column - const token = context.matchBefore(/(\w+\.){0,2}(\w+)?/); - // is no word before the cursor, don't open completions. - if (!token || !token.text) return null; + const getColumnValueCompletions = useCallback( + (cursor: IPosition, token: CodeMirrorToken): CompletionResult => { + const [textBeforeEqual, textAfterEqual] = token.text + .split('=') + .map((s) => s.trim()); + const columnValues = sqlParserRef.current.getColumnValues( + cursor, + textBeforeEqual + ); + + const hasQuote = textAfterEqual.startsWith("'"); + return { + from: token.to - textAfterEqual.length + (hasQuote ? 1 : 0), + options: columnValues.map((v) => ({ + label: `${v}`, + apply: + typeof v === 'number' || hasQuote ? `${v}` : `'${v}'`, + })), + }; + }, + [sqlParserRef] + ); + + const getGeneralCompletions = useCallback( + async ( + context: string, + cursor: IPosition, + token: CodeMirrorToken + ): Promise => { + const tokenText = token.text.toLowerCase(); + const options: Completion[] = []; + if (context === 'column') { + const columns = sqlParserRef.current.getColumnMatches( + cursor, + token.text + ); + options.push( + ...columns.map((column) => ({ + label: column.name, + detail: 'column', + })) + ); + } else if (context === 'table') { + const tableNames = + await sqlParserRef.current.getTableNameMatches(tokenText); + options.push( + ...tableNames.map((tableName) => ({ + label: tableName, + detail: 'table', + })) + ); + } + + // keyword may appear in all contexts + const keywordMatches = + type === 'all' + ? sqlParserRef.current.getKeyWordMatches(tokenText) + : []; + + options.push( + ...keywordMatches.map((keyword) => ({ + label: keyword, + detail: 'keyword', + })) + ); + + let from = token.from; + if (context === 'column') { + from += token.text.lastIndexOf('.') + 1; + } + + return { from, options }; + }, + [sqlParserRef, type] + ); + + const getCompletions = useCallback( + async (context: CompletionContext) => { + if (type === 'none') { + return null; + } + + // Get the token before the cursor, token could be in below foramts + // - column value: column = value (value may be quoted) + const columnValueRegex = /(\w+\.)?\w+\s*=\s*['"]?\w*/; + // - column: schema.table.column, table.column, column + // - table: schema.table, table + // - keyword: any keyword + const generalTokenRegex = /(\w+\.){0,2}\w*/; + + const columnValueToken = context.matchBefore(columnValueRegex); + const generalToken = + !columnValueToken && context.matchBefore(generalTokenRegex); - const cursorPos = context.pos; - const line = context.state.doc.lineAt(cursorPos); - const cursor = { line: line.number - 1, ch: cursorPos - line.from }; + // no token before the cursor, don't open completions. + if (!columnValueToken?.text && !generalToken?.text) return null; - const completions = await autoCompleterRef.current.getCompletions( - cursor, - token - ); - return completions; - }, []); + // Get the cursor position in codemirror v5 format + const cursorPos = context.pos; + const line = context.state.doc.lineAt(cursorPos); + const cursor = { line: line.number - 1, ch: cursorPos - line.from }; + + const sqlParserContext = + sqlParserRef.current.getContextAtPos(cursor); + + // handle the case where the token is a column and the user is trying to type a value in a where clause + if (sqlParserContext === 'column' && columnValueToken?.text) { + return getColumnValueCompletions(cursor, columnValueToken); + } + + return getGeneralCompletions( + sqlParserContext, + cursor, + generalToken + ); + }, + [sqlParserRef, type, getColumnValueCompletions, getGeneralCompletions] + ); const triggerCompletionOnType = () => { return EditorView.updateListener.of((update) => { @@ -51,11 +161,11 @@ export const useAutoCompleteExtension = ({ }; useEffect(() => { - if (autoCompleterRef.current.codeAnalysis && typing && view) { + if (sqlParserRef.current.codeAnalysis && typing && view) { startCompletion(view); setTyping(false); } - }, [autoCompleterRef.current.codeAnalysis, view]); + }, [sqlParserRef.current.codeAnalysis, view]); const extension = useMemo( () => [ diff --git a/querybook/webapp/hooks/queryEditor/extensions/useHoverTooltipExtension.tsx b/querybook/webapp/hooks/queryEditor/extensions/useHoverTooltipExtension.tsx index c5677521c..0b03752ff 100644 --- a/querybook/webapp/hooks/queryEditor/extensions/useHoverTooltipExtension.tsx +++ b/querybook/webapp/hooks/queryEditor/extensions/useHoverTooltipExtension.tsx @@ -8,84 +8,69 @@ import ReactDOM from 'react-dom'; import { Provider } from 'react-redux'; import { FunctionDocumentationTooltipByName } from 'components/CodeMirrorTooltip/FunctionDocumentationTooltip'; +import { TableColumnTooltip } from 'components/CodeMirrorTooltip/TableColumnTooltip'; import { TableTooltipByName } from 'components/CodeMirrorTooltip/TableTooltip'; import { getTokenAtOffset, offsetToPos } from 'lib/codemirror/utils'; -import { ICodeAnalysis, TableToken } from 'lib/sql-helper/sql-lexer'; +import { SqlParser } from 'lib/sql-helper/sql-parser'; import { reduxStore } from 'redux/store'; export const useHoverTooltipExtension = ({ - codeAnalysisRef, + sqlParserRef, metastoreId, language, }: { - codeAnalysisRef: MutableRefObject; + sqlParserRef: MutableRefObject; metastoreId: number; language: string; }) => { - const getTableAtV5Position = useCallback( - (codeAnalysis, v5Pos: { line: number; ch: number }) => { - const { line, ch } = v5Pos; - if (codeAnalysis) { - const tableReferences: TableToken[] = [].concat.apply( - [], - Object.values(codeAnalysis.lineage.references) - ); - - return tableReferences.find((tableInfo) => { - if (tableInfo.line !== line) { - return false; - } - const isSchemaExplicit = - tableInfo.end - tableInfo.start > tableInfo.name.length; - const tablePos = { - from: - tableInfo.start + - (isSchemaExplicit ? tableInfo.schema.length : 0), - to: tableInfo.end, - }; - - return tablePos.from <= ch && tablePos.to >= ch; - }); - } - - return null; - }, - [] - ); - const getTableAtCursor = useCallback( (editorView: EditorView) => { const selection = editorView.state.selection.main; - const v5Pos = offsetToPos(editorView, selection.from); - return getTableAtV5Position(codeAnalysisRef.current, v5Pos); + const v5Pos = offsetToPos(editorView.state, selection.from); + return sqlParserRef.current.getTableAtPos(v5Pos); }, - [codeAnalysisRef, getTableAtV5Position] + [sqlParserRef.current] ); const getHoverTooltips: HoverTooltipSource = useCallback( (view: EditorView, pos: number, side: -1 | 1) => { - const v5Pos = offsetToPos(view, pos); - const table = getTableAtV5Position(codeAnalysisRef.current, v5Pos); + const v5Pos = offsetToPos(view.state, pos); + + const token = getTokenAtOffset(view.state, pos); + if (!token) { + return null; + } - const token = getTokenAtOffset(view, pos); const nextChar = view.state.doc.sliceString(token.to, token.to + 1); + let table = null; + let column = null; + const context = sqlParserRef.current.getContextAtPos(v5Pos); + let tooltipComponent = null; - if (table) { - tooltipComponent = ( - - ); - } else if (nextChar === '(') { + if (nextChar === '(') { tooltipComponent = ( ); + } else if (context === 'table') { + table = sqlParserRef.current.getTableAtPos(v5Pos); + if (table) { + tooltipComponent = ( + + ); + } + } else if (context === 'column') { + column = sqlParserRef.current.getColumnAtPos(v5Pos, token.text); + if (column) { + tooltipComponent = ; + } } if (!tooltipComponent) { @@ -95,7 +80,6 @@ export const useHoverTooltipExtension = ({ return { pos: token.from, end: token.to, - above: true, create: (view: EditorView) => { const container = document.createElement('div'); ReactDOM.render( @@ -109,7 +93,7 @@ export const useHoverTooltipExtension = ({ }, }; }, - [codeAnalysisRef, getTableAtV5Position, language, metastoreId] + [sqlParserRef, language, metastoreId] ); const extension = useMemo( diff --git a/querybook/webapp/hooks/queryEditor/useAutoComplete.ts b/querybook/webapp/hooks/queryEditor/useAutoComplete.ts deleted file mode 100644 index 5c64f249f..000000000 --- a/querybook/webapp/hooks/queryEditor/useAutoComplete.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { useEffect, useMemo, useRef } from 'react'; - -import { - AutoCompleteType, - SqlAutoCompleter, -} from 'lib/sql-helper/sql-autocompleter'; -import { ICodeAnalysis } from 'lib/sql-helper/sql-lexer'; - -export function useAutoComplete( - metastoreId: number, - autoCompleteType: AutoCompleteType, - language: string, - codeAnalysis: ICodeAnalysis -) { - const autoCompleterRef = useRef(); - const autoCompleter = useMemo(() => { - const completer = new SqlAutoCompleter( - language, - metastoreId, - autoCompleteType - ); - autoCompleterRef.current = completer; - return completer; - }, [language, metastoreId, autoCompleteType]); - - useEffect(() => { - autoCompleter.codeAnalysis = codeAnalysis; - }, [codeAnalysis, autoCompleter]); - - return autoCompleterRef; -} diff --git a/querybook/webapp/hooks/queryEditor/useSqlParser.ts b/querybook/webapp/hooks/queryEditor/useSqlParser.ts new file mode 100644 index 000000000..d30023dba --- /dev/null +++ b/querybook/webapp/hooks/queryEditor/useSqlParser.ts @@ -0,0 +1,22 @@ +import { useEffect, useMemo, useRef } from 'react'; + +import { ICodeAnalysis } from 'lib/sql-helper/sql-lexer'; +import { SqlParser } from 'lib/sql-helper/sql-parser'; + +export function useSqlParser( + metastoreId: number, + language: string, + codeAnalysis: ICodeAnalysis +) { + const parserRef = useRef(); + const parser = useMemo(() => { + parserRef.current = new SqlParser(language, metastoreId); + return parserRef.current; + }, [language, metastoreId]); + + useEffect(() => { + parser.codeAnalysis = codeAnalysis; + }, [codeAnalysis, parser]); + + return parserRef; +} diff --git a/querybook/webapp/hooks/redux/useUserQueryEditorConfig.ts b/querybook/webapp/hooks/redux/useUserQueryEditorConfig.ts index cea45625c..8efdb1155 100644 --- a/querybook/webapp/hooks/redux/useUserQueryEditorConfig.ts +++ b/querybook/webapp/hooks/redux/useUserQueryEditorConfig.ts @@ -1,8 +1,8 @@ import { useMemo } from 'react'; import { UserSettingsFontSizeToCSSFontSize } from 'const/font'; +import { AutoCompleteType } from 'hooks/queryEditor/extensions/useAutoCompleteExtension'; import CodeMirror from 'lib/codemirror'; -import { AutoCompleteType } from 'lib/sql-helper/sql-autocompleter'; import { IStoreState } from 'redux/store/types'; import { useShallowSelector } from './useShallowSelector'; diff --git a/querybook/webapp/lib/codemirror/utils.ts b/querybook/webapp/lib/codemirror/utils.ts index f8d86de8e..a69a55506 100644 --- a/querybook/webapp/lib/codemirror/utils.ts +++ b/querybook/webapp/lib/codemirror/utils.ts @@ -1,5 +1,5 @@ import { syntaxTree } from '@codemirror/language'; -import { EditorView } from '@uiw/react-codemirror'; +import { EditorState, EditorView } from '@uiw/react-codemirror'; import { IPosition } from 'lib/sql-helper/sql-lexer'; @@ -17,24 +17,36 @@ export const posToOffset = (editorView: EditorView, pos: IPosition): number => { // convert offset in v6 to codemirror v5 position export const offsetToPos = ( - editorView: EditorView, + editorState: EditorState, offset: number ): IPosition => { - const doc = editorView.state.doc; + const doc = editorState.doc; const line = doc.lineAt(offset); return { line: line.number - 1, ch: offset - line.from }; }; export const getTokenAtOffset = ( - editorView: EditorView, - pos: number -): CodeMirrorToken => { - const tree = syntaxTree(editorView.state); - const node = tree.resolveInner(pos); + editorState: EditorState, + pos: number, + side?: -1 | 0 | 1 +): CodeMirrorToken | null => { + const tree = syntaxTree(editorState); + let node = tree.resolveInner(pos, side); + + if (node.name === 'Statement' || node.parent === null) { + return null; + } + + // Check if the node is part of a CompositeIdentifier + if (node.parent && node.parent.name === 'CompositeIdentifier') { + node = node.parent; + } + + const to = side === -1 ? pos : node.to; return { from: node.from, to: node.to, - text: editorView.state.doc.sliceString(node.from, node.to), + text: editorState.doc.sliceString(node.from, to), }; }; diff --git a/querybook/webapp/lib/sql-helper/sql-autocompleter.ts b/querybook/webapp/lib/sql-helper/sql-autocompleter.ts deleted file mode 100644 index 909716dd5..000000000 --- a/querybook/webapp/lib/sql-helper/sql-autocompleter.ts +++ /dev/null @@ -1,300 +0,0 @@ -import { Completion, CompletionResult } from '@codemirror/autocomplete'; -import { bind } from 'lodash-decorators'; - -import { CodeMirrorToken } from 'lib/codemirror/utils'; -import { ICodeAnalysis, TableToken } from 'lib/sql-helper/sql-lexer'; -import { - getLanguageSetting, - ILanguageSetting, -} from 'lib/sql-helper/sql-setting'; -import { reduxStore } from 'redux/store'; -import { SearchTableResource } from 'resource/search'; - -interface ILineAnalysis { - statementNum: number; - context: string; - reference: TableToken[]; - alias: Record; -} - -export type AutoCompleteType = 'none' | 'schema' | 'all'; - -function findLast(arr: Array<[number, any]>, num: number) { - let index = 0; - // while index is not the last index - while (arr.length > index + 1) { - if (arr[index + 1][0] <= num) { - index++; - } else { - break; - } - } - return arr[index]; -} - -export class SqlAutoCompleter { - private _codeAnalysis?: ICodeAnalysis; - private metastoreId?: number; - private language: string; - private languageSetting: ILanguageSetting; - private keywords?: string[]; - private type: AutoCompleteType; - - public constructor( - language: string, - metastoreId: number = null, - type: AutoCompleteType = 'all' - ) { - this.metastoreId = metastoreId; - this.type = type; - - this._codeAnalysis = null; - - this.language = language; - this.languageSetting = getLanguageSetting(this.language); - } - - public get codeAnalysis() { - return this._codeAnalysis; - } - - @bind - public set codeAnalysis(newCodeAnalysis: ICodeAnalysis) { - this._codeAnalysis = newCodeAnalysis; - } - - public getKeywords() { - if (!this.keywords) { - this.keywords = [...this.languageSetting.keywords]; - } - return this.keywords; - } - - public getCompletions( - cursor: { line: number; ch: number }, - token: CodeMirrorToken | null, - options: { - passive?: boolean; - } = {} - ): Promise { - if (this.type === 'none') { - return Promise.resolve(null); - } - - const passive = !!options['passive']; - - const lineAnalysis: ILineAnalysis = { - context: 'none', - alias: {}, - reference: [], - statementNum: 0, - }; - if (this.codeAnalysis && this.codeAnalysis.editorLines) { - const editorLines = this.codeAnalysis.editorLines; - const line = editorLines[cursor.line]; - if (line != null) { - lineAnalysis.statementNum = findLast( - line.statements, - cursor.ch - )[1]; - lineAnalysis.context = findLast(line.contexts, cursor.ch)[1]; - lineAnalysis.reference = - this.codeAnalysis.lineage.references[ - lineAnalysis.statementNum - ]; - lineAnalysis.alias = - this.codeAnalysis.lineage.aliases[ - lineAnalysis.statementNum - ]; - } - } - - let result: Completion[] = []; - const searchStr = token.text.toLowerCase(); - - return new Promise(async (resolve) => { - if (searchStr.length > 0 || !passive) { - if (searchStr.includes('.')) { - const matches = await this.addHierarchicalContextMatches( - token, - lineAnalysis - ); - result = matches; - } else { - const flatMatches = await this.addFlatContextMatches( - searchStr, - lineAnalysis - ); - const keywatchMatches = this.addKeyWordMatches( - searchStr, - this.getKeywords() - ); - - result = flatMatches.concat(keywatchMatches); - } - } - - let from = token.from; - if (lineAnalysis.context === 'column') { - from += token.text.lastIndexOf('.') + 1; - } - - resolve({ - from, - options: result ?? [], - }); - }); - } - - private prefixMatch(prefix: string, word: string) { - const len = prefix.length; - return word.substring(0, len).toUpperCase() === prefix.toUpperCase(); - } - - private addKeyWordMatches(searchStr: string, wordList: string[]) { - if (searchStr.length < 2 || this.type === 'schema') { - // we don't autosuggest keywords unless it is at least 2 characters long - // if autocomplete type is schema, then keyword is not provided - return []; - } - - const result = []; - for (const word of wordList) { - if (this.prefixMatch(searchStr, word)) { - result.push({ label: word.toUpperCase(), detail: 'keyword' }); - } - } - - // If user already has typed the full keyword, dont show the hint - if ( - result.length === 1 && - searchStr.toUpperCase() === result[0].label.toUpperCase() - ) { - return []; - } - - return result; - } - - private async getTableNamesFromPrefix(prefix: string): Promise { - const metastoreId = this.metastoreId; - if (metastoreId == null) { - return []; - } - - const { data: names } = await SearchTableResource.suggest( - metastoreId, - prefix - ); - return names; - } - - private getColumnsFromPrefix( - prefix: string, - tableNames: Array> - ) { - const { dataSources } = reduxStore.getState(); - - const dataTables = tableNames - .map((table) => `${table.schema}.${table.name}`) - .filter( - (tableName) => - tableName in - (dataSources.dataTableNameToId[this.metastoreId] || {}) - ) - .map( - (tableName) => - dataSources.dataTableNameToId[this.metastoreId][tableName] - ) - .map((tableId) => dataSources.dataTablesById[tableId]); - const columnIds = [].concat(...dataTables.map((table) => table.column)); - const columnNames = columnIds.map( - (id) => dataSources.dataColumnsById[id].name - ); - const filteredColumnNames = columnNames.filter((name) => - name.toLowerCase().startsWith(prefix) - ); - - return filteredColumnNames; - } - - private async addFlatContextMatches( - searchStr: string, - lineAnalysis: ILineAnalysis - ): Promise { - if (lineAnalysis.context === 'table') { - return (await this.getTableNamesFromPrefix(searchStr)).map( - (tableName) => ({ - label: tableName, - detail: 'table', - }) - ); - } else if ( - lineAnalysis.context === 'column' && - lineAnalysis.reference - ) { - return this.getColumnsFromPrefix( - searchStr, - lineAnalysis.reference - ).map((columnName) => ({ - label: columnName, - detail: 'column', - })); - } - - return []; - } - - private async addHierarchicalContextMatches( - token: CodeMirrorToken, - lineAnalysis: ILineAnalysis - ): Promise { - const context = token.text.split('.'); - - if (lineAnalysis.context === 'table') { - const prefix = context.join('.'); - const tableNames = await this.getTableNamesFromPrefix(prefix); - - const results = []; - for (const tableName of tableNames) { - const schemaTableNames = tableName.split('.'); - - if (schemaTableNames.length === 2) { - results.push({ - label: tableName, - detail: 'table', - }); - } - } - return results; - } else if (lineAnalysis.context === 'column') { - const tableNames: Array> = []; - if (context.length === 3) { - tableNames.push({ - schema: context[0], - name: context[1], - }); - } else if (context.length === 2 && lineAnalysis.reference) { - const name = context[0]; - if (name in lineAnalysis.alias) { - const table = lineAnalysis.alias[name]; - tableNames.push(table); - } else { - for (const table of lineAnalysis.reference) { - if (table.name === name) { - tableNames.push(table); - } - } - } - } - - const prefix = context[context.length - 1].toLowerCase(); - return this.getColumnsFromPrefix(prefix, tableNames).map( - (column) => ({ - label: column, - detail: 'column', - }) - ); - } - } -} diff --git a/querybook/webapp/lib/sql-helper/sql-parser.ts b/querybook/webapp/lib/sql-helper/sql-parser.ts new file mode 100644 index 000000000..b03fe5dfd --- /dev/null +++ b/querybook/webapp/lib/sql-helper/sql-parser.ts @@ -0,0 +1,323 @@ +import { bind } from 'lodash-decorators'; + +import { IDataColumn } from 'const/metastore'; +import { ICodeAnalysis, IPosition, TableToken } from 'lib/sql-helper/sql-lexer'; +import { + getLanguageSetting, + ILanguageSetting, +} from 'lib/sql-helper/sql-setting'; +import { reduxStore } from 'redux/store'; +import { SearchTableResource } from 'resource/search'; + +interface ILineAnalysis { + statementNum: number; + context: string; + reference: TableToken[]; + alias: Record; +} + +function findLast(arr: Array<[number, any]>, num: number) { + let index = 0; + // while index is not the last index + while (arr.length > index + 1) { + if (arr[index + 1][0] <= num) { + index++; + } else { + break; + } + } + return arr[index]; +} + +export class SqlParser { + private _codeAnalysis?: ICodeAnalysis; + private metastoreId?: number; + private language: string; + private languageSetting: ILanguageSetting; + private keywords?: string[]; + + public constructor(language: string, metastoreId: number = null) { + this.metastoreId = metastoreId; + + this._codeAnalysis = null; + + this.language = language; + this.languageSetting = getLanguageSetting(this.language); + } + + public get codeAnalysis() { + return this._codeAnalysis; + } + + @bind + public set codeAnalysis(newCodeAnalysis: ICodeAnalysis) { + this._codeAnalysis = newCodeAnalysis; + } + + /** + * Get the context at the given position, it could be 'table', 'column' or 'none' + * @param pos position of the cursor or mouse pointer + * @returns string: 'table', 'column' or 'none' + */ + public getContextAtPos(pos: IPosition): string { + if (!this.codeAnalysis?.editorLines) { + return 'none'; + } + const editorLines = this.codeAnalysis.editorLines; + const line = editorLines[pos.line]; + if (!line) { + return 'none'; + } + + return findLast(line.contexts, pos.ch)[1]; + } + + /** + * Get the table at the given position + * @param pos position of the cursor or mouse pointer + * @returns TableToken if the cursor is on a table, otherwise null + */ + public getTableAtPos(pos: IPosition): TableToken | null { + const { line, ch } = pos; + if (this.codeAnalysis) { + const tableReferences: TableToken[] = [].concat.apply( + [], + Object.values(this.codeAnalysis.lineage.references) + ); + + return tableReferences.find((tableInfo) => { + if (tableInfo.line !== line) { + return false; + } + const isSchemaExplicit = + tableInfo.end - tableInfo.start > tableInfo.name.length; + const tablePos = { + from: + tableInfo.start + + (isSchemaExplicit ? tableInfo.schema.length : 0), + to: tableInfo.end, + }; + + return tablePos.from <= ch && tablePos.to >= ch; + }); + } + + return null; + } + + /** + * Get the column at the given position + * @param pos position of the cursor or mouse pointer + * @param text the token text before or at the cursor + * @returns IDataColumn if the cursor is on a column, otherwise null + */ + public getColumnAtPos(pos: IPosition, text: string): IDataColumn | null { + const columns = this.getColumnMatches(pos, text, true); + if (columns.length === 1) { + return columns[0]; + } + return null; + } + + /** + * Get the distinct column values if the cursor is on a column + * @param cursor position of the cursor or mouse pointer + * @param text the token text before or at the cursor + * @returns Array of column values if the cursor is on a column, otherwise empty array + */ + public getColumnValues( + cursor: IPosition, + text: string + ): Array { + const columns = this.getColumnMatches(cursor, text, true); + + if (columns.length !== 1) return []; + + const colStats = columns[0].stats ?? []; + // find the stat with key="distinct_values" + const distinctValuesStat = colStats.find( + (stat) => stat.key === 'distinct_values' + ); + if (distinctValuesStat?.value instanceof Array) { + return distinctValuesStat?.value; + } + + return []; + } + + /** + * Get the columns match for the current cursor position and given text + * + * @param cursor cursor position + * @param text token text before or at the cursor + * @param exactMatch whether to do exact match or prefix match + */ + public getColumnMatches( + cursor: IPosition, + text: string, + exactMatch: boolean = false + ): IDataColumn[] { + const lineAnalysis: ILineAnalysis = this.getLineAnalysis(cursor); + + const tokenText = text.toLowerCase(); + + if (tokenText.includes('.')) { + const tableNames: Array> = []; + const context = tokenText.split('.'); + // for the case of schema.table.column + if (context.length === 3) { + tableNames.push({ + schema: context[0], + name: context[1], + }); + } else if (context.length === 2 && lineAnalysis.reference) { + const name = context[0]; + if (name in lineAnalysis.alias) { + const table = lineAnalysis.alias[name]; + tableNames.push(table); + } else { + for (const table of lineAnalysis.reference) { + if (table.name === name) { + tableNames.push(table); + } + } + } + } + + const columnName = context[context.length - 1].toLowerCase(); + return this.getColumnsFromPrefix( + columnName, + tableNames, + exactMatch + ); + } else { + return this.getColumnsFromPrefix( + tokenText, + lineAnalysis.reference, + exactMatch + ); + } + } + + public getKeyWordMatches(searchStr: string) { + const keywordList = this.getKeywords(); + + if (!searchStr || searchStr.length < 2 || searchStr.includes('.')) { + // we don't autosuggest keywords unless it is at least 2 characters long + // if autocomplete type is schema, then keyword is not provided + return []; + } + + const result = []; + for (const word of keywordList) { + if (this.prefixMatch(searchStr, word)) { + result.push(word.toUpperCase()); + } + } + + // If user already has typed the full keyword, dont show the hint + if ( + result.length === 1 && + searchStr.toUpperCase() === result[0].toUpperCase() + ) { + return []; + } + + return result; + } + + /** + * Get the table names that match the given prefix + * @param prefix prefix to match + * @returns Array of table names that match the prefix + */ + public async getTableNameMatches(prefix: string): Promise { + const metastoreId = this.metastoreId; + if (metastoreId == null) { + return []; + } + + const { data: tableNames } = await SearchTableResource.suggest( + metastoreId, + prefix + ); + + // Filter out table names that are not in the format of schema.table + return tableNames.filter((tableName) => { + const schemaTableNames = tableName.split('.'); + return schemaTableNames.length === 2; + }); + } + + private getLineAnalysis(cursor: IPosition): ILineAnalysis { + const lineAnalysis: ILineAnalysis = { + context: 'none', + alias: {}, + reference: [], + statementNum: 0, + }; + if (this.codeAnalysis && this.codeAnalysis.editorLines) { + const editorLines = this.codeAnalysis.editorLines; + const line = editorLines[cursor.line]; + if (line != null) { + lineAnalysis.statementNum = findLast( + line.statements, + cursor.ch + )[1]; + lineAnalysis.context = findLast(line.contexts, cursor.ch)[1]; + lineAnalysis.reference = + this.codeAnalysis.lineage.references[ + lineAnalysis.statementNum + ]; + lineAnalysis.alias = + this.codeAnalysis.lineage.aliases[ + lineAnalysis.statementNum + ]; + } + } + + return lineAnalysis; + } + + private getKeywords() { + if (!this.keywords) { + this.keywords = [...this.languageSetting.keywords]; + } + return this.keywords; + } + + private prefixMatch(prefix: string, word: string) { + const len = prefix.length; + return word.substring(0, len).toUpperCase() === prefix.toUpperCase(); + } + + private getColumnsFromPrefix( + prefix: string, + tableNames: Array>, + exactMatch: boolean = false + ) { + const { dataSources } = reduxStore.getState(); + + const dataTables = tableNames + .map((table) => `${table.schema}.${table.name}`) + .filter( + (tableName) => + tableName in + (dataSources.dataTableNameToId[this.metastoreId] || {}) + ) + .map( + (tableName) => + dataSources.dataTableNameToId[this.metastoreId][tableName] + ) + .map((tableId) => dataSources.dataTablesById[tableId]); + const columnIds = [].concat(...dataTables.map((table) => table.column)); + const columns = columnIds.map((id) => dataSources.dataColumnsById[id]); + const filteredColumns = columns.filter((column) => + exactMatch + ? column.name.toLowerCase() === prefix + : column.name.toLowerCase().startsWith(prefix) + ); + + return filteredColumns; + } +}