[Streams] [Dedicated Grok UI] Package / highlight matched parts / suggestions

.github/CODEOWNERS

@@ -459,6 +459,7 @@ src/platform/packages/shared/kbn-field-utils @elastic/kibana-data-discovery
 src/platform/packages/shared/kbn-flot-charts @elastic/kibana-presentation @elastic/stack-monitoring
 src/platform/packages/shared/kbn-ftr-common-functional-services @elastic/kibana-operations @elastic/appex-qa
 src/platform/packages/shared/kbn-ftr-common-functional-ui-services @elastic/appex-qa
+src/platform/packages/shared/kbn-grok-ui @elastic/streams-program-team
 src/platform/packages/shared/kbn-grouping @elastic/response-ops
 src/platform/packages/shared/kbn-guided-onboarding @elastic/appex-sharedux
 src/platform/packages/shared/kbn-i18n @elastic/kibana-core

package.json

@@ -562,6 +562,7 @@
     "@kbn/graph-plugin": "link:x-pack/platform/plugins/private/graph",
     "@kbn/grid-example-plugin": "link:examples/grid_example",
     "@kbn/grid-layout": "link:src/platform/packages/private/kbn-grid-layout",
+    "@kbn/grok-ui": "link:src/platform/packages/shared/kbn-grok-ui",
     "@kbn/grokdebugger-plugin": "link:x-pack/platform/plugins/private/grokdebugger",
     "@kbn/grouping": "link:src/platform/packages/shared/kbn-grouping",
     "@kbn/guided-onboarding": "link:src/platform/packages/shared/kbn-guided-onboarding",
@@ -1211,6 +1212,7 @@
     "oas": "^25.3.0",
     "object-hash": "^3.0.0",
     "object-path-immutable": "^3.1.1",
+    "oniguruma-to-es": "^3.1.1",
     "openai": "^4.72.0",
     "openpgp": "5.10.1",
     "ora": "^4.0.4",

renovate.json

@@ -4016,6 +4016,25 @@
       "minimumReleaseAge": "7 days",
       "enabled": true
     },
+    {
+      "groupName": "Oniguruma to es",
+      "matchDepNames": [
+        "oniguruma-to-es"
+      ],
+      "reviewers": [
+        "team:obs-ux-logs-team"
+      ],
+      "matchBaseBranches": [
+        "main"
+      ],
+      "labels": [
+        "Team:obs-ux-logs",
+        "release_note:skip",
+        "backport:all-open"
+      ],
+      "minimumReleaseAge": "7 days",
+      "enabled": true
+    },
     {
       "groupName": "OpenTelemetry modules",
       "matchDepNames": [

src/platform/packages/shared/kbn-grok-ui/README.md

@@ -0,0 +1,84 @@
+# @kbn/grok-ui
+
+
+- Tools for parsing / converting Grok expressions (into Oniguruma / JS Regex).
+- UI components for working with Grok expressions.
+
+# NOTE
+
+The UI for this is still in the work in progress phase. UI / UX will be refined.
+
+## Usage
+
+You can either use the parsing / conversion tools standalone, or use the UI component which wraps the tools. The UI component offers all of the definitions [defined in the ES repo](https://github.com/elastic/elasticsearch/tree/main/libs/grok/src/main/resources/patterns/ecs-v1).
+
+
+## Tools
+
+First you need a `GrokCollection` which will hold your pattern definitions:
+
+`const collection = new GrokCollection();`
+
+Then you can add your definitions:
+
+```ts
+Object.entries(PATTERN_MAP).forEach(([key, value]) => {
+    collection.addPattern(key, String.raw`${value}`);
+});
+```
+
+Once they're added, resolve your patterns. This converts the pattern placeholders into their matching Oniguruma based on the definitions.
+
+`collection.resolvePatterns();`
+
+Now we can create a `DraftGrokExpression`. This instance can have it's expression changed on the fly to test different samples / expressions, this instance will be passed the collection you created with the pattern definitions.
+
+`const draftGrokExpression = new DraftGrokExpression(collection);`
+
+Once you have an expression you're interested in, you can call:
+
+```ts
+draftGrokExpression.updateExpression(
+    String.raw`^\"(?<rid>[^\"]+)\" \| %{IPORHOST:clientip} (?:-|%{IPORHOST:forwardedfor}) (?:-|%{USER:ident}) (?:-|%{USER:auth}) \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|-)\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes})`
+);
+```
+
+At this point you can grab a Regular Expression instance to use (this will have converted Oniguruma to a native JS Regex):
+
+`const regexp = draftGrokExpression.getRegex();`
+
+If you'd just like the raw regex pattern represented as a string you can call:
+
+`const regexpPattern = draftGrokExpression.getRegexPattern()`
+
+Or you can just call `parse()` to get structured output directly:
+
+```ts
+const parsed = draftGrokExpression.parse([
+    `"uRzbUwp5eZgAAAAaqIAAAAAa" | 5.3.2.1 - - - [24/Feb/2013:13:40:51 +0100] "GET /cpc HTTP/1.1" 302 -`,
+    `"URzbTwp5eZgAAAAWlbUAAAAV" | 4.3.2.7 - - - [14/Feb/2013:13:40:47 +0100] "GET /cpc/finish.do?cd=true&mea_d=0&targetPage=%2Fcpc%2F HTTP/1.1" 200 5264`,
+    `"URzbUwp5eZgAAAAaqIEAAAAa" | 4.3.2.1 - - - [14/Feb/2013:13:40:51 +0100] "GET /cpc/ HTTP/1.1" 402 -`,
+    `"URzbUwp5eZgAAAAWlbYAAAAV" | 4.3.2.1 - - - [14/Feb/2013:13:40:51 +0100] "POST /cpc/ HTTP/1.1" 305 - `,
+]);
+```
+
+## UI component 
+
+This component is built on top of the same tools.
+
+```tsx
+const GrokEditorExample = () => {
+  const [samples, setSamples] = useState('');
+  const [expression, setExpression] = useState('');
+
+  return (
+    <GrokEditor
+      samples={samples}
+      onChangeSamples={setSamples}
+      expression={expression}
+      onChangeExpression={setExpression}
+      onChangeOutput={(output) => console.log(output)}
+    />
+  );
+};
+```

src/platform/packages/shared/kbn-grok-ui/components/grok_editor.tsx

@@ -0,0 +1,238 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { CodeEditor, CodeEditorProps } from '@kbn/code-editor';
+import { monaco } from '@kbn/monaco';
+import React, { useCallback, useEffect, useRef, useState, useMemo } from 'react';
+import { css } from '@emotion/react';
+import { useEuiTheme, EuiCodeBlock } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { escape } from 'lodash';
+import { GrokCollection } from '../models/grok_collection_and_pattern';
+import { DraftGrokExpression } from '../models/draft_grok_expression';
+import { PATTERN_MAP } from '../constants/pattern_map';
+
+interface GrokEditorProps {
+  expression: string;
+  onChangeExpression(expression: string): void;
+  samples: string;
+  onChangeSamples(samples: string): void;
+  onChangeOutput?: (output: Array<Record<string, unknown> | null>) => void;
+}
+
+const samplesLabel = i18n.translate('kbn.grokUi.samplesLabel', { defaultMessage: 'Samples' });
+const expressionLabel = i18n.translate('kbn.grokUi.expressionLabel', {
+  defaultMessage: 'Grok expression',
+});
+const outputLabel = i18n.translate('kbn.grokUi.outputLabel', {
+  defaultMessage: 'Output',
+});
+const semanticNameLabel = i18n.translate('kbn.grokUi.semanticNameLabel', {
+  defaultMessage: 'Semantic name:',
+});
+const patternNameLabel = i18n.translate('kbn.grokUi.patternNameLabel', {
+  defaultMessage: 'Pattern:',
+});
+const typeNameLabel = i18n.translate('kbn.grokUi.typeNameLabel', {
+  defaultMessage: 'Type:',
+});
+
+export const GrokEditor = ({
+  expression,
+  onChangeExpression,
+  samples,
+  onChangeSamples,
+  onChangeOutput,
+}: GrokEditorProps) => {
+  const eui = useEuiTheme();
+
+  const [grokResources] = useState<{
+    collection: GrokCollection;
+    draftGrokExpression: DraftGrokExpression;
+    suggestionProvider: monaco.languages.CompletionItemProvider;
+  }>(() => {
+    const collection = new GrokCollection();
+    Object.entries(PATTERN_MAP).forEach(([key, value]) => {
+      collection.addPattern(key, String.raw`${value}`);
+    });
+    collection.resolvePatterns();
+    const draftGrokExpression = new DraftGrokExpression(collection);
+    const suggestionProvider = collection.getSuggestionProvider();
+
+    return {
+      collection,
+      draftGrokExpression,
+      suggestionProvider,
+    };
+  });
+
+  // Monaco doesn't support dynamic inline styles, so we need to generate static styles.
+  const colourPaletteStyles = useMemo(() => {
+    return grokResources.collection.getColourPaletteStyles();
+  }, [grokResources.collection]);
+
+  const [output, setOutput] = useState<Array<Record<string, unknown> | null> | null>(null);
+
+  // Sets background highlights for matching parts and generates structured output
+  const processGrok = useCallback(() => {
+    const { draftGrokExpression } = grokResources;
+    draftGrokExpression.updateExpression(expression);
+    const regexpPatternSource = draftGrokExpression.getRegex();
+    const fields = draftGrokExpression.getFields();
+    const model = sampleEditorRef.current?.getModel();
+    const lineCount = model?.getLineCount() ?? 0;
+
+    // Overall (continuous) match ranges
+    const overallMatchRanges: monaco.Range[] = [];
+    const captureGroupDecorations: monaco.editor.IModelDeltaDecoration[] = [];
+    const outputResult: Array<Record<string, string | number>> = [];
+
+    for (let i = 1; i <= lineCount; i++) {
+      const line = model?.getLineContent(i) ?? '';
+
+      // Parse can handle multiple lines of samples, but we'll go line by line to share the line content lookup.
+      const parsed = draftGrokExpression.parse([line]);
+      outputResult.push(parsed[0]);
+
+      if (regexpPatternSource) {
+        const regexpPattern = new RegExp(
+          regexpPatternSource.source,
+          // d flag is added to allow for indices tracking
+          regexpPatternSource.flags + 'd'
+        );
+
+        // We expect one match per line (we are not using global matches / flags) or none
+        const match = line.match(regexpPattern);
+
+        // Overall continuous match highlight
+        if (match && match.length > 0) {
+          const matchingText = match[0];
+          const startIndex = match.index;
+
+          if (startIndex !== undefined) {
+            const endIndex = startIndex + matchingText.length + 1;
+            const matchRange = new monaco.Range(i, startIndex, i, endIndex);
+            overallMatchRanges.push(matchRange);
+          }
+        }
+
+        // Semantic (field name) match highlights
+        const matchGroupResults = regexpPattern.exec(line);
+        if (matchGroupResults && matchGroupResults.indices && matchGroupResults.indices.groups) {
+          for (const [key, value] of Object.entries(matchGroupResults.indices.groups)) {
+            if (value) {
+              const fieldDefinition = fields.get(key);
+              const [startIndex, endIndex] = value;
+              const decorationRange = new monaco.Range(i, startIndex + 1, i, endIndex + 1);
+              captureGroupDecorations.push({
+                range: decorationRange,
+                options: {
+                  inlineClassName: colourToClassName(fieldDefinition?.colour),
+                  hoverMessage: [
+                    { value: `${semanticNameLabel} ${fieldDefinition.name}` },
+                    {
+                      value: `${patternNameLabel} ${escape(fieldDefinition.pattern)}`,
+                      supportHtml: true,
+                    },
+                    ...(fieldDefinition.type
+                      ? [{ value: `${typeNameLabel} ${fieldDefinition.type}` }]
+                      : []),
+                  ],
+                },
+              });
+            }
+          }
+        }
+      }
+    }
+
+    const overallMatchDecorations: monaco.editor.IModelDeltaDecoration[] = overallMatchRanges.map(
+      (range) => {
+        return {
+          range,
+          options: {
+            inlineClassName: 'grok-pattern-match',
+          },
+        };
+      }
+    );
+
+    sampleEditorDecorationsCollection.current?.clear();
+    sampleEditorDecorationsCollection.current?.set(
+      overallMatchDecorations.concat(captureGroupDecorations)
+    );
+
+    setOutput(outputResult);
+    onChangeOutput?.(outputResult);
+  }, [grokResources, expression, onChangeOutput]);
+
+  useEffect(() => {
+    processGrok();
+  }, [expression, samples, processGrok]);
+
+  const grokEditorRef = useRef<monaco.editor.IStandaloneCodeEditor | null>(null);
+  const sampleEditorRef = useRef<monaco.editor.IStandaloneCodeEditor | null>(null);
+  const sampleEditorDecorationsCollection =
+    useRef<monaco.editor.IEditorDecorationsCollection | null>(null);
+
+  const onGrokEditorMount: CodeEditorProps['editorDidMount'] = (editor) => {
+    grokEditorRef.current = editor;
+  };
+
+  const onSampleEditorMount: CodeEditorProps['editorDidMount'] = (editor) => {
+    sampleEditorRef.current = editor;
+    sampleEditorDecorationsCollection.current = editor.createDecorationsCollection();
+  };
+
+  const onGrokEditorChange: CodeEditorProps['onChange'] = (value) => {
+    onChangeExpression(value);
+  };
+
+  const onSampleEditorChange: CodeEditorProps['onChange'] = (value) => {
+    onChangeSamples(value);
+  };
+
+  return (
+    <div
+      css={css`
+        .grok-pattern-match {
+          background-color: ${eui.euiTheme.colors.highlight};
+        }
+        ${colourPaletteStyles}
+      `}
+    >
+      {samplesLabel}
+      <CodeEditor
+        languageId="plaintext"
+        value={samples}
+        height="150px"
+        editorDidMount={onSampleEditorMount}
+        onChange={onSampleEditorChange}
+      />
+      {expressionLabel}
+      <CodeEditor
+        languageId="grok"
+        value={expression}
+        height="150px"
+        editorDidMount={onGrokEditorMount}
+        onChange={onGrokEditorChange}
+        suggestionProvider={grokResources.suggestionProvider}
+      />
+      {outputLabel}
+      <EuiCodeBlock language="json" fontSize="s" paddingSize="m" overflowHeight={150}>
+        {JSON.stringify(output, null, 2)}
+      </EuiCodeBlock>
+    </div>
+  );
+};
+
+const colourToClassName = (colour: string) => {
+  const colourWithoutHash = colour.substring(1);
+  return `grok-pattern-match-${colourWithoutHash}`;
+};

src/platform/packages/shared/kbn-grok-ui/components/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export * from './grok_editor';