diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e0517f01a7195..d70d619ea3fd6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -515,6 +515,7 @@ x-pack/test/plugin_functional/plugins/global_search_test @elastic/kibana-core x-pack/platform/plugins/private/graph @elastic/kibana-visualizations examples/grid_example @elastic/kibana-presentation src/platform/packages/private/kbn-grid-layout @elastic/kibana-presentation +src/platform/packages/shared/kbn-grok-ui @elastic/streams-program-team x-pack/platform/plugins/private/grokdebugger @elastic/kibana-management src/platform/packages/shared/kbn-grouping @elastic/response-ops src/platform/packages/shared/kbn-guided-onboarding @elastic/appex-sharedux diff --git a/package.json b/package.json index 46e1f650d8db4..90d8517f35b40 100644 --- a/package.json +++ b/package.json @@ -562,6 +562,7 @@ "@kbn/graph-plugin": "link:x-pack/platform/plugins/private/graph", "@kbn/grid-example-plugin": "link:examples/grid_example", "@kbn/grid-layout": "link:src/platform/packages/private/kbn-grid-layout", + "@kbn/grok-ui": "link:src/platform/packages/shared/kbn-grok-ui", "@kbn/grokdebugger-plugin": "link:x-pack/platform/plugins/private/grokdebugger", "@kbn/grouping": "link:src/platform/packages/shared/kbn-grouping", "@kbn/guided-onboarding": "link:src/platform/packages/shared/kbn-guided-onboarding", @@ -1209,6 +1210,7 @@ "nunjucks": "^3.2.4", "object-hash": "^3.0.0", "object-path-immutable": "^3.1.1", + "oniguruma-to-es": "^3.1.1", "openai": "^4.72.0", "openpgp": "5.10.1", "opn": "^5.5.0", diff --git a/renovate.json b/renovate.json index 28695ddd76253..3f936eda02bf9 100644 --- a/renovate.json +++ b/renovate.json @@ -921,6 +921,25 @@ "minimumReleaseAge": "7 days", "enabled": true }, + { + "groupName": "Oniguruma to es", + "matchDepNames": [ + "oniguruma-to-es" + ], + "reviewers": [ + "team:obs-ux-logs-team" + ], + "matchBaseBranches": [ + "main" + ], + "labels": [ + "Team:obs-ux-logs", + "release_note:skip", + "backport:all-open" + ], + "minimumReleaseAge": "7 days", + "enabled": true + }, { "groupName": "OpenTelemetry modules", "matchDepPrefixes": [ diff --git a/src/platform/packages/shared/kbn-grok-ui/README.md b/src/platform/packages/shared/kbn-grok-ui/README.md new file mode 100644 index 0000000000000..4210ba58e05f1 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/README.md @@ -0,0 +1,84 @@ +# @kbn/grok-ui + + +- Tools for parsing / converting Grok expressions (into Oniguruma / JS Regex). +- UI components for working with Grok expressions. + +# NOTE + +The UI for this is still in the work in progress phase. UI / UX will be refined. + +## Usage + +You can either use the parsing / conversion tools standalone, or use the UI component which wraps the tools. The UI component offers all of the definitions [defined in the ES repo](https://github.com/elastic/elasticsearch/tree/main/libs/grok/src/main/resources/patterns/ecs-v1). + + +## Tools + +First you need a `GrokCollection` which will hold your pattern definitions: + +`const collection = new GrokCollection();` + +Then you can add your definitions: + +```ts +Object.entries(PATTERN_MAP).forEach(([key, value]) => { + collection.addPattern(key, String.raw`${value}`); +}); +``` + +Once they're added, resolve your patterns. This converts the pattern placeholders into their matching Oniguruma based on the definitions. + +`collection.resolvePatterns();` + +Now we can create a `DraftGrokExpression`. This instance can have it's expression changed on the fly to test different samples / expressions, this instance will be passed the collection you created with the pattern definitions. + +`const draftGrokExpression = new DraftGrokExpression(collection);` + +Once you have an expression you're interested in, you can call: + +```ts +draftGrokExpression.updateExpression( + String.raw`^\"(?[^\"]+)\" \| %{IPORHOST:clientip} (?:-|%{IPORHOST:forwardedfor}) (?:-|%{USER:ident}) (?:-|%{USER:auth}) \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|-)\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes})` +); +``` + +At this point you can grab a Regular Expression instance to use (this will have converted Oniguruma to a native JS Regex): + +`const regexp = draftGrokExpression.getRegex();` + +If you'd just like the raw regex pattern represented as a string you can call: + +`const regexpPattern = draftGrokExpression.getRegexPattern()` + +Or you can just call `parse()` to get structured output directly: + +```ts +const parsed = draftGrokExpression.parse([ + `"uRzbUwp5eZgAAAAaqIAAAAAa" | 5.3.2.1 - - - [24/Feb/2013:13:40:51 +0100] "GET /cpc HTTP/1.1" 302 -`, + `"URzbTwp5eZgAAAAWlbUAAAAV" | 4.3.2.7 - - - [14/Feb/2013:13:40:47 +0100] "GET /cpc/finish.do?cd=true&mea_d=0&targetPage=%2Fcpc%2F HTTP/1.1" 200 5264`, + `"URzbUwp5eZgAAAAaqIEAAAAa" | 4.3.2.1 - - - [14/Feb/2013:13:40:51 +0100] "GET /cpc/ HTTP/1.1" 402 -`, + `"URzbUwp5eZgAAAAWlbYAAAAV" | 4.3.2.1 - - - [14/Feb/2013:13:40:51 +0100] "POST /cpc/ HTTP/1.1" 305 - `, +]); +``` + +## UI component + +This component is built on top of the same tools. + +```tsx +const GrokEditorExample = () => { + const [samples, setSamples] = useState(''); + const [expression, setExpression] = useState(''); + + return ( + console.log(output)} + /> + ); +}; +``` \ No newline at end of file diff --git a/src/platform/packages/shared/kbn-grok-ui/components/grok_editor.tsx b/src/platform/packages/shared/kbn-grok-ui/components/grok_editor.tsx new file mode 100644 index 0000000000000..69d251a438f3e --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/components/grok_editor.tsx @@ -0,0 +1,238 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { CodeEditor, CodeEditorProps } from '@kbn/code-editor'; +import { monaco } from '@kbn/monaco'; +import React, { useCallback, useEffect, useRef, useState, useMemo } from 'react'; +import { css } from '@emotion/react'; +import { useEuiTheme, EuiCodeBlock } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { escape } from 'lodash'; +import { GrokCollection } from '../models/grok_collection_and_pattern'; +import { DraftGrokExpression } from '../models/draft_grok_expression'; +import { PATTERN_MAP } from '../constants/pattern_map'; + +interface GrokEditorProps { + expression: string; + onChangeExpression(expression: string): void; + samples: string; + onChangeSamples(samples: string): void; + onChangeOutput?: (output: Array | null>) => void; +} + +const samplesLabel = i18n.translate('kbn.grokUi.samplesLabel', { defaultMessage: 'Samples' }); +const expressionLabel = i18n.translate('kbn.grokUi.expressionLabel', { + defaultMessage: 'Grok expression', +}); +const outputLabel = i18n.translate('kbn.grokUi.outputLabel', { + defaultMessage: 'Output', +}); +const semanticNameLabel = i18n.translate('kbn.grokUi.semanticNameLabel', { + defaultMessage: 'Semantic name:', +}); +const patternNameLabel = i18n.translate('kbn.grokUi.patternNameLabel', { + defaultMessage: 'Pattern:', +}); +const typeNameLabel = i18n.translate('kbn.grokUi.typeNameLabel', { + defaultMessage: 'Type:', +}); + +export const GrokEditor = ({ + expression, + onChangeExpression, + samples, + onChangeSamples, + onChangeOutput, +}: GrokEditorProps) => { + const eui = useEuiTheme(); + + const [grokResources] = useState<{ + collection: GrokCollection; + draftGrokExpression: DraftGrokExpression; + suggestionProvider: monaco.languages.CompletionItemProvider; + }>(() => { + const collection = new GrokCollection(); + Object.entries(PATTERN_MAP).forEach(([key, value]) => { + collection.addPattern(key, String.raw`${value}`); + }); + collection.resolvePatterns(); + const draftGrokExpression = new DraftGrokExpression(collection); + const suggestionProvider = collection.getSuggestionProvider(); + + return { + collection, + draftGrokExpression, + suggestionProvider, + }; + }); + + // Monaco doesn't support dynamic inline styles, so we need to generate static styles. + const colourPaletteStyles = useMemo(() => { + return grokResources.collection.getColourPaletteStyles(); + }, [grokResources.collection]); + + const [output, setOutput] = useState | null> | null>(null); + + // Sets background highlights for matching parts and generates structured output + const processGrok = useCallback(() => { + const { draftGrokExpression } = grokResources; + draftGrokExpression.updateExpression(expression); + const regexpPatternSource = draftGrokExpression.getRegex(); + const fields = draftGrokExpression.getFields(); + const model = sampleEditorRef.current?.getModel(); + const lineCount = model?.getLineCount() ?? 0; + + // Overall (continuous) match ranges + const overallMatchRanges: monaco.Range[] = []; + const captureGroupDecorations: monaco.editor.IModelDeltaDecoration[] = []; + const outputResult: Array> = []; + + for (let i = 1; i <= lineCount; i++) { + const line = model?.getLineContent(i) ?? ''; + + // Parse can handle multiple lines of samples, but we'll go line by line to share the line content lookup. + const parsed = draftGrokExpression.parse([line]); + outputResult.push(parsed[0]); + + if (regexpPatternSource) { + const regexpPattern = new RegExp( + regexpPatternSource.source, + // d flag is added to allow for indices tracking + regexpPatternSource.flags + 'd' + ); + + // We expect one match per line (we are not using global matches / flags) or none + const match = line.match(regexpPattern); + + // Overall continuous match highlight + if (match && match.length > 0) { + const matchingText = match[0]; + const startIndex = match.index; + + if (startIndex !== undefined) { + const endIndex = startIndex + matchingText.length + 1; + const matchRange = new monaco.Range(i, startIndex, i, endIndex); + overallMatchRanges.push(matchRange); + } + } + + // Semantic (field name) match highlights + const matchGroupResults = regexpPattern.exec(line); + if (matchGroupResults && matchGroupResults.indices && matchGroupResults.indices.groups) { + for (const [key, value] of Object.entries(matchGroupResults.indices.groups)) { + if (value) { + const fieldDefinition = fields.get(key); + const [startIndex, endIndex] = value; + const decorationRange = new monaco.Range(i, startIndex + 1, i, endIndex + 1); + captureGroupDecorations.push({ + range: decorationRange, + options: { + inlineClassName: colourToClassName(fieldDefinition?.colour), + hoverMessage: [ + { value: `${semanticNameLabel} ${fieldDefinition.name}` }, + { + value: `${patternNameLabel} ${escape(fieldDefinition.pattern)}`, + supportHtml: true, + }, + ...(fieldDefinition.type + ? [{ value: `${typeNameLabel} ${fieldDefinition.type}` }] + : []), + ], + }, + }); + } + } + } + } + } + + const overallMatchDecorations: monaco.editor.IModelDeltaDecoration[] = overallMatchRanges.map( + (range) => { + return { + range, + options: { + inlineClassName: 'grok-pattern-match', + }, + }; + } + ); + + sampleEditorDecorationsCollection.current?.clear(); + sampleEditorDecorationsCollection.current?.set( + overallMatchDecorations.concat(captureGroupDecorations) + ); + + setOutput(outputResult); + onChangeOutput?.(outputResult); + }, [grokResources, expression, onChangeOutput]); + + useEffect(() => { + processGrok(); + }, [expression, samples, processGrok]); + + const grokEditorRef = useRef(null); + const sampleEditorRef = useRef(null); + const sampleEditorDecorationsCollection = + useRef(null); + + const onGrokEditorMount: CodeEditorProps['editorDidMount'] = (editor) => { + grokEditorRef.current = editor; + }; + + const onSampleEditorMount: CodeEditorProps['editorDidMount'] = (editor) => { + sampleEditorRef.current = editor; + sampleEditorDecorationsCollection.current = editor.createDecorationsCollection(); + }; + + const onGrokEditorChange: CodeEditorProps['onChange'] = (value) => { + onChangeExpression(value); + }; + + const onSampleEditorChange: CodeEditorProps['onChange'] = (value) => { + onChangeSamples(value); + }; + + return ( +
+ {samplesLabel} + + {expressionLabel} + + {outputLabel} + + {JSON.stringify(output, null, 2)} + +
+ ); +}; + +const colourToClassName = (colour: string) => { + const colourWithoutHash = colour.substring(1); + return `grok-pattern-match-${colourWithoutHash}`; +}; diff --git a/src/platform/packages/shared/kbn-grok-ui/components/index.ts b/src/platform/packages/shared/kbn-grok-ui/components/index.ts new file mode 100644 index 0000000000000..c86c12ff0ca79 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/components/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export * from './grok_editor'; diff --git a/src/platform/packages/shared/kbn-grok-ui/constants/pattern_map.ts b/src/platform/packages/shared/kbn-grok-ui/constants/pattern_map.ts new file mode 100644 index 0000000000000..79cd1557d40c1 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/constants/pattern_map.ts @@ -0,0 +1,534 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +// !This file is auto-generated by generate_patterns_cli.ts! +export const PATTERN_MAP = { + S3_REQUEST_LINE: + '(?:%{WORD:http.request.method} %{NOTSPACE:url.original}(?: HTTP/%{NUMBER:http.version})?)', + S3_ACCESS_LOG: + '%{WORD:aws.s3access.bucket_owner} %{NOTSPACE:aws.s3access.bucket} \\%{HTTPDATE:timestamp}\\ (?:-|%{IP:client.ip}) (?:-|%{NOTSPACE:client.user.id}) %{NOTSPACE:aws.s3access.request_id} %{NOTSPACE:aws.s3access.operation} (?:-|%{NOTSPACE:aws.s3access.key}) (?:-|"%{S3_REQUEST_LINE:aws.s3access.request_uri}") (?:-|%{INT:http.response.status_code:int}) (?:-|%{NOTSPACE:aws.s3access.error_code}) (?:-|%{INT:aws.s3access.bytes_sent:long}) (?:-|%{INT:aws.s3access.object_size:long}) (?:-|%{INT:aws.s3access.total_time:int}) (?:-|%{INT:aws.s3access.turn_around_time:int}) "(?:-|%{DATA:http.request.referrer})" "(?:-|%{DATA:user_agent.original})" (?:-|%{NOTSPACE:aws.s3access.version_id})(?: (?:-|%{NOTSPACE:aws.s3access.host_id}) (?:-|%{NOTSPACE:aws.s3access.signature_version}) (?:-|%{NOTSPACE:tls.cipher}) (?:-|%{NOTSPACE:aws.s3access.authentication_type}) (?:-|%{NOTSPACE:aws.s3access.host_header}) (?:-|%{NOTSPACE:aws.s3access.tls_version}))?', + ELB_URIHOST: '%{IPORHOST:url.domain}(?::%{POSINT:url.port:int})?', + ELB_URIPATHQUERY: '%{URIPATH:url.path}(?:\\?%{URIQUERY:url.query})?', + ELB_URIPATHPARAM: '%{ELB_URIPATHQUERY}', + ELB_URI: + '%{URIPROTO:url.scheme}://(?:%{USER:url.username}(?::^@*)?@)?(?:%{ELB_URIHOST})?(?:%{ELB_URIPATHQUERY})?', + ELB_REQUEST_LINE: + '(?:%{WORD:http.request.method} %{ELB_URI:url.original}(?: HTTP/%{NUMBER:http.version})?)', + ELB_V1_HTTP_LOG: + '%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:aws.elb.name} %{IP:source.ip}:%{INT:source.port:int} (?:-|(?:%{IP:aws.elb.backend.ip}:%{INT:aws.elb.backend.port:int})) (?:-1|%{NUMBER:aws.elb.request_processing_time.sec:float}) (?:-1|%{NUMBER:aws.elb.backend_processing_time.sec:float}) (?:-1|%{NUMBER:aws.elb.response_processing_time.sec:float}) %{INT:http.response.status_code:int} (?:-|%{INT:aws.elb.backend.http.response.status_code:int}) %{INT:http.request.body.bytes:long} %{INT:http.response.body.bytes:long} "%{ELB_REQUEST_LINE}"(?: "(?:-|%{DATA:user_agent.original})" (?:-|%{NOTSPACE:tls.cipher}) (?:-|%{NOTSPACE:aws.elb.ssl_protocol}))?', + ELB_ACCESS_LOG: '%{ELB_V1_HTTP_LOG}', + CLOUDFRONT_ACCESS_LOG: + '(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\\t%{TIME})\\t%{WORD:aws.cloudfront.x_edge_location}\\t(?:-|%{INT:destination.bytes:long})\\t%{IPORHOST:source.ip}\\t%{WORD:http.request.method}\\t%{HOSTNAME:url.domain}\\t%{NOTSPACE:url.path}\\t(?:(?:000)|%{INT:http.response.status_code:int})\\t(?:-|%{DATA:http.request.referrer})\\t%{DATA:user_agent.original}\\t(?:-|%{DATA:url.query})\\t(?:-|%{DATA:aws.cloudfront.http.request.cookie})\\t%{WORD:aws.cloudfront.x_edge_result_type}\\t%{NOTSPACE:aws.cloudfront.x_edge_request_id}\\t%{HOSTNAME:aws.cloudfront.http.request.host}\\t%{URIPROTO:network.protocol}\\t(?:-|%{INT:source.bytes:long})\\t%{NUMBER:aws.cloudfront.time_taken:float}\\t(?:-|%{IP:network.forwarded_ip})\\t(?:-|%{DATA:aws.cloudfront.ssl_protocol})\\t(?:-|%{NOTSPACE:tls.cipher})\\t%{WORD:aws.cloudfront.x_edge_response_result_type}(?:\\t(?:-|HTTP/%{NUMBER:http.version})\\t(?:-|%{DATA:aws.cloudfront.fle_status})\\t(?:-|%{DATA:aws.cloudfront.fle_encrypted_fields})\\t%{INT:source.port:int}\\t%{NUMBER:aws.cloudfront.time_to_first_byte:float}\\t(?:-|%{DATA:aws.cloudfront.x_edge_detailed_result_type})\\t(?:-|%{NOTSPACE:http.request.mime_type})\\t(?:-|%{INT:aws.cloudfront.http.request.size:long})\\t(?:-|%{INT:aws.cloudfront.http.request.range.start:long})\\t(?:-|%{INT:aws.cloudfront.http.request.range.end:long}))?', + BACULA_TIMESTAMP: '%{MONTHDAY}-%{MONTH}(?:-%{YEAR})? %{HOUR}:%{MINUTE}', + BACULA_HOST: '%{HOSTNAME}', + BACULA_VOLUME: '%{USER}', + BACULA_DEVICE: '%{USER}', + BACULA_DEVICEPATH: '%{UNIXPATH}', + BACULA_CAPACITY: '%{INT}{1,3}(,%{INT}{3})*', + BACULA_VERSION: '%{USER}', + BACULA_JOB: '%{USER}', + BACULA_LOG_MAX_CAPACITY: + 'User defined maximum volume capacity %{BACULA_CAPACITY:bacula.volume.max_capacity} exceeded on device \\"%{BACULA_DEVICE:bacula.volume.device}\\" \\(%{BACULA_DEVICEPATH:bacula.volume.path}\\).?', + BACULA_LOG_END_VOLUME: + 'End of medium on Volume \\"%{BACULA_VOLUME:bacula.volume.name}\\" Bytes=%{BACULA_CAPACITY:bacula.volume.bytes} Blocks=%{BACULA_CAPACITY:bacula.volume.blocks} at %{BACULA_TIMESTAMP:bacula.timestamp}.', + BACULA_LOG_NEW_VOLUME: 'Created new Volume \\"%{BACULA_VOLUME:bacula.volume.name}\\" in catalog.', + BACULA_LOG_NEW_LABEL: + 'Labeled new Volume \\"%{BACULA_VOLUME:bacula.volume.name}\\" on (?:file )?device \\"%{BACULA_DEVICE:bacula.volume.device}\\" \\(%{BACULA_DEVICEPATH:bacula.volume.path}\\).', + BACULA_LOG_WROTE_LABEL: + 'Wrote label to prelabeled Volume \\"%{BACULA_VOLUME:bacula.volume.name}\\" on device \\"%{BACULA_DEVICE:bacula.volume.device}\\" \\(%{BACULA_DEVICEPATH:bacula.volume.path}\\)', + BACULA_LOG_NEW_MOUNT: + 'New volume \\"%{BACULA_VOLUME:bacula.volume.name}\\" mounted on device \\"%{BACULA_DEVICE:bacula.volume.device}\\" \\(%{BACULA_DEVICEPATH:bacula.volume.path}\\) at %{BACULA_TIMESTAMP:bacula.timestamp}.', + BACULA_LOG_NOOPEN: '\\s*Cannot open %{DATA}: ERR=%{GREEDYDATA:error.message}', + BACULA_LOG_NOOPENDIR: + '\\s*Could not open directory \\"?%{DATA:file.path}\\"?: ERR=%{GREEDYDATA:error.message}', + BACULA_LOG_NOSTAT: '\\s*Could not stat %{DATA:file.path}: ERR=%{GREEDYDATA:error.message}', + BACULA_LOG_NOJOBS: + 'There are no more Jobs associated with Volume \\"%{BACULA_VOLUME:bacula.volume.name}\\". Marking it purged.', + BACULA_LOG_ALL_RECORDS_PRUNED: + '.*?All records pruned from Volume \\"%{BACULA_VOLUME:bacula.volume.name}\\"; marking it \\"Purged\\"', + BACULA_LOG_BEGIN_PRUNE_JOBS: 'Begin pruning Jobs older than %{INT} month %{INT} days .', + BACULA_LOG_BEGIN_PRUNE_FILES: 'Begin pruning Files.', + BACULA_LOG_PRUNED_JOBS: + 'Pruned %{INT} Jobs* for client %{BACULA_HOST:bacula.client.name} from catalog.', + BACULA_LOG_PRUNED_FILES: + 'Pruned Files from %{INT} Jobs* for client %{BACULA_HOST:bacula.client.name} from catalog.', + BACULA_LOG_ENDPRUNE: 'End auto prune.', + BACULA_LOG_STARTJOB: 'Start Backup JobId %{INT}, Job=%{BACULA_JOB:bacula.job.name}', + BACULA_LOG_STARTRESTORE: 'Start Restore Job %{BACULA_JOB:bacula.job.name}', + BACULA_LOG_USEDEVICE: 'Using Device \\"%{BACULA_DEVICE:bacula.volume.device}\\"', + BACULA_LOG_DIFF_FS: + '\\s*%{UNIXPATH} is a different filesystem. Will not descend from %{UNIXPATH} into it.', + BACULA_LOG_JOBEND: + 'Job write elapsed time = %{DATA:bacula.job.elapsed_time}, Transfer rate = %{NUMBER} (K|M|G)? Bytes/second', + BACULA_LOG_NOPRUNE_JOBS: 'No Jobs found to prune.', + BACULA_LOG_NOPRUNE_FILES: 'No Files found to prune.', + BACULA_LOG_VOLUME_PREVWRITTEN: + 'Volume \\"?%{BACULA_VOLUME:bacula.volume.name}\\"? previously written, moving to end of data.', + BACULA_LOG_READYAPPEND: + 'Ready to append to end of Volume \\"%{BACULA_VOLUME:bacula.volume.name}\\" size=%{INT:bacula.volume.size:long}', + BACULA_LOG_CANCELLING: 'Cancelling duplicate JobId=%{INT:bacula.job.other_id}.', + BACULA_LOG_MARKCANCEL: + 'JobId %{INT:bacula.job.id}, Job %{BACULA_JOB:bacula.job.name} marked to be canceled.', + BACULA_LOG_CLIENT_RBJ: + 'shell command: run ClientRunBeforeJob \\"%{GREEDYDATA:bacula.job.client_run_before_command}\\"', + BACULA_LOG_VSS: '(Generate )?VSS (Writer)?', + BACULA_LOG_MAXSTART: 'Fatal [eE]rror: Job canceled because max start delay time exceeded.', + BACULA_LOG_DUPLICATE: + 'Fatal [eE]rror: JobId %{INT:bacula.job.other_id} already running. Duplicate job not allowed.', + BACULA_LOG_NOJOBSTAT: 'Fatal [eE]rror: No Job status returned from FD.', + BACULA_LOG_FATAL_CONN: + 'Fatal [eE]rror: bsock.c:133 Unable to connect to (Client: %{BACULA_HOST:bacula.client.name}|Storage daemon) on %{IPORHOST:client.address}:%{POSINT:client.port:int}. ERR=%{GREEDYDATA:error.message}', + BACULA_LOG_NO_CONNECT: + 'Warning: bsock.c:127 Could not connect to (Client: %{BACULA_HOST:bacula.client.name}|Storage daemon) on %{IPORHOST:client.address}:%{POSINT:client.port:int}. ERR=%{GREEDYDATA:error.message}', + BACULA_LOG_NO_AUTH: + 'Fatal error: Unable to authenticate with File daemon at \\"?%{IPORHOST:client.address}(?::%{POSINT:client.port:int})?\\"?. Possible causes:', + BACULA_LOG_NOSUIT: 'No prior or suitable Full backup found in catalog. Doing FULL backup.', + BACULA_LOG_NOPRIOR: 'No prior Full backup Job record found.', + BACULA_LOG_JOB: '(Error: )?Bacula %{BACULA_HOST} %{BACULA_VERSION} \\(%{BACULA_VERSION}\\):', + BACULA_LOG: + '%{BACULA_TIMESTAMP:timestamp} %{BACULA_HOST:host.hostname}(?: JobId %{INT:bacula.job.id})?:? (%{BACULA_LOG_MAX_CAPACITY}|%{BACULA_LOG_END_VOLUME}|%{BACULA_LOG_NEW_VOLUME}|%{BACULA_LOG_NEW_LABEL}|%{BACULA_LOG_WROTE_LABEL}|%{BACULA_LOG_NEW_MOUNT}|%{BACULA_LOG_NOOPEN}|%{BACULA_LOG_NOOPENDIR}|%{BACULA_LOG_NOSTAT}|%{BACULA_LOG_NOJOBS}|%{BACULA_LOG_ALL_RECORDS_PRUNED}|%{BACULA_LOG_BEGIN_PRUNE_JOBS}|%{BACULA_LOG_BEGIN_PRUNE_FILES}|%{BACULA_LOG_PRUNED_JOBS}|%{BACULA_LOG_PRUNED_FILES}|%{BACULA_LOG_ENDPRUNE}|%{BACULA_LOG_STARTJOB}|%{BACULA_LOG_STARTRESTORE}|%{BACULA_LOG_USEDEVICE}|%{BACULA_LOG_DIFF_FS}|%{BACULA_LOG_JOBEND}|%{BACULA_LOG_NOPRUNE_JOBS}|%{BACULA_LOG_NOPRUNE_FILES}|%{BACULA_LOG_VOLUME_PREVWRITTEN}|%{BACULA_LOG_READYAPPEND}|%{BACULA_LOG_CANCELLING}|%{BACULA_LOG_MARKCANCEL}|%{BACULA_LOG_CLIENT_RBJ}|%{BACULA_LOG_VSS}|%{BACULA_LOG_MAXSTART}|%{BACULA_LOG_DUPLICATE}|%{BACULA_LOG_NOJOBSTAT}|%{BACULA_LOG_FATAL_CONN}|%{BACULA_LOG_NO_CONNECT}|%{BACULA_LOG_NO_AUTH}|%{BACULA_LOG_NOSUIT}|%{BACULA_LOG_JOB}|%{BACULA_LOG_NOPRIOR})', + BACULA_LOGLINE: '%{BACULA_LOG}', + BIND9_TIMESTAMP: '%{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME}', + BIND9_DNSTYPE: + '(?:A|AAAA|CAA|CDNSKEY|CDS|CERT|CNAME|CSYNC|DLV|DNAME|DNSKEY|DS|HINFO|LOC|MX|NAPTR|NS|NSEC|NSEC3|OPENPGPKEY|PTR|RRSIG|RP|SIG|SMIMEA|SOA|SRV|TSIG|TXT|URI)', + BIND9_CATEGORY: '(?:queries)', + BIND9_QUERYLOGBASE: + 'client(:? @0x(?:[0-9A-Fa-f]+))? %{IP:client.ip}#%{POSINT:client.port:int} \\(%{GREEDYDATA:bind.log.question.name}\\): query: %{GREEDYDATA:dns.question.name} (?IN) %{BIND9_DNSTYPE:dns.question.type}(:? %{DATA:bind.log.question.flags})? \\(%{IP:server.ip}\\)', + BIND9_QUERYLOG: + '%{BIND9_TIMESTAMP:timestamp} %{BIND9_CATEGORY:bing.log.category}: %{LOGLEVEL:log.level}: %{BIND9_QUERYLOGBASE}', + BIND9: '%{BIND9_QUERYLOG}', + BRO_BOOL: '[TF]', + BRO_DATA: '[^\\t]+', + BRO_HTTP: + '%{NUMBER:timestamp}\\t%{NOTSPACE:zeek.session_id}\\t%{IP:source.ip}\\t%{INT:source.port:int}\\t%{IP:destination.ip}\\t%{INT:destination.port:int}\\t%{INT:zeek.http.trans_depth:int}\\t(?:-|%{WORD:http.request.method})\\t(?:-|%{BRO_DATA:url.domain})\\t(?:-|%{BRO_DATA:url.original})\\t(?:-|%{BRO_DATA:http.request.referrer})\\t(?:-|%{BRO_DATA:user_agent.original})\\t(?:-|%{NUMBER:http.request.body.bytes:long})\\t(?:-|%{NUMBER:http.response.body.bytes:long})\\t(?:-|%{POSINT:http.response.status_code:int})\\t(?:-|%{DATA:zeek.http.status_msg})\\t(?:-|%{POSINT:zeek.http.info_code:int})\\t(?:-|%{DATA:zeek.http.info_msg})\\t(?:-|%{BRO_DATA:zeek.http.filename})\\t(?:\\(empty\\)|%{BRO_DATA:zeek.http.tags})\\t(?:-|%{BRO_DATA:url.username})\\t(?:-|%{BRO_DATA:url.password})\\t(?:-|%{BRO_DATA:zeek.http.proxied})\\t(?:-|%{BRO_DATA:zeek.http.orig_fuids})\\t(?:-|%{BRO_DATA:http.request.mime_type})\\t(?:-|%{BRO_DATA:zeek.http.resp_fuids})\\t(?:-|%{BRO_DATA:http.response.mime_type})', + BRO_DNS: + '%{NUMBER:timestamp}\\t%{NOTSPACE:zeek.session_id}\\t%{IP:source.ip}\\t%{INT:source.port:int}\\t%{IP:destination.ip}\\t%{INT:destination.port:int}\\t%{WORD:network.transport}\\t(?:-|%{INT:dns.id:int})\\t(?:-|%{BRO_DATA:dns.question.name})\\t(?:-|%{INT:zeek.dns.qclass:int})\\t(?:-|%{BRO_DATA:zeek.dns.qclass_name})\\t(?:-|%{INT:zeek.dns.qtype:int})\\t(?:-|%{BRO_DATA:dns.question.type})\\t(?:-|%{INT:zeek.dns.rcode:int})\\t(?:-|%{BRO_DATA:dns.response_code})\\t(?:-|%{BRO_BOOL:zeek.dns.AA})\\t(?:-|%{BRO_BOOL:zeek.dns.TC})\\t(?:-|%{BRO_BOOL:zeek.dns.RD})\\t(?:-|%{BRO_BOOL:zeek.dns.RA})\\t(?:-|%{NONNEGINT:zeek.dns.Z:int})\\t(?:-|%{BRO_DATA:zeek.dns.answers})\\t(?:-|%{DATA:zeek.dns.TTLs})\\t(?:-|%{BRO_BOOL:zeek.dns.rejected})', + BRO_CONN: + '%{NUMBER:timestamp}\\t%{NOTSPACE:zeek.session_id}\\t%{IP:source.ip}\\t%{INT:source.port:int}\\t%{IP:destination.ip}\\t%{INT:destination.port:int}\\t%{WORD:network.transport}\\t(?:-|%{BRO_DATA:network.protocol})\\t(?:-|%{NUMBER:zeek.connection.duration:float})\\t(?:-|%{INT:zeek.connection.orig_bytes:long})\\t(?:-|%{INT:zeek.connection.resp_bytes:long})\\t(?:-|%{BRO_DATA:zeek.connection.state})\\t(?:-|%{BRO_BOOL:zeek.connection.local_orig})\\t(?:(?:-|%{BRO_BOOL:zeek.connection.local_resp})\\t)?(?:-|%{INT:zeek.connection.missed_bytes:long})\\t(?:-|%{BRO_DATA:zeek.connection.history})\\t(?:-|%{INT:source.packets:long})\\t(?:-|%{INT:source.bytes:long})\\t(?:-|%{INT:destination.packets:long})\\t(?:-|%{INT:destination.bytes:long})\\t(?:\\(empty\\)|%{BRO_DATA:zeek.connection.tunnel_parents})', + BRO_FILES: + '%{NUMBER:timestamp}\\t%{NOTSPACE:zeek.files.fuid}\\t(?:-|%{IP:server.ip})\\t(?:-|%{IP:client.ip})\\t(?:-|%{BRO_DATA:zeek.files.session_ids})\\t(?:-|%{BRO_DATA:zeek.files.source})\\t(?:-|%{INT:zeek.files.depth:int})\\t(?:-|%{BRO_DATA:zeek.files.analyzers})\\t(?:-|%{BRO_DATA:file.mime_type})\\t(?:-|%{BRO_DATA:file.name})\\t(?:-|%{NUMBER:zeek.files.duration:float})\\t(?:-|%{BRO_DATA:zeek.files.local_orig})\\t(?:-|%{BRO_BOOL:zeek.files.is_orig})\\t(?:-|%{INT:zeek.files.seen_bytes:long})\\t(?:-|%{INT:file.size:long})\\t(?:-|%{INT:zeek.files.missing_bytes:long})\\t(?:-|%{INT:zeek.files.overflow_bytes:long})\\t(?:-|%{BRO_BOOL:zeek.files.timedout})\\t(?:-|%{BRO_DATA:zeek.files.parent_fuid})\\t(?:-|%{BRO_DATA:file.hash.md5})\\t(?:-|%{BRO_DATA:file.hash.sha1})\\t(?:-|%{BRO_DATA:file.hash.sha256})\\t(?:-|%{BRO_DATA:zeek.files.extracted})', + EXIM_MSGID: '[0-9A-Za-z]{6}-[0-9A-Za-z]{6}-[0-9A-Za-z]{2}', + EXIM_FLAGS: '(?:<=|=>|->|\\*>|\\*\\*|==|<>|>>)', + EXIM_DATE: '(:?%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME})', + EXIM_PID: '\\[%{POSINT:process.pid:int}\\]', + EXIM_QT: '((\\d+y)?(\\d+w)?(\\d+d)?(\\d+h)?(\\d+m)?(\\d+s)?)', + EXIM_EXCLUDE_TERMS: + '(Message is frozen|(Start|End) queue run| Warning: | retry time not reached | no (IP address|host name) found for (IP address|host) | unexpected disconnection while reading SMTP command | no immediate delivery: |another process is handling this message)', + EXIM_REMOTE_HOST: + '(H=(%{NOTSPACE:source.address} )?(\\(%{NOTSPACE:exim.log.remote_address}\\) )?\\%{IP:source.ip}\\](?::%{POSINT:source.port:int})?)', + EXIM_INTERFACE: '(I=\\[%{IP:destination.ip}\\](?::%{NUMBER:destination.port:int}))', + EXIM_PROTOCOL: '(P=%{NOTSPACE:network.protocol})', + EXIM_MSG_SIZE: '(S=%{NUMBER:exim.log.message.size:int})', + EXIM_HEADER_ID: '(id=%{NOTSPACE:exim.log.header_id})', + EXIM_QUOTED_CONTENT: '(?:\\\\.|[^\\\\"])*', + EXIM_SUBJECT: '(T="%{EXIM_QUOTED_CONTENT:exim.log.message.subject}")', + EXIM_UNKNOWN_FIELD: '(?:[A-Za-z0-9]{1,4}=(?:%{QUOTEDSTRING}|%{NOTSPACE}))', + EXIM_NAMED_FIELDS: + '(?: (?:%{EXIM_REMOTE_HOST}|%{EXIM_INTERFACE}|%{EXIM_PROTOCOL}|%{EXIM_MSG_SIZE}|%{EXIM_HEADER_ID}|%{EXIM_SUBJECT}|%{EXIM_UNKNOWN_FIELD}))*', + EXIM_MESSAGE_ARRIVAL: + '%{EXIM_DATE:timestamp} (?:%{EXIM_PID} )?%{EXIM_MSGID:exim.log.message.id} (?<=) (?[a-z:] )?%{EMAILADDRESS:exim.log.sender.email}%{EXIM_NAMED_FIELDS}(?:(?: from ?)? for %{EMAILADDRESS:exim.log.recipient.email})?', + EXIM: '%{EXIM_MESSAGE_ARRIVAL}', + NETSCREENSESSIONLOG: + '%{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:observer.hostname} %{NOTSPACE:observer.name}: (?NetScreen) device_id=%{WORD:netscreen.device_id} .*?(system-\\w+-%{NONNEGINT:event.code}\\(%{WORD:netscreen.session.type}\\))?: start_time="%{DATA:netscreen.session.start_time}" duration=%{INT:netscreen.session.duration:int} policy_id=%{INT:netscreen.policy_id} service=%{DATA:netscreen.service} proto=%{INT:netscreen.protocol_number:int} src zone=%{WORD:observer.ingress.zone} dst zone=%{WORD:observer.egress.zone} action=%{WORD:event.action} sent=%{INT:source.bytes:long} rcvd=%{INT:destination.bytes:long} src=%{IPORHOST:source.address} dst=%{IPORHOST:destination.address}(?: src_port=%{INT:source.port:int} dst_port=%{INT:destination.port:int})?(?: src-xlated ip=%{IP:source.nat.ip} port=%{INT:source.nat.port:int} dst-xlated ip=%{IP:destination.nat.ip} port=%{INT:destination.nat.port:int})?(?: session_id=%{INT:netscreen.session.id} reason=%{GREEDYDATA:netscreen.session.reason})?', + CISCO_TAGGED_SYSLOG: + '^<%{POSINT:log.syslog.priority:int}>%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:host.hostname})? ?: %%{CISCOTAG:cisco.asa.tag}:', + CISCOTIMESTAMP: '%{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME}', + CISCOTAG: '[A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+)', + CISCO_ACTION: + 'Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted', + CISCO_REASON: + 'Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\\s*)*', + CISCO_DIRECTION: 'Inbound|inbound|Outbound|outbound', + CISCO_INTERVAL: 'first hit|%{INT}-second interval', + CISCO_XLATE_TYPE: 'static|dynamic', + CISCO_HITCOUNT_INTERVAL: + 'hit-cnt %{INT:cisco.asa.hit_count:int} (?:first hit|%{INT:cisco.asa.interval:int}-second interval)', + CISCO_SRC_IP_USER: + '%{NOTSPACE:observer.ingress.interface.name}:%{IP:source.ip}(?:\\(%{DATA:source.user.name}\\))?', + CISCO_DST_IP_USER: + '%{NOTSPACE:observer.egress.interface.name}:%{IP:destination.ip}(?:\\(%{DATA:destination.user.name}\\))?', + CISCO_SRC_HOST_PORT_USER: + '%{NOTSPACE:observer.ingress.interface.name}:(?:(?:%{IP:source.ip})|(?:%{HOSTNAME:source.address}))(?:/%{INT:source.port:int})?(?:\\(%{DATA:source.user.name}\\))?', + CISCO_DST_HOST_PORT_USER: + '%{NOTSPACE:observer.egress.interface.name}:(?:(?:%{IP:destination.ip})|(?:%{HOSTNAME:destination.address}))(?:/%{INT:destination.port:int})?(?:\\(%{DATA:destination.user.name}\\))?', + CISCOFW104001: '\\((?:Primary|Secondary)\\) Switching to ACTIVE - %{GREEDYDATA:event.reason}', + CISCOFW104002: '\\((?:Primary|Secondary)\\) Switching to STANDBY - %{GREEDYDATA:event.reason}', + CISCOFW104003: '\\((?:Primary|Secondary)\\) Switching to FAILED\\.', + CISCOFW104004: '\\((?:Primary|Secondary)\\) Switching to OK\\.', + CISCOFW105003: + '\\((?:Primary|Secondary)\\) Monitoring on [Ii]nterface %{NOTSPACE:network.interface.name} waiting', + CISCOFW105004: + '\\((?:Primary|Secondary)\\) Monitoring on [Ii]nterface %{NOTSPACE:network.interface.name} normal', + CISCOFW105005: + '\\((?:Primary|Secondary)\\) Lost Failover communications with mate on [Ii]nterface %{NOTSPACE:network.interface.name}', + CISCOFW105008: + '\\((?:Primary|Secondary)\\) Testing [Ii]nterface %{NOTSPACE:network.interface.name}', + CISCOFW105009: + '\\((?:Primary|Secondary)\\) Testing on [Ii]nterface %{NOTSPACE:network.interface.name} (?:Passed|Failed)', + CISCOFW106001: + '%{CISCO_DIRECTION:cisco.asa.network.direction} %{WORD:cisco.asa.network.transport} connection %{CISCO_ACTION:cisco.asa.outcome} from %{IP:source.ip}/%{INT:source.port:int} to %{IP:destination.ip}/%{INT:destination.port:int} flags %{DATA:cisco.asa.tcp_flags} on interface %{NOTSPACE:observer.egress.interface.name}', + CISCOFW106006_106007_106010: + '%{CISCO_ACTION:cisco.asa.outcome} %{CISCO_DIRECTION:cisco.asa.network.direction} %{WORD:cisco.asa.network.transport} (?:from|src) %{IP:source.ip}/%{INT:source.port:int}(?:\\(%{DATA:source.user.name}\\))? (?:to|dst) %{IP:destination.ip}/%{INT:destination.port:int}(?:\\(%{DATA:destination.user.name}\\))? (?:(?:on interface %{NOTSPACE:observer.egress.interface.name})|(?:due to %{CISCO_REASON:event.reason}))', + CISCOFW106014: + '%{CISCO_ACTION:cisco.asa.outcome} %{CISCO_DIRECTION:cisco.asa.network.direction} %{WORD:cisco.asa.network.transport} src %{CISCO_SRC_IP_USER} dst %{CISCO_DST_IP_USER}\\s?\\(type %{INT:cisco.asa.icmp_type:int}, code %{INT:cisco.asa.icmp_code:int}\\)', + CISCOFW106015: + '%{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} \\(%{DATA:cisco.asa.rule_name}\\) from %{IP:source.ip}/%{INT:source.port:int} to %{IP:destination.ip}/%{INT:destination.port:int} flags %{DATA:cisco.asa.tcp_flags} on interface %{NOTSPACE:observer.egress.interface.name}', + CISCOFW106021: + '%{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} reverse path check from %{IP:source.ip} to %{IP:destination.ip} on interface %{NOTSPACE:observer.egress.interface.name}', + CISCOFW106023: + '%{CISCO_ACTION:cisco.asa.outcome}(?: protocol)? %{WORD:cisco.asa.network.transport} src %{CISCO_SRC_HOST_PORT_USER} dst %{CISCO_DST_HOST_PORT_USER}( \\(type %{INT:cisco.asa.icmp_type:int}, code %{INT:cisco.asa.icmp_code:int}\\))? by access-group "?%{DATA:cisco.asa.rule_name}"? \\%{DATA:[@metadata.cisco.asa.hashcode1}, %{DATA:@metadata.cisco.asa.hashcode2}\\]', + CISCOFW106100_2_3: + "access-list %{NOTSPACE:cisco.asa.rule_name} %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} for user '%{DATA:user.name}' %{DATA:observer.ingress.interface.name}/%{IP:source.ip}\\(%{INT:source.port:int}\\) -> %{DATA:observer.egress.interface.name}/%{IP:destination.ip}\\(%{INT:destination.port:int}\\) %{CISCO_HITCOUNT_INTERVAL} \\%{DATA:[@metadata.cisco.asa.hashcode1}, %{DATA:@metadata.cisco.asa.hashcode2}\\]", + CISCOFW106100: + 'access-list %{NOTSPACE:cisco.asa.rule_name} %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} %{DATA:observer.ingress.interface.name}/%{IP:source.ip}\\(%{INT:source.port:int}\\)(?:\\(%{DATA:source.user.name}\\))? -> %{DATA:observer.egress.interface.name}/%{IP:destination.ip}\\(%{INT:destination.port:int}\\)(?:\\(%{DATA:source.user.name}\\))? hit-cnt %{INT:cisco.asa.hit_count:int} %{CISCO_INTERVAL} \\%{DATA:[@metadata.cisco.asa.hashcode1}, %{DATA:@metadata.cisco.asa.hashcode2}\\]', + CISCOFW304001: + '%{IP:source.ip}(?:\\(%{DATA:source.user.name}\\))? Accessed URL %{IP:destination.ip}:%{GREEDYDATA:url.original}', + CISCOFW110002: + '%{CISCO_REASON:event.reason} for %{WORD:cisco.asa.network.transport} from %{DATA:observer.ingress.interface.name}:%{IP:source.ip}/%{INT:source.port:int} to %{IP:destination.ip}/%{INT:destination.port:int}', + CISCOFW302010: + '%{INT:cisco.asa.connections.in_use:int} in use, %{INT:cisco.asa.connections.most_used:int} most used', + CISCOFW302013_302014_302015_302016: + '%{CISCO_ACTION:cisco.asa.outcome}(?: %{CISCO_DIRECTION:cisco.asa.network.direction})? %{WORD:cisco.asa.network.transport} connection %{INT:cisco.asa.connection_id} for %{NOTSPACE:observer.ingress.interface.name}:%{IP:source.ip}/%{INT:source.port:int}(?: \\(%{IP:source.nat.ip}/%{INT:source.nat.port:int}\\))?(?:\\(%{DATA:source.user.name?}\\))? to %{NOTSPACE:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int}( \\(%{IP:destination.nat.ip}/%{INT:destination.nat.port:int}\\))?(?:\\(%{DATA:destination.user.name}\\))?( duration %{TIME:cisco.asa.duration} bytes %{INT:network.bytes:long})?(?: %{CISCO_REASON:event.reason})?(?: \\(%{DATA:user.name}\\))?', + CISCOFW302020_302021: + '%{CISCO_ACTION:cisco.asa.outcome}(?: %{CISCO_DIRECTION:cisco.asa.network.direction})? %{WORD:cisco.asa.network.transport} connection for faddr %{IP:destination.ip}/%{INT:cisco.asa.icmp_seq:int}(?:\\(%{DATA:destination.user.name}\\))? gaddr %{IP:source.nat.ip}/%{INT:cisco.asa.icmp_type:int} laddr %{IP:source.ip}/%{INT}(?: \\(%{DATA:source.user.name}\\))?', + CISCOFW305011: + '%{CISCO_ACTION:cisco.asa.outcome} %{CISCO_XLATE_TYPE} %{WORD:cisco.asa.network.transport} translation from %{DATA:observer.ingress.interface.name}:%{IP:source.ip}(/%{INT:source.port:int})?(?:\\(%{DATA:source.user.name}\\))? to %{DATA:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int}', + CISCOFW313001_313004_313008: + '%{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} type=%{INT:cisco.asa.icmp_type:int}, code=%{INT:cisco.asa.icmp_code:int} from %{IP:source.ip} on interface %{NOTSPACE:observer.egress.interface.name}(?: to %{IP:destination.ip})?', + CISCOFW313005: + '%{CISCO_REASON:event.reason} for %{WORD:cisco.asa.network.transport} error message: %{WORD} src %{CISCO_SRC_IP_USER} dst %{CISCO_DST_IP_USER} \\(type %{INT:cisco.asa.icmp_type:int}, code %{INT:cisco.asa.icmp_code:int}\\) on %{NOTSPACE} interface\\.\\s+Original IP payload: %{WORD:cisco.asa.original_ip_payload.network.transport} src %{IP:cisco.asa.original_ip_payload.source.ip}/%{INT:cisco.asa.original_ip_payload.source.port:int}(?:\\(%{DATA:cisco.asa.original_ip_payload.source.user.name}\\))? dst %{IP:cisco.asa.original_ip_payload.destination.ip}/%{INT:cisco.asa.original_ip_payload.destination.port:int}(?:\\(%{DATA:cisco.asa.original_ip_payload.destination.user.name}\\))?', + CISCOFW321001: + "Resource '%{DATA:cisco.asa.resource.name}' limit of %{POSINT:cisco.asa.resource.limit:int} reached for system", + CISCOFW402117: + '%{WORD:cisco.asa.network.type}: Received a non-IPSec packet \\(protocol=\\s?%{WORD:cisco.asa.network.transport}\\) from %{IP:source.ip} to %{IP:destination.ip}\\.?', + CISCOFW402119: + '%{WORD:cisco.asa.network.type}: Received an %{WORD:cisco.asa.ipsec.protocol} packet \\(SPI=\\s?%{DATA:cisco.asa.ipsec.spi}, sequence number=\\s?%{DATA:cisco.asa.ipsec.seq_num}\\) from %{IP:source.ip} \\(user=\\s?%{DATA:source.user.name}\\) to %{IP:destination.ip} that failed anti-replay checking\\.?', + CISCOFW419001: + '%{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} packet from %{NOTSPACE:observer.ingress.interface.name}:%{IP:source.ip}/%{INT:source.port:int} to %{NOTSPACE:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int}, reason: %{GREEDYDATA:event.reason}', + CISCOFW419002: + '%{CISCO_REASON:event.reason} from %{DATA:observer.ingress.interface.name}:%{IP:source.ip}/%{INT:source.port:int} to %{DATA:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int} with different initial sequence number', + CISCOFW500004: + '%{CISCO_REASON:event.reason} for protocol=%{WORD:cisco.asa.network.transport}, from %{IP:source.ip}/%{INT:source.port:int} to %{IP:destination.ip}/%{INT:destination.port:int}', + CISCOFW602303_602304: + '%{WORD:cisco.asa.network.type}: An %{CISCO_DIRECTION:cisco.asa.network.direction} %{DATA:cisco.asa.ipsec.tunnel_type} SA \\(SPI=\\s?%{DATA:cisco.asa.ipsec.spi}\\) between %{IP:source.ip} and %{IP:destination.ip} \\(user=\\s?%{DATA:source.user.name}\\) has been %{CISCO_ACTION:cisco.asa.outcome}', + CISCOFW710001_710002_710003_710005_710006: + '%{WORD:cisco.asa.network.transport} (?:request|access) %{CISCO_ACTION:cisco.asa.outcome} from %{IP:source.ip}/%{INT:source.port:int} to %{DATA:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int}', + CISCOFW713172: + 'Group = %{DATA:cisco.asa.source.group}, IP = %{IP:source.ip}, Automatic NAT Detection Status:\\s+Remote end\\s*%{DATA:@metadata.cisco.asa.remote_nat}\\s*behind a NAT device\\s+This\\s+end\\s*%{DATA:@metadata.cisco.asa.local_nat}\\s*behind a NAT device', + CISCOFW733100: + '\\\\s*%{DATA:[cisco.asa.burst.object}\\s*\\] drop %{DATA:cisco.asa.burst.id} exceeded. Current burst rate is %{INT:cisco.asa.burst.current_rate:int} per second, max configured rate is %{INT:cisco.asa.burst.configured_rate:int}; Current average rate is %{INT:cisco.asa.burst.avg_rate:int} per second, max configured rate is %{INT:cisco.asa.burst.configured_avg_rate:int}; Cumulative total count is %{INT:cisco.asa.burst.cumulative_count:int}', + IPTABLES_TCP_FLAGS: '(CWR |ECE |URG |ACK |PSH |RST |SYN |FIN )*', + IPTABLES_TCP_PART: + '(?:SEQ=%{INT:iptables.tcp.seq:int}\\s+)?(?:ACK=%{INT:iptables.tcp.ack:int}\\s+)?WINDOW=%{INT:iptables.tcp.window:int}\\s+RES=0x%{BASE16NUM:iptables.tcp_reserved_bits}\\s+%{IPTABLES_TCP_FLAGS:iptables.tcp.flags}', + IPTABLES4_FRAG: '(?:(?<= )(?:CE|DF|MF))*', + IPTABLES4_PART: + 'SRC=%{IPV4:source.ip}\\s+DST=%{IPV4:destination.ip}\\s+LEN=(?:%{INT:iptables.length:int})?\\s+TOS=(?:0|0x%{BASE16NUM:iptables.tos})?\\s+PREC=(?:0x%{BASE16NUM:iptables.precedence_bits})?\\s+TTL=(?:%{INT:iptables.ttl:int})?\\s+ID=(?:%{INT:iptables.id})?\\s+(?:%{IPTABLES4_FRAG:iptables.fragment_flags})?(?:\\s+FRAG: %{INT:iptables.fragment_offset:int})?', + IPTABLES6_PART: + 'SRC=%{IPV6:source.ip}\\s+DST=%{IPV6:destination.ip}\\s+LEN=(?:%{INT:iptables.length:int})?\\s+TC=(?:0|0x%{BASE16NUM:iptables.tos})?\\s+HOPLIMIT=(?:%{INT:iptables.ttl:int})?\\s+FLOWLBL=(?:%{INT:iptables.flow_label})?', + IPTABLES: + 'IN=(?:%{NOTSPACE:observer.ingress.interface.name})?\\s+OUT=(?:%{NOTSPACE:observer.egress.interface.name})?\\s+(?:MAC=(?:%{COMMONMAC:destination.mac})?(?::%{COMMONMAC:source.mac})?(?::A-Fa-f0-9{2}:A-Fa-f0-9{2})?\\s+)?(:?%{IPTABLES4_PART}|%{IPTABLES6_PART}).*?PROTO=(?:%{WORD:network.transport})?\\s+SPT=(?:%{INT:source.port:int})?\\s+DPT=(?:%{INT:destination.port:int})?\\s+(?:%{IPTABLES_TCP_PART})?', + SHOREWALL: + '(?:%{SYSLOGTIMESTAMP:timestamp}) (?:%{WORD:observer.hostname}) .*Shorewall:(?:%{WORD:shorewall.firewall.type})?:(?:%{WORD:shorewall.firewall.action})?.*%{IPTABLES}', + SFW2_LOG_PREFIX: 'SFW2\\-INext\\-%{NOTSPACE:suse.firewall.action}', + SFW2: '((?:%{SYSLOGTIMESTAMP:timestamp})|(?:%{TIMESTAMP_ISO8601:timestamp}))\\s*%{HOSTNAME:observer.hostname}.*?%{SFW2_LOG_PREFIX:suse.firewall.log_prefix}\\s*%{IPTABLES}', + USERNAME: '[a-zA-Z0-9._-]+', + USER: '%{USERNAME}', + EMAILLOCALPART: + "[a-zA-Z0-9!#$%&'*+\\-/=?^_`{|}~]{1,64}(?:\\.[a-zA-Z0-9!#$%&'*+\\-/=?^_`{|}~]{1,62}){0,63}", + EMAILADDRESS: '%{EMAILLOCALPART}@%{HOSTNAME}', + INT: '(?:[+-]?(?:[0-9]+))', + BASE10NUM: '(?[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))', + NUMBER: '(?:%{BASE10NUM})', + BASE16NUM: '(?(?"(?>\\\\.|[^\\\\"]+)+"|""|(?>\'(?>\\\\.|[^\\\\\']+)+\')|\'\'|(?>`(?>\\\\.|[^\\\\`]+)+`)|``))', + UUID: '[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}', + URN: "urn:[0-9A-Za-z][0-9A-Za-z-]{0,31}:(?:%[0-9a-fA-F]{2}|[0-9A-Za-z()+,.:=@;$_!*'/?#-])+", + MAC: '(?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})', + CISCOMAC: '(?:(?:[A-Fa-f0-9]{4}\\.){2}[A-Fa-f0-9]{4})', + WINDOWSMAC: '(?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})', + COMMONMAC: '(?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})', + IPV6: '((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?', + IPV4: '(?[A-Za-z]+:|\\\\)(?:\\\\[^\\\\?*]*)+', + URIPROTO: '[A-Za-z]([A-Za-z0-9+\\-.]+)+', + URIHOST: '%{IPORHOST}(?::%{POSINT})?', + URIPATH: "(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\\-]*)+", + URIQUERY: "[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\\-\\[\\]<>]*", + URIPARAM: '\\?%{URIQUERY}', + URIPATHPARAM: '%{URIPATH}(?:\\?%{URIQUERY})?', + URI: '%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATH}(?:\\?%{URIQUERY})?)?', + MONTH: + '\\b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y|i)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\\b', + MONTHNUM: '(?:0?[1-9]|1[0-2])', + MONTHNUM2: '(?:0[1-9]|1[0-2])', + MONTHDAY: '(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])', + DAY: '(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)', + YEAR: '(?>\\d\\d){1,2}', + HOUR: '(?:2[0123]|[01]?[0-9])', + MINUTE: '(?:[0-5][0-9])', + SECOND: '(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)', + TIME: '(?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])', + DATE_US: '%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}', + DATE_EU: '%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}', + ISO8601_TIMEZONE: '(?:Z|[+-]%{HOUR}(?::?%{MINUTE}))', + ISO8601_SECOND: '%{SECOND}', + TIMESTAMP_ISO8601: + '%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?', + DATE: '%{DATE_US}|%{DATE_EU}', + DATESTAMP: '%{DATE}[- ]%{TIME}', + TZ: '(?:[APMCE][SD]T|UTC)', + DATESTAMP_RFC822: '%{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}', + DATESTAMP_RFC2822: '%{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}', + DATESTAMP_OTHER: '%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}', + DATESTAMP_EVENTLOG: '%{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}', + SYSLOGTIMESTAMP: '%{MONTH} +%{MONTHDAY} %{TIME}', + PROG: '[\\x21-\\x5a\\x5c\\x5e-\\x7e]+', + SYSLOGPROG: '%{PROG:process.name}(?:\\[%{POSINT:process.pid:int}\\])?', + SYSLOGHOST: '%{IPORHOST}', + SYSLOGFACILITY: + '<%{NONNEGINT:log.syslog.facility.code:int}.%{NONNEGINT:log.syslog.priority:int}>', + HTTPDATE: '%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}', + QS: '%{QUOTEDSTRING}', + SYSLOGBASE: + '%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:host.hostname} %{SYSLOGPROG}:', + LOGLEVEL: + '([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo?(?:rmation)?|INFO?(?:RMATION)?|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)', + HAPROXYTIME: '(?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])', + HAPROXYDATE: '%{MONTHDAY}/%{MONTH}/%{YEAR}:%{HAPROXYTIME}.%{INT}', + HAPROXYCAPTUREDREQUESTHEADERS: '%{DATA:haproxy.http.request.captured_headers}', + HAPROXYCAPTUREDRESPONSEHEADERS: '%{DATA:haproxy.http.response.captured_headers}', + HAPROXYURI: + '(?:%{URIPROTO:url.scheme}://)?(?:%{USER:url.username}(?::[^@]*)?@)?(?:%{IPORHOST:url.domain}(?::%{POSINT:url.port:int})?)?(?:%{URIPATH:url.path}(?:\\?%{URIQUERY:url.query})?)?', + HAPROXYHTTPREQUESTLINE: + '(?:|(?:%{WORD:http.request.method} %{HAPROXYURI:url.original}(?: HTTP/%{NUMBER:http.version})?))', + HAPROXYHTTPBASE: + '%{IP:source.address}:%{INT:source.port:int} \\[%{HAPROXYDATE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/(?:|%{NOTSPACE:haproxy.server_name}) (?:-1|%{INT:haproxy.http.request.time_wait_ms:int})/(?:-1|%{INT:haproxy.total_waiting_time_ms:int})/(?:-1|%{INT:haproxy.connection_wait_time_ms:int})/(?:-1|%{INT:haproxy.http.request.time_wait_without_data_ms:int})/%{NOTSPACE:haproxy.total_time_ms} %{INT:http.response.status_code:int} %{INT:source.bytes:long} (?:-|%{DATA:haproxy.http.request.captured_cookie}) (?:-|%{DATA:haproxy.http.response.captured_cookie}) %{NOTSPACE:haproxy.termination_state} %{INT:haproxy.connections.active:int}/%{INT:haproxy.connections.frontend:int}/%{INT:haproxy.connections.backend:int}/%{INT:haproxy.connections.server:int}/%{INT:haproxy.connections.retries:int} %{INT:haproxy.server_queue:int}/%{INT:haproxy.backend_queue:int}(?: \\{%{HAPROXYCAPTUREDREQUESTHEADERS}\\}(?: \\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\\})?)?(?: "%{HAPROXYHTTPREQUESTLINE}"?)?', + HAPROXYHTTP: + '(?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp}) %{IPORHOST:host.hostname} %{SYSLOGPROG}: %{HAPROXYHTTPBASE}', + HAPROXYTCP: + '(?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp}) %{IPORHOST:host.hostname} %{SYSLOGPROG}: %{IP:source.address}:%{INT:source.port:int} \\[%{HAPROXYDATE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/(?:|%{NOTSPACE:haproxy.server_name}) (?:-1|%{INT:haproxy.total_waiting_time_ms:int})/(?:-1|%{INT:haproxy.connection_wait_time_ms:int})/%{NOTSPACE:haproxy.total_time_ms} %{INT:source.bytes:long} %{NOTSPACE:haproxy.termination_state} %{INT:haproxy.connections.active:int}/%{INT:haproxy.connections.frontend:int}/%{INT:haproxy.connections.backend:int}/%{INT:haproxy.connections.server:int}/%{INT:haproxy.connections.retries:int} %{INT:haproxy.server_queue:int}/%{INT:haproxy.backend_queue:int}', + HTTPDUSER: '%{EMAILADDRESS}|%{USER}', + HTTPDERROR_DATE: '%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}', + HTTPD_COMMONLOG: + '%{IPORHOST:source.address} (?:-|%{HTTPDUSER:apache.access.user.identity}) (?:-|%{HTTPDUSER:user.name}) \\[%{HTTPDATE:timestamp}\\] "(?:%{WORD:http.request.method} %{NOTSPACE:url.original}(?: HTTP/%{NUMBER:http.version})?|%{DATA})" (?:-|%{INT:http.response.status_code:int}) (?:-|%{INT:http.response.body.bytes:long})', + HTTPD_COMBINEDLOG: + '%{HTTPD_COMMONLOG} "(?:-|%{DATA:http.request.referrer})" "(?:-|%{DATA:user_agent.original})"', + HTTPD20_ERRORLOG: + '\\[%{HTTPDERROR_DATE:timestamp}\\] \\[%{LOGLEVEL:log.level}\\] (?:\\[client %{IPORHOST:source.address}\\] )?%{GREEDYDATA:message}', + HTTPD24_ERRORLOG: + '\\[%{HTTPDERROR_DATE:timestamp}\\] \\[(?:%{WORD:apache.error.module})?:%{LOGLEVEL:log.level}\\] \\[pid %{POSINT:process.pid:long}(:tid %{INT:process.thread.id:int})?\\](?: \\(%{POSINT:apache.error.proxy.error.code?}\\)%{DATA:apache.error.proxy.error.message}:)?(?: \\[client %{IPORHOST:source.address}(?::%{POSINT:source.port:int})?\\])?(?: %{DATA:error.code}:)? %{GREEDYDATA:message}', + HTTPD_ERRORLOG: '%{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}', + COMMONAPACHELOG: '%{HTTPD_COMMONLOG}', + COMBINEDAPACHELOG: '%{HTTPD_COMBINEDLOG}', + JAVACLASS: '(?:[a-zA-Z$_][a-zA-Z$_0-9]*\\.)*[a-zA-Z$_][a-zA-Z$_0-9]*', + JAVAFILE: '(?:[a-zA-Z$_0-9. -]+)', + JAVAMETHOD: '(?:(<(?:cl)?init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)', + JAVASTACKTRACEPART: + '%{SPACE}at %{JAVACLASS:java.log.origin.class.name}\\.%{JAVAMETHOD:log.origin.function}\\(%{JAVAFILE:log.origin.file.name}(?::%{INT:log.origin.file.line:int})?\\)', + JAVATHREAD: '(?:[A-Z]{2}-Processor[\\d]+)', + JAVALOGMESSAGE: '(?:.*)', + CATALINA7_DATESTAMP: '%{MONTH} %{MONTHDAY}, %{YEAR} %{HOUR}:%{MINUTE}:%{SECOND} (?:AM|PM)', + CATALINA7_LOG: + '%{CATALINA7_DATESTAMP:timestamp} %{JAVACLASS:java.log.origin.class.name}(?: %{JAVAMETHOD:log.origin.function})?\\s*(?:%{LOGLEVEL:log.level}:)? %{JAVALOGMESSAGE:message}', + CATALINA8_DATESTAMP: '%{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}:%{SECOND}', + CATALINA8_LOG: + '%{CATALINA8_DATESTAMP:timestamp} %{LOGLEVEL:log.level} \\[%{DATA:java.log.origin.thread.name}\\] %{JAVACLASS:java.log.origin.class.name}\\.(?:%{JAVAMETHOD:log.origin.function})? %{JAVALOGMESSAGE:message}', + CATALINA_DATESTAMP: '(?:%{CATALINA8_DATESTAMP})|(?:%{CATALINA7_DATESTAMP})', + CATALINALOG: '(?:%{CATALINA8_LOG})|(?:%{CATALINA7_LOG})', + TOMCAT7_LOG: '%{CATALINA7_LOG}', + TOMCAT8_LOG: '%{CATALINA8_LOG}', + TOMCATLEGACY_DATESTAMP: + '%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}(?: %{ISO8601_TIMEZONE})?', + TOMCATLEGACY_LOG: + '%{TOMCATLEGACY_DATESTAMP:timestamp} \\| %{LOGLEVEL:log.level} \\| %{JAVACLASS:java.log.origin.class.name} - %{JAVALOGMESSAGE:message}', + TOMCAT_DATESTAMP: + '(?:%{CATALINA8_DATESTAMP})|(?:%{CATALINA7_DATESTAMP})|(?:%{TOMCATLEGACY_DATESTAMP})', + TOMCATLOG: '(?:%{TOMCAT8_LOG})|(?:%{TOMCAT7_LOG})|(?:%{TOMCATLEGACY_LOG})', + RT_FLOW_TAG: '(?:RT_FLOW_SESSION_CREATE|RT_FLOW_SESSION_CLOSE|RT_FLOW_SESSION_DENY)', + RT_FLOW_EVENT: 'RT_FLOW_TAG', + RT_FLOW1: + '%{RT_FLOW_TAG:juniper.srx.tag}: %{GREEDYDATA:juniper.srx.reason}: %{IP:source.ip}/%{INT:source.port:int}->%{IP:destination.ip}/%{INT:destination.port:int} %{DATA:juniper.srx.service_name} %{IP:source.nat.ip}/%{INT:source.nat.port:int}->%{IP:destination.nat.ip}/%{INT:destination.nat.port:int} (?:(?:None)|(?:%{DATA:juniper.srx.src_nat_rule_name})) (?:(?:None)|(?:%{DATA:juniper.srx.dst_nat_rule_name})) %{INT:network.iana_number} %{DATA:rule.name} %{DATA:observer.ingress.zone} %{DATA:observer.egress.zone} %{INT:juniper.srx.session_id} \\d+\\(%{INT:source.bytes:long}\\) \\d+\\(%{INT:destination.bytes:long}\\) %{INT:juniper.srx.elapsed_time:int} .*', + RT_FLOW2: + '%{RT_FLOW_TAG:juniper.srx.tag}: session created %{IP:source.ip}/%{INT:source.port:int}->%{IP:destination.ip}/%{INT:destination.port:int} %{DATA:juniper.srx.service_name} %{IP:source.nat.ip}/%{INT:source.nat.port:int}->%{IP:destination.nat.ip}/%{INT:destination.nat.port:int} (?:(?:None)|(?:%{DATA:juniper.srx.src_nat_rule_name})) (?:(?:None)|(?:%{DATA:juniper.srx.dst_nat_rule_name})) %{INT:network.iana_number} %{DATA:rule.name} %{DATA:observer.ingress.zone} %{DATA:observer.egress.zone} %{INT:juniper.srx.session_id} .*', + RT_FLOW3: + '%{RT_FLOW_TAG:juniper.srx.tag}: session denied %{IP:source.ip}/%{INT:source.port:int}->%{IP:destination.ip}/%{INT:destination.port:int} %{DATA:juniper.srx.service_name} %{INT:network.iana_number}\\(\\d\\) %{DATA:rule.name} %{DATA:observer.ingress.zone} %{DATA:observer.egress.zone} .*', + SYSLOG5424PRINTASCII: '[!-~]+', + SYSLOGBASE2: + '(?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp})(?: %{SYSLOGFACILITY})?(?: %{SYSLOGHOST:host.hostname})?(?: %{SYSLOGPROG}:)?', + SYSLOGPAMSESSION: + '%{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:system.auth.pam.module}\\(%{DATA:system.auth.pam.origin}\\): session %{WORD:system.auth.pam.session_state} for user %{USERNAME:user.name}(?: by %{GREEDYDATA})?', + CRON_ACTION: '[A-Z ]+', + CRONLOG: + '%{SYSLOGBASE} \\(%{USER:user.name}\\) %{CRON_ACTION:system.cron.action} \\(%{DATA:message}\\)', + SYSLOGLINE: '%{SYSLOGBASE2} %{GREEDYDATA:message}', + SYSLOG5424PRI: '<%{NONNEGINT:log.syslog.priority:int}>', + SYSLOG5424SD: '\\[%{DATA}\\]+', + SYSLOG5424BASE: + '%{SYSLOG5424PRI}%{NONNEGINT:system.syslog.version} +(?:-|%{TIMESTAMP_ISO8601:timestamp}) +(?:-|%{IPORHOST:host.hostname}) +(?:-|%{SYSLOG5424PRINTASCII:process.name}) +(?:-|%{POSINT:process.pid:int}) +(?:-|%{SYSLOG5424PRINTASCII:event.code}) +(?:-|%{SYSLOG5424SD:system.syslog.structured_data})?', + SYSLOG5424LINE: '%{SYSLOG5424BASE} +%{GREEDYDATA:message}', + MAVEN_VERSION: '(?:(\\d+)\\.)?(?:(\\d+)\\.)?(\\*|\\d+)(?:[.-](RELEASE|SNAPSHOT))?', + MCOLLECTIVE: + '., \\[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:process.pid:int}\\]%{SPACE}%{LOGLEVEL:log.level}', + MCOLLECTIVEAUDIT: '%{TIMESTAMP_ISO8601:timestamp}:', + MONGO_LOG: '%{SYSLOGTIMESTAMP:timestamp} \\[%{WORD:mongodb.component}\\] %{GREEDYDATA:message}', + MONGO_QUERY: '\\{ (?<={ ).*(?= } ntoreturn:) \\}', + MONGO_SLOWQUERY: + '%{WORD:mongodb.profile.op} %{MONGO_WORDDASH:mongodb.database}\\.%{MONGO_WORDDASH:mongodb.collection} %{WORD}: %{MONGO_QUERY:mongodb.query.original} ntoreturn:%{NONNEGINT:mongodb.profile.ntoreturn:int} ntoskip:%{NONNEGINT:mongodb.profile.ntoskip:int} nscanned:%{NONNEGINT:mongodb.profile.nscanned:int}.*? nreturned:%{NONNEGINT:mongodb.profile.nreturned:int}.*? %{INT:mongodb.profile.duration:int}ms', + MONGO_WORDDASH: '\\b[\\w-]+\\b', + MONGO3_SEVERITY: '\\w', + MONGO3_COMPONENT: '%{WORD}', + MONGO3_LOG: + '%{TIMESTAMP_ISO8601:timestamp} %{MONGO3_SEVERITY:log.level} (?:-|%{MONGO3_COMPONENT:mongodb.component})%{SPACE}(?:\\[%{DATA:mongodb.context}\\])? %{GREEDYDATA:message}', + NAGIOSTIME: '\\[%{NUMBER:timestamp}\\]', + NAGIOS_TYPE_CURRENT_SERVICE_STATE: 'CURRENT SERVICE STATE', + NAGIOS_TYPE_CURRENT_HOST_STATE: 'CURRENT HOST STATE', + NAGIOS_TYPE_SERVICE_NOTIFICATION: 'SERVICE NOTIFICATION', + NAGIOS_TYPE_HOST_NOTIFICATION: 'HOST NOTIFICATION', + NAGIOS_TYPE_SERVICE_ALERT: 'SERVICE ALERT', + NAGIOS_TYPE_HOST_ALERT: 'HOST ALERT', + NAGIOS_TYPE_SERVICE_FLAPPING_ALERT: 'SERVICE FLAPPING ALERT', + NAGIOS_TYPE_HOST_FLAPPING_ALERT: 'HOST FLAPPING ALERT', + NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT: 'SERVICE DOWNTIME ALERT', + NAGIOS_TYPE_HOST_DOWNTIME_ALERT: 'HOST DOWNTIME ALERT', + NAGIOS_TYPE_PASSIVE_SERVICE_CHECK: 'PASSIVE SERVICE CHECK', + NAGIOS_TYPE_PASSIVE_HOST_CHECK: 'PASSIVE HOST CHECK', + NAGIOS_TYPE_SERVICE_EVENT_HANDLER: 'SERVICE EVENT HANDLER', + NAGIOS_TYPE_HOST_EVENT_HANDLER: 'HOST EVENT HANDLER', + NAGIOS_TYPE_EXTERNAL_COMMAND: 'EXTERNAL COMMAND', + NAGIOS_TYPE_TIMEPERIOD_TRANSITION: 'TIMEPERIOD TRANSITION', + NAGIOS_EC_DISABLE_SVC_CHECK: 'DISABLE_SVC_CHECK', + NAGIOS_EC_ENABLE_SVC_CHECK: 'ENABLE_SVC_CHECK', + NAGIOS_EC_DISABLE_HOST_CHECK: 'DISABLE_HOST_CHECK', + NAGIOS_EC_ENABLE_HOST_CHECK: 'ENABLE_HOST_CHECK', + NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT: 'PROCESS_SERVICE_CHECK_RESULT', + NAGIOS_EC_PROCESS_HOST_CHECK_RESULT: 'PROCESS_HOST_CHECK_RESULT', + NAGIOS_EC_SCHEDULE_SERVICE_DOWNTIME: 'SCHEDULE_SERVICE_DOWNTIME', + NAGIOS_EC_SCHEDULE_HOST_DOWNTIME: 'SCHEDULE_HOST_DOWNTIME', + NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS: 'DISABLE_HOST_SVC_NOTIFICATIONS', + NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS: 'ENABLE_HOST_SVC_NOTIFICATIONS', + NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS: 'DISABLE_HOST_NOTIFICATIONS', + NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS: 'ENABLE_HOST_NOTIFICATIONS', + NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS: 'DISABLE_SVC_NOTIFICATIONS', + NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS: 'ENABLE_SVC_NOTIFICATIONS', + NAGIOS_WARNING: 'Warning:%{SPACE}%{GREEDYDATA:message}', + NAGIOS_CURRENT_SERVICE_STATE: + '%{NAGIOS_TYPE_CURRENT_SERVICE_STATE:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{DATA:nagios.log.state_type};%{INT:nagios.log.attempt:int};%{GREEDYDATA:message}', + NAGIOS_CURRENT_HOST_STATE: + '%{NAGIOS_TYPE_CURRENT_HOST_STATE:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{DATA:nagios.log.state_type};%{INT:nagios.log.attempt:int};%{GREEDYDATA:message}', + NAGIOS_SERVICE_NOTIFICATION: + '%{NAGIOS_TYPE_SERVICE_NOTIFICATION:nagios.log.type}: %{DATA:user.name};%{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{DATA:nagios.log.notification_command};%{GREEDYDATA:message}', + NAGIOS_HOST_NOTIFICATION: + '%{NAGIOS_TYPE_HOST_NOTIFICATION:nagios.log.type}: %{DATA:user.name};%{DATA:host.hostname};%{DATA:service.state};%{DATA:nagios.log.notification_command};%{GREEDYDATA:message}', + NAGIOS_SERVICE_ALERT: + '%{NAGIOS_TYPE_SERVICE_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{DATA:nagios.log.state_type};%{INT:nagios.log.attempt:int};%{GREEDYDATA:message}', + NAGIOS_HOST_ALERT: + '%{NAGIOS_TYPE_HOST_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{DATA:nagios.log.state_type};%{INT:nagios.log.attempt:int};%{GREEDYDATA:message}', + NAGIOS_SERVICE_FLAPPING_ALERT: + '%{NAGIOS_TYPE_SERVICE_FLAPPING_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{GREEDYDATA:message}', + NAGIOS_HOST_FLAPPING_ALERT: + '%{NAGIOS_TYPE_HOST_FLAPPING_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{GREEDYDATA:message}', + NAGIOS_SERVICE_DOWNTIME_ALERT: + '%{NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{GREEDYDATA:nagios.log.comment}', + NAGIOS_HOST_DOWNTIME_ALERT: + '%{NAGIOS_TYPE_HOST_DOWNTIME_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{GREEDYDATA:nagios.log.comment}', + NAGIOS_PASSIVE_SERVICE_CHECK: + '%{NAGIOS_TYPE_PASSIVE_SERVICE_CHECK:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{GREEDYDATA:nagios.log.comment}', + NAGIOS_PASSIVE_HOST_CHECK: + '%{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{GREEDYDATA:nagios.log.comment}', + NAGIOS_SERVICE_EVENT_HANDLER: + '%{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{DATA:nagios.log.state_type};%{DATA:nagios.log.event_handler_name}', + NAGIOS_HOST_EVENT_HANDLER: + '%{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{DATA:nagios.log.state_type};%{DATA:nagios.log.event_handler_name}', + NAGIOS_TIMEPERIOD_TRANSITION: + '%{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios.log.type}: %{DATA:service.name};%{NUMBER:nagios.log.period_from:int};%{NUMBER:nagios.log.period_to:int}', + NAGIOS_EC_LINE_DISABLE_SVC_CHECK: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_SVC_CHECK:nagios.log.command};%{DATA:host.hostname};%{DATA:service.name}', + NAGIOS_EC_LINE_DISABLE_HOST_CHECK: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_HOST_CHECK:nagios.log.command};%{DATA:host.hostname}', + NAGIOS_EC_LINE_ENABLE_SVC_CHECK: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_SVC_CHECK:nagios.log.command};%{DATA:host.hostname};%{DATA:service.name}', + NAGIOS_EC_LINE_ENABLE_HOST_CHECK: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_HOST_CHECK:nagios.log.command};%{DATA:host.hostname}', + NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT:nagios.log.command};%{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{GREEDYDATA:nagios.log.check_result}', + NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_PROCESS_HOST_CHECK_RESULT:nagios.log.command};%{DATA:host.hostname};%{DATA:service.state};%{GREEDYDATA:nagios.log.check_result}', + NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS:nagios.log.command};%{GREEDYDATA:host.hostname}', + NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS:nagios.log.command};%{GREEDYDATA:host.hostname}', + NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS:nagios.log.command};%{DATA:host.hostname};%{GREEDYDATA:service.name}', + NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS:nagios.log.command};%{GREEDYDATA:host.hostname}', + NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS:nagios.log.command};%{GREEDYDATA:host.hostname}', + NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS:nagios.log.command};%{DATA:host.hostname};%{GREEDYDATA:service.name}', + NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME: + '%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_SCHEDULE_HOST_DOWNTIME:nagios.log.command};%{DATA:host.hostname};%{NUMBER:nagios.log.start_time};%{NUMBER:nagios.log.end_time};%{NUMBER:nagios.log.fixed};%{NUMBER:nagios.log.trigger_id};%{NUMBER:nagios.log.duration:int};%{DATA:user.name};%{DATA:nagios.log.comment}', + NAGIOSLOGLINE: + '%{NAGIOSTIME} (?:%{NAGIOS_WARNING}|%{NAGIOS_CURRENT_SERVICE_STATE}|%{NAGIOS_CURRENT_HOST_STATE}|%{NAGIOS_SERVICE_NOTIFICATION}|%{NAGIOS_HOST_NOTIFICATION}|%{NAGIOS_SERVICE_ALERT}|%{NAGIOS_HOST_ALERT}|%{NAGIOS_SERVICE_FLAPPING_ALERT}|%{NAGIOS_HOST_FLAPPING_ALERT}|%{NAGIOS_SERVICE_DOWNTIME_ALERT}|%{NAGIOS_HOST_DOWNTIME_ALERT}|%{NAGIOS_PASSIVE_SERVICE_CHECK}|%{NAGIOS_PASSIVE_HOST_CHECK}|%{NAGIOS_SERVICE_EVENT_HANDLER}|%{NAGIOS_HOST_EVENT_HANDLER}|%{NAGIOS_TIMEPERIOD_TRANSITION}|%{NAGIOS_EC_LINE_DISABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_ENABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_DISABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_ENABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT}|%{NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT}|%{NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME}|%{NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS})', + POSTGRESQL: + '%{DATESTAMP:timestamp} %{TZ:event.timezone} %{DATA:user.name} %{GREEDYDATA:postgresql.log.connection_id} %{POSINT:process.pid:int}', + RUUID: '\\h{32}', + RCONTROLLER: '(?[^#]+)#(?\\w+)', + RAILS3HEAD: + '(?m)Started %{WORD:http.request.method} "%{URIPATHPARAM:url.original}" for %{IPORHOST:source.address} at (?%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE})', + RPROCESSING: + '\\W*Processing by %{RCONTROLLER} as (?\\S+)(?:\\W*Parameters: {%{DATA:rails.request.params}}\\W*)?', + RAILS3FOOT: + 'Completed %{POSINT:http.response.status_code:int}%{DATA} in %{NUMBER:rails.request.duration.total:float}ms %{RAILS3PROFILE}%{GREEDYDATA}', + RAILS3PROFILE: + '(?:\\(Views: %{NUMBER:rails.request.duration.view:float}ms \\| ActiveRecord: %{NUMBER:rails.request.duration.active_record:float}ms|\\(ActiveRecord: %{NUMBER:rails.request.duration.active_record:float}ms)?', + RAILS3: + '%{RAILS3HEAD}(?:%{RPROCESSING})?(?(?:%{DATA}\\n)*)(?:%{RAILS3FOOT})?', + REDISTIMESTAMP: '%{MONTHDAY} %{MONTH} %{TIME}', + REDISLOG: '\\[%{POSINT:process.pid:int}\\] %{REDISTIMESTAMP:timestamp} \\*', + REDISMONLOG: + '%{NUMBER:timestamp} \\[%{INT:redis.database.id} %{IP:client.ip}:%{POSINT:client.port:int}\\] "%{WORD:redis.command.name}"\\s?%{GREEDYDATA:redis.command.args}', + RUBY_LOGLEVEL: '(?:DEBUG|FATAL|ERROR|WARN|INFO)', + RUBY_LOGGER: + '[DFEWI], \\[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:process.pid:int}\\] *%{RUBY_LOGLEVEL:log.level} -- +%{DATA:process.name}: %{GREEDYDATA:message}', + SQUID3_STATUS: '(?:%{POSINT:http.response.status_code:int}|0|000)', + SQUID3: + '%{NUMBER:timestamp}\\s+%{NUMBER:squid.request.duration:int}\\s%{IP:source.ip}\\s%{WORD:event.action}/%{SQUID3_STATUS}\\s%{INT:http.response.bytes:long}\\s%{WORD:http.request.method}\\s%{NOTSPACE:url.original}\\s(?:-|%{NOTSPACE:user.name})\\s%{WORD:squid.hierarchy_code}/(?:-|%{IPORHOST:destination.address})\\s(?:-|%{NOTSPACE:http.response.mime_type})', + ZEEK_BOOL: '[TF]', + ZEEK_DATA: '[^\\t]+', + ZEEK_HTTP: + '%{NUMBER:timestamp}\\t%{NOTSPACE:zeek.session_id}\\t%{IP:source.ip}\\t%{INT:source.port:int}\\t%{IP:destination.ip}\\t%{INT:destination.port:int}\\t%{INT:zeek.http.trans_depth:int}\\t(?:-|%{WORD:http.request.method})\\t(?:-|%{ZEEK_DATA:url.domain})\\t(?:-|%{ZEEK_DATA:url.original})\\t(?:-|%{ZEEK_DATA:http.request.referrer})\\t(?:-|%{NUMBER:http.version})\\t(?:-|%{ZEEK_DATA:user_agent.original})\\t(?:-|%{ZEEK_DATA:zeek.http.origin})\\t(?:-|%{NUMBER:http.request.body.bytes:long})\\t(?:-|%{NUMBER:http.response.body.bytes:long})\\t(?:-|%{POSINT:http.response.status_code:int})\\t(?:-|%{DATA:zeek.http.status_msg})\\t(?:-|%{POSINT:zeek.http.info_code:int})\\t(?:-|%{DATA:zeek.http.info_msg})\\t(?:\\(empty\\)|%{ZEEK_DATA:zeek.http.tags})\\t(?:-|%{ZEEK_DATA:url.username})\\t(?:-|%{ZEEK_DATA:url.password})\\t(?:-|%{ZEEK_DATA:zeek.http.proxied})\\t(?:-|%{ZEEK_DATA:zeek.http.orig_fuids})\\t(?:-|%{ZEEK_DATA:zeek.http.orig_filenames})\\t(?:-|%{ZEEK_DATA:http.request.mime_type})\\t(?:-|%{ZEEK_DATA:zeek.http.resp_fuids})\\t(?:-|%{ZEEK_DATA:zeek.http.resp_filenames})\\t(?:-|%{ZEEK_DATA:http.response.mime_type})', + ZEEK_DNS: + '%{NUMBER:timestamp}\\t%{NOTSPACE:zeek.session_id}\\t%{IP:source.ip}\\t%{INT:source.port:int}\\t%{IP:destination.ip}\\t%{INT:destination.port:int}\\t%{WORD:network.transport}\\t(?:-|%{INT:dns.id:int})\\t(?:-|%{NUMBER:zeek.dns.rtt:float})\\t(?:-|%{ZEEK_DATA:dns.question.name})\\t(?:-|%{INT:zeek.dns.qclass:int})\\t(?:-|%{ZEEK_DATA:zeek.dns.qclass_name})\\t(?:-|%{INT:zeek.dns.qtype:int})\\t(?:-|%{ZEEK_DATA:dns.question.type})\\t(?:-|%{INT:zeek.dns.rcode:int})\\t(?:-|%{ZEEK_DATA:dns.response_code})\\t%{ZEEK_BOOL:zeek.dns.AA}\\t%{ZEEK_BOOL:zeek.dns.TC}\\t%{ZEEK_BOOL:zeek.dns.RD}\\t%{ZEEK_BOOL:zeek.dns.RA}\\t%{NONNEGINT:zeek.dns.Z:int}\\t(?:-|%{ZEEK_DATA:zeek.dns.answers})\\t(?:-|%{DATA:zeek.dns.TTLs})\\t(?:-|%{ZEEK_BOOL:zeek.dns.rejected})', + ZEEK_CONN: + '%{NUMBER:timestamp}\\t%{NOTSPACE:zeek.session_id}\\t%{IP:source.ip}\\t%{INT:source.port:int}\\t%{IP:destination.ip}\\t%{INT:destination.port:int}\\t%{WORD:network.transport}\\t(?:-|%{ZEEK_DATA:network.protocol})\\t(?:-|%{NUMBER:zeek.connection.duration:float})\\t(?:-|%{INT:zeek.connection.orig_bytes:long})\\t(?:-|%{INT:zeek.connection.resp_bytes:long})\\t(?:-|%{ZEEK_DATA:zeek.connection.state})\\t(?:-|%{ZEEK_BOOL:zeek.connection.local_orig})\\t(?:-|%{ZEEK_BOOL:zeek.connection.local_resp})\\t(?:-|%{INT:zeek.connection.missed_bytes:long})\\t(?:-|%{ZEEK_DATA:zeek.connection.history})\\t(?:-|%{INT:source.packets:long})\\t(?:-|%{INT:source.bytes:long})\\t(?:-|%{INT:destination.packets:long})\\t(?:-|%{INT:destination.bytes:long})\\t(?:-|%{ZEEK_DATA:zeek.connection.tunnel_parents})(?:\\t(?:-|%{COMMONMAC:source.mac})\\t(?:-|%{COMMONMAC:destination.mac}))?', + ZEEK_FILES_TX_HOSTS: + '(?:-|%{IP:server.ip})|(?%{IP:server.ip}(?:\\s,%{IP})+)', + ZEEK_FILES_RX_HOSTS: + '(?:-|%{IP:client.ip})|(?%{IP:client.ip}(?:\\s,%{IP})+)', + ZEEK_FILES: + '%{NUMBER:timestamp}\\t%{NOTSPACE:zeek.files.fuid}\\t%{ZEEK_FILES_TX_HOSTS}\\t%{ZEEK_FILES_RX_HOSTS}\\t(?:-|%{ZEEK_DATA:zeek.files.session_ids})\\t(?:-|%{ZEEK_DATA:zeek.files.source})\\t(?:-|%{INT:zeek.files.depth:int})\\t(?:-|%{ZEEK_DATA:zeek.files.analyzers})\\t(?:-|%{ZEEK_DATA:file.mime_type})\\t(?:-|%{ZEEK_DATA:file.name})\\t(?:-|%{NUMBER:zeek.files.duration:float})\\t(?:-|%{ZEEK_DATA:zeek.files.local_orig})\\t(?:-|%{ZEEK_BOOL:zeek.files.is_orig})\\t(?:-|%{INT:zeek.files.seen_bytes:long})\\t(?:-|%{INT:file.size:long})\\t(?:-|%{INT:zeek.files.missing_bytes:long})\\t(?:-|%{INT:zeek.files.overflow_bytes:long})\\t(?:-|%{ZEEK_BOOL:zeek.files.timedout})\\t(?:-|%{ZEEK_DATA:zeek.files.parent_fuid})\\t(?:-|%{ZEEK_DATA:file.hash.md5})\\t(?:-|%{ZEEK_DATA:file.hash.sha1})\\t(?:-|%{ZEEK_DATA:file.hash.sha256})\\t(?:-|%{ZEEK_DATA:zeek.files.extracted})(?:\\t(?:-|%{ZEEK_BOOL:zeek.files.extracted_cutoff})\\t(?:-|%{INT:zeek.files.extracted_size:long}))?', +}; diff --git a/src/platform/packages/shared/kbn-grok-ui/index.ts b/src/platform/packages/shared/kbn-grok-ui/index.ts new file mode 100644 index 0000000000000..20253a73d8d67 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export * from './components'; diff --git a/src/platform/packages/shared/kbn-grok-ui/jest.config.js b/src/platform/packages/shared/kbn-grok-ui/jest.config.js new file mode 100644 index 0000000000000..8fe3054fc5457 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/jest.config.js @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../../..', + roots: ['/src/platform/packages/shared/kbn-grok-ui'], +}; diff --git a/src/platform/packages/shared/kbn-grok-ui/kibana.jsonc b/src/platform/packages/shared/kbn-grok-ui/kibana.jsonc new file mode 100644 index 0000000000000..8f3a5ce922e75 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/kibana.jsonc @@ -0,0 +1,7 @@ +{ + "type": "shared-common", + "id": "@kbn/grok-ui", + "owner": "@elastic/streams-program-team", + "group": "platform", + "visibility": "shared" +} diff --git a/src/platform/packages/shared/kbn-grok-ui/models/draft_grok_expression.ts b/src/platform/packages/shared/kbn-grok-ui/models/draft_grok_expression.ts new file mode 100644 index 0000000000000..c82b494eb7bca --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/models/draft_grok_expression.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { GrokCollection, GrokPattern } from './grok_collection_and_pattern'; + +export class DraftGrokExpression { + private expression: string | undefined = undefined; + private grokPattern: GrokPattern; + + constructor(collection: GrokCollection, expression?: string) { + this.grokPattern = new GrokPattern(expression || '', 'DRAFT_GROK_EXPRESSION', collection); + this.grokPattern.resolvePattern(); + } + + public updateExpression = (expression: string) => { + this.expression = expression; + this.grokPattern.updatePattern(this.expression); + this.grokPattern.resolvePattern(true); + }; + + public parse = (samples: string[]) => { + return this.grokPattern.parse(samples, true); + }; + + public getRegex = () => { + return this.grokPattern.getRegex(); + }; + + public getRegexPattern = () => { + return this.grokPattern.getRegexPattern(); + }; + + public getFields = () => { + return this.grokPattern.getFields(); + }; +} diff --git a/src/platform/packages/shared/kbn-grok-ui/models/grok_collection_and_pattern.ts b/src/platform/packages/shared/kbn-grok-ui/models/grok_collection_and_pattern.ts new file mode 100644 index 0000000000000..bc5a84d9eb5b6 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/models/grok_collection_and_pattern.ts @@ -0,0 +1,393 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +/* eslint-disable max-classes-per-file */ + +import { toRegExp, toRegExpDetails } from 'oniguruma-to-es'; +import { monaco } from '@kbn/monaco'; +import { v4 as uuidv4 } from 'uuid'; +import { unflattenObject } from '@kbn/object-utils'; +import { euiPaletteColorBlindBehindText } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +// Grok patterns use this official naming: %{SYNTAX:SEMANTIC:TYPE} + +// Will match %{SYNTAX}, %{SYNTAX:SEMANTIC}, %{SYNTAX:SEMANTIC:TYPE}, and support special characters and dots. +const SUBPATTERNS_REGEX = + /%\{[A-Z0-9_@#$%&*+=\-\.]+(?::[A-Za-z0-9_@#$%&*+=\-\.]+)?(?::[A-Za-z]+)?\}/g; + +// Matches "manual" semantic names in the expression, these are user defined capture groups, e.g. (?the pattern here) +const NESTED_FIELD_NAMES_REGEX = + /(\(\?<([A-Za-z0-9_@#$%&*+=\-\.]+)(?::([A-Za-z0-9_@#$%&*+=\-\.]+))?(?::([A-Za-z]+))?>)|\(\?:|\(\?>|\(\?!|\(\? = new Map(); + private patternKeys: string[] = []; + // NOTE: This doesn't subscribe to EUI_VIS_COLOR_STORE changes at the moment, whilst UI / UX is being finalised. + private colourPalette = euiPaletteColorBlindBehindText({ rotations: 3 }); + private colourIndex = 0; + + public getPattern(id: string) { + return this.patterns.get(id); + } + + public addPattern(id: string, rawPattern: string) { + if (this.patterns.has(id)) { + // eslint-disable-next-line no-console + console.warn('Warning: pattern with ID: %s already exists', id); + } else { + const pattern = new GrokPattern(rawPattern, id, this); + this.patterns.set(id, pattern); + + return pattern; + } + } + + public resolvePatterns() { + this.patterns.forEach((pattern) => { + if (!pattern.isResolved()) { + pattern.resolvePattern(); + } + }); + this.patternKeys = Array.from(this.patterns.keys()); + } + + // Only relevant for Monaco users. + // Can be used with Monaco code editor to provide suggestions. + public getSuggestionProvider = () => { + const provider: monaco.languages.CompletionItemProvider = { + triggerCharacters: ['{'], + provideCompletionItems: (model, position, context, token) => { + const wordUntil = model.getWordUntilPosition(position); + const lineContent = model?.getLineContent(position.lineNumber); + // A relatively simple implementation: when typing %{ the } will autocomplete, so if the next character is } we will provide suggestions. + // Could potentially be more robust. + const nextCharacterIsClosingPatternBracket = + lineContent?.charAt(position.column - 1) === '}'; + + const matchingPatterns = this.patternKeys.filter((key) => + key.toLowerCase().startsWith(wordUntil.word.toLowerCase()) + ); + + const wordRange = new monaco.Range( + position.lineNumber, + wordUntil.startColumn, + position.lineNumber, + wordUntil.endColumn + ); + + const suggestions = matchingPatterns.map((key) => { + return { + label: key, + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: key + (nextCharacterIsClosingPatternBracket ? '' : '}'), + range: wordRange, + }; + }); + + return { + suggestions, + }; + }, + }; + return provider; + }; + + public getColour = () => { + // Loop back to 0 once at the end of the rotations + this.colourIndex = this.colourIndex + 1 <= this.colourPalette.length ? this.colourIndex + 1 : 0; + return this.colourPalette[this.colourIndex]; + }; + + // Only relevant for Monaco users. + // Monaco doesn't support dynamic inline styles, so we need to generate static styles for the colour palette. + public getColourPaletteStyles = () => { + const styles: Record = {}; + for (let $i = 0; $i < this.colourPalette.length; $i++) { + const colour = this.colourPalette[$i]; + const colourWithoutHash = colour.substring(1); + styles[`.grok-pattern-match-${colourWithoutHash}`] = { + backgroundColor: colour, + cursor: 'pointer', + }; + } + return styles; + }; + + public resetColourIndex = () => { + this.colourIndex = 0; + }; +} + +export class GrokPattern { + // The raw pattern, this might be a direct Oniguruma expression, or an expression that contains Grok subpatterns. + // E.g. INT (?:[+-]?(?:[0-9]+)) or MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) + private rawPattern: string; + // Primarily used for mapping capture groups to names / types + private fields; + // The resolved pattern is the raw pattern converted to Oniguruma regex. This includes sub patterns and field names being converted. + private resolvedPattern: string | null = null; + // The regexp is the Oniguruama regex pattern converted to a JS regular expression. + // This will be undefined if the pattern hasn't been resolved / regex hasn't been generated. It will be null if the regex generation failed. + private regexp: RegExp | null | undefined = undefined; + private parentCollection: GrokCollection; + + constructor(rawPattern: string, id: string, collection: GrokCollection) { + // These are keyed to match the regex capturing groups keys, which will be a randomly generated ID. + this.fields = new Map(); + this.rawPattern = rawPattern; + this.parentCollection = collection; + } + + public isResolved() { + return this.resolvedPattern !== null; + } + + public resolvePattern = (forceResolve = false) => { + if (!forceResolve && this.isResolved()) { + return this.resolvedPattern; + } + + this.parentCollection.resetColourIndex(); + this.fields.clear(); + this.resolveSubPatterns(); + this.resolveFieldNames(); + return this.resolvedPattern; + }; + + private resolveSubPatterns = () => { + let rawPattern = this.rawPattern; + const subPatterns = rawPattern.match(SUBPATTERNS_REGEX) || []; + + subPatterns.forEach((matched) => { + // Matched will either be %{SYNTAX}, %{SYNTAX:SEMANTIC}, or %{SYNTAX:SEMANTIC:TYPE} + // Removes %{ } + const withBracketsRemoved = matched.substring(2, matched.length - 1); + const elements = withBracketsRemoved.split(':'); + + // Syntax e.g. INT + const subPatternName = elements[0]; + + // Semantic e.g. mynumber (optional) + const fieldName = elements[1]; + + // Type e.g float (optional) + const fieldType = elements[2]; + + const subPattern = this.parentCollection.getPattern(subPatternName); + + if (!subPattern) { + return; + } + + if (!subPattern.isResolved()) { + subPattern.resolvePattern(); + } + + // Only some patterns will have a semantic / field name, other patterns will be matched but are not captured / part of the structured output. + // The replacements will take something like ${WORD} and replace it with a resolved Oniguruma pattern, e.g. \b\w+\b + if (fieldName) { + // We generate a unique ID for the capture group, this is used to map the capture group to the field metadata. + // Capture groups, for instance, do not support special characters or dots in the name in JavaScript's regex implementation, but this is allowed in Grok. + const generatedId = getGeneratedCaptureGroupId(); + + // As we want to track this semantic / field result we also prefix with a named capture group. + rawPattern = rawPattern.replace( + matched, + '(?<' + generatedId + '>' + subPattern.resolvedPattern! + ')' + ); + const fieldEntry = { + name: fieldName, + type: fieldType && SUPPORTED_TYPE_CONVERSIONS.includes(fieldType) ? fieldType : null, + colour: this.parentCollection.getColour(), + pattern: matched, + }; + this.fields.set(generatedId, fieldEntry); + } else { + // This will form part of the overal pattern and part of the "continuous" match, but the result won't be captured / part of the structured output. + rawPattern = rawPattern.replace(matched, subPattern.resolvedPattern!); + } + }); + + this.resolvedPattern = rawPattern; + }; + + // Resolve manual semantic / field names in the expression provided by capture groups, e.g.: (?[0-9A-F]{10,11}) + private resolveFieldNames = () => { + if (!this.resolvedPattern) { + return; + } + + let nestLevel = 0; + let inRangeDef = 0; + let matched; + + while ((matched = NESTED_FIELD_NAMES_REGEX.exec(this.resolvedPattern)) !== null) { + switch (matched[0]) { + case '(': { + if (!inRangeDef) { + nestLevel = nestLevel + 1; + } + break; + } + case '\\(': + break; // can be ignored + case '\\)': + break; // can be ignored + case ')': { + if (!inRangeDef) { + nestLevel = nestLevel - 1; + } + break; + } + case '[': { + ++inRangeDef; + break; + } + case '\\[': + break; // can be ignored + case '\\]': + break; // can be ignored + case ']': { + --inRangeDef; + break; + } + case '(?:': // fallthrough // group not captured + case '(?>': // fallthrough // atomic group + case '(?!': // fallthrough // negative look-ahead + case '(? [2] is queue_id, [3] is mything (optional), [4] is int (optional) + const generatedId = getGeneratedCaptureGroupId(); + + // This check is so we don't reprocess the field name replacements from resolveSubPatterns + if (!matched[2].includes('_____GENERATED_CAPTURE_GROUP_____')) { + this.fields.set(generatedId, { + name: matched[3] ?? matched[2], + type: + matched[4] && SUPPORTED_TYPE_CONVERSIONS.includes(matched[4]) ? matched[4] : null, + colour: this.parentCollection.getColour(), + pattern: `${CUSTOM_NAMED_CAPTURE_PATTERN_PREFIX} ` + String.raw`${matched[1]}`, + }); + + // (? becomes (? + this.resolvedPattern = this.resolvedPattern.replace( + matched.input, + matched.input.replace(matched[2], generatedId) + ); + } + + break; + } + } + } + }; + + public parse = (samples: string[], regenerateRegex = false) => { + if (!this.isResolved()) { + // eslint-disable-next-line no-console + console.warn('Pattern must be resolved before it can be parsed'); + } + + if (regenerateRegex || !this.regexp) { + this.getRegex(); + } + if (this.regexp) { + return samples.map((sample) => { + const results = this.regexp?.exec(sample); + // Takes our groups, which are keyed by our generated IDs, then we can map these results back to the real metadata we've stored under the fields property. + // E.g. real names and real types. + if (results?.groups) { + return unflattenObject( + Object.entries(results.groups).reduce>( + (acc, [key, value]) => { + const field = this.fields.get(key); + if (!field) return acc; + if (field && field.type === 'int') { + acc[field.name] = parseInt(value, 10); + } else if (field && field.type === 'float') { + acc[field.name] = parseFloat(value); + } else { + acc[field.name] = value; + } + return acc; + }, + {} + ) + ); + } else { + return {}; + } + }); + } else { + return []; + } + }; + + public getRegex = () => { + if (!this.isResolved()) { + this.resolvePattern(); + } + + try { + this.regexp = toRegExp(this.resolvedPattern!); + } catch { + this.regexp = null; + } + + return this.regexp; + }; + + public getRegexPattern = () => { + if (!this.isResolved()) { + this.resolvePattern(); + } + + try { + const regexpDetails = toRegExpDetails(this.resolvedPattern!); + return regexpDetails.pattern; + } catch { + return null; + } + }; + + public updatePattern = (rawPattern: string) => { + this.rawPattern = rawPattern; + this.resolvedPattern = null; + this.regexp = undefined; + }; + + public getFields = () => { + return this.fields; + }; +} + +const getGeneratedCaptureGroupId = () => { + // Named capture groups support underscores, so we replace hyphens with underscores. + // Named capture groups must start with a string, not a number, so we prefix with a letter. p_ here just stands for prefix. + return `p_${uuidv4().replaceAll('-', '_')}${CAPTURE_GROUP_GENERATED_ID_SUFFIX}`; +}; diff --git a/src/platform/packages/shared/kbn-grok-ui/models/grok_models.test.ts b/src/platform/packages/shared/kbn-grok-ui/models/grok_models.test.ts new file mode 100644 index 0000000000000..589d5468b0309 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/models/grok_models.test.ts @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { PATTERN_MAP } from '../constants/pattern_map'; +import { DraftGrokExpression } from './draft_grok_expression'; +import { GrokCollection } from './grok_collection_and_pattern'; + +let grokCollection: GrokCollection; +let draftGrokExpression: DraftGrokExpression; + +describe('Grok models', () => { + beforeAll(() => { + grokCollection = new GrokCollection(); + Object.entries(PATTERN_MAP).forEach(([key, value]) => { + grokCollection.addPattern(key, String.raw`${value}`); + }); + grokCollection.resolvePatterns(); + draftGrokExpression = new DraftGrokExpression(grokCollection); + }); + + it('should parse a simple pattern', () => { + draftGrokExpression.updateExpression('%{NUMBER:bytes} %{NUMBER:response}'); + const parsed = draftGrokExpression.parse(['1234 200']); + expect(parsed).toEqual([{ bytes: '1234', response: '200' }]); + }); + + it('should parse a pattern with a custom pattern', () => { + draftGrokExpression.updateExpression( + '%{NUMBER:bytes} %{LOGLEVEL:level} %{NUMBER:response} (?[0-9A-F]{10,11})' + ); + const parsed = draftGrokExpression.parse(['1234 WARN 200 5555555555']); + expect(parsed).toEqual([ + { bytes: '1234', level: 'WARN', response: '200', queueId: '5555555555' }, + ]); + }); + + it('Should parse multiple samples', () => { + draftGrokExpression.updateExpression('%{WORD:test} %{NUMBER:response}'); + const parsed = draftGrokExpression.parse(['Hello 200', 'Another 404']); + expect(parsed).toEqual([ + { test: 'Hello', response: '200' }, + { test: 'Another', response: '404' }, + ]); + }); + + it('Should parse a pattern using types', () => { + draftGrokExpression.updateExpression( + '%{NUMBER:notype} %{NUMBER:bytes:int} %{NUMBER:large:float}' + ); + const parsed = draftGrokExpression.parse(['578 1234.2323 20034343434.2323']); + expect(parsed).toEqual([{ notype: '578', bytes: 1234, large: 20034343434.2323 }]); + }); + + it('Should parse a pattern with special characters in the semantic', () => { + draftGrokExpression.updateExpression('%{WORD:@someword}'); + const parsed = draftGrokExpression.parse(['test']); + expect(parsed).toEqual([{ '@someword': 'test' }]); + }); + + it('Should support nested semantic names', () => { + draftGrokExpression.updateExpression( + '%{WORD:log.level} %{WORD:log.other} %{WORD:log.extra} %{WORD:log.nested.test}' + ); + const parsed = draftGrokExpression.parse(['level other extra nestedTest']); + expect(parsed).toEqual([ + { log: { level: 'level', other: 'other', extra: 'extra', nested: { test: 'nestedTest' } } }, + ]); + }); + + describe('Should parse complex patterns', () => { + it('Example one', () => { + draftGrokExpression.updateExpression( + String.raw`^\"(?[^\"]+)\" \| %{IPORHOST:clientip} (?:-|%{IPORHOST:forwardedfor}) (?:-|%{USER:ident}) (?:-|%{USER:auth}) \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|-)\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes})` + ); + const parsed = draftGrokExpression.parse([ + `"uRzbUwp5eZgAAAAaqIAAAAAa" | 5.3.2.1 - - - [24/Feb/2013:13:40:51 +0100] "GET /cpc HTTP/1.1" 302 -`, + `"URzbTwp5eZgAAAAWlbUAAAAV" | 4.3.2.7 - - - [14/Feb/2013:13:40:47 +0100] "GET /cpc/finish.do?cd=true&mea_d=0&targetPage=%2Fcpc%2F HTTP/1.1" 200 5264`, + `"URzbUwp5eZgAAAAaqIEAAAAa" | 4.3.2.1 - - - [14/Feb/2013:13:40:51 +0100] "GET /cpc/ HTTP/1.1" 402 -`, + `"URzbUwp5eZgAAAAWlbYAAAAV" | 4.3.2.1 - - - [14/Feb/2013:13:40:51 +0100] "POST /cpc/ HTTP/1.1" 305 - `, + ]); + + expect(parsed).toEqual([ + { + rid: 'uRzbUwp5eZgAAAAaqIAAAAAa', + clientip: '5.3.2.1', + timestamp: '24/Feb/2013:13:40:51 +0100', + verb: 'GET', + request: '/cpc', + httpversion: '1.1', + response: 302, + }, + { + rid: 'URzbTwp5eZgAAAAWlbUAAAAV', + clientip: '4.3.2.7', + timestamp: '14/Feb/2013:13:40:47 +0100', + verb: 'GET', + request: '/cpc/finish.do?cd=true&mea_d=0&targetPage=%2Fcpc%2F', + httpversion: '1.1', + response: 200, + bytes: '5264', + }, + { + rid: 'URzbUwp5eZgAAAAaqIEAAAAa', + clientip: '4.3.2.1', + timestamp: '14/Feb/2013:13:40:51 +0100', + verb: 'GET', + request: '/cpc/', + httpversion: '1.1', + response: 402, + }, + { + rid: 'URzbUwp5eZgAAAAWlbYAAAAV', + clientip: '4.3.2.1', + timestamp: '14/Feb/2013:13:40:51 +0100', + verb: 'POST', + request: '/cpc/', + httpversion: '1.1', + response: 305, + }, + ]); + }); + + it('Example two', () => { + draftGrokExpression.updateExpression( + String.raw`^\[(?%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR})\]\s+(\[%{WORD:loglevel}\]\s+)?%{GREEDYDATA:message}` + ); + const parsed = draftGrokExpression.parse([ + `[Thu Nov 01 21:54:03 2012] [error] [client 1.2.3.4] File does not exist: /usr/local/apache2/htdocs/default/cpc`, + `[Thu Nov 01 21:56:32 2012] [error] (146)Connection refused: proxy: AJP: attempt to connect to 1.2.3.4:8080 (dev1) failed`, + `[Thu Nov 01 21:56:32 2012] [error] ap_proxy_connect_backend disabling worker for (dev1)`, + `[Thu Nov 01 21:56:32 2012] [error] proxy: AJP: failed to make connection to backend: dev1`, + `[Thu Nov 01 21:56:35 2012] [error] (146)Connection refused: proxy: AJP: attempt to connect to 1.2.3.4:8012 (dev1) failed`, + `[Thu Nov 01 21:56:35 2012] [error] ap_proxy_connect_backend disabling worker for (dev1)`, + `[Thu Nov 01 21:56:35 2012] [error] proxy: AJP: failed to make connection to backend: dev1`, + ]); + + expect(parsed).toEqual([ + { + timestamp: 'Thu Nov 01 21:54:03 2012', + loglevel: 'error', + message: '[client 1.2.3.4] File does not exist: /usr/local/apache2/htdocs/default/cpc', + }, + { + timestamp: 'Thu Nov 01 21:56:32 2012', + loglevel: 'error', + message: + '(146)Connection refused: proxy: AJP: attempt to connect to 1.2.3.4:8080 (dev1) failed', + }, + { + timestamp: 'Thu Nov 01 21:56:32 2012', + loglevel: 'error', + message: 'ap_proxy_connect_backend disabling worker for (dev1)', + }, + { + timestamp: 'Thu Nov 01 21:56:32 2012', + loglevel: 'error', + message: 'proxy: AJP: failed to make connection to backend: dev1', + }, + { + timestamp: 'Thu Nov 01 21:56:35 2012', + loglevel: 'error', + message: + '(146)Connection refused: proxy: AJP: attempt to connect to 1.2.3.4:8012 (dev1) failed', + }, + { + timestamp: 'Thu Nov 01 21:56:35 2012', + loglevel: 'error', + message: 'ap_proxy_connect_backend disabling worker for (dev1)', + }, + { + timestamp: 'Thu Nov 01 21:56:35 2012', + loglevel: 'error', + message: 'proxy: AJP: failed to make connection to backend: dev1', + }, + ]); + }); + }); +}); diff --git a/src/platform/packages/shared/kbn-grok-ui/models/index.ts b/src/platform/packages/shared/kbn-grok-ui/models/index.ts new file mode 100644 index 0000000000000..f68d9ef53d4ff --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/models/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export * from './grok_collection_and_pattern'; +export * from './draft_grok_expression'; diff --git a/src/platform/packages/shared/kbn-grok-ui/package.json b/src/platform/packages/shared/kbn-grok-ui/package.json new file mode 100644 index 0000000000000..8392b52c06448 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/grok-ui", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0" +} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/aws b/src/platform/packages/shared/kbn-grok-ui/patterns/aws new file mode 100644 index 0000000000000..35d1467adce08 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/aws @@ -0,0 +1,28 @@ +S3_REQUEST_LINE (?:%{WORD:http.request.method} %{NOTSPACE:url.original}(?: HTTP/%{NUMBER:http.version})?) + +S3_ACCESS_LOG %{WORD:aws.s3access.bucket_owner} %{NOTSPACE:aws.s3access.bucket} \%{HTTPDATE:timestamp}\ (?:-|%{IP:client.ip}) (?:-|%{NOTSPACE:client.user.id}) %{NOTSPACE:aws.s3access.request_id} %{NOTSPACE:aws.s3access.operation} (?:-|%{NOTSPACE:aws.s3access.key}) (?:-|"%{S3_REQUEST_LINE:aws.s3access.request_uri}") (?:-|%{INT:http.response.status_code:int}) (?:-|%{NOTSPACE:aws.s3access.error_code}) (?:-|%{INT:aws.s3access.bytes_sent:long}) (?:-|%{INT:aws.s3access.object_size:long}) (?:-|%{INT:aws.s3access.total_time:int}) (?:-|%{INT:aws.s3access.turn_around_time:int}) "(?:-|%{DATA:http.request.referrer})" "(?:-|%{DATA:user_agent.original})" (?:-|%{NOTSPACE:aws.s3access.version_id})(?: (?:-|%{NOTSPACE:aws.s3access.host_id}) (?:-|%{NOTSPACE:aws.s3access.signature_version}) (?:-|%{NOTSPACE:tls.cipher}) (?:-|%{NOTSPACE:aws.s3access.authentication_type}) (?:-|%{NOTSPACE:aws.s3access.host_header}) (?:-|%{NOTSPACE:aws.s3access.tls_version}))? +# :long - %{INT:aws.s3access.bytes_sent:int} +# :long - %{INT:aws.s3access.object_size:int} + +ELB_URIHOST %{IPORHOST:url.domain}(?::%{POSINT:url.port:int})? +ELB_URIPATHQUERY %{URIPATH:url.path}(?:\?%{URIQUERY:url.query})? +# deprecated - old name: +ELB_URIPATHPARAM %{ELB_URIPATHQUERY} +ELB_URI %{URIPROTO:url.scheme}://(?:%{USER:url.username}(?::^@*)?@)?(?:%{ELB_URIHOST})?(?:%{ELB_URIPATHQUERY})? + +ELB_REQUEST_LINE (?:%{WORD:http.request.method} %{ELB_URI:url.original}(?: HTTP/%{NUMBER:http.version})?) + +# pattern supports 'regular' HTTP ELB format +ELB_V1_HTTP_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:aws.elb.name} %{IP:source.ip}:%{INT:source.port:int} (?:-|(?:%{IP:aws.elb.backend.ip}:%{INT:aws.elb.backend.port:int})) (?:-1|%{NUMBER:aws.elb.request_processing_time.sec:float}) (?:-1|%{NUMBER:aws.elb.backend_processing_time.sec:float}) (?:-1|%{NUMBER:aws.elb.response_processing_time.sec:float}) %{INT:http.response.status_code:int} (?:-|%{INT:aws.elb.backend.http.response.status_code:int}) %{INT:http.request.body.bytes:long} %{INT:http.response.body.bytes:long} "%{ELB_REQUEST_LINE}"(?: "(?:-|%{DATA:user_agent.original})" (?:-|%{NOTSPACE:tls.cipher}) (?:-|%{NOTSPACE:aws.elb.ssl_protocol}))? +# :long - %{INT:http.request.body.bytes:int} +# :long - %{INT:http.response.body.bytes:int} + +ELB_ACCESS_LOG %{ELB_V1_HTTP_LOG} + +# pattern used to match a shorted format, that's why we have the optional part (starting with *http.version*) at the end +CLOUDFRONT_ACCESS_LOG (?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:aws.cloudfront.x_edge_location}\t(?:-|%{INT:destination.bytes:long})\t%{IPORHOST:source.ip}\t%{WORD:http.request.method}\t%{HOSTNAME:url.domain}\t%{NOTSPACE:url.path}\t(?:(?:000)|%{INT:http.response.status_code:int})\t(?:-|%{DATA:http.request.referrer})\t%{DATA:user_agent.original}\t(?:-|%{DATA:url.query})\t(?:-|%{DATA:aws.cloudfront.http.request.cookie})\t%{WORD:aws.cloudfront.x_edge_result_type}\t%{NOTSPACE:aws.cloudfront.x_edge_request_id}\t%{HOSTNAME:aws.cloudfront.http.request.host}\t%{URIPROTO:network.protocol}\t(?:-|%{INT:source.bytes:long})\t%{NUMBER:aws.cloudfront.time_taken:float}\t(?:-|%{IP:network.forwarded_ip})\t(?:-|%{DATA:aws.cloudfront.ssl_protocol})\t(?:-|%{NOTSPACE:tls.cipher})\t%{WORD:aws.cloudfront.x_edge_response_result_type}(?:\t(?:-|HTTP/%{NUMBER:http.version})\t(?:-|%{DATA:aws.cloudfront.fle_status})\t(?:-|%{DATA:aws.cloudfront.fle_encrypted_fields})\t%{INT:source.port:int}\t%{NUMBER:aws.cloudfront.time_to_first_byte:float}\t(?:-|%{DATA:aws.cloudfront.x_edge_detailed_result_type})\t(?:-|%{NOTSPACE:http.request.mime_type})\t(?:-|%{INT:aws.cloudfront.http.request.size:long})\t(?:-|%{INT:aws.cloudfront.http.request.range.start:long})\t(?:-|%{INT:aws.cloudfront.http.request.range.end:long}))? +# :long - %{INT:destination.bytes:int} +# :long - %{INT:source.bytes:int} +# :long - %{INT:aws.cloudfront.http.request.size:int} +# :long - %{INT:aws.cloudfront.http.request.range.start:int} +# :long - %{INT:aws.cloudfront.http.request.range.end:int} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/bacula b/src/platform/packages/shared/kbn-grok-ui/patterns/bacula new file mode 100644 index 0000000000000..169defdecea3a --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/bacula @@ -0,0 +1,53 @@ +BACULA_TIMESTAMP %{MONTHDAY}-%{MONTH}(?:-%{YEAR})? %{HOUR}:%{MINUTE} +BACULA_HOST %{HOSTNAME} +BACULA_VOLUME %{USER} +BACULA_DEVICE %{USER} +BACULA_DEVICEPATH %{UNIXPATH} +BACULA_CAPACITY %{INT}{1,3}(,%{INT}{3})* +BACULA_VERSION %{USER} +BACULA_JOB %{USER} + +BACULA_LOG_MAX_CAPACITY User defined maximum volume capacity %{BACULA_CAPACITY:bacula.volume.max_capacity} exceeded on device \"%{BACULA_DEVICE:bacula.volume.device}\" \(%{BACULA_DEVICEPATH:bacula.volume.path}\).? +BACULA_LOG_END_VOLUME End of medium on Volume \"%{BACULA_VOLUME:bacula.volume.name}\" Bytes=%{BACULA_CAPACITY:bacula.volume.bytes} Blocks=%{BACULA_CAPACITY:bacula.volume.blocks} at %{BACULA_TIMESTAMP:bacula.timestamp}. +BACULA_LOG_NEW_VOLUME Created new Volume \"%{BACULA_VOLUME:bacula.volume.name}\" in catalog. +BACULA_LOG_NEW_LABEL Labeled new Volume \"%{BACULA_VOLUME:bacula.volume.name}\" on (?:file )?device \"%{BACULA_DEVICE:bacula.volume.device}\" \(%{BACULA_DEVICEPATH:bacula.volume.path}\). +BACULA_LOG_WROTE_LABEL Wrote label to prelabeled Volume \"%{BACULA_VOLUME:bacula.volume.name}\" on device \"%{BACULA_DEVICE:bacula.volume.device}\" \(%{BACULA_DEVICEPATH:bacula.volume.path}\) +BACULA_LOG_NEW_MOUNT New volume \"%{BACULA_VOLUME:bacula.volume.name}\" mounted on device \"%{BACULA_DEVICE:bacula.volume.device}\" \(%{BACULA_DEVICEPATH:bacula.volume.path}\) at %{BACULA_TIMESTAMP:bacula.timestamp}. +BACULA_LOG_NOOPEN \s*Cannot open %{DATA}: ERR=%{GREEDYDATA:error.message} +BACULA_LOG_NOOPENDIR \s*Could not open directory \"?%{DATA:file.path}\"?: ERR=%{GREEDYDATA:error.message} +BACULA_LOG_NOSTAT \s*Could not stat %{DATA:file.path}: ERR=%{GREEDYDATA:error.message} +BACULA_LOG_NOJOBS There are no more Jobs associated with Volume \"%{BACULA_VOLUME:bacula.volume.name}\". Marking it purged. +BACULA_LOG_ALL_RECORDS_PRUNED .*?All records pruned from Volume \"%{BACULA_VOLUME:bacula.volume.name}\"; marking it \"Purged\" +BACULA_LOG_BEGIN_PRUNE_JOBS Begin pruning Jobs older than %{INT} month %{INT} days . +BACULA_LOG_BEGIN_PRUNE_FILES Begin pruning Files. +BACULA_LOG_PRUNED_JOBS Pruned %{INT} Jobs* for client %{BACULA_HOST:bacula.client.name} from catalog. +BACULA_LOG_PRUNED_FILES Pruned Files from %{INT} Jobs* for client %{BACULA_HOST:bacula.client.name} from catalog. +BACULA_LOG_ENDPRUNE End auto prune. +BACULA_LOG_STARTJOB Start Backup JobId %{INT}, Job=%{BACULA_JOB:bacula.job.name} +BACULA_LOG_STARTRESTORE Start Restore Job %{BACULA_JOB:bacula.job.name} +BACULA_LOG_USEDEVICE Using Device \"%{BACULA_DEVICE:bacula.volume.device}\" +BACULA_LOG_DIFF_FS \s*%{UNIXPATH} is a different filesystem. Will not descend from %{UNIXPATH} into it. +BACULA_LOG_JOBEND Job write elapsed time = %{DATA:bacula.job.elapsed_time}, Transfer rate = %{NUMBER} (K|M|G)? Bytes/second +BACULA_LOG_NOPRUNE_JOBS No Jobs found to prune. +BACULA_LOG_NOPRUNE_FILES No Files found to prune. +BACULA_LOG_VOLUME_PREVWRITTEN Volume \"?%{BACULA_VOLUME:bacula.volume.name}\"? previously written, moving to end of data. +BACULA_LOG_READYAPPEND Ready to append to end of Volume \"%{BACULA_VOLUME:bacula.volume.name}\" size=%{INT:bacula.volume.size:long} +# :long - %{INT:bacula.volume.size:int} +BACULA_LOG_CANCELLING Cancelling duplicate JobId=%{INT:bacula.job.other_id}. +BACULA_LOG_MARKCANCEL JobId %{INT:bacula.job.id}, Job %{BACULA_JOB:bacula.job.name} marked to be canceled. +BACULA_LOG_CLIENT_RBJ shell command: run ClientRunBeforeJob \"%{GREEDYDATA:bacula.job.client_run_before_command}\" +BACULA_LOG_VSS (Generate )?VSS (Writer)? +BACULA_LOG_MAXSTART Fatal [eE]rror: Job canceled because max start delay time exceeded. +BACULA_LOG_DUPLICATE Fatal [eE]rror: JobId %{INT:bacula.job.other_id} already running. Duplicate job not allowed. +BACULA_LOG_NOJOBSTAT Fatal [eE]rror: No Job status returned from FD. +BACULA_LOG_FATAL_CONN Fatal [eE]rror: bsock.c:133 Unable to connect to (Client: %{BACULA_HOST:bacula.client.name}|Storage daemon) on %{IPORHOST:client.address}:%{POSINT:client.port:int}. ERR=%{GREEDYDATA:error.message} +BACULA_LOG_NO_CONNECT Warning: bsock.c:127 Could not connect to (Client: %{BACULA_HOST:bacula.client.name}|Storage daemon) on %{IPORHOST:client.address}:%{POSINT:client.port:int}. ERR=%{GREEDYDATA:error.message} +BACULA_LOG_NO_AUTH Fatal error: Unable to authenticate with File daemon at \"?%{IPORHOST:client.address}(?::%{POSINT:client.port:int})?\"?. Possible causes: +BACULA_LOG_NOSUIT No prior or suitable Full backup found in catalog. Doing FULL backup. +BACULA_LOG_NOPRIOR No prior Full backup Job record found. + +BACULA_LOG_JOB (Error: )?Bacula %{BACULA_HOST} %{BACULA_VERSION} \(%{BACULA_VERSION}\): + +BACULA_LOG %{BACULA_TIMESTAMP:timestamp} %{BACULA_HOST:host.hostname}(?: JobId %{INT:bacula.job.id})?:? (%{BACULA_LOG_MAX_CAPACITY}|%{BACULA_LOG_END_VOLUME}|%{BACULA_LOG_NEW_VOLUME}|%{BACULA_LOG_NEW_LABEL}|%{BACULA_LOG_WROTE_LABEL}|%{BACULA_LOG_NEW_MOUNT}|%{BACULA_LOG_NOOPEN}|%{BACULA_LOG_NOOPENDIR}|%{BACULA_LOG_NOSTAT}|%{BACULA_LOG_NOJOBS}|%{BACULA_LOG_ALL_RECORDS_PRUNED}|%{BACULA_LOG_BEGIN_PRUNE_JOBS}|%{BACULA_LOG_BEGIN_PRUNE_FILES}|%{BACULA_LOG_PRUNED_JOBS}|%{BACULA_LOG_PRUNED_FILES}|%{BACULA_LOG_ENDPRUNE}|%{BACULA_LOG_STARTJOB}|%{BACULA_LOG_STARTRESTORE}|%{BACULA_LOG_USEDEVICE}|%{BACULA_LOG_DIFF_FS}|%{BACULA_LOG_JOBEND}|%{BACULA_LOG_NOPRUNE_JOBS}|%{BACULA_LOG_NOPRUNE_FILES}|%{BACULA_LOG_VOLUME_PREVWRITTEN}|%{BACULA_LOG_READYAPPEND}|%{BACULA_LOG_CANCELLING}|%{BACULA_LOG_MARKCANCEL}|%{BACULA_LOG_CLIENT_RBJ}|%{BACULA_LOG_VSS}|%{BACULA_LOG_MAXSTART}|%{BACULA_LOG_DUPLICATE}|%{BACULA_LOG_NOJOBSTAT}|%{BACULA_LOG_FATAL_CONN}|%{BACULA_LOG_NO_CONNECT}|%{BACULA_LOG_NO_AUTH}|%{BACULA_LOG_NOSUIT}|%{BACULA_LOG_JOB}|%{BACULA_LOG_NOPRIOR}) +# old (deprecated) name : +BACULA_LOGLINE %{BACULA_LOG} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/bind b/src/platform/packages/shared/kbn-grok-ui/patterns/bind new file mode 100644 index 0000000000000..ec212de118ddb --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/bind @@ -0,0 +1,13 @@ +BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME} + +BIND9_DNSTYPE (?:A|AAAA|CAA|CDNSKEY|CDS|CERT|CNAME|CSYNC|DLV|DNAME|DNSKEY|DS|HINFO|LOC|MX|NAPTR|NS|NSEC|NSEC3|OPENPGPKEY|PTR|RRSIG|RP|SIG|SMIMEA|SOA|SRV|TSIG|TXT|URI) +BIND9_CATEGORY (?:queries) + +# dns.question.class is static - only 'IN' is supported by Bind9 +# bind.log.question.name is expected to be a 'duplicate' (same as the dns.question.name capture) +BIND9_QUERYLOGBASE client(:? @0x(?:[0-9A-Fa-f]+))? %{IP:client.ip}#%{POSINT:client.port:int} \(%{GREEDYDATA:bind.log.question.name}\): query: %{GREEDYDATA:dns.question.name} (?IN) %{BIND9_DNSTYPE:dns.question.type}(:? %{DATA:bind.log.question.flags})? \(%{IP:server.ip}\) + +# for query-logging category and severity are always fixed as "queries: info: " +BIND9_QUERYLOG %{BIND9_TIMESTAMP:timestamp} %{BIND9_CATEGORY:bing.log.category}: %{LOGLEVEL:log.level}: %{BIND9_QUERYLOGBASE} + +BIND9 %{BIND9_QUERYLOG} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/bro b/src/platform/packages/shared/kbn-grok-ui/patterns/bro new file mode 100644 index 0000000000000..dc38d5a7fba59 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/bro @@ -0,0 +1,30 @@ +# supports the 'old' BRO log files, for updated Zeek log format see the patters/ecs-v1/zeek +# https://www.bro.org/sphinx/script-reference/log-files.html + +BRO_BOOL [TF] +BRO_DATA [^\t]+ + +# http.log - old format (before the Zeek rename) : +BRO_HTTP %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{INT:zeek.http.trans_depth:int}\t(?:-|%{WORD:http.request.method})\t(?:-|%{BRO_DATA:url.domain})\t(?:-|%{BRO_DATA:url.original})\t(?:-|%{BRO_DATA:http.request.referrer})\t(?:-|%{BRO_DATA:user_agent.original})\t(?:-|%{NUMBER:http.request.body.bytes:long})\t(?:-|%{NUMBER:http.response.body.bytes:long})\t(?:-|%{POSINT:http.response.status_code:int})\t(?:-|%{DATA:zeek.http.status_msg})\t(?:-|%{POSINT:zeek.http.info_code:int})\t(?:-|%{DATA:zeek.http.info_msg})\t(?:-|%{BRO_DATA:zeek.http.filename})\t(?:\(empty\)|%{BRO_DATA:zeek.http.tags})\t(?:-|%{BRO_DATA:url.username})\t(?:-|%{BRO_DATA:url.password})\t(?:-|%{BRO_DATA:zeek.http.proxied})\t(?:-|%{BRO_DATA:zeek.http.orig_fuids})\t(?:-|%{BRO_DATA:http.request.mime_type})\t(?:-|%{BRO_DATA:zeek.http.resp_fuids})\t(?:-|%{BRO_DATA:http.response.mime_type}) +# :long - %{NUMBER:http.request.body.bytes:int} +# :long - %{NUMBER:http.response.body.bytes:int} + +# dns.log - old format +BRO_DNS %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{WORD:network.transport}\t(?:-|%{INT:dns.id:int})\t(?:-|%{BRO_DATA:dns.question.name})\t(?:-|%{INT:zeek.dns.qclass:int})\t(?:-|%{BRO_DATA:zeek.dns.qclass_name})\t(?:-|%{INT:zeek.dns.qtype:int})\t(?:-|%{BRO_DATA:dns.question.type})\t(?:-|%{INT:zeek.dns.rcode:int})\t(?:-|%{BRO_DATA:dns.response_code})\t(?:-|%{BRO_BOOL:zeek.dns.AA})\t(?:-|%{BRO_BOOL:zeek.dns.TC})\t(?:-|%{BRO_BOOL:zeek.dns.RD})\t(?:-|%{BRO_BOOL:zeek.dns.RA})\t(?:-|%{NONNEGINT:zeek.dns.Z:int})\t(?:-|%{BRO_DATA:zeek.dns.answers})\t(?:-|%{DATA:zeek.dns.TTLs})\t(?:-|%{BRO_BOOL:zeek.dns.rejected}) + +# conn.log - old bro, also supports 'newer' format (optional *zeek.connection.local_resp* flag) compared to non-ecs mode +BRO_CONN %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{WORD:network.transport}\t(?:-|%{BRO_DATA:network.protocol})\t(?:-|%{NUMBER:zeek.connection.duration:float})\t(?:-|%{INT:zeek.connection.orig_bytes:long})\t(?:-|%{INT:zeek.connection.resp_bytes:long})\t(?:-|%{BRO_DATA:zeek.connection.state})\t(?:-|%{BRO_BOOL:zeek.connection.local_orig})\t(?:(?:-|%{BRO_BOOL:zeek.connection.local_resp})\t)?(?:-|%{INT:zeek.connection.missed_bytes:long})\t(?:-|%{BRO_DATA:zeek.connection.history})\t(?:-|%{INT:source.packets:long})\t(?:-|%{INT:source.bytes:long})\t(?:-|%{INT:destination.packets:long})\t(?:-|%{INT:destination.bytes:long})\t(?:\(empty\)|%{BRO_DATA:zeek.connection.tunnel_parents}) +# :long - %{INT:zeek.connection.orig_bytes:int} +# :long - %{INT:zeek.connection.resp_bytes:int} +# :long - %{INT:zeek.connection.missed_bytes:int} +# :long - %{INT:source.packets:int} +# :long - %{INT:source.bytes:int} +# :long - %{INT:destination.packets:int} +# :long - %{INT:destination.bytes:int} + +# files.log - old format +BRO_FILES %{NUMBER:timestamp}\t%{NOTSPACE:zeek.files.fuid}\t(?:-|%{IP:server.ip})\t(?:-|%{IP:client.ip})\t(?:-|%{BRO_DATA:zeek.files.session_ids})\t(?:-|%{BRO_DATA:zeek.files.source})\t(?:-|%{INT:zeek.files.depth:int})\t(?:-|%{BRO_DATA:zeek.files.analyzers})\t(?:-|%{BRO_DATA:file.mime_type})\t(?:-|%{BRO_DATA:file.name})\t(?:-|%{NUMBER:zeek.files.duration:float})\t(?:-|%{BRO_DATA:zeek.files.local_orig})\t(?:-|%{BRO_BOOL:zeek.files.is_orig})\t(?:-|%{INT:zeek.files.seen_bytes:long})\t(?:-|%{INT:file.size:long})\t(?:-|%{INT:zeek.files.missing_bytes:long})\t(?:-|%{INT:zeek.files.overflow_bytes:long})\t(?:-|%{BRO_BOOL:zeek.files.timedout})\t(?:-|%{BRO_DATA:zeek.files.parent_fuid})\t(?:-|%{BRO_DATA:file.hash.md5})\t(?:-|%{BRO_DATA:file.hash.sha1})\t(?:-|%{BRO_DATA:file.hash.sha256})\t(?:-|%{BRO_DATA:zeek.files.extracted}) +# :long - %{INT:zeek.files.seen_bytes:int} +# :long - %{INT:file.size:int} +# :long - %{INT:zeek.files.missing_bytes:int} +# :long - %{INT:zeek.files.overflow_bytes:int} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/exim b/src/platform/packages/shared/kbn-grok-ui/patterns/exim new file mode 100644 index 0000000000000..dba79503c0097 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/exim @@ -0,0 +1,26 @@ +EXIM_MSGID [0-9A-Za-z]{6}-[0-9A-Za-z]{6}-[0-9A-Za-z]{2} +# <= message arrival +# => normal message delivery +# -> additional address in same delivery +# *> delivery suppressed by -N +# ** delivery failed; address bounced +# == delivery deferred; temporary problem +EXIM_FLAGS (?:<=|=>|->|\*>|\*\*|==|<>|>>) +EXIM_DATE (:?%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}) +EXIM_PID \[%{POSINT:process.pid:int}\] +EXIM_QT ((\d+y)?(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?) +EXIM_EXCLUDE_TERMS (Message is frozen|(Start|End) queue run| Warning: | retry time not reached | no (IP address|host name) found for (IP address|host) | unexpected disconnection while reading SMTP command | no immediate delivery: |another process is handling this message) +EXIM_REMOTE_HOST (H=(%{NOTSPACE:source.address} )?(\(%{NOTSPACE:exim.log.remote_address}\) )?\%{IP:source.ip}\](?::%{POSINT:source.port:int})?) +EXIM_INTERFACE (I=\[%{IP:destination.ip}\](?::%{NUMBER:destination.port:int})) +EXIM_PROTOCOL (P=%{NOTSPACE:network.protocol}) +EXIM_MSG_SIZE (S=%{NUMBER:exim.log.message.size:int}) +EXIM_HEADER_ID (id=%{NOTSPACE:exim.log.header_id}) +EXIM_QUOTED_CONTENT (?:\\.|[^\\"])* +EXIM_SUBJECT (T="%{EXIM_QUOTED_CONTENT:exim.log.message.subject}") + +EXIM_UNKNOWN_FIELD (?:[A-Za-z0-9]{1,4}=(?:%{QUOTEDSTRING}|%{NOTSPACE})) +EXIM_NAMED_FIELDS (?: (?:%{EXIM_REMOTE_HOST}|%{EXIM_INTERFACE}|%{EXIM_PROTOCOL}|%{EXIM_MSG_SIZE}|%{EXIM_HEADER_ID}|%{EXIM_SUBJECT}|%{EXIM_UNKNOWN_FIELD}))* + +EXIM_MESSAGE_ARRIVAL %{EXIM_DATE:timestamp} (?:%{EXIM_PID} )?%{EXIM_MSGID:exim.log.message.id} (?<=) (?[a-z:] )?%{EMAILADDRESS:exim.log.sender.email}%{EXIM_NAMED_FIELDS}(?:(?: from ?)? for %{EMAILADDRESS:exim.log.recipient.email})? + +EXIM %{EXIM_MESSAGE_ARRIVAL} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/firewalls b/src/platform/packages/shared/kbn-grok-ui/patterns/firewalls new file mode 100644 index 0000000000000..892b3a506825d --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/firewalls @@ -0,0 +1,111 @@ +# NetScreen firewall logs +NETSCREENSESSIONLOG %{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:observer.hostname} %{NOTSPACE:observer.name}: (?NetScreen) device_id=%{WORD:netscreen.device_id} .*?(system-\w+-%{NONNEGINT:event.code}\(%{WORD:netscreen.session.type}\))?: start_time="%{DATA:netscreen.session.start_time}" duration=%{INT:netscreen.session.duration:int} policy_id=%{INT:netscreen.policy_id} service=%{DATA:netscreen.service} proto=%{INT:netscreen.protocol_number:int} src zone=%{WORD:observer.ingress.zone} dst zone=%{WORD:observer.egress.zone} action=%{WORD:event.action} sent=%{INT:source.bytes:long} rcvd=%{INT:destination.bytes:long} src=%{IPORHOST:source.address} dst=%{IPORHOST:destination.address}(?: src_port=%{INT:source.port:int} dst_port=%{INT:destination.port:int})?(?: src-xlated ip=%{IP:source.nat.ip} port=%{INT:source.nat.port:int} dst-xlated ip=%{IP:destination.nat.ip} port=%{INT:destination.nat.port:int})?(?: session_id=%{INT:netscreen.session.id} reason=%{GREEDYDATA:netscreen.session.reason})? +# :long - %{INT:source.bytes:int} +# :long - %{INT:destination.bytes:int} + +#== Cisco ASA == +CISCO_TAGGED_SYSLOG ^<%{POSINT:log.syslog.priority:int}>%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:host.hostname})? ?: %%{CISCOTAG:cisco.asa.tag}: +CISCOTIMESTAMP %{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME} +CISCOTAG [A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+) +# Common Particles +CISCO_ACTION Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted +CISCO_REASON Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)* +CISCO_DIRECTION Inbound|inbound|Outbound|outbound +CISCO_INTERVAL first hit|%{INT}-second interval +CISCO_XLATE_TYPE static|dynamic +# helpers +CISCO_HITCOUNT_INTERVAL hit-cnt %{INT:cisco.asa.hit_count:int} (?:first hit|%{INT:cisco.asa.interval:int}-second interval) +CISCO_SRC_IP_USER %{NOTSPACE:observer.ingress.interface.name}:%{IP:source.ip}(?:\(%{DATA:source.user.name}\))? +CISCO_DST_IP_USER %{NOTSPACE:observer.egress.interface.name}:%{IP:destination.ip}(?:\(%{DATA:destination.user.name}\))? +CISCO_SRC_HOST_PORT_USER %{NOTSPACE:observer.ingress.interface.name}:(?:(?:%{IP:source.ip})|(?:%{HOSTNAME:source.address}))(?:/%{INT:source.port:int})?(?:\(%{DATA:source.user.name}\))? +CISCO_DST_HOST_PORT_USER %{NOTSPACE:observer.egress.interface.name}:(?:(?:%{IP:destination.ip})|(?:%{HOSTNAME:destination.address}))(?:/%{INT:destination.port:int})?(?:\(%{DATA:destination.user.name}\))? +# ASA-1-104001 +CISCOFW104001 \((?:Primary|Secondary)\) Switching to ACTIVE - %{GREEDYDATA:event.reason} +# ASA-1-104002 +CISCOFW104002 \((?:Primary|Secondary)\) Switching to STANDBY - %{GREEDYDATA:event.reason} +# ASA-1-104003 +CISCOFW104003 \((?:Primary|Secondary)\) Switching to FAILED\. +# ASA-1-104004 +CISCOFW104004 \((?:Primary|Secondary)\) Switching to OK\. +# ASA-1-105003 +CISCOFW105003 \((?:Primary|Secondary)\) Monitoring on [Ii]nterface %{NOTSPACE:network.interface.name} waiting +# ASA-1-105004 +CISCOFW105004 \((?:Primary|Secondary)\) Monitoring on [Ii]nterface %{NOTSPACE:network.interface.name} normal +# ASA-1-105005 +CISCOFW105005 \((?:Primary|Secondary)\) Lost Failover communications with mate on [Ii]nterface %{NOTSPACE:network.interface.name} +# ASA-1-105008 +CISCOFW105008 \((?:Primary|Secondary)\) Testing [Ii]nterface %{NOTSPACE:network.interface.name} +# ASA-1-105009 +CISCOFW105009 \((?:Primary|Secondary)\) Testing on [Ii]nterface %{NOTSPACE:network.interface.name} (?:Passed|Failed) +# ASA-2-106001 +CISCOFW106001 %{CISCO_DIRECTION:cisco.asa.network.direction} %{WORD:cisco.asa.network.transport} connection %{CISCO_ACTION:cisco.asa.outcome} from %{IP:source.ip}/%{INT:source.port:int} to %{IP:destination.ip}/%{INT:destination.port:int} flags %{DATA:cisco.asa.tcp_flags} on interface %{NOTSPACE:observer.egress.interface.name} +# ASA-2-106006, ASA-2-106007, ASA-2-106010 +CISCOFW106006_106007_106010 %{CISCO_ACTION:cisco.asa.outcome} %{CISCO_DIRECTION:cisco.asa.network.direction} %{WORD:cisco.asa.network.transport} (?:from|src) %{IP:source.ip}/%{INT:source.port:int}(?:\(%{DATA:source.user.name}\))? (?:to|dst) %{IP:destination.ip}/%{INT:destination.port:int}(?:\(%{DATA:destination.user.name}\))? (?:(?:on interface %{NOTSPACE:observer.egress.interface.name})|(?:due to %{CISCO_REASON:event.reason})) +# ASA-3-106014 +CISCOFW106014 %{CISCO_ACTION:cisco.asa.outcome} %{CISCO_DIRECTION:cisco.asa.network.direction} %{WORD:cisco.asa.network.transport} src %{CISCO_SRC_IP_USER} dst %{CISCO_DST_IP_USER}\s?\(type %{INT:cisco.asa.icmp_type:int}, code %{INT:cisco.asa.icmp_code:int}\) +# ASA-6-106015 +CISCOFW106015 %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} \(%{DATA:cisco.asa.rule_name}\) from %{IP:source.ip}/%{INT:source.port:int} to %{IP:destination.ip}/%{INT:destination.port:int} flags %{DATA:cisco.asa.tcp_flags} on interface %{NOTSPACE:observer.egress.interface.name} +# ASA-1-106021 +CISCOFW106021 %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} reverse path check from %{IP:source.ip} to %{IP:destination.ip} on interface %{NOTSPACE:observer.egress.interface.name} +# ASA-4-106023 +CISCOFW106023 %{CISCO_ACTION:cisco.asa.outcome}(?: protocol)? %{WORD:cisco.asa.network.transport} src %{CISCO_SRC_HOST_PORT_USER} dst %{CISCO_DST_HOST_PORT_USER}( \(type %{INT:cisco.asa.icmp_type:int}, code %{INT:cisco.asa.icmp_code:int}\))? by access-group "?%{DATA:cisco.asa.rule_name}"? \%{DATA:[@metadata.cisco.asa.hashcode1}, %{DATA:@metadata.cisco.asa.hashcode2}\] +# ASA-4-106100, ASA-4-106102, ASA-4-106103 +CISCOFW106100_2_3 access-list %{NOTSPACE:cisco.asa.rule_name} %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} for user '%{DATA:user.name}' %{DATA:observer.ingress.interface.name}/%{IP:source.ip}\(%{INT:source.port:int}\) -> %{DATA:observer.egress.interface.name}/%{IP:destination.ip}\(%{INT:destination.port:int}\) %{CISCO_HITCOUNT_INTERVAL} \%{DATA:[@metadata.cisco.asa.hashcode1}, %{DATA:@metadata.cisco.asa.hashcode2}\] +# ASA-5-106100 +CISCOFW106100 access-list %{NOTSPACE:cisco.asa.rule_name} %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} %{DATA:observer.ingress.interface.name}/%{IP:source.ip}\(%{INT:source.port:int}\)(?:\(%{DATA:source.user.name}\))? -> %{DATA:observer.egress.interface.name}/%{IP:destination.ip}\(%{INT:destination.port:int}\)(?:\(%{DATA:source.user.name}\))? hit-cnt %{INT:cisco.asa.hit_count:int} %{CISCO_INTERVAL} \%{DATA:[@metadata.cisco.asa.hashcode1}, %{DATA:@metadata.cisco.asa.hashcode2}\] +# ASA-5-304001 +CISCOFW304001 %{IP:source.ip}(?:\(%{DATA:source.user.name}\))? Accessed URL %{IP:destination.ip}:%{GREEDYDATA:url.original} +# ASA-6-110002 +CISCOFW110002 %{CISCO_REASON:event.reason} for %{WORD:cisco.asa.network.transport} from %{DATA:observer.ingress.interface.name}:%{IP:source.ip}/%{INT:source.port:int} to %{IP:destination.ip}/%{INT:destination.port:int} +# ASA-6-302010 +CISCOFW302010 %{INT:cisco.asa.connections.in_use:int} in use, %{INT:cisco.asa.connections.most_used:int} most used +# ASA-6-302013, ASA-6-302014, ASA-6-302015, ASA-6-302016 +CISCOFW302013_302014_302015_302016 %{CISCO_ACTION:cisco.asa.outcome}(?: %{CISCO_DIRECTION:cisco.asa.network.direction})? %{WORD:cisco.asa.network.transport} connection %{INT:cisco.asa.connection_id} for %{NOTSPACE:observer.ingress.interface.name}:%{IP:source.ip}/%{INT:source.port:int}(?: \(%{IP:source.nat.ip}/%{INT:source.nat.port:int}\))?(?:\(%{DATA:source.user.name?}\))? to %{NOTSPACE:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int}( \(%{IP:destination.nat.ip}/%{INT:destination.nat.port:int}\))?(?:\(%{DATA:destination.user.name}\))?( duration %{TIME:cisco.asa.duration} bytes %{INT:network.bytes:long})?(?: %{CISCO_REASON:event.reason})?(?: \(%{DATA:user.name}\))? +# :long - %{INT:network.bytes:int} +# ASA-6-302020, ASA-6-302021 +CISCOFW302020_302021 %{CISCO_ACTION:cisco.asa.outcome}(?: %{CISCO_DIRECTION:cisco.asa.network.direction})? %{WORD:cisco.asa.network.transport} connection for faddr %{IP:destination.ip}/%{INT:cisco.asa.icmp_seq:int}(?:\(%{DATA:destination.user.name}\))? gaddr %{IP:source.nat.ip}/%{INT:cisco.asa.icmp_type:int} laddr %{IP:source.ip}/%{INT}(?: \(%{DATA:source.user.name}\))? +# ASA-6-305011 +CISCOFW305011 %{CISCO_ACTION:cisco.asa.outcome} %{CISCO_XLATE_TYPE} %{WORD:cisco.asa.network.transport} translation from %{DATA:observer.ingress.interface.name}:%{IP:source.ip}(/%{INT:source.port:int})?(?:\(%{DATA:source.user.name}\))? to %{DATA:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int} +# ASA-3-313001, ASA-3-313004, ASA-3-313008 +CISCOFW313001_313004_313008 %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} type=%{INT:cisco.asa.icmp_type:int}, code=%{INT:cisco.asa.icmp_code:int} from %{IP:source.ip} on interface %{NOTSPACE:observer.egress.interface.name}(?: to %{IP:destination.ip})? +# ASA-4-313005 +CISCOFW313005 %{CISCO_REASON:event.reason} for %{WORD:cisco.asa.network.transport} error message: %{WORD} src %{CISCO_SRC_IP_USER} dst %{CISCO_DST_IP_USER} \(type %{INT:cisco.asa.icmp_type:int}, code %{INT:cisco.asa.icmp_code:int}\) on %{NOTSPACE} interface\.\s+Original IP payload: %{WORD:cisco.asa.original_ip_payload.network.transport} src %{IP:cisco.asa.original_ip_payload.source.ip}/%{INT:cisco.asa.original_ip_payload.source.port:int}(?:\(%{DATA:cisco.asa.original_ip_payload.source.user.name}\))? dst %{IP:cisco.asa.original_ip_payload.destination.ip}/%{INT:cisco.asa.original_ip_payload.destination.port:int}(?:\(%{DATA:cisco.asa.original_ip_payload.destination.user.name}\))? +# ASA-5-321001 +CISCOFW321001 Resource '%{DATA:cisco.asa.resource.name}' limit of %{POSINT:cisco.asa.resource.limit:int} reached for system +# ASA-4-402117 +CISCOFW402117 %{WORD:cisco.asa.network.type}: Received a non-IPSec packet \(protocol=\s?%{WORD:cisco.asa.network.transport}\) from %{IP:source.ip} to %{IP:destination.ip}\.? +# ASA-4-402119 +CISCOFW402119 %{WORD:cisco.asa.network.type}: Received an %{WORD:cisco.asa.ipsec.protocol} packet \(SPI=\s?%{DATA:cisco.asa.ipsec.spi}, sequence number=\s?%{DATA:cisco.asa.ipsec.seq_num}\) from %{IP:source.ip} \(user=\s?%{DATA:source.user.name}\) to %{IP:destination.ip} that failed anti-replay checking\.? +# ASA-4-419001 +CISCOFW419001 %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} packet from %{NOTSPACE:observer.ingress.interface.name}:%{IP:source.ip}/%{INT:source.port:int} to %{NOTSPACE:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int}, reason: %{GREEDYDATA:event.reason} +# ASA-4-419002 +CISCOFW419002 %{CISCO_REASON:event.reason} from %{DATA:observer.ingress.interface.name}:%{IP:source.ip}/%{INT:source.port:int} to %{DATA:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int} with different initial sequence number +# ASA-4-500004 +CISCOFW500004 %{CISCO_REASON:event.reason} for protocol=%{WORD:cisco.asa.network.transport}, from %{IP:source.ip}/%{INT:source.port:int} to %{IP:destination.ip}/%{INT:destination.port:int} +# ASA-6-602303, ASA-6-602304 +CISCOFW602303_602304 %{WORD:cisco.asa.network.type}: An %{CISCO_DIRECTION:cisco.asa.network.direction} %{DATA:cisco.asa.ipsec.tunnel_type} SA \(SPI=\s?%{DATA:cisco.asa.ipsec.spi}\) between %{IP:source.ip} and %{IP:destination.ip} \(user=\s?%{DATA:source.user.name}\) has been %{CISCO_ACTION:cisco.asa.outcome} +# ASA-7-710001, ASA-7-710002, ASA-7-710003, ASA-7-710005, ASA-7-710006 +CISCOFW710001_710002_710003_710005_710006 %{WORD:cisco.asa.network.transport} (?:request|access) %{CISCO_ACTION:cisco.asa.outcome} from %{IP:source.ip}/%{INT:source.port:int} to %{DATA:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int} +# ASA-6-713172 +CISCOFW713172 Group = %{DATA:cisco.asa.source.group}, IP = %{IP:source.ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:@metadata.cisco.asa.remote_nat}\s*behind a NAT device\s+This\s+end\s*%{DATA:@metadata.cisco.asa.local_nat}\s*behind a NAT device +# ASA-4-733100 +CISCOFW733100 \\s*%{DATA:[cisco.asa.burst.object}\s*\] drop %{DATA:cisco.asa.burst.id} exceeded. Current burst rate is %{INT:cisco.asa.burst.current_rate:int} per second, max configured rate is %{INT:cisco.asa.burst.configured_rate:int}; Current average rate is %{INT:cisco.asa.burst.avg_rate:int} per second, max configured rate is %{INT:cisco.asa.burst.configured_avg_rate:int}; Cumulative total count is %{INT:cisco.asa.burst.cumulative_count:int} +#== End Cisco ASA == + + +IPTABLES_TCP_FLAGS (CWR |ECE |URG |ACK |PSH |RST |SYN |FIN )* +IPTABLES_TCP_PART (?:SEQ=%{INT:iptables.tcp.seq:int}\s+)?(?:ACK=%{INT:iptables.tcp.ack:int}\s+)?WINDOW=%{INT:iptables.tcp.window:int}\s+RES=0x%{BASE16NUM:iptables.tcp_reserved_bits}\s+%{IPTABLES_TCP_FLAGS:iptables.tcp.flags} + +IPTABLES4_FRAG (?:(?<= )(?:CE|DF|MF))* +IPTABLES4_PART SRC=%{IPV4:source.ip}\s+DST=%{IPV4:destination.ip}\s+LEN=(?:%{INT:iptables.length:int})?\s+TOS=(?:0|0x%{BASE16NUM:iptables.tos})?\s+PREC=(?:0x%{BASE16NUM:iptables.precedence_bits})?\s+TTL=(?:%{INT:iptables.ttl:int})?\s+ID=(?:%{INT:iptables.id})?\s+(?:%{IPTABLES4_FRAG:iptables.fragment_flags})?(?:\s+FRAG: %{INT:iptables.fragment_offset:int})? +IPTABLES6_PART SRC=%{IPV6:source.ip}\s+DST=%{IPV6:destination.ip}\s+LEN=(?:%{INT:iptables.length:int})?\s+TC=(?:0|0x%{BASE16NUM:iptables.tos})?\s+HOPLIMIT=(?:%{INT:iptables.ttl:int})?\s+FLOWLBL=(?:%{INT:iptables.flow_label})? + +IPTABLES IN=(?:%{NOTSPACE:observer.ingress.interface.name})?\s+OUT=(?:%{NOTSPACE:observer.egress.interface.name})?\s+(?:MAC=(?:%{COMMONMAC:destination.mac})?(?::%{COMMONMAC:source.mac})?(?::A-Fa-f0-9{2}:A-Fa-f0-9{2})?\s+)?(:?%{IPTABLES4_PART}|%{IPTABLES6_PART}).*?PROTO=(?:%{WORD:network.transport})?\s+SPT=(?:%{INT:source.port:int})?\s+DPT=(?:%{INT:destination.port:int})?\s+(?:%{IPTABLES_TCP_PART})? + +# Shorewall firewall logs +SHOREWALL (?:%{SYSLOGTIMESTAMP:timestamp}) (?:%{WORD:observer.hostname}) .*Shorewall:(?:%{WORD:shorewall.firewall.type})?:(?:%{WORD:shorewall.firewall.action})?.*%{IPTABLES} +#== End Shorewall +#== SuSE Firewall 2 == +SFW2_LOG_PREFIX SFW2\-INext\-%{NOTSPACE:suse.firewall.action} +SFW2 ((?:%{SYSLOGTIMESTAMP:timestamp})|(?:%{TIMESTAMP_ISO8601:timestamp}))\s*%{HOSTNAME:observer.hostname}.*?%{SFW2_LOG_PREFIX:suse.firewall.log_prefix}\s*%{IPTABLES} +#== End SuSE == diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/grok_patterns b/src/platform/packages/shared/kbn-grok-ui/patterns/grok_patterns new file mode 100644 index 0000000000000..6f58f3ff4750d --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/grok_patterns @@ -0,0 +1,95 @@ +USERNAME [a-zA-Z0-9._-]+ +USER %{USERNAME} +EMAILLOCALPART [a-zA-Z0-9!#$%&'*+\-/=?^_`{|}~]{1,64}(?:\.[a-zA-Z0-9!#$%&'*+\-/=?^_`{|}~]{1,62}){0,63} +EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME} +INT (?:[+-]?(?:[0-9]+)) +BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) +NUMBER (?:%{BASE10NUM}) +BASE16NUM (?(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) +UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} +# URN, allowing use of RFC 2141 section 2.3 reserved characters +URN urn:[0-9A-Za-z][0-9A-Za-z-]{0,31}:(?:%[0-9a-fA-F]{2}|[0-9A-Za-z()+,.:=@;$_!*'/?#-])+ + +# Networking +MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) +CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) +WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) +COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) +IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +IPV4 (?[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ +URIPROTO [A-Za-z]([A-Za-z0-9+\-.]+)+ +URIHOST %{IPORHOST}(?::%{POSINT})? +# uripath comes loosely from RFC1738, but mostly from what Firefox doesn't turn into %XX +URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\-]*)+ +URIQUERY [A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]* +# deprecated (kept due compatibility): +URIPARAM \?%{URIQUERY} +URIPATHPARAM %{URIPATH}(?:\?%{URIQUERY})? +URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATH}(?:\?%{URIQUERY})?)? + +# Months: January, Feb, 3, 03, 12, December +MONTH \b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y|i)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\b +MONTHNUM (?:0?[1-9]|1[0-2]) +MONTHNUM2 (?:0[1-9]|1[0-2]) +MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) + +# Days: Monday, Tue, Thu, etc... +DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) + +# Years? +YEAR (?>\d\d){1,2} +HOUR (?:2[0123]|[01]?[0-9]) +MINUTE (?:[0-5][0-9]) +# '60' is a leap second in most time standards and thus is valid. +SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) +TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) +# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) +DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} +DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} +ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) +ISO8601_SECOND %{SECOND} +TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? +DATE %{DATE_US}|%{DATE_EU} +DATESTAMP %{DATE}[- ]%{TIME} +TZ (?:[APMCE][SD]T|UTC) +DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} +DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} +DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} +DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} + +# Syslog Dates: Month Day HH:MM:SS +SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} +PROG [\x21-\x5a\x5c\x5e-\x7e]+ +SYSLOGPROG %{PROG:process.name}(?:\[%{POSINT:process.pid:int}\])? +SYSLOGHOST %{IPORHOST} +SYSLOGFACILITY <%{NONNEGINT:log.syslog.facility.code:int}.%{NONNEGINT:log.syslog.priority:int}> +HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} + +# Shortcuts +QS %{QUOTEDSTRING} + +# Log formats +SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:host.hostname} %{SYSLOGPROG}: + +# Log Levels +LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo?(?:rmation)?|INFO?(?:RMATION)?|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/haproxy b/src/platform/packages/shared/kbn-grok-ui/patterns/haproxy new file mode 100644 index 0000000000000..f46d4ba945bb3 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/haproxy @@ -0,0 +1,40 @@ + +HAPROXYTIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) +HAPROXYDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{HAPROXYTIME}.%{INT} + +# Override these default patterns to parse out what is captured in your haproxy.cfg +HAPROXYCAPTUREDREQUESTHEADERS %{DATA:haproxy.http.request.captured_headers} +HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:haproxy.http.response.captured_headers} + +# Example: +# These haproxy config lines will add data to the logs that are captured +# by the patterns below. Place them in your custom patterns directory to +# override the defaults. +# +# capture request header Host len 40 +# capture request header X-Forwarded-For len 50 +# capture request header Accept-Language len 50 +# capture request header Referer len 200 +# capture request header User-Agent len 200 +# +# capture response header Content-Type len 30 +# capture response header Content-Encoding len 10 +# capture response header Cache-Control len 200 +# capture response header Last-Modified len 200 +# +# HAPROXYCAPTUREDREQUESTHEADERS %{DATA:haproxy.http.request.host}\|%{DATA:haproxy.http.request.x_forwarded_for}\|%{DATA:haproxy.http.request.accept_language}\|%{DATA:http.request.referrer}\|%{DATA:user_agent.original} +# HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:http.response.mime_type}\|%{DATA:haproxy.http.response.encoding}\|%{DATA:haproxy.http.response.cache_control}\|%{DATA:haproxy.http.response.last_modified} + +HAPROXYURI (?:%{URIPROTO:url.scheme}://)?(?:%{USER:url.username}(?::[^@]*)?@)?(?:%{IPORHOST:url.domain}(?::%{POSINT:url.port:int})?)?(?:%{URIPATH:url.path}(?:\?%{URIQUERY:url.query})?)? + +HAPROXYHTTPREQUESTLINE (?:|(?:%{WORD:http.request.method} %{HAPROXYURI:url.original}(?: HTTP/%{NUMBER:http.version})?)) + +# parse a haproxy 'httplog' line +HAPROXYHTTPBASE %{IP:source.address}:%{INT:source.port:int} \[%{HAPROXYDATE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/(?:|%{NOTSPACE:haproxy.server_name}) (?:-1|%{INT:haproxy.http.request.time_wait_ms:int})/(?:-1|%{INT:haproxy.total_waiting_time_ms:int})/(?:-1|%{INT:haproxy.connection_wait_time_ms:int})/(?:-1|%{INT:haproxy.http.request.time_wait_without_data_ms:int})/%{NOTSPACE:haproxy.total_time_ms} %{INT:http.response.status_code:int} %{INT:source.bytes:long} (?:-|%{DATA:haproxy.http.request.captured_cookie}) (?:-|%{DATA:haproxy.http.response.captured_cookie}) %{NOTSPACE:haproxy.termination_state} %{INT:haproxy.connections.active:int}/%{INT:haproxy.connections.frontend:int}/%{INT:haproxy.connections.backend:int}/%{INT:haproxy.connections.server:int}/%{INT:haproxy.connections.retries:int} %{INT:haproxy.server_queue:int}/%{INT:haproxy.backend_queue:int}(?: \{%{HAPROXYCAPTUREDREQUESTHEADERS}\}(?: \{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?)?(?: "%{HAPROXYHTTPREQUESTLINE}"?)? +# :long - %{INT:source.bytes:int} + +HAPROXYHTTP (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp}) %{IPORHOST:host.hostname} %{SYSLOGPROG}: %{HAPROXYHTTPBASE} + +# parse a haproxy 'tcplog' line +HAPROXYTCP (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp}) %{IPORHOST:host.hostname} %{SYSLOGPROG}: %{IP:source.address}:%{INT:source.port:int} \[%{HAPROXYDATE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/(?:|%{NOTSPACE:haproxy.server_name}) (?:-1|%{INT:haproxy.total_waiting_time_ms:int})/(?:-1|%{INT:haproxy.connection_wait_time_ms:int})/%{NOTSPACE:haproxy.total_time_ms} %{INT:source.bytes:long} %{NOTSPACE:haproxy.termination_state} %{INT:haproxy.connections.active:int}/%{INT:haproxy.connections.frontend:int}/%{INT:haproxy.connections.backend:int}/%{INT:haproxy.connections.server:int}/%{INT:haproxy.connections.retries:int} %{INT:haproxy.server_queue:int}/%{INT:haproxy.backend_queue:int} +# :long - %{INT:source.bytes:int} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/httpd b/src/platform/packages/shared/kbn-grok-ui/patterns/httpd new file mode 100644 index 0000000000000..9b58e5096ad38 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/httpd @@ -0,0 +1,17 @@ +HTTPDUSER %{EMAILADDRESS}|%{USER} +HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR} + +# Log formats +HTTPD_COMMONLOG %{IPORHOST:source.address} (?:-|%{HTTPDUSER:apache.access.user.identity}) (?:-|%{HTTPDUSER:user.name}) \[%{HTTPDATE:timestamp}\] "(?:%{WORD:http.request.method} %{NOTSPACE:url.original}(?: HTTP/%{NUMBER:http.version})?|%{DATA})" (?:-|%{INT:http.response.status_code:int}) (?:-|%{INT:http.response.body.bytes:long}) +# :long - %{INT:http.response.body.bytes:int} +HTTPD_COMBINEDLOG %{HTTPD_COMMONLOG} "(?:-|%{DATA:http.request.referrer})" "(?:-|%{DATA:user_agent.original})" + +# Error logs +HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:log.level}\] (?:\[client %{IPORHOST:source.address}\] )?%{GREEDYDATA:message} +HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[(?:%{WORD:apache.error.module})?:%{LOGLEVEL:log.level}\] \[pid %{POSINT:process.pid:long}(:tid %{INT:process.thread.id:int})?\](?: \(%{POSINT:apache.error.proxy.error.code?}\)%{DATA:apache.error.proxy.error.message}:)?(?: \[client %{IPORHOST:source.address}(?::%{POSINT:source.port:int})?\])?(?: %{DATA:error.code}:)? %{GREEDYDATA:message} +# :long - %{INT:process.thread.id:int} +HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG} + +# Deprecated +COMMONAPACHELOG %{HTTPD_COMMONLOG} +COMBINEDAPACHELOG %{HTTPD_COMBINEDLOG} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/java b/src/platform/packages/shared/kbn-grok-ui/patterns/java new file mode 100644 index 0000000000000..8dd539f6c0283 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/java @@ -0,0 +1,34 @@ +JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]* +#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source' +JAVAFILE (?:[a-zA-Z$_0-9. -]+) +#Allow special , methods +JAVAMETHOD (?:(<(?:cl)?init>)|[a-zA-Z$_][a-zA-Z$_0-9]*) +#Line number is optional in special cases 'Native method' or 'Unknown source' +JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:java.log.origin.class.name}\.%{JAVAMETHOD:log.origin.function}\(%{JAVAFILE:log.origin.file.name}(?::%{INT:log.origin.file.line:int})?\) +# Java Logs +JAVATHREAD (?:[A-Z]{2}-Processor[\d]+) +JAVALOGMESSAGE (?:.*) + +# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM +# matches default logging configuration in Tomcat 4.1, 5.0, 5.5, 6.0, 7.0 +CATALINA7_DATESTAMP %{MONTH} %{MONTHDAY}, %{YEAR} %{HOUR}:%{MINUTE}:%{SECOND} (?:AM|PM) +CATALINA7_LOG %{CATALINA7_DATESTAMP:timestamp} %{JAVACLASS:java.log.origin.class.name}(?: %{JAVAMETHOD:log.origin.function})?\s*(?:%{LOGLEVEL:log.level}:)? %{JAVALOGMESSAGE:message} + +# 31-Jul-2020 16:40:38.578 in Tomcat 8.5/9.0 +CATALINA8_DATESTAMP %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}:%{SECOND} +CATALINA8_LOG %{CATALINA8_DATESTAMP:timestamp} %{LOGLEVEL:log.level} \[%{DATA:java.log.origin.thread.name}\] %{JAVACLASS:java.log.origin.class.name}\.(?:%{JAVAMETHOD:log.origin.function})? %{JAVALOGMESSAGE:message} + +CATALINA_DATESTAMP (?:%{CATALINA8_DATESTAMP})|(?:%{CATALINA7_DATESTAMP}) +CATALINALOG (?:%{CATALINA8_LOG})|(?:%{CATALINA7_LOG}) + +# in Tomcat 5.5, 6.0, 7.0 it is the same as catalina.out logging format +TOMCAT7_LOG %{CATALINA7_LOG} +TOMCAT8_LOG %{CATALINA8_LOG} + +# NOTE: a weird log we started with - not sure what TC version this should match out of the box (due the | delimiters) +TOMCATLEGACY_DATESTAMP %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}(?: %{ISO8601_TIMEZONE})? +TOMCATLEGACY_LOG %{TOMCATLEGACY_DATESTAMP:timestamp} \| %{LOGLEVEL:log.level} \| %{JAVACLASS:java.log.origin.class.name} - %{JAVALOGMESSAGE:message} + +TOMCAT_DATESTAMP (?:%{CATALINA8_DATESTAMP})|(?:%{CATALINA7_DATESTAMP})|(?:%{TOMCATLEGACY_DATESTAMP}) + +TOMCATLOG (?:%{TOMCAT8_LOG})|(?:%{TOMCAT7_LOG})|(?:%{TOMCATLEGACY_LOG}) diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/junos b/src/platform/packages/shared/kbn-grok-ui/patterns/junos new file mode 100644 index 0000000000000..d23d45502aa19 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/junos @@ -0,0 +1,13 @@ +# JUNOS 11.4 RT_FLOW patterns +RT_FLOW_TAG (?:RT_FLOW_SESSION_CREATE|RT_FLOW_SESSION_CLOSE|RT_FLOW_SESSION_DENY) +# deprecated legacy name: +RT_FLOW_EVENT RT_FLOW_TAG + +RT_FLOW1 %{RT_FLOW_TAG:juniper.srx.tag}: %{GREEDYDATA:juniper.srx.reason}: %{IP:source.ip}/%{INT:source.port:int}->%{IP:destination.ip}/%{INT:destination.port:int} %{DATA:juniper.srx.service_name} %{IP:source.nat.ip}/%{INT:source.nat.port:int}->%{IP:destination.nat.ip}/%{INT:destination.nat.port:int} (?:(?:None)|(?:%{DATA:juniper.srx.src_nat_rule_name})) (?:(?:None)|(?:%{DATA:juniper.srx.dst_nat_rule_name})) %{INT:network.iana_number} %{DATA:rule.name} %{DATA:observer.ingress.zone} %{DATA:observer.egress.zone} %{INT:juniper.srx.session_id} \d+\(%{INT:source.bytes:long}\) \d+\(%{INT:destination.bytes:long}\) %{INT:juniper.srx.elapsed_time:int} .* +# :long - %{INT:source.bytes:int} +# :long - %{INT:destination.bytes:int} + +RT_FLOW2 %{RT_FLOW_TAG:juniper.srx.tag}: session created %{IP:source.ip}/%{INT:source.port:int}->%{IP:destination.ip}/%{INT:destination.port:int} %{DATA:juniper.srx.service_name} %{IP:source.nat.ip}/%{INT:source.nat.port:int}->%{IP:destination.nat.ip}/%{INT:destination.nat.port:int} (?:(?:None)|(?:%{DATA:juniper.srx.src_nat_rule_name})) (?:(?:None)|(?:%{DATA:juniper.srx.dst_nat_rule_name})) %{INT:network.iana_number} %{DATA:rule.name} %{DATA:observer.ingress.zone} %{DATA:observer.egress.zone} %{INT:juniper.srx.session_id} .* + +RT_FLOW3 %{RT_FLOW_TAG:juniper.srx.tag}: session denied %{IP:source.ip}/%{INT:source.port:int}->%{IP:destination.ip}/%{INT:destination.port:int} %{DATA:juniper.srx.service_name} %{INT:network.iana_number}\(\d\) %{DATA:rule.name} %{DATA:observer.ingress.zone} %{DATA:observer.egress.zone} .* + diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/linux_syslog b/src/platform/packages/shared/kbn-grok-ui/patterns/linux_syslog new file mode 100644 index 0000000000000..f2582f506c099 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/linux_syslog @@ -0,0 +1,16 @@ +SYSLOG5424PRINTASCII [!-~]+ + +SYSLOGBASE2 (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp})(?: %{SYSLOGFACILITY})?(?: %{SYSLOGHOST:host.hostname})?(?: %{SYSLOGPROG}:)? +SYSLOGPAMSESSION %{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:system.auth.pam.module}\(%{DATA:system.auth.pam.origin}\): session %{WORD:system.auth.pam.session_state} for user %{USERNAME:user.name}(?: by %{GREEDYDATA})? + +CRON_ACTION [A-Z ]+ +CRONLOG %{SYSLOGBASE} \(%{USER:user.name}\) %{CRON_ACTION:system.cron.action} \(%{DATA:message}\) + +SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message} + +# IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424) +SYSLOG5424PRI <%{NONNEGINT:log.syslog.priority:int}> +SYSLOG5424SD \[%{DATA}\]+ +SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:system.syslog.version} +(?:-|%{TIMESTAMP_ISO8601:timestamp}) +(?:-|%{IPORHOST:host.hostname}) +(?:-|%{SYSLOG5424PRINTASCII:process.name}) +(?:-|%{POSINT:process.pid:int}) +(?:-|%{SYSLOG5424PRINTASCII:event.code}) +(?:-|%{SYSLOG5424SD:system.syslog.structured_data})? + +SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:message} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/maven b/src/platform/packages/shared/kbn-grok-ui/patterns/maven new file mode 100644 index 0000000000000..f1dc808871026 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/maven @@ -0,0 +1 @@ +MAVEN_VERSION (?:(\d+)\.)?(?:(\d+)\.)?(\*|\d+)(?:[.-](RELEASE|SNAPSHOT))? diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/mcollective b/src/platform/packages/shared/kbn-grok-ui/patterns/mcollective new file mode 100644 index 0000000000000..f797cbde8a2bd --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/mcollective @@ -0,0 +1,4 @@ +# Remember, these can be multi-line events. +MCOLLECTIVE ., \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:process.pid:int}\]%{SPACE}%{LOGLEVEL:log.level} + +MCOLLECTIVEAUDIT %{TIMESTAMP_ISO8601:timestamp}: diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/mongodb b/src/platform/packages/shared/kbn-grok-ui/patterns/mongodb new file mode 100644 index 0000000000000..7f1c03de61f21 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/mongodb @@ -0,0 +1,7 @@ +MONGO_LOG %{SYSLOGTIMESTAMP:timestamp} \[%{WORD:mongodb.component}\] %{GREEDYDATA:message} +MONGO_QUERY \{ (?<={ ).*(?= } ntoreturn:) \} +MONGO_SLOWQUERY %{WORD:mongodb.profile.op} %{MONGO_WORDDASH:mongodb.database}\.%{MONGO_WORDDASH:mongodb.collection} %{WORD}: %{MONGO_QUERY:mongodb.query.original} ntoreturn:%{NONNEGINT:mongodb.profile.ntoreturn:int} ntoskip:%{NONNEGINT:mongodb.profile.ntoskip:int} nscanned:%{NONNEGINT:mongodb.profile.nscanned:int}.*? nreturned:%{NONNEGINT:mongodb.profile.nreturned:int}.*? %{INT:mongodb.profile.duration:int}ms +MONGO_WORDDASH \b[\w-]+\b +MONGO3_SEVERITY \w +MONGO3_COMPONENT %{WORD} +MONGO3_LOG %{TIMESTAMP_ISO8601:timestamp} %{MONGO3_SEVERITY:log.level} (?:-|%{MONGO3_COMPONENT:mongodb.component})%{SPACE}(?:\[%{DATA:mongodb.context}\])? %{GREEDYDATA:message} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/nagios b/src/platform/packages/shared/kbn-grok-ui/patterns/nagios new file mode 100644 index 0000000000000..d0a3b423b14f7 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/nagios @@ -0,0 +1,124 @@ +################################################################################## +################################################################################## +# Chop Nagios log files to smithereens! +# +# A set of GROK filters to process logfiles generated by Nagios. +# While it does not, this set intends to cover all possible Nagios logs. +# +# Some more work needs to be done to cover all External Commands: +# http://old.nagios.org/developerinfo/externalcommands/commandlist.php +# +# If you need some support on these rules please contact: +# Jelle Smet http://smetj.net +# +################################################################################# +################################################################################# + +NAGIOSTIME \[%{NUMBER:timestamp}\] + +############################################### +######## Begin nagios log types +############################################### +NAGIOS_TYPE_CURRENT_SERVICE_STATE CURRENT SERVICE STATE +NAGIOS_TYPE_CURRENT_HOST_STATE CURRENT HOST STATE + +NAGIOS_TYPE_SERVICE_NOTIFICATION SERVICE NOTIFICATION +NAGIOS_TYPE_HOST_NOTIFICATION HOST NOTIFICATION + +NAGIOS_TYPE_SERVICE_ALERT SERVICE ALERT +NAGIOS_TYPE_HOST_ALERT HOST ALERT + +NAGIOS_TYPE_SERVICE_FLAPPING_ALERT SERVICE FLAPPING ALERT +NAGIOS_TYPE_HOST_FLAPPING_ALERT HOST FLAPPING ALERT + +NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT SERVICE DOWNTIME ALERT +NAGIOS_TYPE_HOST_DOWNTIME_ALERT HOST DOWNTIME ALERT + +NAGIOS_TYPE_PASSIVE_SERVICE_CHECK PASSIVE SERVICE CHECK +NAGIOS_TYPE_PASSIVE_HOST_CHECK PASSIVE HOST CHECK + +NAGIOS_TYPE_SERVICE_EVENT_HANDLER SERVICE EVENT HANDLER +NAGIOS_TYPE_HOST_EVENT_HANDLER HOST EVENT HANDLER + +NAGIOS_TYPE_EXTERNAL_COMMAND EXTERNAL COMMAND +NAGIOS_TYPE_TIMEPERIOD_TRANSITION TIMEPERIOD TRANSITION +############################################### +######## End nagios log types +############################################### + +############################################### +######## Begin external check types +############################################### +NAGIOS_EC_DISABLE_SVC_CHECK DISABLE_SVC_CHECK +NAGIOS_EC_ENABLE_SVC_CHECK ENABLE_SVC_CHECK +NAGIOS_EC_DISABLE_HOST_CHECK DISABLE_HOST_CHECK +NAGIOS_EC_ENABLE_HOST_CHECK ENABLE_HOST_CHECK +NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT PROCESS_SERVICE_CHECK_RESULT +NAGIOS_EC_PROCESS_HOST_CHECK_RESULT PROCESS_HOST_CHECK_RESULT +NAGIOS_EC_SCHEDULE_SERVICE_DOWNTIME SCHEDULE_SERVICE_DOWNTIME +NAGIOS_EC_SCHEDULE_HOST_DOWNTIME SCHEDULE_HOST_DOWNTIME +NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS DISABLE_HOST_SVC_NOTIFICATIONS +NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS ENABLE_HOST_SVC_NOTIFICATIONS +NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS DISABLE_HOST_NOTIFICATIONS +NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS ENABLE_HOST_NOTIFICATIONS +NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS DISABLE_SVC_NOTIFICATIONS +NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS ENABLE_SVC_NOTIFICATIONS +############################################### +######## End external check types +############################################### +NAGIOS_WARNING Warning:%{SPACE}%{GREEDYDATA:message} + +NAGIOS_CURRENT_SERVICE_STATE %{NAGIOS_TYPE_CURRENT_SERVICE_STATE:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{DATA:nagios.log.state_type};%{INT:nagios.log.attempt:int};%{GREEDYDATA:message} +NAGIOS_CURRENT_HOST_STATE %{NAGIOS_TYPE_CURRENT_HOST_STATE:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{DATA:nagios.log.state_type};%{INT:nagios.log.attempt:int};%{GREEDYDATA:message} + +NAGIOS_SERVICE_NOTIFICATION %{NAGIOS_TYPE_SERVICE_NOTIFICATION:nagios.log.type}: %{DATA:user.name};%{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{DATA:nagios.log.notification_command};%{GREEDYDATA:message} +NAGIOS_HOST_NOTIFICATION %{NAGIOS_TYPE_HOST_NOTIFICATION:nagios.log.type}: %{DATA:user.name};%{DATA:host.hostname};%{DATA:service.state};%{DATA:nagios.log.notification_command};%{GREEDYDATA:message} + +NAGIOS_SERVICE_ALERT %{NAGIOS_TYPE_SERVICE_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{DATA:nagios.log.state_type};%{INT:nagios.log.attempt:int};%{GREEDYDATA:message} +NAGIOS_HOST_ALERT %{NAGIOS_TYPE_HOST_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{DATA:nagios.log.state_type};%{INT:nagios.log.attempt:int};%{GREEDYDATA:message} + +NAGIOS_SERVICE_FLAPPING_ALERT %{NAGIOS_TYPE_SERVICE_FLAPPING_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{GREEDYDATA:message} +NAGIOS_HOST_FLAPPING_ALERT %{NAGIOS_TYPE_HOST_FLAPPING_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{GREEDYDATA:message} + +NAGIOS_SERVICE_DOWNTIME_ALERT %{NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{GREEDYDATA:nagios.log.comment} +NAGIOS_HOST_DOWNTIME_ALERT %{NAGIOS_TYPE_HOST_DOWNTIME_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{GREEDYDATA:nagios.log.comment} + +NAGIOS_PASSIVE_SERVICE_CHECK %{NAGIOS_TYPE_PASSIVE_SERVICE_CHECK:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{GREEDYDATA:nagios.log.comment} +NAGIOS_PASSIVE_HOST_CHECK %{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{GREEDYDATA:nagios.log.comment} + +NAGIOS_SERVICE_EVENT_HANDLER %{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{DATA:nagios.log.state_type};%{DATA:nagios.log.event_handler_name} +NAGIOS_HOST_EVENT_HANDLER %{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{DATA:nagios.log.state_type};%{DATA:nagios.log.event_handler_name} + +NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios.log.type}: %{DATA:service.name};%{NUMBER:nagios.log.period_from:int};%{NUMBER:nagios.log.period_to:int} + +#################### +#### External checks +#################### + +#Disable host & service check +NAGIOS_EC_LINE_DISABLE_SVC_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_SVC_CHECK:nagios.log.command};%{DATA:host.hostname};%{DATA:service.name} +NAGIOS_EC_LINE_DISABLE_HOST_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_HOST_CHECK:nagios.log.command};%{DATA:host.hostname} + +#Enable host & service check +NAGIOS_EC_LINE_ENABLE_SVC_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_SVC_CHECK:nagios.log.command};%{DATA:host.hostname};%{DATA:service.name} +NAGIOS_EC_LINE_ENABLE_HOST_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_HOST_CHECK:nagios.log.command};%{DATA:host.hostname} + +#Process host & service check +NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT:nagios.log.command};%{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{GREEDYDATA:nagios.log.check_result} +NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_PROCESS_HOST_CHECK_RESULT:nagios.log.command};%{DATA:host.hostname};%{DATA:service.state};%{GREEDYDATA:nagios.log.check_result} + +#Disable host & service notifications +NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS:nagios.log.command};%{GREEDYDATA:host.hostname} +NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS:nagios.log.command};%{GREEDYDATA:host.hostname} +NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS:nagios.log.command};%{DATA:host.hostname};%{GREEDYDATA:service.name} + +#Enable host & service notifications +NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS:nagios.log.command};%{GREEDYDATA:host.hostname} +NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS:nagios.log.command};%{GREEDYDATA:host.hostname} +NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS:nagios.log.command};%{DATA:host.hostname};%{GREEDYDATA:service.name} + +#Schedule host & service downtime +NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_SCHEDULE_HOST_DOWNTIME:nagios.log.command};%{DATA:host.hostname};%{NUMBER:nagios.log.start_time};%{NUMBER:nagios.log.end_time};%{NUMBER:nagios.log.fixed};%{NUMBER:nagios.log.trigger_id};%{NUMBER:nagios.log.duration:int};%{DATA:user.name};%{DATA:nagios.log.comment} + +#End matching line +NAGIOSLOGLINE %{NAGIOSTIME} (?:%{NAGIOS_WARNING}|%{NAGIOS_CURRENT_SERVICE_STATE}|%{NAGIOS_CURRENT_HOST_STATE}|%{NAGIOS_SERVICE_NOTIFICATION}|%{NAGIOS_HOST_NOTIFICATION}|%{NAGIOS_SERVICE_ALERT}|%{NAGIOS_HOST_ALERT}|%{NAGIOS_SERVICE_FLAPPING_ALERT}|%{NAGIOS_HOST_FLAPPING_ALERT}|%{NAGIOS_SERVICE_DOWNTIME_ALERT}|%{NAGIOS_HOST_DOWNTIME_ALERT}|%{NAGIOS_PASSIVE_SERVICE_CHECK}|%{NAGIOS_PASSIVE_HOST_CHECK}|%{NAGIOS_SERVICE_EVENT_HANDLER}|%{NAGIOS_HOST_EVENT_HANDLER}|%{NAGIOS_TIMEPERIOD_TRANSITION}|%{NAGIOS_EC_LINE_DISABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_ENABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_DISABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_ENABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT}|%{NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT}|%{NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME}|%{NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS}) diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/postgresql b/src/platform/packages/shared/kbn-grok-ui/patterns/postgresql new file mode 100644 index 0000000000000..cbfd5a690c4d2 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/postgresql @@ -0,0 +1,2 @@ +# Default postgresql pg_log format pattern +POSTGRESQL %{DATESTAMP:timestamp} %{TZ:event.timezone} %{DATA:user.name} %{GREEDYDATA:postgresql.log.connection_id} %{POSINT:process.pid:int} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/rails b/src/platform/packages/shared/kbn-grok-ui/patterns/rails new file mode 100644 index 0000000000000..81717d9b8ffbe --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/rails @@ -0,0 +1,13 @@ +RUUID \h{32} +# rails controller with action +RCONTROLLER (?[^#]+)#(?\w+) + +# this will often be the only line: +RAILS3HEAD (?m)Started %{WORD:http.request.method} "%{URIPATHPARAM:url.original}" for %{IPORHOST:source.address} at (?%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}) +# for some a strange reason, params are stripped of {} - not sure that's a good idea. +RPROCESSING \W*Processing by %{RCONTROLLER} as (?\S+)(?:\W*Parameters: {%{DATA:rails.request.params}}\W*)? +RAILS3FOOT Completed %{POSINT:http.response.status_code:int}%{DATA} in %{NUMBER:rails.request.duration.total:float}ms %{RAILS3PROFILE}%{GREEDYDATA} +RAILS3PROFILE (?:\(Views: %{NUMBER:rails.request.duration.view:float}ms \| ActiveRecord: %{NUMBER:rails.request.duration.active_record:float}ms|\(ActiveRecord: %{NUMBER:rails.request.duration.active_record:float}ms)? + +# putting it all together +RAILS3 %{RAILS3HEAD}(?:%{RPROCESSING})?(?(?:%{DATA}\n)*)(?:%{RAILS3FOOT})? diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/redis b/src/platform/packages/shared/kbn-grok-ui/patterns/redis new file mode 100644 index 0000000000000..063290ed80dd9 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/redis @@ -0,0 +1,3 @@ +REDISTIMESTAMP %{MONTHDAY} %{MONTH} %{TIME} +REDISLOG \[%{POSINT:process.pid:int}\] %{REDISTIMESTAMP:timestamp} \* +REDISMONLOG %{NUMBER:timestamp} \[%{INT:redis.database.id} %{IP:client.ip}:%{POSINT:client.port:int}\] "%{WORD:redis.command.name}"\s?%{GREEDYDATA:redis.command.args} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/ruby b/src/platform/packages/shared/kbn-grok-ui/patterns/ruby new file mode 100644 index 0000000000000..2c9a7cedd5146 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/ruby @@ -0,0 +1,2 @@ +RUBY_LOGLEVEL (?:DEBUG|FATAL|ERROR|WARN|INFO) +RUBY_LOGGER [DFEWI], \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:process.pid:int}\] *%{RUBY_LOGLEVEL:log.level} -- +%{DATA:process.name}: %{GREEDYDATA:message} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/squid b/src/platform/packages/shared/kbn-grok-ui/patterns/squid new file mode 100644 index 0000000000000..dfff4f623f095 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/squid @@ -0,0 +1,6 @@ +# Pattern squid3 +# Documentation of squid3 logs formats can be found at the following link: +# http://wiki.squid-cache.org/Features/LogFormat +SQUID3_STATUS (?:%{POSINT:http.response.status_code:int}|0|000) +SQUID3 %{NUMBER:timestamp}\s+%{NUMBER:squid.request.duration:int}\s%{IP:source.ip}\s%{WORD:event.action}/%{SQUID3_STATUS}\s%{INT:http.response.bytes:long}\s%{WORD:http.request.method}\s%{NOTSPACE:url.original}\s(?:-|%{NOTSPACE:user.name})\s%{WORD:squid.hierarchy_code}/(?:-|%{IPORHOST:destination.address})\s(?:-|%{NOTSPACE:http.response.mime_type}) +# :long - %{INT:http.response.bytes:int} diff --git a/src/platform/packages/shared/kbn-grok-ui/patterns/zeek b/src/platform/packages/shared/kbn-grok-ui/patterns/zeek new file mode 100644 index 0000000000000..397e84aa17c35 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/patterns/zeek @@ -0,0 +1,33 @@ +# updated Zeek log matching, for legacy matching see the patters/ecs-v1/bro + +ZEEK_BOOL [TF] +ZEEK_DATA [^\t]+ + +# http.log - the 'new' format (compared to BRO_HTTP) +# has *version* and *origin* fields added and *filename* replaced with *orig_filenames* + *resp_filenames* +ZEEK_HTTP %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{INT:zeek.http.trans_depth:int}\t(?:-|%{WORD:http.request.method})\t(?:-|%{ZEEK_DATA:url.domain})\t(?:-|%{ZEEK_DATA:url.original})\t(?:-|%{ZEEK_DATA:http.request.referrer})\t(?:-|%{NUMBER:http.version})\t(?:-|%{ZEEK_DATA:user_agent.original})\t(?:-|%{ZEEK_DATA:zeek.http.origin})\t(?:-|%{NUMBER:http.request.body.bytes:long})\t(?:-|%{NUMBER:http.response.body.bytes:long})\t(?:-|%{POSINT:http.response.status_code:int})\t(?:-|%{DATA:zeek.http.status_msg})\t(?:-|%{POSINT:zeek.http.info_code:int})\t(?:-|%{DATA:zeek.http.info_msg})\t(?:\(empty\)|%{ZEEK_DATA:zeek.http.tags})\t(?:-|%{ZEEK_DATA:url.username})\t(?:-|%{ZEEK_DATA:url.password})\t(?:-|%{ZEEK_DATA:zeek.http.proxied})\t(?:-|%{ZEEK_DATA:zeek.http.orig_fuids})\t(?:-|%{ZEEK_DATA:zeek.http.orig_filenames})\t(?:-|%{ZEEK_DATA:http.request.mime_type})\t(?:-|%{ZEEK_DATA:zeek.http.resp_fuids})\t(?:-|%{ZEEK_DATA:zeek.http.resp_filenames})\t(?:-|%{ZEEK_DATA:http.response.mime_type}) +# :long - %{NUMBER:http.request.body.bytes:int} +# :long - %{NUMBER:http.response.body.bytes:int} + +# dns.log - 'updated' BRO_DNS format (added *zeek.dns.rtt*) +ZEEK_DNS %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{WORD:network.transport}\t(?:-|%{INT:dns.id:int})\t(?:-|%{NUMBER:zeek.dns.rtt:float})\t(?:-|%{ZEEK_DATA:dns.question.name})\t(?:-|%{INT:zeek.dns.qclass:int})\t(?:-|%{ZEEK_DATA:zeek.dns.qclass_name})\t(?:-|%{INT:zeek.dns.qtype:int})\t(?:-|%{ZEEK_DATA:dns.question.type})\t(?:-|%{INT:zeek.dns.rcode:int})\t(?:-|%{ZEEK_DATA:dns.response_code})\t%{ZEEK_BOOL:zeek.dns.AA}\t%{ZEEK_BOOL:zeek.dns.TC}\t%{ZEEK_BOOL:zeek.dns.RD}\t%{ZEEK_BOOL:zeek.dns.RA}\t%{NONNEGINT:zeek.dns.Z:int}\t(?:-|%{ZEEK_DATA:zeek.dns.answers})\t(?:-|%{DATA:zeek.dns.TTLs})\t(?:-|%{ZEEK_BOOL:zeek.dns.rejected}) + +# conn.log - the 'new' format (requires *zeek.connection.local_resp*, handles `(empty)` as `-` for tunnel_parents, and optional mac adresses) +ZEEK_CONN %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{WORD:network.transport}\t(?:-|%{ZEEK_DATA:network.protocol})\t(?:-|%{NUMBER:zeek.connection.duration:float})\t(?:-|%{INT:zeek.connection.orig_bytes:long})\t(?:-|%{INT:zeek.connection.resp_bytes:long})\t(?:-|%{ZEEK_DATA:zeek.connection.state})\t(?:-|%{ZEEK_BOOL:zeek.connection.local_orig})\t(?:-|%{ZEEK_BOOL:zeek.connection.local_resp})\t(?:-|%{INT:zeek.connection.missed_bytes:long})\t(?:-|%{ZEEK_DATA:zeek.connection.history})\t(?:-|%{INT:source.packets:long})\t(?:-|%{INT:source.bytes:long})\t(?:-|%{INT:destination.packets:long})\t(?:-|%{INT:destination.bytes:long})\t(?:-|%{ZEEK_DATA:zeek.connection.tunnel_parents})(?:\t(?:-|%{COMMONMAC:source.mac})\t(?:-|%{COMMONMAC:destination.mac}))? +# :long - %{INT:zeek.connection.orig_bytes:int} +# :long - %{INT:zeek.connection.resp_bytes:int} +# :long - %{INT:zeek.connection.missed_bytes:int} +# :long - %{INT:source.packets:int} +# :long - %{INT:source.bytes:int} +# :long - %{INT:destination.packets:int} +# :long - %{INT:destination.bytes:int} + +# files.log - updated BRO_FILES format (2 new fields added at the end) +ZEEK_FILES_TX_HOSTS (?:-|%{IP:server.ip})|(?%{IP:server.ip}(?:\s,%{IP})+) +ZEEK_FILES_RX_HOSTS (?:-|%{IP:client.ip})|(?%{IP:client.ip}(?:\s,%{IP})+) +ZEEK_FILES %{NUMBER:timestamp}\t%{NOTSPACE:zeek.files.fuid}\t%{ZEEK_FILES_TX_HOSTS}\t%{ZEEK_FILES_RX_HOSTS}\t(?:-|%{ZEEK_DATA:zeek.files.session_ids})\t(?:-|%{ZEEK_DATA:zeek.files.source})\t(?:-|%{INT:zeek.files.depth:int})\t(?:-|%{ZEEK_DATA:zeek.files.analyzers})\t(?:-|%{ZEEK_DATA:file.mime_type})\t(?:-|%{ZEEK_DATA:file.name})\t(?:-|%{NUMBER:zeek.files.duration:float})\t(?:-|%{ZEEK_DATA:zeek.files.local_orig})\t(?:-|%{ZEEK_BOOL:zeek.files.is_orig})\t(?:-|%{INT:zeek.files.seen_bytes:long})\t(?:-|%{INT:file.size:long})\t(?:-|%{INT:zeek.files.missing_bytes:long})\t(?:-|%{INT:zeek.files.overflow_bytes:long})\t(?:-|%{ZEEK_BOOL:zeek.files.timedout})\t(?:-|%{ZEEK_DATA:zeek.files.parent_fuid})\t(?:-|%{ZEEK_DATA:file.hash.md5})\t(?:-|%{ZEEK_DATA:file.hash.sha1})\t(?:-|%{ZEEK_DATA:file.hash.sha256})\t(?:-|%{ZEEK_DATA:zeek.files.extracted})(?:\t(?:-|%{ZEEK_BOOL:zeek.files.extracted_cutoff})\t(?:-|%{INT:zeek.files.extracted_size:long}))? +# :long - %{INT:zeek.files.seen_bytes:int} +# :long - %{INT:file.size:int} +# :long - %{INT:zeek.files.missing_bytes:int} +# :long - %{INT:zeek.files.overflow_bytes:int} +# :long - %{INT:zeek.files.extracted_size:int} diff --git a/src/platform/packages/shared/kbn-grok-ui/scripts/README.md b/src/platform/packages/shared/kbn-grok-ui/scripts/README.md new file mode 100644 index 0000000000000..44a11d6bc046e --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/scripts/README.md @@ -0,0 +1,2 @@ +Takes the patterns from ./patterns and generates an object that can later be used with a GrokCollection instance. +The patterns held in ./patterns are just a replica of: https://github.com/elastic/elasticsearch/tree/main/libs/grok/src/main/resources/patterns/ecs-v1 \ No newline at end of file diff --git a/src/platform/packages/shared/kbn-grok-ui/scripts/generate_patterns.js b/src/platform/packages/shared/kbn-grok-ui/scripts/generate_patterns.js new file mode 100644 index 0000000000000..093866cb62113 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/scripts/generate_patterns.js @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +require('../../../../../setup_node_env'); +require('./generate_patterns_cli'); diff --git a/src/platform/packages/shared/kbn-grok-ui/scripts/generate_patterns_cli.ts b/src/platform/packages/shared/kbn-grok-ui/scripts/generate_patterns_cli.ts new file mode 100644 index 0000000000000..4271fab207ff6 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/scripts/generate_patterns_cli.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { run } from '@kbn/dev-cli-runner'; +import { REPO_ROOT } from '@kbn/repo-info'; +import { readdirSync, readFileSync, writeFileSync } from 'fs'; +import Path from 'path'; + +const PATTERNS_DIR = Path.resolve(REPO_ROOT, 'src/platform/packages/shared/kbn-grok-ui/patterns'); +const PATTERN_MAP_DIR = Path.resolve( + REPO_ROOT, + 'src/platform/packages/shared/kbn-grok-ui/constants' +); +const patternLineRegex = /^([A-Z0-9_]+)\s+(.+)/; +const splitLineRegex = /\r?\n/; + +run(async ({ log }) => { + const files = readdirSync(PATTERNS_DIR); + + const patternMap: Record = {}; + + const writePatternMap = (patterns: Record) => { + const patternMapContent = ` + // !This file is auto-generated by generate_patterns_cli.ts! + export const PATTERN_MAP = ${JSON.stringify(patterns, null, 2)};`; + writeFileSync(Path.join(PATTERN_MAP_DIR, 'pattern_map.ts'), patternMapContent); + }; + + const handleFile = (fileContent: Buffer, patterns: Record) => { + if (fileContent) { + const lines = fileContent.toString().split(splitLineRegex); + + if (lines && lines.length) { + lines.forEach((line) => { + const elements = patternLineRegex.exec(line); + + if (elements && elements.length > 2) { + patterns[elements[1]] = elements[2]; + } + }); + } + } + }; + + files.forEach((file, index) => { + const fileContent = readFileSync(Path.join(PATTERNS_DIR, file)); + handleFile(fileContent, patternMap); + if (index === files.length - 1) { + writePatternMap(patternMap); + log.success('All patterns generated successfully'); + } + }); +}); diff --git a/src/platform/packages/shared/kbn-grok-ui/tsconfig.json b/src/platform/packages/shared/kbn-grok-ui/tsconfig.json new file mode 100644 index 0000000000000..568bd19aa89e5 --- /dev/null +++ b/src/platform/packages/shared/kbn-grok-ui/tsconfig.json @@ -0,0 +1,28 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node", + "react", + "@testing-library/jest-dom", + "@emotion/react/types/css-prop" + ] + }, + "include": [ + "**/*.ts", + "**/*.tsx" + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/monaco", + "@kbn/dev-cli-runner", + "@kbn/repo-info", + "@kbn/code-editor", + "@kbn/i18n", + "@kbn/object-utils", + ] +} diff --git a/src/platform/packages/shared/kbn-object-utils/index.ts b/src/platform/packages/shared/kbn-object-utils/index.ts index f5051ab8b042c..9b359e9d763fa 100644 --- a/src/platform/packages/shared/kbn-object-utils/index.ts +++ b/src/platform/packages/shared/kbn-object-utils/index.ts @@ -9,3 +9,4 @@ export * from './src/calculate_object_diff'; export * from './src/flatten_object'; +export * from './src/unflatten_object'; diff --git a/src/platform/packages/shared/kbn-object-utils/src/unflatten_object.test.ts b/src/platform/packages/shared/kbn-object-utils/src/unflatten_object.test.ts new file mode 100644 index 0000000000000..5d15ba2ca779a --- /dev/null +++ b/src/platform/packages/shared/kbn-object-utils/src/unflatten_object.test.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { unflattenObject } from './unflatten_object'; + +describe('unflattenObject', () => { + it('unflattens deeply nested objects', () => { + expect(unflattenObject({ 'first.second.third': 'third' })).toEqual({ + first: { + second: { + third: 'third', + }, + }, + }); + }); + + it('does not unflatten arrays', () => { + expect( + unflattenObject({ + simpleArray: ['0', '1', '2'], + complexArray: [{ one: 'one', two: 'two', three: 'three' }], + 'nested.array': [0, 1, 2], + 'complex.nested': [{ one: 'one', two: 'two', 'first.second': 'foo', 'first.third': 'bar' }], + }) + ).toEqual({ + simpleArray: ['0', '1', '2'], + complexArray: [{ one: 'one', two: 'two', three: 'three' }], + nested: { + array: [0, 1, 2], + }, + complex: { + nested: [{ one: 'one', two: 'two', first: { second: 'foo', third: 'bar' } }], + }, + }); + }); + + it('handles null values correctly', () => { + expect( + unflattenObject({ + 'agent.name': null, + }) + ).toEqual({ + agent: { + name: null, + }, + }); + }); +}); diff --git a/src/platform/packages/shared/kbn-object-utils/src/unflatten_object.ts b/src/platform/packages/shared/kbn-object-utils/src/unflatten_object.ts new file mode 100644 index 0000000000000..752fabe0972a9 --- /dev/null +++ b/src/platform/packages/shared/kbn-object-utils/src/unflatten_object.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { set } from '@kbn/safer-lodash-set'; +import { DedotObject } from '@kbn/utility-types'; + +export function unflattenObject>( + source: T, + target: Record = {} +): DedotObject { + // eslint-disable-next-line guard-for-in + for (const key in source) { + const val = source[key as keyof typeof source]; + if (Array.isArray(val)) { + const unflattenedArray = val.map((item: unknown) => { + if (item && typeof item === 'object' && !Array.isArray(item)) { + return unflattenObject(item); + } + return item; + }); + set(target, key, unflattenedArray); + } else { + set(target, key, val); + } + } + + return target as DedotObject; +} diff --git a/src/platform/packages/shared/kbn-object-utils/tsconfig.json b/src/platform/packages/shared/kbn-object-utils/tsconfig.json index 2e04c4a6f5733..3f083b7a24636 100644 --- a/src/platform/packages/shared/kbn-object-utils/tsconfig.json +++ b/src/platform/packages/shared/kbn-object-utils/tsconfig.json @@ -6,5 +6,8 @@ "exclude": ["target/**/*"], "extends": "../../../../../tsconfig.base.json", "include": ["**/*.ts"], - "kbn_references": [] + "kbn_references": [ + "@kbn/safer-lodash-set", + "@kbn/utility-types", + ] } diff --git a/tsconfig.base.json b/tsconfig.base.json index fd60fb2e66f74..ef4e9bca35565 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1024,6 +1024,8 @@ "@kbn/grid-example-plugin/*": ["examples/grid_example/*"], "@kbn/grid-layout": ["src/platform/packages/private/kbn-grid-layout"], "@kbn/grid-layout/*": ["src/platform/packages/private/kbn-grid-layout/*"], + "@kbn/grok-ui": ["src/platform/packages/shared/kbn-grok-ui"], + "@kbn/grok-ui/*": ["src/platform/packages/shared/kbn-grok-ui/*"], "@kbn/grokdebugger-plugin": ["x-pack/platform/plugins/private/grokdebugger"], "@kbn/grokdebugger-plugin/*": ["x-pack/platform/plugins/private/grokdebugger/*"], "@kbn/grouping": ["src/platform/packages/shared/kbn-grouping"], diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/processors/grok/grok_patterns_editor.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/processors/grok/grok_patterns_editor.tsx index a4f37478ec595..b80f614e66d09 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/processors/grok/grok_patterns_editor.tsx +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/processors/grok/grok_patterns_editor.tsx @@ -5,6 +5,7 @@ * 2.0. */ +import React from 'react'; import { useFormContext, useFieldArray, @@ -24,7 +25,6 @@ import { EuiButtonIcon, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import React from 'react'; import { SortableList } from '../../sortable_list'; import { GrokFormState } from '../../types'; import { GrokAiSuggestions } from './grok_ai_suggestions'; diff --git a/yarn.lock b/yarn.lock index d907d52cdb293..e09e67b3e1d2a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5636,6 +5636,10 @@ version "0.0.0" uid "" +"@kbn/grok-ui@link:src/platform/packages/shared/kbn-grok-ui": + version "0.0.0" + uid "" + "@kbn/grokdebugger-plugin@link:x-pack/platform/plugins/private/grokdebugger": version "0.0.0" uid "" @@ -16707,6 +16711,11 @@ emittery@^0.13.1: resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.13.1.tgz#c04b8c3457490e0847ae51fced3af52d338e3dad" integrity sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ== +emoji-regex-xs@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/emoji-regex-xs/-/emoji-regex-xs-1.0.0.tgz#e8af22e5d9dbd7f7f22d280af3d19d2aab5b0724" + integrity sha512-LRlerrMYoIDrT6jgpeZ2YYl/L8EulRTt5hQcYjy5AInh7HWXKimpqx68aknBFpGL2+/IcogTcaydJEgaTmOpDg== + emoji-regex@10.3.0, emoji-regex@^10.2.1, emoji-regex@^10.3.0: version "10.3.0" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-10.3.0.tgz#76998b9268409eb3dae3de989254d456e70cfe23" @@ -24004,6 +24013,15 @@ onetime@^7.0.0: dependencies: mimic-function "^5.0.0" +oniguruma-to-es@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/oniguruma-to-es/-/oniguruma-to-es-3.1.1.tgz#480e4bac4d3bc9439ac0d2124f0725e7a0d76d17" + integrity sha512-bUH8SDvPkH3ho3dvwJwfonjlQ4R80vjyvrU8YpxuROddv55vAEJrTuCuCVUhhsHbtlD9tGGbaNApGQckXhS8iQ== + dependencies: + emoji-regex-xs "^1.0.0" + regex "^6.0.1" + regex-recursion "^6.0.2" + open@^10.0.3: version "10.1.0" resolved "https://registry.yarnpkg.com/open/-/open-10.1.0.tgz#a7795e6e5d519abe4286d9937bb24b51122598e1" @@ -26503,6 +26521,25 @@ regex-not@^1.0.0, regex-not@^1.0.2: extend-shallow "^3.0.2" safe-regex "^1.1.0" +regex-recursion@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/regex-recursion/-/regex-recursion-6.0.2.tgz#a0b1977a74c87f073377b938dbedfab2ea582b33" + integrity sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg== + dependencies: + regex-utilities "^2.3.0" + +regex-utilities@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/regex-utilities/-/regex-utilities-2.3.0.tgz#87163512a15dce2908cf079c8960d5158ff43280" + integrity sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng== + +regex@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/regex/-/regex-6.0.1.tgz#282fa4435d0c700b09c0eb0982b602e05ab6a34f" + integrity sha512-uorlqlzAKjKQZ5P+kTJr3eeJGSVroLKoHmquUj4zHWuR+hEyNqlXsSKlYYF5F4NI6nl7tWCs0apKJ0lmfsXAPA== + dependencies: + regex-utilities "^2.3.0" + regexp-tree@~0.1.1: version "0.1.27" resolved "https://registry.yarnpkg.com/regexp-tree/-/regexp-tree-0.1.27.tgz#2198f0ef54518ffa743fe74d983b56ffd631b6cd"