From 4daf7a6dd335c137a371700e5bd8d30e6119682b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 15 Jun 2021 17:50:02 +0200 Subject: [PATCH 1/3] Set log.original and event.original as message fields --- .../message/builtin_rules/generic.test.ts | 142 ++++++++++++++++++ .../message/builtin_rules/generic.ts | 64 +++++--- 2 files changed, 181 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts index ae5a45c61d3b5..630dce22645e1 100644 --- a/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts +++ b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts @@ -134,6 +134,50 @@ describe('Generic Rules', () => { }); describe('log.original fallback', () => { + test('includes the event.dataset and log.level if present', () => { + const flattenedDocument = { + '@timestamp': ['2016-12-26T16:22:13.000Z'], + 'event.dataset': ['generic.test'], + 'log.level': ['TEST_LEVEL'], + 'log.original': ['TEST_MESSAGE'], + }; + + expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` + Array [ + Object { + "constant": "[", + }, + Object { + "field": "event.dataset", + "highlights": Array [], + "value": Array [ + "generic.test", + ], + }, + Object { + "constant": "][", + }, + Object { + "field": "log.level", + "highlights": Array [], + "value": Array [ + "TEST_LEVEL", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "log.original", + "highlights": Array [], + "value": Array [ + "TEST_MESSAGE", + ], + }, + ] + `); + }); + test('includes the event.dataset if present', () => { const flattenedDocument = { '@timestamp': ['2016-12-26T16:22:13.000Z'], @@ -186,4 +230,102 @@ describe('Generic Rules', () => { `); }); }); + + describe('event.original fallback', () => { + test('includes the event.dataset and log.level if present', () => { + const flattenedDocument = { + '@timestamp': ['2016-12-26T16:22:13.000Z'], + 'event.dataset': ['generic.test'], + 'log.level': ['TEST_LEVEL'], + 'event.original': ['TEST_MESSAGE'], + }; + + expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` + Array [ + Object { + "constant": "[", + }, + Object { + "field": "event.dataset", + "highlights": Array [], + "value": Array [ + "generic.test", + ], + }, + Object { + "constant": "][", + }, + Object { + "field": "log.level", + "highlights": Array [], + "value": Array [ + "TEST_LEVEL", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "event.original", + "highlights": Array [], + "value": Array [ + "TEST_MESSAGE", + ], + }, + ] + `); + }); + + test('includes the event.dataset if present', () => { + const flattenedDocument = { + '@timestamp': ['2016-12-26T16:22:13.000Z'], + 'event.dataset': ['generic.test'], + 'event.original': ['TEST_MESSAGE'], + }; + + expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` + Array [ + Object { + "constant": "[", + }, + Object { + "field": "event.dataset", + "highlights": Array [], + "value": Array [ + "generic.test", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "event.original", + "highlights": Array [], + "value": Array [ + "TEST_MESSAGE", + ], + }, + ] + `); + }); + + test('includes the original message', () => { + const flattenedDocument = { + '@timestamp': ['2016-12-26T16:22:13.000Z'], + 'event.original': ['TEST_MESSAGE'], + }; + + expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` + Array [ + Object { + "field": "event.original", + "highlights": Array [], + "value": Array [ + "TEST_MESSAGE", + ], + }, + ] + `); + }); + }); }); diff --git a/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.ts index c16d65a75b3e0..7f8bba7645f52 100644 --- a/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.ts +++ b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.ts @@ -7,15 +7,17 @@ import { LogMessageFormattingRule } from '../rule_types'; -const BUILTIN_GENERIC_MESSAGE_FIELDS = ['message', '@message']; +const BUILTIN_GENERIC_MESSAGE_FIELDS = ['message', '@message', 'log.original', 'event.original']; -export const getGenericRules = (genericMessageFields: string[]) => [ - ...Array.from(new Set([...genericMessageFields, ...BUILTIN_GENERIC_MESSAGE_FIELDS])).reduce< +export const getGenericRules = (genericMessageFields: string[]) => + Array.from(new Set([...genericMessageFields, ...BUILTIN_GENERIC_MESSAGE_FIELDS])).reduce< LogMessageFormattingRule[] - >((genericRules, fieldName) => [...genericRules, ...createGenericRulesForField(fieldName)], []), + >((genericRules, fieldName) => [...genericRules, ...createGenericRulesForField(fieldName)], []); + +const createGenericRulesForField = (fieldName: string) => [ { when: { - exists: ['event.dataset', 'log.original'], + exists: ['event.dataset', 'log.level', fieldName, 'error.stack_trace.text'], }, format: [ { @@ -24,30 +26,29 @@ export const getGenericRules = (genericMessageFields: string[]) => [ { field: 'event.dataset', }, + { + constant: '][', + }, + { + field: 'log.level', + }, { constant: '] ', }, { - field: 'log.original', + field: fieldName, + }, + { + constant: '\n', }, - ], - }, - { - when: { - exists: ['log.original'], - }, - format: [ { - field: 'log.original', + field: 'error.stack_trace.text', }, ], }, -]; - -const createGenericRulesForField = (fieldName: string) => [ { when: { - exists: ['event.dataset', 'log.level', fieldName, 'error.stack_trace.text'], + exists: ['event.dataset', 'log.level', fieldName], }, format: [ { @@ -68,6 +69,25 @@ const createGenericRulesForField = (fieldName: string) => [ { field: fieldName, }, + ], + }, + { + when: { + exists: ['event.dataset', fieldName, 'error.stack_trace.text'], + }, + format: [ + { + constant: '[', + }, + { + field: 'event.dataset', + }, + { + constant: '] ', + }, + { + field: fieldName, + }, { constant: '\n', }, @@ -78,7 +98,7 @@ const createGenericRulesForField = (fieldName: string) => [ }, { when: { - exists: ['event.dataset', 'log.level', fieldName], + exists: ['event.dataset', fieldName], }, format: [ { @@ -87,12 +107,6 @@ const createGenericRulesForField = (fieldName: string) => [ { field: 'event.dataset', }, - { - constant: '][', - }, - { - field: 'log.level', - }, { constant: '] ', }, From 1683e1330b9f324d0d3bbd709e36b4e9a5a3a200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 15 Jun 2021 20:13:53 +0200 Subject: [PATCH 2/3] Revert decision to unify generic and fallback fields --- .../message/builtin_rules/generic.test.ts | 20 ------- .../message/builtin_rules/generic.ts | 59 ++++++++----------- 2 files changed, 26 insertions(+), 53 deletions(-) diff --git a/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts index 630dce22645e1..f9e8babc45092 100644 --- a/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts +++ b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts @@ -154,16 +154,6 @@ describe('Generic Rules', () => { "generic.test", ], }, - Object { - "constant": "][", - }, - Object { - "field": "log.level", - "highlights": Array [], - "value": Array [ - "TEST_LEVEL", - ], - }, Object { "constant": "] ", }, @@ -252,16 +242,6 @@ describe('Generic Rules', () => { "generic.test", ], }, - Object { - "constant": "][", - }, - Object { - "field": "log.level", - "highlights": Array [], - "value": Array [ - "TEST_LEVEL", - ], - }, Object { "constant": "] ", }, diff --git a/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.ts index 7f8bba7645f52..07b6cf03e2c5d 100644 --- a/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.ts +++ b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.ts @@ -7,12 +7,17 @@ import { LogMessageFormattingRule } from '../rule_types'; -const BUILTIN_GENERIC_MESSAGE_FIELDS = ['message', '@message', 'log.original', 'event.original']; +const BUILTIN_GENERIC_MESSAGE_FIELDS = ['message', '@message']; +const BUILTIN_FALLBACK_MESSAGE_FIELDS = ['log.original', 'event.original']; -export const getGenericRules = (genericMessageFields: string[]) => - Array.from(new Set([...genericMessageFields, ...BUILTIN_GENERIC_MESSAGE_FIELDS])).reduce< - LogMessageFormattingRule[] - >((genericRules, fieldName) => [...genericRules, ...createGenericRulesForField(fieldName)], []); +export const getGenericRules = (genericMessageFields: string[]): LogMessageFormattingRule[] => [ + ...Array.from(new Set([...genericMessageFields, ...BUILTIN_GENERIC_MESSAGE_FIELDS])).flatMap( + createGenericRulesForField + ), + ...BUILTIN_FALLBACK_MESSAGE_FIELDS.filter( + (fieldName) => !genericMessageFields.includes(fieldName) + ).flatMap(createFallbackRulesForField), +]; const createGenericRulesForField = (fieldName: string) => [ { @@ -73,14 +78,14 @@ const createGenericRulesForField = (fieldName: string) => [ }, { when: { - exists: ['event.dataset', fieldName, 'error.stack_trace.text'], + exists: ['log.level', fieldName, 'error.stack_trace.text'], }, format: [ { constant: '[', }, { - field: 'event.dataset', + field: 'log.level', }, { constant: '] ', @@ -98,14 +103,14 @@ const createGenericRulesForField = (fieldName: string) => [ }, { when: { - exists: ['event.dataset', fieldName], + exists: ['log.level', fieldName], }, format: [ { constant: '[', }, { - field: 'event.dataset', + field: 'log.level', }, { constant: '] ', @@ -117,18 +122,9 @@ const createGenericRulesForField = (fieldName: string) => [ }, { when: { - exists: ['log.level', fieldName, 'error.stack_trace.text'], + exists: [fieldName, 'error.stack_trace.text'], }, format: [ - { - constant: '[', - }, - { - field: 'log.level', - }, - { - constant: '] ', - }, { field: fieldName, }, @@ -142,36 +138,33 @@ const createGenericRulesForField = (fieldName: string) => [ }, { when: { - exists: ['log.level', fieldName], + exists: [fieldName], }, format: [ - { - constant: '[', - }, - { - field: 'log.level', - }, - { - constant: '] ', - }, { field: fieldName, }, ], }, +]; + +const createFallbackRulesForField = (fieldName: string) => [ { when: { - exists: [fieldName, 'error.stack_trace.text'], + exists: ['event.dataset', fieldName], }, format: [ { - field: fieldName, + constant: '[', }, { - constant: '\n', + field: 'event.dataset', }, { - field: 'error.stack_trace.text', + constant: '] ', + }, + { + field: fieldName, }, ], }, From cf8bb0c416d5a1d6110428ff24aa0f34294cf112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 16 Jun 2021 12:06:29 +0200 Subject: [PATCH 3/3] Remove outdated test case --- .../message/builtin_rules/generic.test.ts | 68 ------------------- 1 file changed, 68 deletions(-) diff --git a/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts index f9e8babc45092..ba8eab91e3456 100644 --- a/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts +++ b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts @@ -134,40 +134,6 @@ describe('Generic Rules', () => { }); describe('log.original fallback', () => { - test('includes the event.dataset and log.level if present', () => { - const flattenedDocument = { - '@timestamp': ['2016-12-26T16:22:13.000Z'], - 'event.dataset': ['generic.test'], - 'log.level': ['TEST_LEVEL'], - 'log.original': ['TEST_MESSAGE'], - }; - - expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` - Array [ - Object { - "constant": "[", - }, - Object { - "field": "event.dataset", - "highlights": Array [], - "value": Array [ - "generic.test", - ], - }, - Object { - "constant": "] ", - }, - Object { - "field": "log.original", - "highlights": Array [], - "value": Array [ - "TEST_MESSAGE", - ], - }, - ] - `); - }); - test('includes the event.dataset if present', () => { const flattenedDocument = { '@timestamp': ['2016-12-26T16:22:13.000Z'], @@ -222,40 +188,6 @@ describe('Generic Rules', () => { }); describe('event.original fallback', () => { - test('includes the event.dataset and log.level if present', () => { - const flattenedDocument = { - '@timestamp': ['2016-12-26T16:22:13.000Z'], - 'event.dataset': ['generic.test'], - 'log.level': ['TEST_LEVEL'], - 'event.original': ['TEST_MESSAGE'], - }; - - expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` - Array [ - Object { - "constant": "[", - }, - Object { - "field": "event.dataset", - "highlights": Array [], - "value": Array [ - "generic.test", - ], - }, - Object { - "constant": "] ", - }, - Object { - "field": "event.original", - "highlights": Array [], - "value": Array [ - "TEST_MESSAGE", - ], - }, - ] - `); - }); - test('includes the event.dataset if present', () => { const flattenedDocument = { '@timestamp': ['2016-12-26T16:22:13.000Z'],