Skip to content

[7.7] [SIEM] Update beat doc (#61902) #62801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
XavierM merged 1 commit into from
Apr 7, 2020
Merged

[7.7] [SIEM] Update beat doc (#61902) #62801

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 62 additions & 31 deletions x-pack/legacy/plugins/siem/server/lib/index_fields/elasticsearch_adapter.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,77 +23,108 @@ describe('Index Fields', () => {
).toEqual(
sortBy('name', [
{
aggregatable: true,
category: 'base',
description:
'Date/time when the event originated. For log events this is the date/time when the event was generated, and not when it was read. Required field for all events.',
'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.',
example: '2016-05-23T08:05:34.853Z',
indexes: ['auditbeat', 'filebeat', 'packetbeat'],
name: '@timestamp',
searchable: true,
type: 'date',
searchable: true,
aggregatable: true,
category: 'base',
indexes: ['auditbeat', 'filebeat', 'packetbeat'],
},
{
description: 'Each document has an _id that uniquely identifies it',
example: 'Y-6TfmcB0WOhS6qyMv3s',
footnote: '',
group: 1,
level: 'core',
name: '_id',
required: true,
type: 'string',
searchable: true,
aggregatable: false,
readFromDocValues: true,
category: '_id',
indexes: ['auditbeat', 'filebeat', 'packetbeat'],
},
{
description:
'An index is like a ‘database’ in a relational database. It has a mapping which defines multiple types. An index is a logical namespace which maps to one or more primary shards and can have zero or more replica shards.',
example: 'auditbeat-8.0.0-2019.02.19-000001',
footnote: '',
group: 1,
level: 'core',
name: '_index',
required: true,
type: 'string',
searchable: true,
aggregatable: true,
category: 'agent',
readFromDocValues: true,
category: '_index',
indexes: ['auditbeat', 'filebeat', 'packetbeat'],
},
{
description:
'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.',
'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.',
example: '8a4f500f',
indexes: ['auditbeat'],
name: 'agent.ephemeral_id',
searchable: true,
type: 'string',
},
{
searchable: true,
aggregatable: true,
category: 'agent',
indexes: ['filebeat'],
indexes: ['auditbeat'],
},
{
name: 'agent.hostname',
searchable: true,
type: 'string',
},
{
aggregatable: true,
category: 'agent',
indexes: ['filebeat'],
},
{
description:
'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.',
'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.',
example: '8a4f500d',
indexes: ['packetbeat'],
name: 'agent.id',
searchable: true,
type: 'string',
},
{
searchable: true,
aggregatable: true,
category: 'agent',
indexes: ['packetbeat'],
},
{
description:
'Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.',
'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.',
example: 'foo',
indexes: ['auditbeat', 'filebeat'],
name: 'agent.name',
searchable: true,
type: 'string',
},
{
searchable: true,
aggregatable: true,
category: 'agent',
indexes: ['auditbeat', 'filebeat'],
},
{
description:
'Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.',
'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.',
example: 'filebeat',
indexes: ['auditbeat', 'packetbeat'],
name: 'agent.type',
searchable: true,
type: 'string',
},
{
searchable: true,
aggregatable: true,
category: 'agent',
indexes: ['auditbeat', 'packetbeat'],
},
{
description: 'Version of the agent.',
example: '6.0.0-rc2',
indexes: ['auditbeat', 'filebeat'],
name: 'agent.version',
searchable: true,
type: 'string',
searchable: true,
aggregatable: true,
category: 'agent',
indexes: ['auditbeat', 'filebeat'],
},
])
);
Expand Down
55 changes: 39 additions & 16 deletions x-pack/legacy/plugins/siem/server/lib/index_fields/elasticsearch_adapter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { get } from 'lodash/fp';
import { isEmpty, get } from 'lodash/fp';

import { IndexField } from '../../graphql/types';
import {
Expand Down Expand Up @@ -51,6 +51,23 @@ export class ElasticsearchIndexFieldAdapter implements FieldsAdapter {
}
}

const missingFields = [
{
name: '_id',
type: 'string',
searchable: true,
aggregatable: false,
readFromDocValues: true,
},
{
name: '_index',
type: 'string',
searchable: true,
aggregatable: true,
readFromDocValues: true,
},
];

export const formatIndexFields = (
responsesIndexFields: IndexFieldDescriptor[][],
indexesAlias: IndexAlias[]
Expand All @@ -59,20 +76,23 @@ export const formatIndexFields = (
.reduce(
(accumulator: IndexField[], indexFields: IndexFieldDescriptor[], indexesAliasIdx: number) => [
...accumulator,
...indexFields.reduce((itemAccumulator: IndexField[], index: IndexFieldDescriptor) => {
const alias: IndexAlias = indexesAlias[indexesAliasIdx];
const splitName = index.name.split('.');
const category = baseCategoryFields.includes(splitName[0]) ? 'base' : splitName[0];
return [
...itemAccumulator,
{
...(hasDocumentation(alias, index.name) ? getDocumentation(alias, index.name) : {}),
...index,
category,
indexes: [alias],
} as IndexField,
];
}, []),
...[...missingFields, ...indexFields].reduce(
(itemAccumulator: IndexField[], index: IndexFieldDescriptor) => {
const alias: IndexAlias = indexesAlias[indexesAliasIdx];
const splitName = index.name.split('.');
const category = baseCategoryFields.includes(splitName[0]) ? 'base' : splitName[0];
return [
...itemAccumulator,
{
...(hasDocumentation(alias, index.name) ? getDocumentation(alias, index.name) : {}),
...index,
category,
indexes: [alias],
} as IndexField,
];
},
[]
),
],
[]
)
Expand All @@ -84,7 +104,10 @@ export const formatIndexFields = (
...accumulator.slice(0, alreadyExistingIndexField),
{
...existingIndexField,
indexes: [...existingIndexField.indexes, ...indexfield.indexes],
description: isEmpty(existingIndexField.description)
? indexfield.description
: existingIndexField.description,
indexes: Array.from(new Set([...existingIndexField.indexes, ...indexfield.indexes])),
},
...accumulator.slice(alreadyExistingIndexField + 1),
];
Expand Down
Loading