diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index eedc00b5f8a2..b8a84fbd102b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -197,6 +197,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add RabbitMQ module. {pull}12032[12032] - Add new `container` input. {pull}12162[12162] - Add specific date processor to convert timezones so same pipeline can be used when convert_timezone is enabled or disabled. {pull}12253[12253] +- Add MSSQL module {pull}12079[12079] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index b5e3f7121b1e..c5538cc78894 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -34,6 +34,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -7658,6 +7659,33 @@ alias to: message -- +[[exported-fields-mssql]] +== mssql fields + +MS SQL Filebeat Module + + +[float] +== mssql fields + +Fields from the MSSQL log files + + +[float] +== log fields + +Common log fields + + +*`mssql.log.origin`*:: ++ +-- +type: keyword + +Origin of the message, usually the server but it can also be a recovery process + +-- + [[exported-fields-mysql]] == MySQL fields diff --git a/filebeat/docs/modules/mssql.asciidoc b/filebeat/docs/modules/mssql.asciidoc new file mode 100644 index 000000000000..83f83a82549a --- /dev/null +++ b/filebeat/docs/modules/mssql.asciidoc @@ -0,0 +1,63 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-mssql]] +:modulename: mssql +:has-dashboards: false + +== MSSQL module + +The +{modulename}+ module parses error logs created by MSSQL. + +include::../include/what-happens.asciidoc[] + +[float] +=== Compatibility + +include::../include/running-modules.asciidoc[] + +include::../include/configuring-intro.asciidoc[] + +The following example shows how to set paths in the +modules.d/{modulename}.yml+ +file to override the default paths for Træfik logs: + +["source","yaml",subs="attributes"] +----- +- module: mssql + access: + enabled: true + var.paths: ["/var/opt/mssql/log/error*"] +----- + + +To specify the same settings at the command line, you use: + +["source","sh",subs="attributes"] +----- +-M "mssql.access.var.paths=[/var/opt/mssql/log/error*]" +----- + +//set the fileset name used in the included example +:fileset_ex: log + +include::../include/config-option-intro.asciidoc[] + +[float] +==== `log` fileset settings + +include::../include/var-paths.asciidoc[] + +:has-dashboards!: + +:fileset_ex!: + +:modulename!: + + +[float] +=== Fields + +For a description of each field in the module, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index d0ee0f35dffd..dc24b516c9b4 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -17,6 +17,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -50,6 +51,7 @@ include::modules/kafka.asciidoc[] include::modules/kibana.asciidoc[] include::modules/logstash.asciidoc[] include::modules/mongodb.asciidoc[] +include::modules/mssql.asciidoc[] include::modules/mysql.asciidoc[] include::modules/nats.asciidoc[] include::modules/netflow.asciidoc[] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index a3ab3697f2db..4e56c78a9053 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -313,6 +313,16 @@ filebeat.modules: # can be added under this section. #input: +#-------------------------------- Mssql Module -------------------------------- +- module: mssql + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + #-------------------------------- MySQL Module -------------------------------- #- module: mysql # Error logs diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index 0b15c711b356..ef08938cea86 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -13,6 +13,7 @@ import ( _ "github.com/elastic/beats/x-pack/filebeat/module/coredns" _ "github.com/elastic/beats/x-pack/filebeat/module/envoyproxy" _ "github.com/elastic/beats/x-pack/filebeat/module/iptables" + _ "github.com/elastic/beats/x-pack/filebeat/module/mssql" _ "github.com/elastic/beats/x-pack/filebeat/module/netflow" _ "github.com/elastic/beats/x-pack/filebeat/module/panw" _ "github.com/elastic/beats/x-pack/filebeat/module/rabbitmq" diff --git a/x-pack/filebeat/module/mssql/_meta/config.yml b/x-pack/filebeat/module/mssql/_meta/config.yml new file mode 100644 index 000000000000..652ca8910568 --- /dev/null +++ b/x-pack/filebeat/module/mssql/_meta/config.yml @@ -0,0 +1,8 @@ +- module: mssql + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/x-pack/filebeat/module/mssql/_meta/docs.asciidoc b/x-pack/filebeat/module/mssql/_meta/docs.asciidoc new file mode 100644 index 000000000000..3f74f125767b --- /dev/null +++ b/x-pack/filebeat/module/mssql/_meta/docs.asciidoc @@ -0,0 +1,50 @@ +:modulename: mssql +:has-dashboards: false + +== MSSQL module + +The +{modulename}+ module parses error logs created by MSSQL. + +include::../include/what-happens.asciidoc[] + +[float] +=== Compatibility + +include::../include/running-modules.asciidoc[] + +include::../include/configuring-intro.asciidoc[] + +The following example shows how to set paths in the +modules.d/{modulename}.yml+ +file to override the default paths for Træfik logs: + +["source","yaml",subs="attributes"] +----- +- module: mssql + access: + enabled: true + var.paths: ["/var/opt/mssql/log/error*"] +----- + + +To specify the same settings at the command line, you use: + +["source","sh",subs="attributes"] +----- +-M "mssql.access.var.paths=[/var/opt/mssql/log/error*]" +----- + +//set the fileset name used in the included example +:fileset_ex: log + +include::../include/config-option-intro.asciidoc[] + +[float] +==== `log` fileset settings + +include::../include/var-paths.asciidoc[] + +:has-dashboards!: + +:fileset_ex!: + +:modulename!: diff --git a/x-pack/filebeat/module/mssql/_meta/fields.yml b/x-pack/filebeat/module/mssql/_meta/fields.yml new file mode 100644 index 000000000000..3d5557882421 --- /dev/null +++ b/x-pack/filebeat/module/mssql/_meta/fields.yml @@ -0,0 +1,8 @@ +- key: mssql + title: "mssql" + description: MS SQL Filebeat Module + fields: + - name: mssql + type: group + description: Fields from the MSSQL log files + fields: diff --git a/x-pack/filebeat/module/mssql/fields.go b/x-pack/filebeat/module/mssql/fields.go new file mode 100644 index 000000000000..e50a5eaa6799 --- /dev/null +++ b/x-pack/filebeat/module/mssql/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package mssql + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "mssql", asset.ModuleFieldsPri, AssetMssql); err != nil { + panic(err) + } +} + +// AssetMssql returns asset data. +// This is the base64 encoded gzipped contents of module/mssql. +func AssetMssql() string { + return "eJxsj0FugzAQRfec4ivr5gJsK2XVqKo4gYEPtTJm6IxJxe0rKGoJrXf+nv/e+Iwb5xLJ/UMKIMcsLHFa76cCaOmNxTFHHUpcK1RvL7hEYc2QcdV2EhZAFymtlwUAnDGExF/kcvI8skRvOo1b8sC9rHV0pgn5nbhWi0a0RxeFvlX2kr1ItP/JDuBnTUmHjbS0d4N/d/pPshepxT4OD08H3+s6Ae3WbyS6h55PmHwKIvMaOu1OQz1lxIwmDAjiipoIMDZ6p80YTRu6H1TfG984f6q1xVcAAAD//2bhgvE=" +} diff --git a/x-pack/filebeat/module/mssql/log/_meta/fields.yml b/x-pack/filebeat/module/mssql/log/_meta/fields.yml new file mode 100644 index 000000000000..fcfa7618a50b --- /dev/null +++ b/x-pack/filebeat/module/mssql/log/_meta/fields.yml @@ -0,0 +1,7 @@ +- name: log + description: Common log fields + type: group + fields: + - name: origin + description: Origin of the message, usually the server but it can also be a recovery process + type: keyword diff --git a/x-pack/filebeat/module/mssql/log/config/config.yml b/x-pack/filebeat/module/mssql/log/config/config.yml new file mode 100644 index 000000000000..7af7d54f8a1c --- /dev/null +++ b/x-pack/filebeat/module/mssql/log/config/config.yml @@ -0,0 +1,15 @@ +type: log +paths: + {{ range $i, $path := .paths }} +- {{$path}} + {{ end }} +exclude_files: [".gz$"] + +multiline.pattern: '^\d\d' +multiline.negate: true +multiline.match: after + +{{ if .convert_timezone }} +processors: +- add_locale: ~ +{{ end }} diff --git a/x-pack/filebeat/module/mssql/log/ingest/pipeline.json b/x-pack/filebeat/module/mssql/log/ingest/pipeline.json new file mode 100644 index 000000000000..81f01fd61a0a --- /dev/null +++ b/x-pack/filebeat/module/mssql/log/ingest/pipeline.json @@ -0,0 +1,58 @@ +{ + "description": "Pipeline to parse MSSQL logs", + "processors": [ + { + "grok": { + "field": "message", + "patterns": ["%{MSSQL_DATE:date} %{DATA:mssql.log.origin} [ ]*%{GREEDYDATA:msg_temp}"], + "pattern_definitions": { + "MSSQL_DATE":"%{DATA} %{DATA}" + } + } + }, + { + "date": { + "field": "date", + "target_field": "@timestamp", + "formats": ["yyyy-MM-dd HH:mm:ss.SS"], + "ignore_failure": true + } + }, + { + "date": { + "if": "ctx.event.timezone != null", + "field": "@timestamp", + "formats": ["ISO8601"], + "timezone": "{{ event.timezone }}", + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] + } + }, + { + "remove": { + "field":"date", + "ignore_missing": true + } + }, + { + "rename": { + "field": "message", + "target_field": "log.original" + } + }, + { + "rename": { + "field": "msg_temp", + "target_field": "message", + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/x-pack/filebeat/module/mssql/log/manifest.yml b/x-pack/filebeat/module/mssql/log/manifest.yml new file mode 100644 index 000000000000..f4c5f7b66a97 --- /dev/null +++ b/x-pack/filebeat/module/mssql/log/manifest.yml @@ -0,0 +1,22 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/opt/mssql/log/error* + os.darwin: + - /var/opt/mssql/log/error* + os.windows: + - c:\ProgramFiles\Microsoft SQL Server\MSSQL.1MSSQL\LOG\ERRORLOG* + os.linux: + - /var/opt/mssql/log/error* + - name: convert_timezone + default: true + # if ES < 6.1.0, this flag switches to false automatically when evaluating the + # pipeline + min_elasticsearch_version: + version: 6.1.0 + value: false + +ingest_pipeline: ingest/pipeline.json +input: config/config.yml diff --git a/x-pack/filebeat/module/mssql/log/test/test.log b/x-pack/filebeat/module/mssql/log/test/test.log new file mode 100644 index 000000000000..9c9325ece091 --- /dev/null +++ b/x-pack/filebeat/module/mssql/log/test/test.log @@ -0,0 +1,21 @@ +2019-05-03 09:01:09.99 Server Microsoft SQL Server 2017 (RTM-CU13) (KB4466404) - 14.0.3048.4 (X64) + Nov 30 2018 12:57:58 + Copyright (C) 2017 Microsoft Corporation + Developer Edition (64-bit) on Linux (Ubuntu 16.04.5 LTS) +2019-05-03 09:01:09.99 Server UTC adjustment: 0:00 +2019-05-03 09:01:09.99 Server (c) Microsoft Corporation. +2019-05-03 09:01:09.99 Server All rights reserved. +2019-05-03 09:01:10.00 Server Server process ID is 4124. +2019-05-03 09:01:10.00 Server Logging SQL Server messages in file '/var/opt/mssql/log/errorlog'. +2019-05-03 09:01:10.00 Server Registry startup parameters: + -d /var/opt/mssql/data/master.mdf + -l /var/opt/mssql/data/mastlog.ldf + -e /var/opt/mssql/log/errorlog +2019-05-03 09:01:10.00 Server SQL Server detected 1 sockets with 6 cores per socket and 12 logical processors per socket, 12 total logical processors; using 12 logical processors based on SQL Server licensing. This is an informational message; no user action is required. +2019-05-03 09:01:10.00 Server SQL Server is starting at normal priority base (=7). This is an informational message only. No user action is required. +2019-05-03 09:01:10.00 Server Detected 25445 MB of RAM. This is an informational message; no user action is required. +2019-05-03 09:01:10.00 Server Using conventional memory in the memory manager. +2019-05-03 09:01:10.01 Server Large Page Allocated: 32MB +2019-05-03 09:01:10.20 Server Buffer pool extension is already disabled. No action is n +2019-05-03 09:01:11.93 spid22s Service Broker manager has started. +2019-05-03 09:01:12.03 spid6s Recovery is complete. This is an informational message only. No user action is required. diff --git a/x-pack/filebeat/module/mssql/log/test/test.log-expected.json b/x-pack/filebeat/module/mssql/log/test/test.log-expected.json new file mode 100644 index 000000000000..870107e66081 --- /dev/null +++ b/x-pack/filebeat/module/mssql/log/test/test.log-expected.json @@ -0,0 +1,218 @@ +[ + { + "@timestamp": "2019-05-03T09:01:09.990Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 0, + "log.original": "2019-05-03 09:01:09.99 Server Microsoft SQL Server 2017 (RTM-CU13) (KB4466404) - 14.0.3048.4 (X64)\n\tNov 30 2018 12:57:58\n\tCopyright (C) 2017 Microsoft Corporation\n\tDeveloper Edition (64-bit) on Linux (Ubuntu 16.04.5 LTS)", + "message": "Microsoft SQL Server 2017 (RTM-CU13) (KB4466404) - 14.0.3048.4 (X64)", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:09.990Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 226, + "log.original": "2019-05-03 09:01:09.99 Server UTC adjustment: 0:00", + "message": "UTC adjustment: 0:00", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:09.990Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 282, + "log.original": "2019-05-03 09:01:09.99 Server (c) Microsoft Corporation.", + "message": "(c) Microsoft Corporation.", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:09.990Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 344, + "log.original": "2019-05-03 09:01:09.99 Server All rights reserved.", + "message": "All rights reserved.", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:10.000Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 400, + "log.original": "2019-05-03 09:01:10.00 Server Server process ID is 4124.", + "message": "Server process ID is 4124.", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:10.000Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 462, + "log.original": "2019-05-03 09:01:10.00 Server Logging SQL Server messages in file '/var/opt/mssql/log/errorlog'.", + "message": "Logging SQL Server messages in file '/var/opt/mssql/log/errorlog'.", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:10.000Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 564, + "log.original": "2019-05-03 09:01:10.00 Server Registry startup parameters:\n\t -d /var/opt/mssql/data/master.mdf\n\t -l /var/opt/mssql/data/mastlog.ldf\n\t -e /var/opt/mssql/log/errorlog", + "message": "Registry startup parameters:", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:10.000Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 734, + "log.original": "2019-05-03 09:01:10.00 Server SQL Server detected 1 sockets with 6 cores per socket and 12 logical processors per socket, 12 total logical processors; using 12 logical processors based on SQL Server licensing. This is an informational message; no user action is required.", + "message": "SQL Server detected 1 sockets with 6 cores per socket and 12 logical processors per socket, 12 total logical processors; using 12 logical processors based on SQL Server licensing. This is an informational message; no user action is required.", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:10.000Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 1011, + "log.original": "2019-05-03 09:01:10.00 Server SQL Server is starting at normal priority base (=7). This is an informational message only. No user action is required.", + "message": "SQL Server is starting at normal priority base (=7). This is an informational message only. No user action is required.", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:10.000Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 1166, + "log.original": "2019-05-03 09:01:10.00 Server Detected 25445 MB of RAM. This is an informational message; no user action is required.", + "message": "Detected 25445 MB of RAM. This is an informational message; no user action is required.", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:10.000Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 1289, + "log.original": "2019-05-03 09:01:10.00 Server Using conventional memory in the memory manager.", + "message": "Using conventional memory in the memory manager.", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:10.010Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 1373, + "log.original": "2019-05-03 09:01:10.01 Server Large Page Allocated: 32MB", + "message": "Large Page Allocated: 32MB", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:10.200Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 1435, + "log.original": "2019-05-03 09:01:10.20 Server Buffer pool extension is already disabled. No action is n", + "message": "Buffer pool extension is already disabled. No action is n", + "mssql.log.origin": "Server", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:11.930Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 1528, + "log.original": "2019-05-03 09:01:11.93 spid22s Service Broker manager has started.", + "message": "Service Broker manager has started.", + "mssql.log.origin": "spid22s", + "service.type": "mssql" + }, + { + "@timestamp": "2019-05-03T09:01:12.030Z", + "ecs.version": "1.0.0", + "event.dataset": "mssql.log", + "event.module": "mssql", + "event.timezone": "+00:00", + "fileset.name": "log", + "input.type": "log", + "log.offset": 1599, + "log.original": "2019-05-03 09:01:12.03 spid6s Recovery is complete. This is an informational message only. No user action is required.", + "message": "Recovery is complete. This is an informational message only. No user action is required.", + "mssql.log.origin": "spid6s", + "service.type": "mssql" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/modules.d/mssql.yml.disabled b/x-pack/filebeat/modules.d/mssql.yml.disabled new file mode 100644 index 000000000000..5e03b661da8c --- /dev/null +++ b/x-pack/filebeat/modules.d/mssql.yml.disabled @@ -0,0 +1,11 @@ +# Module: mssql +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-mssql.html + +- module: mssql + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: