Skip to content

Commit b18d1f2

Browse files
jsorianojsoriano
authored
Store message from MongoDB json logs in message field (#26338)
1 parent 72b4a23 commit b18d1f2

File tree

5 files changed

+22
-34
lines changed

5 files changed

+22
-34
lines changed

‎filebeat/docs/fields.asciidoc‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -95755,18 +95755,6 @@ alias to: message
9575595755

9575695756
--
9575795757

95758-
*`mongodb.log.msg`*::
95759-
+
95760-
--
95761-
String representing the raw log output message as passed from the server or driver
95762-
95763-
95764-
type: text
95765-
95766-
example: MongoDB starting
95767-
95768-
--
95769-
9577095758
*`mongodb.log.id`*::
9577195759
+
9577295760
--

‎filebeat/module/mongodb/fields.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎filebeat/module/mongodb/log/_meta/fields.yml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
Context of message
1414
example: initandlisten
1515
type: keyword
16-
1716
- name: severity
1817
type: alias
1918
path: log.level
@@ -22,11 +21,6 @@
2221
type: alias
2322
path: message
2423
migration: true
25-
- name: msg
26-
description: >
27-
String representing the raw log output message as passed from the server or driver
28-
example: MongoDB starting
29-
type: text
3024
- name: id
3125
description: >
3226
Integer representing the unique identifier of the log statement

‎filebeat/module/mongodb/log/ingest/pipeline-json.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,12 @@ processors:
1717
- rename:
1818
field: mongodb.log.ctx
1919
target_field: mongodb.log.context
20+
- rename:
21+
field: message
22+
target_field: event.original
23+
- rename:
24+
field: mongodb.log.msg
25+
target_field: message
2026
- append:
2127
field: event.type
2228
value: access

‎filebeat/module/mongodb/log/test/mongodb-ubuntu-4.4.4-json.log-expected.json‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,18 @@
77
"event.dataset": "mongodb.log",
88
"event.kind": "event",
99
"event.module": "mongodb",
10+
"event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:20.349+00:00\"},\"s\":\"I\", \"c\":\"STORAGE\", \"id\":4615611, \"ctx\":\"initandlisten\",\"msg\":\"MongoDB starting\",\"attr\":{\"pid\":1,\"port\":27017,\"dbPath\":\"/data/db\",\"architecture\":\"64-bit\",\"host\":\"6150fe65a89c\"}}",
1011
"event.type": [
1112
"info"
1213
],
1314
"fileset.name": "log",
1415
"input.type": "log",
1516
"log.level": "I",
1617
"log.offset": 0,
17-
"message": "{\"t\":{\"$date\":\"2021-03-22T21:21:20.349+00:00\"},\"s\":\"I\", \"c\":\"STORAGE\", \"id\":4615611, \"ctx\":\"initandlisten\",\"msg\":\"MongoDB starting\",\"attr\":{\"pid\":1,\"port\":27017,\"dbPath\":\"/data/db\",\"architecture\":\"64-bit\",\"host\":\"6150fe65a89c\"}}",
18+
"message": "MongoDB starting",
1819
"mongodb.log.component": "STORAGE",
1920
"mongodb.log.context": "initandlisten",
2021
"mongodb.log.id": 4615611,
21-
"mongodb.log.msg": "MongoDB starting",
2222
"service.type": "mongodb"
2323
},
2424
{
@@ -29,18 +29,18 @@
2929
"event.dataset": "mongodb.log",
3030
"event.kind": "event",
3131
"event.module": "mongodb",
32+
"event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:20.350+00:00\"},\"s\":\"I\", \"c\":\"CONTROL\", \"id\":23403, \"ctx\":\"initandlisten\",\"msg\":\"Build Info\",\"attr\":{\"buildInfo\":{\"version\":\"4.4.4\",\"gitVersion\":\"8db30a63db1a9d84bdcad0c83369623f708e0397\",\"openSSLVersion\":\"OpenSSL 1.1.1 11 Sep 2018\",\"modules\":[],\"allocator\":\"tcmalloc\",\"environment\":{\"distmod\":\"ubuntu1804\",\"distarch\":\"x86_64\",\"target_arch\":\"x86_64\"}}}}",
3233
"event.type": [
3334
"info"
3435
],
3536
"fileset.name": "log",
3637
"input.type": "log",
3738
"log.level": "I",
3839
"log.offset": 231,
39-
"message": "{\"t\":{\"$date\":\"2021-03-22T21:21:20.350+00:00\"},\"s\":\"I\", \"c\":\"CONTROL\", \"id\":23403, \"ctx\":\"initandlisten\",\"msg\":\"Build Info\",\"attr\":{\"buildInfo\":{\"version\":\"4.4.4\",\"gitVersion\":\"8db30a63db1a9d84bdcad0c83369623f708e0397\",\"openSSLVersion\":\"OpenSSL 1.1.1 11 Sep 2018\",\"modules\":[],\"allocator\":\"tcmalloc\",\"environment\":{\"distmod\":\"ubuntu1804\",\"distarch\":\"x86_64\",\"target_arch\":\"x86_64\"}}}}",
40+
"message": "Build Info",
4041
"mongodb.log.component": "CONTROL",
4142
"mongodb.log.context": "initandlisten",
4243
"mongodb.log.id": 23403,
43-
"mongodb.log.msg": "Build Info",
4444
"service.type": "mongodb"
4545
},
4646
{
@@ -51,18 +51,18 @@
5151
"event.dataset": "mongodb.log",
5252
"event.kind": "event",
5353
"event.module": "mongodb",
54+
"event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.240+00:00\"},\"s\":\"I\", \"c\":\"RECOVERY\", \"id\":23987, \"ctx\":\"initandlisten\",\"msg\":\"WiredTiger recoveryTimestamp\",\"attr\":{\"recoveryTimestamp\":{\"$timestamp\":{\"t\":0,\"i\":0}}}}",
5455
"event.type": [
5556
"info"
5657
],
5758
"fileset.name": "log",
5859
"input.type": "log",
5960
"log.level": "I",
6061
"log.offset": 621,
61-
"message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.240+00:00\"},\"s\":\"I\", \"c\":\"RECOVERY\", \"id\":23987, \"ctx\":\"initandlisten\",\"msg\":\"WiredTiger recoveryTimestamp\",\"attr\":{\"recoveryTimestamp\":{\"$timestamp\":{\"t\":0,\"i\":0}}}}",
62+
"message": "WiredTiger recoveryTimestamp",
6263
"mongodb.log.component": "RECOVERY",
6364
"mongodb.log.context": "initandlisten",
6465
"mongodb.log.id": 23987,
65-
"mongodb.log.msg": "WiredTiger recoveryTimestamp",
6666
"service.type": "mongodb"
6767
},
6868
{
@@ -73,18 +73,18 @@
7373
"event.dataset": "mongodb.log",
7474
"event.kind": "event",
7575
"event.module": "mongodb",
76+
"event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.363+00:00\"},\"s\":\"I\", \"c\":\"STORAGE\", \"id\":20320, \"ctx\":\"initandlisten\",\"msg\":\"createCollection\",\"attr\":{\"namespace\":\"admin.system.version\",\"uuidDisposition\":\"provided\",\"uuid\":{\"uuid\":{\"$uuid\":\"b383f03c-b97c-4584-87ae-ab1b8ea399c3\"}},\"options\":{\"uuid\":{\"$uuid\":\"b383f03c-b97c-4584-87ae-ab1b8ea399c3\"}}}}",
7677
"event.type": [
7778
"info"
7879
],
7980
"fileset.name": "log",
8081
"input.type": "log",
8182
"log.level": "I",
8283
"log.offset": 826,
83-
"message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.363+00:00\"},\"s\":\"I\", \"c\":\"STORAGE\", \"id\":20320, \"ctx\":\"initandlisten\",\"msg\":\"createCollection\",\"attr\":{\"namespace\":\"admin.system.version\",\"uuidDisposition\":\"provided\",\"uuid\":{\"uuid\":{\"$uuid\":\"b383f03c-b97c-4584-87ae-ab1b8ea399c3\"}},\"options\":{\"uuid\":{\"$uuid\":\"b383f03c-b97c-4584-87ae-ab1b8ea399c3\"}}}}",
84+
"message": "createCollection",
8485
"mongodb.log.component": "STORAGE",
8586
"mongodb.log.context": "initandlisten",
8687
"mongodb.log.id": 20320,
87-
"mongodb.log.msg": "createCollection",
8888
"service.type": "mongodb"
8989
},
9090
{
@@ -95,18 +95,18 @@
9595
"event.dataset": "mongodb.log",
9696
"event.kind": "event",
9797
"event.module": "mongodb",
98+
"event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.410+00:00\"},\"s\":\"I\", \"c\":\"INDEX\", \"id\":20345, \"ctx\":\"initandlisten\",\"msg\":\"Index build: done building\",\"attr\":{\"buildUUID\":null,\"namespace\":\"admin.system.version\",\"index\":\"_id_\",\"commitTimestamp\":{\"$timestamp\":{\"t\":0,\"i\":0}}}}",
9899
"event.type": [
99100
"info"
100101
],
101102
"fileset.name": "log",
102103
"input.type": "log",
103104
"log.level": "I",
104105
"log.offset": 1167,
105-
"message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.410+00:00\"},\"s\":\"I\", \"c\":\"INDEX\", \"id\":20345, \"ctx\":\"initandlisten\",\"msg\":\"Index build: done building\",\"attr\":{\"buildUUID\":null,\"namespace\":\"admin.system.version\",\"index\":\"_id_\",\"commitTimestamp\":{\"$timestamp\":{\"t\":0,\"i\":0}}}}",
106+
"message": "Index build: done building",
106107
"mongodb.log.component": "INDEX",
107108
"mongodb.log.context": "initandlisten",
108109
"mongodb.log.id": 20345,
109-
"mongodb.log.msg": "Index build: done building",
110110
"service.type": "mongodb"
111111
},
112112
{
@@ -117,18 +117,18 @@
117117
"event.dataset": "mongodb.log",
118118
"event.kind": "event",
119119
"event.module": "mongodb",
120+
"event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.412+00:00\"},\"s\":\"I\", \"c\":\"COMMAND\", \"id\":20459, \"ctx\":\"initandlisten\",\"msg\":\"Setting featureCompatibilityVersion\",\"attr\":{\"newVersion\":\"4.4\"}}",
120121
"event.type": [
121122
"info"
122123
],
123124
"fileset.name": "log",
124125
"input.type": "log",
125126
"log.level": "I",
126127
"log.offset": 1435,
127-
"message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.412+00:00\"},\"s\":\"I\", \"c\":\"COMMAND\", \"id\":20459, \"ctx\":\"initandlisten\",\"msg\":\"Setting featureCompatibilityVersion\",\"attr\":{\"newVersion\":\"4.4\"}}",
128+
"message": "Setting featureCompatibilityVersion",
128129
"mongodb.log.component": "COMMAND",
129130
"mongodb.log.context": "initandlisten",
130131
"mongodb.log.id": 20459,
131-
"mongodb.log.msg": "Setting featureCompatibilityVersion",
132132
"service.type": "mongodb"
133133
},
134134
{
@@ -139,18 +139,18 @@
139139
"event.dataset": "mongodb.log",
140140
"event.kind": "event",
141141
"event.module": "mongodb",
142+
"event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.451+00:00\"},\"s\":\"I\", \"c\":\"FTDC\", \"id\":20625, \"ctx\":\"initandlisten\",\"msg\":\"Initializing full-time diagnostic data capture\",\"attr\":{\"dataDirectory\":\"/data/db/diagnostic.data\"}}",
142143
"event.type": [
143144
"info"
144145
],
145146
"fileset.name": "log",
146147
"input.type": "log",
147148
"log.level": "I",
148149
"log.offset": 1617,
149-
"message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.451+00:00\"},\"s\":\"I\", \"c\":\"FTDC\", \"id\":20625, \"ctx\":\"initandlisten\",\"msg\":\"Initializing full-time diagnostic data capture\",\"attr\":{\"dataDirectory\":\"/data/db/diagnostic.data\"}}",
150+
"message": "Initializing full-time diagnostic data capture",
150151
"mongodb.log.component": "FTDC",
151152
"mongodb.log.context": "initandlisten",
152153
"mongodb.log.id": 20625,
153-
"mongodb.log.msg": "Initializing full-time diagnostic data capture",
154154
"service.type": "mongodb"
155155
}
156-
]
156+
]

0 commit comments

Comments
 (0)