Skip to content

Commit ee61cac

Browse files
adriansrmergify-bot
adriansr
authored and
mergify-bot
committed
ibmmq: Fix timestamp parsing (#29773)
This fixes the timestamp parsing in ibmmq logs: - Date processor format definition was broken for ES 8.0 (extra `a` character). - The header date format in some logs was unsupported. - The Time() field, with correct TZ and higher precission was ignored. (cherry picked from commit fbc33ab)
1 parent 048c9f2 commit ee61cac

File tree

5 files changed

+119
-118
lines changed

5 files changed

+119
-118
lines changed

‎filebeat/tests/system/test_modules.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
"f5.bigipafm",
2626
"fortinet.clientendpoint",
2727
"haproxy.log",
28-
"ibmmq.errorlog",
2928
"icinga.startup",
3029
"imperva.securesphere",
3130
"infoblox.nios",

‎x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ processors:
3030
patterns:
3131
- 'Process\(%{DATA:process.pid}\) User\(%{WORD:user.name}\) Program\(%{DATA:process.title}\)
3232
Host\(%{DATA:host.hostname}\) Installation\(%{WORD:ibmmq.errorlog.installation}\)
33-
VRMF\(%{DATA:service.version}\)( QMgr\(%{DATA:ibmmq.errorlog.qmgr}\))?( Time\(%{TIMESTAMP_ISO8601:@timestamp}\))?(
33+
VRMF\(%{DATA:service.version}\)( QMgr\(%{DATA:ibmmq.errorlog.qmgr}\))?( Time\(%{TIMESTAMP_ISO8601:log_timestamp}\))?(
3434
RemoteHost\(%{DATA:destination.address}\))?( ArithInsert1\(%{DATA:ibmmq.errorlog.arithinsert1}\))?(
3535
ArithInsert2\(%{DATA:ibmmq.errorlog.arithinsert2}\))?( CommentInsert1\(%{DATA:ibmmq.errorlog.commentinsert1}\))?(
3636
CommentInsert2\(%{DATA:ibmmq.errorlog.commentinsert2}\))?( CommentInsert3\(%{DATA:ibmmq.errorlog.commentinsert3}\))?
@@ -41,8 +41,10 @@ processors:
4141
field: log_timestamp
4242
target_field: '@timestamp'
4343
formats:
44-
- MM/dd/yyyy hh:mm:ss aa
44+
- ISO8601
45+
- MM/dd/yyyy hh:mm:ss a
4546
- dd/MM/yyyy HH:mm:ss
47+
- dd.MM.yyyy HH:mm:ss
4648
ignore_failure: true
4749
- append:
4850
field: ibmmq.errorlog.commentinsert

‎x-pack/filebeat/module/ibmmq/errorlog/test/AMQERR01.log-expected.json‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@
6666
"user.name": "felix"
6767
},
6868
{
69-
"@timestamp": "2018-10-11T10:46:25.000Z",
69+
"@timestamp": "2018-10-11T08:46:25.924Z",
7070
"event.dataset": "ibmmq.errorlog",
7171
"event.kind": "event",
7272
"event.module": "ibmmq",
@@ -99,7 +99,7 @@
9999
"user.name": "MUSR_MQADMIN"
100100
},
101101
{
102-
"@timestamp": "2018-10-11T10:46:26.000Z",
102+
"@timestamp": "2018-10-11T08:46:26.343Z",
103103
"event.dataset": "ibmmq.errorlog",
104104
"event.kind": "event",
105105
"event.module": "ibmmq",
@@ -132,7 +132,7 @@
132132
"user.name": "MUSR_MQADMIN"
133133
},
134134
{
135-
"@timestamp": "2018-10-11T10:46:26.000Z",
135+
"@timestamp": "2018-10-11T08:46:26.346Z",
136136
"event.dataset": "ibmmq.errorlog",
137137
"event.kind": "event",
138138
"event.module": "ibmmq",
@@ -264,7 +264,7 @@
264264
"user.name": "felix"
265265
},
266266
{
267-
"@timestamp": "2018-10-28T15:12:07.000Z",
267+
"@timestamp": "2018-10-28T14:12:07.685Z",
268268
"event.dataset": "ibmmq.errorlog",
269269
"event.kind": "event",
270270
"event.module": "ibmmq",
@@ -297,7 +297,7 @@
297297
"user.name": "MUSR_MQADMIN"
298298
},
299299
{
300-
"@timestamp": "2018-10-28T15:12:07.000Z",
300+
"@timestamp": "2018-10-28T14:12:07.789Z",
301301
"event.dataset": "ibmmq.errorlog",
302302
"event.kind": "event",
303303
"event.module": "ibmmq",
@@ -330,7 +330,7 @@
330330
"user.name": "MUSR_MQADMIN"
331331
},
332332
{
333-
"@timestamp": "2018-10-28T15:12:08.000Z",
333+
"@timestamp": "2018-10-28T14:12:08.663Z",
334334
"event.dataset": "ibmmq.errorlog",
335335
"event.kind": "event",
336336
"event.module": "ibmmq",
@@ -363,7 +363,7 @@
363363
"user.name": "MUSR_MQADMIN"
364364
},
365365
{
366-
"@timestamp": "2018-10-28T15:12:08.000Z",
366+
"@timestamp": "2018-10-28T14:12:08.665Z",
367367
"event.dataset": "ibmmq.errorlog",
368368
"event.kind": "event",
369369
"event.module": "ibmmq",
@@ -396,7 +396,7 @@
396396
"user.name": "MUSR_MQADMIN"
397397
},
398398
{
399-
"@timestamp": "2018-10-29T16:48:52.000Z",
399+
"@timestamp": "2018-10-29T15:48:52.594Z",
400400
"event.dataset": "ibmmq.errorlog",
401401
"event.kind": "event",
402402
"event.module": "ibmmq",
@@ -429,7 +429,7 @@
429429
"user.name": "MUSR_MQADMIN"
430430
},
431431
{
432-
"@timestamp": "2018-10-29T16:48:52.000Z",
432+
"@timestamp": "2018-10-29T15:48:52.663Z",
433433
"event.dataset": "ibmmq.errorlog",
434434
"event.kind": "event",
435435
"event.module": "ibmmq",
@@ -462,7 +462,7 @@
462462
"user.name": "MUSR_MQADMIN"
463463
},
464464
{
465-
"@timestamp": "2018-10-29T16:48:53.000Z",
465+
"@timestamp": "2018-10-29T15:48:53.368Z",
466466
"event.dataset": "ibmmq.errorlog",
467467
"event.kind": "event",
468468
"event.module": "ibmmq",
@@ -495,7 +495,7 @@
495495
"user.name": "MUSR_MQADMIN"
496496
},
497497
{
498-
"@timestamp": "2018-10-29T16:48:53.000Z",
498+
"@timestamp": "2018-10-29T15:48:53.369Z",
499499
"event.dataset": "ibmmq.errorlog",
500500
"event.kind": "event",
501501
"event.module": "ibmmq",
@@ -528,7 +528,7 @@
528528
"user.name": "MUSR_MQADMIN"
529529
},
530530
{
531-
"@timestamp": "2018-10-29T16:49:35.000Z",
531+
"@timestamp": "2018-10-29T15:49:35.477Z",
532532
"event.dataset": "ibmmq.errorlog",
533533
"event.kind": "event",
534534
"event.module": "ibmmq",
@@ -561,7 +561,7 @@
561561
"user.name": "MUSR_MQADMIN"
562562
},
563563
{
564-
"@timestamp": "2018-10-29T16:49:35.000Z",
564+
"@timestamp": "2018-10-29T15:49:35.553Z",
565565
"event.dataset": "ibmmq.errorlog",
566566
"event.kind": "event",
567567
"event.module": "ibmmq",
@@ -594,7 +594,7 @@
594594
"user.name": "MUSR_MQADMIN"
595595
},
596596
{
597-
"@timestamp": "2018-10-29T16:49:36.000Z",
597+
"@timestamp": "2018-10-29T15:49:36.447Z",
598598
"event.dataset": "ibmmq.errorlog",
599599
"event.kind": "event",
600600
"event.module": "ibmmq",
@@ -627,7 +627,7 @@
627627
"user.name": "MUSR_MQADMIN"
628628
},
629629
{
630-
"@timestamp": "2018-10-29T16:49:36.000Z",
630+
"@timestamp": "2018-10-29T15:49:36.448Z",
631631
"event.dataset": "ibmmq.errorlog",
632632
"event.kind": "event",
633633
"event.module": "ibmmq",

‎x-pack/filebeat/module/ibmmq/errorlog/test/AMQERR01_QM1.log-expected.json‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[
22
{
3-
"@timestamp": "2022-01-10T10:25:59.012Z",
3+
"@timestamp": "2018-07-13T07:06:00.000Z",
44
"event.dataset": "ibmmq.errorlog",
55
"event.kind": "event",
66
"event.module": "ibmmq",
@@ -33,7 +33,7 @@
3333
"user.name": "felix"
3434
},
3535
{
36-
"@timestamp": "2022-01-10T10:25:59.012Z",
36+
"@timestamp": "2018-07-13T07:06:00.000Z",
3737
"event.dataset": "ibmmq.errorlog",
3838
"event.kind": "event",
3939
"event.module": "ibmmq",
@@ -66,7 +66,7 @@
6666
"user.name": "felix"
6767
},
6868
{
69-
"@timestamp": "2022-01-10T10:25:59.012Z",
69+
"@timestamp": "2018-07-13T07:06:00.000Z",
7070
"event.dataset": "ibmmq.errorlog",
7171
"event.kind": "event",
7272
"event.module": "ibmmq",
@@ -99,7 +99,7 @@
9999
"user.name": "felix"
100100
},
101101
{
102-
"@timestamp": "2022-01-10T10:25:59.012Z",
102+
"@timestamp": "2018-07-13T07:06:01.000Z",
103103
"event.dataset": "ibmmq.errorlog",
104104
"event.kind": "event",
105105
"event.module": "ibmmq",
@@ -132,7 +132,7 @@
132132
"user.name": "felix"
133133
},
134134
{
135-
"@timestamp": "2022-01-10T10:25:59.012Z",
135+
"@timestamp": "2018-07-13T07:06:01.000Z",
136136
"event.dataset": "ibmmq.errorlog",
137137
"event.kind": "event",
138138
"event.module": "ibmmq",
@@ -165,7 +165,7 @@
165165
"user.name": "felix"
166166
},
167167
{
168-
"@timestamp": "2022-01-10T10:25:59.012Z",
168+
"@timestamp": "2018-07-13T07:06:01.000Z",
169169
"event.dataset": "ibmmq.errorlog",
170170
"event.kind": "event",
171171
"event.module": "ibmmq",
@@ -198,7 +198,7 @@
198198
"user.name": "felix"
199199
},
200200
{
201-
"@timestamp": "2022-01-10T10:25:59.012Z",
201+
"@timestamp": "2018-07-13T07:06:01.000Z",
202202
"event.dataset": "ibmmq.errorlog",
203203
"event.kind": "event",
204204
"event.module": "ibmmq",
@@ -231,7 +231,7 @@
231231
"user.name": "felix"
232232
},
233233
{
234-
"@timestamp": "2022-01-10T10:25:59.013Z",
234+
"@timestamp": "2018-07-13T07:06:01.000Z",
235235
"event.dataset": "ibmmq.errorlog",
236236
"event.kind": "event",
237237
"event.module": "ibmmq",
@@ -264,7 +264,7 @@
264264
"user.name": "felix"
265265
},
266266
{
267-
"@timestamp": "2022-01-10T10:25:59.013Z",
267+
"@timestamp": "2018-07-13T07:06:01.000Z",
268268
"event.dataset": "ibmmq.errorlog",
269269
"event.kind": "event",
270270
"event.module": "ibmmq",
@@ -297,7 +297,7 @@
297297
"user.name": "felix"
298298
},
299299
{
300-
"@timestamp": "2022-01-10T10:25:59.013Z",
300+
"@timestamp": "2018-07-13T07:06:01.000Z",
301301
"event.dataset": "ibmmq.errorlog",
302302
"event.kind": "event",
303303
"event.module": "ibmmq",
@@ -330,7 +330,7 @@
330330
"user.name": "felix"
331331
},
332332
{
333-
"@timestamp": "2022-01-10T10:25:59.013Z",
333+
"@timestamp": "2018-07-13T07:06:01.000Z",
334334
"event.dataset": "ibmmq.errorlog",
335335
"event.kind": "event",
336336
"event.module": "ibmmq",
@@ -363,7 +363,7 @@
363363
"user.name": "felix"
364364
},
365365
{
366-
"@timestamp": "2022-01-10T10:25:59.013Z",
366+
"@timestamp": "2018-07-13T07:06:01.000Z",
367367
"event.dataset": "ibmmq.errorlog",
368368
"event.kind": "event",
369369
"event.module": "ibmmq",
@@ -396,7 +396,7 @@
396396
"user.name": "felix"
397397
},
398398
{
399-
"@timestamp": "2022-01-10T10:25:59.013Z",
399+
"@timestamp": "2018-07-13T07:06:01.000Z",
400400
"event.dataset": "ibmmq.errorlog",
401401
"event.kind": "event",
402402
"event.module": "ibmmq",
@@ -429,7 +429,7 @@
429429
"user.name": "felix"
430430
},
431431
{
432-
"@timestamp": "2022-01-10T10:25:59.013Z",
432+
"@timestamp": "2018-07-13T07:06:01.000Z",
433433
"event.dataset": "ibmmq.errorlog",
434434
"event.kind": "event",
435435
"event.module": "ibmmq",
@@ -462,7 +462,7 @@
462462
"user.name": "felix"
463463
},
464464
{
465-
"@timestamp": "2022-01-10T10:25:59.013Z",
465+
"@timestamp": "2018-07-13T07:06:01.000Z",
466466
"event.dataset": "ibmmq.errorlog",
467467
"event.kind": "event",
468468
"event.module": "ibmmq",
@@ -495,7 +495,7 @@
495495
"user.name": "felix"
496496
},
497497
{
498-
"@timestamp": "2022-01-10T10:25:59.013Z",
498+
"@timestamp": "2018-07-13T07:06:02.000Z",
499499
"event.dataset": "ibmmq.errorlog",
500500
"event.kind": "event",
501501
"event.module": "ibmmq",
@@ -528,7 +528,7 @@
528528
"user.name": "felix"
529529
},
530530
{
531-
"@timestamp": "2022-01-10T10:25:59.013Z",
531+
"@timestamp": "2018-07-13T07:06:03.000Z",
532532
"event.dataset": "ibmmq.errorlog",
533533
"event.kind": "event",
534534
"event.module": "ibmmq",
@@ -561,7 +561,7 @@
561561
"user.name": "felix"
562562
},
563563
{
564-
"@timestamp": "2022-01-10T10:25:59.013Z",
564+
"@timestamp": "2018-07-13T07:06:03.000Z",
565565
"event.dataset": "ibmmq.errorlog",
566566
"event.kind": "event",
567567
"event.module": "ibmmq",
@@ -594,7 +594,7 @@
594594
"user.name": "felix"
595595
},
596596
{
597-
"@timestamp": "2022-01-10T10:25:59.014Z",
597+
"@timestamp": "2018-07-13T07:06:03.000Z",
598598
"event.dataset": "ibmmq.errorlog",
599599
"event.kind": "event",
600600
"event.module": "ibmmq",
@@ -627,7 +627,7 @@
627627
"user.name": "felix"
628628
},
629629
{
630-
"@timestamp": "2022-01-10T10:25:59.014Z",
630+
"@timestamp": "2018-07-13T07:06:03.000Z",
631631
"event.dataset": "ibmmq.errorlog",
632632
"event.kind": "event",
633633
"event.module": "ibmmq",
@@ -660,7 +660,7 @@
660660
"user.name": "felix"
661661
},
662662
{
663-
"@timestamp": "2022-01-10T10:25:59.014Z",
663+
"@timestamp": "2018-07-13T07:06:03.000Z",
664664
"event.dataset": "ibmmq.errorlog",
665665
"event.kind": "event",
666666
"event.module": "ibmmq",
@@ -693,7 +693,7 @@
693693
"user.name": "felix"
694694
},
695695
{
696-
"@timestamp": "2022-01-10T10:25:59.014Z",
696+
"@timestamp": "2018-07-13T07:06:03.000Z",
697697
"event.dataset": "ibmmq.errorlog",
698698
"event.kind": "event",
699699
"event.module": "ibmmq",
@@ -726,7 +726,7 @@
726726
"user.name": "felix"
727727
},
728728
{
729-
"@timestamp": "2022-01-10T10:25:59.014Z",
729+
"@timestamp": "2018-07-13T07:06:03.000Z",
730730
"event.dataset": "ibmmq.errorlog",
731731
"event.kind": "event",
732732
"event.module": "ibmmq",

0 commit comments

Comments
 (0)