diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9dbfba0c3aba..8a70e12d8d50 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -230,6 +230,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add convert_timezone to nginx module. {issue}9839[9839] {pull}10148[10148] - Add support for Percona in the `slowlog` fileset of `mysql` module. {issue}6665[6665] {pull}10227[10227] - Added support for ingesting structured Elasticsearch audit logs {pull}10352[10352] +- Added support for ingesting structured Elasticsearch slow logs {pull}10445[10445] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index aea98291493c..8e7d5333596b 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -4327,6 +4327,37 @@ elasticsearch Module +*`elasticsearch.component`*:: ++ +-- +type: keyword + +example: o.e.c.m.MetaDataCreateIndexService + +Elasticsearch component from where the log event originated + +-- + +*`elasticsearch.cluster.uuid`*:: ++ +-- +type: keyword + +example: GmvrbHlNTiSVYiPf8kxg9g + +UUID of the cluster + +-- + +*`elasticsearch.cluster.name`*:: ++ +-- +example: docker-cluster + +Name of the cluster + +-- + *`elasticsearch.node.id`*:: + -- diff --git a/filebeat/module/elasticsearch/_meta/fields.yml b/filebeat/module/elasticsearch/_meta/fields.yml index 3bfea37be8e0..aaeb1037eb18 100644 --- a/filebeat/module/elasticsearch/_meta/fields.yml +++ b/filebeat/module/elasticsearch/_meta/fields.yml @@ -7,6 +7,17 @@ type: group description: > fields: + - name: component + description: "Elasticsearch component from where the log event originated" + example: "o.e.c.m.MetaDataCreateIndexService" + type: keyword + - name: cluster.uuid + description: "UUID of the cluster" + example: "GmvrbHlNTiSVYiPf8kxg9g" + type: keyword + - name: cluster.name + description: "Name of the cluster" + example: "docker-cluster" - name: node.id description: "ID of the node" example: "DSiWcTyeThWtUXLB9J0BMw" diff --git a/filebeat/module/elasticsearch/fields.go b/filebeat/module/elasticsearch/fields.go index 9ec2b7b1d93c..9796731ed2e7 100644 --- a/filebeat/module/elasticsearch/fields.go +++ b/filebeat/module/elasticsearch/fields.go @@ -32,5 +32,5 @@ func init() { // AssetElasticsearch returns asset data. // This is the base64 encoded gzipped contents of module/elasticsearch. func AssetElasticsearch() string { - return "eJzUmllv2zr2wN/7KQ788r8FEv3tJM3cGJgLtG6apuiapZ173UCgqWOJNUUqJGXXt+h3H5D0IsuSvEyb6eSltbic31l4eEjpEEY47QJyog2jGomiySMAwwzHLrRWnrceAUSoqWKZYVJ04Y9HALA6Ft7IKOf4CGDIkEe667ocgiAprouxf2aaYRdiJfNs9qRCxup0xSmFjDBg0eJ5aXjr8jnIIZgEXc9WoR9+JWnmtHx+zT7RmyneJJ/M7b9ePzt71X72ZlLs6yFHOJ1IFVUz2P/WUrwlKW7mGH96+0r/dby9YCYi/Nos+dJ2cd2rZQ4ZxwESc2hQm0MmstzsKr/J+k46i6plk3cX8fPJ4PZq2Pv45B9Pr+n9oBfvYHedEBU1io/mRnddqyna2wskecTMWu9i7K4x/FFoKMdwcWpOpqhWWsrK3CToe8FQyRQmCaMJmIRpwDEKA1KxmAliMOqCQm0OwCgidCaVbQOWhUPGDapWScrSEnZUubXaIEVyLzewHRv5PyWo0DlD4X2Oep0Yfltv8coSuDq/voGn7y/ngx8X1VuMmxANCimyMUYghZO27EYTIgTyxwfAJSU8tCsRfrN93G+3MoFpnWNU5Hxcb7HlPLvbTSHh6UaPk9wkKAyjxD70gxxcqcFqPiacRc5oJCZMrDtzBt7qQivCIcm5sTGxB3uuUQXbKWC7/p+u1OMA2LDYgFEDMKGGjTGMmEJqpJruCy056kboK9sDjFysMIRMMUFZRjgMkEsR69qI6ENrxAZEkNBKax1AyyZXHZIoZaIFdztDW7Wl2GhlUdhg/BAwCfELAr8izeuN24UW5bk2qLqpFMxI9f8pYWIP+yoeZESRdIN97Uq+vboE1xcNqnpztr5ZM9rp//mF0JFgNDn6vj0ZQJpzw8KqxFskN/jVrDXOJ15rLOx+jG4IpkvfBwilqDVGMJjO/NMUQkMpD4/anbOg3QnaJzaKVp4crz053Se0ZultdQNdV+FWsPscwRdSszH1Dvv09+twNDj9eD1+lzy9b5vJ+/HLdx/2yY4erlTcrOPZ6LfTFeh2Cf0eR6KuqZKcX1XrtjVrOJDRtHIw4YyU4yQjJulCYkwWzHW14wMqhUFRDseUxYp4jY3KsRLDlQFhxT68CcINrNrAt5Hqt+qQRJFCXZ5/k2Qtc0UxYNkegnPFdpRmE9SssuB7CFzsAruKtftORRzXyZzLizBT6PfLH1dzzieP6f5zAlz0wBYOGs1MQLBlmZslRFeHZ1n6BgL798IJAp0hZUNG7Z590fMiglLnTTtAhXtgw9ayBaD9Kx79LnpAJefoN+hK0IL7cx8doUZaizbkklRtXVuA9UokC4F2s5IqYiK2FrXcr8iYwJgpkxMOKaEJEw3gmqp8EOppOpA8NGTAMTQsxZ+lB7wnuUawIoAJ0EiliDRQjkRYHfIMPAs4Fr0R3Cgm4gcA34LboWzkniAZhQqHOsyUtHWG4/+J5DeWWWf21LmU6DBA4RAVClvzLJWqR7c1IOfIQ4WaEvFQ1AV7p0SNLD1nYwQ5+ILUaFv6cwSSZXx+YGEatJFZhlG9MpQTrcNccEmih9LES3PxInJbYDqILa1Ps9xx1jJWJeUtGd/7wIDe+1sf47N4QTWUKrXAy1RYgVifsqF0pKsxMmw09JaK2L+SEjI3mkX+MmOESiCvUqCQWKb6v0DJRBkSGintmfwhMG+kIRyQk8zGawnaSKDS1ubGkxf2S3cTpA1RrteQCaaToLLK+DJOQ5WLmiVYr8gGBdxRw6I6klcf38xo8qyw2g6AaCB+ehvlmWTCgMjTAapqWpMoJJEOjbVLaLNMXfLYm/yCqAGJV6w5kwpOqsttMzdUJY1FINsU6HaXOfOPNrFFMFKOrIs91IyzkcuQuProUV26bbJWD7iMY7/1xjUiEyTlzLh3IfsSSQaEcznbbIiI5n5hf+9cy9ox4WhQm9SZMBivXTZvgQmLxWuVd3Js4I8Yl4OpaapQ7M7005BubRpxRPUwi9Mqj8IYy/dpezvuHY8gRoGzwllSmmdE0Omv70HnPDm0Bilq8Au4s9amm707lbmIf6R//7QT/o97eFrW4RfwcYNdq+kWdkM1XhG6ehl47Zpt9obyi43qGFj306I2lmkmRfkmblXcaxkv+63eGy5vGGWAAQ3S4A0a8pwY0lNIDLo3oxaX0dL7l7qNq/Lmpkzkt66qCdejv+mexgVN01ppeRde9OqvVqsvUqtWYfVqWeRssX5AWWUpS2qimHNwuabgopqYyIcQuNBvjCpBEoUa7xtNfo33uT1az6rJWssfn5ycnZ0dVZq/lmJZGobzi6Bgw5uc1QP1Re/A/pMyztmsWKsl7Jy221uWjAsrDezaJ7sBukToylpr5NkbvUIRPCF6NjFGO9D/vhX9ImdxOeEyrk9avt1fw2t/uDgvf3izBtHqH7U7vx+2Tw+Pzm467W77tNs5OTg7Pr7rX7598Q7u+v7rDD9FMIMI7nNU0zvoj8OPr5IvH++gn6JRjLpvQE6D46B9aOcN2qfB0eldv33nqvH+SfAk1XcH7kfojdQ/cb/tmSVhRvc7ZyfHT+yjaYa6f3dgD0fG/8chuDcT/Q+351d/hjcvz9+GL85vei8Xc7gvNHS/Y/u7twL9b59bjvZzq/vtcyslhiYh4dz/HEipzedWtxO0v3//fnfwn6R6W+yXdrK1PB+jWvuKpuiNSmMP0ax6b3N2twZuIHFLjpnFEWn2gssdlZ2x6viO2+1U74hiHdnEYtvr5O0myoVKg6hr2+49WivRtXZ2lLuMzCbp/tM226tOeDmsd8RwAR86BzZxcDlp9vIOS2Y3QvxqFAk9ZwPhue02UweYGEqVkvWX3ftGyTLZNEWlP6AyUxcoJ0d7CPXZaaNYa3yGkf/crA7gaDcAJXPDSpt2+YsV16POyLrdefnX0Ydno7Mvk5PYxOSFEbsZvvSFwIr0y+jH+LZ5Cd40rL1I0iZZ/w4AAP//P6pndQ==" + return "eJzUmllv2zr2wN/7KQ788r8FEv2dpZnGwFygddMkRZc0W6fXDQSaOpZYU6RCUnZ8i373ASnZkbV5mbbT8UsbcTm/s/DwkNIujHHWA+REG0Y1EkWjJwCGGY496Cw97zwBCFBTxRLDpOjBn08AYHksvJNByvEJwIghD3TPddkFQWKsirE/M0uwB6GSaZI/qZGxPF1xSirjRAoUZtFSmqBzssS36A8jJWOYRqgQTITAZQg4sQ1SsZAJYjDoFCbFBxInzijSQ496sfcODXlFDOkrJAbPRYAPV6gmjGJxXKbfGGdTqYIqPk+1QeWlKQsaNbi5OX8FcuQw8wH1ZKfxRA3P+PtrdnX7mV2Mno8fwuNwcxr7VyPNexLjWjSBpGNUu5U+c2lCBui1qP2otO1ZL+PVFftEr2d4HX0yN/96+/L4Tfflu+n6GjuGtdVt5ph8ev9G/3WwvmBmw6Vdsoso171e5ohxHCIxuwa12WUiSc2m8tus76SzhjVAPpyGr6bDm8tR//bZP15c0fthP9zA7joiKmgVH8yN7rrWU3TXF0jSgJlK72LaqTD8WWgop5/i1JzMUC21lJW5tvnF9ponHUYjMBHTlYzTA4Xa7IBRROhEKtsGLPFHjJfW2bIl7Khya71BiuSZXM92bOX/tMiSCu9T1FVi+KPakilL4PLk6hpeXJzPBz8tqrcYNyUaFFJkEwxACiftsRuNiBDIn+4Al5Rw365E+CPL25RwtzKBaZ1iUOR82myxx3k2t5tCwuOVHiepiVAYRol9mA1ycKUGq/mEcBY4o5GQMFF1Zg7esXkVRyTlxsbEFuypRuWtp4Dt+n+6Vo8dYKNiw/JeWQIm1LAJ+gFTSI1Us22hJUfdCn1pe4CRixWGkCgmKEsIhyFyKULdGBED6IzZkAjiW2mdHejY5Kp9EsRMdOBuY2irthQrrSwKG0w2BExEsgWBD0jTZuP2oJPvrr1YCmak+v+YMLGFfRX3EqJIvMK+diXfXJ6D64sGVbM5O9+sGe30//xK6FgwGu1/X58MIE65YX5d4i2SG3wwlcb5xJXGwu7H6IpgOs/6AKEUtcYAhrPcP20hNJJyd7+7d+x197zuoY2ipScHlSdH24RWnt6WN9CqCjeC3acIWSGVj2l22Ke/3/rj4dHt1eRD9OK+a6YXk7MPH7fJjhlcqbip4tnot9MV6DYJ/T5Hoq6okpxf1uu2Nqs/lMGsdjDhjJTjJCEm6kFkTOLNdbXjPSqFWT6J2F/MQkUyjY1KsRbDlQF+zT68CsINrNvA15GabdU+CQKFujz/Kslapoqix5ItBKeKbSjNJqi8suBbCFzsApuK1dXTUJvMubwAE4XZfvnjas755CHdfk6A0z7YwkGjyQV4a5a5SUR0fXiWpa8gsL/XThDoBCkbMWr37NN+JsIrdV61A9S4B1ZsLWsA2l/x6HfaByo5x2yDrgUtuD/NosPXSBvRRlySuq1rDbB+iWQh0G5WUgVMhNailvsNmRCYMGVSwiEmNGKiBVxTlQ59PYuHkvuGDDn6hsX4s/SAC5JqBCsCmACNVIpAA+VIhNUhTSBjAceiV4IbxUT4C8DX4HYoK7mnSMa+wpH2EyVtneH4fyL5tWXWiT11Pkp0GKBwhAqFrXkelWpGtzUg58h9hZoS8auoC/aOiRpbes4mCHL4FanRtvTnCCRJ+PzAwjRoI5MEg2ZlKCda+6ngkgS/SpNMmosXkdoC00GsaX2apI6zkbEuKa/JeJEFBvQvbrIYz+MF1Uiq2AI/psIaxOaUDaUjXYORYaWh11TE/kpKyNRoFmSXGWNUAnmdAoXEMtP/BUomypDQSmnP5L8C81oawgE5SWy8lqCNdLfrHE1GXtgv3U2QNkS5XiMmmI682irj6yT2VSoalmCzIisUcEcNi+pI3ty+y2nSpLDadoBoINn0NsoTyYQBkcZDVPW0JlJIAu0baxffZpmm5LE1+SlRQxIuWTOXCk6qy225G+qSxiKQbQp0u8uc+Ueb2CIYKcfWxRlUztnKZUhYf/SoL91WWasPXIZhtvWGDSIjJOXMuHUhe4YkAcK5zDcbIoK5X9jfG9eydow/HjYmdSYMhpXL5jUwYbF4rfJOjg38MeNyODNtFYrdmX4a0o1NI46oGWZxWuWBH2L5Pm1rx33gAYQoMC+cJaVpQgSd/f4edM6TI2uQoga/gTsbbbrauzOZivBH+veznfB/3MOzsg6/gY9b7FpPt7AbqsmS0OXLwCvX7F7Dl19s1MdA1U9t3wSUxb2V4WO/5XvDbd/0N29ctTc3ZaJs66qbsBr9bfc0Lmja1konc+Fpv/lqtf4itW4V1q+WRc4W1QPKMktZUhvFnIPLioKLamIqf4XAhX4TVBGSwNd432ryK7xP7dE6ryYbLX9weHh8fLxfa/5GisfS0J9fBHkr3uQsH6hP+zv2n5hxzvJirZFw76jbXbNkXFhpaNc+2QzQJUJX1loj52/0CkXwlOh8Ygw2oH++Fv0iZ3E55TJsTlpZe3YNr7PDxUn5m6kKRGew3917vts92t0/vt7r9rpHvb3DneODg7vB+fvXH+BukH2dkU3h5RDefYpqdgeDiX/7Jvp6eweDGI1i1H0DcuQdeN1dO6/XPfL2j+4G3TtXjQ8OvWexvttxf/iZkQaH7m97ZomY0YO948ODZ/bRLEE9uNuxhyOT/cchuDcTg483J5ef/euzk/f+65Pr/tliDveFhh7s2f7urcDg25eOo/3S6X370omJoZFPOM/+HEqpzZdOb8/rfv/+/W7nP0n1ttgv7WSVPB+iqnxFU/RGrbFHaJa9tzq7WwO3kLglx8ziiJS/4HJHZWesJr6DbjfWG6JYR7ax2PYmeZuJcqHSIurKtmcebZToWvc2lPsYmW3Ss6/+bK8m4eWw3hDDBbzvHNjGweW03csbLJnNCPHBKOJnnC2EJ7Zbrg4wMZIqJtWX3dtGyWOyaYvK7IDKTFOgHO5vITTLTivFWuMzDLLPzZoA9jcDUDI1rLRpl79YcT2ajKy7e2d/7X98OT7+Oj0MTUheG7GZ4UtfCCxJPw9+jG/bl+B1y9oLJG2T9e8AAAD//6Kq7D0=" } diff --git a/filebeat/module/elasticsearch/slowlog/config/slowlog.yml b/filebeat/module/elasticsearch/slowlog/config/slowlog.yml index 54c69f3c2450..bcacf308e095 100644 --- a/filebeat/module/elasticsearch/slowlog/config/slowlog.yml +++ b/filebeat/module/elasticsearch/slowlog/config/slowlog.yml @@ -6,7 +6,7 @@ paths: exclude_files: [".gz$"] multiline: - pattern: '^\[?[0-9]{4}-[0-9]{2}-[0-9]{2}' + pattern: '^(\[?[0-9]{4}-[0-9]{2}-[0-9]{2}|{)' negate: true match: after diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json new file mode 100644 index 000000000000..b4dcca93b26e --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json @@ -0,0 +1,108 @@ +{ + "description": "Pipeline for parsing the Elasticsearch slow logs in JSON format.", + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ], + "processors": [ + { + "json": { + "field": "message", + "target_field": "elasticsearch.slowlog" + } + }, + { + "drop": { + "if": "ctx.elasticsearch.slowlog.type != 'index_indexing_slowlog' && ctx.elasticsearch.slowlog.type != 'index_search_slowlog'" + } + }, + { + "remove": { + "field": "elasticsearch.slowlog.type" + } + }, + { + "rename": { + "field": "elasticsearch.slowlog.level", + "target_field": "log.level" + } + }, + { + "rename": { + "field": "elasticsearch.slowlog.component", + "target_field": "elasticsearch.component" + } + }, + { + "dot_expander": { + "field": "cluster.name", + "path": "elasticsearch.slowlog" + } + }, + { + "rename": { + "field": "elasticsearch.slowlog.cluster.name", + "target_field": "elasticsearch.cluster.name" + } + }, + { + "dot_expander": { + "field": "node.name", + "path": "elasticsearch.slowlog" + } + }, + { + "rename": { + "field": "elasticsearch.slowlog.node.name", + "target_field": "elasticsearch.node.name" + } + }, + { + "dot_expander": { + "field": "cluster.uuid", + "path": "elasticsearch.slowlog" + } + }, + { + "rename": { + "field": "elasticsearch.slowlog.cluster.uuid", + "target_field": "elasticsearch.cluster.uuid", + "ignore_missing": true + } + }, + { + "dot_expander": { + "field": "node.id", + "path": "elasticsearch.slowlog" + } + }, + { + "rename": { + "field": "elasticsearch.slowlog.node.id", + "target_field": "elasticsearch.node.id", + "ignore_missing": true + } + }, + { + "grok": { + "field": "elasticsearch.slowlog.message", + "pattern_definitions" : { + "GREEDYMULTILINE" : "(.|\n)*", + "INDEXNAME": "[a-zA-Z0-9_.-]*" + }, + "patterns": [ + "(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?" + ] + } + }, + { + "remove": { + "field": "elasticsearch.slowlog.message" + } + } + ] +} diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json new file mode 100644 index 000000000000..cd3b23f40c99 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json @@ -0,0 +1,33 @@ +{ + "description": "Pipeline for parsing elasticsearch slow logs in plaintext format.", + "processors": [ + { + "grok": { + "field": "message", + "pattern_definitions": { + "GREEDYMULTILINE": "(.|\n)*", + "INDEXNAME": "[a-zA-Z0-9_.-]*" + }, + "patterns": [ + "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})?\\]\\[%{DATA:elasticsearch.slowlog.logger}\\]%{SPACE}\\[%{WORD:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?" + ] + } + }, + { + "split": { + "if": "ctx.elasticsearch.slowlog?.stats != ''", + "field": "elasticsearch.slowlog.stats", + "separator": ",", + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json index 1981c3711f71..91756d80d516 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json @@ -1,70 +1,81 @@ { - "description": "Pipeline for parsing elasticsearch slowlog logs", - "processors": [ - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "grok": { - "field": "message", - "pattern_definitions" : { - "GREEDYMULTILINE" : "(.|\n)*", - "INDEXNAME": "[a-zA-Z0-9_.-]*" - }, - "patterns": [ - "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})?\\]\\[%{DATA:elasticsearch.slowlog.logger}\\]%{SPACE}\\[%{WORD:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:temp.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?" - ] - } - }, - { - "split": { - "if": "ctx.elasticsearch.slowlog?.stats != ''", - "field": "elasticsearch.slowlog.stats", - "separator": ",", - "ignore_missing": true - } - }, - { - "date": { - "field": "elasticsearch.slowlog.timestamp", - "target_field": "@timestamp", - "formats": [ - "ISO8601" - ], - {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >} - "ignore_failure": true - } - }, - { - "remove": { - "field": [ - "elasticsearch.slowlog.timestamp" - ] - } - }, - - { - "script": { - "lang": "painless", - "source": "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)", - "params": { "scale": 1000000 }, - "if": "ctx.temp?.duration != null" - } - }, - { - "remove": { - "field": "temp.duration", - "ignore_missing": true - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] + "description": "Pipeline for parsing elasticsearch slow logs.", + "processors": [ + { + "rename": { + "field": "@timestamp", + "target_field": "event.created" + } + }, + { + "grok": { + "field": "message", + "patterns": [ + "^%{CHAR:first_char}" + ], + "pattern_definitions": { + "CHAR": "." + } + } + }, + { + "pipeline": { + "if": "ctx.first_char != '{'", + "name": "{< IngestPipeline "pipeline-plaintext" >}" + } + }, + { + "pipeline": { + "if": "ctx.first_char == '{'", + "name": "{< IngestPipeline "pipeline-json" >}" + } + }, + { + "date": { + "field": "elasticsearch.slowlog.timestamp", + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ], + {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >} + "ignore_failure": true + } + }, + { + "remove": { + "field": "elasticsearch.slowlog.timestamp" + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = Math.round(ctx.elasticsearch.slowlog.duration * params.scale)", + "params": { + "scale": 1000000 + }, + "if": "ctx.elasticsearch.slowlog?.duration != null" + } + }, + { + "remove": { + "field": "elasticsearch.slowlog.duration", + "ignore_missing": true + } + }, + { + "remove": { + "field": [ + "first_char" + ] + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] } diff --git a/filebeat/module/elasticsearch/slowlog/manifest.yml b/filebeat/module/elasticsearch/slowlog/manifest.yml index 904b5db5ef63..0e839abd39dd 100644 --- a/filebeat/module/elasticsearch/slowlog/manifest.yml +++ b/filebeat/module/elasticsearch/slowlog/manifest.yml @@ -5,12 +5,18 @@ var: default: - /var/log/elasticsearch/*_index_search_slowlog.log - /var/log/elasticsearch/*_index_indexing_slowlog.log + - /var/log/elasticsearch/*_index_search_slowlog.json + - /var/log/elasticsearch/*_index_indexing_slowlog.json os.darwin: - /usr/local/var/lib/elasticsearch/*_index_search_slowlog.log - /usr/local/var/lib/elasticsearch/*_index_indexing_slowlog.log + - /usr/local/var/lib/elasticsearch/*_index_search_slowlog.json + - /usr/local/var/lib/elasticsearch/*_index_indexing_slowlog.json os.windows: - c:/ProgramData/Elastic/Elasticsearch/logs/*_index_search_slowlog.log - c:/ProgramData/Elastic/Elasticsearch/logs/*_index_indexing_slowlog.log + - c:/ProgramData/Elastic/Elasticsearch/logs/*_index_search_slowlog.json + - c:/ProgramData/Elastic/Elasticsearch/logs/*_index_indexing_slowlog.json - name: convert_timezone default: false # if ES < 6.1.0, this flag switches to false automatically when evaluating the @@ -19,5 +25,8 @@ var: version: 6.1.0 value: false -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: + - ingest/pipeline.json + - ingest/pipeline-plaintext.json + - ingest/pipeline-json.json input: config/slowlog.yml diff --git a/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json index 32dda026a239..e5d67738dfca 100644 --- a/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json @@ -74,7 +74,6 @@ "elasticsearch.slowlog.id": "s01HZ2QBk9jw4gtgaFtn", "elasticsearch.slowlog.logger": "index.indexing.slowlog.index", "elasticsearch.slowlog.routing": "", - "elasticsearch.slowlog.source_query": "\n{\n \"@timestamp\":\"2018-07-04T21:27:30.730Z\",\n \"metricset\":{\n \"name\":\"network\",\n \"module\":\"system\",\n \"rtt\":7264},\n \"system\":{\n \"network\":{\n \"name\":\"lo0\",\n \"in\":{\n \"errors\":0,\n \"dropped\":0,\n \"bytes\":77666873,\n \"packets\":244595},\n \"out\":{\n \"packets\":244595,\n \"bytes\":77666873,\n \"errors\":0,\n \"dropped\":0\n }\n }\n },\n \"beat\":{\n \"name\":\"Rados-MacBook-Pro.local\",\n \"hostname\":\"Rados-MacBook-Pro.local\",\n \"version\":\"6.3.0\"\n },\n \"host\":{\n \"name\":\"Rados-MacBook-Pro.local\"\n }\n }", "elasticsearch.slowlog.took": "1.7ms", "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", @@ -82,12 +81,9 @@ "event.module": "elasticsearch", "fileset.name": "slowlog", "input.type": "log", - "log.flags": [ - "multiline" - ], "log.level": "INFO", "log.offset": 1817, - "message": "[2018-07-04T21:51:30,411][INFO ][index.indexing.slowlog.index] [v_VJhjV] [metricbeat-6.3.0-2018.07.04/VLKxBLvUSYuIMKzpacGjRg] took[1.7ms], took_millis[1], type[doc], id[s01HZ2QBk9jw4gtgaFtn], routing[], source[\n{\n \"@timestamp\":\"2018-07-04T21:27:30.730Z\",\n \"metricset\":{\n \"name\":\"network\",\n \"module\":\"system\",\n \"rtt\":7264},\n \"system\":{\n \"network\":{\n \"name\":\"lo0\",\n \"in\":{\n \"errors\":0,\n \"dropped\":0,\n \"bytes\":77666873,\n \"packets\":244595},\n \"out\":{\n \"packets\":244595,\n \"bytes\":77666873,\n \"errors\":0,\n \"dropped\":0\n }\n }\n },\n \"beat\":{\n \"name\":\"Rados-MacBook-Pro.local\",\n \"hostname\":\"Rados-MacBook-Pro.local\",\n \"version\":\"6.3.0\"\n },\n \"host\":{\n \"name\":\"Rados-MacBook-Pro.local\"\n }\n }]", + "message": "[2018-07-04T21:51:30,411][INFO ][index.indexing.slowlog.index] [v_VJhjV] [metricbeat-6.3.0-2018.07.04/VLKxBLvUSYuIMKzpacGjRg] took[1.7ms], took_millis[1], type[doc], id[s01HZ2QBk9jw4gtgaFtn], routing[], source[", "service.type": "elasticsearch" }, { diff --git a/filebeat/module/elasticsearch/slowlog/test/es_index_indexing_slowlog-json.log b/filebeat/module/elasticsearch/slowlog/test/es_index_indexing_slowlog-json.log new file mode 100644 index 000000000000..2fff1458a156 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es_index_indexing_slowlog-json.log @@ -0,0 +1,2 @@ +{"type": "index_indexing_slowlog", "timestamp": "2019-01-29T08:35:54,170+0100", "level": "WARN", "component": "i.i.s.index", "cluster.name": "distribution_run", "node.name": "node-0", "cluster.uuid": "oqKkg2eoQh2P_KrKliI3DA", "node.id": "U7rdLkcqR9eRvOiyLmr_qQ", "message": "[index1/PJlGJFFURIO1zIboktQMUw] took[4.6ms], took_millis[4], type[_doc], id[1], routing[], source[{\"somefield\":\"somevalue\"}]" } +{"type": "index_indexing_slowlog", "timestamp": "2019-01-29T08:35:58,359+0100", "level": "WARN", "component": "i.i.s.index", "cluster.name": "distribution_run", "node.name": "node-0", "cluster.uuid": "oqKkg2eoQh2P_KrKliI3DA", "node.id": "U7rdLkcqR9eRvOiyLmr_qQ", "message": "[index1/PJlGJFFURIO1zIboktQMUw] took[803micros], took_millis[0], type[_doc], id[2], routing[], source[{\"somefield\":\"somevalue\"}]" } diff --git a/filebeat/module/elasticsearch/slowlog/test/es_index_indexing_slowlog-json.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es_index_indexing_slowlog-json.log-expected.json new file mode 100644 index 000000000000..74de3b8a2037 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es_index_indexing_slowlog-json.log-expected.json @@ -0,0 +1,52 @@ +[ + { + "@timestamp": "2019-01-29T07:35:54.170Z", + "ecs.version": "1.0.0-beta2", + "elasticsearch.cluster.name": "distribution_run", + "elasticsearch.cluster.uuid": "oqKkg2eoQh2P_KrKliI3DA", + "elasticsearch.component": "i.i.s.index", + "elasticsearch.index.id": "PJlGJFFURIO1zIboktQMUw", + "elasticsearch.index.name": "index1", + "elasticsearch.node.id": "U7rdLkcqR9eRvOiyLmr_qQ", + "elasticsearch.node.name": "node-0", + "elasticsearch.slowlog.id": "1", + "elasticsearch.slowlog.routing": "", + "elasticsearch.slowlog.source_query": "{\"somefield\":\"somevalue\"}", + "elasticsearch.slowlog.took": "4.6ms", + "elasticsearch.slowlog.type": "_doc", + "event.dataset": "elasticsearch.slowlog", + "event.duration": 4000000, + "event.module": "elasticsearch", + "fileset.name": "slowlog", + "input.type": "log", + "log.level": "WARN", + "log.offset": 0, + "message": "{\"type\": \"index_indexing_slowlog\", \"timestamp\": \"2019-01-29T08:35:54,170+0100\", \"level\": \"WARN\", \"component\": \"i.i.s.index\", \"cluster.name\": \"distribution_run\", \"node.name\": \"node-0\", \"cluster.uuid\": \"oqKkg2eoQh2P_KrKliI3DA\", \"node.id\": \"U7rdLkcqR9eRvOiyLmr_qQ\", \"message\": \"[index1/PJlGJFFURIO1zIboktQMUw] took[4.6ms], took_millis[4], type[_doc], id[1], routing[], source[{\\\"somefield\\\":\\\"somevalue\\\"}]\" }", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2019-01-29T07:35:58.359Z", + "ecs.version": "1.0.0-beta2", + "elasticsearch.cluster.name": "distribution_run", + "elasticsearch.cluster.uuid": "oqKkg2eoQh2P_KrKliI3DA", + "elasticsearch.component": "i.i.s.index", + "elasticsearch.index.id": "PJlGJFFURIO1zIboktQMUw", + "elasticsearch.index.name": "index1", + "elasticsearch.node.id": "U7rdLkcqR9eRvOiyLmr_qQ", + "elasticsearch.node.name": "node-0", + "elasticsearch.slowlog.id": "2", + "elasticsearch.slowlog.routing": "", + "elasticsearch.slowlog.source_query": "{\"somefield\":\"somevalue\"}", + "elasticsearch.slowlog.took": "803micros", + "elasticsearch.slowlog.type": "_doc", + "event.dataset": "elasticsearch.slowlog", + "event.duration": 0, + "event.module": "elasticsearch", + "fileset.name": "slowlog", + "input.type": "log", + "log.level": "WARN", + "log.offset": 409, + "message": "{\"type\": \"index_indexing_slowlog\", \"timestamp\": \"2019-01-29T08:35:58,359+0100\", \"level\": \"WARN\", \"component\": \"i.i.s.index\", \"cluster.name\": \"distribution_run\", \"node.name\": \"node-0\", \"cluster.uuid\": \"oqKkg2eoQh2P_KrKliI3DA\", \"node.id\": \"U7rdLkcqR9eRvOiyLmr_qQ\", \"message\": \"[index1/PJlGJFFURIO1zIboktQMUw] took[803micros], took_millis[0], type[_doc], id[2], routing[], source[{\\\"somefield\\\":\\\"somevalue\\\"}]\" }", + "service.type": "elasticsearch" + } +] \ No newline at end of file diff --git a/filebeat/module/elasticsearch/slowlog/test/es_index_search_slowlog-json.log b/filebeat/module/elasticsearch/slowlog/test/es_index_search_slowlog-json.log new file mode 100644 index 000000000000..6239afaa3ea0 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es_index_search_slowlog-json.log @@ -0,0 +1,3 @@ +{"type": "index_search_slowlog", "timestamp": "2019-01-29T08:31:40,426+0100", "level": "WARN", "component": "i.s.s.query", "cluster.name": "distribution_run", "node.name": "node-0", "cluster.uuid": "oqKkg2eoQh2P_KrKliI3DA", "node.id": "U7rdLkcqR9eRvOiyLmr_qQ", "message": "[index1][0] took[70.4micros], took_millis[0], total_hits[0 hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{}], id[], " } +{"type": "index_search_slowlog", "timestamp": "2019-01-29T08:36:01,675+0100", "level": "WARN", "component": "i.s.s.query", "cluster.name": "distribution_run", "node.name": "node-0", "cluster.uuid": "oqKkg2eoQh2P_KrKliI3DA", "node.id": "U7rdLkcqR9eRvOiyLmr_qQ", "message": "[index1][0] took[731.3micros], took_millis[0], total_hits[2 hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{}], id[], " } +{"type": "index_search_slowlog", "timestamp": "2019-01-29T08:36:01,685+0100", "level": "WARN", "component": "i.s.s.fetch", "cluster.name": "distribution_run", "node.name": "node-0", "cluster.uuid": "oqKkg2eoQh2P_KrKliI3DA", "node.id": "U7rdLkcqR9eRvOiyLmr_qQ", "message": "[index1][0] took[9.9ms], took_millis[9], total_hits[2 hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{}], id[], " } diff --git a/filebeat/module/elasticsearch/slowlog/test/es_index_search_slowlog-json.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es_index_search_slowlog-json.log-expected.json new file mode 100644 index 000000000000..9ea5fff1821a --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es_index_search_slowlog-json.log-expected.json @@ -0,0 +1,65 @@ +[ + { + "@timestamp": "2019-01-29T07:31:40.426Z", + "ecs.version": "1.0.0-beta2", + "elasticsearch.cluster.name": "distribution_run", + "elasticsearch.cluster.uuid": "oqKkg2eoQh2P_KrKliI3DA", + "elasticsearch.component": "i.s.s.query", + "elasticsearch.index.name": "index1", + "elasticsearch.node.id": "U7rdLkcqR9eRvOiyLmr_qQ", + "elasticsearch.node.name": "node-0", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.took": "70.4micros", + "event.dataset": "elasticsearch.slowlog", + "event.duration": 0, + "event.module": "elasticsearch", + "fileset.name": "slowlog", + "input.type": "log", + "log.level": "WARN", + "log.offset": 0, + "message": "{\"type\": \"index_search_slowlog\", \"timestamp\": \"2019-01-29T08:31:40,426+0100\", \"level\": \"WARN\", \"component\": \"i.s.s.query\", \"cluster.name\": \"distribution_run\", \"node.name\": \"node-0\", \"cluster.uuid\": \"oqKkg2eoQh2P_KrKliI3DA\", \"node.id\": \"U7rdLkcqR9eRvOiyLmr_qQ\", \"message\": \"[index1][0] took[70.4micros], took_millis[0], total_hits[0 hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{}], id[], \" }", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2019-01-29T07:36:01.675Z", + "ecs.version": "1.0.0-beta2", + "elasticsearch.cluster.name": "distribution_run", + "elasticsearch.cluster.uuid": "oqKkg2eoQh2P_KrKliI3DA", + "elasticsearch.component": "i.s.s.query", + "elasticsearch.index.name": "index1", + "elasticsearch.node.id": "U7rdLkcqR9eRvOiyLmr_qQ", + "elasticsearch.node.name": "node-0", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.took": "731.3micros", + "event.dataset": "elasticsearch.slowlog", + "event.duration": 0, + "event.module": "elasticsearch", + "fileset.name": "slowlog", + "input.type": "log", + "log.level": "WARN", + "log.offset": 429, + "message": "{\"type\": \"index_search_slowlog\", \"timestamp\": \"2019-01-29T08:36:01,675+0100\", \"level\": \"WARN\", \"component\": \"i.s.s.query\", \"cluster.name\": \"distribution_run\", \"node.name\": \"node-0\", \"cluster.uuid\": \"oqKkg2eoQh2P_KrKliI3DA\", \"node.id\": \"U7rdLkcqR9eRvOiyLmr_qQ\", \"message\": \"[index1][0] took[731.3micros], took_millis[0], total_hits[2 hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{}], id[], \" }", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2019-01-29T07:36:01.685Z", + "ecs.version": "1.0.0-beta2", + "elasticsearch.cluster.name": "distribution_run", + "elasticsearch.cluster.uuid": "oqKkg2eoQh2P_KrKliI3DA", + "elasticsearch.component": "i.s.s.fetch", + "elasticsearch.index.name": "index1", + "elasticsearch.node.id": "U7rdLkcqR9eRvOiyLmr_qQ", + "elasticsearch.node.name": "node-0", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.took": "9.9ms", + "event.dataset": "elasticsearch.slowlog", + "event.duration": 9000000, + "event.module": "elasticsearch", + "fileset.name": "slowlog", + "input.type": "log", + "log.level": "WARN", + "log.offset": 859, + "message": "{\"type\": \"index_search_slowlog\", \"timestamp\": \"2019-01-29T08:36:01,685+0100\", \"level\": \"WARN\", \"component\": \"i.s.s.fetch\", \"cluster.name\": \"distribution_run\", \"node.name\": \"node-0\", \"cluster.uuid\": \"oqKkg2eoQh2P_KrKliI3DA\", \"node.id\": \"U7rdLkcqR9eRvOiyLmr_qQ\", \"message\": \"[index1][0] took[9.9ms], took_millis[9], total_hits[2 hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{}], id[], \" }", + "service.type": "elasticsearch" + } +] \ No newline at end of file diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json index 92559d982bfd..ec13bd7ce587 100644 --- a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json @@ -129,7 +129,6 @@ "elasticsearch.slowlog.id": "s01HZ2QBk9jw4gtgaFtn", "elasticsearch.slowlog.logger": "index.indexing.slowlog.index", "elasticsearch.slowlog.routing": "", - "elasticsearch.slowlog.source_query": "\n{\n \"@timestamp\":\"2018-07-04T21:27:30.730Z\",\n \"metricset\":{\n \"name\":\"network\",\n \"module\":\"system\",\n \"rtt\":7264},\n \"system\":{\n \"network\":{\n \"name\":\"lo0\",\n \"in\":{\n \"errors\":0,\n \"dropped\":0,\n \"bytes\":77666873,\n \"packets\":244595},\n \"out\":{\n \"packets\":244595,\n \"bytes\":77666873,\n \"errors\":0,\n \"dropped\":0\n }\n }\n },\n \"beat\":{\n \"name\":\"Rados-MacBook-Pro.local\",\n \"hostname\":\"Rados-MacBook-Pro.local\",\n \"version\":\"6.3.0\"\n },\n \"host\":{\n \"name\":\"Rados-MacBook-Pro.local\"\n }\n }", "elasticsearch.slowlog.took": "1.7ms", "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", @@ -137,12 +136,9 @@ "event.module": "elasticsearch", "fileset.name": "slowlog", "input.type": "log", - "log.flags": [ - "multiline" - ], "log.level": "INFO", "log.offset": 4766, - "message": "[2018-07-04T21:51:30,411][INFO ][index.indexing.slowlog.index] [v_VJhjV] [metricbeat-6.3.0-2018.07.04/VLKxBLvUSYuIMKzpacGjRg] took[1.7ms], took_millis[1], type[doc], id[s01HZ2QBk9jw4gtgaFtn], routing[], source[\n{\n \"@timestamp\":\"2018-07-04T21:27:30.730Z\",\n \"metricset\":{\n \"name\":\"network\",\n \"module\":\"system\",\n \"rtt\":7264},\n \"system\":{\n \"network\":{\n \"name\":\"lo0\",\n \"in\":{\n \"errors\":0,\n \"dropped\":0,\n \"bytes\":77666873,\n \"packets\":244595},\n \"out\":{\n \"packets\":244595,\n \"bytes\":77666873,\n \"errors\":0,\n \"dropped\":0\n }\n }\n },\n \"beat\":{\n \"name\":\"Rados-MacBook-Pro.local\",\n \"hostname\":\"Rados-MacBook-Pro.local\",\n \"version\":\"6.3.0\"\n },\n \"host\":{\n \"name\":\"Rados-MacBook-Pro.local\"\n }\n }]", + "message": "[2018-07-04T21:51:30,411][INFO ][index.indexing.slowlog.index] [v_VJhjV] [metricbeat-6.3.0-2018.07.04/VLKxBLvUSYuIMKzpacGjRg] took[1.7ms], took_millis[1], type[doc], id[s01HZ2QBk9jw4gtgaFtn], routing[], source[", "service.type": "elasticsearch" } ] \ No newline at end of file