From 0acc1f838d5ca0dbe690d267981815bc5df06907 Mon Sep 17 00:00:00 2001 From: noudAndi Date: Mon, 13 Mar 2017 14:13:23 +0100 Subject: [PATCH] Exported a new haproxy-extractor, which fits the actual default format of haproxy. Main change: - in regex: old "haprxy:" new "haproxy\[.+\]:" - in condition: old: ^haproxy: new: "haproxy\[.+\]:" --- content_pack.json | 527 ++++++++++++++++++++++++---------------------- 1 file changed, 271 insertions(+), 256 deletions(-) diff --git a/content_pack.json b/content_pack.json index 812c52c..0cdad4b 100644 --- a/content_pack.json +++ b/content_pack.json @@ -1,275 +1,290 @@ { - "id" : null, - "name" : "HAProxy HTTP logs", - "description" : "This content pack will launch an UDP input on port 11002 that is able to parse the standard HAProxy HTTP logs. ", - "category" : "HTTP servers", - "inputs" : [ { - "title" : "HAProxy HTTP logs", - "configuration" : { - "override_source" : "", - "recv_buffer_size" : 262144, - "bind_address" : "0.0.0.0", - "port" : 11002 - }, - "type" : "org.graylog2.inputs.raw.udp.RawUDPInput", - "global" : false, - "extractors" : [ { - "title" : "Remote Address", - "type" : "REGEX", - "configuration" : { - "regex_value" : "haproxy:\\s+(\\S+):" + "extractors": [ + { + "title": "Milliseconds spent waiting for the client to send a full HTTP request", + "extractor_type": "split_and_index", + "converters": [ + { + "type": "numeric", + "config": {} + } + ], + "order": 0, + "cursor_strategy": "copy", + "source_field": "timings", + "target_field": "tq", + "extractor_config": { + "index": 1, + "split_by": "/" }, - "converters" : [ ], - "order" : 1, - "cursor_strategy" : "COPY", - "target_field" : "remote_addr", - "source_field" : "message", - "condition_type" : "REGEX", - "condition_value" : "^haproxy:" - }, { - "title" : "Request Timestamp", - "type" : "REGEX", - "configuration" : { - "regex_value" : "haproxy:.+?\\[(.+?)\\]" + "condition_type": "none", + "condition_value": "haproxy:" + }, + { + "title": "Milliseconds spent waiting in queues", + "extractor_type": "split_and_index", + "converters": [ + { + "type": "numeric", + "config": {} + } + ], + "order": 0, + "cursor_strategy": "copy", + "source_field": "timings", + "target_field": "tw", + "extractor_config": { + "index": 2, + "split_by": "/" }, - "converters" : [ { - "type" : "DATE", - "configuration" : { - "date_format" : "dd/MMM/YYYY:HH:mm:ss.SSS" + "condition_type": "none", + "condition_value": "" + }, + { + "title": "Milliseconds spent waiting for the connection to establish to the final server", + "extractor_type": "split_and_index", + "converters": [ + { + "type": "numeric", + "config": {} } - } ], - "order" : 3, - "cursor_strategy" : "COPY", - "target_field" : "timestamp", - "source_field" : "message", - "condition_type" : "REGEX", - "condition_value" : "^haproxy:" - }, { - "title" : "Frontend", - "type" : "REGEX", - "configuration" : { - "regex_value" : "haproxy:.+?\\[.+?\\]\\s(\\S+)" + ], + "order": 0, + "cursor_strategy": "copy", + "source_field": "timings", + "target_field": "tc", + "extractor_config": { + "index": 3, + "split_by": "/" }, - "converters" : [ ], - "order" : 4, - "cursor_strategy" : "COPY", - "target_field" : "frontend", - "source_field" : "message", - "condition_type" : "REGEX", - "condition_value" : "^haproxy:" - }, { - "title" : "Server", - "type" : "REGEX", - "configuration" : { - "regex_value" : "haproxy:.+?\\[.+?\\].+?/(\\S+)" + "condition_type": "none", + "condition_value": "" + }, + { + "title": "Milliseconds spent waiting for the server to send a full HTTP response", + "extractor_type": "split_and_index", + "converters": [ + { + "type": "numeric", + "config": {} + } + ], + "order": 0, + "cursor_strategy": "copy", + "source_field": "timings", + "target_field": "tr", + "extractor_config": { + "index": 4, + "split_by": "/" }, - "converters" : [ ], - "order" : 6, - "cursor_strategy" : "COPY", - "target_field" : "server", - "source_field" : "message", - "condition_type" : "REGEX", - "condition_value" : "^haproxy:" - }, { - "title" : "Backend", - "type" : "REGEX", - "configuration" : { - "regex_value" : "haproxy:.+?\\[.+?\\].+?(\\S+)/" + "condition_type": "none", + "condition_value": "" + }, + { + "title": "Milliseconds elapsed between the accept and the last close", + "extractor_type": "split_and_index", + "converters": [ + { + "type": "numeric", + "config": {} + } + ], + "order": 0, + "cursor_strategy": "copy", + "source_field": "timings", + "target_field": "tt", + "extractor_config": { + "index": 5, + "split_by": "/" }, - "converters" : [ ], - "order" : 5, - "cursor_strategy" : "COPY", - "target_field" : "backend", - "source_field" : "message", - "condition_type" : "REGEX", - "condition_value" : "^haproxy:" - }, { - "title" : "Response Status", - "type" : "REGEX", - "configuration" : { - "regex_value" : "haproxy:.+?\\[.+?\\].+?\\s\\S+\\s\\S+\\s(\\d+)" + "condition_type": "none", + "condition_value": "" + }, + { + "title": "Timings", + "extractor_type": "regex", + "converters": [], + "order": 0, + "cursor_strategy": "copy", + "source_field": "message", + "target_field": "timings", + "extractor_config": { + "regex_value": "haproxy\\[.+\\]:.+?\\[.+?\\].+?\\S+/\\S+\\s(\\S+)" }, - "converters" : [ { - "type" : "NUMERIC", - "configuration" : { } - } ], - "order" : 7, - "cursor_strategy" : "COPY", - "target_field" : "response_status", - "source_field" : "message", - "condition_type" : "REGEX", - "condition_value" : "^haproxy:" - }, { - "title" : "Response Bytes", - "type" : "REGEX", - "configuration" : { - "regex_value" : "haproxy:.+?\\[.+?\\].+?\\s\\S+\\s\\S+\\s\\d+\\s(\\d+)" + "condition_type": "regex", + "condition_value": "haproxy\\[.+\\]:" + }, + { + "title": "Remote Address", + "extractor_type": "regex", + "converters": [], + "order": 1, + "cursor_strategy": "copy", + "source_field": "message", + "target_field": "remote_addr", + "extractor_config": { + "regex_value": "haproxy\\[.+\\]:\\s+(\\S+):" }, - "converters" : [ { - "type" : "NUMERIC", - "configuration" : { } - } ], - "order" : 8, - "cursor_strategy" : "COPY", - "target_field" : "response_bytes", - "source_field" : "message", - "condition_type" : "REGEX", - "condition_value" : "^haproxy:" - }, { - "title" : "Request Verb", - "type" : "REGEX", - "configuration" : { - "regex_value" : "haproxy:.+?\"(\\w+)\\s" + "condition_type": "regex", + "condition_value": "haproxy\\[.+\\]:" + }, + { + "title": "HTTP Version", + "extractor_type": "regex", + "converters": [], + "order": 2, + "cursor_strategy": "copy", + "source_field": "message", + "target_field": "http_version", + "extractor_config": { + "regex_value": "haproxy\\[.+\\]:.+HTTP/(\\S+)\"" }, - "converters" : [ ], - "order" : 9, - "cursor_strategy" : "COPY", - "target_field" : "request_verb", - "source_field" : "message", - "condition_type" : "REGEX", - "condition_value" : "^haproxy:" - }, { - "title" : "Request Path", - "type" : "REGEX", - "configuration" : { - "regex_value" : "haproxy:.+?\"\\w+\\s(\\S+)" + "condition_type": "regex", + "condition_value": "haproxy\\[.+\\]:" + }, + { + "title": "Request Timestamp", + "extractor_type": "regex", + "converters": [ + { + "type": "date", + "config": { + "date_format": "dd/MMM/YYYY:HH:mm:ss.SSS" + } + } + ], + "order": 3, + "cursor_strategy": "copy", + "source_field": "message", + "target_field": "timestamp", + "extractor_config": { + "regex_value": "haproxy\\[.+\\]:.+?\\[(.+?)\\]" }, - "converters" : [ ], - "order" : 10, - "cursor_strategy" : "COPY", - "target_field" : "request_path", - "source_field" : "message", - "condition_type" : "REGEX", - "condition_value" : "^haproxy:" - }, { - "title" : "HTTP Version", - "type" : "REGEX", - "configuration" : { - "regex_value" : "haproxy:.+HTTP/(\\S+)\"" + "condition_type": "regex", + "condition_value": "haproxy\\[.+\\]:" + }, + { + "title": "Frontend", + "extractor_type": "regex", + "converters": [], + "order": 4, + "cursor_strategy": "copy", + "source_field": "message", + "target_field": "frontend", + "extractor_config": { + "regex_value": "haproxy\\[.+\\]:.+?\\[.+?\\]\\s(\\S+)" }, - "converters" : [ ], - "order" : 2, - "cursor_strategy" : "COPY", - "target_field" : "http_version", - "source_field" : "message", - "condition_type" : "REGEX", - "condition_value" : "^haproxy:" - }, { - "title" : "Message", - "type" : "REGEX", - "configuration" : { - "regex_value" : "haproxy:.+\"(.+)\"" + "condition_type": "regex", + "condition_value": "haproxy\\[.+\\]:" + }, + { + "title": "Backend", + "extractor_type": "regex", + "converters": [], + "order": 5, + "cursor_strategy": "copy", + "source_field": "message", + "target_field": "backend", + "extractor_config": { + "regex_value": "haproxy\\[.+\\]:.+?\\[.+?\\].+?(\\S+)/" }, - "converters" : [ ], - "order" : 11, - "cursor_strategy" : "COPY", - "target_field" : "message", - "source_field" : "message", - "condition_type" : "REGEX", - "condition_value" : "^haproxy:" - }, { - "title" : "Timings", - "type" : "REGEX", - "configuration" : { - "regex_value" : "haproxy:.+?\\[.+?\\].+?\\S+/\\S+\\s(\\S+)" + "condition_type": "regex", + "condition_value": "haproxy\\[.+\\]:" + }, + { + "title": "Server", + "extractor_type": "regex", + "converters": [], + "order": 6, + "cursor_strategy": "copy", + "source_field": "message", + "target_field": "server", + "extractor_config": { + "regex_value": "haproxy\\[.+\\]:.+?\\[.+?\\].+?/(\\S+)" }, - "converters" : [ ], - "order" : 0, - "cursor_strategy" : "COPY", - "target_field" : "timings", - "source_field" : "message", - "condition_type" : "REGEX", - "condition_value" : "haproxy:" - }, { - "title" : "Milliseconds spent waiting for the client to send a full HTTP request", - "type" : "SPLIT_AND_INDEX", - "configuration" : { - "index" : 1, - "split_by" : "/" + "condition_type": "regex", + "condition_value": "haproxy\\[.+\\]:" + }, + { + "title": "Response Status", + "extractor_type": "regex", + "converters": [ + { + "type": "numeric", + "config": {} + } + ], + "order": 7, + "cursor_strategy": "copy", + "source_field": "message", + "target_field": "response_status", + "extractor_config": { + "regex_value": "haproxy\\[.+\\]:.+?\\[.+?\\].+?\\s\\S+\\s\\S+\\s(\\d+)" }, - "converters" : [ { - "type" : "NUMERIC", - "configuration" : { } - } ], - "order" : 0, - "cursor_strategy" : "COPY", - "target_field" : "tq", - "source_field" : "timings", - "condition_type" : "NONE", - "condition_value" : "haproxy:" - }, { - "title" : "Milliseconds spent waiting in queues", - "type" : "SPLIT_AND_INDEX", - "configuration" : { - "index" : 2, - "split_by" : "/" + "condition_type": "regex", + "condition_value": "haproxy\\[.+\\]:" + }, + { + "title": "Response Bytes", + "extractor_type": "regex", + "converters": [ + { + "type": "numeric", + "config": {} + } + ], + "order": 8, + "cursor_strategy": "copy", + "source_field": "message", + "target_field": "response_bytes", + "extractor_config": { + "regex_value": "haproxy\\[.+\\]:.+?\\[.+?\\].+?\\s\\S+\\s\\S+\\s\\d+\\s(\\d+)" }, - "converters" : [ { - "type" : "NUMERIC", - "configuration" : { } - } ], - "order" : 0, - "cursor_strategy" : "COPY", - "target_field" : "tw", - "source_field" : "timings", - "condition_type" : "NONE", - "condition_value" : "" - }, { - "title" : "Milliseconds spent waiting for the connection to establish to the final server", - "type" : "SPLIT_AND_INDEX", - "configuration" : { - "index" : 3, - "split_by" : "/" + "condition_type": "regex", + "condition_value": "haproxy\\[.+\\]:" + }, + { + "title": "Request Verb", + "extractor_type": "regex", + "converters": [], + "order": 9, + "cursor_strategy": "copy", + "source_field": "message", + "target_field": "request_verb", + "extractor_config": { + "regex_value": "haproxy\\[.+\\]:.+?\"(\\w+)\\s" }, - "converters" : [ { - "type" : "NUMERIC", - "configuration" : { } - } ], - "order" : 0, - "cursor_strategy" : "COPY", - "target_field" : "tc", - "source_field" : "timings", - "condition_type" : "NONE", - "condition_value" : "" - }, { - "title" : "Milliseconds spent waiting for the server to send a full HTTP response", - "type" : "SPLIT_AND_INDEX", - "configuration" : { - "index" : 4, - "split_by" : "/" + "condition_type": "regex", + "condition_value": "haproxy\\[.+\\]:" + }, + { + "title": "Request Path", + "extractor_type": "regex", + "converters": [], + "order": 10, + "cursor_strategy": "copy", + "source_field": "message", + "target_field": "request_path", + "extractor_config": { + "regex_value": "haproxy\\[.+\\]:.+?\"\\w+\\s(\\S+)" }, - "converters" : [ { - "type" : "NUMERIC", - "configuration" : { } - } ], - "order" : 0, - "cursor_strategy" : "COPY", - "target_field" : "tr", - "source_field" : "timings", - "condition_type" : "NONE", - "condition_value" : "" - }, { - "title" : "Milliseconds elapsed between the accept and the last close", - "type" : "SPLIT_AND_INDEX", - "configuration" : { - "index" : 5, - "split_by" : "/" + "condition_type": "regex", + "condition_value": "haproxy\\[.+\\]:" + }, + { + "title": "Message", + "extractor_type": "regex", + "converters": [], + "order": 11, + "cursor_strategy": "copy", + "source_field": "message", + "target_field": "message", + "extractor_config": { + "regex_value": "haproxy\\[.+\\]:.+\"(.+)\"" }, - "converters" : [ { - "type" : "NUMERIC", - "configuration" : { } - } ], - "order" : 0, - "cursor_strategy" : "COPY", - "target_field" : "tt", - "source_field" : "timings", - "condition_type" : "NONE", - "condition_value" : "" - } ], - "static_fields" : { } - } ], - "streams" : [ ], - "outputs" : [ ], - "dashboards" : [ ], - "grok_patterns" : [ ] -} + "condition_type": "regex", + "condition_value": "haproxy\\[.+\\]:" + } + ], + "version": "2.2.0-SNAPSHOT" +} \ No newline at end of file