merge application and router logs into single pipeline

Author: Schnitzel

lagoon-remote/logs-forwarder-logstash/.lagoon.yml

@@ -118,3 +118,11 @@ objects:
     externalName: ${SERVICE_NAME}.${OPENSHIFT_PROJECT}.svc.cluster.local
     sessionAffinity: None
     type: ExternalName
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: application-logs
+  spec:
+    externalName: ${SERVICE_NAME}.${OPENSHIFT_PROJECT}.svc.cluster.local
+    sessionAffinity: None
+    type: ExternalName

services/logs2logs-db/.lagoon.yml

@@ -186,3 +186,11 @@ objects:
     externalName: ${SERVICE_NAME}.${OPENSHIFT_PROJECT}.svc.cluster.local
     sessionAffinity: None
     type: ExternalName
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: application-logs
+  spec:
+    externalName: ${SERVICE_NAME}.${OPENSHIFT_PROJECT}.svc.cluster.local
+    sessionAffinity: None
+    type: ExternalName

services/logs2logs-db/pipeline/application-logs.conf

@@ -0,0 +1,78 @@
+input {
+  udp {
+    port => 5140
+    queue_size => 5000
+    receive_buffer_bytes => 26214400
+  }
+  lumberjack {
+    port => 5044
+    ssl_certificate => "certs/lumberjack.cert"
+    ssl_key => "certs/lumberjack.key"
+    codec => json
+  }
+}
+
+filter {
+  if [message] =~ /^{.*}$/ {
+    # message is JSON, this is an application log
+    mutate {
+      add_field => { "log-type" => "application-logs" }
+    }
+    json {
+      source => "message"
+    }
+    if ![type] {
+      mutate {
+        add_field => { "type" => "noproject" }
+      }
+    }
+  } else {
+    # Not JSON, therefore a syslog and a router log entry
+    mutate {
+      add_field => { "log-type" => "router-logs" }
+    }
+    grok {
+      match => ["message", "(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{SYSLOGPROG}: %{HAPROXYHTTPBASE}"]
+    }
+    grok {
+      match => ["captured_request_headers", "%{URIHOST:request_header_host}\|%{GREEDYDATA:request_header_useragent}"]
+    }
+    grok {
+      match => ["backend_name", "%{NOTSPACE:haproxy_backend}:%{NOTSPACE:openshift_project}:%{NOTSPACE:openshift_route}"]
+    }
+    grok {
+      match => ["server_name", "pod:%{NOTSPACE:openshift_pod}:%{NOTSPACE:openshift_service}:%{NOTSPACE:openshift_pod_ip}:%{NOTSPACE:openshift_pod_port}"]
+    }
+    if ![openshift_project] {
+      mutate {
+        add_field => { "openshift_project" => "noproject" }
+      }
+    }
+  }
+}
+
+output {
+  # stdout { codec => rubydebug }
+  if [log-type] == "router-logs" {
+    elasticsearch {
+      user => admin
+      password => "${LOGSDB_ADMIN_PASSWORD}"
+      hosts => ["${ELASTICSEARCH_URL}"]
+      index => "router-logs-%{[openshift_project]}-%{+YYYY.MM}"
+      template => "/usr/share/logstash/templates/router-logs.json"
+      template_name => "router-logs"
+      template_overwrite => true
+    }
+  } else {
+    elasticsearch {
+      user => admin
+      password => "${LOGSDB_ADMIN_PASSWORD}"
+      hosts => ["${ELASTICSEARCH_URL}"]
+      index => "application-logs-%{[type]}-%{+YYYY.MM}"
+      template => "/usr/share/logstash/templates/application-logs.json"
+      template_name => "application-logs"
+      template_overwrite => true
+    }
+  }
+
+}

services/logs2logs-db/pipeline/router-application-logs.conf

@@ -1,8 +1,6 @@
 - pipeline.id: lagoon-logs
   path.config: "pipeline/lagoon-logs.conf"
-- pipeline.id: router-logs
-  path.config: "pipeline/router-logs.conf"
+- pipeline.id: router-application-logs
+  path.config: "pipeline/router-application-logs.conf"
 - pipeline.id: service-logs
   path.config: "pipeline/service-logs.conf"
-- pipeline.id: application-logs
-  path.config: "pipeline/application-logs.conf"

services/logs2logs-db/pipeline/router-logs.conf

@@ -0,0 +1,47 @@
+{
+    "template" : "application-logs-*",
+    "version" : 60002,
+    "settings" : {
+      "index.refresh_interval" : "5s",
+      "number_of_shards": 1,
+      "number_of_replicas": 1
+    },
+    "mappings" : {
+      "_default_" : {
+        "dynamic_templates" : [ {
+          "message_field" : {
+            "path_match" : "message",
+            "match_mapping_type" : "string",
+            "mapping" : {
+              "type" : "text",
+              "norms" : false
+            }
+          }
+        }, {
+          "string_fields" : {
+            "match" : "*",
+            "match_mapping_type" : "string",
+            "mapping" : {
+              "type" : "text", "norms" : false,
+              "fields" : {
+                "keyword" : { "type": "keyword", "ignore_above": 256 }
+              }
+            }
+          }
+        } ],
+        "properties" : {
+          "@timestamp": { "type": "date"},
+          "@version": { "type": "keyword"},
+          "geoip"  : {
+            "dynamic": true,
+            "properties" : {
+              "ip": { "type": "ip" },
+              "location" : { "type" : "geo_point" },
+              "latitude" : { "type" : "half_float" },
+              "longitude" : { "type" : "half_float" }
+            }
+          }
+        }
+      }
+    }
+  }