FEATURE: pfSense Suricata logs

salt/elasticsearch/files/ingest/.fleet_final_pipeline-1

@@ -68,7 +68,7 @@
         "field": "_security",
         "ignore_missing": true
       }
-    },
+    }, 
     { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
     { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
     { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },

salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0

@@ -0,0 +1,389 @@
+{
+  "description": "Pipeline for pfSense",
+  "processors": [
+    {
+      "set": {
+        "field": "ecs.version",
+        "value": "8.10.0"
+      }
+    },
+    {
+      "set": {
+        "field": "observer.vendor",
+        "value": "netgate"
+      }
+    },
+    {
+      "set": {
+        "field": "observer.type",
+        "value": "firewall"
+      }
+    },
+    {
+      "rename": {
+        "field": "message",
+        "target_field": "event.original"
+      }
+    },
+    {
+      "set": {
+        "field": "event.kind",
+        "value": "event"
+      }
+    },
+    {
+      "set": {
+        "field": "event.timezone",
+        "value": "{{_tmp.tz_offset}}",
+        "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
+      }
+    },
+    {
+      "grok": {
+        "description": "Parse syslog header",
+        "field": "event.original",
+        "patterns": [
+          "^(%{ECS_SYSLOG_PRI})?%{TIMESTAMP} %{GREEDYDATA:message}"
+        ],
+        "pattern_definitions": {
+          "ECS_SYSLOG_PRI": "<%{NONNEGINT:log.syslog.priority:long}>(\\d )?",
+          "TIMESTAMP": "(?:%{BSD_TIMESTAMP_FORMAT}|%{SYSLOG_TIMESTAMP_FORMAT})",
+          "BSD_TIMESTAMP_FORMAT": "%{SYSLOGTIMESTAMP:_tmp.timestamp}(%{SPACE}%{BSD_PROCNAME}|%{SPACE}%{OBSERVER}%{SPACE}%{BSD_PROCNAME})(\\[%{POSINT:process.pid:long}\\])?:",
+          "BSD_PROCNAME": "(?:\\b%{NAME:process.name}|\\(%{NAME:process.name}\\))",
+          "NAME": "[[[:alnum:]]_-]+",
+          "SYSLOG_TIMESTAMP_FORMAT": "%{TIMESTAMP_ISO8601:_tmp.timestamp8601}%{SPACE}%{OBSERVER}%{SPACE}%{PROCESS}%{SPACE}(%{POSINT:process.pid:long}|-) - (-|%{META})",
+          "TIMESTAMP_ISO8601": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE:event.timezone}?",
+          "OBSERVER": "(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})",
+          "PROCESS": "(\\(%{DATA:process.name}\\)|(?:%{UNIXPATH}*/)?%{BASEPATH:process.name})",
+          "BASEPATH": "[[[:alnum:]]_%!$@:.,+~-]+",
+          "META": "\\[[^\\]]*\\]"
+        }
+      }
+    },
+    {
+      "date": {
+        "if": "ctx._tmp.timestamp8601 != null",
+        "field": "_tmp.timestamp8601",
+        "target_field": "@timestamp",
+        "formats": [
+          "ISO8601"
+        ]
+      }
+    },
+    {
+      "date": {
+        "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null",
+        "field": "_tmp.timestamp",
+        "target_field": "@timestamp",
+        "formats": [
+          "MMM  d HH:mm:ss",
+          "MMM d HH:mm:ss",
+          "MMM dd HH:mm:ss"
+        ],
+        "timezone": "{{ event.timezone }}"
+      }
+    },
+    {
+      "grok": {
+        "description": "Set Event Provider",
+        "field": "process.name",
+        "patterns": [
+          "^%{HYPHENATED_WORDS:event.provider}"
+        ],
+        "pattern_definitions": {
+          "HYPHENATED_WORDS": "\\b[A-Za-z0-9_]+(-[A-Za-z_]+)*\\b"
+        }
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-firewall",
+        "if": "ctx.event.provider == 'filterlog'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-openvpn",
+        "if": "ctx.event.provider == 'openvpn'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-ipsec",
+        "if": "ctx.event.provider == 'charon'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-dhcp",
+        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-unbound",
+        "if": "ctx.event.provider == 'unbound'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-haproxy",
+        "if": "ctx.event.provider == 'haproxy'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-php-fpm",
+        "if": "ctx.event.provider == 'php-fpm'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-squid",
+        "if": "ctx.event.provider == 'squid'"
+      }
+    },
+    {
+     "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-suricata",
+        "if": "ctx.event.provider == 'suricata'"
+      }
+    },
+    {
+      "drop": {
+        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"suricata\"].contains(ctx.event?.provider)"
+      }
+    },
+    {
+      "append": {
+        "field": "event.category",
+        "value": "network",
+        "if": "ctx.network != null"
+      }
+    },
+    {
+      "convert": {
+        "field": "source.address",
+        "target_field": "source.ip",
+        "type": "ip",
+        "ignore_failure": true,
+        "ignore_missing": true
+      }
+    },
+    {
+      "convert": {
+        "field": "destination.address",
+        "target_field": "destination.ip",
+        "type": "ip",
+        "ignore_failure": true,
+        "ignore_missing": true
+      }
+    },
+    {
+      "set": {
+        "field": "network.type",
+        "value": "ipv6",
+        "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")"
+      }
+    },
+    {
+      "set": {
+        "field": "network.type",
+        "value": "ipv4",
+        "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")"
+      }
+    },
+    {
+      "geoip": {
+        "field": "source.ip",
+        "target_field": "source.geo",
+        "ignore_missing": true
+      }
+    },
+    {
+      "geoip": {
+        "field": "destination.ip",
+        "target_field": "destination.geo",
+        "ignore_missing": true
+      }
+    },
+    {
+      "geoip": {
+        "ignore_missing": true,
+        "database_file": "GeoLite2-ASN.mmdb",
+        "field": "source.ip",
+        "target_field": "source.as",
+        "properties": [
+          "asn",
+          "organization_name"
+        ]
+      }
+    },
+    {
+      "geoip": {
+        "database_file": "GeoLite2-ASN.mmdb",
+        "field": "destination.ip",
+        "target_field": "destination.as",
+        "properties": [
+          "asn",
+          "organization_name"
+        ],
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "source.as.asn",
+        "target_field": "source.as.number",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "source.as.organization_name",
+        "target_field": "source.as.organization.name",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "destination.as.asn",
+        "target_field": "destination.as.number",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "destination.as.organization_name",
+        "target_field": "destination.as.organization.name",
+        "ignore_missing": true
+      }
+    },
+    {
+      "community_id": {
+        "target_field": "network.community_id",
+        "ignore_failure": true
+      }
+    },
+    {
+      "grok": {
+        "field": "observer.ingress.interface.name",
+        "patterns": [
+          "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
+        ],
+        "ignore_missing": true,
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "network.vlan.id",
+        "copy_from": "observer.ingress.vlan.id",
+        "ignore_empty_value": true
+      }
+    },
+    {
+      "append": {
+        "field": "related.ip",
+        "value": "{{destination.ip}}",
+        "allow_duplicates": false,
+        "if": "ctx.destination?.ip != null"
+      }
+    },
+    {
+      "append": {
+        "field": "related.ip",
+        "value": "{{source.ip}}",
+        "allow_duplicates": false,
+        "if": "ctx.source?.ip != null"
+      }
+    },
+    {
+      "append": {
+        "field": "related.ip",
+        "value": "{{source.nat.ip}}",
+        "allow_duplicates": false,
+        "if": "ctx.source?.nat?.ip != null"
+      }
+    },
+    {
+      "append": {
+        "field": "related.hosts",
+        "value": "{{destination.domain}}",
+        "if": "ctx.destination?.domain != null"
+      }
+    },
+    {
+      "append": {
+        "field": "related.user",
+        "value": "{{user.name}}",
+        "if": "ctx.user?.name != null"
+      }
+    },
+    {
+      "set": {
+        "field": "network.direction",
+        "value": "{{network.direction}}bound",
+        "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
+      }
+    },
+    {
+      "remove": {
+        "field": [
+          "_tmp"
+        ],
+        "ignore_failure": true
+      }
+    },
+    {
+      "script": {
+        "lang": "painless",
+        "description": "This script processor iterates over the whole document to remove fields with null values.",
+        "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
+      }
+    },
+    {
+      "remove": {
+        "field": "event.original",
+        "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))",
+        "ignore_failure": true,
+        "ignore_missing": true
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log@custom",
+        "ignore_missing_pipeline": true
+      }
+    }
+  ],
+  "on_failure": [
+    {
+      "remove": {
+        "field": [
+          "_tmp"
+        ],
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "event.kind",
+        "value": "pipeline_error"
+      }
+    },
+    {
+      "append": {
+        "field": "error.message",
+        "value": "{{{ _ingest.on_failure_message }}}"
+      }
+    }
+  ],
+  "_meta": {
+    "managed_by": "fleet",
+    "managed": true,
+    "package": {
+      "name": "pfsense"
+    }
+  }
+}