[Cisco Umbrella] Add Audit Log support

packages/cisco_umbrella/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.7.0"
+  changes:
+    - description: Add Audit Logs
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/3332
 - version: "0.6.1"
   changes:
     - description: Fix use of destination.ip instead of source.nat.ip in DNS logs

packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-common-config.yml

@@ -1,5 +1,3 @@
-dynamic_fields:
-  event.ingested: ".*"
 fields:
   tags:
     - preserve_original_event

packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-umbrella-auditlogs.log

@@ -0,0 +1 @@
+"1234567890","2021-07-22 10:46:45","user@domain.com","user", "logexportconfigurations", "update","81.2.69.144","version: 4","version: 5"

packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-umbrella-auditlogs.log-config.yml

@@ -0,0 +1,4 @@
+fields:
+  log:
+    file:
+      path: /test/path/auditlogs

packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-umbrella-auditlogs.log-expected.json

@@ -0,0 +1,65 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2021-07-22T10:46:45.000Z",
+            "cisco": {
+                "umbrella": {
+                    "audit": {
+                        "after": "version: 5",
+                        "before": "version: 4",
+                        "type": "logexportconfigurations"
+                    }
+                }
+            },
+            "ecs": {
+                "version": "8.2.0"
+            },
+            "event": {
+                "action": "update",
+                "category": "configuration",
+                "id": "1234567890",
+                "original": "\"1234567890\",\"2021-07-22 10:46:45\",\"user@domain.com\",\"user\", \"logexportconfigurations\", \"update\",\"81.2.69.144\",\"version: 4\",\"version: 5\"",
+                "type": [
+                    "change"
+                ]
+            },
+            "log": {
+                "file": {
+                    "path": "/test/path/auditlogs"
+                }
+            },
+            "observer": {
+                "product": "Umbrella",
+                "vendor": "Cisco"
+            },
+            "related": {
+                "ip": [
+                    "81.2.69.144"
+                ]
+            },
+            "source": {
+                "address": "81.2.69.144",
+                "geo": {
+                    "city_name": "London",
+                    "continent_name": "Europe",
+                    "country_iso_code": "GB",
+                    "country_name": "United Kingdom",
+                    "location": {
+                        "lat": 51.5142,
+                        "lon": -0.0931
+                    },
+                    "region_iso_code": "GB-ENG",
+                    "region_name": "England"
+                },
+                "ip": "81.2.69.144"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "email": "user@domain.com",
+                "name": "user"
+            }
+        }
+    ]
+}

packages/cisco_umbrella/data_stream/log/agent/stream/aws-s3.yml.hbs

@@ -13,6 +13,7 @@ file_selectors:
   - regex: {{bucket_list_prefix}}/proxylogs/.+
   - regex: {{bucket_list_prefix}}/cloudfirewalllogs/.+
   - regex: {{bucket_list_prefix}}/iplogs/.+
+  - regex: {{bucket_list_prefix}}/auditlogs/.+
 {{/if}}
 {{#if region}}
 default_region: {{region}}

packages/cisco_umbrella/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -131,6 +131,23 @@ processors:
       value: firewall
       if: ctx?.log?.file?.path.contains('cloudfirewalllogs')
 
+  #######################
+  # Audit Logs #
+  #######################
+  - csv:
+      field: event.original
+      target_fields:
+        - event.id
+        - cisco.umbrella._tmp.time
+        - user.email
+        - user.name
+        - cisco.umbrella.audit.type
+        - event.action
+        - source.address
+        - cisco.umbrella.audit.before
+        - cisco.umbrella.audit.after
+      if: ctx?.log?.file?.path.contains('auditlogs')
+
   - uri_parts:
       field: url.full
       ignore_failure: true
@@ -223,6 +240,7 @@ processors:
   - set:
       field: event.category
       value: network
+      if: "!ctx?.log?.file?.path.contains('auditlogs')"
   - append:
       field: event.type
       value: allowed
@@ -235,6 +253,22 @@ processors:
       field: event.type
       value: connection
       if: ctx?.cisco?.umbrella?.action != null
+  - set:
+      field: event.category
+      value: configuration
+      if: "ctx?.log?.file?.path.contains('auditlogs')"
+  - append:
+      field: event.type
+      value: creation
+      if: "ctx?.log?.file?.path.contains('auditlogs') && ctx.event?.action.toLowerCase() == 'create'"
+  - append:
+      field: event.type
+      value: change
+      if: "ctx?.log?.file?.path.contains('auditlogs') && ctx.event?.action.toLowerCase() == 'update'"
+  - append:
+      field: event.type
+      value: deletion
+      if: "ctx?.log?.file?.path.contains('auditlogs') && ctx.event?.action.toLowerCase() == 'delete'"
   # Converting address fields to either ip or domain
   - convert:
       field: source.address

packages/cisco_umbrella/data_stream/log/fields/ecs.yml

@@ -130,6 +130,10 @@
   level: core
   name: source.geo.location
   type: geo_point
+- external: ecs
+  name: source.geo.region_iso_code
+- external: ecs
+  name: source.geo.region_name
 - external: ecs
   name: source.ip
 - external: ecs
@@ -170,6 +174,8 @@
   name: user.id
 - external: ecs
   name: user.name
+- external: ecs
+  name: user.email
 - external: ecs
   name: user_agent.original
 - external: ecs

packages/cisco_umbrella/data_stream/log/fields/fields.yml

@@ -79,21 +79,30 @@
       description: >
         The unique identity of the network tunnel.
 
-    - name: cisco.umbrella.identities
+    - name: identities
+      type: keyword
+    - name: identity_types
+      type: keyword
+    - name: request_method
+      type: keyword
+    - name: dlp_status
       type: keyword
-    - name: cisco.umbrella.identity_types
+    - name: certificate_errors
       type: keyword
-    - name: cisco.umbrella.request_method
+    - name: file_name
       type: keyword
-    - name: cisco.umbrella.dlp_status
+    - name: ruleset_id
       type: keyword
-    - name: cisco.umbrella.certificate_errors
+    - name: rule_id
       type: keyword
-    - name: cisco.umbrella.file_name
+    - name: destination_lists_id
       type: keyword
-    - name: cisco.umbrella.ruleset_id
+    - name: audit.type
       type: keyword
-    - name: cisco.umbrella.rule_id
+      description: Where the change was made, such as settings or a policy.
+    - name: audit.before
       type: keyword
-    - name: cisco.umbrella.destination_lists_id
+      description: The policy or setting before the change was made.
+    - name: audit.after
       type: keyword
+      description: The policy or setting after the change was made.

packages/cisco_umbrella/docs/README.md

@@ -123,26 +123,27 @@ An example event for `log` looks as following:
 | cisco.umbrella.amp_disposition | The status of the files proxied and scanned by Cisco Advanced Malware Protection (AMP) as part of the Umbrella File Inspection feature; can be Clean, Malicious or Unknown. | keyword |
 | cisco.umbrella.amp_malware_name | If Malicious, the name of the malware according to AMP. | keyword |
 | cisco.umbrella.amp_score | The score of the malware from AMP. This field is not currently used and will be blank. | keyword |
+| cisco.umbrella.audit.after | The policy or setting after the change was made. | keyword |
+| cisco.umbrella.audit.before | The policy or setting before the change was made. | keyword |
+| cisco.umbrella.audit.type | Where the change was made, such as settings or a policy. | keyword |
 | cisco.umbrella.av_detections | The detection name according to the antivirus engine used in file inspection. | keyword |
 | cisco.umbrella.blocked_categories | The categories that resulted in the destination being blocked. Available in version 4 and above. | keyword |
 | cisco.umbrella.categories | The security or content categories that the destination matches. | keyword |
-| cisco.umbrella.cisco.umbrella.certificate_errors |  | keyword |
-| cisco.umbrella.cisco.umbrella.destination_lists_id |  | keyword |
-| cisco.umbrella.cisco.umbrella.dlp_status |  | keyword |
-| cisco.umbrella.cisco.umbrella.file_name |  | keyword |
-| cisco.umbrella.cisco.umbrella.identities |  | keyword |
-| cisco.umbrella.cisco.umbrella.identity_types |  | keyword |
-| cisco.umbrella.cisco.umbrella.request_method |  | keyword |
-| cisco.umbrella.cisco.umbrella.rule_id |  | keyword |
-| cisco.umbrella.cisco.umbrella.ruleset_id |  | keyword |
+| cisco.umbrella.certificate_errors |  | keyword |
 | cisco.umbrella.computer_name | The computer name related to the event. | keyword |
 | cisco.umbrella.content_type | The type of web content, typically text/html. | keyword |
 | cisco.umbrella.datacenter | The name of the Umbrella Data Center that processed the user-generated traffic. | keyword |
-| cisco.umbrella.identities | An array of the different identities related to the event. | keyword |
-| cisco.umbrella.identity_types | The type of identity that made the request. For example, Roaming Computer or Network. | keyword |
+| cisco.umbrella.destination_lists_id |  | keyword |
+| cisco.umbrella.dlp_status |  | keyword |
+| cisco.umbrella.file_name |  | keyword |
+| cisco.umbrella.identities |  | keyword |
+| cisco.umbrella.identity_types |  | keyword |
 | cisco.umbrella.origin_id | The unique identity of the network tunnel. | keyword |
 | cisco.umbrella.policy_identity_type | The first identity type matched with this request. Available in version 3 and above. | keyword |
 | cisco.umbrella.puas | A list of all potentially unwanted application (PUA) results for the proxied file as returned by the antivirus scanner. | keyword |
+| cisco.umbrella.request_method |  | keyword |
+| cisco.umbrella.rule_id |  | keyword |
+| cisco.umbrella.ruleset_id |  | keyword |
 | cisco.umbrella.sha_sha256 | Hex digest of the response content. | keyword |
 | client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword |
 | client.registered_domain | The highest registered client domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword |
@@ -202,7 +203,7 @@ An example event for `log` looks as following:
 | host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword |
 | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |
 | host.ip | Host ip addresses. | ip |
-| host.mac | Host mac addresses. | keyword |
+| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |
 | host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword |
 | host.os.build | OS build information. | keyword |
 | host.os.codename | OS codename, if any. | keyword |
@@ -243,6 +244,8 @@ An example event for `log` looks as following:
 | source.geo.country_iso_code | Country ISO code. | keyword |
 | source.geo.country_name | Country name. | keyword |
 | source.geo.location | Longitude and latitude. | geo_point |
+| source.geo.region_iso_code | Region ISO code. | keyword |
+| source.geo.region_name | Region name. | keyword |
 | source.ip | IP address of the source (IPv4 or IPv6). | ip |
 | source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |
 | source.nat.ip | Translated ip of source based NAT sessions (e.g. internal client to internet) Typically connections traversing load balancers, firewalls, or routers. | ip |
@@ -262,6 +265,7 @@ An example event for `log` looks as following:
 | url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword |
 | url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword |
 | user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword |
+| user.email | User email address. | keyword |
 | user.full_name | User's full name, if available. | keyword |
 | user.full_name.text | Multi-field of `user.full_name`. | match_only_text |
 | user.id | Unique identifier of the user. | keyword |

packages/cisco_umbrella/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: cisco_umbrella
 title: Cisco Umbrella
-version: 0.6.1
+version: 0.7.0
 license: basic
 description: Collect logs from Cisco Umbrella with Elastic Agent.
 type: integration