diff --git a/packages/cisco_umbrella/changelog.yml b/packages/cisco_umbrella/changelog.yml index 38accf045ab..995b7d4c565 100644 --- a/packages/cisco_umbrella/changelog.yml +++ b/packages/cisco_umbrella/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.7.0" + changes: + - description: Add Audit Logs + type: enhancement + link: https://github.com/elastic/integrations/pull/3332 - version: "0.6.1" changes: - description: Fix use of destination.ip instead of source.nat.ip in DNS logs diff --git a/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-common-config.yml index 5622947e4b8..4da22641654 100644 --- a/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-common-config.yml +++ b/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -1,5 +1,3 @@ -dynamic_fields: - event.ingested: ".*" fields: tags: - preserve_original_event diff --git a/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-umbrella-auditlogs.log b/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-umbrella-auditlogs.log new file mode 100644 index 00000000000..2aea2423c55 --- /dev/null +++ b/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-umbrella-auditlogs.log @@ -0,0 +1 @@ +"1234567890","2021-07-22 10:46:45","user@domain.com","user", "logexportconfigurations", "update","81.2.69.144","version: 4","version: 5" \ No newline at end of file diff --git a/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-umbrella-auditlogs.log-config.yml b/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-umbrella-auditlogs.log-config.yml new file mode 100644 index 00000000000..190571f5b28 --- /dev/null +++ b/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-umbrella-auditlogs.log-config.yml @@ -0,0 +1,4 @@ +fields: + log: + file: + path: /test/path/auditlogs diff --git a/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-umbrella-auditlogs.log-expected.json b/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-umbrella-auditlogs.log-expected.json new file mode 100644 index 00000000000..00751fb3267 --- /dev/null +++ b/packages/cisco_umbrella/data_stream/log/_dev/test/pipeline/test-umbrella-auditlogs.log-expected.json @@ -0,0 +1,65 @@ +{ + "expected": [ + { + "@timestamp": "2021-07-22T10:46:45.000Z", + "cisco": { + "umbrella": { + "audit": { + "after": "version: 5", + "before": "version: 4", + "type": "logexportconfigurations" + } + } + }, + "ecs": { + "version": "8.2.0" + }, + "event": { + "action": "update", + "category": "configuration", + "id": "1234567890", + "original": "\"1234567890\",\"2021-07-22 10:46:45\",\"user@domain.com\",\"user\", \"logexportconfigurations\", \"update\",\"81.2.69.144\",\"version: 4\",\"version: 5\"", + "type": [ + "change" + ] + }, + "log": { + "file": { + "path": "/test/path/auditlogs" + } + }, + "observer": { + "product": "Umbrella", + "vendor": "Cisco" + }, + "related": { + "ip": [ + "81.2.69.144" + ] + }, + "source": { + "address": "81.2.69.144", + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "email": "user@domain.com", + "name": "user" + } + } + ] +} \ No newline at end of file diff --git a/packages/cisco_umbrella/data_stream/log/agent/stream/aws-s3.yml.hbs b/packages/cisco_umbrella/data_stream/log/agent/stream/aws-s3.yml.hbs index c9394a2dcae..13c48cb3662 100644 --- a/packages/cisco_umbrella/data_stream/log/agent/stream/aws-s3.yml.hbs +++ b/packages/cisco_umbrella/data_stream/log/agent/stream/aws-s3.yml.hbs @@ -13,6 +13,7 @@ file_selectors: - regex: {{bucket_list_prefix}}/proxylogs/.+ - regex: {{bucket_list_prefix}}/cloudfirewalllogs/.+ - regex: {{bucket_list_prefix}}/iplogs/.+ + - regex: {{bucket_list_prefix}}/auditlogs/.+ {{/if}} {{#if region}} default_region: {{region}} diff --git a/packages/cisco_umbrella/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_umbrella/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 80c3f525a22..cadb340add1 100644 --- a/packages/cisco_umbrella/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cisco_umbrella/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -131,6 +131,23 @@ processors: value: firewall if: ctx?.log?.file?.path.contains('cloudfirewalllogs') + ####################### + # Audit Logs # + ####################### + - csv: + field: event.original + target_fields: + - event.id + - cisco.umbrella._tmp.time + - user.email + - user.name + - cisco.umbrella.audit.type + - event.action + - source.address + - cisco.umbrella.audit.before + - cisco.umbrella.audit.after + if: ctx?.log?.file?.path.contains('auditlogs') + - uri_parts: field: url.full ignore_failure: true @@ -163,6 +180,7 @@ processors: target_field: "@timestamp" formats: - "yyyy-MM-dd HH:mm:ss" + - ISO8601 if: ctx?.cisco?.umbrella?._tmp?.time != null ################## # DNS ECS Fields # @@ -223,6 +241,7 @@ processors: - set: field: event.category value: network + if: "!ctx?.log?.file?.path.contains('auditlogs')" - append: field: event.type value: allowed @@ -235,6 +254,22 @@ processors: field: event.type value: connection if: ctx?.cisco?.umbrella?.action != null + - set: + field: event.category + value: configuration + if: "ctx?.log?.file?.path.contains('auditlogs')" + - append: + field: event.type + value: creation + if: "ctx?.log?.file?.path.contains('auditlogs') && ctx.event?.action.toLowerCase() == 'create'" + - append: + field: event.type + value: change + if: "ctx?.log?.file?.path.contains('auditlogs') && ctx.event?.action.toLowerCase() == 'update'" + - append: + field: event.type + value: deletion + if: "ctx?.log?.file?.path.contains('auditlogs') && ctx.event?.action.toLowerCase() == 'delete'" # Converting address fields to either ip or domain - convert: field: source.address diff --git a/packages/cisco_umbrella/data_stream/log/fields/ecs.yml b/packages/cisco_umbrella/data_stream/log/fields/ecs.yml index cac2ed3f9fb..dd96bd34c91 100644 --- a/packages/cisco_umbrella/data_stream/log/fields/ecs.yml +++ b/packages/cisco_umbrella/data_stream/log/fields/ecs.yml @@ -130,6 +130,10 @@ level: core name: source.geo.location type: geo_point +- external: ecs + name: source.geo.region_iso_code +- external: ecs + name: source.geo.region_name - external: ecs name: source.ip - external: ecs @@ -170,6 +174,8 @@ name: user.id - external: ecs name: user.name +- external: ecs + name: user.email - external: ecs name: user_agent.original - external: ecs diff --git a/packages/cisco_umbrella/data_stream/log/fields/fields.yml b/packages/cisco_umbrella/data_stream/log/fields/fields.yml index ee699497280..930527b81dd 100644 --- a/packages/cisco_umbrella/data_stream/log/fields/fields.yml +++ b/packages/cisco_umbrella/data_stream/log/fields/fields.yml @@ -79,21 +79,30 @@ description: > The unique identity of the network tunnel. - - name: cisco.umbrella.identities + - name: identities + type: keyword + - name: identity_types + type: keyword + - name: request_method + type: keyword + - name: dlp_status type: keyword - - name: cisco.umbrella.identity_types + - name: certificate_errors type: keyword - - name: cisco.umbrella.request_method + - name: file_name type: keyword - - name: cisco.umbrella.dlp_status + - name: ruleset_id type: keyword - - name: cisco.umbrella.certificate_errors + - name: rule_id type: keyword - - name: cisco.umbrella.file_name + - name: destination_lists_id type: keyword - - name: cisco.umbrella.ruleset_id + - name: audit.type type: keyword - - name: cisco.umbrella.rule_id + description: Where the change was made, such as settings or a policy. + - name: audit.before type: keyword - - name: cisco.umbrella.destination_lists_id + description: The policy or setting before the change was made. + - name: audit.after type: keyword + description: The policy or setting after the change was made. diff --git a/packages/cisco_umbrella/docs/README.md b/packages/cisco_umbrella/docs/README.md index 6af338e67e9..10abae05905 100644 --- a/packages/cisco_umbrella/docs/README.md +++ b/packages/cisco_umbrella/docs/README.md @@ -123,26 +123,27 @@ An example event for `log` looks as following: | cisco.umbrella.amp_disposition | The status of the files proxied and scanned by Cisco Advanced Malware Protection (AMP) as part of the Umbrella File Inspection feature; can be Clean, Malicious or Unknown. | keyword | | cisco.umbrella.amp_malware_name | If Malicious, the name of the malware according to AMP. | keyword | | cisco.umbrella.amp_score | The score of the malware from AMP. This field is not currently used and will be blank. | keyword | +| cisco.umbrella.audit.after | The policy or setting after the change was made. | keyword | +| cisco.umbrella.audit.before | The policy or setting before the change was made. | keyword | +| cisco.umbrella.audit.type | Where the change was made, such as settings or a policy. | keyword | | cisco.umbrella.av_detections | The detection name according to the antivirus engine used in file inspection. | keyword | | cisco.umbrella.blocked_categories | The categories that resulted in the destination being blocked. Available in version 4 and above. | keyword | | cisco.umbrella.categories | The security or content categories that the destination matches. | keyword | -| cisco.umbrella.cisco.umbrella.certificate_errors | | keyword | -| cisco.umbrella.cisco.umbrella.destination_lists_id | | keyword | -| cisco.umbrella.cisco.umbrella.dlp_status | | keyword | -| cisco.umbrella.cisco.umbrella.file_name | | keyword | -| cisco.umbrella.cisco.umbrella.identities | | keyword | -| cisco.umbrella.cisco.umbrella.identity_types | | keyword | -| cisco.umbrella.cisco.umbrella.request_method | | keyword | -| cisco.umbrella.cisco.umbrella.rule_id | | keyword | -| cisco.umbrella.cisco.umbrella.ruleset_id | | keyword | +| cisco.umbrella.certificate_errors | | keyword | | cisco.umbrella.computer_name | The computer name related to the event. | keyword | | cisco.umbrella.content_type | The type of web content, typically text/html. | keyword | | cisco.umbrella.datacenter | The name of the Umbrella Data Center that processed the user-generated traffic. | keyword | -| cisco.umbrella.identities | An array of the different identities related to the event. | keyword | -| cisco.umbrella.identity_types | The type of identity that made the request. For example, Roaming Computer or Network. | keyword | +| cisco.umbrella.destination_lists_id | | keyword | +| cisco.umbrella.dlp_status | | keyword | +| cisco.umbrella.file_name | | keyword | +| cisco.umbrella.identities | | keyword | +| cisco.umbrella.identity_types | | keyword | | cisco.umbrella.origin_id | The unique identity of the network tunnel. | keyword | | cisco.umbrella.policy_identity_type | The first identity type matched with this request. Available in version 3 and above. | keyword | | cisco.umbrella.puas | A list of all potentially unwanted application (PUA) results for the proxied file as returned by the antivirus scanner. | keyword | +| cisco.umbrella.request_method | | keyword | +| cisco.umbrella.rule_id | | keyword | +| cisco.umbrella.ruleset_id | | keyword | | cisco.umbrella.sha_sha256 | Hex digest of the response content. | keyword | | client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | | client.registered_domain | The highest registered client domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | @@ -202,7 +203,7 @@ An example event for `log` looks as following: | host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | +| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | @@ -243,6 +244,8 @@ An example event for `log` looks as following: | source.geo.country_iso_code | Country ISO code. | keyword | | source.geo.country_name | Country name. | keyword | | source.geo.location | Longitude and latitude. | geo_point | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | | source.ip | IP address of the source (IPv4 or IPv6). | ip | | source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | source.nat.ip | Translated ip of source based NAT sessions (e.g. internal client to internet) Typically connections traversing load balancers, firewalls, or routers. | ip | @@ -262,6 +265,7 @@ An example event for `log` looks as following: | url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | | url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | | user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | +| user.email | User email address. | keyword | | user.full_name | User's full name, if available. | keyword | | user.full_name.text | Multi-field of `user.full_name`. | match_only_text | | user.id | Unique identifier of the user. | keyword | diff --git a/packages/cisco_umbrella/manifest.yml b/packages/cisco_umbrella/manifest.yml index 747188e1a6f..e8cf60233be 100644 --- a/packages/cisco_umbrella/manifest.yml +++ b/packages/cisco_umbrella/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: cisco_umbrella title: Cisco Umbrella -version: 0.6.1 +version: 0.7.0 license: basic description: Collect logs from Cisco Umbrella with Elastic Agent. type: integration