diff --git a/packages/aws/_dev/build/docs/route53.md b/packages/aws/_dev/build/docs/route53.md new file mode 100644 index 00000000000..a625060a7c6 --- /dev/null +++ b/packages/aws/_dev/build/docs/route53.md @@ -0,0 +1,9 @@ +# Route 53 + +## Logs + +### Public Hosted Zone Logs + +{{event "route53_public_logs"}} + +{{fields "route53_public_logs"}} diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 437eb9d9dd6..2f33f74b516 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.8.0" + changes: + - description: Add Route 53 Public Zone Logs Datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/2316 - version: "1.7.1" changes: - description: Regenerate test files using the new GeoIP database diff --git a/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-common-config.yml b/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..5622947e4b8 --- /dev/null +++ b/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-route53.log b/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-route53.log new file mode 100644 index 00000000000..d9dbd185d2d --- /dev/null +++ b/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-route53.log @@ -0,0 +1,5 @@ +1.0 2017-12-13T08:16:02.130Z Z123412341234 example.com A NOERROR UDP FRA6 89.160.20.112 - +1.0 2017-12-13T08:15:50.235Z Z123412341234 example.com AAAA NOERROR TCP IAD12 89.160.20.112 192.168.222.0/24 +1.0 2017-12-13T08:16:03.983Z Z123412341234 example.com ANY NOERROR UDP FRA6 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 2001:db8:abcd::/48 +1.0 2017-12-13T08:15:50.342Z Z123412341234 bad.example.com A NXDOMAIN UDP IAD12 89.160.20.112 192.168.111.0/24 +1.0 2017-12-13T08:16:05.744Z Z123412341234 txt.example.com TXT NOERROR UDP JFK5 89.160.20.112 - \ No newline at end of file diff --git a/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-route53.log-expected.json b/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-route53.log-expected.json new file mode 100644 index 00000000000..7b32e06d087 --- /dev/null +++ b/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-route53.log-expected.json @@ -0,0 +1,375 @@ +{ + "expected": [ + { + "cloud": { + "provider": "aws" + }, + "@timestamp": "2017-12-13T08:16:02.130Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "example.com" + ], + "ip": [ + "89.160.20.112" + ] + }, + "dns": { + "question": { + "name": "example.com", + "registered_domain": "example.com", + "type": "A", + "top_level_domain": "com" + }, + "response_code": "NOERROR" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-E", + "city_name": "Linköping", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Östergötland County", + "location": { + "lon": 15.6167, + "lat": 58.4167 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.112", + "ip": "89.160.20.112" + }, + "event": { + "ingested": "2021-12-21T22:56:17.914514638Z", + "original": "1.0 2017-12-13T08:16:02.130Z Z123412341234 example.com A NOERROR UDP FRA6 89.160.20.112 -", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "route53": { + "hosted_zone_id": "Z123412341234", + "edge_location": "FRA6" + } + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + } + }, + { + "cloud": { + "provider": "aws" + }, + "@timestamp": "2017-12-13T08:15:50.235Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "example.com" + ], + "ip": [ + "89.160.20.112" + ] + }, + "dns": { + "question": { + "name": "example.com", + "registered_domain": "example.com", + "type": "AAAA", + "top_level_domain": "com" + }, + "response_code": "NOERROR" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-E", + "city_name": "Linköping", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Östergötland County", + "location": { + "lon": 15.6167, + "lat": 58.4167 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.112", + "ip": "89.160.20.112" + }, + "event": { + "ingested": "2021-12-21T22:56:17.914525865Z", + "original": "1.0 2017-12-13T08:15:50.235Z Z123412341234 example.com AAAA NOERROR TCP IAD12 89.160.20.112 192.168.222.0/24", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "route53": { + "hosted_zone_id": "Z123412341234", + "edns_client_subnet": "192.168.222.0/24", + "edge_location": "IAD12" + } + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "tcp", + "type": "ipv4", + "iana_number": "6" + } + }, + { + "cloud": { + "provider": "aws" + }, + "@timestamp": "2017-12-13T08:16:03.983Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "example.com" + ], + "ip": [ + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + ] + }, + "dns": { + "question": { + "name": "example.com", + "registered_domain": "example.com", + "type": "ANY", + "top_level_domain": "com" + }, + "response_code": "NOERROR" + }, + "source": { + "geo": { + "continent_name": "Europe", + "country_name": "Norway", + "location": { + "lon": 10.0, + "lat": 62.0 + }, + "country_iso_code": "NO" + }, + "address": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + }, + "event": { + "ingested": "2021-12-21T22:56:17.914528671Z", + "original": "1.0 2017-12-13T08:16:03.983Z Z123412341234 example.com ANY NOERROR UDP FRA6 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 2001:db8:abcd::/48", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "route53": { + "hosted_zone_id": "Z123412341234", + "edns_client_subnet": "2001:db8:abcd::/48", + "edge_location": "FRA6" + } + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv6", + "iana_number": "17" + } + }, + { + "cloud": { + "provider": "aws" + }, + "@timestamp": "2017-12-13T08:15:50.342Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "bad.example.com" + ], + "ip": [ + "89.160.20.112" + ] + }, + "dns": { + "question": { + "name": "bad.example.com", + "subdomain": "bad", + "registered_domain": "example.com", + "type": "A", + "top_level_domain": "com" + }, + "response_code": "NXDOMAIN" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-E", + "city_name": "Linköping", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Östergötland County", + "location": { + "lon": 15.6167, + "lat": 58.4167 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.112", + "ip": "89.160.20.112" + }, + "event": { + "ingested": "2021-12-21T22:56:17.914531179Z", + "original": "1.0 2017-12-13T08:15:50.342Z Z123412341234 bad.example.com A NXDOMAIN UDP IAD12 89.160.20.112 192.168.111.0/24", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "failure" + }, + "aws": { + "route53": { + "hosted_zone_id": "Z123412341234", + "edns_client_subnet": "192.168.111.0/24", + "edge_location": "IAD12" + } + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + } + }, + { + "cloud": { + "provider": "aws" + }, + "@timestamp": "2017-12-13T08:16:05.744Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "txt.example.com" + ], + "ip": [ + "89.160.20.112" + ] + }, + "dns": { + "question": { + "name": "txt.example.com", + "subdomain": "txt", + "registered_domain": "example.com", + "type": "TXT", + "top_level_domain": "com" + }, + "response_code": "NOERROR" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-E", + "city_name": "Linköping", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Östergötland County", + "location": { + "lon": 15.6167, + "lat": 58.4167 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.112", + "ip": "89.160.20.112" + }, + "event": { + "ingested": "2021-12-21T22:56:17.914533595Z", + "original": "1.0 2017-12-13T08:16:05.744Z Z123412341234 txt.example.com TXT NOERROR UDP JFK5 89.160.20.112 -", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "route53": { + "hosted_zone_id": "Z123412341234", + "edge_location": "JFK5" + } + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..c1576eedae0 --- /dev/null +++ b/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,93 @@ +{{#unless log_group_name}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} +{{/if}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} +{{/if}} +{{/unless}} + +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} +{{/if}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} +{{/if}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..76450c33ada --- /dev/null +++ b/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,129 @@ +--- +description: Pipeline for AWS Route53 Logs + +processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' + - set: + field: cloud.provider + value: aws + # - set: + # if: ctx.aws?.vpcflow?.account_id != null + # field: cloud.account.id + # value: '{{aws.vpcflow.account_id}}' + - set: + field: event.kind + value: event + - append: + field: event.category + value: network + - append: + field: event.type + value: protocol + - rename: + field: message + target_field: event.original + ignore_missing: true + - grok: + field: event.original + patterns: + - '%{BASE10NUM} %{TIMESTAMP_ISO8601:_tmp.timestamp} %{DATA:aws.route53.hosted_zone_id} %{HOSTNAME:_tmp.question} %{WORD:dns.question.type} %{WORD:dns.response_code} %{WORD:network.transport} %{EDGE_LOCATION:aws.route53.edge_location} %{IP:source.address} (%{SUBNET:aws.route53.edns_client_subnet}|-)' + pattern_definitions: + EDGE_LOCATION: '[A-Z]{3}\d+' + SUBNET: '%{IP}/[0-9]+' + - date: + field: _tmp.timestamp + target_field: '@timestamp' + ignore_failure: true + formats: + - ISO8601 + - set: + field: event.outcome + value: success + if: ctx.dns?.response_code == "NOERROR" + - set: + field: event.outcome + value: failure + if: ctx.dns?.response_code != "NOERROR" + - registered_domain: + field: _tmp.question + target_field: dns.question + ignore_missing: true + - rename: + field: dns.question.domain + target_field: dns.question.name + ignore_missing: true + - convert: + field: source.address + target_field: source.ip + type: ip + ignore_missing: true + - lowercase: + field: network.transport + ignore_missing: true + - set: + field: network.protocol + value: dns + - set: + field: network.type + value: ipv4 + if: 'ctx.source?.ip != null && ctx.source?.ip.contains(".")' + - set: + field: network.type + value: ipv6 + if: 'ctx.source?.ip != null && ctx.source?.ip.contains(":")' + - set: + field: network.iana_number + value: '6' + if: ctx.network?.transport == "tcp" + - set: + field: network.iana_number + value: '17' + if: ctx.network?.transport == "udp" + # IP Geolocation Lookup + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + # IP Autonomous System (AS) Lookup + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - append: + field: related.ip + value: "{{source.ip}}" + if: ctx.source?.ip != null + - append: + field: related.hosts + value: "{{dns.question.name}}" + if: ctx.dns?.question?.name != null + - remove: + field: + - _tmp + ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true +on_failure: + - set: + field: 'error.message' + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/aws/data_stream/route53_public_logs/fields/agent.yml b/packages/aws/data_stream/route53_public_logs/fields/agent.yml new file mode 100644 index 00000000000..da4e652c53b --- /dev/null +++ b/packages/aws/data_stream/route53_public_logs/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/packages/aws/data_stream/route53_public_logs/fields/base-fields.yml b/packages/aws/data_stream/route53_public_logs/fields/base-fields.yml new file mode 100644 index 00000000000..825d4871f22 --- /dev/null +++ b/packages/aws/data_stream/route53_public_logs/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.route53_public_logs diff --git a/packages/aws/data_stream/route53_public_logs/fields/beats.yml b/packages/aws/data_stream/route53_public_logs/fields/beats.yml new file mode 100644 index 00000000000..3dde4d0b577 --- /dev/null +++ b/packages/aws/data_stream/route53_public_logs/fields/beats.yml @@ -0,0 +1,15 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.file.path + type: keyword + description: Path to the log file. +- name: awscloudwatch.log_stream + type: keyword + description: AWS CloudWatch Log Stream name +- name: awscloudwatch.log_group + type: keyword + description: AWS CloudWatch Log Group name +- name: awscloudwatch.ingestion_time + type: date + description: AWS CloudWatch ingest time diff --git a/packages/aws/data_stream/route53_public_logs/fields/ecs.yml b/packages/aws/data_stream/route53_public_logs/fields/ecs.yml new file mode 100644 index 00000000000..082dc202412 --- /dev/null +++ b/packages/aws/data_stream/route53_public_logs/fields/ecs.yml @@ -0,0 +1,54 @@ +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: message +- external: ecs + name: tags +- external: ecs + name: dns.question.name +- external: ecs + name: dns.question.type +- external: ecs + name: dns.question.registered_domain +- external: ecs + name: dns.question.subdomain +- external: ecs + name: dns.question.top_level_domain +- external: ecs + name: dns.response_code +- external: ecs + name: network.iana_number +- external: ecs + name: network.transport +- external: ecs + name: network.protocol +- external: ecs + name: network.type +- external: ecs + name: related.ip +- external: ecs + name: related.hosts +- external: ecs + name: source.address +- external: ecs + name: source.ip +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name +- name: source.geo.city_name + external: ecs +- name: source.geo.continent_name + external: ecs +- name: source.geo.country_iso_code + external: ecs +- name: source.geo.country_name + external: ecs +- name: source.geo.location + external: ecs +- name: source.geo.region_iso_code + external: ecs +- name: source.geo.region_name + external: ecs diff --git a/packages/aws/data_stream/route53_public_logs/fields/fields.yml b/packages/aws/data_stream/route53_public_logs/fields/fields.yml new file mode 100644 index 00000000000..e4a8a7ad764 --- /dev/null +++ b/packages/aws/data_stream/route53_public_logs/fields/fields.yml @@ -0,0 +1,15 @@ +- name: aws.route53 + type: group + fields: + - name: hosted_zone_id + type: keyword + description: | + The ID of the hosted zone that is associated with all the DNS queries in this log. + - name: edge_location + type: keyword + description: | + The Route 53 edge location that responded to the query. Each edge location is identified by a three-letter code and an arbitrary number, for example, DFW3. The three-letter code typically corresponds with the International Air Transport Association airport code for an airport near the edge location. (These abbreviations might change in the future.) + - name: edns_client_subnet + type: keyword + description: | + A partial IP address for the client that the request originated from, if available from the DNS resolver. diff --git a/packages/aws/data_stream/route53_public_logs/manifest.yml b/packages/aws/data_stream/route53_public_logs/manifest.yml new file mode 100644 index 00000000000..acd3da3d270 --- /dev/null +++ b/packages/aws/data_stream/route53_public_logs/manifest.yml @@ -0,0 +1,106 @@ +title: AWS Route 53 Public Zone Logs +type: logs +streams: + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS Route 53 Logs + description: Collect AWS Route 53 Logs using Cloudwatch + vars: + - name: log_group_arn + type: text + title: Log Group ARN + description: "ARN of the log group to collect logs from. See [Documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-cloudwatch.html#_log_group_arn)." + multi: false + show_user: true + required: false + - name: log_group_name + type: text + title: Log Group Name + description: "Name of the log group to collect logs from. See [Documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-cloudwatch.html#_log_group_name)." + multi: false + show_user: false + required: false + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + description: "The prefix for a group of log group names. See [Documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-cloudwatch.html#_log_group_name_prefix)." + multi: false + show_user: false + required: false + - name: log_streams + type: text + title: Log Streams + description: Required when using `Log Group Name Prefix` or `Log Group Name`. + multi: false + show_user: false + required: false + - name: log_stream_prefix + type: text + title: Log Streams Prefix + description: A list of strings of log streams names that Filebeat collect log events from. + multi: false + show_user: false + required: false + - name: region_name + type: text + title: Region Name + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + multi: false + show_user: false + required: false + - name: start_position + type: text + title: Start Position + description: Allows user to specify if this input should read log files from the beginning or from the end. + multi: false + show_user: false + required: true + default: beginning + - name: scan_frequency + type: text + title: Scan Frequency + description: This config parameter sets how often Filebeat checks for new log events from the specified log group in seconds. Default scan_frequency is 1 minute. + multi: false + show_user: false + required: true + default: 1m + - name: api_timeout + type: text + title: API Timeout + multi: false + required: false + show_user: false + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. + - name: api_sleep + type: text + title: API Sleep + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. FilterLogEvents API has a quota of 5 transactions per second (TPS)/account/Region. By default, api_sleep is 200 ms. + multi: false + show_user: false + required: false + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-route53-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/route53_public_logs/sample_event.json b/packages/aws/data_stream/route53_public_logs/sample_event.json new file mode 100644 index 00000000000..eecc9516125 --- /dev/null +++ b/packages/aws/data_stream/route53_public_logs/sample_event.json @@ -0,0 +1,96 @@ +{ + "awscloudwatch": { + "log_group": "test", + "ingestion_time": "2021-12-06T02:18:20.000Z", + "log_stream": "test" + }, + "agent": { + "name": "docker-fleet-agent", + "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", + "type": "filebeat", + "ephemeral_id": "1cf87179-f6b3-44b0-a46f-3aa6bc0f995f", + "version": "8.0.0" + }, + "elastic_agent": { + "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", + "version": "8.0.0", + "snapshot": true + }, + "dns": { + "response_code": "NOERROR", + "question": { + "registered_domain": "example.com", + "top_level_domain": "com", + "name": "txt.example.com", + "subdomain": "txt", + "type": "TXT" + } + }, + "source": { + "as": { + "number": 721, + "organization": { + "name": "DoD Network Information Center" + } + }, + "address": "55.36.5.7", + "ip": "55.36.5.7" + }, + "tags": [ + "preserve_original_event", + "forwarded", + "aws-route53-logs" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "provider": "aws", + "region": "us-east-1" + }, + "input": { + "type": "aws-cloudwatch" + }, + "@timestamp": "2017-12-13T08:16:05.744Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "txt.example.com" + ], + "ip": [ + "55.36.5.7" + ] + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.route53_public_logs" + }, + "log.file.path": "test/test", + "event": { + "agent_id_status": "verified", + "ingested": "2021-12-06T02:37:25Z", + "original": "1.0 2017-12-13T08:16:05.744Z Z123412341234 txt.example.com TXT NOERROR UDP JFK5 55.36.5.7 -", + "kind": "event", + "id": "36545504503447201576705984279898091551471012413796646912", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "dataset": "aws.route53_public_logs", + "outcome": "success" + }, + "aws": { + "route53": { + "hosted_zone_id": "Z123412341234", + "edge_location": "JFK5" + } + } +} \ No newline at end of file diff --git a/packages/aws/docs/route53.md b/packages/aws/docs/route53.md new file mode 100644 index 00000000000..61e415f6b78 --- /dev/null +++ b/packages/aws/docs/route53.md @@ -0,0 +1,182 @@ +# Route 53 + +## Logs + +### Public Hosted Zone Logs + +An example event for `route53_public` looks as following: + +```json +{ + "awscloudwatch": { + "log_group": "test", + "ingestion_time": "2021-12-06T02:18:20.000Z", + "log_stream": "test" + }, + "agent": { + "name": "docker-fleet-agent", + "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", + "type": "filebeat", + "ephemeral_id": "1cf87179-f6b3-44b0-a46f-3aa6bc0f995f", + "version": "8.0.0" + }, + "elastic_agent": { + "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", + "version": "8.0.0", + "snapshot": true + }, + "dns": { + "response_code": "NOERROR", + "question": { + "registered_domain": "example.com", + "top_level_domain": "com", + "name": "txt.example.com", + "subdomain": "txt", + "type": "TXT" + } + }, + "source": { + "as": { + "number": 721, + "organization": { + "name": "DoD Network Information Center" + } + }, + "address": "55.36.5.7", + "ip": "55.36.5.7" + }, + "tags": [ + "preserve_original_event", + "forwarded", + "aws-route53-logs" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "provider": "aws", + "region": "us-east-1" + }, + "input": { + "type": "aws-cloudwatch" + }, + "@timestamp": "2017-12-13T08:16:05.744Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "txt.example.com" + ], + "ip": [ + "55.36.5.7" + ] + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.route53_public_logs" + }, + "log.file.path": "test/test", + "event": { + "agent_id_status": "verified", + "ingested": "2021-12-06T02:37:25Z", + "original": "1.0 2017-12-13T08:16:05.744Z Z123412341234 txt.example.com TXT NOERROR UDP JFK5 55.36.5.7 -", + "kind": "event", + "id": "36545504503447201576705984279898091551471012413796646912", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "dataset": "aws.route53_public_logs", + "outcome": "success" + }, + "aws": { + "route53": { + "hosted_zone_id": "Z123412341234", + "edge_location": "JFK5" + } + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.route53.edge_location | The Route 53 edge location that responded to the query. Each edge location is identified by a three-letter code and an arbitrary number, for example, DFW3. The three-letter code typically corresponds with the International Air Transport Association airport code for an airport near the edge location. (These abbreviations might change in the future.) | keyword | +| aws.route53.edns_client_subnet | A partial IP address for the client that the request originated from, if available from the DNS resolver. | keyword | +| aws.route53.hosted_zone_id | The ID of the hosted zone that is associated with all the DNS queries in this log. | keyword | +| awscloudwatch.ingestion_time | AWS CloudWatch ingest time | date | +| awscloudwatch.log_group | AWS CloudWatch Log Group name | keyword | +| awscloudwatch.log_stream | AWS CloudWatch Log Stream name | keyword | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | +| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | +| dns.question.subdomain | The subdomain is all of the labels under the registered_domain. If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | +| dns.question.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | +| dns.question.type | The type of record being queried. | keyword | +| dns.response_code | The DNS response code. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Path to the log file. | keyword | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | +| network.protocol | L7 Network protocol name. ex. http, lumberjack, transport protocol. The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| source.as.organization.name | Organization name. | keyword | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| tags | List of keywords used to tag each event. | keyword | + diff --git a/packages/aws/img/logo_route53.svg b/packages/aws/img/logo_route53.svg new file mode 100644 index 00000000000..0da61a46013 --- /dev/null +++ b/packages/aws/img/logo_route53.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 5e1fc0591f4..70f2c75292d 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 1.7.1 +version: 1.8.0 license: basic description: Collect logs and metrics from Amazon Web Services with Elastic Agent. type: integration @@ -519,5 +519,20 @@ policy_templates: title: AWS VPC logo size: 32x32 type: image/svg+xml + - name: route53 + title: AWS Route 53 + description: Collect logs from Amazon Route53 with Elastic Agent + data_streams: + - route53_public_logs + inputs: + - type: aws-cloudwatch + title: Collect logs from Route53 + description: Collecting logs from Route53 using aws-cloudwatch input + input_group: logs + icons: + - src: /img/logo_route53.svg + title: AWS Route53 logo + size: 32x40 + type: image/svg+xml owner: github: elastic/integrations