diff --git a/packages/aws/_dev/build/docs/route53.md b/packages/aws/_dev/build/docs/route53.md index a625060a7c6..4d4c97596a8 100644 --- a/packages/aws/_dev/build/docs/route53.md +++ b/packages/aws/_dev/build/docs/route53.md @@ -1,9 +1,36 @@ # Route 53 +This integration is used to fetch logs from [Route 53](https://aws.amazon.com/route53/). ## Logs ### Public Hosted Zone Logs +The `route53_public_logs` dataset collects information about public DNS queries that Route 53 receives. + +Query logs contain only the queries that DNS resolvers forward to Route 53. If a DNS resolver has already cached the response to a query (such as the IP address for a load balancer for example.com), the resolver will continue to return the cached response without forwarding the query to Route 53 until the TTL for the corresponding record expires. + +Depending on how many DNS queries are submitted for a domain name (example.com) or subdomain name (www.example.com), which resolvers your users are using, and the TTL for the record, query logs might contain information about only one query out of every several thousand queries that are submitted to DNS resolvers. + +See [Route 53 Documentation](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/query-logs.html) for more information + {{event "route53_public_logs"}} {{fields "route53_public_logs"}} + +### Resolver Logs + +The `route53_resolver_logs` dataset collects all DNS queries & responses for: +* Queries that originate in Amazon Virtual Private Cloud VPCs that you specify, as well as the responses to those DNS queries. +* Queries from on-premises resources that use an inbound Resolver endpoint. +* Queries that use an outbound Resolver endpoint for recursive DNS resolution. +* Queries that use Route 53 Resolver DNS Firewall rules to block, allow, or monitor domain lists. + +As is standard for DNS resolvers, resolvers cache DNS queries for a length of time determined by the time-to-live (TTL) for the resolver. The Route 53 Resolver caches queries that originate in your VPCs, and responds from the cache whenever possible to speed up responses. Resolver query logging logs only unique queries, not queries that Resolver is able to respond to from the cache. + +For example, suppose that an EC2 instance in one of the VPCs that a query logging configuration is logging queries for, submits a request for accounting.example.com. Resolver caches the response to that query, and logs the query. If the same instance’s elastic network interface makes a query for accounting.example.com within the TTL of the Resolver’s cache, Resolver responds to the query from the cache. The second query is not logged. + +See [Route 53 Documentation](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/resolver-query-logs.html) for more information + +{{event "route53_resolver_logs"}} + +{{fields "route53_resolver_logs"}} diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 2f33f74b516..77a5a8c1859 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.9.0" + changes: + - description: Add Route 53 Resolver Logs Datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/2341 - version: "1.8.0" changes: - description: Add Route 53 Public Zone Logs Datastream diff --git a/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml index 76450c33ada..c368df15d36 100644 --- a/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml @@ -11,10 +11,6 @@ processors: - set: field: cloud.provider value: aws - # - set: - # if: ctx.aws?.vpcflow?.account_id != null - # field: cloud.account.id - # value: '{{aws.vpcflow.account_id}}' - set: field: event.kind value: event @@ -53,6 +49,7 @@ processors: field: _tmp.question target_field: dns.question ignore_missing: true + if: '!ctx._tmp?.question.endsWith("in-addr.arpa")' - rename: field: dns.question.domain target_field: dns.question.name diff --git a/packages/aws/data_stream/route53_public_logs/manifest.yml b/packages/aws/data_stream/route53_public_logs/manifest.yml index acd3da3d270..eeca62fe833 100644 --- a/packages/aws/data_stream/route53_public_logs/manifest.yml +++ b/packages/aws/data_stream/route53_public_logs/manifest.yml @@ -86,7 +86,7 @@ streams: show_user: false default: - forwarded - - aws-route53-logs + - aws-route53_public-logs - name: processors type: yaml title: Processors diff --git a/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-common-config.yml b/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..5622947e4b8 --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-route53.log b/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-route53.log new file mode 100644 index 00000000000..24de090fe1e --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-route53.log @@ -0,0 +1,34 @@ +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:23Z","query_name":"does-not-exist.abc.com.","query_type":"A","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"48701","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:23Z","query_name":"159.86.31.172.in-addr.arpa.","query_type":"PTR","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"ip-172-31-86-159.ec2.internal.","Type":"PTR","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"47924","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:23Z","query_name":"does-not-exist.example.com.","query_type":"AAAA","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"48701","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"__cloud_init_expected_not_found__.","query_type":"AAAA","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"37272","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"example.invalid.","query_type":"AAAA","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"53211","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"__cloud_init_expected_not_found__.","query_type":"A","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"37272","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"instance-data.","query_type":"A","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"56779","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"instance-data.","query_type":"AAAA","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"56779","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"__cloud_init_expected_not_found__.ec2.internal.","query_type":"AAAA","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"56468","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"example.invalid.","query_type":"A","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"53211","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:26Z","query_name":"amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"s3-r-w.dualstack.us-east-1.amazonaws.com.","Type":"CNAME","Class":"IN"},{"Rdata":"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6","Type":"AAAA","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"43312","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:26Z","query_name":"ec2-instance-connect.us-east-1.amazonaws.com.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[],"srcaddr":"172.31.86.159","srcport":"39727","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:26Z","query_name":"amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"s3-r-w.dualstack.us-east-1.amazonaws.com.","Type":"CNAME","Class":"IN"},{"Rdata":"67.43.156.12","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"43312","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:26Z","query_name":"s3-r-w.dualstack.us-east-1.amazonaws.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"67.43.156.12","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"43312","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:26Z","query_name":"s3-r-w.dualstack.us-east-1.amazonaws.com.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6","Type":"AAAA","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"43312","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:26Z","query_name":"ec2-instance-connect.us-east-1.amazonaws.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"67.43.156.12","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"39727","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:27Z","query_name":"s3-r-w.us-east-1.amazonaws.com.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[],"srcaddr":"172.31.86.159","srcport":"44474","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:27Z","query_name":"s3-r-w.us-east-1.amazonaws.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"81.2.69.143","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"44474","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:27Z","query_name":"amazonlinux-2-repos-us-east-1.s3.us-east-1.amazonaws.com.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"s3-r-w.us-east-1.amazonaws.com.","Type":"CNAME","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"44474","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:27Z","query_name":"amazonlinux-2-repos-us-east-1.s3.us-east-1.amazonaws.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"s3-r-w.us-east-1.amazonaws.com.","Type":"CNAME","Class":"IN"},{"Rdata":"67.43.156.12","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"44474","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:39Z","query_name":"15.22.21.154.in-addr.arpa.","query_type":"PTR","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.160","srcport":"59464","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:49Z","query_name":"1.amazon.pool.ntp.org.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[],"srcaddr":"172.31.86.159","srcport":"46159","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:49Z","query_name":"2.amazon.pool.ntp.org.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6","Type":"AAAA","Class":"IN"},{"Rdata":"2606:4700:f1::1","Type":"AAAA","Class":"IN"},{"Rdata":"2607:f3c8:3803:1::6","Type":"AAAA","Class":"IN"},{"Rdata":"2001:67c:1560:8003::c7","Type":"AAAA","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"49167","transport":"UDP","srcids":{}} | +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:49Z","query_name":"0.amazon.pool.ntp.org.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[],"srcaddr":"172.31.86.159","srcport":"51725","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:49Z","query_name":"0.amazon.pool.ntp.org.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"81.2.69.143","Type":"A","Class":"IN"},{"Rdata":"45.63.54.13","Type":"A","Class":"IN"},{"Rdata":"216.229.4.69","Type":"A","Class":"IN"},{"Rdata":"45.79.111.167","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"51725","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:49Z","query_name":"1.amazon.pool.ntp.org.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"175.16.199.1","Type":"A","Class":"IN"},{"Rdata":"108.61.73.244","Type":"A","Class":"IN"},{"Rdata":"71.43.215.194","Type":"A","Class":"IN"},{"Rdata":"162.159.200.1","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"46159","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:49Z","query_name":"2.amazon.pool.ntp.org.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"81.2.69.143","Type":"A","Class":"IN"},{"Rdata":"216.229.0.50","Type":"A","Class":"IN"},{"Rdata":"192.227.183.3","Type":"A","Class":"IN"},{"Rdata":"162.159.200.1","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"49167","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:47:41Z","query_name":"37.85.255.92.in-addr.arpa.","query_type":"PTR","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"39685","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:48:12Z","query_name":"test.example.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"1.128.3.4","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"58350","transport":"UDP","srcids":{}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:48:12Z","query_name":"test.example.com.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[],"srcaddr":"172.31.86.159","srcport":"38200","transport":"UDP","srcids":{"instance":"i-079c44232510ca8ff"}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:48:30Z","query_name":"249.252.85.54.in-addr.arpa.","query_type":"PTR","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"ec2-54-85-252-249.compute-1.amazonaws.com.","Type":"PTR","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"47882","transport":"UDP","srcids":{"instance":"i-079c44232510ca8ff"}} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:48:33Z","query_name":"abcd.example.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"test.example.com.","Type":"CNAME","Class":"IN"},{"Rdata":"1.128.3.4","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"52785","transport":"UDP","srcids":{"instance":"i-079c44232510ca8ff"}} +{"srcaddr":"81.2.69.143","vpc_id":"vpc-7example","answers":[{"Rdata":"203.0.113.9","Type":"PTR","Class":"IN"}],"firewall_rule_group_id":"rslvr-frg-01234567890abcdef","firewall_rule_action":"BLOCK","query_name":"15.3.4.32.in-addr.arpa.","firewall_domain_list_id":"rslvr-fdl-01234567890abcdef","query_class":"IN","srcids":{"instance":"i-0d15cd0d3example"},"rcode":"NOERROR","query_type":"PTR","transport":"UDP","version":"1.100000","account_id":"111122223333","srcport":"56067","query_timestamp":"2021-02-04T17:51:55Z","region":"us-east-1"} +{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:48:30Z","query_name":"6.c.f.6.a.9.0.e.2.b.9.a.2.f.1.9.2.0.0.4.d.d.a.0.0.4.f.c.2.0.a.2.ip6.arpa","query_type":"PTR","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"ec2-54-85-252-249.compute-1.amazonaws.com.","Type":"PTR","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"47882","transport":"UDP","srcids":{"instance":"i-079c44232510ca8ff"}} \ No newline at end of file diff --git a/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-route53.log-expected.json b/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-route53.log-expected.json new file mode 100644 index 00000000000..e148e2128b8 --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-route53.log-expected.json @@ -0,0 +1,2316 @@ +{ + "expected": [ + { + "dns": { + "question": { + "name": "does-not-exist.abc.com", + "subdomain": "does-not-exist", + "registered_domain": "abc.com", + "type": "A", + "top_level_domain": "com", + "class": "IN" + }, + "response_code": "NXDOMAIN" + }, + "source": { + "port": 48701, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:23.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "does-not-exist.abc.com" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309404301Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:23Z\",\"query_name\":\"does-not-exist.abc.com.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NXDOMAIN\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"48701\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "failure" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "159.86.31.172.in-addr.arpa", + "type": "PTR", + "class": "IN" + }, + "answers": [ + { + "data": "ip-172-31-86-159.ec2.internal", + "type": "PTR", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 47924, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:23.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "ip-172-31-86-159.ec2.internal" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309409081Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:23Z\",\"query_name\":\"159.86.31.172.in-addr.arpa.\",\"query_type\":\"PTR\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"ip-172-31-86-159.ec2.internal.\",\"Type\":\"PTR\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"47924\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "does-not-exist.example.com", + "subdomain": "does-not-exist", + "registered_domain": "example.com", + "type": "AAAA", + "top_level_domain": "com", + "class": "IN" + }, + "response_code": "NXDOMAIN" + }, + "source": { + "port": 48701, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:23.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "does-not-exist.example.com" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309411124Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:23Z\",\"query_name\":\"does-not-exist.example.com.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NXDOMAIN\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"48701\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "failure" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "__cloud_init_expected_not_found__", + "type": "AAAA", + "class": "IN" + }, + "response_code": "NXDOMAIN" + }, + "source": { + "port": 37272, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:24.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "__cloud_init_expected_not_found__" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309413051Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:24Z\",\"query_name\":\"__cloud_init_expected_not_found__.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NXDOMAIN\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"37272\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "failure" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "example.invalid", + "type": "AAAA", + "class": "IN" + }, + "response_code": "NXDOMAIN" + }, + "source": { + "port": 53211, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:24.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "example.invalid" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309414935Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:24Z\",\"query_name\":\"example.invalid.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NXDOMAIN\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"53211\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "failure" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "__cloud_init_expected_not_found__", + "type": "A", + "class": "IN" + }, + "response_code": "NXDOMAIN" + }, + "source": { + "port": 37272, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:24.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "__cloud_init_expected_not_found__" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309416800Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:24Z\",\"query_name\":\"__cloud_init_expected_not_found__.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NXDOMAIN\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"37272\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "failure" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "instance-data", + "type": "A", + "class": "IN" + }, + "response_code": "NXDOMAIN" + }, + "source": { + "port": 56779, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:24.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "instance-data" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309418649Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:24Z\",\"query_name\":\"instance-data.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NXDOMAIN\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"56779\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "failure" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "instance-data", + "type": "AAAA", + "class": "IN" + }, + "response_code": "NXDOMAIN" + }, + "source": { + "port": 56779, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:24.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "instance-data" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309420491Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:24Z\",\"query_name\":\"instance-data.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NXDOMAIN\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"56779\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "failure" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "__cloud_init_expected_not_found__.ec2.internal", + "type": "AAAA", + "class": "IN" + }, + "response_code": "NXDOMAIN" + }, + "source": { + "port": 56468, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:24.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "__cloud_init_expected_not_found__.ec2.internal" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309422339Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:24Z\",\"query_name\":\"__cloud_init_expected_not_found__.ec2.internal.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NXDOMAIN\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"56468\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "failure" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "example.invalid", + "type": "A", + "class": "IN" + }, + "response_code": "NXDOMAIN" + }, + "source": { + "port": 53211, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:24.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "example.invalid" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309424182Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:24Z\",\"query_name\":\"example.invalid.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NXDOMAIN\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"53211\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "failure" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com", + "registered_domain": "amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com", + "type": "AAAA", + "top_level_domain": "s3.dualstack.us-east-1.amazonaws.com", + "class": "IN" + }, + "answers": [ + { + "name": "s3-r-w.dualstack.us-east-1.amazonaws.com", + "data": "s3-r-w.dualstack.us-east-1.amazonaws.com", + "type": "CNAME", + "class": "IN" + }, + { + "data": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "type": "AAAA", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 43312, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:26.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "s3-r-w.dualstack.us-east-1.amazonaws.com", + "amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com" + ], + "ip": [ + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309426062Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:26Z\",\"query_name\":\"amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"s3-r-w.dualstack.us-east-1.amazonaws.com.\",\"Type\":\"CNAME\",\"Class\":\"IN\"},{\"Rdata\":\"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6\",\"Type\":\"AAAA\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"43312\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "ec2-instance-connect.us-east-1.amazonaws.com", + "registered_domain": "ec2-instance-connect.us-east-1.amazonaws.com", + "type": "AAAA", + "top_level_domain": "us-east-1.amazonaws.com", + "class": "IN" + }, + "response_code": "NOERROR" + }, + "source": { + "port": 39727, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:26.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "ec2-instance-connect.us-east-1.amazonaws.com" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309428099Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:26Z\",\"query_name\":\"ec2-instance-connect.us-east-1.amazonaws.com.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"39727\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com", + "registered_domain": "amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com", + "type": "A", + "top_level_domain": "s3.dualstack.us-east-1.amazonaws.com", + "class": "IN" + }, + "answers": [ + { + "name": "s3-r-w.dualstack.us-east-1.amazonaws.com", + "data": "s3-r-w.dualstack.us-east-1.amazonaws.com", + "type": "CNAME", + "class": "IN" + }, + { + "data": "67.43.156.12", + "type": "A", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 43312, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:26.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "s3-r-w.dualstack.us-east-1.amazonaws.com", + "amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com" + ], + "ip": [ + "67.43.156.12", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309429958Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:26Z\",\"query_name\":\"amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"s3-r-w.dualstack.us-east-1.amazonaws.com.\",\"Type\":\"CNAME\",\"Class\":\"IN\"},{\"Rdata\":\"67.43.156.12\",\"Type\":\"A\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"43312\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "s3-r-w.dualstack.us-east-1.amazonaws.com", + "subdomain": "s3-r-w", + "registered_domain": "dualstack.us-east-1.amazonaws.com", + "type": "A", + "top_level_domain": "us-east-1.amazonaws.com", + "class": "IN" + }, + "answers": [ + { + "data": "67.43.156.12", + "type": "A", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 43312, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:26.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "s3-r-w.dualstack.us-east-1.amazonaws.com" + ], + "ip": [ + "67.43.156.12", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309431818Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:26Z\",\"query_name\":\"s3-r-w.dualstack.us-east-1.amazonaws.com.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"67.43.156.12\",\"Type\":\"A\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"43312\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "s3-r-w.dualstack.us-east-1.amazonaws.com", + "subdomain": "s3-r-w", + "registered_domain": "dualstack.us-east-1.amazonaws.com", + "type": "AAAA", + "top_level_domain": "us-east-1.amazonaws.com", + "class": "IN" + }, + "answers": [ + { + "data": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "type": "AAAA", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 43312, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:26.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "s3-r-w.dualstack.us-east-1.amazonaws.com" + ], + "ip": [ + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309433672Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:26Z\",\"query_name\":\"s3-r-w.dualstack.us-east-1.amazonaws.com.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6\",\"Type\":\"AAAA\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"43312\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "ec2-instance-connect.us-east-1.amazonaws.com", + "registered_domain": "ec2-instance-connect.us-east-1.amazonaws.com", + "type": "A", + "top_level_domain": "us-east-1.amazonaws.com", + "class": "IN" + }, + "answers": [ + { + "data": "67.43.156.12", + "type": "A", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 39727, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:26.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "ec2-instance-connect.us-east-1.amazonaws.com" + ], + "ip": [ + "67.43.156.12", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309435526Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:26Z\",\"query_name\":\"ec2-instance-connect.us-east-1.amazonaws.com.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"67.43.156.12\",\"Type\":\"A\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"39727\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "s3-r-w.us-east-1.amazonaws.com", + "registered_domain": "s3-r-w.us-east-1.amazonaws.com", + "type": "AAAA", + "top_level_domain": "us-east-1.amazonaws.com", + "class": "IN" + }, + "response_code": "NOERROR" + }, + "source": { + "port": 44474, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:27.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "s3-r-w.us-east-1.amazonaws.com" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309437515Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:27Z\",\"query_name\":\"s3-r-w.us-east-1.amazonaws.com.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"44474\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "s3-r-w.us-east-1.amazonaws.com", + "registered_domain": "s3-r-w.us-east-1.amazonaws.com", + "type": "A", + "top_level_domain": "us-east-1.amazonaws.com", + "class": "IN" + }, + "answers": [ + { + "data": "81.2.69.143", + "type": "A", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 44474, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:27.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "s3-r-w.us-east-1.amazonaws.com" + ], + "ip": [ + "81.2.69.143", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309439439Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:27Z\",\"query_name\":\"s3-r-w.us-east-1.amazonaws.com.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"81.2.69.143\",\"Type\":\"A\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"44474\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "amazonlinux-2-repos-us-east-1.s3.us-east-1.amazonaws.com", + "subdomain": "amazonlinux-2-repos-us-east-1", + "registered_domain": "s3.us-east-1.amazonaws.com", + "type": "AAAA", + "top_level_domain": "us-east-1.amazonaws.com", + "class": "IN" + }, + "answers": [ + { + "name": "s3-r-w.us-east-1.amazonaws.com", + "data": "s3-r-w.us-east-1.amazonaws.com", + "type": "CNAME", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 44474, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:27.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "s3-r-w.us-east-1.amazonaws.com", + "amazonlinux-2-repos-us-east-1.s3.us-east-1.amazonaws.com" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309441308Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:27Z\",\"query_name\":\"amazonlinux-2-repos-us-east-1.s3.us-east-1.amazonaws.com.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"s3-r-w.us-east-1.amazonaws.com.\",\"Type\":\"CNAME\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"44474\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "amazonlinux-2-repos-us-east-1.s3.us-east-1.amazonaws.com", + "subdomain": "amazonlinux-2-repos-us-east-1", + "registered_domain": "s3.us-east-1.amazonaws.com", + "type": "A", + "top_level_domain": "us-east-1.amazonaws.com", + "class": "IN" + }, + "answers": [ + { + "name": "s3-r-w.us-east-1.amazonaws.com", + "data": "s3-r-w.us-east-1.amazonaws.com", + "type": "CNAME", + "class": "IN" + }, + { + "data": "67.43.156.12", + "type": "A", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 44474, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:27.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "s3-r-w.us-east-1.amazonaws.com", + "amazonlinux-2-repos-us-east-1.s3.us-east-1.amazonaws.com" + ], + "ip": [ + "67.43.156.12", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309443156Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:27Z\",\"query_name\":\"amazonlinux-2-repos-us-east-1.s3.us-east-1.amazonaws.com.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"s3-r-w.us-east-1.amazonaws.com.\",\"Type\":\"CNAME\",\"Class\":\"IN\"},{\"Rdata\":\"67.43.156.12\",\"Type\":\"A\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"44474\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "15.22.21.154.in-addr.arpa", + "type": "PTR", + "class": "IN" + }, + "response_code": "NXDOMAIN" + }, + "source": { + "port": 59464, + "address": "172.31.86.160", + "ip": "172.31.86.160" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:39.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "172.31.86.160", + "154.21.22.15" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309445004Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:39Z\",\"query_name\":\"15.22.21.154.in-addr.arpa.\",\"query_type\":\"PTR\",\"query_class\":\"IN\",\"rcode\":\"NXDOMAIN\",\"answers\":[],\"srcaddr\":\"172.31.86.160\",\"srcport\":\"59464\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "failure" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "1.amazon.pool.ntp.org", + "subdomain": "1.amazon.pool", + "registered_domain": "ntp.org", + "type": "AAAA", + "top_level_domain": "org", + "class": "IN" + }, + "response_code": "NOERROR" + }, + "source": { + "port": 46159, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:49.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "1.amazon.pool.ntp.org" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309446834Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:49Z\",\"query_name\":\"1.amazon.pool.ntp.org.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"46159\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "2.amazon.pool.ntp.org", + "subdomain": "2.amazon.pool", + "registered_domain": "ntp.org", + "type": "AAAA", + "top_level_domain": "org", + "class": "IN" + }, + "answers": [ + { + "data": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "type": "AAAA", + "class": "IN" + }, + { + "data": "2606:4700:f1::1", + "type": "AAAA", + "class": "IN" + }, + { + "data": "2607:f3c8:3803:1::6", + "type": "AAAA", + "class": "IN" + }, + { + "data": "2001:67c:1560:8003::c7", + "type": "AAAA", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 49167, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:49.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "2.amazon.pool.ntp.org" + ], + "ip": [ + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "2606:4700:f1::1", + "2607:f3c8:3803:1::6", + "2001:67c:1560:8003::c7", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309448689Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:49Z\",\"query_name\":\"2.amazon.pool.ntp.org.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6\",\"Type\":\"AAAA\",\"Class\":\"IN\"},{\"Rdata\":\"2606:4700:f1::1\",\"Type\":\"AAAA\",\"Class\":\"IN\"},{\"Rdata\":\"2607:f3c8:3803:1::6\",\"Type\":\"AAAA\",\"Class\":\"IN\"},{\"Rdata\":\"2001:67c:1560:8003::c7\",\"Type\":\"AAAA\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"49167\",\"transport\":\"UDP\",\"srcids\":{}} |", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "0.amazon.pool.ntp.org", + "subdomain": "0.amazon.pool", + "registered_domain": "ntp.org", + "type": "AAAA", + "top_level_domain": "org", + "class": "IN" + }, + "response_code": "NOERROR" + }, + "source": { + "port": 51725, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:49.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "0.amazon.pool.ntp.org" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309450669Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:49Z\",\"query_name\":\"0.amazon.pool.ntp.org.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"51725\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "0.amazon.pool.ntp.org", + "subdomain": "0.amazon.pool", + "registered_domain": "ntp.org", + "type": "A", + "top_level_domain": "org", + "class": "IN" + }, + "answers": [ + { + "data": "81.2.69.143", + "type": "A", + "class": "IN" + }, + { + "data": "45.63.54.13", + "type": "A", + "class": "IN" + }, + { + "data": "216.229.4.69", + "type": "A", + "class": "IN" + }, + { + "data": "45.79.111.167", + "type": "A", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 51725, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:49.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "0.amazon.pool.ntp.org" + ], + "ip": [ + "81.2.69.143", + "45.63.54.13", + "216.229.4.69", + "45.79.111.167", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309452496Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:49Z\",\"query_name\":\"0.amazon.pool.ntp.org.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"81.2.69.143\",\"Type\":\"A\",\"Class\":\"IN\"},{\"Rdata\":\"45.63.54.13\",\"Type\":\"A\",\"Class\":\"IN\"},{\"Rdata\":\"216.229.4.69\",\"Type\":\"A\",\"Class\":\"IN\"},{\"Rdata\":\"45.79.111.167\",\"Type\":\"A\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"51725\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "1.amazon.pool.ntp.org", + "subdomain": "1.amazon.pool", + "registered_domain": "ntp.org", + "type": "A", + "top_level_domain": "org", + "class": "IN" + }, + "answers": [ + { + "data": "175.16.199.1", + "type": "A", + "class": "IN" + }, + { + "data": "108.61.73.244", + "type": "A", + "class": "IN" + }, + { + "data": "71.43.215.194", + "type": "A", + "class": "IN" + }, + { + "data": "162.159.200.1", + "type": "A", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 46159, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:49.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "1.amazon.pool.ntp.org" + ], + "ip": [ + "175.16.199.1", + "108.61.73.244", + "71.43.215.194", + "162.159.200.1", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309454367Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:49Z\",\"query_name\":\"1.amazon.pool.ntp.org.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"175.16.199.1\",\"Type\":\"A\",\"Class\":\"IN\"},{\"Rdata\":\"108.61.73.244\",\"Type\":\"A\",\"Class\":\"IN\"},{\"Rdata\":\"71.43.215.194\",\"Type\":\"A\",\"Class\":\"IN\"},{\"Rdata\":\"162.159.200.1\",\"Type\":\"A\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"46159\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "2.amazon.pool.ntp.org", + "subdomain": "2.amazon.pool", + "registered_domain": "ntp.org", + "type": "A", + "top_level_domain": "org", + "class": "IN" + }, + "answers": [ + { + "data": "81.2.69.143", + "type": "A", + "class": "IN" + }, + { + "data": "216.229.0.50", + "type": "A", + "class": "IN" + }, + { + "data": "192.227.183.3", + "type": "A", + "class": "IN" + }, + { + "data": "162.159.200.1", + "type": "A", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 49167, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:46:49.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "2.amazon.pool.ntp.org" + ], + "ip": [ + "81.2.69.143", + "216.229.0.50", + "192.227.183.3", + "162.159.200.1", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309456250Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:49Z\",\"query_name\":\"2.amazon.pool.ntp.org.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"81.2.69.143\",\"Type\":\"A\",\"Class\":\"IN\"},{\"Rdata\":\"216.229.0.50\",\"Type\":\"A\",\"Class\":\"IN\"},{\"Rdata\":\"192.227.183.3\",\"Type\":\"A\",\"Class\":\"IN\"},{\"Rdata\":\"162.159.200.1\",\"Type\":\"A\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"49167\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "37.85.255.92.in-addr.arpa", + "type": "PTR", + "class": "IN" + }, + "response_code": "NXDOMAIN" + }, + "source": { + "port": 39685, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:47:41.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "172.31.86.159", + "92.255.85.37" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309458116Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:47:41Z\",\"query_name\":\"37.85.255.92.in-addr.arpa.\",\"query_type\":\"PTR\",\"query_class\":\"IN\",\"rcode\":\"NXDOMAIN\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"39685\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "failure" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "test.example.com", + "subdomain": "test", + "registered_domain": "example.com", + "type": "A", + "top_level_domain": "com", + "class": "IN" + }, + "answers": [ + { + "data": "1.128.3.4", + "type": "A", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 58350, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:48:12.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "test.example.com" + ], + "ip": [ + "1.128.3.4", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309459972Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:48:12Z\",\"query_name\":\"test.example.com.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"1.128.3.4\",\"Type\":\"A\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"58350\",\"transport\":\"UDP\",\"srcids\":{}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c" + } + }, + { + "dns": { + "question": { + "name": "test.example.com", + "subdomain": "test", + "registered_domain": "example.com", + "type": "AAAA", + "top_level_domain": "com", + "class": "IN" + }, + "response_code": "NOERROR" + }, + "source": { + "port": 38200, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "instance": { + "id": "i-079c44232510ca8ff" + }, + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:48:12.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "test.example.com" + ], + "ip": [ + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309461953Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:48:12Z\",\"query_name\":\"test.example.com.\",\"query_type\":\"AAAA\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"38200\",\"transport\":\"UDP\",\"srcids\":{\"instance\":\"i-079c44232510ca8ff\"}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c", + "instance_id": "i-079c44232510ca8ff" + } + }, + { + "dns": { + "question": { + "name": "249.252.85.54.in-addr.arpa", + "type": "PTR", + "class": "IN" + }, + "answers": [ + { + "data": "ec2-54-85-252-249.compute-1.amazonaws.com", + "type": "PTR", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 47882, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "instance": { + "id": "i-079c44232510ca8ff" + }, + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:48:30.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "ec2-54-85-252-249.compute-1.amazonaws.com" + ], + "ip": [ + "172.31.86.159", + "54.85.252.249" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309463891Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:48:30Z\",\"query_name\":\"249.252.85.54.in-addr.arpa.\",\"query_type\":\"PTR\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"ec2-54-85-252-249.compute-1.amazonaws.com.\",\"Type\":\"PTR\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"47882\",\"transport\":\"UDP\",\"srcids\":{\"instance\":\"i-079c44232510ca8ff\"}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c", + "instance_id": "i-079c44232510ca8ff" + } + }, + { + "dns": { + "question": { + "name": "abcd.example.com", + "subdomain": "abcd", + "registered_domain": "example.com", + "type": "A", + "top_level_domain": "com", + "class": "IN" + }, + "answers": [ + { + "name": "test.example.com", + "data": "test.example.com", + "type": "CNAME", + "class": "IN" + }, + { + "data": "1.128.3.4", + "type": "A", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 52785, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "instance": { + "id": "i-079c44232510ca8ff" + }, + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:48:33.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "test.example.com", + "abcd.example.com" + ], + "ip": [ + "1.128.3.4", + "172.31.86.159" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309465776Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:48:33Z\",\"query_name\":\"abcd.example.com.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"test.example.com.\",\"Type\":\"CNAME\",\"Class\":\"IN\"},{\"Rdata\":\"1.128.3.4\",\"Type\":\"A\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"52785\",\"transport\":\"UDP\",\"srcids\":{\"instance\":\"i-079c44232510ca8ff\"}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c", + "instance_id": "i-079c44232510ca8ff" + } + }, + { + "dns": { + "question": { + "name": "15.3.4.32.in-addr.arpa", + "type": "PTR", + "class": "IN" + }, + "answers": [ + { + "data": "203.0.113.9", + "type": "PTR", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "GB-ENG", + "city_name": "London", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "region_name": "England", + "location": { + "lon": -0.0931, + "lat": 51.5142 + } + }, + "address": "81.2.69.143", + "port": 56067, + "ip": "81.2.69.143" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "instance": { + "id": "i-0d15cd0d3example" + }, + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "111122223333" + } + }, + "@timestamp": "2021-02-04T17:51:55.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "203.0.113.9" + ], + "ip": [ + "81.2.69.143", + "32.4.3.15" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309467629Z", + "original": "{\"srcaddr\":\"81.2.69.143\",\"vpc_id\":\"vpc-7example\",\"answers\":[{\"Rdata\":\"203.0.113.9\",\"Type\":\"PTR\",\"Class\":\"IN\"}],\"firewall_rule_group_id\":\"rslvr-frg-01234567890abcdef\",\"firewall_rule_action\":\"BLOCK\",\"query_name\":\"15.3.4.32.in-addr.arpa.\",\"firewall_domain_list_id\":\"rslvr-fdl-01234567890abcdef\",\"query_class\":\"IN\",\"srcids\":{\"instance\":\"i-0d15cd0d3example\"},\"rcode\":\"NOERROR\",\"query_type\":\"PTR\",\"transport\":\"UDP\",\"version\":\"1.100000\",\"account_id\":\"111122223333\",\"srcport\":\"56067\",\"query_timestamp\":\"2021-02-04T17:51:55Z\",\"region\":\"us-east-1\"}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "route53": { + "firewall": { + "rule_group": { + "id": "rslvr-frg-01234567890abcdef" + }, + "action": "BLOCK", + "domain_list": { + "id": "rslvr-fdl-01234567890abcdef" + } + } + }, + "vpc_id": "vpc-7example", + "instance_id": "i-0d15cd0d3example" + } + }, + { + "dns": { + "question": { + "name": "6.c.f.6.a.9.0.e.2.b.9.a.2.f.1.9.2.0.0.4.d.d.a.0.0.4.f.c.2.0.a.2.ip6.arpa", + "type": "PTR", + "class": "IN" + }, + "answers": [ + { + "data": "ec2-54-85-252-249.compute-1.amazonaws.com", + "type": "PTR", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "source": { + "port": 47882, + "address": "172.31.86.159", + "ip": "172.31.86.159" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "cloud": { + "instance": { + "id": "i-079c44232510ca8ff" + }, + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "626345947581" + } + }, + "@timestamp": "2021-12-11T22:48:30.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "hosts": [ + "ec2-54-85-252-249.compute-1.amazonaws.com" + ], + "ip": [ + "172.31.86.159", + "2a02:cf40:0add:4002:91f2:a9b2:e09a:6fc6" + ] + }, + "event": { + "ingested": "2021-12-22T16:22:41.309469504Z", + "original": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:48:30Z\",\"query_name\":\"6.c.f.6.a.9.0.e.2.b.9.a.2.f.1.9.2.0.0.4.d.d.a.0.0.4.f.c.2.0.a.2.ip6.arpa\",\"query_type\":\"PTR\",\"query_class\":\"IN\",\"rcode\":\"NOERROR\",\"answers\":[{\"Rdata\":\"ec2-54-85-252-249.compute-1.amazonaws.com.\",\"Type\":\"PTR\",\"Class\":\"IN\"}],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"47882\",\"transport\":\"UDP\",\"srcids\":{\"instance\":\"i-079c44232510ca8ff\"}}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success" + }, + "aws": { + "vpc_id": "vpc-01e31a7c", + "instance_id": "i-079c44232510ca8ff" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..c1576eedae0 --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,93 @@ +{{#unless log_group_name}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} +{{/if}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} +{{/if}} +{{/unless}} + +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} +{{/if}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} +{{/if}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs new file mode 100644 index 00000000000..ccf43bcddc1 --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs @@ -0,0 +1,51 @@ +queue_url: {{queue_url}} +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if visibility_timeout}} +visibility_timeout: {{visibility_timeout}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if max_number_of_messages}} +max_number_of_messages: {{max_number_of_messages}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if fips_enabled}} +fips_enabled: {{fips_enabled}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/aws/data_stream/route53_resolver_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/route53_resolver_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..1daa7af31c8 --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,292 @@ +--- +description: Pipeline for AWS Route53 Logs + +processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- set: + field: ecs.version + value: '1.12.0' +- rename: + field: message + target_field: event.original + ignore_missing: true +- json: + field: event.original + target_field: json +- set: + field: cloud.provider + value: aws +- rename: + field: json.account_id + target_field: cloud.account.id + ignore_missing: true +- date: + field: json.query_timestamp + target_field: '@timestamp' + ignore_failure: true + formats: + - ISO8601 +- rename: + field: json.region + target_field: cloud.region + ignore_missing: true +- rename: + field: json.vpc_id + target_field: aws.vpc_id + ignore_missing: true +- rename: + field: json.srcids.instance + target_field: aws.instance_id + ignore_missing: true +- set: + field: cloud.instance.id + copy_from: aws.instance_id + ignore_empty_value: true +- gsub: + field: json.query_name + pattern: \.$ + replacement: "" + ignore_missing: true +- registered_domain: + field: json.query_name + target_field: dns.question + ignore_missing: true + if: '!ctx.json?.query_name.endsWith("in-addr.arpa") && !ctx.json?.query_name.endsWith("ip6.arpa")' +- rename: + field: dns.question.domain + target_field: dns.question.name + ignore_missing: true +- rename: + field: json.query_name + target_field: dns.question.name + ignore_missing: true + if: ctx.dns?.question?.name == null +- rename: + field: json.query_class + target_field: dns.question.class + ignore_missing: true +- rename: + field: json.query_type + target_field: dns.question.type + ignore_missing: true +- rename: + field: json.rcode + target_field: dns.response_code + ignore_missing: true +- rename: + field: json.answers + target_field: dns.answers + ignore_missing: true +- script: + lang: painless + ignore_failure: true + if: ctx.dns?.answers != null && ctx.dns?.answers instanceof List + source: >- + List answers = new ArrayList(); + for (answer in ctx.dns.answers) { + Map new_answer = new HashMap(); + if(answer?.Class != null) { + new_answer.put("class", answer?.Class); + } + if(answer?.Type != null) { + new_answer.put("type", answer?.Type); + } + if(answer?.Rdata != null) { + new_answer.put("data", answer?.Rdata); + if (new_answer?.data != null && new_answer.data.length() > 0 && new_answer.data.substring(new_answer.data.length() - 1) == '.') { + new_answer.data = new_answer.data.substring(0, new_answer.data.length() - 1); + } + if (new_answer?.type != null && new_answer.type == 'CNAME') { + new_answer.put("name", new_answer?.data); + } + } + answers.add(new_answer); + if(ctx.related == null) { + ctx.put('related', new HashMap()); + } + if(ctx.related?.ip == null) { + ctx.related.put('ip',new ArrayList()); + } + if(ctx.related?.hosts == null) { + ctx.related.put('hosts',new ArrayList()); + } + if(['A','AAAA'].contains(new_answer.type)) { + ctx.related.ip.add(new_answer.data); + } + if(['CNAME', 'PTR'].contains(new_answer.type)) { + ctx.related.hosts.add(new_answer.data); + } + } + ctx.dns.answers = answers; +- rename: + field: json.transport + target_field: network.transport + ignore_missing: true +- lowercase: + field: network.transport + ignore_missing: true +- set: + field: network.iana_number + value: '6' + if: ctx.network?.transport == "tcp" +- set: + field: network.iana_number + value: '17' + if: ctx.network?.transport == "udp" +- set: + field: network.protocol + value: dns +- convert: + field: json.srcport + target_field: source.port + type: long + ignore_missing: true +- rename: + field: json.srcaddr + target_field: source.address + ignore_missing: true +- convert: + field: source.address + target_field: source.ip + type: ip + ignore_missing: true +# IP Geolocation Lookup +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +# IP Autonomous System (AS) Lookup +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- set: + field: network.type + value: ipv4 + if: 'ctx.source?.ip != null && ctx.source?.ip.contains(".")' +- set: + field: network.type + value: ipv6 + if: 'ctx.source?.ip != null && ctx.source?.ip.contains(":")' +- rename: + field: json.firewall_rule_action + target_field: aws.route53.firewall.action + ignore_missing: true +- rename: + field: json.firewall_rule_group_id + target_field: aws.route53.firewall.rule_group.id + ignore_missing: true +- rename: + field: json.firewall_domain_list_id + target_field: aws.route53.firewall.domain_list.id + ignore_missing: true +- set: + field: event.kind + value: event +- append: + field: event.category + value: network +- append: + field: event.type + value: protocol +- set: + field: event.outcome + value: success + if: ctx.dns?.response_code == "NOERROR" +- set: + field: event.outcome + value: failure + if: ctx.dns?.response_code != "NOERROR" +- append: + field: related.ip + value: "{{source.ip}}" + if: ctx.source?.ip != null +- script: + lang: painless + ignore_failure: true + if: ctx.dns?.question?.name != null && ctx.dns?.question?.type == "PTR" + source: >- + String ip; + if(ctx.dns?.question?.name.contains(".in-addr.arpa")) { + List reverse_ip = Arrays.asList(ctx.dns?.question?.name.replace(".in-addr.arpa", "").splitOnToken(".")); + List ip_arr = new ArrayList(); + for (int i = reverse_ip.length; i > 0 ; i--) { + ip_arr.add(reverse_ip[i-1]); + } + ip = String.join(".",ip_arr); + } else if (ctx.dns?.question?.name.contains(".ip6.arpa")) { + List reverse_ip = Arrays.asList(ctx.dns?.question?.name.replace(".ip6.arpa", "").splitOnToken(".")); + List ip_arr = new ArrayList(); + int j = 1; + for (int i = reverse_ip.length; i > 0 ; i--) { + ip_arr.add(reverse_ip[i-1]); + if(j % 4 == 0 && i != 1) { + j = 0; + ip_arr.add(":"); + } + j++; + } + ip = String.join("",ip_arr); + } + if(ctx.related?.ip == null) { + ctx.related.put('ip',new ArrayList()); + } + if(ip != null && !ctx.related?.ip.contains(ip)) { + ctx.related.ip.add(ip); + } +- append: + field: related.hosts + value: "{{dns.question.name}}" + if: ctx.dns?.question?.name != null && ctx.dns?.question?.type != "PTR" +- remove: + field: + - json + ignore_missing: true +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0)); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0)); + } + handleMap(ctx); +- remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true +on_failure: +- set: + field: 'error.message' + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/aws/data_stream/route53_resolver_logs/fields/agent.yml b/packages/aws/data_stream/route53_resolver_logs/fields/agent.yml new file mode 100644 index 00000000000..da4e652c53b --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/packages/aws/data_stream/route53_resolver_logs/fields/base-fields.yml b/packages/aws/data_stream/route53_resolver_logs/fields/base-fields.yml new file mode 100644 index 00000000000..4d7ce7a2f2c --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.route53_resolver_logs diff --git a/packages/aws/data_stream/route53_resolver_logs/fields/beats.yml b/packages/aws/data_stream/route53_resolver_logs/fields/beats.yml new file mode 100644 index 00000000000..3dde4d0b577 --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/fields/beats.yml @@ -0,0 +1,15 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.file.path + type: keyword + description: Path to the log file. +- name: awscloudwatch.log_stream + type: keyword + description: AWS CloudWatch Log Stream name +- name: awscloudwatch.log_group + type: keyword + description: AWS CloudWatch Log Group name +- name: awscloudwatch.ingestion_time + type: date + description: AWS CloudWatch ingest time diff --git a/packages/aws/data_stream/route53_resolver_logs/fields/ecs.yml b/packages/aws/data_stream/route53_resolver_logs/fields/ecs.yml new file mode 100644 index 00000000000..9f910559a55 --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/fields/ecs.yml @@ -0,0 +1,60 @@ +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: message +- external: ecs + name: tags +- external: ecs + name: dns.question.name +- external: ecs + name: dns.question.type +- external: ecs + name: dns.question.registered_domain +- external: ecs + name: dns.question.subdomain +- external: ecs + name: dns.question.top_level_domain +- external: ecs + name: dns.response_code +- external: ecs + name: dns.answers +- external: ecs + name: dns.question.class +- external: ecs + name: network.iana_number +- external: ecs + name: network.transport +- external: ecs + name: network.protocol +- external: ecs + name: network.type +- external: ecs + name: related.ip +- external: ecs + name: related.hosts +- external: ecs + name: source.port +- external: ecs + name: source.address +- external: ecs + name: source.ip +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name +- name: source.geo.city_name + external: ecs +- name: source.geo.continent_name + external: ecs +- name: source.geo.country_iso_code + external: ecs +- name: source.geo.country_name + external: ecs +- name: source.geo.location + external: ecs +- name: source.geo.region_iso_code + external: ecs +- name: source.geo.region_name + external: ecs diff --git a/packages/aws/data_stream/route53_resolver_logs/fields/fields.yml b/packages/aws/data_stream/route53_resolver_logs/fields/fields.yml new file mode 100644 index 00000000000..a944c6c77ba --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/fields/fields.yml @@ -0,0 +1,15 @@ +- name: aws.route53 + type: group + fields: + - name: firewall.rule_group.id + type: keyword + description: | + The ID of the DNS Firewall rule group that matched the domain name in the query. This is populated only if DNS Firewall found a match for a rule with action set to alert or block. + - name: firewall.action + type: keyword + description: | + The action specified by the rule that matched the domain name in the query. This is populated only if DNS Firewall found a match for a rule with action set to alert or block. + - name: firewall.domain_list.id + type: keyword + description: | + The domain list used by the rule that matched the domain name in the query. This is populated only if DNS Firewall found a match for a rule with action set to alert or block. diff --git a/packages/aws/data_stream/route53_resolver_logs/fields/package-fields.yml b/packages/aws/data_stream/route53_resolver_logs/fields/package-fields.yml new file mode 100644 index 00000000000..a342f302759 --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/fields/package-fields.yml @@ -0,0 +1,11 @@ +- name: aws + type: group + fields: + - name: vpc_id + type: keyword + description: | + The ID of the VPC that contains the network interface for which the traffic is recorded. + - name: instance_id + type: keyword + description: |- + The ID of the instance that's associated with network interface for which the traffic is recorded, if the instance is owned by you. diff --git a/packages/aws/data_stream/route53_resolver_logs/manifest.yml b/packages/aws/data_stream/route53_resolver_logs/manifest.yml new file mode 100644 index 00000000000..f55fedda446 --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/manifest.yml @@ -0,0 +1,173 @@ +title: AWS Route 53 Resolver Query Logs +type: logs +streams: + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS Route 53 Resolver Logs + description: Collect AWS Route 53 Resolver Logs using Cloudwatch + vars: + - name: log_group_arn + type: text + title: Log Group ARN + description: "ARN of the log group to collect logs from. See [Documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-cloudwatch.html#_log_group_arn)." + multi: false + show_user: true + required: false + - name: log_group_name + type: text + title: Log Group Name + description: "Name of the log group to collect logs from. See [Documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-cloudwatch.html#_log_group_name)." + multi: false + show_user: false + required: false + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + description: "The prefix for a group of log group names. See [Documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-cloudwatch.html#_log_group_name_prefix)." + multi: false + show_user: false + required: false + - name: log_streams + type: text + title: Log Streams + description: Required when using `Log Group Name Prefix` or `Log Group Name`. + multi: false + show_user: false + required: false + - name: log_stream_prefix + type: text + title: Log Streams Prefix + description: A list of strings of log streams names that Filebeat collect log events from. + multi: false + show_user: false + required: false + - name: region_name + type: text + title: Region Name + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + multi: false + show_user: false + required: false + - name: start_position + type: text + title: Start Position + description: Allows user to specify if this input should read log files from the beginning or from the end. + multi: false + show_user: false + required: true + default: beginning + - name: scan_frequency + type: text + title: Scan Frequency + description: This config parameter sets how often Filebeat checks for new log events from the specified log group in seconds. Default scan_frequency is 1 minute. + multi: false + show_user: false + required: true + default: 1m + - name: api_timeout + type: text + title: API Timeout + multi: false + required: false + show_user: false + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. + - name: api_sleep + type: text + title: API Sleep + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. FilterLogEvents API has a quota of 5 transactions per second (TPS)/account/Region. By default, api_sleep is 200 ms. + multi: false + show_user: false + required: false + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-route53_resolver-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - input: aws-s3 + template_path: aws-s3.yml.hbs + title: AWS Route 53 Resolver Logs + description: Collect AWS Resolver logs using s3 input + vars: + - name: visibility_timeout + type: text + title: Visibility Timeout + multi: false + required: false + show_user: false + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout + type: text + title: API Timeout + multi: false + required: false + show_user: false + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. + - name: queue_url + type: text + title: Queue URL + multi: false + required: true + show_user: true + description: URL of the AWS SQS queue that messages will be received from. + - name: fips_enabled + type: bool + title: Enable S3 FIPS + default: false + multi: false + required: false + show_user: false + description: Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-route53_resolver-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: max_number_of_messages + type: integer + title: Maximum Concurrent SQS Messages + description: The maximum number of SQS messages that can be inflight at any time. + default: 5 + required: false + show_user: false diff --git a/packages/aws/data_stream/route53_resolver_logs/sample_event.json b/packages/aws/data_stream/route53_resolver_logs/sample_event.json new file mode 100644 index 00000000000..4fbc261e6ef --- /dev/null +++ b/packages/aws/data_stream/route53_resolver_logs/sample_event.json @@ -0,0 +1,128 @@ +{ + "@timestamp": "2021-02-04T17:51:55.000Z", + "agent": { + "name": "docker-fleet-agent", + "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", + "type": "filebeat", + "ephemeral_id": "1cf87179-f6b3-44b0-a46f-3aa6bc0f995f", + "version": "8.0.0" + }, + "aws": { + "route53": { + "firewall": { + "rule_group": { + "id": "rslvr-frg-01234567890abcdef" + }, + "action": "BLOCK", + "domain_list": { + "id": "rslvr-fdl-01234567890abcdef" + } + } + }, + "vpc_id": "vpc-7example", + "instance_id": "i-0d15cd0d3example" + }, + "awscloudwatch": { + "log_group": "test", + "ingestion_time": "2021-12-06T02:18:20.000Z", + "log_stream": "test" + }, + "cloud": { + "instance": { + "id": "i-0d15cd0d3example" + }, + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "111122223333" + } + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.route53_public_logs" + }, + "dns": { + "question": { + "name": "15.3.4.32.in-addr.arpa", + "subdomain": "15.3.4", + "registered_domain": "32.in-addr.arpa", + "type": "PTR", + "top_level_domain": "in-addr.arpa", + "class": "IN" + }, + "answers": [ + { + "data": "203.0.113.9", + "type": "PTR", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", + "version": "8.0.0", + "snapshot": true + }, + "event": { + "agent_id_status": "verified", + "ingested": "2021-12-12T00:28:02.201047005Z", + "original": "{\"srcaddr\":\"4.5.64.102\",\"vpc_id\":\"vpc-7example\",\"answers\":[{\"Rdata\":\"203.0.113.9\",\"Type\":\"PTR\",\"Class\":\"IN\"}],\"firewall_rule_group_id\":\"rslvr-frg-01234567890abcdef\",\"firewall_rule_action\":\"BLOCK\",\"query_name\":\"15.3.4.32.in-addr.arpa.\",\"firewall_domain_list_id\":\"rslvr-fdl-01234567890abcdef\",\"query_class\":\"IN\",\"srcids\":{\"instance\":\"i-0d15cd0d3example\"},\"rcode\":\"NOERROR\",\"query_type\":\"PTR\",\"transport\":\"UDP\",\"version\":\"1.100000\",\"account_id\":\"111122223333\",\"srcport\":\"56067\",\"query_timestamp\":\"2021-02-04T17:51:55Z\",\"region\":\"us-east-1\"}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success", + "dataset": "aws.route53_resolver_logs" + }, + "input": { + "type": "aws-cloudwatch" + }, + "log.file.path": "test/test", + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "related": { + "hosts": [ + "15.3.4.32.in-addr.arpa" + ], + "ip": [ + "4.5.64.102" + ] + }, + "source": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 3356, + "organization": { + "name": "Level 3 Parent, LLC" + } + }, + "address": "4.5.64.102", + "port": 56067, + "ip": "4.5.64.102" + }, + "tags": [ + "preserve_original_event", + "forwarded", + "aws-route53_resolver-logs" + ] +} \ No newline at end of file diff --git a/packages/aws/docs/route53.md b/packages/aws/docs/route53.md index 61e415f6b78..b77f148ee8a 100644 --- a/packages/aws/docs/route53.md +++ b/packages/aws/docs/route53.md @@ -1,9 +1,18 @@ # Route 53 +This integration is used to fetch logs from [Route 53](https://aws.amazon.com/route53/). ## Logs ### Public Hosted Zone Logs +The `route53_public_logs` dataset collects information about public DNS queries that Route 53 receives. + +Query logs contain only the queries that DNS resolvers forward to Route 53. If a DNS resolver has already cached the response to a query (such as the IP address for a load balancer for example.com), the resolver will continue to return the cached response without forwarding the query to Route 53 until the TTL for the corresponding record expires. + +Depending on how many DNS queries are submitted for a domain name (example.com) or subdomain name (www.example.com), which resolvers your users are using, and the TTL for the record, query logs might contain information about only one query out of every several thousand queries that are submitted to DNS resolvers. + +See [Route 53 Documentation](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/query-logs.html) for more information + An example event for `route53_public` looks as following: ```json @@ -180,3 +189,231 @@ An example event for `route53_public` looks as following: | source.ip | IP address of the source (IPv4 or IPv6). | ip | | tags | List of keywords used to tag each event. | keyword | + +### Resolver Logs + +The `route53_resolver_logs` dataset collects all DNS queries & responses for: +* Queries that originate in Amazon Virtual Private Cloud VPCs that you specify, as well as the responses to those DNS queries. +* Queries from on-premises resources that use an inbound Resolver endpoint. +* Queries that use an outbound Resolver endpoint for recursive DNS resolution. +* Queries that use Route 53 Resolver DNS Firewall rules to block, allow, or monitor domain lists. + +As is standard for DNS resolvers, resolvers cache DNS queries for a length of time determined by the time-to-live (TTL) for the resolver. The Route 53 Resolver caches queries that originate in your VPCs, and responds from the cache whenever possible to speed up responses. Resolver query logging logs only unique queries, not queries that Resolver is able to respond to from the cache. + +For example, suppose that an EC2 instance in one of the VPCs that a query logging configuration is logging queries for, submits a request for accounting.example.com. Resolver caches the response to that query, and logs the query. If the same instance’s elastic network interface makes a query for accounting.example.com within the TTL of the Resolver’s cache, Resolver responds to the query from the cache. The second query is not logged. + +See [Route 53 Documentation](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/resolver-query-logs.html) for more information + +An example event for `route53_resolver` looks as following: + +```json +{ + "@timestamp": "2021-02-04T17:51:55.000Z", + "agent": { + "name": "docker-fleet-agent", + "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", + "type": "filebeat", + "ephemeral_id": "1cf87179-f6b3-44b0-a46f-3aa6bc0f995f", + "version": "8.0.0" + }, + "aws": { + "route53": { + "firewall": { + "rule_group": { + "id": "rslvr-frg-01234567890abcdef" + }, + "action": "BLOCK", + "domain_list": { + "id": "rslvr-fdl-01234567890abcdef" + } + } + }, + "vpc_id": "vpc-7example", + "instance_id": "i-0d15cd0d3example" + }, + "awscloudwatch": { + "log_group": "test", + "ingestion_time": "2021-12-06T02:18:20.000Z", + "log_stream": "test" + }, + "cloud": { + "instance": { + "id": "i-0d15cd0d3example" + }, + "region": "us-east-1", + "provider": "aws", + "account": { + "id": "111122223333" + } + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.route53_public_logs" + }, + "dns": { + "question": { + "name": "15.3.4.32.in-addr.arpa", + "subdomain": "15.3.4", + "registered_domain": "32.in-addr.arpa", + "type": "PTR", + "top_level_domain": "in-addr.arpa", + "class": "IN" + }, + "answers": [ + { + "data": "203.0.113.9", + "type": "PTR", + "class": "IN" + } + ], + "response_code": "NOERROR" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", + "version": "8.0.0", + "snapshot": true + }, + "event": { + "agent_id_status": "verified", + "ingested": "2021-12-12T00:28:02.201047005Z", + "original": "{\"srcaddr\":\"4.5.64.102\",\"vpc_id\":\"vpc-7example\",\"answers\":[{\"Rdata\":\"203.0.113.9\",\"Type\":\"PTR\",\"Class\":\"IN\"}],\"firewall_rule_group_id\":\"rslvr-frg-01234567890abcdef\",\"firewall_rule_action\":\"BLOCK\",\"query_name\":\"15.3.4.32.in-addr.arpa.\",\"firewall_domain_list_id\":\"rslvr-fdl-01234567890abcdef\",\"query_class\":\"IN\",\"srcids\":{\"instance\":\"i-0d15cd0d3example\"},\"rcode\":\"NOERROR\",\"query_type\":\"PTR\",\"transport\":\"UDP\",\"version\":\"1.100000\",\"account_id\":\"111122223333\",\"srcport\":\"56067\",\"query_timestamp\":\"2021-02-04T17:51:55Z\",\"region\":\"us-east-1\"}", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "kind": "event", + "outcome": "success", + "dataset": "aws.route53_resolver_logs" + }, + "input": { + "type": "aws-cloudwatch" + }, + "log.file.path": "test/test", + "network": { + "protocol": "dns", + "transport": "udp", + "type": "ipv4", + "iana_number": "17" + }, + "related": { + "hosts": [ + "15.3.4.32.in-addr.arpa" + ], + "ip": [ + "4.5.64.102" + ] + }, + "source": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 3356, + "organization": { + "name": "Level 3 Parent, LLC" + } + }, + "address": "4.5.64.102", + "port": 56067, + "ip": "4.5.64.102" + }, + "tags": [ + "preserve_original_event", + "forwarded", + "aws-route53_resolver-logs" + ] +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.instance_id | The ID of the instance that's associated with network interface for which the traffic is recorded, if the instance is owned by you. | keyword | +| aws.route53.firewall.action | The action specified by the rule that matched the domain name in the query. This is populated only if DNS Firewall found a match for a rule with action set to alert or block. | keyword | +| aws.route53.firewall.domain_list.id | The domain list used by the rule that matched the domain name in the query. This is populated only if DNS Firewall found a match for a rule with action set to alert or block. | keyword | +| aws.route53.firewall.rule_group.id | The ID of the DNS Firewall rule group that matched the domain name in the query. This is populated only if DNS Firewall found a match for a rule with action set to alert or block. | keyword | +| aws.vpc_id | The ID of the VPC that contains the network interface for which the traffic is recorded. | keyword | +| awscloudwatch.ingestion_time | AWS CloudWatch ingest time | date | +| awscloudwatch.log_group | AWS CloudWatch Log Group name | keyword | +| awscloudwatch.log_stream | AWS CloudWatch Log Stream name | keyword | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| dns.answers | An array containing an object for each answer section returned by the server. The main keys that should be present in these objects are defined by ECS. Records that have more information may contain more keys than what ECS defines. Not all DNS data sources give all details about DNS answers. At minimum, answer objects must contain the `data` key. If more information is available, map as much of it to ECS as possible, and add any additional fields to the answer objects as custom fields. | object | +| dns.question.class | The class of records being queried. | keyword | +| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | +| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | +| dns.question.subdomain | The subdomain is all of the labels under the registered_domain. If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | +| dns.question.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | +| dns.question.type | The type of record being queried. | keyword | +| dns.response_code | The DNS response code. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Path to the log file. | keyword | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | +| network.protocol | L7 Network protocol name. ex. http, lumberjack, transport protocol. The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| source.as.organization.name | Organization name. | keyword | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | + diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 70f2c75292d..fe79810c111 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 1.8.0 +version: 1.9.0 license: basic description: Collect logs and metrics from Amazon Web Services with Elastic Agent. type: integration @@ -524,6 +524,7 @@ policy_templates: description: Collect logs from Amazon Route53 with Elastic Agent data_streams: - route53_public_logs + - route53_resolver_logs inputs: - type: aws-cloudwatch title: Collect logs from Route53