elastic · kaiyan-sheng · Jan 5, 2022 · Dec 6, 2021 · Dec 22, 2021 · Dec 22, 2021
diff --git a/packages/aws/_dev/build/docs/route53.md b/packages/aws/_dev/build/docs/route53.md
@@ -1,9 +1,36 @@
 # Route 53
 
+This integration is used to fetch logs from [Route 53](https://aws.amazon.com/route53/).
 ## Logs
 
 ### Public Hosted Zone Logs
 
+The `route53_public_logs` dataset collects information about public DNS queries that Route 53 receives.
+
+Query logs contain only the queries that DNS resolvers forward to Route 53. If a DNS resolver has already cached the response to a query (such as the IP address for a load balancer for example.com), the resolver will continue to return the cached response without forwarding the query to Route 53 until the TTL for the corresponding record expires.
+
+Depending on how many DNS queries are submitted for a domain name (example.com) or subdomain name (www.example.com), which resolvers your users are using, and the TTL for the record, query logs might contain information about only one query out of every several thousand queries that are submitted to DNS resolvers.
+
+See [Route 53 Documentation](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/query-logs.html) for more information
+
 {{event "route53_public_logs"}}
 
 {{fields "route53_public_logs"}}
+
+### Resolver Logs
+
+The `route53_resolver_logs` dataset collects all DNS queries & responses for:
+* Queries that originate in Amazon Virtual Private Cloud VPCs that you specify, as well as the responses to those DNS queries.
+* Queries from on-premises resources that use an inbound Resolver endpoint.
+* Queries that use an outbound Resolver endpoint for recursive DNS resolution.
+* Queries that use Route 53 Resolver DNS Firewall rules to block, allow, or monitor domain lists.
+
+As is standard for DNS resolvers, resolvers cache DNS queries for a length of time determined by the time-to-live (TTL) for the resolver. The Route 53 Resolver caches queries that originate in your VPCs, and responds from the cache whenever possible to speed up responses. Resolver query logging logs only unique queries, not queries that Resolver is able to respond to from the cache.
+
+For example, suppose that an EC2 instance in one of the VPCs that a query logging configuration is logging queries for, submits a request for accounting.example.com. Resolver caches the response to that query, and logs the query. If the same instance’s elastic network interface makes a query for accounting.example.com within the TTL of the Resolver’s cache, Resolver responds to the query from the cache. The second query is not logged.
+
+See [Route 53 Documentation](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/resolver-query-logs.html) for more information
+
+{{event "route53_resolver_logs"}}
+
+{{fields "route53_resolver_logs"}}
diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.9.0"
+  changes:
+    - description: Add Route 53 Resolver Logs Datastream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2341
 - version: "1.8.0"
   changes:
     - description: Add Route 53 Public Zone Logs Datastream

diff --git a/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml
@@ -11,10 +11,6 @@ processors:
   - set:
       field: cloud.provider
       value: aws
-  # - set:
-  #     if: ctx.aws?.vpcflow?.account_id != null
-  #     field: cloud.account.id
-  #     value: '{{aws.vpcflow.account_id}}'
   - set:
       field: event.kind
       value: event
@@ -53,6 +49,7 @@ processors:
       field: _tmp.question
       target_field: dns.question
       ignore_missing: true
+      if: '!ctx._tmp?.question.endsWith("in-addr.arpa")'
   - rename:
       field: dns.question.domain
       target_field: dns.question.name

diff --git a/packages/aws/data_stream/route53_public_logs/manifest.yml b/packages/aws/data_stream/route53_public_logs/manifest.yml
@@ -86,7 +86,7 @@ streams:
         show_user: false
         default:
           - forwarded
-          - aws-route53-logs
+          - aws-route53_public-logs
       - name: processors
         type: yaml
         title: Processors

diff --git a/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-common-config.yml b/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-common-config.yml
@@ -0,0 +1,5 @@
+dynamic_fields:
+  event.ingested: ".*"
+fields:
+  tags:
+    - preserve_original_event
diff --git a/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-route53.log b/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-route53.log
@@ -0,0 +1,34 @@
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:23Z","query_name":"does-not-exist.abc.com.","query_type":"A","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"48701","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:23Z","query_name":"159.86.31.172.in-addr.arpa.","query_type":"PTR","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"ip-172-31-86-159.ec2.internal.","Type":"PTR","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"47924","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:23Z","query_name":"does-not-exist.example.com.","query_type":"AAAA","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"48701","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"__cloud_init_expected_not_found__.","query_type":"AAAA","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"37272","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"example.invalid.","query_type":"AAAA","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"53211","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"__cloud_init_expected_not_found__.","query_type":"A","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"37272","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"instance-data.","query_type":"A","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"56779","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"instance-data.","query_type":"AAAA","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"56779","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"__cloud_init_expected_not_found__.ec2.internal.","query_type":"AAAA","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"56468","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:24Z","query_name":"example.invalid.","query_type":"A","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"53211","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:26Z","query_name":"amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"s3-r-w.dualstack.us-east-1.amazonaws.com.","Type":"CNAME","Class":"IN"},{"Rdata":"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6","Type":"AAAA","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"43312","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:26Z","query_name":"ec2-instance-connect.us-east-1.amazonaws.com.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[],"srcaddr":"172.31.86.159","srcport":"39727","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:26Z","query_name":"amazonlinux-2-repos-us-east-1.s3.dualstack.us-east-1.amazonaws.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"s3-r-w.dualstack.us-east-1.amazonaws.com.","Type":"CNAME","Class":"IN"},{"Rdata":"67.43.156.12","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"43312","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:26Z","query_name":"s3-r-w.dualstack.us-east-1.amazonaws.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"67.43.156.12","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"43312","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:26Z","query_name":"s3-r-w.dualstack.us-east-1.amazonaws.com.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6","Type":"AAAA","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"43312","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:26Z","query_name":"ec2-instance-connect.us-east-1.amazonaws.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"67.43.156.12","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"39727","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:27Z","query_name":"s3-r-w.us-east-1.amazonaws.com.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[],"srcaddr":"172.31.86.159","srcport":"44474","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:27Z","query_name":"s3-r-w.us-east-1.amazonaws.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"81.2.69.143","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"44474","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:27Z","query_name":"amazonlinux-2-repos-us-east-1.s3.us-east-1.amazonaws.com.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"s3-r-w.us-east-1.amazonaws.com.","Type":"CNAME","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"44474","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:27Z","query_name":"amazonlinux-2-repos-us-east-1.s3.us-east-1.amazonaws.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"s3-r-w.us-east-1.amazonaws.com.","Type":"CNAME","Class":"IN"},{"Rdata":"67.43.156.12","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"44474","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:39Z","query_name":"15.22.21.154.in-addr.arpa.","query_type":"PTR","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.160","srcport":"59464","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:49Z","query_name":"1.amazon.pool.ntp.org.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[],"srcaddr":"172.31.86.159","srcport":"46159","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:49Z","query_name":"2.amazon.pool.ntp.org.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6","Type":"AAAA","Class":"IN"},{"Rdata":"2606:4700:f1::1","Type":"AAAA","Class":"IN"},{"Rdata":"2607:f3c8:3803:1::6","Type":"AAAA","Class":"IN"},{"Rdata":"2001:67c:1560:8003::c7","Type":"AAAA","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"49167","transport":"UDP","srcids":{}} |
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:49Z","query_name":"0.amazon.pool.ntp.org.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[],"srcaddr":"172.31.86.159","srcport":"51725","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:49Z","query_name":"0.amazon.pool.ntp.org.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"81.2.69.143","Type":"A","Class":"IN"},{"Rdata":"45.63.54.13","Type":"A","Class":"IN"},{"Rdata":"216.229.4.69","Type":"A","Class":"IN"},{"Rdata":"45.79.111.167","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"51725","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:49Z","query_name":"1.amazon.pool.ntp.org.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"175.16.199.1","Type":"A","Class":"IN"},{"Rdata":"108.61.73.244","Type":"A","Class":"IN"},{"Rdata":"71.43.215.194","Type":"A","Class":"IN"},{"Rdata":"162.159.200.1","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"46159","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:46:49Z","query_name":"2.amazon.pool.ntp.org.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"81.2.69.143","Type":"A","Class":"IN"},{"Rdata":"216.229.0.50","Type":"A","Class":"IN"},{"Rdata":"192.227.183.3","Type":"A","Class":"IN"},{"Rdata":"162.159.200.1","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"49167","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:47:41Z","query_name":"37.85.255.92.in-addr.arpa.","query_type":"PTR","query_class":"IN","rcode":"NXDOMAIN","answers":[],"srcaddr":"172.31.86.159","srcport":"39685","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:48:12Z","query_name":"test.example.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"1.128.3.4","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"58350","transport":"UDP","srcids":{}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:48:12Z","query_name":"test.example.com.","query_type":"AAAA","query_class":"IN","rcode":"NOERROR","answers":[],"srcaddr":"172.31.86.159","srcport":"38200","transport":"UDP","srcids":{"instance":"i-079c44232510ca8ff"}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:48:30Z","query_name":"249.252.85.54.in-addr.arpa.","query_type":"PTR","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"ec2-54-85-252-249.compute-1.amazonaws.com.","Type":"PTR","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"47882","transport":"UDP","srcids":{"instance":"i-079c44232510ca8ff"}}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:48:33Z","query_name":"abcd.example.com.","query_type":"A","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"test.example.com.","Type":"CNAME","Class":"IN"},{"Rdata":"1.128.3.4","Type":"A","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"52785","transport":"UDP","srcids":{"instance":"i-079c44232510ca8ff"}}
+{"srcaddr":"81.2.69.143","vpc_id":"vpc-7example","answers":[{"Rdata":"203.0.113.9","Type":"PTR","Class":"IN"}],"firewall_rule_group_id":"rslvr-frg-01234567890abcdef","firewall_rule_action":"BLOCK","query_name":"15.3.4.32.in-addr.arpa.","firewall_domain_list_id":"rslvr-fdl-01234567890abcdef","query_class":"IN","srcids":{"instance":"i-0d15cd0d3example"},"rcode":"NOERROR","query_type":"PTR","transport":"UDP","version":"1.100000","account_id":"111122223333","srcport":"56067","query_timestamp":"2021-02-04T17:51:55Z","region":"us-east-1"}
+{"version":"1.100000","account_id":"626345947581","region":"us-east-1","vpc_id":"vpc-01e31a7c","query_timestamp":"2021-12-11T22:48:30Z","query_name":"6.c.f.6.a.9.0.e.2.b.9.a.2.f.1.9.2.0.0.4.d.d.a.0.0.4.f.c.2.0.a.2.ip6.arpa","query_type":"PTR","query_class":"IN","rcode":"NOERROR","answers":[{"Rdata":"ec2-54-85-252-249.compute-1.amazonaws.com.","Type":"PTR","Class":"IN"}],"srcaddr":"172.31.86.159","srcport":"47882","transport":"UDP","srcids":{"instance":"i-079c44232510ca8ff"}}