[FEATURE] Make sure route53 records are propagated across DNS servers…

… - Prevents negative cache issue (#64)

config/tasks/create_dns_a.yml

@@ -27,11 +27,26 @@
     value: "{{ hostvars[item.hostname]['ansible_host'] }}"
     private_zone: "{{cluster_vars.route53_private_zone | default(true)}}"
     overwrite: true
+    wait: yes
   become: false
   delegate_to: localhost
   run_once: true
   with_items: "{{ cluster_hosts_target }}"
   when: cluster_vars.dns_server=="route53"
+  async: 7200
+  poll: 0
+  register: route53_records
+
+- name: config/dns/a/route53 | Wait for records to bereplicated to all Amazon Route 53 DNS servers
+  async_status:
+    jid: "{{ item.ansible_job_id }}"
+  register: route53_jobs
+  until: route53_jobs.finished
+  delay: 1
+  retries: 300
+  run_once: true
+  with_items: "{{route53_records.results}}"
+  delegate_to: localhost
 
 - name: config/dns/a/clouddns | create/update A records in GCP (clouddns)
   block:
@@ -72,12 +87,6 @@
       with_items: "{{ cluster_hosts_target }}"
       run_once: true
 
-    - name: config/dns/a/dig | A short sleep to allow DNS operation to complete.  Possibly the records are not replicated when the module returns, but without a small sleep, the dig command below will sometimes fail and create a negative cache, which means name won't resolve until the SOA TTL expires.
-      wait_for:
-        timeout: 10
-      delegate_to: localhost
-      run_once: true
-
     - name: "config/dns/a/dig | Check that DNS has updated (or otherwise wait for it to do so)  [Note: lookup('dig', new_fqdn) doesn't work - seems to cache - https://github.com/ansible/ansible/issues/44128].  NOTE: A short TTL on the SOA helps if a negative cache is created.  If this is an external IP, check external DNS (otherwise only the internal VPC IP will be returned)"
       shell: "dig {{new_fqdn}} +short {% if not hostvars[item.hostname].ansible_host | regex_search('^(10.|192.168|172.1[6-9].|172.2[0-9].|172.3[01].).*') %}@{{external_dns_resolver}}{% endif %}"
       register: dig_result