(CAT-1241) - Adding retry when provision failed with Timeout

puppetlabs · Jul 28, 2023 · cccb292 · cccb292
1 parent edf7413
commit cccb292
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 1 deletion.
diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml
@@ -23,7 +23,7 @@ Metrics/AbcSize:
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns, inherit_mode.
 # AllowedMethods: refine
 Metrics/BlockLength:
-  Max: 345
+  Max: 350
 
 # Offense count: 8
 # Configuration parameters: AllowedMethods, AllowedPatterns.

diff --git a/lib/puppet_litmus/rake_tasks.rb b/lib/puppet_litmus/rake_tasks.rb
@@ -75,6 +75,8 @@
   task :provision, [:provisioner, :platform, :inventory_vars] do |_task, args|
     Rake::Task['spec_prep'].invoke
 
+    retry_count = 3
+    current_retry_count = 0
     begin
       spinner = start_spinner("Provisioning #{args[:platform]} using #{args[:provisioner]} provisioner.")
 
@@ -94,6 +96,13 @@
           check_connectivity?(inventory_hash_from_inventory_file, target)
         end
       end
+    rescue PuppetLitmus::RakeHelper::LitmusTimeoutError
+      current_retry_count += 1
+      Rake::Task['litmus:tear_down'].invoke(target_names.first)
+      raise if current_retry_count >= retry_count
+
+      puts "Provision of node #{target_names.first} failed, Retrying #{current_retry_count} of #{retry_count}"
+      retry
     ensure
       stop_spinner(spinner)
     end