Stop deploys if ClusterResourceDiscovery's kubectl calls fail

Shopify · Feb 19, 2020 · e4ce2a5 · e4ce2a5
1 parent e4e6e34
commit e4ce2a5
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 7 deletions.
diff --git a/lib/krane/cluster_resource_discovery.rb b/lib/krane/cluster_resource_discovery.rb
@@ -37,7 +37,7 @@ def prunable_resources(namespaced:)
     def fetch_resources(namespaced: false)
       command = %w(api-resources)
       command << "--namespaced=#{namespaced}"
-      raw, _, st = kubectl.run(*command, output: "wide", attempts: 5,
+      raw, err, st = kubectl.run(*command, output: "wide", attempts: 5,
         use_namespace: false)
       if st.success?
         rows = raw.split("\n")
@@ -59,7 +59,7 @@ def fetch_resources(namespaced: false)
           resource
         end
       else
-        []
+        raise FatalKubeAPIError, "Error retrieving api-resources: #{err}"
       end
     end
 
@@ -68,7 +68,7 @@ def fetch_resources(namespaced: false)
     # kubectl api-versions returns a list of group/version strings e.g. autoscaling/v2beta2
     # A kind may not exist in all versions of the group.
     def fetch_api_versions
-      raw, _, st = kubectl.run("api-versions", attempts: 5, use_namespace: false)
+      raw, err, st = kubectl.run("api-versions", attempts: 5, use_namespace: false)
       # The "core" group is represented by an empty string
       versions = { "" => %w(v1) }
       if st.success?
@@ -78,6 +78,8 @@ def fetch_api_versions
           versions[group] ||= []
           versions[group] << version
         end
+      else
+        raise FatalKubeAPIError, "Error retrieving api-versions: #{err}"
       end
       versions
     end
@@ -97,12 +99,12 @@ def version_for_kind(versions, kind)
     end
 
     def fetch_crds
-      raw_json, _, st = kubectl.run("get", "CustomResourceDefinition", output: "json", attempts: 5,
+      raw_json, err, st = kubectl.run("get", "CustomResourceDefinition", output: "json", attempts: 5,
         use_namespace: false)
       if st.success?
         JSON.parse(raw_json)["items"]
       else
-        []
+        raise FatalKubeAPIError, "Error retrieving CustomResourceDefinition: #{err}"
       end
     end
 

diff --git a/test/integration-serial/serial_deploy_test.rb b/test/integration-serial/serial_deploy_test.rb
@@ -558,4 +558,15 @@ def test_global_deploy_validation_catches_namespaced_cr
       "#{add_unique_prefix_for_test('my-first-mail')} (#{add_unique_prefix_for_test('Mail')})",
     ])
   end
+
+  def test_resource_discovery_stops_deploys_when_kubectl_errs
+    failure_msg = "Stubbed failure reason"
+    Krane::ClusterResourceDiscovery.any_instance.expects(:fetch_resources).raises(Krane::FatalKubeAPIError, failure_msg)
+    assert_deploy_failure(deploy_fixtures("hello-cloud", subset: ["configmap-data.yml"]))
+
+    assert_logs_match_all([
+      "Result: FAILURE",
+      failure_msg,
+    ], in_order: true)
+  end
 end
diff --git a/test/unit/cluster_resource_discovery_test.rb b/test/unit/cluster_resource_discovery_test.rb
@@ -6,8 +6,10 @@ class ClusterResourceDiscoveryTest < Krane::TestCase
 
   def test_fetch_resources_failure
     crd = mocked_cluster_resource_discovery(nil, success: false)
-    resources = crd.fetch_resources
-    assert_equal(resources, [])
+    assert_raises_message(Krane::FatalKubeAPIError, "Error retrieving api-resources:") do
+      crd.fetch_resources
+    end
+
   end
 
   def test_fetch_resources_not_namespaced