Merge pull request #2012 from kosstennbl/cert-task-rework

cert: rework certification tasks, add "essential" argument

src/tasks/cert/cert.cr

@@ -16,12 +16,14 @@ task "cert", ["version", "cert_compatibility", "cert_state", "cert_security", "c
   max_passed = CNFManager::Points.total_max_passed("cert")
   essential_total_passed = CNFManager::Points.total_passed("essential")
   essential_max_passed = CNFManager::Points.total_max_passed("essential")
+  max_passed = essential_max_passed if args.raw.includes? "essential"
   stdout_success "  - #{total_passed} of #{max_passed} total tests passed"
 
   if essential_total_passed >= ESSENTIAL_PASSED_THRESHOLD
     stdout_success "  - #{essential_total_passed} of #{essential_max_passed} essential tests passed"
   else
-    stdout_failure "FAILED: #{essential_total_passed} of #{essential_max_passed} essential tests passed"
+    stdout_failure "  - #{essential_total_passed} of #{essential_max_passed} essential tests passed"
+    stdout_failure "Certification failed! Passing threshold is #{ESSENTIAL_PASSED_THRESHOLD} essential tests"
   end
 
   update_yml("#{CNFManager::Points::Results.file}", "points", total)

src/tasks/cert/cert_compatibility.cr

@@ -6,10 +6,19 @@ require "totem"
 require "../utils/utils.cr"
 
 desc "The CNF test suite checks to see if CNFs support horizontal scaling (across multiple machines) and vertical scaling (between sizes of machines) by using the native K8s kubectl"
-task "cert_compatibility", ["cert_compatibility_title", "helm_chart_valid", "helm_chart_published", "helm_deploy", "increase_decrease_capacity", "rollback"] do |_, args|
-# task "cert_compatibility", ["cert_compatibility_title", "helm_chart_valid", "helm_chart_published", "helm_deploy", "cni_compatible", "increase_decrease_capacity", "rollback"] do |_, args|
-  # stdout_score("compatibility", "Compatibility, Installability, and Upgradeability")
-  stdout_score(["compatibility", "cert"], "Compatibility, Installability, and Upgradeability")
+task "cert_compatibility" do |t, args|
+  puts "Compatibility, Installability & Upgradability Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
+
+  essential_only = args.raw.includes? "essential"
+  tags = ["compatibility", "cert"]
+  tags << "essential" if essential_only
+
+  tasks = CNFManager::Points.tasks_by_tag_intersection(tags)
+  tasks.each do |task|
+    t.invoke(task)
+  end
+
+  stdout_score(tags, "Compatibility, Installability, and Upgradeability")
   case "#{ARGV.join(" ")}" 
   when /cert_compatibility/
     stdout_info "Results have been saved to #{CNFManager::Points::Results.file}".colorize(:green)
@@ -18,5 +27,5 @@ task "cert_compatibility", ["cert_compatibility_title", "helm_chart_valid", "hel
 end
 
 task "cert_compatibility_title" do |_, args|
-  puts "Compatibility, Installability & Upgradability Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
+
 end

src/tasks/cert/cert_configuration.cr

@@ -9,34 +9,21 @@ require "../utils/utils.cr"
 
 desc "Configuration should be managed in a declarative manner, using ConfigMaps, Operators, or other declarative interfaces."
 
-task "cert_configuration", [
-    "cert_configuration_title",
-    "nodeport_not_used",
-    "hostport_not_used",
-    "hardcoded_ip_addresses_in_k8s_runtime_configuration",
-    "secrets_used",
-    "immutable_configmap",
-    "require_labels",
-    "latest_tag",
-    "default_namespace",
-    "operator_installed",
-  ] do |_, args|
-# task "cert_configuration", [
-#     "cert_configuration_title",
-#     "nodeport_not_used",
-#     "secrets_used",
-#     "immutable_configmap",
-#     "require_labels",
-#     "default_namespace"
-#   ] do |_, args|
-  # stdout_score("configuration", "configuration")
-  stdout_score(["configuration", "cert"], "configuration")
+task "cert_configuration" do |t, args|
+  puts "Configuration Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
+
+  essential_only = args.raw.includes? "essential"
+  tags = ["configuration", "cert"]
+  tags << "essential" if essential_only
+
+  tasks = CNFManager::Points.tasks_by_tag_intersection(tags)
+  tasks.each do |task|
+    t.invoke(task)
+  end
+
+  stdout_score(tags, "configuration")
   case "#{ARGV.join(" ")}" 
   when /cert_configuration/
     stdout_info "Results have been saved to #{CNFManager::Points::Results.file}".colorize(:green)
   end
 end
-
-task "cert_configuration_title" do |_, args|
-  puts "Configuration Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
-end

src/tasks/cert/cert_microservice.cr

@@ -9,16 +9,21 @@ require "halite"
 require "totem"
 
 desc "The CNF test suite checks to see if CNFs follows microservice principles"
-task "cert_microservice", ["cert_microservice_title","reasonable_image_size", "reasonable_startup_time", "single_process_type", "service_discovery", "shared_database", "zombie_handled", "sig_term_handled", "specialized_init_system"] do |_, args|
-# task "cert_microservice", ["cert_microservice_title", "reasonable_image_size", "reasonable_startup_time", "service_discovery"] do |_, args|
-  # stdout_score("microservice")
-  stdout_score(["microservice", "cert"], "microservice")
+task "cert_microservice" do |t, args|
+  puts "Microservice Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
+
+  essential_only = args.raw.includes? "essential"
+  tags = ["microservice", "cert"]
+  tags << "essential" if essential_only
+
+  tasks = CNFManager::Points.tasks_by_tag_intersection(tags)
+  tasks.each do |task|
+    t.invoke(task)
+  end
+
+  stdout_score(tags, "microservice")
   case "#{ARGV.join(" ")}" 
   when /cert_microservice/
     stdout_info "Results have been saved to #{CNFManager::Points::Results.file}".colorize(:green)
   end
 end
-
-task "cert_microservice_title" do |_, args|
-  puts "Microservice Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
-end

src/tasks/cert/cert_observability.cr

@@ -6,17 +6,21 @@ require "totem"
 require "../utils/utils.cr"
 
 desc "In order to maintain, debug, and have insight into a protected environment, its infrastructure elements must have the property of being observable. This means these elements must externalize their internal states in some way that lends itself to metrics, tracing, and logging."
-task "cert_observability", ["cert_observability_title", "log_output", "prometheus_traffic", "open_metrics", "routed_logs", "tracing"] do |_, args|
-# task "cert_observability", ["cert_observability_title", "prometheus_traffic", "open_metrics", "routed_logs", "tracing"] do |_, args|
-  # stdout_score("observability", "Observability and Diagnostics")
-  stdout_score(["observability", "cert"], "Observability and Diagnostics")
+task "cert_observability" do |t, args|
+  puts "Observability and Diagnostics Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
+
+  essential_only = args.raw.includes? "essential"
+  tags = ["observability", "cert"]
+  tags << "essential" if essential_only
+
+  tasks = CNFManager::Points.tasks_by_tag_intersection(tags)
+  tasks.each do |task|
+    t.invoke(task)
+  end
+
+  stdout_score(tags, "Observability and Diagnostics")
   case "#{ARGV.join(" ")}" 
   when /cert_observability/
     stdout_info "Results have been saved to #{CNFManager::Points::Results.file}".colorize(:green)
   end
 end
-
-task "cert_observability_title" do |_, args|
-  puts "Observability and Diagnostics Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
-end
-

src/tasks/cert/cert_resilience.cr

@@ -5,30 +5,24 @@ require "colorize"
 require "../utils/utils.cr"
 
 desc "The CNF test suite checks to see if the CNFs are resilient to failures."
- task "cert_resilience", [
-   "cert_resilience_title",
-   "pod_network_latency",
-   "pod_network_corruption",
-   "disk_fill",
-   "pod_delete",
-   "pod_memory_hog",
-   "pod_io_stress",
-   "pod_network_duplication",
-   "liveness",
-   "readiness"
-  ] do |t, args|
-#  task "cert_resilience", ["cert_resilience_title", "pod_network_latency", "pod_network_corruption", "disk_fill", "pod_delete", "pod_memory_hog", "pod_io_stress", "pod_dns_error"] do |t, args|
+ task "cert_resilience" do |t, args|
+  puts "Reliability, Resilience, and Availability Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
+
+  essential_only = args.raw.includes? "essential"
+  tags = ["resilience", "cert"]
+  tags << "essential" if essential_only
+
+  tasks = CNFManager::Points.tasks_by_tag_intersection(tags)
+  tasks.each do |task|
+    t.invoke(task)
+  end
+
   Log.for("verbose").info {  "resilience" } if check_verbose(args)
   VERBOSE_LOGGING.debug "resilience args.raw: #{args.raw}" if check_verbose(args)
   VERBOSE_LOGGING.debug "resilience args.named: #{args.named}" if check_verbose(args)
-  # stdout_score("resilience", "Reliability, Resilience, and Availability")
-  stdout_score(["resilience", "cert"], "Reliability, Resilience, and Availability")
+  stdout_score(tags, "Reliability, Resilience, and Availability")
   case "#{ARGV.join(" ")}" 
   when /cert_resilience/
     stdout_info "Results have been saved to #{CNFManager::Points::Results.file}".colorize(:green)
   end
 end
-
-task "cert_resilience_title" do |_, args|
-  puts "Reliability, Resilience, and Availability Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
-end

src/tasks/cert/cert_security.cr

@@ -6,49 +6,21 @@ require "totem"
 require "../utils/utils.cr"
 
 desc "CNF containers should be isolated from one another and the host.  The CNF Test suite uses tools like Sysdig Inspect and gVisor"
-task "cert_security", [
-   "cert_security_title", 
-    "symlink_file_system",
-    "privilege_escalation",
-    "insecure_capabilities",
-    "cpu_limits",
-    "memory_limits",
-    "linux_hardening",
-    "ingress_egress_blocked",
-    "host_pid_ipc_privileges",
-    "non_root_containers",
-    "privileged_containers",
-    "immutable_file_systems",
-    "hostpath_mounts",
-    "container_sock_mounts",
-    "external_ips",
-    "selinux_options",
-    "sysctls",
-    "host_network",
-    "service_account_mapping",
-    "application_credentials"
-  ] do |_, args|
-# task "cert_security", [
-#   "cert_security_title", 
-#     "symlink_file_system",
-#     "privilege_escalation",
-#     "insecure_capabilities",
-#     "linux_hardening",
-#     "ingress_egress_blocked",
-#     "host_pid_ipc_privileges",
-#     "immutable_file_systems",
-#     "hostpath_mounts",
-#     "external_ips",
-#     "sysctls"
-#   ] do |_, args|
-  # stdout_score("security")
-  stdout_score(["security", "cert"], "security")
+task "cert_security" do |t, args|
+  puts "Security Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
+
+  essential_only = args.raw.includes? "essential"
+  tags = ["security", "cert"]
+  tags << "essential" if essential_only
+
+  tasks = CNFManager::Points.tasks_by_tag_intersection(tags)
+  tasks.each do |task|
+    t.invoke(task)
+  end
+
+  stdout_score(tags, "security")
   case "#{ARGV.join(" ")}" 
   when /cert_security/
     stdout_info "Results have been saved to #{CNFManager::Points::Results.file}".colorize(:green)
   end
 end
-
-task "cert_security_title" do |_, args|
-  puts "Security Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
-end

src/tasks/cert/cert_state.cr

@@ -7,17 +7,21 @@ require "../utils/utils.cr"
 require "kubectl_client"
 
 desc "The CNF test suite checks if state is stored in a custom resource definition or a separate database (e.g. etcd) rather than requiring local storage.  It also checks to see if state is resilient to node failure"
-task "cert_state", ["cert_state_title", "no_local_volume_configuration", "elastic_volumes", "node_drain", "volume_hostpath_not_found"] do |_, args|
-# task "cert_state", ["cert_state_title", "no_local_volume_configuration", "elastic_volumes" ] do |_, args|
-  # stdout_score("state")
-  stdout_score(["state", "cert"], "state")
+task "cert_state" do |t, args|
+  puts "State Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
+
+  essential_only = args.raw.includes? "essential"
+  tags = ["state", "cert"]
+  tags << "essential" if essential_only
+
+  tasks = CNFManager::Points.tasks_by_tag_intersection(tags)
+  tasks.each do |task|
+    t.invoke(task)
+  end
+
+  stdout_score(tags, "state")
   case "#{ARGV.join(" ")}" 
   when /cert_state/
     stdout_info "Results have been saved to #{CNFManager::Points::Results.file}".colorize(:green)
   end
 end
-
-task "cert_state_title" do |_, args|
-  puts "State Tests".colorize(Colorize::ColorRGB.new(0, 255, 255))
-end
-

src/tasks/utils/points.cr

@@ -183,6 +183,16 @@ module CNFManager
       end
     end
 
+    def self.tasks_by_tag_intersection(tags)
+      tasks = tags.reduce([] of String) do |acc, t| 
+        if acc.empty?
+          acc = tasks_by_tag(t)
+        else
+          acc = acc & tasks_by_tag(t)
+        end
+      end
+    end
+
     # Gets the total assigned points for a tag (or all total points) from the results file
     # Usesful for calculation categories total
     def self.total_points(tag=nil)
@@ -191,13 +201,7 @@ module CNFManager
 
     def self.total_points(tags : Array(String) = [] of String)
       if !tags.empty?
-        tasks = tags.reduce([] of String) do |acc, t| 
-          if acc.empty?
-            acc = tasks_by_tag(t)
-          else
-            acc = acc & tasks_by_tag(t)
-          end
-        end
+        tasks = tasks_by_tag_intersection(tags)
       else
         tasks = all_task_test_names
       end
@@ -225,13 +229,7 @@ module CNFManager
     def self.total_passed(tags : Array(String) = [] of String)
       Log.debug { "total_passed: #{tags}" }
       if !tags.empty?
-        tasks = tags.reduce([] of String) do |acc, t| 
-          if acc.empty?
-            acc = tasks_by_tag(t)
-          else
-            acc = acc & tasks_by_tag(t)
-          end
-        end
+        tasks = tasks_by_tag_intersection(tags)
       else
         tasks = all_task_test_names
       end
@@ -276,13 +274,7 @@ module CNFManager
     def self.total_max_points(tags : Array(String) = [] of String)
       Log.debug { "total_max_points tag: #{tags}" }
       if !tags.empty?
-        tasks = tags.reduce([] of String) do |acc, t| 
-          if acc.empty?
-            acc = tasks_by_tag(t)
-          else
-            acc = acc & tasks_by_tag(t)
-          end
-        end
+        tasks = tasks_by_tag_intersection(tags)
       else
         tasks = all_task_test_names
       end
@@ -349,14 +341,7 @@ module CNFManager
     def self.total_max_passed(tags : Array(String) = [] of String)
       Log.debug { "total_max_passed tag: #{tags}" }
       if !tags.empty?
-        tasks = tags.reduce([] of String) do |acc, t| 
-          Log.info { "total_max_passed acc: #{acc}" }
-          if acc.empty?
-            acc = tasks_by_tag(t)
-          else
-            acc = acc & tasks_by_tag(t)
-          end
-        end
+        tasks = tasks_by_tag_intersection(tags)
       else
         tasks = all_task_test_names
       end