Add a kubernetes-render task

README.md

@@ -50,6 +50,10 @@ This repo also includes related tools for [running tasks](#kubernetes-run) and [
 * [Prerequisites](#prerequisites-1)
 * [Usage](#usage-2)
 
+**KUBERNETES-RENDER**
+* [Prerequisites](#prerequisites-2)
+* [Usage](#usage-3)
+
 **DEVELOPMENT**
 * [Setup](#setup)
 * [Running the test suite locally](#running-the-test-suite-locally)
@@ -371,6 +375,34 @@ Based on this specification `kubernetes-run` will create a new pod with the entr
 
 
 
+# kubernetes-render
+
+`kubernetes-render` is a tool for rendering ERB templates to raw Kubernetes YAML. It's useful for seeing what `kubernetes-deploy` does before actually invoking `kubectl` on the rendered YAML. It's also useful for outputting YAML that can be passed to other tools, for validation or introspection purposes.
+
+
+## Prerequisites
+
+ * `kubernetes-render` does __not__ require a running cluster or an active kubernetes context, which is nice if you want to run it in a CI environment, potentially alongside something like https://github.com/garethr/kubeval to make sure your configuration is sound.
+ * Like the other `kubernetes-deploy` commands, `kubernetes-render` requires the `$REVISION` environment variable to be set, and will make it available as `current_sha` in your ERB templates.
+
+## Usage
+
+To render all templates in your template dir, run:
+
+```
+kubernetes-render --template-dir=./path/to/template/dir
+```
+
+To render some templates in a template dir, run kubernetes-render with the names of the templates to render:
+
+```
+kubernetes-render --template-dir=./path/to/template/dir this-template.yaml.erb that-template.yaml.erb
+```
+
+*Options:*
+
+- `--template-dir=DIR`: Used to set the directory to interpret template names relative to. This is often the same directory passed as `--template-dir` when running `kubernetes-deploy` to actually deploy templates. Set `$ENVIRONMENT` instead to use `config/deploy/$ENVIRONMENT`.
+- `--bindings=BINDINGS`: Makes additional variables available to your ERB templates. For example, `kubernetes-render --bindings=color=blue,size=large some-template.yaml.erb` will expose `color` and `size` to `some-template.yaml.erb`.
 
 
 # Development

exe/kubernetes-deploy

@@ -40,23 +40,10 @@ ARGV.options do |opts|
   opts.parse!
 end
 
-if !template_dir && ENV.key?("ENVIRONMENT")
-  template_dir = "config/deploy/#{ENV['ENVIRONMENT']}"
-end
-
-if !template_dir || template_dir.empty?
-  puts "Template directory is unknown. " \
-    "Either specify --template-dir argument or set $ENVIRONMENT to use config/deploy/$ENVIRONMENT as a default path."
-  exit 1
-end
-
-revision = ENV.fetch('REVISION') do
-  puts "ENV['REVISION'] is missing. Please specify the commit SHA"
-  exit 1
-end
-
 namespace = ARGV[0]
 context = ARGV[1]
+template_dir = KubernetesDeploy::OptionsHelper.default_and_check_template_dir(template_dir)
+revision = KubernetesDeploy::OptionsHelper.revision_from_environment
 logger = KubernetesDeploy::FormattedLogger.build(namespace, context, verbose_prefix: verbose_log_prefix)
 
 runner = KubernetesDeploy::DeployTask.new(

exe/kubernetes-render

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'kubernetes-deploy'
+require 'kubernetes-deploy/render_task'
+require 'optparse'
+
+template_dir = nil
+bindings = {}
+
+ARGV.options do |opts|
+  opts.on("--bindings=BINDINGS", "Expose additional variables to ERB templates " \
+    "(format: k1=v1,k2=v2, JSON string or file (JSON or YAML) path prefixed by '@')") do |binds|
+    bindings.merge!(KubernetesDeploy::BindingsParser.parse(binds))
+  end
+  opts.on("--template-dir=DIR", "Set the template dir (default: config/deploy/$ENVIRONMENT)") { |v| template_dir = v }
+  opts.parse!
+end
+
+templates = ARGV
+logger = KubernetesDeploy::FormattedLogger.build(verbose_prefix: false)
+revision = KubernetesDeploy::OptionsHelper.revision_from_environment
+
+runner = KubernetesDeploy::RenderTask.new(
+  logger: logger,
+  current_sha: revision,
+  template_dir: template_dir,
+  bindings: bindings,
+)
+
+success = runner.run(STDOUT, templates)
+exit 1 unless success

lib/kubernetes-deploy.rb

@@ -14,6 +14,7 @@
 require 'kubernetes-deploy/oj'
 require 'kubernetes-deploy/errors'
 require 'kubernetes-deploy/formatted_logger'
+require 'kubernetes-deploy/options_helper'
 require 'kubernetes-deploy/statsd'
 require 'kubernetes-deploy/deploy_task'
 require 'kubernetes-deploy/concurrency'

lib/kubernetes-deploy/resource_discovery.rb → lib/kubernetes-deploy/cluster_resource_discovery.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module KubernetesDeploy
-  class ResourceDiscovery
+  class ClusterResourceDiscovery
     def initialize(namespace:, context:, logger:, namespace_tags:)
       @namespace = namespace
       @context = context

lib/kubernetes-deploy/deploy_task.rb

@@ -38,7 +38,8 @@
 require 'kubernetes-deploy/kubeclient_builder'
 require 'kubernetes-deploy/ejson_secret_provisioner'
 require 'kubernetes-deploy/renderer'
-require 'kubernetes-deploy/resource_discovery'
+require 'kubernetes-deploy/cluster_resource_discovery'
+require 'kubernetes-deploy/template_discovery'
 
 module KubernetesDeploy
   class DeployTask
@@ -184,7 +185,7 @@ def run!(verify_result: true, allow_protected_ns: false, prune: true)
     private
 
     def cluster_resource_discoverer
-      @cluster_resource_discoverer ||= ResourceDiscovery.new(
+      @cluster_resource_discoverer ||= ClusterResourceDiscovery.new(
         namespace: @namespace,
         context: @context,
         logger: @logger,
@@ -255,9 +256,7 @@ def discover_resources
       resources = []
       @logger.info("Discovering templates:")
 
-      Dir.foreach(@template_dir) do |filename|
-        next unless filename.end_with?(".yml.erb", ".yml", ".yaml", ".yaml.erb")
-
+      TemplateDiscovery.new(@template_dir).templates.each do |filename|
         split_templates(filename) do |r_def|
           r = KubernetesResource.build(namespace: @namespace, context: @context, logger: @logger,
                                        definition: r_def, statsd_tags: @namespace_tags)
@@ -276,7 +275,7 @@ def discover_resources
     def split_templates(filename)
       file_content = File.read(File.join(@template_dir, filename))
       rendered_content = @renderer.render_template(filename, file_content)
-      YAML.load_stream(rendered_content) do |doc|
+      YAML.load_stream(rendered_content, "<rendered> #{filename}") do |doc|
         next if doc.blank?
         unless doc.is_a?(Hash)
           raise InvalidTemplateError.new("Template is not a valid Kubernetes manifest",

lib/kubernetes-deploy/errors.rb

@@ -3,6 +3,7 @@ module KubernetesDeploy
   class FatalDeploymentError < StandardError; end
   class FatalKubeAPIError < FatalDeploymentError; end
   class KubectlError < StandardError; end
+  class TaskConfigurationError < FatalDeploymentError; end
 
   class InvalidTemplateError < FatalDeploymentError
     attr_reader :content

lib/kubernetes-deploy/formatted_logger.rb

@@ -6,12 +6,22 @@ module KubernetesDeploy
   class FormattedLogger < Logger
     include DeferredSummaryLogging
 
-    def self.build(namespace, context, stream = $stderr, verbose_prefix: false)
+    def self.build(namespace = nil, context = nil, stream = $stderr, verbose_prefix: false)
       l = new(stream)
       l.level = level_from_env
 
+      middle = if verbose_prefix
+        if namespace.blank?
+          raise ArgumentError, 'Must pass a namespace if logging verbosely'
+        end
+        if context.blank?
+          raise ArgumentError, 'Must pass a context if logging verbosely'
+        end
+
+        "[#{context}][#{namespace}]"
+      end
+
       l.formatter = proc do |severity, datetime, _progname, msg|
-        middle = verbose_prefix ? "[#{context}][#{namespace}]" : ""
         colorized_line = ColorizedString.new("[#{severity}][#{datetime}]#{middle}\t#{msg}\n")
 
         case severity

lib/kubernetes-deploy/options_helper.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module KubernetesDeploy
+  module OptionsHelper
+    def self.default_and_check_template_dir(template_dir)
+      if !template_dir && ENV.key?("ENVIRONMENT")
+        template_dir = "config/deploy/#{ENV['ENVIRONMENT']}"
+      end
+
+      if !template_dir || template_dir.empty?
+        puts "Template directory is unknown. " \
+          "Either specify --template-dir argument or set $ENVIRONMENT to use config/deploy/$ENVIRONMENT " \
+        + "as a default path."
+        exit 1
+      end
+
+      template_dir
+    end
+
+    def self.revision_from_environment
+      ENV.fetch('REVISION') do
+        puts "ENV['REVISION'] is missing. Please specify the commit SHA"
+        exit 1
+      end
+    end
+  end
+end

lib/kubernetes-deploy/render_task.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+require 'tempfile'
+
+require 'kubernetes-deploy/renderer'
+require 'kubernetes-deploy/template_discovery'
+
+module KubernetesDeploy
+  class RenderTask
+    def initialize(logger:, current_sha:, template_dir:, bindings:)
+      @logger = logger
+      @template_dir = template_dir
+      @renderer = KubernetesDeploy::Renderer.new(
+        current_sha: current_sha,
+        bindings: bindings,
+        template_dir: @template_dir,
+        logger: @logger,
+      )
+    end
+
+    def run(*args)
+      run!(*args)
+      true
+    rescue KubernetesDeploy::FatalDeploymentError
+      false
+    end
+
+    def run!(stream, only_filenames = [])
+      @logger.reset
+      @logger.phase_heading("Initializing render task")
+
+      filenames = if only_filenames.empty?
+        TemplateDiscovery.new(@template_dir).templates
+      else
+        only_filenames
+      end
+
+      validate_configuration(filenames)
+      render_filenames(stream, filenames)
+
+      @logger.summary.add_action("Successfully rendered #{filenames.size} template(s)")
+      @logger.print_summary(:success)
+    rescue KubernetesDeploy::FatalDeploymentError
+      @logger.print_summary(:failure)
+      raise
+    end
+
+    private
+
+    def render_filenames(stream, filenames)
+      exceptions = []
+      @logger.phase_heading("Rendering template(s)")
+
+      filenames.each do |filename|
+        begin
+          render_filename(filename, stream)
+        rescue KubernetesDeploy::InvalidTemplateError => exception
+          exceptions << exception
+          log_invalid_template(exception)
+        end
+      end
+
+      unless exceptions.empty?
+        raise exceptions[0]
+      end
+    end
+
+    def render_filename(filename, stream)
+      @logger.info("Rendering #{File.basename(filename)} ...")
+      file_content = File.read(File.join(@template_dir, filename))
+      rendered_content = @renderer.render_template(filename, file_content)
+      YAML.load_stream(rendered_content, "<rendered> #{filename}") do |doc|
+        stream.puts YAML.dump(doc)
+      end
+      @logger.info("Rendered #{File.basename(filename)}")
+    rescue Psych::SyntaxError => exception
+      raise InvalidTemplateError.new("Template is not valid YAML. #{exception.message}", filename: filename)
+    end
+
+    def validate_configuration(filenames)
+      @logger.info("Validating configuration")
+      errors = []
+
+      if filenames.empty?
+        errors << "no templates found in template dir #{@template_dir}"
+      end
+
+      absolute_template_dir = File.expand_path(@template_dir)
+
+      filenames.each do |filename|
+        absolute_file = File.expand_path(File.join(@template_dir, filename))
+        if !File.exist?(absolute_file)
+          errors << "Filename \"#{absolute_file}\" could not be found"
+        elsif !File.file?(absolute_file)
+          errors << "Filename \"#{absolute_file}\" is not a file"
+        elsif !absolute_file.start_with?(absolute_template_dir)
+          errors << "Filename \"#{absolute_file}\" is outside the template directory," \
+          " which was resolved as #{absolute_template_dir}"
+        end
+      end
+
+      unless errors.empty?
+        @logger.summary.add_action("Configuration invalid")
+        @logger.summary.add_paragraph(errors.map { |err| "- #{err}" }.join("\n"))
+        raise KubernetesDeploy::TaskConfigurationError, "Configuration invalid: #{errors.join(', ')}"
+      end
+    end
+
+    def log_invalid_template(exception)
+      @logger.error("Failed to render #{exception.filename}")
+
+      debug_msg = ColorizedString.new("Invalid template: #{exception.filename}\n").red
+      debug_msg += "> Error message: #{exception}"
+      if exception.content
+        debug_msg += "> Template content:\n#{exception.content}"
+      end
+      @logger.summary.add_paragraph(debug_msg)
+    end
+  end
+end

lib/kubernetes-deploy/renderer.rb

@@ -52,7 +52,7 @@ def render_partial(partial, locals)
       template = File.read(partial_path)
       expanded_template = ERB.new(template, nil, '-').result(erb_binding)
 
-      docs = Psych.parse_stream(expanded_template)
+      docs = Psych.parse_stream(expanded_template, partial_path)
       # If the partial contains multiple documents or has an explicit document header,
       # we know it cannot validly be indented in the parent, so return it immediately.
       return expanded_template unless docs.children.one? && docs.children.first.implicit

lib/kubernetes-deploy/runner_task.rb

@@ -8,7 +8,6 @@ module KubernetesDeploy
   class RunnerTask
     include KubeclientBuilder
 
-    class TaskConfigurationError < FatalDeploymentError; end
     class TaskTemplateMissingError < TaskConfigurationError; end
 
     attr_reader :pod_name

lib/kubernetes-deploy/template_discovery.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module KubernetesDeploy
+  class TemplateDiscovery
+    def initialize(template_dir)
+      @template_dir = template_dir
+    end
+
+    def templates
+      Dir.foreach(@template_dir).select do |filename|
+        filename.end_with?(".yml.erb", ".yml", ".yaml", ".yaml.erb")
+      end
+    end
+  end
+end

test/fixtures/invalid/raise_inside.yml.erb

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hello-cloud-raise-data
+  labels:
+    name: hello-cloud-raise-data
+    app: hello-cloud
+data:
+  datapoint1: <% raise RuntimeError, "mock error when evaluating erb" %>

test/fixtures/some-invalid/configmap-data.yml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hello-cloud-configmap-data
+  labels:
+    name: hello-cloud-configmap-data
+    app: hello-cloud
+data:
+  datapoint1: value1
+  datapoint2: value2