add jinja2 template parsing (cloudtools#701)

* add jinja2 template parsing

* fix use with non-blueprint variables and lookups

docs/index.rst

@@ -56,6 +56,7 @@ Contents:
    lookups
    commands
    blueprints
+   templates
    API Docs <api/modules>
 
 

docs/templates.rst

@@ -0,0 +1,23 @@
+==========
+Templates
+==========
+
+CloudFormation templates can be provided via python Blueprints_ or JSON/YAML.
+JSON/YAML templates are specified for stacks via the ``template_path`` config
+option (see `Stacks <config.html#stacks>`_).
+
+Jinja2 Templating
+=================
+
+Templates with a ``.j2`` extension will be parsed using `Jinja2 
+<http://jinja.pocoo.org/>`_. The stacker ``context`` and ``mappings`` objects
+and stack ``variables`` objects are available for use in the template:
+
+.. code-block:: yaml
+
+    Description: TestTemplate
+    Resources:
+      Bucket:
+        Type: AWS::S3::Bucket
+        Properties:
+          BucketName: {{ context.environment.foo }}-{{ variables.myparamname }}

setup.py

@@ -16,6 +16,7 @@
     "PyYAML>=3.13b1",
     "awacs>=0.6.0",
     "gitpython>=2.0,<3.0",
+    "jinja2>=2.7,<3.0",
     "schematics>=2.0.1,<2.1.0",
     "formic2",
     "python-dateutil>=2.0,<3.0",

stacker/blueprints/raw.py

@@ -8,6 +8,8 @@
 import os
 import sys
 
+from jinja2 import Template
+
 from ..util import parse_cloudformation_template
 from ..exceptions import InvalidConfig, UnresolvedVariable
 from .base import Blueprint
@@ -52,16 +54,13 @@ def get_template_params(template):
     return params
 
 
-def resolve_variable(var_name, var_def, provided_variable, blueprint_name):
+def resolve_variable(provided_variable, blueprint_name):
     """Resolve a provided variable value against the variable definition.
 
     This acts as a subset of resolve_variable logic in the base module, leaving
     out everything that doesn't apply to CFN parameters.
 
     Args:
-        var_name (str): The name of the defined variable on a blueprint.
-        var_def (dict): A dictionary representing the defined variables
-            attributes.
         provided_variable (:class:`stacker.variables.Variable`): The variable
             value provided to the blueprint.
         blueprint_name (str): The name of the blueprint that the variable is
@@ -71,8 +70,6 @@ def resolve_variable(var_name, var_def, provided_variable, blueprint_name):
         object: The resolved variable string value.
 
     Raises:
-        MissingVariable: Raised when a variable with no default is not
-            provided a value.
         UnresolvedVariable: Raised when the provided variable is not already
             resolved.
 
@@ -143,20 +140,33 @@ def resolve_variables(self, provided_variables):
         """Resolve the values of the blueprint variables.
 
         This will resolve the values of the template parameters with values
-        from the env file, the config, and any lookups resolved.
+        from the env file, the config, and any lookups resolved. The
+        resolution is run twice, in case the blueprint is jinja2 templated
+        and requires provided variables to render.
 
         Args:
             provided_variables (list of :class:`stacker.variables.Variable`):
                 list of provided variables
 
         """
+        # Pass 1 to set resolved_variables to provided variables
         self.resolved_variables = {}
+        variable_dict = dict((var.name, var) for var in provided_variables)
+        for var_name, _var_def in variable_dict.items():
+            value = resolve_variable(
+                variable_dict.get(var_name),
+                self.name
+            )
+            if value is not None:
+                self.resolved_variables[var_name] = value
+
+        # Pass 2 to render the blueprint and set resolved_variables according
+        # to defined variables
         defined_variables = self.get_parameter_definitions()
+        self.resolved_variables = {}
         variable_dict = dict((var.name, var) for var in provided_variables)
-        for var_name, var_def in defined_variables.items():
+        for var_name, _var_def in defined_variables.items():
             value = resolve_variable(
-                var_name,
-                var_def,
                 variable_dict.get(var_name),
                 self.name
             )
@@ -186,7 +196,16 @@ def rendered(self):
             template_path = get_template_path(self.raw_template_path)
             if template_path:
                 with open(template_path, 'r') as template:
-                    self._rendered = template.read()
+                    if len(os.path.splitext(template_path)) == 2 and (
+                            os.path.splitext(template_path)[1] == '.j2'):
+                        self._rendered = Template(template.read()).render(
+                            context=self.context,
+                            mappings=self.mappings,
+                            name=self.name,
+                            variables=self.resolved_variables
+                        )
+                    else:
+                        self._rendered = template.read()
             else:
                 raise InvalidConfig(
                     'Could not find template %s' % self.raw_template_path

stacker/tests/blueprints/test_raw.py

@@ -12,10 +12,12 @@
 from stacker.blueprints.raw import (
     get_template_params, get_template_path, RawTemplateBlueprint
 )
+from stacker.variables import Variable
 from ..factories import mock_context
 
 RAW_JSON_TEMPLATE_PATH = 'stacker/tests/fixtures/cfn_template.json'
 RAW_YAML_TEMPLATE_PATH = 'stacker/tests/fixtures/cfn_template.yaml'
+RAW_J2_TEMPLATE_PATH = 'stacker/tests/fixtures/cfn_template.json.j2'
 
 
 class TestRawBluePrintHelpers(unittest.TestCase):
@@ -115,6 +117,53 @@ def test_to_json(self):
             expected_json
         )
 
+    def test_j2_to_json(self):
+        """Verify jinja2 template parsing."""
+        expected_json = json.dumps(
+            {
+                "AWSTemplateFormatVersion": "2010-09-09",
+                "Description": "TestTemplate",
+                "Parameters": {
+                    "Param1": {
+                        "Type": "String"
+                    },
+                    "Param2": {
+                        "Default": "default",
+                        "Type": "CommaDelimitedList"
+                    }
+                },
+                "Resources": {
+                    "Dummy": {
+                        "Type": "AWS::CloudFormation::WaitConditionHandle"
+                    }
+                },
+                "Outputs": {
+                    "DummyId": {
+                        "Value": "dummy-bar-param1val-foo-1234"
+                    }
+                }
+            },
+            sort_keys=True,
+            indent=4
+        )
+        blueprint = RawTemplateBlueprint(
+            name="stack1",
+            context=mock_context(
+                extra_config_args={'stacks': [{'name': 'stack1',
+                                               'template_path': 'unused',
+                                               'variables': {
+                                                   'Param1': 'param1val',
+                                                   'bar': 'foo'}}]},
+                environment={'foo': 'bar'}),
+            raw_template_path=RAW_J2_TEMPLATE_PATH
+        )
+        blueprint.resolve_variables([Variable("Param1", "param1val"),
+                                     Variable("bar", "foo")])
+        self.assertEqual(
+            expected_json,
+            blueprint.to_json()
+        )
+
 
 class TestVariables(unittest.TestCase):
     """Test class for blueprint variable methods."""

stacker/tests/factories.py

@@ -32,10 +32,13 @@ def mock_context(namespace="default", extra_config_args=None, **kwargs):
     if extra_config_args:
         config_args.update(extra_config_args)
     config = Config(config_args)
-    environment = kwargs.get("environment", {})
+    if kwargs.get("environment"):
+        return Context(
+            config=config,
+            **kwargs)
     return Context(
         config=config,
-        environment=environment,
+        environment={},
         **kwargs)
 
 

stacker/tests/fixtures/cfn_template.json.j2

@@ -0,0 +1,23 @@
+{
+    "AWSTemplateFormatVersion": "2010-09-09",
+    "Description": "TestTemplate",
+    "Parameters": {
+        "Param1": {
+            "Type": "String"
+        },
+        "Param2": {
+            "Default": "default",
+            "Type": "CommaDelimitedList"
+        }
+    },
+    "Resources": {
+      "Dummy": {
+        "Type": "AWS::CloudFormation::WaitConditionHandle"
+      }
+    },
+    "Outputs": {
+      "DummyId": {
+        "Value": "dummy-{{ context.environment.foo }}-{{ variables.Param1 }}-{{ variables.bar }}-1234"
+      }
+    }
+}