diff --git a/examples/aws-custom-master-key.nix b/examples/aws-custom-master-key.nix new file mode 100644 index 00000000..f46d18d6 --- /dev/null +++ b/examples/aws-custom-master-key.nix @@ -0,0 +1,36 @@ +{ + resources.awsCustomerMasterKey.cmk = + {lib, ...}: + { + alias = "nixops-kms"; + description = "nixops is the best"; + policy = builtins.toJSON + { + Statement= [ + { + Effect= "Allow"; + Principal = "*"; + Action = "*"; + Resource= "*"; + } + ]; + }; + origin = "AWS_KMS"; + deletionWaitPeriod = 7; + region = "us-east-1"; + accessKeyId = "testing"; + tags = { name = "nixops-managed-cmk";}; + }; + resources.ebsVolumes.ebs = + {resources, ...}: + { + region = "us-east-1"; + accessKeyId = "testing"; + size = 50; + volumeType = "gp2"; + kmsKeyId = resources.awsCustomerMasterKey.cmk; + zone = "us-east-1a"; + tags = { name = "nixops"; env = "test";}; + }; + +} \ No newline at end of file diff --git a/nix/aws-customer-master-key.nix b/nix/aws-customer-master-key.nix new file mode 100644 index 00000000..3a41c552 --- /dev/null +++ b/nix/aws-customer-master-key.nix @@ -0,0 +1,71 @@ +{ config, lib, uuid, name, ... }: + +with lib; + +{ + imports = [ ./common-ec2-auth-options.nix ]; + + options = { + + alias = mkOption { + default = "nixops-${uuid}-${name}"; + type = types.str; + description = "Alias of the CMK."; + }; + + keyId = mkOption { + default = ""; + type = types.str; + description = "The globally unique identifier for the CMK. This is set by NixOps"; + }; + + policy = mkOption { + default = null; + type = types.nullOr types.str; + description = '' + The key policy to attach to the CMK. + ''; + }; + + description = mkOption { + default = "CMK created by nixops"; + type = types.str; + description = "A description of the CMK."; + }; + + origin = mkOption { + default = "AWS_KMS"; + type = types.enum [ "AWS_KMS" "AWS_CLOUDHSM" ]; + description = '' + The source of the key material for the CMK. + You cannot change the origin after you create the CMK. + There is also an EXTERNAL option but it is not supported + in nixops yet. + ''; + }; + + customKeyStoreId = mkOption { + default = null; + type = types.nullOr types.str; + description = '' + Creates the CMK in the specified custom key store and the key + material in its associated AWS CloudHSM cluster. To create a CMK + in a custom key store, you must also specify the Origin parameter + with a value of "AWS_CLOUDHSM" . + ''; + }; + + deletionWaitPeriod = mkOption { + default = 30; + type = types.int; + description = '' + The waiting period, specified in number of days. After + the waiting period ends, AWS KMS deletes the customer master key (CMK). + Valid values are between 7 and 30. + ''; + }; + + } // import ./common-ec2-options.nix { inherit lib; }; + + config._type = "aws-customer-master-key"; +} diff --git a/nix/default.nix b/nix/default.nix index fd71ca4f..5f209591 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -44,6 +44,7 @@ awsVPNGateways = evalResources ./aws-vpn-gateway.nix (zipAttrs resourcesByType.awsVPNGateways or []); awsVPNConnections = evalResources ./aws-vpn-connection.nix (zipAttrs resourcesByType.awsVPNConnections or []); awsVPNConnectionRoutes = evalResources ./aws-vpn-connection-route.nix (zipAttrs resourcesByType.awsVPNConnectionRoutes or []); + awsCustomerMasterKey = evalResources ./aws-customer-master-key.nix (zipAttrs resourcesByType.awsCustomerMasterKey or []); }; } diff --git a/nix/ebs-volume.nix b/nix/ebs-volume.nix index b816b887..08d623fd 100644 --- a/nix/ebs-volume.nix +++ b/nix/ebs-volume.nix @@ -1,6 +1,7 @@ { config, lib, uuid, name, ... }: with lib; +with import ./lib.nix lib; { @@ -49,6 +50,17 @@ with lib; ''; }; + kmsKeyId = mkOption { + default = null; + type = with types; nullOr (either types.str (resource "aws-customer-master-key")); + apply = x: if builtins.isString x then x else "res-" + x._name; + description = '' + The identifier of the AWS Key Management Service (AWS KMS) + customer master key (CMK) to use for Amazon EBS encryption. + If this parameter is not specified, your AWS managed CMK for EBS is used. + ''; + }; + } // import ./common-ec2-options.nix { inherit lib; }; config = { diff --git a/nixopsaws/resources/__init__.py b/nixopsaws/resources/__init__.py index 20773100..39264d20 100644 --- a/nixopsaws/resources/__init__.py +++ b/nixopsaws/resources/__init__.py @@ -36,3 +36,4 @@ import vpc_route_table import vpc_route_table_association import vpc_subnet +import aws_customer_master_key diff --git a/nixopsaws/resources/aws_customer_master_key.py b/nixopsaws/resources/aws_customer_master_key.py new file mode 100644 index 00000000..11889226 --- /dev/null +++ b/nixopsaws/resources/aws_customer_master_key.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- + +# Automatic provisioning of AWS CMK. +import os +import boto3 +import botocore + +import nixops.util +import nixops.resources +from nixopsaws.resources.ec2_common import EC2CommonState +import nixopsaws.ec2_utils +from nixops.state import StateDict +from nixops.diff import Diff, Handler + +class awsCustomerMasterKeyDefinition(nixops.resources.ResourceDefinition): + """Definition of an aws customer master key.""" + + @classmethod + def get_type(cls): + return "aws-customer-master-key" + + @classmethod + def get_resource_type(cls): + return "awsCustomerMasterKey" + + def show_type(self): + return "{0}".format(self.get_type()) + + +class awsCustomerMasterKeyState(nixops.resources.DiffEngineResourceState, EC2CommonState): + """State of a CMK.""" + + state = nixops.util.attr_property("state", nixops.resources.ResourceState.MISSING, int) + access_key_id = nixops.util.attr_property("accessKeyId", None) + _reserved_keys = EC2CommonState.COMMON_EC2_RESERVED + ["keyId"] + + @classmethod + def get_type(cls): + return "aws-customer-master-key" + + def __init__(self, depl, name, id): + nixops.resources.DiffEngineResourceState.__init__(self, depl, name, id) + self.keyId = self._state.get('keyId', None) + self.handle_create = Handler(['deletionWaitPeriod', 'origin', 'region', 'customKeyStoreId'], + handle=self.realize_create_cmk) + self.handle_description = Handler(['description'], after=[self.handle_create], + handle=self.realize_update_description) + self.handle_policy = Handler(['policy'], after=[self.handle_create], handle=self.realize_policy) + self.handle_alias = Handler(['alias'], after=[self.handle_create], handle=self.realize_update_alias) + self.handle_tag_update = Handler(['tags'], after=[self.handle_create], handle=self.realize_update_tag) + + def show_type(self): + s = super(awsCustomerMasterKeyState, self).show_type() + region = self._state.get('region', None) + if region: s = "{0} [{1}]".format(s, region) + return s + + @property + def resource_id(self): + return self._state.get('keyId', None) + + def prefix_definition(self, attr): + return {('resources', 'awsCustomerMasterKey'): attr} + + def get_physical_spec(self): + return { 'cmkId': self._state.get('keyId', None)} + + def get_definition_prefix(self): + return "resources.awsCustomerMasterKey." + + def realize_create_cmk(self, allow_recreate): + """Handle both create and recreate of the aws customer master key resource """ + config = self.get_defn() + if self.state == self.UP: + if not allow_recreate: + raise Exception("aws customer master key {} definition changed and it needs to be recreated " + "use --allow-recreate if you want to create a new one".format(self.keyId)) + self.warn("aws customer master key definition changed, recreating...") + self._destroy() + self._client = None + + self._state["region"] = config['region'] + + self.log("creating aws customer master key under region {0}".format(config['region'])) + args = dict( + KeyUsage='ENCRYPT_DECRYPT', + Origin = config['origin'], + ) + if config['customKeyStoreId']: + args['CustomKeyStoreId'] = config['customKeyStoreId'] + cmk = self.get_client(service="kms").create_key(**args) + self.keyId = cmk['KeyMetadata']['KeyId'] + + with self.depl._db: + self.state = self.UP + self._state["keyId"] = self.keyId + self._state["region"] = config['region'] + self._state["origin"] = config['origin'] + self._state["deletionWaitPeriod"] = config['deletionWaitPeriod'] + + def realize_update_description(self, allow_recreate): + config = self.get_defn() + self.get_client(service="kms").update_key_description(KeyId=self.keyId, Description=config['description']) + + with self.depl._db: + self._state['description'] = config['description'] + + def realize_policy(self, allow_recreate): + config = self.get_defn() + self.log("updating `{0}` policy...".format(self.keyId)) + self.get_client(service="kms").put_key_policy(KeyId=self.keyId, PolicyName="default", Policy=config['policy']) + + with self.depl._db: + self._state['policy'] = config['policy'] + + def realize_update_tag(self, allow_recreate): + config = self.get_defn() + tags = config['tags'] + tags.update(self.get_common_tags()) + self.get_client(service="kms").tag_resource(KeyId=self.keyId, Tags=[{"TagKey": k, "TagValue": tags[k]} for k in tags]) + + def realize_update_alias(self, allow_recreate): + config = self.get_defn() + # we don't want to have many alias for a key so we delete the old one before creating a new + if self._state.get('subnetId', None): + self.get_client(service="kms").delete_alias(AliasName="alias/" + self._state['alias']) + self.log("updating `{0}` alias...".format(self.keyId)) + else: + self.log("creating alias for `{0}`...".format(self.keyId)) + self.get_client(service="kms").create_alias(TargetKeyId=self.keyId, AliasName="alias/" + config['alias']) + + with self.depl._db: + self._state['alias'] = config['alias'] + + def _check(self): + if self._state.get('keyId', None) is None: + return + try: + cmk = self.get_client(service="kms").describe_key(KeyId=self._state["keyId"]) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'InvalidKmsID.NotFound': + self.warn("aws customer master key {0} was deleted from outside nixops," + " it needs to be recreated...".format(self._state["keyId"])) + self.cleanup_state() + return + cmk_state = cmk['KeyMetadata']['KeyState'] + if cmk_state == "Enabled": + return + elif cmk_state == "Disabled": + raise Exception("aws customer master key state is {1}, Enable it form the console".format(cmk_state)) + elif cmk_state == "PendingDeletion": + raise Exception("aws customer master key state is {1}, run a destroy operation to sync the state or cancel the deletion.".format(cmk_state)) + else: + raise Exception("aws customer master key state is {1}".format(cmk_state)) + + def _destroy(self): + if self.state != self.UP: return + + self.get_client(service="kms").delete_alias(AliasName="alias/" + self._state['alias']) + self.log("scheduling aws customer master key `{0}` deletion to {1} day(s)..." + .format(self._state['alias'], self._state['deletionWaitPeriod'])) + try: + self.get_client(service="kms").schedule_key_deletion( + KeyId=self._state['keyId'], + PendingWindowInDays=self._state['deletionWaitPeriod']) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'InvalidCmkID.NotFound': + self.warn("aws customer master key {0} was already deleted".format(self._state['keyId'])) + else: + raise e + self.cleanup_state() + + def cleanup_state(self): + with self.depl._db: + self.state = self.MISSING + self._state['keyId'] = None + self._state['region'] = None + self._state['alias'] = None + self._state['policy'] = None + self._state['description'] = None + self._state['origin'] = None + self._state['customKeyStoreId'] = None + self._state['deletionWaitPeriod'] = None + + def destroy(self, wipe=False): + self._destroy() + return True \ No newline at end of file diff --git a/nixopsaws/resources/ebs_volume.py b/nixopsaws/resources/ebs_volume.py index f0ec57e5..86cb1462 100644 --- a/nixopsaws/resources/ebs_volume.py +++ b/nixopsaws/resources/ebs_volume.py @@ -37,6 +37,7 @@ class EBSVolumeState(nixops.resources.ResourceState, ec2_common.EC2CommonState): size = nixops.util.attr_property("ec2.size", None, int) iops = nixops.util.attr_property("ec2.iops", None, int) volume_type = nixops.util.attr_property("ec2.volumeType", None) + kms_key_id = nixops.util.attr_property("ec2.kmsKeyId", None) @classmethod @@ -70,6 +71,10 @@ def connect(self, region): self._conn = nixopsaws.ec2_utils.connect(region, self.access_key_id) return self._conn + def create_after(self, resources, defn): + return {r for r in resources if + isinstance(r, nixopsaws.resources.aws_customer_master_key.awsCustomerMasterKeyState)} + def connect_boto3(self, region): if self._conn_boto3: return self._conn_boto3 self._conn_boto3 = nixopsaws.ec2_utils.connect_ec2_boto3(region, self.access_key_id) @@ -102,6 +107,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): if not self.access_key_id: raise Exception("please set ‘accessKeyId’, $EC2_ACCESS_KEY or $AWS_ACCESS_KEY_ID") + self.connect_boto3(defn.config['region']) self.connect(defn.config['region']) if self._exists(): @@ -123,9 +129,9 @@ def create(self, defn, check, allow_reboot, allow_recreate): self._get_vol(defn.config) else: if defn.config['size'] == 0 and defn.config['snapshot'] != "": - snapshots = self._conn.get_all_snapshots(snapshot_ids=[defn.config['snapshot']]) + snapshots = self._conn_boto3.describe_snapshots(SnapshotIds=[defn.config['snapshot']]) assert len(snapshots) == 1 - defn.config['size'] = snapshots[0].volume_size + defn.config['size'] = snapshots[0]['VolumeSize'] if defn.config['snapshot']: self.log("creating EBS volume of {0} GiB from snapshot ‘{1}’...".format(defn.config['size'], defn.config['snapshot'])) @@ -135,9 +141,23 @@ def create(self, defn, check, allow_reboot, allow_recreate): if defn.config['zone'] is None: raise Exception("please set a zone where the volume will be created") - volume = self._conn.create_volume( - zone=defn.config['zone'], size=defn.config['size'], snapshot=defn.config['snapshot'], - iops=defn.config['iops'], volume_type=defn.config['volumeType']) + args = dict( + AvailabilityZone=defn.config['zone'], + Size=defn.config['size'], + SnapshotId=defn.config['snapshot'], + VolumeType=defn.config['volumeType'] + ) + if defn.config['iops']: + args['Iops'] = defn.config['iops'] + if defn.config['kmsKeyId']: + if defn.config['kmsKeyId'].startswith("res-"): + res = self.depl.get_typed_resource(defn.config['kmsKeyId'][4:].split(".")[0], "aws-customer-master-key") + defn.config['kmsKeyId'] = res.keyId + args['Encrypted']=True + args['KmsKeyId']=defn.config['kmsKeyId'] + + volume = self._conn_boto3.create_volume(**args) + # FIXME: if we crash before the next step, we forget the # volume we just created. Doesn't seem to be anything we # can do about this. @@ -147,7 +167,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.region = defn.config['region'] self.zone = defn.config['zone'] self.size = defn.config['size'] - self.volume_id = volume.id + self.volume_id = volume['VolumeId'] self.iops = defn.config['iops'] self.volume_type = defn.config['volumeType'] @@ -171,10 +191,15 @@ def destroy(self, wipe=False): if wipe: log.warn("wipe is not supported") - self.connect(self.region) - volume = nixopsaws.ec2_utils.get_volume_by_id(self._conn, self.volume_id, allow_missing=True) - if not volume: return True + self.connect_boto3(self.region) + + try: + volume = self._conn_boto3.describe_volumes(VolumeIds=[self.volume_id])['Volumes'][0]['State'] + except botocore.exceptions.ClientError as error: + if error.response['Error']['Code'] == "InvalidVolume.NotFound": + return True + if volume == "deleted": return True if not self.depl.logger.confirm("are you sure you want to destroy EBS volume ‘{0}’?".format(self.name)): return False self.log("destroying EBS volume ‘{0}’...".format(self.volume_id)) - volume.delete() + self._conn_boto3.delete_volume(VolumeId=self.volume_id) return True