examples/aws: deploy script imports cert contents

examples/aws/README.md

@@ -577,15 +577,17 @@ After deploying Cloud Foundry, you can now deploy Diego.
 
 ### Fill in the Property-Overrides Stub
 
-To generate a manifest for the Diego deployment, replace the properties in
+To generate a manifest for the Diego deployment, replace the properties in the
 `$DEPLOYMENT_DIR/stubs/diego/property-overrides.yml` file that are prefixed with `REPLACE_WITH_`.
 
-Here is a summary of the properties that need to be changed:
+Here is a summary of the properties that must be changed:
 
-- Replace REPLACE_WITH_ACTIVE_KEY_LABEL with any desired key name (such as `key-a`).
-- Replace REPLACE_WITH_A_SECURE_PASSPHRASE with a unique passphrase associated with the active key label.
-- Replace the BBS and etcd certificate placeholders with the contents of the files generated in [Configuring Security](#configuring-security).
-- Replace the SSH-Proxy host key with the [host key generated](#generating-ssh-proxy-host-key) above.
+- Replace `REPLACE_WITH_ACTIVE_KEY_LABEL` with any desired key name (such as `key-a`).
+- Replace `REPLACE_WITH_A_SECURE_PASSPHRASE` with a unique passphrase associated with the active key label.
+
+Component log levels and other deployment properties may also be overridden in this stub file.
+
+This stub file also contains the contents of the BBS, etcd, and SSH-Proxy certificates and keys generated above. If those files are regenerated, the `deploy_aws_environment` script will update the property-overrides stub with their new contents.
 
 ### Edit the Instance-Count-Overrides Stub
 

examples/aws/deploy_aws_environment

@@ -22,6 +22,65 @@ EOF
   exit 1
 }
 
+
+indent() {
+  sed -e 's/^/  /'
+}
+
+indent_contents_of() {
+  indent < "$1"
+}
+
+block() {
+  cat <<-EOF
+$1: |
+$(indent_contents_of "$2")
+EOF
+}
+
+cf_credentials() {
+  cat <<-EOF
+# GENERATED: NO TOUCHING
+---
+cf_credentials:
+  ssh_host_key_fingerprint: "$(cat keypair/ssh-proxy-host-key-fingerprint)"
+  consul:
+$(block ca_cert     certs/consul-certs/server-ca.crt | indent | indent)
+$(block agent_cert  certs/consul-certs/agent.crt     | indent | indent)
+$(block agent_key   certs/consul-certs/agent.key     | indent | indent)
+$(block server_cert certs/consul-certs/server.crt    | indent | indent)
+$(block server_key  certs/consul-certs/server.key    | indent | indent)
+  uaa:
+$(block signing_key      keypair/uaa     | indent | indent)
+$(block verification_key keypair/uaa.pub | indent | indent)
+
+EOF
+}
+
+diego_credentials() {
+  cat <<-EOF
+# GENERATED: NO TOUCHING
+---
+diego_credentials:
+$(block diego_ca           certs/diego-ca.crt             | indent)
+$(block ssh_proxy_host_key keypair/ssh-proxy-host-key.pem | indent)
+  bbs:
+$(block client_cert certs/bbs-certs/client.crt  | indent | indent)
+$(block client_key  certs/bbs-certs/client.key  | indent | indent)
+$(block server_cert certs/bbs-certs/server.crt  | indent | indent)
+$(block server_key  certs/bbs-certs/server.key  | indent | indent)
+  etcd:
+$(block client_cert certs/etcd-certs/client.crt | indent | indent)
+$(block client_key  certs/etcd-certs/client.key | indent | indent)
+$(block server_cert certs/etcd-certs/server.crt | indent | indent)
+$(block server_key  certs/etcd-certs/server.key | indent | indent)
+$(block peer_ca     certs/etcd-peer-ca.crt      | indent | indent)
+$(block peer_cert   certs/etcd-certs/peer.crt   | indent | indent)
+$(block peer_key    certs/etcd-certs/peer.key   | indent | indent)
+EOF
+}
+
+
 if [ "$1" == "create" ]; then
   UPDATE_OR_CREATE=create-stack
 elif [ "$1" == "update" ]; then
@@ -66,7 +125,7 @@ fi
 
 # generate stub to be fed into template for cloudformation
 cat > stubs/infrastructure/certificates.yml <<EOF
-# GENERATED, NO TOUCHING
+# GENERATED: NO TOUCHING
 EOF
 
 aws iam get-server-certificate --server-certificate-name cfrouter \
@@ -95,13 +154,13 @@ fi
 
 # generate AWS resources stub for shared purposes
 cat > stubs/aws-resources.yml <<EOF
-# GENERATED, NO TOUCHING
+# GENERATED: NO TOUCHING
 EOF
 
 boosh resources --name $stack_name >> stubs/aws-resources.yml
 
 cat > deployments/bosh-init/bosh-init.yml <<EOF
-# GENERATED, NO TOUCHING
+# GENERATED: NO TOUCHING
 EOF
 
 spiff merge \
@@ -112,30 +171,28 @@ spiff merge \
   >> deployments/bosh-init/bosh-init.yml
 
 bosh-init deploy deployments/bosh-init/bosh-init.yml
-bosh -n target `cat stubs/aws-resources.yml | grep BoshInit | awk '{ gsub(/"/, "", $NF); print $NF }'`
+bosh -n target $(cat stubs/aws-resources.yml | grep BoshInit | awk '{ gsub(/"/, "", $NF); print $NF }')
 
 # generate director uuid stub for template to create deployment stub
 cat > stubs/director-uuid.yml <<EOF
-# GENERATED, NO TOUCHING
+# GENERATED: NO TOUCHING
 ---
 director_uuid: $(bosh status --uuid | tr -d '\n')
 EOF
 
-touch stubs/cf/domain.yml
-
+# generate stub with deployment base domain
 cat > stubs/cf/domain.yml <<EOF
-# GENERATED, NO TOUCHING
+# GENERATED: NO TOUCHING
 EOF
 
 spiff merge $SCRIPT_DIR/templates/cf/domain.yml \
   $SCRIPT_DIR/templates/cf/domain-internal.yml \
-  stubs/domain.yml >> stubs/cf/domain.yml
+  stubs/domain.yml \
+  >> stubs/cf/domain.yml
 
 # generate deployment stub
-touch stubs/cf/stub.yml
-
 cat > stubs/cf/stub.yml <<EOF
-# GENERATED, NO TOUCHING
+# GENERATED: NO TOUCHING
 EOF
 
 spiff merge \
@@ -144,18 +201,38 @@ spiff merge \
   stubs/aws-resources.yml \
   stubs/cf/domain.yml \
   stubs/director-uuid.yml \
+  <(cf_credentials) \
   >> stubs/cf/stub.yml
 
+# copy CF property stub if not already present
 if [ ! -f stubs/cf/properties.yml ]; then
   cp $SCRIPT_DIR/stubs/cf/properties.yml stubs/cf/properties.yml
 fi
 
 mkdir -p stubs/diego
 
-if [ ! -f stubs/diego/property-overrides.yml ]; then
-  cp $SCRIPT_DIR/stubs/diego/property-overrides.yml stubs/diego/property-overrides.yml
+# generate Diego property-override stub with certs
+if [ -f stubs/diego/property-overrides.yml ]; then
+  # update BBS, etcd certs and keys in existing property-overrides stub
+  temp_property_overrides=$(mktemp stubs/diego/property-overrides.yml.XXXXX)
+
+  spiff merge \
+    stubs/diego/property-overrides.yml \
+    $SCRIPT_DIR/templates/diego/property-overrides-internal.yml \
+    <(diego_credentials) \
+    > "${temp_property_overrides}"
+
+  mv "${temp_property_overrides}" stubs/diego/property-overrides.yml
+else
+  # create new property-overrides stub with default overrides
+  spiff merge \
+    $SCRIPT_DIR/templates/diego/property-overrides.yml \
+    $SCRIPT_DIR/templates/diego/property-overrides-internal.yml \
+    <(diego_credentials) \
+    > stubs/diego/property-overrides.yml
 fi
 
+# generate Diego IaaS-settings stub
 spiff merge \
   $SCRIPT_DIR/../../manifest-generation/misc-templates/aws-iaas-settings.yml \
   $SCRIPT_DIR/templates/diego/iaas-settings-internal.yml \

examples/aws/stubs/cf/properties.yml

@@ -4,26 +4,13 @@ meta:
   environment: REPLACE_WITH_CF_DEPLOYMENT_NAME
 
 properties:
-  app_ssh:
-    host_key_fingerprint: REPLACE_WITH_CONTENTS_OF_(DEPLOYMENT_DIR/keypair/ssh-proxy-host-key-fingerprint)
-
   cc:
     staging_upload_password: REPLACE_WITH_STAGING_UPLOAD_PASSWORD
     staging_upload_user: REPLACE_WITH_STAGING_UPLOAD_USER
     bulk_api_password: REPLACE_WITH_BULK_API_PASSWORD
     db_encryption_key: REPLACE_WITH_DATABASE_ENCRYPTION_KEY
 
   consul:
-    ca_cert: |
-      REPLACE_WITH_CONTENTS_OF_(DEPLOYMENT_DIR/certs/consul-certs/server-ca.crt)
-    server_cert: |
-      REPLACE_WITH_CONTENTS_OF_(DEPLOYMENT_DIR/certs/consul-certs/server.crt)
-    server_key: |
-      REPLACE_WITH_CONTENTS_OF_(DEPLOYMENT_DIR/certs/consul-certs/server.key)
-    agent_cert: |
-      REPLACE_WITH_CONTENTS_OF_(DEPLOYMENT_DIR/certs/consul-certs/agent.crt)
-    agent_key: |
-      REPLACE_WITH_CONTENTS_OF_(DEPLOYMENT_DIR/certs/consul-certs/agent.key)
     encrypt_keys:
       - REPLACE_WITH_CONSUL_ENCRYPTION_KEY
 
@@ -66,11 +53,6 @@ properties:
         secret: REPLACE_WITH_SSH_PROXY_PASSWORD
       tcp_router:
         secret: REPLACE_WITH_TCP_ROUTER_PASSWORD
-    jwt:
-      signing_key: |
-        REPLACE_WITH_CONTENTS_OF_(DEPLOYMENT_DIR/keypair/uaa)
-      verification_key: |
-        REPLACE_WITH_CONTENTS_OF_(DEPLOYMENT_DIR/keypair/uaa.pub)
     scim:
       users:
       - admin|REPLACE_WITH_ADMIN_PASSWORD|scim.write,scim.read,openid,cloud_controller.admin,doppler.firehose

examples/aws/templates/cf/stub-internal.yml

@@ -87,6 +87,9 @@ properties:
       availability_zone2: (( meta.zones.z2 ))
       availability_zone3: (( meta.zones.z3 ))
 
+  app_ssh:
+    host_key_fingerprint: (( cf_credentials.ssh_host_key_fingerprint ))
+
   cc:
     resource_pool:
       resource_directory_key: (( Resources.Bucket.AppResources ))
@@ -100,13 +103,27 @@ properties:
   ccdb:
     address: (( properties.databases.address ))
 
+  uaa:
+    jwt:
+      signing_key: (( cf_credentials.uaa.signing_key ))
+      verification_key: (( cf_credentials.uaa.verification_key ))
+
   uaadb:
     address: (( properties.databases.address ))
 
   databases:
     address: (( jobs.postgres_z1.networks.cf1.static_ips.[0] ))
 
+  consul:
+    ca_cert: (( cf_credentials.consul.ca_cert ))
+    server_cert: (( cf_credentials.consul.server_cert ))
+    server_key: (( cf_credentials.consul.server_key ))
+    agent_cert: (( cf_credentials.consul.agent_cert ))
+    agent_key: (( cf_credentials.consul.agent_key ))
+
 Resources: (( merge ))
 Region: (( merge ))
 AccessKeyID: (( merge ))
 SecretAccessKey: (( merge ))
+
+cf_credentials: (( merge ))

examples/aws/templates/cf/stub.yml

@@ -91,6 +91,9 @@ properties:
       availability_zone2: (( merge ))
       availability_zone3: (( merge ))
 
+  app_ssh:
+    host_key_fingerprint: (( merge ))
+
   cc:
     resource_pool:
       resource_directory_key: (( merge ))
@@ -114,6 +117,11 @@ properties:
         name: ccdb
         citext: true
 
+  uaa:
+    jwt:
+      signing_key: (( merge ))
+      verification_key: (( merge ))
+
   uaadb:
     db_scheme: postgresql
     address: (( merge ))
@@ -145,3 +153,10 @@ properties:
       - tag: uaa
         name: uaadb
         citext: true
+
+  consul:
+    ca_cert: (( merge ))
+    server_cert: (( merge ))
+    server_key: (( merge ))
+    agent_cert: (( merge ))
+    agent_key: (( merge ))

examples/aws/templates/diego/property-overrides-internal.yml

@@ -0,0 +1,20 @@
+property_overrides:
+  bbs:
+    ca_cert: (( diego_credentials.diego_ca ))
+    client_cert: (( diego_credentials.bbs.client_cert ))
+    client_key: (( diego_credentials.bbs.client_key ))
+    server_cert: (( diego_credentials.bbs.server_cert ))
+    server_key: (( diego_credentials.bbs.server_key ))
+  etcd:
+    ca_cert: (( diego_credentials.diego_ca ))
+    client_cert: (( diego_credentials.etcd.client_cert ))
+    client_key: (( diego_credentials.etcd.client_key ))
+    peer_ca_cert: (( diego_credentials.etcd.peer_ca ))
+    peer_cert: (( diego_credentials.etcd.peer_cert ))
+    peer_key: (( diego_credentials.etcd.peer_key ))
+    server_cert: (( diego_credentials.etcd.server_cert ))
+    server_key: (( diego_credentials.etcd.server_key ))
+  ssh_proxy:
+    host_key: (( diego_credentials.ssh_proxy_host_key ))
+
+diego_credentials: (( merge ))