diff --git a/installer/pkg/workflow/install.go b/installer/pkg/workflow/install.go index 6360d60260..4470d178d4 100644 --- a/installer/pkg/workflow/install.go +++ b/installer/pkg/workflow/install.go @@ -9,8 +9,8 @@ func NewInstallFullWorkflow(clusterDir string) Workflow { metadata: metadata{clusterDir: clusterDir}, steps: []Step{ readClusterConfigStep, + generateClusterConfigMaps, installAssetsStep, - generateKubeConfigStep, generateIgnConfigStep, installTopologyStep, installTNCCNAMEStep, @@ -30,8 +30,8 @@ func NewInstallAssetsWorkflow(clusterDir string) Workflow { metadata: metadata{clusterDir: clusterDir}, steps: []Step{ readClusterConfigStep, + generateClusterConfigMaps, installAssetsStep, - generateKubeConfigStep, generateIgnConfigStep, }, } diff --git a/installer/pkg/workflow/utils.go b/installer/pkg/workflow/utils.go index 0574d4b276..2c3df2da4d 100644 --- a/installer/pkg/workflow/utils.go +++ b/installer/pkg/workflow/utils.go @@ -75,7 +75,7 @@ func findStepTemplates(stepName, platform string) (string, error) { return "", os.ErrNotExist } -func generateKubeConfigStep(m *metadata) error { +func generateClusterConfigMaps(m *metadata) error { clusterGeneratedPath := filepath.Join(m.clusterDir, generatedPath) if err := os.MkdirAll(clusterGeneratedPath, os.ModeDir|0755); err != nil { return fmt.Errorf("Failed to create cluster generated directory at %s", clusterGeneratedPath) diff --git a/modules/bootkube/assets.tf b/modules/bootkube/assets.tf index a848e1ae95..6a48768446 100644 --- a/modules/bootkube/assets.tf +++ b/modules/bootkube/assets.tf @@ -11,40 +11,7 @@ resource "random_string" "kubelet_bootstrap_token_secret" { upper = false } -# Self-hosted manifests (resources/generated/manifests/) -resource "template_dir" "bootkube" { - source_dir = "${path.module}/resources/manifests" - destination_dir = "./generated/manifests" - - vars { - tectonic_network_operator_image = "${var.container_images["tectonic_network_operator"]}" - tnc_operator_image = "${var.container_images["tnc_operator"]}" - - cloud_provider_config = "${var.cloud_provider_config}" - - root_ca_cert = "${base64encode(var.root_ca_cert_pem)}" - aggregator_ca_cert = "${base64encode(var.aggregator_ca_cert_pem)}" - kube_ca_cert = "${base64encode(var.kube_ca_cert_pem)}" - kube_ca_key = "${base64encode(var.kube_ca_key_pem)}" - kubelet_bootstrap_token_id = "${random_string.kubelet_bootstrap_token_id.result}" - kubelet_bootstrap_token_secret = "${random_string.kubelet_bootstrap_token_secret.result}" - apiserver_key = "${base64encode(var.apiserver_key_pem)}" - apiserver_cert = "${base64encode(var.apiserver_cert_pem)}" - apiserver_proxy_key = "${base64encode(var.apiserver_proxy_key_pem)}" - apiserver_proxy_cert = "${base64encode(var.apiserver_proxy_cert_pem)}" - oidc_ca_cert = "${base64encode(var.oidc_ca_cert)}" - pull_secret = "${base64encode(file(var.pull_secret_path))}" - serviceaccount_pub = "${base64encode(tls_private_key.service_account.public_key_pem)}" - serviceaccount_key = "${base64encode(tls_private_key.service_account.private_key_pem)}" - kube_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - - etcd_ca_cert = "${base64encode(var.etcd_ca_cert_pem)}" - etcd_client_cert = "${base64encode(var.etcd_client_cert_pem)}" - etcd_client_key = "${base64encode(var.etcd_client_key_pem)}" - } -} - -# kubeconfig (resources/generated/auth/kubeconfig) +# kubeconfig (/auth/kubeconfig) data "template_file" "kubeconfig" { template = "${file("${path.module}/resources/kubeconfig")}" @@ -57,12 +24,17 @@ data "template_file" "kubeconfig" { } } -resource "local_file" "kubeconfig" { - content = "${data.template_file.kubeconfig.rendered}" - filename = "./generated/auth/kubeconfig" +data "ignition_file" "kubeconfig" { + filesystem = "root" + path = "/opt/tectonic/auth/kubeconfig" + mode = "0600" + + content { + content = "${data.template_file.kubeconfig.rendered}" + } } -# kubeconfig-kubelet (resources/generated/auth/kubeconfig-kubelet) +# kubeconfig-kubelet data "template_file" "kubeconfig-kubelet" { template = "${file("${path.module}/resources/kubeconfig-kubelet")}" @@ -75,12 +47,17 @@ data "template_file" "kubeconfig-kubelet" { } } -resource "local_file" "kubeconfig-kubelet" { - content = "${data.template_file.kubeconfig-kubelet.rendered}" - filename = "./generated/auth/kubeconfig-kubelet" +data "ignition_file" "kubeconfig-kubelet" { + filesystem = "root" + path = "/opt/tectonic/auth/kubeconfig-kubelet" + mode = "0600" + + content { + content = "${data.template_file.kubeconfig-kubelet.rendered}" + } } -# bootkube.sh (resources/generated/bootkube.sh) +# bootkube.sh data "template_file" "bootkube_sh" { template = "${file("${path.module}/resources/bootkube.sh")}" @@ -91,9 +68,14 @@ data "template_file" "bootkube_sh" { } } -resource "local_file" "bootkube_sh" { - content = "${data.template_file.bootkube_sh.rendered}" - filename = "./generated/bootkube.sh" +data "ignition_file" "bootkube_sh" { + filesystem = "root" + path = "/opt/tectonic/bootkube.sh" + mode = "0755" + + content { + content = "${data.template_file.bootkube_sh.rendered}" + } } # bootkube.service (available as output variable) diff --git a/modules/bootkube/manifests.tf b/modules/bootkube/manifests.tf new file mode 100644 index 0000000000..e1c405294a --- /dev/null +++ b/modules/bootkube/manifests.tf @@ -0,0 +1,72 @@ +variable "manifest_names" { + default = [ + "01-tectonic-namespace.yaml", + "02-ingress-namespace.yaml", + "app-version-kind.yaml", + "app-version-tectonic-network.yaml", + "app-version-tnc.yaml", + "kube-apiserver-secret.yaml", + "kube-cloud-config.yaml", + "kube-controller-manager-secret.yaml", + "kubelet-bootstrap-token.yaml", + "node-config-kind.yaml", + "pull.json", + "tectonic-network-operator.yaml", + "tectonic-node-controller-operator.yaml", + ] +} + +# Self-hosted manifests (resources/generated/manifests/) +data "template_file" "manifest_file_list" { + count = "${length(var.manifest_names)}" + template = "${file("${path.module}/resources/manifests/${var.manifest_names[count.index]}")}" + + vars { + tectonic_network_operator_image = "${var.container_images["tectonic_network_operator"]}" + tnc_operator_image = "${var.container_images["tnc_operator"]}" + + cloud_provider_config = "${var.cloud_provider_config}" + + root_ca_cert = "${base64encode(var.root_ca_cert_pem)}" + aggregator_ca_cert = "${base64encode(var.aggregator_ca_cert_pem)}" + kube_ca_cert = "${base64encode(var.kube_ca_cert_pem)}" + kube_ca_key = "${base64encode(var.kube_ca_key_pem)}" + kubelet_bootstrap_token_id = "${random_string.kubelet_bootstrap_token_id.result}" + kubelet_bootstrap_token_secret = "${random_string.kubelet_bootstrap_token_secret.result}" + apiserver_key = "${base64encode(var.apiserver_key_pem)}" + apiserver_cert = "${base64encode(var.apiserver_cert_pem)}" + apiserver_proxy_key = "${base64encode(var.apiserver_proxy_key_pem)}" + apiserver_proxy_cert = "${base64encode(var.apiserver_proxy_cert_pem)}" + oidc_ca_cert = "${base64encode(var.oidc_ca_cert)}" + pull_secret = "${base64encode(file(var.pull_secret_path))}" + serviceaccount_pub = "${base64encode(tls_private_key.service_account.public_key_pem)}" + serviceaccount_key = "${base64encode(tls_private_key.service_account.private_key_pem)}" + kube_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" + + etcd_ca_cert = "${base64encode(var.etcd_ca_cert_pem)}" + etcd_client_cert = "${base64encode(var.etcd_client_cert_pem)}" + etcd_client_key = "${base64encode(var.etcd_client_key_pem)}" + } +} + +# Ignition entry for every bootkube manifest +# Drops them in /opt/tectonic/manifests/ +data "ignition_file" "manifest_file_list" { + count = "${length(var.manifest_names)}" + filesystem = "root" + mode = "0644" + + path = "/opt/tectonic/manifests/${var.manifest_names[count.index]}" + + content { + content = "${data.template_file.manifest_file_list.*.rendered[count.index]}" + } +} + +# Log the generated manifest files to disk for debugging and user visibility +# Dest: ./generated/manifests/ +resource "local_file" "manifest_files" { + count = "${length(var.manifest_names)}" + filename = "./generated/manifests/${var.manifest_names[count.index]}" + content = "${data.template_file.manifest_file_list.*.rendered[count.index]}" +} diff --git a/modules/bootkube/outputs.tf b/modules/bootkube/outputs.tf index dc1a023a61..bc811b14b0 100644 --- a/modules/bootkube/outputs.tf +++ b/modules/bootkube/outputs.tf @@ -1,49 +1,11 @@ -# This output is meant to be used to inject a dependency on the generated -# assets. As of Terraform v0.9, it is difficult to make a module depend on -# another module (no depends_on, no triggers), or to make a data source -# depend on a module (no depends_on, no triggers, generally no dummy variable). -# -# For instance, using the 'archive_file' data source against the generated -# assets, which is a common use-case, is tricky. There is no mechanism for -# defining explicit dependencies and the only available variables are for the -# source, destination and archive type, leaving little opportunities for us to -# inject a dependency. Thanks to the property described below, this output can -# be used as part of the output filename, in order to enforce the creation of -# the archive after the assets have been properly generated. -# -# Both localfile and template_dir providers compute their IDs by hashing -# the content of the resources on disk. Because this output is computed from the -# combination of all the resources' IDs, it can't be guessed and can only be -# interpolated once the assets have all been created. -output "id" { - value = "${sha1(" - ${local_file.kubeconfig.id} - ${local_file.kubeconfig-kubelet.id} - ${local_file.bootkube_sh.id} - ${template_dir.bootkube.id} - ")}" -} - -output "kubeconfig" { - value = "${data.template_file.kubeconfig.rendered}" -} - output "kubeconfig-kubelet" { value = "${data.template_file.kubeconfig-kubelet.rendered}" } -output "systemd_service_rendered" { - value = "${data.template_file.bootkube_service.rendered}" -} - output "systemd_service_id" { value = "${data.ignition_systemd_unit.bootkube_service.id}" } -output "systemd_path_unit_rendered" { - value = "${data.template_file.bootkube_path_unit.rendered}" -} - output "systemd_path_unit_id" { value = "${data.ignition_systemd_unit.bootkube_path_unit.id}" } @@ -51,3 +13,24 @@ output "systemd_path_unit_id" { output "kube_dns_service_ip" { value = "${cidrhost(var.service_cidr, 10)}" } + +output "kubeconfig_rendered" { + value = "${data.template_file.kubeconfig.rendered}" +} + +output "kubeconfig-kubelet_rendered" { + value = "${data.template_file.kubeconfig-kubelet.rendered}" +} + +output "ignition_file_id_list" { + value = ["${flatten(list( + list( + data.ignition_file.bootkube_sh.id, + data.ignition_file.kubeconfig.id, + data.ignition_file.kubeconfig-kubelet.id, + data.ignition_file.service_account_key.id, + data.ignition_file.service_account_crt.id, + ), + data.ignition_file.manifest_file_list.*.id, + ))}"] +} diff --git a/modules/bootkube/service-account.tf b/modules/bootkube/service-account.tf index 00fbaf11a7..0c4e57186c 100644 --- a/modules/bootkube/service-account.tf +++ b/modules/bootkube/service-account.tf @@ -9,7 +9,27 @@ resource "local_file" "service_account_key" { filename = "./generated/tls/service-account.key" } +data "ignition_file" "service_account_key" { + filesystem = "root" + path = "/opt/tectonic/tls/service-account.key" + mode = "0644" + + content { + content = "${tls_private_key.service_account.private_key_pem}" + } +} + resource "local_file" "service_account_crt" { content = "${tls_private_key.service_account.public_key_pem}" filename = "./generated/tls/service-account.pub" } + +data "ignition_file" "service_account_crt" { + filesystem = "root" + path = "/opt/tectonic/tls/service-account.pub" + mode = "0644" + + content { + content = "${tls_private_key.service_account.public_key_pem}" + } +} diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf index 09292e31a8..4f908db690 100644 --- a/modules/ignition/assets.tf +++ b/modules/ignition/assets.tf @@ -50,7 +50,6 @@ data "template_file" "kubelet" { cloud_provider_config = "${var.cloud_provider_config != "" ? "--cloud-config=/etc/kubernetes/cloud/config" : ""}" cluster_dns_ip = "${var.kube_dns_service_ip}" debug_config = "${var.kubelet_debug_config}" - kubeconfig_fetch_cmd = "${var.kubeconfig_fetch_cmd != "" ? "ExecStartPre=${var.kubeconfig_fetch_cmd}" : ""}" node_label = "${var.kubelet_node_label}" node_taints_param = "${var.kubelet_node_taints != "" ? "--register-with-taints=${var.kubelet_node_taints}" : ""}" } @@ -67,7 +66,6 @@ data "template_file" "k8s_node_bootstrap" { vars { bootstrap_upgrade_cl = "${var.bootstrap_upgrade_cl}" - kubeconfig_fetch_cmd = "${var.kubeconfig_fetch_cmd != "" ? "ExecStartPre=${var.kubeconfig_fetch_cmd}" : ""}" tectonic_torcx_image_url = "${replace(var.container_images["tectonic_torcx"],var.image_re,"$1")}" tectonic_torcx_image_tag = "${replace(var.container_images["tectonic_torcx"],var.image_re,"$2")}" torcx_skip_setup = "false" @@ -81,42 +79,12 @@ data "ignition_systemd_unit" "k8s_node_bootstrap" { content = "${data.template_file.k8s_node_bootstrap.rendered}" } -data "ignition_systemd_unit" "init_assets" { - name = "init-assets.service" - enabled = "${var.assets_location != "" ? true : false}" - content = "${file("${path.module}/resources/services/init-assets.service")}" -} - -data "ignition_systemd_unit" "rm_assets_path_unit" { - name = "rm-assets.path" - enabled = true - content = "${file("${path.module}/resources/paths/rm-assets.path")}" -} - data "ignition_systemd_unit" "rm_assets" { name = "rm-assets.service" - enabled = false + enabled = true content = "${file("${path.module}/resources/services/rm-assets.service")}" } -data "template_file" "s3_puller" { - template = "${file("${path.module}/resources/bin/s3-puller.sh")}" - - vars { - awscli_image = "${var.container_images["awscli"]}" - } -} - -data "ignition_file" "s3_puller" { - filesystem = "root" - path = "/opt/s3-puller.sh" - mode = 0755 - - content { - content = "${data.template_file.s3_puller.rendered}" - } -} - data "ignition_systemd_unit" "locksmithd" { name = "locksmithd.service" mask = true @@ -141,30 +109,6 @@ data "ignition_file" "installer_kubelet_env" { } } -data "template_file" "tx_off" { - template = "${file("${path.module}/resources/services/tx-off.service")}" -} - -data "ignition_systemd_unit" "tx_off" { - name = "tx-off.service" - enabled = true - content = "${data.template_file.tx_off.rendered}" -} - -data "template_file" "azure_udev_rules" { - template = "${file("${path.module}/resources/udev/66-azure-storage.rules")}" -} - -data "ignition_file" "azure_udev_rules" { - filesystem = "root" - path = "/etc/udev/rules.d/66-azure-storage.rules" - mode = 0644 - - content { - content = "${data.template_file.azure_udev_rules.rendered}" - } -} - data "template_file" "coreos_metadata" { template = "${file("${path.module}/resources/dropins/10-metadata.conf")}" @@ -185,24 +129,6 @@ data "ignition_systemd_unit" "coreos_metadata" { ] } -data "template_file" "gcs_puller" { - vars { - gcloudsdk_image = "${var.container_images["gcloudsdk"]}" - } - - template = "${file("${path.module}/resources/bin/gcs-puller.sh")}" -} - -data "ignition_file" "gcs_puller" { - filesystem = "root" - path = "/opt/gcs-puller.sh" - mode = 0755 - - content { - content = "${data.template_file.gcs_puller.rendered}" - } -} - data "ignition_systemd_unit" "iscsi" { name = "iscsid.service" enabled = "${var.iscsi_enabled ? true : false}" diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf index c89b523448..afe62eda8e 100644 --- a/modules/ignition/outputs.tf +++ b/modules/ignition/outputs.tf @@ -1,91 +1,3 @@ -output "max_user_watches_id" { - value = "${data.ignition_file.max_user_watches.id}" -} - -output "max_user_watches_rendered" { - value = "${data.template_file.max_user_watches.rendered}" -} - -output "docker_dropin_id" { - value = "${data.ignition_systemd_unit.docker_dropin.id}" -} - -output "docker_dropin_rendered" { - value = "${data.template_file.docker_dropin.rendered}" -} - -output "kubelet_service_id" { - value = "${data.ignition_systemd_unit.kubelet.id}" -} - -output "kubelet_service_rendered" { - value = "${data.template_file.kubelet.rendered}" -} - -output "k8s_node_bootstrap_service_id" { - value = "${data.ignition_systemd_unit.k8s_node_bootstrap.id}" -} - -output "k8s_node_bootstrap_service_rendered" { - value = "${data.template_file.k8s_node_bootstrap.rendered}" -} - -output "init_assets_service_id" { - value = "${data.ignition_systemd_unit.init_assets.id}" -} - -output "rm_assets_service_id" { - value = "${data.ignition_systemd_unit.rm_assets.id}" -} - -output "rm_assets_path_unit_id" { - value = "${data.ignition_systemd_unit.rm_assets_path_unit.id}" -} - -output "s3_puller_id" { - value = "${data.ignition_file.s3_puller.id}" -} - -output "s3_puller_rendered" { - value = "${data.template_file.s3_puller.rendered}" -} - -output "locksmithd_service_id" { - value = "${data.ignition_systemd_unit.locksmithd.id}" -} - -output "installer_runtime_mappings_id" { - value = "${data.ignition_file.installer_runtime_mappings.id}" -} - -output "installer_runtime_mappings_rendered" { - value = "${data.template_file.installer_runtime_mappings.rendered}" -} - -output "installer_kubelet_env_id" { - value = "${data.ignition_file.installer_kubelet_env.id}" -} - -output "installer_kubelet_env_rendered" { - value = "${data.template_file.installer_kubelet_env.rendered}" -} - -output "tx_off_service_id" { - value = "${data.ignition_systemd_unit.tx_off.id}" -} - -output "tx_off_service_rendered" { - value = "${data.template_file.tx_off.rendered}" -} - -output "azure_udev_rules_id" { - value = "${data.ignition_file.azure_udev_rules.id}" -} - -output "azure_udev_rules_rendered" { - value = "${data.template_file.azure_udev_rules.rendered}" -} - output "etcd_dropin_id_list" { value = "${data.ignition_systemd_unit.etcd.*.id}" } @@ -94,35 +6,10 @@ output "etcd_dropin_rendered_list" { value = "${data.template_file.etcd.*.rendered}" } -output "coreos_metadata_dropin_id" { - value = "${data.ignition_systemd_unit.coreos_metadata.id}" -} - -output "coreos_metadata_dropin_rendered" { - value = "${data.template_file.coreos_metadata.rendered}" -} - -output "gcs_puller_id" { - value = "${data.ignition_file.gcs_puller.id}" -} - -output "update_ca_certificates_dropin_id" { - value = "${data.ignition_systemd_unit.update_ca_certificates_dropin.id}" -} - output "update_ca_certificates_dropin_rendered" { value = "${data.template_file.update_ca_certificates_dropin.rendered}" } -output "ca_cert_id_list" { - value = [ - "${data.ignition_file.root_ca_cert_pem.id}", - "${data.ignition_file.ingress_ca_cert_pem.id}", - "${data.ignition_file.etcd_ca_cert_pem.id}", - "${data.ignition_file.custom_ca_cert_pem.*.id}", - ] -} - output "ca_cert_pem_list" { value = [ "${var.root_ca_cert_pem}", @@ -133,33 +20,41 @@ output "ca_cert_pem_list" { } output "etcd_crt_id_list" { + value = ["${flatten(list( + data.ignition_file.etcd_ca.*.id, + data.ignition_file.etcd_client_key.*.id, + data.ignition_file.etcd_client_crt.*.id, + data.ignition_file.etcd_server_key.*.id, + data.ignition_file.etcd_server_crt.*.id, + data.ignition_file.etcd_peer_key.*.id, + data.ignition_file.etcd_peer_crt.*.id, + ))}"] +} + +output "ignition_file_id_list" { + value = ["${flatten(list( + data.ignition_file.custom_ca_cert_pem.*.id, + list( + data.ignition_file.root_ca_cert_pem.id, + data.ignition_file.ingress_ca_cert_pem.id, + data.ignition_file.etcd_ca_cert_pem.id, + data.ignition_file.installer_kubelet_env.id, + data.ignition_file.installer_runtime_mappings.id, + data.ignition_file.max_user_watches.id, + data.ignition_file.profile_env.id, + data.ignition_file.systemd_default_env.id, + ), + ))}"] +} + +output "ignition_systemd_id_list" { value = [ - "${data.ignition_file.etcd_ca.*.id}", - "${data.ignition_file.etcd_client_key.*.id}", - "${data.ignition_file.etcd_client_crt.*.id}", - "${data.ignition_file.etcd_server_key.*.id}", - "${data.ignition_file.etcd_server_crt.*.id}", - "${data.ignition_file.etcd_peer_key.*.id}", - "${data.ignition_file.etcd_peer_crt.*.id}", + "${data.ignition_systemd_unit.docker_dropin.id}", + "${data.ignition_systemd_unit.kubelet.id}", + "${data.ignition_systemd_unit.locksmithd.id}", + "${data.ignition_systemd_unit.k8s_node_bootstrap.id}", + "${data.ignition_systemd_unit.update_ca_certificates_dropin.id}", + "${data.ignition_systemd_unit.iscsi.id}", + "${data.ignition_systemd_unit.rm_assets.id}", ] } - -output "iscsi_service_id" { - value = "${data.ignition_systemd_unit.iscsi.id}" -} - -output "profile_env_id" { - value = "${data.ignition_file.profile_env.id}" -} - -output "profile_env_rendered" { - value = "${data.template_file.profile_env.rendered}" -} - -output "systemd_default_env_id" { - value = "${data.ignition_file.systemd_default_env.id}" -} - -output "systemd_default_env_rendered" { - value = "${data.template_file.systemd_default_env.rendered}" -} diff --git a/modules/ignition/resources/bin/gcs-puller.sh b/modules/ignition/resources/bin/gcs-puller.sh deleted file mode 100644 index 1c6f97fbcd..0000000000 --- a/modules/ignition/resources/bin/gcs-puller.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC2139,SC2154,SC2034,SC1083 -set -x - -if [ "$#" -ne "2" ]; then - echo "Usage: $0 location destination" - exit 1 -fi - -GSUTIL="docker run --net=host -v $HOME/.config:/.config -v /tmp:/gs ${gcloudsdk_image} gsutil" -ASSETS_ADDRESS=$${1} -ASSETS_NAME=$(basename $${1}) -TARGET_FOLDER=$${2} - -gcs_copy_assets(){ - $${GSUTIL} cp gs://$${ASSETS_ADDRESS} /gs/$${ASSETS_NAME} -} - -gcs_remove_assets(){ - $${GSUTIL} rm gs://$${ASSETS_ADDRESS} -} - -until gcs_copy_assets; do - echo "Could not pull from GCS, retrying in 5 seconds" - sleep 5 -done - -until gcs_remove_assets; do - echo "Could not delete assets from GCS, retrying in 5 seconds" - sleep 5 -done - -/usr/bin/sudo mv /tmp/$${ASSETS_NAME} $${TARGET_FOLDER} diff --git a/modules/ignition/resources/bin/s3-puller.sh b/modules/ignition/resources/bin/s3-puller.sh deleted file mode 100644 index 9bf946c0a7..0000000000 --- a/modules/ignition/resources/bin/s3-puller.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -if [ "$#" -ne "2" ]; then - echo "Usage: $0 location destination" - exit 1 -fi - -# shellcheck disable=SC2034 -LOCATION=$1 -# shellcheck disable=SC1083,SC1001 -LOCATION_ESCAPED=$${1//\//+} -DESTINATION=$2 - -s3_pull() { - # shellcheck disable=SC2086,SC2154,SC2016 - /usr/bin/docker run \ - --volume /tmp:/tmp \ - --network=host \ - --env LOCATION=$LOCATION \ - --env LOCATION_ESCAPED=$LOCATION_ESCAPED \ - --entrypoint=/bin/bash \ - ${awscli_image} \ - -c ' - set -e - set -o pipefail - REGION=$(wget -q -O - http://169.254.169.254/latest/meta-data/placement/availability-zone | sed '"'"'s/[a-zA-Z]$//'"'"') - /usr/bin/aws --region="$REGION" s3 cp s3://"$LOCATION" /tmp/"$LOCATION_ESCAPED" - ' -} - -until s3_pull; do - echo "failed to pull from S3; retrying in 5 seconds" - sleep 5 -done - -/usr/bin/sudo mv /tmp/"$LOCATION_ESCAPED" "$DESTINATION" diff --git a/modules/ignition/resources/paths/rm-assets.path b/modules/ignition/resources/paths/rm-assets.path deleted file mode 100644 index f18bc93bdd..0000000000 --- a/modules/ignition/resources/paths/rm-assets.path +++ /dev/null @@ -1,7 +0,0 @@ -[Unit] -Description=Trigger for rm-assets.service -[Path] -PathExists=/opt/tectonic/manifests -Unit=rm-assets.service -[Install] -WantedBy=multi-user.target diff --git a/modules/ignition/resources/services/init-assets.service b/modules/ignition/resources/services/init-assets.service deleted file mode 100644 index 47a2299881..0000000000 --- a/modules/ignition/resources/services/init-assets.service +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=Download Tectonic Assets -ConditionPathExists=!/opt/init_assets.done -Before=bootkube.service k8s-node-bootstrap.service - -[Service] -Type=oneshot -RemainAfterExit=true -WorkingDirectory=/opt - -User=root -Group=root - -ExecStart=/usr/bin/bash /opt/init-assets.sh -ExecStartPost=/bin/touch /opt/init_assets.done - -[Install] -WantedBy=multi-user.target -RequiredBy=bootkube.service k8s-node-bootstrap.service diff --git a/modules/ignition/resources/services/k8s-node-bootstrap.service b/modules/ignition/resources/services/k8s-node-bootstrap.service index ff543c5401..4927a8cf34 100644 --- a/modules/ignition/resources/services/k8s-node-bootstrap.service +++ b/modules/ignition/resources/services/k8s-node-bootstrap.service @@ -12,7 +12,6 @@ Restart=on-failure RestartSec=10 TimeoutStartSec=1h ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes -${kubeconfig_fetch_cmd} ExecStartPre=/usr/bin/docker run --rm \ --env http_proxy \ --env HTTP_PROXY \ diff --git a/modules/ignition/resources/services/kubelet.service b/modules/ignition/resources/services/kubelet.service index 8b169d6a50..9948a0d9f1 100644 --- a/modules/ignition/resources/services/kubelet.service +++ b/modules/ignition/resources/services/kubelet.service @@ -19,7 +19,6 @@ ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/kubelet/pki -${kubeconfig_fetch_cmd} ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/root-ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid diff --git a/modules/ignition/resources/services/rm-assets.service b/modules/ignition/resources/services/rm-assets.service index 98b34f4e4c..8a6a1d77d7 100644 --- a/modules/ignition/resources/services/rm-assets.service +++ b/modules/ignition/resources/services/rm-assets.service @@ -1,7 +1,6 @@ [Unit] Description=Clean up install assets from S3 ConditionPathExists=!/opt/tectonic/init_rm_assets.done -After=bootkube.service tectonic.service [Service] Type=oneshot @@ -11,7 +10,7 @@ WorkingDirectory=/opt User=root Group=root -ExecStartPre=/usr/bin/bash /opt/rm-assets.sh +ExecStartPre=/usr/bin/bash /opt/tectonic/rm-assets.sh ExecStart=/usr/bin/echo "cleaned up installation assets" ExecStartPost=/bin/touch /opt/tectonic/init_rm_assets.done diff --git a/modules/ignition/variables.tf b/modules/ignition/variables.tf index 31dff24210..63c4bee092 100644 --- a/modules/ignition/variables.tf +++ b/modules/ignition/variables.tf @@ -33,12 +33,6 @@ variable "kubelet_node_taints" { default = "" } -variable "kubeconfig_fetch_cmd" { - type = "string" - description = "(optional) The command that fetches `/etc/kubernetes/kubeconfig`." - default = "" -} - variable "cloud_provider" { type = "string" description = "(optional) The cloud provider to be used for the kubelet." diff --git a/modules/tectonic/assets.tf b/modules/tectonic/assets.tf index ad2d05d959..aa67d518cf 100644 --- a/modules/tectonic/assets.tf +++ b/modules/tectonic/assets.tf @@ -3,68 +3,9 @@ resource "random_id" "cluster_id" { byte_length = 16 } -# Kubernetes Manifests (resources/generated/manifests/) -resource "template_dir" "tectonic" { - source_dir = "${path.module}/resources/manifests" - destination_dir = "./generated/tectonic" - - vars { - addon_resizer_image = "${var.container_images["addon_resizer"]}" - kube_core_operator_image = "${var.container_images["kube_core_operator"]}" - kubernetes_addon_operator_image = "${var.container_images["kubernetes_addon_operator"]}" - tectonic_channel_operator_image = "${var.container_images["tectonic_channel_operator"]}" - tectonic_prometheus_operator_image = "${var.container_images["tectonic_prometheus_operator"]}" - tectonic_cluo_operator_image = "${var.container_images["tectonic_cluo_operator"]}" - tectonic_alm_operator_image = "${var.container_images["tectonic_alm_operator"]}" - tectonic_ingress_controller_operator_image = "${var.container_images["tectonic_ingress_controller_operator"]}" - tectonic_utility_operator_image = "${var.container_images["tectonic_utility_operator"]}" - - tectonic_monitoring_auth_base_image = "${var.container_base_images["tectonic_monitoring_auth"]}" - config_reload_base_image = "${var.container_base_images["config_reload"]}" - addon_resizer_base_image = "${var.container_base_images["addon_resizer"]}" - kube_state_metrics_base_image = "${var.container_base_images["kube_state_metrics"]}" - prometheus_operator_base_image = "${var.container_base_images["prometheus_operator"]}" - prometheus_config_reload_base_image = "${var.container_base_images["prometheus_config_reload"]}" - prometheus_base_image = "${var.container_base_images["prometheus"]}" - alertmanager_base_image = "${var.container_base_images["alertmanager"]}" - node_exporter_base_image = "${var.container_base_images["node_exporter"]}" - grafana_base_image = "${var.container_base_images["grafana"]}" - grafana_watcher_base_image = "${var.container_base_images["grafana_watcher"]}" - kube_rbac_proxy_base_image = "${var.container_base_images["kube_rbac_proxy"]}" - - monitoring_version = "${var.versions["monitoring"]}" - tectonic_version = "${var.versions["tectonic"]}" - tectonic_cluo_operator_version = "${var.versions["cluo"]}" - tectonic_alm_operator_version = "${var.versions["alm"]}" - - license = "${base64encode(file(var.license_path))}" - pull_secret = "${base64encode(file(var.pull_secret_path))}" - - update_server = "${var.update_server}" - update_channel = "${var.update_channel}" - update_app_id = "${var.update_app_id}" - - admin_email = "${lower(var.admin_email)}" - - base_address = "${var.base_address}" - - ingress_ca_cert = "${base64encode(var.ingress_ca_cert_pem)}" - ingress_tls_cert = "${base64encode(var.ingress_cert_pem)}" - ingress_tls_key = "${base64encode(var.ingress_key_pem)}" - - identity_server_tls_cert = "${base64encode(var.identity_server_cert_pem)}" - identity_server_tls_key = "${base64encode(var.identity_server_key_pem)}" - identity_server_ca_cert = "${base64encode(var.identity_server_ca_cert)}" - identity_client_tls_cert = "${base64encode(var.identity_client_cert_pem)}" - identity_client_tls_key = "${base64encode(var.identity_client_key_pem)}" - identity_client_ca_cert = "${base64encode(var.identity_client_ca_cert)}" - - platform = "${var.platform}" - } -} - -# tectonic.sh (resources/generated/tectonic.sh) -data "template_file" "tectonic" { +# tectonic.sh (/opt/tectonic/tectonic.sh) +data "template_file" "tectonic_sh" { + // TODO: This doesn't need to be a template anymore template = "${file("${path.module}/resources/tectonic.sh")}" vars { @@ -72,13 +13,18 @@ data "template_file" "tectonic" { } } -resource "local_file" "tectonic" { - content = "${data.template_file.tectonic.rendered}" - filename = "./generated/tectonic.sh" +data "ignition_file" "tectonic_sh" { + filesystem = "root" + mode = "0755" + path = "/opt/tectonic/tectonic.sh" + + content { + content = "${data.template_file.tectonic_sh.rendered}" + } } -# tectonic.sh (resources/generated/tectonic-wrapper.sh) -data "template_file" "tectonic_wrapper" { +# (/opt/tectonic/tectonic-wrapper.sh) +data "template_file" "tectonic_wrapper_sh" { template = "${file("${path.module}/resources/tectonic-wrapper.sh")}" vars { @@ -86,9 +32,14 @@ data "template_file" "tectonic_wrapper" { } } -resource "local_file" "tectonic_wrapper" { - content = "${data.template_file.tectonic_wrapper.rendered}" - filename = "./generated/tectonic-wrapper.sh" +data "ignition_file" "tectonic_wrapper_sh" { + filesystem = "root" + mode = "0755" + path = "/opt/tectonic/tectonic-wrapper.sh" + + content { + content = "${data.template_file.tectonic_wrapper_sh.rendered}" + } } # tectonic.service (available as output variable) diff --git a/modules/tectonic/manifests.tf b/modules/tectonic/manifests.tf new file mode 100644 index 0000000000..52929a7f6d --- /dev/null +++ b/modules/tectonic/manifests.tf @@ -0,0 +1,124 @@ +#List of all tectonic manifests +# To update this list, issue this command from the repo root: +# find modules/tectonic/resources/manifests -type f -printf ' "%P",\n' | sort +variable "manifest_names" { + default = [ + "ingress/cluster-config.yaml", + "ingress/pull.json", + "ingress/README.md", + "ingress/svc-account.yaml", + "rbac/binding-admin.yaml", + "rbac/binding-discovery.yaml", + "rbac/role-admin.yaml", + "rbac/role-user.yaml", + "secrets/ca-cert.yaml", + "secrets/identity-grpc-client.yaml", + "secrets/identity-grpc-server.yaml", + "secrets/ingress-tls.yaml", + "secrets/license.json", + "secrets/pull.json", + "updater/app-version-kind.yaml", + "updater/app_versions/app-version-kube-core.yaml", + "updater/app_versions/app-version-kubernetes-addon.yaml", + "updater/app_versions/app-version-tectonic-alm.yaml", + "updater/app_versions/app-version-tectonic-cluo.yaml", + "updater/app_versions/app-version-tectonic-cluster.yaml", + "updater/app_versions/app-version-tectonic-ingress.yaml", + "updater/app_versions/app-version-tectonic-monitoring.yaml", + "updater/app_versions/app-version-tectonic-utility.yaml", + "updater/migration-status-kind.yaml", + "updater/operators/kube-core-operator.yaml", + "updater/operators/kubernetes-addon-operator.yaml", + "updater/operators/tectonic-alm-operator.yaml", + "updater/operators/tectonic-channel-operator.yaml", + "updater/operators/tectonic-cluo-operator.yaml", + "updater/operators/tectonic-ingress-controller-operator.yaml", + "updater/operators/tectonic-prometheus-operator.yaml", + "updater/operators/tectonic-utility-operator.yaml", + "updater/tectonic-channel-operator-config.yaml", + "updater/tectonic-channel-operator-kind.yaml", + "updater/tectonic-monitoring-config.yaml", + ] +} + +# Kubernetes Manifests, rendered +data "template_file" "manifest_file_list" { + count = "${length(var.manifest_names)}" + template = "${file("${path.module}/resources/manifests/${var.manifest_names[count.index]}")}" + + vars { + addon_resizer_image = "${var.container_images["addon_resizer"]}" + kube_core_operator_image = "${var.container_images["kube_core_operator"]}" + kubernetes_addon_operator_image = "${var.container_images["kubernetes_addon_operator"]}" + tectonic_channel_operator_image = "${var.container_images["tectonic_channel_operator"]}" + tectonic_prometheus_operator_image = "${var.container_images["tectonic_prometheus_operator"]}" + tectonic_cluo_operator_image = "${var.container_images["tectonic_cluo_operator"]}" + tectonic_alm_operator_image = "${var.container_images["tectonic_alm_operator"]}" + tectonic_ingress_controller_operator_image = "${var.container_images["tectonic_ingress_controller_operator"]}" + tectonic_utility_operator_image = "${var.container_images["tectonic_utility_operator"]}" + + tectonic_monitoring_auth_base_image = "${var.container_base_images["tectonic_monitoring_auth"]}" + config_reload_base_image = "${var.container_base_images["config_reload"]}" + addon_resizer_base_image = "${var.container_base_images["addon_resizer"]}" + kube_state_metrics_base_image = "${var.container_base_images["kube_state_metrics"]}" + prometheus_operator_base_image = "${var.container_base_images["prometheus_operator"]}" + prometheus_config_reload_base_image = "${var.container_base_images["prometheus_config_reload"]}" + prometheus_base_image = "${var.container_base_images["prometheus"]}" + alertmanager_base_image = "${var.container_base_images["alertmanager"]}" + node_exporter_base_image = "${var.container_base_images["node_exporter"]}" + grafana_base_image = "${var.container_base_images["grafana"]}" + grafana_watcher_base_image = "${var.container_base_images["grafana_watcher"]}" + kube_rbac_proxy_base_image = "${var.container_base_images["kube_rbac_proxy"]}" + + monitoring_version = "${var.versions["monitoring"]}" + tectonic_version = "${var.versions["tectonic"]}" + tectonic_cluo_operator_version = "${var.versions["cluo"]}" + tectonic_alm_operator_version = "${var.versions["alm"]}" + + license = "${base64encode(file(var.license_path))}" + pull_secret = "${base64encode(file(var.pull_secret_path))}" + + update_server = "${var.update_server}" + update_channel = "${var.update_channel}" + update_app_id = "${var.update_app_id}" + + admin_email = "${lower(var.admin_email)}" + + base_address = "${var.base_address}" + + ingress_ca_cert = "${base64encode(var.ingress_ca_cert_pem)}" + ingress_tls_cert = "${base64encode(var.ingress_cert_pem)}" + ingress_tls_key = "${base64encode(var.ingress_key_pem)}" + + identity_server_tls_cert = "${base64encode(var.identity_server_cert_pem)}" + identity_server_tls_key = "${base64encode(var.identity_server_key_pem)}" + identity_server_ca_cert = "${base64encode(var.identity_server_ca_cert)}" + identity_client_tls_cert = "${base64encode(var.identity_client_cert_pem)}" + identity_client_tls_key = "${base64encode(var.identity_client_key_pem)}" + identity_client_ca_cert = "${base64encode(var.identity_client_ca_cert)}" + + platform = "${var.platform}" + } +} + +# Ignition entry for every tectonic manifest +# Drops them in /opt/tectonic/tectonic/ +data "ignition_file" "tectonic_manifest_list" { + count = "${length(var.manifest_names)}" + filesystem = "root" + mode = "0644" + + path = "/opt/tectonic/tectonic/${var.manifest_names[count.index]}" + + content { + content = "${data.template_file.manifest_file_list.*.rendered[count.index]}" + } +} + +# Log the generated manifest files to disk for debugging and user visibility +# Dest: ./generated/tectonic/ +resource "local_file" "manifest_files" { + count = "${length(var.manifest_names)}" + filename = "./generated/tectonic/${var.manifest_names[count.index]}" + content = "${data.template_file.manifest_file_list.*.rendered[count.index]}" +} diff --git a/modules/tectonic/output.tf b/modules/tectonic/output.tf index bab58cc33c..d071c2eeb9 100644 --- a/modules/tectonic/output.tf +++ b/modules/tectonic/output.tf @@ -1,24 +1,3 @@ -# This output is meant to be used to inject a dependency on the generated -# assets. As of Terraform v0.9, it is difficult to make a module depend on -# another module (no depends_on, no triggers), or to make a data source -# depend on a module (no depends_on, no triggers, generally no dummy variable). -# -# For instance, using the 'archive_file' data source against the generated -# assets, which is a common use-case, is tricky. There is no mechanism for -# defining explicit dependencies and the only available variables are for the -# source, destination and archive type, leaving little opportunities for us to -# inject a dependency. Thanks to the property described below, this output can -# be used as part of the output filename, in order to enforce the creation of -# the archive after the assets have been properly generated. -# -# Both localfile and template_dir providers compute their IDs by hashing -# the content of the resources on disk. Because this output is computed from the -# combination of all the resources' IDs, it can't be guessed and can only be -# interpolated once the assets have all been created. -output "id" { - value = "${sha1("${template_dir.tectonic.id} ${local_file.tectonic.id}")}" -} - output "systemd_service_rendered" { value = "${data.template_file.tectonic_service.rendered}" } @@ -35,6 +14,16 @@ output "systemd_path_unit_id" { value = "${data.ignition_systemd_unit.tectonic_path.id}" } +output "ignition_file_id_list" { + value = ["${flatten(list( + list( + data.ignition_file.tectonic_sh.id, + data.ignition_file.tectonic_wrapper_sh.id, + ), + data.ignition_file.tectonic_manifest_list.*.id, + ))}"] +} + output "cluster_id" { value = "${format( "%s-%s-%s-%s-%s", diff --git a/modules/tls/ca/self-signed/assets.tf b/modules/tls/ca/self-signed/assets.tf index a0545f71cf..da268ff3aa 100644 --- a/modules/tls/ca/self-signed/assets.tf +++ b/modules/tls/ca/self-signed/assets.tf @@ -3,27 +3,92 @@ resource "local_file" "root_ca_cert" { filename = "./generated/tls/root-ca.crt" } +data "ignition_file" "root_ca_cert" { + filesystem = "root" + mode = "0644" + path = "/opt/tectonic/tls/root-ca.crt" + + content { + content = "${var.root_ca_cert_pem == "" ? join("", tls_self_signed_cert.root_ca.*.cert_pem) : var.root_ca_cert_pem}" + } +} + resource "local_file" "kube_ca_key" { content = "${tls_private_key.kube_ca.private_key_pem}" filename = "./generated/tls/kube-ca.key" } +data "ignition_file" "kube_ca_key" { + filesystem = "root" + mode = "0600" + + content { + content = "${tls_private_key.kube_ca.private_key_pem}" + } + + path = "/opt/tectonic/tls/kube-ca.key" +} + resource "local_file" "kube_ca_cert" { content = "${tls_locally_signed_cert.kube_ca.cert_pem}" filename = "./generated/tls/kube-ca.crt" } +data "ignition_file" "kube_ca_cert" { + filesystem = "root" + mode = "0600" + + content { + content = "${tls_locally_signed_cert.kube_ca.cert_pem}" + } + + path = "/opt/tectonic/tls/kube-ca.crt" +} + resource "local_file" "aggregator_ca_key" { content = "${tls_private_key.aggregator_ca.private_key_pem}" filename = "./generated/tls/aggregator-ca.key" } +data "ignition_file" "aggregator_ca_key" { + filesystem = "root" + mode = "0600" + + content { + content = "${tls_private_key.aggregator_ca.private_key_pem}" + } + + path = "/opt/tectonic/tls/aggregator-ca.key" +} + resource "local_file" "aggregator_ca_cert" { content = "${tls_locally_signed_cert.aggregator_ca.cert_pem}" filename = "./generated/tls/aggregator-ca.crt" } +data "ignition_file" "aggregator_ca_cert" { + filesystem = "root" + mode = "0644" + + content { + content = "${tls_locally_signed_cert.aggregator_ca.cert_pem}" + } + + path = "/opt/tectonic/tls/aggregator-ca.crt" +} + resource "local_file" "etcd_ca_cert" { content = "${tls_locally_signed_cert.etcd_ca.cert_pem}" filename = "./generated/tls/etcd-client-ca.crt" } + +data "ignition_file" "etcd_ca_cert" { + filesystem = "root" + mode = "0644" + + content { + content = "${tls_locally_signed_cert.etcd_ca.cert_pem}" + } + + path = "/opt/tectonic/tls/etcd-client-ca.crt" +} diff --git a/modules/tls/ca/self-signed/outputs.tf b/modules/tls/ca/self-signed/outputs.tf index c7fbd0dbe6..5bf9af7b43 100644 --- a/modules/tls/ca/self-signed/outputs.tf +++ b/modules/tls/ca/self-signed/outputs.tf @@ -50,3 +50,14 @@ output "id" { )} ")}" } + +output "ignition_file_id_list" { + value = [ + "${data.ignition_file.root_ca_cert.id}", + "${data.ignition_file.kube_ca_key.id}", + "${data.ignition_file.kube_ca_cert.id}", + "${data.ignition_file.aggregator_ca_key.id}", + "${data.ignition_file.aggregator_ca_cert.id}", + "${data.ignition_file.etcd_ca_cert.id}", + ] +} diff --git a/modules/tls/etcd/assets.tf b/modules/tls/etcd/assets.tf index 8a478477e2..390330502d 100644 --- a/modules/tls/etcd/assets.tf +++ b/modules/tls/etcd/assets.tf @@ -3,11 +3,33 @@ resource "local_file" "etcd_client_cert" { filename = "./generated/tls/etcd-client.crt" } +data "ignition_file" "etcd_client_cert" { + filesystem = "root" + mode = "0644" + + content { + content = "${tls_locally_signed_cert.etcd_client.cert_pem}" + } + + path = "/opt/tectonic/tls/etcd-client.crt" +} + resource "local_file" "etcd_client_key" { content = "${tls_private_key.etcd_client.private_key_pem}" filename = "./generated/tls/etcd-client.key" } +data "ignition_file" "etcd_client_key" { + filesystem = "root" + mode = "0600" + + content { + content = "${tls_private_key.etcd_client.private_key_pem}" + } + + path = "/opt/tectonic/tls/etcd-client.key" +} + resource "local_file" "etcd_server_cert" { content = "${tls_locally_signed_cert.etcd_server.cert_pem}" filename = "./generated/tls/etcd/server.crt" diff --git a/modules/tls/etcd/outputs.tf b/modules/tls/etcd/outputs.tf index a0cfff883c..3e1b64400f 100644 --- a/modules/tls/etcd/outputs.tf +++ b/modules/tls/etcd/outputs.tf @@ -34,3 +34,10 @@ output "id" { )} ")}" } + +output "ignition_file_id_list" { + value = [ + "${data.ignition_file.etcd_client_cert.id}", + "${data.ignition_file.etcd_client_key.id}", + ] +} diff --git a/modules/tls/kube/assets.tf b/modules/tls/kube/assets.tf index 00634fa3b1..169061464b 100644 --- a/modules/tls/kube/assets.tf +++ b/modules/tls/kube/assets.tf @@ -3,6 +3,17 @@ resource "local_file" "apiserver_key" { filename = "./generated/tls/apiserver.key" } +data "ignition_file" "apiserver_key" { + filesystem = "root" + mode = "0600" + + content { + content = "${tls_private_key.apiserver.private_key_pem}" + } + + path = "/opt/tectonic/tls/apiserver.key" +} + data "template_file" "apiserver_cert" { template = "${join("", list(tls_locally_signed_cert.apiserver.cert_pem, var.kube_ca_cert_pem))}" } @@ -12,22 +23,77 @@ resource "local_file" "apiserver_cert" { filename = "./generated/tls/apiserver.crt" } +data "ignition_file" "apiserver_cert" { + filesystem = "root" + mode = "0644" + + content { + content = "${data.template_file.apiserver_cert.rendered}" + } + + path = "/opt/tectonic/tls/apiserver.crt" +} + resource "local_file" "apiserver_proxy_key" { content = "${tls_private_key.apiserver_proxy.private_key_pem}" filename = "./generated/tls/apiserver-proxy.key" } +data "ignition_file" "apiserver_proxy_key" { + filesystem = "root" + mode = "0600" + + content { + content = "${tls_private_key.apiserver_proxy.private_key_pem}" + } + + path = "/opt/tectonic/tls/apiserver-proxy.key" +} + resource "local_file" "apiserver_proxy_cert" { content = "${tls_locally_signed_cert.apiserver_proxy.cert_pem}" filename = "./generated/tls/apiserver-proxy.crt" } +data "ignition_file" "apiserver_proxy_cert" { + filesystem = "root" + mode = "0644" + + content { + content = "${tls_locally_signed_cert.apiserver_proxy.cert_pem}" + } + + path = "/opt/tectonic/tls/apiserver-proxy.crt" +} + resource "local_file" "admin_key" { content = "${tls_private_key.admin.private_key_pem}" filename = "./generated/tls/admin.key" } +data "ignition_file" "admin_key" { + filesystem = "root" + mode = "0600" + + content { + content = "${tls_private_key.admin.private_key_pem}" + } + + path = "/opt/tectonic/tls/admin.key" +} + resource "local_file" "admin_cert" { content = "${tls_locally_signed_cert.admin.cert_pem}" filename = "./generated/tls/admin.crt" } + +data "ignition_file" "admin_cert" { + filesystem = "root" + mode = "0644" + + content { + content = "${tls_locally_signed_cert.admin.cert_pem}" + } + + path = "/opt/tectonic/tls/admin.crt" +} diff --git a/modules/tls/kube/outputs.tf b/modules/tls/kube/outputs.tf index 042252a591..429e1ae423 100644 --- a/modules/tls/kube/outputs.tf +++ b/modules/tls/kube/outputs.tf @@ -22,6 +22,17 @@ output "apiserver_proxy_key_pem" { value = "${tls_private_key.apiserver_proxy.private_key_pem}" } +output "ignition_file_id_list" { + value = [ + "${data.ignition_file.apiserver_key.id}", + "${data.ignition_file.apiserver_cert.id}", + "${data.ignition_file.apiserver_proxy_key.id}", + "${data.ignition_file.apiserver_proxy_cert.id}", + "${data.ignition_file.admin_key.id}", + "${data.ignition_file.admin_cert.id}", + ] +} + output "id" { value = "${sha1(" ${join(" ", diff --git a/steps/assets/ignition-bootstrap.tf b/steps/assets/ignition-bootstrap.tf index 43793424d3..db5826ed91 100644 --- a/steps/assets/ignition-bootstrap.tf +++ b/steps/assets/ignition-bootstrap.tf @@ -1,7 +1,6 @@ module "ignition_bootstrap" { source = "../../modules/ignition" - assets_location = "${local.bucket_name}/${local.bucket_assets_key}" base_domain = "${var.tectonic_base_domain}" bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}" cloud_provider = "aws" @@ -31,58 +30,50 @@ module "ignition_bootstrap" { no_proxy = "${var.tectonic_no_proxy}" } -data "ignition_config" "bootstrap" { - files = ["${compact(list( - data.ignition_file.init_assets.id, - data.ignition_file.rm_assets.id, - module.ignition_bootstrap.installer_kubelet_env_id, - module.ignition_bootstrap.installer_runtime_mappings_id, - module.ignition_bootstrap.max_user_watches_id, - module.ignition_bootstrap.s3_puller_id, - module.ignition_bootstrap.profile_env_id, - module.ignition_bootstrap.systemd_default_env_id, - ))}", - "${module.ignition_bootstrap.ca_cert_id_list}", - ] +# The cluster configs written by the install binary +data "ignition_file" "kube-system_cluster_config" { + filesystem = "root" + mode = "0644" + path = "/opt/tectonic/manifests/cluster-config.yaml" - systemd = ["${compact(list( - module.ignition_bootstrap.docker_dropin_id, - module.ignition_bootstrap.locksmithd_service_id, - module.ignition_bootstrap.kubelet_service_id, - module.ignition_bootstrap.k8s_node_bootstrap_service_id, - module.ignition_bootstrap.init_assets_service_id, - module.ignition_bootstrap.rm_assets_service_id, - module.ignition_bootstrap.rm_assets_path_unit_id, - module.bootkube.systemd_service_id, - module.bootkube.systemd_path_unit_id, - module.tectonic.systemd_service_id, - module.tectonic.systemd_path_unit_id, - module.ignition_bootstrap.update_ca_certificates_dropin_id, - module.ignition_bootstrap.iscsi_service_id, - ))}"] + content { + content = "${file("./generated/manifests/cluster-config.yaml")}" + } } -data "template_file" "init_assets" { - template = "${file("${path.module}/resources/init-assets.sh")}" +data "ignition_file" "tectonic_cluster_config" { + filesystem = "root" + mode = "0644" + path = "/opt/tectonic/tectonic/cluster-config.yaml" - vars { - cluster_name = "${var.tectonic_cluster_name}" - awscli_image = "${var.tectonic_container_images["awscli"]}" - assets_s3_location = "${local.bucket_name}/${local.bucket_assets_key}" + content { + content = "${file("./generated/tectonic/cluster-config.yaml")}" } } -data "ignition_file" "init_assets" { +data "ignition_file" "tnco_config" { filesystem = "root" - path = "/opt/init-assets.sh" - mode = 0755 + mode = "0644" + path = "/opt/tectonic/tnco-config.yaml" content { - content = "${data.template_file.init_assets.rendered}" + content = "${file("./generated/tnco-config.yaml")}" } } -data "template_file" "rm_assets" { +data "ignition_file" "kco_config" { + filesystem = "root" + mode = "0644" + path = "/opt/tectonic/kco-config.yaml" + + content { + content = "${file("./generated/kco-config.yaml")}" + } +} + +# Removing assets is platform-specific +# But it must be installed in /opt/tectonic/rm-assets.sh +data "template_file" "rm_assets_sh" { template = "${file("${path.module}/resources/rm-assets.sh")}" vars { @@ -92,12 +83,40 @@ data "template_file" "rm_assets" { } } -data "ignition_file" "rm_assets" { +data "ignition_file" "rm_assets_sh" { filesystem = "root" - path = "/opt/rm-assets.sh" - mode = 0755 + path = "/opt/tectonic/rm-assets.sh" + mode = "0700" content { - content = "${data.template_file.rm_assets.rendered}" + content = "${data.template_file.rm_assets_sh.rendered}" } } + +data "ignition_config" "bootstrap" { + files = ["${compact(flatten(list( + list( + data.ignition_file.kube-system_cluster_config.id, + data.ignition_file.tectonic_cluster_config.id, + data.ignition_file.tnco_config.id, + data.ignition_file.kco_config.id, + data.ignition_file.rm_assets_sh.id, + ), + module.ignition_bootstrap.ignition_file_id_list, + module.bootkube.ignition_file_id_list, + module.tectonic.ignition_file_id_list, + module.ca_certs.ignition_file_id_list, + module.kube_certs.ignition_file_id_list, + module.etcd_certs.ignition_file_id_list, + )))}"] + + systemd = ["${compact(flatten(list( + list( + module.bootkube.systemd_service_id, + module.bootkube.systemd_path_unit_id, + module.tectonic.systemd_service_id, + module.tectonic.systemd_path_unit_id, + ), + module.ignition_bootstrap.ignition_systemd_id_list, + )))}"] +} diff --git a/steps/assets/local-files.tf b/steps/assets/local-files.tf new file mode 100644 index 0000000000..9fc87c6599 --- /dev/null +++ b/steps/assets/local-files.tf @@ -0,0 +1,22 @@ +locals { + auth_dir = "./generated/auth" + kubeconfig_kubelet_path = "${local.auth_dir}/kubeconfig-kubelet" + kubeconfig_path = "${local.auth_dir}/kubeconfig" +} + +# some files we want rendered to disk for other use +resource "local_file" "kubeconfig-kubelet" { + content = "${module.bootkube.kubeconfig-kubelet_rendered}" + filename = "${local.kubeconfig_kubelet_path}" +} + +resource "local_file" "kubeconfig" { + content = "${module.bootkube.kubeconfig_rendered}" + filename = "${local.kubeconfig_path}" +} + +# Generally not used except for debugging +resource "local_file" "bootstrap_ignition" { + content = "${data.ignition_config.bootstrap.rendered}" + filename = "ignition-bootstrap.ign" +} diff --git a/steps/assets/resources/init-assets.sh b/steps/assets/resources/init-assets.sh deleted file mode 100644 index 8b39463049..0000000000 --- a/steps/assets/resources/init-assets.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -# Download the assets from S3. -# shellcheck disable=SC2154 -/usr/bin/bash /opt/s3-puller.sh "${assets_s3_location}" /var/tmp/tectonic.zip -unzip -o -d /var/tmp/tectonic/ /var/tmp/tectonic.zip -rm /var/tmp/tectonic.zip -# make files in /opt/tectonic available atomically -mv /var/tmp/tectonic /opt/tectonic - -exit 0 diff --git a/steps/assets/resources/rm-assets.sh b/steps/assets/resources/rm-assets.sh index bbf9ef4064..546b474e4a 100644 --- a/steps/assets/resources/rm-assets.sh +++ b/steps/assets/resources/rm-assets.sh @@ -5,7 +5,7 @@ s3_clean() { # instead of simply removing the remote assets.zip, # overwrite it with a zero byte file, such that terraform doesn't # detect deletion but rather a simple change which it can ignore. - touch /tmp/assets.zip + touch /tmp/empty # shellcheck disable=SC2086,SC2154,SC2016 /usr/bin/docker run \ @@ -17,9 +17,9 @@ s3_clean() { -c ' set -e set -o pipefail + set -u REGION=$(wget -q -O - http://169.254.169.254/latest/meta-data/placement/availability-zone | sed '"'"'s/[a-zA-Z]$//'"'"') - /usr/bin/aws --region="$REGION" s3 cp /tmp/assets.zip s3://"$LOCATION/assets.zip" - /usr/bin/aws --region="$REGION" s3 cp /tmp/assets.zip s3://"$LOCATION/config/master" + /usr/bin/aws --region="$REGION" s3 cp /tmp/empty s3://"$LOCATION/config/bootstrap" ' } diff --git a/steps/topology/main.tf b/steps/topology/main.tf index ef80da79b8..5a81cea73b 100644 --- a/steps/topology/main.tf +++ b/steps/topology/main.tf @@ -32,8 +32,7 @@ resource "aws_route53_zone" "tectonic_int" { } module "vpc" { - source = "../../modules/aws/vpc" - depends_on = ["${aws_s3_bucket_object.tectonic_assets.id}"] + source = "../../modules/aws/vpc" base_domain = "${var.tectonic_base_domain}" cidr_block = "${var.tectonic_aws_vpc_cidr_block}" diff --git a/steps/topology/s3-assets.tf b/steps/topology/s3-assets.tf index ada2c17cb8..62a5a45804 100644 --- a/steps/topology/s3-assets.tf +++ b/steps/topology/s3-assets.tf @@ -1,3 +1,10 @@ +## Create the S3 bucket where we'll upload the initial ignition file +# This needs a bit of a special setup. Because Ignition is fetching the +# configuration over the tnc LB pointed to the s3 bucket, it doesn't have any +# identity. So, the file needs to be public. But then we'd expose secrets, +# so the public ignition file just has an ignition redirect to a s3:// url, +# which ignition can fetch directly with authentication. + resource "aws_s3_bucket" "tectonic" { # This bucket name must match the CNAME # https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#VirtualHostingCustomURLs @@ -16,20 +23,18 @@ resource "aws_s3_bucket" "tectonic" { } } -# Bootkube / Tectonic assets -resource "aws_s3_bucket_object" "tectonic_assets" { - bucket = "${aws_s3_bucket.tectonic.bucket}" - key = "assets.zip" - source = "${data.archive_file.assets.output_path}" - acl = "private" +# The real ignition contents, with secrets. +# Must be private. The node will zero this out as soon as it boots. +resource "aws_s3_bucket_object" "ignition_bootstrap_real" { + bucket = "${aws_s3_bucket.tectonic.bucket}" + key = "config/bootstrap" + content = "${local.ignition_bootstrap}" + acl = "private" - # To be on par with the current Tectonic installer, we only do server-side - # encryption, using AES256. Eventually, we should start using KMS-based - # client-side encryption. server_side_encryption = "AES256" tags = "${merge(map( - "Name", "${var.tectonic_cluster_name}-tectonic-assets", + "Name", "${var.tectonic_cluster_name}-ignition-master", "KubernetesCluster", "${var.tectonic_cluster_name}", "tectonicClusterID", "${var.tectonic_cluster_id}" ), var.tectonic_aws_extra_tags)}" @@ -39,32 +44,20 @@ resource "aws_s3_bucket_object" "tectonic_assets" { } } -data "archive_file" "assets" { - type = "zip" - source_dir = "./generated/" - - # Because the archive_file provider is a data source, depends_on can't be - # used to guarantee that the tectonic/bootkube modules have generated - # all the assets on disk before trying to archive them. Instead, we use their - # ID outputs, that are only computed once the assets have actually been - # written to disk. We re-hash the IDs (or dedicated module outputs, like module.bootkube.content_hash) - # to make the filename shorter, since there is no security nor collision risk anyways. - # - # Additionally, data sources do not support managing any lifecycle whatsoever, - # and therefore, the archive is never deleted. To avoid cluttering the module - # folder, we write it in the Terraform managed hidden folder `.terraform`. - output_path = "./.terraform/generated_${sha1("${var.tectonic_cluster_id}")}.zip" +# The public ignition configuration +data "ignition_config" "bootstrap_redirect" { + replace { + source = "s3://${aws_s3_bucket.tectonic.bucket}/config/bootstrap" + } } -# Ignition +# The public ignition object. resource "aws_s3_bucket_object" "ignition_bootstrap" { bucket = "${aws_s3_bucket.tectonic.bucket}" key = "config/master" - content = "${local.ignition_bootstrap}" + content = "${data.ignition_config.bootstrap_redirect.rendered}" acl = "public-read" - # TODO: Lock down permissions. - # At the minute this is pulic (so accessible via http) so joiners nodes can reach the TNC using the same url server_side_encryption = "AES256" tags = "${merge(map( diff --git a/tests/rspec/lib/aws_cluster.rb b/tests/rspec/lib/aws_cluster.rb index 7c156f8e08..69be7c676e 100644 --- a/tests/rspec/lib/aws_cluster.rb +++ b/tests/rspec/lib/aws_cluster.rb @@ -27,7 +27,7 @@ def initialize(config_file) @name = @config_file.cluster_name @build_path = File.join(File.dirname(ENV['RELEASE_TARBALL_PATH']), "tectonic/#{@name}") @manifest_path = File.join(@build_path, 'generated') - @kubeconfig = File.join(manifest_path, 'auth/kubeconfig') + @kubeconfig = File.join(@build_path, 'generated/auth/kubeconfig') @role_credentials = nil @role_credentials = AWSIAM.assume_role(@aws_region) if ENV.key?('TECTONIC_INSTALLER_ROLE')