diff --git a/Documentation/dev/node-bootstrap-flow.md b/Documentation/dev/node-bootstrap-flow.md index 023d7e9619..aa364f7a2a 100644 --- a/Documentation/dev/node-bootstrap-flow.md +++ b/Documentation/dev/node-bootstrap-flow.md @@ -118,9 +118,15 @@ ExecStartPost=/bin/touch /opt/tectonic/init_tectonic.done Tectonic service unit is not enabled by default. It is instead triggered by a path unit, which waits for assets written synchronously by terraform. This service waits for bootkube process to be *completed* via systemd dependency. -It is a oneshot service, thus marked as started only once the script return with success. +It is a oneshot service, thus marked as started only once the script returns with success. It is skipped on further boots, as the condition-path exists. +### `rm-assets.path` and `rm-assets.service` + +This service waits for the bootkube and tectonic process to be completed. +It is a oneshot service, thus marked as started only once the script returns with success. +This is an optional service only present on platforms which pull assets from block storage. + ## Diagram This is a visual simplified representation of the overall bootstrapping flow. @@ -135,29 +141,31 @@ Legend: * b.s -> bootkube.service * t.p -> tectonic.path * t.s -> tectonic.service - -.------------------------------------------------------------------------------------------------------------------. -| | -| Provision cloud/userdata +----------+ Provision files | -| ,----------------------------------------------o| TF |o-----------------.------------------------. | -| | +----------+ | | | -| | v v | -| | +----------+ +-----+ +-------+ | -| | .--->| (reboot) |----. | b.p | | t.p | | -| | | +----------+ | +-----+ +-------+ | -| V | | o o | -| +-------+ | v Before +------------+ Before | Trigger Trigger | | -| | IGN | | *---------->| k.s |o--------. | | | -| +-------+ o ^ +------------+ | v v | -| | +----------+ | ^ | | +-----+ Before +-------+ | -| '------>| knb.s |o--------------' | v '--->| b.s |o--------------->| t.s | | -| Enable +----------+ '------' +-----+ +-------+ | -| ^ | | -| | v | -| '----' o o | -| | | | -| * First boot | * Each boot | * First boot | -| * All nodes | * All nodes | * Bootkube master | -| | | | -'----------------------------------------------o----------------------------o--------------------------------------' + * rm.p -> rm-assets.path + * rm.s -> rm-assets.service + +.---------------------------------------------------------------------------------------------------------------------------------------+ +| | +| Provision cloud/userdata +----------+ Provision files | +| ,----------------------------------------------o| TF |o-----------------.------------------------.-----------------+ | +| | +----------+ | | | | +| | v v v | +| | +----------+ +-----+ +-------+ +------+ | +| | .--->| (reboot) |----. | b.p | | t.p | | rm.p | | +| | | +----------+ | +-----+ +-------+ +------+ | +| V | | o o o | +| +-------+ | v Before +------------+ Before | Trigger Trigger | Trigger | | +| | IGN | | *---------->| k.s |o--------. | | | | +| +-------+ o ^ +------------+ | v v v | +| | +----------+ | ^ | | +-----+ Before +-------+ Before +-----+ | +| '------>| knb.s |o--------------' | v '--->| b.s |o--------------->| t.s |--------> |rm.s | | +| Enable +----------+ '------' +-----+ +-------+ +-----+ | +| ^ | | +| | v | +| '----' o o | +| | | | +| * First boot | * Each boot | * First boot | +| * All nodes | * All nodes | * Bootkube master | +| | | | +'----------------------------------------------o----------------------------o-----------------------------------------------------------+ ``` diff --git a/modules/aws/master-asg/ignition.tf b/modules/aws/master-asg/ignition.tf index 4005cfe4bc..19364f23a1 100644 --- a/modules/aws/master-asg/ignition.tf +++ b/modules/aws/master-asg/ignition.tf @@ -19,6 +19,7 @@ data "ignition_config" "main" { var.ign_tectonic_service_id, var.ign_bootkube_path_unit_id, var.ign_tectonic_path_unit_id, + var.ign_rm_assets_path_unit_id, ))}"] } diff --git a/modules/aws/master-asg/master.tf b/modules/aws/master-asg/master.tf index fe1419115b..afc83084b2 100644 --- a/modules/aws/master-asg/master.tf +++ b/modules/aws/master-asg/master.tf @@ -153,7 +153,7 @@ resource "aws_iam_role_policy" "master_policy" { "s3:GetObject", "s3:HeadObject", "s3:ListBucket", - "s3:DeleteObject" + "s3:PutObject" ], "Resource": "arn:aws:s3:::*", "Effect": "Allow" diff --git a/modules/aws/master-asg/resources/rm-assets.sh b/modules/aws/master-asg/resources/rm-assets.sh index 2b8330c13f..01d2ec4a21 100644 --- a/modules/aws/master-asg/resources/rm-assets.sh +++ b/modules/aws/master-asg/resources/rm-assets.sh @@ -2,34 +2,25 @@ set -e s3_clean() { - # Delete Install assets from S3 - # shellcheck disable=SC2086,SC2154,SC2016 - /usr/bin/docker run \ - --volume /tmp:/tmp \ - --network=host \ - --env LOCATION="${assets_s3_location}" \ - --entrypoint=/bin/bash \ - ${awscli_image} \ - -c ' - REGION=$(wget -q -O - http://169.254.169.254/latest/meta-data/placement/availability-zone | sed '"'"'s/[a-zA-Z]$//'"'"') - usr/bin/aws --region=$${REGION} s3 rm s3://$${LOCATION} - ' -} - -# shellcheck disable=SC2086,SC2154 -/usr/bin/docker run \ - --volume /run/metadata:/run/metadata \ - --volume /opt/detect-master.sh:/detect-master.sh:ro \ - --network=host \ - --env CLUSTER_NAME=${cluster_name} \ - --entrypoint=/detect-master.sh \ - ${awscli_image} + # instead of simply removing the remote assets.zip, + # overwrite it with a zero byte file, such that terraform doesn't + # detect deletion but rather a simple change which it can ignore. + touch /tmp/assets.zip -# Don't do anything if cluster is still in startup -STARTUP=$(cat /run/metadata/master) -if [ "$STARTUP" == "true" ]; then - exit 0 -fi + # shellcheck disable=SC2086,SC2154,SC2016 + /usr/bin/docker run \ + --volume /tmp:/tmp \ + --network=host \ + --env LOCATION="${assets_s3_location}" \ + --entrypoint=/bin/bash \ + ${awscli_image} \ + -c ' + set -e + set -o pipefail + REGION=$(wget -q -O - http://169.254.169.254/latest/meta-data/placement/availability-zone | sed '"'"'s/[a-zA-Z]$//'"'"') + /usr/bin/aws --region="$REGION" s3 cp /tmp/assets.zip s3://"$LOCATION" + ' +} until s3_clean; do echo "failed to clean up S3 assets. retrying in 5 seconds." diff --git a/modules/aws/master-asg/variables.tf b/modules/aws/master-asg/variables.tf index a7ad1a03c4..c2a3cd2104 100644 --- a/modules/aws/master-asg/variables.tf +++ b/modules/aws/master-asg/variables.tf @@ -129,3 +129,7 @@ variable "ign_init_assets_service_id" { variable "ign_rm_assets_service_id" { type = "string" } + +variable "ign_rm_assets_path_unit_id" { + type = "string" +} diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf index 8fe965f544..1d1910938c 100644 --- a/modules/ignition/assets.tf +++ b/modules/ignition/assets.tf @@ -74,9 +74,15 @@ data "ignition_systemd_unit" "init_assets" { content = "${file("${path.module}/resources/services/init-assets.service")}" } +data "ignition_systemd_unit" "rm_assets_path_unit" { + name = "rm-assets.path" + enable = true + content = "${file("${path.module}/resources/paths/rm-assets.path")}" +} + data "ignition_systemd_unit" "rm_assets" { name = "rm-assets.service" - enable = "${var.assets_location != "" ? true : false}" + enable = false content = "${file("${path.module}/resources/services/rm-assets.service")}" } diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf index 6b8ef1f0db..7b2ea81caa 100644 --- a/modules/ignition/outputs.tf +++ b/modules/ignition/outputs.tf @@ -38,6 +38,10 @@ output "rm_assets_service_id" { value = "${data.ignition_systemd_unit.rm_assets.id}" } +output "rm_assets_path_unit_id" { + value = "${data.ignition_systemd_unit.rm_assets_path_unit.id}" +} + output "s3_puller_id" { value = "${data.ignition_file.s3_puller.id}" } diff --git a/modules/ignition/resources/paths/rm-assets.path b/modules/ignition/resources/paths/rm-assets.path new file mode 100644 index 0000000000..f18bc93bdd --- /dev/null +++ b/modules/ignition/resources/paths/rm-assets.path @@ -0,0 +1,7 @@ +[Unit] +Description=Trigger for rm-assets.service +[Path] +PathExists=/opt/tectonic/manifests +Unit=rm-assets.service +[Install] +WantedBy=multi-user.target diff --git a/modules/ignition/resources/services/rm-assets.service b/modules/ignition/resources/services/rm-assets.service index e25b41b0b4..98b34f4e4c 100644 --- a/modules/ignition/resources/services/rm-assets.service +++ b/modules/ignition/resources/services/rm-assets.service @@ -1,7 +1,7 @@ [Unit] Description=Clean up install assets from S3 -ConditionPathExists=/opt/tectonic/init_tectonic.done -After=tectonic.service +ConditionPathExists=!/opt/tectonic/init_rm_assets.done +After=bootkube.service tectonic.service [Service] Type=oneshot @@ -13,6 +13,7 @@ Group=root ExecStartPre=/usr/bin/bash /opt/rm-assets.sh ExecStart=/usr/bin/echo "cleaned up installation assets" +ExecStartPost=/bin/touch /opt/tectonic/init_rm_assets.done [Install] WantedBy=multi-user.target diff --git a/modules/tectonic/resources/tectonic.sh b/modules/tectonic/resources/tectonic.sh index 2ac1739d0c..d6fb76f1b4 100755 --- a/modules/tectonic/resources/tectonic.sh +++ b/modules/tectonic/resources/tectonic.sh @@ -104,7 +104,16 @@ wait_for_pods() { asset_cleanup() { echo "Cleaning up installation assets" - rm -rf "$${ASSETS_PATH:?}/"* + + # shellcheck disable=SC2034 + for d in "manifests" "auth" "bootstrap-manifests" "net-manifests" "tectonic" "tls"; do + rm -rf "$${ASSETS_PATH:?}/$${d:?}/"* + done + + # shellcheck disable=SC2034 + for f in "bootkube.sh" "tectonic.sh" "tectonic-wrapper.sh"; do + rm -f "$${ASSETS_PATH:?}/$${f:?}" + done } # chdir into the assets path directory diff --git a/platforms/aws/main.tf b/platforms/aws/main.tf index c5471cec8c..2cd59cb333 100644 --- a/platforms/aws/main.tf +++ b/platforms/aws/main.tf @@ -132,6 +132,7 @@ module "masters" { ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" + ign_rm_assets_path_unit_id = "${module.ignition_masters.rm_assets_path_unit_id}" ign_rm_assets_service_id = "${module.ignition_masters.rm_assets_service_id}" ign_s3_puller_id = "${module.ignition_masters.s3_puller_id}" ign_tectonic_path_unit_id = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}" diff --git a/platforms/aws/s3.tf b/platforms/aws/s3.tf index 7ff3260260..9fa6d7c154 100644 --- a/platforms/aws/s3.tf +++ b/platforms/aws/s3.tf @@ -15,6 +15,10 @@ resource "aws_s3_bucket" "tectonic" { "KubernetesCluster", "${var.tectonic_cluster_name}", "tectonicClusterID", "${module.tectonic.cluster_id}" ), var.tectonic_aws_extra_tags)}" + + lifecycle { + ignore_changes = ["*"] + } } # Bootkube / Tectonic assets @@ -34,6 +38,10 @@ resource "aws_s3_bucket_object" "tectonic_assets" { "KubernetesCluster", "${var.tectonic_cluster_name}", "tectonicClusterID", "${module.tectonic.cluster_id}" ), var.tectonic_aws_extra_tags)}" + + lifecycle { + ignore_changes = ["*"] + } } # kubeconfig