From 669a3d5737b5e275173f7584eaf2f550934848e0 Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Tue, 22 Aug 2017 11:46:57 +0200 Subject: [PATCH 01/13] modules/ignition: unify max-user-watches Fixes partially INST-132 --- modules/aws/ignition/ignition.tf | 12 +----------- modules/aws/ignition/variables-ignition.tf | 1 + modules/azure/master-as/ignition-master.tf | 12 +----------- modules/azure/master-as/variables-ignition.tf | 1 + modules/azure/worker-as/ignition-worker.tf | 12 +----------- modules/azure/worker-as/variables-ignition.tf | 1 + modules/ignition/assets.tf | 13 +++++++++++++ modules/ignition/outputs.import | 5 +++++ modules/ignition/outputs.tf | 7 +++++++ .../resources/sysctl.d/max-user-watches.conf | 1 + modules/openstack/nodes/ignition.tf | 12 +----------- modules/openstack/nodes/variables-ignition.tf | 1 + modules/vmware/node/ignition.tf | 12 +----------- modules/vmware/node/variables-ignition.tf | 1 + platforms/aws/main.tf | 8 ++++++++ platforms/azure/main.tf | 8 ++++++++ platforms/metal/cl/bootkube-controller.yaml.tmpl | 4 ++-- platforms/metal/cl/bootkube-worker.yaml.tmpl | 4 ++-- platforms/metal/matchers.tf | 8 ++++++++ platforms/openstack/neutron/main.tf | 8 ++++++++ platforms/vmware/main.tf | 8 ++++++++ 21 files changed, 80 insertions(+), 59 deletions(-) create mode 120000 modules/aws/ignition/variables-ignition.tf create mode 120000 modules/azure/master-as/variables-ignition.tf create mode 120000 modules/azure/worker-as/variables-ignition.tf create mode 100644 modules/ignition/assets.tf create mode 100644 modules/ignition/outputs.import create mode 100644 modules/ignition/outputs.tf create mode 100644 modules/ignition/resources/sysctl.d/max-user-watches.conf create mode 120000 modules/openstack/nodes/variables-ignition.tf create mode 120000 modules/vmware/node/variables-ignition.tf diff --git a/modules/aws/ignition/ignition.tf b/modules/aws/ignition/ignition.tf index 6e4bc6b30e..0f897ad52c 100644 --- a/modules/aws/ignition/ignition.tf +++ b/modules/aws/ignition/ignition.tf @@ -1,6 +1,6 @@ data "ignition_config" "main" { files = [ - "${data.ignition_file.max_user_watches.id}", + "${var.ign_max_user_watches_id}", "${data.ignition_file.s3_puller.id}", "${data.ignition_file.init_assets.id}", "${data.ignition_file.detect_master.id}", @@ -69,16 +69,6 @@ data "ignition_systemd_unit" "kubelet_env" { content = "${data.template_file.kubelet_env.rendered}" } -data "ignition_file" "max_user_watches" { - filesystem = "root" - path = "/etc/sysctl.d/max-user-watches.conf" - mode = 0644 - - content { - content = "fs.inotify.max_user_watches=16184" - } -} - data "template_file" "s3_puller" { template = "${file("${path.module}/resources/s3-puller.sh")}" diff --git a/modules/aws/ignition/variables-ignition.tf b/modules/aws/ignition/variables-ignition.tf new file mode 120000 index 0000000000..e22a48c662 --- /dev/null +++ b/modules/aws/ignition/variables-ignition.tf @@ -0,0 +1 @@ +../../ignition/outputs.import \ No newline at end of file diff --git a/modules/azure/master-as/ignition-master.tf b/modules/azure/master-as/ignition-master.tf index 9d16c93d43..5420d6f370 100644 --- a/modules/azure/master-as/ignition-master.tf +++ b/modules/azure/master-as/ignition-master.tf @@ -3,7 +3,7 @@ data "ignition_config" "master" { "${data.ignition_file.kubeconfig.id}", "${data.ignition_file.kubelet_env.id}", "${module.azure_udev-rules.udev-rules_id}", - "${data.ignition_file.max_user_watches.id}", + "${var.ign_max_user_watches_id}", "${data.ignition_file.cloud_provider_config.id}", ] @@ -87,16 +87,6 @@ EOF } } -data "ignition_file" "max_user_watches" { - filesystem = "root" - path = "/etc/sysctl.d/max-user-watches.conf" - mode = 0644 - - content { - content = "fs.inotify.max_user_watches=16184" - } -} - data "ignition_file" "cloud_provider_config" { filesystem = "root" path = "/etc/kubernetes/cloud/config" diff --git a/modules/azure/master-as/variables-ignition.tf b/modules/azure/master-as/variables-ignition.tf new file mode 120000 index 0000000000..e22a48c662 --- /dev/null +++ b/modules/azure/master-as/variables-ignition.tf @@ -0,0 +1 @@ +../../ignition/outputs.import \ No newline at end of file diff --git a/modules/azure/worker-as/ignition-worker.tf b/modules/azure/worker-as/ignition-worker.tf index f0809147c1..1347085a0e 100644 --- a/modules/azure/worker-as/ignition-worker.tf +++ b/modules/azure/worker-as/ignition-worker.tf @@ -3,7 +3,7 @@ data "ignition_config" "worker" { "${data.ignition_file.kubeconfig.id}", "${data.ignition_file.kubelet-env.id}", "${module.azure_udev-rules.udev-rules_id}", - "${data.ignition_file.max-user-watches.id}", + "${var.ign_max_user_watches_id}", "${data.ignition_file.cloud-provider-config.id}", ] @@ -86,16 +86,6 @@ EOF } } -data "ignition_file" "max-user-watches" { - filesystem = "root" - path = "/etc/sysctl.d/max-user-watches.conf" - mode = 0644 - - content { - content = "fs.inotify.max_user_watches=16184" - } -} - data "ignition_file" "cloud-provider-config" { filesystem = "root" path = "/etc/kubernetes/cloud/config" diff --git a/modules/azure/worker-as/variables-ignition.tf b/modules/azure/worker-as/variables-ignition.tf new file mode 120000 index 0000000000..e22a48c662 --- /dev/null +++ b/modules/azure/worker-as/variables-ignition.tf @@ -0,0 +1 @@ +../../ignition/outputs.import \ No newline at end of file diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf new file mode 100644 index 0000000000..8159f1d2da --- /dev/null +++ b/modules/ignition/assets.tf @@ -0,0 +1,13 @@ +data "template_file" "max_user_watches" { + template = "${file("${path.module}/resources/sysctl.d/max-user-watches.conf")}" +} + +data "ignition_file" "max_user_watches" { + filesystem = "root" + path = "/etc/sysctl.d/max-user-watches.conf" + mode = 0644 + + content { + content = "${data.template_file.max_user_watches.rendered}" + } +} diff --git a/modules/ignition/outputs.import b/modules/ignition/outputs.import new file mode 100644 index 0000000000..3fb898093e --- /dev/null +++ b/modules/ignition/outputs.import @@ -0,0 +1,5 @@ +# This file is supposed to be symlinked in consuming modules + +variable "ign_max_user_watches_id" { + type = "string" +} diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf new file mode 100644 index 0000000000..6486c8e9e9 --- /dev/null +++ b/modules/ignition/outputs.tf @@ -0,0 +1,7 @@ +output "max_user_watches_id" { + value = "${data.ignition_file.max_user_watches.id}" +} + +output "max_user_watches_rendered" { + value = "${data.template_file.max_user_watches.rendered}" +} diff --git a/modules/ignition/resources/sysctl.d/max-user-watches.conf b/modules/ignition/resources/sysctl.d/max-user-watches.conf new file mode 100644 index 0000000000..8a14449976 --- /dev/null +++ b/modules/ignition/resources/sysctl.d/max-user-watches.conf @@ -0,0 +1 @@ +fs.inotify.max_user_watches=16184 diff --git a/modules/openstack/nodes/ignition.tf b/modules/openstack/nodes/ignition.tf index 80b55ce0d3..ebaf2122e0 100644 --- a/modules/openstack/nodes/ignition.tf +++ b/modules/openstack/nodes/ignition.tf @@ -8,7 +8,7 @@ data "ignition_config" "node" { files = [ "${data.ignition_file.kubeconfig.id}", "${data.ignition_file.kubelet-env.id}", - "${data.ignition_file.max_user_watches.id}", + "${var.ign_max_user_watches_id}", "${data.ignition_file.resolv_conf.id}", "${data.ignition_file.hostname.*.id[count.index]}", ] @@ -107,16 +107,6 @@ EOF } } -data "ignition_file" "max_user_watches" { - filesystem = "root" - path = "/etc/sysctl.d/max-user-watches.conf" - mode = 0644 - - content { - content = "fs.inotify.max_user_watches=16184" - } -} - data "ignition_systemd_unit" "bootkube" { name = "bootkube.service" content = "${var.bootkube_service}" diff --git a/modules/openstack/nodes/variables-ignition.tf b/modules/openstack/nodes/variables-ignition.tf new file mode 120000 index 0000000000..e22a48c662 --- /dev/null +++ b/modules/openstack/nodes/variables-ignition.tf @@ -0,0 +1 @@ +../../ignition/outputs.import \ No newline at end of file diff --git a/modules/vmware/node/ignition.tf b/modules/vmware/node/ignition.tf index 6e673569ad..9cdcd409f3 100644 --- a/modules/vmware/node/ignition.tf +++ b/modules/vmware/node/ignition.tf @@ -6,7 +6,7 @@ data "ignition_config" "node" { ] files = [ - "${data.ignition_file.max-user-watches.id}", + "${var.ign_max_user_watches_id}", "${data.ignition_file.node_hostname.*.id[count.index]}", "${data.ignition_file.kubelet-env.id}", ] @@ -80,16 +80,6 @@ data "ignition_systemd_unit" "kubelet-env" { content = "${data.template_file.kubelet-env.rendered}" } -data "ignition_file" "max-user-watches" { - filesystem = "root" - path = "/etc/sysctl.d/max-user-watches.conf" - mode = 0644 - - content { - content = "fs.inotify.max_user_watches=16184" - } -} - data "ignition_systemd_unit" "bootkube" { name = "bootkube.service" content = "${var.bootkube_service}" diff --git a/modules/vmware/node/variables-ignition.tf b/modules/vmware/node/variables-ignition.tf new file mode 120000 index 0000000000..e22a48c662 --- /dev/null +++ b/modules/vmware/node/variables-ignition.tf @@ -0,0 +1 @@ +../../ignition/outputs.import \ No newline at end of file diff --git a/platforms/aws/main.tf b/platforms/aws/main.tf index 8a7ce6ec91..8f4e75e51f 100644 --- a/platforms/aws/main.tf +++ b/platforms/aws/main.tf @@ -1,3 +1,7 @@ +module "ignition" { + source = "../../modules/ignition" +} + provider "aws" { region = "${var.tectonic_aws_region}" } @@ -96,6 +100,8 @@ module "ignition_masters" { tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" cluster_name = "${var.tectonic_cluster_name}" image_re = "${var.tectonic_image_re}" + + ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" } module "masters" { @@ -144,6 +150,8 @@ module "ignition_workers" { tectonic_service = "" cluster_name = "" image_re = "${var.tectonic_image_re}" + + ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" } module "workers" { diff --git a/platforms/azure/main.tf b/platforms/azure/main.tf index 21e547b974..07db221ca1 100644 --- a/platforms/azure/main.tf +++ b/platforms/azure/main.tf @@ -1,3 +1,7 @@ +module "ignition" { + source = "../../modules/ignition" +} + provider "azurerm" { environment = "${var.tectonic_azure_cloud_environment}" client_secret = "${var.tectonic_azure_client_secret}" @@ -125,6 +129,8 @@ module "masters" { cl_channel = "${var.tectonic_cl_channel}" extra_tags = "${var.tectonic_azure_extra_tags}" + + ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" } module "workers" { @@ -154,6 +160,8 @@ module "workers" { cl_channel = "${var.tectonic_cl_channel}" extra_tags = "${var.tectonic_azure_extra_tags}" + + ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" } module "dns" { diff --git a/platforms/metal/cl/bootkube-controller.yaml.tmpl b/platforms/metal/cl/bootkube-controller.yaml.tmpl index 64920de43a..fc7e8e8fdc 100644 --- a/platforms/metal/cl/bootkube-controller.yaml.tmpl +++ b/platforms/metal/cl/bootkube-controller.yaml.tmpl @@ -143,9 +143,9 @@ storage: {{.domain_name}} - path: /etc/sysctl.d/max-user-watches.conf filesystem: root + mode: 0644 contents: - inline: | - fs.inotify.max_user_watches=16184 + inline: {{.ign_max_user_watches_json}} passwd: users: - name: core diff --git a/platforms/metal/cl/bootkube-worker.yaml.tmpl b/platforms/metal/cl/bootkube-worker.yaml.tmpl index 4c3fc0d1dc..ad361c9f5c 100644 --- a/platforms/metal/cl/bootkube-worker.yaml.tmpl +++ b/platforms/metal/cl/bootkube-worker.yaml.tmpl @@ -98,9 +98,9 @@ storage: {{.domain_name}} - path: /etc/sysctl.d/max-user-watches.conf filesystem: root + mode: 0644 contents: - inline: | - fs.inotify.max_user_watches=16184 + inline: {{.ign_max_user_watches_json}} passwd: users: - name: core diff --git a/platforms/metal/matchers.tf b/platforms/metal/matchers.tf index acb1deb3fa..69771b18e2 100644 --- a/platforms/metal/matchers.tf +++ b/platforms/metal/matchers.tf @@ -1,3 +1,7 @@ +module "ignition" { + source = "../../modules/ignition" +} + // Install CoreOS to disk resource "matchbox_group" "coreos_install" { count = "${length(var.tectonic_metal_controller_names) + length(var.tectonic_metal_worker_names)}" @@ -54,6 +58,8 @@ resource "matchbox_group" "controller" { etcd_image_tag = "v${var.tectonic_versions["etcd"]}" kubelet_image_url = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$1")}" kubelet_image_tag = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}" + + ign_max_user_watches_json = "${jsonencode(module.ignition.max_user_watches_rendered)}" } } @@ -77,5 +83,7 @@ resource "matchbox_group" "worker" { kubelet_image_url = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$1")}" kubelet_image_tag = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}" kube_version_image = "${var.tectonic_container_images["kube_version"]}" + + ign_max_user_watches_json = "${jsonencode(module.ignition.max_user_watches_rendered)}" } } diff --git a/platforms/openstack/neutron/main.tf b/platforms/openstack/neutron/main.tf index 57a9879889..be3177db99 100644 --- a/platforms/openstack/neutron/main.tf +++ b/platforms/openstack/neutron/main.tf @@ -1,3 +1,7 @@ +module "ignition" { + source = "../../../modules/ignition" +} + module "bootkube" { source = "../../../modules/bootkube" cloud_provider = "" @@ -140,6 +144,8 @@ EOF kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }" tectonic_experimental = "${var.tectonic_experimental}" tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" + + ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" } module "worker_nodes" { @@ -164,6 +170,8 @@ EOF node_taints = "" kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }" tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" + + ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" } module "secrets" { diff --git a/platforms/vmware/main.tf b/platforms/vmware/main.tf index d563b49bbe..3671819d37 100644 --- a/platforms/vmware/main.tf +++ b/platforms/vmware/main.tf @@ -1,3 +1,7 @@ +module "ignition" { + source = "../../modules/ignition" +} + module "etcd" { source = "../../modules/vmware/etcd" instance_count = "${var.tectonic_experimental ? 0 : var.tectonic_etcd_count }" @@ -65,6 +69,8 @@ module "masters" { kubeconfig = "${module.bootkube.kubeconfig}" private_key = "${var.tectonic_vmware_ssh_private_key_path}" image_re = "${var.tectonic_image_re}" + + ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" } module "workers" { @@ -99,4 +105,6 @@ module "workers" { kubeconfig = "${module.bootkube.kubeconfig}" private_key = "${var.tectonic_vmware_ssh_private_key_path}" image_re = "${var.tectonic_image_re}" + + ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" } From e7b4e220f49d5fb0133507f3b97fdf20ee94da35 Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Tue, 22 Aug 2017 16:18:57 +0200 Subject: [PATCH 02/13] modules/ignition: unify docker dropin Fixes partially INST-132 --- modules/aws/ignition/ignition.tf | 14 +------------- modules/azure/master-as/ignition-master.tf | 14 +------------- modules/azure/worker-as/ignition-worker.tf | 14 +------------- modules/ignition/assets.tf | 16 ++++++++++++++++ modules/ignition/outputs.import | 4 ++++ modules/ignition/outputs.tf | 8 ++++++++ .../resources/dropins/10-dockeropts.conf | 2 ++ modules/openstack/nodes/ignition.tf | 14 +------------- modules/vmware/node/ignition.tf | 14 +------------- platforms/aws/main.tf | 2 ++ platforms/azure/main.tf | 2 ++ platforms/metal/matchers.tf | 2 ++ platforms/openstack/neutron/main.tf | 2 ++ platforms/vmware/main.tf | 2 ++ 14 files changed, 45 insertions(+), 65 deletions(-) create mode 100644 modules/ignition/resources/dropins/10-dockeropts.conf diff --git a/modules/aws/ignition/ignition.tf b/modules/aws/ignition/ignition.tf index 0f897ad52c..f71f2ca1a3 100644 --- a/modules/aws/ignition/ignition.tf +++ b/modules/aws/ignition/ignition.tf @@ -7,7 +7,7 @@ data "ignition_config" "main" { ] systemd = [ - "${data.ignition_systemd_unit.docker.id}", + "${var.ign_docker_dropin_id}", "${data.ignition_systemd_unit.locksmithd.id}", "${data.ignition_systemd_unit.kubelet.id}", "${data.ignition_systemd_unit.kubelet_env.id}", @@ -17,18 +17,6 @@ data "ignition_config" "main" { ] } -data "ignition_systemd_unit" "docker" { - name = "docker.service" - enable = true - - dropin = [ - { - name = "10-dockeropts.conf" - content = "[Service]\nEnvironment=\"DOCKER_OPTS=--log-opt max-size=50m --log-opt max-file=3\"\n" - }, - ] -} - data "ignition_systemd_unit" "locksmithd" { name = "locksmithd.service" mask = true diff --git a/modules/azure/master-as/ignition-master.tf b/modules/azure/master-as/ignition-master.tf index 5420d6f370..cce158cc53 100644 --- a/modules/azure/master-as/ignition-master.tf +++ b/modules/azure/master-as/ignition-master.tf @@ -8,7 +8,7 @@ data "ignition_config" "master" { ] systemd = [ - "${data.ignition_systemd_unit.docker.id}", + "${var.ign_docker_dropin_id}", "${data.ignition_systemd_unit.locksmithd.id}", "${data.ignition_systemd_unit.kubelet_master.id}", "${data.ignition_systemd_unit.tectonic.id}", @@ -29,18 +29,6 @@ data "ignition_user" "core" { ] } -data "ignition_systemd_unit" "docker" { - name = "docker.service" - enable = true - - dropin = [ - { - name = "10-dockeropts.conf" - content = "[Service]\nEnvironment=\"DOCKER_OPTS=--log-opt max-size=50m --log-opt max-file=3\"\n" - }, - ] -} - data "ignition_systemd_unit" "locksmithd" { name = "locksmithd.service" mask = true diff --git a/modules/azure/worker-as/ignition-worker.tf b/modules/azure/worker-as/ignition-worker.tf index 1347085a0e..f79f7e41c9 100644 --- a/modules/azure/worker-as/ignition-worker.tf +++ b/modules/azure/worker-as/ignition-worker.tf @@ -8,7 +8,7 @@ data "ignition_config" "worker" { ] systemd = [ - "${data.ignition_systemd_unit.docker.id}", + "${var.ign_docker_dropin_id}", "${data.ignition_systemd_unit.locksmithd.id}", "${data.ignition_systemd_unit.kubelet-worker.id}", "${module.net_ignition.tx-off_id}", @@ -19,18 +19,6 @@ data "ignition_config" "worker" { ] } -data "ignition_systemd_unit" "docker" { - name = "docker.service" - enable = true - - dropin = [ - { - name = "10-dockeropts.conf" - content = "[Service]\nEnvironment=\"DOCKER_OPTS=--log-opt max-size=50m --log-opt max-file=3\"\n" - }, - ] -} - data "ignition_systemd_unit" "locksmithd" { name = "locksmithd.service" mask = true diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf index 8159f1d2da..fb0884d744 100644 --- a/modules/ignition/assets.tf +++ b/modules/ignition/assets.tf @@ -11,3 +11,19 @@ data "ignition_file" "max_user_watches" { content = "${data.template_file.max_user_watches.rendered}" } } + +data "template_file" "docker_dropin" { + template = "${file("${path.module}/resources/dropins/10-dockeropts.conf")}" +} + +data "ignition_systemd_unit" "docker_dropin" { + name = "docker.service" + enable = true + + dropin = [ + { + name = "10-dockeropts.conf" + content = "${data.template_file.docker_dropin.rendered}" + }, + ] +} diff --git a/modules/ignition/outputs.import b/modules/ignition/outputs.import index 3fb898093e..4ba3adf54e 100644 --- a/modules/ignition/outputs.import +++ b/modules/ignition/outputs.import @@ -3,3 +3,7 @@ variable "ign_max_user_watches_id" { type = "string" } + +variable "ign_docker_dropin_id" { + type = "string" +} diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf index 6486c8e9e9..7aef1d36dc 100644 --- a/modules/ignition/outputs.tf +++ b/modules/ignition/outputs.tf @@ -5,3 +5,11 @@ output "max_user_watches_id" { output "max_user_watches_rendered" { value = "${data.template_file.max_user_watches.rendered}" } + +output "docker_dropin_id" { + value = "${data.ignition_systemd_unit.docker_dropin.id}" +} + +output "docker_dropin_rendered" { + value = "${data.template_file.docker_dropin.rendered}" +} diff --git a/modules/ignition/resources/dropins/10-dockeropts.conf b/modules/ignition/resources/dropins/10-dockeropts.conf new file mode 100644 index 0000000000..bf5b180932 --- /dev/null +++ b/modules/ignition/resources/dropins/10-dockeropts.conf @@ -0,0 +1,2 @@ +[Service] +Environment="DOCKER_OPTS=--log-opt max-size=50m --log-opt max-file=3" diff --git a/modules/openstack/nodes/ignition.tf b/modules/openstack/nodes/ignition.tf index ebaf2122e0..4f190d6ffb 100644 --- a/modules/openstack/nodes/ignition.tf +++ b/modules/openstack/nodes/ignition.tf @@ -14,7 +14,7 @@ data "ignition_config" "node" { ] systemd = [ - "${data.ignition_systemd_unit.docker.id}", + "${var.ign_docker_dropin_id}", "${data.ignition_systemd_unit.locksmithd.id}", "${data.ignition_systemd_unit.kubelet.id}", "${data.ignition_systemd_unit.bootkube.id}", @@ -50,18 +50,6 @@ data "ignition_file" "hostname" { } } -data "ignition_systemd_unit" "docker" { - name = "docker.service" - enable = true - - dropin = [ - { - name = "10-dockeropts.conf" - content = "[Service]\nEnvironment=\"DOCKER_OPTS=--log-opt max-size=50m --log-opt max-file=3\"\n" - }, - ] -} - data "ignition_systemd_unit" "locksmithd" { name = "locksmithd.service" mask = true diff --git a/modules/vmware/node/ignition.tf b/modules/vmware/node/ignition.tf index 9cdcd409f3..654d00ee8f 100644 --- a/modules/vmware/node/ignition.tf +++ b/modules/vmware/node/ignition.tf @@ -12,7 +12,7 @@ data "ignition_config" "node" { ] systemd = [ - "${data.ignition_systemd_unit.docker.id}", + "${var.ign_docker_dropin_id}", "${data.ignition_systemd_unit.locksmithd.id}", "${data.ignition_systemd_unit.kubelet.id}", "${data.ignition_systemd_unit.kubelet-env.id}", @@ -30,18 +30,6 @@ data "ignition_user" "core" { ssh_authorized_keys = ["${var.core_public_keys}"] } -data "ignition_systemd_unit" "docker" { - name = "docker.service" - enable = true - - dropin = [ - { - name = "10-dockeropts.conf" - content = "[Service]\nEnvironment=\"DOCKER_OPTS=--log-opt max-size=50m --log-opt max-file=3\"\n" - }, - ] -} - data "ignition_systemd_unit" "locksmithd" { name = "locksmithd.service" mask = true diff --git a/platforms/aws/main.tf b/platforms/aws/main.tf index 8f4e75e51f..d34870e689 100644 --- a/platforms/aws/main.tf +++ b/platforms/aws/main.tf @@ -102,6 +102,7 @@ module "ignition_masters" { image_re = "${var.tectonic_image_re}" ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition.docker_dropin_id}" } module "masters" { @@ -152,6 +153,7 @@ module "ignition_workers" { image_re = "${var.tectonic_image_re}" ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition.docker_dropin_id}" } module "workers" { diff --git a/platforms/azure/main.tf b/platforms/azure/main.tf index 07db221ca1..7c78a34692 100644 --- a/platforms/azure/main.tf +++ b/platforms/azure/main.tf @@ -131,6 +131,7 @@ module "masters" { extra_tags = "${var.tectonic_azure_extra_tags}" ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition.docker_dropin_id}" } module "workers" { @@ -162,6 +163,7 @@ module "workers" { extra_tags = "${var.tectonic_azure_extra_tags}" ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition.docker_dropin_id}" } module "dns" { diff --git a/platforms/metal/matchers.tf b/platforms/metal/matchers.tf index 69771b18e2..713ba2d90a 100644 --- a/platforms/metal/matchers.tf +++ b/platforms/metal/matchers.tf @@ -60,6 +60,7 @@ resource "matchbox_group" "controller" { kubelet_image_tag = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}" ign_max_user_watches_json = "${jsonencode(module.ignition.max_user_watches_rendered)}" + ign_docker_dropin_json = "${jsonencode(module.ignition.docker_dropin_rendered)}" } } @@ -85,5 +86,6 @@ resource "matchbox_group" "worker" { kube_version_image = "${var.tectonic_container_images["kube_version"]}" ign_max_user_watches_json = "${jsonencode(module.ignition.max_user_watches_rendered)}" + ign_docker_dropin_json = "${jsonencode(module.ignition.docker_dropin_rendered)}" } } diff --git a/platforms/openstack/neutron/main.tf b/platforms/openstack/neutron/main.tf index be3177db99..87a18a1bbf 100644 --- a/platforms/openstack/neutron/main.tf +++ b/platforms/openstack/neutron/main.tf @@ -146,6 +146,7 @@ EOF tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition.docker_dropin_id}" } module "worker_nodes" { @@ -172,6 +173,7 @@ EOF tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition.docker_dropin_id}" } module "secrets" { diff --git a/platforms/vmware/main.tf b/platforms/vmware/main.tf index 3671819d37..ca81614d52 100644 --- a/platforms/vmware/main.tf +++ b/platforms/vmware/main.tf @@ -71,6 +71,7 @@ module "masters" { image_re = "${var.tectonic_image_re}" ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition.docker_dropin_id}" } module "workers" { @@ -107,4 +108,5 @@ module "workers" { image_re = "${var.tectonic_image_re}" ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition.docker_dropin_id}" } From 76f06e4294f55ef5242a5273aace97e79fdca1fc Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Wed, 23 Aug 2017 10:58:31 +0200 Subject: [PATCH 03/13] modules/ignition: unify kubelet --- modules/aws/ignition/outputs.tf | 3 - .../resources/services/kubelet.service | 45 ------ modules/aws/ignition/variables.tf | 60 -------- .../aws/{ignition => master-asg}/ignition.tf | 59 +------ modules/aws/master-asg/master.tf | 2 +- .../resources/detect-master.sh | 0 .../resources/init-assets.sh | 0 .../resources/services/init-assets.service | 0 .../variables-ignition.tf | 0 modules/aws/master-asg/variables.tf | 125 +++++++++------ modules/aws/worker-asg/ignition.tf | 18 +++ modules/aws/worker-asg/variables-ignition.tf | 1 + modules/aws/worker-asg/variables.tf | 13 +- modules/aws/worker-asg/worker.tf | 2 +- modules/azure/master-as/ignition-master.tf | 20 +-- .../resources/master-kubelet.service | 44 ------ modules/azure/master-as/variables.tf | 93 ++++------- modules/azure/worker-as/ignition-worker.tf | 19 +-- .../resources/worker-kubelet.service | 43 ------ modules/azure/worker-as/variables.tf | 85 ++++------ modules/ignition/assets.tf | 55 +++++++ modules/ignition/outputs.import | 4 + modules/ignition/outputs.tf | 24 +++ .../resources/bin}/s3-puller.sh | 0 .../resources/services/kubelet-env.service | 2 +- .../resources/services}/kubelet.service | 15 +- modules/ignition/variables.tf | 50 ++++++ modules/openstack/nodes/ignition.tf | 19 +-- modules/openstack/nodes/variables.tf | 70 +++------ modules/vmware/node/ignition.tf | 19 +-- .../node/resources/services/kubelet.service | 43 ------ modules/vmware/node/variables.tf | 118 ++++++-------- platforms/aws/main.tf | 145 ++++++++---------- platforms/azure/main.tf | 120 ++++++++------- .../metal/cl/bootkube-controller.yaml.tmpl | 42 +---- platforms/metal/cl/bootkube-worker.yaml.tmpl | 53 +------ platforms/metal/matchers.tf | 41 +++-- platforms/openstack/neutron/main.tf | 89 ++++++----- platforms/vmware/main.tf | 54 ++++--- 39 files changed, 641 insertions(+), 954 deletions(-) delete mode 100644 modules/aws/ignition/outputs.tf delete mode 100644 modules/aws/ignition/resources/services/kubelet.service delete mode 100644 modules/aws/ignition/variables.tf rename modules/aws/{ignition => master-asg}/ignition.tf (51%) rename modules/aws/{ignition => master-asg}/resources/detect-master.sh (100%) rename modules/aws/{ignition => master-asg}/resources/init-assets.sh (100%) rename modules/aws/{ignition => master-asg}/resources/services/init-assets.service (100%) rename modules/aws/{ignition => master-asg}/variables-ignition.tf (100%) create mode 100644 modules/aws/worker-asg/ignition.tf create mode 120000 modules/aws/worker-asg/variables-ignition.tf delete mode 100644 modules/azure/master-as/resources/master-kubelet.service delete mode 100644 modules/azure/worker-as/resources/worker-kubelet.service rename modules/{aws/ignition/resources => ignition/resources/bin}/s3-puller.sh (100%) rename modules/{aws => }/ignition/resources/services/kubelet-env.service (86%) rename modules/{openstack/nodes/resources => ignition/resources/services}/kubelet.service (89%) create mode 100644 modules/ignition/variables.tf delete mode 100644 modules/vmware/node/resources/services/kubelet.service diff --git a/modules/aws/ignition/outputs.tf b/modules/aws/ignition/outputs.tf deleted file mode 100644 index 808eca355c..0000000000 --- a/modules/aws/ignition/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "ignition" { - value = "${data.ignition_config.main.rendered}" -} diff --git a/modules/aws/ignition/resources/services/kubelet.service b/modules/aws/ignition/resources/services/kubelet.service deleted file mode 100644 index b74cb13ee1..0000000000 --- a/modules/aws/ignition/resources/services/kubelet.service +++ /dev/null @@ -1,45 +0,0 @@ -[Unit] -Description=Kubelet via Hyperkube ACI - -[Service] -EnvironmentFile=/etc/kubernetes/kubelet.env -Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log" - -ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests \ - /srv/kubernetes/manifests /etc/kubernetes/checkpoint-secrets \ - /etc/kubernetes/cni/net.d /var/lib/cni -ExecStartPre=/usr/bin/bash -c "/opt/s3-puller.sh ${kubeconfig_s3_location} /etc/kubernetes/kubeconfig" -ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" -ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - -ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --require-kubeconfig \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --allow-privileged \ - --node-labels=${node_label} \ - ${node_taints_param} \ - ${cni_bin_dir_flag} \ - --minimum-container-ttl-duration=6m0s \ - --cluster-dns=${cluster_dns_ip} \ - --cluster-domain=cluster.local \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --anonymous-auth=false \ - --cloud-provider=aws -ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid - -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/modules/aws/ignition/variables.tf b/modules/aws/ignition/variables.tf deleted file mode 100644 index 599230e95e..0000000000 --- a/modules/aws/ignition/variables.tf +++ /dev/null @@ -1,60 +0,0 @@ -variable "container_images" { - description = "Container images to use" - type = "map" -} - -variable "assets_s3_location" { - type = "string" - description = "Location on S3 of the Bootkube/Tectonic assets to use (bucket/key)" -} - -variable "kubeconfig_s3_location" { - type = "string" - description = "Location on S3 of the kubeconfig file to use (bucket/key)" -} - -variable "kube_dns_service_ip" { - type = "string" - description = "Service IP used to reach kube-dns" -} - -variable "kubelet_node_label" { - type = "string" - description = "Label that Kubelet will apply on the node" -} - -variable "kubelet_node_taints" { - type = "string" - description = "Taints that Kubelet will apply on the node" -} - -variable "kubelet_cni_bin_dir" { - type = "string" -} - -variable "bootkube_service" { - type = "string" - description = "The content of the bootkube systemd service unit" -} - -variable "tectonic_service" { - type = "string" - description = "The content of the tectonic installer systemd service unit" -} - -variable "tectonic_service_disabled" { - description = "Specifies whether the tectonic installer systemd unit will be disabled. If true, no tectonic assets will be deployed" - default = false -} - -variable "cluster_name" { - type = "string" -} - -variable "image_re" { - description = < /etc/kubernetes/ca.crt" -ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid -ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --require-kubeconfig \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --allow-privileged \ - --node-labels=${node_label} \ - ${node_taints_param} \ - ${cni_bin_dir_flag} \ - --minimum-container-ttl-duration=6m0s \ - --cluster_dns=${cluster_dns} \ - --cluster_domain=cluster.local \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --anonymous-auth=false \ - --cloud-provider="${cloud_provider}" \ - --cloud-config=/etc/kubernetes/cloud/config -ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/modules/azure/master-as/variables.tf b/modules/azure/master-as/variables.tf index 042eb26226..7428e28b61 100644 --- a/modules/azure/master-as/variables.tf +++ b/modules/azure/master-as/variables.tf @@ -1,46 +1,30 @@ -// Location is the Azure Location (East US, West US, etc) -variable "location" { - type = "string" -} - -variable "resource_group_name" { - type = "string" -} - -variable "cluster_id" { - type = "string" -} - -// VM Size name -variable "vm_size" { - type = "string" +variable "bootkube_service" { + type = "string" + description = "The content of the bootkube systemd service unit" } -// Storage account type -variable "storage_type" { +variable "cl_channel" { type = "string" } -variable "storage_id" { +variable "cloud_provider_config" { type = "string" } -// The base DNS domain of the cluster. -// Example: `azure.dev.coreos.systems` -variable "base_domain" { +variable "cluster_id" { type = "string" } -// The name of the cluster. variable "cluster_name" { - type = "string" + type = "string" + description = "The name of the cluster." } -variable "public_ssh_key" { - type = "string" +variable "extra_tags" { + type = "map" } -variable "virtual_network" { +variable "kube_image_tag" { type = "string" } @@ -48,48 +32,40 @@ variable "kube_image_url" { type = "string" } -variable "kube_image_tag" { - type = "string" -} - variable "kubeconfig_content" { type = "string" } -// Count of master nodes to be created. -variable "master_count" { - type = "string" -} - -variable "tectonic_kube_dns_service_ip" { - type = "string" +variable "location" { + type = "string" + description = "Location is the Azure Location (East US, West US, etc)" } -variable "cloud_provider" { - type = "string" - default = "azure" +variable "master_count" { + type = "string" + description = "Count of master nodes to be created." } -variable "cloud_provider_config" { - description = "Content of cloud provider config" - type = "string" +variable "network_interface_ids" { + type = "list" + description = "List of NICs to use for master VMs" } -variable "kubelet_node_label" { +variable "public_ssh_key" { type = "string" } -variable "kubelet_node_taints" { +variable "resource_group_name" { type = "string" } -variable "kubelet_cni_bin_dir" { +variable "storage_id" { type = "string" } -variable "bootkube_service" { +variable "storage_type" { type = "string" - description = "The content of the bootkube systemd service unit" + description = "Storage account type" } variable "tectonic_service" { @@ -102,20 +78,7 @@ variable "tectonic_service_disabled" { default = false } -variable "network_interface_ids" { - type = "list" - description = "List of NICs to use for master VMs" -} - -variable "versions" { - description = "(internal) Versions of the components to use" - type = "map" -} - -variable "cl_channel" { - type = "string" -} - -variable "extra_tags" { - type = "map" +variable "vm_size" { + type = "string" + description = "VM Size name" } diff --git a/modules/azure/worker-as/ignition-worker.tf b/modules/azure/worker-as/ignition-worker.tf index f79f7e41c9..9e3f789fed 100644 --- a/modules/azure/worker-as/ignition-worker.tf +++ b/modules/azure/worker-as/ignition-worker.tf @@ -10,7 +10,7 @@ data "ignition_config" "worker" { systemd = [ "${var.ign_docker_dropin_id}", "${data.ignition_systemd_unit.locksmithd.id}", - "${data.ignition_systemd_unit.kubelet-worker.id}", + "${var.ign_kubelet_service_id}", "${module.net_ignition.tx-off_id}", ] @@ -24,23 +24,6 @@ data "ignition_systemd_unit" "locksmithd" { mask = true } -data "template_file" "kubelet-worker" { - template = "${file("${path.module}/resources/worker-kubelet.service")}" - - vars { - node_label = "${var.kubelet_node_label}" - cloud_provider = "${var.cloud_provider}" - cluster_dns = "${var.tectonic_kube_dns_service_ip}" - cni_bin_dir_flag = "${var.kubelet_cni_bin_dir != "" ? "--cni-bin-dir=${var.kubelet_cni_bin_dir}" : ""}" - } -} - -data "ignition_systemd_unit" "kubelet-worker" { - name = "kubelet.service" - enable = true - content = "${data.template_file.kubelet-worker.rendered}" -} - data "ignition_file" "kubeconfig" { filesystem = "root" path = "/etc/kubernetes/kubeconfig" diff --git a/modules/azure/worker-as/resources/worker-kubelet.service b/modules/azure/worker-as/resources/worker-kubelet.service deleted file mode 100644 index 025a3f68db..0000000000 --- a/modules/azure/worker-as/resources/worker-kubelet.service +++ /dev/null @@ -1,43 +0,0 @@ -[Unit] -Description=Kubelet via Hyperkube ACI - -[Service] -Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log" -EnvironmentFile=/etc/kubernetes/kubelet.env -ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests -ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests -ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets -ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d -ExecStartPre=/bin/mkdir -p /var/lib/cni -ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" -ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid -ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --require-kubeconfig \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --allow-privileged \ - --node-labels=${node_label} \ - ${cni_bin_dir_flag} \ - --minimum-container-ttl-duration=6m0s \ - --cluster_dns=${cluster_dns} \ - --cluster_domain=cluster.local \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --anonymous-auth=false \ - --cloud-provider="${cloud_provider}" \ - --cloud-config=/etc/kubernetes/cloud/config -ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/modules/azure/worker-as/variables.tf b/modules/azure/worker-as/variables.tf index e5fd95b515..aba15bae9d 100644 --- a/modules/azure/worker-as/variables.tf +++ b/modules/azure/worker-as/variables.tf @@ -1,45 +1,31 @@ -// Location is the Azure Location (East US, West US, etc) -variable "location" { - type = "string" -} - -variable "resource_group_name" { - type = "string" -} - -variable "cluster_id" { - type = "string" -} - -// VM Size name -variable "vm_size" { +variable "cl_channel" { type = "string" } -// Storage account type -variable "storage_type" { - type = "string" +variable "cloud_provider" { + type = "string" + default = "azure" } -variable "storage_id" { - type = "string" +variable "cloud_provider_config" { + description = "Content of cloud provider config" + type = "string" } -// Count of worker nodes to be created. -variable "worker_count" { +variable "cluster_id" { type = "string" } -// The name of the cluster. variable "cluster_name" { - type = "string" + type = "string" + description = "The name of the cluster." } -variable "public_ssh_key" { - type = "string" +variable "extra_tags" { + type = "map" } -variable "virtual_network" { +variable "kube_image_tag" { type = "string" } @@ -47,51 +33,48 @@ variable "kube_image_url" { type = "string" } -variable "kube_image_tag" { - type = "string" -} - variable "kubeconfig_content" { type = "string" default = "" } -variable "tectonic_kube_dns_service_ip" { - type = "string" +variable "location" { + type = "string" + description = "Location is the Azure Location (East US, West US, etc)" } -variable "cloud_provider" { - type = "string" - default = "azure" +variable "network_interface_ids" { + type = "list" + description = "List of NICs to use for master VMs" } -variable "cloud_provider_config" { - description = "Content of cloud provider config" - type = "string" +variable "public_ssh_key" { + type = "string" } -variable "kubelet_node_label" { +variable "resource_group_name" { type = "string" } -variable "network_interface_ids" { - type = "list" - description = "List of NICs to use for master VMs" +variable "storage_id" { + type = "string" } -variable "versions" { - description = "(internal) Versions of the components to use" - type = "map" +variable "storage_type" { + type = "string" + description = "Storage account type" } -variable "cl_channel" { +variable "tectonic_kube_dns_service_ip" { type = "string" } -variable "kubelet_cni_bin_dir" { - type = "string" +variable "vm_size" { + type = "string" + description = "VM Size name" } -variable "extra_tags" { - type = "map" +variable "worker_count" { + type = "string" + description = "Count of worker nodes to be created." } diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf index fb0884d744..673ba133a8 100644 --- a/modules/ignition/assets.tf +++ b/modules/ignition/assets.tf @@ -27,3 +27,58 @@ data "ignition_systemd_unit" "docker_dropin" { }, ] } + +data "template_file" "kubelet" { + template = "${file("${path.module}/resources/services/kubelet.service")}" + + vars { + cloud_provider = "${var.cloud_provider != "" ? "--cloud-provider=${var.cloud_provider}" : ""}" + cloud_provider_config = "${var.cloud_provider_config != "" ? "--cloud-config=/etc/kubernetes/cloud/config" : ""}" + cluster_dns_ip = "${var.kube_dns_service_ip}" + cni_bin_dir_flag = "${var.kubelet_cni_bin_dir != "" ? "--cni-bin-dir=${var.kubelet_cni_bin_dir}" : ""}" + kubeconfig_fetch_cmd = "${var.kubeconfig_fetch_cmd != "" ? "ExecStartPre=${var.kubeconfig_fetch_cmd}" : ""}" + node_label = "${var.kubelet_node_label}" + node_taints_param = "${var.kubelet_node_taints != "" ? "--register-with-taints=${var.kubelet_node_taints}" : ""}" + } +} + +data "ignition_systemd_unit" "kubelet" { + name = "kubelet.service" + enable = true + content = "${data.template_file.kubelet.rendered}" +} + +data "template_file" "kubelet_env" { + template = "${file("${path.module}/resources/services/kubelet-env.service")}" + + vars { + kube_version_image_url = "${replace(var.container_images["kube_version"],var.image_re,"$1")}" + kube_version_image_tag = "${replace(var.container_images["kube_version"],var.image_re,"$2")}" + kubelet_image_url = "${replace(var.container_images["hyperkube"],var.image_re,"$1")}" + kubeconfig_fetch_cmd = "${var.kubeconfig_fetch_cmd != "" ? "ExecStartPre=${var.kubeconfig_fetch_cmd}" : ""}" + } +} + +data "ignition_systemd_unit" "kubelet_env" { + name = "kubelet-env.service" + enable = true + content = "${data.template_file.kubelet_env.rendered}" +} + +data "template_file" "s3_puller" { + template = "${file("${path.module}/resources/bin/s3-puller.sh")}" + + vars { + awscli_image = "${var.container_images["awscli"]}" + } +} + +data "ignition_file" "s3_puller" { + filesystem = "root" + path = "/opt/s3-puller.sh" + mode = 0755 + + content { + content = "${data.template_file.s3_puller.rendered}" + } +} diff --git a/modules/ignition/outputs.import b/modules/ignition/outputs.import index 4ba3adf54e..c7ddbe10cb 100644 --- a/modules/ignition/outputs.import +++ b/modules/ignition/outputs.import @@ -7,3 +7,7 @@ variable "ign_max_user_watches_id" { variable "ign_docker_dropin_id" { type = "string" } + +variable "ign_kubelet_service_id" { + type = "string" +} diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf index 7aef1d36dc..c4ecb7fdf6 100644 --- a/modules/ignition/outputs.tf +++ b/modules/ignition/outputs.tf @@ -13,3 +13,27 @@ output "docker_dropin_id" { output "docker_dropin_rendered" { value = "${data.template_file.docker_dropin.rendered}" } + +output "kubelet_service_id" { + value = "${data.ignition_systemd_unit.kubelet.id}" +} + +output "kubelet_service_rendered" { + value = "${data.template_file.kubelet.rendered}" +} + +output "kubelet_env_service_id" { + value = "${data.ignition_systemd_unit.kubelet_env.id}" +} + +output "kubelet_env_service_rendered" { + value = "${data.template_file.kubelet_env.rendered}" +} + +output "s3_puller_id" { + value = "${data.ignition_file.s3_puller.id}" +} + +output "s3_puller_rendered" { + value = "${data.template_file.s3_puller.rendered}" +} diff --git a/modules/aws/ignition/resources/s3-puller.sh b/modules/ignition/resources/bin/s3-puller.sh similarity index 100% rename from modules/aws/ignition/resources/s3-puller.sh rename to modules/ignition/resources/bin/s3-puller.sh diff --git a/modules/aws/ignition/resources/services/kubelet-env.service b/modules/ignition/resources/services/kubelet-env.service similarity index 86% rename from modules/aws/ignition/resources/services/kubelet-env.service rename to modules/ignition/resources/services/kubelet-env.service index e9b6515502..bdbec3f50b 100644 --- a/modules/aws/ignition/resources/services/kubelet-env.service +++ b/modules/ignition/resources/services/kubelet-env.service @@ -5,7 +5,7 @@ ConditionPathExists=!/etc/kubernetes/kubelet.env [Service] ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes -ExecStartPre=/usr/bin/bash -c "/opt/s3-puller.sh ${kubeconfig_s3_location} /etc/kubernetes/kubeconfig" +${kubeconfig_fetch_cmd} ExecStartPre=/usr/bin/bash -c "docker run --rm -v /etc/kubernetes:/etc/kubernetes ${kube_version_image_url}:${kube_version_image_tag} --kubeconfig=/etc/kubernetes/kubeconfig > /etc/kubernetes/kube.version" ExecStart=/usr/bin/bash -c "echo KUBELET_IMAGE_URL=${kubelet_image_url} > /etc/kubernetes/kubelet.env; echo KUBELET_IMAGE_TAG=$(tr '+' '_' < /etc/kubernetes/kube.version) >> /etc/kubernetes/kubelet.env; rm /etc/kubernetes/kube.version" Restart=on-failure diff --git a/modules/openstack/nodes/resources/kubelet.service b/modules/ignition/resources/services/kubelet.service similarity index 89% rename from modules/openstack/nodes/resources/kubelet.service rename to modules/ignition/resources/services/kubelet.service index 95575624c7..b9add6965b 100644 --- a/modules/openstack/nodes/resources/kubelet.service +++ b/modules/ignition/resources/services/kubelet.service @@ -2,6 +2,7 @@ Description=Kubelet via Hyperkube ACI [Service] +EnvironmentFile=/etc/kubernetes/kubelet.env Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ --volume=resolv,kind=host,source=/etc/resolv.conf \ --mount volume=resolv,target=/etc/resolv.conf \ @@ -9,14 +10,16 @@ Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ --mount volume=var-lib-cni,target=/var/lib/cni \ --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log" -EnvironmentFile=/etc/kubernetes/kubelet.env + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /var/lib/cni +${kubeconfig_fetch_cmd} ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=/usr/lib/coreos/kubelet-wrapper \ --kubeconfig=/etc/kubernetes/kubeconfig \ --require-kubeconfig \ @@ -26,15 +29,19 @@ ExecStart=/usr/lib/coreos/kubelet-wrapper \ --exit-on-lock-contention \ --pod-manifest-path=/etc/kubernetes/manifests \ --allow-privileged \ - --node-labels=${node_labels} \ + --node-labels=${node_label} \ ${node_taints_param} \ ${cni_bin_dir_flag} \ --minimum-container-ttl-duration=6m0s \ - --cluster_dns=${cluster_dns} \ - --cluster_domain=cluster.local \ + --cluster-dns=${cluster_dns_ip} \ + --cluster-domain=cluster.local \ --client-ca-file=/etc/kubernetes/ca.crt \ + ${cloud_provider} \ + ${cloud_provider_config} \ --anonymous-auth=false + ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + Restart=always RestartSec=10 diff --git a/modules/ignition/variables.tf b/modules/ignition/variables.tf new file mode 100644 index 0000000000..55f4f00f5a --- /dev/null +++ b/modules/ignition/variables.tf @@ -0,0 +1,50 @@ +variable "container_images" { + description = "Container images to use" + type = "map" +} + +variable "image_re" { + description = < /etc/kubernetes/ca.crt" -ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - -ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --require-kubeconfig \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --allow-privileged \ - --node-labels=${node_label} \ - ${node_taints_param} \ - ${cni_bin_dir_flag} \ - --minimum-container-ttl-duration=6m0s \ - --cluster-dns=${cluster_dns_ip} \ - --cluster-domain=cluster.local \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --anonymous-auth=false -ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid - -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/modules/vmware/node/variables.tf b/modules/vmware/node/variables.tf index 41124769f4..de724eae99 100644 --- a/modules/vmware/node/variables.tf +++ b/modules/vmware/node/variables.tf @@ -1,39 +1,44 @@ -variable "instance_count" { - type = "string" - description = "Number of nodes to be created." -} - variable "base_domain" { type = "string" } +variable "bootkube_service" { + type = "string" + description = "The content of the bootkube systemd service unit" +} + variable "container_images" { description = "Container images to use" type = "map" } -variable "kube_dns_service_ip" { +variable "image_re" { + description = "(internal) Regular expression used to extract repo and tag components from image strings" type = "string" - description = "Service IP used to reach kube-dns" } -variable "kubelet_node_label" { +variable "instance_count" { type = "string" - description = "Label that Kubelet will apply on the node" + description = "Number of nodes to be created." } -variable "kubelet_node_taints" { - type = "string" - description = "Taints that Kubelet will apply on the node" +variable "kube_image_tag" { + type = "string" } -variable "kubelet_cni_bin_dir" { +variable "kube_image_url" { type = "string" } -variable "bootkube_service" { +variable "kubeconfig" { type = "string" - description = "The content of the bootkube systemd service unit" + description = "Contents of Kubeconfig" +} + +variable "private_key" { + type = "string" + description = "SSH private key file in .pem format corresponding to tectonic_vmware_ssh_authorized_key. If not provided, SSH agent will be used." + default = "" } variable "tectonic_service" { @@ -46,99 +51,72 @@ variable "tectonic_service_disabled" { default = false } -variable dns_server { - type = "string" - description = "DNS Server of the nodes" -} - -variable ip_address { - type = "map" - description = "IP Address of the node" -} - -variable gateway { +variable "vmware_folder" { type = "string" - description = "Gateway of the node" -} - -variable hostname { - type = "map" - description = "Hostname of the node" + description = "Name of the VMware folder to create objects in" } -variable core_public_keys { +variable "core_public_keys" { type = "list" description = "Public Key for Core User" } -variable vmware_datacenter { - type = "string" - description = "vSphere Datacenter to create VMs in" -} - -variable vmware_cluster { +variable "dns_server" { type = "string" - description = "vSphere Cluster to create VMs in" + description = "DNS Server of the nodes" } -variable vm_vcpu { +variable "gateway" { type = "string" - description = "VMs vCPU count" + description = "Gateway of the node" } -variable vm_memory { - type = "string" - description = "VMs Memory size in MB" +variable "hostname" { + type = "map" + description = "Hostname of the node" } -variable vm_network_label { - type = "string" - description = "VMs PortGroup" +variable "ip_address" { + type = "map" + description = "IP Address of the node" } -variable vm_disk_datastore { +variable "vm_disk_datastore" { type = "string" description = "Datastore to create VM(s) in " } -variable vm_disk_template { +variable "vm_disk_template" { type = "string" description = "Disk template to use for cloning CoreOS Container Linux" } -variable vm_disk_template_folder { +variable "vm_disk_template_folder" { type = "string" description = "vSphere Folder CoreOS Container Linux is located in" } -variable "vmware_folder" { +variable "vm_memory" { type = "string" - description = "Name of the VMware folder to create objects in" -} - -variable "kube_image_url" { - type = "string" + description = "VMs Memory size in MB" } -variable "kube_image_tag" { - type = "string" +variable "vm_network_label" { + type = "string" + description = "VMs PortGroup" } -variable "kubeconfig" { +variable "vm_vcpu" { type = "string" - description = "Contents of Kubeconfig" + description = "VMs vCPU count" } -variable "private_key" { +variable "vmware_cluster" { type = "string" - description = "SSH private key file in .pem format corresponding to tectonic_vmware_ssh_authorized_key. If not provided, SSH agent will be used." - default = "" + description = "vSphere Cluster to create VMs in" } -variable "image_re" { - description = < /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --require-kubeconfig \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --anonymous-auth=false \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --allow-privileged \ - --hostname-override={{.domain_name}} \ - --node-labels=node-role.kubernetes.io/master \ - {{.cni_bin_dir_flag}} \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ - --cluster_dns={{.k8s_dns_service_ip}} \ - --cluster_domain=cluster.local - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid - Restart=always - RestartSec=10 - [Install] - WantedBy=multi-user.target + contents: {{.ign_kubelet_service_json}} - name: bootkube.service contents: | [Unit] diff --git a/platforms/metal/cl/bootkube-worker.yaml.tmpl b/platforms/metal/cl/bootkube-worker.yaml.tmpl index ad361c9f5c..01c0116a13 100644 --- a/platforms/metal/cl/bootkube-worker.yaml.tmpl +++ b/platforms/metal/cl/bootkube-worker.yaml.tmpl @@ -29,58 +29,9 @@ systemd: RequiredBy=kubelet.service - name: kubelet-env.service enable: true - contents: | - [Unit] - Description=Determine the Kubelet Image Version - ConditionPathExists=!/etc/kubernetes/kubelet.env - [Service] - ExecStartPre=/usr/bin/bash -c "docker run --rm -v /etc/kubernetes:/etc/kubernetes {{.kube_version_image}} --kubeconfig=/etc/kubernetes/kubeconfig > /etc/kubernetes/kube.version" - ExecStart=/usr/bin/bash -c "echo KUBELET_IMAGE_URL={{.kubelet_image_url}} > /etc/kubernetes/kubelet.env; echo KUBELET_IMAGE_TAG=$(tr '+' '_' < /etc/kubernetes/kube.version) >> /etc/kubernetes/kubelet.env; rm /etc/kubernetes/kube.version" - Restart=on-failure - RestartSec=10 - [Install] - WantedBy=multi-user.target + contents: {{.ign_kubelet_env_service_json}} - name: kubelet.service - contents: | - [Unit] - Description=Kubelet via Hyperkube ACI - [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log" - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets - ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests - ExecStartPre=/bin/mkdir -p /var/lib/cni - ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --require-kubeconfig \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --anonymous-auth=false \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --allow-privileged \ - --hostname-override={{.domain_name}} \ - --node-labels=node-role.kubernetes.io/node \ - {{.cni_bin_dir_flag}} \ - --cluster_dns={{.k8s_dns_service_ip}} \ - --cluster_domain=cluster.local - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid - Restart=always - RestartSec=5 - [Install] - WantedBy=multi-user.target + contents: {{.ign_kubelet_service_json}} storage: files: diff --git a/platforms/metal/matchers.tf b/platforms/metal/matchers.tf index 713ba2d90a..e4e509c154 100644 --- a/platforms/metal/matchers.tf +++ b/platforms/metal/matchers.tf @@ -1,7 +1,3 @@ -module "ignition" { - source = "../../modules/ignition" -} - // Install CoreOS to disk resource "matchbox_group" "coreos_install" { count = "${length(var.tectonic_metal_controller_names) + length(var.tectonic_metal_worker_names)}" @@ -23,6 +19,17 @@ resource "matchbox_group" "coreos_install" { // DO NOT PLACE SECRETS IN USER-DATA +module "ignition_masters" { + source = "../../modules/ignition" + + container_images = "${var.tectonic_container_images}" + image_re = "${var.tectonic_image_re}" + kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" + kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }" + kubelet_node_label = "node-role.kubernetes.io/master" + kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule" +} + resource "matchbox_group" "controller" { count = "${length(var.tectonic_metal_controller_names)}" name = "${format("%s-%s", var.tectonic_cluster_name, element(var.tectonic_metal_controller_names, count.index))}" @@ -35,8 +42,6 @@ resource "matchbox_group" "controller" { metadata { domain_name = "${element(var.tectonic_metal_controller_domains, count.index)}" - k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" - cni_bin_dir_flag = "${var.tectonic_calico_network_policy ? "--cni-bin-dir=/var/lib/cni/bin" : "" }" ssh_authorized_key = "${var.tectonic_ssh_authorized_key}" exclude_tectonic = "${var.tectonic_vanilla_k8s}" @@ -59,11 +64,23 @@ resource "matchbox_group" "controller" { kubelet_image_url = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$1")}" kubelet_image_tag = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}" - ign_max_user_watches_json = "${jsonencode(module.ignition.max_user_watches_rendered)}" - ign_docker_dropin_json = "${jsonencode(module.ignition.docker_dropin_rendered)}" + ign_max_user_watches_json = "${jsonencode(module.ignition_masters.max_user_watches_rendered)}" + ign_docker_dropin_json = "${jsonencode(module.ignition_masters.docker_dropin_rendered)}" + ign_kubelet_service_json = "${jsonencode(module.ignition_masters.kubelet_service_rendered)}" } } +module "ignition_workers" { + source = "../../modules/ignition" + + container_images = "${var.tectonic_container_images}" + image_re = "${var.tectonic_image_re}" + kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" + kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }" + kubelet_node_label = "node-role.kubernetes.io/node" + kubelet_node_taints = "" +} + resource "matchbox_group" "worker" { count = "${length(var.tectonic_metal_worker_names)}" name = "${format("%s-%s", var.tectonic_cluster_name, element(var.tectonic_metal_worker_names, count.index))}" @@ -76,8 +93,6 @@ resource "matchbox_group" "worker" { metadata { domain_name = "${element(var.tectonic_metal_worker_domains, count.index)}" - k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" - cni_bin_dir_flag = "${var.tectonic_calico_network_policy ? "--cni-bin-dir=/var/lib/cni/bin" : "" }" ssh_authorized_key = "${var.tectonic_ssh_authorized_key}" # extra data @@ -85,7 +100,9 @@ resource "matchbox_group" "worker" { kubelet_image_tag = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}" kube_version_image = "${var.tectonic_container_images["kube_version"]}" - ign_max_user_watches_json = "${jsonencode(module.ignition.max_user_watches_rendered)}" - ign_docker_dropin_json = "${jsonencode(module.ignition.docker_dropin_rendered)}" + ign_docker_dropin_json = "${jsonencode(module.ignition_workers.docker_dropin_rendered)}" + ign_kubelet_env_service_json = "${jsonencode(module.ignition_workers.kubelet_env_service_rendered)}" + ign_kubelet_service_json = "${jsonencode(module.ignition_workers.kubelet_service_rendered)}" + ign_max_user_watches_json = "${jsonencode(module.ignition_workers.max_user_watches_rendered)}" } } diff --git a/platforms/openstack/neutron/main.tf b/platforms/openstack/neutron/main.tf index 87a18a1bbf..60c873bf4b 100644 --- a/platforms/openstack/neutron/main.tf +++ b/platforms/openstack/neutron/main.tf @@ -1,7 +1,3 @@ -module "ignition" { - source = "../../../modules/ignition" -} - module "bootkube" { source = "../../../modules/bootkube" cloud_provider = "" @@ -121,6 +117,17 @@ data "null_data_source" "local" { } } +module "ignition_masters" { + source = "../../../modules/ignition" + + container_images = "${var.tectonic_container_images}" + image_re = "${var.tectonic_image_re}" + kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" + kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }" + kubelet_node_label = "node-role.kubernetes.io/master" + kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule" +} + module "master_nodes" { source = "../../../modules/openstack/nodes" @@ -129,24 +136,31 @@ search ${var.tectonic_base_domain} ${join("\n", formatlist("nameserver %s", var.tectonic_openstack_dns_nameservers))} EOF - kubeconfig_content = "${module.bootkube.kubeconfig}" - cluster_name = "${var.tectonic_cluster_name}" - instance_count = "${var.tectonic_master_count}" - kube_image_url = "${data.null_data_source.local.outputs.kube_image_url}" - kube_image_tag = "${data.null_data_source.local.outputs.kube_image_tag}" - tectonic_kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" - core_public_keys = ["${module.secrets.core_public_key_openssh}"] - bootkube_service = "${module.bootkube.systemd_service}" - tectonic_service = "${module.tectonic.systemd_service}" - hostname_infix = "master" - node_labels = "node-role.kubernetes.io/master" - node_taints = "node-role.kubernetes.io/master=:NoSchedule" - kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }" - tectonic_experimental = "${var.tectonic_experimental}" - tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" - - ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" - ign_docker_dropin_id = "${module.ignition.docker_dropin_id}" + bootkube_service = "${module.bootkube.systemd_service}" + cluster_name = "${var.tectonic_cluster_name}" + core_public_keys = ["${module.secrets.core_public_key_openssh}"] + hostname_infix = "master" + instance_count = "${var.tectonic_master_count}" + kube_image_tag = "${data.null_data_source.local.outputs.kube_image_tag}" + kube_image_url = "${data.null_data_source.local.outputs.kube_image_url}" + kubeconfig_content = "${module.bootkube.kubeconfig}" + tectonic_service = "${module.tectonic.systemd_service}" + tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" + + ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" + ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" + ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" +} + +module "ignition_workers" { + source = "../../../modules/ignition" + + container_images = "${var.tectonic_container_images}" + image_re = "${var.tectonic_image_re}" + kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" + kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }" + kubelet_node_label = "node-role.kubernetes.io/node" + kubelet_node_taints = "" } module "worker_nodes" { @@ -157,23 +171,20 @@ search ${var.tectonic_base_domain} ${join("\n", formatlist("nameserver %s", var.tectonic_openstack_dns_nameservers))} EOF - kubeconfig_content = "${module.bootkube.kubeconfig}" - cluster_name = "${var.tectonic_cluster_name}" - instance_count = "${var.tectonic_worker_count}" - kube_image_url = "${data.null_data_source.local.outputs.kube_image_url}" - kube_image_tag = "${data.null_data_source.local.outputs.kube_image_tag}" - tectonic_kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" - core_public_keys = ["${module.secrets.core_public_key_openssh}"] - bootkube_service = "" - tectonic_service = "" - hostname_infix = "worker" - node_labels = "node-role.kubernetes.io/node" - node_taints = "" - kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }" - tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" - - ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" - ign_docker_dropin_id = "${module.ignition.docker_dropin_id}" + bootkube_service = "" + cluster_name = "${var.tectonic_cluster_name}" + core_public_keys = ["${module.secrets.core_public_key_openssh}"] + hostname_infix = "worker" + instance_count = "${var.tectonic_worker_count}" + kube_image_tag = "${data.null_data_source.local.outputs.kube_image_tag}" + kube_image_url = "${data.null_data_source.local.outputs.kube_image_url}" + kubeconfig_content = "${module.bootkube.kubeconfig}" + tectonic_service = "" + tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" + + ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}" + ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" + ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" } module "secrets" { diff --git a/platforms/vmware/main.tf b/platforms/vmware/main.tf index ca81614d52..d61a4abf32 100644 --- a/platforms/vmware/main.tf +++ b/platforms/vmware/main.tf @@ -1,7 +1,3 @@ -module "ignition" { - source = "../../modules/ignition" -} - module "etcd" { source = "../../modules/vmware/etcd" instance_count = "${var.tectonic_experimental ? 0 : var.tectonic_etcd_count }" @@ -36,6 +32,17 @@ module "etcd" { vmware_folder = "${vsphere_folder.tectonic_vsphere_folder.path}" } +module "ignition_masters" { + source = "../../modules/ignition" + + container_images = "${var.tectonic_container_images}" + image_re = "${var.tectonic_image_re}" + kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" + kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }" + kubelet_node_label = "node-role.kubernetes.io/master" + kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule" +} + module "masters" { source = "../../modules/vmware/node" instance_count = "${var.tectonic_master_count}" @@ -46,10 +53,6 @@ module "masters" { ip_address = "${var.tectonic_vmware_master_ip}" gateway = "${var.tectonic_vmware_master_gateway}" - kubelet_node_label = "node-role.kubernetes.io/master" - kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule" - kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }" - kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" container_images = "${var.tectonic_container_images}" bootkube_service = "${module.bootkube.systemd_service}" tectonic_service = "${module.tectonic.systemd_service}" @@ -70,8 +73,20 @@ module "masters" { private_key = "${var.tectonic_vmware_ssh_private_key_path}" image_re = "${var.tectonic_image_re}" - ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" - ign_docker_dropin_id = "${module.ignition.docker_dropin_id}" + ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" + ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" + ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" +} + +module "ignition_workers" { + source = "../../modules/ignition" + + container_images = "${var.tectonic_container_images}" + image_re = "${var.tectonic_image_re}" + kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" + kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }" + kubelet_node_label = "node-role.kubernetes.io/node" + kubelet_node_taints = "" } module "workers" { @@ -84,15 +99,11 @@ module "workers" { ip_address = "${var.tectonic_vmware_worker_ip}" gateway = "${var.tectonic_vmware_worker_gateway}" - kubelet_node_label = "node-role.kubernetes.io/node" - kubelet_node_taints = "" - kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }" - kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" - container_images = "${var.tectonic_container_images}" - bootkube_service = "" - tectonic_service = "" - kube_image_url = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$1")}" - kube_image_tag = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}" + container_images = "${var.tectonic_container_images}" + bootkube_service = "" + tectonic_service = "" + kube_image_url = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$1")}" + kube_image_tag = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}" vmware_datacenter = "${var.tectonic_vmware_datacenter}" vmware_cluster = "${var.tectonic_vmware_cluster}" @@ -107,6 +118,7 @@ module "workers" { private_key = "${var.tectonic_vmware_ssh_private_key_path}" image_re = "${var.tectonic_image_re}" - ign_max_user_watches_id = "${module.ignition.max_user_watches_id}" - ign_docker_dropin_id = "${module.ignition.docker_dropin_id}" + ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" + ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" + ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" } From b5fc1d10422eace85039f6a0d2b6162242581084 Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Thu, 24 Aug 2017 15:29:48 +0200 Subject: [PATCH 04/13] modules/ignition: unify locksmithd service --- modules/aws/master-asg/ignition.tf | 7 +------ modules/aws/worker-asg/ignition.tf | 7 +------ modules/azure/master-as/ignition-master.tf | 7 +------ modules/azure/worker-as/ignition-worker.tf | 7 +------ modules/ignition/assets.tf | 5 +++++ modules/ignition/outputs.import | 4 ++++ modules/ignition/outputs.tf | 4 ++++ modules/openstack/nodes/ignition.tf | 7 +------ modules/vmware/node/ignition.tf | 7 +------ platforms/aws/main.tf | 8 +++++--- platforms/azure/main.tf | 16 +++++++++------- platforms/openstack/neutron/main.tf | 14 ++++++++------ platforms/vmware/main.tf | 14 ++++++++------ 13 files changed, 49 insertions(+), 58 deletions(-) diff --git a/modules/aws/master-asg/ignition.tf b/modules/aws/master-asg/ignition.tf index 4e61da77d5..1c6436065a 100644 --- a/modules/aws/master-asg/ignition.tf +++ b/modules/aws/master-asg/ignition.tf @@ -8,7 +8,7 @@ data "ignition_config" "main" { systemd = [ "${var.ign_docker_dropin_id}", - "${data.ignition_systemd_unit.locksmithd.id}", + "${var.ign_locksmithd_service_id}", "${var.ign_kubelet_service_id}", "${var.ign_s3_kubelet_env_service_id}", "${data.ignition_systemd_unit.init_assets.id}", @@ -17,11 +17,6 @@ data "ignition_config" "main" { ] } -data "ignition_systemd_unit" "locksmithd" { - name = "locksmithd.service" - mask = true -} - data "ignition_file" "detect_master" { filesystem = "root" path = "/opt/detect-master.sh" diff --git a/modules/aws/worker-asg/ignition.tf b/modules/aws/worker-asg/ignition.tf index 60169a4f04..f097a90044 100644 --- a/modules/aws/worker-asg/ignition.tf +++ b/modules/aws/worker-asg/ignition.tf @@ -6,13 +6,8 @@ data "ignition_config" "main" { systemd = [ "${var.ign_docker_dropin_id}", - "${data.ignition_systemd_unit.locksmithd.id}", + "${var.ign_locksmithd_service_id}", "${var.ign_kubelet_service_id}", "${var.ign_s3_kubelet_env_service_id}", ] } - -data "ignition_systemd_unit" "locksmithd" { - name = "locksmithd.service" - mask = true -} diff --git a/modules/azure/master-as/ignition-master.tf b/modules/azure/master-as/ignition-master.tf index 106097b24b..40f929b992 100644 --- a/modules/azure/master-as/ignition-master.tf +++ b/modules/azure/master-as/ignition-master.tf @@ -9,7 +9,7 @@ data "ignition_config" "master" { systemd = [ "${var.ign_docker_dropin_id}", - "${data.ignition_systemd_unit.locksmithd.id}", + "${var.ign_locksmithd_service_id}", "${var.ign_kubelet_service_id}", "${data.ignition_systemd_unit.tectonic.id}", "${data.ignition_systemd_unit.bootkube.id}", @@ -29,11 +29,6 @@ data "ignition_user" "core" { ] } -data "ignition_systemd_unit" "locksmithd" { - name = "locksmithd.service" - mask = true -} - data "ignition_file" "kubeconfig" { filesystem = "root" path = "/etc/kubernetes/kubeconfig" diff --git a/modules/azure/worker-as/ignition-worker.tf b/modules/azure/worker-as/ignition-worker.tf index 9e3f789fed..81972ee3e3 100644 --- a/modules/azure/worker-as/ignition-worker.tf +++ b/modules/azure/worker-as/ignition-worker.tf @@ -9,7 +9,7 @@ data "ignition_config" "worker" { systemd = [ "${var.ign_docker_dropin_id}", - "${data.ignition_systemd_unit.locksmithd.id}", + "${var.ign_locksmithd_service_id}", "${var.ign_kubelet_service_id}", "${module.net_ignition.tx-off_id}", ] @@ -19,11 +19,6 @@ data "ignition_config" "worker" { ] } -data "ignition_systemd_unit" "locksmithd" { - name = "locksmithd.service" - mask = true -} - data "ignition_file" "kubeconfig" { filesystem = "root" path = "/etc/kubernetes/kubeconfig" diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf index 673ba133a8..2ed448917e 100644 --- a/modules/ignition/assets.tf +++ b/modules/ignition/assets.tf @@ -82,3 +82,8 @@ data "ignition_file" "s3_puller" { content = "${data.template_file.s3_puller.rendered}" } } + +data "ignition_systemd_unit" "locksmithd" { + name = "locksmithd.service" + mask = true +} diff --git a/modules/ignition/outputs.import b/modules/ignition/outputs.import index c7ddbe10cb..90bfd3139d 100644 --- a/modules/ignition/outputs.import +++ b/modules/ignition/outputs.import @@ -11,3 +11,7 @@ variable "ign_docker_dropin_id" { variable "ign_kubelet_service_id" { type = "string" } + +variable "ign_locksmithd_service_id" { + type = "string" +} diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf index c4ecb7fdf6..bb4329481b 100644 --- a/modules/ignition/outputs.tf +++ b/modules/ignition/outputs.tf @@ -37,3 +37,7 @@ output "s3_puller_id" { output "s3_puller_rendered" { value = "${data.template_file.s3_puller.rendered}" } + +output "locksmithd_service_id" { + value = "${data.ignition_systemd_unit.locksmithd.id}" +} diff --git a/modules/openstack/nodes/ignition.tf b/modules/openstack/nodes/ignition.tf index 88f7ca5389..a34328eb56 100644 --- a/modules/openstack/nodes/ignition.tf +++ b/modules/openstack/nodes/ignition.tf @@ -15,7 +15,7 @@ data "ignition_config" "node" { systemd = [ "${var.ign_docker_dropin_id}", - "${data.ignition_systemd_unit.locksmithd.id}", + "${var.ign_locksmithd_service_id}", "${var.ign_kubelet_service_id}", "${data.ignition_systemd_unit.bootkube.id}", "${data.ignition_systemd_unit.tectonic.id}", @@ -50,11 +50,6 @@ data "ignition_file" "hostname" { } } -data "ignition_systemd_unit" "locksmithd" { - name = "locksmithd.service" - mask = true -} - data "ignition_file" "kubeconfig" { filesystem = "root" path = "/etc/kubernetes/kubeconfig" diff --git a/modules/vmware/node/ignition.tf b/modules/vmware/node/ignition.tf index 8321f93f6d..11e3462794 100644 --- a/modules/vmware/node/ignition.tf +++ b/modules/vmware/node/ignition.tf @@ -13,7 +13,7 @@ data "ignition_config" "node" { systemd = [ "${var.ign_docker_dropin_id}", - "${data.ignition_systemd_unit.locksmithd.id}", + "${var.ign_locksmithd_service_id}", "${var.ign_kubelet_service_id}", "${data.ignition_systemd_unit.kubelet-env.id}", "${data.ignition_systemd_unit.bootkube.id}", @@ -30,11 +30,6 @@ data "ignition_user" "core" { ssh_authorized_keys = ["${var.core_public_keys}"] } -data "ignition_systemd_unit" "locksmithd" { - name = "locksmithd.service" - mask = true -} - data "template_file" "kubelet-env" { template = "${file("${path.module}/resources/services/kubelet-env.service")}" diff --git a/platforms/aws/main.tf b/platforms/aws/main.tf index c166570770..134c3f9c29 100644 --- a/platforms/aws/main.tf +++ b/platforms/aws/main.tf @@ -127,9 +127,10 @@ module "masters" { ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" + ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" - ign_s3_puller_id = "${module.ignition_masters.s3_puller_id}" ign_s3_kubelet_env_service_id = "${module.ignition_masters.kubelet_env_service_id}" + ign_s3_puller_id = "${module.ignition_masters.s3_puller_id}" } module "ignition_workers" { @@ -164,9 +165,10 @@ module "workers" { vpc_id = "${module.vpc.vpc_id}" worker_iam_role = "${var.tectonic_aws_worker_iam_role_name}" - ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}" ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" - ign_s3_puller_id = "${module.ignition_workers.s3_puller_id}" + ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" + ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" ign_s3_kubelet_env_service_id = "${module.ignition_workers.kubelet_env_service_id}" + ign_s3_puller_id = "${module.ignition_workers.s3_puller_id}" } diff --git a/platforms/azure/main.tf b/platforms/azure/main.tf index dcd4df5094..b5ff608e20 100644 --- a/platforms/azure/main.tf +++ b/platforms/azure/main.tf @@ -129,10 +129,11 @@ module "masters" { tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" vm_size = "${var.tectonic_azure_master_vm_size}" - ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" - ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" - ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" - ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" + ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" + ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" + ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" + ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" + ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" } module "ignition_workers" { @@ -169,9 +170,10 @@ module "workers" { vm_size = "${var.tectonic_azure_worker_vm_size}" worker_count = "${var.tectonic_worker_count}" - ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}" - ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" - ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" + ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}" + ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" + ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" + ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" } module "dns" { diff --git a/platforms/openstack/neutron/main.tf b/platforms/openstack/neutron/main.tf index 60c873bf4b..d02f449d22 100644 --- a/platforms/openstack/neutron/main.tf +++ b/platforms/openstack/neutron/main.tf @@ -147,9 +147,10 @@ EOF tectonic_service = "${module.tectonic.systemd_service}" tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" - ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" - ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" - ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" + ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" + ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" + ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" } module "ignition_workers" { @@ -182,9 +183,10 @@ EOF tectonic_service = "" tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" - ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}" - ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" - ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}" + ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" + ign_locksmithd_service_id = "${module.ignition_workers.locksmithd_service_id}" + ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" } module "secrets" { diff --git a/platforms/vmware/main.tf b/platforms/vmware/main.tf index d61a4abf32..970cbd0742 100644 --- a/platforms/vmware/main.tf +++ b/platforms/vmware/main.tf @@ -73,9 +73,10 @@ module "masters" { private_key = "${var.tectonic_vmware_ssh_private_key_path}" image_re = "${var.tectonic_image_re}" - ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" - ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" - ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" + ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" + ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" + ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" } module "ignition_workers" { @@ -118,7 +119,8 @@ module "workers" { private_key = "${var.tectonic_vmware_ssh_private_key_path}" image_re = "${var.tectonic_image_re}" - ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" - ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" - ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" + ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" + ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" + ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" } From 6554738e631b05ac7bfd11ebceee63fabe895bde Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Thu, 24 Aug 2017 16:28:22 +0200 Subject: [PATCH 05/13] modules/ignition: unify kubelet.env --- modules/azure/master-as/ignition-master.tf | 15 +-------- modules/azure/master-as/variables.tf | 6 +--- modules/azure/worker-as/ignition-worker.tf | 15 +-------- modules/azure/worker-as/variables.tf | 6 +--- modules/ignition/assets.tf | 23 +++++++++++-- modules/ignition/outputs.tf | 10 +++++- .../ignition/resources/kubernetes/kubelet.env | 2 ++ modules/openstack/nodes/ignition.tf | 15 +-------- modules/openstack/nodes/variables.tf | 32 ++++++++----------- modules/vmware/node/ignition.tf | 15 +-------- modules/vmware/node/variables.tf | 12 +++---- platforms/azure/main.tf | 6 ++-- .../metal/cl/bootkube-controller.yaml.tmpl | 4 +-- platforms/metal/matchers.tf | 5 ++- platforms/openstack/neutron/main.tf | 6 ++-- platforms/vmware/main.tf | 6 ++-- 16 files changed, 64 insertions(+), 114 deletions(-) create mode 100644 modules/ignition/resources/kubernetes/kubelet.env diff --git a/modules/azure/master-as/ignition-master.tf b/modules/azure/master-as/ignition-master.tf index 40f929b992..671ce21c4d 100644 --- a/modules/azure/master-as/ignition-master.tf +++ b/modules/azure/master-as/ignition-master.tf @@ -1,7 +1,7 @@ data "ignition_config" "master" { files = [ "${data.ignition_file.kubeconfig.id}", - "${data.ignition_file.kubelet_env.id}", + "${var.ign_kubelet_env_id}", "${module.azure_udev-rules.udev-rules_id}", "${var.ign_max_user_watches_id}", "${data.ignition_file.cloud_provider_config.id}", @@ -39,19 +39,6 @@ data "ignition_file" "kubeconfig" { } } -data "ignition_file" "kubelet_env" { - filesystem = "root" - path = "/etc/kubernetes/kubelet.env" - mode = 0644 - - content { - content = < Date: Fri, 25 Aug 2017 09:40:29 +0200 Subject: [PATCH 06/13] modules/ignition: unify tx-off service --- modules/azure/master-as/ignition-master.tf | 6 +----- modules/azure/master-as/variables.tf | 4 ++++ modules/azure/worker-as/ignition-worker.tf | 6 +----- modules/azure/worker-as/variables.tf | 4 ++++ modules/ignition/assets.tf | 10 ++++++++++ modules/ignition/outputs.tf | 8 ++++++++ .../resources/services}/tx-off.service | 0 modules/net/ignition/ignition.tf | 5 ----- modules/net/ignition/outputs.tf | 3 --- platforms/azure/main.tf | 2 ++ 10 files changed, 30 insertions(+), 18 deletions(-) rename modules/{net/ignition/resources => ignition/resources/services}/tx-off.service (100%) delete mode 100644 modules/net/ignition/ignition.tf delete mode 100644 modules/net/ignition/outputs.tf diff --git a/modules/azure/master-as/ignition-master.tf b/modules/azure/master-as/ignition-master.tf index 671ce21c4d..68da7186af 100644 --- a/modules/azure/master-as/ignition-master.tf +++ b/modules/azure/master-as/ignition-master.tf @@ -13,7 +13,7 @@ data "ignition_config" "master" { "${var.ign_kubelet_service_id}", "${data.ignition_systemd_unit.tectonic.id}", "${data.ignition_systemd_unit.bootkube.id}", - "${module.net_ignition.tx-off_id}", + "${var.ign_tx_off_service_id}", ] users = [ @@ -60,10 +60,6 @@ data "ignition_systemd_unit" "tectonic" { content = "${var.tectonic_service}" } -module "net_ignition" { - source = "../../net/ignition" -} - module "azure_udev-rules" { source = "../udev-rules" } diff --git a/modules/azure/master-as/variables.tf b/modules/azure/master-as/variables.tf index 0c691ac6bc..75c50a9207 100644 --- a/modules/azure/master-as/variables.tf +++ b/modules/azure/master-as/variables.tf @@ -28,6 +28,10 @@ variable "ign_kubelet_env_id" { type = "string" } +variable "ign_tx_off_service_id" { + type = "string" +} + variable "kubeconfig_content" { type = "string" } diff --git a/modules/azure/worker-as/ignition-worker.tf b/modules/azure/worker-as/ignition-worker.tf index a4942cc0c7..d2f23b19c3 100644 --- a/modules/azure/worker-as/ignition-worker.tf +++ b/modules/azure/worker-as/ignition-worker.tf @@ -11,7 +11,7 @@ data "ignition_config" "worker" { "${var.ign_docker_dropin_id}", "${var.ign_locksmithd_service_id}", "${var.ign_kubelet_service_id}", - "${module.net_ignition.tx-off_id}", + "${var.ign_tx_off_service_id}", ] users = [ @@ -72,10 +72,6 @@ data "ignition_user" "core" { ] } -module "net_ignition" { - source = "../../net/ignition" -} - module "azure_udev-rules" { source = "../udev-rules" } diff --git a/modules/azure/worker-as/variables.tf b/modules/azure/worker-as/variables.tf index 9803a83416..b1ab2ef20a 100644 --- a/modules/azure/worker-as/variables.tf +++ b/modules/azure/worker-as/variables.tf @@ -29,6 +29,10 @@ variable "ign_kubelet_env_id" { type = "string" } +variable "ign_tx_off_service_id" { + type = "string" +} + variable "kubeconfig_content" { type = "string" default = "" diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf index 75e347f616..728cfb4bdf 100644 --- a/modules/ignition/assets.tf +++ b/modules/ignition/assets.tf @@ -106,3 +106,13 @@ data "ignition_file" "kubelet_env" { content = "${data.template_file.kubelet_env.rendered}" } } + +data "template_file" "tx_off" { + template = "${file("${path.module}/resources/services/tx-off.service")}" +} + +data "ignition_systemd_unit" "tx_off" { + name = "tx-off.service" + enable = true + content = "${data.template_file.tx_off.rendered}" +} diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf index 2a5687e80e..ba474e32e7 100644 --- a/modules/ignition/outputs.tf +++ b/modules/ignition/outputs.tf @@ -49,3 +49,11 @@ output "kubelet_env_id" { output "kubelet_env_rendered" { value = "${data.template_file.kubelet_env.rendered}" } + +output "tx_off_service_id" { + value = "${data.ignition_systemd_unit.tx_off.id}" +} + +output "tx_off_service_rendered" { + value = "${data.template_file.tx_off.rendered}" +} diff --git a/modules/net/ignition/resources/tx-off.service b/modules/ignition/resources/services/tx-off.service similarity index 100% rename from modules/net/ignition/resources/tx-off.service rename to modules/ignition/resources/services/tx-off.service diff --git a/modules/net/ignition/ignition.tf b/modules/net/ignition/ignition.tf deleted file mode 100644 index ad43b476e8..0000000000 --- a/modules/net/ignition/ignition.tf +++ /dev/null @@ -1,5 +0,0 @@ -data "ignition_systemd_unit" "tx-off" { - name = "tx-off.service" - enable = true - content = "${file("${path.module}/resources/tx-off.service")}" -} diff --git a/modules/net/ignition/outputs.tf b/modules/net/ignition/outputs.tf deleted file mode 100644 index 1d1ef717e6..0000000000 --- a/modules/net/ignition/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "tx-off_id" { - value = "${data.ignition_systemd_unit.tx-off.id}" -} diff --git a/platforms/azure/main.tf b/platforms/azure/main.tf index 8d214a5b74..c30d653628 100644 --- a/platforms/azure/main.tf +++ b/platforms/azure/main.tf @@ -133,6 +133,7 @@ module "masters" { ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" + ign_tx_off_service_id = "${module.ignition_masters.tx_off_service_id}" } module "ignition_workers" { @@ -172,6 +173,7 @@ module "workers" { ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" + ign_tx_off_service_id = "${module.ignition_workers.tx_off_service_id}" } module "dns" { From 89fbde331c874c0d7f4c6b1bfcdbf99be5694b02 Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Fri, 25 Aug 2017 12:38:04 +0200 Subject: [PATCH 07/13] modules/ignition: unify 66-azure-storage-rules --- modules/azure/master-as/ignition-master.tf | 6 +----- modules/azure/master-as/variables.tf | 4 ++++ modules/azure/worker-as/ignition-worker.tf | 6 +----- modules/azure/worker-as/variables.tf | 4 ++++ modules/ignition/assets.tf | 14 ++++++++++++++ modules/ignition/outputs.tf | 8 ++++++++ .../resources/udev}/66-azure-storage.rules | 0 platforms/azure/main.tf | 2 ++ 8 files changed, 34 insertions(+), 10 deletions(-) rename modules/{azure/udev-rules/resources => ignition/resources/udev}/66-azure-storage.rules (100%) diff --git a/modules/azure/master-as/ignition-master.tf b/modules/azure/master-as/ignition-master.tf index 68da7186af..48961df633 100644 --- a/modules/azure/master-as/ignition-master.tf +++ b/modules/azure/master-as/ignition-master.tf @@ -2,7 +2,7 @@ data "ignition_config" "master" { files = [ "${data.ignition_file.kubeconfig.id}", "${var.ign_kubelet_env_id}", - "${module.azure_udev-rules.udev-rules_id}", + "${var.ign_azure_udev_rules_id}", "${var.ign_max_user_watches_id}", "${data.ignition_file.cloud_provider_config.id}", ] @@ -59,7 +59,3 @@ data "ignition_systemd_unit" "tectonic" { enable = "${var.tectonic_service_disabled == 0 ? true : false}" content = "${var.tectonic_service}" } - -module "azure_udev-rules" { - source = "../udev-rules" -} diff --git a/modules/azure/master-as/variables.tf b/modules/azure/master-as/variables.tf index 75c50a9207..75b57dafc8 100644 --- a/modules/azure/master-as/variables.tf +++ b/modules/azure/master-as/variables.tf @@ -24,6 +24,10 @@ variable "extra_tags" { type = "map" } +variable "ign_azure_udev_rules_id" { + type = "string" +} + variable "ign_kubelet_env_id" { type = "string" } diff --git a/modules/azure/worker-as/ignition-worker.tf b/modules/azure/worker-as/ignition-worker.tf index d2f23b19c3..1644ff64fa 100644 --- a/modules/azure/worker-as/ignition-worker.tf +++ b/modules/azure/worker-as/ignition-worker.tf @@ -2,7 +2,7 @@ data "ignition_config" "worker" { files = [ "${data.ignition_file.kubeconfig.id}", "${var.ign_kubelet_env_id}", - "${module.azure_udev-rules.udev-rules_id}", + "${var.ign_azure_udev_rules_id}", "${var.ign_max_user_watches_id}", "${data.ignition_file.cloud-provider-config.id}", ] @@ -71,7 +71,3 @@ data "ignition_user" "core" { "${file(var.public_ssh_key)}", ] } - -module "azure_udev-rules" { - source = "../udev-rules" -} diff --git a/modules/azure/worker-as/variables.tf b/modules/azure/worker-as/variables.tf index b1ab2ef20a..edcef948be 100644 --- a/modules/azure/worker-as/variables.tf +++ b/modules/azure/worker-as/variables.tf @@ -25,6 +25,10 @@ variable "extra_tags" { type = "map" } +variable "ign_azure_udev_rules_id" { + type = "string" +} + variable "ign_kubelet_env_id" { type = "string" } diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf index 728cfb4bdf..e409c5cdcf 100644 --- a/modules/ignition/assets.tf +++ b/modules/ignition/assets.tf @@ -116,3 +116,17 @@ data "ignition_systemd_unit" "tx_off" { enable = true content = "${data.template_file.tx_off.rendered}" } + +data "template_file" "azure_udev_rules" { + template = "${file("${path.module}/resources/udev/66-azure-storage.rules")}" +} + +data "ignition_file" "azure_udev_rules" { + filesystem = "root" + path = "/etc/udev/rules.d/66-azure-storage.rules" + mode = 0644 + + content { + content = "${data.template_file.azure_udev_rules.rendered}" + } +} diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf index ba474e32e7..80a662b0a6 100644 --- a/modules/ignition/outputs.tf +++ b/modules/ignition/outputs.tf @@ -57,3 +57,11 @@ output "tx_off_service_id" { output "tx_off_service_rendered" { value = "${data.template_file.tx_off.rendered}" } + +output "azure_udev_rules_id" { + value = "${data.ignition_file.azure_udev_rules.id}" +} + +output "azure_udev_rules_rendered" { + value = "${data.template_file.azure_udev_rules.rendered}" +} diff --git a/modules/azure/udev-rules/resources/66-azure-storage.rules b/modules/ignition/resources/udev/66-azure-storage.rules similarity index 100% rename from modules/azure/udev-rules/resources/66-azure-storage.rules rename to modules/ignition/resources/udev/66-azure-storage.rules diff --git a/platforms/azure/main.tf b/platforms/azure/main.tf index c30d653628..d8b38612d2 100644 --- a/platforms/azure/main.tf +++ b/platforms/azure/main.tf @@ -127,6 +127,7 @@ module "masters" { tectonic_service_disabled = "${var.tectonic_vanilla_k8s}" vm_size = "${var.tectonic_azure_master_vm_size}" + ign_azure_udev_rules_id = "${module.ignition_masters.azure_udev_rules_id}" ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" ign_kubelet_env_id = "${module.ignition_masters.kubelet_env_id}" @@ -168,6 +169,7 @@ module "workers" { vm_size = "${var.tectonic_azure_worker_vm_size}" worker_count = "${var.tectonic_worker_count}" + ign_azure_udev_rules_id = "${module.ignition_workers.azure_udev_rules_id}" ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}" ign_kubelet_env_id = "${module.ignition_workers.kubelet_env_id}" ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" From 7cd1e1472999803cf22868d22f9c8b78b254d4af Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Mon, 28 Aug 2017 09:48:53 +0200 Subject: [PATCH 08/13] moduls/azure: disable stale tectonic service on workers ... it is not needed there, only masters are being used for bootstrapping. --- modules/azure/worker-as/ignition-worker.tf | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/modules/azure/worker-as/ignition-worker.tf b/modules/azure/worker-as/ignition-worker.tf index 1644ff64fa..e7280212f1 100644 --- a/modules/azure/worker-as/ignition-worker.tf +++ b/modules/azure/worker-as/ignition-worker.tf @@ -49,21 +49,6 @@ data "ignition_file" "cloud-provider-config" { } } -data "ignition_systemd_unit" "tectonic" { - name = "tectonic.service" - enable = true - - content = < Date: Mon, 28 Aug 2017 10:19:51 +0200 Subject: [PATCH 09/13] modules/azure: remove stale azure_udev_rules definition --- modules/azure/worker-as/ignition-worker.tf | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/modules/azure/worker-as/ignition-worker.tf b/modules/azure/worker-as/ignition-worker.tf index e7280212f1..d792d4a880 100644 --- a/modules/azure/worker-as/ignition-worker.tf +++ b/modules/azure/worker-as/ignition-worker.tf @@ -29,16 +29,6 @@ data "ignition_file" "kubeconfig" { } } -data "ignition_file" "azure_udev_rules" { - filesystem = "root" - path = "/etc/udev/rules.d/66-azure-storage.rules" - mode = 0644 - - content { - content = "${file("${path.module}/resources/66-azure-storage.rules")}" - } -} - data "ignition_file" "cloud-provider-config" { filesystem = "root" path = "/etc/kubernetes/cloud/config" From d21d81ceedfd8fbbb8960be5ac7ff9fe5ba855c3 Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Mon, 28 Aug 2017 11:36:37 +0200 Subject: [PATCH 10/13] platforms/metal: unify dockeropts with other platforms This adds the same docker options dropin as all the other platforms. Fixes INST-38 --- platforms/metal/cl/bootkube-controller.yaml.tmpl | 3 +++ platforms/metal/cl/bootkube-worker.yaml.tmpl | 3 +++ platforms/metal/matchers.tf | 2 ++ 3 files changed, 8 insertions(+) diff --git a/platforms/metal/cl/bootkube-controller.yaml.tmpl b/platforms/metal/cl/bootkube-controller.yaml.tmpl index 6bfe92a284..592c252e57 100644 --- a/platforms/metal/cl/bootkube-controller.yaml.tmpl +++ b/platforms/metal/cl/bootkube-controller.yaml.tmpl @@ -26,6 +26,9 @@ systemd: {{ end }} - name: docker.service enable: true + dropins: + - name: 10-dockeropts.conf + contents: {{.ign_docker_dropin_json}} - name: locksmithd.service mask: true - name: kubelet.path diff --git a/platforms/metal/cl/bootkube-worker.yaml.tmpl b/platforms/metal/cl/bootkube-worker.yaml.tmpl index 01c0116a13..4aa574e855 100644 --- a/platforms/metal/cl/bootkube-worker.yaml.tmpl +++ b/platforms/metal/cl/bootkube-worker.yaml.tmpl @@ -3,6 +3,9 @@ systemd: units: - name: docker.service enable: true + dropins: + - name: 10-dockeropts.conf + contents: {{.ign_docker_dropin_json}} - name: locksmithd.service mask: true - name: kubelet.path diff --git a/platforms/metal/matchers.tf b/platforms/metal/matchers.tf index 61e48866bc..f6b098e3a7 100644 --- a/platforms/metal/matchers.tf +++ b/platforms/metal/matchers.tf @@ -64,6 +64,7 @@ resource "matchbox_group" "controller" { kubelet_image_url = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$1")}" kubelet_image_tag = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}" + ign_docker_dropin_json = "${jsonencode(module.ignition_masters.docker_dropin_rendered)}" ign_kubelet_env_json = "${jsonencode(module.ignition_masters.kubelet_env_rendered)}" ign_kubelet_service_json = "${jsonencode(module.ignition_masters.kubelet_service_rendered)}" ign_max_user_watches_json = "${jsonencode(module.ignition_masters.max_user_watches_rendered)}" @@ -100,6 +101,7 @@ resource "matchbox_group" "worker" { kubelet_image_tag = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}" kube_version_image = "${var.tectonic_container_images["kube_version"]}" + ign_docker_dropin_json = "${jsonencode(module.ignition_masters.docker_dropin_rendered)}" ign_kubelet_env_service_json = "${jsonencode(module.ignition_workers.kubelet_env_service_rendered)}" ign_kubelet_service_json = "${jsonencode(module.ignition_workers.kubelet_service_rendered)}" ign_max_user_watches_json = "${jsonencode(module.ignition_workers.max_user_watches_rendered)}" From 821c36391a96448b44e942dfdbece088ba517001 Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Mon, 28 Aug 2017 12:02:35 +0200 Subject: [PATCH 11/13] platforms/*: s/ignition_masters/ignition_workers --- platforms/metal/matchers.tf | 2 +- platforms/vmware/main.tf | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/platforms/metal/matchers.tf b/platforms/metal/matchers.tf index f6b098e3a7..f015c221e5 100644 --- a/platforms/metal/matchers.tf +++ b/platforms/metal/matchers.tf @@ -101,7 +101,7 @@ resource "matchbox_group" "worker" { kubelet_image_tag = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}" kube_version_image = "${var.tectonic_container_images["kube_version"]}" - ign_docker_dropin_json = "${jsonencode(module.ignition_masters.docker_dropin_rendered)}" + ign_docker_dropin_json = "${jsonencode(module.ignition_workers.docker_dropin_rendered)}" ign_kubelet_env_service_json = "${jsonencode(module.ignition_workers.kubelet_env_service_rendered)}" ign_kubelet_service_json = "${jsonencode(module.ignition_workers.kubelet_service_rendered)}" ign_max_user_watches_json = "${jsonencode(module.ignition_workers.max_user_watches_rendered)}" diff --git a/platforms/vmware/main.tf b/platforms/vmware/main.tf index f476221dcf..9f83b081fd 100644 --- a/platforms/vmware/main.tf +++ b/platforms/vmware/main.tf @@ -116,9 +116,9 @@ module "workers" { private_key = "${var.tectonic_vmware_ssh_private_key_path}" image_re = "${var.tectonic_image_re}" - ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" - ign_kubelet_env_id = "${module.ignition_masters.kubelet_env_id}" - ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" - ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" - ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}" + ign_kubelet_env_id = "${module.ignition_workers.kubelet_env_id}" + ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" + ign_locksmithd_service_id = "${module.ignition_workers.locksmithd_service_id}" + ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" } From 9b948d615cca47b3f0d74afe03e17c93b2ff1dc7 Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Mon, 28 Aug 2017 12:03:55 +0200 Subject: [PATCH 12/13] modules/ignition: apply nn-foo.conf scheme for max-user-watches.conf Fixes https://github.com/coreos/tectonic-installer/pull/1743#discussion_r134488295 --- modules/ignition/assets.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf index e409c5cdcf..9f7390062b 100644 --- a/modules/ignition/assets.tf +++ b/modules/ignition/assets.tf @@ -4,7 +4,7 @@ data "template_file" "max_user_watches" { data "ignition_file" "max_user_watches" { filesystem = "root" - path = "/etc/sysctl.d/max-user-watches.conf" + path = "/etc/sysctl.d/10-max-user-watches.conf" mode = 0644 content { From e28a6449aa07eb95c78e181bf145dcc3ad6e4a60 Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Mon, 28 Aug 2017 17:46:06 +0200 Subject: [PATCH 13/13] modules/vmware: remove stale kubelet-env declaration --- modules/vmware/node/ignition.tf | 18 +-------------- .../resources/services/kubelet-env.service | 14 ------------ modules/vmware/node/variables.tf | 5 +++++ platforms/vmware/main.tf | 22 ++++++++++--------- 4 files changed, 18 insertions(+), 41 deletions(-) delete mode 100644 modules/vmware/node/resources/services/kubelet-env.service diff --git a/modules/vmware/node/ignition.tf b/modules/vmware/node/ignition.tf index aa75b4782a..17bc653566 100644 --- a/modules/vmware/node/ignition.tf +++ b/modules/vmware/node/ignition.tf @@ -15,7 +15,7 @@ data "ignition_config" "node" { "${var.ign_docker_dropin_id}", "${var.ign_locksmithd_service_id}", "${var.ign_kubelet_service_id}", - "${data.ignition_systemd_unit.kubelet-env.id}", + "${var.ign_kubelet_env_service_id}", "${data.ignition_systemd_unit.bootkube.id}", "${data.ignition_systemd_unit.tectonic.id}", ] @@ -30,22 +30,6 @@ data "ignition_user" "core" { ssh_authorized_keys = ["${var.core_public_keys}"] } -data "template_file" "kubelet-env" { - template = "${file("${path.module}/resources/services/kubelet-env.service")}" - - vars { - kube_version_image_url = "${replace(var.container_images["kube_version"],var.image_re,"$1")}" - kube_version_image_tag = "${replace(var.container_images["kube_version"],var.image_re,"$2")}" - kubelet_image_url = "${replace(var.container_images["hyperkube"],var.image_re,"$1")}" - } -} - -data "ignition_systemd_unit" "kubelet-env" { - name = "kubelet-env.service" - enable = true - content = "${data.template_file.kubelet-env.rendered}" -} - data "ignition_systemd_unit" "bootkube" { name = "bootkube.service" content = "${var.bootkube_service}" diff --git a/modules/vmware/node/resources/services/kubelet-env.service b/modules/vmware/node/resources/services/kubelet-env.service deleted file mode 100644 index 9a5100a332..0000000000 --- a/modules/vmware/node/resources/services/kubelet-env.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Determine the Kubelet Image Version -ConditionPathExists=!/etc/kubernetes/kubelet.env - -[Service] - -ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes -ExecStartPre=/usr/bin/bash -c "docker run --rm -v /etc/kubernetes/kubeconfig:/kubeconfig ${kube_version_image_url}:${kube_version_image_tag} --kubeconfig=/kubeconfig > /etc/kubernetes/kube.version" -ExecStart=/usr/bin/bash -c "echo KUBELET_IMAGE_URL=${kubelet_image_url} > /etc/kubernetes/kubelet.env; echo KUBELET_IMAGE_TAG=$(tr '+' '_' < /etc/kubernetes/kube.version) >> /etc/kubernetes/kubelet.env; rm /etc/kubernetes/kube.version" -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/modules/vmware/node/variables.tf b/modules/vmware/node/variables.tf index 096ffe53f5..8b80a01a30 100644 --- a/modules/vmware/node/variables.tf +++ b/modules/vmware/node/variables.tf @@ -116,3 +116,8 @@ variable "vmware_datacenter" { type = "string" description = "vSphere Datacenter to create VMs in" } + +variable "ign_kubelet_env_service_id" { + type = "string" + description = "The kubelet env service to use" +} diff --git a/platforms/vmware/main.tf b/platforms/vmware/main.tf index 9f83b081fd..9ac18eebe5 100644 --- a/platforms/vmware/main.tf +++ b/platforms/vmware/main.tf @@ -71,11 +71,12 @@ module "masters" { private_key = "${var.tectonic_vmware_ssh_private_key_path}" image_re = "${var.tectonic_image_re}" - ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" - ign_kubelet_env_id = "${module.ignition_masters.kubelet_env_id}" - ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" - ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" - ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" + ign_kubelet_env_id = "${module.ignition_masters.kubelet_env_id}" + ign_kubelet_env_service_id = "${module.ignition_masters.kubelet_env_service_id}" + ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" + ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" + ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" } module "ignition_workers" { @@ -116,9 +117,10 @@ module "workers" { private_key = "${var.tectonic_vmware_ssh_private_key_path}" image_re = "${var.tectonic_image_re}" - ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}" - ign_kubelet_env_id = "${module.ignition_workers.kubelet_env_id}" - ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" - ign_locksmithd_service_id = "${module.ignition_workers.locksmithd_service_id}" - ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" + ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}" + ign_kubelet_env_id = "${module.ignition_workers.kubelet_env_id}" + ign_kubelet_env_service_id = "${module.ignition_workers.kubelet_env_service_id}" + ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" + ign_locksmithd_service_id = "${module.ignition_workers.locksmithd_service_id}" + ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" }