From fff30dd518e65f3f1a68ca4ef61ca30bf3f37425 Mon Sep 17 00:00:00 2001 From: Enxebre Date: Wed, 25 Oct 2017 17:33:18 +0200 Subject: [PATCH 1/5] platform/gcp: add region dynamic node availability and ssh bootstrap process --- modules/gcp/master-igm/ignition.tf | 23 +---- modules/gcp/master-igm/master.tf | 13 ++- modules/gcp/master-igm/outputs.tf | 2 +- .../gcp/master-igm/resources/init-assets.sh | 19 ---- modules/gcp/master-igm/variables.tf | 21 +--- modules/gcp/network/loadbalancer.tf | 49 ++++++++- modules/gcp/network/outputs.tf | 6 +- modules/gcp/worker-igm/ignition.tf | 12 ++- modules/gcp/worker-igm/variables.tf | 12 +-- modules/gcp/worker-igm/worker.tf | 8 +- modules/ignition/assets.tf | 14 --- modules/ignition/outputs.tf | 8 -- modules/ignition/resources/bin/gcs-puller.sh | 17 ---- platforms/gcp/bootstrap.tf | 16 +++ platforms/gcp/gcs.tf | 39 -------- platforms/gcp/main.tf | 99 ++++++------------- platforms/gcp/variables.tf | 5 - tests/rspec/lib/gcp_cluster.rb | 2 +- tests/smoke/gcp/vars/gcp.tfvars.json | 1 - 19 files changed, 138 insertions(+), 228 deletions(-) delete mode 100644 modules/gcp/master-igm/resources/init-assets.sh delete mode 100644 modules/ignition/resources/bin/gcs-puller.sh create mode 100644 platforms/gcp/bootstrap.tf delete mode 100644 platforms/gcp/gcs.tf diff --git a/modules/gcp/master-igm/ignition.tf b/modules/gcp/master-igm/ignition.tf index 0651f0cec4..71765373d8 100644 --- a/modules/gcp/master-igm/ignition.tf +++ b/modules/gcp/master-igm/ignition.tf @@ -1,8 +1,7 @@ data "ignition_config" "main" { files = [ + "${data.ignition_file.kubeconfig.id}", "${var.ign_max_user_watches_id}", - "${var.ign_gcs_puller_id}", - "${data.ignition_file.init_assets.id}", "${var.ign_installer_kubelet_env_id}", ] @@ -11,7 +10,6 @@ data "ignition_config" "main" { var.ign_locksmithd_service_id, var.ign_kubelet_service_id, var.ign_k8s_node_bootstrap_service_id, - var.ign_init_assets_service_id, var.ign_bootkube_service_id, var.ign_tectonic_service_id, var.ign_bootkube_path_unit_id, @@ -19,23 +17,12 @@ data "ignition_config" "main" { ))}"] } -data "template_file" "init_assets" { - template = "${file("${path.module}/resources/init-assets.sh")}" - - vars { - cluster_name = "${var.cluster_name}" - assets_gcs_location = "${var.assets_gcs_location}" - kubelet_image_url = "${replace(var.container_images["hyperkube"],var.image_re,"$1")}" - kubelet_image_tag = "${replace(var.container_images["hyperkube"],var.image_re,"$2")}" - } -} - -data "ignition_file" "init_assets" { +data "ignition_file" "kubeconfig" { filesystem = "root" - path = "/opt/init-assets.sh" - mode = 0755 + path = "/etc/kubernetes/kubeconfig" + mode = 0644 content { - content = "${data.template_file.init_assets.rendered}" + content = "${var.kubeconfig_content}" } } diff --git a/modules/gcp/master-igm/master.tf b/modules/gcp/master-igm/master.tf index fb91b6ec83..dd04a010ce 100644 --- a/modules/gcp/master-igm/master.tf +++ b/modules/gcp/master-igm/master.tf @@ -47,12 +47,17 @@ resource "google_compute_instance_template" "master-it" { } } -resource "google_compute_instance_group_manager" "master-igm" { - count = "${var.instance_count}" - target_size = 1 +resource "google_compute_region_instance_group_manager" "master-igm" { + count = 1 + region = "${var.region}" + target_size = "${var.instance_count}" name = "${var.cluster_name}-master-igm-${count.index}" - zone = "${element(var.zone_list, count.index)}" instance_template = "${google_compute_instance_template.master-it.self_link}" target_pools = ["${var.master_targetpool_self_link}"] base_instance_name = "mstr" + + named_port { + name = "https" + port = 443 + } } diff --git a/modules/gcp/master-igm/outputs.tf b/modules/gcp/master-igm/outputs.tf index a6e53b3866..022f249904 100644 --- a/modules/gcp/master-igm/outputs.tf +++ b/modules/gcp/master-igm/outputs.tf @@ -15,5 +15,5 @@ limitations under the License. */ output "instance_group" { - value = ["${google_compute_instance_group_manager.master-igm.*.instance_group}"] + value = ["${google_compute_region_instance_group_manager.master-igm.*.instance_group}"] } diff --git a/modules/gcp/master-igm/resources/init-assets.sh b/modules/gcp/master-igm/resources/init-assets.sh deleted file mode 100644 index 7701f5c7a6..0000000000 --- a/modules/gcp/master-igm/resources/init-assets.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -e - -# Download the assets from GCS -# shellcheck disable=SC2086,SC2154 -/usr/bin/bash /opt/gcs-puller.sh ${assets_gcs_location} /var/tmp/tectonic.zip -unzip -o -d /var/tmp/tectonic/ /var/tmp/tectonic.zip -rm /var/tmp/tectonic.zip -# make files in /opt/tectonic available atomically -mv /var/tmp/tectonic /opt/tectonic - -# Populate the kubelet.env file. -mkdir -p /etc/kubernetes -# shellcheck disable=SC2154 -echo "KUBELET_IMAGE_URL=${kubelet_image_url}" > /etc/kubernetes/kubelet.env -# shellcheck disable=SC2154 -echo "KUBELET_IMAGE_TAG=${kubelet_image_tag}" >> /etc/kubernetes/kubelet.env - -exit 0 diff --git a/modules/gcp/master-igm/variables.tf b/modules/gcp/master-igm/variables.tf index 5b826319c3..1b0af8a2fd 100644 --- a/modules/gcp/master-igm/variables.tf +++ b/modules/gcp/master-igm/variables.tf @@ -26,10 +26,6 @@ variable "region" { type = "string" } -variable "zone_list" { - type = "list" -} - variable "machine_type" { type = "string" } @@ -74,19 +70,6 @@ variable "ign_tectonic_path_unit_id" { type = "string" } -variable "ign_gcs_puller_id" { - type = "string" -} - -variable "ign_init_assets_service_id" { - type = "string" -} - -variable "assets_gcs_location" { - type = "string" - description = "Location on gcs of the Bootkube/Tectonic assets to use (bucket/key)" -} - variable "container_images" { description = "Container images to use" type = "map" @@ -100,3 +83,7 @@ variable "image_re" { variable "public_ssh_key" { default = "" } + +variable "kubeconfig_content" { + type = "string" +} diff --git a/modules/gcp/network/loadbalancer.tf b/modules/gcp/network/loadbalancer.tf index 598c4cb86d..8fbf653aa2 100644 --- a/modules/gcp/network/loadbalancer.tf +++ b/modules/gcp/network/loadbalancer.tf @@ -19,14 +19,57 @@ resource "google_compute_http_health_check" "worker-hc" { check_interval_sec = 1 } -resource "google_compute_address" "masters-ip" { +// api-server/masters lb +// We need to use a global lb for bootstraping +// because of https://issuetracker.google.com/issues/67366622 +resource "google_compute_global_address" "masters-ip" { + name = "${var.cluster_name}-masters-ip" +} + +resource "google_compute_global_forwarding_rule" "api-external-fwd-rule" { + name = "${var.cluster_name}-api-external-fwd-rule" + target = "${google_compute_target_tcp_proxy.api-external-tcp-proxy.self_link}" + ip_address = "${google_compute_global_address.masters-ip.address}" + port_range = "443" +} + +resource "google_compute_target_tcp_proxy" "api-external-tcp-proxy" { + name = "${var.cluster_name}-api-external-tcp-proxy" + backend_service = "${google_compute_backend_service.api-backend-service.self_link}" +} + +resource "google_compute_backend_service" "api-backend-service" { + name = "${var.cluster_name}-api-backend-service" + protocol = "TCP" + port_name = "https" + timeout_sec = 10 + session_affinity = "NONE" + + backend { + group = "${var.master_instance_group[0]}" + } + + health_checks = ["${google_compute_health_check.api-health-check.self_link}"] +} + +resource "google_compute_health_check" "api-health-check" { + name = "${var.cluster_name}-api-health-check" + timeout_sec = 1 + check_interval_sec = 1 + + ssl_health_check { + port = "443" + } +} + +resource "google_compute_address" "ssh-masters-ip" { name = "${var.cluster_name}-masters-ip" } resource "google_compute_forwarding_rule" "api-external-fwd-rule" { load_balancing_scheme = "EXTERNAL" name = "${var.cluster_name}-api-external-fwd-rule" - ip_address = "${google_compute_address.masters-ip.address}" + ip_address = "${google_compute_address.ssh-masters-ip.address}" region = "${var.gcp_region}" target = "${google_compute_target_pool.master-targetpool.self_link}" port_range = "443" @@ -39,7 +82,7 @@ resource "google_compute_address" "ingress-ip" { resource "google_compute_forwarding_rule" "api-external-ssh-fwd-rule" { load_balancing_scheme = "EXTERNAL" name = "${var.cluster_name}-api-external-ssh-fwd-rule" - ip_address = "${google_compute_address.masters-ip.address}" + ip_address = "${google_compute_address.ssh-masters-ip.address}" region = "${var.gcp_region}" target = "${google_compute_target_pool.master-targetpool.self_link}" port_range = "22" diff --git a/modules/gcp/network/outputs.tf b/modules/gcp/network/outputs.tf index b550d097ef..addbeda45d 100644 --- a/modules/gcp/network/outputs.tf +++ b/modules/gcp/network/outputs.tf @@ -15,7 +15,11 @@ limitations under the License. */ output "master_ip" { - value = "${google_compute_address.masters-ip.address}" + value = "${google_compute_global_address.masters-ip.address}" +} + +output "ssh_master_ip" { + value = "${google_compute_address.ssh-masters-ip.address}" } output "ingress_ip" { diff --git a/modules/gcp/worker-igm/ignition.tf b/modules/gcp/worker-igm/ignition.tf index 4677a32c8d..7f0ac14495 100644 --- a/modules/gcp/worker-igm/ignition.tf +++ b/modules/gcp/worker-igm/ignition.tf @@ -1,7 +1,7 @@ data "ignition_config" "main" { files = [ + "${data.ignition_file.kubeconfig.id}", "${var.ign_max_user_watches_id}", - "${var.ign_gcs_puller_id}", "${var.ign_installer_kubelet_env_id}", ] @@ -12,3 +12,13 @@ data "ignition_config" "main" { "${var.ign_kubelet_service_id}", ] } + +data "ignition_file" "kubeconfig" { + filesystem = "root" + path = "/etc/kubernetes/kubeconfig" + mode = 0644 + + content { + content = "${var.kubeconfig_content}" + } +} diff --git a/modules/gcp/worker-igm/variables.tf b/modules/gcp/worker-igm/variables.tf index 02010a6f78..e3de1ff321 100644 --- a/modules/gcp/worker-igm/variables.tf +++ b/modules/gcp/worker-igm/variables.tf @@ -26,10 +26,6 @@ variable "region" { type = "string" } -variable "zone_list" { - type = "list" -} - variable "machine_type" { type = "string" } @@ -56,10 +52,10 @@ variable "disk_size" { description = "The size of the volume in gigabytes for the root block device." } -variable "ign_gcs_puller_id" { - type = "string" -} - variable "public_ssh_key" { default = "" } + +variable "kubeconfig_content" { + type = "string" +} diff --git a/modules/gcp/worker-igm/worker.tf b/modules/gcp/worker-igm/worker.tf index b7da04abed..88846c0cf0 100644 --- a/modules/gcp/worker-igm/worker.tf +++ b/modules/gcp/worker-igm/worker.tf @@ -47,11 +47,11 @@ resource "google_compute_instance_template" "worker-it" { } } -resource "google_compute_instance_group_manager" "worker-igm" { - count = "${var.instance_count}" - target_size = 1 +resource "google_compute_region_instance_group_manager" "worker-igm" { + count = 1 + region = "${var.region}" + target_size = "${var.instance_count}" name = "${var.cluster_name}-worker-igm-${count.index}" - zone = "${element(var.zone_list, count.index)}" instance_template = "${google_compute_instance_template.worker-it.self_link}" target_pools = ["${var.worker_targetpool_self_link}"] base_instance_name = "wrkr" diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf index edcdffb858..32de84dab0 100644 --- a/modules/ignition/assets.tf +++ b/modules/ignition/assets.tf @@ -91,20 +91,6 @@ data "ignition_file" "s3_puller" { } } -data "template_file" "gcs_puller" { - template = "${file("${path.module}/resources/bin/gcs-puller.sh")}" -} - -data "ignition_file" "gcs_puller" { - filesystem = "root" - path = "/opt/gcs-puller.sh" - mode = 0755 - - content { - content = "${data.template_file.gcs_puller.rendered}" - } -} - data "ignition_systemd_unit" "locksmithd" { name = "locksmithd.service" mask = true diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf index 4690ad7be6..6a2fa2028e 100644 --- a/modules/ignition/outputs.tf +++ b/modules/ignition/outputs.tf @@ -42,14 +42,6 @@ output "s3_puller_rendered" { value = "${data.template_file.s3_puller.rendered}" } -output "gcs_puller_id" { - value = "${data.ignition_file.gcs_puller.id}" -} - -output "gcs_puller_rendered" { - value = "${data.template_file.gcs_puller.rendered}" -} - output "locksmithd_service_id" { value = "${data.ignition_systemd_unit.locksmithd.id}" } diff --git a/modules/ignition/resources/bin/gcs-puller.sh b/modules/ignition/resources/bin/gcs-puller.sh deleted file mode 100644 index 8aa9ff4a8a..0000000000 --- a/modules/ignition/resources/bin/gcs-puller.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -x - -if [ "$#" -ne "2" ]; then - echo "Usage: $0 location destination" - exit 1 -fi - -docker pull google/cloud-sdk > /dev/null -# shellcheck disable=SC2034,SC1083 -gsutil="docker run -t --net=host -v /tmp:/gs google/cloud-sdk gsutil" -# shellcheck disable=SC2034,SC1083 -assets=$(basename $${1}) -# shellcheck disable=SC2034,SC1083 -$${gsutil} cp gs://$${1} /gs/$${assets} -# shellcheck disable=SC2034,SC1083 -/usr/bin/sudo mv /tmp/$${assets} $${2} diff --git a/platforms/gcp/bootstrap.tf b/platforms/gcp/bootstrap.tf new file mode 100644 index 0000000000..7d4afdcf1f --- /dev/null +++ b/platforms/gcp/bootstrap.tf @@ -0,0 +1,16 @@ +module "bootstrapper" { + source = "../../modules/bootstrap-ssh" + + _dependencies = [ + "${module.masters.instance_group}", + "${module.etcd.etcd_ip_addresses}", + "${module.etcd_certs.id}", + "${module.bootkube.id}", + "${module.tectonic.id}", + "${module.flannel_vxlan.id}", + "${module.calico.id}", + "${module.canal.id}", + ] + + bootstrapping_host = "${module.network.ssh_master_ip}" +} diff --git a/platforms/gcp/gcs.tf b/platforms/gcp/gcs.tf deleted file mode 100644 index d5ee8f10d2..0000000000 --- a/platforms/gcp/gcs.tf +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2017 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -resource "google_storage_bucket" "tectonic" { - name = "${var.tectonic_cluster_name}-${var.tectonic_gcp_region}-${var.tectonic_gcp_ext_google_managedzone_name}" - location = "${upper(var.tectonic_gcp_region)}" - storage_class = "REGIONAL" -} - -resource "google_storage_bucket_acl" "private_acl" { - bucket = "${google_storage_bucket.tectonic.name}" - predefined_acl = "projectprivate" -} - -# Bootkube / Tectonic assets -resource "google_storage_bucket_object" "tectonic-assets" { - name = "assets.zip" - bucket = "${google_storage_bucket.tectonic.name}" - source = "${data.archive_file.assets.output_path}" -} - -# kubeconfig -resource "google_storage_bucket_object" "kubeconfig" { - name = "kubeconfig" - bucket = "${google_storage_bucket.tectonic.name}" - content = "${module.bootkube.kubeconfig}" -} diff --git a/platforms/gcp/main.tf b/platforms/gcp/main.tf index 14027e7a49..50291ddf4b 100644 --- a/platforms/gcp/main.tf +++ b/platforms/gcp/main.tf @@ -38,45 +38,14 @@ module "network" { cluster_name = "${var.tectonic_cluster_name}" master_instance_group = "${module.masters.instance_group}" - - # VPC layout settings. - # - # The following parameters control the layout of the VPC accross availability zones. - # Two modes are available: - # A. Explicitly configure a list of AZs + associated subnet CIDRs - # B. Let the module calculate subnets accross a set number of AZs - # - # To enable mode A, make sure "tectonic_gcp_az_count" variable IS NOT SET to any value - # and instead configure a set of AZs + CIDRs for masters and workers using the - # "tectonic_gcp_master_custom_subnets" and "tectonic_gcp_worker_custom_subnets" variables. - # - # To enable mode B, make sure that "tectonic_gcp_master_custom_subnets" and "tectonic_gcp_worker_custom_subnets" - # ARE NOT SET. Instead, set the desired number of VPC AZs using "tectonic_gcp_az_count" variable. - - # These counts could be deducted by length(keys(var.tectonic_gcp_master_custom_subnets)) - # but there is a restriction on passing computed values as counts. This approach works around that. - #master_az_count = "${var.tectonic_gcp_az_count == "" ? "${length(keys(var.tectonic_gcp_master_custom_subnets))}" : var.tectonic_gcp_az_count}" - #worker_az_count = "${var.tectonic_gcp_az_count == "" ? "${length(keys(var.tectonic_gcp_worker_custom_subnets))}" : var.tectonic_gcp_az_count}" - # The appending of the "padding" element is required as workaround since the function - # element() won't work on empty lists. See https://github.com/hashicorp/terraform/issues/11210 - #master_subnets = "${concat(values(var.tectonic_gcp_master_custom_subnets),list("padding"))}" - #worker_subnets = "${concat(values(var.tectonic_gcp_worker_custom_subnets),list("padding"))}" - # The split() / join() trick works around the limitation of tenrary operator expressions - # only being able to return strings. - #master_azs = ["${ split("|", "${length(keys(var.tectonic_gcp_master_custom_subnets))}" > 0 ? - # join("|", keys(var.tectonic_gcp_master_custom_subnets)) : - # join("|", data.gcp_availability_zones.azs.names) - #)}"] - #worker_azs = ["${ split("|", "${length(keys(var.tectonic_gcp_worker_custom_subnets))}" > 0 ? - # join("|", keys(var.tectonic_gcp_worker_custom_subnets)) : - # join("|", data.gcp_availability_zones.azs.names) - #)}"] } +data "google_compute_zones" "available" {} + module "etcd" { source = "../../modules/gcp/etcd" - instance_count = "${var.tectonic_experimental ? 0 : var.tectonic_etcd_count > 0 ? var.tectonic_etcd_count : length(var.tectonic_gcp_zones) == 5 ? 5 : 3}" - zone_list = "${var.tectonic_gcp_zones}" + instance_count = "${var.tectonic_experimental ? 0 : var.tectonic_etcd_count > 0 ? var.tectonic_etcd_count : length(data.google_compute_zones.available.names) == 5 ? 5 : 3}" + zone_list = "${data.google_compute_zones.available.names}" machine_type = "${var.tectonic_gcp_etcd_gce_type}" managed_zone_name = "${var.tectonic_gcp_ext_google_managedzone_name}" cluster_name = "${var.tectonic_cluster_name}" @@ -106,13 +75,12 @@ module "etcd" { module "masters" { source = "../../modules/gcp/master-igm" - region = "${var.tectonic_gcp_region}" - instance_count = "${var.tectonic_master_count}" - zone_list = "${var.tectonic_gcp_zones}" - machine_type = "${var.tectonic_gcp_master_gce_type}" - cluster_name = "${var.tectonic_cluster_name}" - public_ssh_key = "${var.tectonic_gcp_ssh_key}" - assets_gcs_location = "${google_storage_bucket.tectonic.name}/${google_storage_bucket_object.tectonic-assets.name}" + region = "${var.tectonic_gcp_region}" + instance_count = "${var.tectonic_master_count}" + machine_type = "${var.tectonic_gcp_master_gce_type}" + cluster_name = "${var.tectonic_cluster_name}" + public_ssh_key = "${var.tectonic_gcp_ssh_key}" + kubeconfig_content = "${module.bootkube.kubeconfig}" master_subnetwork_name = "${module.network.master_subnetwork_name}" master_targetpool_self_link = "${module.network.master_targetpool_self_link}" @@ -127,11 +95,9 @@ module "masters" { ign_bootkube_service_id = "${module.bootkube.systemd_service_id}" ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" - ign_init_assets_service_id = "${module.ignition_masters.init_assets_service_id}" ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" ign_installer_kubelet_env_id = "${module.ignition_masters.installer_kubelet_env_id}" - ign_gcs_puller_id = "${module.ignition_masters.gcs_puller_id}" ign_tectonic_path_unit_id = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}" ign_tectonic_service_id = "${module.tectonic.systemd_service_id}" image_re = "${var.tectonic_image_re}" @@ -141,12 +107,12 @@ module "masters" { module "workers" { source = "../../modules/gcp/worker-igm" - region = "${var.tectonic_gcp_region}" - instance_count = "${var.tectonic_worker_count}" - zone_list = "${var.tectonic_gcp_zones}" - machine_type = "${var.tectonic_gcp_worker_gce_type}" - cluster_name = "${var.tectonic_cluster_name}" - public_ssh_key = "${var.tectonic_gcp_ssh_key}" + region = "${var.tectonic_gcp_region}" + instance_count = "${var.tectonic_worker_count}" + machine_type = "${var.tectonic_gcp_worker_gce_type}" + cluster_name = "${var.tectonic_cluster_name}" + public_ssh_key = "${var.tectonic_gcp_ssh_key}" + kubeconfig_content = "${module.bootkube.kubeconfig}" worker_subnetwork_name = "${module.network.worker_subnetwork_name}" worker_targetpool_self_link = "${module.network.worker_targetpool_self_link}" @@ -163,23 +129,22 @@ module "workers" { ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" ign_installer_kubelet_env_id = "${module.ignition_workers.installer_kubelet_env_id}" - ign_gcs_puller_id = "${module.ignition_workers.gcs_puller_id}" } module "ignition_masters" { source = "../../modules/ignition" - cluster_name = "${var.tectonic_cluster_name}" - bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}" - tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}" - container_images = "${var.tectonic_container_images}" - image_re = "${var.tectonic_image_re}" - kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" - kubeconfig_fetch_cmd = "/opt/gcs-puller.sh ${google_storage_bucket.tectonic.name}/${google_storage_bucket_object.kubeconfig.name} /etc/kubernetes/kubeconfig" - kubelet_cni_bin_dir = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }" - kubelet_node_label = "node-role.kubernetes.io/master" - kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule" - assets_location = "${google_storage_bucket.tectonic.name}/${google_storage_bucket_object.tectonic-assets.name}" + cluster_name = "${var.tectonic_cluster_name}" + bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}" + tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}" + container_images = "${var.tectonic_container_images}" + image_re = "${var.tectonic_image_re}" + kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" + + kubelet_cni_bin_dir = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }" + kubelet_node_label = "node-role.kubernetes.io/master" + kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule" + etcd_advertise_name_list = "${data.template_file.etcd_hostname_list.*.rendered}" etcd_count = "${length(data.template_file.etcd_hostname_list.*.id)}" etcd_initial_cluster_list = "${data.template_file.etcd_hostname_list.*.rendered}" @@ -195,10 +160,10 @@ module "ignition_workers" { container_images = "${var.tectonic_container_images}" image_re = "${var.tectonic_image_re}" kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" - kubeconfig_fetch_cmd = "/opt/gcs-puller.sh ${google_storage_bucket.tectonic.name}/${google_storage_bucket_object.kubeconfig.name} /etc/kubernetes/kubeconfig" - kubelet_cni_bin_dir = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }" - kubelet_node_label = "node-role.kubernetes.io/node" - kubelet_node_taints = "" + + kubelet_cni_bin_dir = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }" + kubelet_node_label = "node-role.kubernetes.io/node" + kubelet_node_taints = "" } module "dns" { @@ -208,7 +173,7 @@ module "dns" { etcd_dns_enabled = "${!var.tectonic_experimental && length(compact(var.tectonic_etcd_servers)) == 0}" tls_enabled = "${var.tectonic_etcd_tls_enabled}" external_endpoints = ["${compact(var.tectonic_etcd_servers)}"] - etcd_instance_count = "${var.tectonic_experimental ? 0 : var.tectonic_etcd_count > 0 ? var.tectonic_etcd_count : length(var.tectonic_gcp_zones) == 5 ? 5 : 3}" + etcd_instance_count = "${var.tectonic_experimental ? 0 : var.tectonic_etcd_count > 0 ? var.tectonic_etcd_count : length(data.google_compute_zones.available.names) == 5 ? 5 : 3}" managed_zone_name = "${var.tectonic_gcp_ext_google_managedzone_name}" etcd_ip_addresses = "${module.etcd.etcd_ip_addresses}" base_domain = "${var.tectonic_base_domain}" diff --git a/platforms/gcp/variables.tf b/platforms/gcp/variables.tf index c0fb527abb..ecc537d00d 100644 --- a/platforms/gcp/variables.tf +++ b/platforms/gcp/variables.tf @@ -34,11 +34,6 @@ variable "tectonic_gcp_region" { description = "The GCP region to use. Some regions only have 2 zones." } -variable "tectonic_gcp_zones" { - type = "list" - description = "List of two or more zones to use from specified GCP region." -} - variable "tectonic_gcp_master_gce_type" { type = "string" description = "Instance size for the master node(s). Example: `n1-standard-2`." diff --git a/tests/rspec/lib/gcp_cluster.rb b/tests/rspec/lib/gcp_cluster.rb index 28ca621e93..ccdf3ab4d7 100644 --- a/tests/rspec/lib/gcp_cluster.rb +++ b/tests/rspec/lib/gcp_cluster.rb @@ -30,7 +30,7 @@ def credentials_defined? def master_ip_addresses ip_addresses = [] Dir.chdir(@build_path) do - ip_address = `echo module.network.master_ip | terraform console ../../platforms/gcp`.chomp + ip_address = `echo module.network.ssh_master_ip | terraform console ../../platforms/gcp`.chomp if ip_address.empty? raise 'should get the master_ip_address to use in the tests.' end diff --git a/tests/smoke/gcp/vars/gcp.tfvars.json b/tests/smoke/gcp/vars/gcp.tfvars.json index 8ee08c7f9e..604bba645b 100644 --- a/tests/smoke/gcp/vars/gcp.tfvars.json +++ b/tests/smoke/gcp/vars/gcp.tfvars.json @@ -1,7 +1,6 @@ { "tectonic_bootstrap_upgrade_cl": "false", "tectonic_gcp_region": "us-central1", - "tectonic_gcp_zones" : ["us-central1-a","us-central1-b"], "tectonic_gcp_ext_google_managedzone_name": "testing", "tectonic_gcp_ssh_key": "~/.ssh/id_rsa.pub", "tectonic_base_domain": "tectonic.gcp.dev.coreos.systems", From 60ed9680d04364f59076d9fce2e220b89f602285 Mon Sep 17 00:00:00 2001 From: Enxebre Date: Thu, 26 Oct 2017 17:17:17 +0200 Subject: [PATCH 2/5] platform/gcp: add jenkins config --- Jenkinsfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index d45f153541..786d5490e0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,6 +9,7 @@ creds = [ file(credentialsId: 'tectonic-license', variable: 'TF_VAR_tectonic_license_path'), file(credentialsId: 'tectonic-pull', variable: 'TF_VAR_tectonic_pull_secret_path'), + file(credentialsId: 'GCP-APPLICATION', variable: 'GOOGLE_APPLICATION_CREDENTIALS'), [ $class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'tectonic-jenkins-installer' @@ -72,6 +73,11 @@ pipeline { defaultValue: true, description: '' ) + booleanParam( + name: 'PLATFORM/GCP', + defaultValue: true, + description: '' + ) booleanParam( name: 'PLATFORM/BARE_METAL', defaultValue: true, @@ -197,6 +203,7 @@ pipeline { GRAFITI_DELETER_ROLE = 'grafiti-deleter' TF_VAR_tectonic_container_images = "${params.hyperkube_image}" TF_VAR_tectonic_container_linux_version = "${params.container_linux_version}" + GOOGLE_PROJECT = "tectonic-installer" } steps { script { @@ -226,6 +233,10 @@ pipeline { builds['azure_example'] = runRSpecTest('spec/azure_example_spec.rb', '') } + if (params."PLATFORM/GCP") { + builds['gcp'] = runRSpecTest('spec/gcp_spec.rb', '') + } + if (params."PLATFORM/BARE_METAL") { builds['bare_metal'] = { node('worker && bare-metal') { From ef47c56398eab87a54aebc5903bc96b5f869be17 Mon Sep 17 00:00:00 2001 From: Enxebre Date: Thu, 26 Oct 2017 17:28:22 +0200 Subject: [PATCH 3/5] platform/gcp: update docs and add namespace for instances --- Documentation/variables/gcp.md | 1 - examples/terraform.tfvars.gcp | 3 --- modules/gcp/master-igm/master.tf | 2 +- modules/gcp/worker-igm/worker.tf | 2 +- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Documentation/variables/gcp.md b/Documentation/variables/gcp.md index 9148d0564e..bff3087af1 100644 --- a/Documentation/variables/gcp.md +++ b/Documentation/variables/gcp.md @@ -19,5 +19,4 @@ This document gives an overview of variables used in the Google Cloud platform o | tectonic_gcp_worker_disk_size | The size of the disk in gigabytes for the root block device of worker nodes. | string | `30` | | tectonic_gcp_worker_disktype | The type of disk (pd-standard or pd-ssd) for the worker nodes. | string | `pd-standard` | | tectonic_gcp_worker_gce_type | Instance size for the worker node(s). Example: `n1-standard-2`. | string | `n1-standard-2` | -| tectonic_gcp_zones | List of two or more zones to use from specified GCP region. | list | - | diff --git a/examples/terraform.tfvars.gcp b/examples/terraform.tfvars.gcp index 5fe70a8d68..1b3b0db115 100644 --- a/examples/terraform.tfvars.gcp +++ b/examples/terraform.tfvars.gcp @@ -157,9 +157,6 @@ tectonic_gcp_worker_disktype = "pd-standard" // Instance size for the worker node(s). Example: `n1-standard-2`. tectonic_gcp_worker_gce_type = "n1-standard-2" -// List of two or more zones to use from specified GCP region. -tectonic_gcp_zones = "" - // The path to the tectonic licence file. // You can download the Tectonic license file from your Account overview page at [1]. // diff --git a/modules/gcp/master-igm/master.tf b/modules/gcp/master-igm/master.tf index dd04a010ce..049210809a 100644 --- a/modules/gcp/master-igm/master.tf +++ b/modules/gcp/master-igm/master.tf @@ -54,7 +54,7 @@ resource "google_compute_region_instance_group_manager" "master-igm" { name = "${var.cluster_name}-master-igm-${count.index}" instance_template = "${google_compute_instance_template.master-it.self_link}" target_pools = ["${var.master_targetpool_self_link}"] - base_instance_name = "mstr" + base_instance_name = "${var.cluster_name}-mstr" named_port { name = "https" diff --git a/modules/gcp/worker-igm/worker.tf b/modules/gcp/worker-igm/worker.tf index 88846c0cf0..84384bab55 100644 --- a/modules/gcp/worker-igm/worker.tf +++ b/modules/gcp/worker-igm/worker.tf @@ -54,5 +54,5 @@ resource "google_compute_region_instance_group_manager" "worker-igm" { name = "${var.cluster_name}-worker-igm-${count.index}" instance_template = "${google_compute_instance_template.worker-it.self_link}" target_pools = ["${var.worker_targetpool_self_link}"] - base_instance_name = "wrkr" + base_instance_name = "${var.cluster_name}-wrkr" } From 383967a0e5e0eabbf285cc4fe78978473e98362f Mon Sep 17 00:00:00 2001 From: Enxebre Date: Thu, 26 Oct 2017 18:15:31 +0200 Subject: [PATCH 4/5] platform/gcp: pin container linux variable for tests --- tests/smoke/gcp/vars/gcp.tfvars.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/smoke/gcp/vars/gcp.tfvars.json b/tests/smoke/gcp/vars/gcp.tfvars.json index 604bba645b..90ec4105c6 100644 --- a/tests/smoke/gcp/vars/gcp.tfvars.json +++ b/tests/smoke/gcp/vars/gcp.tfvars.json @@ -1,5 +1,6 @@ { "tectonic_bootstrap_upgrade_cl": "false", + "tectonic_container_linux_version": "latest", "tectonic_gcp_region": "us-central1", "tectonic_gcp_ext_google_managedzone_name": "testing", "tectonic_gcp_ssh_key": "~/.ssh/id_rsa.pub", From 6856be82cda431dbe6181b11a9908baa0fdac09f Mon Sep 17 00:00:00 2001 From: Enxebre Date: Fri, 27 Oct 2017 16:09:38 +0200 Subject: [PATCH 5/5] platform/gcp: add worker and master to instance name --- modules/gcp/master-igm/master.tf | 2 +- modules/gcp/worker-igm/worker.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/gcp/master-igm/master.tf b/modules/gcp/master-igm/master.tf index 049210809a..e12537c4f2 100644 --- a/modules/gcp/master-igm/master.tf +++ b/modules/gcp/master-igm/master.tf @@ -54,7 +54,7 @@ resource "google_compute_region_instance_group_manager" "master-igm" { name = "${var.cluster_name}-master-igm-${count.index}" instance_template = "${google_compute_instance_template.master-it.self_link}" target_pools = ["${var.master_targetpool_self_link}"] - base_instance_name = "${var.cluster_name}-mstr" + base_instance_name = "${var.cluster_name}-master" named_port { name = "https" diff --git a/modules/gcp/worker-igm/worker.tf b/modules/gcp/worker-igm/worker.tf index 84384bab55..830f541fb0 100644 --- a/modules/gcp/worker-igm/worker.tf +++ b/modules/gcp/worker-igm/worker.tf @@ -54,5 +54,5 @@ resource "google_compute_region_instance_group_manager" "worker-igm" { name = "${var.cluster_name}-worker-igm-${count.index}" instance_template = "${google_compute_instance_template.worker-it.self_link}" target_pools = ["${var.worker_targetpool_self_link}"] - base_instance_name = "${var.cluster_name}-wrkr" + base_instance_name = "${var.cluster_name}-worker" }