Improvements (#175)

Replaced Update and Create commands with Apply
Added flag to pass in gke cluster and shared vpc TF state storage buckets
Refactored the k8s api check to use a connect gw cred

Co-authored-by: Nick Eberts <[email protected]>

Author: knee-berts

README.md

@@ -46,10 +46,10 @@ curl -sLSf -o ./gkekitctl https://github.com/GoogleCloudPlatform/gke-poc-toolkit
 ./gkekitctl init
 ```
 
-7. **Run `gkekitctl create` to run the Toolkit.** By default, this command sets up a single-cluster GKE environment. ([Configuration here](cli/pkg/cli_init/samples/default-config.yaml)). Enter your project ID when prompted.
+7. **Run `gkekitctl apply` to run the Toolkit.** By default, this command sets up a single-cluster GKE environment. ([Configuration here](cli/pkg/cli_init/samples/default-config.yaml)). Enter your project ID when prompted.
 
 ```shell
-./gkekitctl create
+./gkekitctl apply
 ```
 ```shell
 # expected output 
@@ -73,9 +73,13 @@ time="2022-02-04T21:58:00Z" level=info msg="🌎 5 Namespaces found in cluster=g
 
 ## Update
 
-If you want to update your environment change the config file and run the update command. This is a great way to add or remove clusters.
+If you want to update your environment change the config file and re-run the apply command. This is a great way to add or remove clusters.
 ```bash
-./gkekitctl update --config <config file name>
+## Local tf state 
+./gkekitctl apply --config <"config file name">
+
+## If you are using remote TF state 
+./gkekitctl apply --config <"config file name"> --gkestate <"bucket name used for state file"> --vpcstate <"if using sharevpc, bucket name for shared vpc state file">
 ```
 
 ## Clean up 

cli/cmd/create.go renamed to cli/cmd/apply.go

@@ -26,26 +26,43 @@ import (
 )
 
 // createCmd represents the create command
-var createCmd = &cobra.Command{
-	Use:   "create",
-	Short: "Create GKE Demo Environment",
-	Example: ` gkekitctl create
-	gkekitctl create --config <file.yaml>`,
+var applyCmd = &cobra.Command{
+	Use:   "apply",
+	Short: "Create or Update GKE Demo Environment",
+	Example: ` gkekitctl apply
+	gkekitctl apply --config <file.yaml>`,
 	Run: func(cmd *cobra.Command, args []string) {
+		bucketNameClusters, err := cmd.Flags().GetString("gkestate")
+		if err != nil {
+			log.Errorf("🚨Failed to get bucketNameClusters value from flag: %s", err)
+		}
+		if bucketNameClusters != "" {
+			log.Infof("✅ Terraform state storage bucket for clusters is %s", bucketNameClusters)
+		}
+		bucketNameSharedVPC, err := cmd.Flags().GetString("vpcstate")
+		if err != nil {
+			log.Errorf("🚨Failed to get bucketNameSharedVPC value from flag: %s", err)
+		}
+		if bucketNameSharedVPC != "" {
+			log.Infof("✅ Terraform state storage bucket for shared VPC is %s", bucketNameSharedVPC)
+		}
+
 		log.Info("👟 Started config validation...")
 		conf := config.InitConf(cfgFile)
-		log.Info("👟 Started generating TFVars...")
 
 		// Send user analytics - async
 		if conf.SendAnalytics {
 			go analytics.SendAnalytics(conf, Version, GitCommit)
 		}
 
+		log.Info("👟 Started generating TFVars...")
 		config.GenerateTfvars(conf)
+
 		log.Info("👟 Started configuring TF State...")
-		err := config.CheckTfStateType(conf)
+		err = config.CheckTfStateType(conf, bucketNameClusters, bucketNameSharedVPC)
+
 		if err != nil {
-			log.Errorf("🚨 Failed checking TF state type: %s", err)
+			log.Errorf("🚨 Failed setting up TF state: %s", err)
 		} else {
 			log.Info("✅ TF state configured successfully.")
 		}
@@ -78,5 +95,10 @@ var createCmd = &cobra.Command{
 }
 
 func init() {
-	rootCmd.AddCommand(createCmd)
+	var bucketNameClusters string
+	var bucketNameSharedVPC string
+
+	rootCmd.AddCommand(applyCmd)
+	applyCmd.Flags().StringVarP(&bucketNameClusters, "gkestate", "g", "", "GKE Tf State bucket name")
+	applyCmd.Flags().StringVarP(&bucketNameSharedVPC, "vpcstate", "v", "", "Shared VPC Tf State bucket name")
 }

cli/cmd/root.go

@@ -61,7 +61,7 @@ func init() {
 	// Cobra supports persistent flags, which, if defined here,
 	// will be global for your application.
 
-	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is /.gkekitctl.yaml)")
+	rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file (default is /.gkekitctl.yaml)")
 
 	// Cobra also supports local flags, which will only run
 	// when this action is called directly.

cli/cmd/update.go

@@ -18,6 +18,7 @@ package config
 
 import (
 	"context"
+	"fmt"
 	"html/template"
 	"os"
 	"strings"
@@ -27,40 +28,60 @@ import (
 	"github.com/thanhpk/randstr"
 )
 
-func CheckTfStateType(conf *Config) error {
+func CheckTfStateType(conf *Config, bucketNameClusters string, bucketNameSharedVPC string) error {
 	if conf.TerraformState == "cloud" {
-		bucketNames := [3]string{}
 		if conf.VpcConfig.VpcType == "shared" {
-			bucketNameSharedVPC := "tf-state-sharedvpc-" + strings.ToLower(randstr.String(6))
-			err := createTfBackend(conf.VpcConfig.VpcProjectID, bucketNameSharedVPC, "shared_vpc/backend.tf")
+			if bucketNameSharedVPC == "" {
+				bucketNameSharedVPC := "tf-state-sharedvpc-" + strings.ToLower(randstr.String(6))
+				err := createTfStorage(conf.VpcConfig.VpcProjectID, bucketNameSharedVPC)
+				if err != nil {
+					return err
+				}
+			}
+			err := createTfBackend(bucketNameSharedVPC, "shared_vpc/backend.tf")
+			if err != nil {
+				return err
+			}
+		}
+		if bucketNameClusters == "" {
+			log.Infof("Clusters Bucket Name: %s", bucketNameClusters)
+			bucketNameClusters := "tf-state-clusters-" + strings.ToLower(randstr.String(6))
+			err := createTfStorage(conf.ClustersProjectID, bucketNameClusters)
 			if err != nil {
 				return err
 			}
 		}
-		bucketNameClusters := "tf-state-clusters-" + strings.ToLower(randstr.String(6))
-		err := createTfBackend(conf.ClustersProjectID, bucketNameClusters, "cluster_build/backend.tf")
+		err := createTfBackend(bucketNameClusters, "cluster_build/backend.tf")
 		if err != nil {
 			return err
 		}
-		log.Print(bucketNames)
 		return nil
 	}
 	return nil
 }
 
 // func CreateTfStateBucket(projectId string, bucketName string) error {
-func createTfBackend(projectId string, bucketName string, fileLocation string) error {
+func createTfStorage(projectId string, bucketName string) error {
 	ctx := context.Background()
 	c, err := storage.NewClient(ctx)
 	if err != nil {
-		log.Fatalf("error creating storage client: %s", err)
 		return err
 	}
-	err = c.Bucket(bucketName).Create(ctx, projectId, nil)
-	if err != nil {
-		log.Fatalf("error creating storage bucket: %s", err)
-		return err
+	attrs, err := c.Bucket(bucketName).Attrs(ctx)
+	if err == storage.ErrBucketNotExist {
+		log.Infof("Creating storage bucket: %s ", bucketName)
+		err = c.Bucket(bucketName).Create(ctx, projectId, nil)
+		if err != nil {
+			log.Fatalf("error creating storage bucket: %s", err)
+			return err
+		}
+	} else {
+		fmt.Printf("The bucket exists and has attributes: %#v\n", attrs)
 	}
+	return err
+}
+
+func createTfBackend(bucketName string, fileLocation string) error {
 	vars := make(map[string]interface{})
 	vars["TfStateBucket"] = bucketName
 	tmpl, err := template.ParseFiles("templates/terraform_backend.tf.tmpl")

cli/pkg/config/tf_state.go

@@ -18,11 +18,12 @@ package lifecycle
 
 import (
 	"context"
-	"encoding/base64"
 	"fmt"
 	"gkekitctl/pkg/config"
+	"strconv"
 
 	log "github.com/sirupsen/logrus"
+	"google.golang.org/api/cloudresourcemanager/v1"
 	"google.golang.org/api/container/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -33,6 +34,11 @@ import (
 
 func GenerateKubeConfig(conf *config.Config) (*api.Config, error) {
 	projectId := conf.ClustersProjectID
+	projectNumber, err := GetProjectNumber(projectId)
+	if err != nil {
+		return api.NewConfig(), fmt.Errorf("GetProjectNumber: %w", err)
+	}
+
 	log.Infof("Clusters Project ID is %s", projectId)
 	ctx := context.Background()
 
@@ -57,21 +63,12 @@ func GenerateKubeConfig(conf *config.Config) (*api.Config, error) {
 	}
 
 	for _, f := range resp.Clusters {
-		name := fmt.Sprintf("gke_%s_%s_%s", projectId, f.Zone, f.Name)
+		name := fmt.Sprintf("connectgateway_%s_global_%s-membership", projectId, f.Name)
+		server := fmt.Sprintf("https://connectgateway.googleapis.com/v1beta1/projects/%s/locations/global/gkeMemberships/%s-membership", projectNumber, f.Name)
 		log.Infof("Connecting to cluster: %s,", name)
-		cert, err := base64.StdEncoding.DecodeString(f.MasterAuth.ClusterCaCertificate)
-		if err != nil {
-			return &ret, fmt.Errorf("invalid certificate cluster=%s cert=%s: %w", name, f.MasterAuth.ClusterCaCertificate, err)
-		}
-		// example: gke_my-project_us-central1-b_cluster-1 => https://XX.XX.XX.XX
-		proxy := ""
-		if conf.PrivateEndpoint {
-			proxy = "http://localhost:8888"
-		}
+
 		ret.Clusters[name] = &api.Cluster{
-			CertificateAuthorityData: cert,
-			Server:                   "https://" + f.Endpoint,
-			ProxyURL:                 proxy,
+			Server: server,
 		}
 		// Just reuse the context name as an auth name.
 		ret.Contexts[name] = &api.Context{
@@ -82,18 +79,15 @@ func GenerateKubeConfig(conf *config.Config) (*api.Config, error) {
 		ret.AuthInfos[name] = &api.AuthInfo{
 			AuthProvider: &api.AuthProviderConfig{
 				Name: "gcp",
-				Config: map[string]string{
-					"scopes": "https://www.googleapis.com/auth/cloud-platform",
-				},
 			},
 		}
 	}
 
 	// Write kubeconfig to YAML file
-	// err = clientcmd.WriteToFile(ret, "kubeconfig")
-	// if err != nil {
-	// 	return &ret, err
-	// }
+	err = clientcmd.WriteToFile(ret, "kubeconfig")
+	if err != nil {
+		return &ret, err
+	}
 	return &ret, nil
 }
 
@@ -125,6 +119,19 @@ func ListNamespaces(kubeConfig *api.Config) error {
 	return nil
 }
 
+func GetProjectNumber(project_id string) (string, error) {
+	ctx := context.Background()
+	cloudresourcemanagerService, err := cloudresourcemanager.NewService(ctx)
+	if err != nil {
+		fmt.Errorf("cloudresourcemanager.NewService: %w", err)
+	}
+	project, err := cloudresourcemanagerService.Projects.Get(project_id).Do()
+	if err != nil {
+		fmt.Errorf("cloudresourcemanager.NewService: %w", err)
+	}
+	return strconv.FormatInt(project.ProjectNumber, 10), nil
+}
+
 // Kubectl apply using client.go
 // func Apply(config *rest.Config, clusterName string, fileName []byte) error {
 

cli/pkg/lifecycle/kubernetes_helpers.go

@@ -116,7 +116,7 @@ steps:
     gcloud auth application-default print-access-token
     gcloud config set project $$TEST_PROJECT_ID
     gcloud auth list
-    ./gkekitctl create --config samples/default-config.yaml
+    ./gkekitctl apply --config samples/default-config.yaml
 timeout: "60m"
 availableSecrets:
   secretManager:

test/e2e/cloudbuild-default-config.yaml

@@ -117,7 +117,7 @@ steps:
     gcloud auth application-default print-access-token
     gcloud config set project $$TEST_PROJECT_ID
     gcloud auth list
-    ./gkekitctl create --config samples/module-bypass-multi-clusters-networking-acm-standalone-vpc.yaml
+    ./gkekitctl apply --config samples/module-bypass-multi-clusters-networking-acm-standalone-vpc.yaml
 timeout: "60m"
 availableSecrets:
   secretManager:

test/e2e/cloudbuild-module-bypass-multi-clusters-networking-acm-standalone-vpc.yaml

@@ -129,7 +129,7 @@ steps:
     gcloud auth application-default print-access-token
     gcloud config set project $$TEST_PROJECT_ID
     gcloud auth list
-    ./gkekitctl create --config samples/multi-clusters-shared-vpc.yaml
+    ./gkekitctl apply --config samples/multi-clusters-shared-vpc.yaml
 timeout: "60m"
 availableSecrets:
   secretManager:

test/e2e/cloudbuild-multi-clusters-acm-shared-vpc.yaml

@@ -117,7 +117,7 @@ steps:
     gcloud auth application-default print-access-token
     gcloud config set project $$TEST_PROJECT_ID
     gcloud auth list
-    ./gkekitctl create --config samples/multi-clusters-standalone-vpc.yaml
+    ./gkekitctl apply --config samples/multi-clusters-standalone-vpc.yaml
 timeout: "60m"
 availableSecrets:
   secretManager:

test/e2e/cloudbuild-multi-clusters-acm-standalone-vpc.yaml

@@ -128,7 +128,7 @@ steps:
     gcloud auth application-default print-access-token
     gcloud config set project $$TEST_PROJECT_ID
     gcloud auth list
-    ./gkekitctl create --config samples/multi-clusters-networking-acm-shared-vpc.yaml
+    ./gkekitctl apply --config samples/multi-clusters-networking-acm-shared-vpc.yaml
 timeout: "60m"
 availableSecrets:
   secretManager: