From 1680afb1c1a499b42348d26e7d8beee168375474 Mon Sep 17 00:00:00 2001 From: Nicholas Eberts Date: Fri, 6 May 2022 16:56:02 -0400 Subject: [PATCH] Enable Fleet Workload ID pools (#164) * Added ASM resource label logic to the cluster module * Added ASM resource label logic to the cluster module * Added ASM resource label logic to the cluster module * Added Authority to cluster config. * added issuer version * Added both versions of the gateway crds * Cleaned up unused code in the CLI * Added tracing to ASM config map Co-authored-by: Nick Eberts --- cli/cmd/create.go | 16 -- cli/pkg/anthos/configsync.go | 240 ++++++++---------- cli/pkg/anthos/kubernetes_helpers.go | 85 +++---- cli/pkg/anthos/multi_cluster_gateway.go | 43 ---- .../asm/manifests/asm-access-logs.yaml | 7 +- terraform/modules/hub/hub.tf | 5 +- terraform/modules/mcg/scripts/install_crds.sh | 10 +- 7 files changed, 168 insertions(+), 238 deletions(-) delete mode 100644 cli/pkg/anthos/multi_cluster_gateway.go diff --git a/cli/cmd/create.go b/cli/cmd/create.go index 0be839e1..c48fb630 100644 --- a/cli/cmd/create.go +++ b/cli/cmd/create.go @@ -76,22 +76,6 @@ var createCmd = &cobra.Command{ log.Info("✅ Clusters API access check passed.") } - // Init Multi-cluster Gateway - // if conf.MultiClusterGateway { - // err := anthos.InitMCG(kc) - // if err != nil { - // log.Errorf("🚨 Failed to initialize Multi-cluster Gateway CRDs: %s", err) - // } else { - // log.Info("✅ MultiCluster Gateway CRDs installed successfully.") - // } - // } - - // Run Anthos modules if anthos features are enabled - // if conf.ConfigSync || conf.MultiClusterGateway { - // lifecycle.InitTF("anthos", tfStateBucket[2]) - // lifecycle.ApplyTF("anthos") - // } - // Init ACM (either ConfigSync or ConfigSync plus PolicyController) if conf.ConfigSync { err := anthos.InitACM(conf, kc) diff --git a/cli/pkg/anthos/configsync.go b/cli/pkg/anthos/configsync.go index 51667a09..535d26b6 100644 --- a/cli/pkg/anthos/configsync.go +++ b/cli/pkg/anthos/configsync.go @@ -17,29 +17,11 @@ limitations under the License. package anthos import ( - "context" - "fmt" - "io/ioutil" - "strings" - "gkekitctl/pkg/config" log "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" - "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // register GCP auth provider "k8s.io/client-go/tools/clientcmd/api" - - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "encoding/pem" - "os" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/clientcmd" - - "golang.org/x/crypto/ssh" ) // If ConfigSync is enabled, this function runs after TF apply and does the following: @@ -60,121 +42,121 @@ func InitACM(conf *config.Config, kc *api.Config) error { log.Infof("✔️ Kubeconfig generated: %+v", kc) // Verify access to Kubernetes API on all clusters - log.Info("☸️ Verifying Kubernetes API access for all clusters...") - err = ListNamespaces(kc) - if err != nil { - return err - } + // log.Info("☸️ Verifying Kubernetes API access for all clusters...") + // err = ListNamespaces(kc) + // if err != nil { + // return err + // } return nil } // ssh-keygen // Source: https://stackoverflow.com/questions/21151714/go-generate-an-ssh-public-key -func InitSSH() error { - privateKeyPath := "id_rsa" - pubKeyPath := "id_rsa.pub" - - // make ssh dir if not exists - err := os.MkdirAll("ssh", 0700) - if err != nil { - log.Warnf("Error making ssh dir: %v", err) - } - - // ssh keygen to local dir - log.Info("Generating private key") - privateKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return err - } - - // generate and write private key as PEM - log.Info("Encoding private key") - privateKeyFile, err := os.Create(privateKeyPath) - defer privateKeyFile.Close() - if err != nil { - return err - } - privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)} - if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil { - return err - } - - // generate and write public key - log.Info("Generating public key") - pub, err := ssh.NewPublicKey(&privateKey.PublicKey) - if err != nil { - return err - } - return ioutil.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0655) -} - -func PromptUser(conf *config.Config) error { - // read public key as string - bytes, err := ioutil.ReadFile("id_rsa.pub") - if err != nil { - return err - } - pubKey := string(bytes) - - // Prompt user to register ssh public key to their Cloud Source Repositories - log.Info("💻 Copy the key below to the clipboard, then open this link to register: https://source.cloud.google.com/user/ssh_keys") - log.Info(pubKey) - log.Info("Once you've registered the key, press enter to continue...") - fmt.Scanln() // wait for Enter Key - - // Prompt user to run make start proxy - if conf.PrivateEndpoint { - log.Info("⚠️ Your clusters have Private Endpoints. Please open another terminal tab and run the following command to proxy via your GCE Bastion Host.") - log.Infof("gcloud beta compute ssh gke-tk-bastion --tunnel-through-iap --project %s --zone %s-b -- -4 -L8888:127.0.0.1:8888", conf.ClustersProjectID, conf.ClustersConfig[0].Region) - log.Info("Once the proxy tunnel is running, press enter to continue...") - fmt.Scanln() // wait for Enter Key - } - return nil -} +// func InitSSH() error { +// privateKeyPath := "id_rsa" +// pubKeyPath := "id_rsa.pub" + +// // make ssh dir if not exists +// err := os.MkdirAll("ssh", 0700) +// if err != nil { +// log.Warnf("Error making ssh dir: %v", err) +// } + +// // ssh keygen to local dir +// log.Info("Generating private key") +// privateKey, err := rsa.GenerateKey(rand.Reader, 4096) +// if err != nil { +// return err +// } + +// // generate and write private key as PEM +// log.Info("Encoding private key") +// privateKeyFile, err := os.Create(privateKeyPath) +// defer privateKeyFile.Close() +// if err != nil { +// return err +// } +// privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)} +// if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil { +// return err +// } + +// // generate and write public key +// log.Info("Generating public key") +// pub, err := ssh.NewPublicKey(&privateKey.PublicKey) +// if err != nil { +// return err +// } +// return ioutil.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0655) +// } + +// func PromptUser(conf *config.Config) error { +// // read public key as string +// bytes, err := ioutil.ReadFile("id_rsa.pub") +// if err != nil { +// return err +// } +// pubKey := string(bytes) + +// // Prompt user to register ssh public key to their Cloud Source Repositories +// log.Info("💻 Copy the key below to the clipboard, then open this link to register: https://source.cloud.google.com/user/ssh_keys") +// log.Info(pubKey) +// log.Info("Once you've registered the key, press enter to continue...") +// fmt.Scanln() // wait for Enter Key + +// // Prompt user to run make start proxy +// if conf.PrivateEndpoint { +// log.Info("⚠️ Your clusters have Private Endpoints. Please open another terminal tab and run the following command to proxy via your GCE Bastion Host.") +// log.Infof("gcloud beta compute ssh gke-tk-bastion --tunnel-through-iap --project %s --zone %s-b -- -4 -L8888:127.0.0.1:8888", conf.ClustersProjectID, conf.ClustersConfig[0].Region) +// log.Info("Once the proxy tunnel is running, press enter to continue...") +// fmt.Scanln() // wait for Enter Key +// } +// return nil +// } // Create gitcreds Secret in every cluster, using the contents of the id_rsa file as data. -func CreateGitCredsSecret(kubeConfig *api.Config) error { - ctx := context.Background() - - // Get string value of private key - privateKey, err := ioutil.ReadFile("id_rsa") - if err != nil { - return fmt.Errorf("Failed to read id_rsa from file: %v", err) - } - privateKeyString := string(privateKey) - // log.Infof("Private key string is the contents below:") - // log.Info(privateKeyString) - - for clusterName := range kubeConfig.Clusters { - cfg, err := clientcmd.NewNonInteractiveClientConfig(*kubeConfig, clusterName, &clientcmd.ConfigOverrides{CurrentContext: clusterName}, nil).ClientConfig() - if err != nil { - return fmt.Errorf("failed to create Kubernetes configuration cluster=%s: %w", clusterName, err) - } - - k8s, err := kubernetes.NewForConfig(cfg) - if err != nil { - return fmt.Errorf("failed to create Kubernetes client cluster=%s: %w", clusterName, err) - } - - err = WaitForNamespace(k8s, ctx, "config-management-system", clusterName) - if err != nil { - return fmt.Errorf("config management system namespace is not ready on cluster=%s: %w", clusterName, err) - } - - _, err = k8s.CoreV1().Secrets("config-management-system").Create(ctx, &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "git-creds", - }, - StringData: map[string]string{"ssh": privateKeyString}, - }, metav1.CreateOptions{}) - - if err != nil { - if strings.Contains(err.Error(), "already exists") { - log.Warn("git-creds secret already exists") - } else { - return fmt.Errorf("failed to create gitcreds secret- cluster=%s: %w", clusterName, err) - } - } - } - return nil -} +// func CreateGitCredsSecret(kubeConfig *api.Config) error { +// ctx := context.Background() + +// // Get string value of private key +// privateKey, err := ioutil.ReadFile("id_rsa") +// if err != nil { +// return fmt.Errorf("Failed to read id_rsa from file: %v", err) +// } +// privateKeyString := string(privateKey) +// // log.Infof("Private key string is the contents below:") +// // log.Info(privateKeyString) + +// for clusterName := range kubeConfig.Clusters { +// cfg, err := clientcmd.NewNonInteractiveClientConfig(*kubeConfig, clusterName, &clientcmd.ConfigOverrides{CurrentContext: clusterName}, nil).ClientConfig() +// if err != nil { +// return fmt.Errorf("failed to create Kubernetes configuration cluster=%s: %w", clusterName, err) +// } + +// k8s, err := kubernetes.NewForConfig(cfg) +// if err != nil { +// return fmt.Errorf("failed to create Kubernetes client cluster=%s: %w", clusterName, err) +// } + +// err = WaitForNamespace(k8s, ctx, "config-management-system", clusterName) +// if err != nil { +// return fmt.Errorf("config management system namespace is not ready on cluster=%s: %w", clusterName, err) +// } + +// _, err = k8s.CoreV1().Secrets("config-management-system").Create(ctx, &v1.Secret{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "git-creds", +// }, +// StringData: map[string]string{"ssh": privateKeyString}, +// }, metav1.CreateOptions{}) + +// if err != nil { +// if strings.Contains(err.Error(), "already exists") { +// log.Warn("git-creds secret already exists") +// } else { +// return fmt.Errorf("failed to create gitcreds secret- cluster=%s: %w", clusterName, err) +// } +// } +// } +// return nil +// } diff --git a/cli/pkg/anthos/kubernetes_helpers.go b/cli/pkg/anthos/kubernetes_helpers.go index c62b8074..2fbf3d7d 100644 --- a/cli/pkg/anthos/kubernetes_helpers.go +++ b/cli/pkg/anthos/kubernetes_helpers.go @@ -22,16 +22,11 @@ import ( "fmt" "gkekitctl/pkg/config" - "github.com/pytimer/k8sutil/apply" log "github.com/sirupsen/logrus" "google.golang.org/api/container/v1" - clientgo "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/discovery" - "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd/api" ) @@ -95,10 +90,10 @@ func GenerateKubeConfig(conf *config.Config) (*api.Config, error) { } // Write kubeconfig to YAML file - err = clientcmd.WriteToFile(ret, "kubeconfig") - if err != nil { - return &ret, err - } + // err = clientcmd.WriteToFile(ret, "kubeconfig") + // if err != nil { + // return &ret, err + // } return &ret, nil } @@ -131,44 +126,44 @@ func ListNamespaces(kubeConfig *api.Config) error { } // Kubectl apply using client.go -func Apply(config *rest.Config, clusterName string, fileName []byte) error { +// func Apply(config *rest.Config, clusterName string, fileName []byte) error { - dynamicClient, err := dynamic.NewForConfig(config) - if err != nil { - return fmt.Errorf("failed to setup dynamic client for cluster=%s: %w", clusterName, err) - } - discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) - if err != nil { - return fmt.Errorf("failed to setup diecovery client for cluster=%s: %w", clusterName, err) - } +// dynamicClient, err := dynamic.NewForConfig(config) +// if err != nil { +// return fmt.Errorf("failed to setup dynamic client for cluster=%s: %w", clusterName, err) +// } +// discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) +// if err != nil { +// return fmt.Errorf("failed to setup diecovery client for cluster=%s: %w", clusterName, err) +// } - applyOptions := apply.NewApplyOptions(dynamicClient, discoveryClient) - if err := applyOptions.Apply(context.TODO(), fileName); err != nil { - return fmt.Errorf("failed to create apply gateway crd cluster=%s: %w", clusterName, err) - } +// applyOptions := apply.NewApplyOptions(dynamicClient, discoveryClient) +// if err := applyOptions.Apply(context.TODO(), fileName); err != nil { +// return fmt.Errorf("failed to create apply gateway crd cluster=%s: %w", clusterName, err) +// } - return nil -} +// return nil +// } // check namespace and watch if not created -func WaitForNamespace(k8s *kubernetes.Clientset, ctx context.Context, nameSpace string, clusterName string) error { - ns, err := k8s.CoreV1().Namespaces().Get(ctx, "config-management-system", metav1.GetOptions{}) - timeout := int64(120) - if clientgo.IsNotFound(err) { - log.Infof("%s was not found on cluster=%s: %v", nameSpace, clusterName, err) - ns, err := k8s.CoreV1().Namespaces().Watch(ctx, metav1.ListOptions{ - FieldSelector: "metadata.name=" + nameSpace, - Watch: true, - TimeoutSeconds: &timeout, - }) - if err != nil { - return fmt.Errorf("failed watch on namespace %s on cluster=%s: %v", nameSpace, clusterName, err) - } - log.Infof("%s is ready on cluster: %s", ns, clusterName) - - } else if err != nil { - return fmt.Errorf("%s namespace on cluster=%s: %w", nameSpace, clusterName, err) - } - log.Infof("%s is ready on cluster: %s", ns, clusterName) - return nil -} +// func WaitForNamespace(k8s *kubernetes.Clientset, ctx context.Context, nameSpace string, clusterName string) error { +// ns, err := k8s.CoreV1().Namespaces().Get(ctx, "config-management-system", metav1.GetOptions{}) +// timeout := int64(120) +// if clientgo.IsNotFound(err) { +// log.Infof("%s was not found on cluster=%s: %v", nameSpace, clusterName, err) +// ns, err := k8s.CoreV1().Namespaces().Watch(ctx, metav1.ListOptions{ +// FieldSelector: "metadata.name=" + nameSpace, +// Watch: true, +// TimeoutSeconds: &timeout, +// }) +// if err != nil { +// return fmt.Errorf("failed watch on namespace %s on cluster=%s: %v", nameSpace, clusterName, err) +// } +// log.Infof("%s is ready on cluster: %s", ns, clusterName) + +// } else if err != nil { +// return fmt.Errorf("%s namespace on cluster=%s: %w", nameSpace, clusterName, err) +// } +// log.Infof("%s is ready on cluster: %s", ns, clusterName) +// return nil +// } diff --git a/cli/pkg/anthos/multi_cluster_gateway.go b/cli/pkg/anthos/multi_cluster_gateway.go deleted file mode 100644 index b13ec664..00000000 --- a/cli/pkg/anthos/multi_cluster_gateway.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright © 2020 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package anthos - -import ( - "fmt" - "io/ioutil" - - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/clientcmd/api" -) - -func InitMCG(kubeConfig *api.Config) error { - mcgConfigs, err := ioutil.ReadFile("templates/gateway_api_crds.yaml") - if err != nil { - return fmt.Errorf("failed to read in gateway api crd yaml: %w", err) - } - for clusterName := range kubeConfig.Clusters { - cfg, err := clientcmd.NewNonInteractiveClientConfig(*kubeConfig, clusterName, &clientcmd.ConfigOverrides{CurrentContext: clusterName}, nil).ClientConfig() - if err != nil { - return fmt.Errorf("failed to create Kubernetes configuration cluster=%s: %w", clusterName, err) - } - - if err := Apply(cfg, clusterName, mcgConfigs); err != nil { - return fmt.Errorf("failed to apply gateway api crd to cluster=%s: %w", clusterName, err) - } - } - return nil -} diff --git a/terraform/modules/asm/manifests/asm-access-logs.yaml b/terraform/modules/asm/manifests/asm-access-logs.yaml index ed399a38..cead1d4a 100644 --- a/terraform/modules/asm/manifests/asm-access-logs.yaml +++ b/terraform/modules/asm/manifests/asm-access-logs.yaml @@ -4,6 +4,9 @@ metadata: name: istio-asm-managed namespace: istio-system data: - mesh: | + mesh: |- accessLogFile: /dev/stdout - accessLogEncoding: JSON \ No newline at end of file + accessLogEncoding: JSON + defaultConfig: + tracing: + stackdriver: {} \ No newline at end of file diff --git a/terraform/modules/hub/hub.tf b/terraform/modules/hub/hub.tf index f5847898..f7c4b471 100644 --- a/terraform/modules/hub/hub.tf +++ b/terraform/modules/hub/hub.tf @@ -17,4 +17,7 @@ resource "google_gke_hub_membership" "membership" { resource_link = "//container.googleapis.com/projects/${var.project_id}/locations/${each.value.region}/clusters/${each.key}" } } -} \ No newline at end of file + authority { + issuer = "https://container.googleapis.com/v1/projects/${var.project_id}/locations/${each.value.region}/clusters/${each.key}" + } +} \ No newline at end of file diff --git a/terraform/modules/mcg/scripts/install_crds.sh b/terraform/modules/mcg/scripts/install_crds.sh index 76265390..e9da274a 100755 --- a/terraform/modules/mcg/scripts/install_crds.sh +++ b/terraform/modules/mcg/scripts/install_crds.sh @@ -12,8 +12,14 @@ export KUBECONFIG=${WORKDIR}/tempkubeconfig # Install Gateway API CRDs echo -e "Installing GatewayAPI CRDs" -kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=${GATEWAY_API_VERSION}" \ -| kubectl apply --kubeconfig ${KUBECONFIG} --context=gke_${PROJECT_ID}_${LOCATION}_${CLUSTER} -f - 2>&1 >/dev/null + +# Gateway api crds need to be installed in a specific order, this is temporary. +# kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=${GATEWAY_API_VERSION}" \ +# | kubectl apply --kubeconfig ${KUBECONFIG} --context=gke_${PROJECT_ID}_${LOCATION}_${CLUSTER} -f - 2>&1 >/dev/null + +kubectl apply -k "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v0.4.2" --kubeconfig ${KUBECONFIG} --context=gke_${PROJECT_ID}_${LOCATION}_${CLUSTER} + +kubectl apply -k "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v0.3.0" --kubeconfig ${KUBECONFIG} --context=gke_${PROJECT_ID}_${LOCATION}_${CLUSTER} # Verify CRD is established in the cluster echo -e "Validating GatewayAPI CRD creation"