diff --git a/docs/pages/includes/helm-reference/zz_generated.tbot.mdx b/docs/pages/includes/helm-reference/zz_generated.tbot.mdx index d9a5e36600eec..6a6de55db3ebc 100644 --- a/docs/pages/includes/helm-reference/zz_generated.tbot.mdx +++ b/docs/pages/includes/helm-reference/zz_generated.tbot.mdx @@ -35,7 +35,7 @@ This must contain the port number, usually 443 or 3080 for Proxy Service. Connecting to the Proxy Service is the most common and recommended way to connect to Teleport. This is mandatory to connect to Teleport Enterprise (Cloud) -This setting is mutually exclusive with teleportProxyAddress and is ignored if `customConfig` is set. +This setting is mutually exclusive with teleportProxyAddress and is ignored if `tbotConfig` is set. For example: ```yaml @@ -54,7 +54,7 @@ should be used when you are deploying the bot in the same Kubernetes cluster tha Helm release and have direct access to the Auth Service. Else, you should prefer connecting via the Proxy Service. -This setting is mutually exclusive with teleportProxyAddress and is ignored if `customConfig` is set. +This setting is mutually exclusive with teleportProxyAddress and is ignored if `tbotConfig` is set. For example: ```yaml @@ -64,7 +64,7 @@ teleportAuthAddress: "teleport-auth.teleport-namespace.svc.cluster.local:3025" ## `defaultOutput` `defaultOutput` controls the default output configured for the tbot agent. -Ignored if `customConfig` is set. +Ignored if `tbotConfig` is set. ### `defaultOutput.enabled` @@ -74,6 +74,102 @@ Ignored if `customConfig` is set. `defaultOutput.enabled` controls whether the default output is enabled. +## `argocd` + +`argocd` configures tbot to synchronize Teleport-managed Kubernetes clusters +to Argo CD. +Ignored if `tbotConfig` is set. + +### `argocd.enabled` + +| Type | Default | +|------|---------| +| `bool` | `false` | + +`argocd.enabled` controls whether the Argo CD output is enabled. + +### `argocd.clusterSelectors` + +| Type | Default | +|------|---------| +| `list` | `[]` | + +`argocd.clusterSelectors` determines which Kubernetes clusters will +be synchronized to Argo CD. + +For example: +```yaml +clusterSelectors: + - name: my-cluster-1 + - labels: + environment: production +``` + +### `argocd.secretNamespace` + +| Type | Default | +|------|---------| +| `string` | `""` | + +`argocd.secretNamespace` determines to which Kubernetes namespace +cluster secrets will be written (it must be the namespace in which Argo CD +is running). Defaults to the current namespace. + +### `argocd.secretNamePrefix` + +| Type | Default | +|------|---------| +| `string` | `""` | + +`argocd.secretNamePrefix` overrides the string that cluster secret +names will be prefixed with. Defaults to "teleport.argocd-cluster". + +### `argocd.secretLabels` + +| Type | Default | +|------|---------| +| `object` | `{}` | + +`argocd.secretLabels` provides a set of labels that will be applied +to cluster secrets. + +### `argocd.secretAnnotations` + +| Type | Default | +|------|---------| +| `object` | `{}` | + +`argocd.secretAnnotations` provides a set of annotations that will +be applied to cluster secrets. + +### `argocd.project` + +| Type | Default | +|------|---------| +| `string` | `""` | + +`argocd.project` sets the Argo CD project with which the Kubernetes +clusters will be associated. + +### `argocd.namespaces` + +| Type | Default | +|------|---------| +| `list` | `[]` | + +`argocd.namespaces` controls which Kubernetes namespaces the Argo CD +clusters will be allowed to operate on. + +### `argocd.clusterResources` + +| Type | Default | +|------|---------| +| `bool` | `false` | + +`argocd.clusterResources` determines whether the Argo CD cluster is +allowed to operate on cluster-scoped resources (only when `argocd.namespaces` +is non-empty). + ## `persistence` `persistence` controls how the tbot agent stores its data. @@ -102,7 +198,7 @@ use the more specific configuration values throughout this chart. `outputs` contains additional outputs to configure for the tbot agent. These should be in the same format as the `outputs` field in the tbot.yaml. -Ignored if `customConfig` is set. +Ignored if `tbotConfig` is set. ## `services` @@ -112,7 +208,7 @@ Ignored if `customConfig` is set. `services` contains additional services to configure for the tbot agent. These should be in the same format as the `services` field in the tbot.yaml. -Ignored if `customConfig` is set. +Ignored if `tbotConfig` is set. ## `joinMethod` @@ -122,7 +218,7 @@ Ignored if `customConfig` is set. `joinMethod` describes how tbot joins the Teleport cluster. See [the join method reference](../../reference/join-methods.mdx) for a list fo supported values and detailed explanations. -Ignored if `customConfig` is set. +Ignored if `tbotConfig` is set. ## `token` @@ -132,7 +228,7 @@ Ignored if `customConfig` is set. `token` is the name of the token used by tbot to join the Teleport cluster. This value is not sensitive unless the `joinMethod` is set to `"token"`. -Ignored if `customConfig` is set. +Ignored if `tbotConfig` is set. ## `teleportVersionOverride` diff --git a/examples/chart/tbot/.lint/argocd.yaml b/examples/chart/tbot/.lint/argocd.yaml new file mode 100644 index 0000000000000..2538d6d9f5238 --- /dev/null +++ b/examples/chart/tbot/.lint/argocd.yaml @@ -0,0 +1,21 @@ +clusterName: "test.teleport.sh" +teleportProxyAddress: "test.teleport.sh:443" +token: "my-token" +defaultOutput: + enabled: false +argocd: + enabled: true + clusterSelectors: + - name: foo + - labels: + foo: bar + secretNamespace: my-namespace + secretLabels: + baz: qux + secretAnnotations: + chunky: bacon + project: my-argo-project + namespaces: + - dev + - prod + clusterResources: true diff --git a/examples/chart/tbot/templates/_config.tpl b/examples/chart/tbot/templates/_config.tpl index d8c9aff491862..3c0e25b23e6ac 100644 --- a/examples/chart/tbot/templates/_config.tpl +++ b/examples/chart/tbot/templates/_config.tpl @@ -31,7 +31,7 @@ storage: {{- else }} {{- required "'persistence' must be 'secret' or 'disabled'" "" }} {{- end }} -{{- if or (.Values.defaultOutput.enabled) (.Values.outputs) }} +{{- if or (.Values.defaultOutput.enabled) (.Values.argocd.enabled) (.Values.outputs) }} outputs: {{- if .Values.defaultOutput.enabled }} - type: identity @@ -39,6 +39,36 @@ outputs: type: kubernetes_secret name: {{ include "tbot.defaultOutputName" . }} {{- end }} +{{- if .Values.argocd.enabled }} + - type: kubernetes/argo-cd + {{- if .Values.argocd.clusterSelectors }} + selectors: + {{- toYaml .Values.argocd.clusterSelectors | nindent 8 }} + {{- else }} + {{- required "'argocd.clusterSelectors' must be provided if `argocd.enabled' is true" "" }} + {{- end }} + {{- if .Values.argocd.secretNamespace }} + secret_namespace: {{ .Values.argocd.secretNamespace }} + {{- end }} + {{- if .Values.argocd.secretLabels }} + secret_labels: + {{- toYaml .Values.argocd.secretLabels | nindent 8 }} + {{- end }} + {{- if .Values.argocd.secretAnnotations }} + secret_annotations: + {{- toYaml .Values.argocd.secretAnnotations | nindent 8 }} + {{- end }} + {{- if .Values.argocd.project }} + project: {{ .Values.argocd.project }} + {{- end }} + {{- if .Values.argocd.namespaces }} + namespaces: + {{- toYaml .Values.argocd.namespaces | nindent 8 }} + {{- end }} + {{- if .Values.argocd.clusterResources }} + cluster_resources: {{ .Values.argocd.clusterResources }} + {{- end }} +{{- end }} {{- if .Values.outputs }} {{- toYaml .Values.outputs | nindent 2}} {{- end }} diff --git a/examples/chart/tbot/tests/__snapshot__/config_test.yaml.snap b/examples/chart/tbot/tests/__snapshot__/config_test.yaml.snap index 2dbfb81532f58..9b97cab754ab5 100644 --- a/examples/chart/tbot/tests/__snapshot__/config_test.yaml.snap +++ b/examples/chart/tbot/tests/__snapshot__/config_test.yaml.snap @@ -25,6 +25,39 @@ it should match the snapshot (custom): metadata: name: RELEASE-NAME-tbot namespace: NAMESPACE +should match the snapshot (argocd): + 1: | + apiVersion: v1 + data: + tbot.yaml: |- + onboarding: + join_method: kubernetes + token: my-token + outputs: + - cluster_resources: true + namespaces: + - dev + - prod + project: my-argo-project + secret_annotations: + chunky: bacon + secret_labels: + baz: qux + secret_namespace: my-namespace + selectors: + - name: foo + - labels: + foo: bar + type: kubernetes/argo-cd + proxy_server: test.teleport.sh:443 + storage: + name: RELEASE-NAME-tbot + type: kubernetes_secret + version: v2 + kind: ConfigMap + metadata: + name: RELEASE-NAME-tbot + namespace: NAMESPACE should match the snapshot (full): 1: | apiVersion: v1 diff --git a/examples/chart/tbot/tests/config_test.yaml b/examples/chart/tbot/tests/config_test.yaml index 6db9a736bf1bc..313baa79abb1d 100644 --- a/examples/chart/tbot/tests/config_test.yaml +++ b/examples/chart/tbot/tests/config_test.yaml @@ -29,3 +29,8 @@ tests: path: /buzz asserts: - matchSnapshot: {} + - it: should match the snapshot (argocd) + values: + - ../.lint/argocd.yaml + asserts: + - matchSnapshot: {} diff --git a/examples/chart/tbot/values.yaml b/examples/chart/tbot/values.yaml index c0e4b69139b72..5dd816010299b 100644 --- a/examples/chart/tbot/values.yaml +++ b/examples/chart/tbot/values.yaml @@ -18,7 +18,7 @@ fullnameOverride: "" # Connecting to the Proxy Service is the most common and recommended way to connect to Teleport. # This is mandatory to connect to Teleport Enterprise (Cloud) # -# This setting is mutually exclusive with teleportProxyAddress and is ignored if `customConfig` is set. +# This setting is mutually exclusive with teleportProxyAddress and is ignored if `tbotConfig` is set. # # For example: # ```yaml @@ -31,7 +31,7 @@ teleportProxyAddress: "" # Helm release and have direct access to the Auth Service. # Else, you should prefer connecting via the Proxy Service. # -# This setting is mutually exclusive with teleportProxyAddress and is ignored if `customConfig` is set. +# This setting is mutually exclusive with teleportProxyAddress and is ignored if `tbotConfig` is set. # # For example: # ```yaml @@ -40,11 +40,52 @@ teleportProxyAddress: "" teleportAuthAddress: "" # defaultOutput -- controls the default output configured for the tbot agent. -# Ignored if `customConfig` is set. +# Ignored if `tbotConfig` is set. defaultOutput: # defaultOutput.enabled(bool) -- controls whether the default output is enabled. enabled: true +# argocd -- configures tbot to synchronize Teleport-managed Kubernetes clusters +# to Argo CD. +# Ignored if `tbotConfig` is set. +argocd: + # argocd.enabled(bool) -- controls whether the Argo CD output is enabled. + enabled: false + # argocd.clusterSelectors(list) -- determines which Kubernetes clusters will + # be synchronized to Argo CD. + # + # For example: + # ```yaml + # clusterSelectors: + # - name: my-cluster-1 + # - labels: + # environment: production + # ``` + clusterSelectors: [] + # argocd.secretNamespace(string) -- determines to which Kubernetes namespace + # cluster secrets will be written (it must be the namespace in which Argo CD + # is running). Defaults to the current namespace. + secretNamespace: "" + # argocd.secretNamePrefix(string) -- overrides the string that cluster secret + # names will be prefixed with. Defaults to "teleport.argocd-cluster". + secretNamePrefix: "" + # argocd.secretLabels(object) -- provides a set of labels that will be applied + # to cluster secrets. + secretLabels: {} + # argocd.secretAnnotations(object) -- provides a set of annotations that will + # be applied to cluster secrets. + secretAnnotations: {} + # argocd.project(string) -- sets the Argo CD project with which the Kubernetes + # clusters will be associated. + project: "" + # argocd.namespaces(list) -- controls which Kubernetes namespaces the Argo CD + # clusters will be allowed to operate on. + namespaces: [] + # argocd.clusterResources(bool) -- determines whether the Argo CD cluster is + # allowed to operate on cluster-scoped resources (only when `argocd.namespaces` + # is non-empty). + clusterResources: false + # persistence -- controls how the tbot agent stores its data. # # Options: @@ -61,22 +102,22 @@ tbotConfig: {} # outputs(list) -- contains additional outputs to configure for the tbot agent. # These should be in the same format as the `outputs` field in the tbot.yaml. -# Ignored if `customConfig` is set. +# Ignored if `tbotConfig` is set. outputs: [] # services(list) -- contains additional services to configure for the tbot agent. # These should be in the same format as the `services` field in the tbot.yaml. -# Ignored if `customConfig` is set. +# Ignored if `tbotConfig` is set. services: [] # joinMethod(string) -- describes how tbot joins the Teleport cluster. # See [the join method reference](../../reference/join-methods.mdx) for a list fo supported values and detailed explanations. -# Ignored if `customConfig` is set. +# Ignored if `tbotConfig` is set. joinMethod: "kubernetes" # token(string) -- is the name of the token used by tbot to join the Teleport cluster. # This value is not sensitive unless the `joinMethod` is set to `"token"`. -# Ignored if `customConfig` is set. +# Ignored if `tbotConfig` is set. token: "" # teleportVersionOverride(string) -- controls the tbot image version deployed by diff --git a/integrations/terraform-mwi/provider/kubernetes_data_source.go b/integrations/terraform-mwi/provider/kubernetes_data_source.go index 060ea884f25db..0796d532974cb 100644 --- a/integrations/terraform-mwi/provider/kubernetes_data_source.go +++ b/integrations/terraform-mwi/provider/kubernetes_data_source.go @@ -194,7 +194,6 @@ func (d *KubernetesDataSource) Read( }, DisableExecPlugin: true, }, - bot.DefaultCredentialLifetime, ), } if err := botCfg.CheckAndSetDefaults(); err != nil { diff --git a/integrations/terraform-mwi/provider/kubernetes_ephemeral_resource.go b/integrations/terraform-mwi/provider/kubernetes_ephemeral_resource.go index 19a7ce36789b4..e1fff25f3aea7 100644 --- a/integrations/terraform-mwi/provider/kubernetes_ephemeral_resource.go +++ b/integrations/terraform-mwi/provider/kubernetes_ephemeral_resource.go @@ -196,7 +196,6 @@ func (r *KubernetesEphemeralResource) Open( }, DisableExecPlugin: true, }, - bot.DefaultCredentialLifetime, ), } if err := botCfg.CheckAndSetDefaults(); err != nil { diff --git a/lib/tbot/config/config.go b/lib/tbot/config/config.go index 71e5686c991e6..59d930a947fce 100644 --- a/lib/tbot/config/config.go +++ b/lib/tbot/config/config.go @@ -381,6 +381,12 @@ func (o *ServiceConfigs) UnmarshalYAML(node *yaml.Node) error { return trace.Wrap(err) } out = append(out, v) + case k8s.ArgoCDOutputServiceType: + v := &k8s.ArgoCDOutputConfig{} + if err := node.Decode(v); err != nil { + return trace.Wrap(err) + } + out = append(out, v) case ssh.HostOutputServiceType: v := &ssh.HostOutputConfig{} if err := v.UnmarshalConfig(unmarshalContext, node); err != nil { diff --git a/lib/tbot/services/k8s/argocd_output.go b/lib/tbot/services/k8s/argocd_output.go new file mode 100644 index 0000000000000..6ea755994270a --- /dev/null +++ b/lib/tbot/services/k8s/argocd_output.go @@ -0,0 +1,420 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package k8s + +import ( + "bytes" + "cmp" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "log/slog" + "maps" + "strconv" + "strings" + "time" + + "github.com/gravitational/trace" + corev1 "k8s.io/api/core/v1" + kubeerrors "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/gravitational/teleport" + apiclient "github.com/gravitational/teleport/api/client" + "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/api/utils" + "github.com/gravitational/teleport/lib/kube/kubeconfig" + "github.com/gravitational/teleport/lib/tbot/bot" + "github.com/gravitational/teleport/lib/tbot/bot/connection" + "github.com/gravitational/teleport/lib/tbot/client" + "github.com/gravitational/teleport/lib/tbot/identity" + "github.com/gravitational/teleport/lib/tbot/internal" + "github.com/gravitational/teleport/lib/tbot/readyz" +) + +// ArgoCDServiceBuilder builds a new ArgoCDOutput. +func ArgoCDServiceBuilder(cfg *ArgoCDOutputConfig, opts ...ArgoCDServiceOption) bot.ServiceBuilder { + return func(deps bot.ServiceDependencies) (bot.Service, error) { + if err := cfg.CheckAndSetDefaults(); err != nil { + return nil, trace.Wrap(err) + } + + svc := &ArgoCDOutput{ + cfg: cfg, + defaultCredentialLifetime: bot.DefaultCredentialLifetime, + proxyPinger: deps.ProxyPinger, + client: deps.Client, + identityGenerator: deps.IdentityGenerator, + clientBuilder: deps.ClientBuilder, + reloadCh: deps.ReloadCh, + botIdentityReadyCh: deps.BotIdentityReadyCh, + } + + for _, opt := range opts { + opt.applyToArgoOutput(svc) + } + + // If no k8s client is provided, we attempt to create one from the + // environment. + if svc.k8s == nil { + var err error + if svc.k8s, err = newKubernetesClient(); err != nil { + return nil, trace.Wrap(err, "creating Kubernetes client") + } + } + + svc.log = deps.LoggerForService(svc) + svc.statusReporter = deps.StatusRegistry.AddService(svc.String()) + + if svc.alpnUpgradeCache == nil { + svc.alpnUpgradeCache = internal.NewALPNUpgradeCache(svc.log) + } + + return svc, nil + } +} + +// ArgoCDServiceOption is an option that can be provided to customize the service. +type ArgoCDServiceOption interface{ applyToArgoOutput(*ArgoCDOutput) } + +func (opt DefaultCredentialLifetimeOption) applyToArgoOutput(o *ArgoCDOutput) { + o.defaultCredentialLifetime = opt.lifetime +} + +func (opt KubernetesClientOption) applyToArgoOutput(o *ArgoCDOutput) { o.k8s = opt.client } +func (opt InsecureOption) applyToArgoOutput(o *ArgoCDOutput) { o.insecure = opt.insecure } +func (opt ALPNUpgradeCacheOption) applyToArgoOutput(o *ArgoCDOutput) { o.alpnUpgradeCache = opt.cache } + +// ArgoCDOutput registers Kubernetes clusters with ArgoCD by writing their +// details and the client certificate, etc. as Kubernetes secrets. +type ArgoCDOutput struct { + cfg *ArgoCDOutputConfig + defaultCredentialLifetime bot.CredentialLifetime + k8s kubernetes.Interface + + log *slog.Logger + statusReporter readyz.Reporter + reloadCh <-chan struct{} + + identityGenerator *identity.Generator + clientBuilder *client.Builder + proxyPinger connection.ProxyPinger + + client *apiclient.Client + botIdentityReadyCh <-chan struct{} + alpnUpgradeCache *internal.ALPNUpgradeCache + insecure bool +} + +// String returns the human-readable representation of the service that will be +// used in logs and the `/readyz` endpoints. +func (s *ArgoCDOutput) String() string { + if s.cfg.Name != "" { + return s.cfg.Name + } + var selectors []string + for _, s := range s.cfg.Selectors { + selectors = append(selectors, s.String()) + } + return fmt.Sprintf("kubernetes-argo-cd-output (%s)", strings.Join(selectors, ", ")) +} + +// Run periodically refreshes the cluster credentials. +func (s *ArgoCDOutput) Run(ctx context.Context) error { + err := internal.RunOnInterval(ctx, internal.RunOnIntervalConfig{ + Service: s.String(), + Name: "output-renewal", + F: func(ctx context.Context) error { + err := s.generate(ctx) + + // If the Teleport proxy is behind a TLS-terminating load balancer, + // generate will return a NotImplemented error. We return nil here + // because we do not want to RunOnInterval to retry. + // + // While we could have generate return nil in this case instead, we + // do want to surface it as a hard error in one-shot mode. + if trace.IsNotImplemented(err) { + s.log.ErrorContext(ctx, "Failed to generate Argo CD cluster credentials", "error", err) + return nil + } + + return err + }, + Interval: cmp.Or(s.cfg.CredentialLifetime, s.defaultCredentialLifetime).RenewalInterval, + RetryLimit: internal.RenewalRetryLimit, + Log: s.log, + ReloadCh: s.reloadCh, + IdentityReadyCh: s.botIdentityReadyCh, + StatusReporter: s.statusReporter, + }) + return trace.Wrap(err) +} + +// OneShot generates cluster credentials once and exits. +func (s *ArgoCDOutput) OneShot(ctx context.Context) error { + return s.generate(ctx) +} + +func (s *ArgoCDOutput) generate(ctx context.Context) error { + ctx, span := tracer.Start( + ctx, + "ArgoCDOutput/generate", + ) + defer span.End() + + clusters, err := s.discoverClusters(ctx) + if err != nil { + return trace.Wrap(err, "discovering clusters") + } + + var errors []error + for _, cluster := range clusters { + secret, err := s.renderSecret(cluster) + if err != nil { + errors = append(errors, trace.Wrap(err, "rendering cluster secret")) + continue + } + if err := s.writeSecret(ctx, secret); err != nil { + errors = append(errors, trace.Wrap(err, "writing cluster secret")) + continue + } + } + return trace.NewAggregate(errors...) +} + +func (s *ArgoCDOutput) discoverClusters(ctx context.Context) ([]*argoClusterCredentials, error) { + effectiveLifetime := cmp.Or(s.cfg.CredentialLifetime, s.defaultCredentialLifetime) + id, err := s.identityGenerator.GenerateFacade(ctx, + identity.WithLifetime(effectiveLifetime.TTL, effectiveLifetime.RenewalInterval), + identity.WithLogger(s.log), + ) + if err != nil { + return nil, trace.Wrap(err, "generating identity") + } + + impersonatedClient, err := s.clientBuilder.Build(ctx, id) + if err != nil { + return nil, trace.Wrap(err) + } + defer impersonatedClient.Close() + + clusters, err := fetchAllMatchingKubeClusters(ctx, impersonatedClient, s.cfg.Selectors) + if err != nil { + return nil, trace.Wrap(err) + } + + var clusterNames []string + for _, c := range clusters { + clusterNames = append(clusterNames, c.GetName()) + } + clusterNames = utils.Deduplicate(clusterNames) + + s.log.InfoContext( + ctx, + "Generated identity for Kubernetes access", + "matched_cluster_count", len(clusterNames), + "identity", id.Get(), + ) + proxyPong, err := s.proxyPinger.Ping(ctx) + if err != nil { + return nil, trace.Wrap(err, "pinging proxy to determine connection pathway") + } + + proxyAddr, kubeSNI, err := selectKubeConnectionMethod(proxyPong) + if err != nil { + return nil, trace.Wrap(err) + } + + if proxyPong.Proxy.TLSRoutingEnabled { + required, err := s.alpnUpgradeCache.IsUpgradeRequired(ctx, proxyAddr, s.insecure) + if err != nil { + return nil, trace.Wrap(err) + } + if required { + return nil, trace.NotImplemented( + "Teleport proxy %q appears to be behind a TLS-terminating load balancer that does not support ALPN. "+ + "The %q service does not support this configuration, but you may be able to work around it by "+ + "running a local proxy with `tbot proxy kube` and configuring the cluster in Argo CD manually.", + proxyAddr, + ArgoCDOutputServiceType, + ) + } + } + + hostCAs, err := s.client.GetCertAuthorities(ctx, types.HostCA, false) + if err != nil { + return nil, trace.Wrap(err) + } + + keyRing, err := internal.NewClientKeyRing(id.Get(), hostCAs) + if err != nil { + return nil, trace.Wrap(err) + } + + clusterCAs, err := keyRing.RootClusterCAs() + if err != nil { + return nil, trace.Wrap(err) + } + caBytes := bytes.Join(clusterCAs, []byte("\n")) + if len(caBytes) == 0 { + return nil, trace.BadParameter("TLS trusted CAs missing in provided credentials") + } + + credentials := make([]*argoClusterCredentials, len(clusterNames)) + for idx, clusterName := range clusterNames { + credentials[idx] = &argoClusterCredentials{ + teleportClusterName: proxyPong.ClusterName, + kubeClusterName: clusterName, + addr: fmt.Sprintf( + "%s/v1/teleport/%s/%s", + proxyAddr, + encodePathComponent(proxyPong.ClusterName), + encodePathComponent(clusterName), + ), + tlsClientConfig: argoTLSClientConfig{ + CAData: caBytes, + CertData: keyRing.TLSCert, + KeyData: keyRing.TLSPrivateKey.PrivateKeyPEM(), + ServerName: kubeSNI, + }, + botName: id.Get().TLSIdentity.BotName, + } + } + return credentials, nil +} + +type argoClusterCredentials struct { + teleportClusterName string + kubeClusterName string + addr string + tlsClientConfig argoTLSClientConfig + botName string +} + +type argoTLSClientConfig struct { + CAData []byte `json:"caData"` + CertData []byte `json:"certData"` + KeyData []byte `json:"keyData"` + ServerName string `json:"serverName,omitempty"` +} + +func (s *ArgoCDOutput) renderSecret(cluster *argoClusterCredentials) (*corev1.Secret, error) { + labels := map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + } + for k, v := range s.cfg.SecretLabels { + // Do not overwrite any of "our" labels. + if _, ok := labels[k]; !ok { + labels[k] = v + } + } + + annotations := map[string]string{ + "teleport.dev/bot-name": cluster.botName, + "teleport.dev/kubernetes-cluster-name": cluster.kubeClusterName, + "teleport.dev/updated": time.Now().Format(time.RFC3339), + "teleport.dev/tbot-version": teleport.Version, + "teleport.dev/teleport-cluster-name": cluster.teleportClusterName, + } + for k, v := range s.cfg.SecretAnnotations { + // Do not overwrite any of "our" annotations. + if _, ok := annotations[k]; !ok { + annotations[k] = v + } + } + + configJSON, err := json.Marshal(struct { + TLSClientConfig argoTLSClientConfig `json:"tlsClientConfig"` + }{cluster.tlsClientConfig}) + if err != nil { + return nil, trace.Wrap(err, "marshaling cluster credentials") + } + + data := map[string][]byte{ + "name": []byte(kubeconfig.ContextName(cluster.teleportClusterName, cluster.kubeClusterName)), + "server": []byte(cluster.addr), + "config": configJSON, + } + + if s.cfg.Project != "" { + data["project"] = []byte(s.cfg.Project) + } + + if len(s.cfg.Namespaces) != 0 { + data["namespaces"] = []byte(strings.Join(s.cfg.Namespaces, ",")) + data["clusterResources"] = []byte(strconv.FormatBool(s.cfg.ClusterResources)) + } + + return &corev1.Secret{ + Type: corev1.SecretTypeOpaque, + ObjectMeta: v1.ObjectMeta{ + Name: s.secretName(cluster), + Namespace: s.cfg.SecretNamespace, + Labels: labels, + Annotations: annotations, + }, + Data: data, + }, nil +} + +func (s *ArgoCDOutput) secretName(cluster *argoClusterCredentials) string { + h := sha256.New() + _, _ = h.Write([]byte(cluster.teleportClusterName)) + _, _ = h.Write([]byte(cluster.kubeClusterName)) + return fmt.Sprintf("%s.%x", s.cfg.SecretNamePrefix, h.Sum(nil)[:8]) +} + +func (s *ArgoCDOutput) writeSecret(ctx context.Context, secret *corev1.Secret) error { + fullName := fmt.Sprintf("%s/%s", secret.GetNamespace(), secret.GetName()) + client := s.k8s.CoreV1().Secrets(secret.GetNamespace()) + + existing, err := client.Get(ctx, secret.GetName(), v1.GetOptions{}) + if kubeerrors.IsNotFound(err) { + // Secret is new, create it. + if _, err := client.Create(ctx, secret, v1.CreateOptions{ + FieldManager: "tbot", + }); err != nil { + return trace.Wrap(err, "creating secret: %s", fullName) + } + return nil + } else if err != nil { + // Failed to read the secret. + return trace.Wrap(err, "reading secret: %s", fullName) + } + + // Secret exists, update it. + secret.SetResourceVersion(secret.ResourceVersion) + + annotations := make(map[string]string) + maps.Copy(annotations, existing.Annotations) + maps.Copy(annotations, secret.Annotations) + secret.SetAnnotations(annotations) + + // We use Update rather than Apply or Patch here because Argo CD will also + // write to the secret (e.g. to add its own annotations or edit the config) + // so it's likely we'd need to "force" apply our changes anyway. + if _, err := client.Update(ctx, secret, v1.UpdateOptions{ + FieldManager: "tbot", + }); err != nil { + return trace.Wrap(err, "updating secret: %s", fullName) + } + return nil +} diff --git a/lib/tbot/services/k8s/argocd_output_config.go b/lib/tbot/services/k8s/argocd_output_config.go new file mode 100644 index 0000000000000..1ee9e8cf18f8d --- /dev/null +++ b/lib/tbot/services/k8s/argocd_output_config.go @@ -0,0 +1,143 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package k8s + +import ( + "cmp" + "os" + + "github.com/gravitational/trace" + "k8s.io/apimachinery/pkg/api/validation" + + "github.com/gravitational/teleport/lib/tbot/bot" + "github.com/gravitational/teleport/lib/tbot/internal/encoding" +) + +const ArgoCDOutputServiceType = "kubernetes/argo-cd" + +// ArgoCDOutputConfig contains configuration for the service that registers +// Kubernetes cluster credentials in Argo CD. +// +// See: https://argo-cd.readthedocs.io/en/stable/operator-manual/declarative-setup/#clusters +type ArgoCDOutputConfig struct { + // Name of the service for logs and the /readyz endpoint. + Name string `yaml:"name,omitempty"` + + // CredentialLifetime contains configuration for how long credentials will + // last and the frequency at which they'll be renewed. + CredentialLifetime bot.CredentialLifetime `yaml:",inline"` + + // Selectors is a list of selectors used to determine which Kubernetes + // clusters will be registered in Argo CD. + Selectors []*KubernetesSelector `yaml:"selectors,omitempty"` + + // SecretNamespace is the namespace to which cluster secrets will be written. + // By default, it will use the `POD_NAMESPACE` environment variable, or if + // that is empty: "default". + SecretNamespace string `yaml:"secret_namespace,omitempty"` + + // SecretNamePrefix is the prefix that will be applied to Kubernetes secret + // names. The rest of the name will be derived from the name of the target + // cluster. Defaults to: "teleport.argocd-cluster". + SecretNamePrefix string `yaml:"secret_name_prefix,omitempty"` + + // SecretLabels is a set of labels that will be applied to the created + // Kubernetes secrets (in addition to the labels added for Argo's benefit). + SecretLabels map[string]string `yaml:"secret_labels,omitempty"` + + // SecretLabels is a set of annotations that will be applied to the created + // Kubernetes secrets (in addition to tbot's own annotations). + SecretAnnotations map[string]string `yaml:"secret_annotations,omitempty"` + + // Project is the Argo CD project with which the Kubernetes cluster + // credentials will be associated. + Project string `yaml:"project,omitempty"` + + // Namespaces are the Kubernetes namespaces the created Argo CD cluster + // credentials will be allowed to operate on. + Namespaces []string `yaml:"namespaces,omitempty"` + + // ClusterResources determines whether the created Argo CD cluster + // credentials will be allowed to operate on cluster-scoped resources (only + // when Namespaces is non-empty). + ClusterResources bool `yaml:"cluster_resources,omitempty"` +} + +// GetName returns the user-given name of the service, used for validation purposes. +func (o *ArgoCDOutputConfig) GetName() string { + return o.Name +} + +// CheckAndSetDefaults validates the service configuration and sets any default +// values. +func (o *ArgoCDOutputConfig) CheckAndSetDefaults() error { + if len(o.Selectors) == 0 { + return trace.BadParameter("at least one selector is required") + } + for idx, selector := range o.Selectors { + if err := selector.CheckAndSetDefaults(); err != nil { + return trace.Wrap(err, "validating selectors[%d]", idx) + } + } + + o.SecretNamespace = cmp.Or( + o.SecretNamespace, + os.Getenv(kubernetesNamespaceEnv), + "default", + ) + + if o.SecretNamePrefix == "" { + o.SecretNamePrefix = "teleport.argocd-cluster" + } else { + if len(validation.NameIsDNSSubdomain(o.SecretNamePrefix, true)) != 0 { + return trace.BadParameter("secret_name_prefix may only include lowercase letters, numbers, '-' and '.' characters") + } + } + + for idx, ns := range o.Namespaces { + if ns == "" { + return trace.BadParameter("namespaces[%d] cannot be blank", idx) + } + if len(validation.ValidateNamespaceName(ns, false)) != 0 { + return trace.BadParameter("namespaces[%d] is not a valid namespace name", idx) + } + } + + if o.ClusterResources && len(o.Namespaces) == 0 { + return trace.BadParameter("cluster_resources is only applicable if namespaces is also set") + } + + return nil +} + +// Type returns the service type string. +func (s *ArgoCDOutputConfig) Type() string { + return ArgoCDOutputServiceType +} + +// MarshalYAML marshals the configuration to YAML. +func (s *ArgoCDOutputConfig) MarshalYAML() (any, error) { + type raw ArgoCDOutputConfig + return encoding.WithTypeHeader((*raw)(s), ArgoCDOutputServiceType) +} + +// GetCredentialLifetime returns the service's credential lifetime. +func (o *ArgoCDOutputConfig) GetCredentialLifetime() bot.CredentialLifetime { + return o.CredentialLifetime +} diff --git a/lib/tbot/services/k8s/argocd_output_config_test.go b/lib/tbot/services/k8s/argocd_output_config_test.go new file mode 100644 index 0000000000000..63be98bc3ede5 --- /dev/null +++ b/lib/tbot/services/k8s/argocd_output_config_test.go @@ -0,0 +1,204 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package k8s + +import ( + "testing" + "time" + + "github.com/gravitational/teleport/lib/tbot/bot" +) + +func TestArgoCDOutput_YAML(t *testing.T) { + tests := []testYAMLCase[ArgoCDOutputConfig]{ + { + name: "full", + in: ArgoCDOutputConfig{ + Name: "my-argo-service", + Selectors: []*KubernetesSelector{ + { + Name: "foo", + Labels: map[string]string{}, + }, + { + Labels: map[string]string{ + "foo": "bar", + }, + }, + }, + CredentialLifetime: bot.CredentialLifetime{ + TTL: 1 * time.Minute, + RenewalInterval: 30 * time.Second, + }, + SecretNamespace: "argocd", + SecretNamePrefix: "my-argo-cluster-", + SecretLabels: map[string]string{ + "my-label": "value", + }, + SecretAnnotations: map[string]string{ + "my-annotation": "value", + }, + Project: "super-secret-project", + Namespaces: []string{"prod", "dev"}, + ClusterResources: true, + }, + }, + { + name: "minimal", + in: ArgoCDOutputConfig{ + Selectors: []*KubernetesSelector{ + { + Name: "foo", + Labels: map[string]string{}, + }, + }, + }, + }, + } + testYAML(t, tests) +} + +func TestArgoCDConfig_CheckAndSetDefaults(t *testing.T) { + t.Setenv("POD_NAMESPACE", "my-pod-namespace") + + tests := []testCheckAndSetDefaultsCase[*ArgoCDOutputConfig]{ + { + name: "valid_with_name_selector", + in: func() *ArgoCDOutputConfig { + return &ArgoCDOutputConfig{ + Selectors: []*KubernetesSelector{ + {Name: "foo", Labels: make(map[string]string)}, + }, + SecretNamespace: "argocd", + SecretNamePrefix: "argo-cluster", + } + }, + }, + { + name: "valid_with_labels", + in: func() *ArgoCDOutputConfig { + return &ArgoCDOutputConfig{ + Selectors: []*KubernetesSelector{ + {Labels: map[string]string{"foo": "bar"}}, + }, + SecretNamespace: "argocd", + SecretNamePrefix: "argo-cluster", + } + }, + }, + { + name: "no_selectors", + in: func() *ArgoCDOutputConfig { + return &ArgoCDOutputConfig{ + Selectors: []*KubernetesSelector{}, + SecretNamespace: "argocd", + SecretNamePrefix: "argo-cluster", + } + }, + wantErr: "at least one selector is required", + }, + { + name: "invalid_selectors", + in: func() *ArgoCDOutputConfig { + return &ArgoCDOutputConfig{ + Selectors: []*KubernetesSelector{ + {}, + }, + SecretNamespace: "argocd", + SecretNamePrefix: "argo-cluster", + } + }, + wantErr: "one of 'name' and 'labels' must be specified", + }, + { + name: "invalid_secret_name_prefix", + in: func() *ArgoCDOutputConfig { + return &ArgoCDOutputConfig{ + Selectors: []*KubernetesSelector{ + {Labels: map[string]string{"foo": "bar"}}, + }, + SecretNamespace: "argocd", + SecretNamePrefix: "NOT VALID", + } + }, + wantErr: "secret_name_prefix may only include lowercase letters, numbers, '-' and '.' characters", + }, + { + name: "empty_namespace", + in: func() *ArgoCDOutputConfig { + return &ArgoCDOutputConfig{ + Selectors: []*KubernetesSelector{ + {Labels: map[string]string{"foo": "bar"}}, + }, + SecretNamespace: "argocd", + SecretNamePrefix: "argo-cluster", + Namespaces: []string{""}, + } + }, + wantErr: "namespaces[0] cannot be blank", + }, + { + name: "invalid_namespaces", + in: func() *ArgoCDOutputConfig { + return &ArgoCDOutputConfig{ + Selectors: []*KubernetesSelector{ + {Labels: map[string]string{"foo": "bar"}}, + }, + SecretNamespace: "argocd", + SecretNamePrefix: "argo-cluster", + Namespaces: []string{"foo,"}, + } + }, + wantErr: "namespaces[0] is not a valid namespace name", + }, + { + name: "cluster_resources_but_no_namespaces", + in: func() *ArgoCDOutputConfig { + return &ArgoCDOutputConfig{ + Selectors: []*KubernetesSelector{ + {Labels: map[string]string{"foo": "bar"}}, + }, + SecretNamespace: "argocd", + SecretNamePrefix: "argo-cluster", + Namespaces: []string{}, + ClusterResources: true, + } + }, + wantErr: "cluster_resources is only applicable if namespaces is also set", + }, + { + name: "defaults", + in: func() *ArgoCDOutputConfig { + return &ArgoCDOutputConfig{ + Selectors: []*KubernetesSelector{ + {Labels: map[string]string{"foo": "bar"}}, + }, + } + }, + want: &ArgoCDOutputConfig{ + Selectors: []*KubernetesSelector{ + {Labels: map[string]string{"foo": "bar"}}, + }, + SecretNamespace: "my-pod-namespace", + SecretNamePrefix: "teleport.argocd-cluster", + }, + }, + } + testCheckAndSetDefaults(t, tests) +} diff --git a/lib/tbot/services/k8s/argocd_output_test.go b/lib/tbot/services/k8s/argocd_output_test.go new file mode 100644 index 0000000000000..aed414a47fe60 --- /dev/null +++ b/lib/tbot/services/k8s/argocd_output_test.go @@ -0,0 +1,240 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package k8s + +import ( + "encoding/json" + "net/url" + "strings" + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/lib/service/servicecfg" + "github.com/gravitational/teleport/lib/tbot/bot" + "github.com/gravitational/teleport/lib/tbot/bot/connection" + "github.com/gravitational/teleport/lib/tbot/bot/destination" + "github.com/gravitational/teleport/lib/utils/log/logtest" + "github.com/gravitational/teleport/tool/teleport/testenv" +) + +func TestArgoCDOutput_String(t *testing.T) { + t.Parallel() + + svc := &ArgoCDOutput{ + cfg: &ArgoCDOutputConfig{ + Selectors: []*KubernetesSelector{ + {Name: "cluster-1"}, + {Name: "cluster-2"}, + { + Labels: map[string]string{ + "env": "prod", + "region": "eu", + }, + }, + }, + }, + } + require.Equal(t, "kubernetes-argo-cd-output (name=cluster-1, name=cluster-2, labels={env=prod, region=eu})", svc.String()) +} + +func TestArgoCDOutput_EndToEnd(t *testing.T) { + t.Parallel() + + ctx := t.Context() + log := logtest.NewLogger() + + // Spin up a test server. + process, err := testenv.NewTeleportProcess( + t.TempDir(), + defaultTestServerOpts(log), + testenv.WithProxyKube(), + testenv.WithAuthConfig(func(auth *servicecfg.AuthConfig) { + auth.NetworkingConfig.SetProxyListenerMode(types.ProxyListenerMode_Multiplex) + }), + ) + require.NoError(t, err) + + rootClient, err := testenv.NewDefaultAuthClient(process) + require.NoError(t, err) + + registerCluster := func(t *testing.T, name string) { + t.Helper() + + kubeCluster, err := types.NewKubernetesClusterV3( + types.Metadata{ + Name: name, + Labels: map[string]string{"department": "engineering"}, + }, + types.KubernetesClusterSpecV3{}, + ) + require.NoError(t, err) + + kubeServer, err := types.NewKubernetesServerV3FromCluster(kubeCluster, "host", "1234") + require.NoError(t, err) + + _, err = rootClient.UpsertKubernetesServer(ctx, kubeServer) + require.NoError(t, err) + } + + // Register a kubernetes cluster. + registerCluster(t, "kube-cluster-1") + + // Create a role giving the bot access to the kubernetes cluster. + role, err := types.NewRole("bot-role", types.RoleSpecV6{ + Allow: types.RoleConditions{ + KubernetesLabels: types.Labels{"*": []string{"*"}}, + KubeGroups: []string{"system:masters"}, + KubeUsers: []string{"kubernetes-user"}, + }, + }) + require.NoError(t, err) + + _, err = rootClient.UpsertRole(ctx, role) + require.NoError(t, err) + + // Create the service. + k8s := fake.NewClientset() + service := ArgoCDServiceBuilder( + &ArgoCDOutputConfig{ + SecretNamePrefix: "my-cluster", + SecretNamespace: "argocd", + SecretLabels: map[string]string{ + "team": "billing", + }, + SecretAnnotations: map[string]string{ + "managed-by": "ninjas", + }, + Selectors: []*KubernetesSelector{ + {Labels: map[string]string{"department": "engineering"}}, + }, + Project: "my-argo-project", + Namespaces: []string{"prod", "dev"}, + ClusterResources: true, + }, + WithKubernetesClient(k8s), + ) + + proxyAddr, err := process.ProxyWebAddr() + require.NoError(t, err) + + onboarding, _ := makeBot(t, rootClient, "argo-bot", role.GetName()) + + botConfig := bot.Config{ + InternalStorage: destination.NewMemory(), + Connection: connection.Config{ + Address: proxyAddr.Addr, + AddressKind: connection.AddressKindProxy, + Insecure: true, + }, + Logger: log, + Onboarding: *onboarding, + Services: []bot.ServiceBuilder{service}, + } + + // Run the bot in one-shot mode. + b, err := bot.New(botConfig) + require.NoError(t, err) + require.NoError(t, b.OneShot(ctx)) + + // Expect the cluster credentials to have been written to a secret. + list, err := k8s.CoreV1(). + Secrets("argocd"). + List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, list.Items, 1) + + secret := list.Items[0] + + // Check we apply the secret name prefix. + require.True(t, strings.HasPrefix(secret.Name, "my-cluster")) + + // Check we set the correct labels on the secret. + require.Equal(t, + map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "team": "billing", + }, + secret.Labels, + ) + + // Check the name and other top-level fields. + expectedData := map[string]string{ + "name": "root-kube-cluster-1", + "project": "my-argo-project", + "namespaces": "prod,dev", + "clusterResources": "true", + } + for k, v := range expectedData { + require.Equal(t, v, string(secret.Data[k])) + } + + // Check the server addr. + server := string(secret.Data["server"]) + serverURL, err := url.Parse(server) + require.NoError(t, err) + + _, port, _ := strings.Cut(proxyAddr.Addr, ":") + require.Equal(t, port, serverURL.Port()) + require.Equal(t, "/v1/teleport/cm9vdA/a3ViZS1jbHVzdGVyLTE", serverURL.Path) + + // Check the config. + var config map[string]any + require.NoError(t, json.Unmarshal(secret.Data["config"], &config)) + + tlsConfig := config["tlsClientConfig"].(map[string]any) + require.Equal(t, + "kube-teleport-proxy-alpn.teleport.cluster.local", + tlsConfig["serverName"], + ) + + // Check the CA Certificates, Client Certificate, and Private Key were set. + require.NotEmpty(t, tlsConfig["caData"]) + require.NotEmpty(t, tlsConfig["certData"]) + require.NotEmpty(t, tlsConfig["keyData"]) + + expectedAnnotations := map[string]string{ + "teleport.dev/bot-name": "argo-bot", + "teleport.dev/kubernetes-cluster-name": "kube-cluster-1", + "teleport.dev/tbot-version": teleport.Version, + "teleport.dev/teleport-cluster-name": "root", + "managed-by": "ninjas", + } + for k, v := range expectedAnnotations { + require.Equal(t, v, secret.Annotations[k]) + } + + // Add another cluster and run the bot again. + registerCluster(t, "kube-cluster-2") + + b, err = bot.New(botConfig) + require.NoError(t, err) + require.NoError(t, b.OneShot(ctx)) + + // Expect another secret to have been written. + list, err = k8s.CoreV1(). + Secrets("argocd"). + List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, list.Items, 2) +} diff --git a/lib/tbot/services/k8s/helpers_test.go b/lib/tbot/services/k8s/helpers_test.go index f4807e901d1ce..178dcc70af0d8 100644 --- a/lib/tbot/services/k8s/helpers_test.go +++ b/lib/tbot/services/k8s/helpers_test.go @@ -20,14 +20,26 @@ package k8s import ( "bytes" + "log/slog" + "net" + "strconv" "testing" + "time" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" + headerv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1" + machineidv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/machineid/v1" "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/lib/auth/authclient" + "github.com/gravitational/teleport/lib/defaults" + "github.com/gravitational/teleport/lib/service/servicecfg" + "github.com/gravitational/teleport/lib/tbot/bot/onboarding" "github.com/gravitational/teleport/lib/tbot/internal" + "github.com/gravitational/teleport/lib/utils" "github.com/gravitational/teleport/lib/utils/testutils/golden" + "github.com/gravitational/teleport/tool/teleport/testenv" ) type testYAMLCase[T any] struct { @@ -112,3 +124,58 @@ func newMockDiscoveredKubeCluster(t *testing.T, name, discoveredName string) *ty require.NoError(t, err) return kubeCluster } + +// makeBot creates a server-side bot and returns joining parameters. +func makeBot(t *testing.T, client *authclient.Client, name string, roles ...string) (*onboarding.Config, *machineidv1pb.Bot) { + ctx := t.Context() + t.Helper() + + b, err := client.BotServiceClient().CreateBot(ctx, &machineidv1pb.CreateBotRequest{ + Bot: &machineidv1pb.Bot{ + Kind: types.KindBot, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: name, + }, + Spec: &machineidv1pb.BotSpec{ + Roles: roles, + }, + }, + }) + require.NoError(t, err) + + tokenName, err := utils.CryptoRandomHex(defaults.TokenLenBytes) + require.NoError(t, err) + tok, err := types.NewProvisionTokenFromSpec( + tokenName, + time.Now().Add(10*time.Minute), + types.ProvisionTokenSpecV2{ + Roles: []types.SystemRole{types.RoleBot}, + BotName: b.Metadata.Name, + }) + require.NoError(t, err) + err = client.CreateToken(ctx, tok) + require.NoError(t, err) + + return &onboarding.Config{ + TokenValue: tok.GetName(), + JoinMethod: types.JoinMethodToken, + }, b +} + +func defaultTestServerOpts(log *slog.Logger) testenv.TestServerOptFunc { + return func(o *testenv.TestServersOpts) error { + testenv.WithClusterName("root")(o) + testenv.WithConfig(func(cfg *servicecfg.Config) { + cfg.Logger = log + cfg.Proxy.PublicAddrs = []utils.NetAddr{ + {AddrNetwork: "tcp", Addr: net.JoinHostPort("localhost", strconv.Itoa(cfg.Proxy.WebAddr.Port(0)))}, + } + cfg.Proxy.TunnelPublicAddrs = []utils.NetAddr{ + cfg.Proxy.ReverseTunnelListenAddr, + } + })(o) + + return nil + } +} diff --git a/lib/tbot/services/k8s/k8s.go b/lib/tbot/services/k8s/k8s.go index de3e73c2ab768..f5d894e69f49f 100644 --- a/lib/tbot/services/k8s/k8s.go +++ b/lib/tbot/services/k8s/k8s.go @@ -20,8 +20,11 @@ package k8s import ( "go.opentelemetry.io/otel" + "k8s.io/client-go/kubernetes" "github.com/gravitational/teleport" + "github.com/gravitational/teleport/lib/tbot/bot" + "github.com/gravitational/teleport/lib/tbot/internal" logutils "github.com/gravitational/teleport/lib/utils/log" ) @@ -29,3 +32,36 @@ var ( tracer = otel.Tracer("github.com/gravitational/teleport/lib/tbot/services/k8s") log = logutils.NewPackageLogger(teleport.ComponentKey, teleport.ComponentTBot) ) + +// WithDefaultCredentialLifetime sets the service's default credential lifetime. +func WithDefaultCredentialLifetime(lifetime bot.CredentialLifetime) DefaultCredentialLifetimeOption { + return DefaultCredentialLifetimeOption{lifetime} +} + +// DefaultCredentialLifetimeOption is returned from WithDefaultCredentialLifetime. +type DefaultCredentialLifetimeOption struct{ lifetime bot.CredentialLifetime } + +// WithKubernetesClient sets the service's Kubernetes client. It's used in tests. +func WithKubernetesClient(k8s kubernetes.Interface) KubernetesClientOption { + return KubernetesClientOption{k8s} +} + +// KubernetesClientOption is returned from WithKubernetesClient. +type KubernetesClientOption struct{ client kubernetes.Interface } + +// WithInsecure controls whether the service will verify proxy certificates. +func WithInsecure(insecure bool) InsecureOption { + return InsecureOption{insecure} +} + +// InsecureOption is returned from WithInsecure. +type InsecureOption struct{ insecure bool } + +// WithALPNUpgradeCache sets the service's ALPN upgrade cache so that it can be +// shared with other services. +func WithALPNUpgradeCache(cache *internal.ALPNUpgradeCache) ALPNUpgradeCacheOption { + return ALPNUpgradeCacheOption{cache} +} + +// ALPNUpgradeCacheOption is returned from WithALPNUpgradeCache. +type ALPNUpgradeCacheOption struct{ cache *internal.ALPNUpgradeCache } diff --git a/lib/tbot/services/k8s/output_v1.go b/lib/tbot/services/k8s/output_v1.go index 97c0ef0798173..e847cbf4402e7 100644 --- a/lib/tbot/services/k8s/output_v1.go +++ b/lib/tbot/services/k8s/output_v1.go @@ -51,7 +51,7 @@ import ( const defaultKubeconfigPath = "kubeconfig.yaml" -func OutputV1ServiceBuilder(cfg *OutputV1Config, defaultCredentialLifetime bot.CredentialLifetime) bot.ServiceBuilder { +func OutputV1ServiceBuilder(cfg *OutputV1Config, opts ...OutputV1Option) bot.ServiceBuilder { return func(deps bot.ServiceDependencies) (bot.Service, error) { if err := cfg.CheckAndSetDefaults(); err != nil { return nil, trace.Wrap(err) @@ -59,7 +59,7 @@ func OutputV1ServiceBuilder(cfg *OutputV1Config, defaultCredentialLifetime bot.C svc := &OutputV1Service{ botAuthClient: deps.Client, botIdentityReadyCh: deps.BotIdentityReadyCh, - defaultCredentialLifetime: defaultCredentialLifetime, + defaultCredentialLifetime: bot.DefaultCredentialLifetime, cfg: cfg, proxyPinger: deps.ProxyPinger, reloadCh: deps.ReloadCh, @@ -67,12 +67,22 @@ func OutputV1ServiceBuilder(cfg *OutputV1Config, defaultCredentialLifetime bot.C identityGenerator: deps.IdentityGenerator, clientBuilder: deps.ClientBuilder, } + for _, opt := range opts { + opt.applyToV1Output(svc) + } svc.log = deps.LoggerForService(svc) svc.statusReporter = deps.StatusRegistry.AddService(svc.String()) return svc, nil } } +// OutputV1Option is an option that can be provided to customize the service. +type OutputV1Option interface{ applyToV1Output(*OutputV1Service) } + +func (opt DefaultCredentialLifetimeOption) applyToV1Output(o *OutputV1Service) { + o.defaultCredentialLifetime = opt.lifetime +} + // OutputV1Service produces credentials which can be used to connect to // a Kubernetes Cluster through teleport. type OutputV1Service struct { diff --git a/lib/tbot/services/k8s/output_v2.go b/lib/tbot/services/k8s/output_v2.go index e26813c40cec7..2be83feaf2313 100644 --- a/lib/tbot/services/k8s/output_v2.go +++ b/lib/tbot/services/k8s/output_v2.go @@ -51,7 +51,7 @@ import ( logutils "github.com/gravitational/teleport/lib/utils/log" ) -func OutputV2ServiceBuilder(cfg *OutputV2Config, defaultCredentialLifetime bot.CredentialLifetime) bot.ServiceBuilder { +func OutputV2ServiceBuilder(cfg *OutputV2Config, opts ...OutputV2Option) bot.ServiceBuilder { return func(deps bot.ServiceDependencies) (bot.Service, error) { if err := cfg.CheckAndSetDefaults(); err != nil { return nil, trace.Wrap(err) @@ -59,7 +59,7 @@ func OutputV2ServiceBuilder(cfg *OutputV2Config, defaultCredentialLifetime bot.C svc := &OutputV2Service{ botAuthClient: deps.Client, botIdentityReadyCh: deps.BotIdentityReadyCh, - defaultCredentialLifetime: defaultCredentialLifetime, + defaultCredentialLifetime: bot.DefaultCredentialLifetime, cfg: cfg, proxyPinger: deps.ProxyPinger, reloadCh: deps.ReloadCh, @@ -67,12 +67,22 @@ func OutputV2ServiceBuilder(cfg *OutputV2Config, defaultCredentialLifetime bot.C identityGenerator: deps.IdentityGenerator, clientBuilder: deps.ClientBuilder, } + for _, opt := range opts { + opt.applyToV2Output(svc) + } svc.log = deps.LoggerForService(svc) svc.statusReporter = deps.StatusRegistry.AddService(svc.String()) return svc, nil } } +// OutputV1Option is an option that can be provided to customize the service. +type OutputV2Option interface{ applyToV2Output(*OutputV2Service) } + +func (opt DefaultCredentialLifetimeOption) applyToV2Output(o *OutputV2Service) { + o.defaultCredentialLifetime = opt.lifetime +} + // OutputV2Service produces credentials which can be used to connect to a // Kubernetes Cluster through teleport. type OutputV2Service struct { diff --git a/lib/tbot/services/k8s/output_v2_config.go b/lib/tbot/services/k8s/output_v2_config.go index 4350046392703..14b01706903d3 100644 --- a/lib/tbot/services/k8s/output_v2_config.go +++ b/lib/tbot/services/k8s/output_v2_config.go @@ -20,6 +20,9 @@ package k8s import ( "context" + "fmt" + "slices" + "strings" "github.com/gravitational/trace" "gopkg.in/yaml.v3" @@ -139,6 +142,23 @@ type KubernetesSelector struct { Labels map[string]string `yaml:"labels,omitempty"` } +// String returns a human-readable representation of the selector for logs. +func (s *KubernetesSelector) String() string { + switch { + case s.Name != "": + return fmt.Sprintf("name=%s", s.Name) + case len(s.Labels) != 0: + labels := make([]string, 0, len(s.Labels)) + for k, v := range s.Labels { + labels = append(labels, k+"="+v) + } + slices.Sort(labels) + return fmt.Sprintf("labels={%s}", strings.Join(labels, ", ")) + default: + return "" + } +} + func (s *KubernetesSelector) CheckAndSetDefaults() error { if s.Name == "" && len(s.Labels) == 0 { return trace.BadParameter("selectors: one of 'name' and 'labels' must be specified") diff --git a/lib/tbot/services/k8s/secret_destination.go b/lib/tbot/services/k8s/secret_destination.go index 3bbb58762fd61..f796b3cb255ec 100644 --- a/lib/tbot/services/k8s/secret_destination.go +++ b/lib/tbot/services/k8s/secret_destination.go @@ -146,14 +146,8 @@ func (dks *SecretDestination) Init(ctx context.Context, subdirs []string) error // If no k8s client is injected, we attempt to create one from the // environment. if dks.k8s == nil { - // BuildConfigFromFlags falls back to InClusterConfig if both params - // are empty. This means KUBECONFIG takes precedence. - clientCfg, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) - if err != nil { - return trace.Wrap(err) - } - dks.k8s, err = kubernetes.NewForConfig(clientCfg) - if err != nil { + var err error + if dks.k8s, err = newKubernetesClient(); err != nil { return trace.Wrap(err) } } @@ -293,3 +287,17 @@ func (dks *SecretDestination) MarshalYAML() (any, error) { func (dks *SecretDestination) IsPersistent() bool { return true } + +func newKubernetesClient() (*kubernetes.Clientset, error) { + // BuildConfigFromFlags falls back to InClusterConfig if both params + // are empty. This means KUBECONFIG takes precedence. + clientCfg, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) + if err != nil { + return nil, trace.Wrap(err) + } + k8s, err := kubernetes.NewForConfig(clientCfg) + if err != nil { + return nil, trace.Wrap(err) + } + return k8s, nil +} diff --git a/lib/tbot/services/k8s/secret_destination_test.go b/lib/tbot/services/k8s/secret_destination_test.go index 95bd262e63ed7..a0fd935795702 100644 --- a/lib/tbot/services/k8s/secret_destination_test.go +++ b/lib/tbot/services/k8s/secret_destination_test.go @@ -23,49 +23,13 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - kubeerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/fake" - core "k8s.io/client-go/testing" ) func TestDestinationKubernetesSecret(t *testing.T) { t.Setenv("POD_NAMESPACE", "test-namespace") - // Hack a reactor into the Kubernetes client-go fake client set as it - // doesn't currently support Apply :) - // https://github.com/kubernetes/kubernetes/issues/99953 - fakeClientSet := func(objects ...runtime.Object) *fake.Clientset { - f := fake.NewSimpleClientset(objects...) - f.PrependReactor("patch", "secrets", func(action core.Action) (handled bool, ret runtime.Object, err error) { - pa := action.(core.PatchAction) - if pa.GetPatchType() == types.ApplyPatchType { - react := core.ObjectReaction(f.Tracker()) - _, _, err := react( - core.NewGetAction(pa.GetResource(), pa.GetNamespace(), pa.GetName()), - ) - if kubeerrors.IsNotFound(err) { - _, _, err = react( - core.NewCreateAction(pa.GetResource(), pa.GetNamespace(), &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: pa.GetName(), - Namespace: pa.GetNamespace(), - }, - }), - ) - if err != nil { - return false, nil, err - } - } - return react(action) - } - return false, nil, nil - }) - return f - } - tests := []struct { name string dest *SecretDestination @@ -76,7 +40,7 @@ func TestDestinationKubernetesSecret(t *testing.T) { name: "no existing secret", dest: &SecretDestination{ Name: "my-secret", - k8s: fakeClientSet(), + k8s: fake.NewClientset(), }, }, { @@ -87,14 +51,14 @@ func TestDestinationKubernetesSecret(t *testing.T) { "key": "value", "bar": "baz", }, - k8s: fakeClientSet(), + k8s: fake.NewClientset(), }, }, { name: "existing secret", dest: &SecretDestination{ Name: "my-secret", - k8s: fakeClientSet(&corev1.Secret{ + k8s: fake.NewClientset(&corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "my-secret", Namespace: "test-namespace", diff --git a/lib/tbot/services/k8s/testdata/TestArgoCDOutput_YAML/full.golden b/lib/tbot/services/k8s/testdata/TestArgoCDOutput_YAML/full.golden new file mode 100644 index 0000000000000..e750a49a738ff --- /dev/null +++ b/lib/tbot/services/k8s/testdata/TestArgoCDOutput_YAML/full.golden @@ -0,0 +1,19 @@ +type: kubernetes/argo-cd +name: my-argo-service +credential_ttl: 1m0s +renewal_interval: 30s +selectors: + - name: foo + - labels: + foo: bar +secret_namespace: argocd +secret_name_prefix: my-argo-cluster- +secret_labels: + my-label: value +secret_annotations: + my-annotation: value +project: super-secret-project +namespaces: + - prod + - dev +cluster_resources: true diff --git a/lib/tbot/services/k8s/testdata/TestArgoCDOutput_YAML/minimal.golden b/lib/tbot/services/k8s/testdata/TestArgoCDOutput_YAML/minimal.golden new file mode 100644 index 0000000000000..a2fbedb367a4f --- /dev/null +++ b/lib/tbot/services/k8s/testdata/TestArgoCDOutput_YAML/minimal.golden @@ -0,0 +1,3 @@ +type: kubernetes/argo-cd +selectors: + - name: foo diff --git a/lib/tbot/tbot.go b/lib/tbot/tbot.go index 35aeb6ab09220..2cf02770f081c 100644 --- a/lib/tbot/tbot.go +++ b/lib/tbot/tbot.go @@ -216,9 +216,16 @@ func (b *Bot) Run(ctx context.Context) (err error) { case *ssh.MultiplexerConfig: services = append(services, ssh.MultiplexerServiceBuilder(svcCfg, alpnUpgradeCache, b.cfg.ConnectionConfig(), b.cfg.CredentialLifetime, clientMetrics)) case *k8s.OutputV1Config: - services = append(services, k8s.OutputV1ServiceBuilder(svcCfg, b.cfg.CredentialLifetime)) + services = append(services, k8s.OutputV1ServiceBuilder(svcCfg, k8s.WithDefaultCredentialLifetime(b.cfg.CredentialLifetime))) case *k8s.OutputV2Config: - services = append(services, k8s.OutputV2ServiceBuilder(svcCfg, b.cfg.CredentialLifetime)) + services = append(services, k8s.OutputV2ServiceBuilder(svcCfg, k8s.WithDefaultCredentialLifetime(b.cfg.CredentialLifetime))) + case *k8s.ArgoCDOutputConfig: + services = append(services, k8s.ArgoCDServiceBuilder( + svcCfg, + k8s.WithDefaultCredentialLifetime(b.cfg.CredentialLifetime), + k8s.WithInsecure(b.cfg.ConnectionConfig().Insecure), + k8s.WithALPNUpgradeCache(alpnUpgradeCache), + )) case *ssh.HostOutputConfig: services = append(services, ssh.HostOutputServiceBuilder(svcCfg, b.cfg.CredentialLifetime)) case *application.OutputConfig: