diff --git a/Makefile b/Makefile index b3e52b56cca02..9ad7c42aebce1 100644 --- a/Makefile +++ b/Makefile @@ -363,8 +363,9 @@ lint-sh: # Lints all the Helm charts found in directories under examples/chart and exits on failure # If there is a .lint directory inside, the chart gets linted once for each .yaml file in that directory -# We use yamllint's 'relaxed' configuration as it's more compatible with Helm output and will only error on +# We inherit yamllint's 'relaxed' configuration as it's more compatible with Helm output and will only error on # show-stopping issues. Kubernetes' YAML parser is not particularly fussy. +# If errors are found, the file is printed with line numbers to aid in debugging. .PHONY: lint-helm lint-helm: @if ! type yamllint 2>&1 >/dev/null; then \ @@ -373,16 +374,21 @@ lint-helm: exit 0; \ fi; \ for CHART in $$(find examples/chart -mindepth 1 -maxdepth 1 -type d); do \ - if [ -d $$CHART/.lint ]; then \ - for VALUES in $$CHART/.lint/*.yaml; do \ - echo "$$CHART: $$VALUES"; \ - helm lint --strict $$CHART -f $$VALUES || exit 1; \ - helm template test $$CHART -f $$VALUES | yamllint -d relaxed - || exit 1; \ + if [ -d $${CHART}/.lint ]; then \ + for VALUES in $${CHART}/.lint/*.yaml; do \ + export HELM_TEMP=$$(mktemp); \ + echo -n "Using values from '$${VALUES}': "; \ + yamllint -c examples/chart/.lint-config.yaml $${VALUES} || { cat -En $${VALUES}; exit 1; }; \ + helm lint --strict $${CHART} -f $${VALUES} || exit 1; \ + helm template test $${CHART} -f $${VALUES} 1>$${HELM_TEMP} || exit 1; \ + yamllint -c examples/chart/.lint-config.yaml $${HELM_TEMP} || { cat -En $${HELM_TEMP}; exit 1; }; \ done \ else \ - helm lint --strict $$CHART || exit 1; \ - helm template test $$CHART 1>/dev/null || exit 1; \ - fi \ + export HELM_TEMP=$$(mktemp); \ + helm lint --strict $${CHART} || exit 1; \ + helm template test $${CHART} 1>$${HELM_TEMP} || exit 1; \ + yamllint -c examples/chart/.lint-config.yaml $${HELM_TEMP} || { cat -En $${HELM_TEMP}; exit 1; }; \ + fi; \ done # This rule triggers re-generation of version.go and gitref.go if Makefile changes diff --git a/examples/chart/.lint-config.yaml b/examples/chart/.lint-config.yaml new file mode 100644 index 0000000000000..524ca96807df0 --- /dev/null +++ b/examples/chart/.lint-config.yaml @@ -0,0 +1,4 @@ +extends: relaxed +rules: + line-length: + max: 120 diff --git a/examples/chart/teleport-auto-trustedcluster/Chart.yaml b/examples/chart/teleport-auto-trustedcluster/Chart.yaml index 39d552b1b9528..6eb4ba3ef2f32 100644 --- a/examples/chart/teleport-auto-trustedcluster/Chart.yaml +++ b/examples/chart/teleport-auto-trustedcluster/Chart.yaml @@ -1,8 +1,9 @@ name: teleport-auto-trustedcluster apiVersion: v2 -version: 0.0.8 +version: 0.0.9 appVersion: "6" -description: Teleport trusted cluster installation which automatically joins itself back to the provided root cluster. +description: "[deprecated] Teleport trusted cluster installation which automatically joins itself back to the provided root cluster." icon: https://goteleport.com/images/logos/logo-teleport-square.svg keywords: - Teleport Enterprise +deprecated: true diff --git a/examples/chart/teleport-auto-trustedcluster/templates/service.yaml b/examples/chart/teleport-auto-trustedcluster/templates/service.yaml index e54a008f69a60..190db258e17b3 100644 --- a/examples/chart/teleport-auto-trustedcluster/templates/service.yaml +++ b/examples/chart/teleport-auto-trustedcluster/templates/service.yaml @@ -15,8 +15,8 @@ spec: type: {{ .Values.service.type }} ports: {{- range $key, $value := .Values.service.ports }} - - name: {{ $key }} -{{ toYaml $value | indent 6 }} + - name: {{ $key }} +{{ toYaml $value | indent 4 }} {{- end }} {{- if and (semverCompare ">=1.7-0" .Capabilities.KubeVersion.GitVersion) (.Values.service.externalTrafficPolicy) }} externalTrafficPolicy: "{{ .Values.service.externalTrafficPolicy }}" diff --git a/examples/chart/teleport-cluster/.lint/lint-no-acme.yaml b/examples/chart/teleport-cluster/.lint/acme-off.yaml similarity index 54% rename from examples/chart/teleport-cluster/.lint/lint-no-acme.yaml rename to examples/chart/teleport-cluster/.lint/acme-off.yaml index 47a090280333a..29a9052737d1d 100644 --- a/examples/chart/teleport-cluster/.lint/lint-no-acme.yaml +++ b/examples/chart/teleport-cluster/.lint/acme-off.yaml @@ -1 +1,3 @@ clusterName: test-cluster-name +extraArgs: +- "--insecure" diff --git a/examples/chart/teleport-cluster/.lint/acme-on.yaml b/examples/chart/teleport-cluster/.lint/acme-on.yaml new file mode 100644 index 0000000000000..02821dc7cad17 --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/acme-on.yaml @@ -0,0 +1,3 @@ +clusterName: test-acme-cluster +acme: true +acmeEmail: test@email.com diff --git a/examples/chart/teleport-cluster/.lint/lint-acme.yaml b/examples/chart/teleport-cluster/.lint/acme-uri-staging.yaml similarity index 100% rename from examples/chart/teleport-cluster/.lint/lint-acme.yaml rename to examples/chart/teleport-cluster/.lint/acme-uri-staging.yaml diff --git a/examples/chart/teleport-cluster/.lint/affinity.yaml b/examples/chart/teleport-cluster/.lint/affinity.yaml new file mode 100644 index 0000000000000..e984e7d098edb --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/affinity.yaml @@ -0,0 +1,29 @@ +clusterName: test-gcp-cluster +chartMode: gcp +gcp: + projectId: gcpproj-123456 + backendTable: test-teleport-firestore-storage-collection + auditLogTable: test-teleport-firestore-auditlog-collection + sessionRecordingBucket: test-gcp-session-storage-bucket +highAvailability: + replicaCount: 2 +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: gravitational.io/dedicated + operator: In + values: + - teleport + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - teleport + topologyKey: kubernetes.io/hostname + weight: 1 diff --git a/examples/chart/teleport-cluster/.lint/annotations.yaml b/examples/chart/teleport-cluster/.lint/annotations.yaml new file mode 100644 index 0000000000000..4e9fce5f68d98 --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/annotations.yaml @@ -0,0 +1,17 @@ +clusterName: helm-lint +annotations: + config: + kubernetes.io/config: "test-annotation" + kubernetes.io/config-different: 2 + deployment: + kubernetes.io/deployment: "test-annotation" + kubernetes.io/deployment-different: 3 + pod: + kubernetes.io/pod: "test-annotation" + kubernetes.io/pod-different: 4 + service: + kubernetes.io/service: "test-annotation" + kubernetes.io/service-different: 5 + serviceAccount: + kubernetes.io/serviceaccount: "test-annotation" + kubernetes.io/serviceaccount-different: 6 diff --git a/examples/chart/teleport-cluster/.lint/aws-ha-acme.yaml b/examples/chart/teleport-cluster/.lint/aws-ha-acme.yaml new file mode 100644 index 0000000000000..c2c4d2e31b534 --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/aws-ha-acme.yaml @@ -0,0 +1,14 @@ +clusterName: test-aws-cluster +chartMode: aws +aws: + region: us-west-2 + backendTable: test-dynamodb-backend-table + auditLogTable: test-dynamodb-auditlog-table + sessionRecordingBucket: test-s3-session-storage-bucket +highAvailability: + replicaCount: 3 + certManager: + enabled: true + issuerName: letsencrypt-production +labels: + env: aws diff --git a/examples/chart/teleport-cluster/.lint/aws-ha-antiaffinity.yaml b/examples/chart/teleport-cluster/.lint/aws-ha-antiaffinity.yaml new file mode 100644 index 0000000000000..0e639a246cd94 --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/aws-ha-antiaffinity.yaml @@ -0,0 +1,12 @@ +clusterName: test-aws-cluster +chartMode: aws +aws: + region: us-west-2 + backendTable: test-dynamodb-backend-table + auditLogTable: test-dynamodb-auditlog-table + sessionRecordingBucket: test-s3-session-storage-bucket +highAvailability: + replicaCount: 3 + requireAntiAffinity: true +labels: + env: aws diff --git a/examples/chart/teleport-cluster/.lint/aws-ha.yaml b/examples/chart/teleport-cluster/.lint/aws-ha.yaml new file mode 100644 index 0000000000000..5bb212035d68d --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/aws-ha.yaml @@ -0,0 +1,11 @@ +clusterName: test-aws-cluster +chartMode: aws +aws: + region: us-west-2 + backendTable: test-dynamodb-backend-table + auditLogTable: test-dynamodb-auditlog-table + sessionRecordingBucket: test-s3-session-storage-bucket +highAvailability: + replicaCount: 3 +labels: + env: aws diff --git a/examples/chart/teleport-cluster/.lint/aws.yaml b/examples/chart/teleport-cluster/.lint/aws.yaml new file mode 100644 index 0000000000000..0c822e3299fc5 --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/aws.yaml @@ -0,0 +1,11 @@ +clusterName: test-aws-cluster +chartMode: aws +aws: + region: us-west-2 + backendTable: test-dynamodb-backend-table + auditLogTable: test-dynamodb-auditlog-table + sessionRecordingBucket: test-s3-session-storage-bucket +acme: true +acmeEmail: test@email.com +labels: + env: aws diff --git a/examples/chart/teleport-cluster/.lint/gcp-ha-acme.yaml b/examples/chart/teleport-cluster/.lint/gcp-ha-acme.yaml new file mode 100644 index 0000000000000..d122907071d57 --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/gcp-ha-acme.yaml @@ -0,0 +1,14 @@ +clusterName: test-gcp-cluster +chartMode: gcp +gcp: + projectId: gcpproj-123456 + backendTable: test-teleport-firestore-storage-collection + auditLogTable: test-teleport-firestore-auditlog-collection + sessionRecordingBucket: test-gcp-session-storage-bucket +highAvailability: + replicaCount: 3 + certManager: + enabled: true + issuerName: letsencrypt-production +labels: + env: gcp diff --git a/examples/chart/teleport-cluster/.lint/gcp-ha-antiaffinity.yaml b/examples/chart/teleport-cluster/.lint/gcp-ha-antiaffinity.yaml new file mode 100644 index 0000000000000..9743cad827876 --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/gcp-ha-antiaffinity.yaml @@ -0,0 +1,12 @@ +clusterName: test-gcp-cluster +chartMode: gcp +gcp: + projectId: gcpproj-123456 + backendTable: test-teleport-firestore-storage-collection + auditLogTable: test-teleport-firestore-auditlog-collection + sessionRecordingBucket: test-gcp-session-storage-bucket +highAvailability: + replicaCount: 3 + requireAntiAffinity: true +labels: + env: gcp diff --git a/examples/chart/teleport-cluster/.lint/gcp-ha.yaml b/examples/chart/teleport-cluster/.lint/gcp-ha.yaml new file mode 100644 index 0000000000000..26b43d4d0f0e1 --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/gcp-ha.yaml @@ -0,0 +1,11 @@ +clusterName: test-gcp-cluster +chartMode: gcp +gcp: + projectId: gcpproj-123456 + backendTable: test-teleport-firestore-storage-collection + auditLogTable: test-teleport-firestore-auditlog-collection + sessionRecordingBucket: test-gcp-session-storage-bucket +highAvailability: + replicaCount: 3 +labels: + env: gcp diff --git a/examples/chart/teleport-cluster/.lint/gcp.yaml b/examples/chart/teleport-cluster/.lint/gcp.yaml new file mode 100644 index 0000000000000..56a395bd8ab61 --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/gcp.yaml @@ -0,0 +1,11 @@ +clusterName: test-gcp-cluster +chartMode: gcp +gcp: + projectId: gcpproj-123456 + backendTable: test-teleport-firestore-storage-collection + auditLogTable: test-teleport-firestore-auditlog-collection + sessionRecordingBucket: test-gcp-session-storage-bucket +acme: true +acmeEmail: test@email.com +labels: + env: gcp diff --git a/examples/chart/teleport-cluster/.lint/initcontainers.yaml b/examples/chart/teleport-cluster/.lint/initcontainers.yaml new file mode 100644 index 0000000000000..24de49e5ee2ee --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/initcontainers.yaml @@ -0,0 +1,5 @@ +clusterName: helm-lint +initContainers: +- name: "teleport-init" + image: "alpine" + args: ["echo test"] diff --git a/examples/chart/teleport-cluster/.lint/resources.yaml b/examples/chart/teleport-cluster/.lint/resources.yaml new file mode 100644 index 0000000000000..070a85c77cdbc --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/resources.yaml @@ -0,0 +1,10 @@ +clusterName: helm-lint +# These are just sample values to test the chart. +# They are not intended to be guidelines or suggestions for running teleport. +resources: + limits: + cpu: 2 + memory: 4Gi + requests: + cpu: 1 + memory: 2Gi diff --git a/examples/chart/teleport-cluster/.lint/standalone-customsize.yaml b/examples/chart/teleport-cluster/.lint/standalone-customsize.yaml new file mode 100644 index 0000000000000..345999323a716 --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/standalone-customsize.yaml @@ -0,0 +1,8 @@ +clusterName: test-standalone-cluster +chartMode: standalone +standalone: + existingClaimName: teleport-storage +acme: true +acmeEmail: test@email.com +labels: + env: standalone diff --git a/examples/chart/teleport-cluster/.lint/standalone-existingpvc.yaml b/examples/chart/teleport-cluster/.lint/standalone-existingpvc.yaml new file mode 100644 index 0000000000000..e8acbe8cf9848 --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/standalone-existingpvc.yaml @@ -0,0 +1,8 @@ +clusterName: test-standalone-cluster +chartMode: standalone +standalone: + volumeSize: 50Gi +acme: true +acmeEmail: test@email.com +labels: + env: standalone diff --git a/examples/chart/teleport-cluster/.lint/tolerations.yaml b/examples/chart/teleport-cluster/.lint/tolerations.yaml new file mode 100644 index 0000000000000..69d4161781cb9 --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/tolerations.yaml @@ -0,0 +1,18 @@ +clusterName: test-aws-cluster +chartMode: aws +aws: + region: us-west-2 + backendTable: test-dynamodb-backend-table + auditLogTable: test-dynamodb-auditlog-table + sessionRecordingBucket: test-s3-session-storage-bucket +highAvailability: + replicaCount: 3 +tolerations: +- key: "dedicated" + operator: "Equal" + value: "teleport" + effect: "NoExecute" +- key: "dedicated" + operator: "Equal" + value: "teleport" + effect: "NoSchedule" diff --git a/examples/chart/teleport-cluster/.lint/lint-versionoverride.yaml b/examples/chart/teleport-cluster/.lint/version-override.yaml similarity index 100% rename from examples/chart/teleport-cluster/.lint/lint-versionoverride.yaml rename to examples/chart/teleport-cluster/.lint/version-override.yaml diff --git a/examples/chart/teleport-cluster/.lint/volumes.yaml b/examples/chart/teleport-cluster/.lint/volumes.yaml new file mode 100644 index 0000000000000..83bb8b8dbe95c --- /dev/null +++ b/examples/chart/teleport-cluster/.lint/volumes.yaml @@ -0,0 +1,8 @@ +clusterName: helm-lint +extraVolumeMounts: +- name: "my-mount" + path: "/path/to/mount" +extraVolumes: +- name: "my-mount" + secret: + secretName: "mySecret" diff --git a/examples/chart/teleport-cluster/Chart.yaml b/examples/chart/teleport-cluster/Chart.yaml index 5e97367389338..507cbd29a40db 100644 --- a/examples/chart/teleport-cluster/Chart.yaml +++ b/examples/chart/teleport-cluster/Chart.yaml @@ -1,6 +1,6 @@ name: teleport-cluster apiVersion: v2 -version: 6.0.0 +version: "6" appVersion: "6" description: Teleport is a unified access plane for your infrastructure icon: https://goteleport.com/images/logos/logo-teleport-square.svg diff --git a/examples/chart/teleport-cluster/templates/NOTES.txt b/examples/chart/teleport-cluster/templates/NOTES.txt new file mode 100644 index 0000000000000..b84de6d0f8d39 --- /dev/null +++ b/examples/chart/teleport-cluster/templates/NOTES.txt @@ -0,0 +1,16 @@ +{{- if .Values.highAvailability.certManager.enabled }} +You have enabled cert-manager support in high availability mode. + +There may be a short delay before Teleport pods start while an ACME certificate is issued. +You can check the status of the certificate with `kubectl -n {{ .Release.Namespace }} describe certificate/{{ .Release.Name }}` + +NOTE: For certificates to be provisioned, you must also install cert-manager (https://cert-manager.io/docs/) and configure an appropriate + Issuer with access to your DNS provider to handle DNS01 challenges (https://cert-manager.io/docs/configuration/acme/dns01/#supported-dns01-providers) + +For more information, please see the Helm guides in the Teleport docs (https://goteleport.com/docs/kubernetes-access/helm/guides/) +{{- else if (gt (int .Values.highAvailability.replicaCount) 1) }} +You have requested more than 1 replica but have not enabled cert-manager support (highAvailability.certManager.enabled=true) to get ACME certificates. +Your Teleport cluster will not be properly accessible by remote nodes until TLS certificates with the correct clusterName ({{ .Values.clusterName }}) are configured. + +For more information, please see the Helm guides in the Teleport docs (https://goteleport.com/docs/kubernetes-access/helm/guides/) +{{- end }} \ No newline at end of file diff --git a/examples/chart/teleport-cluster/templates/certificate.yaml b/examples/chart/teleport-cluster/templates/certificate.yaml new file mode 100644 index 0000000000000..e4e7f75b0e085 --- /dev/null +++ b/examples/chart/teleport-cluster/templates/certificate.yaml @@ -0,0 +1,17 @@ +{{- if .Values.highAvailability.certManager.enabled }} + {{- $domain:= (required "clusterName is required in chartValues when certManager is enabled" .Values.clusterName) }} + {{- $domainWildcard := printf "*.%s" (required "clusterName is required in chartValues when certManager is enabled" .Values.clusterName) }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +spec: + secretName: teleport-tls + dnsNames: + - {{ quote $domain }} + - {{ quote $domainWildcard }} + issuerRef: + name: {{ required "highAvailability.certManager.issuerName is required in chart values" .Values.highAvailability.certManager.issuerName }} + kind: {{ required "highAvailability.certManager.issuerKind is required in chart values" .Values.highAvailability.certManager.issuerKind }} +{{- end }} diff --git a/examples/chart/teleport-cluster/templates/config.yaml b/examples/chart/teleport-cluster/templates/config.yaml index f0e93db6a2593..f3dd01021c037 100644 --- a/examples/chart/teleport-cluster/templates/config.yaml +++ b/examples/chart/teleport-cluster/templates/config.yaml @@ -1,39 +1,65 @@ -{{- if not .Values.customConfig -}} +{{- if not (eq .Values.chartMode "custom") -}} apiVersion: v1 kind: ConfigMap metadata: name: {{ .Release.Name }} + {{- if .Values.annotations.config }} + annotations: + {{- toYaml .Values.annotations.config | nindent 4 }} + {{- end }} data: teleport.yaml: | teleport: log: severity: {{ .Values.logLevel }} output: stderr + {{- if eq .Values.chartMode "aws" }} + storage: + type: dynamodb + region: {{ required "aws.region is required in chart values" .Values.aws.region }} + table_name: {{ required "aws.backendTable is required in chart values" .Values.aws.backendTable }} + audit_events_uri: ['dynamodb://{{ required "aws.auditLogTable is required in chart values" .Values.aws.auditLogTable }}'] + audit_sessions_uri: s3://{{ required "aws.sessionRecordingBucket is required in chart values" .Values.aws.sessionRecordingBucket }} + {{- else if eq .Values.chartMode "gcp" }} + storage: + type: firestore + project_id: {{ required "gcp.projectId is required in chart values" .Values.gcp.projectId }} + collection_name: {{ required "gcp.backendTable is required in chart values" .Values.gcp.backendTable }} + {{- if .Values.gcp.credentialSecretName }} + credentials_path: /etc/teleport-secrets/gcp-credentials.json + {{- end }} + audit_events_uri: ['firestore://{{ required "gcp.auditLogTable is required in chart values" .Values.gcp.auditLogTable }}?projectID={{ required "gcp.projectId is required in chart values" .Values.gcp.projectId }}&credentialsPath=/etc/teleport-secrets/gcp-credentials.json'] + audit_sessions_uri: "gs://{{ required "gcp.sessionRecordingBucket is required in chart values" .Values.gcp.sessionRecordingBucket }}?projectID={{ required "gcp.projectId is required in chart values" .Values.gcp.projectId }}&credentialsPath=/etc/teleport-secrets/gcp-credentials.json" + {{- end }} auth_service: enabled: true cluster_name: {{ required "clusterName is required in chart values" .Values.clusterName }} -{{- if .Values.enterprise }} - license_file: '/var/lib/license/license-enterprise.pem' -{{- end }} + {{- if .Values.enterprise }} + license_file: '/var/lib/license/license.pem' + {{- end }} kubernetes_service: enabled: true listen_addr: 0.0.0.0:3027 + {{- if .Values.labels }} labels: -{{- if .Values.labels }} -{{ toYaml .Values.labels | indent 8 }} -{{- end }} + {{ toYaml .Values.labels | indent 8 }} + {{- end }} proxy_service: public_addr: '{{ required "clusterName is required in chart values" .Values.clusterName }}:443' kube_listen_addr: 0.0.0.0:3026 enabled: true - {{- if .Values.acme }} + {{- if .Values.highAvailability.certManager.enabled }} + https_keypairs: + - key_file: /etc/teleport-tls/tls.key + cert_file: /etc/teleport-tls/tls.crt + {{- else if .Values.acme }} acme: enabled: {{ .Values.acme }} - email: {{ .Values.acmeEmail }} - {{- if .Values.acmeURI }} + email: {{ required "acmeEmail is required in chart values" .Values.acmeEmail }} + {{- if .Values.acmeURI }} uri: {{ .Values.acmeURI }} - {{- end }} - {{- end }} + {{- end }} + {{- end }} ssh_service: enabled: false {{- end -}} \ No newline at end of file diff --git a/examples/chart/teleport-cluster/templates/deployment.yaml b/examples/chart/teleport-cluster/templates/deployment.yaml index 9e31ca1b72b9b..8ce8a6274f843 100644 --- a/examples/chart/teleport-cluster/templates/deployment.yaml +++ b/examples/chart/teleport-cluster/templates/deployment.yaml @@ -1,3 +1,9 @@ +{{- if and (.Values.acme) (gt (int .Values.highAvailability.replicaCount) 1) }} +{{- fail "Cannot enable built-in ACME support with more than one replica, use highAvailability.certManager.enabled instead" }} +{{- end }} +{{- if and (eq .Values.chartMode "standalone") (gt (int .Values.highAvailability.replicaCount) 1) }} +{{- fail "Cannot enable multiple replicas in standalone mode, use a different chartMode which supports high availability - see README and docs" }} +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: @@ -5,8 +11,16 @@ metadata: namespace: {{ .Release.Namespace }} labels: app: {{ .Release.Name }} + {{- if .Values.annotations.deployment }} + annotations: + {{- toYaml .Values.annotations.deployment | nindent 4 }} + {{- end }} spec: - replicas: {{ .Values.replicaCount }} + {{- if not (eq .Values.chartMode "standalone") }} + replicas: {{ .Values.highAvailability.replicaCount }} + {{- else }} + replicas: 1 + {{- end }} selector: matchLabels: app: {{ .Release.Name }} @@ -15,17 +29,79 @@ spec: annotations: # ConfigMap checksum, to recreate the pod on config changes. checksum/config: {{ include (print $.Template.BasePath "/config.yaml") . | sha256sum }} +{{- if .Values.annotations.pod }} + {{- toYaml .Values.annotations.pod | nindent 8 }} +{{- end }} labels: app: {{ .Release.Name }} spec: + {{- if or .Values.affinity (gt (int .Values.highAvailability.replicaCount) 1) }} + affinity: + {{- if .Values.affinity }} + {{- if .Values.highAvailability.requireAntiAffinity }} + {{- fail "Cannot use highAvailability.requireAntiAffinity when affinity is also set in chart values - unset one or the other" }} + {{- end }} + {{- toYaml .Values.affinity | nindent 8 }} + {{- else }} + podAntiAffinity: + {{- if .Values.highAvailability.requireAntiAffinity }} + requiredDuringSchedulingIgnoredDuringExecution: + {{- else if gt (int .Values.highAvailability.replicaCount) 1 }} + preferredDuringSchedulingIgnoredDuringExecution: + {{- end }} + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ .Release.Name }} + topologyKey: "kubernetes.io/hostname" + {{- end }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: + {{- toYaml .Values.tolerations | nindent 6 }} + {{- end }} +{{- if .Values.initContainers }} + initContainers: {{- toYaml .Values.initContainers | nindent 6 }} + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.enterprise }} + - mountPath: /var/lib/license + name: "license" + readOnly: true + {{- end }} + {{- if eq .Values.chartMode "gcp" }} + - mountPath: /etc/teleport-secrets + name: "gcp-credentials" + readOnly: true + {{- end }} + - mountPath: /etc/teleport + name: "config" + readOnly: true + - mountPath: /var/lib/teleport + name: "data" + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} +{{- end }} containers: - name: "teleport" - image: "{{ if .Values.enterprise }}{{ .Values.enterpriseImage }}{{ else }}{{ .Values.image }}{{ end }}:{{ if .Values.teleportVersionOverride }}{{ .Values.teleportVersionOverride }}{{ else }}{{ .Chart.AppVersion }}{{ end }}" + image: "{{ if .Values.enterprise }}{{ .Values.enterpriseImage }}{{ else }}{{ .Values.image }}{{ end }}:{{ if .Values.teleportVersionOverride }}{{ .Values.teleportVersionOverride }}{{ else }}{{ .Chart.Version }}{{ end }}" + imagePullPolicy: {{ .Values.imagePullPolicy }} args: - "--diag-addr=0.0.0.0:3000" {{- if .Values.insecureSkipProxyTLSVerify }} - "--insecure" {{- end }} + {{- if .Values.extraArgs }} + {{- toYaml .Values.extraArgs | nindent 8 }} + {{- end }} ports: - name: diag containerPort: 3000 @@ -44,27 +120,61 @@ spec: initialDelaySeconds: 5 # wait 5s for agent to register periodSeconds: 5 # poll health every 5s failureThreshold: 12 # consider agent unhealthy after 60s (12 * 5s) +{{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} +{{- end }} volumeMounts: {{- if .Values.enterprise }} - mountPath: /var/lib/license name: "license" readOnly: true +{{- end }} +{{- if eq .Values.chartMode "gcp" }} + - mountPath: /etc/teleport-secrets + name: "gcp-credentials" + readOnly: true +{{- end }} +{{- if .Values.highAvailability.certManager.enabled }} + - mountPath: /etc/teleport-tls + name: "teleport-tls" + readOnly: true {{- end }} - mountPath: /etc/teleport name: "config" readOnly: true - mountPath: /var/lib/teleport name: "data" +{{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 8 }} +{{- end }} volumes: {{- if .Values.enterprise }} - name: license secret: secretName: "license" +{{- end }} +{{- if .Values.gcp.credentialSecretName }} + - name: gcp-credentials + secret: + secretName: {{ required "gcp.credentialSecretName is required in chart values" .Values.gcp.credentialSecretName }} +{{- end }} +{{- if .Values.highAvailability.certManager.enabled }} + - name: teleport-tls + secret: + secretName: teleport-tls {{- end }} - name: "config" configMap: name: {{ .Release.Name }} - name: "data" + {{- if eq .Values.chartMode "standalone" }} persistentVolumeClaim: - claimName: {{ .Release.Name }} + claimName: {{ if .Values.standalone.existingClaimName }}{{ .Values.standalone.existingClaimName }}{{ else }}{{ .Release.Name }}{{ end }} + {{- else }} + emptyDir: {} + {{- end }} +{{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 6 }} +{{- end }} serviceAccountName: {{ .Release.Name }} diff --git a/examples/chart/teleport-cluster/templates/pvc.yaml b/examples/chart/teleport-cluster/templates/pvc.yaml index ccf13c28376fb..7e01b3c36d12e 100644 --- a/examples/chart/teleport-cluster/templates/pvc.yaml +++ b/examples/chart/teleport-cluster/templates/pvc.yaml @@ -1,3 +1,4 @@ +{{- if and (eq .Values.chartMode "standalone") (not .Values.standalone.existingClaimName) }} apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -10,4 +11,5 @@ spec: - ReadWriteOnce resources: requests: - storage: 10Gi + storage: {{ required "standalone.volumeSize is required in chart values" .Values.standalone.volumeSize }} +{{- end }} diff --git a/examples/chart/teleport-cluster/templates/service.yaml b/examples/chart/teleport-cluster/templates/service.yaml index ef671ba001708..de332eb635315 100644 --- a/examples/chart/teleport-cluster/templates/service.yaml +++ b/examples/chart/teleport-cluster/templates/service.yaml @@ -5,6 +5,17 @@ metadata: namespace: {{ .Release.Namespace }} labels: app: {{ .Release.Name }} + {{- if (or (.Values.annotations.service) (eq .Values.chartMode "aws")) }} + annotations: + {{- if .Values.annotations.service }} + {{- toYaml .Values.annotations.service | nindent 4 }} + {{- end }} + {{- if eq .Values.chartMode "aws" }} + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp + service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" + service.beta.kubernetes.io/aws-load-balancer-type: nlb + {{- end }} + {{- end }} spec: type: LoadBalancer ports: diff --git a/examples/chart/teleport-cluster/templates/serviceaccount.yaml b/examples/chart/teleport-cluster/templates/serviceaccount.yaml index 81c0d7ed4d2ed..9b600660f2485 100644 --- a/examples/chart/teleport-cluster/templates/serviceaccount.yaml +++ b/examples/chart/teleport-cluster/templates/serviceaccount.yaml @@ -3,3 +3,7 @@ kind: ServiceAccount metadata: name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} +{{- if .Values.annotations.serviceAccount }} + annotations: +{{- toYaml .Values.annotations.serviceAccount | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/examples/chart/teleport-cluster/values.schema.json b/examples/chart/teleport-cluster/values.schema.json new file mode 100644 index 0000000000000..943d8cd90f33d --- /dev/null +++ b/examples/chart/teleport-cluster/values.schema.json @@ -0,0 +1,301 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "required": [ + "clusterName", + "enterprise", + "podSecurityPolicy", + "labels", + "chartMode", + "highAvailability", + "image", + "enterpriseImage", + "logLevel", + "affinity", + "annotations", + "extraVolumes", + "extraVolumeMounts", + "imagePullPolicy", + "initContainers", + "resources", + "tolerations" + ], + "properties": { + "clusterName": { + "$id": "#/properties/clusterName", + "type": "string", + "default": "" + }, + "teleportVersionOverride": { + "$id": "#/properties/teleportVersionOverride", + "type": "string", + "default": "" + }, + "acme": { + "$id": "#/properties/acme", + "type": "boolean", + "default": false + }, + "acmeEmail": { + "$id": "#/properties/acmeEmail", + "type": "string", + "default": "" + }, + "acmeURI": { + "$id": "#/properties/acmeURI", + "type": "string", + "default": "" + }, + "enterprise": { + "$id": "#/properties/enterprise", + "type": "boolean", + "default": false + }, + "podSecurityPolicy": { + "$id": "#/properties/podSecurityPolicy", + "type": "object", + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "$id": "#/properties/podSecurityPolicy/properties/enabled", + "type": "boolean", + "default": true + } + } + }, + "labels": { + "$id": "#/properties/labels", + "type": "object", + "default": {} + }, + "chartMode": { + "$id": "#/properties/chartMode", + "type": "string", + "enum": ["standalone", "aws", "gcp", "custom"], + "default": "standalone" + }, + "standalone": { + "$id": "#/properties/standalone", + "type": "object", + "required": [ + "volumeSize" + ], + "properties": { + "existingClaimName": { + "$id": "#/properties/standalone/properties/existingClaimName", + "type": "string", + "default": "" + }, + "volumeSize": { + "$id": "#/properties/standalone/properties/volumeSize", + "type": "string", + "default": "" + } + } + }, + "aws": { + "$id": "#/properties/aws", + "type": "object", + "properties": { + "region": { + "$id": "#/properties/aws/properties/region", + "type": "string", + "default": "" + }, + "backendTable": { + "$id": "#/properties/aws/properties/backendTable", + "type": "string", + "default": "" + }, + "auditLogTable": { + "$id": "#/properties/aws/properties/auditLogTable", + "type": "string", + "default": "" + }, + "sessionRecordingBucket": { + "$id": "#/properties/aws/properties/sessionRecordingBucket", + "type": "string", + "default": "" + } + } + }, + "gcp": { + "$id": "#/properties/gcp", + "type": "object", + "required": [ + "credentialSecretName" + ], + "properties": { + "projectId": { + "$id": "#/properties/gcp/properties/projectId", + "type": "string", + "default": "" + }, + "backendTable": { + "$id": "#/properties/gcp/properties/backendTable", + "type": "string", + "default": "" + }, + "auditLogTable": { + "$id": "#/properties/gcp/properties/auditLogTable", + "type": "string", + "default": "" + }, + "sessionRecordingBucket": { + "$id": "#/properties/gcp/properties/sessionRecordingBucket", + "type": "string", + "default": "" + }, + "credentialSecretName": { + "$id": "#/properties/gcp/properties/credentialSecretName", + "type": "string", + "default": "teleport-gcp-credentials" + } + } + }, + "highAvailability": { + "$id": "#/properties/highAvailability", + "type": "object", + "required": [ + "replicaCount", + "requireAntiAffinity", + "certManager" + ], + "properties": { + "replicaCount": { + "$id": "#/properties/highAvailability/properties/replicaCount", + "type": "integer", + "default": 1 + }, + "requireAntiAffinity": { + "$id": "#/properties/highAvailability/properties/requireAntiAffinity", + "type": "boolean", + "default": false + }, + "certManager": { + "$id": "#/properties/highAvailability/properties/certManager", + "type": "object", + "required": [ + "enabled", + "issuerName", + "issuerKind" + ], + "properties": { + "enabled": { + "$id": "#/properties/highAvailability/properties/certManager/properties/enabled", + "type": "boolean", + "default": "false" + }, + "issuerName": { + "$id": "#/properties/highAvailability/properties/certManager/properties/issuerName", + "type": "string", + "default": "" + }, + "issuerKind": { + "$id": "#/properties/highAvailability/properties/certManager/properties/issuerKind", + "type": "string", + "default": "Issuer" + } + } + } + } + }, + "image": { + "$id": "#/properties/image", + "type": "string", + "default": "quay.io/gravitational/teleport" + }, + "enterpriseImage": { + "$id": "#/properties/enterpriseImage", + "type": "string", + "default": "quay.io/gravitational/teleport-ent" + }, + "logLevel": { + "$id": "#/properties/logLevel", + "type": "string", + "enum": ["DEBUG", "INFO", "WARN", "WARNING", "ERROR"], + "default": "INFO" + }, + "affinity": { + "$id": "#/properties/affinity", + "type": "object", + "default": {} + }, + "annotations": { + "$id": "#/properties/annotations", + "type": "object", + "required": [ + "config", + "deployment", + "pod", + "service", + "serviceAccount" + ], + "properties": { + "config": { + "$id": "#/properties/annotations/properties/config", + "type": "object", + "default": {} + }, + "deployment": { + "$id": "#/properties/annotations/properties/deployment", + "type": "object", + "default": {} + }, + "pod": { + "$id": "#/properties/annotations/properties/pod", + "type": "object", + "default": {} + }, + "service": { + "$id": "#/properties/annotations/properties/service", + "type": "object", + "default": {} + }, + "serviceAccount": { + "$id": "#/properties/annotations/properties/serviceAccount", + "type": "object", + "default": {} + } + } + }, + "extraArgs": { + "$id": "#/properties/extraArgs", + "type": "array", + "default": [] + }, + "extraVolumes": { + "$id": "#/properties/extraVolumes", + "type": "array", + "default": [] + }, + "extraVolumeMounts": { + "$id": "#/properties/extraVolumeMounts", + "type": "array", + "default": [] + }, + "imagePullPolicy": { + "$id": "#/properties/imagePullPolicy", + "type": "string", + "enum": ["Never", "IfNotPresent", "Always"], + "default": "IfNotPresent" + }, + "initContainers": { + "$id": "#/properties/initContainers", + "type": "array", + "default": [] + }, + "resources": { + "$id": "#/properties/resources", + "type": "object", + "default": {} + }, + "tolerations": { + "$id": "#/properties/tolerations", + "type": "array", + "default": [] + } + } +} \ No newline at end of file diff --git a/examples/chart/teleport-cluster/values.yaml b/examples/chart/teleport-cluster/values.yaml index 2e70ba957c4d6..9cdf598d72b73 100644 --- a/examples/chart/teleport-cluster/values.yaml +++ b/examples/chart/teleport-cluster/values.yaml @@ -3,25 +3,31 @@ ################################################## # clusterName is a unique cluster name. -# This value cannot be changed after your cluster starts. -# Use fully qualified domain name to access your cluster, +# This value cannot be changed after your cluster starts without rebuilding it from scratch. +# We recommend using the fully qualified domain name that you use to access your cluster, # for example: teleport.example.com -clusterName: +clusterName: "" ################################################## # Values that you may need to change. ################################################## -# Version of teleport image, if different from appVersion in Chart.yaml. +# Version of teleport image, if different from chart version in Chart.yaml. teleportVersionOverride: "" # ACME is a protocol for getting Web X.509 certificates -# acme enables acme protocol +# Note: ACME can only be used for single-instance clusters. It is not suitable for use in HA configurations. +# Setting acme to 'true' enables the ACME protocol and will attempt to get a free TLS certificate from Let's Encrypt. +# Setting acme to 'false' (the default) will cause Teleport to generate and use self-signed certificates for its web UI. acme: false +# acmeEmail is the email address to provide during certificate registration (this is a Let's Encrypt requirement) acmeEmail: "" +# acmeURI is the ACME server to use for getting certificates. The default is to use Let's Encrypt's production server. acmeURI: "" # Set enterprise to true to use enterprise image +# You will need to download your Enterprise license from the Teleport dashboard and create a secret to use this: +# kubectl -n ${TELEPORT_NAMESPACE?} create secret generic license --from-file=/path/to/downloaded/license.pem enterprise: false # If true, create & use Pod Security Policy resources @@ -29,25 +35,168 @@ enterprise: false podSecurityPolicy: enabled: true -# Set customConfig to true to use a custom config. Create -# a config resource, name it as release name and place it in the chart namespace. -customConfig: false - # Labels is a map of key-value pairs about this cluster labels: {} +# Mode to deploy the chart in. The default is "standalone". Options: +# - "standalone": will deploy a Teleport container running auth and proxy services with a PersistentVolumeClaim for storage. +# - "aws": will deploy a Teleport container running auth and proxy services using DynamoDB for backend/audit log storage and S3 for session recordings. (1) +# - "gcp": will deploy a Teleport container running auth and proxy services using Firestore for backend/audit log storage and Google Cloud storage for session recordings. (2) +# - "custom": will deploy a Teleport container using a teleport.yaml config file that you provide. (3) +# (1) To use "aws" mode, you must also configure the "aws" section below. +# (2) To use "gcp" mode, you must also configure the "gcp" section below. +# (3) When set to "custom", you must create a ConfigMap containing a 'teleport.yaml' key with an inline Teleport YAML config, +# give it the same name as the Helm release and place it in the chart namespace. +# kubectl -n ${TELEPORT_NAMESPACE?} create configmap ${HELM_RELEASE_NAME?} --from-file=teleport.yaml +chartMode: standalone + +################################################################ +# Standalone-specific settings (only used in "standalone" mode) +################################################################ +standalone: + # Leave blank to automatically create a PersistentVolumeClaim for Teleport storage. + # If you would like to use a pre-existing PersistentVolumeClaim, put its name here. + existingClaimName: "" + # Size of persistent volume to request when created by Teleport. + # Ignored if existingClaimName is provided. + volumeSize: 10Gi + +################################################## +# AWS-specific settings (only used in "aws" mode) +################################################## +aws: + # The AWS region where the DynamoDB tables are located. + region: "" + # The DynamoDB table name to use for backend storage. Teleport will attempt to create this table automatically if it does not exist. + # The container will need an appropriately-provisioned IAM role with permissions to create DynamoDB tables. + backendTable: "" + # The DynamoDB table name to use for audit log storage. Teleport will attempt to create this table automatically if it does not exist. + # The container will need an appropriately-provisioned IAM role with permissions to create DynamoDB tables. + # This MUST NOT be the same table name as used for 'backendTable' as the schemas are different. + auditLogTable: "" + # The S3 bucket name to use for recorded session storage. Teleport will attempt to create this bucket automatically if it does not exist. + # The container will need an appropriately-provisioned IAM role with permissions to create S3 buckets. + sessionRecordingBucket: "" + +################################################## +# GCP-specific settings (only used in "gcp" mode) +################################################## +gcp: + # The project name being used for the GCP account where Teleport is running. + # See https://support.google.com/googleapi/answer/7014113?hl=en + projectId: "" + # The Firestore collection to use for backend storage. Teleport will attempt to create this collection automatically if it does not exist. + # Either of the following must be true: + # - The container will need an appropriately-provisioned IAM role/service account with permissions to create Firestore collections + # - The service account credentials provided via 'credentialSecretName' will need permissions to create Firestore collections. + backendTable: "" + # The Firestore collection to use for audit log storage. Teleport will attempt to create this collection automatically if it does not exist. + # Either of the following must be true: + # - The container will need an appropriately-provisioned IAM role/service account with permissions to create Firestore collections + # - The service account credentials provided via 'credentialSecretName' will need permissions to create Firestore collections. + # This MUST NOT be the same collection name as used for 'backendTable' as the schemas are different. + auditLogTable: "" + # The Google storage bucket name to use for recorded session storage. This bucket must already exist in the Google account being used. + sessionRecordingBucket: "" + # The name of the Kubernetes secret used to store the Google credentials. + # You will need to create this secret manually. It must contain a JSON file from Google with the credentials that Teleport will use. + # You can override this to a blank value if the worker node running Teleport already has a service account which grants access. + credentialSecretName: teleport-gcp-credentials + +# Settings for high availability. These are not used in "standalone" mode. +# When using "custom" mode, you must use highly-available storage (etcd, DynamoDB or Firestore) for multiple replicas to be supported. +# Manually configuring NFS-based storage or ReadWriteMany volume claims is NOT supported and will result in errors. +highAvailability: + # Set to >1 for a high availability mode where multiple Teleport pods will be deployed and connections will be load balanced between them. + # Note: this will disable the use of ACME certs. + replicaCount: 1 + # Setting 'requireAntiAffinity' to true will use 'requiredDuringSchedulingIgnoredDuringExecution' to require that multiple Teleport pods must not be scheduled on the + # same physical host. This will result in Teleport pods failing to be scheduled in very small clusters or during node downtime, so should be used with caution. + # Setting 'requireAntiAffinity' to false (the default) uses 'preferredDuringSchedulingIgnoredDuringExecution' to make this a soft requirement. + # This setting only has any effect when replicaCount is greater than 1. + requireAntiAffinity: false + # Settings for cert-manager (can be used for provisioning TLS certs in HA mode) + certManager: + # If set to true, use cert-manager to get certificates for Teleport to use for TLS termination + enabled: false + # Name of the Issuer/ClusterIssuer to use for certs + # NOTE: You will always need to create this yourself when certManager.enabled is true. + issuerName: "" + # Kind of Issuer that cert-manager should look for. + # This defaults to 'Issuer' to keep everything contained within the teleport namespace. + issuerKind: Issuer + + ################################################## # Values that you shouldn't need to change. ################################################## -# Container image for the agent. +# Container image for the cluster. image: quay.io/gravitational/teleport # Enterprise version of the image enterpriseImage: quay.io/gravitational/teleport-ent -# Number of replicas for the agent deployment. -replicaCount: 1 # Log level for the Teleport process. # Available log levels are: DEBUG, INFO, WARNING, ERROR. # The default is INFO, which is recommended in production. # DEBUG is useful during first-time setup or to see more detailed logs for debugging. logLevel: INFO + + +################################## +# Extra Kubernetes configuration # +################################## + +# Affinity for pod assignment +# https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +# NOTE: If affinity is set here, highAvailability.requireAntiAffinity cannot also be used - you can only set one or the other. +affinity: {} + +# Kubernetes annotations to apply +# https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +annotations: + # Annotations for the ConfigMap (note: these will not be applied in 'custom' mode) + config: {} + # Annotations for the Deployment + deployment: {} + # Annotations for each Pod in the Deployment + pod: {} + # Annotations for the Service object + service: {} + # Annotations for the ServiceAccount object + serviceAccount: {} + +# Extra arguments to pass to 'teleport start' for the main Teleport pod +extraArgs: [] + +# Extra volumes to mount into the Teleport pods +# https://kubernetes.io/docs/concepts/storage/volumes/ +extraVolumes: [] +#- name: myvolume +# secret: +# secretName: testSecret + +# Extra volume mounts corresponding to the volumes mounted above +extraVolumeMounts: [] +#- name: myvolume +# path: /path/on/host + +# Allow the imagePullPolicy to be overridden +imagePullPolicy: IfNotPresent + +# A list of initContainers to run before each Teleport pod starts +# https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +initContainers: [] +#- name: "teleport-init" +# image: "alpine" +# args: ["echo test"] + +# Resources to request for each pod in the deployment +# https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +resources: {} +# requests: +# cpu: "1" +# memory: "2Gi" + +# Tolerations for pod assignment +# https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] diff --git a/examples/chart/teleport-daemonset/Chart.yaml b/examples/chart/teleport-daemonset/Chart.yaml index 1ea965f2d9015..6bd55ee4bc07b 100644 --- a/examples/chart/teleport-daemonset/Chart.yaml +++ b/examples/chart/teleport-daemonset/Chart.yaml @@ -1,8 +1,9 @@ name: teleport-daemonset apiVersion: v2 -version: 0.0.8 +version: 0.0.9 appVersion: "6" -description: Teleport daemonset installation which provides some access to underlying Kubernetes nodes. +description: "[deprecated] Teleport daemonset installation which provides some access to underlying Kubernetes nodes." icon: https://goteleport.com/images/logos/logo-teleport-square.svg keywords: - Teleport Enterprise +deprecated: true diff --git a/examples/chart/teleport-daemonset/templates/service.yaml b/examples/chart/teleport-daemonset/templates/service.yaml index 8b8f4e5295350..de1f5075237fe 100644 --- a/examples/chart/teleport-daemonset/templates/service.yaml +++ b/examples/chart/teleport-daemonset/templates/service.yaml @@ -15,8 +15,8 @@ spec: type: {{ .Values.service.type }} ports: {{- range $key, $value := .Values.service.ports }} - - name: {{ $key }} -{{ toYaml $value | indent 6 }} + - name: {{ $key }} +{{ toYaml $value | indent 4 }} {{- end }} {{- if and (semverCompare ">=1.7-0" .Capabilities.KubeVersion.GitVersion) (.Values.service.externalTrafficPolicy) }} externalTrafficPolicy: "{{ .Values.service.externalTrafficPolicy }}" @@ -41,8 +41,8 @@ metadata: spec: type: NodePort ports: - - port: {{ .Values.service.ports.authssh.port }} - nodePort: 32025 + - port: {{ .Values.service.ports.authssh.port }} + nodePort: 32025 selector: app: {{ template "teleport.name" . }} release: {{ .Release.Name }} diff --git a/examples/chart/teleport-kube-agent/.lint/affinity.yaml b/examples/chart/teleport-kube-agent/.lint/affinity.yaml new file mode 100644 index 0000000000000..a961974768b22 --- /dev/null +++ b/examples/chart/teleport-kube-agent/.lint/affinity.yaml @@ -0,0 +1,24 @@ +authToken: auth-token +proxyAddr: proxy.example.com:3080 +roles: kube +kubeClusterName: test-kube-cluster +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: gravitational.io/dedicated + operator: In + values: + - teleport + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - teleport + topologyKey: kubernetes.io/hostname + weight: 1 diff --git a/examples/chart/teleport-kube-agent/.lint/lint-values-all-v5.yaml b/examples/chart/teleport-kube-agent/.lint/all-v5.yaml similarity index 100% rename from examples/chart/teleport-kube-agent/.lint/lint-values-all-v5.yaml rename to examples/chart/teleport-kube-agent/.lint/all-v5.yaml diff --git a/examples/chart/teleport-kube-agent/.lint/lint-values-all-v6.yaml b/examples/chart/teleport-kube-agent/.lint/all-v6.yaml similarity index 57% rename from examples/chart/teleport-kube-agent/.lint/lint-values-all-v6.yaml rename to examples/chart/teleport-kube-agent/.lint/all-v6.yaml index ffae292ba6066..7b8f28bce3cc0 100644 --- a/examples/chart/teleport-kube-agent/.lint/lint-values-all-v6.yaml +++ b/examples/chart/teleport-kube-agent/.lint/all-v6.yaml @@ -15,3 +15,13 @@ databases: protocol: "postgres" labels: database: staging +annotations: + config: + kubernetes.io/config: "test-annotation" + kubernetes.io/config-different: 2 + deployment: + kubernetes.io/deployment: "test-annotation" + kubernetes.io/deployment-different: 3 + pod: + kubernetes.io/pod: "test-annotation" + kubernetes.io/pod-different: 4 diff --git a/examples/chart/teleport-kube-agent/.lint/annotations.yaml b/examples/chart/teleport-kube-agent/.lint/annotations.yaml new file mode 100644 index 0000000000000..7e68a0d2f09ed --- /dev/null +++ b/examples/chart/teleport-kube-agent/.lint/annotations.yaml @@ -0,0 +1,14 @@ +authToken: auth-token +proxyAddr: proxy.example.com:3080 +roles: kube +kubeClusterName: test-kube-cluster +annotations: + config: + kubernetes.io/config: "test-annotation" + kubernetes.io/config-different: 2 + deployment: + kubernetes.io/deployment: "test-annotation" + kubernetes.io/deployment-different: 3 + pod: + kubernetes.io/pod: "test-annotation" + kubernetes.io/pod-different: 4 diff --git a/examples/chart/teleport-kube-agent/.lint/lint-values-backwards-compatibility.yaml b/examples/chart/teleport-kube-agent/.lint/backwards-compatibility.yaml similarity index 100% rename from examples/chart/teleport-kube-agent/.lint/lint-values-backwards-compatibility.yaml rename to examples/chart/teleport-kube-agent/.lint/backwards-compatibility.yaml diff --git a/examples/chart/teleport-kube-agent/.lint/lint-values-db.yaml b/examples/chart/teleport-kube-agent/.lint/db.yaml similarity index 100% rename from examples/chart/teleport-kube-agent/.lint/lint-values-db.yaml rename to examples/chart/teleport-kube-agent/.lint/db.yaml diff --git a/examples/chart/teleport-kube-agent/.lint/initcontainers.yaml b/examples/chart/teleport-kube-agent/.lint/initcontainers.yaml new file mode 100644 index 0000000000000..a8d7a2af74f49 --- /dev/null +++ b/examples/chart/teleport-kube-agent/.lint/initcontainers.yaml @@ -0,0 +1,17 @@ +authToken: auth-token +proxyAddr: proxy.example.com:3080 +roles: kube +kubeClusterName: test-kube-cluster +initContainers: +- name: "teleport-init" + image: "alpine" + args: ["echo test"] +# These are just sample values to test the chart. +# They are not intended to be guidelines or suggestions for running teleport. +resources: + limits: + cpu: 2 + memory: 4Gi + requests: + cpu: 1 + memory: 2Gi diff --git a/examples/chart/teleport-kube-agent/.lint/resources.yaml b/examples/chart/teleport-kube-agent/.lint/resources.yaml new file mode 100644 index 0000000000000..bd0ccf428486d --- /dev/null +++ b/examples/chart/teleport-kube-agent/.lint/resources.yaml @@ -0,0 +1,13 @@ +authToken: auth-token +proxyAddr: proxy.example.com:3080 +roles: kube +kubeClusterName: test-kube-cluster +# These are just sample values to test the chart. +# They are not intended to be guidelines or suggestions for running teleport. +resources: + limits: + cpu: 2 + memory: 4Gi + requests: + cpu: 1 + memory: 2Gi diff --git a/examples/chart/teleport-kube-agent/.lint/tolerations.yaml b/examples/chart/teleport-kube-agent/.lint/tolerations.yaml new file mode 100644 index 0000000000000..87abf137c5ff3 --- /dev/null +++ b/examples/chart/teleport-kube-agent/.lint/tolerations.yaml @@ -0,0 +1,13 @@ +authToken: auth-token +proxyAddr: proxy.example.com:3080 +roles: kube +kubeClusterName: test-kube-cluster +tolerations: +- key: "dedicated" + operator: "Equal" + value: "teleport" + effect: "NoExecute" +- key: "dedicated" + operator: "Equal" + value: "teleport" + effect: "NoSchedule" diff --git a/examples/chart/teleport-kube-agent/.lint/volumes.yaml b/examples/chart/teleport-kube-agent/.lint/volumes.yaml new file mode 100644 index 0000000000000..14bdc49390f50 --- /dev/null +++ b/examples/chart/teleport-kube-agent/.lint/volumes.yaml @@ -0,0 +1,11 @@ +authToken: auth-token +proxyAddr: proxy.example.com:3080 +roles: kube +kubeClusterName: test-kube-cluster +extraVolumeMounts: +- name: "my-mount" + path: "/path/to/mount" +extraVolumes: +- name: "my-mount" + secret: + secretName: "mySecret" diff --git a/examples/chart/teleport-kube-agent/Chart.yaml b/examples/chart/teleport-kube-agent/Chart.yaml index 09c3206602212..7c4f440dafae9 100644 --- a/examples/chart/teleport-kube-agent/Chart.yaml +++ b/examples/chart/teleport-kube-agent/Chart.yaml @@ -1,6 +1,6 @@ name: teleport-kube-agent apiVersion: v2 -version: 0.0.4 +version: "6" appVersion: "6" description: Teleport provides a secure SSH and Kubernetes remote access solution that doesn't get in the way. icon: https://goteleport.com/images/logos/logo-teleport-square.svg diff --git a/examples/chart/teleport-kube-agent/templates/config.yaml b/examples/chart/teleport-kube-agent/templates/config.yaml index 277d6e03defaf..0bfdfb556c830 100644 --- a/examples/chart/teleport-kube-agent/templates/config.yaml +++ b/examples/chart/teleport-kube-agent/templates/config.yaml @@ -2,6 +2,10 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ .Release.Name }} + {{- if .Values.annotations.config }} + annotations: + {{- toYaml .Values.annotations.config | nindent 4 }} + {{- end }} data: teleport.yaml: | teleport: diff --git a/examples/chart/teleport-kube-agent/templates/deployment.yaml b/examples/chart/teleport-kube-agent/templates/deployment.yaml index 8dd2ce5e0eba3..06d68bb0d22a5 100644 --- a/examples/chart/teleport-kube-agent/templates/deployment.yaml +++ b/examples/chart/teleport-kube-agent/templates/deployment.yaml @@ -1,7 +1,7 @@ {{- if .Values.teleportVersionOverride }} {{- $_ := set . "teleportVersion" .Values.teleportVersionOverride }} {{- else }} - {{- $_ := set . "teleportVersion" .Chart.AppVersion }} + {{- $_ := set . "teleportVersion" .Chart.Version }} {{- end }} apiVersion: apps/v1 kind: Deployment @@ -10,6 +10,10 @@ metadata: namespace: {{ .Release.Namespace }} labels: app: {{ .Release.Name }} + {{- if .Values.annotations.deployment }} + annotations: + {{- toYaml .Values.annotations.deployment | nindent 4 }} + {{- end }} spec: replicas: {{ .Values.replicaCount }} selector: @@ -20,9 +24,47 @@ spec: annotations: # ConfigMap checksum, to recreate the pod on config changes. checksum/config: {{ include (print $.Template.BasePath "/config.yaml") . | sha256sum }} +{{- if .Values.annotations.pod }} + {{- toYaml .Values.annotations.pod | nindent 8 }} +{{- end }} labels: app: {{ .Release.Name }} spec: + {{- if .Values.affinity }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: + {{- toYaml .Values.tolerations | nindent 6 }} + {{- end }} +{{- if .Values.initContainers }} + initContainers: {{- toYaml .Values.initContainers | nindent 6 }} + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 9807 + volumeMounts: + - mountPath: /etc/teleport + name: "config" + readOnly: true + - mountPath: /etc/teleport-secrets + name: "auth-token" + readOnly: true + - mountPath: /var/lib/teleport + name: "data" + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} +{{- end }} containers: - name: "teleport" image: "{{ if .Values.enterprise }}{{ .Values.enterpriseImage }}{{ else }}{{ .Values.image }}{{ end }}:{{ .teleportVersion }}" @@ -57,6 +99,10 @@ spec: initialDelaySeconds: 5 # wait 5s for agent to register periodSeconds: 5 # poll health every 5s failureThreshold: 12 # consider agent unhealthy after 60s (12 * 5s) +{{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} +{{- end }} volumeMounts: - mountPath: /etc/teleport name: "config" @@ -66,6 +112,9 @@ spec: readOnly: true - mountPath: /var/lib/teleport name: "data" +{{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 8 }} +{{- end }} volumes: - name: "config" configMap: @@ -75,4 +124,7 @@ spec: secretName: {{ .Values.secretName }} - name: "data" emptyDir: {} +{{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 6 }} +{{- end }} serviceAccountName: {{ .Values.serviceAccountName }} diff --git a/examples/chart/teleport-kube-agent/values.schema.json b/examples/chart/teleport-kube-agent/values.schema.json new file mode 100644 index 0000000000000..699177719637c --- /dev/null +++ b/examples/chart/teleport-kube-agent/values.schema.json @@ -0,0 +1,207 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "required": [ + "authToken", + "proxyAddr", + "roles", + "kubeClusterName", + "apps", + "databases", + "teleportVersionOverride", + "insecureSkipProxyTLSVerify", + "podSecurityPolicy", + "labels", + "image", + "replicaCount", + "clusterRoleName", + "clusterRoleBindingName", + "serviceAccountName", + "secretName", + "logLevel", + "affinity", + "annotations", + "extraVolumes", + "extraVolumeMounts", + "imagePullPolicy", + "initContainers", + "resources", + "tolerations" + ], + "properties": { + "authToken": { + "$id": "#/properties/authToken", + "type": "string", + "default": "" + }, + "proxyAddr": { + "$id": "#/properties/proxyAddr", + "type": "string", + "default": "" + }, + "roles": { + "$id": "#/properties/roles", + "type": "string", + "default": "kube" + }, + "kubeClusterName": { + "$id": "#/properties/kubeClusterName", + "type": "string", + "default": "" + }, + "apps": { + "$id": "#/properties/apps", + "type": "array", + "default": [], + "required": [ + "name", + "uri" + ], + "properties": { + "name": { + "$id": "#/properties/apps/name", + "type": "string", + "default": "" + }, + "uri": { + "$id": "#/properties/apps/uri", + "type": "string", + "default": "" + }, + "additionalProperties": true + } + }, + "databases": { + "$id": "#/properties/databases", + "type": "array", + "default": [] + }, + "teleportVersionOverride": { + "$id": "#/properties/teleportVersionOverride", + "type": "string", + "default": "" + }, + "insecureSkipProxyTLSVerify": { + "$id": "#/properties/insecureSkipProxyTLSVerify", + "type": "boolean", + "default": false + }, + "podSecurityPolicy": { + "$id": "#/properties/podSecurityPolicy", + "type": "object", + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "$id": "#/properties/podSecurityPolicy/properties/enabled", + "type": "boolean", + "default": true + } + } + }, + "labels": { + "$id": "#/properties/labels", + "type": "object", + "default": {} + }, + "image": { + "$id": "#/properties/image", + "type": "string", + "default": "quay.io/gravitational/teleport" + }, + "replicaCount": { + "$id": "#/properties/replicaCount", + "type": "integer", + "default": 1 + }, + "clusterRoleName": { + "$id": "#/properties/clusterRoleName", + "type": "string", + "default": "teleport-kube-agent" + }, + "clusterRoleBindingName": { + "$id": "#/properties/clusterRoleBindingName", + "type": "string", + "default": "teleport-kube-agent" + }, + "serviceAccountName": { + "$id": "#/properties/serviceAccountName", + "type": "string", + "default": "teleport-kube-agent" + }, + "secretName": { + "$id": "#/properties/secretName", + "type": "string", + "default": "teleport-kube-agent-join-token" + }, + "logLevel": { + "$id": "#/properties/logLevel", + "type": "string", + "enum": ["DEBUG", "INFO", "WARN", "WARNING", "ERROR"], + "default": "INFO" + }, + "affinity": { + "$id": "#/properties/affinity", + "type": "object", + "default": {} + }, + "annotations": { + "$id": "#/properties/annotations", + "type": "object", + "required": [ + "config", + "deployment", + "pod" + ], + "properties": { + "config": { + "$id": "#/properties/annotations/properties/config", + "type": "object", + "default": {} + }, + "deployment": { + "$id": "#/properties/annotations/properties/deployment", + "type": "object", + "default": {} + }, + "pod": { + "$id": "#/properties/annotations/properties/pod", + "type": "object", + "default": {} + } + } + }, + "extraVolumes": { + "$id": "#/properties/extraVolumes", + "type": "array", + "default": [] + }, + "extraVolumeMounts": { + "$id": "#/properties/extraVolumeMounts", + "type": "array", + "default": [] + }, + "imagePullPolicy": { + "$id": "#/properties/imagePullPolicy", + "type": "string", + "enum": ["Never", "IfNotPresent", "Always"], + "default": "IfNotPresent" + }, + "initContainers": { + "$id": "#/properties/initContainers", + "type": "array", + "default": [] + }, + "resources": { + "$id": "#/properties/resources", + "type": "object", + "default": {} + }, + "tolerations": { + "$id": "#/properties/tolerations", + "type": "array", + "default": [] + } + } +} \ No newline at end of file diff --git a/examples/chart/teleport-kube-agent/values.yaml b/examples/chart/teleport-kube-agent/values.yaml index 888f8f7cad2e5..9ed815461d53a 100644 --- a/examples/chart/teleport-kube-agent/values.yaml +++ b/examples/chart/teleport-kube-agent/values.yaml @@ -5,18 +5,18 @@ # Join token for the cluster. # Leave empty if the secret "teleport-kube-agent-join-token" # has been created before and contains a valid join token -authToken: +authToken: "" # Address of the teleport proxy with port (usually :3080). -proxyAddr: +proxyAddr: "" # Comma-separated list of roles to enable (any of: kube,db,app) -roles: +roles: "kube" ################################################################ # Values that must be provided if Kubernetes access is enabled. ################################################################ # Name for this kubernetes cluster to be used by teleport users. -kubeClusterName: +kubeClusterName: "" ################################################################ # Values that must be provided if Application access is enabled. @@ -26,7 +26,7 @@ kubeClusterName: # apps: # - name: grafana # uri: http://localhost:3000 -apps: +apps: [] ################################################################ # Values that must be provided if Database access is enabled. @@ -37,7 +37,7 @@ apps: # - name: aurora # uri: "postgres-aurora-instance-1.xxx.us-east-1.rds.amazonaws.com:5432" # protocol: "postgres" -databases: +databases: [] ################################################################ # Values that you may need to change. @@ -79,3 +79,54 @@ secretName: teleport-kube-agent-join-token # The default is INFO, which is recommended in production. # DEBUG is useful during first-time setup or to see more detailed logs for debugging. logLevel: INFO + +################################## +# Extra Kubernetes configuration # +################################## + +# Affinity for pod assignment +# https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +affinity: {} + +# Kubernetes annotations to apply +# https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +annotations: + # Annotations for the ConfigMap + config: {} + # Annotations for the Deployment + deployment: {} + # Annotations for each Pod in the Deployment + pod: {} + +# Extra volumes to mount into the Teleport pods +# https://kubernetes.io/docs/concepts/storage/volumes/ +extraVolumes: [] +#- name: myvolume +# secret: +# secretName: testSecret + +# Extra volume mounts corresponding to the volumes mounted above +extraVolumeMounts: [] +#- name: myvolume +# path: /path/on/host + +# Allow the imagePullPolicy to be overridden +imagePullPolicy: IfNotPresent + +# A list of initContainers to run before each Teleport pod starts +# https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +initContainers: [] +#- name: "teleport-init" +# image: "alpine" +# args: ["echo test"] + +# Resources to request for each pod in the deployment +# https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +resources: {} +# requests: +# cpu: "1" +# memory: "2Gi" + +# Tolerations for pod assignment +# https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] diff --git a/examples/chart/teleport/Chart.yaml b/examples/chart/teleport/Chart.yaml index 790f72d18ab9b..dbf50a3f11f57 100644 --- a/examples/chart/teleport/Chart.yaml +++ b/examples/chart/teleport/Chart.yaml @@ -1,8 +1,9 @@ name: teleport apiVersion: v2 -version: 0.0.12 +version: 0.0.13 appVersion: "6" -description: Teleport provides a secure SSH and Kubernetes remote access solution that doesn't get in the way. +description: "[This chart is deprecated, use teleport-cluster instead]" icon: https://goteleport.com/images/logos/logo-teleport-square.svg keywords: - Teleport +deprecated: true diff --git a/examples/chart/teleport/templates/service.yaml b/examples/chart/teleport/templates/service.yaml index 93189b058aac1..a616a0d8d5356 100644 --- a/examples/chart/teleport/templates/service.yaml +++ b/examples/chart/teleport/templates/service.yaml @@ -19,10 +19,10 @@ spec: {{- end }} ports: {{- range $key, $value := .Values.service.ports }} -{{ if or (not $.Values.config.highAvailability) (and ($.Values.config.highAvailability) (not (eq $key "authssh"))) }} - - name: {{ $key }} -{{ toYaml $value | indent 6 }} - {{ end }} +{{- if or (not $.Values.config.highAvailability) (and ($.Values.config.highAvailability) (not (eq $key "authssh"))) }} + - name: {{ $key }} +{{ toYaml $value | indent 4 }} +{{- end }} {{- end }} {{- if and (semverCompare ">=1.7-0" .Capabilities.KubeVersion.GitVersion) (.Values.service.externalTrafficPolicy) }} externalTrafficPolicy: "{{ .Values.service.externalTrafficPolicy }}" diff --git a/examples/chart/teleport/templates/serviceaccount.yaml b/examples/chart/teleport/templates/serviceaccount.yaml index fe5f4c394ff48..8468e9211b9eb 100644 --- a/examples/chart/teleport/templates/serviceaccount.yaml +++ b/examples/chart/teleport/templates/serviceaccount.yaml @@ -3,7 +3,8 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ template "teleport.serviceAccountName" . }} - labels: {{ include "teleport.labels" . | nindent 4 }} + labels: + {{- include "teleport.labels" . | nindent 4 }} {{- if .Values.serviceAccount.annotations }} annotations: {{ toYaml .Values.serviceAccount.annotations | nindent 4 }} {{- end }} diff --git a/version.mk b/version.mk index 7d2f5d8ab5e85..09741da6f05c3 100644 --- a/version.mk +++ b/version.mk @@ -17,6 +17,23 @@ func init() { Gitref = \"$(GITREF)\"} " # setver updates version.go and gitref.go with VERSION and GITREF vars # .PHONY:setver -setver: +setver: helm-version @printf $(VERSION_GO) | gofmt > version.go @printf $(GITREF_GO) | gofmt > gitref.go + +# helm-version automatically updates the versions of Helm charts to match the version set in the Makefile, +# so that chart versions are also kept in sync when the Teleport version is updated for a release. +# If the version contains '-dev' (as it does on the master branch, or for development builds) then we get the latest +# published major version number by parsing a sorted list of git tags instead, to make deploying the chart from master +# work as expected. Version numbers are quoted as a string because Helm otherwise treats dotted decimals as floats. +.PHONY:helm-version +helm-version: + @if ! echo "$${VERSION}" | grep -q "\-dev"; then \ + export CHART_VERSION=$${VERSION}; \ + else \ + export CHART_VERSION=$$(git ls-remote --tags https://github.com/gravitational/teleport | cut -d'/' -f3 | grep -Ev '(alpha|beta|dev|rc)' | sort -rV | head -n1 | cut -d. -f1 | tr -d '^v'); \ + fi; \ + for CHART in teleport-cluster teleport-kube-agent; do \ + sed -i "s_^version:\ .*_version: \"$${CHART_VERSION}\"_g" examples/chart/$${CHART}/Chart.yaml || exit 1; \ + sed -i "s_^appVersion:\ .*_appVersion: \"$${CHART_VERSION}\"_g" examples/chart/$${CHART}/Chart.yaml || exit 1; \ + done \ No newline at end of file