diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index 922f8189c1e4..9ea1dcac0ac1 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -59,7 +59,7 @@ jobs: name: E2E Tests runs-on: ubuntu-latest timeout-minutes: 25 - needs: [ tests, argoexec-image ] + needs: [ argoexec-image ] env: KUBECONFIG: /home/runner/.kubeconfig strategy: @@ -86,7 +86,19 @@ jobs: profile: minimal - test: test-python-sdk profile: minimal + - test: test-executor + install_k3s_version: v1.21.2+k3s1 + profile: minimal + - test: test-corefunctional + install_k3s_version: v1.21.2+k3s1 + profile: minimal + - test: test-functional + install_k3s_version: v1.21.2+k3s1 + profile: minimal steps: + - name: Install socat + # needed by Kubernetes v1.25 + run: sudo apt-get -y install socat - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: @@ -105,7 +117,7 @@ jobs: cache: pip - name: Install and start K3S run: | - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.21.2+k3s1 INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC=--docker K3S_KUBECONFIG_MODE=644 sh - + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${{matrix.install_k3s_version}} INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC=--docker K3S_KUBECONFIG_MODE=644 sh - until kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml cluster-info ; do sleep 10s ; done cp /etc/rancher/k3s/k3s.yaml /home/runner/.kubeconfig echo "- name: fake_token_user" >> $KUBECONFIG @@ -133,11 +145,13 @@ jobs: - run: make cli STATIC_FILES=false if: ${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} name: Build CLI + - run: ./hack/port-forward.sh + name: Start port forward - run: make start PROFILE=${{matrix.profile}} AUTH_MODE=client STATIC_FILES=false LOG_LEVEL=info API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} UI=false LOGS=false > /tmp/argo.log 2>&1 & name: Start controller/API - run: make wait timeout-minutes: 4 - name: Wait for MinIO/MySQL etc to be ready + name: Wait for controller to be up - name: Run tests ${{matrix.test}} # https://github.com/marketplace/actions/retry-step uses: nick-fields/retry@v2.8.2 @@ -145,11 +159,40 @@ jobs: timeout_minutes: 20 max_attempts: 2 command: make ${{matrix.test}} E2E_SUITE_TIMEOUT=20m STATIC_FILES=false + - if: ${{ failure() }} + name: MinIO/MySQL deployment + run: | + set -eux + kubectl get deploy + kubectl describe deploy + - if: ${{ failure() }} + name: MinIO/MySQL pods + run: | + set -eux + kubectl get pods -l '!workflows.argoproj.io/workflow' + kubectl describe pods -l '!workflows.argoproj.io/workflow' + - if: ${{ failure() }} + name: MinIO/MySQL logs + run: kubectl logs -l '!workflows.argoproj.io/workflow' --prefix - if: ${{ failure() }} name: Controller/API logs run: | [ -e /tmp/argo.log ] && cat /tmp/argo.log - + - if: ${{ failure() }} + name: Workflows + run: | + set -eux + kubectl get wf + kubectl describe wf + - if: ${{ failure() }} + name: Workflow pods + run: | + set -eux + kubectl get pods -l workflows.argoproj.io/workflow + kubectl describe pods -l workflows.argoproj.io/workflow + - if: ${{ failure() }} + name: Wait container logs + run: kubectl logs -c wait -l workflows.argoproj.io/workflow --prefix codegen: name: Codegen runs-on: ubuntu-latest diff --git a/.spelling b/.spelling index e0278299d787..4e90c7203423 100644 --- a/.spelling +++ b/.spelling @@ -183,6 +183,7 @@ v1.0 v1.1 v1.2 v1.3 +v1.24 v2 v2.10 v2.11 diff --git a/docs/access-token.md b/docs/access-token.md index 63ecb459d823..b199509f86a5 100644 --- a/docs/access-token.md +++ b/docs/access-token.md @@ -33,11 +33,24 @@ kubectl create rolebinding jenkins --role=jenkins --serviceaccount=argo:jenkins ## Token Creation -You now need to get a token: +You now need to create a secret to hold your token: ```bash -SECRET=$(kubectl get sa jenkins -o=jsonpath='{.secrets[0].name}') -ARGO_TOKEN="Bearer $(kubectl get secret $SECRET -o=jsonpath='{.data.token}' | base64 --decode)" + kubectl apply -f - < /dev/null - - echo "$ARGO_TOKEN" + while true; do + TOKEN=$(kubectl get secret jenkins.service-account-token -o=jsonpath='{.data.token}' | base64 --decode) + if [ "$TOKEN" != "" ]; then + echo "Bearer $TOKEN" + exit + fi + sleep 1 + done ;; *) exit 1 diff --git a/hack/free-port.sh b/hack/free-port.sh index b94962d221a6..73e49e93dfad 100755 --- a/hack/free-port.sh +++ b/hack/free-port.sh @@ -1,6 +1,10 @@ -#!/bin/sh -set -eu +#!/usr/bin/env bash +set -eu -o pipefail port=$1 -lsof -s TCP:LISTEN -i ":$port" | grep -v PID | awk '{print $2}' | xargs -L 1 kill || true +pids=$(lsof -t -s TCP:LISTEN -i ":$port" || true) + +if [ "$pids" != "" ]; then + kill $pids +fi diff --git a/hack/port-forward.sh b/hack/port-forward.sh index a1c9479603b2..6a8bb086840b 100755 --- a/hack/port-forward.sh +++ b/hack/port-forward.sh @@ -8,7 +8,7 @@ pf() { dest_port=${3:-"$port"} ./hack/free-port.sh $port echo "port-forward $resource $port" - kubectl -n argo port-forward "svc/$resource" "$port:$dest_port" > /dev/null & + kubectl -n argo port-forward "svc/$resource" "$port:$dest_port" & until lsof -i ":$port" > /dev/null ; do sleep 1 ; done } diff --git a/manifests/quick-start-minimal.yaml b/manifests/quick-start-minimal.yaml index 35a816df2fdc..521123a193ab 100644 --- a/manifests/quick-start-minimal.yaml +++ b/manifests/quick-start-minimal.yaml @@ -1461,6 +1461,22 @@ stringData: --- apiVersion: v1 kind: Secret +metadata: + annotations: + kubernetes.io/service-account.name: default + name: default.service-account-token +type: kubernetes.io/service-account-token +--- +apiVersion: v1 +kind: Secret +metadata: + annotations: + kubernetes.io/service-account.name: github.com + name: github.com.service-account-token +type: kubernetes.io/service-account-token +--- +apiVersion: v1 +kind: Secret metadata: labels: app: httpbin @@ -1685,6 +1701,7 @@ spec: labels: app: httpbin spec: + automountServiceAccountToken: false containers: - image: kennethreitz/httpbin livenessProbe: @@ -1719,6 +1736,7 @@ spec: labels: app: minio spec: + automountServiceAccountToken: false containers: - command: - minio diff --git a/manifests/quick-start-mysql.yaml b/manifests/quick-start-mysql.yaml index 652a1257209e..98c3126878b2 100644 --- a/manifests/quick-start-mysql.yaml +++ b/manifests/quick-start-mysql.yaml @@ -1491,6 +1491,22 @@ stringData: --- apiVersion: v1 kind: Secret +metadata: + annotations: + kubernetes.io/service-account.name: default + name: default.service-account-token +type: kubernetes.io/service-account-token +--- +apiVersion: v1 +kind: Secret +metadata: + annotations: + kubernetes.io/service-account.name: github.com + name: github.com.service-account-token +type: kubernetes.io/service-account-token +--- +apiVersion: v1 +kind: Secret metadata: labels: app: httpbin @@ -1729,6 +1745,7 @@ spec: labels: app: httpbin spec: + automountServiceAccountToken: false containers: - image: kennethreitz/httpbin livenessProbe: @@ -1763,6 +1780,7 @@ spec: labels: app: minio spec: + automountServiceAccountToken: false containers: - command: - minio @@ -1818,6 +1836,7 @@ spec: app: mysql name: mysql spec: + automountServiceAccountToken: false containers: - env: - name: MYSQL_USER @@ -1833,17 +1852,10 @@ spec: ports: - containerPort: 3306 readinessProbe: - exec: - command: - - mysql - - -u - - mysql - - -ppassword - - argo - - -e - - SELECT 1 - initialDelaySeconds: 15 - timeoutSeconds: 2 + initialDelaySeconds: 30 + periodSeconds: 10 + tcpSocket: + port: 3306 nodeSelector: kubernetes.io/os: linux --- diff --git a/manifests/quick-start-postgres.yaml b/manifests/quick-start-postgres.yaml index f3172ff9b851..b1e88792200e 100644 --- a/manifests/quick-start-postgres.yaml +++ b/manifests/quick-start-postgres.yaml @@ -1491,6 +1491,22 @@ stringData: --- apiVersion: v1 kind: Secret +metadata: + annotations: + kubernetes.io/service-account.name: default + name: default.service-account-token +type: kubernetes.io/service-account-token +--- +apiVersion: v1 +kind: Secret +metadata: + annotations: + kubernetes.io/service-account.name: github.com + name: github.com.service-account-token +type: kubernetes.io/service-account-token +--- +apiVersion: v1 +kind: Secret metadata: labels: app: httpbin @@ -1729,6 +1745,7 @@ spec: labels: app: httpbin spec: + automountServiceAccountToken: false containers: - image: kennethreitz/httpbin livenessProbe: @@ -1763,6 +1780,7 @@ spec: labels: app: minio spec: + automountServiceAccountToken: false containers: - command: - minio diff --git a/manifests/quick-start/base/default.service-account-token-secret.yaml b/manifests/quick-start/base/default.service-account-token-secret.yaml new file mode 100644 index 000000000000..86a6e71ee798 --- /dev/null +++ b/manifests/quick-start/base/default.service-account-token-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: default.service-account-token + annotations: + kubernetes.io/service-account.name: default +type: kubernetes.io/service-account-token \ No newline at end of file diff --git a/manifests/quick-start/base/httpbin/httpbin-deploy.yaml b/manifests/quick-start/base/httpbin/httpbin-deploy.yaml index 86ac25bff648..222a86c04eae 100644 --- a/manifests/quick-start/base/httpbin/httpbin-deploy.yaml +++ b/manifests/quick-start/base/httpbin/httpbin-deploy.yaml @@ -13,6 +13,7 @@ spec: labels: app: httpbin spec: + automountServiceAccountToken: false containers: - name: main image: kennethreitz/httpbin diff --git a/manifests/quick-start/base/kustomization.yaml b/manifests/quick-start/base/kustomization.yaml index 325f94d4b12e..803d90c11e1a 100644 --- a/manifests/quick-start/base/kustomization.yaml +++ b/manifests/quick-start/base/kustomization.yaml @@ -6,6 +6,7 @@ resources: - minio - httpbin - webhooks + - default.service-account-token-secret.yaml - argo-server-sso-secret.yaml - executor/emissary/executor-role.yaml - executor-default-rolebinding.yaml diff --git a/manifests/quick-start/base/minio/minio-deploy.yaml b/manifests/quick-start/base/minio/minio-deploy.yaml index 849522b4550c..7a3e8550898a 100644 --- a/manifests/quick-start/base/minio/minio-deploy.yaml +++ b/manifests/quick-start/base/minio/minio-deploy.yaml @@ -13,6 +13,7 @@ spec: labels: app: minio spec: + automountServiceAccountToken: false containers: - name: main image: minio/minio diff --git a/manifests/quick-start/base/webhooks/github.com-secret.yaml b/manifests/quick-start/base/webhooks/github.com-secret.yaml new file mode 100644 index 000000000000..0ba9ffb60d38 --- /dev/null +++ b/manifests/quick-start/base/webhooks/github.com-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: github.com.service-account-token + annotations: + kubernetes.io/service-account.name: github.com +type: kubernetes.io/service-account-token \ No newline at end of file diff --git a/manifests/quick-start/base/webhooks/kustomization.yaml b/manifests/quick-start/base/webhooks/kustomization.yaml index ffef982a7067..dde4e6c1fd7d 100644 --- a/manifests/quick-start/base/webhooks/kustomization.yaml +++ b/manifests/quick-start/base/webhooks/kustomization.yaml @@ -4,5 +4,6 @@ kind: Kustomization resources: - submit-workflow-template-role.yaml - github.com-sa.yaml + - github.com-secret.yaml - github.com-rolebinding.yaml - argo-workflows-webhook-clients-secret.yaml diff --git a/manifests/quick-start/mysql/mysql-deployment.yaml b/manifests/quick-start/mysql/mysql-deployment.yaml index 63a4224d9ece..02f49afe0191 100644 --- a/manifests/quick-start/mysql/mysql-deployment.yaml +++ b/manifests/quick-start/mysql/mysql-deployment.yaml @@ -14,6 +14,7 @@ spec: labels: app: mysql spec: + automountServiceAccountToken: false containers: - name: main image: mysql:8 @@ -29,9 +30,9 @@ spec: ports: - containerPort: 3306 readinessProbe: - exec: - command: ["mysql", "-u", "mysql", "-ppassword", "argo", "-e", "SELECT 1"] - initialDelaySeconds: 15 - timeoutSeconds: 2 + tcpSocket: + port: 3306 + initialDelaySeconds: 30 + periodSeconds: 10 nodeSelector: kubernetes.io/os: linux diff --git a/mkdocs.yml b/mkdocs.yml index cfdbe6502207..d636b8144152 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -215,6 +215,7 @@ nav: - workflow-executors.md - workflow-restrictions.md - sidecar-injection.md + - manually-create-secrets.md - Argo Server: - argo-server.md - argo-server-auth-mode.md diff --git a/server/auth/gatekeeper.go b/server/auth/gatekeeper.go index 588ae4fef78c..193676303b47 100644 --- a/server/auth/gatekeeper.go +++ b/server/auth/gatekeeper.go @@ -8,6 +8,8 @@ import ( "sort" "strconv" + "github.com/argoproj/argo-workflows/v3/util/secrets" + eventsource "github.com/argoproj/argo-events/pkg/client/eventsource/clientset/versioned" sensor "github.com/argoproj/argo-events/pkg/client/sensor/clientset/versioned" log "github.com/sirupsen/logrus" @@ -316,10 +318,8 @@ func (s *gatekeeper) rbacAuthorization(ctx context.Context, claims *types.Claims } func (s *gatekeeper) authorizationForServiceAccount(ctx context.Context, serviceAccount *corev1.ServiceAccount) (string, error) { - if len(serviceAccount.Secrets) == 0 { - return "", fmt.Errorf("expected at least one secret for SSO RBAC service account: %s", serviceAccount.GetName()) - } - secret, err := s.cache.GetSecret(ctx, serviceAccount.GetNamespace(), serviceAccount.Secrets[0].Name) + secretName := secrets.TokenNameForServiceAccount(serviceAccount) + secret, err := s.cache.GetSecret(ctx, serviceAccount.GetNamespace(), secretName) if err != nil { return "", fmt.Errorf("failed to get service account secret: %w", err) } diff --git a/server/auth/webhook/interceptor.go b/server/auth/webhook/interceptor.go index 6b88fab93745..551dbf2e9b1c 100644 --- a/server/auth/webhook/interceptor.go +++ b/server/auth/webhook/interceptor.go @@ -7,6 +7,8 @@ import ( "net/http" "strings" + "github.com/argoproj/argo-workflows/v3/util/secrets" + log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -84,10 +86,7 @@ func addWebhookAuthorization(r *http.Request, kube kubernetes.Interface) error { if err != nil { return fmt.Errorf("failed to get service account \"%s\": %w", serviceAccountName, err) } - if len(serviceAccount.Secrets) == 0 { - return fmt.Errorf("failed to get secret for service account \"%s\": no secrets", serviceAccountName) - } - tokenSecret, err := secretsInterface.Get(ctx, serviceAccount.Secrets[0].Name, metav1.GetOptions{}) + tokenSecret, err := secretsInterface.Get(ctx, secrets.TokenNameForServiceAccount(serviceAccount), metav1.GetOptions{}) if err != nil { return fmt.Errorf("failed to get token secret \"%s\": %w", tokenSecret, err) } diff --git a/test/e2e/agent_test.go b/test/e2e/agent_test.go index 83dda64d3bee..7c3f06cb486d 100644 --- a/test/e2e/agent_test.go +++ b/test/e2e/agent_test.go @@ -110,15 +110,15 @@ spec: - - name: http-status-is-201-fails template: http-status-is-201 arguments: - parameters: [{name: url, value: "https://httpstat.us/200"}] + parameters: [{name: url, value: "http://httpbin:9100/status/200"}] - name: http-status-is-201-succeeds template: http-status-is-201 arguments: - parameters: [{name: url, value: "https://httpstat.us/201"}] + parameters: [{name: url, value: "http://httpbin:9100/status/201"}] - name: http-body-contains-google-fails template: http-body-contains-google arguments: - parameters: [{name: url, value: "https://httpstat.us/200"}] + parameters: [{name: url, value: "http://httpbin:9100/status/200"}] - name: http-body-contains-google-succeeds template: http-body-contains-google arguments: diff --git a/test/e2e/argo_server_test.go b/test/e2e/argo_server_test.go index 561abdc9d42c..5f328d47e00f 100644 --- a/test/e2e/argo_server_test.go +++ b/test/e2e/argo_server_test.go @@ -14,6 +14,8 @@ import ( "testing" "time" + "github.com/argoproj/argo-workflows/v3/util/secrets" + "github.com/gavv/httpexpect/v2" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -377,31 +379,25 @@ func (s *ArgoServerSuite) TestMultiCookieAuth() { Status(200) } -func (s *ArgoServerSuite) TestPermission() { - nsName := fixtures.Namespace - // Create good serviceaccount - goodSaName := "argotestgood" - goodSa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: goodSaName}} +func (s *ArgoServerSuite) createServiceAccount(name string) { ctx := context.Background() - s.Run("CreateGoodSA", func() { - _, err := s.KubeClient.CoreV1().ServiceAccounts(nsName).Create(ctx, goodSa, metav1.CreateOptions{}) - assert.NoError(s.T(), err) + _, err := s.KubeClient.CoreV1().ServiceAccounts(fixtures.Namespace).Create(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name}}, metav1.CreateOptions{}) + assert.NoError(s.T(), err) + secret, err := s.KubeClient.CoreV1().Secrets(fixtures.Namespace).Create(ctx, secrets.NewTokenSecret(name), metav1.CreateOptions{}) + assert.NoError(s.T(), err) + s.T().Cleanup(func() { + _ = s.KubeClient.CoreV1().Secrets(fixtures.Namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{}) + _ = s.KubeClient.CoreV1().ServiceAccounts(fixtures.Namespace).Delete(ctx, name, metav1.DeleteOptions{}) }) - defer func() { - // Clean up created sa - _ = s.KubeClient.CoreV1().ServiceAccounts(nsName).Delete(ctx, goodSaName, metav1.DeleteOptions{}) - }() +} - // Create bad serviceaccount +func (s *ArgoServerSuite) TestPermission() { + ctx := context.Background() + nsName := fixtures.Namespace + goodSaName := "argotestgood" + s.createServiceAccount(goodSaName) badSaName := "argotestbad" - badSa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: badSaName}} - s.Run("CreateBadSA", func() { - _, err := s.KubeClient.CoreV1().ServiceAccounts(nsName).Create(ctx, badSa, metav1.CreateOptions{}) - assert.NoError(s.T(), err) - }) - defer func() { - _ = s.KubeClient.CoreV1().ServiceAccounts(nsName).Delete(ctx, badSaName, metav1.DeleteOptions{}) - }() + s.createServiceAccount(badSaName) // Create RBAC Role var roleName string @@ -445,7 +441,7 @@ func (s *ArgoServerSuite) TestPermission() { s.Run("GetGoodSAToken", func() { sAccount, err := s.KubeClient.CoreV1().ServiceAccounts(nsName).Get(ctx, goodSaName, metav1.GetOptions{}) if assert.NoError(s.T(), err) { - secretName := sAccount.Secrets[0].Name + secretName := secrets.TokenNameForServiceAccount(sAccount) secret, err := s.KubeClient.CoreV1().Secrets(nsName).Get(ctx, secretName, metav1.GetOptions{}) assert.NoError(s.T(), err) goodToken = string(secret.Data["token"]) @@ -457,7 +453,7 @@ func (s *ArgoServerSuite) TestPermission() { s.Run("GetBadSAToken", func() { sAccount, err := s.KubeClient.CoreV1().ServiceAccounts(nsName).Get(ctx, badSaName, metav1.GetOptions{}) assert.NoError(s.T(), err) - secretName := sAccount.Secrets[0].Name + secretName := secrets.TokenNameForServiceAccount(sAccount) secret, err := s.KubeClient.CoreV1().Secrets(nsName).Get(ctx, secretName, metav1.GetOptions{}) assert.NoError(s.T(), err) badToken = string(secret.Data["token"]) diff --git a/test/e2e/artifacts_test.go b/test/e2e/artifacts_test.go index dd5390f98289..bdc64be927f5 100644 --- a/test/e2e/artifacts_test.go +++ b/test/e2e/artifacts_test.go @@ -143,7 +143,7 @@ func (s *ArtifactsSuite) TestArtifactGC() { } else { fmt.Printf("verifying artifact %s is not deleted at completion time\n", expectedArtifact.key) then.ExpectArtifactByKey(expectedArtifact.key, expectedArtifact.bucketName, func(t *testing.T, object minio.ObjectInfo, err error) { - assert.Nil(t, err) + assert.NoError(t, err) }) } } diff --git a/test/e2e/fixtures/e2e_suite.go b/test/e2e/fixtures/e2e_suite.go index e1435fed1add..e189c918df96 100644 --- a/test/e2e/fixtures/e2e_suite.go +++ b/test/e2e/fixtures/e2e_suite.go @@ -5,9 +5,10 @@ import ( "encoding/base64" "fmt" "os" - "strings" "time" + "github.com/argoproj/argo-workflows/v3/util/secrets" + "github.com/TwiN/go-color" "github.com/stretchr/testify/suite" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -189,16 +190,11 @@ func (s *E2ESuite) GetServiceAccountToken() (string, error) { } ctx := context.Background() - secretList, err := clientset.CoreV1().Secrets("argo").List(ctx, metav1.ListOptions{}) + sec, err := clientset.CoreV1().Secrets(Namespace).Get(ctx, secrets.TokenName("argo-server"), metav1.GetOptions{}) if err != nil { return "", err } - for _, sec := range secretList.Items { - if strings.HasPrefix(sec.Name, "argo-server-token") { - return string(sec.Data["token"]), nil - } - } - return "", nil + return string(sec.Data["token"]), nil } func (s *E2ESuite) Given() *Given { diff --git a/test/e2e/manifests/minimal/kustomization.yaml b/test/e2e/manifests/minimal/kustomization.yaml index e29f4ca34dcc..b1a5e2110728 100644 --- a/test/e2e/manifests/minimal/kustomization.yaml +++ b/test/e2e/manifests/minimal/kustomization.yaml @@ -7,6 +7,7 @@ resources: - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_eventsources.yaml - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_sensors.yaml - ../mixins/argo-workflows-agent-ca-certificates.yaml +- ../mixins/argo-server.service-account-token-secret.yaml patchesStrategicMerge: - ../mixins/argo-server-deployment.yaml diff --git a/test/e2e/manifests/mixins/argo-server.service-account-token-secret.yaml b/test/e2e/manifests/mixins/argo-server.service-account-token-secret.yaml new file mode 100644 index 000000000000..6a92851b419f --- /dev/null +++ b/test/e2e/manifests/mixins/argo-server.service-account-token-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: argo-server.service-account-token + annotations: + kubernetes.io/service-account.name: argo-server +type: kubernetes.io/service-account-token \ No newline at end of file diff --git a/test/e2e/manifests/mysql/kustomization.yaml b/test/e2e/manifests/mysql/kustomization.yaml index 032ae2248a3f..37e04d03591c 100644 --- a/test/e2e/manifests/mysql/kustomization.yaml +++ b/test/e2e/manifests/mysql/kustomization.yaml @@ -6,6 +6,7 @@ resources: - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_eventbus.yaml - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_eventsources.yaml - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_sensors.yaml +- ../mixins/argo-server.service-account-token-secret.yaml patchesStrategicMerge: - ../mixins/argo-server-deployment.yaml diff --git a/test/e2e/manifests/plugins/hello-executor-plugin.service-account-token-secret.yaml b/test/e2e/manifests/plugins/hello-executor-plugin.service-account-token-secret.yaml new file mode 100644 index 000000000000..391fa7086c99 --- /dev/null +++ b/test/e2e/manifests/plugins/hello-executor-plugin.service-account-token-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: hello-executor-plugin.service-account-token + annotations: + kubernetes.io/service-account.name: hello-executor-plugin +type: kubernetes.io/service-account-token \ No newline at end of file diff --git a/test/e2e/manifests/plugins/kustomization.yaml b/test/e2e/manifests/plugins/kustomization.yaml index d47461ea77b9..83d9d8ec06fc 100644 --- a/test/e2e/manifests/plugins/kustomization.yaml +++ b/test/e2e/manifests/plugins/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - ../minimal - hello-executor-plugin-serviceaccount.yaml + - hello-executor-plugin.service-account-token-secret.yaml - hello-executor-plugin-configmap.yaml commonLabels: diff --git a/test/util/serviceaccount.go b/test/util/serviceaccount.go index 799b6cb3d0e1..950a2376eb69 100644 --- a/test/util/serviceaccount.go +++ b/test/util/serviceaccount.go @@ -3,6 +3,8 @@ package util import ( "context" + "github.com/argoproj/argo-workflows/v3/util/secrets" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -10,24 +12,12 @@ import ( // CreateServiceAccountWithToken creates a service account with a given name with a service account token. // Need to use this function to simulate the actual behavior of Kubernetes API server with the fake client. -func CreateServiceAccountWithToken(ctx context.Context, clientset kubernetes.Interface, namespace, name, tokenName string) (*corev1.ServiceAccount, error) { +func CreateServiceAccountWithToken(ctx context.Context, clientset kubernetes.Interface, namespace, name string) (*corev1.ServiceAccount, error) { sa, err := clientset.CoreV1().ServiceAccounts(namespace).Create(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name}}, metav1.CreateOptions{}) if err != nil { return nil, err } - token, err := clientset.CoreV1().Secrets(namespace).Create(ctx, &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: tokenName, - Annotations: map[string]string{ - corev1.ServiceAccountNameKey: sa.Name, - corev1.ServiceAccountUIDKey: string(sa.UID), - }, - }, Type: corev1.SecretTypeServiceAccountToken, - }, + _, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secrets.NewTokenSecret(name), metav1.CreateOptions{}) - if err != nil { - return nil, err - } - sa.Secrets = []corev1.ObjectReference{{Name: token.Name}} - return clientset.CoreV1().ServiceAccounts(namespace).Update(ctx, sa, metav1.UpdateOptions{}) + return sa, err } diff --git a/util/secrets/secret_name.go b/util/secrets/secret_name.go new file mode 100644 index 000000000000..dc8730a40572 --- /dev/null +++ b/util/secrets/secret_name.go @@ -0,0 +1,24 @@ +package secrets + +import ( + "fmt" + + "github.com/argoproj/argo-workflows/v3/workflow/common" + + corev1 "k8s.io/api/core/v1" +) + +// TokenNameForServiceAccount returns the name of the secret container the access token for the service account +func TokenNameForServiceAccount(sa *corev1.ServiceAccount) string { + if len(sa.Secrets) > 0 { + return sa.Secrets[0].Name + } + if v, ok := sa.Annotations[common.AnnotationKeyServiceAccountTokenName]; ok { + return v + } + return TokenName(sa.Name) +} + +func TokenName(name string) string { + return fmt.Sprintf("%s.service-account-token", name) +} diff --git a/util/secrets/secret_name_test.go b/util/secrets/secret_name_test.go new file mode 100644 index 000000000000..9a81214dda10 --- /dev/null +++ b/util/secrets/secret_name_test.go @@ -0,0 +1,46 @@ +package secrets + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestServiceAccountTokenName(t *testing.T) { + type args struct { + sa *corev1.ServiceAccount + } + tests := []struct { + name string + args args + want string + }{ + { + "discovery by secret (Kubernetes =v1.24)", + args{&corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{"workflows.argoproj.io/service-account-token.name": "my-token"}}, + }}, + "my-token", + }, + { + "discovery by name (Kubernetes >=v1.24)", + args{&corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: "my-name"}, + }}, + "my-name.service-account-token", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := TokenNameForServiceAccount(tt.args.sa); got != tt.want { + t.Errorf("ServiceAccountTokenName() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/util/secrets/secrets.go b/util/secrets/secrets.go new file mode 100644 index 000000000000..1009b6087767 --- /dev/null +++ b/util/secrets/secrets.go @@ -0,0 +1,17 @@ +package secrets + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NewTokenSecret creates a new secret struct. +func NewTokenSecret(name string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: TokenName(name), + Annotations: map[string]string{corev1.ServiceAccountNameKey: name}, + }, + Type: corev1.SecretTypeServiceAccountToken, + } +} diff --git a/workflow/common/common.go b/workflow/common/common.go index 90275b31c8ed..72d61279e041 100644 --- a/workflow/common/common.go +++ b/workflow/common/common.go @@ -15,6 +15,10 @@ const ( // AnnotationKeyDefaultContainer is the annotation that specify container that will be used by default in case of kubectl commands for example AnnotationKeyDefaultContainer = "kubectl.kubernetes.io/default-container" + // AnnotationKeyServiceAccountTokenName is used to name the secret that containers the service account token name. + // It is intentially named similar to ` `kubernetes.io/service-account.name`. + AnnotationKeyServiceAccountTokenName = workflow.WorkflowFullName + "/service-account-token.name" + // AnnotationKeyNodeID is the ID of the node. // Historically, the pod name was the same as the node ID. // Therefore, if it does not exist, then the node ID is the pod name. diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go index 082a6580fccf..db6031509dca 100644 --- a/workflow/controller/operator.go +++ b/workflow/controller/operator.go @@ -15,6 +15,8 @@ import ( "sync" "time" + "github.com/argoproj/argo-workflows/v3/util/secrets" + "github.com/antonmedv/expr" "github.com/argoproj/pkg/humanize" argokubeerr "github.com/argoproj/pkg/kube/errors" @@ -3742,10 +3744,7 @@ func (woc *wfOperationCtx) getServiceAccountTokenName(ctx context.Context, name if err != nil { return "", err } - if len(account.Secrets) == 0 { - return "", fmt.Errorf("service account %s/%s does not have any secrets", account.Namespace, account.Name) - } - return account.Secrets[0].Name, nil + return secrets.TokenNameForServiceAccount(account), nil } // setWfPodNamesAnnotation sets an annotation on a workflow with the pod naming diff --git a/workflow/controller/workflowpod_test.go b/workflow/controller/workflowpod_test.go index a109b853e837..128808b005bb 100644 --- a/workflow/controller/workflowpod_test.go +++ b/workflow/controller/workflowpod_test.go @@ -224,7 +224,7 @@ func TestTmplServiceAccount(t *testing.T) { func TestWFLevelAutomountServiceAccountToken(t *testing.T) { woc := newWoc() ctx := context.Background() - _, err := util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "foo", "foo-token") + _, err := util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "foo") assert.NoError(t, err) falseValue := false @@ -246,7 +246,7 @@ func TestWFLevelAutomountServiceAccountToken(t *testing.T) { func TestTmplLevelAutomountServiceAccountToken(t *testing.T) { woc := newWoc() ctx := context.Background() - _, err := util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "foo", "foo-token") + _, err := util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "foo") assert.NoError(t, err) trueValue := true @@ -280,7 +280,7 @@ func verifyServiceAccountTokenVolumeMount(t *testing.T, ctr apiv1.Container, vol func TestWFLevelExecutorServiceAccountName(t *testing.T) { woc := newWoc() ctx := context.Background() - _, err := util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "foo", "foo-token") + _, err := util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "foo") assert.NoError(t, err) woc.execWf.Spec.Executor = &wfv1.ExecutorConfig{ServiceAccountName: "foo"} @@ -294,7 +294,6 @@ func TestWFLevelExecutorServiceAccountName(t *testing.T) { assert.Len(t, pods.Items, 1) pod := pods.Items[0] assert.Equal(t, "exec-sa-token", pod.Spec.Volumes[2].Name) - assert.Equal(t, "foo-token", pod.Spec.Volumes[2].VolumeSource.Secret.SecretName) waitCtr := pod.Spec.Containers[0] verifyServiceAccountTokenVolumeMount(t, waitCtr, "exec-sa-token", "/var/run/secrets/kubernetes.io/serviceaccount") @@ -304,9 +303,9 @@ func TestWFLevelExecutorServiceAccountName(t *testing.T) { func TestTmplLevelExecutorServiceAccountName(t *testing.T) { woc := newWoc() ctx := context.Background() - _, err := util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "foo", "foo-token") + _, err := util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "foo") assert.NoError(t, err) - _, err = util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "tmpl", "tmpl-token") + _, err = util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "tmpl") assert.NoError(t, err) woc.execWf.Spec.Executor = &wfv1.ExecutorConfig{ServiceAccountName: "foo"} @@ -321,7 +320,6 @@ func TestTmplLevelExecutorServiceAccountName(t *testing.T) { assert.Len(t, pods.Items, 1) pod := pods.Items[0] assert.Equal(t, "exec-sa-token", pod.Spec.Volumes[2].Name) - assert.Equal(t, "tmpl-token", pod.Spec.Volumes[2].VolumeSource.Secret.SecretName) waitCtr := pod.Spec.Containers[0] verifyServiceAccountTokenVolumeMount(t, waitCtr, "exec-sa-token", "/var/run/secrets/kubernetes.io/serviceaccount") @@ -332,9 +330,9 @@ func TestTmplLevelExecutorSecurityContext(t *testing.T) { var user int64 = 1000 ctx := context.Background() woc := newWoc() - _, err := util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "foo", "foo-token") + _, err := util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "foo") assert.NoError(t, err) - _, err = util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "tmpl", "tmpl-token") + _, err = util.CreateServiceAccountWithToken(ctx, woc.controller.kubeclientset, "", "tmpl") assert.NoError(t, err) woc.controller.Config.Executor = &apiv1.Container{SecurityContext: &apiv1.SecurityContext{RunAsUser: &user}}