test: Use 1password for test secrets (ddev#6536)

.github/workflows/tests.yml

@@ -67,9 +67,7 @@ jobs:
 
     env:
       CGO_ENABLED: 0
-      DDEV_ACQUIA_SSH_KEY: ${{ secrets.DDEV_ACQUIA_SSH_KEY }}
       DDEV_NONINTERACTIVE: "true"
-      DDEV_PANTHEON_SSH_KEY: ${{ secrets.DDEV_PANTHEON_SSH_KEY }}
       DDEV_SKIP_NODEJS_TEST: "true"
       GOTEST_SHORT: "12" # 12 is drupal10; means in TestFullSiteSetup we only use drupal10
       DDEV_TEST_WEBSERVER_TYPE: ${{ matrix.webserver }}
@@ -125,13 +123,22 @@ jobs:
           echo "MAKE_TARGET=test" >> $GITHUB_ENV
           echo "TESTARGS=-failfast -run '(TestDdevFullSite.*|TestDdevImportFiles|TestDdevAllDatabases|TestComposerCreateCmd|Test.*(Push|Pull)|TestAutocomplet)'" >> $GITHUB_ENV
           echo "GOTEST_SHORT=" >> $GITHUB_ENV
-          echo "DDEV_PLATFORM_API_TOKEN=${{ secrets.DDEV_PLATFORM_API_TOKEN }}" >> $GITHUB_ENV
-          echo "DDEV_UPSUN_API_TOKEN=${{ secrets.DDEV_UPSUN_API_TOKEN }}" >> $GITHUB_ENV
-          echo "DDEV_PANTHEON_API_TOKEN=${{ secrets.DDEV_PANTHEON_API_TOKEN }}" >> $GITHUB_ENV
-          echo "DDEV_ALLOW_ACQUIA_PUSH=${{ secrets.DDEV_ALLOW_ACQUIA_PUSH }}" >> $GITHUB_ENV
-          echo "DDEV_ACQUIA_API_KEY=${{ secrets.DDEV_ACQUIA_API_KEY }}" >> $GITHUB_ENV
-          echo "DDEV_ACQUIA_API_SECRET=${{ secrets.DDEV_ACQUIA_API_SECRET }}" >> $GITHUB_ENV
-          echo "DDEV_LAGOON_SSH_KEY=${{ secrets.DDEV_LAGOON_SSH_KEY }}" >> $GITHUB_ENV
+        if: ${{ matrix.pull-push-test-platforms }}
+
+      - name: Load 1password secret(s) for push-pull-test-platforms
+        uses: 1password/load-secrets-action@v2
+        with:
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.TESTS_SERVICE_ACCOUNT_TOKEN }}"
+          DDEV_ACQUIA_API_KEY: "op://test-secrets/DDEV_ACQUIA_API_KEY/credential"
+          DDEV_ACQUIA_API_SECRET: "op://test-secrets/DDEV_ACQUIA_API_SECRET/credential"
+          DDEV_ACQUIA_SSH_KEY: "op://test-secrets/DDEV_ACQUIA_SSH_KEY/private key?ssh-format=openssh"
+          DDEV_LAGOON_SSH_KEY: "op://test-secrets/DDEV_LAGOON_SSH_KEY/private key?ssh-format=openssh"
+          DDEV_PANTHEON_API_TOKEN: "op://test-secrets/DDEV_PANTHEON_API_TOKEN/credential"
+          DDEV_PANTHEON_SSH_KEY: "op://test-secrets/DDEV_PANTHEON_SSH_KEY/private key?ssh-format=openssh"
+          DDEV_PLATFORM_API_TOKEN: "op://test-secrets/DDEV_PLATFORM_API_TOKEN/credential"
+          DDEV_UPSUN_API_TOKEN: "op://test-secrets/DDEV_UPSUN_API_TOKEN/credential"
         if: ${{ matrix.pull-push-test-platforms }}
 
       - name: Override environment variables for plain nginx

cmd/ddev/cmd/auth-ssh.go

@@ -70,7 +70,7 @@ var AuthSSHCommand = &cobra.Command{
 		}
 		sshKeyPath = util.WindowsPathToCygwinPath(sshKeyPath)
 
-		dockerCmd := []string{"run", "-it", "--rm", "--volumes-from=" + ddevapp.SSHAuthName, "--user=" + uidStr, "--entrypoint=", "--mount=type=bind,src=" + sshKeyPath + ",dst=/tmp/sshtmp", versionconstants.SSHAuthImage + ":" + versionconstants.SSHAuthTag + "-built", "bash", "-c", `cp -r /tmp/sshtmp ~/.ssh && chmod -R go-rwx ~/.ssh && cd ~/.ssh && ssh-add $(file * | awk -F: "/private key/ { print \$1 }")`}
+		dockerCmd := []string{"run", "-it", "--rm", "--volumes-from=" + ddevapp.SSHAuthName, "--user=" + uidStr, "--entrypoint=", "--mount=type=bind,src=" + sshKeyPath + ",dst=/tmp/sshtmp", versionconstants.SSHAuthImage + ":" + versionconstants.SSHAuthTag + "-built", "bash", "-c", `cp -r /tmp/sshtmp ~/.ssh && chmod -R go-rwx ~/.ssh && cd ~/.ssh && grep -l '^-----BEGIN .* PRIVATE KEY-----' * | xargs -d '\n' ssh-add`}
 
 		err = exec.RunInteractiveCommand("docker", dockerCmd)
 

docs/content/developers/secret-management.md

@@ -0,0 +1,9 @@
+# Secret Management
+
+Most secrets used in our build process are managed via 1Password. The technique is documented in [developer.1password.com](https://developer.1password.com/docs/ci-cd/github-actions/).
+
+## Pull and Push Secrets
+
+Secrets for the TestPlatformPull, TestAcquiaPull, and similar tests are in the `test-secrets` vault in the DDEV 1Password instance. They can be rotated and otherwise managed there.
+
+The `test-secrets` vault allows access to the service account [`tests`](https://team-ddev.1password.com/developer-tools/infrastructure-secrets/serviceaccount/76URGQSBMVEUHPEWVHQRFJ4N3Q) whose auth token is in "1Password Service Account Auth Token: Tests"

markdown-link-check.json

@@ -62,6 +62,9 @@
     },
     {
       "pattern": "^https://aur.archlinux.org"
+    },
+    {
+      "pattern": "^https://team-ddev.1password.com"
     }
   ],
   "httpHeaders": [

mkdocs.yml

@@ -224,3 +224,4 @@ nav:
       - developers/writing-style-guide.md
       - developers/remote-config.md
       - developers/maintainers.md
+      - developers/secret-management.md

pkg/ddevapp/providerAcquia_test.go

@@ -48,8 +48,6 @@ func TestAcquiaPull(t *testing.T) {
 		t.Skipf("No DDEV_ACQUIA_SSH_KEY env var has been set. Skipping %v", t.Name())
 	}
 
-	sshkey = strings.Replace(sshkey, "<SPLIT>", "\n", -1)
-
 	require.True(t, isPullSiteValid(acquiaPullSiteURL, acquiaSiteExpectation), "acquiaPullSiteURL %s isn't working right", acquiaPullSiteURL)
 	// Set up tests and give ourselves a working directory.
 	assert := asrt.New(t)
@@ -142,7 +140,6 @@ func TestAcquiaPush(t *testing.T) {
 	if sshkey = os.Getenv("DDEV_ACQUIA_SSH_KEY"); sshkey == "" {
 		t.Skipf("No DDEV_ACQUIA_SSH_KEY env var has been set. Skipping %v", t.Name())
 	}
-	sshkey = strings.Replace(sshkey, "<SPLIT>", "\n", -1)
 
 	// Set up tests and give ourselves a working directory.
 	assert := asrt.New(t)

pkg/ddevapp/providerLagoon_test.go

@@ -27,7 +27,7 @@ const lagoonProjectName = "amazeeio-ddev"
 const lagoonPullTestSiteEnvironment = "pull"
 const lagoonPushTestSiteEnvironment = "push"
 
-// TODO: Change this to the actual dediicated pull environment
+// TODO: Change this to the actual dedicated pull environment
 const lagoonPullSiteURL = "https://nginx.pull.amazeeio-ddev.us2.amazee.io/"
 const lagoonSiteExpectation = "Super easy vegetarian pasta"
 
@@ -38,7 +38,6 @@ func lagoonSetupSSHKey(t *testing.T) string {
 	if sshkey = os.Getenv("DDEV_LAGOON_SSH_KEY"); sshkey == "" {
 		t.Skipf("No DDEV_LAGOON_SSH_KEY env var has been set. Skipping %v", t.Name())
 	}
-	sshkey = strings.Replace(sshkey, "<SPLIT>", "\n", -1)
 	return sshkey + "\n"
 }
 

pkg/ddevapp/providerPantheon_test.go

@@ -135,7 +135,6 @@ func TestPantheonPush(t *testing.T) {
 	if sshkey = os.Getenv("DDEV_PANTHEON_SSH_KEY"); sshkey == "" {
 		t.Skipf("No DDEV_PANTHEON_SSH_KEY env var has been set. Skipping %v", t.Name())
 	}
-	sshkey = strings.Replace(sshkey, "<SPLIT>", "\n", -1)
 
 	// Set up tests and give ourselves a working directory.
 	assert := asrt.New(t)
@@ -269,10 +268,23 @@ func setupSSHKey(t *testing.T, privateKey string, expectScriptDir string) error
 	// Provide an SSH key for `ddev auth ssh`
 	err := os.Mkdir("sshtest", 0755)
 	require.NoError(t, err)
+	// If the first line is empty, discard it
+	if privateKey[0] == '\n' {
+		privateKey = privateKey[1:]
+	}
+	if privateKey[len(privateKey)-1] != '\n' {
+		privateKey = privateKey + "\n"
+	}
+	//l := len(privateKey)
+	//t.Logf("privateKey starts with character '%v' string '%s' keytype '%s' and ends with '%s'. The last character is '%v'", privateKey[0], privateKey[0:30], privateKey[5:40], privateKey[l-26:], privateKey[l-1])
 	err = os.WriteFile(filepath.Join("sshtest", "id_rsa_test"), []byte(privateKey), 0600)
 	require.NoError(t, err)
+	//out, err2 := exec.RunHostCommand("file", filepath.Join("sshtest", "id_rsa_test"))
+	//require.NoError(t, err2)
+	//t.Logf("result of file on id_rsa_test=%s", out)
 	out, err := exec.RunHostCommand("expect", filepath.Join(expectScriptDir, "ddevauthssh.expect"), DdevBin, "./sshtest")
-	require.NoError(t, err, "out=%s", out)
+	pwd, _ := os.Getwd()
+	require.NoError(t, err, "failed to RunHostCommand expect script in dir=%s, out=%s", pwd, out)
 	require.Contains(t, out, "Identity added:")
 	return nil
 }

scripts/read-1password-secrets.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Read secrets into environment from 1password
+
+for item in DDEV_ACQUIA_API_KEY DDEV_ACQUIA_API_SECRET DDEV_PANTHEON_API_TOKEN DDEV_PLATFORM_API_TOKEN DDEV_UPSUN_API_TOKEN; do
+  printf "export ${item}=$(op item get ${item} --field=credential --reveal)\n"
+done >/tmp/1penv.sh
+
+for item in DDEV_ACQUIA_SSH_KEY DDEV_LAGOON_SSH_KEY DDEV_PANTHEON_SSH_KEY; do
+  printf "export ${item}=$(op item get --field='private key' --reveal ${item})\n"
+done >> /tmp/1penv.sh
+
+echo "You can now 'source /tmp/1penv.sh' to get the new environment variables"