Add external secret extension implementation

cmd/server/flags.go

@@ -289,6 +289,16 @@ var flags = append([]cli.Flag{
 		Name:    "registry-extension-endpoint",
 		Usage:   "url used for calling registry service endpoint",
 	},
+	&cli.StringFlag{
+		Sources: cli.EnvVars("WOODPECKER_SECRET_EXTENSION_ENDPOINT"),
+		Name:    "secret-extension-endpoint",
+		Usage:   "url used for calling external secret service endpoint",
+	},
+	&cli.BoolFlag{
+		Sources: cli.EnvVars("WOODPECKER_SECRET_EXTENSION_NETRC"),
+		Name:    "secret-extension-netrc",
+		Usage:   "include netrc credentials in requests to secret service endpoint",
+	},
 	&cli.StringFlag{
 		Sources: cli.EnvVars("WOODPECKER_EXTENSIONS_ALLOWED_HOSTS"),
 		Name:    "extensions-allowed-hosts",

cmd/server/openapi/docs.go

@@ -5292,6 +5292,12 @@ const docTemplate = `{
                 "require_approval": {
                     "$ref": "#/definitions/model.ApprovalMode"
                 },
+                "secret_extension_endpoint": {
+                    "type": "string"
+                },
+                "secret_extension_netrc": {
+                    "type": "boolean"
+                },
                 "timeout": {
                     "type": "integer"
                 },
@@ -5394,6 +5400,12 @@ const docTemplate = `{
                 "require_approval": {
                     "$ref": "#/definitions/model.ApprovalMode"
                 },
+                "secret_extension_endpoint": {
+                    "type": "string"
+                },
+                "secret_extension_netrc": {
+                    "type": "boolean"
+                },
                 "timeout": {
                     "type": "integer"
                 },
@@ -5447,6 +5459,12 @@ const docTemplate = `{
                 "require_approval": {
                     "type": "string"
                 },
+                "secret_extension_endpoint": {
+                    "type": "string"
+                },
+                "secret_extension_netrc": {
+                    "type": "boolean"
+                },
                 "timeout": {
                     "type": "integer"
                 },

docs/docs/20-usage/72-extensions/55-secret-extension.md

@@ -0,0 +1,166 @@
+# Secret extension
+
+Woodpecker uses the secret extension to get secrets from an external service. You can configure an HTTP endpoint in the repository settings in the extensions tab.
+
+Using such an extension can be useful if you want to:
+
+- Centralize secret management (e.g. HashiCorp Vault, AWS Secrets Manager)
+- Dynamically generate secrets per pipeline
+
+## Security
+
+:::warning
+As Woodpecker will pass private information like tokens and will execute the returned configuration, it is extremely important to secure the external extension. Therefore Woodpecker signs every request. Read more about it in the security section.
+:::
+
+## Global configuration
+
+In addition to the ability to configure the extension per repository, you can also configure a global endpoint in the Woodpecker server configuration. This can be useful if you want to use the extension for all repositories. Be careful if
+you share your Woodpecker server with others as they will also use your secret extension.
+
+If both the global and the repo-level extension return a secret with the same name, it will use the secret from the repo extension.
+
+```ini title="Server"
+WOODPECKER_SECRET_EXTENSION_ENDPOINT=https://example.com/secrets
+WOODPECKER_SECRET_EXTENSION_NETRC=false
+```
+
+## How it works
+
+When a pipeline is triggered, Woodpecker will fetch secrets from your service. The extension secrets are merged with the secrets configured directly in Woodpecker, with extension secrets taking priority by name. If the extension is unavailable, Woodpecker falls back to the locally configured secrets.
+
+### Request
+
+The extension receives an HTTP POST request with the following JSON payload:
+
+:::info
+The `netrc` field is only included in the request when the global `WOODPECKER_SECRET_EXTENSION_NETRC` is set to `true` (default: `false`) or the per-repo "Send netrc credentials" is checked.
+:::
+
+```ts
+class Request {
+  repo: Repo;
+  pipeline: Pipeline;
+  netrc?: Netrc; // only included when netrc sending is enabled (see above)
+}
+```
+
+Checkout the following models for more information:
+
+- [repo model](https://github.com/woodpecker-ci/woodpecker/blob/main/server/model/repo.go)
+- [pipeline model](https://github.com/woodpecker-ci/woodpecker/blob/main/server/model/pipeline.go)
+- [netrc model](https://github.com/woodpecker-ci/woodpecker/blob/main/server/model/netrc.go)
+
+:::tip
+The `netrc` data is pretty powerful as it contains credentials to access the repository. You can use this to clone the repository or even use the forge (Github or Gitlab, ...) API to get more information about the repository.
+:::
+
+Example request:
+
+```json
+// Please check the latest structure in the models mentioned above.
+// This example is likely outdated.
+
+{
+  "repo": {
+    "id": 100,
+    "uid": "",
+    "user_id": 0,
+    "namespace": "",
+    "name": "woodpecker-test-pipeline",
+    "slug": "",
+    "scm": "git",
+    "git_http_url": "",
+    "git_ssh_url": "",
+    "link": "",
+    "default_branch": "",
+    "private": true,
+    "visibility": "private",
+    "active": true,
+    "config": "",
+    "trusted": false,
+    "protected": false,
+    "ignore_forks": false,
+    "ignore_pulls": false,
+    "cancel_pulls": false,
+    "timeout": 60,
+    "counter": 0,
+    "synced": 0,
+    "created": 0,
+    "updated": 0,
+    "version": 0
+  },
+  "pipeline": {
+    "author": "myUser",
+    "author_avatar": "https://myforge.com/avatars/d6b3f7787a685fcdf2a44e2c685c7e03",
+    "author_email": "my@email.com",
+    "branch": "main",
+    "changed_files": ["some-filename.txt"],
+    "commit": "2fff90f8d288a4640e90f05049fe30e61a14fd50",
+    "created_at": 0,
+    "deploy_to": "",
+    "enqueued_at": 0,
+    "error": "",
+    "event": "push",
+    "finished_at": 0,
+    "id": 0,
+    "link_url": "https://myforge.com/myUser/woodpecker-testpipe/commit/2fff90f8d288a4640e90f05049fe30e61a14fd50",
+    "message": "test old config\n",
+    "number": 0,
+    "parent": 0,
+    "ref": "refs/heads/main",
+    "refspec": "",
+    "clone_url": "",
+    "reviewed_at": 0,
+    "reviewed_by": "",
+    "sender": "myUser",
+    "signed": false,
+    "started_at": 0,
+    "status": "",
+    "timestamp": 1645962783,
+    "title": "",
+    "updated_at": 0,
+    "verified": false
+  },
+  "netrc": {
+    "machine": "myforge.com",
+    "login": "myUser",
+    "password": "forge-access-token"
+  }
+}
+// Note: the "netrc" field is omitted when netrc sending is not enabled.
+```
+
+### Response
+
+The extension should respond with a JSON object containing a `secrets` array.
+If the extension wants to keep the existing secrets without adding any, it can respond with HTTP status `204 No Content`.
+
+```ts
+class Response {
+  secrets: {
+    name: string; // the secret name, matched by from_secret in pipeline config
+    value: string; // the secret value
+    images?: string[]; // optional: restrict to specific plugins
+    events?: string[]; // optional: restrict to specific pipeline events
+  }[];
+}
+```
+
+Example response:
+
+```json
+{
+  "secrets": [
+    {
+      "name": "docker_password",
+      "value": "your-secret-password-123"
+    },
+    {
+      "name": "deploy_token",
+      "value": "super-secret-token",
+      "events": ["push", "tag"]
+    }
+  ]
+}
+```

docs/docs/20-usage/72-extensions/index.md

@@ -6,6 +6,7 @@ There is currently one type of extension available:
 
 - [Configuration extension](./40-configuration-extension.md) to modify or generate pipeline configurations on the fly.
 - [Registry extension](./50-registry-extension.md) to get registry credentials from the extension.
+- [Secret extension](./55-secret-extension.md) to get secrets from an external service.
 
 ## Security
 

docs/docs/30-administration/10-configuration/10-server.md

@@ -993,6 +993,28 @@ If you enable this, all repos will exclusively use the global config service end
 
 ---
 
+### SECRET_EXTENSION_ENDPOINT
+
+- Name: `WOODPECKER_SECRET_EXTENSION_ENDPOINT`
+- Default: none
+
+Specify a secret extension endpoint, see [Secret Extension](../../20-usage/72-extensions/55-secret-extension.md)
+
+---
+
+### SECRET_EXTENSION_NETRC
+
+- Name: `WOODPECKER_SECRET_EXTENSION_NETRC`
+- Default: false
+
+Send `netrc` to the secret extension endpoint.
+
+:::warning
+The `netrc` data is pretty powerful as it contains credentials to access the repository. You can use this to clone the repository or even use the forge API to get more information about the repository.
+:::
+
+---
+
 ### REGISTRY_EXTENSION_ENDPOINT
 
 - Name: `WOODPECKER_REGISTRY_EXTENSION_ENDPOINT`

server/api/hook_test.go

@@ -97,7 +97,7 @@ func TestHook(t *testing.T) {
 	_forge.On("Netrc", mock.Anything, mock.Anything).Return(&model.Netrc{}, nil)
 	_store.On("GetPipelineLastBefore", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil)
 	_manager.On("SecretServiceFromRepo", repo).Return(_secretService)
-	_secretService.On("SecretListPipeline", repo, mock.Anything, mock.Anything).Return(nil, nil)
+	_secretService.On("SecretListPipeline", mock.Anything, repo, mock.Anything, mock.Anything).Return(nil, nil)
 	_manager.On("RegistryServiceFromRepo", repo).Return(_registryService)
 	_registryService.On("RegistryListPipeline", mock.Anything, repo, mock.Anything).Return(nil, nil)
 	_manager.On("EnvironmentService").Return(nil)

server/api/pipeline_test.go

@@ -304,7 +304,7 @@ func TestCreatePipeline(t *testing.T) {
 		}, nil).Maybe()
 		mockForge.On("Status", mock.Anything, fakeUser, fakeRepo, mock.Anything, mock.Anything).Return(nil).Maybe()
 
-		mockSecretService.On("SecretListPipeline", fakeRepo, mock.Anything).Return([]*model.Secret{}, nil).Maybe()
+		mockSecretService.On("SecretListPipeline", mock.Anything, fakeRepo, mock.Anything, mock.Anything).Return([]*model.Secret{}, nil).Maybe()
 		mockRegistryService.On("RegistryListPipeline", mock.Anything, fakeRepo, mock.Anything).Return([]*model.Registry{}, nil).Maybe()
 
 		mockManager := manager_mocks.NewMockManager(t)
@@ -377,7 +377,7 @@ func TestCreatePipeline(t *testing.T) {
 		}, nil).Maybe()
 
 		mockForge.On("Status", mock.Anything, fakeUser, fakeRepo, mock.Anything, mock.Anything).Return(nil).Maybe()
-		mockSecretService.On("SecretListPipeline", fakeRepo, mock.Anything).Return([]*model.Secret{}, nil).Maybe()
+		mockSecretService.On("SecretListPipeline", mock.Anything, fakeRepo, mock.Anything, mock.Anything).Return([]*model.Secret{}, nil).Maybe()
 		mockRegistryService.On("RegistryListPipeline", mock.Anything, fakeRepo, mock.Anything).Return([]*model.Registry{}, nil).Maybe()
 
 		mockManager := manager_mocks.NewMockManager(t)

server/api/repo.go

@@ -299,6 +299,12 @@ func PatchRepo(c *gin.Context) {
 	if in.RegistryExtensionEndpoint != nil {
 		repo.RegistryExtensionEndpoint = *in.RegistryExtensionEndpoint
 	}
+	if in.SecretExtensionEndpoint != nil {
+		repo.SecretExtensionEndpoint = *in.SecretExtensionEndpoint
+	}
+	if in.SecretExtensionNetrc != nil {
+		repo.SecretExtensionNetrc = *in.SecretExtensionNetrc
+	}
 
 	err := _store.UpdateRepo(repo)
 	if err != nil {

server/model/repo.go

@@ -75,6 +75,8 @@ type Repo struct {
 	ConfigExtensionEndpoint      string               `json:"config_extension_endpoint"       xorm:"varchar(500) 'config_extension_endpoint'"`
 	ConfigExtensionExclusive     bool                 `json:"config_extension_exclusive"      xorm:"DEFAULT FALSE 'config_extension_exclusive'"`
 	RegistryExtensionEndpoint    string               `json:"registry_extension_endpoint"     xorm:"varchar(500) 'registry_extension_endpoint'"`
+	SecretExtensionEndpoint      string               `json:"secret_extension_endpoint"       xorm:"varchar(500) 'secret_extension_endpoint'"`
+	SecretExtensionNetrc         bool                 `json:"secret_extension_netrc"          xorm:"DEFAULT FALSE 'secret_extension_netrc'"`
 } //	@name	Repo
 
 // TableName return database table name for xorm.
@@ -148,6 +150,8 @@ type RepoPatch struct {
 	ConfigExtensionEndpoint      *string                    `json:"config_extension_endpoint,omitempty"`
 	ConfigExtensionExclusive     *bool                      `json:"config_extension_exclusive"`
 	RegistryExtensionEndpoint    *string                    `json:"registry_extension_endpoint,omitempty"`
+	SecretExtensionEndpoint      *string                    `json:"secret_extension_endpoint,omitempty"`
+	SecretExtensionNetrc         *bool                      `json:"secret_extension_netrc,omitempty"`
 } //	@name	RepoPatch
 
 type ForgeRemoteID string

server/pipeline/items.go

@@ -46,7 +46,7 @@ func parsePipeline(ctx context.Context, forge forge.Forge, store store.Store, cu
 	}
 
 	secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo)
-	secs, err := secretService.SecretListPipeline(repo, currentPipeline)
+	secs, err := secretService.SecretListPipeline(ctx, repo, currentPipeline, netrc)
 	if err != nil {
 		log.Error().Err(err).Msgf("error getting secrets for %s#%d", repo.FullName, currentPipeline.Number)
 	}

server/pipeline/items_test.go

@@ -132,7 +132,7 @@ steps:
 	server.Config.Services.Manager = mockManager
 
 	secretService := secret_service_mocks.NewMockService(t)
-	secretService.On("SecretListPipeline", mock.Anything, mock.Anything).Return([]*model.Secret{
+	secretService.On("SecretListPipeline", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]*model.Secret{
 		{
 			Name:  "hello",
 			Value: "secret world",

server/services/manager.go

@@ -87,7 +87,7 @@ func NewManager(c *cli.Command, store store.Store, setupForge SetupForge) (Manag
 		signaturePrivateKey: signaturePrivateKey,
 		signaturePublicKey:  signaturePublicKey,
 		store:               store,
-		secret:              setupSecretService(store),
+		secret:              setupSecretService(store, c.String("secret-extension-endpoint"), client, c.Bool("secret-extension-netrc")),
 		registry:            setupRegistryService(store, c.String("docker-config"), c.String("registry-extension-endpoint"), client),
 		config:              configService,
 		environment:         environment.Parse(c.StringSlice("environment")),
@@ -101,7 +101,11 @@ func (m *manager) SignaturePublicKey() crypto.PublicKey {
 	return m.signaturePublicKey
 }
 
-func (m *manager) SecretServiceFromRepo(_ *model.Repo) secret.Service {
+func (m *manager) SecretServiceFromRepo(repo *model.Repo) secret.Service {
+	if repo.SecretExtensionEndpoint != "" {
+		return secret.NewCombined(m.secret, secret.NewHTTP(strings.TrimRight(repo.SecretExtensionEndpoint, "/"), m.client, repo.SecretExtensionNetrc))
+	}
+
 	return m.SecretService()
 }