External service

cmd/compose/compose.go

@@ -72,17 +72,16 @@ const (
 )
 
 // rawEnv load a dot env file using docker/cli key=value parser, without attempt to interpolate or evaluate values
-func rawEnv(r io.Reader, filename string, lookup func(key string) (string, bool)) (map[string]string, error) {
+func rawEnv(r io.Reader, filename string, vars map[string]string, lookup func(key string) (string, bool)) error {
 	lines, err := kvfile.ParseFromReader(r, lookup)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse env_file %s: %w", filename, err)
+		return fmt.Errorf("failed to parse env_file %s: %w", filename, err)
 	}
-	vars := types.Mapping{}
 	for _, line := range lines {
 		key, value, _ := strings.Cut(line, "=")
 		vars[key] = value
 	}
-	return vars, nil
+	return nil
 }
 
 func init() {

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/Microsoft/go-winio v0.6.2
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
 	github.com/buger/goterm v1.0.4
-	github.com/compose-spec/compose-go/v2 v2.4.10-0.20250319114556-312596f4c1fe
+	github.com/compose-spec/compose-go/v2 v2.5.0
 	github.com/containerd/containerd/v2 v2.0.4
 	github.com/containerd/platforms v1.0.0-rc.1
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
@@ -205,3 +205,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace github.com/compose-spec/compose-go/v2 => github.com/glours/compose-go/v2 v2.0.0-20250403082600-80aa75f06535

go.sum

@@ -83,8 +83,6 @@ github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004 h1:lkAMpLVBDaj17e
 github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=
-github.com/compose-spec/compose-go/v2 v2.4.10-0.20250319114556-312596f4c1fe h1:gl5+6pDRe/b8tbqJOXvNOZWNQe4aFLymlMV0iqFp9GI=
-github.com/compose-spec/compose-go/v2 v2.4.10-0.20250319114556-312596f4c1fe/go.mod h1:6k5l/0TxCg0/2uLEhRVEsoBWBprS2uvZi32J7xub3lo=
 github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo=
 github.com/containerd/cgroups/v3 v3.0.5/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins=
 github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro=
@@ -169,6 +167,8 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ
 github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
 github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/glours/compose-go/v2 v2.0.0-20250403082600-80aa75f06535 h1:S/P6v3QxsMpkKn+2OSMPNkfSkadSjSHoMGAc/eBZgMU=
+github.com/glours/compose-go/v2 v2.0.0-20250403082600-80aa75f06535/go.mod h1:vPlkN0i+0LjLf9rv52lodNMUTJF5YHVfHVGLLIP67NA=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=

pkg/compose/build.go

@@ -263,7 +263,7 @@ func (s *composeService) build(ctx context.Context, project *types.Project, opti
 
 func (s *composeService) ensureImagesExists(ctx context.Context, project *types.Project, buildOpts *api.BuildOptions, quietPull bool) error {
 	for name, service := range project.Services {
-		if service.Image == "" && service.Build == nil {
+		if service.Provider == nil && service.Image == "" && service.Build == nil {
 			return fmt.Errorf("invalid service %q. Must specify either image or build", name)
 		}
 	}

pkg/compose/convergence.go

@@ -110,6 +110,9 @@ func (c *convergence) apply(ctx context.Context, project *types.Project, options
 }
 
 func (c *convergence) ensureService(ctx context.Context, project *types.Project, service types.ServiceConfig, recreate string, inherit bool, timeout *time.Duration) error { //nolint:gocyclo
+	if service.Provider != nil {
+		return c.service.runPlugin(ctx, project, service, "up")
+	}
 	expected, err := getScale(service)
 	if err != nil {
 		return err

pkg/compose/down.go

@@ -83,8 +83,11 @@ func (s *composeService) down(ctx context.Context, projectName string, options a
 	}
 
 	err = InReverseDependencyOrder(ctx, project, func(c context.Context, service string) error {
-		serviceContainers := containers.filter(isService(service))
 		serv := project.Services[service]
+		if serv.Provider != nil {
+			return s.runPlugin(ctx, project, serv, "down")
+		}
+		serviceContainers := containers.filter(isService(service))
 		err := s.removeContainers(ctx, serviceContainers, &serv, options.Timeout, options.Volumes)
 		return err
 	}, WithRootNodesAndDown(options.Services))

pkg/compose/plugins.go

@@ -0,0 +1,149 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package compose
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/compose-spec/compose-go/v2/types"
+	"github.com/docker/cli/cli-plugins/manager"
+	"github.com/docker/cli/cli-plugins/socket"
+	"github.com/docker/compose/v2/pkg/progress"
+	"github.com/docker/docker/errdefs"
+	"github.com/spf13/cobra"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
+	"golang.org/x/sync/errgroup"
+)
+
+type JsonMessage struct {
+	Type    string `json:"type"`
+	Message string `json:"message"`
+}
+
+const (
+	ErrorType  = "error"
+	InfoType   = "info"
+	SetEnvType = "setenv"
+)
+
+func (s *composeService) runPlugin(ctx context.Context, project *types.Project, service types.ServiceConfig, command string) error { //nolint:gocyclo
+	x := *service.Provider
+
+	// Only support Docker CLI plugins for first iteration. Could support any binary from PATH
+	plugin, err := manager.GetPlugin(x.Type, s.dockerCli, &cobra.Command{})
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			return fmt.Errorf("unsupported external service type %s", x.Type)
+		}
+		return err
+	}
+
+	args := []string{"compose", "--project-name", project.Name, command}
+	for k, v := range x.Options {
+		args = append(args, fmt.Sprintf("--%s=%s", k, v))
+	}
+
+	cmd := exec.CommandContext(ctx, plugin.Path, args...)
+	// Remove DOCKER_CLI_PLUGIN... variable so plugin can detect it run standalone
+	cmd.Env = filter(os.Environ(), manager.ReexecEnvvar)
+
+	// Use docker/cli mechanism to propagate termination signal to child process
+	server, err := socket.NewPluginServer(nil)
+	if err != nil {
+		defer server.Close() //nolint:errcheck
+		cmd.Cancel = server.Close
+		cmd.Env = replace(cmd.Env, socket.EnvKey, server.Addr().String())
+	}
+
+	cmd.Env = append(cmd.Env, fmt.Sprintf("DOCKER_CONTEXT=%s", s.dockerCli.CurrentContext()))
+
+	// propagate opentelemetry context to child process, see https://github.com/open-telemetry/oteps/blob/main/text/0258-env-context-baggage-carriers.md
+	carrier := propagation.MapCarrier{}
+	otel.GetTextMapPropagator().Inject(ctx, &carrier)
+	cmd.Env = append(cmd.Env, types.Mapping(carrier).Values()...)
+
+	eg := errgroup.Group{}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return err
+	}
+	eg.Go(cmd.Wait)
+
+	decoder := json.NewDecoder(stdout)
+	defer func() { _ = stdout.Close() }()
+
+	variables := types.Mapping{}
+
+	pw := progress.ContextWriter(ctx)
+	pw.Event(progress.CreatingEvent(service.Name))
+	for {
+		var msg JsonMessage
+		err = decoder.Decode(&msg)
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		switch msg.Type {
+		case ErrorType:
+			pw.Event(progress.ErrorMessageEvent(service.Name, "error"))
+			return errors.New(msg.Message)
+		case InfoType:
+			pw.Event(progress.ErrorMessageEvent(service.Name, msg.Message))
+		case SetEnvType:
+			key, val, found := strings.Cut(msg.Message, "=")
+			if !found {
+				return fmt.Errorf("invalid response from plugin: %s", msg.Message)
+			}
+			variables[key] = val
+		default:
+			return fmt.Errorf("invalid response from plugin: %s", msg.Type)
+		}
+	}
+
+	err = eg.Wait()
+	if err != nil {
+		pw.Event(progress.ErrorMessageEvent(service.Name, err.Error()))
+		return fmt.Errorf("failed to create external service: %s", err.Error())
+	}
+	pw.Event(progress.CreatedEvent(service.Name))
+
+	prefix := strings.ToUpper(service.Name) + "_"
+	for name, s := range project.Services {
+		if _, ok := s.DependsOn[service.Name]; ok {
+			for key, val := range variables {
+				s.Environment[prefix+key] = &val
+			}
+			project.Services[name] = s
+		}
+	}
+	return nil
+}

pkg/compose/pull.go

@@ -335,6 +335,9 @@ func (s *composeService) pullRequiredImages(ctx context.Context, project *types.
 }
 
 func mustPull(service types.ServiceConfig, images map[string]api.ImageSummary) (bool, error) {
+	if service.Provider != nil {
+		return false, nil
+	}
 	if service.Image == "" {
 		return false, nil
 	}