[v0.11] cherry-picks for v0.11.1

Dockerfile

@@ -2,13 +2,12 @@
 
 ARG GO_VERSION=1.20
 ARG XX_VERSION=1.2.1
-ARG DOCKERD_VERSION=20.10.14
+
+ARG DOCKER_VERSION=24.0.2
 ARG GOTESTSUM_VERSION=v1.9.0
 ARG REGISTRY_VERSION=2.8.0
 ARG BUILDKIT_VERSION=v0.11.6
 
-FROM docker:$DOCKERD_VERSION AS dockerd-release
-
 # xx is a helper for cross-compilation
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 
@@ -25,6 +24,22 @@ FROM registry:$REGISTRY_VERSION AS registry
 
 FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit
 
+FROM gobase AS docker
+ARG TARGETPLATFORM
+ARG DOCKER_VERSION
+WORKDIR /opt/docker
+RUN DOCKER_ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
+    "linux/amd64")   echo "x86_64"  ;; \
+    "linux/arm/v6")  echo "armel"   ;; \
+    "linux/arm/v7")  echo "armhf"   ;; \
+    "linux/arm64")   echo "aarch64" ;; \
+    "linux/ppc64le") echo "ppc64le" ;; \
+    "linux/s390x")   echo "s390x"   ;; \
+    *)               echo ""        ;; esac) \
+  && echo "DOCKER_ARCH=$DOCKER_ARCH" \
+  && wget -qO- "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-${DOCKER_VERSION}.tgz" | tar xvz --strip 1
+RUN ./dockerd --version && ./containerd --version && ./ctr --version && ./runc --version
+
 FROM gobase AS gotestsum
 ARG GOTESTSUM_VERSION
 ENV GOFLAGS=
@@ -77,9 +92,20 @@ FROM binaries-$TARGETOS AS binaries
 ARG BUILDKIT_SBOM_SCAN_STAGE=true
 
 FROM gobase AS integration-test-base
-RUN apk add --no-cache docker runc containerd
+# https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies
+RUN apk add --no-cache \
+      btrfs-progs \
+      e2fsprogs \
+      e2fsprogs-extra \
+      ip6tables \
+      iptables \
+      openssl \
+      shadow-uidmap \
+      xfsprogs \
+      xz
 COPY --link --from=gotestsum /out/gotestsum /usr/bin/
 COPY --link --from=registry /bin/registry /usr/bin/
+COPY --link --from=docker /opt/docker/* /usr/bin/
 COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/
 COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/
 COPY --link --from=binaries /buildx /usr/bin/
@@ -102,7 +128,7 @@ FROM scratch AS release
 COPY --from=releaser /out/ /
 
 # Shell
-FROM docker:$DOCKERD_VERSION AS dockerd-release
+FROM docker:$DOCKER_VERSION AS dockerd-release
 FROM alpine AS shell
 RUN apk add --no-cache iptables tmux git vim less openssh
 RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/local/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx

bake/compose.go

@@ -37,6 +37,7 @@ func ParseCompose(cfgs []compose.ConfigFile, envs map[string]string) (*Config, e
 	}, func(options *loader.Options) {
 		options.SetProjectName("bake", false)
 		options.SkipNormalization = true
+		options.Profiles = []string{"*"}
 	})
 	if err != nil {
 		return nil, err

bake/compose_test.go

@@ -36,6 +36,8 @@ services:
         - token
         - aws
   webapp2:
+    profiles:
+      - test
     build:
       context: ./dir
       dockerfile_inline: |

build/build.go

@@ -642,7 +642,7 @@ func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Op
 	}
 
 	// setup extrahosts
-	extraHosts, err := toBuildkitExtraHosts(opt.ExtraHosts, nodeDriver.IsMobyDriver())
+	extraHosts, err := toBuildkitExtraHosts(ctx, opt.ExtraHosts, nodeDriver)
 	if err != nil {
 		return nil, nil, err
 	}

build/utils.go

@@ -3,10 +3,12 @@ package build
 import (
 	"archive/tar"
 	"bytes"
+	"context"
 	"net"
 	"os"
 	"strings"
 
+	"github.com/docker/buildx/driver"
 	"github.com/docker/cli/opts"
 	"github.com/docker/docker/builder/remotecontext/urlutil"
 	"github.com/moby/buildkit/util/gitutil"
@@ -57,7 +59,7 @@ func isArchive(header []byte) bool {
 }
 
 // toBuildkitExtraHosts converts hosts from docker key:value format to buildkit's csv format
-func toBuildkitExtraHosts(inp []string, mobyDriver bool) (string, error) {
+func toBuildkitExtraHosts(ctx context.Context, inp []string, nodeDriver *driver.DriverHandle) (string, error) {
 	if len(inp) == 0 {
 		return "", nil
 	}
@@ -67,11 +69,16 @@ func toBuildkitExtraHosts(inp []string, mobyDriver bool) (string, error) {
 		if !ok || host == "" || ip == "" {
 			return "", errors.Errorf("invalid host %s", h)
 		}
-		// Skip IP address validation for "host-gateway" string with moby driver
-		if !mobyDriver || ip != mobyHostGatewayName {
-			if net.ParseIP(ip) == nil {
-				return "", errors.Errorf("invalid host %s", h)
+		// If the IP Address is a "host-gateway", replace this value with the
+		// IP address provided by the worker's label.
+		if ip == mobyHostGatewayName {
+			hgip, err := nodeDriver.HostGatewayIP(ctx)
+			if err != nil {
+				return "", errors.Wrap(err, "unable to derive the IP value for host-gateway")
 			}
+			ip = hgip.String()
+		} else if net.ParseIP(ip) == nil {
+			return "", errors.Errorf("invalid host %s", h)
 		}
 		hosts = append(hosts, host+"="+ip)
 	}

commands/build.go

@@ -487,7 +487,8 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 	flags.StringArrayVar(&options.platforms, "platform", platformsDefault, "Set target platform for build")
 
 	if isExperimental() {
-		flags.StringVar(&options.printFunc, "print", "", "Print result of information request (e.g., outline, targets) [experimental]")
+		flags.StringVar(&options.printFunc, "print", "", "Print result of information request (e.g., outline, targets)")
+		flags.SetAnnotation("print", "experimentalCLI", nil)
 	}
 
 	flags.BoolVar(&options.exportPush, "push", false, `Shorthand for "--output=type=registry"`)
@@ -514,10 +515,14 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 	flags.StringVar(&options.provenance, "provenance", "", `Shorthand for "--attest=type=provenance"`)
 
 	if isExperimental() {
-		flags.StringVar(&invokeFlag, "invoke", "", "Invoke a command after the build [experimental]")
-		flags.StringVar(&options.Root, "root", "", "Specify root directory of server to connect [experimental]")
-		flags.BoolVar(&options.Detach, "detach", false, "Detach buildx server (supported only on linux) [experimental]")
-		flags.StringVar(&options.ServerConfig, "server-config", "", "Specify buildx server config file (used only when launching new server) [experimental]")
+		flags.StringVar(&invokeFlag, "invoke", "", "Invoke a command after the build")
+		flags.SetAnnotation("invoke", "experimentalCLI", nil)
+		flags.StringVar(&options.Root, "root", "", "Specify root directory of server to connect")
+		flags.SetAnnotation("root", "experimentalCLI", nil)
+		flags.BoolVar(&options.Detach, "detach", false, "Detach buildx server (supported only on linux)")
+		flags.SetAnnotation("detach", "experimentalCLI", nil)
+		flags.StringVar(&options.ServerConfig, "server-config", "", "Specify buildx server config file (used only when launching new server)")
+		flags.SetAnnotation("server-config", "experimentalCLI", nil)
 	}
 
 	// hidden flags
@@ -540,6 +545,7 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 	flags.BoolVar(&ignoreBool, "squash", false, "Squash newly built layers into a single new layer")
 	flags.MarkHidden("squash")
 	flags.SetAnnotation("squash", "flag-warn", []string{"experimental flag squash is removed with BuildKit. You should squash inside build using a multi-stage Dockerfile for efficiency."})
+	flags.SetAnnotation("squash", "experimentalCLI", nil)
 
 	flags.StringVarP(&ignore, "memory", "m", "", "Memory limit")
 	flags.MarkHidden("memory")

commands/debug-shell.go

@@ -24,6 +24,9 @@ func debugShellCmd(dockerCli command.Cli) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "debug-shell",
 		Short: "Start a monitor",
+		Annotations: map[string]string{
+			"experimentalCLI": "",
+		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			printer, err := progress.NewPrinter(context.TODO(), os.Stderr, os.Stderr, progressMode)
 			if err != nil {
@@ -55,9 +58,15 @@ func debugShellCmd(dockerCli command.Cli) *cobra.Command {
 
 	flags := cmd.Flags()
 
-	flags.StringVar(&options.Root, "root", "", "Specify root directory of server to connect [experimental]")
-	flags.BoolVar(&options.Detach, "detach", runtime.GOOS == "linux", "Detach buildx server (supported only on linux) [experimental]")
-	flags.StringVar(&options.ServerConfig, "server-config", "", "Specify buildx server config file (used only when launching new server) [experimental]")
+	flags.StringVar(&options.Root, "root", "", "Specify root directory of server to connect")
+	flags.SetAnnotation("root", "experimentalCLI", nil)
+
+	flags.BoolVar(&options.Detach, "detach", runtime.GOOS == "linux", "Detach buildx server (supported only on linux)")
+	flags.SetAnnotation("detach", "experimentalCLI", nil)
+
+	flags.StringVar(&options.ServerConfig, "server-config", "", "Specify buildx server config file (used only when launching new server)")
+	flags.SetAnnotation("server-config", "experimentalCLI", nil)
+
 	flags.StringVar(&progressMode, "progress", "auto", `Set type of progress output ("auto", "plain", "tty"). Use plain to show container output`)
 
 	return cmd

controller/build/build.go

@@ -54,6 +54,7 @@ func RunBuild(ctx context.Context, dockerCli command.Cli, in controllerapi.Build
 			NamedContexts:  contexts,
 		},
 		BuildArgs:     in.BuildArgs,
+		CgroupParent:  in.CgroupParent,
 		ExtraHosts:    in.ExtraHosts,
 		Labels:        in.Labels,
 		NetworkMode:   in.NetworkMode,

docs/reference/buildx.md

@@ -9,24 +9,21 @@ Extended build capabilities with BuildKit
 
 ### Subcommands
 
-| Name                                           | Description                                |
-|:-----------------------------------------------|:-------------------------------------------|
-| [`_INTERNAL_SERVE`](buildx__INTERNAL_SERVE.md) |                                            |
-| [`bake`](buildx_bake.md)                       | Build from a file                          |
-| [`build`](buildx_build.md)                     | Start a build                              |
-| [`create`](buildx_create.md)                   | Create a new builder instance              |
-| [`debug-shell`](buildx_debug-shell.md)         | Start a monitor                            |
-| [`du`](buildx_du.md)                           | Disk usage                                 |
-| [`imagetools`](buildx_imagetools.md)           | Commands to work on images in registry     |
-| [`inspect`](buildx_inspect.md)                 | Inspect current builder instance           |
-| [`install`](buildx_install.md)                 | Install buildx as a 'docker builder' alias |
-| [`ls`](buildx_ls.md)                           | List builder instances                     |
-| [`prune`](buildx_prune.md)                     | Remove build cache                         |
-| [`rm`](buildx_rm.md)                           | Remove a builder instance                  |
-| [`stop`](buildx_stop.md)                       | Stop builder instance                      |
-| [`uninstall`](buildx_uninstall.md)             | Uninstall the 'docker builder' alias       |
-| [`use`](buildx_use.md)                         | Set the current builder instance           |
-| [`version`](buildx_version.md)                 | Show buildx version information            |
+| Name                                   | Description                            |
+|:---------------------------------------|:---------------------------------------|
+| [`bake`](buildx_bake.md)               | Build from a file                      |
+| [`build`](buildx_build.md)             | Start a build                          |
+| [`create`](buildx_create.md)           | Create a new builder instance          |
+| [`debug-shell`](buildx_debug-shell.md) | Start a monitor                        |
+| [`du`](buildx_du.md)                   | Disk usage                             |
+| [`imagetools`](buildx_imagetools.md)   | Commands to work on images in registry |
+| [`inspect`](buildx_inspect.md)         | Inspect current builder instance       |
+| [`ls`](buildx_ls.md)                   | List builder instances                 |
+| [`prune`](buildx_prune.md)             | Remove build cache                     |
+| [`rm`](buildx_rm.md)                   | Remove a builder instance              |
+| [`stop`](buildx_stop.md)               | Stop builder instance                  |
+| [`use`](buildx_use.md)                 | Set the current builder instance       |
+| [`version`](buildx_version.md)         | Show buildx version information        |
 
 
 ### Options