From 6c4958bcd933cb76e5f743c2699ea6dfb936fa4f Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 10 Nov 2022 15:27:13 +0900 Subject: [PATCH 1/4] integration: fix TestSecurityModeSysfs on cgroup v2 Signed-off-by: Akihiro Suda (cherry picked from commit 030483aecf63f7094153e59b4d7ade979035668d) Signed-off-by: Sebastiaan van Stijn --- client/client_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 33ed640fcecc..d489a7071ee7 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -660,7 +660,12 @@ func testSecurityModeSysfs(t *testing.T, sb integration.Sandbox) { require.NoError(t, err) defer c.Close() - command := `mkdir /sys/fs/cgroup/cpuset/securitytest` + cg := "/sys/fs/cgroup/cpuset/securitytest" // cgroup v1 + if _, err := os.Stat("/sys/fs/cgroup/cpuset"); errors.Is(err, os.ErrNotExist) { + cg = "/sys/fs/cgroup/securitytest" // cgroup v2 + } + + command := "mkdir " + cg st := llb.Image("busybox:latest"). Run(llb.Shlex(command), llb.Security(mode)) @@ -675,7 +680,7 @@ func testSecurityModeSysfs(t *testing.T, sb integration.Sandbox) { if secMode == securitySandbox { require.Error(t, err) require.Contains(t, err.Error(), "executor failed running") - require.Contains(t, err.Error(), "mkdir /sys/fs/cgroup/cpuset/securitytest") + require.Contains(t, err.Error(), "mkdir "+cg) } else { require.NoError(t, err) } From 55b69a2e4da06db793f7186bf6a224197e33e499 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 27 Dec 2022 18:10:34 +0100 Subject: [PATCH 2/4] frontend: fix testMultiStageImplicitFrom to account for busybox changes It looks like there's some changes between `busybox:1.34.0` and up; version 1.34.0 of the image did not have a `/usr/bin` directory (only `/usr/sbin`); docker run --rm -it busybox:1.34.0 ls -al /usr/ total 12 drwxr-xr-x 3 root root 4096 Sep 13 2021 . drwxr-xr-x 1 root root 4096 Dec 27 14:45 .. drwxr-xr-x 2 daemon daemon 4096 Sep 13 2021 sbin But 1.34.1 and up do; docker run --rm -it busybox:1.34.1 ls -al usr/ total 16 drwxr-xr-x 4 root root 4096 Dec 21 18:28 . drwxr-xr-x 1 root root 4096 Dec 27 14:44 .. drwxr-xr-x 2 root root 4096 Dec 21 18:28 bin drwxr-xr-x 2 daemon daemon 4096 Dec 21 18:28 sbin It's not immediately apparent what caused this change, or if it's in busybox itself, or in the official image only; https://github.com/mirror/busybox/compare/1_34_0...1_34_1 But either way, this change caused a test to fail: sandbox.go:238: time="2022-12-27T13:45:25.294022820Z" level=debug msg="> creating 4gr5bno8rj7l3k7h9jxe3jhal [/bin/sh -c mkdir /usr/bin && echo -n foo > /usr/bin/go]" span="[golang 2/2] RUN mkdir /usr/bin && echo -n foo > /usr/bin/go" sandbox.go:238: time="2022-12-27T13:45:25.433886983Z" level=debug msg="sandbox set key processing took 70.062631ms for container 5b4o358g2ryquk4s6ami38gqo" sandbox.go:238: mkdir: can't create directory '/usr/bin': File exists Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 34f9898f3112cac7c899e41d45ecdeb1502c3131) Signed-off-by: Sebastiaan van Stijn --- frontend/dockerfile/dockerfile_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 882679f9376f..1b01ae927b03 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -3595,7 +3595,7 @@ COPY --from=busybox /etc/passwd test dockerfile = []byte(` FROM busybox AS golang -RUN mkdir /usr/bin && echo -n foo > /usr/bin/go +RUN mkdir -p /usr/bin && echo -n foo > /usr/bin/go FROM scratch COPY --from=golang /usr/bin/go go From 6078223dab858ca86ed81a91016b399b882e4dab Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 10 Nov 2022 10:22:58 +0900 Subject: [PATCH 3/4] integration: add ENTRYPOINT to fix nested cgroup v2 See https://github.com/moby/moby/blob/v20.10.21/hack/dind#L28-L38 Fix issue 3265 Signed-off-by: Akihiro Suda (cherry picked from commit e5516c22a08f25a15dc7477b12259039767b09f2) --- Dockerfile | 4 ++++ hack/test | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8a049eed4045..2c7a1f66cc13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -259,6 +259,10 @@ RUN apt-get --no-install-recommends install -y uidmap sudo vim iptables fuse \ && mkdir -m 0700 -p /run/user/1000 \ && chown -R user /run/user/1000 /home/user \ && update-alternatives --set iptables /usr/sbin/iptables-legacy +# The entrypoint script is needed for enabling nested cgroup v2 (https://github.com/moby/buildkit/issues/3265#issuecomment-1309631736) +RUN curl -Ls https://raw.githubusercontent.com/moby/moby/v20.10.21/hack/dind > /docker-entrypoint.sh \ + && chmod 0755 /docker-entrypoint.sh +ENTRYPOINT ["/docker-entrypoint.sh"] # musl is needed to directly use the registry binary that is built on alpine ENV BUILDKIT_INTEGRATION_CONTAINERD_EXTRA="containerd-1.3=/opt/containerd-alt/bin" ENV BUILDKIT_INTEGRATION_SNAPSHOTTER=stargz diff --git a/hack/test b/hack/test index f1d365e52675..89c43519799a 100755 --- a/hack/test +++ b/hack/test @@ -78,7 +78,7 @@ fi if [ "$TEST_GATEWAY" == 1 ]; then # Build-test "github.com/moby/buildkit/frontend/gateway/client", which isn't otherwise built by CI # It really only needs buildkit-base. We have integration-tests in $iid, which is a direct child of buildkit-base. - cid=$(docker create --rm --volumes-from=$cacheVolume $iid go build -v ./frontend/gateway/client) + cid=$(docker create --rm --volumes-from=$cacheVolume --entrypoint="" $iid go build -v ./frontend/gateway/client) docker start -a $cid fi From 53788532fe9a130162fbb0066cc64bf631a627af Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 3 Jan 2023 11:41:52 +0100 Subject: [PATCH 4/4] [0.8] integration: skip TestSecurityModeSysfs (insecure mode) This branch is only used for vendoring in moby/moby, which doesn't use this. ``` === CONT TestIntegration/TestSecurityModeSysfs/worker=oci/secmode=insecure client_test.go:685: Error Trace: client_test.go:685 run.go:171 Error: Received unexpected error: rpc error: code = Unknown desc = executor failed running [mkdir /sys/fs/cgroup/securitytest]: exit code: 1 github.com/moby/buildkit/util/stack.Enable /src/util/stack/stack.go:77 github.com/moby/buildkit/util/grpcerrors.FromGRPC /src/util/grpcerrors/grpcerrors.go:188 github.com/moby/buildkit/util/grpcerrors.UnaryClientInterceptor /src/util/grpcerrors/intercept.go:41 google.golang.org/grpc.(*ClientConn).Invoke /src/vendor/google.golang.org/grpc/call.go:35 github.com/moby/buildkit/api/services/control.(*controlClient).Solve /src/api/services/control/control.pb.go:1321 github.com/moby/buildkit/client.(*Client).solve.func2 /src/client/solve.go:201 golang.org/x/sync/errgroup.(*Group).Go.func1 /src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:1571 failed to solve github.com/moby/buildkit/client.(*Client).solve.func2 /src/client/solve.go:214 golang.org/x/sync/errgroup.(*Group).Go.func1 /src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:1571 Test: TestIntegration/TestSecurityModeSysfs/worker=oci/secmode=insecure ... ... sandbox.go:223: time="2023-01-03T10:10:16Z" level=debug msg="> creating 501jemvwbacokctnpm9779p7y [mkdir /sys/fs/cgroup/securitytest]" sandbox.go:223: mkdir: can't create directory '/sys/fs/cgroup/securitytest': File exists ``` Signed-off-by: CrazyMax --- client/client_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/client_test.go b/client/client_test.go index d489a7071ee7..7d99f43d170a 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -651,6 +651,7 @@ func testSecurityModeSysfs(t *testing.T, sb integration.Sandbox) { if secMode == securitySandbox { allowedEntitlements = []entitlements.Entitlement{} } else { + t.Skip("Failing on 0.8 branch; see https://github.com/moby/buildkit/pull/3439") skipDockerd(t, sb) mode = llb.SecurityModeInsecure allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}