Sourcepolicy

api/services/control/control.proto

@@ -8,6 +8,7 @@ import "github.com/moby/buildkit/solver/pb/ops.proto";
 import "github.com/moby/buildkit/api/types/worker.proto";
 // import "github.com/containerd/containerd/api/types/descriptor.proto";
 import "github.com/gogo/googleapis/google/rpc/status.proto";
+import "github.com/moby/buildkit/sourcepolicy/pb/policy.proto";
 
 option (gogoproto.sizer_all) = true;
 option (gogoproto.marshaler_all) = true;
@@ -68,6 +69,7 @@ message SolveRequest {
 	repeated string Entitlements = 9 [(gogoproto.customtype) = "github.com/moby/buildkit/util/entitlements.Entitlement" ];
 	map<string, pb.Definition> FrontendInputs = 10;
 	bool Internal = 11; // Internal builds are not recorded in build history
+	moby.buildkit.v1.sourcepolicy.Policy SourcePolicy = 12;
 }
 
 message CacheOptions {
@@ -230,4 +232,4 @@ message BuildResultInfo {
 message Exporter {
 	string Type = 1;
 	map<string, string> Attrs = 2;
-}
+}

client/client_test.go

@@ -47,6 +47,9 @@ import (
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/solver/result"
+	"github.com/moby/buildkit/sourcepolicy"
+	sourcepolicypb "github.com/moby/buildkit/sourcepolicy/pb"
+	spb "github.com/moby/buildkit/sourcepolicy/pb"
 	"github.com/moby/buildkit/util/attestation"
 	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
 	"github.com/moby/buildkit/util/contentutil"
@@ -188,6 +191,7 @@ func TestIntegration(t *testing.T) {
 		testMultipleCacheExports,
 		testMountStubsDirectory,
 		testMountStubsTimestamp,
+		testSourcePolicy,
 	)
 }
 
@@ -8473,3 +8477,132 @@ func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser
 		return wc, nil
 	}
 }
+
+func testSourcePolicy(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+		st := llb.Image("busybox:1.34.1-uclibc").File(
+			llb.Copy(llb.HTTP("https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md"),
+				"README.md", "README.md"))
+		def, err := st.Marshal(sb.Context())
+		if err != nil {
+			return nil, err
+		}
+		return c.Solve(ctx, gateway.SolveRequest{
+			Definition: def.ToPB(),
+		})
+	}
+
+	type testCase struct {
+		srcPol      *sourcepolicypb.Policy
+		expectedErr string
+	}
+	testCases := []testCase{
+		{
+			// Valid
+			srcPol: &sourcepolicypb.Policy{
+				Rules: []*sourcepolicypb.Rule{
+					{
+						Action: sourcepolicypb.PolicyAction_CONVERT,
+						Selector: &sourcepolicypb.Selector{
+							Identifier: "docker-image://docker.io/library/busybox:1.34.1-uclibc",
+						},
+						Updates: &sourcepolicypb.Update{
+							Identifier: "docker-image://docker.io/library/busybox:1.34.1-uclibc@sha256:3614ca5eacf0a3a1bcc361c939202a974b4902b9334ff36eb29ffe9011aaad83",
+						},
+					},
+					{
+						Action: sourcepolicypb.PolicyAction_CONVERT,
+						Selector: &sourcepolicypb.Selector{
+							Identifier: "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+						},
+						Updates: &sourcepolicypb.Update{
+							Identifier: "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+							Attrs:      map[string]string{"http.checksum": "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53"},
+						},
+					},
+				},
+			},
+			expectedErr: "",
+		},
+		{
+			// Invalid docker-image source
+			srcPol: &sourcepolicypb.Policy{
+				Rules: []*sourcepolicypb.Rule{
+					{
+						Action: sourcepolicypb.PolicyAction_CONVERT,
+						Selector: &sourcepolicypb.Selector{
+							Identifier: "docker-image://docker.io/library/busybox:1.34.1-uclibc",
+						},
+						Updates: &sourcepolicypb.Update{
+							Identifier: "docker-image://docker.io/library/busybox:1.34.1-uclibc@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", // invalid
+						},
+					},
+				},
+			},
+			expectedErr: "docker.io/library/busybox:1.34.1-uclibc@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: not found",
+		},
+		{
+			// Invalid http source
+			srcPol: &sourcepolicypb.Policy{
+				Rules: []*sourcepolicypb.Rule{
+					{
+						Action: sourcepolicypb.PolicyAction_CONVERT,
+						Selector: &sourcepolicypb.Selector{
+							Identifier: "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+						},
+						Updates: &sourcepolicypb.Update{
+							Attrs: map[string]string{pb.AttrHTTPChecksum: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, // invalid
+						},
+					},
+				},
+			},
+			expectedErr: "digest mismatch sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53: sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+		},
+	}
+	for i, tc := range testCases {
+		tc := tc
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			_, err = c.Build(sb.Context(), SolveOpt{SourcePolicy: tc.srcPol}, "", frontend, nil)
+			if tc.expectedErr == "" {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.expectedErr)
+			}
+		})
+	}
+
+	t.Run("Frontend policies", func(t *testing.T) {
+		denied := "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md"
+		frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+			st := llb.Image("busybox:1.34.1-uclibc").File(
+				llb.Copy(llb.HTTP(denied),
+					"README.md", "README.md"))
+			def, err := st.Marshal(sb.Context())
+			if err != nil {
+				return nil, err
+			}
+			return c.Solve(ctx, gateway.SolveRequest{
+				Definition: def.ToPB(),
+				SourcePolicies: []*spb.Policy{{
+					Rules: []*spb.Rule{
+						{
+							Action: spb.PolicyAction_DENY,
+							Selector: &spb.Selector{
+								Identifier: denied,
+							},
+						},
+					},
+				}},
+			})
+		}
+
+		_, err = c.Build(sb.Context(), SolveOpt{}, "", frontend, nil)
+		require.ErrorContains(t, err, sourcepolicy.ErrSourceDenied.Error())
+	})
+}

client/llb/llbtest/platform_test.go

@@ -27,7 +27,7 @@ func TestCustomPlatform(t *testing.T) {
 	def, err := s.Marshal(context.TODO())
 	require.NoError(t, err)
 
-	e, err := llbsolver.Load(def.ToPB())
+	e, err := llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 
 	require.Equal(t, depth(e), 5)
@@ -56,7 +56,7 @@ func TestDefaultPlatform(t *testing.T) {
 	def, err := s.Marshal(context.TODO())
 	require.NoError(t, err)
 
-	e, err := llbsolver.Load(def.ToPB())
+	e, err := llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 
 	require.Equal(t, depth(e), 2)
@@ -80,7 +80,7 @@ func TestPlatformOnMarshal(t *testing.T) {
 	def, err := s.Marshal(context.TODO(), llb.Windows)
 	require.NoError(t, err)
 
-	e, err := llbsolver.Load(def.ToPB())
+	e, err := llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 
 	expected := ocispecs.Platform{OS: "windows", Architecture: "amd64"}
@@ -100,7 +100,7 @@ func TestPlatformMixed(t *testing.T) {
 	def, err := s1.Marshal(context.TODO(), llb.LinuxAmd64)
 	require.NoError(t, err)
 
-	e, err := llbsolver.Load(def.ToPB())
+	e, err := llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 
 	require.Equal(t, depth(e), 4)
@@ -129,7 +129,7 @@ func TestFallbackPath(t *testing.T) {
 	// the cap.
 	def, err := llb.Scratch().Run(llb.Shlex("cmd")).Marshal(context.TODO(), llb.LinuxAmd64)
 	require.NoError(t, err)
-	e, err := llbsolver.Load(def.ToPB())
+	e, err := llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 	require.False(t, def.Metadata[e.Vertex.Digest()].Caps[pb.CapExecMetaSetsDefaultPath])
 	_, ok := getenv(e, "PATH")
@@ -141,7 +141,7 @@ func TestFallbackPath(t *testing.T) {
 	require.Error(t, cs.Supports(pb.CapExecMetaSetsDefaultPath))
 	def, err = llb.Scratch().Run(llb.Shlex("cmd")).Marshal(context.TODO(), llb.LinuxAmd64, llb.WithCaps(cs))
 	require.NoError(t, err)
-	e, err = llbsolver.Load(def.ToPB())
+	e, err = llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 	require.False(t, def.Metadata[e.Vertex.Digest()].Caps[pb.CapExecMetaSetsDefaultPath])
 	v, ok := getenv(e, "PATH")
@@ -155,7 +155,7 @@ func TestFallbackPath(t *testing.T) {
 	require.NoError(t, cs.Supports(pb.CapExecMetaSetsDefaultPath))
 	def, err = llb.Scratch().Run(llb.Shlex("cmd")).Marshal(context.TODO(), llb.LinuxAmd64, llb.WithCaps(cs))
 	require.NoError(t, err)
-	e, err = llbsolver.Load(def.ToPB())
+	e, err = llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 	require.True(t, def.Metadata[e.Vertex.Digest()].Caps[pb.CapExecMetaSetsDefaultPath])
 	_, ok = getenv(e, "PATH")
@@ -171,7 +171,7 @@ func TestFallbackPath(t *testing.T) {
 	} {
 		def, err = llb.Scratch().AddEnv("PATH", "foo").Run(llb.Shlex("cmd")).Marshal(context.TODO(), append(cos, llb.LinuxAmd64)...)
 		require.NoError(t, err)
-		e, err = llbsolver.Load(def.ToPB())
+		e, err = llbsolver.Load(context.TODO(), def.ToPB(), nil)
 		require.NoError(t, err)
 		// pb.CapExecMetaSetsDefaultPath setting is irrelevant (and variable).
 		v, ok = getenv(e, "PATH")

client/solve.go

@@ -22,6 +22,7 @@ import (
 	"github.com/moby/buildkit/session/filesync"
 	"github.com/moby/buildkit/session/grpchijack"
 	"github.com/moby/buildkit/solver/pb"
+	spb "github.com/moby/buildkit/sourcepolicy/pb"
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/entitlements"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -47,6 +48,7 @@ type SolveOpt struct {
 	SharedSession         *session.Session // TODO: refactor to better session syncing
 	SessionPreInitialized bool             // TODO: refactor to better session syncing
 	Internal              bool
+	SourcePolicy          *spb.Policy
 }
 
 type ExportEntry struct {
@@ -261,6 +263,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			Cache:          cacheOpt.options,
 			Entitlements:   opt.AllowedEntitlements,
 			Internal:       opt.Internal,
+			SourcePolicy:   opt.SourcePolicy,
 		})
 		if err != nil {
 			return errors.Wrap(err, "failed to solve")

cmd/buildctl/build.go

@@ -20,6 +20,7 @@ import (
 	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/moby/buildkit/session/sshforward/sshprovider"
 	"github.com/moby/buildkit/solver/pb"
+	spb "github.com/moby/buildkit/sourcepolicy/pb"
 	"github.com/moby/buildkit/util/progress/progresswriter"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
@@ -95,6 +96,10 @@ var buildCommand = cli.Command{
 			Name:  "metadata-file",
 			Usage: "Output build metadata (e.g., image digest) to a file as JSON",
 		},
+		cli.StringFlag{
+			Name:  "source-policy-file",
+			Usage: "Read source policy file from a JSON file",
+		},
 	},
 }
 
@@ -189,6 +194,19 @@ func buildAction(clicontext *cli.Context) error {
 		return err
 	}
 
+	var srcPol *spb.Policy
+	if srcPolFile := clicontext.String("source-policy-file"); srcPolFile != "" {
+		b, err := os.ReadFile(srcPolFile)
+		if err != nil {
+			return err
+		}
+		var srcPolStruct spb.Policy
+		if err := json.Unmarshal(b, &srcPolStruct); err != nil {
+			return errors.Wrapf(err, "failed to unmarshal source-policy-file %q", srcPolFile)
+		}
+		srcPol = &srcPolStruct
+	}
+
 	eg, ctx := errgroup.WithContext(bccommon.CommandContext(clicontext))
 
 	solveOpt := client.SolveOpt{
@@ -201,6 +219,7 @@ func buildAction(clicontext *cli.Context) error {
 		CacheImports:        cacheImports,
 		Session:             attachable,
 		AllowedEntitlements: allowed,
+		SourcePolicy:        srcPol,
 	}
 
 	solveOpt.FrontendAttrs, err = build.ParseOpt(clicontext.StringSlice("opt"))

control/control.go

@@ -412,7 +412,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
 	}, llbsolver.ExporterRequest{
 		Exporter:       expi,
 		CacheExporters: cacheExporters,
-	}, req.Entitlements, procs, req.Internal)
+	}, req.Entitlements, procs, req.Internal, req.SourcePolicy)
 	if err != nil {
 		return nil, err
 	}

docs/build-repro.md

@@ -127,3 +127,43 @@ jq '.' metadata.json
   "containerimage.digest": "sha256:..."
 }
 ```
+
+### Reproducing the pinned dependencies
+
+<!-- TODO: s/master/v0.11/ after the release -->
+Reproducing the pinned dependencies is supported in the master branch of BuildKit.
+
+e.g.,
+```bash
+buildctl build --frontend dockerfile.v0 --local dockerfile=. --local context=. --source-policy-file policy.json
+```
+
+An example `policy.json`:
+```json
+{
+  "rules": [
+    {
+      "action": "CONVERT",
+      "source": {
+        "type": "docker-image",
+        "identifier": "docker.io/library/alpine:latest"
+      },
+      "destination": {
+        "identifier": "docker-image://docker.io/library/alpine:latest@sha256:4edbd2beb5f78b1014028f4fbb99f3237d9561100b6881aabbf5acce2c4f9454"
+      }
+    },
+    {
+      "action": "CONVERT",
+      "source": {
+        "type": "http",
+        "identifier": "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md"
+      },
+      "destination": {
+        "attrs": {"http.checksum": "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53"}
+      }
+    }
+  ]
+}
+```
+
+Any source type is supported, but how to pin a source depends on the type.

frontend/gateway/client/client.go

@@ -8,6 +8,7 @@ import (
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/solver/result"
+	spb "github.com/moby/buildkit/sourcepolicy/pb"
 	"github.com/moby/buildkit/util/apicaps"
 	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -128,6 +129,7 @@ type SolveRequest struct {
 	FrontendOpt    map[string]string
 	FrontendInputs map[string]*pb.Definition
 	CacheImports   []CacheOptionsEntry
+	SourcePolicies []*spb.Policy
 }
 
 type CacheOptionsEntry struct {

frontend/gateway/forwarder/forward.go

@@ -62,6 +62,7 @@ func (c *bridgeClient) Solve(ctx context.Context, req client.SolveRequest) (*cli
 		FrontendOpt:    req.FrontendOpt,
 		FrontendInputs: req.FrontendInputs,
 		CacheImports:   req.CacheImports,
+		SourcePolicies: req.SourcePolicies,
 	}, c.sid)
 	if err != nil {
 		return nil, c.wrapSolveError(err)