feat: index referrers for image manifests (#41)

Signed-off-by: wangxiaoxuan273 <[email protected]>

manifest/ocischema/builder.go

@@ -22,6 +22,10 @@ type Builder struct {
 	// calls to AppendReference.
 	layers []distribution.Descriptor
 
+	// Subject specifies the descriptor of another manifest. This value is
+	// used by the referrers API.
+	subject *distribution.Descriptor
+
 	// Annotations contains arbitrary metadata relating to the targeted content.
 	annotations map[string]string
 
@@ -32,10 +36,11 @@ type Builder struct {
 // NewManifestBuilder is used to build new manifests for the current schema
 // version. It takes a BlobService so it can publish the configuration blob
 // as part of the Build process, and annotations.
-func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder {
+func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, subject *distribution.Descriptor, annotations map[string]string) distribution.ManifestBuilder {
 	mb := &Builder{
 		bs:          bs,
 		configJSON:  make([]byte, len(configJSON)),
+		subject:     subject,
 		annotations: annotations,
 		mediaType:   v1.MediaTypeImageManifest,
 	}
@@ -63,6 +68,7 @@ func (mb *Builder) Build(ctx context.Context) (distribution.Manifest, error) {
 			MediaType:     mb.mediaType,
 		},
 		Layers:      make([]distribution.Descriptor, len(mb.layers)),
+		Subject:     mb.subject,
 		Annotations: mb.annotations,
 	}
 	copy(m.Layers, mb.layers)

manifest/ocischema/builder_test.go

@@ -126,7 +126,7 @@ func TestBuilder(t *testing.T) {
 	annotations := map[string]string{"hot": "potato"}
 
 	bs := &mockBlobService{descriptors: make(map[digest.Digest]distribution.Descriptor)}
-	builder := NewManifestBuilder(bs, imgJSON, annotations)
+	builder := NewManifestBuilder(bs, imgJSON, nil, annotations)
 
 	for _, d := range descriptors {
 		if err := builder.AppendReference(d); err != nil {

manifest/ocischema/manifest.go

@@ -51,6 +51,10 @@ type Manifest struct {
 	// configuration.
 	Layers []distribution.Descriptor `json:"layers"`
 
+	// Subject specifies the descriptor of another manifest. This value is
+	// used by the referrers API.
+	Subject *distribution.Descriptor `json:"subject,omitempty"`
+
 	// Annotations contains arbitrary metadata for the image manifest.
 	Annotations map[string]string `json:"annotations,omitempty"`
 }
@@ -60,6 +64,9 @@ func (m Manifest) References() []distribution.Descriptor {
 	references := make([]distribution.Descriptor, 0, 1+len(m.Layers))
 	references = append(references, m.Config)
 	references = append(references, m.Layers...)
+	if m.Subject != nil {
+		references = append(references, *m.Subject)
+	}
 	return references
 }
 

registry/storage/manifeststore_test.go

@@ -407,7 +407,7 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
 	// Build a manifest and store it and its layers in the registry
 
 	blobStore := env.repository.Blobs(ctx)
-	builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{})
+	builder := ocischema.NewManifestBuilder(blobStore, []byte{}, nil, map[string]string{})
 	err = builder.(*ocischema.Builder).SetMediaType(imageMediaType)
 	if err != nil {
 		t.Fatal(err)

registry/storage/ociartifactmanifesthandler.go

@@ -130,9 +130,13 @@ func (ms *ociArtifactManifestHandler) indexReferrers(ctx context.Context, dm *oc
 	//  but need to consider the max path length in different os
 	subjectRevision := dm.Subject.Digest
 
-	referrersLinkPath, err := pathFor(referrersLinkPathSpec{name: ms.repository.Named().Name(), revision: revision, subjectRevision: subjectRevision})
+	return indexWithSubject(ctx, ms.repository.Named().Name(), revision, subjectRevision, ms.storageDriver)
+}
+
+func indexWithSubject(ctx context.Context, repo string, revision digest.Digest, subjectRevision digest.Digest, sd driver.StorageDriver) error {
+	referrersLinkPath, err := pathFor(referrersLinkPathSpec{name: repo, revision: revision, subjectRevision: subjectRevision})
 	if err != nil {
 		return fmt.Errorf("failed to generate referrers link path for %v", revision)
 	}
-	return ms.storageDriver.PutContent(ctx, referrersLinkPath, []byte(revision.String()))
+	return sd.PutContent(ctx, referrersLinkPath, []byte(revision.String()))
 }

registry/storage/ocimanifesthandler.go

@@ -7,17 +7,21 @@ import (
 
 	"github.com/distribution/distribution/v3"
 	dcontext "github.com/distribution/distribution/v3/context"
+	"github.com/distribution/distribution/v3/manifest/manifestlist"
 	"github.com/distribution/distribution/v3/manifest/ocischema"
+	"github.com/distribution/distribution/v3/manifest/schema2"
+	"github.com/distribution/distribution/v3/registry/storage/driver"
 	"github.com/opencontainers/go-digest"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-//ocischemaManifestHandler is a ManifestHandler that covers ocischema manifests.
+// ocischemaManifestHandler is a ManifestHandler that covers ocischema manifests.
 type ocischemaManifestHandler struct {
-	repository   distribution.Repository
-	blobStore    distribution.BlobStore
-	ctx          context.Context
-	manifestURLs manifestURLs
+	repository    distribution.Repository
+	blobStore     distribution.BlobStore
+	ctx           context.Context
+	manifestURLs  manifestURLs
+	storageDriver driver.StorageDriver
 }
 
 var _ ManifestHandler = &ocischemaManifestHandler{}
@@ -56,6 +60,12 @@ func (ms *ocischemaManifestHandler) Put(ctx context.Context, manifest distributi
 		return "", err
 	}
 
+	err = ms.indexReferrers(ctx, m, revision.Digest)
+	if err != nil {
+		dcontext.GetLogger(ctx).Errorf("error indexing referrers: %v", err)
+		return "", err
+	}
+
 	return revision.Digest, nil
 }
 
@@ -109,7 +119,7 @@ func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst oc
 				}
 			}
 
-		case v1.MediaTypeImageManifest:
+		case v1.MediaTypeImageManifest, v1.MediaTypeArtifactManifest, v1.MediaTypeImageIndex, schema2.MediaTypeManifest, manifestlist.MediaTypeManifestList:
 			var exists bool
 			exists, err = manifestService.Exists(ctx, descriptor.Digest)
 			if err != nil || !exists {
@@ -141,3 +151,16 @@ func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst oc
 
 	return nil
 }
+
+// indexReferrers indexes the subject of the given revision in its referrers index store.
+func (ms *ocischemaManifestHandler) indexReferrers(ctx context.Context, dm *ocischema.DeserializedManifest, revision digest.Digest) error {
+	if dm.Subject == nil {
+		return nil
+	}
+
+	// [TODO] We can use artifact type in the link path to support filtering by artifact type
+	//  but need to consider the max path length in different os
+	subjectRevision := dm.Subject.Digest
+
+	return indexWithSubject(ctx, ms.repository.Named().Name(), revision, subjectRevision, ms.storageDriver)
+}

registry/storage/registry.go

@@ -283,10 +283,11 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M
 			blobStore:  blobStore,
 		},
 		ocischemaHandler: &ocischemaManifestHandler{
-			ctx:          ctx,
-			repository:   repo,
-			blobStore:    blobStore,
-			manifestURLs: repo.registry.manifestURLs,
+			ctx:           ctx,
+			repository:    repo,
+			blobStore:     blobStore,
+			manifestURLs:  repo.registry.manifestURLs,
+			storageDriver: repo.driver,
 		},
 		ociartifactHandler: &ociArtifactManifestHandler{
 			repository:    repo,