From ad70be14757a3661349abf2df5884391e0cb8901 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang <103478229+wangxiaoxuan273@users.noreply.github.com> Date: Fri, 25 Nov 2022 12:43:44 +0800 Subject: [PATCH] feat: index referrers for image manifests (#41) Signed-off-by: wangxiaoxuan273 --- manifest/ocischema/builder.go | 8 ++++- manifest/ocischema/builder_test.go | 2 +- manifest/ocischema/manifest.go | 7 ++++ registry/storage/manifeststore_test.go | 2 +- .../storage/ociartifactmanifesthandler.go | 8 +++-- registry/storage/ocimanifesthandler.go | 35 +++++++++++++++---- registry/storage/registry.go | 9 ++--- 7 files changed, 56 insertions(+), 15 deletions(-) diff --git a/manifest/ocischema/builder.go b/manifest/ocischema/builder.go index 7d99c184f33..618a0fd51b0 100644 --- a/manifest/ocischema/builder.go +++ b/manifest/ocischema/builder.go @@ -22,6 +22,10 @@ type Builder struct { // calls to AppendReference. layers []distribution.Descriptor + // Subject specifies the descriptor of another manifest. This value is + // used by the referrers API. + subject *distribution.Descriptor + // Annotations contains arbitrary metadata relating to the targeted content. annotations map[string]string @@ -32,10 +36,11 @@ type Builder struct { // NewManifestBuilder is used to build new manifests for the current schema // version. It takes a BlobService so it can publish the configuration blob // as part of the Build process, and annotations. -func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder { +func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, subject *distribution.Descriptor, annotations map[string]string) distribution.ManifestBuilder { mb := &Builder{ bs: bs, configJSON: make([]byte, len(configJSON)), + subject: subject, annotations: annotations, mediaType: v1.MediaTypeImageManifest, } @@ -63,6 +68,7 @@ func (mb *Builder) Build(ctx context.Context) (distribution.Manifest, error) { MediaType: mb.mediaType, }, Layers: make([]distribution.Descriptor, len(mb.layers)), + Subject: mb.subject, Annotations: mb.annotations, } copy(m.Layers, mb.layers) diff --git a/manifest/ocischema/builder_test.go b/manifest/ocischema/builder_test.go index 2e511393c9e..66b9f3ec7d2 100644 --- a/manifest/ocischema/builder_test.go +++ b/manifest/ocischema/builder_test.go @@ -126,7 +126,7 @@ func TestBuilder(t *testing.T) { annotations := map[string]string{"hot": "potato"} bs := &mockBlobService{descriptors: make(map[digest.Digest]distribution.Descriptor)} - builder := NewManifestBuilder(bs, imgJSON, annotations) + builder := NewManifestBuilder(bs, imgJSON, nil, annotations) for _, d := range descriptors { if err := builder.AppendReference(d); err != nil { diff --git a/manifest/ocischema/manifest.go b/manifest/ocischema/manifest.go index e76e490705c..31c20455bde 100644 --- a/manifest/ocischema/manifest.go +++ b/manifest/ocischema/manifest.go @@ -51,6 +51,10 @@ type Manifest struct { // configuration. Layers []distribution.Descriptor `json:"layers"` + // Subject specifies the descriptor of another manifest. This value is + // used by the referrers API. + Subject *distribution.Descriptor `json:"subject,omitempty"` + // Annotations contains arbitrary metadata for the image manifest. Annotations map[string]string `json:"annotations,omitempty"` } @@ -60,6 +64,9 @@ func (m Manifest) References() []distribution.Descriptor { references := make([]distribution.Descriptor, 0, 1+len(m.Layers)) references = append(references, m.Config) references = append(references, m.Layers...) + if m.Subject != nil { + references = append(references, *m.Subject) + } return references } diff --git a/registry/storage/manifeststore_test.go b/registry/storage/manifeststore_test.go index 33a39c4ff4e..0b3fea6f087 100644 --- a/registry/storage/manifeststore_test.go +++ b/registry/storage/manifeststore_test.go @@ -407,7 +407,7 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo // Build a manifest and store it and its layers in the registry blobStore := env.repository.Blobs(ctx) - builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{}) + builder := ocischema.NewManifestBuilder(blobStore, []byte{}, nil, map[string]string{}) err = builder.(*ocischema.Builder).SetMediaType(imageMediaType) if err != nil { t.Fatal(err) diff --git a/registry/storage/ociartifactmanifesthandler.go b/registry/storage/ociartifactmanifesthandler.go index 37c3d39239f..52752006760 100644 --- a/registry/storage/ociartifactmanifesthandler.go +++ b/registry/storage/ociartifactmanifesthandler.go @@ -130,9 +130,13 @@ func (ms *ociArtifactManifestHandler) indexReferrers(ctx context.Context, dm *oc // but need to consider the max path length in different os subjectRevision := dm.Subject.Digest - referrersLinkPath, err := pathFor(referrersLinkPathSpec{name: ms.repository.Named().Name(), revision: revision, subjectRevision: subjectRevision}) + return indexWithSubject(ctx, ms.repository.Named().Name(), revision, subjectRevision, ms.storageDriver) +} + +func indexWithSubject(ctx context.Context, repo string, revision digest.Digest, subjectRevision digest.Digest, sd driver.StorageDriver) error { + referrersLinkPath, err := pathFor(referrersLinkPathSpec{name: repo, revision: revision, subjectRevision: subjectRevision}) if err != nil { return fmt.Errorf("failed to generate referrers link path for %v", revision) } - return ms.storageDriver.PutContent(ctx, referrersLinkPath, []byte(revision.String())) + return sd.PutContent(ctx, referrersLinkPath, []byte(revision.String())) } diff --git a/registry/storage/ocimanifesthandler.go b/registry/storage/ocimanifesthandler.go index 2735a03eca7..05706719c46 100644 --- a/registry/storage/ocimanifesthandler.go +++ b/registry/storage/ocimanifesthandler.go @@ -7,17 +7,21 @@ import ( "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" + "github.com/distribution/distribution/v3/manifest/manifestlist" "github.com/distribution/distribution/v3/manifest/ocischema" + "github.com/distribution/distribution/v3/manifest/schema2" + "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" v1 "github.com/opencontainers/image-spec/specs-go/v1" ) -//ocischemaManifestHandler is a ManifestHandler that covers ocischema manifests. +// ocischemaManifestHandler is a ManifestHandler that covers ocischema manifests. type ocischemaManifestHandler struct { - repository distribution.Repository - blobStore distribution.BlobStore - ctx context.Context - manifestURLs manifestURLs + repository distribution.Repository + blobStore distribution.BlobStore + ctx context.Context + manifestURLs manifestURLs + storageDriver driver.StorageDriver } var _ ManifestHandler = &ocischemaManifestHandler{} @@ -56,6 +60,12 @@ func (ms *ocischemaManifestHandler) Put(ctx context.Context, manifest distributi return "", err } + err = ms.indexReferrers(ctx, m, revision.Digest) + if err != nil { + dcontext.GetLogger(ctx).Errorf("error indexing referrers: %v", err) + return "", err + } + return revision.Digest, nil } @@ -109,7 +119,7 @@ func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst oc } } - case v1.MediaTypeImageManifest: + case v1.MediaTypeImageManifest, v1.MediaTypeArtifactManifest, v1.MediaTypeImageIndex, schema2.MediaTypeManifest, manifestlist.MediaTypeManifestList: var exists bool exists, err = manifestService.Exists(ctx, descriptor.Digest) if err != nil || !exists { @@ -141,3 +151,16 @@ func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst oc return nil } + +// indexReferrers indexes the subject of the given revision in its referrers index store. +func (ms *ocischemaManifestHandler) indexReferrers(ctx context.Context, dm *ocischema.DeserializedManifest, revision digest.Digest) error { + if dm.Subject == nil { + return nil + } + + // [TODO] We can use artifact type in the link path to support filtering by artifact type + // but need to consider the max path length in different os + subjectRevision := dm.Subject.Digest + + return indexWithSubject(ctx, ms.repository.Named().Name(), revision, subjectRevision, ms.storageDriver) +} diff --git a/registry/storage/registry.go b/registry/storage/registry.go index de316845dbc..a8eb0bb8a3d 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -283,10 +283,11 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M blobStore: blobStore, }, ocischemaHandler: &ocischemaManifestHandler{ - ctx: ctx, - repository: repo, - blobStore: blobStore, - manifestURLs: repo.registry.manifestURLs, + ctx: ctx, + repository: repo, + blobStore: blobStore, + manifestURLs: repo.registry.manifestURLs, + storageDriver: repo.driver, }, ociartifactHandler: &ociArtifactManifestHandler{ repository: repo,