Merge pull request #16 from akashsinghal/addReferrerGC

Add Referrers Garbage Collection

.github/workflows/oras-release.yml

@@ -4,7 +4,7 @@ on:
   push:
     tags:
       - "v[0-9]+.[0-9]+.[0-9]+-alpha"
-      - "v[0-9]+.[0-9]+.[0-9]+-rc"
+      - "v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"
 
 jobs:
   publish:

docs/extensions.md

@@ -0,0 +1,117 @@
+---
+description: High level discussion of extensions
+keywords: registry, extension, handlers, repository, distribution, artifacts
+title: Extensions
+---
+
+This document serves as a high level discussion of the implementation of the extensions framework defined in the [OCI Distribution spec](https://github.com/opencontainers/distribution-spec/tree/main/extensions).
+
+## Extension Interface
+
+The `Extension` interface is introduced in the new `extension` package. It defines methods to access the extension's namespace-specific attributes such as the Name, Url defining the extension namespace, and the Description of the namespace. It defines route enumeration at the Registry and Repository level. It also encases the `ExtendedStorage` interface which defines the methods requires to extend the underlying storage functionality of the registry. 
+
+```
+type Extension interface {
+	storage.ExtendedStorage
+	// GetRepositoryRoutes returns a list of extension routes scoped at a repository level
+	GetRepositoryRoutes() []ExtensionRoute
+	// GetRegistryRoutes returns a list of extension routes scoped at a registry level
+	GetRegistryRoutes() []ExtensionRoute
+	// GetNamespaceName returns the name associated with the namespace
+	GetNamespaceName() string
+	// GetNamespaceUrl returns the url link to the documentation where the namespace's extension and endpoints are defined
+	GetNamespaceUrl() string
+	// GetNamespaceDescription returns the description associated with the namespace
+	GetNamespaceDescription() string
+}
+```
+
+The `ExtendedStorage` interface defines methods that specify storage-specific handlers. Each extension will implement a handler extending the functionality. The interface can be expanded in the future to consider new handler types.
+`GetManifestHandlers` is used to return new `ManifestHandlers` defined by each of the extensions.
+`GetGarbageCollectionHandlers` is used to return `GCExtensionHandler` implemented by each extension.
+
+```
+type ExtendedStorage interface {
+	// GetManifestHandlers returns the list of manifest handlers that handle custom manifest formats supported by the extension
+	GetManifestHandlers(
+		repo Repository,
+		blobStore BlobStore) []ManifestHandler
+    // GetGarbageCollectHandlers returns the GCExtensionHandlers that handles custom garbage collection behavior for the extension.
+	GetGarbageCollectionHandlers() []GCExtensionHandler
+}
+```
+
+The `GCExtensionHandler` interface defines three methods that are used in the garbage colection mark and sweep process. The `Mark` method is invoked for each `GCExtensionHandler` after the existing mark process finishes in `MarkAndSweep`. It is used to determine if the manifest and blobs should have their temporary ref count incremented in the case of an artifact manifest, or if the manifest and it's referrers should be recursively indexed for deletion in the case of a non-artifact manifest. `OnManifestDelete` is invoked to extend the `RemoveManifest` functionality for the `Vacuum`. New or special-cased manifests may require custom manifest deletion which can be defined with this method. `SweepBlobs` is used to add artifact manifest/blobs to the original `markSet`. These blobs are retained after determining their ref count is still positive. 
+
+```
+type GCExtensionHandler interface {
+	Mark(ctx context.Context,
+		repository distribution.Repository,
+		storageDriver driver.StorageDriver,
+		registry distribution.Namespace,
+		manifest distribution.Manifest,
+		manifestDigest digest.Digest,
+		dryRun bool,
+		removeUntagged bool) (bool, error)
+	OnManifestDelete(ctx context.Context,
+		storageDriver driver.StorageDriver,
+		registry distribution.Namespace,
+		dgst digest.Digest,
+		repositoryName string) error
+	SweepBlobs(ctx context.Context,
+		markSet map[digest.Digest]struct{}) map[digest.Digest]struct{}
+}
+```
+
+## Registering Extensions
+
+Extensions are defined in the configuration yaml. 
+
+### Sample Extension Configuration YAML
+```
+# Configuration for extensions. It follows the below schema
+# extensions
+#   namespace:
+#     configuration for the extension and its components in any schema specific to that namespace
+extensions:
+  oci: 
+    ext:
+      - discover # enable the discovery extension
+```
+
+Each `Extension` defined must call the `RegisterExtension` method to register an extension initialization function with the extension namespace name. The registered extension list is then used during configuration parsing to get and initialize the specified extension. (`GetExtension`)
+
+```
+// InitExtension is the initialize function for creating the extension namespace
+type InitExtension func(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Extension, error)
+
+// RegisterExtension is used to register an InitExtension for
+// an extension with the given name.
+func RegisterExtension(name string, initFunc InitExtension)
+
+// GetExtension constructs an extension with the given options using the given name.
+func GetExtension(ctx context.Context, name string, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Extension, error)
+```
+
+Each `Extension` defines an `ExtensionRoute` which contains the new `<namespace>/<extension>/<component>` route attributes. Furthermore, the route `Descriptor` and `Dispatcher` are used to register the new route to the application. 
+
+```
+type ExtensionRoute struct {
+	// Namespace is the name of the extension namespace
+	Namespace string
+	// Extension is the name of the extension under the namespace
+	Extension string
+	// Component is the name of the component under the extension
+	Component string
+	// Descriptor is the route descriptor that gives its path
+	Descriptor v2.RouteDescriptor
+	// Dispatcher if present signifies that the route is http route with a dispatcher
+	Dispatcher RouteDispatchFunc
+}
+
+// RouteDispatchFunc is the http route dispatcher used by the extension route handlers
+type RouteDispatchFunc func(extContext *ExtensionContext, r *http.Request) http.Handler
+```
+
+
+

docs/garbage-collection.md

@@ -121,4 +121,4 @@ blob eligible for deletion: sha256:7e15ce58ccb2181a8fced7709e9893206f0937cc9543b
 blob eligible for deletion: sha256:87192bdbe00f8f2a62527f36bb4c7c7f4eaf9307e4b87e8334fb6abec1765bcb
 blob eligible for deletion: sha256:b549a9959a664038fc35c155a95742cf12297672ca0ae35735ec027d55bf4e97
 blob eligible for deletion: sha256:f251d679a7c61455f06d793e43c06786d7766c88b8c24edf242b2c08e3c3f599
-```
+```

registry/extension/distribution/manifests.go

@@ -19,7 +19,7 @@ type manifestsGetAPIResponse struct {
 
 // manifestHandler handles requests for manifests under a manifest name.
 type manifestHandler struct {
-	*extension.Context
+	*extension.ExtensionContext
 	storageDriver driver.StorageDriver
 }
 

registry/extension/distribution/registry.go

@@ -34,7 +34,7 @@ type distributionOptions struct {
 }
 
 // newDistNamespace creates a new extension namespace with the name "distribution"
-func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Namespace, error) {
+func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Extension, error) {
 
 	optionsYaml, err := yaml.Marshal(options)
 	if err != nil {
@@ -67,7 +67,7 @@ func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, o
 
 func init() {
 	// register the extension namespace.
-	extension.Register(namespaceName, newDistNamespace)
+	extension.RegisterExtension(namespaceName, newDistNamespace)
 }
 
 // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store.
@@ -76,12 +76,17 @@ func (o *distributionNamespace) GetManifestHandlers(repo distribution.Repository
 	return []storage.ManifestHandler{}
 }
 
+func (o *distributionNamespace) GetGarbageCollectionHandlers() []storage.GCExtensionHandler {
+	// This extension doesn't extend any garbage collection operations.
+	return []storage.GCExtensionHandler{}
+}
+
 // GetRepositoryRoutes returns a list of extension routes scoped at a repository level
-func (d *distributionNamespace) GetRepositoryRoutes() []extension.Route {
-	var routes []extension.Route
+func (d *distributionNamespace) GetRepositoryRoutes() []extension.ExtensionRoute {
+	var routes []extension.ExtensionRoute
 
 	if d.manifestsEnabled {
-		routes = append(routes, extension.Route{
+		routes = append(routes, extension.ExtensionRoute{
 			Namespace: namespaceName,
 			Extension: extensionName,
 			Component: manifestsComponentName,
@@ -100,7 +105,7 @@ func (d *distributionNamespace) GetRepositoryRoutes() []extension.Route {
 	}
 
 	if d.tagHistoryEnabled {
-		routes = append(routes, extension.Route{
+		routes = append(routes, extension.ExtensionRoute{
 			Namespace: namespaceName,
 			Extension: extensionName,
 			Component: tagHistoryComponentName,
@@ -134,7 +139,7 @@ func (d *distributionNamespace) GetRepositoryRoutes() []extension.Route {
 
 // GetRegistryRoutes returns a list of extension routes scoped at a registry level
 // There are no registry scoped routes exposed by this namespace
-func (d *distributionNamespace) GetRegistryRoutes() []extension.Route {
+func (d *distributionNamespace) GetRegistryRoutes() []extension.ExtensionRoute {
 	return nil
 }
 
@@ -153,21 +158,21 @@ func (d *distributionNamespace) GetNamespaceDescription() string {
 	return namespaceDescription
 }
 
-func (d *distributionNamespace) tagHistoryDispatcher(ctx *extension.Context, r *http.Request) http.Handler {
+func (d *distributionNamespace) tagHistoryDispatcher(ctx *extension.ExtensionContext, r *http.Request) http.Handler {
 	tagHistoryHandler := &tagHistoryHandler{
-		Context:       ctx,
-		storageDriver: d.storageDriver,
+		ExtensionContext: ctx,
+		storageDriver:    d.storageDriver,
 	}
 
 	return handlers.MethodHandler{
 		"GET": http.HandlerFunc(tagHistoryHandler.getTagManifestDigests),
 	}
 }
 
-func (d *distributionNamespace) manifestsDispatcher(ctx *extension.Context, r *http.Request) http.Handler {
+func (d *distributionNamespace) manifestsDispatcher(ctx *extension.ExtensionContext, r *http.Request) http.Handler {
 	manifestsHandler := &manifestHandler{
-		Context:       ctx,
-		storageDriver: d.storageDriver,
+		ExtensionContext: ctx,
+		storageDriver:    d.storageDriver,
 	}
 
 	return handlers.MethodHandler{

registry/extension/distribution/taghistory.go

@@ -20,7 +20,7 @@ type tagHistoryAPIResponse struct {
 
 // manifestHandler handles requests for manifests under a manifest name.
 type tagHistoryHandler struct {
-	*extension.Context
+	*extension.ExtensionContext
 	storageDriver driver.StorageDriver
 }
 

registry/extension/extension.go

@@ -1,7 +1,7 @@
 package extension
 
 import (
-	c "context"
+	"context"
 	"fmt"
 	"net/http"
 
@@ -13,9 +13,9 @@ import (
 	"github.com/distribution/distribution/v3/registry/storage/driver"
 )
 
-// Context contains the request specific context for use in across handlers.
-type Context struct {
-	c.Context
+// ExtensionContext contains the request specific context for use in across handlers.
+type ExtensionContext struct {
+	context.Context
 
 	// Registry is the base namespace that is used by all extension namespaces
 	Registry distribution.Namespace
@@ -26,10 +26,10 @@ type Context struct {
 }
 
 // RouteDispatchFunc is the http route dispatcher used by the extension route handlers
-type RouteDispatchFunc func(extContext *Context, r *http.Request) http.Handler
+type RouteDispatchFunc func(extContext *ExtensionContext, r *http.Request) http.Handler
 
-// Route describes an extension route.
-type Route struct {
+// ExtensionRoute describes an extension route.
+type ExtensionRoute struct {
 	// Namespace is the name of the extension namespace
 	Namespace string
 	// Extension is the name of the extension under the namespace
@@ -42,13 +42,14 @@ type Route struct {
 	Dispatcher RouteDispatchFunc
 }
 
-// Namespace is the namespace that is used to define extensions to the distribution.
-type Namespace interface {
+// Extension is the interface that is used to define extensions to the distribution.
+type Extension interface {
 	storage.ExtendedStorage
+	// ExtensionService
 	// GetRepositoryRoutes returns a list of extension routes scoped at a repository level
-	GetRepositoryRoutes() []Route
+	GetRepositoryRoutes() []ExtensionRoute
 	// GetRegistryRoutes returns a list of extension routes scoped at a registry level
-	GetRegistryRoutes() []Route
+	GetRegistryRoutes() []ExtensionRoute
 	// GetNamespaceName returns the name associated with the namespace
 	GetNamespaceName() string
 	// GetNamespaceUrl returns the url link to the documentation where the namespace's extension and endpoints are defined
@@ -57,8 +58,8 @@ type Namespace interface {
 	GetNamespaceDescription() string
 }
 
-// InitExtensionNamespace is the initialize function for creating the extension namespace
-type InitExtensionNamespace func(ctx c.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Namespace, error)
+// InitExtension is the initialize function for creating the extension namespace
+type InitExtension func(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Extension, error)
 
 // EnumerateExtension specifies extension information at the namespace level
 type EnumerateExtension struct {
@@ -68,10 +69,10 @@ type EnumerateExtension struct {
 	Endpoints   []string `json:"endpoints"`
 }
 
-var extensions map[string]InitExtensionNamespace
-var extensionsNamespaces map[string]Namespace
+var extensions map[string]InitExtension
+var extensionsNamespaces map[string]Extension
 
-func EnumerateRegistered(ctx Context) (enumeratedExtensions []EnumerateExtension) {
+func EnumerateRegistered(ctx ExtensionContext) (enumeratedExtensions []EnumerateExtension) {
 	for _, namespace := range extensionsNamespaces {
 		enumerateExtension := EnumerateExtension{
 			Name:        fmt.Sprintf("_%s", namespace.GetNamespaceName()),
@@ -102,11 +103,11 @@ func EnumerateRegistered(ctx Context) (enumeratedExtensions []EnumerateExtension
 	return enumeratedExtensions
 }
 
-// Register is used to register an InitExtensionNamespace for
-// an extension namespace with the given name.
-func Register(name string, initFunc InitExtensionNamespace) {
+// RegisterExtension is used to register an InitExtension for
+// an extension with the given name.
+func RegisterExtension(name string, initFunc InitExtension) {
 	if extensions == nil {
-		extensions = make(map[string]InitExtensionNamespace)
+		extensions = make(map[string]InitExtension)
 	}
 
 	if _, exists := extensions[name]; exists {
@@ -116,11 +117,11 @@ func Register(name string, initFunc InitExtensionNamespace) {
 	extensions[name] = initFunc
 }
 
-// Get constructs an extension namespace with the given options using the given name.
-func Get(ctx c.Context, name string, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Namespace, error) {
+// GetExtension constructs an extension with the given options using the given name.
+func GetExtension(ctx context.Context, name string, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Extension, error) {
 	if extensions != nil {
 		if extensionsNamespaces == nil {
-			extensionsNamespaces = make(map[string]Namespace)
+			extensionsNamespaces = make(map[string]Extension)
 		}
 
 		if initFunc, exists := extensions[name]; exists {

registry/extension/oci/discover.go

@@ -16,7 +16,7 @@ type discoverGetAPIResponse struct {
 
 // extensionHandler handles requests for manifests under a manifest name.
 type extensionHandler struct {
-	*extension.Context
+	*extension.ExtensionContext
 	storageDriver driver.StorageDriver
 }
 
@@ -26,7 +26,7 @@ func (eh *extensionHandler) getExtensions(w http.ResponseWriter, r *http.Request
 	w.Header().Set("Content-Type", "application/json")
 
 	// get list of extension information seperated at the namespace level
-	enumeratedExtensions := extension.EnumerateRegistered(*eh.Context)
+	enumeratedExtensions := extension.EnumerateRegistered(*eh.ExtensionContext)
 
 	// remove the oci extension so it's not returned by discover
 	ociExtensionName := fmt.Sprintf("_%s", namespaceName)