Populate content/index.json manifest list

Signed-off-by: Clarence "Sparr" Risher <[email protected]>
awslabs · Aug 14, 2023 · 9b3e9f6 · 9b3e9f6
1 parent bde429d
commit 9b3e9f6
Show file tree

Hide file tree

Showing 9 changed files with 62 additions and 26 deletions.
diff --git a/cmd/soci/commands/create.go b/cmd/soci/commands/create.go
@@ -86,10 +86,11 @@ var CreateCommand = cli.Command{
 			return err
 		}
 
-		ctx, blobStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+		ctx, blobStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 		if err != nil {
 			return err
 		}
+		defer done(ctx)
 
 		ps, err := internal.GetPlatforms(ctx, cliContext, srcImg, cs)
 		if err != nil {

diff --git a/cmd/soci/commands/index/info.go b/cmd/soci/commands/index/info.go
@@ -53,10 +53,13 @@ var infoCommand = cli.Command{
 		}
 		ctx, cancel := context.WithTimeout(context.Background(), cliContext.GlobalDuration("timeout"))
 		defer cancel()
-		ctx, store, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+
+		ctx, store, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 		if err != nil {
 			return err
 		}
+		defer done(ctx)
+
 		reader, err := store.Fetch(ctx, v1.Descriptor{Digest: digest})
 		if err != nil {
 			return err

diff --git a/cmd/soci/commands/index/rm.go b/cmd/soci/commands/index/rm.go
@@ -47,10 +47,11 @@ var rmCommand = cli.Command{
 		}
 
 		ctx := context.Background()
-		ctx, contentStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+		ctx, contentStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 		if err != nil {
 			return fmt.Errorf("cannot create local content store: %w", err)
 		}
+		defer done(ctx)
 
 		db, err := soci.NewDB(soci.ArtifactsDbPath())
 		if err != nil {

diff --git a/cmd/soci/commands/push.go b/cmd/soci/commands/push.go
@@ -132,10 +132,11 @@ if they are available in the snapshotter's local content store.
 			}, nil
 		}
 
-		ctx, src, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+		ctx, src, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 		if err != nil {
 			return fmt.Errorf("cannot create local content store: %w", err)
 		}
+		defer done(ctx)
 
 		dst.Client = authClient
 		dst.PlainHTTP = cliContext.Bool("plain-http")

diff --git a/cmd/soci/commands/rebuild_db.go b/cmd/soci/commands/rebuild_db.go
@@ -39,10 +39,11 @@ var RebuildDBCommand = cli.Command{
 		if err != nil {
 			return err
 		}
-		ctx, blobStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+		ctx, blobStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 		if err != nil {
 			return err
 		}
+		defer done(ctx)
 
 		contentStorePath, err := store.GetContentStorePath(store.ContentStoreType(cliContext.GlobalString("content-store")))
 		if err != nil {

diff --git a/cmd/soci/commands/ztoc/get-file.go b/cmd/soci/commands/ztoc/get-file.go
@@ -87,10 +87,11 @@ var getFileCommand = cli.Command{
 }
 
 func getZtoc(ctx context.Context, cliContext *cli.Context, d digest.Digest) (*ztoc.Ztoc, error) {
-	ctx, blobStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+	ctx, blobStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 	if err != nil {
 		return nil, err
 	}
+	defer done(ctx)
 
 	reader, err := blobStore.Fetch(ctx, v1.Descriptor{Digest: d})
 	if err != nil {

diff --git a/cmd/soci/commands/ztoc/info.go b/cmd/soci/commands/ztoc/info.go
@@ -72,10 +72,13 @@ var infoCommand = cli.Command{
 		}
 		ctx, cancel := context.WithTimeout(context.Background(), cliContext.GlobalDuration("timeout"))
 		defer cancel()
-		ctx, store, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+
+		ctx, store, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 		if err != nil {
 			return err
 		}
+		defer done(ctx)
+
 		reader, err := store.Fetch(ctx, v1.Descriptor{Digest: digest})
 		if err != nil {
 			return err

diff --git a/fs/fs.go b/fs/fs.go
@@ -141,10 +141,12 @@ func NewFilesystem(ctx context.Context, root string, cfg config.FSConfig, opts .
 			return docker.ConfigureDefaultRegistries(docker.WithPlainHTTP(docker.MatchLocalhost))(refspec.Hostname())
 		})
 	}
-	ctx, store, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cfg.ContentStoreConfig.Type)), store.WithNamespace(cfg.ContentStoreConfig.Namespace))
+	ctx, store, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cfg.ContentStoreConfig.Type)), store.WithNamespace(cfg.ContentStoreConfig.Namespace))
 	if err != nil {
 		return nil, fmt.Errorf("cannot create content store: %w", err)
 	}
+	// FIXME: This runs far too soon. Writing index.json for the soci content store should happen after each change to the store and/or when the snapshotter shuts down.
+	defer done(ctx)
 
 	var bgFetcher *bf.BackgroundFetcher
 

diff --git a/soci/store/store.go b/soci/store/store.go
@@ -136,20 +136,20 @@ type CleanupFunc func(context.Context) error
 
 func nopCleanup(context.Context) error { return nil }
 
-func NewContentStore(ctx context.Context, opts ...Option) (context.Context, Store, error) {
+func NewContentStore(ctx context.Context, opts ...Option) (context.Context, Store, CleanupFunc, error) {
 	storeConfig := NewStoreConfig(opts...)
 
 	contentStoreType, err := CanonicalizeContentStoreType(ContentStoreType(storeConfig.Type))
 	if err != nil {
-		return ctx, nil, err
+		return ctx, nil, nopCleanup, err
 	}
 	switch contentStoreType {
 	case ContainerdContentStoreType:
 		return NewContainerdStore(ctx, storeConfig)
 	case SociContentStoreType:
 		return NewSociStore(ctx)
 	}
-	return ctx, nil, errors.New("unexpectedly reached end of NewContentStore")
+	return ctx, nil, nopCleanup, errors.New("unexpectedly reached end of NewContentStore")
 }
 
 // SociStore wraps oci.Store and adds or stubs additional functionality of the Store interface.
@@ -161,9 +161,22 @@ type SociStore struct {
 var _ Store = (*SociStore)(nil)
 
 // NewSociStore creates a sociStore.
-func NewSociStore(ctx context.Context) (context.Context, *SociStore, error) {
+func NewSociStore(ctx context.Context) (context.Context, *SociStore, CleanupFunc, error) {
 	store, err := oci.New(DefaultSociContentStorePath)
-	return ctx, &SociStore{store}, err
+	return ctx, &SociStore{store}, func(_ context.Context) error { return store.SaveIndex() }, err
+}
+
+// Push adds a new content item to the sociStore then tags it for indexing
+func (s *SociStore) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error {
+	if err := s.Store.Push(ctx, expected, reader); err != nil {
+		return err
+	}
+
+	if err := s.Tag(ctx, expected, expected.Digest.String()); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // Label is a no-op for sociStore until sociStore and ArtifactsDb are better integrated.
@@ -181,33 +194,38 @@ func (s *SociStore) BatchOpen(ctx context.Context) (context.Context, CleanupFunc
 	return ctx, nopCleanup, nil
 }
 
-type ContainerdStore struct {
+// Cleanup on a sociStore will update the content/index.json file
+func (s *SociStore) Cleanup(ctx context.Context) error {
+	return s.SaveIndex()
+}
+
+type containerdStore struct {
 	config.ContentStoreConfig
 	client *containerd.Client
 }
 
-// assert that ContainerdStore implements Store
-var _ Store = (*ContainerdStore)(nil)
+// assert that containerdStore implements Store
+var _ Store = (*containerdStore)(nil)
 
-func NewContainerdStore(ctx context.Context, storeConfig config.ContentStoreConfig) (context.Context, *ContainerdStore, error) {
+func NewContainerdStore(ctx context.Context, storeConfig config.ContentStoreConfig) (context.Context, *containerdStore, CleanupFunc, error) {
 	client, err := containerd.New(config.DefaultImageServiceAddress)
 	if err != nil {
-		return ctx, nil, fmt.Errorf("could not connect to containerd socket for content store access: %w", err)
+		return ctx, nil, nopCleanup, fmt.Errorf("could not connect to containerd socket for content store access: %w", err)
 	}
 
 	ctx = namespaces.WithNamespace(ctx, storeConfig.Namespace)
 
-	containerdStore := ContainerdStore{
+	containerdStore := containerdStore{
 		client: client,
 	}
 
 	containerdStore.ContentStoreConfig = storeConfig
 
-	return ctx, &containerdStore, nil
+	return ctx, &containerdStore, nopCleanup, nil
 }
 
 // Exists returns true iff the described content exists.
-func (s *ContainerdStore) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) {
+func (s *containerdStore) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) {
 	ctx = namespaces.WithNamespace(ctx, s.Namespace)
 	cs := s.client.ContentStore()
 	_, err := cs.Info(ctx, target.Digest)
@@ -226,7 +244,7 @@ type sectionReaderAt struct {
 }
 
 // Fetch fetches the content identified by the descriptor.
-func (s *ContainerdStore) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) {
+func (s *containerdStore) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) {
 	ctx = namespaces.WithNamespace(ctx, s.Namespace)
 	cs := s.client.ContentStore()
 	ra, err := cs.ReaderAt(ctx, target)
@@ -238,7 +256,7 @@ func (s *ContainerdStore) Fetch(ctx context.Context, target ocispec.Descriptor)
 
 // Push pushes the content, matching the expected descriptor.
 // This should be done within a Batch and followed by Label calls to prevent garbage collection.
-func (s *ContainerdStore) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error {
+func (s *containerdStore) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error {
 	ctx = namespaces.WithNamespace(ctx, s.Namespace)
 	exists, err := s.Exists(ctx, expected)
 	if err != nil {
@@ -302,7 +320,7 @@ func LabelGCRefContent(ctx context.Context, store Store, target ocispec.Descript
 }
 
 // Label creates or updates the named label with the given value.
-func (s *ContainerdStore) Label(ctx context.Context, target ocispec.Descriptor, name string, value string) error {
+func (s *containerdStore) Label(ctx context.Context, target ocispec.Descriptor, name string, value string) error {
 	ctx = namespaces.WithNamespace(ctx, s.Namespace)
 	cs := s.client.ContentStore()
 	info := content.Info{
@@ -318,18 +336,23 @@ func (s *ContainerdStore) Label(ctx context.Context, target ocispec.Descriptor,
 }
 
 // Delete removes the described content.
-func (s *ContainerdStore) Delete(ctx context.Context, dgst digest.Digest) error {
+func (s *containerdStore) Delete(ctx context.Context, dgst digest.Digest) error {
 	ctx = namespaces.WithNamespace(ctx, s.Namespace)
 	cs := s.client.ContentStore()
 	return cs.Delete(ctx, dgst)
 }
 
 // BatchOpen creates a lease, ensuring that no content created within the batch will be garbage collected.
 // It returns a cleanup function that ends the lease, which should be called after content is created and labeled.
-func (s *ContainerdStore) BatchOpen(ctx context.Context) (context.Context, CleanupFunc, error) {
+func (s *containerdStore) BatchOpen(ctx context.Context) (context.Context, CleanupFunc, error) {
 	ctx, leaseDone, err := s.client.WithLease(ctx)
 	if err != nil {
 		return ctx, nopCleanup, fmt.Errorf("unable to open batch: %w", err)
 	}
 	return ctx, leaseDone, nil
 }
+
+// Cleanup is a no-op for containerdStore
+func (s *containerdStore) Cleanup(ctx context.Context) error {
+	return nil
+}