Populate content/index.json manifest list

Signed-off-by: Clarence "Sparr" Risher <[email protected]>
awslabs · Aug 14, 2023 · 8413676 · 8413676
1 parent bde429d
commit 8413676
Show file tree

Hide file tree

Showing 9 changed files with 48 additions and 16 deletions.
diff --git a/cmd/soci/commands/create.go b/cmd/soci/commands/create.go
@@ -86,10 +86,11 @@ var CreateCommand = cli.Command{
 			return err
 		}
 
-		ctx, blobStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+		ctx, blobStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 		if err != nil {
 			return err
 		}
+		defer done(ctx)
 
 		ps, err := internal.GetPlatforms(ctx, cliContext, srcImg, cs)
 		if err != nil {

diff --git a/cmd/soci/commands/index/info.go b/cmd/soci/commands/index/info.go
@@ -53,10 +53,13 @@ var infoCommand = cli.Command{
 		}
 		ctx, cancel := context.WithTimeout(context.Background(), cliContext.GlobalDuration("timeout"))
 		defer cancel()
-		ctx, store, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+
+		ctx, store, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 		if err != nil {
 			return err
 		}
+		defer done(ctx)
+
 		reader, err := store.Fetch(ctx, v1.Descriptor{Digest: digest})
 		if err != nil {
 			return err

diff --git a/cmd/soci/commands/index/rm.go b/cmd/soci/commands/index/rm.go
@@ -47,10 +47,11 @@ var rmCommand = cli.Command{
 		}
 
 		ctx := context.Background()
-		ctx, contentStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+		ctx, contentStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 		if err != nil {
 			return fmt.Errorf("cannot create local content store: %w", err)
 		}
+		defer done(ctx)
 
 		db, err := soci.NewDB(soci.ArtifactsDbPath())
 		if err != nil {

diff --git a/cmd/soci/commands/push.go b/cmd/soci/commands/push.go
@@ -132,10 +132,11 @@ if they are available in the snapshotter's local content store.
 			}, nil
 		}
 
-		ctx, src, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+		ctx, src, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 		if err != nil {
 			return fmt.Errorf("cannot create local content store: %w", err)
 		}
+		defer done(ctx)
 
 		dst.Client = authClient
 		dst.PlainHTTP = cliContext.Bool("plain-http")

diff --git a/cmd/soci/commands/rebuild_db.go b/cmd/soci/commands/rebuild_db.go
@@ -39,10 +39,11 @@ var RebuildDBCommand = cli.Command{
 		if err != nil {
 			return err
 		}
-		ctx, blobStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+		ctx, blobStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 		if err != nil {
 			return err
 		}
+		defer done(ctx)
 
 		contentStorePath, err := store.GetContentStorePath(store.ContentStoreType(cliContext.GlobalString("content-store")))
 		if err != nil {

diff --git a/cmd/soci/commands/ztoc/get-file.go b/cmd/soci/commands/ztoc/get-file.go
@@ -87,10 +87,11 @@ var getFileCommand = cli.Command{
 }
 
 func getZtoc(ctx context.Context, cliContext *cli.Context, d digest.Digest) (*ztoc.Ztoc, error) {
-	ctx, blobStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+	ctx, blobStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 	if err != nil {
 		return nil, err
 	}
+	defer done(ctx)
 
 	reader, err := blobStore.Fetch(ctx, v1.Descriptor{Digest: d})
 	if err != nil {

diff --git a/cmd/soci/commands/ztoc/info.go b/cmd/soci/commands/ztoc/info.go
@@ -72,10 +72,13 @@ var infoCommand = cli.Command{
 		}
 		ctx, cancel := context.WithTimeout(context.Background(), cliContext.GlobalDuration("timeout"))
 		defer cancel()
-		ctx, store, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
+
+		ctx, store, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
 		if err != nil {
 			return err
 		}
+		defer done(ctx)
+
 		reader, err := store.Fetch(ctx, v1.Descriptor{Digest: digest})
 		if err != nil {
 			return err

diff --git a/fs/fs.go b/fs/fs.go
@@ -141,10 +141,12 @@ func NewFilesystem(ctx context.Context, root string, cfg config.FSConfig, opts .
 			return docker.ConfigureDefaultRegistries(docker.WithPlainHTTP(docker.MatchLocalhost))(refspec.Hostname())
 		})
 	}
-	ctx, store, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cfg.ContentStoreConfig.Type)), store.WithNamespace(cfg.ContentStoreConfig.Namespace))
+	ctx, store, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cfg.ContentStoreConfig.Type)), store.WithNamespace(cfg.ContentStoreConfig.Namespace))
 	if err != nil {
 		return nil, fmt.Errorf("cannot create content store: %w", err)
 	}
+	// FIXME: This runs far too soon. Writing index.json for the soci content store should happen after each change to the store and/or when the snapshotter shuts down.
+	defer done(ctx)
 
 	var bgFetcher *bf.BackgroundFetcher
 

diff --git a/soci/store/store.go b/soci/store/store.go
@@ -136,20 +136,20 @@ type CleanupFunc func(context.Context) error
 
 func nopCleanup(context.Context) error { return nil }
 
-func NewContentStore(ctx context.Context, opts ...Option) (context.Context, Store, error) {
+func NewContentStore(ctx context.Context, opts ...Option) (context.Context, Store, CleanupFunc, error) {
 	storeConfig := NewStoreConfig(opts...)
 
 	contentStoreType, err := CanonicalizeContentStoreType(ContentStoreType(storeConfig.Type))
 	if err != nil {
-		return ctx, nil, err
+		return ctx, nil, nopCleanup, err
 	}
 	switch contentStoreType {
 	case ContainerdContentStoreType:
 		return NewContainerdStore(ctx, storeConfig)
 	case SociContentStoreType:
 		return NewSociStore(ctx)
 	}
-	return ctx, nil, errors.New("unexpectedly reached end of NewContentStore")
+	return ctx, nil, nopCleanup, errors.New("unexpectedly reached end of NewContentStore")
 }
 
 // SociStore wraps oci.Store and adds or stubs additional functionality of the Store interface.
@@ -161,9 +161,18 @@ type SociStore struct {
 var _ Store = (*SociStore)(nil)
 
 // NewSociStore creates a sociStore.
-func NewSociStore(ctx context.Context) (context.Context, *SociStore, error) {
+func NewSociStore(ctx context.Context) (context.Context, *SociStore, CleanupFunc, error) {
 	store, err := oci.New(DefaultSociContentStorePath)
-	return ctx, &SociStore{store}, err
+	return ctx, &SociStore{store}, func(_ context.Context) error { return store.SaveIndex() }, err
+}
+
+// Push adds a new content item to the sociStore then tags it for indexing
+func (s *SociStore) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error {
+	if err := s.Store.Push(ctx, expected, reader); err != nil {
+		return err
+	}
+
+	return s.Tag(ctx, expected, expected.Digest.String())
 }
 
 // Label is a no-op for sociStore until sociStore and ArtifactsDb are better integrated.
@@ -181,6 +190,11 @@ func (s *SociStore) BatchOpen(ctx context.Context) (context.Context, CleanupFunc
 	return ctx, nopCleanup, nil
 }
 
+// Cleanup on a sociStore will update the content/index.json file
+func (s *SociStore) Cleanup(ctx context.Context) error {
+	return s.SaveIndex()
+}
+
 type ContainerdStore struct {
 	config.ContentStoreConfig
 	client *containerd.Client
@@ -189,10 +203,10 @@ type ContainerdStore struct {
 // assert that ContainerdStore implements Store
 var _ Store = (*ContainerdStore)(nil)
 
-func NewContainerdStore(ctx context.Context, storeConfig config.ContentStoreConfig) (context.Context, *ContainerdStore, error) {
+func NewContainerdStore(ctx context.Context, storeConfig config.ContentStoreConfig) (context.Context, *ContainerdStore, CleanupFunc, error) {
 	client, err := containerd.New(config.DefaultImageServiceAddress)
 	if err != nil {
-		return ctx, nil, fmt.Errorf("could not connect to containerd socket for content store access: %w", err)
+		return ctx, nil, nopCleanup, fmt.Errorf("could not connect to containerd socket for content store access: %w", err)
 	}
 
 	ctx = namespaces.WithNamespace(ctx, storeConfig.Namespace)
@@ -203,7 +217,7 @@ func NewContainerdStore(ctx context.Context, storeConfig config.ContentStoreConf
 
 	containerdStore.ContentStoreConfig = storeConfig
 
-	return ctx, &containerdStore, nil
+	return ctx, &containerdStore, nopCleanup, nil
 }
 
 // Exists returns true iff the described content exists.
@@ -333,3 +347,8 @@ func (s *ContainerdStore) BatchOpen(ctx context.Context) (context.Context, Clean
 	}
 	return ctx, leaseDone, nil
 }
+
+// Cleanup is a no-op for ContainerdStore
+func (s *ContainerdStore) Cleanup(ctx context.Context) error {
+	return nil
+}