diff --git a/copy/copy.go b/copy/copy.go index 512e643b9c..fb7a8f8e92 100644 --- a/copy/copy.go +++ b/copy/copy.go @@ -16,7 +16,7 @@ import ( "github.com/containers/image/v5/image" internalblobinfocache "github.com/containers/image/v5/internal/blobinfocache" "github.com/containers/image/v5/internal/pkg/platform" - internalTypes "github.com/containers/image/v5/internal/types" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/blobinfocache" "github.com/containers/image/v5/pkg/compression" @@ -1233,9 +1233,9 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to // Note: the storage destination optimizes the committing of // layers which requires passing the index of the layer. // Hence, we need to special case and cast. - dest, ok := ic.c.dest.(internalTypes.ImageDestinationWithOptions) + dest, ok := ic.c.dest.(private.ImageDestination) if ok { - options := internalTypes.TryReusingBlobOptions{ + options := private.TryReusingBlobOptions{ Cache: ic.c.blobInfoCache, CanSubstitute: ic.canSubstituteBlobs, SrcRef: srcRef, @@ -1286,8 +1286,8 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to // of the source file are not known yet and must be fetched. // Attempt a partial only when the source allows to retrieve a blob partially and // the destination has support for it. - imgSource, okSource := ic.c.rawSource.(internalTypes.ImageSourceSeekable) - imgDest, okDest := ic.c.dest.(internalTypes.ImageDestinationPartial) + imgSource, okSource := ic.c.rawSource.(private.ImageSourceSeekable) + imgDest, okDest := ic.c.dest.(private.ImageDestinationPartial) if okSource && okDest && !diffIDIsNeeded { if reused, blobInfo := func() (bool, types.BlobInfo) { // A scope for defer bar := ic.c.createProgressBar(pool, true, srcInfo, "blob", "done") @@ -1662,9 +1662,9 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr // Note: the storage destination optimizes the committing of layers // which requires passing the index of the layer. Hence, we need to // special case and cast. - dest, ok := c.dest.(internalTypes.ImageDestinationWithOptions) + dest, ok := c.dest.(private.ImageDestination) if ok { - options := internalTypes.PutBlobOptions{ + options := private.PutBlobOptions{ Cache: c.blobInfoCache, IsConfig: isConfig, EmptyLayer: emptyLayer, diff --git a/copy/progress_reader.go b/copy/progress_reader.go index 42f490d326..3c1b952836 100644 --- a/copy/progress_reader.go +++ b/copy/progress_reader.go @@ -5,7 +5,7 @@ import ( "io" "time" - internalTypes "github.com/containers/image/v5/internal/types" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/types" ) @@ -84,14 +84,14 @@ func (r *progressReader) Read(p []byte) (int, error) { // are received. type imageSourceSeekableProxy struct { // source is the seekable input to read from. - source internalTypes.ImageSourceSeekable + source private.ImageSourceSeekable // progress is the chan where the total number of bytes read so far are reported. progress chan int64 } // GetBlobAt reads from the ImageSourceSeekable and report how many bytes were received // to the progress chan. -func (s imageSourceSeekableProxy) GetBlobAt(ctx context.Context, bInfo types.BlobInfo, chunks []internalTypes.ImageSourceChunk) (chan io.ReadCloser, chan error, error) { +func (s imageSourceSeekableProxy) GetBlobAt(ctx context.Context, bInfo types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) { rc, errs, err := s.source.GetBlobAt(ctx, bInfo, chunks) if err == nil { total := int64(0) diff --git a/docker/docker_image_src.go b/docker/docker_image_src.go index 314e9b394e..70266438e8 100644 --- a/docker/docker_image_src.go +++ b/docker/docker_image_src.go @@ -16,7 +16,7 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/internal/iolimits" - internalTypes "github.com/containers/image/v5/internal/types" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/sysregistriesv2" "github.com/containers/image/v5/types" @@ -288,7 +288,7 @@ func (s *dockerImageSource) HasThreadSafeGetBlob() bool { } // splitHTTP200ResponseToPartial splits a 200 response in multiple streams as specified by the chunks -func splitHTTP200ResponseToPartial(streams chan io.ReadCloser, errs chan error, body io.ReadCloser, chunks []internalTypes.ImageSourceChunk) { +func splitHTTP200ResponseToPartial(streams chan io.ReadCloser, errs chan error, body io.ReadCloser, chunks []private.ImageSourceChunk) { defer close(streams) defer close(errs) currentOffset := uint64(0) @@ -322,7 +322,7 @@ func splitHTTP200ResponseToPartial(streams chan io.ReadCloser, errs chan error, } // handle206Response reads a 206 response and send each part as a separate ReadCloser to the streams chan. -func handle206Response(streams chan io.ReadCloser, errs chan error, body io.ReadCloser, chunks []internalTypes.ImageSourceChunk, mediaType string, params map[string]string) { +func handle206Response(streams chan io.ReadCloser, errs chan error, body io.ReadCloser, chunks []private.ImageSourceChunk, mediaType string, params map[string]string) { defer close(streams) defer close(errs) if !strings.HasPrefix(mediaType, "multipart/") { @@ -359,7 +359,7 @@ func handle206Response(streams chan io.ReadCloser, errs chan error, body io.Read // GetBlobAt returns a stream for the specified blob. // The specified chunks must be not overlapping and sorted by their offset. -func (s *dockerImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []internalTypes.ImageSourceChunk) (chan io.ReadCloser, chan error, error) { +func (s *dockerImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) { headers := make(map[string][]string) var rangeVals []string @@ -401,7 +401,7 @@ func (s *dockerImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo, return streams, errs, nil case http.StatusBadRequest: res.Body.Close() - return nil, nil, internalTypes.BadPartialRequestError{Status: res.Status} + return nil, nil, private.BadPartialRequestError{Status: res.Status} default: err := httpResponseToError(res, "Error fetching partial blob") if err == nil { diff --git a/docker/docker_image_src_test.go b/docker/docker_image_src_test.go index b6a0594101..30b40f3b2a 100644 --- a/docker/docker_image_src_test.go +++ b/docker/docker_image_src_test.go @@ -13,12 +13,14 @@ import ( "strings" "testing" - internalTypes "github.com/containers/image/v5/internal/types" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +var _ private.ImageSourceSeekable = (*dockerImageSource)(nil) + func TestDockerImageSourceReference(t *testing.T) { manifestPathRegex := regexp.MustCompile("^/v2/.*/manifests/latest$") @@ -130,7 +132,7 @@ func TestSplitHTTP200ResponseToPartial(t *testing.T) { defer body.Close() streams := make(chan io.ReadCloser) errs := make(chan error) - chunks := []internalTypes.ImageSourceChunk{ + chunks := []private.ImageSourceChunk{ {Offset: 1, Length: 2}, {Offset: 4, Length: 1}, } @@ -150,7 +152,7 @@ func TestHandle206Response(t *testing.T) { defer body.Close() streams := make(chan io.ReadCloser) errs := make(chan error) - chunks := []internalTypes.ImageSourceChunk{ + chunks := []private.ImageSourceChunk{ {Offset: 1, Length: 2}, {Offset: 4, Length: 1}, } @@ -171,7 +173,7 @@ func TestHandle206Response(t *testing.T) { defer body.Close() streams = make(chan io.ReadCloser) errs = make(chan error) - chunks = []internalTypes.ImageSourceChunk{{Offset: 100, Length: 5}} + chunks = []private.ImageSourceChunk{{Offset: 100, Length: 5}} mediaType = "text/plain" params = map[string]string{} go handle206Response(streams, errs, body, chunks, mediaType, params) diff --git a/internal/private/private.go b/internal/private/private.go new file mode 100644 index 0000000000..80f296bccc --- /dev/null +++ b/internal/private/private.go @@ -0,0 +1,90 @@ +package private + +import ( + "context" + "io" + + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/types" +) + +// ImageDestination is an internal extension to the types.ImageDestination +// interface. +type ImageDestination interface { + types.ImageDestination + + // PutBlobWithOptions writes contents of stream and returns data representing the result. + // inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. + // inputInfo.Size is the expected length of stream, if known. + // inputInfo.MediaType describes the blob format, if known. + // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available + // to any other readers for download using the supplied digest. + // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. + PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options PutBlobOptions) (types.BlobInfo, error) + + // TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination + // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). + // info.Digest must not be empty. + // If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may + // include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be + // reflected in the manifest that will be written. + // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. + TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options TryReusingBlobOptions) (bool, types.BlobInfo, error) +} + +// PutBlobOptions are used in PutBlobWithOptions. +type PutBlobOptions struct { + // Cache to look up blob infos. + Cache types.BlobInfoCache + // Denotes whether the blob is a config or not. + IsConfig bool + // Indicates an empty layer. + EmptyLayer bool + // The corresponding index in the layer slice. + LayerIndex *int +} + +// TryReusingBlobOptions are used in TryReusingBlobWithOptions. +type TryReusingBlobOptions struct { + // Cache to look up blob infos. + Cache types.BlobInfoCache + // Use an equivalent of the desired blob. + CanSubstitute bool + // Indicates an empty layer. + EmptyLayer bool + // The corresponding index in the layer slice. + LayerIndex *int + // The reference of the image that contains the target blob. + SrcRef reference.Named +} + +// ImageSourceChunk is a portion of a blob. +// This API is experimental and can be changed without bumping the major version number. +type ImageSourceChunk struct { + Offset uint64 + Length uint64 +} + +// ImageSourceSeekable is an image source that permits to fetch chunks of the entire blob. +// This API is experimental and can be changed without bumping the major version number. +type ImageSourceSeekable interface { + // GetBlobAt returns a stream for the specified blob. + // The specified chunks must be not overlapping and sorted by their offset. + GetBlobAt(context.Context, types.BlobInfo, []ImageSourceChunk) (chan io.ReadCloser, chan error, error) +} + +// ImageDestinationPartial is a service to store a blob by requesting the missing chunks to a ImageSourceSeekable. +// This API is experimental and can be changed without bumping the major version number. +type ImageDestinationPartial interface { + // PutBlobPartial writes contents of stream and returns data representing the result. + PutBlobPartial(ctx context.Context, stream ImageSourceSeekable, srcInfo types.BlobInfo, cache types.BlobInfoCache) (types.BlobInfo, error) +} + +// BadPartialRequestError is returned by ImageSourceSeekable.GetBlobAt on an invalid request. +type BadPartialRequestError struct { + Status string +} + +func (e BadPartialRequestError) Error() string { + return e.Status +} diff --git a/internal/types/types.go b/internal/types/types.go deleted file mode 100644 index 388f8cf3b4..0000000000 --- a/internal/types/types.go +++ /dev/null @@ -1,91 +0,0 @@ -package types - -import ( - "context" - "io" - - "github.com/containers/image/v5/docker/reference" - publicTypes "github.com/containers/image/v5/types" -) - -// ImageDestinationWithOptions is an internal extension to the ImageDestination -// interface. -type ImageDestinationWithOptions interface { - publicTypes.ImageDestination - - // PutBlobWithOptions is a wrapper around PutBlob. If - // options.LayerIndex is set, the blob will be committed directly. - // Either by the calling goroutine or by another goroutine already - // committing layers. - // - // Please note that TryReusingBlobWithOptions and PutBlobWithOptions - // *must* be used the together. Mixing the two with non "WithOptions" - // functions is not supported. - PutBlobWithOptions(ctx context.Context, stream io.Reader, blobinfo publicTypes.BlobInfo, options PutBlobOptions) (publicTypes.BlobInfo, error) - - // TryReusingBlobWithOptions is a wrapper around TryReusingBlob. If - // options.LayerIndex is set, the reused blob will be recoreded as - // already pulled. - // - // Please note that TryReusingBlobWithOptions and PutBlobWithOptions - // *must* be used the together. Mixing the two with non "WithOptions" - // functions is not supported. - TryReusingBlobWithOptions(ctx context.Context, blobinfo publicTypes.BlobInfo, options TryReusingBlobOptions) (bool, publicTypes.BlobInfo, error) -} - -// PutBlobOptions are used in PutBlobWithOptions. -type PutBlobOptions struct { - // Cache to look up blob infos. - Cache publicTypes.BlobInfoCache - // Denotes whether the blob is a config or not. - IsConfig bool - // Indicates an empty layer. - EmptyLayer bool - // The corresponding index in the layer slice. - LayerIndex *int -} - -// TryReusingBlobOptions are used in TryReusingBlobWithOptions. -type TryReusingBlobOptions struct { - // Cache to look up blob infos. - Cache publicTypes.BlobInfoCache - // Use an equivalent of the desired blob. - CanSubstitute bool - // Indicates an empty layer. - EmptyLayer bool - // The corresponding index in the layer slice. - LayerIndex *int - // The reference of the image that contains the target blob. - SrcRef reference.Named -} - -// ImageSourceChunk is a portion of a blob. -// This API is experimental and can be changed without bumping the major version number. -type ImageSourceChunk struct { - Offset uint64 - Length uint64 -} - -// ImageSourceSeekable is an image source that permits to fetch chunks of the entire blob. -// This API is experimental and can be changed without bumping the major version number. -type ImageSourceSeekable interface { - // GetBlobAt returns a stream for the specified blob. - // The specified chunks must be not overlapping and sorted by their offset. - GetBlobAt(context.Context, publicTypes.BlobInfo, []ImageSourceChunk) (chan io.ReadCloser, chan error, error) -} - -// ImageDestinationPartial is a service to store a blob by requesting the missing chunks to a ImageSourceSeekable. -// This API is experimental and can be changed without bumping the major version number. -type ImageDestinationPartial interface { - // PutBlobPartial writes contents of stream and returns data representing the result. - PutBlobPartial(ctx context.Context, stream ImageSourceSeekable, srcInfo publicTypes.BlobInfo, cache publicTypes.BlobInfoCache) (publicTypes.BlobInfo, error) -} - -// BadPartialRequestError is returned by ImageSourceSeekable.GetBlobAt on an invalid request. -type BadPartialRequestError struct { - Status string -} - -func (e BadPartialRequestError) Error() string { - return e.Status -} diff --git a/storage/storage_image.go b/storage/storage_image.go index 7329ef6eee..a77345c5a9 100644 --- a/storage/storage_image.go +++ b/storage/storage_image.go @@ -18,9 +18,9 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/image" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/internal/putblobdigest" "github.com/containers/image/v5/internal/tmpdir" - internalTypes "github.com/containers/image/v5/internal/types" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/blobinfocache/none" "github.com/containers/image/v5/types" @@ -446,14 +446,14 @@ func (s *storageImageDestination) computeNextBlobCacheFile() string { return filepath.Join(s.directory, fmt.Sprintf("%d", atomic.AddInt32(&s.nextTempFileID, 1))) } -// PutBlobWithOptions is a wrapper around PutBlob. If options.LayerIndex is -// set, the blob will be committed directly. Either by the calling goroutine -// or by another goroutine already committing layers. -// -// Please not that TryReusingBlobWithOptions and PutBlobWithOptions *must* be -// used the together. Mixing the two with non "WithOptions" functions is not -// supported. -func (s *storageImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, options internalTypes.PutBlobOptions) (types.BlobInfo, error) { +// PutBlobWithOptions writes contents of stream and returns data representing the result. +// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. +// inputInfo.Size is the expected length of stream, if known. +// inputInfo.MediaType describes the blob format, if known. +// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available +// to any other readers for download using the supplied digest. +// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. +func (s *storageImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, options private.PutBlobOptions) (types.BlobInfo, error) { info, err := s.PutBlob(ctx, stream, blobinfo, options.Cache, options.IsConfig) if err != nil { return info, err @@ -542,14 +542,14 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader, }, nil } -// TryReusingBlobWithOptions is a wrapper around TryReusingBlob. If -// options.LayerIndex is set, the reused blob will be recoreded as already -// pulled. -// -// Please not that TryReusingBlobWithOptions and PutBlobWithOptions *must* be -// used the together. Mixing the two with the non "WithOptions" functions -// is not supported. -func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context, blobinfo types.BlobInfo, options internalTypes.TryReusingBlobOptions) (bool, types.BlobInfo, error) { +// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination +// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). +// info.Digest must not be empty. +// If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may +// include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be +// reflected in the manifest that will be written. +// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. +func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context, blobinfo types.BlobInfo, options private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { reused, info, err := s.tryReusingBlobWithSrcRef(ctx, blobinfo, options.Cache, options.CanSubstitute, options.SrcRef) if err != nil || !reused || options.LayerIndex == nil { return reused, info, err @@ -586,23 +586,23 @@ func (s *storageImageDestination) tryReusingBlobWithSrcRef(ctx context.Context, } type zstdFetcher struct { - stream internalTypes.ImageSourceSeekable + stream private.ImageSourceSeekable ctx context.Context blobInfo types.BlobInfo } // GetBlobAt converts from chunked.GetBlobAt to ImageSourceSeekable.GetBlobAt. func (f *zstdFetcher) GetBlobAt(chunks []chunked.ImageSourceChunk) (chan io.ReadCloser, chan error, error) { - var newChunks []internalTypes.ImageSourceChunk + var newChunks []private.ImageSourceChunk for _, v := range chunks { - i := internalTypes.ImageSourceChunk{ + i := private.ImageSourceChunk{ Offset: v.Offset, Length: v.Length, } newChunks = append(newChunks, i) } rc, errs, err := f.stream.GetBlobAt(f.ctx, f.blobInfo, newChunks) - if _, ok := err.(internalTypes.BadPartialRequestError); ok { + if _, ok := err.(private.BadPartialRequestError); ok { err = chunked.ErrBadRequest{} } return rc, errs, err @@ -611,7 +611,7 @@ func (f *zstdFetcher) GetBlobAt(chunks []chunked.ImageSourceChunk) (chan io.Read // PutBlobPartial attempts to create a blob using the data that is already present at the destination storage. stream is accessed // in a non-sequential way to retrieve the missing chunks. -func (s *storageImageDestination) PutBlobPartial(ctx context.Context, stream internalTypes.ImageSourceSeekable, srcInfo types.BlobInfo, cache types.BlobInfoCache) (types.BlobInfo, error) { +func (s *storageImageDestination) PutBlobPartial(ctx context.Context, stream private.ImageSourceSeekable, srcInfo types.BlobInfo, cache types.BlobInfoCache) (types.BlobInfo, error) { fetcher := zstdFetcher{ stream: stream, ctx: ctx, diff --git a/storage/storage_test.go b/storage/storage_test.go index 3709c1fca7..8551b0d936 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "github.com/containers/image/v5/internal/private" imanifest "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/blobinfocache/memory" "github.com/containers/image/v5/types" @@ -36,11 +37,13 @@ import ( ) var ( - topwd = "" - _ types.ImageDestination = &storageImageDestination{} - _ types.ImageSource = &storageImageSource{} - _ types.ImageReference = &storageReference{} - _ types.ImageTransport = &storageTransport{} + topwd = "" + _ types.ImageDestination = &storageImageDestination{} + _ private.ImageDestination = (*storageImageDestination)(nil) + _ private.ImageDestinationPartial = (*storageImageDestination)(nil) + _ types.ImageSource = &storageImageSource{} + _ types.ImageReference = &storageReference{} + _ types.ImageTransport = &storageTransport{} ) const (