Skip to content

Vendor containers/{image,storage} for faster compression and parallel pulls #1244

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rhatdan merged 3 commits into from
Jan 5, 2019
Merged

Vendor containers/{image,storage} for faster compression and parallel pulls #1244

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
103a2ba
vendor: use faster version instead compress/gzip
rhatdan Dec 18, 2018
d88807e
blobcache.bats: adjust explicit push tests
vrothberg Dec 20, 2018
08e9423
vendor parallel-copy from containers/image
vrothberg Dec 19, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions image.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -557,6 +557,10 @@ func (i *containerImageSource) LayerInfosForCopy(ctx context.Context) ([]types.B
return nil, nil
}

func (i *containerImageSource) HasThreadSafeGetBlob() bool {
return false
}

func (i *containerImageSource) GetBlob(ctx context.Context, blob types.BlobInfo, cache types.BlobInfoCache) (reader io.ReadCloser, size int64, err error) {
if blob.Digest == i.configDigest {
logrus.Debugf("start reading config")
Expand Down
8 changes: 8 additions & 0 deletions pkg/blobcache/blobcache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,10 @@ func (s *blobCacheSource) GetManifest(ctx context.Context, instanceDigest *diges
return s.source.GetManifest(ctx, instanceDigest)
}

func (s *blobCacheSource) HasThreadSafeGetBlob() bool {
return false
}

func (s *blobCacheSource) GetBlob(ctx context.Context, blobinfo types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
present, size, err := s.reference.HasBlob(blobinfo)
if err != nil {
Expand Down Expand Up @@ -398,6 +402,10 @@ func saveStream(wg *sync.WaitGroup, decompressReader io.ReadCloser, tempFile *os
}
}

func (s *blobCacheDestination) HasThreadSafePutBlob() bool {
return false
}

func (d *blobCacheDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) {
var tempfile *os.File
var err error
Expand Down
54 changes: 0 additions & 54 deletions tests/blobcache.bats
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,33 +187,6 @@ load helpers
run buildah build-using-dockerfile -t ${target} --pull-always --blob-cache=${blobcachedir} --signature-policy ${TESTSDIR}/policy.json ${TESTSDIR}/bud/add-file
echo "$output"
[ "$status" -eq 0 ]
# Try to push the image without the blob cache.
doomeddir=${TESTDIR}/doomed
mkdir -p ${doomeddir}
run buildah push --signature-policy ${TESTSDIR}/policy.json ${target} dir:${doomeddir}
echo "$output"
[ "$status" -eq 0 ]
# Look for layer blobs in the destination that match the ones in the cache.
matched=0
unmatched=0
for content in ${doomeddir}/* ; do
match=false
for blob in ${blobcachedir}/* ; do
if cmp -s ${content} ${blob} ; then
echo $(file ${blob}) and ${content} have the same contents, was cached
match=true
break
fi
done
if ${match} ; then
matched=$(( ${matched} + 1 ))
else
unmatched=$(( ${unmatched} + 1 ))
echo ${content} was not cached
fi
done
[ ${matched} -eq 4 ] # the base layer, our new layer, config, and manifest should all match the cache
[ ${unmatched} -eq 1 ] # the only mismatch should be "version"
# Now try to push the image using the blob cache.
destdir=${TESTDIR}/dest
mkdir -p ${destdir}
Expand Down Expand Up @@ -251,33 +224,6 @@ load helpers
run buildah build-using-dockerfile -t ${target} -D --pull-always --blob-cache=${blobcachedir} --signature-policy ${TESTSDIR}/policy.json ${TESTSDIR}/bud/add-file
echo "$output"
[ "$status" -eq 0 ]
# Try to push the image without the blob cache.
doomeddir=${TESTDIR}/doomed
mkdir -p ${doomeddir}
run buildah push --signature-policy ${TESTSDIR}/policy.json ${target} dir:${doomeddir}
echo "$output"
[ "$status" -eq 0 ]
# Look for layer blobs in the destination that match the ones in the cache.
matched=0
unmatched=0
for content in ${doomeddir}/* ; do
match=false
for blob in ${blobcachedir}/* ; do
if cmp -s ${content} ${blob} ; then
echo $(file ${blob}) and ${content} have the same contents, was cached
match=true
break
fi
done
if ${match} ; then
matched=$(( ${matched} + 1 ))
else
unmatched=$(( ${unmatched} + 1 ))
echo ${content} was not cached
fi
done
[ ${matched} -eq 2 ] # our new layer (written at build-time) and config should match the cache
[ ${unmatched} -eq 3 ] # expected mismatches should be "version", and the base layers, which had to be recompressed
# Now try to push the image using the blob cache.
destdir=${TESTDIR}/dest
mkdir -p ${destdir}
Expand Down
12 changes: 8 additions & 4 deletions vendor.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ github.com/blang/semver master
github.com/BurntSushi/toml master
github.com/containerd/continuity master
github.com/containernetworking/cni v0.7.0-alpha1
github.com/containers/image d53afe179b381fafb427e6b9cf9b1996a98c1067
github.com/containers/image 0c6cc8e1420001ae39fa89d308b3b3bc5ee81c57
github.com/boltdb/bolt master
github.com/containers/libpod c8eaf59d5f4bec249db8134c6a9fcfbcac792519
github.com/containers/storage db40f96d853dfced60c563e61fb66ba231ce7c8d
github.com/containers/storage 60a692f7ce891feb91ce0eda87bd06bfd5651dff
github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716
github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00
github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1
Expand All @@ -32,7 +32,7 @@ github.com/mistifyio/go-zfs master
github.com/moby/moby f8806b18b4b92c5e1980f6e11c917fad201cd73c
github.com/mtrmac/gpgme master
github.com/Nvveen/Gotty master
github.com/opencontainers/go-digest aa2ec055abd10d26d539eb630a92241b781ce4bc
github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7
github.com/opencontainers/image-spec v1.0.0
github.com/opencontainers/runc master
github.com/opencontainers/runtime-spec v1.0.0
Expand All @@ -55,10 +55,14 @@ github.com/xeipuuv/gojsonreference master
github.com/xeipuuv/gojsonschema master
golang.org/x/crypto master
golang.org/x/net master
golang.org/x/sync 42b317875d0fa942474b76e1b46a6060d720ae6e
golang.org/x/sys master
golang.org/x/text master
gopkg.in/cheggaaa/pb.v1 v1.0.13
gopkg.in/cheggaaa/pb.v1 v1.0.27
gopkg.in/yaml.v2 cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b
k8s.io/apimachinery master
k8s.io/client-go master
k8s.io/kubernetes master
github.com/klauspost/pgzip v1.2.1
github.com/klauspost/compress v1.4.1
github.com/klauspost/cpuid v1.2.0
Loading