Skip to content

exporter: workaround containerd client issue with grpc size limits #5785

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

exporter: workaround containerd client issue with grpc size limits #5785

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 26 additions & 6 deletions exporter/containerimage/writer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session
}
idxDone := progress.OneOff(ctx, "exporting manifest list "+idxDigest.String())

if err := content.WriteBlob(ctx, ic.opt.ContentStore, idxDigest.String(), bytes.NewReader(idxBytes), idxDesc, content.WithLabels(labels)); err != nil {
if err := content.WriteBlob(ctx, ic.opt.ContentStore, idxDigest.String(), newBytesReader(idxBytes), idxDesc, content.WithLabels(labels)); err != nil {
return nil, idxDone(errors.Wrapf(err, "error writing manifest list blob %s", idxDigest))
}
idxDone(nil)
Expand Down Expand Up @@ -530,7 +530,7 @@ func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, opts *Ima
}
mfstDone := progress.OneOff(ctx, "exporting manifest "+mfstDigest.String())

if err := content.WriteBlob(ctx, ic.opt.ContentStore, mfstDigest.String(), bytes.NewReader(mfstJSON), mfstDesc, content.WithLabels((labels))); err != nil {
if err := content.WriteBlob(ctx, ic.opt.ContentStore, mfstDigest.String(), newBytesReader(mfstJSON), mfstDesc, content.WithLabels((labels))); err != nil {
return nil, nil, mfstDone(errors.Wrapf(err, "error writing manifest blob %s", mfstDigest))
}
mfstDone(nil)
Expand All @@ -542,7 +542,7 @@ func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, opts *Ima
}
configDone := progress.OneOff(ctx, "exporting config "+configDigest.String())

if err := content.WriteBlob(ctx, ic.opt.ContentStore, configDigest.String(), bytes.NewReader(config), configDesc); err != nil {
if err := content.WriteBlob(ctx, ic.opt.ContentStore, configDigest.String(), newBytesReader(config), configDesc); err != nil {
return nil, nil, configDone(errors.Wrap(err, "error writing config blob"))
}
configDone(nil)
Expand Down Expand Up @@ -584,7 +584,7 @@ func (ic *ImageWriter) commitAttestationsManifest(ctx context.Context, opts *Ima
},
}

if err := content.WriteBlob(ctx, ic.opt.ContentStore, digest.String(), bytes.NewReader(data), desc); err != nil {
if err := content.WriteBlob(ctx, ic.opt.ContentStore, digest.String(), newBytesReader(data), desc); err != nil {
return nil, errors.Wrapf(err, "error writing data blob %s", digest)
}
layers[i] = desc
Expand Down Expand Up @@ -641,10 +641,10 @@ func (ic *ImageWriter) commitAttestationsManifest(ctx context.Context, opts *Ima
}

done := progress.OneOff(ctx, "exporting attestation manifest "+mfstDigest.String())
if err := content.WriteBlob(ctx, ic.opt.ContentStore, mfstDigest.String(), bytes.NewReader(mfstJSON), mfstDesc, content.WithLabels((labels))); err != nil {
if err := content.WriteBlob(ctx, ic.opt.ContentStore, mfstDigest.String(), newBytesReader(mfstJSON), mfstDesc, content.WithLabels((labels))); err != nil {
return nil, done(errors.Wrapf(err, "error writing manifest blob %s", mfstDigest))
}
if err := content.WriteBlob(ctx, ic.opt.ContentStore, configDesc.Digest.String(), bytes.NewReader(config), configDesc); err != nil {
if err := content.WriteBlob(ctx, ic.opt.ContentStore, configDesc.Digest.String(), newBytesReader(config), configDesc); err != nil {
return nil, done(errors.Wrap(err, "error writing config blob"))
}
done(nil)
Expand Down Expand Up @@ -947,3 +947,23 @@ func getRefMetadata(ref cache.ImmutableRef, limit int) []refMetadata {
}
return metas
}

// bytesReader is a wrapper around bytes.Reader that only exposes the io.Reader
// interface. This is to prevent any accidental usage of WriteTo that causes the
// entire buffer to be written at once.
//
// This is needed to workaround https://github.com/containerd/containerd/issues/11440.
// When that is fixed, this can be removed.
type bytesReader struct {
r *bytes.Reader
}

func newBytesReader(p []byte) *bytesReader {
return &bytesReader{
r: bytes.NewReader(p),
}
}

func (r *bytesReader) Read(p []byte) (n int, err error) {
return r.r.Read(p)
}
Loading