bundle download & decompression: added Context argument

to allow stopping using API when these operations are running

Author: redbeam

cmd/crc-embedder/cmd/embed.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path"
@@ -162,7 +163,7 @@ func downloadDataFiles(goos string, components []string, destDir string) ([]stri
 		if !shouldDownload(components, componentName) {
 			continue
 		}
-		filename, err := download.Download(dl.url, destDir, dl.permissions, nil)
+		filename, err := download.Download(context.TODO(), dl.url, destDir, dl.permissions, nil)
 		if err != nil {
 			return nil, err
 		}

pkg/compress/compress_test.go

@@ -1,6 +1,7 @@
 package compress
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -33,7 +34,7 @@ func testCompress(t *testing.T, baseDir string) {
 
 	destDir := t.TempDir()
 
-	fileList, err := extract.Uncompress(testArchiveName, destDir)
+	fileList, err := extract.Uncompress(context.TODO(), testArchiveName, destDir)
 	require.NoError(t, err)
 
 	_, d := filepath.Split(baseDir)

pkg/crc/cache/cache.go

@@ -1,6 +1,7 @@
 package cache
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -121,7 +122,7 @@ func (c *Cache) cacheExecutable() error {
 	// Check the file is tarball or not
 	if isTarball(assetTmpFile) {
 		// Extract the tarball and put it the cache directory.
-		extractedFiles, err = extract.UncompressWithFilter(assetTmpFile, tmpDir,
+		extractedFiles, err = extract.UncompressWithFilter(context.TODO(), assetTmpFile, tmpDir,
 			func(filename string) bool { return filepath.Base(filename) == c.GetExecutableName() })
 		if err != nil {
 			return errors.Wrapf(err, "Cannot uncompress '%s'", assetTmpFile)
@@ -153,7 +154,7 @@ func (c *Cache) getExecutable(destDir string) (string, error) {
 	destPath := filepath.Join(destDir, archiveName)
 	err := embed.Extract(archiveName, destPath)
 	if err != nil {
-		return download.Download(c.archiveURL, destDir, 0600, nil)
+		return download.Download(context.TODO(), c.archiveURL, destDir, 0600, nil)
 	}
 
 	return destPath, err

pkg/crc/image/image.go

@@ -53,7 +53,7 @@ func (img *imageHandler) policyContext() (*signature.PolicyContext, error) {
 }
 
 // copyImage pulls the image from the registry and puts it to destination path
-func (img *imageHandler) copyImage(destPath string, reportWriter io.Writer) (*v1.Manifest, error) {
+func (img *imageHandler) copyImage(ctx context.Context, destPath string, reportWriter io.Writer) (*v1.Manifest, error) {
 	// Source Image from docker transport
 	srcImg := img.imageURI
 	srcRef, err := docker.ParseReference(srcImg)
@@ -71,7 +71,10 @@ func (img *imageHandler) copyImage(destPath string, reportWriter io.Writer) (*v1
 		return nil, err
 	}
 
-	manifestData, err := copy.Image(context.Background(), policyContext,
+	if ctx == nil {
+		panic("ctx is nil, this should not happen")
+	}
+	manifestData, err := copy.Image(ctx, policyContext,
 		destRef, srcRef, &copy.Options{
 			ReportWriter: reportWriter,
 		})
@@ -113,7 +116,7 @@ func GetPresetName(imageName string) crcpreset.Preset {
 	return preset
 }
 
-func PullBundle(imageURI string) (string, error) {
+func PullBundle(ctx context.Context, imageURI string) (string, error) {
 	imgHandler := imageHandler{
 		imageURI: strings.TrimPrefix(imageURI, "docker:"),
 	}
@@ -122,7 +125,7 @@ func PullBundle(imageURI string) (string, error) {
 		return "", err
 	}
 	defer os.RemoveAll(destDir)
-	imgManifest, err := imgHandler.copyImage(destDir, os.Stdout)
+	imgManifest, err := imgHandler.copyImage(ctx, destDir, os.Stdout)
 	if err != nil {
 		return "", err
 	}
@@ -132,7 +135,7 @@ func PullBundle(imageURI string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	fileList, err := extract.Uncompress(filepath.Join(destDir, imgLayer), constants.MachineCacheDir)
+	fileList, err := extract.Uncompress(ctx, filepath.Join(destDir, imgLayer), constants.MachineCacheDir)
 	if err != nil {
 		return "", err
 	}

pkg/crc/machine/bundle/metadata.go

@@ -1,6 +1,7 @@
 package bundle
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -337,16 +338,15 @@ func getVerifiedHash(url string, file string) (string, error) {
 	return "", fmt.Errorf("%s hash is missing or shasums are malformed", file)
 }
 
-func downloadDefault(preset crcPreset.Preset) (string, error) {
+func downloadDefault(ctx context.Context, preset crcPreset.Preset) (string, error) {
 	downloadInfo, err := getBundleDownloadInfo(preset)
 	if err != nil {
 		return "", err
 	}
-
-	return downloadInfo.Download(constants.GetDefaultBundlePath(preset), 0664)
+	return downloadInfo.Download(ctx, constants.GetDefaultBundlePath(preset), 0664)
 }
 
-func Download(preset crcPreset.Preset, bundleURI string, enableBundleQuayFallback bool) (string, error) {
+func Download(ctx context.Context, preset crcPreset.Preset, bundleURI string, enableBundleQuayFallback bool) (string, error) {
 	// If we are asked to download
 	// ~/.crc/cache/crc_podman_libvirt_4.1.1.crcbundle, this means we want
 	// are downloading the default bundle for this release. This uses a
@@ -355,23 +355,23 @@ func Download(preset crcPreset.Preset, bundleURI string, enableBundleQuayFallbac
 	if bundleURI == constants.GetDefaultBundlePath(preset) {
 		switch preset {
 		case crcPreset.OpenShift, crcPreset.Microshift:
-			downloadedBundlePath, err := downloadDefault(preset)
+			downloadedBundlePath, err := downloadDefault(ctx, preset)
 			if err != nil && enableBundleQuayFallback {
 				logging.Info("Unable to download bundle from mirror, falling back to quay")
-				return image.PullBundle(constants.GetDefaultBundleImageRegistry(preset))
+				return image.PullBundle(ctx, constants.GetDefaultBundleImageRegistry(preset))
 			}
 			return downloadedBundlePath, err
 		case crcPreset.OKD:
 			fallthrough
 		default:
-			return image.PullBundle(constants.GetDefaultBundleImageRegistry(preset))
+			return image.PullBundle(ctx, constants.GetDefaultBundleImageRegistry(preset))
 		}
 	}
 	switch {
 	case strings.HasPrefix(bundleURI, "http://"), strings.HasPrefix(bundleURI, "https://"):
-		return download.Download(bundleURI, constants.MachineCacheDir, 0644, nil)
+		return download.Download(ctx, bundleURI, constants.MachineCacheDir, 0644, nil)
 	case strings.HasPrefix(bundleURI, "docker://"):
-		return image.PullBundle(bundleURI)
+		return image.PullBundle(ctx, bundleURI)
 	}
 	// the `bundleURI` parameter turned out to be a local path
 	return bundleURI, nil

pkg/crc/machine/bundle/repository.go

@@ -124,7 +124,7 @@ func (bundle *CrcBundleInfo) createSymlinkOrCopyPodmanRemote(binDir string) erro
 	return bundle.copyExecutableFromBundle(binDir, PodmanExecutable, constants.PodmanRemoteExecutableName)
 }
 
-func (repo *Repository) Extract(path string) error {
+func (repo *Repository) Extract(ctx context.Context, path string) error {
 	bundleName := filepath.Base(path)
 
 	tmpDir := filepath.Join(repo.CacheDir, "tmp-extract")
@@ -133,7 +133,7 @@ func (repo *Repository) Extract(path string) error {
 		_ = os.RemoveAll(tmpDir) // clean up after using it
 	}()
 
-	if _, err := extract.Uncompress(path, tmpDir); err != nil {
+	if _, err := extract.Uncompress(ctx, path, tmpDir); err != nil {
 		return err
 	}
 
@@ -198,8 +198,8 @@ func Use(bundleName string) (*CrcBundleInfo, error) {
 	return defaultRepo.Use(bundleName)
 }
 
-func Extract(path string) (*CrcBundleInfo, error) {
-	if err := defaultRepo.Extract(path); err != nil {
+func Extract(ctx context.Context, path string) (*CrcBundleInfo, error) {
+	if err := defaultRepo.Extract(ctx, path); err != nil {
 		return nil, err
 	}
 	return defaultRepo.Get(filepath.Base(path))

pkg/crc/machine/bundle/repository_test.go

@@ -1,6 +1,7 @@
 package bundle
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -39,7 +40,7 @@ func TestExtract(t *testing.T) {
 		OcBinDir: ocBinDir,
 	}
 
-	assert.NoError(t, repo.Extract(filepath.Join("testdata", testBundle(t))))
+	assert.NoError(t, repo.Extract(context.TODO(), filepath.Join("testdata", testBundle(t))))
 
 	bundle, err := repo.Get(testBundle(t))
 	assert.NoError(t, err)

pkg/crc/machine/start.go

@@ -14,7 +14,7 @@ import (
 	"github.com/crc-org/crc/v2/pkg/crc/cluster"
 	"github.com/crc-org/crc/v2/pkg/crc/constants"
 	crcerrors "github.com/crc-org/crc/v2/pkg/crc/errors"
-	logging "github.com/crc-org/crc/v2/pkg/crc/logging"
+	"github.com/crc-org/crc/v2/pkg/crc/logging"
 	"github.com/crc-org/crc/v2/pkg/crc/machine/bundle"
 	"github.com/crc-org/crc/v2/pkg/crc/machine/config"
 	"github.com/crc-org/crc/v2/pkg/crc/machine/state"
@@ -41,20 +41,20 @@ import (
 
 const minimumMemoryForMonitoring = 14336
 
-func getCrcBundleInfo(preset crcPreset.Preset, bundleName, bundlePath string, enableBundleQuayFallback bool) (*bundle.CrcBundleInfo, error) {
+func getCrcBundleInfo(ctx context.Context, preset crcPreset.Preset, bundleName, bundlePath string, enableBundleQuayFallback bool) (*bundle.CrcBundleInfo, error) {
 	bundleInfo, err := bundle.Use(bundleName)
 	if err == nil {
 		logging.Infof("Loading bundle: %s...", bundleName)
 		return bundleInfo, nil
 	}
 	logging.Debugf("Failed to load bundle %s: %v", bundleName, err)
 	logging.Infof("Downloading bundle: %s...", bundleName)
-	bundlePath, err = bundle.Download(preset, bundlePath, enableBundleQuayFallback)
+	bundlePath, err = bundle.Download(ctx, preset, bundlePath, enableBundleQuayFallback)
 	if err != nil {
 		return nil, err
 	}
 	logging.Infof("Extracting bundle: %s...", bundleName)
-	if _, err := bundle.Extract(bundlePath); err != nil {
+	if _, err := bundle.Extract(ctx, bundlePath); err != nil {
 		return nil, err
 	}
 	return bundle.Use(bundleName)
@@ -279,7 +279,7 @@ func (client *client) Start(ctx context.Context, startConfig types.StartConfig)
 	}
 
 	bundleName := bundle.GetBundleNameWithoutExtension(bundle.GetBundleNameFromURI(startConfig.BundlePath))
-	crcBundleMetadata, err := getCrcBundleInfo(startConfig.Preset, bundleName, startConfig.BundlePath, startConfig.EnableBundleQuayFallback)
+	crcBundleMetadata, err := getCrcBundleInfo(ctx, startConfig.Preset, bundleName, startConfig.BundlePath, startConfig.EnableBundleQuayFallback)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error getting bundle metadata")
 	}

pkg/crc/preflight/preflight_checks_common.go

@@ -1,6 +1,7 @@
 package preflight
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -112,12 +113,12 @@ func fixBundleExtracted(bundlePath string, preset crcpreset.Preset, enableBundle
 		}
 		var err error
 		logging.Infof("Downloading bundle: %s...", bundlePath)
-		if bundlePath, err = bundle.Download(preset, bundlePath, enableBundleQuayFallback); err != nil {
+		if bundlePath, err = bundle.Download(context.TODO(), preset, bundlePath, enableBundleQuayFallback); err != nil {
 			return err
 		}
 
 		logging.Infof("Uncompressing %s", bundlePath)
-		if _, err := bundle.Extract(bundlePath); err != nil {
+		if _, err := bundle.Extract(context.TODO(), bundlePath); err != nil {
 			if errors.Is(err, os.ErrNotExist) {
 				return errors.Wrap(err, "Use `crc setup -b <bundle-path>`")
 			}

pkg/download/download.go

@@ -1,6 +1,7 @@
 package download
 
 import (
+	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
@@ -60,7 +61,7 @@ loop:
 
 // Download function takes sha256sum as hex decoded byte
 // something like hex.DecodeString("33daf4c03f86120fdfdc66bddf6bfff4661c7ca11c5d")
-func Download(uri, destination string, mode os.FileMode, sha256sum []byte) (string, error) {
+func Download(ctx context.Context, uri, destination string, mode os.FileMode, sha256sum []byte) (string, error) {
 	logging.Debugf("Downloading %s to %s", uri, destination)
 
 	client := grab.NewClient()
@@ -70,6 +71,12 @@ func Download(uri, destination string, mode os.FileMode, sha256sum []byte) (stri
 	if err != nil {
 		return "", errors.Wrapf(err, "unable to get request from %s", uri)
 	}
+
+	if ctx == nil {
+		panic("ctx is nil, this should not happen")
+	}
+	req = req.WithContext(ctx)
+
 	if sha256sum != nil {
 		req.SetChecksum(sha256.New(), sha256sum, true)
 	}
@@ -131,12 +138,12 @@ func NewRemoteFile(uri, sha256sum string) *RemoteFile {
 
 }
 
-func (r *RemoteFile) Download(bundlePath string, mode os.FileMode) (string, error) {
-	sha256, err := hex.DecodeString(r.sha256sum)
+func (r *RemoteFile) Download(ctx context.Context, bundlePath string, mode os.FileMode) (string, error) {
+	sha256bytes, err := hex.DecodeString(r.sha256sum)
 	if err != nil {
 		return "", err
 	}
-	return Download(r.URI, bundlePath, mode, sha256)
+	return Download(ctx, r.URI, bundlePath, mode, sha256bytes)
 }
 
 func (r *RemoteFile) GetSha256Sum() string {