diff --git a/analyzer.go b/analyzer.go index ccdc435c2..16c7c1c73 100644 --- a/analyzer.go +++ b/analyzer.go @@ -1,6 +1,7 @@ package lifecycle import ( + "fmt" "os" "github.com/buildpack/imgutil" @@ -17,93 +18,119 @@ type Analyzer struct { SkipLayers bool } -func (a *Analyzer) Analyze(image imgutil.Image) (AnalyzedMetadata, error) { +// Analyze restores metadata for launch and cache layers into the layers directory. +// If a usable cache is not provided, Analyze will not restore any cache=true layer metadata. +func (a *Analyzer) Analyze(image imgutil.Image, cache Cache) (*AnalyzedMetadata, error) { imageID, err := a.getImageIdentifier(image) if err != nil { - return AnalyzedMetadata{}, errors.Wrap(err, "retrieve image identifier") + return nil, errors.Wrap(err, "retrieving image identifier") } - var data LayersMetadata + var appMeta LayersMetadata // continue even if the label cannot be decoded - if err := DecodeLabel(image, LayerMetadataLabel, &data); err != nil { - data = LayersMetadata{} + if err := DecodeLabel(image, LayerMetadataLabel, &appMeta); err != nil { + appMeta = LayersMetadata{} } - if !a.SkipLayers { - for _, buildpack := range a.Buildpacks { - bpLayersDir, err := readBuildpackLayersDir(a.LayersDir, buildpack) - if err != nil { - return AnalyzedMetadata{}, err - } + if a.SkipLayers { + a.Logger.Infof("Skipping buildpack layer analysis") + return &AnalyzedMetadata{ + Image: imageID, + Metadata: appMeta, + }, nil + } + + // Create empty cache metadata in case a usable cache is not provided. + var cacheMeta CacheMetadata + if cache != nil { + var err error + cacheMeta, err = cache.RetrieveMetadata() + if err != nil { + return nil, errors.Wrap(err, "retrieving cache metadata") + } + } else { + a.Logger.Debug("Usable cache not provided, using empty cache metadata.") + } - metadataLayers := data.MetadataForBuildpack(buildpack.ID).Layers - for _, cachedLayer := range bpLayersDir.layers { - cacheType := cachedLayer.classifyCache(metadataLayers) - switch cacheType { - case cacheStaleNoMetadata: - a.Logger.Infof("Removing stale cached launch layer '%s', not in metadata \n", cachedLayer.Identifier()) - if err := cachedLayer.remove(); err != nil { - return AnalyzedMetadata{}, err - } - case cacheStaleWrongSHA: - a.Logger.Infof("Removing stale cached launch layer '%s'", cachedLayer.Identifier()) - if err := cachedLayer.remove(); err != nil { - return AnalyzedMetadata{}, err - } - case cacheMalformed: - a.Logger.Infof("Removing malformed cached layer '%s'", cachedLayer.Identifier()) - if err := cachedLayer.remove(); err != nil { - return AnalyzedMetadata{}, err - } - case cacheNotForLaunch: - a.Logger.Infof("Using cached layer '%s'", cachedLayer.Identifier()) - case cacheValid: - a.Logger.Infof("Using cached launch layer '%s'", cachedLayer.Identifier()) - a.Logger.Infof("Rewriting metadata for layer '%s'", cachedLayer.Identifier()) - if err := cachedLayer.writeMetadata(metadataLayers); err != nil { - return AnalyzedMetadata{}, err - } - } + for _, buildpack := range a.Buildpacks { + buildpackDir, err := readBuildpackLayersDir(a.LayersDir, buildpack) + if err != nil { + return nil, errors.Wrap(err, "reading buildpack layer directory") + } + + // Restore metadata for launch=true layers. + // The restorer step will restore the layer data for cache=true layers if possible or delete the layer. + appLayers := appMeta.MetadataForBuildpack(buildpack.ID).Layers + for name, layer := range appLayers { + identifier := fmt.Sprintf("%s:%s", buildpack.ID, name) + if !layer.Launch { + a.Logger.Debugf("Not restoring metadata for %q, marked as launch=false", identifier) + continue + } + if layer.Build && !layer.Cache { + a.Logger.Debugf("Not restoring metadata for %q, marked as build=true, cache=false", identifier) + continue } + a.Logger.Infof("Restoring metadata for %q from app image", identifier) + if err := a.writeLayerMetadata(buildpackDir, name, layer); err != nil { + return nil, err + } + } - for lmd, data := range metadataLayers { - if !data.Build && !data.Cache { - layer := bpLayersDir.newBPLayer(lmd) - a.Logger.Infof("Writing metadata for uncached layer '%s'", layer.Identifier()) - if err := layer.writeMetadata(metadataLayers); err != nil { - return AnalyzedMetadata{}, err - } - } + // Restore metadata for cache=true layers. + // The restorer step will restore the layer data if possible or delete the layer. + cachedLayers := cacheMeta.MetadataForBuildpack(buildpack.ID).Layers + for name, layer := range cachedLayers { + identifier := fmt.Sprintf("%s:%s", buildpack.ID, name) + if !layer.Cache { + a.Logger.Debugf("Not restoring %q from cache, marked as cache=false", identifier) + continue + } + // If launch=true, the metadata was restored from the app image or the layer is stale. + if layer.Launch { + a.Logger.Debugf("Not restoring %q from cache, marked as launch=true", identifier) + continue + } + a.Logger.Infof("Restoring metadata for %q from cache", identifier) + if err := a.writeLayerMetadata(buildpackDir, name, layer); err != nil { + return nil, err } } - } else { - a.Logger.Infof("Skipping buildpack layer analysis") } // if analyzer is running as root it needs to fix the ownership of the layers dir if current := os.Getuid(); current == 0 { if err := recursiveChown(a.LayersDir, a.UID, a.GID); err != nil { - return AnalyzedMetadata{}, errors.Wrapf(err, "chowning layers dir to '%d/%d'", a.UID, a.GID) + return nil, errors.Wrapf(err, "chowning layers dir to '%d/%d'", a.UID, a.GID) } } - return AnalyzedMetadata{ + return &AnalyzedMetadata{ Image: imageID, - Metadata: data, + Metadata: appMeta, }, nil } func (a *Analyzer) getImageIdentifier(image imgutil.Image) (*ImageIdentifier, error) { if !image.Found() { - a.Logger.Warnf("Image '%s' not found", image.Name()) + a.Logger.Warnf("Image %q not found", image.Name()) return nil, nil } identifier, err := image.Identifier() if err != nil { return nil, err } - a.Logger.Debugf("Analyzing image '%s'", identifier.String()) + a.Logger.Debugf("Analyzing image %q", identifier.String()) return &ImageIdentifier{ Reference: identifier.String(), }, nil } + +func (a *Analyzer) writeLayerMetadata(buildpackDir bpLayersDir, name string, metadata BuildpackLayerMetadata) error { + layer := buildpackDir.newBPLayer(name) + a.Logger.Debugf("Writing layer metadata for %q", layer.Identifier()) + if err := layer.writeMetadata(metadata); err != nil { + return err + } + return layer.writeSha(metadata.SHA) +} diff --git a/analyzer_test.go b/analyzer_test.go index a540a1e78..cb7e2f24e 100644 --- a/analyzer_test.go +++ b/analyzer_test.go @@ -17,6 +17,8 @@ import ( "github.com/sclevine/spec/report" "github.com/buildpack/lifecycle" + "github.com/buildpack/lifecycle/cache" + "github.com/buildpack/lifecycle/cmd" h "github.com/buildpack/lifecycle/testhelpers" "github.com/buildpack/lifecycle/testmock" ) @@ -30,11 +32,13 @@ func TestAnalyzer(t *testing.T) { func testAnalyzer(t *testing.T, when spec.G, it spec.S) { var ( - analyzer *lifecycle.Analyzer - mockCtrl *gomock.Controller - layerDir string - appDir string - tmpDir string + analyzer *lifecycle.Analyzer + mockCtrl *gomock.Controller + layerDir string + appDir string + tmpDir string + cacheDir string + testCache lifecycle.Cache ) it.Before(func() { @@ -46,6 +50,12 @@ func testAnalyzer(t *testing.T, when spec.G, it spec.S) { layerDir, err = ioutil.TempDir("", "lifecycle-layer-dir") h.AssertNil(t, err) + cacheDir, err = ioutil.TempDir("", "some-cache-dir") + h.AssertNil(t, err) + + testCache, err = cache.NewVolumeCache(cacheDir) + h.AssertNil(t, err) + appDir = filepath.Join(layerDir, "some-app-dir") analyzer = &lifecycle.Analyzer{ Buildpacks: []lifecycle.Buildpack{{ID: "metadata.buildpack"}, {ID: "no.cache.buildpack"}, {ID: "no.metadata.buildpack"}}, @@ -56,12 +66,17 @@ func testAnalyzer(t *testing.T, when spec.G, it spec.S) { UID: 1234, GID: 4321, } + if testing.Verbose() { + analyzer.Logger = cmd.Logger + cmd.SetLogLevel("debug") + } mockCtrl = gomock.NewController(t) }) it.After(func() { - os.RemoveAll(tmpDir) - os.RemoveAll(layerDir) + h.AssertNil(t, os.RemoveAll(tmpDir)) + h.AssertNil(t, os.RemoveAll(layerDir)) + h.AssertNil(t, os.RemoveAll(cacheDir)) mockCtrl.Finish() }) @@ -85,515 +100,388 @@ func testAnalyzer(t *testing.T, when spec.G, it spec.S) { }) when("image exists", func() { - when("image label has compatible metadata", func() { - it.Before(func() { - metadataLabel := `{ - "buildpacks": [ - { - "key": "metadata.buildpack", - "layers": { - "valid-launch": { - "data": { - "akey": "avalue", - "bkey": "bvalue" - }, - "sha": "valid-launch-layer-sha", - "launch": true - }, - "valid-launch-build": { - "data": { - "some-key": "val-from-metadata", - "some-other-key": "val-from-metadata" - }, - "sha": "valid-launch-build-sha", - "launch": true, - "build": true - }, - "stale-launch": { - "data": { - "version": "1234" - }, - "sha": "new-sha", - "launch": true - }, - "stale-launch-build": { - "data": { - "some": "metadata" - }, - "sha": "new-launch-build-sha", - "build": true, - "launch": true - }, - "launch-cache": { - "data": { - "some": "metadata" - }, - "sha": "launch-cache-sha", - "cache": true, - "launch": true - } - } - }, - { - "key": "no.cache.buildpack", - "layers": { - "go": { - "data": { - "version": "1.10" - } - } - } - } - ] -}` - h.AssertNil(t, - image.SetLabel( - "io.buildpacks.lifecycle.metadata", - metadataLabel, - )) - h.AssertNil(t, json.Unmarshal([]byte(metadataLabel), &appImageMetadata)) - }) - - it("should use labels to populate the layer dir", func() { - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } + it.Before(func() { + metadata := h.MustReadFile(t, filepath.Join("testdata", "analyzer", "app_metadata.json")) + h.AssertNil(t, image.SetLabel("io.buildpacks.lifecycle.metadata", string(metadata))) + h.AssertNil(t, json.Unmarshal(metadata, &appImageMetadata)) + }) - for _, data := range []struct{ name, expected string }{ - {"metadata.buildpack/valid-launch.toml", `[metadata] - akey = "avalue" - bkey = "bvalue"`}, - {"metadata.buildpack/stale-launch.toml", `[metadata] - version = "1234"`}, - {"no.cache.buildpack/go.toml", `[metadata] - version = "1.10"`}, - } { - if txt, err := ioutil.ReadFile(filepath.Join(layerDir, data.name)); err != nil { - t.Fatalf("Error: %s\n", err) - } else if !strings.Contains(string(txt), data.expected) { - t.Fatalf(`Error: expected "%s" to contain "%s"`, string(txt), data.expected) - } - } - }) + it("restores layer metadata", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - it("should only write layer TOML files that correspond to detected buildpacks", func() { - analyzer.Buildpacks = []lifecycle.Buildpack{{ID: "no.cache.buildpack"}} + for _, data := range []struct{ name, want string }{ + {"metadata.buildpack/launch.toml", "[metadata]\n launch-key = \"launch-value\""}, + {"metadata.buildpack/launch-build-cache.toml", "[metadata]\n launch-build-cache-key = \"launch-build-cache-value\""}, + {"metadata.buildpack/launch-cache.toml", "[metadata]\n launch-cache-key = \"launch-cache-value\""}, + {"no.cache.buildpack/some-layer.toml", "[metadata]\n some-layer-key = \"some-layer-value\""}, + } { + got := h.MustReadFile(t, filepath.Join(layerDir, data.name)) + h.AssertStringContains(t, string(got), data.want) + } + }) - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } + it("restores layer sha files", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if txt, err := ioutil.ReadFile(filepath.Join(layerDir, "no.cache.buildpack", "go.toml")); err != nil { - t.Fatalf("Error: %s\n", err) - } else if !strings.Contains(string(txt), `[metadata] - version = "1.10"`) { - t.Fatalf(`Error: expected "%s" to be toml encoded go.toml`, txt) - } + for _, data := range []struct{ name, want string }{ + {"metadata.buildpack/launch.sha", "launch-sha"}, + {"metadata.buildpack/launch-build-cache.sha", "launch-build-cache-sha"}, + {"metadata.buildpack/launch-cache.sha", "launch-cache-sha"}, + {"no.cache.buildpack/some-layer.sha", "some-layer-sha"}, + } { + got := h.MustReadFile(t, filepath.Join(layerDir, data.name)) + h.AssertStringContains(t, string(got), data.want) + } + }) - if _, err := os.Stat(filepath.Join(layerDir, "metadata.buildpack")); !os.IsNotExist(err) { - t.Fatalf(`Error: expected /layer/metadata.buildpack to not exist`) - } - }) + it("does not restore launch=false layer metadata", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - it("should return the analyzed metadata", func() { - md, err := analyzer.Analyze(image) - h.AssertNil(t, err) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack", "launch-false.toml")) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack", "launch-false.sha")) + }) - h.AssertEq(t, md.Image.Reference, "s0m3D1g3sT") - h.AssertEq(t, md.Metadata, appImageMetadata) - }) + it("does not restore build=true, cache=false layer metadata", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - when("there is a launch/build layer that isn't cached", func() { - it("should not restore the metadata", func() { - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } - if _, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack/stale-launch-build.toml")); !os.IsNotExist(err) { - t.Fatalf("Found unexpected metadata for stale-launch-build layer") - } - }) - }) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack", "launch-build.sha")) + }) - when("there is a cache=true layer in the metadata but not in the cache", func() { - it("should not restore the metadata", func() { - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } - if _, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "launch-cache.toml")); !os.IsNotExist(err) { - t.Fatalf("Found unexpected metadata for launch-cache layer") - } - }) + when("subset of buildpacks are detected", func() { + it.Before(func() { + analyzer.Buildpacks = []lifecycle.Buildpack{{ID: "no.cache.buildpack"}} }) + it("restores layers for detected buildpacks", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - when("there are cached launch layers", func() { - it("leaves the layers", func() { - // copy to layerDir - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) + path := filepath.Join(layerDir, "no.cache.buildpack", "some-layer.toml") + got := h.MustReadFile(t, path) + want := "[metadata]\n some-layer-key = \"some-layer-value\"" - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } + h.AssertStringContains(t, string(got), want) + }) + it("does not restore layers for undetected buildpacks", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if txt, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "valid-launch", "valid-launch-file")); err != nil { - t.Fatalf("Error: %s\n", err) - } else if string(txt) != "valid-launch cached file" { - t.Fatalf("Error: expected cached node file to remain") - } - }) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack")) }) + }) - when("there are cached launch layers", func() { - it("leaves the layer dir and updates the metadata", func() { - // copy to layerDir - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) + it("returns the analyzed metadata", func() { + md, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } + h.AssertEq(t, md.Image.Reference, "s0m3D1g3sT") + h.AssertEq(t, md.Metadata, appImageMetadata) + }) - if txt, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "valid-launch.toml")); err != nil { - t.Fatalf("Error: %s\n", err) - } else { - expected := ` -[metadata] - akey = "avalue" - bkey = "bvalue" -` - if !strings.Contains(string(txt), expected) { - t.Fatalf(`Error: expected "%s" to contain "%s"`, string(txt), expected) - } - } - }) - }) + when("cache exists", func() { + it.Before(func() { + metadata := h.MustReadFile(t, filepath.Join("testdata", "analyzer", "cache_metadata.json")) + var cacheMetadata lifecycle.CacheMetadata + h.AssertNil(t, json.Unmarshal(metadata, &cacheMetadata)) + h.AssertNil(t, testCache.SetMetadata(cacheMetadata)) + h.AssertNil(t, testCache.Commit()) - when("there are cached launch/build layers", func() { - it("leaves the layer dir and updates the metadata", func() { - // copy to layerDir - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) + analyzer.Buildpacks = append(analyzer.Buildpacks, lifecycle.Buildpack{ID: "escaped/buildpack/id"}) + }) - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } + it("restores app and cache layer metadata", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if txt, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "valid-launch-build.toml")); err != nil { - t.Fatalf("Error: %s\n", err) - } else { - expected := ` -[metadata] - some-key = "val-from-metadata" - some-other-key = "val-from-metadata"` - if !strings.Contains(string(txt), expected) { - t.Fatalf("Error: expected metadata to be rewritten \nExpected:\n%s\n\nTo Contain:\n"+ - "%s\n", string(txt), expected) - } - } - }) + for _, data := range []struct{ name, want string }{ + // App layers. + {"metadata.buildpack/launch.toml", "[metadata]\n launch-key = \"launch-value\""}, + {"metadata.buildpack/launch-build-cache.toml", "[metadata]\n launch-build-cache-key = \"launch-build-cache-value\""}, + {"metadata.buildpack/launch-cache.toml", "[metadata]\n launch-cache-key = \"launch-cache-value\""}, + {"no.cache.buildpack/some-layer.toml", "[metadata]\n some-layer-key = \"some-layer-value\""}, + // Cache-image-only layers. + {"metadata.buildpack/cache.toml", "[metadata]\n cache-key = \"cache-value\""}, + } { + got := h.MustReadFile(t, filepath.Join(layerDir, data.name)) + h.AssertStringContains(t, string(got), data.want) + } }) - when("there are cached build layers", func() { - it("leaves the layers", func() { - // copy to layerDir - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) - - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } + it("restores app and cache layer sha files, prefers app sha", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if txt, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "build-layer", "build-layer-file")); err != nil { - t.Fatalf("Error: %s\n", err) - } else if string(txt) != "build-layer-file-contents" { - t.Fatalf("Error: expected cached node file to remain") - } - }) + for _, data := range []struct{ name, want string }{ + {"metadata.buildpack/launch.sha", "launch-sha"}, + {"metadata.buildpack/launch-build-cache.sha", "launch-build-cache-sha"}, + {"metadata.buildpack/launch-cache.sha", "launch-cache-sha"}, + {"no.cache.buildpack/some-layer.sha", "some-layer-sha"}, + // Cache-image-only layers. + {"metadata.buildpack/cache.sha", "cache-sha"}, + } { + got := h.MustReadFile(t, filepath.Join(layerDir, data.name)) + h.AssertStringContains(t, string(got), data.want) + } }) - when("there are stale cached launch layers", func() { - it("removes the layer dir and rewrites the metadata", func() { - // copy to layerDir - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) + it("does not overwrite metadata from app image", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) + for _, name := range []string{ + "metadata.buildpack/launch-build-cache.toml", + "metadata.buildpack/launch-cache.toml", + } { + got := h.MustReadFile(t, filepath.Join(layerDir, name)) + avoid := "[metadata]\n cache-only-key = \"cache-only-value\"" + if strings.Contains(string(got), avoid) { + t.Errorf("Expected %q to not contain %q, got %q", name, avoid, got) } + } + }) - var err error - if _, err = ioutil.ReadDir(filepath.Join(layerDir, "metadata.buildpack", "node_modules")); !os.IsNotExist(err) { - t.Fatalf("Found stale node_modules layer dir, it should not exist") - } - if txt, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "stale-launch.toml")); err != nil { - t.Fatalf("failed to read stale-launch.toml: %s", err) - } else if !strings.Contains(string(txt), `[metadata] - version = "1234"`) { - t.Fatalf(`Error: expected "%s" to be equal %s`, txt, `metadata.version = "1234"`) - } + it("does not overwrite sha from app image", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if _, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "stale-launch.sha")); !os.IsNotExist(err) { - t.Fatalf("Found stale stale-launch.sha, it should be removed") + for _, name := range []string{ + "metadata.buildpack/launch-build-cache.sha", + "metadata.buildpack/launch-cache.sha", + } { + got := h.MustReadFile(t, filepath.Join(layerDir, name)) + avoid := "old-sha" + if strings.Contains(string(got), avoid) { + t.Errorf("Expected %q to not contain %q, got %q", name, avoid, got) } - }) + } }) - when("there are malformed layers", func() { - it("removes the layer", func() { - // copy to layerDir - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) - - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } + it("does not restore cache=true layers for non-selected groups", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - var err error - if _, err = ioutil.ReadDir(filepath.Join(layerDir, "metadata.buildpack", "bad-layer")); !os.IsNotExist(err) { - t.Fatalf("Found bad-layer layer dir, it should be removed") - } - if _, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "bad-layer.toml")); !os.IsNotExist(err) { - t.Fatalf("found bad-layer.toml, it should be removed") - } - if _, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "bad-layer.sha")); !os.IsNotExist(err) { - t.Fatalf("Found stale bad-layer.sha, it should be removed") - } - }) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "no.group.buildpack")) }) - when("there are stale cached launch/build layers", func() { - it("removes the layer dir and metadata", func() { - // copy to layerDir - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) - - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } + it("does not restore launch=true layer metadata", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - var err error - if _, err = ioutil.ReadDir(filepath.Join(layerDir, "metadata.buildpack", "stale-launch-build")); !os.IsNotExist(err) { - t.Fatalf("Found stale stale-launch-build layer dir, it should not exist") - } + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack", "launch-cache-not-in-app.toml")) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack", "launch-cache-not-in-app.sha")) + }) - if _, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "stale-launch-build.toml")); !os.IsNotExist(err) { - t.Fatalf("Found stale stale-launch-build.toml, it should be removed") - } + it("does not restore cache=false layer metadata", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if _, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "stale-launch-build.sha")); !os.IsNotExist(err) { - t.Fatalf("Found stale stale-launch-build.sha, it should be removed") - } - }) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack", "cache-false.toml")) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack", "cache-false.sha")) }) - when("there cached launch layers that are missing from metadata", func() { - it("removes the layer dir and metadata", func() { - // copy to layerDir - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) + it("restores escaped buildpack layer metadata", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } + path := filepath.Join(layerDir, "escaped_buildpack_id", "escaped-bp-layer.toml") + got := h.MustReadFile(t, path) + want := "[metadata]\n escaped-bp-layer-key = \"escaped-bp-layer-value\"" - var err error - if _, err = ioutil.ReadDir(filepath.Join(layerDir, "metadata.buildpack", "old-layer")); !os.IsNotExist(err) { - t.Fatalf("Found stale old-layer layer dir, it should not exist") - } - - if _, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "old-layer.toml")); !os.IsNotExist(err) { - t.Fatalf("Found stale old-layer.toml, it should be removed") - } + h.AssertStringContains(t, string(got), want) + }) - if _, err := ioutil.ReadFile(filepath.Join(layerDir, "metadata.buildpack", "old-layer.sha")); !os.IsNotExist(err) { - t.Fatalf("Found stale old-layer.sha, it should be removed") - } + when("subset of buildpacks are detected", func() { + it.Before(func() { + analyzer.Buildpacks = []lifecycle.Buildpack{{ID: "no.group.buildpack"}} }) - }) - when("there are cached layers for a buildpack that is missing from the group", func() { - it("does not remove app layers", func() { - // copy to layerDir - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) + it("restores layers for detected buildpacks", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } + path := filepath.Join(layerDir, "no.group.buildpack", "some-layer.toml") + got := h.MustReadFile(t, path) + want := "[metadata]\n some-layer-key = \"some-layer-value\"" - appFile := filepath.Join(layerDir, "some-app-dir", "appfile") - if txt, err := ioutil.ReadFile(appFile); err != nil { - t.Fatalf("Error: %s\n", err) - } else if !strings.Contains(string(txt), "appFile file contents") { - t.Fatalf(`Error: expected "%s" to still exist`, appFile) - } + h.AssertStringContains(t, string(got), want) }) + it("does not restore layers for undetected buildpacks", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - it("does not remove remaining layerDir files", func() { - // copy to layerDir - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) - - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } - - appFile := filepath.Join(layerDir, "config.toml") - if txt, err := ioutil.ReadFile(appFile); err != nil { - t.Fatalf("Error: %s\n", err) - } else if !strings.Contains(string(txt), "someNoneLayer = \"file\"") { - t.Fatalf(`Error: expected "%s" to still exist`, appFile) - } + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack")) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "escaped_buildpack_id")) }) }) + }) - when("there are cached non launch layers for a buildpack that is missing from metadata", func() { - it("keeps the layers", func() { - // copy to layerDir - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) + when("analyzer is running as root", func() { + it.Before(func() { + if os.Getuid() != 0 { + t.Skip("Skipped when not running as root") + } + }) - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } + it("chowns new files to CNB_USER_ID:CNB_GROUP_ID", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) + h.AssertUIDGID(t, layerDir, 1234, 4321) + h.AssertUIDGID(t, filepath.Join(layerDir, "metadata.buildpack", "launch.toml"), 1234, 4321) + h.AssertUIDGID(t, filepath.Join(layerDir, "no.cache.buildpack"), 1234, 4321) + h.AssertUIDGID(t, filepath.Join(layerDir, "no.cache.buildpack", "some-layer.toml"), 1234, 4321) + }) + }) - buildLayerFile := filepath.Join(layerDir, "no.metadata.buildpack", "buildlayer", "buildlayerfile") - if txt, err := ioutil.ReadFile(buildLayerFile); err != nil { - t.Fatalf("Error: %s\n", err) - } else if !strings.Contains(string(txt), "buildlayer file contents") { - t.Fatalf(`Error: expected "%s" to still exist`, buildLayerFile) - } - }) + when("skip-layers is true", func() { + it.Before(func() { + analyzer.SkipLayers = true }) - when("there are cached non launch for a buildpack that is missing from metadata", func() { - it("removes the layers", func() { - // copy to layerDir - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) + it("should return the analyzed metadata", func() { + md, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if _, err := analyzer.Analyze(image); err != nil { - t.Fatalf("Error: %s\n", err) - } + h.AssertEq(t, md.Image.Reference, "s0m3D1g3sT") + h.AssertEq(t, md.Metadata, appImageMetadata) + }) - var err error - if _, err = ioutil.ReadDir(filepath.Join(layerDir, "no.metadata.buildpack", "launchlayer")); !os.IsNotExist(err) { - t.Fatalf("Found stale launchlayer layer dir, it should not exist") - } + it("does not write buildpack layer metadata", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if _, err := ioutil.ReadFile(filepath.Join(layerDir, "no.metadata.buildpack", "launchlayer.toml")); !os.IsNotExist(err) { - t.Fatalf("Found stale launchlayer.toml, it should be removed") - } - }) + files, err := ioutil.ReadDir(layerDir) + h.AssertNil(t, err) + h.AssertEq(t, len(files), 0) }) + }) + }) - when("analyzer is running as root", func() { - it.Before(func() { - if os.Getuid() != 0 { - t.Skip("Skipped when not running as root") - } - }) + when("image is not found", func() { + it.Before(func() { + h.AssertNil(t, image.Delete()) + }) - it("chowns new files to CNB_USER_ID:CNB_GROUP_ID", func() { - _, err := analyzer.Analyze(image) - h.AssertNil(t, err) - h.AssertUIDGID(t, layerDir, 1234, 4321) - h.AssertUIDGID(t, filepath.Join(layerDir, "metadata.buildpack", "valid-launch.toml"), 1234, 4321) - h.AssertUIDGID(t, filepath.Join(layerDir, "no.cache.buildpack"), 1234, 4321) - h.AssertUIDGID(t, filepath.Join(layerDir, "no.cache.buildpack", "go.toml"), 1234, 4321) - }) + when("cache exists", func() { + it.Before(func() { + metadata := h.MustReadFile(t, filepath.Join("testdata", "analyzer", "cache_metadata.json")) + var cacheMetadata lifecycle.CacheMetadata + h.AssertNil(t, json.Unmarshal(metadata, &cacheMetadata)) + h.AssertNil(t, testCache.SetMetadata(cacheMetadata)) + h.AssertNil(t, testCache.Commit()) + + analyzer.Buildpacks = append(analyzer.Buildpacks, lifecycle.Buildpack{ID: "escaped/buildpack/id"}) }) - when("skip-layers is true", func() { - it.Before(func() { - analyzer.SkipLayers = true - }) + it("restores cache=true layer metadata", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - it("should return the analyzed metadata", func() { - md, err := analyzer.Analyze(image) - h.AssertNil(t, err) + for _, data := range []struct{ name, want string }{ + {"metadata.buildpack/cache.toml", "[metadata]\n cache-key = \"cache-value\""}, + } { + got := h.MustReadFile(t, filepath.Join(layerDir, data.name)) + h.AssertStringContains(t, string(got), data.want) + } + }) - h.AssertEq(t, md.Image.Reference, "s0m3D1g3sT") - h.AssertEq(t, md.Metadata, appImageMetadata) - }) + it("does not restore launch=true layer metadata", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - it("does not write buildpack layers", func() { - _, err := analyzer.Analyze(image) - h.AssertNil(t, err) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack", "launch-cache.toml")) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack", "launch-build-cache.toml")) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack", "launch-cache-not-in-app.toml")) + }) - fileInfos, err := ioutil.ReadDir(layerDir) - h.AssertNil(t, err) + it("does not restore cache=false layer metadata", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - h.AssertEq(t, len(fileInfos), 0) - }) + h.AssertPathDoesNotExist(t, filepath.Join(layerDir, "metadata.buildpack", "cache-false.toml")) }) - }) - }) - when("the image cannot be found", func() { - var notFoundImage *fakes.Image + it("returns a nil image in the analyzed metadata", func() { + md, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - it.Before(func() { - notFoundImage = fakes.NewImage("image-repo-name", "", nil) - h.AssertNil(t, notFoundImage.Delete()) + h.AssertNil(t, md.Image) + h.AssertEq(t, md.Metadata, lifecycle.LayersMetadata{}) + }) }) + when("cache is empty", func() { + it("does not restore any metadata", func() { + _, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - it("clears the cached launch layers", func() { - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) - _, err := analyzer.Analyze(notFoundImage) - h.AssertNil(t, err) + files, err := ioutil.ReadDir(layerDir) + h.AssertNil(t, err) + h.AssertEq(t, len(files), 0) + }) + it("returns a nil image in the analyzed metadata", func() { + md, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) - if _, err := ioutil.ReadDir(filepath.Join(layerDir, "no.metadata.buildpack", "launchlayer")); !os.IsNotExist(err) { - t.Fatalf("Found stale launchlayer cache, it should not exist") - } - if _, err := ioutil.ReadDir(filepath.Join(layerDir, "metadata.buildpack", "stale-launch-build")); !os.IsNotExist(err) { - t.Fatalf("Found stale stale-launch-build cache, it should not exist") - } - if _, err := ioutil.ReadDir(filepath.Join(layerDir, "some-app-dir")); err != nil { - t.Fatalf("Missing some-app-dir") - } + h.AssertNil(t, md.Image) + h.AssertEq(t, md.Metadata, lifecycle.LayersMetadata{}) + }) }) + when("cache is not provided", func() { + it("does not restore any metadata", func() { + _, err := analyzer.Analyze(image, nil) + h.AssertNil(t, err) - it("should return a nil image in the analyzed metadata", func() { - md, err := analyzer.Analyze(notFoundImage) - h.AssertNil(t, err) + files, err := ioutil.ReadDir(layerDir) + h.AssertNil(t, err) + h.AssertEq(t, len(files), 0) + }) + it("returns a nil image in the analyzed metadata", func() { + md, err := analyzer.Analyze(image, nil) + h.AssertNil(t, err) - h.AssertNil(t, md.Image) - h.AssertEq(t, md.Metadata, lifecycle.LayersMetadata{}) + h.AssertNil(t, md.Image) + h.AssertEq(t, md.Metadata, lifecycle.LayersMetadata{}) + }) }) }) - when("the image does not have the required label", func() { + when("image does not have metadata label", func() { it.Before(func() { h.AssertNil(t, image.SetLabel("io.buildpacks.lifecycle.metadata", "")) }) - - it("returns", func() { - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) - - _, err := analyzer.Analyze(image) + it("does not restore any metadata", func() { + _, err := analyzer.Analyze(image, testCache) h.AssertNil(t, err) - if _, err := ioutil.ReadDir(filepath.Join(layerDir, "no.metadata.buildpack", "launchlayer")); !os.IsNotExist(err) { - t.Fatalf("Found stale launchlayer cache, it should not exist") - } - if _, err := ioutil.ReadDir(filepath.Join(layerDir, "metadata.buildpack", "stale-launch-build")); !os.IsNotExist(err) { - t.Fatalf("Found stale stale-launch-build cache, it should not exist") - } - if _, err := ioutil.ReadDir(filepath.Join(layerDir, "some-app-dir")); err != nil { - t.Fatalf("Missing some-app-dir") - } + files, err := ioutil.ReadDir(layerDir) + h.AssertNil(t, err) + h.AssertEq(t, len(files), 0) + }) + it("returns empty analyzed metadata", func() { + md, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) + h.AssertEq(t, md.Metadata, lifecycle.LayersMetadata{}) }) }) - when("the image label has incompatible metadata", func() { + when("image has incompatible metadata", func() { it.Before(func() { h.AssertNil(t, image.SetLabel("io.buildpacks.lifecycle.metadata", `{["bad", "metadata"]}`)) }) - - it("returns", func() { - h.RecursiveCopy(t, filepath.Join("testdata", "analyzer", "cached-layers"), layerDir) - - _, err := analyzer.Analyze(image) + it("does not restore any metadata", func() { + _, err := analyzer.Analyze(image, testCache) h.AssertNil(t, err) - if _, err := ioutil.ReadDir(filepath.Join(layerDir, "no.metadata.buildpack", "launchlayer")); !os.IsNotExist(err) { - t.Fatalf("Found stale launchlayer cache, it should not exist") - } - if _, err := ioutil.ReadDir(filepath.Join(layerDir, "metadata.buildpack", "stale-launch-build")); !os.IsNotExist(err) { - t.Fatalf("Found stale stale-launch-build cache, it should not exist") - } - if _, err := ioutil.ReadDir(filepath.Join(layerDir, "some-app-dir")); err != nil { - t.Fatalf("Missing some-app-dir") - } + files, err := ioutil.ReadDir(layerDir) + h.AssertNil(t, err) + h.AssertEq(t, len(files), 0) + }) + it("returns empty analyzed metadata", func() { + md, err := analyzer.Analyze(image, testCache) + h.AssertNil(t, err) + h.AssertEq(t, md.Metadata, lifecycle.LayersMetadata{}) }) }) }) diff --git a/cache/image_cache.go b/cache/image_cache.go index 0cdbc4fe2..eb528390d 100644 --- a/cache/image_cache.go +++ b/cache/image_cache.go @@ -6,9 +6,12 @@ import ( "io" "github.com/buildpack/imgutil" + "github.com/buildpack/imgutil/remote" "github.com/pkg/errors" "github.com/buildpack/lifecycle" + + "github.com/google/go-containerregistry/pkg/authn" ) const MetadataLabel = "io.buildpacks.lifecycle.cache.metadata" @@ -26,6 +29,26 @@ func NewImageCache(origImage imgutil.Image, newImage imgutil.Image) *ImageCache } } +func NewImageCacheFromName(name string, keychain authn.Keychain) (*ImageCache, error) { + origImage, err := remote.NewImage( + name, + keychain, + remote.FromBaseImage(name), + ) + if err != nil { + return nil, fmt.Errorf("accessing cache image %q: %v", name, err) + } + emptyImage, err := remote.NewImage( + name, + keychain, + remote.WithPreviousImage(name), + ) + if err != nil { + return nil, fmt.Errorf("creating new cache image %q: %v", name, err) + } + return NewImageCache(origImage, emptyImage), nil +} + func (c *ImageCache) Name() string { return c.origImage.Name() } diff --git a/cmd/analyzer/main.go b/cmd/analyzer/main.go index 899a77cc9..917fb3f87 100644 --- a/cmd/analyzer/main.go +++ b/cmd/analyzer/main.go @@ -15,34 +15,39 @@ import ( "github.com/buildpack/lifecycle" "github.com/buildpack/lifecycle/auth" + "github.com/buildpack/lifecycle/cache" "github.com/buildpack/lifecycle/cmd" ) var ( - analyzedPath string - appDir string - gid int - groupPath string - layersDir string - repoName string - skipLayers bool - uid int - useDaemon bool - useHelpers bool - printVersion bool - logLevel string + analyzedPath string + appDir string + cacheDir string + cacheImageTag string + groupPath string + imageName string + layersDir string + skipLayers bool + useDaemon bool + useHelpers bool + uid int + gid int + printVersion bool + logLevel string ) func init() { cmd.FlagAnalyzedPath(&analyzedPath) cmd.FlagAppDir(&appDir) - cmd.FlagGID(&gid) + cmd.FlagCacheDir(&cacheDir) + cmd.FlagCacheImage(&cacheImageTag) cmd.FlagGroupPath(&groupPath) cmd.FlagLayersDir(&layersDir) - cmd.FlagUID(&uid) - cmd.FlagUseDaemon(&useDaemon) - cmd.FlagUseCredHelpers(&useHelpers) cmd.FlagSkipLayers(&skipLayers) + cmd.FlagUseCredHelpers(&useHelpers) + cmd.FlagUseDaemon(&useDaemon) + cmd.FlagUID(&uid) + cmd.FlagGID(&gid) cmd.FlagVersion(&printVersion) cmd.FlagLogLevel(&logLevel) } @@ -67,13 +72,17 @@ func main() { if flag.Arg(0) == "" { cmd.Exit(cmd.FailErrCode(errors.New("image argument is required"), cmd.CodeInvalidArgs, "parse arguments")) } - repoName = flag.Arg(0) + imageName = flag.Arg(0) + + if !skipLayers && cacheImageTag == "" && cacheDir == "" { + cmd.Logger.Warn("Not restoring cached layer data, no cache flag specified.") + } cmd.Exit(analyzer()) } func analyzer() error { if useHelpers { - if err := lifecycle.SetupCredHelpers(filepath.Join(os.Getenv("HOME"), ".docker"), repoName); err != nil { + if err := lifecycle.SetupCredHelpers(filepath.Join(os.Getenv("HOME"), ".docker"), imageName); err != nil { return cmd.FailErr(err, "setup credential helpers") } } @@ -101,25 +110,38 @@ func analyzer() error { return cmd.FailErr(err, "create docker client") } img, err = local.NewImage( - repoName, + imageName, dockerClient, - local.FromBaseImage(repoName), + local.FromBaseImage(imageName), ) if err != nil { return cmd.FailErr(err, "access previous image") } } else { img, err = remote.NewImage( - repoName, + imageName, auth.EnvKeychain(cmd.EnvRegistryAuth), - remote.FromBaseImage(repoName), + remote.FromBaseImage(imageName), ) if err != nil { return cmd.FailErr(err, "access previous image") } } - md, err := analyzer.Analyze(img) + var cacheStore lifecycle.Cache + if cacheImageTag != "" { + cacheStore, err = cache.NewImageCacheFromName(cacheImageTag, auth.EnvKeychain(cmd.EnvRegistryAuth)) + if err != nil { + return cmd.FailErr(err, "create image cache") + } + } else if cacheDir != "" { + cacheStore, err = cache.NewVolumeCache(cacheDir) + if err != nil { + return cmd.FailErr(err, "create volume cache") + } + } + + md, err := analyzer.Analyze(img, cacheStore) if err != nil { return cmd.FailErrCode(err, cmd.CodeFailed, "analyze") } diff --git a/cmd/restorer/main.go b/cmd/restorer/main.go index 2f2a367d9..d5ffd21d4 100644 --- a/cmd/restorer/main.go +++ b/cmd/restorer/main.go @@ -6,8 +6,6 @@ import ( "io/ioutil" "log" - "github.com/buildpack/imgutil/remote" - "github.com/buildpack/lifecycle" "github.com/buildpack/lifecycle/auth" "github.com/buildpack/lifecycle/cache" @@ -15,10 +13,10 @@ import ( ) var ( - cacheImageTag string cacheDir string - layersDir string + cacheImageTag string groupPath string + layersDir string uid int gid int printVersion bool @@ -26,10 +24,10 @@ var ( ) func init() { - cmd.FlagLayersDir(&layersDir) - cmd.FlagCacheImage(&cacheImageTag) cmd.FlagCacheDir(&cacheDir) + cmd.FlagCacheImage(&cacheImageTag) cmd.FlagGroupPath(&groupPath) + cmd.FlagLayersDir(&layersDir) cmd.FlagUID(&uid) cmd.FlagGID(&gid) cmd.FlagVersion(&printVersion) @@ -37,7 +35,7 @@ func init() { } func main() { - // suppress output from libraries, lifecycle will not use standard logger + // Suppress output from libraries, lifecycle will not use standard logger. log.SetOutput(ioutil.Discard) flag.Parse() @@ -54,7 +52,7 @@ func main() { cmd.Exit(cmd.FailErrCode(errors.New("received unexpected args"), cmd.CodeInvalidArgs, "parse arguments")) } if cacheImageTag == "" && cacheDir == "" { - cmd.Exit(cmd.FailErrCode(errors.New("must supply either -image or -path"), cmd.CodeInvalidArgs, "parse arguments")) + cmd.Logger.Warn("Not restoring cached layer data, no cache flag specified.") } cmd.Exit(restore()) } @@ -75,30 +73,11 @@ func restore() error { var cacheStore lifecycle.Cache if cacheImageTag != "" { - origCacheImage, err := remote.NewImage( - cacheImageTag, - auth.EnvKeychain(cmd.EnvRegistryAuth), - remote.FromBaseImage(cacheImageTag), - ) - if err != nil { - return cmd.FailErr(err, "accessing cache image") - } - - emptyImage, err := remote.NewImage( - cacheImageTag, - auth.EnvKeychain(cmd.EnvRegistryAuth), - remote.WithPreviousImage(cacheImageTag), - ) + cacheStore, err = cache.NewImageCacheFromName(cacheImageTag, auth.EnvKeychain(cmd.EnvRegistryAuth)) if err != nil { - return cmd.FailErr(err, "creating empty image") + return cmd.FailErr(err, "create image cache") } - - cacheStore = cache.NewImageCache( - origCacheImage, - emptyImage, - ) - } else { - var err error + } else if cacheDir != "" { cacheStore, err = cache.NewVolumeCache(cacheDir) if err != nil { return cmd.FailErr(err, "create volume cache") diff --git a/go.mod b/go.mod index eb65a04c6..6d5ba8fc3 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/pkg/errors v0.8.1 github.com/sclevine/spec v1.2.0 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect + golang.org/x/sync v0.0.0-20190423024810-112230192c58 google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8 // indirect ) diff --git a/layers.go b/layers.go index ae3b96bd9..a3d1a5436 100644 --- a/layers.go +++ b/layers.go @@ -86,41 +86,10 @@ func (bd *bpLayersDir) newBPLayer(name string) *bpLayer { } } -type cacheType int - -const ( - cacheStaleNoMetadata cacheType = iota - cacheStaleWrongSHA - cacheNotForLaunch // we can't determine whether the cache is stale for launch=false layers - cacheValid - cacheMalformed -) - type bpLayer struct { layer } -func (bp *bpLayer) classifyCache(metadataLayers map[string]BuildpackLayerMetadata) cacheType { - cachedLayer, err := bp.read() - if err != nil { - return cacheMalformed - } - if !cachedLayer.Launch { - return cacheNotForLaunch - } - if metadataLayers == nil { - return cacheStaleNoMetadata - } - layerMetadata, ok := metadataLayers[bp.name()] - if !ok { - return cacheStaleNoMetadata - } - if layerMetadata.SHA != cachedLayer.SHA { - return cacheStaleWrongSHA - } - return cacheValid -} - func (bp *bpLayer) read() (BuildpackLayerMetadata, error) { var data BuildpackLayerMetadata tomlPath := bp.path + ".toml" @@ -158,8 +127,7 @@ func (bp *bpLayer) remove() error { return nil } -func (bp *bpLayer) writeMetadata(metadataLayers map[string]BuildpackLayerMetadata) error { - layerMetadata := metadataLayers[bp.name()] +func (bp *bpLayer) writeMetadata(metadata BuildpackLayerMetadata) error { path := filepath.Join(bp.path + ".toml") if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil { return err @@ -169,7 +137,7 @@ func (bp *bpLayer) writeMetadata(metadataLayers map[string]BuildpackLayerMetadat return err } defer fh.Close() - return toml.NewEncoder(fh).Encode(layerMetadata.BuildpackLayerMetadataFile) + return toml.NewEncoder(fh).Encode(metadata.BuildpackLayerMetadataFile) } func (bp *bpLayer) hasLocalContents() bool { diff --git a/restorer.go b/restorer.go index 85ab304e8..72ad5c2b8 100644 --- a/restorer.go +++ b/restorer.go @@ -4,6 +4,7 @@ import ( "os" "github.com/pkg/errors" + "golang.org/x/sync/errgroup" "github.com/buildpack/lifecycle/archive" ) @@ -16,33 +17,60 @@ type Restorer struct { GID int } +// Restore attempts to restore layer data for cache=true layers, removing the layer when unsuccessful. +// If a usable cache is not provided, Restore will remove all cache=true layer metadata. func (r *Restorer) Restore(cache Cache) error { - meta, err := cache.RetrieveMetadata() - if err != nil { - return err - } - - if len(meta.Buildpacks) == 0 { - r.Logger.Infof("Cache '%s': metadata not found, nothing to restore", cache.Name()) - return nil + // Create empty cache metadata in case a usable cache is not provided. + var meta CacheMetadata + if cache != nil { + var err error + meta, err = cache.RetrieveMetadata() + if err != nil { + return errors.Wrapf(err, "retrieving cache metadata") + } + } else { + r.Logger.Debug("Usable cache not provided, using empty cache metadata.") } - for _, bp := range r.Buildpacks { - layersDir, err := readBuildpackLayersDir(r.LayersDir, bp) + var g errgroup.Group + for _, buildpack := range r.Buildpacks { + buildpackDir, err := readBuildpackLayersDir(r.LayersDir, buildpack) if err != nil { - return err + return errors.Wrapf(err, "reading buildpack layer directory") } - bpMD := meta.MetadataForBuildpack(bp.ID) - for name, layer := range bpMD.Layers { - if !layer.Cache { + + cachedLayers := meta.MetadataForBuildpack(buildpack.ID).Layers + for _, bpLayer := range buildpackDir.findLayers(cached) { + name := bpLayer.name() + cachedLayer, exists := cachedLayers[name] + if !exists { + r.Logger.Infof("Removing %q, not in cache", bpLayer.Identifier()) + if err := bpLayer.remove(); err != nil { + return errors.Wrapf(err, "removing layer") + } continue } - - if err := r.restoreLayer(name, bpMD, layer, layersDir, cache); err != nil { - return err + data, err := bpLayer.read() + if err != nil { + return errors.Wrapf(err, "reading layer") + } + if data.SHA != cachedLayer.SHA { + r.Logger.Infof("Removing %q, wrong sha", bpLayer.Identifier()) + r.Logger.Debugf("Layer sha: %q, cache sha: %q", data.SHA, cachedLayer.SHA) + if err := bpLayer.remove(); err != nil { + return errors.Wrapf(err, "removing layer") + } + } else { + r.Logger.Infof("Restoring data for %q from cache", bpLayer.Identifier()) + g.Go(func() error { + return r.restoreLayer(cache, cachedLayer.SHA) + }) } } } + if err := g.Wait(); err != nil { + return errors.Wrap(err, "restoring data") + } // if restorer is running as root it needs to fix the ownership of the layers dir if current := os.Getuid(); current == -1 { @@ -55,21 +83,13 @@ func (r *Restorer) Restore(cache Cache) error { return nil } -func (r *Restorer) restoreLayer(name string, bpMD BuildpackLayersMetadata, layer BuildpackLayerMetadata, layersDir bpLayersDir, cache Cache) error { - bpLayer := layersDir.newBPLayer(name) - - r.Logger.Infof("Restoring cached layer '%s'", bpLayer.Identifier()) - if err := bpLayer.writeMetadata(bpMD.Layers); err != nil { - return err - } - - if layer.Launch { - if err := bpLayer.writeSha(layer.SHA); err != nil { - return err - } +func (r *Restorer) restoreLayer(cache Cache, sha string) error { + // Sanity check to prevent panic. + if cache == nil { + return errors.New("restoring layer: cache not provided") } - - rc, err := cache.RetrieveLayer(layer.SHA) + r.Logger.Debugf("Retrieving data for %q", sha) + rc, err := cache.RetrieveLayer(sha) if err != nil { return err } diff --git a/restorer_test.go b/restorer_test.go index 5a392f462..56421f764 100644 --- a/restorer_test.go +++ b/restorer_test.go @@ -5,17 +5,18 @@ import ( "io/ioutil" "os" "path/filepath" - "strings" "testing" "github.com/apex/log" "github.com/apex/log/handlers/discard" + "github.com/pkg/errors" "github.com/sclevine/spec" "github.com/sclevine/spec/report" "github.com/buildpack/lifecycle" "github.com/buildpack/lifecycle/archive" "github.com/buildpack/lifecycle/cache" + "github.com/buildpack/lifecycle/cmd" h "github.com/buildpack/lifecycle/testhelpers" ) @@ -54,6 +55,10 @@ func testRestorer(t *testing.T, when spec.G, it spec.S) { UID: 1234, GID: 4321, } + if testing.Verbose() { + restorer.Logger = cmd.Logger + cmd.SetLogLevel("debug") + } }) it.After(func() { @@ -61,9 +66,69 @@ func testRestorer(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, os.RemoveAll(cacheDir)) }) - when("there is no previous cache", func() { - it("does nothing", func() { - h.AssertNil(t, restorer.Restore(testCache)) + when("there is an no cache", func() { + when("there is a cache=true layer", func() { + it.Before(func() { + meta := "cache=true" + h.AssertNil(t, writeLayer(layersDir, "buildpack.id", "cache-true", meta, "cache-only-layer-sha")) + h.AssertNil(t, restorer.Restore(nil)) + }) + + it("removes metadata and sha file", func() { + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-true.toml")) + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-true.sha")) + }) + it("does not restore layer data", func() { + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-true")) + }) + }) + when("there is a cache=false layer", func() { + it.Before(func() { + meta := "cache=false" + h.AssertNil(t, writeLayer(layersDir, "buildpack.id", "cache-false", meta, "cache-false-layer-sha")) + h.AssertNil(t, restorer.Restore(testCache)) + }) + + it("keeps metadata and sha file", func() { + h.AssertPathExists(t, filepath.Join(layersDir, "buildpack.id", "cache-false.toml")) + h.AssertPathExists(t, filepath.Join(layersDir, "buildpack.id", "cache-false.sha")) + }) + it("does not restore layer data", func() { + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-false")) + }) + }) + }) + + when("there is an empty cache", func() { + when("there is a cache=true layer", func() { + it.Before(func() { + meta := "cache=true" + h.AssertNil(t, writeLayer(layersDir, "buildpack.id", "cache-true", meta, "cache-only-layer-sha")) + h.AssertNil(t, restorer.Restore(testCache)) + }) + + it("removes metadata and sha file", func() { + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-true.toml")) + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-true.sha")) + }) + it("does not restore layer data", func() { + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-true")) + }) + }) + when("there is a cache=false layer", func() { + it.Before(func() { + meta := "cache=false" + h.AssertNil(t, writeLayer(layersDir, "buildpack.id", "cache-false", meta, "cache-false-layer-sha")) + h.AssertNil(t, restorer.Restore(testCache)) + }) + + it("keeps metadata and sha file", func() { + h.AssertPathExists(t, filepath.Join(layersDir, "buildpack.id", "cache-false.toml")) + h.AssertPathExists(t, filepath.Join(layersDir, "buildpack.id", "cache-false.sha")) + }) + it("does not restore layer data", func() { + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-false")) + }) }) }) @@ -124,58 +189,49 @@ func testRestorer(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, os.Mkdir(layersDir, 0777)) contents := fmt.Sprintf(`{ - "buildpacks": [ - { - "key": "buildpack.id", - "layers": { - "cache-only": { - "data": { - "cache-only-key": "cache-only-val" - }, - "cache": true, - "sha": "%s" - }, - "cache-false": { - "data": { - "cache-false-key": "cache-false-val" - }, - "cache": false, - "sha": "%s" - }, - "cache-launch": { - "data": { - "cache-launch-key": "cache-launch-val" - }, - "cache": true, - "launch": true, - "sha": "%s" - } - } - }, { - "key": "nogroup.buildpack.id", - "layers": { - "some-layer": { - "data": { - "no-group-key": "no-group-val" - }, - "cache": true, - "sha": "%s" - } - } - }, { - "key": "escaped/buildpack/id", - "layers": { - "escaped-bp-layer": { - "data": { - "escaped-bp-key": "escaped-bp-val" - }, - "cache": true, - "sha": "%s" - } - } - } - ] - }`, cacheOnlyLayerSHA, cacheFalseLayerSHA, cacheLaunchLayerSHA, noGroupLayerSHA, escapedLayerSHA) + "buildpacks": [ + { + "key": "buildpack.id", + "layers": { + "cache-false": { + "cache": false, + "sha": "%s" + }, + "cache-launch": { + "cache": true, + "launch": true, + "sha": "%s" + }, + "cache-only": { + "cache": true, + "data": { + "some-key": "some-value" + }, + "sha": "%s" + } + } + }, + { + "key": "nogroup.buildpack.id", + "layers": { + "some-layer": { + "cache": true, + "sha": "%s" + } + } + }, + { + "key": "escaped/buildpack/id", + "layers": { + "escaped-bp-layer": { + "cache": true, + "sha": "%s" + } + } + } + ] +} +`, cacheFalseLayerSHA, cacheLaunchLayerSHA, cacheOnlyLayerSHA, noGroupLayerSHA, escapedLayerSHA) err = ioutil.WriteFile( filepath.Join(cacheDir, "committed", "io.buildpacks.lifecycle.cache.metadata"), @@ -189,89 +245,187 @@ func testRestorer(t *testing.T, when spec.G, it spec.S) { os.RemoveAll(tarTempDir) }) - it("restores cached layers", func() { - h.AssertNil(t, restorer.Restore(testCache)) - expectedMetadata := `[metadata] + when("there is a cache=true layer", func() { + var meta string + it.Before(func() { + meta = `cache=true +[metadata] cache-only-key = "cache-only-val"` - if txt, err := ioutil.ReadFile(filepath.Join(layersDir, "buildpack.id", "cache-only.toml")); err != nil { - t.Fatalf("failed to read cache-only.toml: %s", err) - } else if !strings.Contains(string(txt), expectedMetadata) { - t.Fatalf(`Error: expected '%s' to contain '%s'`, txt, expectedMetadata) - } - - expectedText := "echo text from cache-only layer" - if txt, err := ioutil.ReadFile(filepath.Join(layersDir, "buildpack.id", "cache-only", "file-from-cache-only-layer")); err != nil { - t.Fatalf("failed to read file-from-cache-only-layer: %s", err) - } else if !strings.Contains(string(txt), expectedText) { - t.Fatalf(`Error: expected '%s' to contain '%s'`, txt, expectedText) - } + h.AssertNil(t, writeLayer(layersDir, "buildpack.id", "cache-only", meta, cacheOnlyLayerSHA)) + h.AssertNil(t, restorer.Restore(testCache)) + }) + + it("keeps layer metadatata", func() { + got := h.MustReadFile(t, filepath.Join(layersDir, "buildpack.id", "cache-only.toml")) + h.AssertEq(t, string(got), meta) + }) + it("keeps layer sha", func() { + got := h.MustReadFile(t, filepath.Join(layersDir, "buildpack.id", "cache-only.sha")) + h.AssertEq(t, string(got), cacheOnlyLayerSHA) + }) + it("restores data", func() { + got := h.MustReadFile(t, filepath.Join(layersDir, "buildpack.id", "cache-only", "file-from-cache-only-layer")) + want := "echo text from cache-only layer\n" + h.AssertEq(t, string(got), want) + }) }) - it("write a .sha file for launch layers", func() { - h.AssertNil(t, restorer.Restore(testCache)) - expectedMetadata := `[metadata] - cache-launch-key = "cache-launch-val"` - if txt, err := ioutil.ReadFile(filepath.Join(layersDir, "buildpack.id", "cache-launch.toml")); err != nil { - t.Fatalf("failed to read cache-launch.toml: %s", err) - } else if !strings.Contains(string(txt), expectedMetadata) { - t.Fatalf(`Error: expected '%s' to contain '%s'`, txt, expectedMetadata) - } - - expectedText := "echo text from cache launch layer" - if txt, err := ioutil.ReadFile(filepath.Join(layersDir, "buildpack.id", "cache-launch", "file-from-cache-launch-layer")); err != nil { - t.Fatalf("failed to read file-from-cache-launch-layer: %s", err) - } else if !strings.Contains(string(txt), expectedText) { - t.Fatalf(`Error: expected '%s' to contain '%s'`, txt, expectedText) - } - - if sha, err := ioutil.ReadFile(filepath.Join(layersDir, "buildpack.id", "cache-launch.sha")); err != nil { - t.Fatalf("failed to read cache-launch.sha: %s", err) - } else if string(sha) != cacheLaunchLayerSHA { - t.Fatalf(`Error: expected '%s' to be equal to '%s'`, sha, cacheLaunchLayerSHA) - } + when("there is a cache=false layer", func() { + var meta string + it.Before(func() { + meta = `cache=false +[metadata] + cache-false-key = "cache-false-val"` + h.AssertNil(t, writeLayer(layersDir, "buildpack.id", "cache-false", meta, cacheFalseLayerSHA)) + h.AssertNil(t, restorer.Restore(testCache)) + }) + + it("keeps layer metadatata", func() { + got := h.MustReadFile(t, filepath.Join(layersDir, "buildpack.id", "cache-false.toml")) + h.AssertEq(t, string(got), meta) + }) + it("keeps layer sha", func() { + got := h.MustReadFile(t, filepath.Join(layersDir, "buildpack.id", "cache-false.sha")) + h.AssertEq(t, string(got), cacheFalseLayerSHA) + }) + it("does not restore data", func() { + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-false")) + }) }) - it("doesn't restore cache false layers", func() { - h.AssertNil(t, restorer.Restore(testCache)) - if _, err := os.Stat(filepath.Join(layersDir, "buildpack.id", "cache-false.toml")); !os.IsNotExist(err) { - t.Fatal("Error: cache-false.toml should not have been restored") - } + when("there is a cache=true layer with wrong sha", func() { + it.Before(func() { + meta := "cache=true" + h.AssertNil(t, writeLayer(layersDir, "buildpack.id", "cache-true", meta, "some-made-up-sha")) + h.AssertNil(t, restorer.Restore(testCache)) + }) - if _, err := os.Stat(filepath.Join(layersDir, "buildpack.id", "cache-false")); !os.IsNotExist(err) { - t.Fatal("Error: cache-false layer dir should not have been restored") - } + it("removes metadata and sha file", func() { + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-true.toml")) + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-true.sha")) + }) + it("does not restore layer data", func() { + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-true")) + }) }) - it("doesn't restore layers from buildpacks that aren't in the group", func() { - h.AssertNil(t, restorer.Restore(testCache)) - if _, err := os.Stat(filepath.Join(layersDir, "nogroup.buildpack.id")); !os.IsNotExist(err) { - t.Fatal("Error: expected nogroup.buildpack.id not to be restored") - } + when("there is a cache=true layer not in cache", func() { + it.Before(func() { + meta := "cache=true" + h.AssertNil(t, writeLayer(layersDir, "buildpack.id", "cache-layer-not-in-cache", meta, "some-made-up-sha")) + h.AssertNil(t, restorer.Restore(testCache)) + }) + + it("removes metadata and sha file", func() { + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-layer-not-in-cache.toml")) + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-layer-not-in-cache.sha")) + }) + it("does not restore layer data", func() { + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "buildpack.id", "cache-layer-not-in-cache")) + }) }) - it("escapes buildpack IDs when restoring buildpack layers", func() { - h.AssertNil(t, restorer.Restore(testCache)) - expectedMetadata := `[metadata] + when("there is a cache=true escaped layer", func() { + var meta string + it.Before(func() { + meta = `cache=true +[metadata] escaped-bp-key = "escaped-bp-val"` - if txt, err := ioutil.ReadFile(filepath.Join(layersDir, "escaped_buildpack_id", "escaped-bp-layer.toml")); err != nil { - t.Fatalf("failed to read escaped-bp-layer.toml: %s", err) - } else if !strings.Contains(string(txt), expectedMetadata) { - t.Fatalf(`Error: expected '%s' to contain '%s'`, txt, expectedMetadata) - } - - expectedText := "echo text from escaped bp layer" - if txt, err := ioutil.ReadFile(filepath.Join(layersDir, "escaped_buildpack_id", "escaped-bp-layer", "file-from-escaped-bp")); err != nil { - t.Fatalf("failed to read file-from-escaped-bp: %s", err) - } else if !strings.Contains(string(txt), expectedText) { - t.Fatalf(`Error: expected '%s' to contain '%s'`, txt, expectedText) - } + h.AssertNil(t, writeLayer(layersDir, "escaped_buildpack_id", "escaped-bp-layer", meta, escapedLayerSHA)) + h.AssertNil(t, restorer.Restore(testCache)) + }) + + it("keeps layer metadatata", func() { + got := h.MustReadFile(t, filepath.Join(layersDir, "escaped_buildpack_id", "escaped-bp-layer.toml")) + h.AssertEq(t, string(got), meta) + }) + it("keeps layer sha", func() { + got := h.MustReadFile(t, filepath.Join(layersDir, "escaped_buildpack_id", "escaped-bp-layer.sha")) + h.AssertEq(t, string(got), escapedLayerSHA) + }) + it("restores data", func() { + got := h.MustReadFile(t, filepath.Join(layersDir, "escaped_buildpack_id", "escaped-bp-layer", "file-from-escaped-bp")) + want := "echo text from escaped bp layer\n" + h.AssertEq(t, string(got), want) + }) + }) + + when("there is a cache=true layer in cache but not in group", func() { + it.Before(func() { + meta := "cache=true" + h.AssertNil(t, writeLayer(layersDir, "nogroup.buildpack.id", "some-layer", meta, noGroupLayerSHA)) + h.AssertNil(t, restorer.Restore(testCache)) + }) + it("does not restore layer data", func() { + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "nogroup.buildpack.id", "some-layer")) + }) + + when("the buildpack is detected", func() { + it.Before(func() { + restorer.Buildpacks = []lifecycle.Buildpack{{ID: "nogroup.buildpack.id"}} + h.AssertNil(t, restorer.Restore(testCache)) + }) + + it("keeps metadata and sha file", func() { + h.AssertPathExists(t, filepath.Join(layersDir, "nogroup.buildpack.id", "some-layer.toml")) + h.AssertPathExists(t, filepath.Join(layersDir, "nogroup.buildpack.id", "some-layer.sha")) + }) + it("restores data", func() { + got := h.MustReadFile(t, filepath.Join(layersDir, "nogroup.buildpack.id", "some-layer", "file-from-some-layer")) + want := "echo text from some layer\n" + h.AssertEq(t, string(got), want) + }) + }) + }) + + when("there are multiple cache=true layers", func() { + it.Before(func() { + meta := "cache=true" + h.AssertNil(t, writeLayer(layersDir, "buildpack.id", "cache-only", meta, cacheOnlyLayerSHA)) + meta = "cache=true\nlaunch=true" + h.AssertNil(t, writeLayer(layersDir, "buildpack.id", "cache-launch", meta, cacheLaunchLayerSHA)) + meta = "cache=true" + h.AssertNil(t, writeLayer(layersDir, "escaped_buildpack_id", "escaped-bp-layer", meta, escapedLayerSHA)) + + h.AssertNil(t, restorer.Restore(testCache)) + }) + + it("keeps layer metadatata for all layers", func() { + got := h.MustReadFile(t, filepath.Join(layersDir, "buildpack.id", "cache-only.toml")) + h.AssertEq(t, string(got), "cache=true") + got = h.MustReadFile(t, filepath.Join(layersDir, "buildpack.id", "cache-launch.toml")) + h.AssertEq(t, string(got), "cache=true\nlaunch=true") + got = h.MustReadFile(t, filepath.Join(layersDir, "escaped_buildpack_id", "escaped-bp-layer.toml")) + h.AssertEq(t, string(got), "cache=true") + }) + it("keeps layer sha for all layers", func() { + got := h.MustReadFile(t, filepath.Join(layersDir, "buildpack.id", "cache-only.sha")) + h.AssertEq(t, string(got), cacheOnlyLayerSHA) + got = h.MustReadFile(t, filepath.Join(layersDir, "buildpack.id", "cache-launch.sha")) + h.AssertEq(t, string(got), cacheLaunchLayerSHA) + got = h.MustReadFile(t, filepath.Join(layersDir, "escaped_buildpack_id", "escaped-bp-layer.sha")) + h.AssertEq(t, string(got), escapedLayerSHA) + }) + it("restores data for all layers", func() { + got := h.MustReadFile(t, filepath.Join(layersDir, "buildpack.id", "cache-only", "file-from-cache-only-layer")) + want := "echo text from cache-only layer\n" + h.AssertEq(t, string(got), want) + got = h.MustReadFile(t, filepath.Join(layersDir, "buildpack.id", "cache-launch", "file-from-cache-launch-layer")) + want = "echo text from cache launch layer\n" + h.AssertEq(t, string(got), want) + got = h.MustReadFile(t, filepath.Join(layersDir, "escaped_buildpack_id", "escaped-bp-layer", "file-from-escaped-bp")) + want = "echo text from escaped bp layer\n" + h.AssertEq(t, string(got), want) + }) }) when("restorer is running as root", func() { it.Before(func() { if os.Getuid() != 0 { - t.Skip() + t.Skip("Skipped when not running as root") } + meta := "cache=true\nlaunch=false" + h.AssertNil(t, writeLayer(layersDir, "buildpack.id", "cache-launch", meta, cacheLaunchLayerSHA)) }) it("recursively chowns the layers dir to CNB_USER_ID:CNB_GROUP_ID", func() { @@ -294,3 +448,39 @@ func addLayerFromPath(t *testing.T, tarTempDir, layerPath string, c lifecycle.Ca h.AssertNil(t, c.AddLayerFile(sha, tarPath)) return sha } + +func writeLayer(layersDir, buildpack, name, metadata, sha string) error { + buildpackDir := filepath.Join(layersDir, buildpack) + if err := os.MkdirAll(buildpackDir, 0755); err != nil { + return errors.Wrapf(err, "creating buildpack layer directory") + } + metadataPath := filepath.Join(buildpackDir, name+".toml") + if err := ioutil.WriteFile(metadataPath, []byte(metadata), 0755); err != nil { + return errors.Wrapf(err, "writing metadata file") + } + shaPath := filepath.Join(buildpackDir, name+".sha") + if err := ioutil.WriteFile(shaPath, []byte(sha), 0755); err != nil { + return errors.Wrapf(err, "writing sha file") + } + return nil +} + +func TestWriteLayer(t *testing.T) { + layersDir, err := ioutil.TempDir("", "test-write-layer") + if err != nil { + t.Fatalf("Failed to create temporary directory: %v", err) + } + defer os.RemoveAll(layersDir) + + writeLayer(layersDir, "test-buildpack", "test-layer", "test-metadata", "test-sha") + + got := h.MustReadFile(t, filepath.Join(layersDir, "test-buildpack", "test-layer.toml")) + want := "test-metadata" + h.AssertEq(t, string(got), want) + + got = h.MustReadFile(t, filepath.Join(layersDir, "test-buildpack", "test-layer.sha")) + want = "test-sha" + h.AssertEq(t, string(got), want) + + h.AssertPathDoesNotExist(t, filepath.Join(layersDir, "test-buildpack", "test-layer")) +} diff --git a/testdata/analyzer/app_metadata.json b/testdata/analyzer/app_metadata.json new file mode 100644 index 000000000..64aa830b0 --- /dev/null +++ b/testdata/analyzer/app_metadata.json @@ -0,0 +1,59 @@ +{ + "buildpacks": [ + { + "key": "metadata.buildpack", + "layers": { + "launch": { + "data": { + "launch-key": "launch-value" + }, + "launch": true, + "sha": "launch-sha" + }, + "launch-build": { + "build": true, + "data": { + "launch-build-key": "launch-build-value" + }, + "launch": true, + "sha": "launch-build-sha" + }, + "launch-build-cache": { + "build": true, + "cache": true, + "data": { + "launch-build-cache-key": "launch-build-cache-value" + }, + "launch": true, + "sha": "launch-build-cache-sha" + }, + "launch-cache": { + "cache": true, + "data": { + "launch-cache-key": "launch-cache-value" + }, + "launch": true, + "sha": "launch-cache-sha" + }, + "launch-false": { + "data": { + "launch-false-key": "launch-false-value" + }, + "sha": "launch-false-sha" + } + } + }, + { + "key": "no.cache.buildpack", + "layers": { + "some-layer": { + "data": { + "some-layer-key": "some-layer-value" + }, + "launch": true, + "sha": "some-layer-sha" + } + } + } + ] +} diff --git a/testdata/analyzer/cache_metadata.json b/testdata/analyzer/cache_metadata.json new file mode 100644 index 000000000..cc982aa90 --- /dev/null +++ b/testdata/analyzer/cache_metadata.json @@ -0,0 +1,76 @@ +{ + "buildpacks": [ + { + "key": "metadata.buildpack", + "layers": { + "cache": { + "cache": true, + "data": { + "cache-key": "cache-value" + }, + "sha": "cache-sha" + }, + "cache-false": { + "cache": false, + "data": { + "cache-false-key": "cache-false-value" + }, + "sha": "cache-false-sha" + }, + "launch-build-cache": { + "build": true, + "cache": true, + "data": { + "launch-build-cache-key": "cache-specific-value", + "cache-only-key": "cache-only-value" + }, + "launch": true, + "sha": "launch-build-cache-old-sha" + }, + "launch-cache": { + "cache": true, + "data": { + "launch-cache-key": "cache-specific-value", + "cache-only-key": "cache-only-value" + }, + "launch": true, + "sha": "launch-cache-old-sha" + }, + "launch-cache-not-in-app": { + "cache": true, + "data": { + "launch-cache-key": "cache-specific-value", + "cache-only-key": "cache-only-value" + }, + "launch": true, + "sha": "launch-cache-not-in-app-sha" + } + + } + }, + { + "key": "no.group.buildpack", + "layers": { + "some-layer": { + "cache": true, + "data": { + "some-layer-key": "some-layer-value" + }, + "sha": "some-layer-sha" + } + } + }, + { + "key": "escaped/buildpack/id", + "layers": { + "escaped-bp-layer": { + "cache": true, + "data": { + "escaped-bp-layer-key": "escaped-bp-layer-value" + }, + "sha": "escaped-bp-layer-sha" + } + } + } + ] +} diff --git a/testdata/analyzer/cached-layers/config.toml b/testdata/analyzer/cached-layers/config.toml deleted file mode 100644 index 15c93ad25..000000000 --- a/testdata/analyzer/cached-layers/config.toml +++ /dev/null @@ -1 +0,0 @@ -someNoneLayer = "file" \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/bad-layer.sha b/testdata/analyzer/cached-layers/metadata.buildpack/bad-layer.sha deleted file mode 100644 index e69de29bb..000000000 diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/bad-layer.toml b/testdata/analyzer/cached-layers/metadata.buildpack/bad-layer.toml deleted file mode 100644 index 1f033ad83..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/bad-layer.toml +++ /dev/null @@ -1 +0,0 @@ -{"not":"toml"} \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/bad-layer/bad-layer-contents b/testdata/analyzer/cached-layers/metadata.buildpack/bad-layer/bad-layer-contents deleted file mode 100644 index e69de29bb..000000000 diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/build-layer.toml b/testdata/analyzer/cached-layers/metadata.buildpack/build-layer.toml deleted file mode 100644 index 823633c11..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/build-layer.toml +++ /dev/null @@ -1 +0,0 @@ -launch = false \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/build-layer/build-layer-file b/testdata/analyzer/cached-layers/metadata.buildpack/build-layer/build-layer-file deleted file mode 100644 index 0182ad264..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/build-layer/build-layer-file +++ /dev/null @@ -1 +0,0 @@ -build-layer-file-contents \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/old-layer.sha b/testdata/analyzer/cached-layers/metadata.buildpack/old-layer.sha deleted file mode 100644 index e69de29bb..000000000 diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/old-layer.toml b/testdata/analyzer/cached-layers/metadata.buildpack/old-layer.toml deleted file mode 100644 index 9391ee6e1..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/old-layer.toml +++ /dev/null @@ -1 +0,0 @@ -launch = true \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch-build.sha b/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch-build.sha deleted file mode 100644 index 27465f5b0..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch-build.sha +++ /dev/null @@ -1 +0,0 @@ -old-launch-build-sha \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch-build.toml b/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch-build.toml deleted file mode 100644 index 012a3b78f..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch-build.toml +++ /dev/null @@ -1,2 +0,0 @@ -launch=true -build=true \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch-build/stale-launch-build-file b/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch-build/stale-launch-build-file deleted file mode 100644 index 4ed400df8..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch-build/stale-launch-build-file +++ /dev/null @@ -1 +0,0 @@ -stale-launch-build-file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch.sha b/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch.sha deleted file mode 100644 index 37aaffcc7..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch.sha +++ /dev/null @@ -1 +0,0 @@ -stale-node-modules-sha \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch.toml b/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch.toml deleted file mode 100644 index 618912ab0..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/stale-launch.toml +++ /dev/null @@ -1,4 +0,0 @@ -launch = true - -[metadata] - stalekey = "staleval" \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch-build.sha b/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch-build.sha deleted file mode 100644 index 3db914166..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch-build.sha +++ /dev/null @@ -1 +0,0 @@ -valid-launch-build-sha \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch-build.toml b/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch-build.toml deleted file mode 100644 index 4b97184fc..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch-build.toml +++ /dev/null @@ -1,6 +0,0 @@ -build = true -launch = true - -[metadata] - some-key = "val-from-cache" - some-other-key = "val-from-cache" \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch-build/valid-launch-build-file b/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch-build/valid-launch-build-file deleted file mode 100644 index de892e59d..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch-build/valid-launch-build-file +++ /dev/null @@ -1 +0,0 @@ -valid launch build \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch.sha b/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch.sha deleted file mode 100644 index b4fbee3b6..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch.sha +++ /dev/null @@ -1 +0,0 @@ -valid-launch-layer-sha \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch.toml b/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch.toml deleted file mode 100644 index ae1f7a3de..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch.toml +++ /dev/null @@ -1,5 +0,0 @@ -build = false -launch = true - -[metadata] - somekey = "somevalue" \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch/valid-launch-file b/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch/valid-launch-file deleted file mode 100644 index b29be30f4..000000000 --- a/testdata/analyzer/cached-layers/metadata.buildpack/valid-launch/valid-launch-file +++ /dev/null @@ -1 +0,0 @@ -valid-launch cached file \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/no.metadata.buildpack/buildlayer.toml b/testdata/analyzer/cached-layers/no.metadata.buildpack/buildlayer.toml deleted file mode 100644 index 823633c11..000000000 --- a/testdata/analyzer/cached-layers/no.metadata.buildpack/buildlayer.toml +++ /dev/null @@ -1 +0,0 @@ -launch = false \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/no.metadata.buildpack/buildlayer/buildlayerfile b/testdata/analyzer/cached-layers/no.metadata.buildpack/buildlayer/buildlayerfile deleted file mode 100644 index a73464e93..000000000 --- a/testdata/analyzer/cached-layers/no.metadata.buildpack/buildlayer/buildlayerfile +++ /dev/null @@ -1 +0,0 @@ -buildlayer file contents \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/no.metadata.buildpack/launchlayer.toml b/testdata/analyzer/cached-layers/no.metadata.buildpack/launchlayer.toml deleted file mode 100644 index 9391ee6e1..000000000 --- a/testdata/analyzer/cached-layers/no.metadata.buildpack/launchlayer.toml +++ /dev/null @@ -1 +0,0 @@ -launch = true \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/no.metadata.buildpack/launchlayer/launchlayerfile b/testdata/analyzer/cached-layers/no.metadata.buildpack/launchlayer/launchlayerfile deleted file mode 100644 index c477b000b..000000000 --- a/testdata/analyzer/cached-layers/no.metadata.buildpack/launchlayer/launchlayerfile +++ /dev/null @@ -1 +0,0 @@ -launch-layer-contents \ No newline at end of file diff --git a/testdata/analyzer/cached-layers/some-app-dir/appfile b/testdata/analyzer/cached-layers/some-app-dir/appfile deleted file mode 100644 index e2fe86f03..000000000 --- a/testdata/analyzer/cached-layers/some-app-dir/appfile +++ /dev/null @@ -1 +0,0 @@ -appFile file contents \ No newline at end of file diff --git a/testhelpers/testhelpers.go b/testhelpers/testhelpers.go index 91f231f88..7e1c39b0b 100644 --- a/testhelpers/testhelpers.go +++ b/testhelpers/testhelpers.go @@ -116,6 +116,24 @@ func AssertJSONEq(t *testing.T, expected, actual string) { AssertEq(t, expectedJSONAsInterface, actualJSONAsInterface) } +func AssertPathExists(t *testing.T, path string) { + _, err := os.Stat(path) + if os.IsNotExist(err) { + t.Errorf("Expected %q to exist", path) + } else if err != nil { + t.Fatalf("Error stating %q: %v", path, err) + } +} + +func AssertPathDoesNotExist(t *testing.T, path string) { + _, err := os.Stat(path) + if err == nil { + t.Errorf("Expected %q to not exist", path) + } else if !os.IsNotExist(err) { + t.Fatalf("Error stating %q: %v", path, err) + } +} + func isNil(value interface{}) bool { return value == nil || (reflect.TypeOf(value).Kind() == reflect.Ptr && reflect.ValueOf(value).IsNil()) } @@ -306,3 +324,11 @@ func RandomLayer(t *testing.T, tmpDir string) (path string, sha string, contents return path, "sha256:" + sha, contentsBuf.Bytes() } + +func MustReadFile(t *testing.T, path string) []byte { + data, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Error reading %q: %v", path, err) + } + return data +}