Drop whiteout/opaque files and warn when pkg lacks Name/Version

internal/formats/github/encoder.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/anchore/packageurl-go"
 	"github.com/anchore/syft/internal"
-	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/sbom"
 	"github.com/anchore/syft/syft/source"
@@ -148,7 +147,6 @@ func toGithubManifests(s *sbom.SBOM) Manifests {
 func dependencyName(p pkg.Package) string {
 	purl, err := packageurl.FromString(p.PURL)
 	if err != nil {
-		log.Warnf("Invalid PURL for package: '%s' PURL: '%s' (%w)", p.Name, p.PURL, err)
 		return ""
 	}
 	// don't use qualifiers for this

syft/pkg/cataloger/python/package_cataloger.go

@@ -8,7 +8,6 @@ import (
 	"path/filepath"
 
 	"github.com/anchore/syft/internal"
-
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/pkg"
 

syft/pkg/package.go

@@ -33,6 +33,14 @@ func (p *Package) OverrideID(id artifact.ID) {
 }
 
 func (p *Package) SetID() {
+	if p.Name == "" {
+		log.Warnf("%s: missing package name, that is necessary for further metadata extraction. Please take a look at %s", p.FoundBy, p.Locations.ToSlice())
+	}
+
+	if p.Version == "" {
+		log.Warnf("%s: missing package version, that is necessary for further metadata extraction. Please take a look at %s", p.FoundBy, p.Locations.ToSlice())
+	}
+
 	id, err := artifact.IDByHash(p)
 	if err != nil {
 		// TODO: what to do in this case?

syft/pkg/python_package_metadata_test.go

@@ -1,11 +1,12 @@
 package pkg
 
 import (
-	"github.com/anchore/syft/syft/linux"
-	"github.com/sergi/go-diff/diffmatchpatch"
 	"strings"
 	"testing"
 
+	"github.com/anchore/syft/syft/linux"
+	"github.com/sergi/go-diff/diffmatchpatch"
+
 	"github.com/go-test/deep"
 )
 

syft/source/directory_resolver.go

@@ -80,7 +80,7 @@ func newDirectoryResolver(root string, pathFilters ...pathFilterFn) (*directoryR
 		currentWdRelativeToRoot: currentWdRelRoot,
 		fileTree:                filetree.NewFileTree(),
 		metadata:                make(map[file.ID]FileMetadata),
-		pathFilterFns:           append([]pathFilterFn{isUnallowableFileType, isUnixSystemRuntimePath}, pathFilters...),
+		pathFilterFns:           append([]pathFilterFn{isUnallowableFileType, isUnixSystemRuntimePath, isWhiteout, isOpaque}, pathFilters...),
 		refsByMIMEType:          make(map[string][]file.Reference),
 		errPaths:                make(map[string]error),
 	}
@@ -511,6 +511,14 @@ func isUnixSystemRuntimePath(path string, _ os.FileInfo) bool {
 	return internal.HasAnyOfPrefixes(path, unixSystemRuntimePrefixes...)
 }
 
+func isWhiteout(path string, _ os.FileInfo) bool {
+	return file.Path(path).IsWhiteout()
+}
+
+func isOpaque(path string, _ os.FileInfo) bool {
+	return file.Path(path).IsDirWhiteout()
+}
+
 func isUnallowableFileType(_ string, info os.FileInfo) bool {
 	if info == nil {
 		// we can't filter out by filetype for non-existent files

syft/source/directory_resolver_test.go

@@ -24,6 +24,26 @@ import (
 	"github.com/wagoodman/go-progress"
 )
 
+func TestDirectoryResolver_excludeWhiteoutAndOpaqueFiles(t *testing.T) {
+	dir := "./test-fixtures/image-whiteout-opaque/"
+	var allFiles []string
+
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		assert.NoError(t, err)
+		if !info.IsDir() {
+			allFiles = append(allFiles, path)
+		}
+		return nil
+	})
+	assert.NoError(t, err)
+
+	resolver, err := newDirectoryResolver(dir)
+	assert.NoError(t, err)
+
+	assert.Len(t, allFiles, 3)
+	assert.Len(t, resolver.fileTree.AllFiles(), 1)
+}
+
 func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) {
 	cases := []struct {
 		name         string
@@ -886,3 +906,63 @@ func TestDirectoryResolver_indexPath(t *testing.T) {
 		})
 	})
 }
+
+func Test_isWhiteout(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  bool
+	}{
+		{
+			name:  "in-folder",
+			input: "/usr/local/.wh.file-2.txt",
+			want:  true,
+		},
+		{
+			name:  "just-file-name",
+			input: ".wh.file-2.txt",
+			want:  true,
+		},
+		{
+			name:  "not whiteout",
+			input: "file-1.txt",
+			want:  false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var fi os.FileInfo
+			assert.Equalf(t, tt.want, isWhiteout(tt.input, fi), "isWhiteout(%v, %v)", tt.input, fi)
+		})
+	}
+}
+
+func Test_isOpaque(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  bool
+	}{
+		{
+			name:  "in-path",
+			input: "bin/.wh..wh..opq",
+			want:  true,
+		},
+		{
+			name:  "just-file-name",
+			input: ".wh..wh..opq",
+			want:  true,
+		},
+		{
+			name:  "not-opaque",
+			input: "file-1.txt",
+			want:  false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var fi os.FileInfo
+			assert.Equalf(t, tt.want, isOpaque(tt.input, fi), "isWhiteout(%v, %v)", tt.input, fi)
+		})
+	}
+}

syft/source/test-fixtures/image-whiteout-opaque/file-1.txt

@@ -0,0 +1 @@
+content

test/cli/package_missing_data_test.go

@@ -0,0 +1,36 @@
+package cli
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/anchore/syft/syft"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPackageMissingNameAndVersion(t *testing.T) {
+	formats := syft.FormatIDs()
+	commonAssertions := []traitAssertion{
+		func(tb testing.TB, _, stderr string, _ int) {
+			tb.Helper()
+			assert.Contains(tb, stderr, "python-package-cataloger: missing package name, that is necessary for further metadata extraction")
+			assert.Contains(tb, stderr, "python-package-cataloger: missing package version, that is necessary for further metadata extraction")
+		},
+		assertSuccessfulReturnCode,
+	}
+
+	for _, o := range formats {
+		t.Run(fmt.Sprintf("format:%s", o), func(t *testing.T) {
+			cmd, stdout, stderr := runSyft(t, nil, "dir:./test-fixtures/image-empty-files/", "-o", string(o))
+			for _, traitFn := range commonAssertions {
+				traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode())
+			}
+			if t.Failed() {
+				t.Log("STDOUT:\n", stdout)
+				t.Log("STDERR:\n", stderr)
+				t.Log("COMMAND:", strings.Join(cmd.Args, " "))
+			}
+		})
+	}
+}

test/cli/test-fixtures/image-empty-files/Dockerfile

@@ -0,0 +1,2 @@
+FROM scratch
+COPY pkgs/ .

test/cli/test-fixtures/image-empty-files/pkgs/broken-1.0.0-py3.8.egg-info

@@ -0,0 +1 @@
+broken

test/integration/catalog_packages_test.go

@@ -118,7 +118,7 @@ func TestPkgCoverageImage(t *testing.T) {
 			if pkgCount != len(c.pkgInfo)+c.duplicates {
 				t.Logf("Discovered packages of type %+v", c.pkgType)
 				for a := range sbom.Artifacts.PackageCatalog.Enumerate(c.pkgType) {
-					t.Log("   ", a)
+					t.Log("   ", a, a.Locations)
 				}
 				t.Fatalf("unexpected package count: %d!=%d", pkgCount, len(c.pkgInfo))
 			}

test/integration/utils_test.go

@@ -1,9 +1,10 @@
 package integration
 
 import (
-	"github.com/stretchr/testify/require"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/anchore/syft/syft/pkg/cataloger"
 
 	"github.com/anchore/syft/syft/sbom"