feat: add option for including full license content in the SBOM

cmd/syft/internal/options/catalog.go

@@ -158,8 +158,9 @@ func (cfg Catalog) ToFilesConfig() filecataloging.Config {
 
 func (cfg Catalog) ToLicenseConfig() cataloging.LicenseConfig {
 	return cataloging.LicenseConfig{
-		IncludeUnkownLicenseContent: cfg.License.IncludeUnknownLicenseContent,
-		Coverage:                    cfg.License.LicenseCoverage,
+		IncludeFullText:              cfg.License.IncludeFullText,
+		IncludeUnknownLicenseContent: cfg.License.IncludeUnknownLicenseContent,
+		Coverage:                     cfg.License.LicenseCoverage,
 	}
 }
 

cmd/syft/internal/options/license.go

@@ -5,6 +5,7 @@ import (
 )
 
 type licenseConfig struct {
+	IncludeFullText              bool    `yaml:"include-full-text" json:"include-full-text" mapstructure:"include-full-text"`
 	IncludeUnknownLicenseContent bool    `yaml:"include-unknown-license-content" json:"include-unknown-license-content" mapstructure:"include-unknown-license-content"`
 	LicenseCoverage              float64 `yaml:"license-coverage" json:"license-coverage" mapstructure:"license-coverage"`
 }
@@ -14,6 +15,7 @@ var _ interface {
 } = (*licenseConfig)(nil)
 
 func (o *licenseConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
+	descriptions.Add(&o.IncludeFullText, `include the content of a license in the SBOM for all cases where a cataloger has license content access`)
 	descriptions.Add(&o.IncludeUnknownLicenseContent, `include the content of a license in the SBOM when syft
 cannot determine a valid SPDX ID for the given license`)
 	descriptions.Add(&o.LicenseCoverage, `adjust the percent as a fraction of the total text, in normalized words, that
@@ -22,6 +24,7 @@ matches any valid license for the given inputs, expressed as a percentage across
 
 func defaultLicenseConfig() licenseConfig {
 	return licenseConfig{
+		IncludeFullText:              false,
 		IncludeUnknownLicenseContent: false,
 		LicenseCoverage:              75,
 	}

internal/licenses/context_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestSetContextLicenseScanner(t *testing.T) {
-	scanner := testScanner(true)
+	scanner := testScanner(true, false)
 	ctx := context.Background()
 	ctx = SetContextLicenseScanner(ctx, scanner)
 
@@ -20,7 +20,7 @@ func TestSetContextLicenseScanner(t *testing.T) {
 }
 
 func TestIsContextLicenseScannerSet(t *testing.T) {
-	scanner := testScanner(true)
+	scanner := testScanner(true, false)
 	ctx := context.Background()
 	require.False(t, IsContextLicenseScannerSet(ctx))
 
@@ -30,7 +30,7 @@ func TestIsContextLicenseScannerSet(t *testing.T) {
 
 func TestContextLicenseScanner(t *testing.T) {
 	t.Run("with scanner", func(t *testing.T) {
-		scanner := testScanner(true)
+		scanner := testScanner(true, false)
 		ctx := SetContextLicenseScanner(context.Background(), scanner)
 		s, err := ContextLicenseScanner(ctx)
 		if err != nil || s != scanner {

internal/licenses/scanner.go

@@ -13,28 +13,40 @@ import (
 )
 
 const (
-	DefaultCoverageThreshold     = 75 // determined by experimentation
-	DefaultIncludeLicenseContent = false
+	DefaultCoverageThreshold            = 75 // determined by experimentation
+	DefaultIncludeUnknownLicenseContent = false
+	DefaultIncludeFullText              = false
 )
 
+type ID struct {
+	LicenseID string
+	Offset    Offset
+}
+
+type Offset struct {
+	Start, End int
+}
+
 type Scanner interface {
-	IdentifyLicenseIDs(context.Context, io.Reader) ([]string, []byte, error)
+	IdentifyLicenseIDs(context.Context, io.Reader) ([]ID, []byte, error)
 	FileSearch(context.Context, file.LocationReadCloser) ([]file.License, error)
 	PkgSearch(context.Context, file.LocationReadCloser) ([]pkg.License, error)
 }
 
 var _ Scanner = (*scanner)(nil)
 
 type scanner struct {
-	coverageThreshold     float64 // between 0 and 100
-	includeLicenseContent bool
-	scanner               func([]byte) licensecheck.Coverage
+	coverageThreshold            float64 // between 0 and 100
+	includeUnknownLicenseContent bool
+	includeFullText              bool
+	scanner                      func([]byte) licensecheck.Coverage
 }
 
 type ScannerConfig struct {
-	CoverageThreshold     float64
-	IncludeLicenseContent bool
-	Scanner               func([]byte) licensecheck.Coverage
+	CoverageThreshold            float64
+	IncludeUnknownLicenseContent bool
+	IncludeFullText              bool
+	Scanner                      func([]byte) licensecheck.Coverage
 }
 
 type Option func(*scanner)
@@ -45,9 +57,15 @@ func WithCoverage(coverage float64) Option {
 	}
 }
 
-func WithIncludeLicenseContent(includeLicenseContent bool) Option {
+func WithIncludeUnknownLicenseContent(includeUnknownLicenseContent bool) Option {
+	return func(s *scanner) {
+		s.includeUnknownLicenseContent = includeUnknownLicenseContent
+	}
+}
+
+func WithIncludeFullText(includeFullText bool) Option {
 	return func(s *scanner) {
-		s.includeLicenseContent = includeLicenseContent
+		s.includeFullText = includeFullText
 	}
 }
 
@@ -59,9 +77,10 @@ func NewDefaultScanner(o ...Option) (Scanner, error) {
 		return nil, fmt.Errorf("unable to create default license scanner: %w", err)
 	}
 	newScanner := &scanner{
-		coverageThreshold:     DefaultCoverageThreshold,
-		includeLicenseContent: DefaultIncludeLicenseContent,
-		scanner:               s.Scan,
+		coverageThreshold:            DefaultCoverageThreshold,
+		includeUnknownLicenseContent: DefaultIncludeUnknownLicenseContent,
+		includeFullText:              DefaultIncludeFullText,
+		scanner:                      s.Scan,
 	}
 
 	for _, opt := range o {
@@ -78,8 +97,9 @@ func NewScanner(c *ScannerConfig) (Scanner, error) {
 	}
 
 	return &scanner{
-		coverageThreshold:     c.CoverageThreshold,
-		includeLicenseContent: c.IncludeLicenseContent,
-		scanner:               c.Scanner,
+		coverageThreshold:            c.CoverageThreshold,
+		includeFullText:              c.IncludeFullText,
+		includeUnknownLicenseContent: c.IncludeUnknownLicenseContent,
+		scanner:                      c.Scanner,
 	}, nil
 }

internal/licenses/scanner_test.go

@@ -13,64 +13,89 @@ import (
 func TestIdentifyLicenseIDs(t *testing.T) {
 	type expectation struct {
 		yieldError bool
-		ids        []string
+		ids        []ID
 		content    []byte
 	}
 	tests := []struct {
-		name     string
-		in       string
-		expected expectation
+		name                         string
+		in                           string
+		includeUnknownLicenseContent bool
+		includeFullText              bool
+		expected                     expectation
 	}{
 		{
-			name: "apache license 2.0",
-			in:   `test-fixtures/apache-license-2.0`,
+			name:            "apache license 2.0 with content offset and correct content",
+			in:              `test-fixtures/apache-license-2.0`,
+			includeFullText: true,
 			expected: expectation{
 				yieldError: false,
-				ids:        []string{"Apache-2.0"},
-				content:    nil,
+				ids:        []ID{{LicenseID: "Apache-2.0", Offset: Offset{Start: 0, End: 11324}}},
+				content:    mustOpen("test-fixtures/apache-license-2.0"),
 			},
 		},
 		{
-			name: "custom license includes content for IdentifyLicenseIDs",
-			in:   "test-fixtures/nvidia-software-and-cuda-supplement",
+			name:                         "custom license returns content for IdentifyLicenseIDs",
+			in:                           "test-fixtures/nvidia-software-and-cuda-supplement",
+			includeUnknownLicenseContent: true,
 			expected: expectation{
 				yieldError: false,
-				ids:        []string{},
+				ids:        []ID{},
 				content:    mustOpen("test-fixtures/nvidia-software-and-cuda-supplement"),
 			},
 		},
+		{
+			name:            "Identify multiple license IDs. They should be deduplicated and contain content evidence.",
+			in:              `test-fixtures/multi-license`,
+			includeFullText: true,
+			expected: expectation{
+				yieldError: false,
+				ids: []ID{
+					{LicenseID: "MIT", Offset: Offset{Start: 758, End: 1844}},
+					{LicenseID: "NCSA", Offset: Offset{Start: 1925, End: 3463}},
+					{LicenseID: "MIT", Offset: Offset{Start: 3708, End: 4932}},
+					{LicenseID: "Apache-2.0", Offset: Offset{Start: 5021, End: 16378}},
+					{LicenseID: "Zlib", Offset: Offset{Start: 16484, End: 17390}},
+					{LicenseID: "Unlicense", Offset: Offset{Start: 17497, End: 18707}},
+					{LicenseID: "BSD-2-Clause", Offset: Offset{Start: 18908, End: 20298}},
+					{LicenseID: "BSD-3-Clause", Offset: Offset{Start: 20440, End: 21952}},
+					{LicenseID: "BSD-2-Clause", Offset: Offset{Start: 22033, End: 23335}},
+				},
+				content: mustOpen("test-fixtures/multi-license"),
+			},
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			content, err := os.ReadFile(test.in)
 			require.NoError(t, err)
-			ids, content, err := testScanner(false).IdentifyLicenseIDs(context.TODO(), bytes.NewReader(content))
+			ids, content, err := testScanner(test.includeUnknownLicenseContent, test.includeFullText).IdentifyLicenseIDs(context.TODO(), bytes.NewReader(content))
 			if test.expected.yieldError {
 				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
+				return
+			}
+			require.NoError(t, err)
 
-				require.Len(t, ids, len(test.expected.ids))
-				require.Len(t, content, len(test.expected.content))
+			require.Len(t, ids, len(test.expected.ids))
+			require.Len(t, content, len(test.expected.content))
 
-				if len(test.expected.ids) > 0 {
-					require.Equal(t, ids, test.expected.ids)
-				}
+			if len(test.expected.ids) > 0 {
+				require.Equal(t, ids, test.expected.ids)
+			}
 
-				if len(test.expected.content) > 0 {
-					require.Equal(t, content, test.expected.content)
-				}
+			if len(test.expected.content) > 0 {
+				require.Equal(t, content, test.expected.content)
 			}
 		})
 	}
 }
 
-func testScanner(includeLicenseContent bool) Scanner {
+func testScanner(includeUnknownLicenseContent, includeFullText bool) Scanner {
 	return &scanner{
-		coverageThreshold:     DefaultCoverageThreshold,
-		includeLicenseContent: includeLicenseContent,
-		scanner:               licensecheck.Scan,
+		coverageThreshold:            DefaultCoverageThreshold,
+		includeUnknownLicenseContent: includeUnknownLicenseContent,
+		includeFullText:              includeFullText,
+		scanner:                      licensecheck.Scan,
 	}
 }