From 5d9a29abc21b2619efcd1a8ad1c523a11439b401 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Thu, 27 Jan 2022 16:53:27 -0500 Subject: [PATCH 1/5] Generate CycloneDX SBOMs using our own JSON generation --- go.mod | 1 + go.sum | 4 +- internal/sbom/cyclonedx.go | 168 +++++++++++++++++++++++++++++++++++++ pkg/build/gobuild.go | 15 ++++ pkg/build/options.go | 9 ++ pkg/commands/resolver.go | 2 + vendor/modules.txt | 3 +- 7 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 internal/sbom/cyclonedx.go diff --git a/go.mod b/go.mod index c0f1a7190d..2185c0ec7d 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/spf13/cobra v1.3.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.10.1 + golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/tools v0.1.9 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b diff --git a/go.sum b/go.sum index 770d551f1b..291e9ac6fc 100644 --- a/go.sum +++ b/go.sum @@ -1987,8 +1987,9 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce h1:Roh6XWxHFKrPgC/EQhVubSAGQ6Ozk6IdxHSzt1mR0EI= +golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2114,6 +2115,7 @@ golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211111160137-58aab5ef257a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127074510-2fabfed7e28f h1:o66Bv9+w/vuk7Krcig9jZqD01FP7BL8OliFqqw0xzPI= golang.org/x/net v0.0.0-20220127074510-2fabfed7e28f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go new file mode 100644 index 0000000000..f70355bb3b --- /dev/null +++ b/internal/sbom/cyclonedx.go @@ -0,0 +1,168 @@ +// Copyright 2022 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sbom + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "strings" +) + +func bomRef(path, version string) string { + return fmt.Sprintf("pkg:golang/%s@%s?type=module", path, version) +} + +func h1ToSHA256(s string) string { + if !strings.HasPrefix(s, "h1:") { + return "" + } + b, err := base64.StdEncoding.DecodeString(s[3:]) + if err != nil { + return "" + } + return hex.EncodeToString(b) +} + +func GenerateCycloneDX(mod []byte) ([]byte, error) { + bi := &BuildInfo{} + if err := bi.UnmarshalText(mod); err != nil { + return nil, err + } + + doc := document{ + BOMFormat: "CycloneDX", + SpecVersion: "1.4", + Version: 1, + Metadata: metadata{ + Component: component{ + BOMRef: bomRef(bi.Main.Path, bi.Main.Version), + Type: "application", + Name: bi.Main.Path, + Version: bi.Main.Version, + Purl: bomRef(bi.Main.Path, bi.Main.Version), + ExternalReferences: []externalReference{{ + URL: bi.Main.Path, + Type: "vcs", + }}, + }, + Properties: []property{{ + Name: "cdx:gomod:binary:hash:SHA-256", + Value: h1ToSHA256(bi.Main.Sum), + }, { + Name: "cdx:gomod:binary:name", + Value: "out", + }, { + Name: "cdx:gomod:build:env:GOVERSION", + Value: bi.GoVersion, + }}, + // TODO: all the other hashes + // TODO: include bi.Settings? + }, + Dependencies: []dependency{{ + Ref: bomRef(bi.Main.Path, bi.Main.Version), + }}, + Compositions: []composition{{ + Aggregate: "complete", + Dependencies: []string{ + bomRef(bi.Main.Path, bi.Main.Version), + }, + }, { + Aggregate: "unknown", + Dependencies: []string{}, + }}, + } + for _, dep := range bi.Deps { + // Don't include replaced deps + if dep.Replace != nil { + continue + } + doc.Components = append(doc.Components, component{ + BOMRef: bomRef(dep.Path, dep.Version), + Type: "library", + Name: dep.Path, + Scope: "required", + Hashes: []hash{{ + Alg: "SHA-256", + Content: h1ToSHA256(dep.Sum), + }}, + Purl: bomRef(dep.Path, dep.Version), + ExternalReferences: []externalReference{{ + URL: dep.Path, + Type: "vcs", + }}, + }) + doc.Dependencies[0].DependsOn = append(doc.Dependencies[0].DependsOn, bomRef(dep.Path, dep.Version)) + doc.Dependencies = append(doc.Dependencies, dependency{ + Ref: bomRef(dep.Path, dep.Version), + }) + + doc.Compositions[1].Dependencies = append(doc.Compositions[1].Dependencies, bomRef(dep.Path, dep.Version)) + } + + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetIndent("", " ") + if err := enc.Encode(doc); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +type document struct { + BOMFormat string `json:"bomFormat"` + SpecVersion string `json:"specVersion"` + Version int `json:"version"` + Metadata metadata `json:"metadata"` + Components []component `json:"component,omitempty"` + Dependencies []dependency `json:"dependency,omitempty"` + Compositions []composition `json:"compositions,omitempty"` +} +type metadata struct { + Component component `json:"component"` + Properties []property `json:"properties,omitempty"` +} +type component struct { + BOMRef string `json:"bom-ref"` + Type string `json:"type"` + Name string `json:"name"` + Version string `json:"version"` + Scope string `json:"scope"` + Hashes []hash `json:"hashes"` + Purl string `json:"purl"` + ExternalReferences []externalReference `json:"externalReferences"` +} +type hash struct { + Alg string `json:"alg"` + Content string `json:"content"` +} +type externalReference struct { + URL string `json:"url"` + Type string `json:"type"` +} +type property struct { + Name string `json:"name"` + Value string `json:"value"` +} +type dependency struct { + Ref string `json:"ref"` + DependsOn []string `json:"dependsOn,omitempty"` +} +type composition struct { + Aggregate string `json:"aggregate"` + Dependencies []string `json:"dependencies,omitempty"` +} diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go index 03a376746c..5db012771e 100644 --- a/pkg/build/gobuild.go +++ b/pkg/build/gobuild.go @@ -325,6 +325,21 @@ func spdx(version string) sbomber { } } +func cycloneDX() sbomber { + return func(ctx context.Context, file string, appPath string, img v1.Image) ([]byte, types.MediaType, error) { + b, _, err := goversionm(ctx, file, appPath, img) + if err != nil { + return nil, "", err + } + + b, err = sbom.GenerateCycloneDX(b) + if err != nil { + return nil, "", err + } + return b, ctypes.CycloneDXMediaType, nil + } +} + // buildEnv creates the environment variables used by the `go build` command. // From `os/exec.Cmd`: If Env contains duplicate environment keys, only the last // value in the slice for each duplicate key is used. diff --git a/pkg/build/options.go b/pkg/build/options.go index 2712fe2543..e29bca5ba1 100644 --- a/pkg/build/options.go +++ b/pkg/build/options.go @@ -144,6 +144,15 @@ func WithSPDX(version string) Option { } } +// WithCycloneDX is a functional option to direct ko to use CycloneDX for SBOM +// format. +func WithCycloneDX() Option { + return func(gbo *gobuildOpener) error { + gbo.sbom = cycloneDX() + return nil + } +} + // withSBOMber is a functional option for overriding the way SBOMs // are generated. func withSBOMber(sbom sbomber) Option { diff --git a/pkg/commands/resolver.go b/pkg/commands/resolver.go index 8156242cae..e183718581 100644 --- a/pkg/commands/resolver.go +++ b/pkg/commands/resolver.go @@ -104,6 +104,8 @@ func gobuildOptions(bo *options.BuildOptions) ([]build.Option, error) { opts = append(opts, build.WithDisabledSBOM()) case "go.version-m": opts = append(opts, build.WithGoVersionSBOM()) + case "cyclonedx": + opts = append(opts, build.WithCycloneDX()) default: // "spdx" opts = append(opts, build.WithSPDX(version())) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 98e6d1b24d..e7e8f6e25e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -315,7 +315,8 @@ github.com/subosito/gotenv github.com/theupdateframework/go-tuf/encrypted # github.com/vbatts/tar-split v0.11.2 github.com/vbatts/tar-split/archive/tar -# golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 +# golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce +## explicit golang.org/x/crypto/internal/poly1305 golang.org/x/crypto/internal/subtle golang.org/x/crypto/nacl/secretbox From 5a6d45f0290408c66c43759d81b45f2ac099d32a Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Wed, 9 Feb 2022 17:12:36 -0500 Subject: [PATCH 2/5] fix some errors --- internal/sbom/cyclonedx.go | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index f70355bb3b..78bef0aeb3 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -56,21 +56,16 @@ func GenerateCycloneDX(mod []byte) ([]byte, error) { Version: bi.Main.Version, Purl: bomRef(bi.Main.Path, bi.Main.Version), ExternalReferences: []externalReference{{ - URL: bi.Main.Path, + URL: "https://" + bi.Main.Path, Type: "vcs", }}, }, Properties: []property{{ - Name: "cdx:gomod:binary:hash:SHA-256", - Value: h1ToSHA256(bi.Main.Sum), - }, { Name: "cdx:gomod:binary:name", Value: "out", - }, { - Name: "cdx:gomod:build:env:GOVERSION", - Value: bi.GoVersion, }}, - // TODO: all the other hashes + // TODO: include all hashes + // TODO: include go version // TODO: include bi.Settings? }, Dependencies: []dependency{{ @@ -92,17 +87,18 @@ func GenerateCycloneDX(mod []byte) ([]byte, error) { continue } doc.Components = append(doc.Components, component{ - BOMRef: bomRef(dep.Path, dep.Version), - Type: "library", - Name: dep.Path, - Scope: "required", + BOMRef: bomRef(dep.Path, dep.Version), + Type: "library", + Name: dep.Path, + Version: dep.Version, + Scope: "required", Hashes: []hash{{ Alg: "SHA-256", Content: h1ToSHA256(dep.Sum), }}, Purl: bomRef(dep.Path, dep.Version), ExternalReferences: []externalReference{{ - URL: dep.Path, + URL: "https://" + dep.Path, Type: "vcs", }}, }) @@ -128,8 +124,8 @@ type document struct { SpecVersion string `json:"specVersion"` Version int `json:"version"` Metadata metadata `json:"metadata"` - Components []component `json:"component,omitempty"` - Dependencies []dependency `json:"dependency,omitempty"` + Components []component `json:"components,omitempty"` + Dependencies []dependency `json:"dependencies,omitempty"` Compositions []composition `json:"compositions,omitempty"` } type metadata struct { @@ -141,8 +137,8 @@ type component struct { Type string `json:"type"` Name string `json:"name"` Version string `json:"version"` - Scope string `json:"scope"` - Hashes []hash `json:"hashes"` + Scope string `json:"scope,omitempty"` + Hashes []hash `json:"hashes,omitempty"` Purl string `json:"purl"` ExternalReferences []externalReference `json:"externalReferences"` } From 695c7130964341bb6723055f3a4b07cbff75e4b1 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Wed, 9 Feb 2022 18:27:41 -0500 Subject: [PATCH 3/5] Add support to ko deps --- pkg/commands/deps.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/commands/deps.go b/pkg/commands/deps.go index 16ee603159..3ddb1d7dca 100644 --- a/pkg/commands/deps.go +++ b/pkg/commands/deps.go @@ -50,7 +50,7 @@ If the image was not built using ko, or if it was built without embedding depend ctx := cmd.Context() switch sbomType { - case "spdx", "go.version-m": + case "cyclonedx", "spdx", "go.version-m": default: return fmt.Errorf("invalid sbom type %q: must be spdx or go.version-m", sbomType) } @@ -139,6 +139,12 @@ If the image was not built using ko, or if it was built without embedding depend return err } io.Copy(os.Stdout, bytes.NewReader(b)) + case "cyclonedx": + b, err := sbom.GenerateCycloneDX(mod) + if err != nil { + return err + } + io.Copy(os.Stdout, bytes.NewReader(b)) case "go.version-m": io.Copy(os.Stdout, bytes.NewReader(mod)) } From 23a499c6871fbc44ffe14f088faff3495f2e17ef Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Wed, 9 Feb 2022 20:45:12 -0500 Subject: [PATCH 4/5] Add e2e SBOM validation --- .github/workflows/sbom.yaml | 98 +++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 .github/workflows/sbom.yaml diff --git a/.github/workflows/sbom.yaml b/.github/workflows/sbom.yaml new file mode 100644 index 0000000000..843f76fea0 --- /dev/null +++ b/.github/workflows/sbom.yaml @@ -0,0 +1,98 @@ +name: Validate SBOMs + +on: + pull_request: + branches: ['main'] + +jobs: + go-version-m: + name: Generate go version -m + runs-on: ubuntu-latest + + env: + KO_DOCKER_REPO: localhost:1338 + + steps: + - uses: actions/setup-go@v2 + with: + go-version: '1.17.x' + - name: Install cmd/registry + run: | + go install github.com/google/go-containerregistry/cmd/registry@latest + registry & + - uses: actions/checkout@v2 + + - name: Generate + run: | + img=$(go run ./ build ./) + go run ./ deps $img --sbom=go.version-m > gomod.txt + cat gomod.txt + + cyclonedx: + name: Validate CycloneDX SBOM + runs-on: ubuntu-latest + + env: + KO_DOCKER_REPO: localhost:1338 + + steps: + - uses: actions/setup-go@v2 + with: + go-version: '1.17.x' + - name: Install cmd/registry + run: | + go install github.com/google/go-containerregistry/cmd/registry@latest + registry & + - uses: actions/checkout@v2 + + - name: Install CycloneDX + run: | + wget https://github.com/CycloneDX/cyclonedx-cli/releases/download/v0.22.0/cyclonedx-linux-x64 + chmod +x cyclonedx-linux-x64 + + - name: Generate and Validate + run: | + img=$(go run ./ build ./) + go run ./ deps $img --sbom=cyclonedx > sbom.json + ./cyclonedx-linux-x64 validate --input-file=sbom.json --fail-on-errors + + - uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: sbom.json + path: sbom.json + + spdx: + name: Validate SPDX SBOM + runs-on: ubuntu-latest + + env: + KO_DOCKER_REPO: localhost:1338 + + steps: + - uses: actions/setup-go@v2 + with: + go-version: '1.17.x' + - name: Install cmd/registry + run: | + go install github.com/google/go-containerregistry/cmd/registry@latest + registry & + - uses: actions/checkout@v2 + + - name: Install SPDX Tools + run: | + wget https://github.com/spdx/tools/releases/download/v2.2.7/spdx-tools-2.2.7.zip + unzip spdx-tools-2.2.7.zip + + - name: Generate and Validate + run: | + img=$(go run ./ build ./) + go run ./ deps $img --sbom=spdx > sbom.txt + + java -jar ./spdx-tools-2.2.7-jar-with-dependencies.jar Verify sbom.txt + + - uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: sbom.txt + path: sbom.txt From a619255d81058a8b37e0a518dc5697948ab1de9f Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Thu, 10 Feb 2022 08:19:06 -0500 Subject: [PATCH 5/5] ignore empty hashes (why are hashes empty?) --- internal/sbom/cyclonedx.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 78bef0aeb3..8704f0bf41 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -86,22 +86,25 @@ func GenerateCycloneDX(mod []byte) ([]byte, error) { if dep.Replace != nil { continue } - doc.Components = append(doc.Components, component{ + comp := component{ BOMRef: bomRef(dep.Path, dep.Version), Type: "library", Name: dep.Path, Version: dep.Version, Scope: "required", - Hashes: []hash{{ - Alg: "SHA-256", - Content: h1ToSHA256(dep.Sum), - }}, - Purl: bomRef(dep.Path, dep.Version), + Purl: bomRef(dep.Path, dep.Version), ExternalReferences: []externalReference{{ URL: "https://" + dep.Path, Type: "vcs", }}, - }) + } + if dep.Sum != "" { + comp.Hashes = []hash{{ + Alg: "SHA-256", + Content: h1ToSHA256(dep.Sum), + }} + } + doc.Components = append(doc.Components, comp) doc.Dependencies[0].DependsOn = append(doc.Dependencies[0].DependsOn, bomRef(dep.Path, dep.Version)) doc.Dependencies = append(doc.Dependencies, dependency{ Ref: bomRef(dep.Path, dep.Version),