From d290e5d356582ab56e48c8b4c180a4a10f776659 Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Mon, 14 Aug 2023 14:01:53 +0300 Subject: [PATCH 1/3] ci/cd: workflow rework * introduce trivy scanning * add new 'release' workflow that publishes images * split 'ci' flow into two: 'ci' & 'devel' * 'devel' flow publishes images * workflows share common parts (build, e2e, publish etc.) Signed-off-by: Tuomas Katila --- .github/workflows/ci.yaml | 141 ++------------------ .github/workflows/devel.yaml | 37 +++++ .github/workflows/e2e.yaml | 12 ++ .github/workflows/lib-build.yaml | 51 +++++++ .github/workflows/{e2e.yml => lib-e2e.yaml} | 12 +- .github/workflows/lib-publish.yaml | 61 +++++++++ .github/workflows/lib-trivy.yaml | 132 ++++++++++++++++++ .github/workflows/lib-validate.yaml | 71 ++++++++++ .github/workflows/release.yaml | 39 ++++++ .github/workflows/template/trivy-csv.tpl | 29 ++++ Makefile | 15 +-- 11 files changed, 450 insertions(+), 150 deletions(-) create mode 100644 .github/workflows/devel.yaml create mode 100644 .github/workflows/e2e.yaml create mode 100644 .github/workflows/lib-build.yaml rename .github/workflows/{e2e.yml => lib-e2e.yaml} (93%) create mode 100644 .github/workflows/lib-publish.yaml create mode 100644 .github/workflows/lib-trivy.yaml create mode 100644 .github/workflows/lib-validate.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/template/trivy-csv.tpl diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 405acfec1..e4b452de4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,142 +2,29 @@ name: CI on: push: branches: - - main - 'release-*' pull_request: branches: - main - 'release-*' -env: - K8S_VERSION: 1.27.1 permissions: contents: read + pull-requests: read jobs: + trivy: + uses: "./.github/workflows/lib-trivy.yaml" - docs: - name: Check docs are buildable - runs-on: ubuntu-22.04 - steps: - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y python3-venv - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up doc directory - run: | - mkdir $HOME/output - touch $HOME/output/.nojekyll - - name: Build latest - run: | - GITHUB_SHA=$(git rev-parse HEAD) - export GITHUB_SHA - rm -rf _work/venv - make vhtml - mv _build/html/* $HOME/output/ - - golangci: - permissions: - contents: read # for actions/checkout to fetch code - pull-requests: read # for golangci/golangci-lint-action to fetch pull requests - name: lint - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version-file: go.mod - check-latest: true - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - version: v1.52.1 - args: -v --timeout 5m + validate: + uses: "./.github/workflows/lib-validate.yaml" build: - name: Build and check device plugins - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version-file: go.mod - check-latest: true - - name: Check Dockerfiles - run: make check-dockerfiles - - run: make go-mod-tidy - - run: make BUILDTAGS=kerneldrv - - run: make test BUILDTAGS=kerneldrv - - name: Install envtest tool and run envtest - run: | - go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest - setup-envtest use ${K8S_VERSION} - - run: | - KUBEBUILDER_ASSETS=$(setup-envtest use -i -p path ${K8S_VERSION}) make envtest - - run: make check-github-actions - - name: Codecov report - run: bash <(curl -s https://codecov.io/bash) - - image: - name: Build image - runs-on: ubuntu-22.04 - strategy: - matrix: - image: - - intel-fpga-admissionwebhook - - intel-fpga-initcontainer - - intel-gpu-fakedev - - intel-gpu-initcontainer - - intel-gpu-plugin - - intel-fpga-plugin - - intel-qat-initcontainer - - intel-qat-plugin - - intel-qat-plugin-kerneldrv - - intel-deviceplugin-operator - - intel-sgx-admissionwebhook - - intel-sgx-plugin - - intel-sgx-initcontainer - - intel-dsa-plugin - - intel-iaa-plugin - - intel-idxd-config-initcontainer - - intel-dlb-plugin - - intel-dlb-initcontainer - - intel-xpumanager-sidecar - - # Demo images - - crypto-perf - - accel-config-demo - - intel-opencl-icd - - opae-nlb-demo - - openssl-qat-engine - - sgx-sdk-demo - - sgx-aesmd-demo - - dlb-dpdk-demo - - dlb-libdlb-demo - builder: [buildah, docker] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - - run: make -e vendor - - name: Build image - env: - IMAGE_NAME: ${{ matrix.image }} - BUILDER_NAME: ${{ matrix.builder }} - run: | - make ${IMAGE_NAME} BUILDER=${BUILDER_NAME} - - terrascan: - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Install terrascan - run: | - curl -sL "$(curl -sL https://api.github.com/repos/tenable/terrascan/releases/latest | grep -o -E "https://.+?_Linux_x86_64.tar.gz")" > terrascan.tar.gz - tar -xf terrascan.tar.gz terrascan && rm terrascan.tar.gz - install terrascan /usr/local/bin && rm terrascan - - name: Run Terrascan - run: make terrascan - + needs: + - trivy + - validate + uses: "./.github/workflows/lib-build.yaml" + + e2e: + needs: + - build + uses: "./.github/workflows/lib-e2e.yaml" diff --git a/.github/workflows/devel.yaml b/.github/workflows/devel.yaml new file mode 100644 index 000000000..1e9f319d1 --- /dev/null +++ b/.github/workflows/devel.yaml @@ -0,0 +1,37 @@ +name: Devel +on: + push: + branches: + - main +permissions: + contents: read + pull-requests: read + security-events: write + +jobs: + trivy: + uses: "./.github/workflows/lib-trivy.yaml" + with: + upload-to-github-security-tab: true + + validate: + uses: "./.github/workflows/lib-validate.yaml" + + build: + needs: + - validate + - trivy + uses: "./.github/workflows/lib-build.yaml" + + e2e: + needs: + - build + uses: "./.github/workflows/lib-e2e.yaml" + + # devel image push + publish: + needs: + - e2e + - build + uses: "./.github/workflows/lib-publish.yaml" + secrets: inherit diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 000000000..54b3db241 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,12 @@ +name: e2e +on: + schedule: + - cron: '0 4 * * *' + +permissions: + contents: read + pull-requests: read + +jobs: + e2e: + uses: "./.github/workflows/lib-e2e.yaml" diff --git a/.github/workflows/lib-build.yaml b/.github/workflows/lib-build.yaml new file mode 100644 index 000000000..239a8172a --- /dev/null +++ b/.github/workflows/lib-build.yaml @@ -0,0 +1,51 @@ +name: build +on: + workflow_call: +jobs: + image: + name: Build image + runs-on: ubuntu-22.04 + strategy: + matrix: + image: + - intel-fpga-admissionwebhook + - intel-fpga-initcontainer + - intel-gpu-fakedev + - intel-gpu-initcontainer + - intel-gpu-plugin + - intel-fpga-plugin + - intel-qat-initcontainer + - intel-qat-plugin + - intel-qat-plugin-kerneldrv + - intel-deviceplugin-operator + - intel-sgx-admissionwebhook + - intel-sgx-plugin + - intel-sgx-initcontainer + - intel-dsa-plugin + - intel-iaa-plugin + - intel-idxd-config-initcontainer + - intel-dlb-plugin + - intel-dlb-initcontainer + - intel-xpumanager-sidecar + + # # Demo images + - crypto-perf + - accel-config-demo + - intel-opencl-icd + - opae-nlb-demo + - openssl-qat-engine + - sgx-sdk-demo + - sgx-aesmd-demo + - dlb-dpdk-demo + - dlb-libdlb-demo + builder: [buildah, docker] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + - run: make -e vendor + - name: Build image + env: + IMAGE_NAME: ${{ matrix.image }} + BUILDER_NAME: ${{ matrix.builder }} + run: | + make ${IMAGE_NAME} BUILDER=${BUILDER_NAME} diff --git a/.github/workflows/e2e.yml b/.github/workflows/lib-e2e.yaml similarity index 93% rename from .github/workflows/e2e.yml rename to .github/workflows/lib-e2e.yaml index e21ba810d..e9809a64a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/lib-e2e.yaml @@ -1,16 +1,6 @@ name: e2e on: - workflow_dispatch: - schedule: - - cron: '0 4 * * *' - pull_request: - branches: - - main - - 'release-*' - push: - branches: - - main - - 'release-*' + workflow_call: permissions: contents: read diff --git a/.github/workflows/lib-publish.yaml b/.github/workflows/lib-publish.yaml new file mode 100644 index 000000000..c9c1edb4c --- /dev/null +++ b/.github/workflows/lib-publish.yaml @@ -0,0 +1,61 @@ +name: publish +on: + workflow_call: + inputs: + image_tag: + default: "devel" + required: false + type: string +jobs: + image: + name: Build image + runs-on: ubuntu-22.04 + strategy: + matrix: + image: + - intel-fpga-admissionwebhook + - intel-fpga-initcontainer + - intel-gpu-initcontainer + - intel-gpu-plugin + - intel-fpga-plugin + - intel-qat-initcontainer + - intel-qat-plugin + - intel-qat-plugin-kerneldrv + - intel-deviceplugin-operator + - intel-sgx-admissionwebhook + - intel-sgx-plugin + - intel-sgx-initcontainer + - intel-dsa-plugin + - intel-iaa-plugin + - intel-idxd-config-initcontainer + - intel-dlb-plugin + - intel-dlb-initcontainer + - intel-xpumanager-sidecar + + # # Demo images + - crypto-perf + - opae-nlb-demo + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + - run: make -e vendor + - name: Build image + env: + IMAGE_NAME: ${{ matrix.image }} + run: | + REG=intel/ make ${IMAGE_NAME} BUILDER=docker + - name: Trivy scan for image + uses: aquasecurity/trivy-action@master + with: + scan-type: image + image-ref: intel/${{ matrix.image }}:${{ inputs.image_tag }} + exit-code: 1 + - name: Test image base layer + run: IMG=intel/${{ matrix.image }}:${{ inputs.image_tag }} make test-image-base-layer BUILDER=docker + - name: Login + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Push + run: docker push intel/${{ matrix.image }}:${{ inputs.image_tag }} diff --git a/.github/workflows/lib-trivy.yaml b/.github/workflows/lib-trivy.yaml new file mode 100644 index 000000000..f843dd960 --- /dev/null +++ b/.github/workflows/lib-trivy.yaml @@ -0,0 +1,132 @@ +name: trivy +on: + workflow_call: + inputs: + upload-to-github-security-tab: + default: false + required: false + type: boolean + export-csv: + default: false + required: false + type: boolean + deployments: + default: true + required: false + type: boolean + dockerfiles: + default: true + required: false + type: boolean + +jobs: + trivy-scan-deployments: + name: Scan deployments + if: ${{ inputs.deployments }} + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Run Trivy in config mode for deployments + uses: aquasecurity/trivy-action@master + with: + scan-type: config + scan-ref: deployments/ + exit-code: 1 + severity: CRITICAL,HIGH + + trivy-scan-dockerfiles: + name: Scan Dockerfiles + if: ${{ inputs.dockerfiles }} + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Run Trivy in config mode for dockerfiles + uses: aquasecurity/trivy-action@master + with: + scan-type: config + scan-ref: build/docker/ + exit-code: 1 + severity: CRITICAL,HIGH + + trivy-scan-licenses: + runs-on: ubuntu-22.04 + name: Scan licenses + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Run Trivy in fs mode + uses: aquasecurity/trivy-action@master + with: + scan-type: fs + scan-ref: . + exit-code: 1 + scanners: license + severity: "UNKNOWN,MEDIUM,HIGH,CRITICAL" + + trivy-scan-vulns: + runs-on: ubuntu-22.04 + name: Scan vulnerabilities + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Run Trivy in fs mode + continue-on-error: true + uses: aquasecurity/trivy-action@master + with: + scan-type: fs + scan-ref: . + exit-code: 1 + list-all-pkgs: true + format: json + output: trivy-report.json + + - name: Show report in human-readable format + uses: aquasecurity/trivy-action@master + with: + scan-type: convert + vuln-type: '' + severity: '' + image-ref: trivy-report.json + format: table + + - name: Convert report to sarif + if: ${{ inputs.upload-to-github-security-tab }} + uses: aquasecurity/trivy-action@master + with: + scan-type: convert + vuln-type: '' + severity: '' + image-ref: trivy-report.json + format: sarif + output: trivy-report.sarif + + - name: Upload sarif report to GitHub Security tab + if: ${{ inputs.upload-to-github-security-tab }} + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: trivy-report.sarif + + - name: Convert report to csv + if: ${{ inputs.export-csv }} + uses: aquasecurity/trivy-action@master + with: + scan-type: convert + vuln-type: '' + severity: '' + image-ref: trivy-report.json + format: template + template: "@.github/workflows/template/trivy-csv.tpl" + output: trivy-report.csv + + - name: Upload CSV report as an artifact + if: ${{ inputs.export-csv }} + uses: actions/upload-artifact@v3 + with: + name: trivy-report + path: trivy-report.csv \ No newline at end of file diff --git a/.github/workflows/lib-validate.yaml b/.github/workflows/lib-validate.yaml new file mode 100644 index 000000000..d8c141ec6 --- /dev/null +++ b/.github/workflows/lib-validate.yaml @@ -0,0 +1,71 @@ +name: validate +on: + workflow_call: + +env: + K8S_VERSION: 1.27.1 +jobs: + docs: + name: Check docs are buildable + runs-on: ubuntu-22.04 + steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y python3-venv + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up doc directory + run: | + mkdir $HOME/output + touch $HOME/output/.nojekyll + - name: Build latest + run: | + GITHUB_SHA=$(git rev-parse HEAD) + export GITHUB_SHA + rm -rf _work/venv + make vhtml + mv _build/html/* $HOME/output/ + + golangci: + permissions: + contents: read # for actions/checkout to fetch code + pull-requests: read # for golangci/golangci-lint-action to fetch pull requests + name: lint + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version-file: go.mod + check-latest: true + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.52.1 + args: -v --timeout 5m + + build: + name: Build and check device plugins + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version-file: go.mod + check-latest: true + - name: Check Dockerfiles + run: make check-dockerfiles + - run: make go-mod-tidy + - run: make BUILDTAGS=kerneldrv + - run: make test BUILDTAGS=kerneldrv + - name: Install envtest tool and run envtest + run: | + go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + setup-envtest use ${K8S_VERSION} + - run: | + KUBEBUILDER_ASSETS=$(setup-envtest use -i -p path ${K8S_VERSION}) make envtest + - run: make check-github-actions + - name: Codecov report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..4968e06a0 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,39 @@ +name: Release +on: + push: + tags: + - 'v*' +permissions: + contents: read + pull-requests: read + +jobs: + tag_fix: + name: Prepare image tag + runs-on: ubuntu-22.04 + outputs: + fixed_tag: ${{ steps.fix.outputs.tag }} + env: + TAGNAME: ${{ github.ref_name }} + steps: + - id: fix + # remove first character (v) + run: echo "tag=${TAGNAME:1}" >> "$GITHUB_OUTPUT" + + trivy: + name: Trivy + uses: "./.github/workflows/lib-trivy.yaml" + with: + deployments: false + dockerfiles: false + export-csv: true + + build: + name: Build & Publish + needs: + - trivy + - tag_fix + uses: "./.github/workflows/lib-publish.yaml" + secrets: inherit + with: + image_tag: ${{ needs.tag_fix.outputs.fixed_tag }} diff --git a/.github/workflows/template/trivy-csv.tpl b/.github/workflows/template/trivy-csv.tpl new file mode 100644 index 000000000..0c1e50744 --- /dev/null +++ b/.github/workflows/template/trivy-csv.tpl @@ -0,0 +1,29 @@ +{{ range . }} +Trivy Vulnerability Scan Results ({{- .Target -}}) +VulnerabilityID,Severity,CVSS Score,Title,Library,Vulnerable Version,Fixed Version,Information URL,Triage Information +{{ range .Vulnerabilities }} + {{- .VulnerabilityID }}, + {{- .Severity }}, + {{- range $key, $value := .CVSS }} + {{- if (eq $key "nvd") }} + {{- .V3Score -}} + {{- end }} + {{- end }}, + {{- quote .Title }}, + {{- quote .PkgName }}, + {{- quote .InstalledVersion }}, + {{- quote .FixedVersion }}, + {{- .PrimaryURL }} +{{ else -}} + No vulnerabilities found at this time. +{{ end }} +Trivy Dependency Scan Results ({{ .Target }}) +ID,Name,Version,Notes +{{ range .Packages -}} + {{- quote .ID }}, + {{- quote .Name }}, + {{- quote .Version }} +{{ else -}} + No dependencies found at this time. +{{ end }} +{{ end }} diff --git a/Makefile b/Makefile index 65d293fb1..f1f82a85c 100644 --- a/Makefile +++ b/Makefile @@ -184,11 +184,8 @@ dockertemplates = build/docker/templates images = $(shell basename -s .Dockerfile.in -a $(dockertemplates)/*.Dockerfile.in) dockerfiles = $(shell basename -s .in -a $(dockertemplates)/*.Dockerfile.in | xargs -I"{}" echo build/docker/{}) - -skipbaselayercheck = intel-vpu-plugin intel-qat-plugin-kerneldrv intel-idxd-config-initcontainer -distroless_images = $(patsubst %,$(REG)%\:$(TAG),$(filter-out $(skipbaselayercheck),$(images))) test-image-base-layer: - @for img in $(distroless_images); do scripts/test-image-base-layer.sh $$img $(BUILDER) || exit 1; done + @scripts/test-image-base-layer.sh $(IMG) $(BUILDER) $(dockerfiles): $(dockertemplates)/*.Dockerfile.in $(dockerlib)/*.docker @cat $(dockerlib)/default_header.docker $(dockertemplates)/$(shell basename $@.in) \ @@ -221,12 +218,6 @@ $(demos): demos: $(demos) -image_tags = $(patsubst %,$(REG)%\:$(TAG),$(images) $(demos)) -$(image_tags): - @docker push $@ - -push: test-image-base-layer $(image_tags) - set-version: @scripts/set-version.sh $(TAG) @@ -238,9 +229,9 @@ skip_images_source := ubuntu-demo-openvino intel-vpu-plugin skip_images := $(subst $(space),$(comma),$(addprefix ",$(addsuffix ", $(skip_images_source)))) check-github-actions: - @python3 -c 'import sys, yaml, json; json.dump(yaml.load(sys.stdin, Loader=yaml.SafeLoader), sys.stdout)' < .github/workflows/ci.yaml | \ + @python3 -c 'import sys, yaml, json; json.dump(yaml.load(sys.stdin, Loader=yaml.SafeLoader), sys.stdout)' < .github/workflows/lib-build.yaml | \ jq -e '$(images_json) - [$(skip_images)] - .jobs.image.strategy.matrix.image == []' > /dev/null || \ - (echo "Make sure all images are listed in .github/workflows/ci.yaml"; exit 1) + (echo "Make sure all images are listed in .github/workflows/lib-build.yaml"; exit 1) .PHONY: all format test lint build images $(cmds) $(images) lock-images vendor pre-pull set-version check-github-actions envtest fixture update-fixture install-tools test-image-base-layer From 0e9c4e07357c4354c96efaea8daed5209bf4dfed Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Fri, 25 Aug 2023 15:59:37 +0300 Subject: [PATCH 2/3] gitignore: add vendor and licenses Signed-off-by: Tuomas Katila --- .gitignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index d448e9d05..7f2ae6a77 100644 --- a/.gitignore +++ b/.gitignore @@ -32,5 +32,6 @@ deployments/fpga_admissionwebhook/base/intel-fpga-webhook-certs-secret _build _work -*.tgz -charts/operator/crds +vendor/* +licenses/* + From ff32f361fb1ef9b7af166c0234e0606e5dd43668 Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Fri, 25 Aug 2023 16:06:50 +0300 Subject: [PATCH 3/3] trivy: add ignore file for necessary things Signed-off-by: Tuomas Katila --- .trivyignore | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .trivyignore diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 000000000..c75ce363b --- /dev/null +++ b/.trivyignore @@ -0,0 +1,28 @@ +# Image user should not be ‘root’ +# Running containers with ‘root’ user can lead to a container escape situation. +# It is a best practice to run containers as non-root users, which can be done by +# adding # a ‘USER’ statement to the Dockerfile. +AVD-DS-0002 + +# Privileged containers share namespaces with the host system and do not offer any security. +# They should be used exclusively for system containers that require high # privileges. +# initcontainers require privileged access +AVD-KSV-0017 + +# Sharing the host’s network namespace permits processes in the pod to communicate with +# processes bound to the host’s loopback adapter. +# sgx single-node demo deployment uses hostNetwork: true to be able to use the default setting for PCCS URL from containers +AVD-KSV-0009 + +# Do not allow privilege escalation from node proxy +# Check whether role permits privilege escalation from node proxy +# gpu plugin in kubelet mode requires "nodes/proxy" resource access +AVD-KSV-0047 + +# Do not allow update/create of a malicious pod +# Check whether role permits update/create of a malicious pod +# device plugin operator requires access to daemonset creation etc. +AVD-KSV-0048 + +# Device plugins do not use any CSIs +## CVE-2019-11255