From 1d705c4e38fe68a0a0dc1768d2088d6730307294 Mon Sep 17 00:00:00 2001 From: DavidSpek Date: Wed, 20 Jan 2021 20:10:01 +0100 Subject: [PATCH 1/6] add dependabot config script --- .github/base_dependabot.yml.tmp | 8 ++++ .github/depend_template.yml | 6 +++ .github/dependabot.yml | 67 +++++++++++++++++++++++++++++++++ create_dependabot.sh | 24 ++++++++++++ 4 files changed, 105 insertions(+) create mode 100644 .github/base_dependabot.yml.tmp create mode 100644 .github/depend_template.yml create mode 100644 .github/dependabot.yml create mode 100755 create_dependabot.sh diff --git a/.github/base_dependabot.yml.tmp b/.github/base_dependabot.yml.tmp new file mode 100644 index 000000000..0b52908e3 --- /dev/null +++ b/.github/base_dependabot.yml.tmp @@ -0,0 +1,8 @@ +version: 2 +updates: + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "daily" + # Disable version updates for npm dependencies + open-pull-requests-limit: 10 diff --git a/.github/depend_template.yml b/.github/depend_template.yml new file mode 100644 index 000000000..e4436c57e --- /dev/null +++ b/.github/depend_template.yml @@ -0,0 +1,6 @@ +package-ecosystem: "" +directory: "/" +schedule: + interval: "daily" +# Disable version updates for npm dependencies +open-pull-requests-limit: 10 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..3e8bf0917 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,67 @@ +version: 2 +updates: + - package-ecosystem: docker + directory: /apps-cd + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: docker + directory: /go + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: docker + directory: /images + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: docker + directory: /notebook_testing + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: docker + directory: /test-infra/auto-deploy + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: npm + directory: /py/kubeflow/testing/node-license-tools + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: pip + directory: /apps-cd + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: pip + directory: /py + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: pip + directory: /py/kubeflow/testing + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: pip + directory: /test-infra/auto-deploy + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: gomod + directory: /go + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: gomod + directory: /go/cmd/nomos-wait + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: gomod + directory: /tests + schedule: + interval: daily + open-pull-requests-limit: 10 diff --git a/create_dependabot.sh b/create_dependabot.sh new file mode 100755 index 000000000..b5055a1ac --- /dev/null +++ b/create_dependabot.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +rm .github/dependabot.yml +cp .github/base_dependabot.yml.tmp .github/dependabot.yml + +for directory in $(dirname $(find . -type f -name "*ockerfile*") | sort -u | cut -c2-); do + if [[ ${directory} != *"node_modules"* ]]; then + yq eval -i ".updates += {\"package-ecosystem\":\"docker\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10}" .github/dependabot.yml + fi +done + +for directory in $(dirname $(find . -type f -name "package*.json" -not -path "./*node_modules*") | sort -u | cut -c2-); do + yq eval -i ".updates += {\"package-ecosystem\":\"npm\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10}" .github/dependabot.yml +done + +for directory in $(dirname $(find . -type f -name "*requirements.txt" -not -path "./*node_modules*") | sort -u | cut -c2-); do + yq eval -i ".updates += {\"package-ecosystem\":\"pip\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10}" .github/dependabot.yml +done + +for directory in $(dirname $(find . -type f -name "go.*" -not -path "./*node_modules*") | sort -u | cut -c2-); do + yq eval -i ".updates += {\"package-ecosystem\":\"gomod\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10}" .github/dependabot.yml +done + +yq eval -i 'del(.updates[0])' .github/dependabot.yml \ No newline at end of file From 9325cf403be18037624ea4f9e0ba6e4480bca3a7 Mon Sep 17 00:00:00 2001 From: DavidSpek Date: Thu, 21 Jan 2021 18:09:52 +0100 Subject: [PATCH 2/6] add asssignees, create doc and makefile and cleanup --- .github/base_dependabot.yml.tmp | 8 -- .github/depend_template.yml | 6 -- .github/dependabot.yml | 125 ++++++++++++++++++++++---------- Makefile | 4 + README.md | 5 ++ create_dependabot.sh | 24 ------ hack/create_dependabot.sh | 63 ++++++++++++++++ 7 files changed, 158 insertions(+), 77 deletions(-) delete mode 100644 .github/base_dependabot.yml.tmp delete mode 100644 .github/depend_template.yml delete mode 100755 create_dependabot.sh create mode 100755 hack/create_dependabot.sh diff --git a/.github/base_dependabot.yml.tmp b/.github/base_dependabot.yml.tmp deleted file mode 100644 index 0b52908e3..000000000 --- a/.github/base_dependabot.yml.tmp +++ /dev/null @@ -1,8 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "npm" - directory: "/" - schedule: - interval: "daily" - # Disable version updates for npm dependencies - open-pull-requests-limit: 10 diff --git a/.github/depend_template.yml b/.github/depend_template.yml deleted file mode 100644 index e4436c57e..000000000 --- a/.github/depend_template.yml +++ /dev/null @@ -1,6 +0,0 @@ -package-ecosystem: "" -directory: "/" -schedule: - interval: "daily" -# Disable version updates for npm dependencies -open-pull-requests-limit: 10 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3e8bf0917..c849a8514 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,67 +1,114 @@ version: 2 updates: - - package-ecosystem: docker - directory: /apps-cd + - package-ecosystem: "docker" + directory: "./apps-cd" schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 - - package-ecosystem: docker - directory: /go + assignees: + - bobgy + - jlewi + - package-ecosystem: "docker" + directory: "./go" schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 - - package-ecosystem: docker - directory: /images + assignees: + - bobgy + - jlewi + - package-ecosystem: "docker" + directory: "./images" schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 - - package-ecosystem: docker - directory: /notebook_testing + assignees: + - bobgy + - jlewi + - package-ecosystem: "docker" + directory: "./notebook_testing" schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 - - package-ecosystem: docker - directory: /test-infra/auto-deploy + assignees: + - bobgy + - jlewi + - package-ecosystem: "docker" + directory: "./test-infra/auto-deploy" schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 - - package-ecosystem: npm - directory: /py/kubeflow/testing/node-license-tools + assignees: + - bobgy + - jlewi + - package-ecosystem: "npm" + directory: "." schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 - - package-ecosystem: pip - directory: /apps-cd + assignees: + - bobgy + - jlewi + - package-ecosystem: "npm" + directory: "./py/kubeflow/testing/node-license-tools" schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 - - package-ecosystem: pip - directory: /py + assignees: + - bobgy + - jlewi + - package-ecosystem: "pip" + directory: "./apps-cd" schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 - - package-ecosystem: pip - directory: /py/kubeflow/testing + assignees: + - bobgy + - jlewi + - package-ecosystem: "pip" + directory: "./py" schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 - - package-ecosystem: pip - directory: /test-infra/auto-deploy + assignees: + - bobgy + - jlewi + - package-ecosystem: "pip" + directory: "./py/kubeflow/testing" schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 - - package-ecosystem: gomod - directory: /go + assignees: + - bobgy + - jlewi + - package-ecosystem: "pip" + directory: "./test-infra/auto-deploy" schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 - - package-ecosystem: gomod - directory: /go/cmd/nomos-wait + assignees: + - bobgy + - jlewi + - package-ecosystem: "gomod" + directory: "./go" schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 - - package-ecosystem: gomod - directory: /tests + assignees: + - bobgy + - jlewi + - package-ecosystem: "gomod" + directory: "./go/cmd/nomos-wait" schedule: - interval: daily + interval: "daily" open-pull-requests-limit: 10 + assignees: + - bobgy + - jlewi + - package-ecosystem: "gomod" + directory: "./tests" + schedule: + interval: "daily" + open-pull-requests-limit: 10 + assignees: + - bobgy + - jlewi diff --git a/Makefile b/Makefile index b7ab23035..7bdadc5c2 100644 --- a/Makefile +++ b/Makefile @@ -64,3 +64,7 @@ debug-rebuild-and-run: make hydrate && git add . && git commit -m "Latest" && git push jlewi cd ./go/cmd/nomos-wait && go run . kubectl --context=kf-ci-v1 create -f ./tekton/runs/nb-test-run.yaml + +build-dependabot: + chmod +x hack/create_dependabot.sh + hack/create_dependabot.sh \ No newline at end of file diff --git a/README.md b/README.md index 562e07a51..33eacc210 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ - [Step Image](#step-image) - [Checking out code](#checking-out-code) - [Building Docker Images](#building-docker-images) + - [Creating dependabot config yaml for this repo](#Creating-dependabot-config-yaml-for-this-repo) @@ -1108,3 +1109,7 @@ is * TAG used for the images * Argo workflow should define the image paths and tag so that subsequent steps can use the newly built images + +## Creating dependabot config yaml for this repo + +In an effort to use the most current versions and mitigate vulnerable software dependencies and base images, a script was created to properly configure dependabot. The script scans the repository for directories containing files listing such dependencies, and matches the found folders to the relevant `OWNERS` files. It then goes on to generate the `.github/dependabot.yml` file which tells dependabot which directories it needs to scan and for what package ecosystems. When a dependency update is found, dependabot will create a pull request to update the dependency and assign the relevant owners. If changes are made to the repository that add new dependency listing files, the script will need to be run so that `.github/dependabot.yml` is updated to reflect these changes. To manually run the script, execute `make build-dependabot` from the root of this repository. diff --git a/create_dependabot.sh b/create_dependabot.sh deleted file mode 100755 index b5055a1ac..000000000 --- a/create_dependabot.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -rm .github/dependabot.yml -cp .github/base_dependabot.yml.tmp .github/dependabot.yml - -for directory in $(dirname $(find . -type f -name "*ockerfile*") | sort -u | cut -c2-); do - if [[ ${directory} != *"node_modules"* ]]; then - yq eval -i ".updates += {\"package-ecosystem\":\"docker\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10}" .github/dependabot.yml - fi -done - -for directory in $(dirname $(find . -type f -name "package*.json" -not -path "./*node_modules*") | sort -u | cut -c2-); do - yq eval -i ".updates += {\"package-ecosystem\":\"npm\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10}" .github/dependabot.yml -done - -for directory in $(dirname $(find . -type f -name "*requirements.txt" -not -path "./*node_modules*") | sort -u | cut -c2-); do - yq eval -i ".updates += {\"package-ecosystem\":\"pip\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10}" .github/dependabot.yml -done - -for directory in $(dirname $(find . -type f -name "go.*" -not -path "./*node_modules*") | sort -u | cut -c2-); do - yq eval -i ".updates += {\"package-ecosystem\":\"gomod\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10}" .github/dependabot.yml -done - -yq eval -i 'del(.updates[0])' .github/dependabot.yml \ No newline at end of file diff --git a/hack/create_dependabot.sh b/hack/create_dependabot.sh new file mode 100755 index 000000000..7289b771b --- /dev/null +++ b/hack/create_dependabot.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +yq eval --null-input '.version = 2 | .updates = []' > .github/dependabot.yml + +for directory in $(dirname $(find . -type f -name "*ockerfile*") | sort -u); do + if [[ ${directory} != *"node_modules"* ]]; then + if ! [[ "$(dirname $(find ./* -type f -name "OWNERS") | sort -u)[@]" == ${directory} ]]; then + assignees=$(yq eval -j -I=0 '.approvers' ./OWNERS) + yq eval -i ".updates += {\"package-ecosystem\":\"docker\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml + else + for owners in $(find ./* -type f -name "OWNERS" | sort -u); do + if [[ ${directory} == "$(dirname ${owners})" ]]; then + assignees=$(yq eval -j -I=0 '.approvers' ${owners}) + yq eval -i ".updates += {\"package-ecosystem\":\"docker\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml + fi + done + fi + fi +done + +for directory in $(dirname $(find . -type f -name "package*.json" -not -path "./*node_modules*") | sort -u); do + if ! [[ "$(dirname $(find ./* -type f -name "OWNERS") | sort -u)[@]" == ${directory} ]]; then + assignees=$(yq eval -j -I=0 '.approvers' ./OWNERS) + yq eval -i ".updates += {\"package-ecosystem\":\"npm\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml + else + for owners in $(find ./* -type f -name "OWNERS" | sort -u); do + if [[ ${directory} == "$(dirname ${owners})" ]]; then + assignees=$(yq eval -j -I=0 '.approvers' ${owners}) + yq eval -i ".updates += {\"package-ecosystem\":\"npm\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml + fi + done + fi +done + +for directory in $(dirname $(find . -type f -name "*requirements.txt" -not -path "./*node_modules*") | sort -u); do + if ! [[ "$(dirname $(find ./* -type f -name "OWNERS") | sort -u)[@]" == ${directory} ]]; then + assignees=$(yq eval -j -I=0 '.approvers' ./OWNERS) + yq eval -i ".updates += {\"package-ecosystem\":\"pip\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml + else + for owners in $(find ./* -type f -name "OWNERS" | sort -u); do + if [[ ${directory} == "$(dirname ${owners})" ]]; then + assignees=$(yq eval -j -I=0 '.approvers' ${owners}) + yq eval -i ".updates += {\"package-ecosystem\":\"pip\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml + fi + done + fi +done + +for directory in $(dirname $(find . -type f -name "go.*" -not -path "./*node_modules*") | sort -u); do + if ! [[ "$(dirname $(find ./* -type f -name "OWNERS") | sort -u)[@]" == ${directory} ]]; then + assignees=$(yq eval -j -I=0 '.approvers' ./OWNERS) + yq eval -i ".updates += {\"package-ecosystem\":\"gomod\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml + else + for owners in $(find ./* -type f -name "OWNERS" | sort -u); do + if [[ ${directory} == "$(dirname ${owners})" ]]; then + assignees=$(yq eval -j -I=0 '.approvers' ${owners}) + yq eval -i ".updates += {\"package-ecosystem\":\"gomod\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml + fi + done + fi +done + +yq eval -i '... style="" | .updates[].directory style="double" | .updates[].package-ecosystem style="double" | .updates[].schedule.interval style="double"' .github/dependabot.yml From e22a906019c0e9331e9f0262a966512fb81915e5 Mon Sep 17 00:00:00 2001 From: DavidSpek Date: Thu, 21 Jan 2021 18:29:38 +0100 Subject: [PATCH 3/6] small fixup for node_modules folder selection --- hack/create_dependabot.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/hack/create_dependabot.sh b/hack/create_dependabot.sh index 7289b771b..8d2857ce4 100755 --- a/hack/create_dependabot.sh +++ b/hack/create_dependabot.sh @@ -2,8 +2,7 @@ yq eval --null-input '.version = 2 | .updates = []' > .github/dependabot.yml -for directory in $(dirname $(find . -type f -name "*ockerfile*") | sort -u); do - if [[ ${directory} != *"node_modules"* ]]; then +for directory in $(dirname $(find . -type f -name "*ockerfile*" -not -path "./*node_modules*") | sort -u); do if ! [[ "$(dirname $(find ./* -type f -name "OWNERS") | sort -u)[@]" == ${directory} ]]; then assignees=$(yq eval -j -I=0 '.approvers' ./OWNERS) yq eval -i ".updates += {\"package-ecosystem\":\"docker\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml @@ -15,20 +14,21 @@ for directory in $(dirname $(find . -type f -name "*ockerfile*") | sort -u); do fi done fi - fi done for directory in $(dirname $(find . -type f -name "package*.json" -not -path "./*node_modules*") | sort -u); do - if ! [[ "$(dirname $(find ./* -type f -name "OWNERS") | sort -u)[@]" == ${directory} ]]; then - assignees=$(yq eval -j -I=0 '.approvers' ./OWNERS) - yq eval -i ".updates += {\"package-ecosystem\":\"npm\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml - else - for owners in $(find ./* -type f -name "OWNERS" | sort -u); do - if [[ ${directory} == "$(dirname ${owners})" ]]; then - assignees=$(yq eval -j -I=0 '.approvers' ${owners}) - yq eval -i ".updates += {\"package-ecosystem\":\"npm\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml - fi - done + if [[ ${directory} != *"dist"* ]]; then + if ! [[ "$(dirname $(find ./* -type f -name "OWNERS") | sort -u)[@]" == ${directory} ]]; then + assignees=$(yq eval -j -I=0 '.approvers' ./OWNERS) + yq eval -i ".updates += {\"package-ecosystem\":\"npm\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml + else + for owners in $(find ./* -type f -name "OWNERS" | sort -u); do + if [[ ${directory} == "$(dirname ${owners})" ]]; then + assignees=$(yq eval -j -I=0 '.approvers' ${owners}) + yq eval -i ".updates += {\"package-ecosystem\":\"npm\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml + fi + done + fi fi done From 6c25a9fddbf527255a45ed76d513cafaef01325e Mon Sep 17 00:00:00 2001 From: DavidSpek Date: Fri, 22 Jan 2021 13:26:14 +0100 Subject: [PATCH 4/6] change script to python --- .github/dependabot.yml | 262 ++++++++++++++++++++++---------------- Makefile | 3 +- hack/create_dependabot.py | 96 ++++++++++++++ hack/create_dependabot.sh | 63 --------- 4 files changed, 246 insertions(+), 178 deletions(-) create mode 100644 hack/create_dependabot.py delete mode 100755 hack/create_dependabot.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c849a8514..9f45a3787 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,114 +1,150 @@ -version: 2 updates: - - package-ecosystem: "docker" - directory: "./apps-cd" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "docker" - directory: "./go" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "docker" - directory: "./images" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "docker" - directory: "./notebook_testing" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "docker" - directory: "./test-infra/auto-deploy" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "npm" - directory: "." - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "npm" - directory: "./py/kubeflow/testing/node-license-tools" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "pip" - directory: "./apps-cd" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "pip" - directory: "./py" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "pip" - directory: "./py/kubeflow/testing" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "pip" - directory: "./test-infra/auto-deploy" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "gomod" - directory: "./go" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "gomod" - directory: "./go/cmd/nomos-wait" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi - - package-ecosystem: "gomod" - directory: "./tests" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - assignees: - - bobgy - - jlewi +- assignees: + - bobgy + - jlewi + directory: notebook_testing + open-pull-requests-limit: 10 + package-ecosystem: docker + reviewers: + - Jeffwan + - pingsutw + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + directory: test-infra/auto-deploy + open-pull-requests-limit: 10 + package-ecosystem: docker + reviewers: + - Jeffwan + - pingsutw + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + - PatrickXYS + directory: images + open-pull-requests-limit: 10 + package-ecosystem: docker + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + directory: apps-cd + open-pull-requests-limit: 10 + package-ecosystem: docker + reviewers: + - Jeffwan + - pingsutw + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + directory: go + open-pull-requests-limit: 10 + package-ecosystem: docker + reviewers: + - Jeffwan + - pingsutw + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + directory: . + open-pull-requests-limit: 10 + package-ecosystem: npm + reviewers: + - Jeffwan + - pingsutw + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + - PatrickXYS + directory: py/kubeflow/testing/node-license-tools + open-pull-requests-limit: 10 + package-ecosystem: npm + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + directory: test-infra/auto-deploy + open-pull-requests-limit: 10 + package-ecosystem: pip + reviewers: + - Jeffwan + - pingsutw + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + directory: apps-cd + open-pull-requests-limit: 10 + package-ecosystem: pip + reviewers: + - Jeffwan + - pingsutw + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + directory: py + open-pull-requests-limit: 10 + package-ecosystem: pip + reviewers: + - Jeffwan + - pingsutw + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + - PatrickXYS + directory: py/kubeflow/testing + open-pull-requests-limit: 10 + package-ecosystem: pip + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + directory: go + open-pull-requests-limit: 10 + package-ecosystem: gomod + reviewers: + - Jeffwan + - pingsutw + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + directory: go/cmd/nomos-wait + open-pull-requests-limit: 10 + package-ecosystem: gomod + reviewers: + - Jeffwan + - pingsutw + schedule: + interval: daily +- assignees: + - bobgy + - jlewi + directory: tests + open-pull-requests-limit: 10 + package-ecosystem: gomod + reviewers: + - Jeffwan + - pingsutw + schedule: + interval: daily +version: 2 diff --git a/Makefile b/Makefile index 7bdadc5c2..556fb8382 100644 --- a/Makefile +++ b/Makefile @@ -66,5 +66,4 @@ debug-rebuild-and-run: kubectl --context=kf-ci-v1 create -f ./tekton/runs/nb-test-run.yaml build-dependabot: - chmod +x hack/create_dependabot.sh - hack/create_dependabot.sh \ No newline at end of file + python3 hack/create_dependabot.py \ No newline at end of file diff --git a/hack/create_dependabot.py b/hack/create_dependabot.py new file mode 100644 index 000000000..fc9e5dac9 --- /dev/null +++ b/hack/create_dependabot.py @@ -0,0 +1,96 @@ +import yaml +import collections +from pathlib import Path + +repo_path = Path(__file__).parents[1] + +print(repo_path.anchor) + +dependabot = {} +dependabot['version'] = 2 +dependabot['updates'] = [] +ignored_folders = ['node_modules', 'dist', '.git', 'deprecated'] + +def get_owners(path): + while not Path(path/'OWNERS').is_file(): + path = path.parent.absolute() + with open(path/'OWNERS') as owner_file: + owners = yaml.load(owner_file) + return owners + +def get_docker_paths(): + dockerfile_list = list(repo_path.glob('**/*ockerfile*')) + docker_clean_list = [] + for dockerfile in dockerfile_list: + if all(x not in str(dockerfile) for x in ignored_folders): + if dockerfile.parents[0] not in docker_clean_list: + docker_clean_list.append(dockerfile.parents[0]) + return docker_clean_list + +print(get_docker_paths()) + +def get_npm_paths(): + npm_list = list(repo_path.glob('**/package*.json')) + npm_clean_list = [] + for npm_file in npm_list: + if all(x not in str(npm_file) for x in ignored_folders): + if npm_file.parents[0] not in npm_clean_list: + npm_clean_list.append(npm_file.parents[0]) + return npm_clean_list + +def get_pip_paths(): + pip_list = list(repo_path.glob('**/*requirements.txt')) + pip_clean_list = [] + for pip_file in pip_list: + if all(x not in str(pip_file) for x in ignored_folders): + if pip_file.parents[0] not in pip_clean_list: + pip_clean_list.append(pip_file.parents[0]) + return pip_clean_list + +def get_go_paths(): + go_list = list(repo_path.glob('**/go.*')) + go_clean_list = [] + for go_file in go_list: + if all(x not in str(go_file) for x in ignored_folders): + if go_file.parents[0] not in go_clean_list: + go_clean_list.append(go_file.parents[0]) + return go_clean_list + +def append_updates(ecosystem, directory, assignees, reviewers=None): + config = {} + config['package-ecosystem'] = ecosystem + config['directory'] = directory + config['schedule']= {} + config['schedule']['interval'] = 'daily' + config['open-pull-requests-limit'] = 10 + config['assignees'] = assignees + if reviewers: + config['reviewers'] = reviewers + dependabot['updates'].append(config) + +for docker_path in get_docker_paths(): + string_path = str(docker_path) + assignees = get_owners(docker_path).get('approvers') + reviewers = get_owners(docker_path).get('reviewers') + append_updates('docker', string_path, assignees, reviewers) + +for npm_path in get_npm_paths(): + string_path = str(npm_path) + assignees = get_owners(npm_path).get('approvers') + reviewers = get_owners(npm_path).get('reviewers') + append_updates('npm', string_path, assignees, reviewers) + +for pip_path in get_pip_paths(): + string_path = str(pip_path) + assignees = get_owners(pip_path).get('approvers') + reviewers = get_owners(pip_path).get('reviewers') + append_updates('pip', string_path, assignees, reviewers) + +for go_path in get_go_paths(): + string_path = str(go_path) + assignees = get_owners(go_path).get('approvers') + reviewers = get_owners(go_path).get('reviewers') + append_updates('gomod', string_path, assignees, reviewers) + +with open('.github/dependabot.yml', 'w') as outfile: + yaml.dump(dependabot, outfile, default_flow_style=False) \ No newline at end of file diff --git a/hack/create_dependabot.sh b/hack/create_dependabot.sh deleted file mode 100755 index 8d2857ce4..000000000 --- a/hack/create_dependabot.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -yq eval --null-input '.version = 2 | .updates = []' > .github/dependabot.yml - -for directory in $(dirname $(find . -type f -name "*ockerfile*" -not -path "./*node_modules*") | sort -u); do - if ! [[ "$(dirname $(find ./* -type f -name "OWNERS") | sort -u)[@]" == ${directory} ]]; then - assignees=$(yq eval -j -I=0 '.approvers' ./OWNERS) - yq eval -i ".updates += {\"package-ecosystem\":\"docker\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml - else - for owners in $(find ./* -type f -name "OWNERS" | sort -u); do - if [[ ${directory} == "$(dirname ${owners})" ]]; then - assignees=$(yq eval -j -I=0 '.approvers' ${owners}) - yq eval -i ".updates += {\"package-ecosystem\":\"docker\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml - fi - done - fi -done - -for directory in $(dirname $(find . -type f -name "package*.json" -not -path "./*node_modules*") | sort -u); do - if [[ ${directory} != *"dist"* ]]; then - if ! [[ "$(dirname $(find ./* -type f -name "OWNERS") | sort -u)[@]" == ${directory} ]]; then - assignees=$(yq eval -j -I=0 '.approvers' ./OWNERS) - yq eval -i ".updates += {\"package-ecosystem\":\"npm\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml - else - for owners in $(find ./* -type f -name "OWNERS" | sort -u); do - if [[ ${directory} == "$(dirname ${owners})" ]]; then - assignees=$(yq eval -j -I=0 '.approvers' ${owners}) - yq eval -i ".updates += {\"package-ecosystem\":\"npm\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml - fi - done - fi - fi -done - -for directory in $(dirname $(find . -type f -name "*requirements.txt" -not -path "./*node_modules*") | sort -u); do - if ! [[ "$(dirname $(find ./* -type f -name "OWNERS") | sort -u)[@]" == ${directory} ]]; then - assignees=$(yq eval -j -I=0 '.approvers' ./OWNERS) - yq eval -i ".updates += {\"package-ecosystem\":\"pip\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml - else - for owners in $(find ./* -type f -name "OWNERS" | sort -u); do - if [[ ${directory} == "$(dirname ${owners})" ]]; then - assignees=$(yq eval -j -I=0 '.approvers' ${owners}) - yq eval -i ".updates += {\"package-ecosystem\":\"pip\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml - fi - done - fi -done - -for directory in $(dirname $(find . -type f -name "go.*" -not -path "./*node_modules*") | sort -u); do - if ! [[ "$(dirname $(find ./* -type f -name "OWNERS") | sort -u)[@]" == ${directory} ]]; then - assignees=$(yq eval -j -I=0 '.approvers' ./OWNERS) - yq eval -i ".updates += {\"package-ecosystem\":\"gomod\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml - else - for owners in $(find ./* -type f -name "OWNERS" | sort -u); do - if [[ ${directory} == "$(dirname ${owners})" ]]; then - assignees=$(yq eval -j -I=0 '.approvers' ${owners}) - yq eval -i ".updates += {\"package-ecosystem\":\"gomod\",\"directory\":\"${directory}\",\"schedule\":{\"interval\":\"daily\"},\"open-pull-requests-limit\":10,\"assignees\":${assignees}}" .github/dependabot.yml - fi - done - fi -done - -yq eval -i '... style="" | .updates[].directory style="double" | .updates[].package-ecosystem style="double" | .updates[].schedule.interval style="double"' .github/dependabot.yml From f0197602f0aca673a44888b4876d9438adebaf7c Mon Sep 17 00:00:00 2001 From: DavidSpek Date: Fri, 22 Jan 2021 19:20:12 +0100 Subject: [PATCH 5/6] add requested changes --- README.md | 11 ++++++- hack/create_dependabot.py | 61 ++++++++++++++++++++++----------------- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 33eacc210..29e3323e2 100644 --- a/README.md +++ b/README.md @@ -1112,4 +1112,13 @@ is ## Creating dependabot config yaml for this repo -In an effort to use the most current versions and mitigate vulnerable software dependencies and base images, a script was created to properly configure dependabot. The script scans the repository for directories containing files listing such dependencies, and matches the found folders to the relevant `OWNERS` files. It then goes on to generate the `.github/dependabot.yml` file which tells dependabot which directories it needs to scan and for what package ecosystems. When a dependency update is found, dependabot will create a pull request to update the dependency and assign the relevant owners. If changes are made to the repository that add new dependency listing files, the script will need to be run so that `.github/dependabot.yml` is updated to reflect these changes. To manually run the script, execute `make build-dependabot` from the root of this repository. +To use the most current versions and mitigate vulnerable software dependencies and base images, we configure dependabot for desired funtionality. + +* The way dependabot works as below: + 1. We uses a script to scan the repository for directories containing files listing such dependencies, and matches the found folders to the relevant `OWNERS` files + 2. Then it generate the `.github/dependabot.yml` file which tells dependabot which directories it needs to scan and for what package ecosystems. + 3. When a dependency update is found, dependabot will create a pull request to update the dependency and assign the relevant owners and reviewers. + +To generate a new dependabot configuration when dependency listing files are moved or created, the script can be run by executing `make build-dependabot` from the root of this repository. + +More details about dependabot and its configuration can be found here (https://docs.github.com/en/github/managing-security-vulnerabilities/managing-vulnerabilities-in-your-projects-dependencies) diff --git a/hack/create_dependabot.py b/hack/create_dependabot.py index fc9e5dac9..0119965ba 100644 --- a/hack/create_dependabot.py +++ b/hack/create_dependabot.py @@ -68,29 +68,38 @@ def append_updates(ecosystem, directory, assignees, reviewers=None): config['reviewers'] = reviewers dependabot['updates'].append(config) -for docker_path in get_docker_paths(): - string_path = str(docker_path) - assignees = get_owners(docker_path).get('approvers') - reviewers = get_owners(docker_path).get('reviewers') - append_updates('docker', string_path, assignees, reviewers) - -for npm_path in get_npm_paths(): - string_path = str(npm_path) - assignees = get_owners(npm_path).get('approvers') - reviewers = get_owners(npm_path).get('reviewers') - append_updates('npm', string_path, assignees, reviewers) - -for pip_path in get_pip_paths(): - string_path = str(pip_path) - assignees = get_owners(pip_path).get('approvers') - reviewers = get_owners(pip_path).get('reviewers') - append_updates('pip', string_path, assignees, reviewers) - -for go_path in get_go_paths(): - string_path = str(go_path) - assignees = get_owners(go_path).get('approvers') - reviewers = get_owners(go_path).get('reviewers') - append_updates('gomod', string_path, assignees, reviewers) - -with open('.github/dependabot.yml', 'w') as outfile: - yaml.dump(dependabot, outfile, default_flow_style=False) \ No newline at end of file +def main(): + for docker_path in get_docker_paths(): + string_path = str(docker_path) + assignees = get_owners(docker_path).get('approvers') + reviewers = get_owners(docker_path).get('reviewers') + append_updates('docker', string_path, assignees, reviewers) + + for npm_path in get_npm_paths(): + string_path = str(npm_path) + assignees = get_owners(npm_path).get('approvers') + reviewers = get_owners(npm_path).get('reviewers') + append_updates('npm', string_path, assignees, reviewers) + + for pip_path in get_pip_paths(): + string_path = str(pip_path) + assignees = get_owners(pip_path).get('approvers') + reviewers = get_owners(pip_path).get('reviewers') + append_updates('pip', string_path, assignees, reviewers) + + for go_path in get_go_paths(): + string_path = str(go_path) + assignees = get_owners(go_path).get('approvers') + reviewers = get_owners(go_path).get('reviewers') + append_updates('gomod', string_path, assignees, reviewers) + + with open('.github/dependabot.yml', 'w') as outfile: + yaml.dump(dependabot, outfile, default_flow_style=False) + + print(get_docker_paths()) + print(get_npm_paths()) + print(get_pip_paths()) + print(get_go_paths) + +if __name__ == "__main__": + main() \ No newline at end of file From f1fd7988ed2e0c7f9dc753978760d6c4e3d1462f Mon Sep 17 00:00:00 2001 From: DavidSpek Date: Fri, 22 Jan 2021 19:30:56 +0100 Subject: [PATCH 6/6] Fix last nit --- hack/create_dependabot.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/hack/create_dependabot.py b/hack/create_dependabot.py index 0119965ba..a83f04de5 100644 --- a/hack/create_dependabot.py +++ b/hack/create_dependabot.py @@ -2,10 +2,6 @@ import collections from pathlib import Path -repo_path = Path(__file__).parents[1] - -print(repo_path.anchor) - dependabot = {} dependabot['version'] = 2 dependabot['updates'] = [] @@ -27,8 +23,6 @@ def get_docker_paths(): docker_clean_list.append(dockerfile.parents[0]) return docker_clean_list -print(get_docker_paths()) - def get_npm_paths(): npm_list = list(repo_path.glob('**/package*.json')) npm_clean_list = [] @@ -99,7 +93,8 @@ def main(): print(get_docker_paths()) print(get_npm_paths()) print(get_pip_paths()) - print(get_go_paths) + print(get_go_paths()) if __name__ == "__main__": + repo_path = Path(__file__).parents[1] main() \ No newline at end of file