add dependabot config script

.github/dependabot.yml

@@ -0,0 +1,150 @@
+updates:
+- assignees:
+  - bobgy
+  - jlewi
+  directory: notebook_testing
+  open-pull-requests-limit: 10
+  package-ecosystem: docker
+  reviewers:
+  - Jeffwan
+  - pingsutw
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  directory: test-infra/auto-deploy
+  open-pull-requests-limit: 10
+  package-ecosystem: docker
+  reviewers:
+  - Jeffwan
+  - pingsutw
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  - PatrickXYS
+  directory: images
+  open-pull-requests-limit: 10
+  package-ecosystem: docker
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  directory: apps-cd
+  open-pull-requests-limit: 10
+  package-ecosystem: docker
+  reviewers:
+  - Jeffwan
+  - pingsutw
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  directory: go
+  open-pull-requests-limit: 10
+  package-ecosystem: docker
+  reviewers:
+  - Jeffwan
+  - pingsutw
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  directory: .
+  open-pull-requests-limit: 10
+  package-ecosystem: npm
+  reviewers:
+  - Jeffwan
+  - pingsutw
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  - PatrickXYS
+  directory: py/kubeflow/testing/node-license-tools
+  open-pull-requests-limit: 10
+  package-ecosystem: npm
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  directory: test-infra/auto-deploy
+  open-pull-requests-limit: 10
+  package-ecosystem: pip
+  reviewers:
+  - Jeffwan
+  - pingsutw
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  directory: apps-cd
+  open-pull-requests-limit: 10
+  package-ecosystem: pip
+  reviewers:
+  - Jeffwan
+  - pingsutw
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  directory: py
+  open-pull-requests-limit: 10
+  package-ecosystem: pip
+  reviewers:
+  - Jeffwan
+  - pingsutw
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  - PatrickXYS
+  directory: py/kubeflow/testing
+  open-pull-requests-limit: 10
+  package-ecosystem: pip
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  directory: go
+  open-pull-requests-limit: 10
+  package-ecosystem: gomod
+  reviewers:
+  - Jeffwan
+  - pingsutw
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  directory: go/cmd/nomos-wait
+  open-pull-requests-limit: 10
+  package-ecosystem: gomod
+  reviewers:
+  - Jeffwan
+  - pingsutw
+  schedule:
+    interval: daily
+- assignees:
+  - bobgy
+  - jlewi
+  directory: tests
+  open-pull-requests-limit: 10
+  package-ecosystem: gomod
+  reviewers:
+  - Jeffwan
+  - pingsutw
+  schedule:
+    interval: daily
+version: 2

Makefile

@@ -64,3 +64,6 @@ debug-rebuild-and-run:
 	make hydrate && git add . && git commit -m "Latest" && git push jlewi
 	cd ./go/cmd/nomos-wait && go run .
 	kubectl --context=kf-ci-v1 create -f ./tekton/runs/nb-test-run.yaml
+
+build-dependabot:
+	python3 hack/create_dependabot.py

README.md

@@ -42,6 +42,7 @@
     - [Step Image](#step-image)
     - [Checking out code](#checking-out-code)
     - [Building Docker Images](#building-docker-images)
+  - [Creating dependabot config yaml for this repo](#Creating-dependabot-config-yaml-for-this-repo)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
@@ -1108,3 +1109,16 @@ is
   * TAG used for the images
 
 * Argo workflow should define the image paths and tag so that subsequent steps can use the newly built images
+
+## Creating dependabot config yaml for this repo
+
+To use the most current versions and mitigate vulnerable software dependencies and base images, we configure dependabot for desired funtionality.
+
+* The way dependabot works as below:
+  1. We uses a script to scan the repository for directories containing files listing such dependencies, and matches the found folders to the relevant `OWNERS` files
+  2. Then it generate the `.github/dependabot.yml` file which tells dependabot which directories it needs to scan and for what package ecosystems.
+  3. When a dependency update is found, dependabot will create a pull request to update the dependency and assign the relevant owners and reviewers.
+
+To generate a new dependabot configuration when dependency listing files are moved or created, the script can be run by executing `make build-dependabot` from the root of this repository.
+
+More details about dependabot and its configuration can be found here (https://docs.github.com/en/github/managing-security-vulnerabilities/managing-vulnerabilities-in-your-projects-dependencies)

hack/create_dependabot.py

@@ -0,0 +1,105 @@
+import yaml
+import collections
+from pathlib import Path
+
+repo_path = Path(__file__).parents[1]
+
+print(repo_path.anchor)
+
+dependabot = {}
+dependabot['version'] = 2
+dependabot['updates'] = []
+ignored_folders = ['node_modules', 'dist', '.git', 'deprecated']
+
+def get_owners(path):
+    while not Path(path/'OWNERS').is_file():
+        path = path.parent.absolute()
+    with open(path/'OWNERS') as owner_file:
+        owners = yaml.load(owner_file)
+        return owners
+
+def get_docker_paths():
+    dockerfile_list = list(repo_path.glob('**/*ockerfile*'))
+    docker_clean_list = []
+    for dockerfile in dockerfile_list:
+        if all(x not in str(dockerfile) for x in ignored_folders):
+            if dockerfile.parents[0] not in docker_clean_list:
+                docker_clean_list.append(dockerfile.parents[0])
+    return docker_clean_list
+
+print(get_docker_paths())
+
+def get_npm_paths():
+    npm_list = list(repo_path.glob('**/package*.json'))
+    npm_clean_list = []
+    for npm_file in npm_list:
+        if all(x not in str(npm_file) for x in ignored_folders):
+            if npm_file.parents[0] not in npm_clean_list:
+                npm_clean_list.append(npm_file.parents[0])
+    return npm_clean_list
+
+def get_pip_paths():
+    pip_list = list(repo_path.glob('**/*requirements.txt'))
+    pip_clean_list = []
+    for pip_file in pip_list:
+        if all(x not in str(pip_file) for x in ignored_folders):
+            if pip_file.parents[0] not in pip_clean_list:
+                pip_clean_list.append(pip_file.parents[0])
+    return pip_clean_list
+
+def get_go_paths():
+    go_list = list(repo_path.glob('**/go.*'))
+    go_clean_list = []
+    for go_file in go_list:
+        if all(x not in str(go_file) for x in ignored_folders):
+            if go_file.parents[0] not in go_clean_list:
+                go_clean_list.append(go_file.parents[0])
+    return go_clean_list
+
+def append_updates(ecosystem, directory, assignees, reviewers=None):
+    config = {}
+    config['package-ecosystem'] = ecosystem
+    config['directory'] = directory
+    config['schedule']= {}
+    config['schedule']['interval'] = 'daily'
+    config['open-pull-requests-limit'] = 10
+    config['assignees'] = assignees
+    if reviewers:
+        config['reviewers'] = reviewers
+    dependabot['updates'].append(config)
+
+def main():
+    for docker_path in get_docker_paths():
+        string_path = str(docker_path)
+        assignees = get_owners(docker_path).get('approvers')
+        reviewers = get_owners(docker_path).get('reviewers')
+        append_updates('docker', string_path, assignees, reviewers)
+
+    for npm_path in get_npm_paths():
+        string_path = str(npm_path)
+        assignees = get_owners(npm_path).get('approvers')
+        reviewers = get_owners(npm_path).get('reviewers')
+        append_updates('npm', string_path, assignees, reviewers)
+
+    for pip_path in get_pip_paths():
+        string_path = str(pip_path)
+        assignees = get_owners(pip_path).get('approvers')
+        reviewers = get_owners(pip_path).get('reviewers')
+        append_updates('pip', string_path, assignees, reviewers)
+
+    for go_path in get_go_paths():
+        string_path = str(go_path)
+        assignees = get_owners(go_path).get('approvers')
+        reviewers = get_owners(go_path).get('reviewers')
+        append_updates('gomod', string_path, assignees, reviewers)
+
+    with open('.github/dependabot.yml', 'w') as outfile:
+        yaml.dump(dependabot, outfile, default_flow_style=False)
+
+    print(get_docker_paths())
+    print(get_npm_paths())
+    print(get_pip_paths())
+    print(get_go_paths)
+
+if __name__ == "__main__":
+    main()