From 9548df5e068911e9f77887916d6287c1afe58da3 Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Fri, 24 Aug 2018 12:28:54 -0700 Subject: [PATCH 01/42] Revert "Update cat plugin to use new JSON cat api" This reverts commit b4d4f9e0e595a8a8ce9369ba1cb05493d7346e67. --- prow/plugins/cat/cat.go | 73 +++++++++++---------- prow/plugins/cat/cat_test.go | 119 ++++++++++++++++++++++++++--------- 2 files changed, 130 insertions(+), 62 deletions(-) diff --git a/prow/plugins/cat/cat.go b/prow/plugins/cat/cat.go index 1a2f88764512..81806fb77c2b 100644 --- a/prow/plugins/cat/cat.go +++ b/prow/plugins/cat/cat.go @@ -18,7 +18,7 @@ limitations under the License. package cat import ( - "encoding/json" + "encoding/xml" "errors" "fmt" "io/ioutil" @@ -39,7 +39,7 @@ import ( var ( match = regexp.MustCompile(`(?mi)^/meow( .+)?\s*$`) meow = &realClowder{ - url: "https://api.thecatapi.com/api/images/get?format=json", + url: "http://thecatapi.com/api/images/get?format=xml&results_per_page=1", } ) @@ -61,7 +61,7 @@ func helpProvider(config *plugins.Configuration, enabledRepos []string) (*plugin Description: "Add a cat image to the issue", Featured: false, WhoCanUse: "Anyone", - Examples: []string{"/meow"}, + Examples: []string{"/meow", "/meow caturday"}, }) return pluginHelp, nil } @@ -71,7 +71,7 @@ type githubClient interface { } type clowder interface { - readCat() (string, error) + readCat(string) (string, error) } type realClowder struct { @@ -103,34 +103,44 @@ func (c *realClowder) setKey(keyPath string, log *logrus.Entry) { } type catResult struct { - Id string `json:"id"` - URL string `json:"url"` + Source string `xml:"data>images>image>source_url"` + Image string `xml:"data>images>image>url"` } func (cr catResult) Format() (string, error) { - if cr.URL == "" { + if cr.Source == "" { + return "", errors.New("empty source_url") + } + if cr.Image == "" { return "", errors.New("empty image url") } - url, err := url.Parse(cr.URL) + src, err := url.Parse(cr.Source) + if err != nil { + return "", fmt.Errorf("invalid source_url %s: %v", cr.Source, err) + } + img, err := url.Parse(cr.Image) if err != nil { - return "", fmt.Errorf("invalid image url %s: %v", cr.URL, err) + return "", fmt.Errorf("invalid image url %s: %v", cr.Image, err) } - return fmt.Sprintf("[![cat image](%s)](%s)", url, url), nil + return fmt.Sprintf("[![cat image](%s)](%s)", img, src), nil } -func (r *realClowder) Url() string { +func (r *realClowder) Url(category string) string { r.lock.RLock() defer r.lock.RUnlock() uri := string(r.url) + if category != "" { + uri += "&category=" + url.QueryEscape(category) + } if r.key != "" { uri += "&api_key=" + url.QueryEscape(r.key) } return uri } -func (r *realClowder) readCat() (string, error) { - uri := r.Url() +func (r *realClowder) readCat(category string) (string, error) { + uri := r.Url(category) resp, err := http.Get(uri) if err != nil { return "", fmt.Errorf("could not read cat from %s: %v", uri, err) @@ -139,23 +149,19 @@ func (r *realClowder) readCat() (string, error) { if sc := resp.StatusCode; sc > 299 || sc < 200 { return "", fmt.Errorf("failing %d response from %s", sc, uri) } - cats := make([]catResult, 0) - if err = json.NewDecoder(resp.Body).Decode(&cats); err != nil { + var a catResult + if err = xml.NewDecoder(resp.Body).Decode(&a); err != nil { return "", err } - if len(cats) < 1 { - return "", fmt.Errorf("no cats in response from %s", uri) - } - var a catResult = cats[0] - if a.URL == "" { + if a.Image == "" { return "", fmt.Errorf("no image url in response from %s", uri) } // checking size, GitHub doesn't support big images - toobig, err := github.ImageTooBig(a.URL) + toobig, err := github.ImageTooBig(a.Image) if err != nil { - return "", fmt.Errorf("could not validate image size %s: %v", a.URL, err) + return "", fmt.Errorf("could not validate image size %s: %v", a.Image, err) } else if toobig { - return "", fmt.Errorf("longcat is too long: %s", a.URL) + return "", fmt.Errorf("longcat is too long: %s", a.Image) } return a.Format() } @@ -184,19 +190,17 @@ func handle(gc githubClient, log *logrus.Entry, e *github.GenericCommentEvent, c // Now that we know this is a relevant event we can set the key. setKey() + category := mat[1] + if len(category) > 1 { + category = category[1:] + } + org := e.Repo.Owner.Login repo := e.Repo.Name number := e.Number - // Provide an error message when a category is requested - // TODO: remove after a grace period - if len(strings.TrimSpace(mat[1])) > 0 { - msg := "**chomp** CATegories are no longer supported" - return gc.CreateComment(org, repo, number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, msg)) - } - for i := 0; i < 3; i++ { - resp, err := c.readCat() + resp, err := c.readCat(category) if err != nil { log.WithError(err).Error("Failed to get cat img") continue @@ -204,7 +208,12 @@ func handle(gc githubClient, log *logrus.Entry, e *github.GenericCommentEvent, c return gc.CreateComment(org, repo, number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, resp)) } - msg := "https://thecatapi.com appears to be down" + var msg string + if category != "" { + msg = "Bad category. Please see http://thecatapi.com/api/categories/list" + } else { + msg = "http://thecatapi.com appears to be down" + } if err := gc.CreateComment(org, repo, number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, msg)); err != nil { log.WithError(err).Error("Failed to leave comment") } diff --git a/prow/plugins/cat/cat_test.go b/prow/plugins/cat/cat_test.go index f1eef4a1ddf2..6e6020af0a6b 100644 --- a/prow/plugins/cat/cat_test.go +++ b/prow/plugins/cat/cat_test.go @@ -17,6 +17,7 @@ limitations under the License. package cat import ( + "errors" "flag" "fmt" "io" @@ -35,21 +36,25 @@ import ( type fakeClowder string var human = flag.Bool("human", false, "Enable to run additional manual tests") +var category = flag.String("category", "", "Request a particular category if set") var path = flag.String("key-path", "", "Path to api key if set") -func (c fakeClowder) readCat() (string, error) { +func (c fakeClowder) readCat(category string) (string, error) { + if category == "error" { + return "", errors.New(string(c)) + } return fmt.Sprintf("![fake cat image](%s)", c), nil } func TestRealCat(t *testing.T) { if !*human { - t.Skip("Real cats disabled for automation. Manual users can add --human") + t.Skip("Real cats disabled for automation. Manual users can add --human [--category=foo]") } if *path != "" { meow.setKey(*path, logrus.WithField("plugin", pluginName)) } - if cat, err := meow.readCat(); err != nil { + if cat, err := meow.readCat(*category); err != nil { t.Errorf("Could not read cats from %#v: %v", meow, err) } else { fmt.Println(cat) @@ -58,10 +63,12 @@ func TestRealCat(t *testing.T) { func TestUrl(t *testing.T) { cases := []struct { - name string - url string - key string - require []string + name string + url string + category string + key string + require []string + deny []string }{ { name: "only url", @@ -72,6 +79,21 @@ func TestUrl(t *testing.T) { url: "http://foo", key: "blah", require: []string{"api_key=blah"}, + deny: []string{"category="}, + }, + { + name: "category", + url: "http://foo", + category: "bar", + require: []string{"category=bar"}, + deny: []string{"api_key="}, + }, + { + name: "category and key", + url: "http://foo", + category: "this", + key: "that", + require: []string{"category=this", "api_key=that", "&"}, }, } @@ -80,12 +102,17 @@ func TestUrl(t *testing.T) { url: tc.url, key: tc.key, } - url := rc.Url() + url := rc.Url(tc.category) for _, r := range tc.require { if !strings.Contains(url, r) { t.Errorf("%s: %s does not contain %s", tc.name, url, r) } } + for _, d := range tc.deny { + if strings.Contains(url, d) { + t.Errorf("%s: %s contained unexpected %s", tc.name, url, d) + } + } } } @@ -94,29 +121,45 @@ func TestFormat(t *testing.T) { basicURL := "http://example.com" testcases := []struct { name string - url string + src string + img string err bool }{ { name: "basically works", - url: basicURL, + src: basicURL, + img: basicURL, err: false, }, + { + name: "empty source", + src: "", + img: basicURL, + err: true, + }, { name: "empty image", - url: "", + src: basicURL, + img: "", + err: true, + }, + { + name: "bad source", + src: "http://this is not a url", + img: basicURL, err: true, }, { name: "bad image", - url: "http://still a bad url", + src: basicURL, + img: "http://still a bad url", err: true, }, } for _, tc := range testcases { ret, err := catResult{ - Id: tc.name, - URL: tc.url, + Source: tc.src, + Image: tc.img, }.Format() switch { case tc.err: @@ -153,7 +196,8 @@ func TestHttpResponse(t *testing.T) { // create test cases for handling http responses img := ts2.URL + "/cat.jpg" bigimg := ts2.URL + "/bigcat.jpg" - validResponse := fmt.Sprintf(`[{"id":"valid","url":"%s"}]`, img) + src := "http://localhost?kind=source_url" + validResponse := fmt.Sprintf(`%s%s`, img, src) var testcases = []struct { name string path string @@ -164,13 +208,13 @@ func TestHttpResponse(t *testing.T) { { name: "valid", path: "/valid", - response: validResponse, + response: fmt.Sprintf(`%s%s`, img, src), valid: true, }, { name: "image too big", path: "/too-big", - response: fmt.Sprintf(`[{"id":"toobig","url":"%s"}]`, bigimg), + response: fmt.Sprintf(`%s%s`, bigimg, src), }, { name: "return-406", @@ -190,9 +234,9 @@ Available variants: `, }, { - name: "no-cats-in-json", - path: "/no-cats-in-json", - response: "[]", + name: "no-image-in-xml", + path: "/no-image-in-xml", + response: "", }, } @@ -226,7 +270,7 @@ Available variants: // run test for each case for _, testcase := range testcases { fakemeow := &realClowder{url: ts.URL + testcase.path} - cat, err := fakemeow.readCat() + cat, err := fakemeow.readCat(*category) if testcase.valid && err != nil { t.Errorf("For case %s, didn't expect error: %v", testcase.name, err) } else if !testcase.valid && err == nil { @@ -255,7 +299,10 @@ Available variants: } if c := fc.IssueComments[5][0]; !strings.Contains(c.Body, img) { t.Errorf("missing image url: %s from comment: %v", img, c) + } else if !strings.Contains(c.Body, src) { + t.Errorf("missing source url: %s from comment: %v", src, c) } + } // Small, unit tests @@ -267,7 +314,7 @@ func TestCats(t *testing.T) { state string pr bool shouldComment bool - shouldImage bool + shouldError bool }{ { name: "ignore edited comment", @@ -275,7 +322,7 @@ func TestCats(t *testing.T) { action: github.GenericCommentActionEdited, body: "/meow", shouldComment: false, - shouldImage: true, + shouldError: false, }, { name: "leave cat on pr", @@ -284,7 +331,7 @@ func TestCats(t *testing.T) { body: "/meow", pr: true, shouldComment: true, - shouldImage: true, + shouldError: false, }, { name: "leave cat on issue", @@ -292,7 +339,7 @@ func TestCats(t *testing.T) { action: github.GenericCommentActionCreated, body: "/meow", shouldComment: true, - shouldImage: true, + shouldError: false, }, { name: "leave cat on issue, trailing space", @@ -300,7 +347,7 @@ func TestCats(t *testing.T) { action: github.GenericCommentActionCreated, body: "/meow \r", shouldComment: true, - shouldImage: true, + shouldError: false, }, { name: "categorical cat", @@ -308,7 +355,15 @@ func TestCats(t *testing.T) { action: github.GenericCommentActionCreated, body: "/meow clothes", shouldComment: true, - shouldImage: false, + shouldError: false, + }, + { + name: "bad cat", + state: "open", + action: github.GenericCommentActionCreated, + body: "/meow error", + shouldComment: true, + shouldError: true, }, } for _, tc := range testcases { @@ -323,18 +378,22 @@ func TestCats(t *testing.T) { IsPR: tc.pr, } err := handle(fc, logrus.WithField("plugin", pluginName), e, fakeClowder("tubbs"), func() {}) - if err != nil { + if !tc.shouldError && err != nil { t.Errorf("%s: didn't expect error: %v", tc.name, err) continue + } else if tc.shouldError && err == nil { + t.Errorf("%s: expected an error to occur", tc.name) + continue } if tc.shouldComment && len(fc.IssueComments[5]) != 1 { t.Errorf("%s: should have commented.", tc.name) } else if tc.shouldComment { + shouldImage := !tc.shouldError body := fc.IssueComments[5][0].Body hasImage := strings.Contains(body, "![") - if hasImage && !tc.shouldImage { + if hasImage && !shouldImage { t.Errorf("%s: unexpected image in %s", tc.name, body) - } else if !hasImage && tc.shouldImage { + } else if !hasImage && shouldImage { t.Errorf("%s: no image in %s", tc.name, body) } } else if !tc.shouldComment && len(fc.IssueComments[5]) != 0 { From b47409abd9aa2597f6916f57a88bf0666e5ab97d Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Fri, 24 Aug 2018 12:42:56 -0700 Subject: [PATCH 02/42] Use https and use JSON instead of XML --- prow/plugins/cat/cat.go | 20 ++++++++++++-------- prow/plugins/cat/cat_test.go | 17 +++++++++++------ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/prow/plugins/cat/cat.go b/prow/plugins/cat/cat.go index 81806fb77c2b..5dec1da607af 100644 --- a/prow/plugins/cat/cat.go +++ b/prow/plugins/cat/cat.go @@ -18,7 +18,7 @@ limitations under the License. package cat import ( - "encoding/xml" + "encoding/json" "errors" "fmt" "io/ioutil" @@ -39,7 +39,7 @@ import ( var ( match = regexp.MustCompile(`(?mi)^/meow( .+)?\s*$`) meow = &realClowder{ - url: "http://thecatapi.com/api/images/get?format=xml&results_per_page=1", + url: "https://api.thecatapi.com/api/images/get?format=json&results_per_page=1", } ) @@ -103,8 +103,8 @@ func (c *realClowder) setKey(keyPath string, log *logrus.Entry) { } type catResult struct { - Source string `xml:"data>images>image>source_url"` - Image string `xml:"data>images>image>url"` + Source string `json:"source_url"` + Image string `json:"url"` } func (cr catResult) Format() (string, error) { @@ -149,10 +149,14 @@ func (r *realClowder) readCat(category string) (string, error) { if sc := resp.StatusCode; sc > 299 || sc < 200 { return "", fmt.Errorf("failing %d response from %s", sc, uri) } - var a catResult - if err = xml.NewDecoder(resp.Body).Decode(&a); err != nil { + cats := make([]catResult, 0) + if err = json.NewDecoder(resp.Body).Decode(&cats); err != nil { return "", err } + if len(cats) < 1 { + return "", fmt.Errorf("no cats in response from %s", uri) + } + a := cats[0] if a.Image == "" { return "", fmt.Errorf("no image url in response from %s", uri) } @@ -210,9 +214,9 @@ func handle(gc githubClient, log *logrus.Entry, e *github.GenericCommentEvent, c var msg string if category != "" { - msg = "Bad category. Please see http://thecatapi.com/api/categories/list" + msg = "Bad category. Please see https://api.thecatapi.com/api/categories/list" } else { - msg = "http://thecatapi.com appears to be down" + msg = "https://thecatapi.com appears to be down" } if err := gc.CreateComment(org, repo, number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, msg)); err != nil { log.WithError(err).Error("Failed to leave comment") diff --git a/prow/plugins/cat/cat_test.go b/prow/plugins/cat/cat_test.go index 6e6020af0a6b..ed93c4c6d6f5 100644 --- a/prow/plugins/cat/cat_test.go +++ b/prow/plugins/cat/cat_test.go @@ -197,7 +197,7 @@ func TestHttpResponse(t *testing.T) { img := ts2.URL + "/cat.jpg" bigimg := ts2.URL + "/bigcat.jpg" src := "http://localhost?kind=source_url" - validResponse := fmt.Sprintf(`%s%s`, img, src) + validResponse := fmt.Sprintf(`[{"id":"valid","url":"%s","source_url":"%s"}]`, img, src) var testcases = []struct { name string path string @@ -208,13 +208,13 @@ func TestHttpResponse(t *testing.T) { { name: "valid", path: "/valid", - response: fmt.Sprintf(`%s%s`, img, src), + response: validResponse, valid: true, }, { name: "image too big", path: "/too-big", - response: fmt.Sprintf(`%s%s`, bigimg, src), + response: fmt.Sprintf(`[{"id":"toobig","url":"%s","source_url":"%s"}]`, bigimg, src), }, { name: "return-406", @@ -234,9 +234,14 @@ Available variants: `, }, { - name: "no-image-in-xml", - path: "/no-image-in-xml", - response: "", + name: "no-cats-in-json", + path: "/no-cats-in-json", + response: "[]", + }, + { + name: "no-image-in-json", + path: "/no-image-in-json", + response: "[{}]", }, } From 4e378958c49d3e1d5c6b37e6962a610536a48099 Mon Sep 17 00:00:00 2001 From: Sen Lu Date: Wed, 29 Aug 2018 17:14:11 -0700 Subject: [PATCH 03/42] bulk move all other presubmit jobs --- .../k8s-multicluster-ingress/OWNERS | 2 + .../k8s-multicluster-ingress-config.yaml | 118 +++++++++ config/jobs/kubernetes-sigs/poseidon/OWNERS | 5 + .../poseidon/poseidon-config.yaml | 76 ++++++ .../kubernetes-sigs/testing_frameworks/OWNERS | 4 + .../testing-frameworks-config.yaml | 19 ++ .../jobs/kubernetes/cluster-registry/OWNERS | 5 + .../cluster-registry-config.yaml | 49 ++++ config/jobs/kubernetes/community/OWNERS | 8 + .../community/community-presubmit.yaml | 19 ++ config/jobs/kubernetes/heapster/OWNERS | 9 + .../kubernetes/heapster/heapster-config.yaml | 34 +++ config/tests/jobs/jobs_test.go | 4 +- jobs/config.json | 156 ------------ jobs/config_test.py | 6 +- prow/config.yaml | 234 ------------------ 16 files changed, 352 insertions(+), 396 deletions(-) create mode 100644 config/jobs/GoogleCloudPlatform/k8s-multicluster-ingress/OWNERS create mode 100644 config/jobs/GoogleCloudPlatform/k8s-multicluster-ingress/k8s-multicluster-ingress-config.yaml create mode 100644 config/jobs/kubernetes-sigs/poseidon/OWNERS create mode 100644 config/jobs/kubernetes-sigs/poseidon/poseidon-config.yaml create mode 100644 config/jobs/kubernetes-sigs/testing_frameworks/OWNERS create mode 100644 config/jobs/kubernetes-sigs/testing_frameworks/testing-frameworks-config.yaml create mode 100644 config/jobs/kubernetes/cluster-registry/OWNERS create mode 100644 config/jobs/kubernetes/cluster-registry/cluster-registry-config.yaml create mode 100644 config/jobs/kubernetes/community/OWNERS create mode 100644 config/jobs/kubernetes/community/community-presubmit.yaml create mode 100644 config/jobs/kubernetes/heapster/OWNERS create mode 100644 config/jobs/kubernetes/heapster/heapster-config.yaml diff --git a/config/jobs/GoogleCloudPlatform/k8s-multicluster-ingress/OWNERS b/config/jobs/GoogleCloudPlatform/k8s-multicluster-ingress/OWNERS new file mode 100644 index 000000000000..5cfdaa0c8de9 --- /dev/null +++ b/config/jobs/GoogleCloudPlatform/k8s-multicluster-ingress/OWNERS @@ -0,0 +1,2 @@ +approvers: +- nikhiljindal diff --git a/config/jobs/GoogleCloudPlatform/k8s-multicluster-ingress/k8s-multicluster-ingress-config.yaml b/config/jobs/GoogleCloudPlatform/k8s-multicluster-ingress/k8s-multicluster-ingress-config.yaml new file mode 100644 index 000000000000..4f27e41bb5d0 --- /dev/null +++ b/config/jobs/GoogleCloudPlatform/k8s-multicluster-ingress/k8s-multicluster-ingress-config.yaml @@ -0,0 +1,118 @@ +presubmits: + GoogleCloudPlatform/k8s-multicluster-ingress: + - name: pull-kubernetes-multicluster-ingress-test + branches: + - master + always_run: true + labels: + preset-service-account: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - "--repo=github.com/GoogleCloudPlatform/$(REPO_NAME)=$(PULL_REFS)" + - "--root=/go/src" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - --scenario=execute + - -- + - make + - -- + - -j + - "5" + - test + - coveralls + - vet + - fmt + - lint + volumeMounts: + - name: coveralls + mountPath: /etc/coveralls-token + readOnly: true + volumes: + - name: coveralls + secret: + secretName: k8s-multicluster-ingress-coveralls-token + +postsubmits: + GoogleCloudPlatform/k8s-multicluster-ingress: + - name: ci-kubemci-image-push + branches: + - master + labels: + preset-service-account: "true" + preset-dind-enabled: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - "--repo=k8s.io/test-infra=master" + - "--root=/go/src/" + - "--timeout=90" + - --scenario=execute + - -- + - make + - -- + - -C + - images/kubemci + - push-latest + # docker-in-docker needs privileged mode + securityContext: + privileged: true + +periodics: +- interval: 6h + name: ci-kubernetes-multicluster-ingress-test + labels: + preset-service-account: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - --repo=github.com/GoogleCloudPlatform/k8s-multicluster-ingress=master + - --root=/go/src + - --timeout=50 + - --service-account=/etc/service-account/service-account.json + - --upload=gs://kubernetes-jenkins/logs + - --scenario=execute + - -- + - make + - -- + - -j + - "5" + - test + - coveralls + - vet + - fmt + - lint + volumeMounts: + - name: coveralls + mountPath: /etc/coveralls-token + readOnly: true + volumes: + - name: coveralls + secret: + secretName: k8s-multicluster-ingress-coveralls-token + +- interval: 60m + name: ci-kubemci-ingress-conformance + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=110 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --env=GCE_GLBC_IMAGE=gcr.io/k8s-ingress-image-push/ingress-gce-glbc-amd64:master + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-project-type=ingress-project + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.focus=\[Feature:kubemci\] --minStartupPods=8 + - --timeout=90m + image: gcr.io/k8s-testimages/e2e-kubemci:latest + imagePullPolicy: Always diff --git a/config/jobs/kubernetes-sigs/poseidon/OWNERS b/config/jobs/kubernetes-sigs/poseidon/OWNERS new file mode 100644 index 000000000000..490fb779f7c0 --- /dev/null +++ b/config/jobs/kubernetes-sigs/poseidon/OWNERS @@ -0,0 +1,5 @@ +approvers: +- ICGog +- ms705 +- shivramsrivastava +- m1093782566 diff --git a/config/jobs/kubernetes-sigs/poseidon/poseidon-config.yaml b/config/jobs/kubernetes-sigs/poseidon/poseidon-config.yaml new file mode 100644 index 000000000000..d634db650741 --- /dev/null +++ b/config/jobs/kubernetes-sigs/poseidon/poseidon-config.yaml @@ -0,0 +1,76 @@ +presubmits: + kubernetes-sigs/poseidon: + - name: ci-poseidon-e2e-gce + always_run: true + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-dind-enabled: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - "--job=$(JOB_NAME)" + - "--repo=github.com/$(REPO_OWNER)/$(REPO_NAME)=$(PULL_REFS)" + - "--root=/go/src" + - "--service-account=/etc/service-account/service-account.json" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - "--timeout=110" + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --cluster= + - --extract=ci/latest + - --gcp-node-image=ubuntu + - --provider=gce + - --test=false + - --test-cmd=../test/e2e-poseidon-gce.sh + - --timeout=60m + securityContext: + privileged: true + - name: pull-poseidon-bazel + always_run: true + labels: + preset-service-account: "true" + preset-bazel-scratch-dir: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-1.10 + args: + - "--job=$(JOB_NAME)" + - "--repo=github.com/$(REPO_OWNER)/$(REPO_NAME)=$(PULL_REFS)" + - "--service-account=/etc/service-account/service-account.json" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - "--timeout=45" + - --scenario=kubernetes_bazel + - -- + - --build=//cmd/poseidon + - --test=//... -//hack/... -//vendor/... -//test/e2e/... + - --test-args=--build_tag_filters=-e2e + - --test-args=--test_tag_filters=-e2e + - --test-args=--flaky_test_attempts=3 + # Bazel needs privileged mode in order to sandbox builds. + securityContext: + privileged: true + resources: + requests: + memory: "2Gi" + - name: pull-poseidon-verify + branches: + - master + always_run: true + labels: + preset-service-account: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/gcloud-in-go:v20171113-192bec25 + args: + - "--repo=github.com/$(REPO_OWNER)/$(REPO_NAME)=$(PULL_REFS)" + - "--root=/go/src" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - "--clean" + - "--timeout=45" + - --scenario=execute + - -- + - make + - verify diff --git a/config/jobs/kubernetes-sigs/testing_frameworks/OWNERS b/config/jobs/kubernetes-sigs/testing_frameworks/OWNERS new file mode 100644 index 000000000000..49795b03e41c --- /dev/null +++ b/config/jobs/kubernetes-sigs/testing_frameworks/OWNERS @@ -0,0 +1,4 @@ +approvers: +- apelisse +- hoegaarden +- totherme diff --git a/config/jobs/kubernetes-sigs/testing_frameworks/testing-frameworks-config.yaml b/config/jobs/kubernetes-sigs/testing_frameworks/testing-frameworks-config.yaml new file mode 100644 index 000000000000..bf2104bdc30a --- /dev/null +++ b/config/jobs/kubernetes-sigs/testing_frameworks/testing-frameworks-config.yaml @@ -0,0 +1,19 @@ +presubmits: + kubernetes-sigs/testing_frameworks: + - name: pull-frameworks-test + branches: + - master + always_run: true + skip_report: false + labels: + preset-service-account: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/gcloud-in-go:v20171113-192bec25 + args: + - "--repo=sigs.k8s.io/$(REPO_NAME)=$(PULL_REFS)" + - "--root=/go/src" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - --scenario=execute + - -- + - ./bin/test-on-prow.sh diff --git a/config/jobs/kubernetes/cluster-registry/OWNERS b/config/jobs/kubernetes/cluster-registry/OWNERS new file mode 100644 index 000000000000..f4259084b821 --- /dev/null +++ b/config/jobs/kubernetes/cluster-registry/OWNERS @@ -0,0 +1,5 @@ +approvers: +- font +- madhusudancs +- perotinus +- pmorie diff --git a/config/jobs/kubernetes/cluster-registry/cluster-registry-config.yaml b/config/jobs/kubernetes/cluster-registry/cluster-registry-config.yaml new file mode 100644 index 000000000000..b2115f0e5399 --- /dev/null +++ b/config/jobs/kubernetes/cluster-registry/cluster-registry-config.yaml @@ -0,0 +1,49 @@ +presubmits: + kubernetes/cluster-registry: + - name: pull-cluster-registry-verify-gensrc + always_run: true + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-bazel-scratch-dir: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - "--job=$(JOB_NAME)" + - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" + - "--service-account=/etc/service-account/service-account.json" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - --scenario=execute + - -- + - ./hack/verify-all-gensrc.sh + # Bazel needs privileged mode in order to sandbox builds. + securityContext: + privileged: true + resources: + requests: + memory: "2Gi" + - name: pull-cluster-registry-verify-gosrc + always_run: true + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-1.8 + args: + - "--job=$(JOB_NAME)" + - "--root=/go/src" + - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" + - "--service-account=/etc/service-account/service-account.json" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - --scenario=execute + - -- + - ./hack/verify-go-src.sh + env: + # Bazel needs privileged mode in order to sandbox builds. + securityContext: + privileged: true + resources: + requests: + memory: "2Gi" diff --git a/config/jobs/kubernetes/community/OWNERS b/config/jobs/kubernetes/community/OWNERS new file mode 100644 index 000000000000..2a8a92f063d4 --- /dev/null +++ b/config/jobs/kubernetes/community/OWNERS @@ -0,0 +1,8 @@ +approvers: +- calebamiles +- castrojo +- cblecker +- grodrigues3 +- idvoretskyi +- jdumars +- parispittman diff --git a/config/jobs/kubernetes/community/community-presubmit.yaml b/config/jobs/kubernetes/community/community-presubmit.yaml new file mode 100644 index 000000000000..f438cf696e9c --- /dev/null +++ b/config/jobs/kubernetes/community/community-presubmit.yaml @@ -0,0 +1,19 @@ +presubmits: + kubernetes/community: + - name: pull-community-verify + branches: + - master + always_run: true + labels: + preset-service-account: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/gcloud-in-go:v20171113-192bec25 + args: + - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" + - "--root=/go/src" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - --scenario=execute + - -- + - make + - verify diff --git a/config/jobs/kubernetes/heapster/OWNERS b/config/jobs/kubernetes/heapster/OWNERS new file mode 100644 index 000000000000..59e898f89318 --- /dev/null +++ b/config/jobs/kubernetes/heapster/OWNERS @@ -0,0 +1,9 @@ +approvers: +- DirectXMan12 +- piosz +- x13n +- kawych +- loburm +- mwieglus +- huangyuqi +- andyxning diff --git a/config/jobs/kubernetes/heapster/heapster-config.yaml b/config/jobs/kubernetes/heapster/heapster-config.yaml new file mode 100644 index 000000000000..2908c0d988d4 --- /dev/null +++ b/config/jobs/kubernetes/heapster/heapster-config.yaml @@ -0,0 +1,34 @@ +presubmits: + kubernetes/heapster: + - name: pull-heapster-e2e + always_run: true + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-dind-enabled: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" + - "--root=/go/src" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - --scenario=kubernetes_heapster + securityContext: + privileged: true + - name: pull-heapster-unit + always_run: true + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" + - "--root=/go/src" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - --scenario=execute + - -- + - make + - test-unit diff --git a/config/tests/jobs/jobs_test.go b/config/tests/jobs/jobs_test.go index fe6d301093e1..9fe99cec6056 100644 --- a/config/tests/jobs/jobs_test.go +++ b/config/tests/jobs/jobs_test.go @@ -789,7 +789,9 @@ func checkScenarioArgs(jobName, imageName string, args []string) error { } if !scenarioArgs { - return fmt.Errorf("job %s: set --scenario and will need scenario args", jobName) + if scenario != "kubernetes_heapster" { // this scenario does not have any args + return fmt.Errorf("job %s: set --scenario=%s and will need scenario args", jobName, scenario) + } } } diff --git a/jobs/config.json b/jobs/config.json index 1c51b59ec0c8..5fc403856529 100644 --- a/jobs/config.json +++ b/jobs/config.json @@ -67,36 +67,6 @@ "sig-node" ] }, - "ci-kubemci-image-push": { - "args": [ - "make", - "--", - "-C", - "images/kubemci", - "push-latest" - ], - "scenario": "execute", - "sigOwners": [ - "sig-multicluster" - ] - }, - "ci-kubemci-ingress-conformance": { - "args": [ - "--check-leaked-resources", - "--env=GCE_GLBC_IMAGE=gcr.io/k8s-ingress-image-push/ingress-gce-glbc-amd64:master", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-project-type=ingress-project", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:kubemci\\] --minStartupPods=8", - "--timeout=90m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-multicluster" - ] - }, "ci-kubernetes-e2e-autoscaling-vpa-actuation": { "args": [ "--check-leaked-resources", @@ -647,23 +617,6 @@ "sig-instrumentation" ] }, - "ci-kubernetes-multicluster-ingress-test": { - "args": [ - "make", - "--", - "-j", - "5", - "test", - "coveralls", - "vet", - "fmt", - "lint" - ], - "scenario": "execute", - "sigOwners": [ - "sig-multicluster" - ] - }, "ci-kustomize-periodic-default-gke": { "args": [ "--check-leaked-resources", @@ -684,22 +637,6 @@ "sig-cli" ] }, - "ci-poseidon-e2e-gce": { - "args": [ - "--check-leaked-resources", - "--cluster=", - "--extract=ci/latest", - "--gcp-node-image=ubuntu", - "--provider=gce", - "--test=false", - "--test-cmd=../test/e2e-poseidon-gce.sh", - "--timeout=60m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-scheduling" - ] - }, "fake-branch": { "args": [ "./fake/fake-branch.sh" @@ -727,34 +664,6 @@ "sig-testing" ] }, - "pull-cluster-registry-verify-gensrc": { - "args": [ - "./hack/verify-all-gensrc.sh" - ], - "scenario": "execute", - "sigOwners": [ - "sig-multicluster" - ] - }, - "pull-cluster-registry-verify-gosrc": { - "args": [ - "./hack/verify-go-src.sh" - ], - "scenario": "execute", - "sigOwners": [ - "sig-multicluster" - ] - }, - "pull-community-verify": { - "args": [ - "make", - "verify" - ], - "scenario": "execute", - "sigOwners": [ - "sig-contributor-experience" - ] - }, "pull-cri-containerd-build": { "args": [ "./test/build.sh" @@ -789,48 +698,6 @@ "sig-node" ] }, - "pull-frameworks-test": { - "args": [ - "./bin/test-on-prow.sh" - ], - "scenario": "execute", - "sigOwners": [ - "sig-testing" - ] - }, - "pull-heapster-e2e": { - "scenario": "kubernetes_heapster", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "pull-heapster-unit": { - "args": [ - "make", - "test-unit" - ], - "scenario": "execute", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "pull-kubernetes-multicluster-ingress-test": { - "args": [ - "make", - "--", - "-j", - "5", - "test", - "coveralls", - "vet", - "fmt", - "lint" - ], - "scenario": "execute", - "sigOwners": [ - "sig-multicluster" - ] - }, "pull-kubernetes-node-e2e": { "args": [ "--deployment=node", @@ -863,29 +730,6 @@ "sig-node" ] }, - "pull-poseidon-bazel": { - "args": [ - "--build=//cmd/poseidon", - "--test=//... -//hack/... -//vendor/... -//test/e2e/...", - "--test-args=--build_tag_filters=-e2e", - "--test-args=--test_tag_filters=-e2e", - "--test-args=--flaky_test_attempts=3" - ], - "scenario": "kubernetes_bazel", - "sigOwners": [ - "sig-scheduling" - ] - }, - "pull-poseidon-verify": { - "args": [ - "make", - "verify" - ], - "scenario": "execute", - "sigOwners": [ - "sig-scheduling" - ] - }, "random_job": { "scenario": "execute", "sigOwners": [ diff --git a/jobs/config_test.py b/jobs/config_test.py index ae9dcd119130..a0664506d746 100755 --- a/jobs/config_test.py +++ b/jobs/config_test.py @@ -143,13 +143,9 @@ def load_prow_yaml(self, path): for item in doc.get('periodics'): self.add_prow_job(item) - if 'postsubmits' not in doc: - self.fail('No postsubmits in prow config!') - self.presubmits = doc.get('presubmits') - postsubmits = doc.get('postsubmits') - for _repo, joblist in self.presubmits.items() + postsubmits.items(): + for _repo, joblist in self.presubmits.items(): for job in joblist: self.add_prow_job(job) diff --git a/prow/config.yaml b/prow/config.yaml index 0d845159fee1..da989f7bf5c2 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -517,115 +517,6 @@ presets: mountPath: /docker-graph presubmits: - kubernetes/community: - - name: pull-community-verify - branches: - - master - always_run: true - labels: - preset-service-account: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/gcloud-in-go:v20171113-192bec25 - args: - - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" - - "--root=/go/src" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - GoogleCloudPlatform/k8s-multicluster-ingress: - - name: pull-kubernetes-multicluster-ingress-test - branches: - - master - always_run: true - labels: - preset-service-account: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--repo=github.com/GoogleCloudPlatform/$(REPO_NAME)=$(PULL_REFS)" - - "--root=/go/src" - - "--upload=gs://kubernetes-jenkins/pr-logs" - volumeMounts: - - name: coveralls - mountPath: /etc/coveralls-token - readOnly: true - volumes: - - name: coveralls - secret: - secretName: k8s-multicluster-ingress-coveralls-token - - kubernetes/heapster: - - name: pull-heapster-e2e - always_run: true - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-dind-enabled: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" - - "--root=/go/src" - - "--upload=gs://kubernetes-jenkins/pr-logs" - securityContext: - privileged: true - - name: pull-heapster-unit - always_run: true - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" - - "--root=/go/src" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - kubernetes/cluster-registry: - - name: pull-cluster-registry-verify-gensrc - always_run: true - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-bazel-scratch-dir: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--job=$(JOB_NAME)" - - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" - - "--service-account=/etc/service-account/service-account.json" - - "--upload=gs://kubernetes-jenkins/pr-logs" - # Bazel needs privileged mode in order to sandbox builds. - securityContext: - privileged: true - resources: - requests: - memory: "2Gi" - - name: pull-cluster-registry-verify-gosrc - always_run: true - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-1.8 - args: - - "--job=$(JOB_NAME)" - - "--root=/go/src" - - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" - - "--service-account=/etc/service-account/service-account.json" - - "--upload=gs://kubernetes-jenkins/pr-logs" - env: - # Bazel needs privileged mode in order to sandbox builds. - securityContext: - privileged: true - resources: - requests: - memory: "2Gi" containerd/cri: - name: pull-cri-containerd-build always_run: true @@ -800,96 +691,6 @@ presubmits: requests: memory: "6Gi" - kubernetes-sigs/testing_frameworks: - - name: pull-frameworks-test - branches: - - master - always_run: true - skip_report: false - labels: - preset-service-account: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/gcloud-in-go:v20171113-192bec25 - args: - - "--repo=sigs.k8s.io/$(REPO_NAME)=$(PULL_REFS)" - - "--root=/go/src" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - kubernetes-sigs/poseidon: - - name: ci-poseidon-e2e-gce - always_run: true - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-dind-enabled: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--job=$(JOB_NAME)" - - "--repo=github.com/$(REPO_OWNER)/$(REPO_NAME)=$(PULL_REFS)" - - "--root=/go/src" - - "--service-account=/etc/service-account/service-account.json" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - "--timeout=110" - securityContext: - privileged: true - - name: pull-poseidon-bazel - always_run: true - labels: - preset-service-account: "true" - preset-bazel-scratch-dir: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-1.10 - args: - - "--job=$(JOB_NAME)" - - "--repo=github.com/$(REPO_OWNER)/$(REPO_NAME)=$(PULL_REFS)" - - "--service-account=/etc/service-account/service-account.json" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - "--timeout=45" - # Bazel needs privileged mode in order to sandbox builds. - securityContext: - privileged: true - resources: - requests: - memory: "2Gi" - - name: pull-poseidon-verify - branches: - - master - always_run: true - labels: - preset-service-account: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/gcloud-in-go:v20171113-192bec25 - args: - - "--repo=github.com/$(REPO_OWNER)/$(REPO_NAME)=$(PULL_REFS)" - - "--root=/go/src" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - "--clean" - - "--timeout=45" - -postsubmits: - GoogleCloudPlatform/k8s-multicluster-ingress: - - name: ci-kubemci-image-push - branches: - - master - labels: - preset-service-account: "true" - preset-dind-enabled: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--repo=k8s.io/test-infra=master" - - "--root=/go/src/" - - "--timeout=90" - # docker-in-docker needs privileged mode - securityContext: - privileged: true - periodics: - interval: 2h name: application-periodic-default-gke @@ -961,19 +762,6 @@ periodics: - name: GOPATH value: /go -- interval: 60m - name: ci-kubemci-ingress-conformance - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=110 - - --bare - image: gcr.io/k8s-testimages/e2e-kubemci:latest - imagePullPolicy: Always - - name: ci-kubernetes-e2e-autoscaling-vpa-actuation interval: 2h labels: @@ -1384,28 +1172,6 @@ periodics: - --bare image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master -- interval: 6h - name: ci-kubernetes-multicluster-ingress-test - labels: - preset-service-account: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - --repo=github.com/GoogleCloudPlatform/k8s-multicluster-ingress=master - - --root=/go/src - - --timeout=50 - - --service-account=/etc/service-account/service-account.json - - --upload=gs://kubernetes-jenkins/logs - volumeMounts: - - name: coveralls - mountPath: /etc/coveralls-token - readOnly: true - volumes: - - name: coveralls - secret: - secretName: k8s-multicluster-ingress-coveralls-token - - interval: 2h name: ci-kustomize-periodic-default-gke labels: From 18c6766d003c85a3ad49528b6c6e3fd2850bb351 Mon Sep 17 00:00:00 2001 From: James Munnelly Date: Thu, 30 Aug 2018 14:30:25 +0100 Subject: [PATCH 04/42] Explicitly set goarch and goos on go_image targets --- ghproxy/BUILD.bazel | 3 +++ prow/cmd/artifact-uploader/BUILD.bazel | 12 +++++++++++ prow/cmd/branchprotector/BUILD.bazel | 8 ++++++-- prow/cmd/build/BUILD.bazel | 8 ++++++-- prow/cmd/clonerefs/BUILD.bazel | 15 ++++++++++++-- prow/cmd/deck/BUILD.bazel | 6 +++++- prow/cmd/entrypoint/BUILD.bazel | 28 ++++++++++++++++++-------- prow/cmd/gcsupload/BUILD.bazel | 12 +++++++++++ prow/cmd/gerrit/BUILD.bazel | 17 ++++++++++++---- prow/cmd/hook/BUILD.bazel | 6 +++++- prow/cmd/horologium/BUILD.bazel | 6 +++++- prow/cmd/initupload/BUILD.bazel | 16 ++++++++++++--- prow/cmd/jenkins-operator/BUILD.bazel | 6 +++++- prow/cmd/peribolos/BUILD.bazel | 8 ++++++-- prow/cmd/plank/BUILD.bazel | 5 ++++- prow/cmd/sidecar/BUILD.bazel | 16 ++++++++++++--- prow/cmd/sinker/BUILD.bazel | 6 +++++- prow/cmd/splice/BUILD.bazel | 6 +++++- prow/cmd/tide/BUILD.bazel | 5 ++++- prow/cmd/tot/BUILD.bazel | 6 +++++- 20 files changed, 160 insertions(+), 35 deletions(-) diff --git a/ghproxy/BUILD.bazel b/ghproxy/BUILD.bazel index 51a3633307da..336078173847 100644 --- a/ghproxy/BUILD.bazel +++ b/ghproxy/BUILD.bazel @@ -24,7 +24,10 @@ go_image( name = "image", base = "@alpine-base//image", embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", pure = "on", + visibility = ["//visibility:public"], ) go_library( diff --git a/prow/cmd/artifact-uploader/BUILD.bazel b/prow/cmd/artifact-uploader/BUILD.bazel index 570e2b1044de..705849335698 100644 --- a/prow/cmd/artifact-uploader/BUILD.bazel +++ b/prow/cmd/artifact-uploader/BUILD.bazel @@ -18,6 +18,18 @@ go_library( ], ) +load("@io_bazel_rules_docker//go:image.bzl", "go_image") + +go_image( + name = "image", + base = "@alpine-base//image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], +) + go_binary( name = "artifact-uploader", embed = [":go_default_library"], diff --git a/prow/cmd/branchprotector/BUILD.bazel b/prow/cmd/branchprotector/BUILD.bazel index b8e3c228e201..e50c7a3be07f 100644 --- a/prow/cmd/branchprotector/BUILD.bazel +++ b/prow/cmd/branchprotector/BUILD.bazel @@ -19,8 +19,12 @@ load("@io_bazel_rules_docker//go:image.bzl", "go_image") go_image( name = "image", - binary = ":branchprotector", - visibility = ["//prow:__subpackages__"], + base = "@alpine-base//image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], ) load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") diff --git a/prow/cmd/build/BUILD.bazel b/prow/cmd/build/BUILD.bazel index e7bd98c2a166..f46053dfa203 100644 --- a/prow/cmd/build/BUILD.bazel +++ b/prow/cmd/build/BUILD.bazel @@ -16,8 +16,12 @@ load("@io_bazel_rules_docker//go:image.bzl", "go_image") go_image( name = "image", - binary = ":build", - visibility = ["//prow:__subpackages__"], + base = "@alpine-base//image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], ) load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") diff --git a/prow/cmd/clonerefs/BUILD.bazel b/prow/cmd/clonerefs/BUILD.bazel index 4e732bb9da80..a2df27ad759b 100644 --- a/prow/cmd/clonerefs/BUILD.bazel +++ b/prow/cmd/clonerefs/BUILD.bazel @@ -15,10 +15,21 @@ go_library( ) container_image( - name = "image", + name = "base-image", base = "@git-base//image", directory = "/", - files = [":clonerefs"], + visibility = ["//visibility:public"], +) + +load("@io_bazel_rules_docker//go:image.bzl", "go_image") + +go_image( + name = "image", + base = ":base-image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", visibility = ["//visibility:public"], ) diff --git a/prow/cmd/deck/BUILD.bazel b/prow/cmd/deck/BUILD.bazel index 0c80ef89b14d..5df9788099eb 100644 --- a/prow/cmd/deck/BUILD.bazel +++ b/prow/cmd/deck/BUILD.bazel @@ -30,7 +30,11 @@ container_image( go_image( name = "image", base = ":asset-base", - binary = ":deck", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], ) go_binary( diff --git a/prow/cmd/entrypoint/BUILD.bazel b/prow/cmd/entrypoint/BUILD.bazel index 020ac77a56ac..9814589615bf 100644 --- a/prow/cmd/entrypoint/BUILD.bazel +++ b/prow/cmd/entrypoint/BUILD.bazel @@ -14,6 +14,26 @@ go_library( ], ) +load("@io_bazel_rules_docker//go:image.bzl", "go_image") + +container_image( + name = "base-image", + # TODO: this base image is *different* to the one defined in the Dockerfile? + base = "@git-base//image", + directory = "/", + visibility = ["//visibility:private"], +) + +go_image( + name = "image", + base = ":base-image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], +) + filegroup( name = "package-srcs", srcs = glob(["**"]), @@ -28,14 +48,6 @@ filegroup( visibility = ["//visibility:public"], ) -container_image( - name = "image", - base = "@git-base//image", - directory = "/", - files = [":entrypoint"], - visibility = ["//visibility:public"], -) - go_binary( name = "entrypoint", embed = [":go_default_library"], diff --git a/prow/cmd/gcsupload/BUILD.bazel b/prow/cmd/gcsupload/BUILD.bazel index a736b91f3564..6cbb3ff89d2a 100644 --- a/prow/cmd/gcsupload/BUILD.bazel +++ b/prow/cmd/gcsupload/BUILD.bazel @@ -15,6 +15,18 @@ go_library( ], ) +load("@io_bazel_rules_docker//go:image.bzl", "go_image") + +go_image( + name = "image", + base = "@alpine-base//image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], +) + go_binary( name = "gcsupload", embed = [":go_default_library"], diff --git a/prow/cmd/gerrit/BUILD.bazel b/prow/cmd/gerrit/BUILD.bazel index 7fd0c011ef48..b7a83c4c918c 100644 --- a/prow/cmd/gerrit/BUILD.bazel +++ b/prow/cmd/gerrit/BUILD.bazel @@ -1,17 +1,26 @@ package(default_visibility = ["//visibility:public"]) -load("@io_bazel_rules_docker//container:image.bzl", "container_image") load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@io_bazel_rules_docker//container:image.bzl", "container_image") load("@io_bazel_rules_docker//docker:docker.bzl", "container_push") +load("@io_bazel_rules_docker//go:image.bzl", "go_image") container_image( - name = "image", + name = "base-image", base = "@python//image", - cmd = ["./gerrit"], files = [ "third_party/git-cookie-authdaemon", - ":gerrit", ], + visibility = ["//visibility:private"], +) + +go_image( + name = "image", + base = ":base-image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", visibility = ["//visibility:public"], ) diff --git a/prow/cmd/hook/BUILD.bazel b/prow/cmd/hook/BUILD.bazel index f95e3944a0be..7d8a438b511a 100644 --- a/prow/cmd/hook/BUILD.bazel +++ b/prow/cmd/hook/BUILD.bazel @@ -6,7 +6,11 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") go_image( name = "image", base = "@git-base//image", - binary = ":hook", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], ) go_binary( diff --git a/prow/cmd/horologium/BUILD.bazel b/prow/cmd/horologium/BUILD.bazel index ef08d1bc48b6..0e0620d103b8 100644 --- a/prow/cmd/horologium/BUILD.bazel +++ b/prow/cmd/horologium/BUILD.bazel @@ -11,7 +11,11 @@ load( go_image( name = "image", base = "@alpine-base//image", - binary = ":horologium", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], ) go_binary( diff --git a/prow/cmd/initupload/BUILD.bazel b/prow/cmd/initupload/BUILD.bazel index e6d7932bc5c5..47c8e700ae71 100644 --- a/prow/cmd/initupload/BUILD.bazel +++ b/prow/cmd/initupload/BUILD.bazel @@ -1,11 +1,21 @@ -load("@io_bazel_rules_docker//container:image.bzl", "container_image") load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@io_bazel_rules_docker//go:image.bzl", "go_image") +load("@io_bazel_rules_docker//container:image.bzl", "container_image") container_image( - name = "image", + name = "base-image", base = "@alpine-base//image", directory = "/", - files = [":initupload"], + visibility = ["//visibility:private"], +) + +go_image( + name = "image", + base = ":base-image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", visibility = ["//visibility:public"], ) diff --git a/prow/cmd/jenkins-operator/BUILD.bazel b/prow/cmd/jenkins-operator/BUILD.bazel index c334cba1ef38..af7cfea997a3 100644 --- a/prow/cmd/jenkins-operator/BUILD.bazel +++ b/prow/cmd/jenkins-operator/BUILD.bazel @@ -13,7 +13,11 @@ load( go_image( name = "image", base = "@alpine-base//image", - binary = ":jenkins-operator", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], ) go_library( diff --git a/prow/cmd/peribolos/BUILD.bazel b/prow/cmd/peribolos/BUILD.bazel index 29f001c6f83c..ba9454b379dd 100644 --- a/prow/cmd/peribolos/BUILD.bazel +++ b/prow/cmd/peribolos/BUILD.bazel @@ -123,6 +123,10 @@ load("@io_bazel_rules_docker//go:image.bzl", "go_image") go_image( name = "image", - binary = ":peribolos", - visibility = ["//prow:__subpackages__"], + base = "@alpine-base//image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], ) diff --git a/prow/cmd/plank/BUILD.bazel b/prow/cmd/plank/BUILD.bazel index a0a1afce761d..0204983b3ff4 100644 --- a/prow/cmd/plank/BUILD.bazel +++ b/prow/cmd/plank/BUILD.bazel @@ -6,7 +6,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") go_image( name = "image", base = "@alpine-base//image", - binary = ":plank", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", visibility = ["//visibility:public"], ) diff --git a/prow/cmd/sidecar/BUILD.bazel b/prow/cmd/sidecar/BUILD.bazel index 5d95b69084c6..c4fbf0bb2bed 100644 --- a/prow/cmd/sidecar/BUILD.bazel +++ b/prow/cmd/sidecar/BUILD.bazel @@ -1,5 +1,6 @@ -load("@io_bazel_rules_docker//container:image.bzl", "container_image") load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@io_bazel_rules_docker//go:image.bzl", "go_image") +load("@io_bazel_rules_docker//container:image.bzl", "container_image") go_library( name = "go_default_library", @@ -15,10 +16,19 @@ go_library( ) container_image( - name = "image", + name = "base-image", base = "@git-base//image", directory = "/", - files = [":sidecar"], + visibility = ["//visibility:private"], +) + +go_image( + name = "image", + base = ":base-image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", visibility = ["//visibility:public"], ) diff --git a/prow/cmd/sinker/BUILD.bazel b/prow/cmd/sinker/BUILD.bazel index e837154e4d3f..01ec32b7d930 100644 --- a/prow/cmd/sinker/BUILD.bazel +++ b/prow/cmd/sinker/BUILD.bazel @@ -11,7 +11,11 @@ load( go_image( name = "image", base = "@alpine-base//image", - binary = ":sinker", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], ) go_binary( diff --git a/prow/cmd/splice/BUILD.bazel b/prow/cmd/splice/BUILD.bazel index ec7231e13e27..f3032d8bcd57 100644 --- a/prow/cmd/splice/BUILD.bazel +++ b/prow/cmd/splice/BUILD.bazel @@ -11,7 +11,11 @@ load( go_image( name = "image", base = "@git-base//image", - binary = ":splice", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], ) go_binary( diff --git a/prow/cmd/tide/BUILD.bazel b/prow/cmd/tide/BUILD.bazel index 5505a5574884..9c31e8fe5a84 100644 --- a/prow/cmd/tide/BUILD.bazel +++ b/prow/cmd/tide/BUILD.bazel @@ -8,7 +8,10 @@ load( go_image( name = "image", base = "@git-base//image", - binary = ":tide", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", visibility = ["//visibility:public"], ) diff --git a/prow/cmd/tot/BUILD.bazel b/prow/cmd/tot/BUILD.bazel index c528ba763aab..2a3701bf6177 100644 --- a/prow/cmd/tot/BUILD.bazel +++ b/prow/cmd/tot/BUILD.bazel @@ -11,7 +11,11 @@ load( go_image( name = "image", base = "@alpine-base//image", - binary = ":tot", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], ) go_binary( From 2471da249739b1a67ba3c992b204c0469fa73cd8 Mon Sep 17 00:00:00 2001 From: James Munnelly Date: Thu, 30 Aug 2018 18:32:18 +0100 Subject: [PATCH 05/42] Revert changes to podutils --- prow/cmd/clonerefs/BUILD.bazel | 15 ++------------- prow/cmd/entrypoint/BUILD.bazel | 28 ++++++++-------------------- prow/cmd/initupload/BUILD.bazel | 16 +++------------- prow/cmd/sidecar/BUILD.bazel | 16 +++------------- 4 files changed, 16 insertions(+), 59 deletions(-) diff --git a/prow/cmd/clonerefs/BUILD.bazel b/prow/cmd/clonerefs/BUILD.bazel index a2df27ad759b..4e732bb9da80 100644 --- a/prow/cmd/clonerefs/BUILD.bazel +++ b/prow/cmd/clonerefs/BUILD.bazel @@ -15,21 +15,10 @@ go_library( ) container_image( - name = "base-image", + name = "image", base = "@git-base//image", directory = "/", - visibility = ["//visibility:public"], -) - -load("@io_bazel_rules_docker//go:image.bzl", "go_image") - -go_image( - name = "image", - base = ":base-image", - embed = [":go_default_library"], - goarch = "amd64", - goos = "linux", - pure = "on", + files = [":clonerefs"], visibility = ["//visibility:public"], ) diff --git a/prow/cmd/entrypoint/BUILD.bazel b/prow/cmd/entrypoint/BUILD.bazel index 9814589615bf..020ac77a56ac 100644 --- a/prow/cmd/entrypoint/BUILD.bazel +++ b/prow/cmd/entrypoint/BUILD.bazel @@ -14,26 +14,6 @@ go_library( ], ) -load("@io_bazel_rules_docker//go:image.bzl", "go_image") - -container_image( - name = "base-image", - # TODO: this base image is *different* to the one defined in the Dockerfile? - base = "@git-base//image", - directory = "/", - visibility = ["//visibility:private"], -) - -go_image( - name = "image", - base = ":base-image", - embed = [":go_default_library"], - goarch = "amd64", - goos = "linux", - pure = "on", - visibility = ["//visibility:public"], -) - filegroup( name = "package-srcs", srcs = glob(["**"]), @@ -48,6 +28,14 @@ filegroup( visibility = ["//visibility:public"], ) +container_image( + name = "image", + base = "@git-base//image", + directory = "/", + files = [":entrypoint"], + visibility = ["//visibility:public"], +) + go_binary( name = "entrypoint", embed = [":go_default_library"], diff --git a/prow/cmd/initupload/BUILD.bazel b/prow/cmd/initupload/BUILD.bazel index 47c8e700ae71..e6d7932bc5c5 100644 --- a/prow/cmd/initupload/BUILD.bazel +++ b/prow/cmd/initupload/BUILD.bazel @@ -1,21 +1,11 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") -load("@io_bazel_rules_docker//go:image.bzl", "go_image") load("@io_bazel_rules_docker//container:image.bzl", "container_image") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") container_image( - name = "base-image", + name = "image", base = "@alpine-base//image", directory = "/", - visibility = ["//visibility:private"], -) - -go_image( - name = "image", - base = ":base-image", - embed = [":go_default_library"], - goarch = "amd64", - goos = "linux", - pure = "on", + files = [":initupload"], visibility = ["//visibility:public"], ) diff --git a/prow/cmd/sidecar/BUILD.bazel b/prow/cmd/sidecar/BUILD.bazel index c4fbf0bb2bed..5d95b69084c6 100644 --- a/prow/cmd/sidecar/BUILD.bazel +++ b/prow/cmd/sidecar/BUILD.bazel @@ -1,6 +1,5 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") -load("@io_bazel_rules_docker//go:image.bzl", "go_image") load("@io_bazel_rules_docker//container:image.bzl", "container_image") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "go_default_library", @@ -16,19 +15,10 @@ go_library( ) container_image( - name = "base-image", + name = "image", base = "@git-base//image", directory = "/", - visibility = ["//visibility:private"], -) - -go_image( - name = "image", - base = ":base-image", - embed = [":go_default_library"], - goarch = "amd64", - goos = "linux", - pure = "on", + files = [":sidecar"], visibility = ["//visibility:public"], ) From 7fd840c399a60f7dea892987a63f6f95f8dfa535 Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Thu, 30 Aug 2018 09:39:15 -0700 Subject: [PATCH 06/42] Remove regex validation from deck endpoints This regex isn't used inside of deck so it's possible, for instance, to have a job name containing `.' which will be impossible to review the logs for as the `deck` endpoint disallows a `.' in the job name. Signed-off-by: Steve Kuznetsov --- prow/cmd/deck/main.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/prow/cmd/deck/main.go b/prow/cmd/deck/main.go index 39411f2109ba..c4b185d430b7 100644 --- a/prow/cmd/deck/main.go +++ b/prow/cmd/deck/main.go @@ -27,7 +27,6 @@ import ( "net/http" "net/url" "path" - "regexp" "time" "github.com/NYTimes/gziphandler" @@ -97,11 +96,6 @@ func gatherOptions() options { return o } -var ( - // Matches letters, numbers, hyphens, and underscores. - objReg = regexp.MustCompile(`^[\w-]+$`) -) - func main() { o := gatherOptions() if err := o.Validate(); err != nil { @@ -499,16 +493,10 @@ func validateLogRequest(r *http.Request) error { id := r.URL.Query().Get("id") if job == "" { - return errors.New("Missing job query") + return errors.New("request did not provide the 'job' query parameter") } if id == "" { - return errors.New("Missing ID query") - } - if !objReg.MatchString(job) { - return fmt.Errorf("Invalid job query: %s", job) - } - if !objReg.MatchString(id) { - return fmt.Errorf("Invalid ID query: %s", id) + return errors.New("request did not provide the 'id' query parameter") } return nil } @@ -520,8 +508,8 @@ type pjClient interface { func handleRerun(kc pjClient) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { name := r.URL.Query().Get("prowjob") - if !objReg.MatchString(name) { - http.Error(w, "Invalid ProwJob query", http.StatusBadRequest) + if name == "" { + http.Error(w, "request did not provide the 'name' query parameter", http.StatusBadRequest) return } pj, err := kc.GetProwJob(name) From 89312a909e98ea5a17c4f1fb3b345698c27c057f Mon Sep 17 00:00:00 2001 From: Nikolaos Moraitis Date: Thu, 30 Aug 2018 20:38:43 +0200 Subject: [PATCH 07/42] increase waiting time in Secret Agent's test. --- prow/config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow/config/config_test.go b/prow/config/config_test.go index 20aad9000a18..8ab25be8a768 100644 --- a/prow/config/config_test.go +++ b/prow/config/config_test.go @@ -1026,7 +1026,7 @@ func TestSecretAgentLoading(t *testing.T) { tempSecret, changedTokenValue, tempSecretValue)) } else { // Secret agent needs some time to update the values. So wait and retry. - time.Sleep(100 * time.Millisecond) + time.Sleep(400 * time.Millisecond) } } else { break From ff2c0cbe15a8670182cd0c271379b866623d3775 Mon Sep 17 00:00:00 2001 From: Christoph Blecker Date: Thu, 30 Aug 2018 11:51:08 -0700 Subject: [PATCH 08/42] Add shrug label --- label_sync/labels.md | 1 + label_sync/labels.yaml | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/label_sync/labels.md b/label_sync/labels.md index 52d34f53cfad..dc6f407c3bef 100644 --- a/label_sync/labels.md +++ b/label_sync/labels.md @@ -106,6 +106,7 @@ larger set of contributors to apply/remove them. | `wg/multitenancy` | Categorizes an issue or PR as relevant to wg-multitenancy.| anyone | [label](https://git.k8s.io/test-infra/prow/plugins/label) | | `wg/policy` | Categorizes an issue or PR as relevant to wg-policy.| anyone | [label](https://git.k8s.io/test-infra/prow/plugins/label) | | `wg/resource-management` | Categorizes an issue or PR as relevant to wg-resource-management.| anyone | [label](https://git.k8s.io/test-infra/prow/plugins/label) | +| `¯\_(ツ)_/¯` | ¯\_(ツ)_/¯| humans | [shrug](https://git.k8s.io/test-infra/prow/plugins/shrug) | ## Labels that apply to all repos, only for issues diff --git a/label_sync/labels.yaml b/label_sync/labels.yaml index 07cf5a3cd74a..828766e48078 100644 --- a/label_sync/labels.yaml +++ b/label_sync/labels.yaml @@ -657,6 +657,12 @@ default: target: both prowPlugin: milestonestatus addedBy: "members of a configurable github team. default: [@kubernetes/kubernetes-milestone-maintainers](https://github.com/orgs/kubernetes/teams/kubernetes-milestone-maintainers/members)" + - color: f9d0c4 + description: "¯\\_(ツ)_/¯" + name: "¯\\_(ツ)_/¯" + target: both + prowPlugin: shrug + addedBy: humans repos: kubernetes/kubernetes: labels: @@ -836,4 +842,4 @@ repos: description: Issues or PRs related to prow's pod-utilities component name: area/prow/pod-utilities target: both - addedBy: label \ No newline at end of file + addedBy: label From 9b82306a97c7c575d497ca51ff000d532643fd98 Mon Sep 17 00:00:00 2001 From: Sen Lu Date: Thu, 30 Aug 2018 14:29:06 -0700 Subject: [PATCH 09/42] remove trailing quote --- config/jobs/gke/gke-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jobs/gke/gke-prod.yaml b/config/jobs/gke/gke-prod.yaml index dd9000957637..9aaf4d95dcec 100644 --- a/config/jobs/gke/gke-prod.yaml +++ b/config/jobs/gke/gke-prod.yaml @@ -22,7 +22,7 @@ periodics: - --gke-environment=prod - --provider=gke - --test_args=--ginkgo.focus=\[Slow\]|\[Serial\]|\[Disruptive\] --ginkgo.skip=\[Flaky\]|\[Feature:.+\] --minStartupPods=8 - - --timeout=600m" + - --timeout=600m image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - interval: 30m From 2b5b367ac58764a9f98b7d2874127a8d5eac8d4a Mon Sep 17 00:00:00 2001 From: Sen Lu Date: Tue, 28 Aug 2018 16:34:46 -0700 Subject: [PATCH 10/42] migrate rest of the sig-node jobs --- config/jobs/containerd/cri/OWNERS | 4 + .../cri/containerd-cri-presubmit-jobs.yaml | 99 +++++ .../generated-security-jobs.yaml | 351 ++++++++++-------- .../jobs/kubernetes/sig-node/node-docker.yaml | 77 ++++ .../sig-node/sig-node-presubmit.yaml | 136 +++++++ jobs/config.json | 114 ------ prow/config.yaml | 223 ----------- 7 files changed, 513 insertions(+), 491 deletions(-) create mode 100644 config/jobs/containerd/cri/OWNERS create mode 100644 config/jobs/containerd/cri/containerd-cri-presubmit-jobs.yaml create mode 100644 config/jobs/kubernetes/sig-node/node-docker.yaml create mode 100644 config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml diff --git a/config/jobs/containerd/cri/OWNERS b/config/jobs/containerd/cri/OWNERS new file mode 100644 index 000000000000..f3cf631494ac --- /dev/null +++ b/config/jobs/containerd/cri/OWNERS @@ -0,0 +1,4 @@ +approvers: +- yujuhong +- Random-Liu +- dchen1107 diff --git a/config/jobs/containerd/cri/containerd-cri-presubmit-jobs.yaml b/config/jobs/containerd/cri/containerd-cri-presubmit-jobs.yaml new file mode 100644 index 000000000000..762e2018a3af --- /dev/null +++ b/config/jobs/containerd/cri/containerd-cri-presubmit-jobs.yaml @@ -0,0 +1,99 @@ +presubmits: + containerd/cri: + - name: pull-cri-containerd-build + always_run: true + skip_branches: + - v0.1 + labels: + preset-service-account: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-experimental + args: + - "--repo=github.com/containerd/$(REPO_NAME)=$(PULL_REFS)" + - "--root=/go/src" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - --scenario=execute + - -- + - ./test/build.sh + run_after_success: + - name: pull-cri-containerd-node-e2e + branches: + - master + always_run: true + skip_report: false + max_concurrency: 8 + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - --root=/go/src + - "--job=$(JOB_NAME)" + - "--repo=k8s.io/kubernetes" + - "--repo=github.com/containerd/$(REPO_NAME)=$(PULL_REFS)" + - "--service-account=/etc/service-account/service-account.json" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - "--timeout=90" + - --scenario=kubernetes_e2e + - -- + - --deployment=node + - --gcp-project=cri-c8d-pr-node-e2e + - --gcp-zone=us-central1-f + - '--node-test-args=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --container-runtime-process-name=/home/containerd/usr/local/bin/containerd --container-runtime-pid-file= --kubelet-flags="--cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/containerd.service" --extra-log="{\"name\": \"containerd.log\", \"journalctl\": [\"-u\", \"containerd\"]}"' + - --node-tests=true + - --provider=gce + - --test_args=--nodes=8 --focus="\[NodeConformance\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]" --flakeAttempts=2 + - --timeout=65m + - "--node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/containerd/cri-master/image-config.yaml -node-env=PULL_REFS=$(PULL_REFS)" + + - name: pull-cri-containerd-node-e2e + branches: + - release/1.0 + always_run: true + skip_report: false + max_concurrency: 8 + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - --root=/go/src + - "--job=$(JOB_NAME)" + - "--repo=k8s.io/kubernetes=release-1.10" + - "--repo=github.com/containerd/$(REPO_NAME)=$(PULL_REFS)" + - "--service-account=/etc/service-account/service-account.json" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - "--timeout=90" + - --scenario=kubernetes_e2e + - -- + - --deployment=node + - --gcp-project=cri-c8d-pr-node-e2e + - --gcp-zone=us-central1-f + - '--node-test-args=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --container-runtime-process-name=/home/containerd/usr/local/bin/containerd --container-runtime-pid-file= --kubelet-flags="--cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/containerd.service" --extra-log="{\"name\": \"containerd.log\", \"journalctl\": [\"-u\", \"containerd\"]}"' + - --node-tests=true + - --provider=gce + - --test_args=--nodes=8 --focus="\[NodeConformance\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]" --flakeAttempts=2 + - --timeout=65m + - "--node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/containerd/cri-master/image-config.yaml -node-env=PULL_REFS=$(PULL_REFS)" + + - name: pull-cri-containerd-verify + always_run: true + skip_branches: + - v0.1 + labels: + preset-service-account: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-experimental + args: + - "--repo=github.com/containerd/$(REPO_NAME)=$(PULL_REFS)" + - "--root=/go/src" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - --scenario=execute + - -- + - ./test/verify.sh diff --git a/config/jobs/kubernetes-security/generated-security-jobs.yaml b/config/jobs/kubernetes-security/generated-security-jobs.yaml index 0bc24af8e8d8..0ad12501c17d 100644 --- a/config/jobs/kubernetes-security/generated-security-jobs.yaml +++ b/config/jobs/kubernetes-security/generated-security-jobs.yaml @@ -2,160 +2,6 @@ # see genjobs.go, which you can run with hack/update-config.sh presubmits: kubernetes-security/kubernetes: - - agent: kubernetes - always_run: true - cluster: security - context: pull-security-kubernetes-node-e2e - labels: - preset-k8s-ssh: "true" - preset-service-account: "true" - max_concurrency: 12 - name: pull-security-kubernetes-node-e2e - rerun_command: /test pull-security-kubernetes-node-e2e - skip_branches: - - release-1.10 - - release-1.9 - spec: - containers: - - args: - - --ssh=/etc/ssh-security/ssh-security - - --root=/go/src - - --job=$(JOB_NAME) - - --repo=github.com/kubernetes-security/$(REPO_NAME)=$(PULL_REFS) - - --service-account=/etc/service-account/service-account.json - - --upload=gs://kubernetes-security-prow/pr-logs - - --timeout=90 - - -- - - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - name: "" - resources: - requests: - memory: 6Gi - volumeMounts: - - mountPath: /etc/ssh-security - name: ssh-security - volumes: - - name: ssh-security - secret: - defaultMode: 256 - secretName: ssh-security - trigger: (?m)^/test( | .* )pull-security-kubernetes-node-e2e,?($|\s.*) - - agent: kubernetes - always_run: true - branches: - - release-1.10 - cluster: security - context: pull-security-kubernetes-node-e2e - labels: - preset-k8s-ssh: "true" - preset-service-account: "true" - max_concurrency: 12 - name: pull-security-kubernetes-node-e2e - rerun_command: /test pull-security-kubernetes-node-e2e - spec: - containers: - - args: - - --ssh=/etc/ssh-security/ssh-security - - --root=/go/src - - --job=$(JOB_NAME) - - --repo=github.com/kubernetes-security/$(REPO_NAME)=$(PULL_REFS) - - --service-account=/etc/service-account/service-account.json - - --upload=gs://kubernetes-security-prow/pr-logs - - --timeout=90 - - -- - - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-1.10 - name: "" - resources: - requests: - memory: 6Gi - volumeMounts: - - mountPath: /etc/ssh-security - name: ssh-security - volumes: - - name: ssh-security - secret: - defaultMode: 256 - secretName: ssh-security - trigger: (?m)^/test( | .* )pull-security-kubernetes-node-e2e,?($|\s.*) - - agent: kubernetes - always_run: true - branches: - - release-1.9 - cluster: security - context: pull-security-kubernetes-node-e2e - labels: - preset-k8s-ssh: "true" - preset-service-account: "true" - max_concurrency: 12 - name: pull-security-kubernetes-node-e2e - rerun_command: /test pull-security-kubernetes-node-e2e - spec: - containers: - - args: - - --ssh=/etc/ssh-security/ssh-security - - --root=/go/src - - --job=$(JOB_NAME) - - --repo=github.com/kubernetes-security/$(REPO_NAME)=$(PULL_REFS) - - --service-account=/etc/service-account/service-account.json - - --upload=gs://kubernetes-security-prow/pr-logs - - --timeout=90 - - -- - - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-1.9 - name: "" - resources: - requests: - memory: 6Gi - volumeMounts: - - mountPath: /etc/ssh-security - name: ssh-security - volumes: - - name: ssh-security - secret: - defaultMode: 256 - secretName: ssh-security - trigger: (?m)^/test( | .* )pull-security-kubernetes-node-e2e,?($|\s.*) - - agent: kubernetes - always_run: false - branches: - - master - cluster: security - context: pull-security-kubernetes-node-e2e-containerd - labels: - preset-k8s-ssh: "true" - preset-service-account: "true" - max_concurrency: 12 - name: pull-security-kubernetes-node-e2e-containerd - rerun_command: /test pull-security-kubernetes-node-e2e-containerd - spec: - containers: - - args: - - --ssh=/etc/ssh-security/ssh-security - - --root=/go/src - - --job=$(JOB_NAME) - - --repo=github.com/kubernetes-security/$(REPO_NAME)=$(PULL_REFS) - - --repo=github.com/containerd/cri - - --service-account=/etc/service-account/service-account.json - - --upload=gs://kubernetes-security-prow/pr-logs - - --timeout=90 - - -- - - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/containerd/cri-master/image-config.yaml - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - name: "" - resources: - requests: - memory: 6Gi - volumeMounts: - - mountPath: /etc/ssh-security - name: ssh-security - volumes: - - name: ssh-security - secret: - defaultMode: 256 - secretName: ssh-security - trigger: (?m)^/test( | .* )pull-security-kubernetes-node-e2e-containerd,?($|\s.*) - agent: kubernetes always_run: true cluster: security @@ -1126,6 +972,203 @@ presubmits: defaultMode: 256 secretName: ssh-security trigger: (?m)^/test( | .* )pull-security-kubernetes-local-e2e-containerized,?($|\s.*) + - agent: kubernetes + always_run: true + cluster: security + context: pull-security-kubernetes-node-e2e + labels: + preset-k8s-ssh: "true" + preset-service-account: "true" + max_concurrency: 12 + name: pull-security-kubernetes-node-e2e + rerun_command: /test pull-security-kubernetes-node-e2e + skip_branches: + - release-1.10 + - release-1.9 + spec: + containers: + - args: + - --ssh=/etc/ssh-security/ssh-security + - --root=/go/src + - --job=$(JOB_NAME) + - --repo=github.com/kubernetes-security/$(REPO_NAME)=$(PULL_REFS) + - --service-account=/etc/service-account/service-account.json + - --upload=gs://kubernetes-security-prow/pr-logs + - --timeout=90 + - --scenario=kubernetes_e2e + - -- + - --deployment=node + - --gcp-project=k8s-jkns-pr-node-e2e + - --gcp-zone=us-central1-f + - --node-test-args=--kubelet-flags="--cgroups-per-qos=true --cgroup-root=/" + - --node-tests=true + - --provider=gce + - --test_args=--nodes=8 --focus="\[NodeConformance\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]" + --flakeAttempts=2 + - --timeout=65m + - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml + image: gcr.io/k8s-testimages/kubekins-e2e:v20180827-b09cde7d7-master + name: "" + resources: + requests: + memory: 6Gi + volumeMounts: + - mountPath: /etc/ssh-security + name: ssh-security + volumes: + - name: ssh-security + secret: + defaultMode: 256 + secretName: ssh-security + trigger: (?m)^/test( | .* )pull-security-kubernetes-node-e2e,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - release-1.10 + cluster: security + context: pull-security-kubernetes-node-e2e + labels: + preset-k8s-ssh: "true" + preset-service-account: "true" + max_concurrency: 12 + name: pull-security-kubernetes-node-e2e + rerun_command: /test pull-security-kubernetes-node-e2e + spec: + containers: + - args: + - --ssh=/etc/ssh-security/ssh-security + - --root=/go/src + - --job=$(JOB_NAME) + - --repo=github.com/kubernetes-security/$(REPO_NAME)=$(PULL_REFS) + - --service-account=/etc/service-account/service-account.json + - --upload=gs://kubernetes-security-prow/pr-logs + - --timeout=90 + - --scenario=kubernetes_e2e + - -- + - --deployment=node + - --gcp-project=k8s-jkns-pr-node-e2e + - --gcp-zone=us-central1-f + - --node-test-args=--kubelet-flags="--cgroups-per-qos=true --cgroup-root=/" + - --node-tests=true + - --provider=gce + - --test_args=--nodes=8 --focus="\[NodeConformance\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]" + --flakeAttempts=2 + - --timeout=65m + - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml + image: gcr.io/k8s-testimages/kubekins-e2e:v20180827-b09cde7d7-1.10 + name: "" + resources: + requests: + memory: 6Gi + volumeMounts: + - mountPath: /etc/ssh-security + name: ssh-security + volumes: + - name: ssh-security + secret: + defaultMode: 256 + secretName: ssh-security + trigger: (?m)^/test( | .* )pull-security-kubernetes-node-e2e,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - release-1.9 + cluster: security + context: pull-security-kubernetes-node-e2e + labels: + preset-k8s-ssh: "true" + preset-service-account: "true" + max_concurrency: 12 + name: pull-security-kubernetes-node-e2e + rerun_command: /test pull-security-kubernetes-node-e2e + spec: + containers: + - args: + - --ssh=/etc/ssh-security/ssh-security + - --root=/go/src + - --job=$(JOB_NAME) + - --repo=github.com/kubernetes-security/$(REPO_NAME)=$(PULL_REFS) + - --service-account=/etc/service-account/service-account.json + - --upload=gs://kubernetes-security-prow/pr-logs + - --timeout=90 + - --scenario=kubernetes_e2e + - -- + - --deployment=node + - --gcp-project=k8s-jkns-pr-node-e2e + - --gcp-zone=us-central1-f + - --node-test-args=--kubelet-flags="--cgroups-per-qos=true --cgroup-root=/" + - --node-tests=true + - --provider=gce + - --test_args=--nodes=8 --focus="\[NodeConformance\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]" + --flakeAttempts=2 + - --timeout=65m + - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml + image: gcr.io/k8s-testimages/kubekins-e2e:v20180827-b09cde7d7-1.9 + name: "" + resources: + requests: + memory: 6Gi + volumeMounts: + - mountPath: /etc/ssh-security + name: ssh-security + volumes: + - name: ssh-security + secret: + defaultMode: 256 + secretName: ssh-security + trigger: (?m)^/test( | .* )pull-security-kubernetes-node-e2e,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - master + cluster: security + context: pull-security-kubernetes-node-e2e-containerd + labels: + preset-k8s-ssh: "true" + preset-service-account: "true" + max_concurrency: 12 + name: pull-security-kubernetes-node-e2e-containerd + rerun_command: /test pull-security-kubernetes-node-e2e-containerd + spec: + containers: + - args: + - --ssh=/etc/ssh-security/ssh-security + - --root=/go/src + - --job=$(JOB_NAME) + - --repo=github.com/kubernetes-security/$(REPO_NAME)=$(PULL_REFS) + - --repo=github.com/containerd/cri + - --service-account=/etc/service-account/service-account.json + - --upload=gs://kubernetes-security-prow/pr-logs + - --timeout=90 + - --scenario=kubernetes_e2e + - -- + - --deployment=node + - --gcp-project=k8s-c8d-pr-node-e2e + - --gcp-zone=us-central1-f + - '--node-test-args=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock + --container-runtime-process-name=/home/containerd/usr/local/bin/containerd --container-runtime-pid-file= + --kubelet-flags="--cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/containerd.service" + --extra-log="{\"name\": \"containerd.log\", \"journalctl\": [\"-u\", \"containerd\"]}"' + - --node-tests=true + - --provider=gce + - --test_args=--nodes=8 --focus="\[NodeConformance\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]" + --flakeAttempts=2 + - --timeout=65m + - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/containerd/cri-master/image-config.yaml + image: gcr.io/k8s-testimages/kubekins-e2e:v20180827-b09cde7d7-master + name: "" + resources: + requests: + memory: 6Gi + volumeMounts: + - mountPath: /etc/ssh-security + name: ssh-security + volumes: + - name: ssh-security + secret: + defaultMode: 256 + secretName: ssh-security + trigger: (?m)^/test( | .* )pull-security-kubernetes-node-e2e-containerd,?($|\s.*) - agent: kubernetes always_run: false cluster: security diff --git a/config/jobs/kubernetes/sig-node/node-docker.yaml b/config/jobs/kubernetes/sig-node/node-docker.yaml new file mode 100644 index 000000000000..52f607397b36 --- /dev/null +++ b/config/jobs/kubernetes/sig-node/node-docker.yaml @@ -0,0 +1,77 @@ +periodics: +- name: ci-docker-node-conformance + interval: 2h + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180827-b09cde7d7-master + args: + - --repo=k8s.io/kubernetes=master + - --timeout=90 + - --root=/go/src + - --scenario=kubernetes_e2e + - -- + - --deployment=node + - --gcp-zone=us-central1-f + - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml + - --node-test-args=--kubelet-flags="--cgroups-per-qos=true --cgroup-root=/" + - --node-tests=true + - --provider=gce + - --test_args=--nodes=8 --focus="\[NodeConformance\]" --skip="\[Flaky\]|\[Serial\]" + - --timeout=65m + env: + - name: GOPATH + value: /go + +- name: ci-docker-node-features + interval: 4h + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180827-b09cde7d7-master + args: + - --repo=k8s.io/kubernetes=master + - --timeout=200 + - --root=/go/src + - --scenario=kubernetes_e2e + - -- + - --deployment=node + - --gcp-zone=us-central1-f + - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml + - --node-test-args=--kubelet-flags="--cgroups-per-qos=true --cgroup-root=/" + - --node-tests=true + - --provider=gce + - --test_args=--nodes=8 --focus="\[NodeFeature:.+\]" --skip="\[Flaky\]|\[Serial\]" + - --timeout=180m + env: + - name: GOPATH + value: /go + +- name: ci-docker-node-legacy + interval: 2h + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180827-b09cde7d7-master + args: + - --repo=k8s.io/kubernetes=master + - --timeout=200 + - --root=/go/src + - -- + - --deployment=node + - --gcp-zone=us-central1-f + - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml + - --node-test-args=--kubelet-flags="--cgroups-per-qos=true --cgroup-root=/" + - --node-tests=true + - --provider=gce + - --test_args=--nodes=1 --focus="\[Legacy:Docker\]" --skip="\[Flaky\]" + - --timeout=180m + env: + - name: GOPATH + value: /go \ No newline at end of file diff --git a/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml b/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml new file mode 100644 index 000000000000..4989110d5c43 --- /dev/null +++ b/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml @@ -0,0 +1,136 @@ +presubmits: + kubernetes/kubernetes: + - name: pull-kubernetes-node-e2e + skip_branches: + - release-1.10 # per-release image + - release-1.9 # per-release image + always_run: true + max_concurrency: 12 + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180827-b09cde7d7-master + args: + - --root=/go/src + - "--job=$(JOB_NAME)" + - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" + - "--service-account=/etc/service-account/service-account.json" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - "--timeout=90" + - --scenario=kubernetes_e2e + - -- # end bootstrap args, scenario args below + - --deployment=node + - --gcp-project=k8s-jkns-pr-node-e2e + - --gcp-zone=us-central1-f + - --node-test-args=--kubelet-flags="--cgroups-per-qos=true --cgroup-root=/" + - --node-tests=true + - --provider=gce + - --test_args=--nodes=8 --focus="\[NodeConformance\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]" --flakeAttempts=2 + - --timeout=65m + - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml + ports: + resources: + requests: + memory: "6Gi" + - name: pull-kubernetes-node-e2e + branches: + - release-1.10 # per-release image + always_run: true + max_concurrency: 12 + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180827-b09cde7d7-1.10 + args: + - --root=/go/src + - "--job=$(JOB_NAME)" + - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" + - "--service-account=/etc/service-account/service-account.json" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - "--timeout=90" + - --scenario=kubernetes_e2e + - -- # end bootstrap args, scenario args below + - --deployment=node + - --gcp-project=k8s-jkns-pr-node-e2e + - --gcp-zone=us-central1-f + - --node-test-args=--kubelet-flags="--cgroups-per-qos=true --cgroup-root=/" + - --node-tests=true + - --provider=gce + - --test_args=--nodes=8 --focus="\[NodeConformance\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]" --flakeAttempts=2 + - --timeout=65m + - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml + ports: + resources: + requests: + memory: "6Gi" + - name: pull-kubernetes-node-e2e + branches: + - release-1.9 # per-release image + always_run: true + max_concurrency: 12 + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180827-b09cde7d7-1.9 + args: + - --root=/go/src + - "--job=$(JOB_NAME)" + - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" + - "--service-account=/etc/service-account/service-account.json" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - "--timeout=90" + - --scenario=kubernetes_e2e + - -- # end bootstrap args, scenario args below + - --deployment=node + - --gcp-project=k8s-jkns-pr-node-e2e + - --gcp-zone=us-central1-f + - --node-test-args=--kubelet-flags="--cgroups-per-qos=true --cgroup-root=/" + - --node-tests=true + - --provider=gce + - --test_args=--nodes=8 --focus="\[NodeConformance\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]" --flakeAttempts=2 + - --timeout=65m + - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml + ports: + resources: + requests: + memory: "6Gi" + + - name: pull-kubernetes-node-e2e-containerd + branches: + - master + always_run: false + max_concurrency: 12 + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180827-b09cde7d7-master + args: + - --root=/go/src + - "--job=$(JOB_NAME)" + - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" + - "--repo=github.com/containerd/cri" + - "--service-account=/etc/service-account/service-account.json" + - "--upload=gs://kubernetes-jenkins/pr-logs" + - "--timeout=90" + - --scenario=kubernetes_e2e + - -- + - --deployment=node + - --gcp-project=k8s-c8d-pr-node-e2e + - --gcp-zone=us-central1-f + - '--node-test-args=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --container-runtime-process-name=/home/containerd/usr/local/bin/containerd --container-runtime-pid-file= --kubelet-flags="--cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/containerd.service" --extra-log="{\"name\": \"containerd.log\", \"journalctl\": [\"-u\", \"containerd\"]}"' + - --node-tests=true + - --provider=gce + - --test_args=--nodes=8 --focus="\[NodeConformance\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]" --flakeAttempts=2 + - --timeout=65m + - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/containerd/cri-master/image-config.yaml + resources: + requests: + memory: "6Gi" \ No newline at end of file diff --git a/jobs/config.json b/jobs/config.json index 5fc403856529..43377f7b4675 100644 --- a/jobs/config.json +++ b/jobs/config.json @@ -19,54 +19,6 @@ "sig-apps" ] }, - "ci-docker-node-conformance": { - "args": [ - "--deployment=node", - "--gcp-zone=us-central1-f", - "--node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml", - "--node-test-args=--kubelet-flags=\"--cgroups-per-qos=true --cgroup-root=/\"", - "--node-tests=true", - "--provider=gce", - "--test_args=--nodes=8 --focus=\"\\[NodeConformance\\]\" --skip=\"\\[Flaky\\]|\\[Serial\\]\"", - "--timeout=65m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-node" - ] - }, - "ci-docker-node-features": { - "args": [ - "--deployment=node", - "--gcp-zone=us-central1-f", - "--node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml", - "--node-test-args=--kubelet-flags=\"--cgroups-per-qos=true --cgroup-root=/\"", - "--node-tests=true", - "--provider=gce", - "--test_args=--nodes=8 --focus=\"\\[NodeFeature:.+\\]\" --skip=\"\\[Flaky\\]|\\[Serial\\]\"", - "--timeout=180m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-node" - ] - }, - "ci-docker-node-legacy": { - "args": [ - "--deployment=node", - "--gcp-zone=us-central1-f", - "--node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml", - "--node-test-args=--kubelet-flags=\"--cgroups-per-qos=true --cgroup-root=/\"", - "--node-tests=true", - "--provider=gce", - "--test_args=--nodes=1 --focus=\"\\[Legacy:Docker\\]\" --skip=\"\\[Flaky\\]\"", - "--timeout=180m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-node" - ] - }, "ci-kubernetes-e2e-autoscaling-vpa-actuation": { "args": [ "--check-leaked-resources", @@ -664,72 +616,6 @@ "sig-testing" ] }, - "pull-cri-containerd-build": { - "args": [ - "./test/build.sh" - ], - "scenario": "execute", - "sigOwners": [ - "sig-node" - ] - }, - "pull-cri-containerd-node-e2e": { - "args": [ - "--deployment=node", - "--gcp-project=cri-c8d-pr-node-e2e", - "--gcp-zone=us-central1-f", - "--node-test-args=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --container-runtime-process-name=/home/containerd/usr/local/bin/containerd --container-runtime-pid-file= --kubelet-flags=\"--cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/containerd.service\" --extra-log=\"{\\\"name\\\": \\\"containerd.log\\\", \\\"journalctl\\\": [\\\"-u\\\", \\\"containerd\\\"]}\"", - "--node-tests=true", - "--provider=gce", - "--test_args=--nodes=8 --focus=\"\\[NodeConformance\\]\" --skip=\"\\[Flaky\\]|\\[Slow\\]|\\[Serial\\]\" --flakeAttempts=2", - "--timeout=65m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-node" - ] - }, - "pull-cri-containerd-verify": { - "args": [ - "./test/verify.sh" - ], - "scenario": "execute", - "sigOwners": [ - "sig-node" - ] - }, - "pull-kubernetes-node-e2e": { - "args": [ - "--deployment=node", - "--gcp-project=k8s-jkns-pr-node-e2e", - "--gcp-zone=us-central1-f", - "--node-test-args=--kubelet-flags=\"--cgroups-per-qos=true --cgroup-root=/\"", - "--node-tests=true", - "--provider=gce", - "--test_args=--nodes=8 --focus=\"\\[NodeConformance\\]\" --skip=\"\\[Flaky\\]|\\[Slow\\]|\\[Serial\\]\" --flakeAttempts=2", - "--timeout=65m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-node" - ] - }, - "pull-kubernetes-node-e2e-containerd": { - "args": [ - "--deployment=node", - "--gcp-project=k8s-c8d-pr-node-e2e", - "--gcp-zone=us-central1-f", - "--node-test-args=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --container-runtime-process-name=/home/containerd/usr/local/bin/containerd --container-runtime-pid-file= --kubelet-flags=\"--cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/containerd.service\" --extra-log=\"{\\\"name\\\": \\\"containerd.log\\\", \\\"journalctl\\\": [\\\"-u\\\", \\\"containerd\\\"]}\"", - "--node-tests=true", - "--provider=gce", - "--test_args=--nodes=8 --focus=\"\\[NodeConformance\\]\" --skip=\"\\[Flaky\\]|\\[Slow\\]|\\[Serial\\]\" --flakeAttempts=2", - "--timeout=65m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-node" - ] - }, "random_job": { "scenario": "execute", "sigOwners": [ diff --git a/prow/config.yaml b/prow/config.yaml index da989f7bf5c2..8dc584f41e46 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -516,181 +516,6 @@ presets: - name: docker-graph mountPath: /docker-graph -presubmits: - containerd/cri: - - name: pull-cri-containerd-build - always_run: true - skip_branches: - - v0.1 - labels: - preset-service-account: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-experimental - args: - - "--repo=github.com/containerd/$(REPO_NAME)=$(PULL_REFS)" - - "--root=/go/src" - - "--upload=gs://kubernetes-jenkins/pr-logs" - run_after_success: - - name: pull-cri-containerd-node-e2e - branches: - - master - always_run: true - skip_report: false - max_concurrency: 8 - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - --root=/go/src - - "--job=$(JOB_NAME)" - - "--repo=k8s.io/kubernetes" - - "--repo=github.com/containerd/$(REPO_NAME)=$(PULL_REFS)" - - "--service-account=/etc/service-account/service-account.json" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - "--timeout=90" - - "--" - - "--node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/containerd/cri-master/image-config.yaml -node-env=PULL_REFS=$(PULL_REFS)" - - name: pull-cri-containerd-node-e2e - branches: - - release/1.0 - always_run: true - skip_report: false - max_concurrency: 8 - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - --root=/go/src - - "--job=$(JOB_NAME)" - - "--repo=k8s.io/kubernetes=release-1.10" - - "--repo=github.com/containerd/$(REPO_NAME)=$(PULL_REFS)" - - "--service-account=/etc/service-account/service-account.json" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - "--timeout=90" - - "--" - - "--node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/containerd/cri-master/image-config.yaml -node-env=PULL_REFS=$(PULL_REFS)" - - - name: pull-cri-containerd-verify - always_run: true - skip_branches: - - v0.1 - labels: - preset-service-account: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-experimental - args: - - "--repo=github.com/containerd/$(REPO_NAME)=$(PULL_REFS)" - - "--root=/go/src" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - kubernetes/kubernetes: - - name: pull-kubernetes-node-e2e - skip_branches: - - release-1.10 # per-release image - - release-1.9 # per-release image - always_run: true - max_concurrency: 12 - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - --root=/go/src - - "--job=$(JOB_NAME)" - - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" - - "--service-account=/etc/service-account/service-account.json" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - "--timeout=90" - - "--" # end bootstrap args, scenario args below - - "--node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml" - ports: - resources: - requests: - memory: "6Gi" - - name: pull-kubernetes-node-e2e - branches: - - release-1.10 # per-release image - always_run: true - max_concurrency: 12 - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-1.10 - args: - - --root=/go/src - - "--job=$(JOB_NAME)" - - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" - - "--service-account=/etc/service-account/service-account.json" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - "--timeout=90" - - "--" # end bootstrap args, scenario args below - - "--node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml" - ports: - resources: - requests: - memory: "6Gi" - - name: pull-kubernetes-node-e2e - branches: - - release-1.9 # per-release image - always_run: true - max_concurrency: 12 - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-1.9 - args: - - --root=/go/src - - "--job=$(JOB_NAME)" - - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" - - "--service-account=/etc/service-account/service-account.json" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - "--timeout=90" - - "--" # end bootstrap args, scenario args below - - "--node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/image-config.yaml" - ports: - resources: - requests: - memory: "6Gi" - - - name: pull-kubernetes-node-e2e-containerd - branches: - - master - always_run: false - max_concurrency: 12 - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - --root=/go/src - - "--job=$(JOB_NAME)" - - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)" - - "--repo=github.com/containerd/cri" - - "--service-account=/etc/service-account/service-account.json" - - "--upload=gs://kubernetes-jenkins/pr-logs" - - "--timeout=90" - - -- - - "--node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/containerd/cri-master/image-config.yaml" - resources: - requests: - memory: "6Gi" - periodics: - interval: 2h name: application-periodic-default-gke @@ -714,54 +539,6 @@ periodics: cpu: 6 memory: "8Gi" -- name: ci-docker-node-conformance - interval: 2h - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - --repo=k8s.io/kubernetes=master - - --timeout=90 - - --root=/go/src - env: - - name: GOPATH - value: /go - -- name: ci-docker-node-features - interval: 4h - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - --repo=k8s.io/kubernetes=master - - --timeout=200 - - --root=/go/src - env: - - name: GOPATH - value: /go - -- name: ci-docker-node-legacy - interval: 2h - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - --repo=k8s.io/kubernetes=master - - --timeout=200 - - --root=/go/src - env: - - name: GOPATH - value: /go - - name: ci-kubernetes-e2e-autoscaling-vpa-actuation interval: 2h labels: From 3cdd954088e4d144ae42291b317372243d51cdc9 Mon Sep 17 00:00:00 2001 From: Sen Lu Date: Thu, 30 Aug 2018 14:39:49 -0700 Subject: [PATCH 11/42] no more presubmit jobs in config.yaml --- jobs/config_test.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/jobs/config_test.py b/jobs/config_test.py index a0664506d746..6b5e069584be 100755 --- a/jobs/config_test.py +++ b/jobs/config_test.py @@ -65,7 +65,6 @@ class JobTest(unittest.TestCase): realjobs = {} prowjobs = [] - presubmits = [] @property def jobs(self): @@ -137,18 +136,9 @@ def load_prow_yaml(self, path): if 'periodics' not in doc: self.fail('No periodics in prow config!') - if 'presubmits' not in doc: - self.fail('No presubmits in prow config!') - for item in doc.get('periodics'): self.add_prow_job(item) - self.presubmits = doc.get('presubmits') - - for _repo, joblist in self.presubmits.items(): - for job in joblist: - self.add_prow_job(job) - def get_real_bootstrap_job(self, job): key = os.path.splitext(job.strip())[0] if not key in self.realjobs: From 89f5fda7328170e35c12b474eef900a1dfd86767 Mon Sep 17 00:00:00 2001 From: James Munnelly Date: Thu, 30 Aug 2018 22:43:42 +0100 Subject: [PATCH 12/42] Update bazel image rules in external-plugins --- prow/external-plugins/cherrypicker/BUILD.bazel | 11 +++++++++++ prow/external-plugins/needs-rebase/BUILD.bazel | 6 +++++- prow/external-plugins/refresh/BUILD.bazel | 11 +++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/prow/external-plugins/cherrypicker/BUILD.bazel b/prow/external-plugins/cherrypicker/BUILD.bazel index 772d82ad5ae5..94779ad1d8ba 100644 --- a/prow/external-plugins/cherrypicker/BUILD.bazel +++ b/prow/external-plugins/cherrypicker/BUILD.bazel @@ -1,4 +1,5 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") +load("@io_bazel_rules_docker//go:image.bzl", "go_image") go_library( name = "go_default_library", @@ -21,6 +22,16 @@ go_library( ], ) +go_image( + name = "image", + base = "@git-base//image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], +) + go_binary( name = "cherrypicker", embed = [":go_default_library"], diff --git a/prow/external-plugins/needs-rebase/BUILD.bazel b/prow/external-plugins/needs-rebase/BUILD.bazel index 4829f81c2eca..87cbdfe36e50 100644 --- a/prow/external-plugins/needs-rebase/BUILD.bazel +++ b/prow/external-plugins/needs-rebase/BUILD.bazel @@ -19,7 +19,11 @@ docker_push( go_image( name = "image", - binary = ":needs-rebase", + base = "@git-base//image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", visibility = ["//visibility:public"], ) diff --git a/prow/external-plugins/refresh/BUILD.bazel b/prow/external-plugins/refresh/BUILD.bazel index 2560dc5437fa..9420c7b5bb9b 100644 --- a/prow/external-plugins/refresh/BUILD.bazel +++ b/prow/external-plugins/refresh/BUILD.bazel @@ -1,4 +1,5 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@io_bazel_rules_docker//go:image.bzl", "go_image") go_library( name = "go_default_library", @@ -23,6 +24,16 @@ go_library( ], ) +go_image( + name = "image", + base = "@alpine-base//image", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], +) + go_binary( name = "refresh", embed = [":go_default_library"], From 8672f13f49e877e6909abce23e6643982acaee6d Mon Sep 17 00:00:00 2001 From: Sen Lu Date: Thu, 30 Aug 2018 14:51:37 -0700 Subject: [PATCH 13/42] fixing prow test targets --- .../jobs/kubernetes/sig-node/node-docker.yaml | 2 +- .../sig-node/sig-node-presubmit.yaml | 2 +- .../kubernetes/sig-testing/integration.yaml | 4 +- config/tests/jobs/jobs_test.go | 30 +++++++++ prow/config/BUILD.bazel | 4 +- prow/config/jobs_test.go | 61 +------------------ 6 files changed, 37 insertions(+), 66 deletions(-) diff --git a/config/jobs/kubernetes/sig-node/node-docker.yaml b/config/jobs/kubernetes/sig-node/node-docker.yaml index 52f607397b36..ed780f9209f8 100644 --- a/config/jobs/kubernetes/sig-node/node-docker.yaml +++ b/config/jobs/kubernetes/sig-node/node-docker.yaml @@ -74,4 +74,4 @@ periodics: - --timeout=180m env: - name: GOPATH - value: /go \ No newline at end of file + value: /go diff --git a/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml b/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml index 4989110d5c43..b5124ed4871a 100644 --- a/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml +++ b/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml @@ -133,4 +133,4 @@ presubmits: - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/containerd/cri-master/image-config.yaml resources: requests: - memory: "6Gi" \ No newline at end of file + memory: "6Gi" diff --git a/config/jobs/kubernetes/sig-testing/integration.yaml b/config/jobs/kubernetes/sig-testing/integration.yaml index 9dc5ff55d369..90f94a06b142 100644 --- a/config/jobs/kubernetes/sig-testing/integration.yaml +++ b/config/jobs/kubernetes/sig-testing/integration.yaml @@ -133,7 +133,7 @@ periodics: - image: gcr.io/k8s-testimages/bootstrap:v20180730-c3257f6c0 args: - --repo=k8s.io/kubernetes=release-1.10 - - --timeout=100" + - --timeout=100 - --scenario=kubernetes_verify - -- - --branch=release-1.10 @@ -157,7 +157,7 @@ periodics: - image: gcr.io/k8s-testimages/bootstrap:v20180730-c3257f6c0 args: - --repo=k8s.io/kubernetes=release-1.9 - - --timeout=100" + - --timeout=100 - --scenario=kubernetes_verify - -- - --branch=release-1.9 diff --git a/config/tests/jobs/jobs_test.go b/config/tests/jobs/jobs_test.go index 9fe99cec6056..0c900a40ae6b 100644 --- a/config/tests/jobs/jobs_test.go +++ b/config/tests/jobs/jobs_test.go @@ -31,6 +31,7 @@ import ( "regexp" "strings" "testing" + "time" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -910,6 +911,35 @@ func checkScenarioArgs(jobName, imageName string, args []string) error { } } + // timeout should be valid + bootstrap_timeout := 0 * time.Minute + kubetest_timeout := 0 * time.Minute + var err error + kubetest := false + for _, arg := range args { + if strings.HasPrefix(arg, "--timeout=") { + timeout := strings.SplitN(arg, "=", 2)[1] + if kubetest { + if kubetest_timeout, err = time.ParseDuration(timeout); err != nil { + return fmt.Errorf("jobs %s - invalid kubetest timeout : arg %s", jobName, arg) + } + } else { + if bootstrap_timeout, err = time.ParseDuration(timeout + "m"); err != nil { + return fmt.Errorf("jobs %s - invalid bootstrap timeout : arg %s", jobName, arg) + } + } + } + + if arg == "--" { + kubetest = true + } + } + + if bootstrap_timeout.Minutes()-kubetest_timeout.Minutes() < 20.0 { + return fmt.Errorf( + "jobs %s - kubetest timeout(%v), bootstrap timeout(%v): bootstrap timeout need to be 20min more than kubetest timeout!", jobName, kubetest_timeout, bootstrap_timeout) + } + return nil } diff --git a/prow/config/BUILD.bazel b/prow/config/BUILD.bazel index c87347a64ae1..825ed65780b1 100644 --- a/prow/config/BUILD.bazel +++ b/prow/config/BUILD.bazel @@ -15,10 +15,8 @@ go_test( "tide_test.go", ], data = [ - "//jobs", - "//mungegithub:configmaps", + "//config:prowjobs", "//prow:configs", - "//scenarios", ], embed = [":go_default_library"], deps = [ diff --git a/prow/config/jobs_test.go b/prow/config/jobs_test.go index f7fdc069c35c..b4b50acaac26 100644 --- a/prow/config/jobs_test.go +++ b/prow/config/jobs_test.go @@ -17,32 +17,20 @@ limitations under the License. package config import ( - "encoding/json" "flag" "fmt" - "io/ioutil" "os" "regexp" - "strings" "testing" "k8s.io/test-infra/prow/kube" ) var c *Config -var cj configJSON var configPath = flag.String("config", "../config.yaml", "Path to prow config") -var jobConfigPath = flag.String("job-config", "", "Path to prow job config") -var configJSONPath = flag.String("config-json", "../../jobs/config.json", "Path to prow job config") +var jobConfigPath = flag.String("job-config", "../../config/jobs", "Path to prow job config") var podRe = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) -type configJSON map[string]map[string]interface{} - -type JSONJob struct { - Scenario string `json:"scenario"` - Args []string `json:"args"` -} - // Consistent but meaningless order. func flattenJobs(jobs []Presubmit) []Presubmit { ret := jobs @@ -75,19 +63,6 @@ func checkOverlapBrancher(b1, b2 Brancher) bool { return false } -func readConfigJSON(path string) (config configJSON, err error) { - raw, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - config = configJSON{} - err = json.Unmarshal(raw, &config) - if err != nil { - return nil, err - } - return config, nil -} - func TestMain(m *testing.M) { flag.Parse() if *configPath == "" { @@ -102,29 +77,14 @@ func TestMain(m *testing.M) { } c = conf - if *configJSONPath != "" { - cj, err = readConfigJSON(*configJSONPath) - if err != nil { - fmt.Printf("Could not load jobs config: %v", err) - os.Exit(1) - } - } - os.Exit(m.Run()) } -// TODO(spxtr): Some of this is generic prowjob stuff and some of this is k8s- -// specific. Figure out which is which and split this up. func TestPresubmits(t *testing.T) { if len(c.Presubmits) == 0 { t.Fatalf("No jobs found in presubmit.yaml.") } - b, err := ioutil.ReadFile("../../jobs/config.json") - if err != nil { - t.Fatalf("Could not load jobs/config.json: %v", err) - } - var bootstrapConfig map[string]JSONJob - json.Unmarshal(b, &bootstrapConfig) + for _, rootJobs := range c.Presubmits { jobs := flattenJobs(rootJobs) for i, job := range jobs { @@ -163,23 +123,6 @@ func TestPresubmits(t *testing.T) { } } } - var scenario string - job.Name = strings.Replace(job.Name, "pull-security-kubernetes", "pull-kubernetes", 1) - if j, present := bootstrapConfig[job.Name]; present { - scenario = fmt.Sprintf("scenarios/%s.py", j.Scenario) - } - - // Ensure that jobs have a shell script of the same name. - if s, err := os.Stat(fmt.Sprintf("../../%s", scenario)); err != nil { - t.Errorf("Cannot find test-infra/%s for %s", scenario, job.Name) - } else { - if s.Mode()&0111 == 0 { - t.Errorf("Not executable: test-infra/%s (%o)", scenario, s.Mode()&0777) - } - if s.Mode()&0444 == 0 { - t.Errorf("Not readable: test-infra/%s (%o)", scenario, s.Mode()&0777) - } - } } } } From eabe4316473f679681d0af0cd8b47560a4f40099 Mon Sep 17 00:00:00 2001 From: Cole Wagner Date: Thu, 30 Aug 2018 15:38:10 -0700 Subject: [PATCH 14/42] Include 'require-matching-label' in hook binary. --- prow/hook/BUILD.bazel | 1 + prow/hook/plugins.go | 1 + 2 files changed, 2 insertions(+) diff --git a/prow/hook/BUILD.bazel b/prow/hook/BUILD.bazel index 6f0a7975e2a0..de0b80c29260 100644 --- a/prow/hook/BUILD.bazel +++ b/prow/hook/BUILD.bazel @@ -57,6 +57,7 @@ go_library( "//prow/plugins/override:go_default_library", "//prow/plugins/owners-label:go_default_library", "//prow/plugins/releasenote:go_default_library", + "//prow/plugins/require-matching-label:go_default_library", "//prow/plugins/requiresig:go_default_library", "//prow/plugins/shrug:go_default_library", "//prow/plugins/sigmention:go_default_library", diff --git a/prow/hook/plugins.go b/prow/hook/plugins.go index deb5d09a820d..e3cab164fddc 100644 --- a/prow/hook/plugins.go +++ b/prow/hook/plugins.go @@ -41,6 +41,7 @@ import ( _ "k8s.io/test-infra/prow/plugins/override" _ "k8s.io/test-infra/prow/plugins/owners-label" _ "k8s.io/test-infra/prow/plugins/releasenote" + _ "k8s.io/test-infra/prow/plugins/require-matching-label" _ "k8s.io/test-infra/prow/plugins/requiresig" _ "k8s.io/test-infra/prow/plugins/shrug" _ "k8s.io/test-infra/prow/plugins/sigmention" From 0cf3629964adfb4e1f4d6290e8a42b6a0e31a857 Mon Sep 17 00:00:00 2001 From: Cole Wagner Date: Thu, 30 Aug 2018 16:11:02 -0700 Subject: [PATCH 15/42] Bump Prow to version 'v20180830-eabe43164'. --- prow/cluster/branchprotector_cronjob.yaml | 2 +- prow/cluster/deck_deployment.yaml | 2 +- prow/cluster/hook_deployment.yaml | 2 +- prow/cluster/horologium_deployment.yaml | 2 +- prow/cluster/needs-rebase_deployment.yaml | 2 +- prow/cluster/plank_deployment.yaml | 2 +- prow/cluster/sinker_deployment.yaml | 2 +- prow/cluster/splice_deployment.yaml | 2 +- prow/cluster/starter.yaml | 10 +++++----- prow/cluster/tide_deployment.yaml | 2 +- prow/cluster/tot_deployment.yaml | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/prow/cluster/branchprotector_cronjob.yaml b/prow/cluster/branchprotector_cronjob.yaml index c2ac5efa2341..bd1afbd4099b 100644 --- a/prow/cluster/branchprotector_cronjob.yaml +++ b/prow/cluster/branchprotector_cronjob.yaml @@ -14,7 +14,7 @@ spec: spec: containers: - name: branchprotector - image: gcr.io/k8s-prow/branchprotector:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/branchprotector:v20180830-eabe43164 args: - --config-path=/etc/config/config.yaml - --job-config-path=/etc/job-config diff --git a/prow/cluster/deck_deployment.yaml b/prow/cluster/deck_deployment.yaml index 160b03d1df34..2d91dd53e7e8 100644 --- a/prow/cluster/deck_deployment.yaml +++ b/prow/cluster/deck_deployment.yaml @@ -34,7 +34,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: deck - image: gcr.io/k8s-prow/deck:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/deck:v20180830-eabe43164 imagePullPolicy: Always ports: - name: http diff --git a/prow/cluster/hook_deployment.yaml b/prow/cluster/hook_deployment.yaml index 1429f929548a..b10e711a4b03 100644 --- a/prow/cluster/hook_deployment.yaml +++ b/prow/cluster/hook_deployment.yaml @@ -34,7 +34,7 @@ spec: terminationGracePeriodSeconds: 180 containers: - name: hook - image: gcr.io/k8s-prow/hook:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/hook:v20180830-eabe43164 imagePullPolicy: Always args: - --dry-run=false diff --git a/prow/cluster/horologium_deployment.yaml b/prow/cluster/horologium_deployment.yaml index 7298ae7e7c38..143075254e55 100644 --- a/prow/cluster/horologium_deployment.yaml +++ b/prow/cluster/horologium_deployment.yaml @@ -29,7 +29,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: horologium - image: gcr.io/k8s-prow/horologium:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/horologium:v20180830-eabe43164 args: - --job-config-path=/etc/job-config volumeMounts: diff --git a/prow/cluster/needs-rebase_deployment.yaml b/prow/cluster/needs-rebase_deployment.yaml index 85f346587bed..a2035aa4593b 100644 --- a/prow/cluster/needs-rebase_deployment.yaml +++ b/prow/cluster/needs-rebase_deployment.yaml @@ -28,7 +28,7 @@ spec: terminationGracePeriodSeconds: 180 containers: - name: needs-rebase - image: gcr.io/k8s-prow/needs-rebase:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/needs-rebase:v20180830-eabe43164 imagePullPolicy: Always args: - --dry-run=false diff --git a/prow/cluster/plank_deployment.yaml b/prow/cluster/plank_deployment.yaml index 66839e45fd4b..fae8928048cb 100644 --- a/prow/cluster/plank_deployment.yaml +++ b/prow/cluster/plank_deployment.yaml @@ -28,7 +28,7 @@ spec: # serviceAccountName: "plank" # Uncomment for use with RBAC containers: - name: plank - image: gcr.io/k8s-prow/plank:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/plank:v20180830-eabe43164 args: - --tot-url=http://tot - --build-cluster=/etc/cluster/cluster diff --git a/prow/cluster/sinker_deployment.yaml b/prow/cluster/sinker_deployment.yaml index 03b3c59dec6f..35aa1b19daad 100644 --- a/prow/cluster/sinker_deployment.yaml +++ b/prow/cluster/sinker_deployment.yaml @@ -17,7 +17,7 @@ spec: args: - --build-cluster=/etc/cluster/cluster - --job-config-path=/etc/job-config - image: gcr.io/k8s-prow/sinker:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/sinker:v20180830-eabe43164 volumeMounts: - mountPath: /etc/cluster name: cluster diff --git a/prow/cluster/splice_deployment.yaml b/prow/cluster/splice_deployment.yaml index 83100fba18a8..a89713b41873 100644 --- a/prow/cluster/splice_deployment.yaml +++ b/prow/cluster/splice_deployment.yaml @@ -13,7 +13,7 @@ spec: spec: containers: - name: splice - image: gcr.io/k8s-prow/splice:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/splice:v20180830-eabe43164 args: - --job-config-path=/etc/job-config volumeMounts: diff --git a/prow/cluster/starter.yaml b/prow/cluster/starter.yaml index 517215a11862..008bd497566d 100644 --- a/prow/cluster/starter.yaml +++ b/prow/cluster/starter.yaml @@ -97,7 +97,7 @@ spec: terminationGracePeriodSeconds: 180 containers: - name: hook - image: gcr.io/k8s-prow/hook:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/hook:v20180830-eabe43164 imagePullPolicy: Always args: - --dry-run=false @@ -158,7 +158,7 @@ spec: serviceAccountName: "plank" containers: - name: plank - image: gcr.io/k8s-prow/plank:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/plank:v20180830-eabe43164 args: - --dry-run=false volumeMounts: @@ -192,7 +192,7 @@ spec: serviceAccountName: "sinker" containers: - name: sinker - image: gcr.io/k8s-prow/sinker:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/sinker:v20180830-eabe43164 volumeMounts: - name: config mountPath: /etc/config @@ -224,7 +224,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: deck - image: gcr.io/k8s-prow/deck:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/deck:v20180830-eabe43164 args: - --hook-url=http://hook:8888/plugin-help ports: @@ -268,7 +268,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: horologium - image: gcr.io/k8s-prow/horologium:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/horologium:v20180830-eabe43164 volumeMounts: - name: config mountPath: /etc/config diff --git a/prow/cluster/tide_deployment.yaml b/prow/cluster/tide_deployment.yaml index 2e1853ed6e01..5575bed5c4c8 100644 --- a/prow/cluster/tide_deployment.yaml +++ b/prow/cluster/tide_deployment.yaml @@ -28,7 +28,7 @@ spec: # serviceAccountName: "tide" # Uncomment for use with RBAC containers: - name: tide - image: gcr.io/k8s-prow/tide:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/tide:v20180830-eabe43164 args: - --dry-run=false - --github-endpoint=http://ghproxy diff --git a/prow/cluster/tot_deployment.yaml b/prow/cluster/tot_deployment.yaml index 82e38a33511b..4c0332bcb375 100644 --- a/prow/cluster/tot_deployment.yaml +++ b/prow/cluster/tot_deployment.yaml @@ -57,7 +57,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: tot - image: gcr.io/k8s-prow/tot:v20180827-b14cf3e25 + image: gcr.io/k8s-prow/tot:v20180830-eabe43164 imagePullPolicy: Always args: - -storage=/store/tot.json From 4494b5e52a8e06168817c2b3d79c6f49d0a519e6 Mon Sep 17 00:00:00 2001 From: cheyang Date: Fri, 31 Aug 2018 13:09:54 +0800 Subject: [PATCH 16/42] prow: Add kubeflow/arena repository --- config/jobs/kubeflow/kubeflow-postsubmits.yaml | 10 ++++++++++ config/jobs/kubeflow/kubeflow-presubmits.yaml | 11 +++++++++++ testgrid/cmd/configurator/config_test.go | 1 + testgrid/config.yaml | 11 +++++++++++ 4 files changed, 33 insertions(+) diff --git a/config/jobs/kubeflow/kubeflow-postsubmits.yaml b/config/jobs/kubeflow/kubeflow-postsubmits.yaml index 62989a74f836..9a2a48e772fd 100644 --- a/config/jobs/kubeflow/kubeflow-postsubmits.yaml +++ b/config/jobs/kubeflow/kubeflow-postsubmits.yaml @@ -159,3 +159,13 @@ postsubmits: - image: gcr.io/kubeflow-ci/test-worker:latest imagePullPolicy: Always + kubeflow/arena: + - name: kubeflow-arena-postsubmit + branches: + - master + labels: + preset-service-account: "true" + spec: + containers: + - image: gcr.io/kubeflow-ci/test-worker:latest + imagePullPolicy: Always diff --git a/config/jobs/kubeflow/kubeflow-presubmits.yaml b/config/jobs/kubeflow/kubeflow-presubmits.yaml index b9276828bf75..e17239f4838f 100644 --- a/config/jobs/kubeflow/kubeflow-presubmits.yaml +++ b/config/jobs/kubeflow/kubeflow-presubmits.yaml @@ -187,3 +187,14 @@ presubmits: - image: gcr.io/kubeflow-ci/test-worker:latest imagePullPolicy: Always + kubeflow/arena: + - name: kubeflow-arena-presubmit + always_run: true # Run for every PR, or only when requested. + branches: + - master + labels: + preset-service-account: "true" + spec: + containers: + - image: gcr.io/kubeflow-ci/test-worker:latest + imagePullPolicy: Always diff --git a/testgrid/cmd/configurator/config_test.go b/testgrid/cmd/configurator/config_test.go index 1edc795c6fc6..8ce3743a2a87 100644 --- a/testgrid/cmd/configurator/config_test.go +++ b/testgrid/cmd/configurator/config_test.go @@ -370,6 +370,7 @@ func TestJobsTestgridEntryMatch(t *testing.T) { "bazelbuild/rules_k8s", "google/cadvisor", "helm/charts", + "kubeflow/arena", "kubeflow/caffe2-operator", "kubeflow/examples", "kubeflow/experimental-beagle", diff --git a/testgrid/config.yaml b/testgrid/config.yaml index 91c9649e2244..2955e083637e 100644 --- a/testgrid/config.yaml +++ b/testgrid/config.yaml @@ -2104,6 +2104,11 @@ test_groups: num_columns_recent: 30 - name: kubeflow-chainer-operator-postsubmit gcs_prefix: kubernetes-jenkins/logs/kubeflow_chainer-operator/kubeflow-chainer-operator-postsubmit +- name: kubeflow-arena-presubmit + gcs_prefix: kubernetes-jenkins/pr-logs/directory/kubeflow-arena-presubmit + num_columns_recent: 30 +- name: kubeflow-arena-postsubmit + gcs_prefix: kubernetes-jenkins/logs/kubeflow_arena/kubeflow-arena-postsubmit - name: tf-minigo-periodic gcs_prefix: kubernetes-jenkins/logs/tf-minigo-periodic - name: tf-minigo-presubmit @@ -4755,6 +4760,12 @@ dashboards: - name: kubeflow-chainer-operator-presubmit description: Presubmit tests for kubeflow/chainer-operator. test_group_name: kubeflow-chainer-operator-presubmit + - name: kubeflow-arena-postsubmit + description: Postsubmit tests for kubeflow/arena. + test_group_name: kubeflow-arena-postsubmit + - name: kubeflow-arena-presubmit + description: Presubmit tests for kubeflow/arena. + test_group_name: kubeflow-arena-presubmit - name: spark-periodic-default-gke description: Periodic builds and testing of Apache Spark on default version of GKE. test_group_name: spark-periodic-default-gke From b3696212eeb364b1eab1cd0fe77ef617d05cf929 Mon Sep 17 00:00:00 2001 From: Christoph Blecker Date: Fri, 31 Aug 2018 11:35:16 -0700 Subject: [PATCH 17/42] Unify kubernetes/kubectl tide merge requirements --- prow/config.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/prow/config.yaml b/prow/config.yaml index 8dc584f41e46..17cc0022c593 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -275,6 +275,7 @@ tide: - kubernetes/kube-openapi - kubernetes/kube-state-metrics - kubernetes/kubeadm + - kubernetes/kubectl - kubernetes/kubernetes-docs-ja - kubernetes/kubernetes-docs-ko - kubernetes/kubernetes-docs-zh @@ -344,15 +345,6 @@ tide: - do-not-merge/work-in-progress - needs-ok-to-test - needs-rebase - - repos: - - kubernetes/kubectl - labels: - - "cncf-cla: yes" - missingLabels: - - do-not-merge/hold - - do-not-merge/invalid-owners-file - - do-not-merge/work-in-progress - reviewApprovedRequired: true - repos: - kubernetes-sigs/federation-v2 - kubernetes-sigs/kubeadm-dind-cluster From ed8085c51ad8a8078c8bc8da13c90571f26f81cb Mon Sep 17 00:00:00 2001 From: Christoph Blecker Date: Fri, 31 Aug 2018 11:36:29 -0700 Subject: [PATCH 18/42] Move kubernetes-sigs repos into main tide query --- prow/config.yaml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/prow/config.yaml b/prow/config.yaml index 17cc0022c593..fa3ba27a414e 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -331,6 +331,10 @@ tide: - kubernetes-incubator/kubespray - kubernetes-incubator/service-catalog - client-go/unofficial-docs + - kubernetes-sigs/federation-v2 + - kubernetes-sigs/kubeadm-dind-cluster + - kubernetes-sigs/testing_frameworks + - kubernetes-sigs/poseidon labels: - lgtm - approved @@ -345,20 +349,6 @@ tide: - do-not-merge/work-in-progress - needs-ok-to-test - needs-rebase - - repos: - - kubernetes-sigs/federation-v2 - - kubernetes-sigs/kubeadm-dind-cluster - - kubernetes-sigs/testing_frameworks - - kubernetes-sigs/poseidon - labels: - - lgtm - - approved - - "cncf-cla: yes" - missingLabels: - - do-not-merge/hold - - do-not-merge/invalid-owners-file - - do-not-merge/work-in-progress - - needs-ok-to-test merge_method: helm/charts: squash kubeflow: squash From 0e5b6f431a856573addbb13abd2a825753acde88 Mon Sep 17 00:00:00 2001 From: Christoph Blecker Date: Fri, 31 Aug 2018 11:38:10 -0700 Subject: [PATCH 19/42] alpha sort tide query lists --- prow/config.yaml | 100 +++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/prow/config.yaml b/prow/config.yaml index fa3ba27a414e..52bc3051d693 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -252,6 +252,49 @@ tide: - needs-ok-to-test - needs-rebase - repos: + - client-go/unofficial-docs + - kubernetes-client/csharp + - kubernetes-client/gen + - kubernetes-client/go + - kubernetes-client/go-base + - kubernetes-client/haskell + - kubernetes-client/java + - kubernetes-client/javascript + - kubernetes-client/python + - kubernetes-client/python-base + - kubernetes-client/ruby + - kubernetes-csi/csi-test + - kubernetes-csi/docs + - kubernetes-csi/driver-registrar + - kubernetes-csi/drivers + - kubernetes-csi/external-attacher + - kubernetes-csi/external-provisioner + - kubernetes-csi/external-snapshotter + - kubernetes-csi/kubernetes-csi.github.io + - kubernetes-csi/livenessprobe + - kubernetes-incubator/ip-masq-agent + - kubernetes-incubator/kubespray + - kubernetes-incubator/service-catalog + - kubernetes-sigs/application + - kubernetes-sigs/architecture-tracking + - kubernetes-sigs/aws-encryption-provider + - kubernetes-sigs/cluster-api + - kubernetes-sigs/cluster-api-provider-aws + - kubernetes-sigs/cluster-api-provider-gcp + - kubernetes-sigs/cluster-api-provider-openstack + - kubernetes-sigs/cluster-api-provider-vsphere + - kubernetes-sigs/contributor-playground + - kubernetes-sigs/contributor-site + - kubernetes-sigs/controller-runtime + - kubernetes-sigs/controller-tools + - kubernetes-sigs/federation-v2 + - kubernetes-sigs/gcp-compute-persistent-disk-csi-driver + - kubernetes-sigs/gcp-filestore-csi-driver + - kubernetes-sigs/kubeadm-dind-cluster + - kubernetes-sigs/kubebuilder + - kubernetes-sigs/kustomize + - kubernetes-sigs/poseidon + - kubernetes-sigs/testing_frameworks - kubernetes/client-go - kubernetes/cloud-provider-aws - kubernetes/cloud-provider-azure @@ -292,49 +335,6 @@ tide: - kubernetes/test-infra - kubernetes/utils - kubernetes/website - - kubernetes-client/csharp - - kubernetes-client/gen - - kubernetes-client/go - - kubernetes-client/go-base - - kubernetes-client/haskell - - kubernetes-client/java - - kubernetes-client/javascript - - kubernetes-client/python - - kubernetes-client/python-base - - kubernetes-client/ruby - - kubernetes-csi/csi-test - - kubernetes-csi/docs - - kubernetes-csi/driver-registrar - - kubernetes-csi/drivers - - kubernetes-csi/external-attacher - - kubernetes-csi/external-provisioner - - kubernetes-csi/external-snapshotter - - kubernetes-csi/kubernetes-csi.github.io - - kubernetes-csi/livenessprobe - - kubernetes-sigs/application - - kubernetes-sigs/architecture-tracking - - kubernetes-sigs/aws-encryption-provider - - kubernetes-sigs/cluster-api - - kubernetes-sigs/cluster-api-provider-aws - - kubernetes-sigs/cluster-api-provider-gcp - - kubernetes-sigs/cluster-api-provider-openstack - - kubernetes-sigs/cluster-api-provider-vsphere - - kubernetes-sigs/contributor-playground - - kubernetes-sigs/contributor-site - - kubernetes-sigs/controller-runtime - - kubernetes-sigs/controller-tools - - kubernetes-sigs/gcp-compute-persistent-disk-csi-driver - - kubernetes-sigs/gcp-filestore-csi-driver - - kubernetes-sigs/kubebuilder - - kubernetes-sigs/kustomize - - kubernetes-incubator/ip-masq-agent - - kubernetes-incubator/kubespray - - kubernetes-incubator/service-catalog - - client-go/unofficial-docs - - kubernetes-sigs/federation-v2 - - kubernetes-sigs/kubeadm-dind-cluster - - kubernetes-sigs/testing_frameworks - - kubernetes-sigs/poseidon labels: - lgtm - approved @@ -352,19 +352,19 @@ tide: merge_method: helm/charts: squash kubeflow: squash - kubernetes/dashboard: squash - kubernetes/kube-deploy: squash - kubernetes/website: squash - kubernetes/kubernetes-docs-ja: squash - kubernetes/kubernetes-docs-ko: squash kubernetes-client/csharp: squash kubernetes-client/gen: squash - kubernetes-sigs/cluster-api: squash + kubernetes-incubator/service-catalog: squash kubernetes-sigs/cluster-api-provider-aws: squash kubernetes-sigs/cluster-api-provider-gcp: squash kubernetes-sigs/cluster-api-provider-openstack: squash kubernetes-sigs/cluster-api-provider-vsphere: squash - kubernetes-incubator/service-catalog: squash + kubernetes-sigs/cluster-api: squash + kubernetes/dashboard: squash + kubernetes/kube-deploy: squash + kubernetes/kubernetes-docs-ja: squash + kubernetes/kubernetes-docs-ko: squash + kubernetes/website: squash target_url: https://prow.k8s.io/tide.html blocker_label: merge-blocker From 91f4c1bdaec3a8aea106a11c861d4ad0ea4483cc Mon Sep 17 00:00:00 2001 From: Shyam Jeedigunta Date: Fri, 31 Aug 2018 19:20:59 +0200 Subject: [PATCH 20/42] Use boskos for kubemark canary job --- boskos/resources.yaml | 1 + config/jobs/kubernetes/sig-testing/kubetest-canaries.yaml | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/boskos/resources.yaml b/boskos/resources.yaml index 5ff65a28aa68..2d35c280d4c2 100644 --- a/boskos/resources.yaml +++ b/boskos/resources.yaml @@ -352,6 +352,7 @@ resources: - k8s-jenkins-charts - k8s-jenkins-cvm - k8s-jenkins-garbagecollector + - k8s-jenkins-gci-kubemark - k8s-jkns-e2e-bazel - k8s-jkns-e2e-etcd3 - k8s-jkns-e2e-gce diff --git a/config/jobs/kubernetes/sig-testing/kubetest-canaries.yaml b/config/jobs/kubernetes/sig-testing/kubetest-canaries.yaml index c42b79c1733d..7c0256553020 100644 --- a/config/jobs/kubernetes/sig-testing/kubetest-canaries.yaml +++ b/config/jobs/kubernetes/sig-testing/kubetest-canaries.yaml @@ -283,8 +283,6 @@ periodics: - --gcp-master-size=n1-standard-2 - --gcp-node-size=n1-standard-4 - --gcp-nodes=3 - - --gcp-project=k8s-jenkins-gci-kubemark - - --gcp-zone=us-central1-f - --kubemark - --kubemark-nodes=100 - --provider=gce From 4804ce031459a615ec7cb5d438d4fa9c0c0f800d Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 31 Aug 2018 12:34:18 -0700 Subject: [PATCH 21/42] add back gcp zone to kubemark canary --- config/jobs/kubernetes/sig-testing/kubetest-canaries.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/jobs/kubernetes/sig-testing/kubetest-canaries.yaml b/config/jobs/kubernetes/sig-testing/kubetest-canaries.yaml index 7c0256553020..06c5b91e972e 100644 --- a/config/jobs/kubernetes/sig-testing/kubetest-canaries.yaml +++ b/config/jobs/kubernetes/sig-testing/kubetest-canaries.yaml @@ -283,6 +283,7 @@ periodics: - --gcp-master-size=n1-standard-2 - --gcp-node-size=n1-standard-4 - --gcp-nodes=3 + - --gcp-zone=us-central1-f - --kubemark - --kubemark-nodes=100 - --provider=gce From 2cd7e50e350445373c12ea87d73229276e086a66 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 31 Aug 2018 14:05:59 -0700 Subject: [PATCH 22/42] load welcome plugin in hook --- prow/hook/BUILD.bazel | 1 + prow/hook/plugins.go | 1 + 2 files changed, 2 insertions(+) diff --git a/prow/hook/BUILD.bazel b/prow/hook/BUILD.bazel index de0b80c29260..868e2763e569 100644 --- a/prow/hook/BUILD.bazel +++ b/prow/hook/BUILD.bazel @@ -68,6 +68,7 @@ go_library( "//prow/plugins/trigger:go_default_library", "//prow/plugins/updateconfig:go_default_library", "//prow/plugins/verify-owners:go_default_library", + "//prow/plugins/welcome:go_default_library", "//prow/plugins/wip:go_default_library", "//prow/plugins/yuks:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", diff --git a/prow/hook/plugins.go b/prow/hook/plugins.go index e3cab164fddc..f63f4c62c5b3 100644 --- a/prow/hook/plugins.go +++ b/prow/hook/plugins.go @@ -52,6 +52,7 @@ import ( _ "k8s.io/test-infra/prow/plugins/trigger" _ "k8s.io/test-infra/prow/plugins/updateconfig" _ "k8s.io/test-infra/prow/plugins/verify-owners" + _ "k8s.io/test-infra/prow/plugins/welcome" _ "k8s.io/test-infra/prow/plugins/wip" _ "k8s.io/test-infra/prow/plugins/yuks" ) From 047e668073c308c3160de534a3bd39b7e6d98e7c Mon Sep 17 00:00:00 2001 From: Sen Lu Date: Thu, 30 Aug 2018 17:33:37 -0700 Subject: [PATCH 23/42] remove rest of the goodies to config/jobs --- .../spark-integration/OWNERS | 2 + .../spark-integration/spark-config.yaml | 72 ++ config/jobs/kubernetes-sigs/kustomize/OWNERS | 8 + .../kustomize/kustomize-config.yaml | 29 + .../jobs/kubernetes/sig-api-machinery/OWNERS | 3 + .../sig-api-machinery-config.yaml | 23 + config/jobs/kubernetes/sig-apps/OWNERS | 4 + .../kubernetes/sig-apps/sig-apps-config.yaml | 77 ++ config/jobs/kubernetes/sig-autoscaling/OWNERS | 3 + .../sig-autoscaling-config.yaml | 313 ++++++++ .../k8s-upgrade-gce.yaml | 2 +- .../sig-cluster-lifecycle-misc.yaml | 70 ++ .../kubernetes/sig-instrumentation/OWNERS | 3 + .../sig-instrumentation-config.yaml | 276 +++++++ config/jobs/kubernetes/sig-scheduling/OWNERS | 3 + .../sig-scheduling/sig-scheduling-config.yaml | 22 + jobs/BUILD.bazel | 26 - jobs/config.json | 630 ---------------- jobs/config_test.py | 709 ------------------ jobs/env_gc.py | 77 -- jobs/move_extract.py | 98 --- prow/config.yaml | 492 ------------ testgrid/config.yaml | 5 - 23 files changed, 909 insertions(+), 2038 deletions(-) create mode 100644 config/jobs/apache-spark-on-k8s/spark-integration/OWNERS create mode 100644 config/jobs/apache-spark-on-k8s/spark-integration/spark-config.yaml create mode 100644 config/jobs/kubernetes-sigs/kustomize/OWNERS create mode 100644 config/jobs/kubernetes-sigs/kustomize/kustomize-config.yaml create mode 100644 config/jobs/kubernetes/sig-api-machinery/OWNERS create mode 100644 config/jobs/kubernetes/sig-api-machinery/sig-api-machinery-config.yaml create mode 100644 config/jobs/kubernetes/sig-apps/OWNERS create mode 100644 config/jobs/kubernetes/sig-apps/sig-apps-config.yaml create mode 100644 config/jobs/kubernetes/sig-autoscaling/OWNERS create mode 100644 config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-config.yaml create mode 100644 config/jobs/kubernetes/sig-cluster-lifecycle/sig-cluster-lifecycle-misc.yaml create mode 100644 config/jobs/kubernetes/sig-instrumentation/OWNERS create mode 100644 config/jobs/kubernetes/sig-instrumentation/sig-instrumentation-config.yaml create mode 100644 config/jobs/kubernetes/sig-scheduling/OWNERS create mode 100644 config/jobs/kubernetes/sig-scheduling/sig-scheduling-config.yaml delete mode 100755 jobs/config_test.py delete mode 100644 jobs/env_gc.py delete mode 100755 jobs/move_extract.py diff --git a/config/jobs/apache-spark-on-k8s/spark-integration/OWNERS b/config/jobs/apache-spark-on-k8s/spark-integration/OWNERS new file mode 100644 index 000000000000..f00a2b6606b1 --- /dev/null +++ b/config/jobs/apache-spark-on-k8s/spark-integration/OWNERS @@ -0,0 +1,2 @@ +approvers: +- foxish diff --git a/config/jobs/apache-spark-on-k8s/spark-integration/spark-config.yaml b/config/jobs/apache-spark-on-k8s/spark-integration/spark-config.yaml new file mode 100644 index 000000000000..46bf1cd86a9d --- /dev/null +++ b/config/jobs/apache-spark-on-k8s/spark-integration/spark-config.yaml @@ -0,0 +1,72 @@ +periodics: +- interval: 2h + name: spark-periodic-default-gke + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-dind-enabled: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - "--job=$(JOB_NAME)" + - "--repo=github.com/apache-spark-on-k8s/spark-integration=master" + - "--upload=gs://kubernetes-jenkins/logs/" + - "--timeout=150" + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --cluster= + - --deployment=gke + - --extract=gke + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --gke-environment=prod + - --provider=gke + - --test=false + - --test-cmd=../e2e/e2e-prow.sh + - --test-cmd-name=spark-integration + - --timeout=120m + # docker-in-docker needs privileged mode + securityContext: + privileged: true + resources: + requests: + cpu: 6 + memory: "8Gi" + +- interval: 2h + name: spark-periodic-latest-gke + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-dind-enabled: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - "--job=$(JOB_NAME)" + - "--repo=github.com/apache-spark-on-k8s/spark-integration=master" + - "--upload=gs://kubernetes-jenkins/logs/" + - "--timeout=150" + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --cluster= + - --deployment=gke + - --extract=gke-latest + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --gke-environment=prod + - --provider=gke + - --test=false + - --test-cmd=../e2e/e2e-prow.sh + - --test-cmd-name=spark-integration + - --timeout=120m + # docker-in-docker needs privileged mode + securityContext: + privileged: true + resources: + requests: + cpu: 6 + memory: "8Gi" diff --git a/config/jobs/kubernetes-sigs/kustomize/OWNERS b/config/jobs/kubernetes-sigs/kustomize/OWNERS new file mode 100644 index 000000000000..d47900722a15 --- /dev/null +++ b/config/jobs/kubernetes-sigs/kustomize/OWNERS @@ -0,0 +1,8 @@ +approvers: +- grodrigues3 +- droot +- justinsb +- liujingfang1 +- mengqiy +- monopole +- pwittrock diff --git a/config/jobs/kubernetes-sigs/kustomize/kustomize-config.yaml b/config/jobs/kubernetes-sigs/kustomize/kustomize-config.yaml new file mode 100644 index 000000000000..88f397b7e462 --- /dev/null +++ b/config/jobs/kubernetes-sigs/kustomize/kustomize-config.yaml @@ -0,0 +1,29 @@ +periodics: +- interval: 2h + name: ci-kustomize-periodic-default-gke + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - "--job=$(JOB_NAME)" + - "--root=/go/src" + - "--repo=github.com/kubernetes-sigs/kustomize" + - "--upload=gs://kubernetes-jenkins/logs/" + - "--timeout=150" + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --cluster= + - --deployment=gke + - --extract=gke + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --gke-environment=prod + - --provider=gke + - --test=false + - --test-cmd=../examples/integration_tests.sh + - --test-cmd-name=kustomize-integration + - --timeout=120m diff --git a/config/jobs/kubernetes/sig-api-machinery/OWNERS b/config/jobs/kubernetes/sig-api-machinery/OWNERS new file mode 100644 index 000000000000..d40a829111d1 --- /dev/null +++ b/config/jobs/kubernetes/sig-api-machinery/OWNERS @@ -0,0 +1,3 @@ +approvers: +- lavalamp +- deads2k diff --git a/config/jobs/kubernetes/sig-api-machinery/sig-api-machinery-config.yaml b/config/jobs/kubernetes/sig-api-machinery/sig-api-machinery-config.yaml new file mode 100644 index 000000000000..9cc542bb0b08 --- /dev/null +++ b/config/jobs/kubernetes/sig-api-machinery/sig-api-machinery-config.yaml @@ -0,0 +1,23 @@ +periodics: +- interval: 30m + name: ci-kubernetes-e2e-gci-gce-proto + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=70 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --env=STORAGE_MEDIA_TYPE=application/vnd.kubernetes.protobuf + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --ginkgo-parallel=25 + - --provider=gce + - --test_args=--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\] --minStartupPods=8 + - --timeout=50m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master diff --git a/config/jobs/kubernetes/sig-apps/OWNERS b/config/jobs/kubernetes/sig-apps/OWNERS new file mode 100644 index 000000000000..5be21a273a82 --- /dev/null +++ b/config/jobs/kubernetes/sig-apps/OWNERS @@ -0,0 +1,4 @@ +approvers: +- mattfarina +- prydonius +- kow3ns diff --git a/config/jobs/kubernetes/sig-apps/sig-apps-config.yaml b/config/jobs/kubernetes/sig-apps/sig-apps-config.yaml new file mode 100644 index 000000000000..9a7e2ddf98a8 --- /dev/null +++ b/config/jobs/kubernetes/sig-apps/sig-apps-config.yaml @@ -0,0 +1,77 @@ +periodics: +- interval: 2h + name: application-periodic-default-gke + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-dind-enabled: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - --repo=github.com/kubernetes-sigs/application=master + - --upload=gs://kubernetes-jenkins/logs/ + - --timeout=150 + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --cluster= + - --deployment=gke + - --extract=gke + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --gke-environment=prod + - --provider=gke + - --test=false + - --test-cmd=../e2e/e2e-prow.sh + - --test-cmd-name=application-e2e + - --timeout=120m + # docker-in-docker needs privileged mode + securityContext: + privileged: true + resources: + requests: + cpu: 6 + memory: "8Gi" + +- interval: 30m + name: ci-kubernetes-e2e-gce-taint-evict + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=80 + - --bare + - --scenario=kubernetes_e2e + - -- + - --env-file=jobs/env/ci-kubernetes-e2e-gce-taint-evict.env + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.focus=\[Feature:TaintEviction\] --minStartupPods=8 + - --timeout=50m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + + +- interval: 30m + name: ci-kubernetes-e2e-gci-gce-statefulset + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=110 + - --bare + - --scenario=kubernetes_e2e + - -- + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.focus=\[Feature:StatefulSet\] --minStartupPods=8 + - --timeout=90m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master diff --git a/config/jobs/kubernetes/sig-autoscaling/OWNERS b/config/jobs/kubernetes/sig-autoscaling/OWNERS new file mode 100644 index 000000000000..a609698e1d92 --- /dev/null +++ b/config/jobs/kubernetes/sig-autoscaling/OWNERS @@ -0,0 +1,3 @@ +approvers: +- directxman12 +- mwielgus diff --git a/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-config.yaml b/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-config.yaml new file mode 100644 index 000000000000..b9d98f1c1ef3 --- /dev/null +++ b/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-config.yaml @@ -0,0 +1,313 @@ +periodics: +- name: ci-kubernetes-e2e-autoscaling-vpa-actuation + interval: 2h + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-dind-enabled: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - --repo=k8s.io/autoscaler=master + - --root=/go/src + - --timeout=80 + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --provider=gce + - --test=false + - --test-cmd=../vertical-pod-autoscaler/hack/run-e2e.sh + - --test-cmd-args=actuation + - --timeout=60m + # docker-in-docker needs privileged mode + securityContext: + privileged: true + +- name: ci-kubernetes-e2e-autoscaling-vpa-admission-controller + interval: 2h + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-dind-enabled: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - --repo=k8s.io/autoscaler=master + - --root=/go/src + - --timeout=80 + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --provider=gce + - --test=false + - --test-cmd=../vertical-pod-autoscaler/hack/run-e2e.sh + - --test-cmd-args=admission-controller + - --timeout=60m + # docker-in-docker needs privileged mode + securityContext: + privileged: true + +- name: ci-kubernetes-e2e-autoscaling-vpa-full + interval: 2h + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-dind-enabled: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - --repo=k8s.io/autoscaler=master + - --root=/go/src + - --timeout=80 + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --provider=gce + - --test=false + - --test-cmd=../vertical-pod-autoscaler/hack/run-e2e.sh + - --test-cmd-args=full-vpa + - --timeout=60m + # docker-in-docker needs privileged mode + securityContext: + privileged: true + +- name: ci-kubernetes-e2e-autoscaling-vpa-recommender + interval: 2h + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-dind-enabled: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - --repo=k8s.io/autoscaler=master + - --root=/go/src + - --timeout=80 + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --provider=gce + - --test=false + - --test-cmd=../vertical-pod-autoscaler/hack/run-e2e.sh + - --test-cmd-args=recommender + - --timeout=60m + # docker-in-docker needs privileged mode + securityContext: + privileged: true + +- name: ci-kubernetes-e2e-autoscaling-vpa-updater + interval: 2h + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-dind-enabled: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - --repo=k8s.io/autoscaler=master + - --root=/go/src + - --timeout=80 + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --provider=gce + - --test=false + - --test-cmd=../vertical-pod-autoscaler/hack/run-e2e.sh + - --test-cmd-args=updater + - --timeout=60m + # docker-in-docker needs privileged mode + securityContext: + privileged: true + +- interval: 30m + name: ci-kubernetes-e2e-gci-gce-autoscaling + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=350 + - --bare + - --scenario=kubernetes_e2e + - -- + - --cluster=ca + - --env-file=jobs/env/ci-kubernetes-e2e-gci-gce-autoscaling.env + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-nodes=3 + - --gcp-project=k8s-jkns-gci-autoscaling + - --gcp-zone=us-central1-f + - --provider=gce + - --runtime-config=scheduling.k8s.io/v1alpha1=true + - --test_args=--ginkgo.focus=\[Feature:ClusterSizeAutoscalingScaleUp\]|\[Feature:ClusterSizeAutoscalingScaleDown\]|\[Feature:InitialResources\] --ginkgo.skip=\[Flaky\] --minStartupPods=8 + - --timeout=300m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gci-gce-autoscaling-hpa + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=350 + - --bare + - --scenario=kubernetes_e2e + - -- + - --cluster=hpa + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-project=k8s-jkns-gci-autoscaling + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.focus=\[Feature:CustomMetricsAutoscaling\] --ginkgo.skip=\[Flaky\] --minStartupPods=8 + - --timeout=300m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gci-gce-autoscaling-migs + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=350 + - --bare + - --scenario=kubernetes_e2e + - -- + - --cluster=ca + - --env-file=jobs/env/ci-kubernetes-e2e-gci-gce-autoscaling-migs.env + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-nodes=3 + - --gcp-project=k8s-jkns-gci-autoscaling-migs + - --gcp-zone=us-central1-f + - --provider=gce + - --runtime-config=scheduling.k8s.io/v1alpha1=true + - --test_args=--ginkgo.focus=\[Feature:ClusterSizeAutoscalingScaleUp\]|\[Feature:ClusterSizeAutoscalingScaleDown\] --ginkgo.skip=\[Flaky\] --minStartupPods=8 + - --timeout=300m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gci-gce-autoscaling-migs-hpa + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=350 + - --bare + - --scenario=kubernetes_e2e + - -- + - --cluster=hpa + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-project=k8s-jkns-gci-autoscaling-migs + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.focus=\[Feature:CustomMetricsAutoscaling\] --ginkgo.skip=\[Flaky\] --minStartupPods=8 + - --timeout=300m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gci-gke-autoscaling + labels: + preset-gke-alpha-service: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=620 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --cluster=ca + - --deployment=gke + - --extract=ci/latest + - --gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging + - --gcp-node-image=gci + - --gcp-project=k8s-e2e-gci-gke-autoscaling + - --gcp-zone=us-central1-f + - --gke-environment=test + - --provider=gke + - --test_args=--ginkgo.focus=\[Feature:ClusterSizeAutoscalingScaleUp\]|\[Feature:ClusterSizeAutoscalingScaleDown\]|\[Feature:ClusterSizeAutoscalingScaleWithNAP\] --ginkgo.skip=\[Flaky\] --minStartupPods=8 + - --timeout=600m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gci-gke-autoscaling-hpa + labels: + preset-gke-alpha-service: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=440 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --cluster=hpa + - --deployment=gke + - --extract=ci/latest + - --gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging + - --gcp-node-image=gci + - --gcp-project=k8s-e2e-gci-gke-autoscaling + - --gcp-zone=us-central1-f + - --gke-environment=test + - --provider=gke + - --test_args=--ginkgo.focus=\[Feature:CustomMetricsAutoscaling\] --ginkgo.skip=\[Flaky\] --minStartupPods=8 + - --timeout=420m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gci-gke-autoscaling-regional + labels: + preset-gke-alpha-service: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=440 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --deployment=gke + - --extract=ci/latest + - --gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging + - --gcp-node-image=gci + - --gcp-project=k8s-jkns-gci-autoscaling-regio + - --gcp-region=us-central1 + - --gke-command-group=beta + - --gke-environment=test + - --gke-single-zone-node-instance-group=false + - --provider=gke + - --test_args=--gce-multizone=true --ginkgo.focus=\[Feature:ClusterSizeAutoscalingScaleUp\]|\[Feature:ClusterSizeAutoscalingScaleDown\] --ginkgo.skip=\[Flaky\] --minStartupPods=8 + - --timeout=400m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master diff --git a/config/jobs/kubernetes/sig-cluster-lifecycle/k8s-upgrade-gce.yaml b/config/jobs/kubernetes/sig-cluster-lifecycle/k8s-upgrade-gce.yaml index 25242fde814e..9db2833b3dab 100644 --- a/config/jobs/kubernetes/sig-cluster-lifecycle/k8s-upgrade-gce.yaml +++ b/config/jobs/kubernetes/sig-cluster-lifecycle/k8s-upgrade-gce.yaml @@ -464,4 +464,4 @@ periodics: - --test_args=--kubectl-path=../../../../kubernetes_skew/cluster/kubectl.sh --minStartupPods=8 - --timeout=900m - --upgrade_args=--ginkgo.focus=\[Feature:MasterUpgrade\] --upgrade-target=ci/latest - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master \ No newline at end of file + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master diff --git a/config/jobs/kubernetes/sig-cluster-lifecycle/sig-cluster-lifecycle-misc.yaml b/config/jobs/kubernetes/sig-cluster-lifecycle/sig-cluster-lifecycle-misc.yaml new file mode 100644 index 000000000000..05429d8b971b --- /dev/null +++ b/config/jobs/kubernetes/sig-cluster-lifecycle/sig-cluster-lifecycle-misc.yaml @@ -0,0 +1,70 @@ +periodics: +- interval: 2h + name: ci-kubernetes-e2e-gce-gci-latest-rollback-etcd + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=110 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --check-version-skew=false + - --env-file=jobs/env/ci-kubernetes-e2e-gce-gci-latest-rollback-etcd.env + - --extract=ci/latest + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --provider=gce + - --test=false + - --timeout=60m + - --upgrade_args=--ginkgo.focus=\[Feature:EtcdUpgrade\] --etcd-upgrade-storage=etcd2 --etcd-upgrade-version=2.2.1 + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gce-ha-master + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=240 + - --bare + - --scenario=kubernetes_e2e + - -- + - --env=MULTIZONE=true + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-project=kubernetes-ha-master + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.focus=\[Feature:HAMaster\] --minStartupPods=8 + - --timeout=220m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 24h + name: ci-kubernetes-e2e-gce-min-node-permissions + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=520 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --env-file=jobs/env/ci-kubernetes-e2e-gce-min-node-permissions.env + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-project=k8s-min-node-permissions + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.skip=\[Flaky\]|\[Feature:.+\] --minStartupPods=8 + - --timeout=500m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master diff --git a/config/jobs/kubernetes/sig-instrumentation/OWNERS b/config/jobs/kubernetes/sig-instrumentation/OWNERS new file mode 100644 index 000000000000..3fd111f66388 --- /dev/null +++ b/config/jobs/kubernetes/sig-instrumentation/OWNERS @@ -0,0 +1,3 @@ +approvers: +- piosz +- brancz diff --git a/config/jobs/kubernetes/sig-instrumentation/sig-instrumentation-config.yaml b/config/jobs/kubernetes/sig-instrumentation/sig-instrumentation-config.yaml new file mode 100644 index 000000000000..f58fd0079618 --- /dev/null +++ b/config/jobs/kubernetes/sig-instrumentation/sig-instrumentation-config.yaml @@ -0,0 +1,276 @@ +periodics: +- interval: 30m + name: ci-kubernetes-e2e-gce-influxdb + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=70 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --env-file=jobs/env/ci-kubernetes-e2e-gce-influxdb.env + - --extract=ci/latest + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.focus=\[Feature:InfluxdbMonitoring\] + - --timeout=50m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gce-prometheus + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=70 + - --bare + - --scenario=kubernetes_e2e + - -- + - --env-file=jobs/env/ci-kubernetes-e2e-gce-prometheus.env + - --extract=ci/latest + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.focus=\[Feature:PrometheusMonitoring\] --minStartupPods=8 + - --timeout=50m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gce-stackdriver + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=70 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --env-file=jobs/env/ci-kubernetes-e2e-gce-stackdriver.env + - --extract=ci/latest + - --gcp-project=k8s-jkns-e2e-gce-stackdriver + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.focus=\[Feature:StackdriverMonitoring\]|\[Feature:StackdriverCustomMetrics\]|\[Feature:StackdriverMetadataAgent\]|\[Feature:StackdriverExternalMetrics\] --minStartupPods=8 + - --timeout=50m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gci-gce-es-logging + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=110 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --env-file=jobs/env/ci-kubernetes-e2e-gci-gce-es-logging.env + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.focus=\[Feature:Elasticsearch\] --minStartupPods=8 + - --timeout=90m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gci-gce-sd-logging + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=1340 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --env-file=jobs/env/ci-kubernetes-e2e-gci-gce-sd-logging.env + - --extract=ci/latest + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.focus=\[Feature:StackdriverLogging\] --minStartupPods=8 + - --timeout=1320m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gke-sd-logging-gci-beta + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=1400 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --deployment=gke + - --extract=ci/k8s-beta + - --gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --gke-environment=test + - --provider=gke + - --test_args=--ginkgo.focus=\[Feature:StackdriverLogging\] --minStartupPods=8 + - --timeout=1320m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gke-sd-logging-gci-latest + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=1400 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --deployment=gke + - --extract=ci/latest + - --gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --gke-environment=test + - --provider=gke + - --test_args=--ginkgo.focus=\[Feature:StackdriverLogging\] --minStartupPods=8 + - --timeout=1320m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gke-sd-logging-gci-stable1 + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=1400 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --deployment=gke + - --extract=ci/k8s-stable1 + - --gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging + - --gcp-node-image=gci + - --gcp-zone=us-central1-f + - --gke-environment=test + - --provider=gke + - --test_args=--ginkgo.focus=\[Feature:StackdriverLogging\] --minStartupPods=8 + - --timeout=1320m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gke-sd-logging-ubuntu-beta + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=1400 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --deployment=gke + - --extract=ci/k8s-beta + - --gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging + - --gcp-node-image=ubuntu + - --gcp-zone=us-central1-f + - --gke-environment=test + - --provider=gke + - --test_args=--ginkgo.focus=\[Feature:StackdriverLogging\] --minStartupPods=8 + - --timeout=1320m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gke-sd-logging-ubuntu-latest + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=1400 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --deployment=gke + - --extract=ci/latest + - --gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging + - --gcp-node-image=ubuntu + - --gcp-zone=us-central1-f + - --gke-environment=test + - --provider=gke + - --test_args=--ginkgo.focus=\[Feature:StackdriverLogging\] --minStartupPods=8 + - --timeout=1320m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gke-sd-logging-ubuntu-stable1 + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=1400 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --deployment=gke + - --extract=ci/k8s-stable1 + - --gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging + - --gcp-node-image=ubuntu + - --gcp-zone=us-central1-f + - --gke-environment=test + - --provider=gke + - --test_args=--ginkgo.focus=\[Feature:StackdriverLogging\] --minStartupPods=8 + - --timeout=1320m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + +- interval: 30m + name: ci-kubernetes-e2e-gke-stackdriver + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=70 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --deployment=gke + - --extract=ci/latest + - --gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging + - --gcp-node-image=gci + - --gcp-project=k8s-jkns-e2e-gke-stackdriver + - --gcp-zone=us-central1-f + - --gke-environment=test + - --provider=gke + - --test_args=--ginkgo.focus=\[Feature:StackdriverMonitoring\]|\[Feature:StackdriverCustomMetrics\]|\[Feature:StackdriverMetadataAgent\]|\[Feature:StackdriverExternalMetrics\] --minStartupPods=8 + - --timeout=50m + image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master diff --git a/config/jobs/kubernetes/sig-scheduling/OWNERS b/config/jobs/kubernetes/sig-scheduling/OWNERS new file mode 100644 index 000000000000..76f5c3b96679 --- /dev/null +++ b/config/jobs/kubernetes/sig-scheduling/OWNERS @@ -0,0 +1,3 @@ +approvers: +- bsalamat +- k82cn diff --git a/config/jobs/kubernetes/sig-scheduling/sig-scheduling-config.yaml b/config/jobs/kubernetes/sig-scheduling/sig-scheduling-config.yaml new file mode 100644 index 000000000000..285d8b1be451 --- /dev/null +++ b/config/jobs/kubernetes/sig-scheduling/sig-scheduling-config.yaml @@ -0,0 +1,22 @@ +periodics: +- name: ci-kubernetes-e2e-gce-pod-preemption + interval: 2h + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + spec: + containers: + - args: + - --timeout=300 + - --bare + - --scenario=kubernetes_e2e + - -- + - --check-leaked-resources + - --env=KUBE_FEATURE_GATES=PodPriority=true + - --extract=ci/latest + - --gcp-zone=us-central1-f + - --provider=gce + - --test_args=--ginkgo.focus=\[Feature:PodPreemption\] --minStartupPods=8 + - --timeout=180m + image: gcr.io/k8s-testimages/kubekins-e2e:latest-master + imagePullPolicy: Always diff --git a/jobs/BUILD.bazel b/jobs/BUILD.bazel index c754e2d9409a..252d55d8c852 100644 --- a/jobs/BUILD.bazel +++ b/jobs/BUILD.bazel @@ -1,34 +1,8 @@ package(default_visibility = ["//visibility:public"]) -py_test( - name = "config_test", - srcs = [ - "config_test.py", - ], - data = [ - "//boskos:resources.yaml", - "//jobs", - "//prow:config.yaml", - "//scenarios", - ], - deps = [ - ":env_gc", - "@yaml", - ], -) - -py_binary( - name = "env_gc", - srcs = ["env_gc.py"], - data = [ - ":jobs", - ], -) - filegroup( name = "jobs", srcs = glob([ - "**/*.sh", "**/*.env", "**/*.json", ]), diff --git a/jobs/config.json b/jobs/config.json index 43377f7b4675..4602880fd7f7 100644 --- a/jobs/config.json +++ b/jobs/config.json @@ -1,594 +1,4 @@ { - "application-periodic-default-gke": { - "args": [ - "--check-leaked-resources", - "--cluster=", - "--deployment=gke", - "--extract=gke", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--gke-environment=prod", - "--provider=gke", - "--test=false", - "--test-cmd=../e2e/e2e-prow.sh", - "--test-cmd-name=application-e2e", - "--timeout=120m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-apps" - ] - }, - "ci-kubernetes-e2e-autoscaling-vpa-actuation": { - "args": [ - "--check-leaked-resources", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test=false", - "--test-cmd=../vertical-pod-autoscaler/hack/run-e2e.sh", - "--test-cmd-args=actuation", - "--timeout=60m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-autoscaling" - ] - }, - "ci-kubernetes-e2e-autoscaling-vpa-admission-controller": { - "args": [ - "--check-leaked-resources", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test=false", - "--test-cmd=../vertical-pod-autoscaler/hack/run-e2e.sh", - "--test-cmd-args=admission-controller", - "--timeout=60m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-autoscaling" - ] - }, - "ci-kubernetes-e2e-autoscaling-vpa-full": { - "args": [ - "--check-leaked-resources", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test=false", - "--test-cmd=../vertical-pod-autoscaler/hack/run-e2e.sh", - "--test-cmd-args=full-vpa", - "--timeout=60m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-autoscaling" - ] - }, - "ci-kubernetes-e2e-autoscaling-vpa-recommender": { - "args": [ - "--check-leaked-resources", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test=false", - "--test-cmd=../vertical-pod-autoscaler/hack/run-e2e.sh", - "--test-cmd-args=recommender", - "--timeout=60m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-autoscaling" - ] - }, - "ci-kubernetes-e2e-autoscaling-vpa-updater": { - "args": [ - "--check-leaked-resources", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test=false", - "--test-cmd=../vertical-pod-autoscaler/hack/run-e2e.sh", - "--test-cmd-args=updater", - "--timeout=60m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-autoscaling" - ] - }, - "ci-kubernetes-e2e-gce-gci-latest-rollback-etcd": { - "args": [ - "--check-leaked-resources", - "--check-version-skew=false", - "--env-file=jobs/env/ci-kubernetes-e2e-gce-gci-latest-rollback-etcd.env", - "--extract=ci/latest", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test=false", - "--timeout=60m", - "--upgrade_args=--ginkgo.focus=\\[Feature:EtcdUpgrade\\] --etcd-upgrade-storage=etcd2 --etcd-upgrade-version=2.2.1" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-cluster-lifecycle" - ] - }, - "ci-kubernetes-e2e-gce-ha-master": { - "args": [ - "--env=MULTIZONE=true", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-project=kubernetes-ha-master", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:HAMaster\\] --minStartupPods=8", - "--timeout=220m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-cluster-lifecycle" - ] - }, - "ci-kubernetes-e2e-gce-influxdb": { - "args": [ - "--check-leaked-resources", - "--env-file=jobs/env/ci-kubernetes-e2e-gce-influxdb.env", - "--extract=ci/latest", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:InfluxdbMonitoring\\]", - "--timeout=50m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "ci-kubernetes-e2e-gce-min-node-permissions": { - "args": [ - "--check-leaked-resources", - "--env-file=jobs/env/ci-kubernetes-e2e-gce-min-node-permissions.env", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-project=k8s-min-node-permissions", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.skip=\\[Flaky\\]|\\[Feature:.+\\] --minStartupPods=8", - "--timeout=500m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-cluster-lifecycle" - ] - }, - "ci-kubernetes-e2e-gce-pod-preemption": { - "args": [ - "--check-leaked-resources", - "--env=KUBE_FEATURE_GATES=PodPriority=true", - "--extract=ci/latest", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:PodPreemption\\] --minStartupPods=8", - "--timeout=180m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-scheduling" - ] - }, - "ci-kubernetes-e2e-gce-prometheus": { - "args": [ - "--env-file=jobs/env/ci-kubernetes-e2e-gce-prometheus.env", - "--extract=ci/latest", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:PrometheusMonitoring\\] --minStartupPods=8", - "--timeout=50m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "ci-kubernetes-e2e-gce-stackdriver": { - "args": [ - "--check-leaked-resources", - "--env-file=jobs/env/ci-kubernetes-e2e-gce-stackdriver.env", - "--extract=ci/latest", - "--gcp-project=k8s-jkns-e2e-gce-stackdriver", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:StackdriverMonitoring\\]|\\[Feature:StackdriverCustomMetrics\\]|\\[Feature:StackdriverMetadataAgent\\]|\\[Feature:StackdriverExternalMetrics\\] --minStartupPods=8", - "--timeout=50m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "ci-kubernetes-e2e-gce-taint-evict": { - "args": [ - "--env-file=jobs/env/ci-kubernetes-e2e-gce-taint-evict.env", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:TaintEviction\\] --minStartupPods=8", - "--timeout=50m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-apps" - ] - }, - "ci-kubernetes-e2e-gci-gce-autoscaling": { - "args": [ - "--cluster=ca", - "--env-file=jobs/env/ci-kubernetes-e2e-gci-gce-autoscaling.env", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-nodes=3", - "--gcp-project=k8s-jkns-gci-autoscaling", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--runtime-config=scheduling.k8s.io/v1alpha1=true", - "--test_args=--ginkgo.focus=\\[Feature:ClusterSizeAutoscalingScaleUp\\]|\\[Feature:ClusterSizeAutoscalingScaleDown\\]|\\[Feature:InitialResources\\] --ginkgo.skip=\\[Flaky\\] --minStartupPods=8", - "--timeout=300m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-autoscaling" - ] - }, - "ci-kubernetes-e2e-gci-gce-autoscaling-hpa": { - "args": [ - "--cluster=hpa", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-project=k8s-jkns-gci-autoscaling", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:CustomMetricsAutoscaling\\] --ginkgo.skip=\\[Flaky\\] --minStartupPods=8", - "--timeout=300m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-autoscaling" - ] - }, - "ci-kubernetes-e2e-gci-gce-autoscaling-migs": { - "args": [ - "--cluster=ca", - "--env-file=jobs/env/ci-kubernetes-e2e-gci-gce-autoscaling-migs.env", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-nodes=3", - "--gcp-project=k8s-jkns-gci-autoscaling-migs", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--runtime-config=scheduling.k8s.io/v1alpha1=true", - "--test_args=--ginkgo.focus=\\[Feature:ClusterSizeAutoscalingScaleUp\\]|\\[Feature:ClusterSizeAutoscalingScaleDown\\] --ginkgo.skip=\\[Flaky\\] --minStartupPods=8", - "--timeout=300m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-autoscaling" - ] - }, - "ci-kubernetes-e2e-gci-gce-autoscaling-migs-hpa": { - "args": [ - "--cluster=hpa", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-project=k8s-jkns-gci-autoscaling-migs", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:CustomMetricsAutoscaling\\] --ginkgo.skip=\\[Flaky\\] --minStartupPods=8", - "--timeout=300m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-autoscaling" - ] - }, - "ci-kubernetes-e2e-gci-gce-es-logging": { - "args": [ - "--check-leaked-resources", - "--env-file=jobs/env/ci-kubernetes-e2e-gci-gce-es-logging.env", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:Elasticsearch\\] --minStartupPods=8", - "--timeout=90m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "ci-kubernetes-e2e-gci-gce-garbage": { - "_comment": "TODO: remove this job, kubernetes/kubernetes#49262 removed the Feature:GarbageCollector tag", - "args": [ - "--check-leaked-resources", - "--cluster=gc-feature", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:GarbageCollector\\] --minStartupPods=8", - "--timeout=600m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-api-machinery" - ] - }, - "ci-kubernetes-e2e-gci-gce-proto": { - "args": [ - "--check-leaked-resources", - "--env=STORAGE_MEDIA_TYPE=application/vnd.kubernetes.protobuf", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--ginkgo-parallel=25", - "--provider=gce", - "--test_args=--ginkgo.skip=\\[Slow\\]|\\[Serial\\]|\\[Disruptive\\]|\\[Flaky\\]|\\[Feature:.+\\] --minStartupPods=8", - "--timeout=50m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-api-machinery" - ] - }, - "ci-kubernetes-e2e-gci-gce-sd-logging": { - "args": [ - "--check-leaked-resources", - "--env-file=jobs/env/ci-kubernetes-e2e-gci-gce-sd-logging.env", - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:StackdriverLogging\\] --minStartupPods=8", - "--timeout=1320m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "ci-kubernetes-e2e-gci-gce-statefulset": { - "args": [ - "--extract=ci/latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--provider=gce", - "--test_args=--ginkgo.focus=\\[Feature:StatefulSet\\] --minStartupPods=8", - "--timeout=90m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-apps" - ] - }, - "ci-kubernetes-e2e-gci-gke-autoscaling": { - "args": [ - "--check-leaked-resources", - "--cluster=ca", - "--deployment=gke", - "--extract=ci/latest", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=gci", - "--gcp-project=k8s-e2e-gci-gke-autoscaling", - "--gcp-zone=us-central1-f", - "--gke-environment=test", - "--provider=gke", - "--test_args=--ginkgo.focus=\\[Feature:ClusterSizeAutoscalingScaleUp\\]|\\[Feature:ClusterSizeAutoscalingScaleDown\\]|\\[Feature:ClusterSizeAutoscalingScaleWithNAP\\] --ginkgo.skip=\\[Flaky\\] --minStartupPods=8", - "--timeout=600m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-autoscaling" - ] - }, - "ci-kubernetes-e2e-gci-gke-autoscaling-hpa": { - "args": [ - "--check-leaked-resources", - "--cluster=hpa", - "--deployment=gke", - "--extract=ci/latest", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=gci", - "--gcp-project=k8s-e2e-gci-gke-autoscaling", - "--gcp-zone=us-central1-f", - "--gke-environment=test", - "--provider=gke", - "--test_args=--ginkgo.focus=\\[Feature:CustomMetricsAutoscaling\\] --ginkgo.skip=\\[Flaky\\] --minStartupPods=8", - "--timeout=420m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-autoscaling" - ] - }, - "ci-kubernetes-e2e-gci-gke-autoscaling-regional": { - "args": [ - "--check-leaked-resources", - "--deployment=gke", - "--extract=ci/latest", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=gci", - "--gcp-project=k8s-jkns-gci-autoscaling-regio", - "--gcp-region=us-central1", - "--gke-command-group=beta", - "--gke-environment=test", - "--gke-single-zone-node-instance-group=false", - "--provider=gke", - "--test_args=--gce-multizone=true --ginkgo.focus=\\[Feature:ClusterSizeAutoscalingScaleUp\\]|\\[Feature:ClusterSizeAutoscalingScaleDown\\] --ginkgo.skip=\\[Flaky\\] --minStartupPods=8", - "--timeout=400m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-autoscaling" - ] - }, - "ci-kubernetes-e2e-gke-sd-logging-gci-beta": { - "args": [ - "--check-leaked-resources", - "--deployment=gke", - "--extract=ci/k8s-beta", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--gke-environment=test", - "--provider=gke", - "--test_args=--ginkgo.focus=\\[Feature:StackdriverLogging\\] --minStartupPods=8", - "--timeout=1320m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "ci-kubernetes-e2e-gke-sd-logging-gci-latest": { - "args": [ - "--check-leaked-resources", - "--deployment=gke", - "--extract=ci/latest", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--gke-environment=test", - "--provider=gke", - "--test_args=--ginkgo.focus=\\[Feature:StackdriverLogging\\] --minStartupPods=8", - "--timeout=1320m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "ci-kubernetes-e2e-gke-sd-logging-gci-stable1": { - "args": [ - "--check-leaked-resources", - "--deployment=gke", - "--extract=ci/k8s-stable1", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--gke-environment=test", - "--provider=gke", - "--test_args=--ginkgo.focus=\\[Feature:StackdriverLogging\\] --minStartupPods=8", - "--timeout=1320m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "ci-kubernetes-e2e-gke-sd-logging-ubuntu-beta": { - "args": [ - "--check-leaked-resources", - "--deployment=gke", - "--extract=ci/k8s-beta", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=ubuntu", - "--gcp-zone=us-central1-f", - "--gke-environment=test", - "--provider=gke", - "--test_args=--ginkgo.focus=\\[Feature:StackdriverLogging\\] --minStartupPods=8", - "--timeout=1320m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "ci-kubernetes-e2e-gke-sd-logging-ubuntu-latest": { - "args": [ - "--check-leaked-resources", - "--deployment=gke", - "--extract=ci/latest", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=ubuntu", - "--gcp-zone=us-central1-f", - "--gke-environment=test", - "--provider=gke", - "--test_args=--ginkgo.focus=\\[Feature:StackdriverLogging\\] --minStartupPods=8", - "--timeout=1320m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "ci-kubernetes-e2e-gke-sd-logging-ubuntu-stable1": { - "args": [ - "--check-leaked-resources", - "--deployment=gke", - "--extract=ci/k8s-stable1", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=ubuntu", - "--gcp-zone=us-central1-f", - "--gke-environment=test", - "--provider=gke", - "--test_args=--ginkgo.focus=\\[Feature:StackdriverLogging\\] --minStartupPods=8", - "--timeout=1320m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "ci-kubernetes-e2e-gke-stackdriver": { - "args": [ - "--check-leaked-resources", - "--deployment=gke", - "--extract=ci/latest", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=gci", - "--gcp-project=k8s-jkns-e2e-gke-stackdriver", - "--gcp-zone=us-central1-f", - "--gke-environment=test", - "--provider=gke", - "--test_args=--ginkgo.focus=\\[Feature:StackdriverMonitoring\\]|\\[Feature:StackdriverCustomMetrics\\]|\\[Feature:StackdriverMetadataAgent\\]|\\[Feature:StackdriverExternalMetrics\\] --minStartupPods=8", - "--timeout=50m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-instrumentation" - ] - }, - "ci-kustomize-periodic-default-gke": { - "args": [ - "--check-leaked-resources", - "--cluster=", - "--deployment=gke", - "--extract=gke", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--gke-environment=prod", - "--provider=gke", - "--test=false", - "--test-cmd=../examples/integration_tests.sh", - "--test-cmd-name=kustomize-integration", - "--timeout=120m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-cli" - ] - }, "fake-branch": { "args": [ "./fake/fake-branch.sh" @@ -621,45 +31,5 @@ "sigOwners": [ "sig-testing" ] - }, - "spark-periodic-default-gke": { - "args": [ - "--check-leaked-resources", - "--cluster=", - "--deployment=gke", - "--extract=gke", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--gke-environment=prod", - "--provider=gke", - "--test=false", - "--test-cmd=../e2e/e2e-prow.sh", - "--test-cmd-name=spark-integration", - "--timeout=120m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-big-data" - ] - }, - "spark-periodic-latest-gke": { - "args": [ - "--check-leaked-resources", - "--cluster=", - "--deployment=gke", - "--extract=gke-latest", - "--gcp-node-image=gci", - "--gcp-zone=us-central1-f", - "--gke-environment=prod", - "--provider=gke", - "--test=false", - "--test-cmd=../e2e/e2e-prow.sh", - "--test-cmd-name=spark-integration", - "--timeout=120m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-big-data" - ] } } diff --git a/jobs/config_test.py b/jobs/config_test.py deleted file mode 100755 index 6b5e069584be..000000000000 --- a/jobs/config_test.py +++ /dev/null @@ -1,709 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for config.json and Prow configuration.""" - - -import unittest - -import collections -import json -import os -import re - -import yaml - -# pylint: disable=too-many-public-methods, too-many-branches, too-many-locals, too-many-statements - -ORIG_CWD = os.getcwd() - -def test_infra(*paths): - """Return path relative to root of test-infra repo.""" - return os.path.join(ORIG_CWD, os.path.dirname(__file__), '..', *paths) - -def get_required_jobs(): - required_jobs = set() - configs_dir = test_infra('mungegithub', 'submit-queue', 'deployment') - for root, _, files in os.walk(configs_dir): - for file_name in files: - if file_name == 'configmap.yaml': - path = os.path.join(root, file_name) - with open(path) as fp: - conf = yaml.safe_load(fp) - for job in conf.get('required-retest-contexts', '').split(','): - if job: - required_jobs.add(job) - return required_jobs - -class JobTest(unittest.TestCase): - - excludes = [ - 'BUILD.bazel', # For bazel - 'config.json', # For --json mode - 'validOwners.json', # Contains a list of current sigs; sigs are allowed to own jobs - 'config_test.py', # Script for testing config.json and Prow config. - 'env_gc.py', # Tool script to garbage collect unused .env files. - 'move_extract.py', - ] - # also exclude .pyc - excludes.extend(e + 'c' for e in excludes if e.endswith('.py')) - - prow_config = '../prow/config.yaml' - - realjobs = {} - prowjobs = [] - - @property - def jobs(self): - """[(job, job_path)] sequence""" - for path, _, filenames in os.walk(test_infra('jobs/env')): - for job in [f for f in filenames if f not in self.excludes]: - job_path = os.path.join(path, job) - yield job, job_path - - # TODO(krzyzacy): disabled as we currently have multiple source of truth. - # We also should migrate shared env files into presets. - #def test_orphaned_env(self): - # orphans = env_gc.find_orphans() - # if orphans: - # self.fail('the following .env files are not referenced ' + - # 'in config.json, please run `bazel run //jobs:env_gc`: ' + - # ' '.join(orphans)) - - def check_job_template(self, tmpl): - builders = tmpl.get('builders') - if not isinstance(builders, list): - self.fail(tmpl) - self.assertEquals(1, len(builders), builders) - shell = builders[0] - if not isinstance(shell, dict): - self.fail(tmpl) - self.assertEquals(1, len(shell), tmpl) - if 'raw' in shell: - self.assertEquals('maintenance-all-{suffix}', tmpl['name']) - return - cmd = shell.get('shell') - if not isinstance(cmd, basestring): - self.fail(tmpl) - self.assertIn('--service-account=', cmd) - self.assertIn('--upload=', cmd) - if 'kubernetes-security' in cmd: - self.assertIn('--upload=\'gs://kubernetes-security-jenkins/pr-logs\'', cmd) - elif '${{PULL_REFS}}' in cmd: - self.assertIn('--upload=\'gs://kubernetes-jenkins/pr-logs\'', cmd) - else: - self.assertIn('--upload=\'gs://kubernetes-jenkins/logs\'', cmd) - - def add_prow_job(self, job): - name = job.get('name') - real_job = {} - real_job['name'] = name - if 'spec' in job: - spec = job.get('spec') - for container in spec.get('containers'): - if 'args' in container: - for arg in container.get('args'): - match = re.match(r'[\'\"]?--timeout=(\d+)', arg) - if match: - real_job['timeout'] = match.group(1) - if 'pull-' not in name and name in self.realjobs and name not in self.prowjobs: - self.fail('CI job %s exist in both Jenkins and Prow config!' % name) - if name not in self.realjobs: - self.realjobs[name] = real_job - self.prowjobs.append(name) - if 'run_after_success' in job: - for sub in job.get('run_after_success'): - self.add_prow_job(sub) - - def load_prow_yaml(self, path): - with open(os.path.join( - os.path.dirname(__file__), path)) as fp: - doc = yaml.safe_load(fp) - - if 'periodics' not in doc: - self.fail('No periodics in prow config!') - - for item in doc.get('periodics'): - self.add_prow_job(item) - - def get_real_bootstrap_job(self, job): - key = os.path.splitext(job.strip())[0] - if not key in self.realjobs: - self.load_prow_yaml(self.prow_config) - self.assertIn(key, sorted(self.realjobs)) # sorted for clearer error message - return self.realjobs.get(key) - - def test_valid_timeout(self): - """All e2e jobs has 20min or more container timeout than kubetest timeout.""" - bad_jobs = set() - with open(test_infra('jobs/config.json')) as fp: - config = json.loads(fp.read()) - - for job in config: - if config.get(job, {}).get('scenario') != 'kubernetes_e2e': - continue - realjob = self.get_real_bootstrap_job(job) - self.assertTrue(realjob) - self.assertIn('timeout', realjob, job) - container_timeout = int(realjob['timeout']) - - kubetest_timeout = None - for arg in config[job]['args']: - mat = re.match(r'--timeout=(\d+)m', arg) - if not mat: - continue - kubetest_timeout = int(mat.group(1)) - if kubetest_timeout is None: - self.fail('Missing timeout: %s' % job) - if kubetest_timeout > container_timeout: - bad_jobs.add((job, kubetest_timeout, container_timeout)) - elif kubetest_timeout + 20 > container_timeout: - bad_jobs.add(( - 'insufficient kubetest leeway', - job, kubetest_timeout, container_timeout - )) - if bad_jobs: - self.fail( - 'jobs: %s, ' - 'prow timeout need to be at least 20min longer than timeout in config.json' - % ('\n'.join(str(s) for s in bad_jobs)) - ) - - def test_valid_job_config_json(self): - """Validate jobs/config.json.""" - # bootstrap integration test scripts - ignore = [ - 'fake-failure', - 'fake-branch', - 'fake-pr', - 'random_job', - ] - - self.load_prow_yaml(self.prow_config) - config = test_infra('jobs/config.json') - owners = test_infra('jobs/validOwners.json') - with open(config) as fp, open(owners) as ownfp: - config = json.loads(fp.read()) - valid_owners = json.loads(ownfp.read()) - for job in config: - if job not in ignore: - self.assertTrue(job in self.prowjobs or job in self.realjobs, - '%s must have a matching jenkins/prow entry' % job) - - # ownership assertions - self.assertIn('sigOwners', config[job], job) - self.assertIsInstance(config[job]['sigOwners'], list, job) - self.assertTrue(config[job]['sigOwners'], job) # non-empty - owners = config[job]['sigOwners'] - for owner in owners: - self.assertIsInstance(owner, basestring, job) - self.assertIn(owner, valid_owners, job) - - # env assertions - self.assertTrue('scenario' in config[job], job) - scenario = test_infra('scenarios/%s.py' % config[job]['scenario']) - self.assertTrue(os.path.isfile(scenario), job) - self.assertTrue(os.access(scenario, os.X_OK|os.R_OK), job) - args = config[job].get('args', []) - use_shared_build_in_args = False - extract_in_args = False - build_in_args = False - for arg in args: - if arg.startswith('--use-shared-build'): - use_shared_build_in_args = True - elif arg.startswith('--build'): - build_in_args = True - elif arg.startswith('--extract'): - extract_in_args = True - match = re.match(r'--env-file=([^\"]+)\.env', arg) - if match: - env_path = match.group(1) - self.assertTrue(env_path.startswith('jobs/'), env_path) - path = test_infra('%s.env' % env_path) - self.assertTrue( - os.path.isfile(path), - '%s does not exist for %s' % (path, job)) - elif 'kops' not in job: - match = re.match(r'--cluster=([^\"]+)', arg) - if match: - cluster = match.group(1) - self.assertLessEqual( - len(cluster), 23, - 'Job %r, --cluster should be 23 chars or fewer' % job - ) - # these args should not be combined: - # --use-shared-build and (--build or --extract) - self.assertFalse(use_shared_build_in_args and build_in_args) - self.assertFalse(use_shared_build_in_args and extract_in_args) - if config[job]['scenario'] == 'kubernetes_e2e': - if job in self.prowjobs: - for arg in args: - # --mode=local is default now - self.assertNotIn('--mode', arg, job) - else: - self.assertIn('--mode=docker', args, job) - for arg in args: - if "--env=" in arg: - self._check_env(job, arg.split("=", 1)[1]) - if '--provider=gke' in args: - self.assertTrue('--deployment=gke' in args, - '%s must use --deployment=gke' % job) - self.assertFalse(any('--gcp-master-image' in a for a in args), - '%s cannot use --gcp-master-image on GKE' % job) - self.assertFalse(any('--gcp-nodes' in a for a in args), - '%s cannot use --gcp-nodes on GKE' % job) - if '--deployment=gke' in args: - self.assertTrue(any('--gcp-node-image' in a for a in args), job) - self.assertNotIn('--charts-tests', args) # Use --charts - if any('--check_version_skew' in a for a in args): - self.fail('Use --check-version-skew, not --check_version_skew in %s' % job) - if '--check-leaked-resources=true' in args: - self.fail('Use --check-leaked-resources (no value) in %s' % job) - if '--check-leaked-resources==false' in args: - self.fail( - 'Remove --check-leaked-resources=false (default value) from %s' % job) - if ( - '--env-file=jobs/pull-kubernetes-e2e.env' in args - and '--check-leaked-resources' in args): - self.fail('PR job %s should not check for resource leaks' % job) - # Consider deleting any job with --check-leaked-resources=false - if ( - '--provider=gce' not in args - and '--provider=gke' not in args - and '--check-leaked-resources' in args - and 'generated' not in config[job].get('tags', [])): - self.fail('Only GCP jobs can --check-leaked-resources, not %s' % job) - if '--mode=local' in args: - self.fail('--mode=local is default now, drop that for %s' % job) - - extracts = [a for a in args if '--extract=' in a] - shared_builds = [a for a in args if '--use-shared-build' in a] - node_e2e = [a for a in args if '--deployment=node' in a] - local_e2e = [a for a in args if '--deployment=local' in a] - builds = [a for a in args if '--build' in a] - if shared_builds and extracts: - self.fail(('e2e jobs cannot have --use-shared-build' - ' and --extract: %s %s') % (job, args)) - elif not extracts and not shared_builds and not node_e2e: - # we should at least have --build and --stage - if not builds: - self.fail(('e2e job needs --extract or' - ' --use-shared-build or' - ' --build: %s %s') % (job, args)) - - if shared_builds or node_e2e: - expected = 0 - elif builds and not extracts: - expected = 0 - elif 'ingress' in job: - expected = 1 - elif any(s in job for s in [ - 'upgrade', 'skew', 'downgrade', 'rollback', - 'ci-kubernetes-e2e-gce-canary', - ]): - expected = 2 - else: - expected = 1 - if len(extracts) != expected: - self.fail('Wrong number of --extract args (%d != %d) in %s' % ( - len(extracts), expected, job)) - - has_image_family = any( - [x for x in args if x.startswith('--image-family')]) - has_image_project = any( - [x for x in args if x.startswith('--image-project')]) - docker_mode = any( - [x for x in args if x.startswith('--mode=docker')]) - if ( - (has_image_family or has_image_project) - and docker_mode): - self.fail('--image-family / --image-project is not ' - 'supported in docker mode: %s' % job) - if has_image_family != has_image_project: - self.fail('--image-family and --image-project must be' - 'both set or unset: %s' % job) - - if job.startswith('pull-kubernetes-') and not node_e2e and not local_e2e: - if 'gke' in job: - stage = 'gs://kubernetes-release-dev/ci' - suffix = True - elif 'kubeadm' in job: - # kubeadm-based jobs use out-of-band .deb artifacts, - # not the --stage flag. - continue - else: - stage = 'gs://kubernetes-release-pull/ci/%s' % job - suffix = False - if not shared_builds: - self.assertIn('--stage=%s' % stage, args) - self.assertEquals( - suffix, - any('--stage-suffix=' in a for a in args), - ('--stage-suffix=', suffix, job, args)) - - - def test_valid_env(self): - for job, job_path in self.jobs: - with open(job_path) as fp: - data = fp.read() - if 'kops' in job: # TODO(fejta): update this one too - continue - self.assertNotIn( - 'JENKINS_USE_LOCAL_BINARIES=', - data, - 'Send --extract=local to config.json, not JENKINS_USE_LOCAL_BINARIES in %s' % job) - self.assertNotIn( - 'JENKINS_USE_EXISTING_BINARIES=', - data, - 'Send --extract=local to config.json, not JENKINS_USE_EXISTING_BINARIES in %s' % job) # pylint: disable=line-too-long - - def test_only_jobs(self): - """Ensure that everything in jobs/ is a valid job name and script.""" - for job, job_path in self.jobs: - # Jobs should have simple names: letters, numbers, -, . - self.assertTrue(re.match(r'[.0-9a-z-_]+.env', job), job) - # Jobs should point to a real, executable file - # Note: it is easy to forget to chmod +x - self.assertTrue(os.path.isfile(job_path), job_path) - self.assertTrue(os.access(job_path, os.R_OK), job_path) - - def test_all_project_are_unique(self): - # pylint: disable=line-too-long - allowed_list = { - # The cos image validation jobs intentionally share projects. - 'ci-kubernetes-e2e-gce-cosdev-k8sdev-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosdev-k8sdev-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosdev-k8sdev-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosdev-k8sstable1-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosdev-k8sstable1-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosdev-k8sstable1-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosdev-k8sbeta-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosdev-k8sbeta-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosdev-k8sbeta-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sdev-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sdev-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sdev-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sbeta-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sbeta-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sbeta-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sstable1-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sstable1-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sstable1-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sstable2-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sstable2-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sstable2-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sstable3-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sstable3-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosbeta-k8sstable3-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sdev-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sdev-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sdev-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sbeta-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sbeta-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sbeta-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sstable1-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sstable1-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sstable1-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sstable2-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sstable2-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sstable2-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sstable3-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sstable3-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2e-gce-cosstable1-k8sstable3-slow': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2enode-cosbeta-k8sdev-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2enode-cosbeta-k8sdev-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2enode-cosbeta-k8sbeta-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2enode-cosbeta-k8sbeta-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2enode-cosbeta-k8sstable1-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2enode-cosbeta-k8sstable1-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2enode-cosbeta-k8sstable2-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2enode-cosbeta-k8sstable2-serial': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2enode-cosbeta-k8sstable3-default': 'ci-kubernetes-e2e-gce-cos*', - 'ci-kubernetes-e2enode-cosbeta-k8sstable3-serial': 'ci-kubernetes-e2e-gce-cos*', - - # The ubuntu image validation jobs intentionally share projects. - 'ci-kubernetes-e2enode-ubuntu1-k8sbeta-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu1-k8sbeta-serial': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu1-k8sstable1-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu1-k8sstable1-serial': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu1-k8sstable2-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu1-k8sstable2-serial': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu1-k8sstable3-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu1-k8sstable3-serial': 'ci-kubernetes-e2e-ubuntu-node*', - - 'ci-kubernetes-e2e-gce-ubuntu1-k8sbeta-default': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu1-k8sbeta-serial': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu1-k8sbeta-slow': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable1-default': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable1-serial': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable1-slow': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable2-default': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable2-serial': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable2-slow': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable3-default': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable3-serial': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable3-slow': 'ci-kubernetes-e2e-gce-ubuntu*', - - 'ci-kubernetes-e2enode-ubuntu2-k8sbeta-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu2-k8sbeta-serial': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu2-k8sstable1-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu2-k8sstable1-serial': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu2-k8sstable2-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu2-k8sstable2-serial': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu2-k8sstable3-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', - 'ci-kubernetes-e2enode-ubuntu2-k8sstable3-serial': 'ci-kubernetes-e2e-ubuntu-node*', - - 'ci-kubernetes-e2e-gce-ubuntu2-k8sbeta-default': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu2-k8sbeta-serial': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu2-k8sbeta-slow': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable1-default': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable1-serial': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable1-slow': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable2-default': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable2-serial': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable2-slow': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable3-default': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable3-serial': 'ci-kubernetes-e2e-gce-ubuntu*', - 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable3-slow': 'ci-kubernetes-e2e-gce-ubuntu*', - - # The release branch scalability jobs intentionally share projects. - 'ci-kubernetes-e2e-gci-gce-scalability-stable2': 'ci-kubernetes-e2e-gci-gce-scalability-release-*', - 'ci-kubernetes-e2e-gci-gce-scalability-stable1': 'ci-kubernetes-e2e-gci-gce-scalability-release-*', - 'ci-kubernetes-e2e-gce-scalability': 'ci-kubernetes-e2e-gce-scalability-*', - 'ci-kubernetes-e2e-gce-scalability-canary': 'ci-kubernetes-e2e-gce-scalability-*', - # TODO(fejta): remove these (found while migrating jobs) - 'ci-kubernetes-kubemark-100-gce': 'ci-kubernetes-kubemark-*', - 'ci-kubernetes-kubemark-100-canary': 'ci-kubernetes-kubemark-*', - 'ci-kubernetes-kubemark-5-gce-last-release': 'ci-kubernetes-kubemark-*', - 'ci-kubernetes-kubemark-high-density-100-gce': 'ci-kubernetes-kubemark-*', - 'ci-kubernetes-kubemark-gce-scale': 'ci-kubernetes-scale-*', - 'pull-kubernetes-kubemark-e2e-gce-big': 'ci-kubernetes-scale-*', - 'pull-kubernetes-kubemark-e2e-gce-scale': 'ci-kubernetes-scale-*', - 'pull-kubernetes-e2e-gce-100-performance': 'ci-kubernetes-scale-*', - 'pull-kubernetes-e2e-gce-big-performance': 'ci-kubernetes-scale-*', - 'pull-kubernetes-e2e-gce-large-performance': 'ci-kubernetes-scale-*', - 'ci-kubernetes-e2e-gce-large-manual-up': 'ci-kubernetes-scale-*', - 'ci-kubernetes-e2e-gce-large-manual-down': 'ci-kubernetes-scale-*', - 'ci-kubernetes-e2e-gce-large-correctness': 'ci-kubernetes-scale-*', - 'ci-kubernetes-e2e-gce-large-performance': 'ci-kubernetes-scale-*', - 'ci-kubernetes-e2e-gce-scale-correctness': 'ci-kubernetes-scale-*', - 'ci-kubernetes-e2e-gce-scale-performance': 'ci-kubernetes-scale-*', - 'ci-kubernetes-e2e-gke-large-correctness': 'ci-kubernetes-scale-*', - 'ci-kubernetes-e2e-gke-large-performance': 'ci-kubernetes-scale-*', - 'ci-kubernetes-e2e-gke-large-performance-regional': 'ci-kubernetes-scale-*', - 'ci-kubernetes-e2e-gke-large-deploy': 'ci-kubernetes-scale-*', - 'ci-kubernetes-e2e-gke-large-teardown': 'ci-kubernetes-scale-*', - 'ci-kubernetes-e2e-gke-scale-correctness': 'ci-kubernetes-scale-*', - 'pull-kubernetes-e2e-gce': 'pull-kubernetes-e2e-gce-*', - 'pull-kubernetes-e2e-gce-canary': 'pull-kubernetes-e2e-gce-*', - 'ci-kubernetes-e2e-gce': 'ci-kubernetes-e2e-gce-*', - 'ci-kubernetes-e2e-gce-canary': 'ci-kubernetes-e2e-gce-*', - 'ci-kubernetes-node-kubelet-serial': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-orphans': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-serial-cpu-manager': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-features': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-flaky': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-conformance': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-benchmark': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-stable1': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-stable2': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-stable3': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-alpha': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-beta': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-beta-features': 'ci-kubernetes-node-kubelet-*', - 'ci-kubernetes-node-kubelet-non-cri-1-6': 'ci-kubernetes-node-kubelet-*', - # The cri-containerd validation node e2e jobs intentionally share projects. - 'ci-cri-containerd-node-e2e': 'cri-containerd-node-e2e-*', - 'ci-cri-containerd-node-e2e-serial': 'cri-containerd-node-e2e-*', - 'ci-cri-containerd-node-e2e-features': 'cri-containerd-node-e2e-*', - 'ci-cri-containerd-node-e2e-flaky': 'cri-containerd-node-e2e-*', - 'ci-cri-containerd-node-e2e-benchmark': 'cri-containerd-node-e2e-*', - 'ci-containerd-node-e2e': 'cri-containerd-node-e2e-*', - 'ci-containerd-node-e2e-1-1': 'cri-containerd-node-e2e-*', - 'ci-containerd-node-e2e-features': 'cri-containerd-node-e2e-*', - # ci-cri-containerd-e2e-gce-stackdriver intentionally share projects with - # ci-kubernetes-e2e-gce-stackdriver. - 'ci-kubernetes-e2e-gce-stackdriver': 'k8s-jkns-e2e-gce-stackdriver', - 'ci-cri-containerd-e2e-gce-stackdriver': 'k8s-jkns-e2e-gce-stackdriver', - # ingress-GCE e2e jobs - 'pull-ingress-gce-e2e': 'e2e-ingress-gce', - 'ci-ingress-gce-e2e': 'e2e-ingress-gce', - # sig-autoscaling jobs intentionally share projetcs - 'ci-kubernetes-e2e-gci-gce-autoscaling-hpa':'ci-kubernetes-e2e-gci-gce-autoscaling', - 'ci-kubernetes-e2e-gci-gce-autoscaling-migs-hpa':'ci-kubernetes-e2e-gci-gce-autoscaling-migs', - 'ci-kubernetes-e2e-gci-gke-autoscaling-hpa':'ci-kubernetes-e2e-gci-gke-autoscaling', - # gpu+autoscaling jobs intentionally share projects with gpu tests - 'ci-kubernetes-e2e-gci-gke-autoscaling-gpu-v100': 'ci-kubernetes-e2e-gke-staging-latest-device-plugin-gpu-v100', - } - # pylint: enable=line-too-long - projects = collections.defaultdict(set) - boskos = [] - with open(test_infra('boskos/resources.yaml')) as fp: - boskos_config = yaml.safe_load(fp) - for rtype in boskos_config['resources']: - if 'project' in rtype['type']: - for name in rtype['names']: - boskos.append(name) - - with open(test_infra('jobs/config.json')) as fp: - job_config = json.load(fp) - for job in job_config: - project = '' - cfg = job_config.get(job.rsplit('.', 1)[0], {}) - if cfg.get('scenario') == 'kubernetes_e2e': - for arg in cfg.get('args', []): - if not arg.startswith('--gcp-project='): - continue - project = arg.split('=', 1)[1] - if project: - if project in boskos: - self.fail('Project %s cannot be in boskos/resources.yaml!' % project) - projects[project].add(allowed_list.get(job, job)) - - duplicates = [(p, j) for p, j in projects.items() if len(j) > 1] - if duplicates: - self.fail('Jobs duplicate projects:\n %s' % ( - '\n '.join('%s: %s' % t for t in duplicates))) - - def test_jobs_do_not_source_shell(self): - for job, job_path in self.jobs: - with open(job_path) as fp: - script = fp.read() - self.assertFalse(re.search(r'\Wsource ', script), job) - self.assertNotIn('\n. ', script, job) - - def _check_env(self, job, setting): - if not re.match(r'[0-9A-Z_]+=[^\n]*', setting): - self.fail('[%r]: Env %r: need to follow FOO=BAR pattern' % (job, setting)) - if '#' in setting: - self.fail('[%r]: Env %r: No inline comments' % (job, setting)) - if '"' in setting or '\'' in setting: - self.fail('[%r]: Env %r: No quote in env' % (job, setting)) - if '$' in setting: - self.fail('[%r]: Env %r: Please resolve variables in env' % (job, setting)) - if '{' in setting or '}' in setting: - self.fail('[%r]: Env %r: { and } are not allowed in env' % (job, setting)) - # also test for https://github.com/kubernetes/test-infra/issues/2829 - # TODO(fejta): sort this list - black = [ - ('CHARTS_TEST=', '--charts-tests'), - ('CLUSTER_IP_RANGE=', '--test_args=--cluster-ip-range=FOO'), - ('CLOUDSDK_BUCKET=', '--gcp-cloud-sdk=gs://foo'), - ('CLUSTER_NAME=', '--cluster=FOO'), - ('E2E_CLEAN_START=', '--test_args=--clean-start=true'), - ('E2E_DOWN=', '--down=true|false'), - ('E2E_MIN_STARTUP_PODS=', '--test_args=--minStartupPods=FOO'), - ('E2E_NAME=', '--cluster=whatever'), - ('E2E_PUBLISH_PATH=', '--publish=gs://FOO'), - ('E2E_REPORT_DIR=', '--test_args=--report-dir=FOO'), - ('E2E_REPORT_PREFIX=', '--test_args=--report-prefix=FOO'), - ('E2E_TEST=', '--test=true|false'), - ('E2E_UPGRADE_TEST=', '--upgrade_args=FOO'), - ('E2E_UP=', '--up=true|false'), - ('E2E_OPT=', 'Send kubetest the flags directly'), - ('FAIL_ON_GCP_RESOURCE_LEAK=', '--check-leaked-resources=true|false'), - ('FEDERATION_DOWN=', '--down=true|false'), - ('FEDERATION_UP=', '--up=true|false'), - ('GINKGO_PARALLEL=', '--ginkgo-parallel=# (1 for serial)'), - ('GINKGO_PARALLEL_NODES=', '--ginkgo-parallel=# (1 for serial)'), - ('GINKGO_TEST_ARGS=', '--test_args=FOO'), - ('GINKGO_UPGRADE_TEST_ARGS=', '--upgrade_args=FOO'), - ('JENKINS_FEDERATION_PREFIX=', '--stage=gs://FOO'), - ('JENKINS_GCI_PATCH_K8S=', 'Unused, see --extract docs'), - ('JENKINS_PUBLISHED_VERSION=', '--extract=V'), - ('JENKINS_PUBLISHED_SKEW_VERSION=', '--extract=V'), - ('JENKINS_USE_SKEW_KUBECTL=', 'SKEW_KUBECTL=y'), - ('JENKINS_USE_SKEW_TESTS=', '--skew'), - ('JENKINS_SOAK_MODE', '--soak'), - ('JENKINS_SOAK_PREFIX', '--stage=gs://FOO'), - ('JENKINS_USE_EXISTING_BINARIES=', '--extract=local'), - ('JENKINS_USE_LOCAL_BINARIES=', '--extract=none'), - ('JENKINS_USE_SERVER_VERSION=', '--extract=gke'), - ('JENKINS_USE_GCI_VERSION=', '--extract=gci/FAMILY'), - ('JENKINS_USE_GCI_HEAD_IMAGE_FAMILY=', '--extract=gci/FAMILY'), - ('KUBE_GKE_NETWORK=', '--gcp-network=FOO'), - ('KUBE_GCE_NETWORK=', '--gcp-network=FOO'), - ('KUBE_GCE_ZONE=', '--gcp-zone=FOO'), - ('KUBEKINS_TIMEOUT=', '--timeout=XXm'), - ('KUBEMARK_TEST_ARGS=', '--test_args=FOO'), - ('KUBEMARK_TESTS=', '--test_args=--ginkgo.focus=FOO'), - ('KUBEMARK_MASTER_SIZE=', '--kubemark-master-size=FOO'), - ('KUBEMARK_NUM_NODES=', '--kubemark-nodes=FOO'), - ('KUBE_OS_DISTRIBUTION=', '--gcp-node-image=FOO and --gcp-master-image=FOO'), - ('KUBE_NODE_OS_DISTRIBUTION=', '--gcp-node-image=FOO'), - ('KUBE_MASTER_OS_DISTRIBUTION=', '--gcp-master-image=FOO'), - ('KUBERNETES_PROVIDER=', '--provider=FOO'), - ('PERF_TESTS=', '--perf'), - ('PROJECT=', '--gcp-project=FOO'), - ('SKEW_KUBECTL=', '--test_args=--kubectl-path=FOO'), - ('USE_KUBEMARK=', '--kubemark'), - ('ZONE=', '--gcp-zone=FOO'), - ] - for env, fix in black: - if 'kops' in job and env in [ - 'JENKINS_PUBLISHED_VERSION=', - 'JENKINS_USE_LOCAL_BINARIES=', - 'GINKGO_TEST_ARGS=', - 'KUBERNETES_PROVIDER=', - ]: - continue # TODO(fejta): migrate kops jobs - if setting.startswith(env): - self.fail('[%s]: Env %s: Convert %s to use %s in jobs/config.json' % ( - job, setting, env, fix)) - - def test_envs_no_export(self): - for job, job_path in self.jobs: - if not job.endswith('.env'): - continue - with open(job_path) as fp: - lines = list(fp) - for line in lines: - line = line.strip() - self.assertFalse(line.endswith('\\')) - if not line: - continue - if line.startswith('#'): - continue - self._check_env(job, line) - - def test_envs_non_empty(self): - bad = [] - for job, job_path in self.jobs: - if not job.endswith('.env'): - continue - with open(job_path) as fp: - lines = list(fp) - for line in lines: - line = line.strip() - if line and not line.startswith('#'): - break - else: - bad.append(job) - if bad: - self.fail('%s is empty, please remove the file(s)' % bad) - - def test_no_bad_vars_in_jobs(self): - """Searches for jobs that contain ${{VAR}}""" - for job, job_path in self.jobs: - with open(job_path) as fp: - script = fp.read() - bad_vars = re.findall(r'(\${{.+}})', script) - if bad_vars: - self.fail('Job %s contains bad bash variables: %s' % (job, ' '.join(bad_vars))) - -if __name__ == '__main__': - unittest.main() diff --git a/jobs/env_gc.py b/jobs/env_gc.py deleted file mode 100644 index 35a4f115de84..000000000000 --- a/jobs/env_gc.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Garbage collect unused .env files.""" - -import argparse -import os -import json - -ORIG_CWD = os.getcwd() # Checkout changes cwd - -def test_infra(*paths): - """Return path relative to root of test-infra repo.""" - return os.path.join(ORIG_CWD, os.path.dirname(__file__), '..', *paths) - -def find_orphans(): - with open(test_infra('jobs/config.json'), 'r') as fp: - configs = json.loads(fp.read()) - - used = {} - unused = [] - for _, config in configs.items(): - if config['scenario'] == 'execute' or 'args' not in config: - continue - for arg in config['args']: - if (arg.startswith('--env-file=') or - arg.startswith('--properties=')): - used[arg.split('=')[1]] = True - - basepath = test_infra() - for root, _, files in os.walk(test_infra('jobs')): - for name in files: - if name.endswith('.env'): - path = os.path.join(root, name).replace(basepath + '/', '', 1) - if path not in used: - unused.append(path) - - return unused - - -def deep_unlink(path): - """Bazel symlinks to the git client, try readlink() before the unlink().""" - try: - under = os.path.join(os.path.dirname(path), os.readlink(path)) - os.unlink(under) - except OSError: - pass - os.unlink(path) - -def unlink_orphans(): - orphans = find_orphans() - for path in orphans: - print "Deleting unused .env file: {}".format(path) - deep_unlink(test_infra(path)) - if orphans: - print ('\nNote: If this is a git tree, ' + - 'use "git add -u" to stage orphan deletions') - -if __name__ == '__main__': - PARSER = argparse.ArgumentParser( - description='Delete jobs/*.env files not in jobs/config.json') - ARGS = PARSER.parse_args() - - unlink_orphans() diff --git a/jobs/move_extract.py b/jobs/move_extract.py deleted file mode 100755 index ce1cea38699b..000000000000 --- a/jobs/move_extract.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Migrate --extract flag from an JENKINS_FOO env to a scenario flag.""" - -import json -import os -import re -import sys - -ORIG_CWD = os.getcwd() # Checkout changes cwd - -def test_infra(*paths): - """Return path relative to root of test-infra repo.""" - return os.path.join(ORIG_CWD, os.path.dirname(__file__), '..', *paths) - -def sort(): - """Sort config.json alphabetically.""" - # pylint: disable=too-many-branches,too-many-statements,too-many-locals - with open(test_infra('jobs/config.json'), 'r+') as fp: - configs = json.loads(fp.read()) - regexp = re.compile('|'.join([ - r'^KUBE_OS_DISTRIBUTION=(.*)$', - ])) - problems = [] - for job, values in configs.items(): - if values.get('scenario') != 'kubernetes_e2e': - continue - if 'args' not in values: - continue - args = values['args'] - if any('None' in a for a in args): - problems.append('Bad flag with None: %s' % job) - continue - - if not os.path.isfile(test_infra('jobs/env/%s.env' % job)): - continue - with open(test_infra('jobs/env/%s.env' % job)) as fp: - env = fp.read() - lines = [] - mod = False - os_image = '' - for line in env.split('\n'): - mat = regexp.search(line) - if mat: - os_image = mat.group(1) - mod = True - continue - lines.append(line) - if not mod: - continue - - args = list(args) - if os_image: - args.append('--gcp-node-image=%s' % os_image) - args.append('--gcp-master-image=%s' % os_image) - flags = set() - okay = False - for arg in args: - try: - flag, _ = arg.split('=', 1) - except ValueError: - flag = '' - if flag and flag not in ['--env-file', '--extract']: - if flag in flags: - problems.append('Multiple %s in %s' % (flag, job)) - break - flags.add(flag) - else: - okay = True - if not okay: - continue - values['args'] = args - with open(test_infra('jobs/env/%s.env' % job), 'w') as fp: - fp.write('\n'.join(lines)) - with open(test_infra('jobs/config.json'), 'w') as fp: - fp.write(json.dumps(configs, sort_keys=True, indent=2, separators=(',', ': '))) - fp.write('\n') - if not problems: - sys.exit(0) - print >>sys.stderr, '%d problems' % len(problems) - print '\n'.join(problems) - -if __name__ == '__main__': - sort() diff --git a/prow/config.yaml b/prow/config.yaml index 8dc584f41e46..d0c3818e6175 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -515,495 +515,3 @@ presets: volumeMounts: - name: docker-graph mountPath: /docker-graph - -periodics: -- interval: 2h - name: application-periodic-default-gke - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-dind-enabled: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--job=$(JOB_NAME)" - - "--repo=github.com/kubernetes-sigs/application=master" - - "--upload=gs://kubernetes-jenkins/logs/" - - "--timeout=150" - # docker-in-docker needs privileged mode - securityContext: - privileged: true - resources: - requests: - cpu: 6 - memory: "8Gi" - -- name: ci-kubernetes-e2e-autoscaling-vpa-actuation - interval: 2h - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-dind-enabled: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--repo=k8s.io/autoscaler=master" - - "--root=/go/src" - - "--timeout=80" - # docker-in-docker needs privileged mode - securityContext: - privileged: true - -- name: ci-kubernetes-e2e-autoscaling-vpa-admission-controller - interval: 2h - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-dind-enabled: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--repo=k8s.io/autoscaler=master" - - "--root=/go/src" - - "--timeout=80" - # docker-in-docker needs privileged mode - securityContext: - privileged: true - -- name: ci-kubernetes-e2e-autoscaling-vpa-full - interval: 2h - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-dind-enabled: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--repo=k8s.io/autoscaler=master" - - "--root=/go/src" - - "--timeout=80" - # docker-in-docker needs privileged mode - securityContext: - privileged: true - -- name: ci-kubernetes-e2e-autoscaling-vpa-recommender - interval: 2h - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-dind-enabled: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--repo=k8s.io/autoscaler=master" - - "--root=/go/src" - - "--timeout=80" - # docker-in-docker needs privileged mode - securityContext: - privileged: true - -- name: ci-kubernetes-e2e-autoscaling-vpa-updater - interval: 2h - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-dind-enabled: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--repo=k8s.io/autoscaler=master" - - "--root=/go/src" - - "--timeout=80" - # docker-in-docker needs privileged mode - securityContext: - privileged: true - -- interval: 2h - name: ci-kubernetes-e2e-gce-gci-latest-rollback-etcd - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=110 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gce-ha-master - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=240 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gce-influxdb - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=70 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 24h - name: ci-kubernetes-e2e-gce-min-node-permissions - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=520 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- name: ci-kubernetes-e2e-gce-pod-preemption - interval: 2h - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=300 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:latest-master - imagePullPolicy: Always - -- interval: 30m - name: ci-kubernetes-e2e-gce-prometheus - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=70 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gce-stackdriver - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=70 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gce-taint-evict - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=80 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gci-gce-autoscaling - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=350 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gci-gce-autoscaling-hpa - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=350 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gci-gce-autoscaling-migs - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=350 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gci-gce-autoscaling-migs-hpa - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=350 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gci-gce-es-logging - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=110 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gci-gce-garbage - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=620 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gci-gce-proto - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=70 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gci-gce-sd-logging - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=1340 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gci-gce-statefulset - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=110 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gci-gke-autoscaling - labels: - preset-gke-alpha-service: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=620 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gci-gke-autoscaling-hpa - labels: - preset-gke-alpha-service: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=440 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gci-gke-autoscaling-regional - labels: - preset-gke-alpha-service: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=440 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gke-sd-logging-gci-beta - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=1400 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gke-sd-logging-gci-latest - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=1400 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gke-sd-logging-gci-stable1 - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=1400 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gke-sd-logging-ubuntu-beta - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=1400 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gke-sd-logging-ubuntu-latest - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=1400 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gke-sd-logging-ubuntu-stable1 - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=1400 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 30m - name: ci-kubernetes-e2e-gke-stackdriver - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - args: - - --timeout=70 - - --bare - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - -- interval: 2h - name: ci-kustomize-periodic-default-gke - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--job=$(JOB_NAME)" - - "--root=/go/src" - - "--repo=github.com/kubernetes-sigs/kustomize" - - "--upload=gs://kubernetes-jenkins/logs/" - - "--timeout=150" - -- interval: 2h - name: spark-periodic-default-gke - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-dind-enabled: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--job=$(JOB_NAME)" - - "--repo=github.com/apache-spark-on-k8s/spark-integration=master" - - "--upload=gs://kubernetes-jenkins/logs/" - - "--timeout=150" - # docker-in-docker needs privileged mode - securityContext: - privileged: true - resources: - requests: - cpu: 6 - memory: "8Gi" - -- interval: 2h - name: spark-periodic-latest-gke - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-dind-enabled: "true" - spec: - containers: - - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master - args: - - "--job=$(JOB_NAME)" - - "--repo=github.com/apache-spark-on-k8s/spark-integration=master" - - "--upload=gs://kubernetes-jenkins/logs/" - - "--timeout=150" - # docker-in-docker needs privileged mode - securityContext: - privileged: true - resources: - requests: - cpu: 6 - memory: "8Gi" diff --git a/testgrid/config.yaml b/testgrid/config.yaml index 91c9649e2244..e14593a36e28 100644 --- a/testgrid/config.yaml +++ b/testgrid/config.yaml @@ -1460,9 +1460,6 @@ test_groups: gcs_prefix: kubernetes-multiarch-e2e-results/logs/kubeadm-luxas-scaleway-arm64 - name: periodic-multi-platform-kubeadm-packet-arm64 gcs_prefix: kubernetes-multiarch-e2e-results/logs/kubeadm-luxas-packet-arm64 -# garbage collector tests -- name: ci-kubernetes-e2e-gci-gce-garbage - gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-gci-gce-garbage # kubeadm tests - name: ci-kubernetes-e2e-kubeadm-gce gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-kubeadm-gce @@ -4179,8 +4176,6 @@ dashboards: test_group_name: ci-kops-build - name: debian-unstable test_group_name: ci-kubernetes-build-debian-unstable - - name: gci-garbage-collector - test_group_name: ci-kubernetes-e2e-gci-gce-garbage - name: sig-release-1.10-all dashboard_tab: From 872376f2d95b33ca236eceeef939321d693c4b16 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 31 Aug 2018 14:20:00 -0700 Subject: [PATCH 24/42] bump prow to v20180831-52e265274 --- prow/cluster/branchprotector_cronjob.yaml | 2 +- prow/cluster/deck_deployment.yaml | 2 +- prow/cluster/hook_deployment.yaml | 2 +- prow/cluster/horologium_deployment.yaml | 2 +- prow/cluster/needs-rebase_deployment.yaml | 2 +- prow/cluster/plank_deployment.yaml | 2 +- prow/cluster/sinker_deployment.yaml | 2 +- prow/cluster/splice_deployment.yaml | 2 +- prow/cluster/starter.yaml | 10 +++++----- prow/cluster/tide_deployment.yaml | 2 +- prow/cluster/tot_deployment.yaml | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/prow/cluster/branchprotector_cronjob.yaml b/prow/cluster/branchprotector_cronjob.yaml index bd1afbd4099b..3f97f05558b1 100644 --- a/prow/cluster/branchprotector_cronjob.yaml +++ b/prow/cluster/branchprotector_cronjob.yaml @@ -14,7 +14,7 @@ spec: spec: containers: - name: branchprotector - image: gcr.io/k8s-prow/branchprotector:v20180830-eabe43164 + image: gcr.io/k8s-prow/branchprotector:v20180831-52e265274 args: - --config-path=/etc/config/config.yaml - --job-config-path=/etc/job-config diff --git a/prow/cluster/deck_deployment.yaml b/prow/cluster/deck_deployment.yaml index 2d91dd53e7e8..b676ab8cba8f 100644 --- a/prow/cluster/deck_deployment.yaml +++ b/prow/cluster/deck_deployment.yaml @@ -34,7 +34,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: deck - image: gcr.io/k8s-prow/deck:v20180830-eabe43164 + image: gcr.io/k8s-prow/deck:v20180831-52e265274 imagePullPolicy: Always ports: - name: http diff --git a/prow/cluster/hook_deployment.yaml b/prow/cluster/hook_deployment.yaml index b10e711a4b03..ce65cd479416 100644 --- a/prow/cluster/hook_deployment.yaml +++ b/prow/cluster/hook_deployment.yaml @@ -34,7 +34,7 @@ spec: terminationGracePeriodSeconds: 180 containers: - name: hook - image: gcr.io/k8s-prow/hook:v20180830-eabe43164 + image: gcr.io/k8s-prow/hook:v20180831-52e265274 imagePullPolicy: Always args: - --dry-run=false diff --git a/prow/cluster/horologium_deployment.yaml b/prow/cluster/horologium_deployment.yaml index 143075254e55..b869c5e9ce89 100644 --- a/prow/cluster/horologium_deployment.yaml +++ b/prow/cluster/horologium_deployment.yaml @@ -29,7 +29,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: horologium - image: gcr.io/k8s-prow/horologium:v20180830-eabe43164 + image: gcr.io/k8s-prow/horologium:v20180831-52e265274 args: - --job-config-path=/etc/job-config volumeMounts: diff --git a/prow/cluster/needs-rebase_deployment.yaml b/prow/cluster/needs-rebase_deployment.yaml index a2035aa4593b..247b265bbeb5 100644 --- a/prow/cluster/needs-rebase_deployment.yaml +++ b/prow/cluster/needs-rebase_deployment.yaml @@ -28,7 +28,7 @@ spec: terminationGracePeriodSeconds: 180 containers: - name: needs-rebase - image: gcr.io/k8s-prow/needs-rebase:v20180830-eabe43164 + image: gcr.io/k8s-prow/needs-rebase:v20180831-52e265274 imagePullPolicy: Always args: - --dry-run=false diff --git a/prow/cluster/plank_deployment.yaml b/prow/cluster/plank_deployment.yaml index fae8928048cb..afeb2df7e346 100644 --- a/prow/cluster/plank_deployment.yaml +++ b/prow/cluster/plank_deployment.yaml @@ -28,7 +28,7 @@ spec: # serviceAccountName: "plank" # Uncomment for use with RBAC containers: - name: plank - image: gcr.io/k8s-prow/plank:v20180830-eabe43164 + image: gcr.io/k8s-prow/plank:v20180831-52e265274 args: - --tot-url=http://tot - --build-cluster=/etc/cluster/cluster diff --git a/prow/cluster/sinker_deployment.yaml b/prow/cluster/sinker_deployment.yaml index 35aa1b19daad..b91b079fef40 100644 --- a/prow/cluster/sinker_deployment.yaml +++ b/prow/cluster/sinker_deployment.yaml @@ -17,7 +17,7 @@ spec: args: - --build-cluster=/etc/cluster/cluster - --job-config-path=/etc/job-config - image: gcr.io/k8s-prow/sinker:v20180830-eabe43164 + image: gcr.io/k8s-prow/sinker:v20180831-52e265274 volumeMounts: - mountPath: /etc/cluster name: cluster diff --git a/prow/cluster/splice_deployment.yaml b/prow/cluster/splice_deployment.yaml index a89713b41873..bef2ef74e589 100644 --- a/prow/cluster/splice_deployment.yaml +++ b/prow/cluster/splice_deployment.yaml @@ -13,7 +13,7 @@ spec: spec: containers: - name: splice - image: gcr.io/k8s-prow/splice:v20180830-eabe43164 + image: gcr.io/k8s-prow/splice:v20180831-52e265274 args: - --job-config-path=/etc/job-config volumeMounts: diff --git a/prow/cluster/starter.yaml b/prow/cluster/starter.yaml index 008bd497566d..a31167a76e8c 100644 --- a/prow/cluster/starter.yaml +++ b/prow/cluster/starter.yaml @@ -97,7 +97,7 @@ spec: terminationGracePeriodSeconds: 180 containers: - name: hook - image: gcr.io/k8s-prow/hook:v20180830-eabe43164 + image: gcr.io/k8s-prow/hook:v20180831-52e265274 imagePullPolicy: Always args: - --dry-run=false @@ -158,7 +158,7 @@ spec: serviceAccountName: "plank" containers: - name: plank - image: gcr.io/k8s-prow/plank:v20180830-eabe43164 + image: gcr.io/k8s-prow/plank:v20180831-52e265274 args: - --dry-run=false volumeMounts: @@ -192,7 +192,7 @@ spec: serviceAccountName: "sinker" containers: - name: sinker - image: gcr.io/k8s-prow/sinker:v20180830-eabe43164 + image: gcr.io/k8s-prow/sinker:v20180831-52e265274 volumeMounts: - name: config mountPath: /etc/config @@ -224,7 +224,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: deck - image: gcr.io/k8s-prow/deck:v20180830-eabe43164 + image: gcr.io/k8s-prow/deck:v20180831-52e265274 args: - --hook-url=http://hook:8888/plugin-help ports: @@ -268,7 +268,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: horologium - image: gcr.io/k8s-prow/horologium:v20180830-eabe43164 + image: gcr.io/k8s-prow/horologium:v20180831-52e265274 volumeMounts: - name: config mountPath: /etc/config diff --git a/prow/cluster/tide_deployment.yaml b/prow/cluster/tide_deployment.yaml index 5575bed5c4c8..1bdd2af35359 100644 --- a/prow/cluster/tide_deployment.yaml +++ b/prow/cluster/tide_deployment.yaml @@ -28,7 +28,7 @@ spec: # serviceAccountName: "tide" # Uncomment for use with RBAC containers: - name: tide - image: gcr.io/k8s-prow/tide:v20180830-eabe43164 + image: gcr.io/k8s-prow/tide:v20180831-52e265274 args: - --dry-run=false - --github-endpoint=http://ghproxy diff --git a/prow/cluster/tot_deployment.yaml b/prow/cluster/tot_deployment.yaml index 4c0332bcb375..0d332e9e862f 100644 --- a/prow/cluster/tot_deployment.yaml +++ b/prow/cluster/tot_deployment.yaml @@ -57,7 +57,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: tot - image: gcr.io/k8s-prow/tot:v20180830-eabe43164 + image: gcr.io/k8s-prow/tot:v20180831-52e265274 imagePullPolicy: Always args: - -storage=/store/tot.json From f1c0e7fd0b85b0d6f8032266dc58a0338609fed8 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 31 Aug 2018 13:54:55 -0700 Subject: [PATCH 25/42] enable welcome plugin on test-infra for testing --- prow/plugins.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/prow/plugins.yaml b/prow/plugins.yaml index 38f72a3e3dc1..54a2044f9292 100644 --- a/prow/plugins.yaml +++ b/prow/plugins.yaml @@ -199,6 +199,9 @@ config_updater: config/jobs/**/*.yaml: name: job-config +welcome: + message_template: "Welcome @{{.AuthorLogin}}! It looks like this is your first PR to {{.Org}}/{{.Repo}} 🎉" + plugins: bazelbuild/rules_k8s: - approve # Allow OWNERS to /approve @@ -440,6 +443,7 @@ plugins: - override - trigger - verify-owners + - welcome kubernetes/utils: - approve From fdd03a92203526dea234230e2aab5a675ba0b145 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 31 Aug 2018 15:19:48 -0700 Subject: [PATCH 26/42] welcome: check for <= 1 PRs --- prow/plugins/welcome/welcome.go | 4 ++-- prow/plugins/welcome/welcome_test.go | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/prow/plugins/welcome/welcome.go b/prow/plugins/welcome/welcome.go index 62a780672a12..a5c355cc8579 100644 --- a/prow/plugins/welcome/welcome.go +++ b/prow/plugins/welcome/welcome.go @@ -96,8 +96,8 @@ func handlePR(c client, pre github.PullRequestEvent, welcomeTemplate string) err return err } - // if there is exactly one result, this is the first! post the welcome comment - if len(issues) == 1 { + // if there are no results, this is the first! post the welcome comment + if len(issues) == 0 || len(issues) == 1 && issues[0].Number == pre.Number { // load the template, and run it over the PR info parsedTemplate, err := template.New("welcome").Parse(welcomeTemplate) if err != nil { diff --git a/prow/plugins/welcome/welcome_test.go b/prow/plugins/welcome/welcome_test.go index a057fd05d750..ca8a60718b54 100644 --- a/prow/plugins/welcome/welcome_test.go +++ b/prow/plugins/welcome/welcome_test.go @@ -139,6 +139,7 @@ func TestHandlePR(t *testing.T) { author string prNumber int prAction github.PullRequestEventAction + addPR bool expectComment bool }{ { @@ -168,6 +169,16 @@ func TestHandlePR(t *testing.T) { prNumber: 50, expectComment: true, }, + { + name: "new contributor and API recorded PR already", + repoOwner: "kubernetes", + repoName: "test-infra", + author: "newContributor", + prAction: github.PullRequestActionOpened, + prNumber: 50, + expectComment: true, + addPR: true, + }, { name: "new contributor, not PR open event", repoOwner: "kubernetes", @@ -188,8 +199,10 @@ func TestHandlePR(t *testing.T) { fc.ClearComments() event := makeFakePullRequestEvent(tc.repoOwner, tc.repoName, tc.author, tc.prNumber, tc.prAction) - // make sure the PR in the event is recorded - fc.AddPR(tc.repoOwner, tc.repoName, tc.author, tc.prNumber) + if tc.addPR { + // make sure the PR in the event is recorded + fc.AddPR(tc.repoOwner, tc.repoName, tc.author, tc.prNumber) + } // try handling it if err := handlePR(c, event, testWelcomeTemplate); err != nil { From 49a3290844896eb3f55c74ea7814174de3332d94 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 31 Aug 2018 16:24:04 -0700 Subject: [PATCH 27/42] bump prow to v20180831-d506e8661 --- prow/cluster/branchprotector_cronjob.yaml | 2 +- prow/cluster/deck_deployment.yaml | 2 +- prow/cluster/hook_deployment.yaml | 2 +- prow/cluster/horologium_deployment.yaml | 2 +- prow/cluster/needs-rebase_deployment.yaml | 2 +- prow/cluster/plank_deployment.yaml | 2 +- prow/cluster/sinker_deployment.yaml | 2 +- prow/cluster/splice_deployment.yaml | 2 +- prow/cluster/starter.yaml | 10 +++++----- prow/cluster/tide_deployment.yaml | 2 +- prow/cluster/tot_deployment.yaml | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/prow/cluster/branchprotector_cronjob.yaml b/prow/cluster/branchprotector_cronjob.yaml index 3f97f05558b1..cb2fe26087dc 100644 --- a/prow/cluster/branchprotector_cronjob.yaml +++ b/prow/cluster/branchprotector_cronjob.yaml @@ -14,7 +14,7 @@ spec: spec: containers: - name: branchprotector - image: gcr.io/k8s-prow/branchprotector:v20180831-52e265274 + image: gcr.io/k8s-prow/branchprotector:v20180831-d506e8661 args: - --config-path=/etc/config/config.yaml - --job-config-path=/etc/job-config diff --git a/prow/cluster/deck_deployment.yaml b/prow/cluster/deck_deployment.yaml index b676ab8cba8f..dcd130b8b9b2 100644 --- a/prow/cluster/deck_deployment.yaml +++ b/prow/cluster/deck_deployment.yaml @@ -34,7 +34,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: deck - image: gcr.io/k8s-prow/deck:v20180831-52e265274 + image: gcr.io/k8s-prow/deck:v20180831-d506e8661 imagePullPolicy: Always ports: - name: http diff --git a/prow/cluster/hook_deployment.yaml b/prow/cluster/hook_deployment.yaml index ce65cd479416..5ab621b71a25 100644 --- a/prow/cluster/hook_deployment.yaml +++ b/prow/cluster/hook_deployment.yaml @@ -34,7 +34,7 @@ spec: terminationGracePeriodSeconds: 180 containers: - name: hook - image: gcr.io/k8s-prow/hook:v20180831-52e265274 + image: gcr.io/k8s-prow/hook:v20180831-d506e8661 imagePullPolicy: Always args: - --dry-run=false diff --git a/prow/cluster/horologium_deployment.yaml b/prow/cluster/horologium_deployment.yaml index b869c5e9ce89..9cc861d6a748 100644 --- a/prow/cluster/horologium_deployment.yaml +++ b/prow/cluster/horologium_deployment.yaml @@ -29,7 +29,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: horologium - image: gcr.io/k8s-prow/horologium:v20180831-52e265274 + image: gcr.io/k8s-prow/horologium:v20180831-d506e8661 args: - --job-config-path=/etc/job-config volumeMounts: diff --git a/prow/cluster/needs-rebase_deployment.yaml b/prow/cluster/needs-rebase_deployment.yaml index 247b265bbeb5..a90a133d4add 100644 --- a/prow/cluster/needs-rebase_deployment.yaml +++ b/prow/cluster/needs-rebase_deployment.yaml @@ -28,7 +28,7 @@ spec: terminationGracePeriodSeconds: 180 containers: - name: needs-rebase - image: gcr.io/k8s-prow/needs-rebase:v20180831-52e265274 + image: gcr.io/k8s-prow/needs-rebase:v20180831-d506e8661 imagePullPolicy: Always args: - --dry-run=false diff --git a/prow/cluster/plank_deployment.yaml b/prow/cluster/plank_deployment.yaml index afeb2df7e346..324065147eaf 100644 --- a/prow/cluster/plank_deployment.yaml +++ b/prow/cluster/plank_deployment.yaml @@ -28,7 +28,7 @@ spec: # serviceAccountName: "plank" # Uncomment for use with RBAC containers: - name: plank - image: gcr.io/k8s-prow/plank:v20180831-52e265274 + image: gcr.io/k8s-prow/plank:v20180831-d506e8661 args: - --tot-url=http://tot - --build-cluster=/etc/cluster/cluster diff --git a/prow/cluster/sinker_deployment.yaml b/prow/cluster/sinker_deployment.yaml index b91b079fef40..0a863642cad9 100644 --- a/prow/cluster/sinker_deployment.yaml +++ b/prow/cluster/sinker_deployment.yaml @@ -17,7 +17,7 @@ spec: args: - --build-cluster=/etc/cluster/cluster - --job-config-path=/etc/job-config - image: gcr.io/k8s-prow/sinker:v20180831-52e265274 + image: gcr.io/k8s-prow/sinker:v20180831-d506e8661 volumeMounts: - mountPath: /etc/cluster name: cluster diff --git a/prow/cluster/splice_deployment.yaml b/prow/cluster/splice_deployment.yaml index bef2ef74e589..a0f88e0cec7a 100644 --- a/prow/cluster/splice_deployment.yaml +++ b/prow/cluster/splice_deployment.yaml @@ -13,7 +13,7 @@ spec: spec: containers: - name: splice - image: gcr.io/k8s-prow/splice:v20180831-52e265274 + image: gcr.io/k8s-prow/splice:v20180831-d506e8661 args: - --job-config-path=/etc/job-config volumeMounts: diff --git a/prow/cluster/starter.yaml b/prow/cluster/starter.yaml index a31167a76e8c..11fd8cd5f8ea 100644 --- a/prow/cluster/starter.yaml +++ b/prow/cluster/starter.yaml @@ -97,7 +97,7 @@ spec: terminationGracePeriodSeconds: 180 containers: - name: hook - image: gcr.io/k8s-prow/hook:v20180831-52e265274 + image: gcr.io/k8s-prow/hook:v20180831-d506e8661 imagePullPolicy: Always args: - --dry-run=false @@ -158,7 +158,7 @@ spec: serviceAccountName: "plank" containers: - name: plank - image: gcr.io/k8s-prow/plank:v20180831-52e265274 + image: gcr.io/k8s-prow/plank:v20180831-d506e8661 args: - --dry-run=false volumeMounts: @@ -192,7 +192,7 @@ spec: serviceAccountName: "sinker" containers: - name: sinker - image: gcr.io/k8s-prow/sinker:v20180831-52e265274 + image: gcr.io/k8s-prow/sinker:v20180831-d506e8661 volumeMounts: - name: config mountPath: /etc/config @@ -224,7 +224,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: deck - image: gcr.io/k8s-prow/deck:v20180831-52e265274 + image: gcr.io/k8s-prow/deck:v20180831-d506e8661 args: - --hook-url=http://hook:8888/plugin-help ports: @@ -268,7 +268,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: horologium - image: gcr.io/k8s-prow/horologium:v20180831-52e265274 + image: gcr.io/k8s-prow/horologium:v20180831-d506e8661 volumeMounts: - name: config mountPath: /etc/config diff --git a/prow/cluster/tide_deployment.yaml b/prow/cluster/tide_deployment.yaml index 1bdd2af35359..f9cb4d415adc 100644 --- a/prow/cluster/tide_deployment.yaml +++ b/prow/cluster/tide_deployment.yaml @@ -28,7 +28,7 @@ spec: # serviceAccountName: "tide" # Uncomment for use with RBAC containers: - name: tide - image: gcr.io/k8s-prow/tide:v20180831-52e265274 + image: gcr.io/k8s-prow/tide:v20180831-d506e8661 args: - --dry-run=false - --github-endpoint=http://ghproxy diff --git a/prow/cluster/tot_deployment.yaml b/prow/cluster/tot_deployment.yaml index 0d332e9e862f..f888d5edc7a2 100644 --- a/prow/cluster/tot_deployment.yaml +++ b/prow/cluster/tot_deployment.yaml @@ -57,7 +57,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: tot - image: gcr.io/k8s-prow/tot:v20180831-52e265274 + image: gcr.io/k8s-prow/tot:v20180831-d506e8661 imagePullPolicy: Always args: - -storage=/store/tot.json From 1b3cdb46135908d4d4e2d32e74a7221ca4d408f3 Mon Sep 17 00:00:00 2001 From: Sen Lu Date: Fri, 31 Aug 2018 15:44:38 -0700 Subject: [PATCH 28/42] clean up config.json reference in tests, update docs --- README.md | 58 +++++++++++++++++--- config/README.md | 16 ++++-- config/jobs/kubernetes-security/genjobs.go | 62 +++++----------------- config/tests/jobs/jobs_test.go | 50 +++-------------- hack/update-config.sh | 1 - hack/verify-config.sh | 2 - kubetest/README.md | 3 +- prow/README.md | 1 + prow/config/jobtests/job_config_test.go | 35 ------------ 9 files changed, 90 insertions(+), 138 deletions(-) diff --git a/README.md b/README.md index 4768d41b264c..77a84afabe9a 100644 --- a/README.md +++ b/README.md @@ -28,18 +28,61 @@ the different services interact. ## Automated testing -Test anything with the following pattern: +### If you are using kubekins | bootstrap + +Assume your job looks something like + +```yaml + - name: foo-bar-test + interval: 1h + agent: kubernetes + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:latest-master + args: + - --repo=github.com/foo/bar + - --timeout=90 + - --scenario=execute + - -- + - make + - test +``` + +You can see both images use the [same entrypoint script](/images/bootstrap/entrypoint.sh): +```sh +/usr/local/bin/runner.sh \ + ./test-infra/jenkins/bootstrap.py \ + --job="${JOB_NAME}" \ + --service-account="${GOOGLE_APPLICATION_CREDENTIALS}" \ + --upload='gs://kubernetes-jenkins/logs' \ + "$@" ``` + +So to mimic the run locally, you can dump all the args to the entrypoint script, like: + +```sh git clone https://github.com/kubernetes/test-infra -test-infra/jenkins/bootstrap.py --job=J --repo=R --service-account=S.json --upload=gs://B +test-infra/jenkins/bootstrap.py --job=foo-bar-test \ + --repo=github.com/foo/bar \ + --service-account=S.json \ + --upload=gs://B \ + --timeout=90 \ + --scenario=execute \ + -- \ + make \ + test ``` -The `--job=J` flag specifies what test job to run. -The `--repo=R` (or `--bare`) flag controls what we check out from git. +where `--service-account` is the service account you want to activate for your job, and +`--upload` is the gcs bucket where you want to upload your job results to. -Anyone can reconfigure our CI system with a test-infra PR that updates the -appropriate files. Detailed instructions follow: +### If you are using podutils: + +Unfortunately there's no easy way to test it locally - you can follow [getting started](/prow/getting_started.md) +to schedule a job against your own prow cluster. + +We are working on have a utility to run the job locally - https://github.com/kubernetes/test-infra/issues/6590 ### E2E Testing @@ -47,6 +90,9 @@ Our e2e testing uses [kubetest](/kubetest) to build/deploy/test kubernetes clusters on various providers. Please see those documents for additional details about this tool as well as e2e testing generally. +Anyone can reconfigure our CI system with a test-infra PR that updates the +appropriate files. Detailed instructions follow: + ### Create a new job Create a PR in this repo to add/update/remove a job or suite. Specifically diff --git a/config/README.md b/config/README.md index 9d534b0fd55a..ae4724a99bc1 100644 --- a/config/README.md +++ b/config/README.md @@ -2,9 +2,19 @@ This is a central place for Kubernetes-project specific configs for other tools in this repo. -### UNDER MIGRATION - -Directory structure: +## Directory structure: [jobs/](./jobs) : job configs for Kubernetes Prow deployment, potentially testgrid configs as well [tests](./tests) : validation tests for the configs + +## Adding new jobs to Prow: + +1. Find or create an org/repo directory under config/jobs, eg: config/jobs/kubernetes-sigs/kustomize for jobs related to https://github.com/kubernetes-sigs/kustomize. + +1. Create an OWNERS file and add appropriate approver/reviewer for your job. + +1. Add a *.yaml file (the base name has to be unique), and follow https://github.com/kubernetes/test-infra/tree/master/prow#how-to-add-new-jobs for adding new prowjobs. + +Also please read [`create-a-new-job`] + +[`create-a-new-job`]: /README.md#create-a-new-job diff --git a/config/jobs/kubernetes-security/genjobs.go b/config/jobs/kubernetes-security/genjobs.go index b566c3d3a9a6..8df64a51d831 100644 --- a/config/jobs/kubernetes-security/genjobs.go +++ b/config/jobs/kubernetes-security/genjobs.go @@ -27,7 +27,6 @@ package main import ( "bytes" - "encoding/json" "fmt" "io" "io/ioutil" @@ -49,41 +48,7 @@ import ( var configPath = flag.String("config", "", "path to prow/config.yaml, defaults to $PWD/../../prow/config.yaml") var jobsPath = flag.String("jobs", "", "path to prowjobs, defaults to $PWD/../") -var configJSONPath = flag.String("config-json", "", "path to jobs/config.json, defaults to $PWD/../../jobs/config.json") -var outputPath = flag.String("output", "", "path to output the generated jobs to, defaults to $PWD/generated-security-jobs.json") - -// config.json is the worst but contains useful information :-( -type configJSON map[string]map[string]interface{} - -func (c configJSON) ScenarioForJob(jobName string) string { - if scenario, ok := c[jobName]["scenario"]; ok { - return scenario.(string) - } - return "" -} - -func (c configJSON) ArgsForJob(jobName string) []string { - res := []string{} - if args, ok := c[jobName]["args"]; ok { - for _, arg := range args.([]interface{}) { - res = append(res, arg.(string)) - } - } - return res -} - -func readConfigJSON(path string) (config configJSON, err error) { - raw, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - config = configJSON{} - err = json.Unmarshal(raw, &config) - if err != nil { - return nil, err - } - return config, nil -} +var outputPath = flag.String("output", "", "path to output the generated jobs to, defaults to $PWD/generated-security-jobs.yaml") // remove merged presets from a podspec func undoPreset(preset *config.Preset, labels map[string]string, pod *v1.PodSpec) { @@ -154,7 +119,7 @@ func undoPresubmitPresets(presets []config.Preset, presubmit *config.Presubmit) // convert a kubernetes/kubernetes job to a kubernetes-security/kubernetes job // dropLabels should be a set of "k: v" strings // xref: prow/config/config_test.go replace(...) -func convertJobToSecurityJob(j *config.Presubmit, dropLabels sets.String, jobsConfig configJSON) { +func convertJobToSecurityJob(j *config.Presubmit, dropLabels sets.String) { // filter out the unwanted labels if len(j.Labels) > 0 { filteredLabels := make(map[string]string) @@ -210,9 +175,15 @@ func convertJobToSecurityJob(j *config.Presubmit, dropLabels sets.String, jobsCo // check for scenario specific tweaks // NOTE: jobs are remapped to their original name in bootstrap to de-dupe config + scenario := "" + for _, arg := range container.Args { + if strings.HasPrefix(arg, "--scenario=") { + scenario = strings.TrimPrefix(arg, "--scenario=") + } + } // check if we need to change staging artifact location for bazel-build and e2es - if jobsConfig.ScenarioForJob(originalName) == "kubernetes_bazel" { - for _, arg := range jobsConfig.ArgsForJob(originalName) { + if scenario == "kubernetes_bazel" { + for _, arg := range container.Args { if strings.HasPrefix(arg, "--release") { needGCSFlag = true needGCSSharedFlag = true @@ -221,8 +192,8 @@ func convertJobToSecurityJob(j *config.Presubmit, dropLabels sets.String, jobsCo } } - if jobsConfig.ScenarioForJob(originalName) == "kubernetes_e2e" { - for _, arg := range jobsConfig.ArgsForJob(originalName) { + if scenario == "kubernetes_e2e" { + for _, arg := range container.Args { if strings.HasPrefix(arg, "--stage") { needStagingFlag = true } else if strings.HasPrefix(arg, "--use-shared-build") { @@ -269,7 +240,7 @@ func convertJobToSecurityJob(j *config.Presubmit, dropLabels sets.String, jobsCo } // done with this job, check for run_after_success for i := range j.RunAfterSuccess { - convertJobToSecurityJob(&j.RunAfterSuccess[i], dropLabels, jobsConfig) + convertJobToSecurityJob(&j.RunAfterSuccess[i], dropLabels) } } @@ -331,9 +302,6 @@ func main() { if *jobsPath == "" { *jobsPath = pwd + "/../" } - if *configJSONPath == "" { - *configJSONPath = pwd + "/../../jobs/config.json" - } if *outputPath == "" { *outputPath = pwd + "/generated-security-jobs.yaml" } @@ -342,8 +310,6 @@ func main() { if err != nil { log.Fatalf("Failed to read config file: %v", err) } - // read in jobs config - jobsConfig, err := readConfigJSON(*configJSONPath) // create temp file to write updated config f, err := ioutil.TempFile(filepath.Dir(*configPath), "temp") @@ -368,7 +334,7 @@ func main() { // undo merged presets, this needs to occur first! undoPresubmitPresets(parsed.Presets, job) // now convert the job - convertJobToSecurityJob(job, dropLabels, jobsConfig) + convertJobToSecurityJob(job, dropLabels) jobBytes, err := yaml.Marshal(job) if err != nil { log.Fatalf("Failed to marshal job: %v", err) diff --git a/config/tests/jobs/jobs_test.go b/config/tests/jobs/jobs_test.go index 0c900a40ae6b..d1ea33bc5bad 100644 --- a/config/tests/jobs/jobs_test.go +++ b/config/tests/jobs/jobs_test.go @@ -22,11 +22,9 @@ package tests import ( "bytes" - "encoding/json" "errors" "flag" "fmt" - "io/ioutil" "os" "regexp" "strings" @@ -40,39 +38,14 @@ import ( "k8s.io/test-infra/prow/kube" ) -// config.json is the worst but contains useful information :-( -type configJSON map[string]map[string]interface{} - var configPath = flag.String("config", "../../../prow/config.yaml", "Path to prow config") var jobConfigPath = flag.String("job-config", "../../jobs", "Path to prow job config") -var configJSONPath = flag.String("config-json", "../../../jobs/config.json", "Path to prow job config") var gubernatorPath = flag.String("gubernator-path", "https://k8s-gubernator.appspot.com", "Path to linked gubernator") var bucket = flag.String("bucket", "kubernetes-jenkins", "Gcs bucket for log upload") var k8sProw = flag.Bool("k8s-prow", true, "If the config is for k8s prow cluster") -func (c configJSON) ScenarioForJob(jobName string) string { - if scenario, ok := c[jobName]["scenario"]; ok { - return scenario.(string) - } - return "" -} - -func readConfigJSON(path string) (config configJSON, err error) { - raw, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - config = configJSON{} - err = json.Unmarshal(raw, &config) - if err != nil { - return nil, err - } - return config, nil -} - // Loaded at TestMain. var c *cfg.Config -var cj configJSON func TestMain(m *testing.M) { flag.Parse() @@ -88,14 +61,6 @@ func TestMain(m *testing.M) { } c = conf - if *configJSONPath != "" { - cj, err = readConfigJSON(*configJSONPath) - if err != nil { - fmt.Printf("Could not load jobs config: %v", err) - os.Exit(1) - } - } - os.Exit(m.Run()) } @@ -659,8 +624,14 @@ func checkKubekinsPresets(jobName string, spec *v1.PodSpec, labels, validLabels } } - configJSONJobName := strings.Replace(jobName, "pull-kubernetes", "pull-security-kubernetes", -1) - if cj.ScenarioForJob(configJSONJobName) == "kubenetes_e2e" { + scenario := "" + for _, arg := range container.Args { + if strings.HasPrefix(arg, "--scenario=") { + scenario = strings.TrimPrefix(arg, "--scenario=") + } + } + + if scenario == "kubenetes_e2e" { ssh = false for key, val := range labels { if (key == "preset-k8s-ssh" || key == "preset-aws-ssh") && val == "true" { @@ -770,11 +741,6 @@ func checkScenarioArgs(jobName, imageName string, args []string) error { entry = strings.Replace(entry, "pull-security-kubernetes", "pull-kubernetes", -1) } - if _, ok := cj[entry]; ok { - // the unit test is handled in jobs/config_test.py - return nil - } - if !scenarioArgs { if strings.Contains(imageName, "kubekins-e2e") || strings.Contains(imageName, "bootstrap") || diff --git a/hack/update-config.sh b/hack/update-config.sh index 25368bacdbfe..52b5e5fe5312 100755 --- a/hack/update-config.sh +++ b/hack/update-config.sh @@ -21,6 +21,5 @@ TESTINFRA_ROOT=$(git rev-parse --show-toplevel) bazel run //config/jobs/kubernetes-security:genjobs -- \ "--config=${TESTINFRA_ROOT}/prow/config.yaml" \ -"--config-json=${TESTINFRA_ROOT}/jobs/config.json" \ "--jobs=${TESTINFRA_ROOT}/config/jobs" \ "--output=${TESTINFRA_ROOT}/config/jobs/kubernetes-security/generated-security-jobs.yaml" diff --git a/hack/verify-config.sh b/hack/verify-config.sh index 518d2b320368..b89fdaae2b43 100755 --- a/hack/verify-config.sh +++ b/hack/verify-config.sh @@ -21,7 +21,6 @@ set -o xtrace TESTINFRA_ROOT=$(git rev-parse --show-toplevel) PROW_CONFIG="${TESTINFRA_ROOT}/prow/config.yaml" -JOBS_CONFIG="${TESTINFRA_ROOT}/jobs/config.json" JOBS_DIR="${TESTINFRA_ROOT}/config/jobs" TMP_CONFIG=$(mktemp) TMP_GENERATED_JOBS=$(mktemp) @@ -31,7 +30,6 @@ cp "${PROW_CONFIG}" "${TMP_CONFIG}" bazel run //config/jobs/kubernetes-security:genjobs -- \ "--config=${PROW_CONFIG}" \ -"--config-json=${JOBS_CONFIG}" \ "--jobs=${JOBS_DIR}" \ "--output=${TMP_GENERATED_JOBS}" diff --git a/kubetest/README.md b/kubetest/README.md index 8e92d18d0500..e87468a490cf 100644 --- a/kubetest/README.md +++ b/kubetest/README.md @@ -195,7 +195,7 @@ upgrading from v1 to v2. The command does not run other e2e tests after completi the upgrade tests. If you want to run the e2e tests, specify also `--test` and `--test_args` flags. -Tips: CI upgrade tests listed at `jobs/config.json` show flags used in the real CI +Tips: CI upgrade tests listed at [sig-cluster-lifecycle config] show flags used in the real CI test environment, which is a good source to learn more about how the flags are used. ### Staging @@ -223,3 +223,4 @@ right path, the command below does the same as above: [kubekins-e2e]: /images/kubekins-e2e [kubekins-e2e-prow]: /images/e2e-prow [prow]: /prow +[sig-cluster-lifecycle config]: /config/jobs/kubernetes/sig-cluster-lifecycle diff --git a/prow/README.md b/prow/README.md index 6c75338075b5..7b254d435894 100644 --- a/prow/README.md +++ b/prow/README.md @@ -239,4 +239,5 @@ for them. ### KubeCon 2018 EU [Automation and the Kubernetes Contributor Experience](https://www.youtube.com/watch?v=BsIC7gPkH5M) + [SIG Testing Deep Dive](https://www.youtube.com/watch?v=M32NIHRKaOI) diff --git a/prow/config/jobtests/job_config_test.go b/prow/config/jobtests/job_config_test.go index 6e5a761b07ad..9aeae345d54c 100644 --- a/prow/config/jobtests/job_config_test.go +++ b/prow/config/jobtests/job_config_test.go @@ -17,10 +17,8 @@ limitations under the License. package jobtests import ( - "encoding/json" "flag" "fmt" - "io/ioutil" "os" "testing" @@ -30,36 +28,11 @@ import ( cfg "k8s.io/test-infra/prow/config" ) -// config.json is the worst but contains useful information :-( -type configJSON map[string]map[string]interface{} - var configPath = flag.String("config", "../../config.yaml", "Path to prow config") var jobConfigPath = flag.String("job-config", "../../../config/jobs", "Path to prow job config") -var configJSONPath = flag.String("config-json", "../../../jobs/config.json", "Path to prow job config") - -func (c configJSON) ScenarioForJob(jobName string) string { - if scenario, ok := c[jobName]["scenario"]; ok { - return scenario.(string) - } - return "" -} - -func readConfigJSON(path string) (config configJSON, err error) { - raw, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - config = configJSON{} - err = json.Unmarshal(raw, &config) - if err != nil { - return nil, err - } - return config, nil -} // Loaded at TestMain. var c *cfg.Config -var cj configJSON func TestMain(m *testing.M) { flag.Parse() @@ -75,14 +48,6 @@ func TestMain(m *testing.M) { } c = conf - if *configJSONPath != "" { - cj, err = readConfigJSON(*configJSONPath) - if err != nil { - fmt.Printf("Could not load jobs config: %v", err) - os.Exit(1) - } - } - os.Exit(m.Run()) } From 2039248412f424428cb08d5cabbe645fb5569487 Mon Sep 17 00:00:00 2001 From: James Munnelly Date: Sat, 1 Sep 2018 01:04:31 +0100 Subject: [PATCH 29/42] Make barnacle go_library public --- images/bootstrap/barnacle/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/bootstrap/barnacle/BUILD.bazel b/images/bootstrap/barnacle/BUILD.bazel index 389199947d8f..de0e7461df40 100644 --- a/images/bootstrap/barnacle/BUILD.bazel +++ b/images/bootstrap/barnacle/BUILD.bazel @@ -4,7 +4,7 @@ go_library( name = "go_default_library", srcs = ["main.go"], importpath = "k8s.io/test-infra/images/bootstrap/barnacle", - visibility = ["//visibility:private"], + visibility = ["//visibility:public"], deps = [ "//vendor/github.com/docker/docker/api/types:go_default_library", "//vendor/github.com/docker/docker/api/types/filters:go_default_library", From 389b00589c813c327103c877d20d05fe7b3604f2 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 31 Aug 2018 22:11:51 -0700 Subject: [PATCH 30/42] overhaul kind docs --- kind/README.md | 24 ++++++++++++------ kind/docs/base-image.md | 27 ++++++++++++++++++++ kind/docs/design.md | 56 ++++++++++++++++++++++++++++++----------- kind/docs/node-image.md | 48 +++++++++++++++++++++++------------ 4 files changed, 118 insertions(+), 37 deletions(-) create mode 100644 kind/docs/base-image.md diff --git a/kind/README.md b/kind/README.md index 1cf7acc45f3e..66dfdc2b14db 100644 --- a/kind/README.md +++ b/kind/README.md @@ -1,20 +1,20 @@ # `kind` - **K**ubernetes **IN** **D**ocker -## WARNING: `kind` is still a work in progress! See [docs/todo.md](./docs/todo.md) +## WARNING: `kind` is still a work in progress! See [docs/todo.md][todo] `kind` is a toolset for running local Kubernetes clusters using Docker container "nodes". `kind` is designed to be suitable for testing Kubernetes, initially targeting the conformance suite. It consists of: - - Go [packages](./pkg) implementing [cluster creation](./pkg/cluster), [image build](./pkg/build), etc. - - A command line interface ([`kind`](./cmd/kind)) built on these packages. - - Docker [image(s)](./images) written to run systemd, Kubernetes, etc. - - [`kubetest`](https://github.com/kubernetes/test-infra/tree/master/kubetest) integration also built on these packages (WIP) + - Go [packages][packages] implementing [cluster creation][cluster package], [image build][build package], etc. + - A command line interface ([`kind`][kind cli]) built on these packages. + - Docker [image(s)][images] written to run systemd, Kubernetes, etc. + - [`kubetest`][kubetest] integration also built on these packages (WIP) -Kind bootstraps each "node" with [kubeadm](https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm/). +Kind bootstraps each "node" with [kubeadm][kubeadm]. -For more details see [the design documentation](./docs/design.md). +For more details see [the design documentation][design doc]. ## Building @@ -33,3 +33,13 @@ For more usage, run `kind --help` or `kind [command] --help`. `kind build base` will build the base image. `kind build node` will build the node image. + +[todo]: ./docs/todo.md +[packages]: ./pkg +[cluster package]: ./pkg/cluster +[build package]: ./pkg/build +[kind cli]: ./main.go +[images]: ./images +[kubetest]: https://github.com/kubernetes/test-infra/tree/master/kubetest +[kubeadm]: https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm/ +[design doc]: ./docs/design.md diff --git a/kind/docs/base-image.md b/kind/docs/base-image.md new file mode 100644 index 000000000000..68aa10fbc19c --- /dev/null +++ b/kind/docs/base-image.md @@ -0,0 +1,27 @@ +# The Base Image + +The ["base" image][base image] is a small-ish Docker image for running +nested containers, systemd, and kubernetes components. + +To do this we need to set up an environment that will meet the CRI +(currently just docker) and systemd's particular needs. Documentation for each +step we take is inline to the image's [Dockerfile][dockerfile]), +but essentially: + +- we preinstall tools / packages expected by systemd / Docker / Kubernetes other +than Kubernetes itself + +- we install a custom entrypoint that allows us to perform some actions before +the container truly boots + +- we set up a systemd service to forward journal logs to the container tty + +- we do a few tricks to minimize unnecessary services and inform systemd that it +is in docker (see the [Dockerfile][dockerfile]) + +This image is based on a minimal debian image (currently `k8s.gcr.io/debian-base`) +due to high availability of tooling. +We strive to minimize the image size where possible. + +[base image]: ./../images/base +[dockerfile]: ./../images/base/Dockerfile diff --git a/kind/docs/design.md b/kind/docs/design.md index 647d4f91431e..30689ac1b822 100644 --- a/kind/docs/design.md +++ b/kind/docs/design.md @@ -1,7 +1,7 @@ # Design This is the root design documentation for `kind`. See also the project -[README.md](./../README.md). +[README.md][README.md]. ## Overview @@ -12,10 +12,10 @@ Kubernetes "clusters" where each "node" is a Docker container. `kind` is divided into go packages implementing most of the functionality, a command line for users, and a "node" base image. The intent is that the `kind` the suite of packages should eventually be importable and reusable by other -tools (e.g. [kubetest](https://github.com/kubernetes/test-infra/tree/master/kubetest)) +tools (e.g. [kubetest][kubetest]) while the CLI provides a quick way to use and debug these packages. -For [the original proposal](https://docs.google.com/document/d/1VL0shYfKl7goy5Zj4Rghpixbye4M8zs_N2gWoQTSKh0/) by [Q-Lee](https://github.com/q-lee) see [the kubernetes-sig-testing post](https://groups.google.com/d/msg/kubernetes-sig-testing/uVkosorBnVc/8DDC3qvMAwAJ) (NOTE: this document is shared with [kubernetes-sig-testing](https://groups.google.com/forum/#!forum/kubernetes-sig-testing)). +For [the original proposal][original proposal] by [Q-Lee][q-lee] see [the kubernetes-sig-testing post][sig-testing-post] (NOTE: this document is shared with [kubernetes-sig-testing][kubernetes-sig-testing]). In short `kind` targets local clusters for testing purposes. While not all testing can be performed without "real" clusters in "the cloud" with provider @@ -29,7 +29,7 @@ enabled CCMs, enough can that we want something that: ## Clusters -Clusters are managed by logic in [`pkg/cluster`](./../pkg/cluster), which the +Clusters are managed by logic in [`pkg/cluster`][pkg/cluster], which the `kind` cli wraps. Each "cluster" is identified by an internal but well-known [docker object label](https://docs.docker.com/config/labels-custom-metadata/) key, with the cluster @@ -47,26 +47,54 @@ capable of detecting this from the containers and providing helpers to use it. ## Images To run Kubernetes in a container, we first need suitable container image(s). -A single standard base layer is used, containing basic utilities like systemd, -certificates, mount, etc. +A single standard [base layer][base-image.md] is used, containing basic +utilities like systemd, certificates, mount, etc. Installing Kubernetes etc. is performed on top of this image, and may be cached in pre-built images. We expect to provide images with releases already installed for use in integrating against Kubernetes. -For more see [node-image.md](./node-image.md). +For more see [node-image.md][node-image.md]. -## Nodes +## Cluster Lifecycle -### Lifecycle +### Cluster Creation Each "node" runs as a docker container. Each container initially boots to a -pseudo "paused" state, with [the entrypoint](./images/node/entrypoint) -waiting for `SIGUSR1`. This allows us to manipulate and inspect the container -with `docker exec ...` and other tools prior to starting systemd and -all of the components. +pseudo "paused" state, with [the entrypoint][entrypoint] waiting for `SIGUSR1`. +This allows us to manipulate and inspect the container with `docker exec ...` +and other tools prior to starting systemd and all of the components. + +This setup includes fixing mounts and pre-loading saved docker images. Once the nodes are sufficiently prepared, we signal the entrypoint to actually "boot" the node. -# TODO(bentheelder): elaborate on bootup, installation as this stabilizes +We then wait for the Docker service to be ready on the node before running +`kubeadm` to initialize the node. + +Once kubeadm has booted, we export the [KUBECONFIG][kubeconfig], then apply +an [overlay network][overlay network]. + +At this point users can test Kubernetes by using the exported kubeconfig. + + +### Cluster Deletion + +All "node" containers in the cluster are tagged with docker labels identifying +the cluster by the chosen cluster name (default is "1"), to delete a cluster +we can simply list and delete containers with this label. + + +[README.md]: ./../README.md +[kubetest]: https://github.com/kubernetes/test-infra/tree/master/kubetest +[original proposal]: https://docs.google.com/document/d/1VL0shYfKl7goy5Zj4Rghpixbye4M8zs_N2gWoQTSKh0/ +[q-lee]: https://github.com/q-lee +[sig-testing-post]: https://groups.google.com/d/msg/kubernetes-sig-testing/uVkosorBnVc/8DDC3qvMAwAJ +[kubernetes-sig-testing]: https://groups.google.com/forum/#!forum/kubernetes-sig-testing +[pkg/cluster]: ./../pkg/cluster +[base-image.md]: ./base-image.md +[node-image.md]: ./node-image.md +[entrypoint]: ./images/node/entrypoint +[kubeconfig]: https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/ +[overlay network]: https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#pod-network diff --git a/kind/docs/node-image.md b/kind/docs/node-image.md index cd29d2e0d0c2..87ce89ab7ef7 100644 --- a/kind/docs/node-image.md +++ b/kind/docs/node-image.md @@ -1,24 +1,40 @@ # The Node Image -The ["node" image](./../images/node) is a small-ish Docker image for running -nested containers, systemd, and kubernetes components. +The ["node" image][node image] is a Docker image for running +nested containers, systemd, and Kubernetes components. -To do this we need to set up an environment that will meet the CRI -(currently just docker) and systemd's particular needs. Documentation for each -step we take is inline to the image's [Dockerfile](./../images/node/Dockerfile)), -but essentially: +This image is built on top of the ["base" image][base image]. -- we preinstall tools / packages expected by systemd / Docker / Kubernetes other -than Kubernetes itself +Logic for building both can be found in [`pkg/build`][build package], +and they can be built with `kind build base` and `kind build node` respectively. -- we install a custom entrypoint that allows us to perform some actions before -the container truly boots +## Design -- we set up a systemd service to forward journal logs to the container tty +Other than the requirement that this image inherits from the "base" image, which +provides most of the tools statically needed for a kubernetes deployment +(eg `systemd`), variants of this image have the following properties: -- we do a few tricks to minimize unnecessary services and inform systemd that it -is in docker (see the [Dockerfile](./../images/node/Dockerfile)) +- `/kind/images/` contains various `*.tar` files which are +[Docker image archives][docker image archives], +these images will be loaded by the cluster tooling prior to running `kubeadm` -This image is based on a minimal debian image (currently `k8s.gcr.io/debian-base`) -due to high availability of tooling. -We strive to minimize the image size where possible. \ No newline at end of file +- `kubeadm`, `kubectl`, `kubelet` are in the path + +- A [systemd service][systemd service] is enabled for `kubelet`, and is +configured to not fail on swap being enabled. (we must do the latter because +swap is inherited from the host and we don't want to force users to disable swap +before using `kind`) + +- `/kind/version` is a regular text file containing the `gitVersion` of the +installed Kubernetes build + +These properties are used by the [cluster][cluster package] tooling to boot +each "node" container with [kubeadm][kubeadm]. + +[node image]: ./../images/node +[base image]: ./base-image.md +[build package]: ./../pkg/build +[cluster package]: ./../pkg/cluster +[docker image archives]: https://docs.docker.com/engine/reference/commandline/save/ +[systemd service]: https://www.freedesktop.org/software/systemd/man/systemd.service.html +[kubeadm]: https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm/ From 870aa8b3533cb3100751adb26ba4056911971aa3 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 31 Aug 2018 23:25:02 -0700 Subject: [PATCH 31/42] leverage make quick-release-images --- kind/pkg/build/kube/dockerbuildbits.go | 80 +++++++++++++++++++++----- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/kind/pkg/build/kube/dockerbuildbits.go b/kind/pkg/build/kube/dockerbuildbits.go index e5a9aa26d6d3..8e91b303ead0 100644 --- a/kind/pkg/build/kube/dockerbuildbits.go +++ b/kind/pkg/build/kube/dockerbuildbits.go @@ -27,7 +27,9 @@ import ( "k8s.io/test-infra/kind/pkg/exec" ) -// DockerBuildBits implements Bits for a local docke-ized make / bash build +// TODO(bentheelder): plumb through arch + +// DockerBuildBits implements Bits for a local docker-ized make / bash build type DockerBuildBits struct { kubeRoot string paths map[string]string @@ -75,7 +77,6 @@ func NewDockerBuildBits(kubeRoot string) (bits Bits, err error) { // Build implements Bits.Build func (b *DockerBuildBits) Build() error { - // TODO(bentheelder): support other modes of building // cd to k8s source cwd, err := os.Getwd() if err != nil { @@ -91,43 +92,94 @@ func (b *DockerBuildBits) Build() error { return err } + // the PR that added `make quick-release-images` added this script, + // we can use it's presence to detect if that target exists + // TODO(bentheelder): drop support for building without this once we've + // dropped older releases or gotten support for `make quick-release-iamges` + // back ported to them ... + releaseImagesSH := filepath.Join( + b.kubeRoot, "build", "release-images.sh", + ) + // if we can't find the script, use the non `make quick-release-images` build + if _, err := os.Stat(releaseImagesSH); err != nil { + return b.buildBash() + } + // otherwise leverage `make quick-release-images` + return b.build() +} + +// binary and image build when we have `make quick-release-images` support +func (b *DockerBuildBits) build() error { // build binaries cmd := exec.Command("build/run.sh", "make", "all") what := []string{ + // binaries we use directly "cmd/kubeadm", "cmd/kubectl", "cmd/kubelet", + } + cmd.Args = append(cmd.Args, + "WHAT="+strings.Join(what, " "), + "KUBE_BUILD_PLATFORMS=linux/amd64", + // ensure the build isn't especially noisy.. + "KUBE_VERBOSE=0", + ) + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Debug = true + cmd.InheritOutput = true + if err := cmd.Run(); err != nil { + return errors.Wrap(err, "failed to build binaries") + } + + // build images + cmd = exec.Command("make", "quick-release-images", "KUBE_BUILD_HYPERKUBE=n") + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Debug = true + cmd.InheritOutput = true + if err := cmd.Run(); err != nil { + return errors.Wrap(err, "failed to build images") + } + return nil +} + +// binary and image build when we don't have `make quick-release-images` support +func (b *DockerBuildBits) buildBash() error { + // build binaries + cmd := exec.Command("build/run.sh", "make", "all") + what := []string{ + // binaries we use directly + "cmd/kubeadm", + "cmd/kubectl", + "cmd/kubelet", + // docker image wrapped binaries "cmd/cloud-controller-manager", "cmd/kube-apiserver", "cmd/kube-controller-manager", "cmd/kube-scheduler", "cmd/kube-proxy", + // we don't need this one, but the image build wraps it... + "vendor/k8s.io/kube-aggregator", } cmd.Args = append(cmd.Args, "WHAT="+strings.Join(what, " "), "KUBE_BUILD_PLATFORMS=linux/amd64", ) cmd.Env = append(cmd.Env, os.Environ()...) + // ensure the build isn't especially noisy.. cmd.Env = append(cmd.Env, "KUBE_VERBOSE=0") cmd.Debug = true cmd.InheritOutput = true - err = cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return errors.Wrap(err, "failed to build binaries") } - // TODO(bentheelder): this is perhaps a bit overkill - // the build will fail if they are already present though - // We should find what `make quick-release` does and mimic that - err = os.RemoveAll(filepath.Join( + // mimic `make quick-release` internals, clear previous images + if err := os.RemoveAll(filepath.Join( ".", "_output", "release-images", "amd64", - )) - if err != nil { + )); err != nil { return errors.Wrap(err, "failed to remove old release-images") } // build images - // TODO(bentheelder): there has to be a better way to do this, but the - // closest seems to be make quick-release, which builds more than we need buildImages := []string{ "source build/common.sh;", "source hack/lib/version.sh;", @@ -140,11 +192,9 @@ func (b *DockerBuildBits) Build() error { cmd.Env = append(cmd.Env, "KUBE_BUILD_HYPERKUBE=n") cmd.Debug = true cmd.InheritOutput = true - err = cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return errors.Wrap(err, "failed to build images") } - return nil } From 768bce4f9d28c84ea81f888d46cd28705d6413be Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 31 Aug 2018 23:29:30 -0700 Subject: [PATCH 32/42] move paths maps to Paths() --- kind/pkg/build/kube/bazelbuildbits.go | 44 ++++++++++----------- kind/pkg/build/kube/dockerbuildbits.go | 55 ++++++++++++-------------- 2 files changed, 45 insertions(+), 54 deletions(-) diff --git a/kind/pkg/build/kube/bazelbuildbits.go b/kind/pkg/build/kube/bazelbuildbits.go index 99ebaeec228d..1f867b2a5406 100644 --- a/kind/pkg/build/kube/bazelbuildbits.go +++ b/kind/pkg/build/kube/bazelbuildbits.go @@ -28,7 +28,6 @@ import ( // BazelBuildBits implements Bits for a local Bazel build type BazelBuildBits struct { kubeRoot string - paths map[string]string } var _ Bits = &BazelBuildBits{} @@ -40,28 +39,9 @@ func init() { // NewBazelBuildBits returns a new Bits backed by bazel build, // given kubeRoot, the path to the kubernetes source directory func NewBazelBuildBits(kubeRoot string) (bits Bits, err error) { - // https://docs.bazel.build/versions/master/output_directories.html - binDir := filepath.Join(kubeRoot, "bazel-bin") - buildDir := filepath.Join(binDir, "build") - bits = &BazelBuildBits{ + return &BazelBuildBits{ kubeRoot: kubeRoot, - paths: map[string]string{ - // debians - filepath.Join(buildDir, "debs", "kubeadm.deb"): "debs/kubeadm.deb", - filepath.Join(buildDir, "debs", "kubelet.deb"): "debs/kubelet.deb", - filepath.Join(buildDir, "debs", "kubectl.deb"): "debs/kubectl.deb", - filepath.Join(buildDir, "debs", "kubernetes-cni.deb"): "debs/kubernetes-cni.deb", - filepath.Join(buildDir, "debs", "cri-tools.deb"): "debs/cri-tools.deb", - // docker images - filepath.Join(buildDir, "kube-apiserver.tar"): "images/kube-apiserver.tar", - filepath.Join(buildDir, "kube-controller-manager.tar"): "images/kube-controller-manager.tar", - filepath.Join(buildDir, "kube-scheduler.tar"): "images/kube-scheduler.tar", - filepath.Join(buildDir, "kube-proxy.tar"): "images/kube-proxy.tar", - // version files - filepath.Join(kubeRoot, "_output", "git_version"): "version", - }, - } - return bits, nil + }, nil } // Build implements Bits.Build @@ -98,8 +78,24 @@ func (b *BazelBuildBits) Build() error { // Paths implements Bits.Paths func (b *BazelBuildBits) Paths() map[string]string { - // TODO(bentheelder): maybe copy the map before returning /shrug - return b.paths + // https://docs.bazel.build/versions/master/output_directories.html + binDir := filepath.Join(b.kubeRoot, "bazel-bin") + buildDir := filepath.Join(binDir, "build") + return map[string]string{ + // debians + filepath.Join(buildDir, "debs", "kubeadm.deb"): "debs/kubeadm.deb", + filepath.Join(buildDir, "debs", "kubelet.deb"): "debs/kubelet.deb", + filepath.Join(buildDir, "debs", "kubectl.deb"): "debs/kubectl.deb", + filepath.Join(buildDir, "debs", "kubernetes-cni.deb"): "debs/kubernetes-cni.deb", + filepath.Join(buildDir, "debs", "cri-tools.deb"): "debs/cri-tools.deb", + // docker images + filepath.Join(buildDir, "kube-apiserver.tar"): "images/kube-apiserver.tar", + filepath.Join(buildDir, "kube-controller-manager.tar"): "images/kube-controller-manager.tar", + filepath.Join(buildDir, "kube-scheduler.tar"): "images/kube-scheduler.tar", + filepath.Join(buildDir, "kube-proxy.tar"): "images/kube-proxy.tar", + // version file + filepath.Join(b.kubeRoot, "_output", "git_version"): "version", + } } // Install implements Bits.Install diff --git a/kind/pkg/build/kube/dockerbuildbits.go b/kind/pkg/build/kube/dockerbuildbits.go index 8e91b303ead0..f75646d51c7c 100644 --- a/kind/pkg/build/kube/dockerbuildbits.go +++ b/kind/pkg/build/kube/dockerbuildbits.go @@ -32,7 +32,6 @@ import ( // DockerBuildBits implements Bits for a local docker-ized make / bash build type DockerBuildBits struct { kubeRoot string - paths map[string]string } var _ Bits = &DockerBuildBits{} @@ -45,34 +44,9 @@ func init() { // NewDockerBuildBits returns a new Bits backed by the docker-ized build, // given kubeRoot, the path to the kubernetes source directory func NewDockerBuildBits(kubeRoot string) (bits Bits, err error) { - // https://docs.Docker.build/versions/master/output_directories.html - binDir := filepath.Join(kubeRoot, - "_output", "dockerized", "bin", "linux", "amd64", - ) - imageDir := filepath.Join(kubeRoot, - "_output", "release-images", "amd64", - ) - bits = &DockerBuildBits{ + return &DockerBuildBits{ kubeRoot: kubeRoot, - paths: map[string]string{ - // binaries (hyperkube) - filepath.Join(binDir, "kubeadm"): "bin/kubeadm", - filepath.Join(binDir, "kubelet"): "bin/kubelet", - filepath.Join(binDir, "kubectl"): "bin/kubectl", - // docker images - filepath.Join(imageDir, "kube-apiserver.tar"): "images/kube-apiserver.tar", - filepath.Join(imageDir, "kube-controller-manager.tar"): "images/kube-controller-manager.tar", - filepath.Join(imageDir, "kube-scheduler.tar"): "images/kube-scheduler.tar", - filepath.Join(imageDir, "kube-proxy.tar"): "images/kube-proxy.tar", - // version files - filepath.Join(kubeRoot, "_output", "git_version"): "version", - // borrow kubelet service files from bazel debians - // TODO(bentheelder): probably we should use our own config instead :-) - filepath.Join(kubeRoot, "build", "debs", "kubelet.service"): "systemd/kubelet.service", - filepath.Join(kubeRoot, "build", "debs", "10-kubeadm.conf"): "systemd/10-kubeadm.conf", - }, - } - return bits, nil + }, nil } // Build implements Bits.Build @@ -200,8 +174,29 @@ func (b *DockerBuildBits) buildBash() error { // Paths implements Bits.Paths func (b *DockerBuildBits) Paths() map[string]string { - // TODO(bentheelder): maybe copy the map before returning /shrug - return b.paths + binDir := filepath.Join(b.kubeRoot, + "_output", "dockerized", "bin", "linux", "amd64", + ) + imageDir := filepath.Join(b.kubeRoot, + "_output", "release-images", "amd64", + ) + return map[string]string{ + // binaries (hyperkube) + filepath.Join(binDir, "kubeadm"): "bin/kubeadm", + filepath.Join(binDir, "kubelet"): "bin/kubelet", + filepath.Join(binDir, "kubectl"): "bin/kubectl", + // docker images + filepath.Join(imageDir, "kube-apiserver.tar"): "images/kube-apiserver.tar", + filepath.Join(imageDir, "kube-controller-manager.tar"): "images/kube-controller-manager.tar", + filepath.Join(imageDir, "kube-scheduler.tar"): "images/kube-scheduler.tar", + filepath.Join(imageDir, "kube-proxy.tar"): "images/kube-proxy.tar", + // version file + filepath.Join(b.kubeRoot, "_output", "git_version"): "version", + // borrow kubelet service files from bazel debians + // TODO(bentheelder): probably we should use our own config instead :-) + filepath.Join(b.kubeRoot, "build", "debs", "kubelet.service"): "systemd/kubelet.service", + filepath.Join(b.kubeRoot, "build", "debs", "10-kubeadm.conf"): "systemd/10-kubeadm.conf", + } } // Install implements Bits.Install From da35db1509c3ca6b11f8d12119aa811f6752ab0e Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 31 Aug 2018 23:31:28 -0700 Subject: [PATCH 33/42] cleanup bits comments --- kind/pkg/build/kube/aptbits.go | 3 ++- kind/pkg/build/kube/bits.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kind/pkg/build/kube/aptbits.go b/kind/pkg/build/kube/aptbits.go index 1b3a0fa1c13b..6dfc83645c58 100644 --- a/kind/pkg/build/kube/aptbits.go +++ b/kind/pkg/build/kube/aptbits.go @@ -69,7 +69,7 @@ EOF` log.Errorf("Installing Kubernetes packages failed! %v", err) return err } - // get version to version file + // get version for version file lines, err := install.CombinedOutputLines("/bin/sh", "-c", `kubelet --version`) if err != nil { log.Errorf("Failed to get Kubernetes version! %v", err) @@ -80,6 +80,7 @@ EOF` log.Errorf("Failed to parse Kubernetes version with unexpected output: %v", lines) return fmt.Errorf("failed to parse Kubernetes version") } + // write version file version := strings.SplitN(lines[0], " ", 2)[1] if err := install.Run("/bin/sh", "-c", fmt.Sprintf(`echo "%s" >> /kind/version`, version)); err != nil { log.Errorf("Failed to get Kubernetes version! %v", err) diff --git a/kind/pkg/build/kube/bits.go b/kind/pkg/build/kube/bits.go index b68329f90415..2410d6476629 100644 --- a/kind/pkg/build/kube/bits.go +++ b/kind/pkg/build/kube/bits.go @@ -53,9 +53,9 @@ type InstallContext interface { // NewNamedBits returns a new Bits by named implementation // currently this includes: +// "apt" -> NewAptBits(kubeRoot) // "bazel" -> NewBazelBuildBits(kubeRoot) // "docker" or "make" -> NewDockerBuildBits(kubeRoot) -// "apt" -> NewAptBits(kubeRoot) func NewNamedBits(name string, kubeRoot string) (bits Bits, err error) { bitsImpls.Lock() fn, ok := bitsImpls.impls[name] From ff425f580a085b4fa10a1d9758fe6eab0f04d2b3 Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Mon, 3 Sep 2018 08:12:17 +0200 Subject: [PATCH 34/42] Use pre.PullRequest.Head.SHA instead of MergeSHA for review --- prow/plugins/verify-owners/verify-owners.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prow/plugins/verify-owners/verify-owners.go b/prow/plugins/verify-owners/verify-owners.go index c0304579a407..83dbd2a6b178 100644 --- a/prow/plugins/verify-owners/verify-owners.go +++ b/prow/plugins/verify-owners/verify-owners.go @@ -204,8 +204,8 @@ func handle(ghc githubClient, gc *git.Client, log *logrus.Entry, pre *github.Pul Action: github.Comment, Comments: comments, } - if pre.PullRequest.MergeSHA != nil { - draftReview.CommitSHA = *pre.PullRequest.MergeSHA + if pre.PullRequest.Head.SHA != "" { + draftReview.CommitSHA = pre.PullRequest.Head.SHA } err := ghc.CreateReview(org, repo, pre.Number, draftReview) if err != nil { From fac9eee61a6daf2f2564d50df2f2c876f25f1821 Mon Sep 17 00:00:00 2001 From: Shyam Jeedigunta Date: Mon, 3 Sep 2018 09:58:35 +0200 Subject: [PATCH 35/42] Revert "Bump node-schedulable-timeout for large clusters till #67823 is fixed" This reverts commit bc8b9baa32ab13d2c95c8c4d3454f1c17b633b89. --- .../sig-scalability/sig-scalability-periodic-jobs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/jobs/kubernetes/sig-scalability/sig-scalability-periodic-jobs.yaml b/config/jobs/kubernetes/sig-scalability/sig-scalability-periodic-jobs.yaml index 0ef12daa8af9..6f8706214faa 100644 --- a/config/jobs/kubernetes/sig-scalability/sig-scalability-periodic-jobs.yaml +++ b/config/jobs/kubernetes/sig-scalability/sig-scalability-periodic-jobs.yaml @@ -267,7 +267,7 @@ periodics: - --gcp-zone=us-east1-b - --ginkgo-parallel=40 - --provider=gce - - --test_args=--ginkgo.flakeAttempts=2 --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[DisabledForLargeClusters\] --node-schedulable-timeout=90m --minStartupPods=8 + - --test_args=--ginkgo.flakeAttempts=2 --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[DisabledForLargeClusters\] --minStartupPods=8 - --timeout=570m - --use-logexporter image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master @@ -302,7 +302,7 @@ periodics: - --gcp-project=kubernetes-scale - --gcp-zone=us-east1-b - --provider=gce - - --test_args=--ginkgo.focus=\[Feature:Performance\] --node-schedulable-timeout=90m --minStartupPods=8 --gather-resource-usage=master --gather-metrics-at-teardown=master + - --test_args=--ginkgo.focus=\[Feature:Performance\] --minStartupPods=8 --gather-resource-usage=master --gather-metrics-at-teardown=master - --timeout=1290m - --use-logexporter image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master From 03747c3bf7a7e99facf97d7ae2ea188d33f336b2 Mon Sep 17 00:00:00 2001 From: Krzysztof Siedlecki Date: Mon, 3 Sep 2018 10:40:45 +0200 Subject: [PATCH 36/42] updating clusterloader2 job config --- .../sig-scalability/sig-scalability-periodic-jobs.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/jobs/kubernetes/sig-scalability/sig-scalability-periodic-jobs.yaml b/config/jobs/kubernetes/sig-scalability/sig-scalability-periodic-jobs.yaml index 0ef12daa8af9..0e17bc067141 100644 --- a/config/jobs/kubernetes/sig-scalability/sig-scalability-periodic-jobs.yaml +++ b/config/jobs/kubernetes/sig-scalability/sig-scalability-periodic-jobs.yaml @@ -711,7 +711,6 @@ periodics: - --cluster=clusterloader - --extract=ci/latest - --gcp-node-image=gci - - --gcp-project=k8s-jkns-clusterloader - --gcp-zone=us-central1-f - --provider=gce - --test=false @@ -739,7 +738,6 @@ periodics: - --cluster=clusterloader2 - --extract=ci/latest - --gcp-nodes=3 - - --gcp-project=k8s-jkns-clusterloader - --gcp-zone=us-central1-f - --provider=gce - --test=false @@ -747,6 +745,7 @@ periodics: - --test-cmd-args=cluster-loader2 - --test-cmd-args=--testconfig=testing/density/config.yaml - --test-cmd-args=--testconfig=testing/load/config.yaml + - --test-cmd-args=--nodes=3 - --test-cmd-args=--report-dir=/workspace/_artifacts - --test-cmd-name=ClusterLoaderV2 - --timeout=60m From 8571f92529f7d0af5ec46299f20f3ad965ccc3ca Mon Sep 17 00:00:00 2001 From: Krzysztof Siedlecki Date: Mon, 3 Sep 2018 12:06:32 +0200 Subject: [PATCH 37/42] adding k8s-jkns-clusterloader project to boskos --- boskos/resources.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/boskos/resources.yaml b/boskos/resources.yaml index 2d35c280d4c2..60fc2ca012f7 100644 --- a/boskos/resources.yaml +++ b/boskos/resources.yaml @@ -353,6 +353,7 @@ resources: - k8s-jenkins-cvm - k8s-jenkins-garbagecollector - k8s-jenkins-gci-kubemark + - k8s-jkns-clusterloader - k8s-jkns-e2e-bazel - k8s-jkns-e2e-etcd3 - k8s-jkns-e2e-gce From d9e5be83e9971dbcfc477785e8f41aabdc97a8ab Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Tue, 4 Sep 2018 12:40:20 +0200 Subject: [PATCH 38/42] testgrid: fix CRI-O links Signed-off-by: Antonio Murdaca --- testgrid/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgrid/config.yaml b/testgrid/config.yaml index e9b08ab808a8..4eacfff04518 100644 --- a/testgrid/config.yaml +++ b/testgrid/config.yaml @@ -5311,14 +5311,14 @@ dashboards: - name: crio-e2e-fedora test_group_name: test_pull_request_crio_e2e_fedora open_test_template: - url: https://gcsweb-ci.svc.ci.openshift.org/gcs///artifacts/artifacts/junit_01.xml + url: https://gcsweb-ci.svc.ci.openshift.org/gcs// results_url_template: url: https://gcsweb-ci.svc.ci.openshift.org/gcs/ base_options: width=10 - name: crio-e2e-rhel test_group_name: test_pull_request_crio_e2e_rhel open_test_template: - url: https://gcsweb-ci.svc.ci.openshift.org/gcs///artifacts/artifacts/junit_01.xml + url: https://gcsweb-ci.svc.ci.openshift.org/gcs// results_url_template: url: https://gcsweb-ci.svc.ci.openshift.org/gcs/ base_options: width=10 From 539b519d7f252201b26f5093e778e17caf58f642 Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Thu, 30 Aug 2018 10:47:33 -0700 Subject: [PATCH 39/42] Generate job URL using pod-utils for decorated jobs When we are decorating a job with the pod utilities, we also know in `plank` the exact place that the artifacts will go, so there is no need to use a report template. In the future, when all jobs are decorated, we can remove the template entirely. Signed-off-by: Steve Kuznetsov --- prow/config/config.go | 17 ++++++++- prow/gcsupload/run.go | 33 +++++++++++------ prow/plank/BUILD.bazel | 2 ++ prow/plank/controller.go | 34 +++++++++++------- prow/plank/controller_test.go | 68 +++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 25 deletions(-) diff --git a/prow/config/config.go b/prow/config/config.go index 4b388dd4a4b4..1bcb767e847e 100644 --- a/prow/config/config.go +++ b/prow/config/config.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "io/ioutil" + "net/url" "os" "path/filepath" "regexp" @@ -32,7 +33,7 @@ import ( "github.com/ghodss/yaml" "github.com/sirupsen/logrus" - cron "gopkg.in/robfig/cron.v2" + "gopkg.in/robfig/cron.v2" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" @@ -168,6 +169,9 @@ type Plank struct { // DefaultDecorationConfig are defaults for shared fields for ProwJobs // that request to have their PodSpecs decorated DefaultDecorationConfig *kube.DecorationConfig `json:"default_decoration_config,omitempty"` + // JobURLPrefix is the host and path prefix under + // which job details will be viewable + JobURLPrefix string `json:"job_url_prefix,omitempty"` } // Gerrit is config for the gerrit controller. @@ -277,6 +281,9 @@ func Load(prowConfig, jobConfig string) (c *Config, err error) { if err := c.finalizeJobConfig(); err != nil { return nil, err } + if err := c.validateComponentConfig(); err != nil { + return nil, err + } if err := c.validateJobConfig(); err != nil { return nil, err } @@ -552,6 +559,14 @@ func (c *Config) finalizeJobConfig() error { return nil } +// validateComponentConfig validates the infrastructure component configuration +func (c *Config) validateComponentConfig() error { + if _, err := url.Parse(c.Plank.JobURLPrefix); c.Plank.JobURLPrefix != "" && err != nil { + return fmt.Errorf("plank declares an invalid job URL prefix %q: %v", c.Plank.JobURLPrefix, err) + } + return nil +} + // validateJobConfig validates if all the jobspecs/presets are valid // if you are mutating the jobs, please add it to finalizeJobConfig above func (c *Config) validateJobConfig() error { diff --git a/prow/gcsupload/run.go b/prow/gcsupload/run.go index 5539da088ef9..9bfbf6a52fa0 100644 --- a/prow/gcsupload/run.go +++ b/prow/gcsupload/run.go @@ -61,17 +61,7 @@ func (o Options) Run(spec *downwardapi.JobSpec, extra map[string]gcs.UploadFunc) } func (o Options) assembleTargets(spec *downwardapi.JobSpec, extra map[string]gcs.UploadFunc) map[string]gcs.UploadFunc { - builder := builderForStrategy(o.PathStrategy, o.DefaultOrg, o.DefaultRepo) - jobBasePath := gcs.PathForSpec(spec, builder) - if o.PathPrefix != "" { - jobBasePath = path.Join(o.PathPrefix, jobBasePath) - } - var gcsPath string - if o.SubDir == "" { - gcsPath = jobBasePath - } else { - gcsPath = path.Join(jobBasePath, o.SubDir) - } + jobBasePath, gcsPath, builder := PathsForJob(o.GCSConfiguration, spec, o.SubDir) uploadTargets := map[string]gcs.UploadFunc{} @@ -113,6 +103,27 @@ func (o Options) assembleTargets(spec *downwardapi.JobSpec, extra map[string]gcs return uploadTargets } +// PathsForJob determines the following for a job: +// - path in GCS under the bucket where job artifacts will be uploaded for: +// - the job +// - this specific run of the job (if any subdir is present) +// The builder for the job is also returned for use in other path resolution. +func PathsForJob(options *kube.GCSConfiguration, spec *downwardapi.JobSpec, subdir string) (string, string, gcs.RepoPathBuilder) { + builder := builderForStrategy(options.PathStrategy, options.DefaultOrg, options.DefaultRepo) + jobBasePath := gcs.PathForSpec(spec, builder) + if options.PathPrefix != "" { + jobBasePath = path.Join(options.PathPrefix, jobBasePath) + } + var gcsPath string + if subdir == "" { + gcsPath = jobBasePath + } else { + gcsPath = path.Join(jobBasePath, subdir) + } + + return jobBasePath, gcsPath, builder +} + func builderForStrategy(strategy, defaultOrg, defaultRepo string) gcs.RepoPathBuilder { var builder gcs.RepoPathBuilder switch strategy { diff --git a/prow/plank/BUILD.bazel b/prow/plank/BUILD.bazel index 5891306db954..a0b2e7c3773c 100644 --- a/prow/plank/BUILD.bazel +++ b/prow/plank/BUILD.bazel @@ -28,10 +28,12 @@ go_library( importpath = "k8s.io/test-infra/prow/plank", deps = [ "//prow/config:go_default_library", + "//prow/gcsupload:go_default_library", "//prow/github:go_default_library", "//prow/kube:go_default_library", "//prow/pjutil:go_default_library", "//prow/pod-utils/decorate:go_default_library", + "//prow/pod-utils/downwardapi:go_default_library", "//prow/report:go_default_library", "//vendor/github.com/bwmarrin/snowflake:go_default_library", "//vendor/github.com/sirupsen/logrus:go_default_library", diff --git a/prow/plank/controller.go b/prow/plank/controller.go index baca63ac9c81..d383db42053e 100644 --- a/prow/plank/controller.go +++ b/prow/plank/controller.go @@ -19,6 +19,7 @@ package plank import ( "bytes" "fmt" + "path" "strings" "sync" "time" @@ -26,12 +27,13 @@ import ( "github.com/bwmarrin/snowflake" "github.com/sirupsen/logrus" "k8s.io/api/core/v1" - "k8s.io/test-infra/prow/config" + "k8s.io/test-infra/prow/gcsupload" "k8s.io/test-infra/prow/github" "k8s.io/test-infra/prow/kube" "k8s.io/test-infra/prow/pjutil" "k8s.io/test-infra/prow/pod-utils/decorate" + "k8s.io/test-infra/prow/pod-utils/downwardapi" reportlib "k8s.io/test-infra/prow/report" ) @@ -408,12 +410,7 @@ func (c *Controller) syncPendingJob(pj kube.ProwJob, pm map[string]kube.Pod, rep } } - var b bytes.Buffer - if err := c.ca.Config().Plank.JobURLTemplate.Execute(&b, &pj); err != nil { - c.log.WithFields(pjutil.ProwJobFields(&pj)).Errorf("error executing URL template: %v", err) - } else { - pj.Status.URL = b.String() - } + pj.Status.URL = jobURL(c.ca.Config().Plank, pj, c.log) reports <- pj @@ -465,12 +462,7 @@ func (c *Controller) syncTriggeredJob(pj kube.ProwJob, pm map[string]kube.Pod, r pj.Status.State = kube.PendingState pj.Status.PodName = pn pj.Status.Description = "Job triggered." - var b bytes.Buffer - if err := c.ca.Config().Plank.JobURLTemplate.Execute(&b, &pj); err != nil { - c.log.WithFields(pjutil.ProwJobFields(&pj)).Errorf("error executing URL template: %v", err) - } else { - pj.Status.URL = b.String() - } + pj.Status.URL = jobURL(c.ca.Config().Plank, pj, c.log) } reports <- pj if prevState != pj.Status.State { @@ -559,3 +551,19 @@ func (c *Controller) RunAfterSuccessCanRun(parent, child *kube.ProwJob, ca confi } return ps.RunsAgainstChanges(changes) } + +func jobURL(plank config.Plank, pj kube.ProwJob, log *logrus.Entry) string { + if pj.Spec.DecorationConfig != nil && plank.JobURLPrefix != "" { + spec := downwardapi.NewJobSpec(pj.Spec, pj.Status.BuildID, pj.Name) + gcsConfig := pj.Spec.DecorationConfig.GCSConfiguration + _, gcsPath, _ := gcsupload.PathsForJob(gcsConfig, &spec, "") + return path.Join(plank.JobURLPrefix, gcsConfig.Bucket, gcsPath) + } + var b bytes.Buffer + if err := plank.JobURLTemplate.Execute(&b, &pj); err != nil { + log.WithFields(pjutil.ProwJobFields(&pj)).Errorf("error executing URL template: %v", err) + } else { + return b.String() + } + return "" +} diff --git a/prow/plank/controller_test.go b/prow/plank/controller_test.go index f1c32bd5630d..a89f6f0d47d5 100644 --- a/prow/plank/controller_test.go +++ b/prow/plank/controller_test.go @@ -1421,3 +1421,71 @@ func TestMaxConcurrencyWithNewlyTriggeredJobs(t *testing.T) { } } } + +func TestJobURL(t *testing.T) { + var testCases = []struct { + name string + plank config.Plank + pj kube.ProwJob + expected string + }{ + { + name: "non-decorated job uses template", + plank: config.Plank{ + Controller: config.Controller{ + JobURLTemplate: template.Must(template.New("test").Parse("{{.Spec.Type}}")), + }, + }, + pj: kube.ProwJob{Spec: kube.ProwJobSpec{Type: kube.PeriodicJob}}, + expected: "periodic", + }, + { + name: "non-decorated job with broken template gives empty string", + plank: config.Plank{ + Controller: config.Controller{ + JobURLTemplate: template.Must(template.New("test").Parse("{{.Garbage}}")), + }, + }, + pj: kube.ProwJob{}, + expected: "", + }, + { + name: "decorated job without prefix uses template", + plank: config.Plank{ + Controller: config.Controller{ + JobURLTemplate: template.Must(template.New("test").Parse("{{.Spec.Type}}")), + }, + }, + pj: kube.ProwJob{Spec: kube.ProwJobSpec{Type: kube.PeriodicJob}}, + expected: "periodic", + }, + { + name: "decorated job with prefix uses gcslib", + plank: config.Plank{ + JobURLPrefix: "https://gubernator.com/build", + }, + pj: kube.ProwJob{Spec: kube.ProwJobSpec{ + Type: kube.PresubmitJob, + Refs: &kube.Refs{ + Org: "org", + Repo: "repo", + Pulls: []kube.Pull{{Number: 1}}, + }, + DecorationConfig: &kube.DecorationConfig{GCSConfiguration: &kube.GCSConfiguration{ + Bucket: "bucket", + PathStrategy: kube.PathStrategyExplicit, + }}, + }}, + expected: "https:/gubernator.com/build/bucket/pr-logs/pull/org_repo/1", + }, + } + + logger := logrus.New() + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + if actual, expected := jobURL(testCase.plank, testCase.pj, logger.WithField("name", testCase.name)), testCase.expected; actual != expected { + t.Errorf("%s: expected URL to be %q but got %q", testCase.name, expected, actual) + } + }) + } +} From 1855e0849bbf89763ac05c06cbdebdf2d5e6eded Mon Sep 17 00:00:00 2001 From: Sen Lu Date: Tue, 4 Sep 2018 10:54:18 -0700 Subject: [PATCH 40/42] skip some preset checks --- config/tests/jobs/jobs_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/tests/jobs/jobs_test.go b/config/tests/jobs/jobs_test.go index d1ea33bc5bad..17a47f7deb14 100644 --- a/config/tests/jobs/jobs_test.go +++ b/config/tests/jobs/jobs_test.go @@ -679,6 +679,10 @@ func TestValidPresets(t *testing.T) { } } + if !*k8sProw { + return + } + for _, presubmit := range c.AllPresubmits(nil) { if presubmit.Spec != nil && !presubmit.Decorate { if err := checkKubekinsPresets(presubmit.Name, presubmit.Spec, presubmit.Labels, validLabels); err != nil { From 453aea26c3ff447a11a710ff3000175e48d01b10 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 4 Sep 2018 13:04:45 -0700 Subject: [PATCH 41/42] add an e2e job for kind --- config/jobs/kubernetes/sig-testing/dind.yaml | 47 ++++++++++++ experiment/kind-e2e.sh | 78 ++++++++++++++++++++ testgrid/config.yaml | 6 ++ 3 files changed, 131 insertions(+) create mode 100644 experiment/kind-e2e.sh diff --git a/config/jobs/kubernetes/sig-testing/dind.yaml b/config/jobs/kubernetes/sig-testing/dind.yaml index cc0d068e0da1..63814eb3468e 100644 --- a/config/jobs/kubernetes/sig-testing/dind.yaml +++ b/config/jobs/kubernetes/sig-testing/dind.yaml @@ -97,6 +97,53 @@ periodics: # We can adjust this after we know the requirements. memory: "6Gi" volumes: + - name: modules + hostPath: + path: /lib/modules + type: Directory + - name: cgroup + hostPath: + path: /sys/fs/cgroup + type: Directory +- interval: 1h + name: ci-kubernetes-kind-conformance-parallel + labels: + preset-service-account: "true" + preset-bazel-scratch-dir: "true" + preset-dind-enabled: "true" + env: + # run tests in parallel + - name: "PARALLEL" + value: "true" + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:v20180829-29e661965-master + args: + - "--job=$(JOB_NAME)" + - "--root=/go/src" + - "--repo=k8s.io/kubernetes=master" + - "--repo=k8s.io/test-infra=master" + - "--service-account=/etc/service-account/service-account.json" + - "--upload=gs://kubernetes-jenkins/logs" + - "--scenario=execute" + - "--" + - "./../test-infra/experiment/kind-e2e.sh" + # Bazel needs privileged mode in order to sandbox builds. + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + volumeMounts: + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /sys/fs/cgroup + name: cgroup + resources: + requests: + # We can adjust this after we know the requirements. + memory: "6Gi" + volumes: - name: modules hostPath: path: /lib/modules diff --git a/experiment/kind-e2e.sh b/experiment/kind-e2e.sh new file mode 100644 index 000000000000..a3662ce05897 --- /dev/null +++ b/experiment/kind-e2e.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# hack script for running a kind e2e +# TODO(bentheelder): replace this with kubetest integration +# Usage: SKIP="ginkgo skip regex" FOCUS="ginkgo focus regex" kind-e2e.sh + +set -o errexit +set -o nounset +set -o pipefail + +# get and isntall `kind` to tempdir +TMP_GOPATH=$(mktemp -d) +trap "rm -rf ${TMP_GOPATH}" EXIT +go get k8s.io/test-infra/kind +PATH="${TMP_GOPATH}/bin:${PATH}" + +# build the base image +# TODO(bentheelder): eliminate this once we publish this image +kind build base +# build the node image w/ kubernetes +kind build node + +# make sure we have e2e requirements +make -C "${GOPATH}/src/k8s.io/kubernetes" all WHAT="cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo" + +# ginkgo regexes +FOCUS="${FOCUS:-"\\[Conformance\\]"}" +SKIP="${SKIP:-"Alpha|Kubectl|\\[(Disruptive|Feature:[^\\]]+|Flaky)\\]"}" + +# arguments to kubetest for the e2e +KUBETEST_ARGS="--provider=skeleton --test --test_args=\"--ginkgo.focus=${FOCUS} --ginkgo.skip=${SKIP}\" --dump=$HOME/make-logs/ --check-version-skew=false" + +# if we set PARALLEL=true, then skip serial tests and add --ginkgo-parallel to the args +PARALLEL="{PARALLEL:-false}" +if [[ "${PARALLEL}" == "true" ]]; then + SKIP="${SKIP}|\\[Serial\\]" + KUBETEST_ARGS="${KUBETEST_ARGS} --ginkgo-parallel" +fi + +# disable errexit so we can manually cleanup +set +o errexit + +# run kind create, if it fails clean up and exit failure +if ! kind create +then + kind delete + exit 1 +fi + +# export the KUBECONFIG +# TODO(bentheelder): provide a `kind` command that can be eval'ed instead +export KUBECONFIG="${HOME}/.config/kind-config-1" + +# run kubetest, if it fails clean up and exit failure +if ! kubetest "${KUBETEST_ARGS}" +then + kind delete + exit 1 +fi + +# re-enable errexit now that we aren't trying to do any catch and cleanup +set -o errexit + +# delete the cluster +kind delete diff --git a/testgrid/config.yaml b/testgrid/config.yaml index 4eacfff04518..94ab86b0d368 100644 --- a/testgrid/config.yaml +++ b/testgrid/config.yaml @@ -2745,6 +2745,9 @@ test_groups: - name: ci-kubernetes-dind-conformance gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-dind-conformance num_columns_recent: 3 +- name: ci-kubernetes-kind-conformance-parallel + gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-kind-conformance-parallel + num_columns_recent: 3 # cloud-provider-openstack e2e conformance tests # name format: PR trigger ci-presubmit-${zuul-job-name} @@ -5775,6 +5778,9 @@ dashboards: - name: dind description: Runs e2e tests against a dind cluster. test_group_name: pull-kubernetes-dind-e2e + - name: ci-kind-conformance-parallel + description: Runs parallel conformance e2e tests against a dind cluster. + test_group_name: ci-kubernetes-kind-conformance-parallel - name: bazel description: Runs bazel test //... on the test-infra repo. test_group_name: ci-test-infra-bazel From 4d50d30c7e58931c808a8f86b115771c0624440c Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 4 Sep 2018 13:52:23 -0700 Subject: [PATCH 42/42] make script executable --- experiment/kind-e2e.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 experiment/kind-e2e.sh diff --git a/experiment/kind-e2e.sh b/experiment/kind-e2e.sh old mode 100644 new mode 100755