diff --git a/cmd/ci-operator-prowgen/main_test.go b/cmd/ci-operator-prowgen/main_test.go index 1b949477..bb96fd27 100644 --- a/cmd/ci-operator-prowgen/main_test.go +++ b/cmd/ci-operator-prowgen/main_test.go @@ -702,12 +702,9 @@ tests: - agent: kubernetes branches: - ^branch$ - context: "" decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: branch-ci-super-duper-branch-images @@ -743,9 +740,7 @@ tests: context: ci/prow/images decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: pull-ci-super-duper-branch-images @@ -779,9 +774,7 @@ tests: context: ci/prow/unit decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: pull-ci-super-duper-branch-unit @@ -857,9 +850,7 @@ tests: - branch decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s name: branch-ci-super-duper-branch-do-not-overwrite spec: containers: @@ -892,9 +883,7 @@ tests: context: ci/prow/rhel-images decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" ci-operator.openshift.io/variant: rhel @@ -929,9 +918,7 @@ tests: context: ci/prow/rhel-unit decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" ci-operator.openshift.io/variant: rhel @@ -965,12 +952,9 @@ tests: - agent: kubernetes branches: - branch - context: "" decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s name: branch-ci-super-duper-branch-do-not-overwrite spec: containers: @@ -996,12 +980,9 @@ tests: - agent: kubernetes branches: - ^branch$ - context: "" decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" ci-operator.openshift.io/variant: rhel @@ -1075,9 +1056,7 @@ tests: - agent: kubernetes decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s name: branch-ci-super-duper-branch-do-not-overwrite spec: containers: @@ -1110,9 +1089,7 @@ tests: context: ci/prow/images decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: pull-ci-super-duper-branch-images @@ -1146,9 +1123,7 @@ tests: context: ci/prow/unit decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: pull-ci-super-duper-branch-unit @@ -1179,12 +1154,9 @@ tests: prowExpectedPostsubmitYAML: []byte(`postsubmits: super/duper: - agent: kubernetes - context: "" decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s name: branch-ci-super-duper-branch-do-not-overwrite spec: containers: @@ -1210,12 +1182,9 @@ tests: - agent: kubernetes branches: - ^branch$ - context: "" decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: branch-ci-super-duper-branch-images diff --git a/go.mod b/go.mod index 5435b56b..483b1602 100644 --- a/go.mod +++ b/go.mod @@ -24,11 +24,10 @@ require ( golang.org/x/build v0.0.0-20190314133821-5284462c4bec // indirect golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 - golang.org/x/text v0.3.2 // indirect google.golang.org/api v0.3.2 gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect k8s.io/api v0.0.0-20181128191700-6db15a15d2d3 k8s.io/apimachinery v0.0.0-20181128191346-49ce2735e507 k8s.io/client-go v9.0.0+incompatible - k8s.io/test-infra v0.0.0-20190427170421-2228dd155e83 + k8s.io/test-infra v0.0.0-20190507120946-f699675303ae ) diff --git a/go.sum b/go.sum index 64f9624f..111fc508 100644 --- a/go.sum +++ b/go.sum @@ -540,6 +540,8 @@ k8s.io/test-infra v0.0.0-20190419141755-2128cd49ec49 h1:PQA4pBitfxlRosLsKLHvsWqi k8s.io/test-infra v0.0.0-20190419141755-2128cd49ec49/go.mod h1:efXtKqqgOwqGIGXp/07+8yYqU6gLruGy1ZCsJP92WIY= k8s.io/test-infra v0.0.0-20190427170421-2228dd155e83 h1:ejcxndaP8JPFnLe0iBJnxAh9c0bZasVCBqQ7AdnoINw= k8s.io/test-infra v0.0.0-20190427170421-2228dd155e83/go.mod h1:au9sNJng6j7WArFJTbSQUPgxpFnSCwJkOyPc4lyBHuI= +k8s.io/test-infra v0.0.0-20190507120946-f699675303ae h1:ZQT2WAlw3rQ4ZvirLbc+yT6KdYU9H+Cr/7PCq6iIZB4= +k8s.io/test-infra v0.0.0-20190507120946-f699675303ae/go.mod h1:2wpqH1meBDeotw8vmDrRw9FLbmDb/PZNwsW/5GxI0Jc= k8s.io/utils v0.0.0-20181019225348-5e321f9a457c/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= mvdan.cc/xurls/v2 v2.0.0/go.mod h1:2/webFPYOXN9jp/lzuj0zuAVlF+9g4KPFJANH1oJhRU= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= diff --git a/test/prowgen-integration/data/output/jobs/super/duper/super-duper-master-postsubmits.yaml b/test/prowgen-integration/data/output/jobs/super/duper/super-duper-master-postsubmits.yaml index 30a30fd6..d538814d 100644 --- a/test/prowgen-integration/data/output/jobs/super/duper/super-duper-master-postsubmits.yaml +++ b/test/prowgen-integration/data/output/jobs/super/duper/super-duper-master-postsubmits.yaml @@ -3,12 +3,9 @@ postsubmits: - agent: kubernetes branches: - ^master$ - context: "" decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: branch-ci-super-duper-master-images @@ -37,7 +34,6 @@ postsubmits: - agent: jenkins branches: - master - context: "" labels: master: ci.openshift.redhat.com name: branch-ci-super-duper-master-legacy diff --git a/test/prowgen-integration/data/output/jobs/super/duper/super-duper-master-presubmits.yaml b/test/prowgen-integration/data/output/jobs/super/duper/super-duper-master-presubmits.yaml index acb9d5bd..5ab8c364 100644 --- a/test/prowgen-integration/data/output/jobs/super/duper/super-duper-master-presubmits.yaml +++ b/test/prowgen-integration/data/output/jobs/super/duper/super-duper-master-presubmits.yaml @@ -7,9 +7,7 @@ presubmits: context: ci/prow/e2e decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: pull-ci-super-duper-master-e2e @@ -70,9 +68,7 @@ presubmits: context: ci/prow/images decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" max_concurrency: 100 @@ -121,9 +117,7 @@ presubmits: context: ci/prow/lint decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: pull-ci-super-duper-master-lint @@ -157,9 +151,7 @@ presubmits: context: ci/prow/unit decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: pull-ci-super-duper-master-unit diff --git a/test/prowgen-integration/data/output/jobs/super/duper/super-duper-release-3.11-postsubmits.yaml b/test/prowgen-integration/data/output/jobs/super/duper/super-duper-release-3.11-postsubmits.yaml index 76ae08ff..2a657714 100644 --- a/test/prowgen-integration/data/output/jobs/super/duper/super-duper-release-3.11-postsubmits.yaml +++ b/test/prowgen-integration/data/output/jobs/super/duper/super-duper-release-3.11-postsubmits.yaml @@ -3,12 +3,9 @@ postsubmits: - agent: kubernetes branches: - ^release-3\.11$ - context: "" decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: branch-ci-super-duper-release-3.11-images diff --git a/test/prowgen-integration/data/output/jobs/super/duper/super-duper-release-3.11-presubmits.yaml b/test/prowgen-integration/data/output/jobs/super/duper/super-duper-release-3.11-presubmits.yaml index 61d5a19b..aea99fd6 100644 --- a/test/prowgen-integration/data/output/jobs/super/duper/super-duper-release-3.11-presubmits.yaml +++ b/test/prowgen-integration/data/output/jobs/super/duper/super-duper-release-3.11-presubmits.yaml @@ -7,9 +7,7 @@ presubmits: context: ci/prow/images decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: pull-ci-super-duper-release-3.11-images @@ -53,9 +51,7 @@ presubmits: context: ci/prow/lint decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: pull-ci-super-duper-release-3.11-lint @@ -89,9 +85,7 @@ presubmits: context: ci/prow/unit decorate: true decoration_config: - grace_period: 0s skip_cloning: true - timeout: 0s labels: ci-operator.openshift.io/prowgen-controlled: "true" name: pull-ci-super-duper-release-3.11-unit diff --git a/vendor/k8s.io/test-infra/prow/apis/prowjobs/v1/types.go b/vendor/k8s.io/test-infra/prow/apis/prowjobs/v1/types.go index f3f31382..5f711327 100644 --- a/vendor/k8s.io/test-infra/prow/apis/prowjobs/v1/types.go +++ b/vendor/k8s.io/test-infra/prow/apis/prowjobs/v1/types.go @@ -198,11 +198,11 @@ func (d *Duration) MarshalJSON() ([]byte, error) { type DecorationConfig struct { // Timeout is how long the pod utilities will wait // before aborting a job with SIGINT. - Timeout Duration `json:"timeout,omitempty"` + Timeout *Duration `json:"timeout,omitempty"` // GracePeriod is how long the pod utilities will wait // after sending SIGINT to send SIGKILL when aborting // a job. Only applicable if decorating the PodSpec. - GracePeriod Duration `json:"grace_period,omitempty"` + GracePeriod *Duration `json:"grace_period,omitempty"` // UtilityImages holds pull specs for utility container // images used to decorate a PodSpec. @@ -246,10 +246,10 @@ func (d *DecorationConfig) ApplyDefault(def *DecorationConfig) *DecorationConfig merged.UtilityImages = merged.UtilityImages.ApplyDefault(def.UtilityImages) merged.GCSConfiguration = merged.GCSConfiguration.ApplyDefault(def.GCSConfiguration) - if merged.Timeout.Duration == 0 { + if merged.Timeout == nil { merged.Timeout = def.Timeout } - if merged.GracePeriod.Duration == 0 { + if merged.GracePeriod == nil { merged.GracePeriod = def.GracePeriod } if merged.GCSCredentialsSecret == "" { @@ -305,6 +305,13 @@ func (d *DecorationConfig) Validate() error { return nil } +func (d *Duration) Get() time.Duration { + if d == nil { + return 0 + } + return d.Duration +} + // UtilityImages holds pull specs for the utility images // to be used for a job type UtilityImages struct { diff --git a/vendor/k8s.io/test-infra/prow/apis/prowjobs/v1/zz_generated.deepcopy.go b/vendor/k8s.io/test-infra/prow/apis/prowjobs/v1/zz_generated.deepcopy.go index a6cdb850..be2fa4c0 100644 --- a/vendor/k8s.io/test-infra/prow/apis/prowjobs/v1/zz_generated.deepcopy.go +++ b/vendor/k8s.io/test-infra/prow/apis/prowjobs/v1/zz_generated.deepcopy.go @@ -30,8 +30,16 @@ import ( // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DecorationConfig) DeepCopyInto(out *DecorationConfig) { *out = *in - out.Timeout = in.Timeout - out.GracePeriod = in.GracePeriod + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(Duration) + **out = **in + } + if in.GracePeriod != nil { + in, out := &in.GracePeriod, &out.GracePeriod + *out = new(Duration) + **out = **in + } if in.UtilityImages != nil { in, out := &in.UtilityImages, &out.UtilityImages *out = new(UtilityImages) diff --git a/vendor/k8s.io/test-infra/prow/config/config.go b/vendor/k8s.io/test-infra/prow/config/config.go index 18ed11c1..d243525c 100644 --- a/vendor/k8s.io/test-infra/prow/config/config.go +++ b/vendor/k8s.io/test-infra/prow/config/config.go @@ -79,6 +79,7 @@ type ProwConfig struct { BranchProtection BranchProtection `json:"branch-protection,omitempty"` Gerrit Gerrit `json:"gerrit,omitempty"` GitHubReporter GitHubReporter `json:"github_reporter,omitempty"` + SlackReporter *SlackReporter `json:"slack_reporter,omitempty"` // TODO: Move this out of the main config. JenkinsOperators []JenkinsOperator `json:"jenkins_operators,omitempty"` @@ -379,6 +380,36 @@ type GitHubOptions struct { LinkURL *url.URL } +// SlackReporter represents the config for the Slack reporter +type SlackReporter struct { + JobTypesToReport []prowapi.ProwJobType `json:"job_types_to_report"` + JobStatesToReport []prowapi.ProwJobState `json:"job_states_to_report"` + Channel string `json:"channel"` + ReportTemplate string `json:"report_template"` +} + +func (cfg *SlackReporter) DefaultAndValidate() error { + // Default ReportTemplate + if cfg.ReportTemplate == "" { + cfg.ReportTemplate = `Job {{.Spec.Job}} of type {{.Spec.Type}} ended with state {{.Status.State}}. <{{.Status.URL}}|View logs>` + } + + if cfg.Channel == "" { + return errors.New("channel must be set") + } + + // Validate ReportTemplate + tmpl, err := template.New("").Parse(cfg.ReportTemplate) + if err != nil { + return fmt.Errorf("failed to parse template: %v", err) + } + if err := tmpl.Execute(&bytes.Buffer{}, &prowapi.ProwJob{}); err != nil { + return fmt.Errorf("failed to execute report_template: %v", err) + } + + return nil +} + // Load loads and parses the config at path. func Load(prowConfig, jobConfig string) (c *Config, err error) { // we never want config loading to take down the prow components @@ -403,52 +434,27 @@ func Load(prowConfig, jobConfig string) (c *Config, err error) { return c, nil } -// loadConfig loads one or multiple config files and returns a config object. -func loadConfig(prowConfig, jobConfig string) (*Config, error) { - stat, err := os.Stat(prowConfig) +// ReadJobConfig reads the JobConfig yaml, but does not expand or validate it. +func ReadJobConfig(jobConfig string) (JobConfig, error) { + stat, err := os.Stat(jobConfig) if err != nil { - return nil, err - } - - if stat.IsDir() { - return nil, fmt.Errorf("prowConfig cannot be a dir - %s", prowConfig) - } - - var nc Config - if err := yamlToConfig(prowConfig, &nc); err != nil { - return nil, err - } - if err := parseProwConfig(&nc); err != nil { - return nil, err - } - - // TODO(krzyzacy): temporary allow empty jobconfig - // also temporary allow job config in prow config - if jobConfig == "" { - return &nc, nil - } - - stat, err = os.Stat(jobConfig) - if err != nil { - return nil, err + return JobConfig{}, err } if !stat.IsDir() { // still support a single file var jc JobConfig if err := yamlToConfig(jobConfig, &jc); err != nil { - return nil, err + return JobConfig{}, err } - if err := nc.mergeJobConfig(jc); err != nil { - return nil, err - } - return &nc, nil + return jc, nil } // we need to ensure all config files have unique basenames, // since updateconfig plugin will use basename as a key in the configmap uniqueBasenames := sets.String{} + jc := JobConfig{} err = filepath.Walk(jobConfig, func(path string, info os.FileInfo, err error) error { if err != nil { logrus.WithError(err).Errorf("walking path %q.", path) @@ -483,10 +489,47 @@ func loadConfig(prowConfig, jobConfig string) (*Config, error) { if err := yamlToConfig(path, &subConfig); err != nil { return err } - return nc.mergeJobConfig(subConfig) + jc, err = mergeJobConfigs(jc, subConfig) + return err }) if err != nil { + return JobConfig{}, err + } + + return jc, nil +} + +// loadConfig loads one or multiple config files and returns a config object. +func loadConfig(prowConfig, jobConfig string) (*Config, error) { + stat, err := os.Stat(prowConfig) + if err != nil { + return nil, err + } + + if stat.IsDir() { + return nil, fmt.Errorf("prowConfig cannot be a dir - %s", prowConfig) + } + + var nc Config + if err := yamlToConfig(prowConfig, &nc); err != nil { + return nil, err + } + if err := parseProwConfig(&nc); err != nil { + return nil, err + } + + // TODO(krzyzacy): temporary allow empty jobconfig + // also temporary allow job config in prow config + if jobConfig == "" { + return &nc, nil + } + + jc, err := ReadJobConfig(jobConfig) + if err != nil { + return nil, err + } + if err := nc.mergeJobConfig(jc); err != nil { return nil, err } @@ -558,16 +601,34 @@ func ReadFileMaybeGZIP(path string) ([]byte, error) { return ioutil.ReadAll(gzipReader) } -// mergeConfig merges two JobConfig together +func (c *Config) mergeJobConfig(jc JobConfig) error { + m, err := mergeJobConfigs(JobConfig{ + Presets: c.Presets, + Presubmits: c.Presubmits, + Periodics: c.Periodics, + Postsubmits: c.Postsubmits, + }, jc) + if err != nil { + return err + } + c.Presets = m.Presets + c.Presubmits = m.Presubmits + c.Periodics = m.Periodics + c.Postsubmits = m.Postsubmits + return nil +} + +// mergeJobConfigs merges two JobConfig together // It will try to merge: // - Presubmits // - Postsubmits // - Periodics // - PodPresets -func (c *Config) mergeJobConfig(jc JobConfig) error { +func mergeJobConfigs(a, b JobConfig) (JobConfig, error) { // Merge everything // *** Presets *** - c.Presets = append(c.Presets, jc.Presets...) + c := JobConfig{} + c.Presets = append(a.Presets, b.Presets...) // validate no duplicated preset key-value pairs validLabels := map[string]bool{} @@ -575,32 +636,33 @@ func (c *Config) mergeJobConfig(jc JobConfig) error { for label, val := range preset.Labels { pair := label + ":" + val if _, ok := validLabels[pair]; ok { - return fmt.Errorf("duplicated preset 'label:value' pair : %s", pair) + return JobConfig{}, fmt.Errorf("duplicated preset 'label:value' pair : %s", pair) } validLabels[pair] = true } } // *** Periodics *** - c.Periodics = append(c.Periodics, jc.Periodics...) + c.Periodics = append(a.Periodics, b.Periodics...) // *** Presubmits *** - if c.Presubmits == nil { - c.Presubmits = make(map[string][]Presubmit) + c.Presubmits = make(map[string][]Presubmit) + for repo, jobs := range a.Presubmits { + c.Presubmits[repo] = jobs } - for repo, jobs := range jc.Presubmits { + for repo, jobs := range b.Presubmits { c.Presubmits[repo] = append(c.Presubmits[repo], jobs...) } // *** Postsubmits *** - if c.Postsubmits == nil { - c.Postsubmits = make(map[string][]Postsubmit) + c.Postsubmits = make(map[string][]Postsubmit) + for repo, jobs := range a.Postsubmits { + c.Postsubmits[repo] = jobs } - for repo, jobs := range jc.Postsubmits { + for repo, jobs := range b.Postsubmits { c.Postsubmits[repo] = append(c.Postsubmits[repo], jobs...) } - - return nil + return c, nil } func setPresubmitDecorationDefaults(c *Config, ps *Presubmit) { @@ -701,6 +763,12 @@ func (c *Config) validateComponentConfig() error { return fmt.Errorf(`Invalid value for Planks job_url_prefix_config["%s"]: %v`, k, err) } } + + if c.SlackReporter != nil { + if err := c.SlackReporter.DefaultAndValidate(); err != nil { + return fmt.Errorf("failed to validate slackreporter config: %v", err) + } + } return nil } @@ -1020,6 +1088,30 @@ func parseProwConfig(c *Config) error { } } + for name, templates := range c.Tide.MergeTemplate { + if templates.TitleTemplate != "" { + titleTemplate, err := template.New("CommitTitle").Parse(templates.TitleTemplate) + + if err != nil { + return fmt.Errorf("parsing template for commit title: %v", err) + } + + templates.Title = titleTemplate + } + + if templates.BodyTemplate != "" { + bodyTemplate, err := template.New("CommitBody").Parse(templates.BodyTemplate) + + if err != nil { + return fmt.Errorf("parsing template for commit body: %v", err) + } + + templates.Body = bodyTemplate + } + + c.Tide.MergeTemplate[name] = templates + } + for i, tq := range c.Tide.Queries { if err := tq.Validate(); err != nil { return fmt.Errorf("tide query (index %d) is invalid: %v", i, err) diff --git a/vendor/k8s.io/test-infra/prow/config/jobs.go b/vendor/k8s.io/test-infra/prow/config/jobs.go index d347aecc..421a51b5 100644 --- a/vendor/k8s.io/test-infra/prow/config/jobs.go +++ b/vendor/k8s.io/test-infra/prow/config/jobs.go @@ -84,8 +84,8 @@ type JobBase struct { Labels map[string]string `json:"labels,omitempty"` // MaximumConcurrency of this job, 0 implies no limit. MaxConcurrency int `json:"max_concurrency,omitempty"` - // Agent that will take care of running this job. - Agent string `json:"agent"` + // Agent that will take care of running this job. Defaults to "kubernetes" + Agent string `json:"agent,omitempty"` // Cluster is the alias of the cluster to run this job in. // (Default: kube.DefaultClusterAlias) Cluster string `json:"cluster,omitempty"` @@ -104,6 +104,8 @@ type JobBase struct { Spec *v1.PodSpec `json:"spec,omitempty"` // BuildSpec is the Knative build spec used if Agent is knative-build. BuildSpec *buildv1alpha1.BuildSpec `json:"build_spec,omitempty"` + // Annotations are unused by prow itself, but provide a space to configure other automation. + Annotations map[string]string `json:"annotations,omitempty"` UtilityConfig } @@ -122,12 +124,12 @@ type Presubmit struct { // e.g. `@k8s-bot e2e test this` // RerunCommand must also be specified if this field is specified. // (Default: `(?m)^/test (?:.*? )?(?: .*?)?$`) - Trigger string `json:"trigger"` + Trigger string `json:"trigger,omitempty"` // The RerunCommand to give users. Must match Trigger. // Trigger must also be specified if this field is specified. // (Default: `/test `) - RerunCommand string `json:"rerun_command"` + RerunCommand string `json:"rerun_command,omitempty"` Brancher @@ -160,9 +162,9 @@ type Periodic struct { JobBase // (deprecated)Interval to wait between two runs of the job. - Interval string `json:"interval"` + Interval string `json:"interval,omitempty"` // Cron representation of job trigger time - Cron string `json:"cron"` + Cron string `json:"cron,omitempty"` // Tags for config entries Tags []string `json:"tags,omitempty"` @@ -209,7 +211,8 @@ type RegexpChangeMatcher struct { type Reporter struct { // Context is the name of the GitHub status context for the job. - Context string `json:"context"` + // Defaults: the same as the name of the job. + Context string `json:"context,omitempty"` // SkipReport skips commenting and setting status on GitHub. SkipReport bool `json:"skip_report,omitempty"` } @@ -249,8 +252,13 @@ func (br Brancher) Intersects(other Brancher) bool { } return false } - if !baseBranches.Intersection(sets.NewString(other.SkipBranches...)).Equal(baseBranches) { - return true + + // Actually test our branches against the other brancher - if there are regex skip lists, simple comparison + // is insufficient. + for _, b := range baseBranches.List() { + if other.ShouldRun(b) { + return true + } } return false } diff --git a/vendor/k8s.io/test-infra/prow/config/tide.go b/vendor/k8s.io/test-infra/prow/config/tide.go index c37d64fe..0bfed946 100644 --- a/vendor/k8s.io/test-infra/prow/config/tide.go +++ b/vendor/k8s.io/test-infra/prow/config/tide.go @@ -21,6 +21,7 @@ import ( "fmt" "strings" "sync" + "text/template" "time" "github.com/sirupsen/logrus" @@ -62,6 +63,15 @@ type TideContextPolicyOptions struct { Orgs map[string]TideOrgContextPolicy `json:"orgs,omitempty"` } +// TideMergeCommitTemplate holds templates to use for merge commits. +type TideMergeCommitTemplate struct { + TitleTemplate string `json:"title,omitempty"` + BodyTemplate string `json:"body,omitempty"` + + Title *template.Template `json:"-"` + Body *template.Template `json:"-"` +} + // Tide is config for the tide pool. type Tide struct { // SyncPeriodString compiles into SyncPeriod at load time. @@ -81,6 +91,11 @@ type Tide struct { // the default method of merge. Valid options are squash, rebase, and merge. MergeType map[string]github.PullRequestMergeType `json:"merge_method,omitempty"` + // A key/value pair of an org/repo as the key and Go template to override + // the default merge commit title and/or message. Template is passed the + // PullRequest struct (prow/github/types.go#PullRequest) + MergeTemplate map[string]TideMergeCommitTemplate `json:"merge_commit_template,omitempty"` + // URL for tide status contexts. // We can consider allowing this to be set separately for separate repos, or // allowing it to be a template. @@ -140,6 +155,18 @@ func (t *Tide) MergeMethod(org, repo string) github.PullRequestMergeType { return v } +// MergeCommitTemplate returns a struct with Go template string(s) or nil +func (t *Tide) MergeCommitTemplate(org, repo string) TideMergeCommitTemplate { + name := org + "/" + repo + + v, ok := t.MergeTemplate[name] + if !ok { + return t.MergeTemplate[org] + } + + return v +} + // TideQuery is turned into a GitHub search query. See the docs for details: // https://help.github.com/articles/searching-issues-and-pull-requests/ type TideQuery struct { diff --git a/vendor/k8s.io/test-infra/prow/gcsupload/run.go b/vendor/k8s.io/test-infra/prow/gcsupload/run.go index d5d7a702..07f738bd 100644 --- a/vendor/k8s.io/test-infra/prow/gcsupload/run.go +++ b/vendor/k8s.io/test-infra/prow/gcsupload/run.go @@ -77,7 +77,9 @@ func (o Options) assembleTargets(spec *downwardapi.JobSpec, extra map[string]gcs if latestBuilds := gcs.LatestBuildForSpec(spec, builder); len(latestBuilds) > 0 { for _, latestBuild := range latestBuilds { - uploadTargets[latestBuild] = gcs.DataUpload(strings.NewReader(spec.BuildID)) + dir, filename := path.Split(latestBuild) + metadataFromFileName, attrs := gcs.AttributesFromFileName(filename) + uploadTargets[path.Join(dir, metadataFromFileName)] = gcs.DataUploadWithAttributes(strings.NewReader(spec.BuildID), attrs) } } @@ -90,12 +92,13 @@ func (o Options) assembleTargets(spec *downwardapi.JobSpec, extra map[string]gcs if info.IsDir() { gatherArtifacts(item, gcsPath, info.Name(), uploadTargets) } else { - destination := path.Join(gcsPath, info.Name()) + metadataFromFileName, attrs := gcs.AttributesFromFileName(info.Name()) + destination := path.Join(gcsPath, metadataFromFileName) if _, exists := uploadTargets[destination]; exists { logrus.Warnf("Encountered duplicate upload of %s, skipping...", destination) continue } - uploadTargets[destination] = gcs.FileUpload(item) + uploadTargets[destination] = gcs.FileUploadWithAttributes(item, attrs) } } @@ -153,13 +156,15 @@ func gatherArtifacts(artifactDir, gcsPath, subDir string, uploadTargets map[stri // this error as we can be certain it won't occur and best- // effort upload is OK in any case if relPath, err := filepath.Rel(artifactDir, fspath); err == nil { - destination := path.Join(gcsPath, subDir, relPath) + dir, filename := path.Split(path.Join(gcsPath, subDir, relPath)) + metadataFromFileName, attrs := gcs.AttributesFromFileName(filename) + destination := path.Join(dir, metadataFromFileName) if _, exists := uploadTargets[destination]; exists { logrus.Warnf("Encountered duplicate upload of %s, skipping...", destination) return nil } logrus.Printf("Found %s in artifact directory. Uploading as %s\n", fspath, destination) - uploadTargets[destination] = gcs.FileUpload(fspath) + uploadTargets[destination] = gcs.FileUploadWithAttributes(fspath, attrs) } else { logrus.Warnf("Encountered error in relative path calculation for %s under %s: %v", fspath, artifactDir, err) } diff --git a/vendor/k8s.io/test-infra/prow/github/client.go b/vendor/k8s.io/test-infra/prow/github/client.go index 4b526a9c..aa121435 100644 --- a/vendor/k8s.io/test-infra/prow/github/client.go +++ b/vendor/k8s.io/test-infra/prow/github/client.go @@ -2706,7 +2706,7 @@ func (c *Client) GetColumnProjectCard(columnID int, cardNumber int) (*ProjectCar if c.fake { return nil, nil } - path := fmt.Sprintf("/projects/columns/:%d/cards", columnID) + path := fmt.Sprintf("/projects/columns/%d/cards", columnID) var cards []ProjectCard err := c.readPaginatedResults( path, @@ -2736,7 +2736,7 @@ func (c *Client) MoveProjectCard(projectCardID int, newColumnID int) error { c.log("MoveProjectCard", projectCardID, newColumnID) _, err := c.request(&request{ method: http.MethodPost, - path: fmt.Sprintf("/projects/columns/cards/:%d/moves", projectCardID), + path: fmt.Sprintf("/projects/columns/cards/%d/moves", projectCardID), accept: "application/vnd.github.symmetra-preview+json", // allow the description field -- https://developer.github.com/changes/2018-02-22-label-description-search-preview/ requestBody: fmt.Sprintf("{column_id: %d}", newColumnID), exitCodes: []int{201}, diff --git a/vendor/k8s.io/test-infra/prow/kube/BUILD.bazel b/vendor/k8s.io/test-infra/prow/kube/BUILD.bazel index 19268f75..a79b2f92 100644 --- a/vendor/k8s.io/test-infra/prow/kube/BUILD.bazel +++ b/vendor/k8s.io/test-infra/prow/kube/BUILD.bazel @@ -47,7 +47,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/plugin/pkg/client/auth:go_default_library", diff --git a/vendor/k8s.io/test-infra/prow/kube/client.go b/vendor/k8s.io/test-infra/prow/kube/client.go index fb7ab198..0a51f954 100644 --- a/vendor/k8s.io/test-infra/prow/kube/client.go +++ b/vendor/k8s.io/test-infra/prow/kube/client.go @@ -21,7 +21,6 @@ import ( "crypto/tls" "crypto/x509" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -32,7 +31,6 @@ import ( "github.com/sirupsen/logrus" "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/yaml" prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1" @@ -75,17 +73,6 @@ type Client struct { token string namespace string fake bool - - hiddenReposProvider func() []string - hiddenOnly bool -} - -// SetHiddenReposProvider takes a continuation that fetches a list of orgs and repos for -// which PJs should not be returned. -// NOTE: This function is not thread safe and should be called before the client is in use. -func (c *Client) SetHiddenReposProvider(p func() []string, hiddenOnly bool) { - c.hiddenReposProvider = p - c.hiddenOnly = hiddenOnly } // Namespace returns a copy of the client pointing at the specified namespace. @@ -479,26 +466,6 @@ func (c *Client) CreateProwJob(j prowapi.ProwJob) (prowapi.ProwJob, error) { return retJob, err } -func (c *Client) getHiddenRepos() sets.String { - if c.hiddenReposProvider == nil { - return nil - } - return sets.NewString(c.hiddenReposProvider()...) -} - -func shouldHide(pj *prowapi.ProwJob, hiddenRepos sets.String, showHiddenOnly bool) bool { - if pj.Spec.Refs == nil { - // periodic jobs do not have refs and therefore cannot be - // hidden by the org/repo mechanism - return false - } - shouldHide := hiddenRepos.HasAny(fmt.Sprintf("%s/%s", pj.Spec.Refs.Org, pj.Spec.Refs.Repo), pj.Spec.Refs.Org) - if showHiddenOnly { - return !shouldHide - } - return shouldHide -} - // GetProwJob returns the prowjob at name in the client's specified namespace. // // Analogous to kubectl get prowjob/NAME --namespace=client.namespace @@ -508,13 +475,6 @@ func (c *Client) GetProwJob(name string) (prowapi.ProwJob, error) { err := c.request(&request{ path: fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs/%s", c.namespace, name), }, &pj) - if err == nil && shouldHide(&pj, c.getHiddenRepos(), c.hiddenOnly) { - pj = prowapi.ProwJob{} - // Revealing the existence of this prow job is ok because the pj name cannot be used to - // retrieve the pj itself. Furthermore, a timing attack could differentiate true 404s from - // 404s returned when a hidden pj is queried so returning a 404 wouldn't hide the pj's existence. - err = errors.New("403 ProwJob is hidden") - } return pj, err } @@ -532,12 +492,9 @@ func (c *Client) ListProwJobs(selector string) ([]prowapi.ProwJob, error) { query: map[string]string{"labelSelector": selector}, }, &jl) if err == nil { - hidden := c.getHiddenRepos() var pjs []prowapi.ProwJob for _, pj := range jl.Items { - if !shouldHide(&pj, hidden, c.hiddenOnly) { - pjs = append(pjs, pj) - } + pjs = append(pjs, pj) } jl.Items = pjs } diff --git a/vendor/k8s.io/test-infra/prow/kube/config.go b/vendor/k8s.io/test-infra/prow/kube/config.go index fcea0413..e5e390c6 100644 --- a/vendor/k8s.io/test-infra/prow/kube/config.go +++ b/vendor/k8s.io/test-infra/prow/kube/config.go @@ -50,12 +50,12 @@ func kubeConfigs(kubeconfig string) (map[string]rest.Config, string, error) { } configs := map[string]rest.Config{} for context := range cfg.Contexts { - logrus.Infof("* %s", context) contextCfg, err := clientcmd.NewNonInteractiveClientConfig(*cfg, context, &clientcmd.ConfigOverrides{}, loader).ClientConfig() if err != nil { return nil, "", fmt.Errorf("create %s client: %v", context, err) } configs[context] = *contextCfg + logrus.Infof("Parsed kubeconfig context: %s", context) } return configs, cfg.CurrentContext, nil } @@ -144,7 +144,7 @@ func LoadClusterConfigs(kubeconfig, buildCluster string) (map[string]rest.Config // This will work if we are running inside kubernetes localCfg, err := localConfig() if err != nil { - logrus.WithError(err).Warn("Failed to create in-cluster config") + logrus.WithError(err).Warn("Could not create in-cluster config (expected when running outside the cluster).") } kubeCfgs, currentContext, err := kubeConfigs(kubeconfig) diff --git a/vendor/k8s.io/test-infra/prow/plugins/verify-owners/BUILD.bazel b/vendor/k8s.io/test-infra/prow/plugins/verify-owners/BUILD.bazel index 6743e7e9..cef3577c 100644 --- a/vendor/k8s.io/test-infra/prow/plugins/verify-owners/BUILD.bazel +++ b/vendor/k8s.io/test-infra/prow/plugins/verify-owners/BUILD.bazel @@ -12,7 +12,6 @@ go_library( "//prow/pluginhelp:go_default_library", "//prow/plugins:go_default_library", "//prow/plugins/golint:go_default_library", - "//prow/plugins/trigger:go_default_library", "//prow/repoowners:go_default_library", "//vendor/github.com/sirupsen/logrus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/vendor/k8s.io/test-infra/prow/plugins/verify-owners/verify-owners.go b/vendor/k8s.io/test-infra/prow/plugins/verify-owners/verify-owners.go index 7bbe37d5..cae3f793 100644 --- a/vendor/k8s.io/test-infra/prow/plugins/verify-owners/verify-owners.go +++ b/vendor/k8s.io/test-infra/prow/plugins/verify-owners/verify-owners.go @@ -19,7 +19,6 @@ package verifyowners import ( "fmt" "io/ioutil" - "os" "path/filepath" "regexp" "strconv" @@ -34,16 +33,13 @@ import ( "k8s.io/test-infra/prow/pluginhelp" "k8s.io/test-infra/prow/plugins" "k8s.io/test-infra/prow/plugins/golint" - "k8s.io/test-infra/prow/plugins/trigger" "k8s.io/test-infra/prow/repoowners" ) const ( // PluginName defines this plugin's registered name. - PluginName = "verify-owners" - ownersFileName = "OWNERS" - ownersAliasesFileName = "OWNERS_ALIASES" - nonCollaboratorResponseFormat = "The following users are mentioned in %s file(s) but are not members of the %s org." + PluginName = "verify-owners" + ownersFileName = "OWNERS" ) func init() { @@ -52,7 +48,7 @@ func init() { func helpProvider(config *plugins.Configuration, enabledRepos []string) (*pluginhelp.PluginHelp, error) { pluginHelp := &pluginhelp.PluginHelp{ - Description: fmt.Sprintf("The verify-owners plugin validates %s and %s files and ensures that they always contain collaborators of the org, if they are modified in a PR. On validation failure it automatically adds the '%s' label to the PR, and a review comment on the incriminating file(s).", ownersFileName, ownersAliasesFileName, labels.InvalidOwners), + Description: fmt.Sprintf("The verify-owners plugin validates %s files if they are modified in a PR. On validation failure it automatically adds the '%s' label to the PR, and a review comment on the incriminating file(s).", ownersFileName, labels.InvalidOwners), } if config.Owners.LabelsBlackList != nil { pluginHelp.Config = map[string]string{ @@ -69,30 +65,18 @@ type ownersClient interface { } type githubClient interface { - IsCollaborator(owner, repo, login string) (bool, error) - IsMember(org, user string) (bool, error) AddLabel(org, repo string, number int, label string) error CreateComment(owner, repo string, number int, comment string) error CreateReview(org, repo string, number int, r github.DraftReview) error GetPullRequestChanges(org, repo string, number int) ([]github.PullRequestChange, error) RemoveLabel(owner, repo string, number int, label string) error - GetIssueLabels(org, repo string, number int) ([]github.Label, error) -} - -type commentPruner interface { - PruneComments(shouldPrune func(github.IssueComment) bool) } func handlePullRequest(pc plugins.Agent, pre github.PullRequestEvent) error { if pre.Action != github.PullRequestActionOpened && pre.Action != github.PullRequestActionReopened && pre.Action != github.PullRequestActionSynchronize { return nil } - - cp, err := pc.CommentPruner() - if err != nil { - return err - } - return handle(pc.GitHubClient, pc.GitClient, pc.Logger, &pre, pc.PluginConfig.Owners.LabelsBlackList, pc.PluginConfig.TriggerFor(pre.Repo.Owner.Login, pre.Repo.Name), cp) + return handle(pc.GitHubClient, pc.GitClient, pc.Logger, &pre, pc.PluginConfig.Owners.LabelsBlackList) } type messageWithLine struct { @@ -100,10 +84,9 @@ type messageWithLine struct { message string } -func handle(ghc githubClient, gc *git.Client, log *logrus.Entry, pre *github.PullRequestEvent, labelsBlackList []string, triggerConfig plugins.Trigger, cp commentPruner) error { +func handle(ghc githubClient, gc *git.Client, log *logrus.Entry, pre *github.PullRequestEvent, labelsBlackList []string) error { org := pre.Repo.Owner.Login repo := pre.Repo.Name - number := pre.Number wrongOwnersFiles := map[string]messageWithLine{} // Get changes. @@ -119,18 +102,7 @@ func handle(ghc githubClient, gc *git.Client, log *logrus.Entry, pre *github.Pul modifiedOwnersFiles = append(modifiedOwnersFiles, change) } } - - // Check if the OWNERS_ALIASES file was modified. - var modifiedOwnerAliasesFile github.PullRequestChange - var ownerAliasesModified bool - for _, change := range changes { - if change.Filename == ownersAliasesFileName { - modifiedOwnerAliasesFile = change - ownerAliasesModified = true - } - } - - if len(modifiedOwnersFiles) == 0 && !ownerAliasesModified { + if len(modifiedOwnersFiles) == 0 { return nil } @@ -154,50 +126,27 @@ func handle(ghc githubClient, gc *git.Client, log *logrus.Entry, pre *github.Pul } } - // If OWNERS_ALIASES file exists, get all aliases. - // If the file was modified, check for non trusted users in the newly added owners. - nonTrustedUsers, repoAliases, err := nonTrustedUsersInOwnersAliases(ghc, log, triggerConfig, org, repo, r.Dir, modifiedOwnerAliasesFile.Patch, ownerAliasesModified) - if err != nil { - return err - } - - // Check if OWNERS files have the correct config and if they do, - // check if all newly added owners are trusted users. + // Check each OWNERS file. for _, c := range modifiedOwnersFiles { + // Try to load OWNERS file. path := filepath.Join(r.Dir, c.Filename) b, err := ioutil.ReadFile(path) if err != nil { log.WithError(err).Warningf("Failed to read %s.", path) return nil } - msg, owners := parseOwnersFile(b, c, log, labelsBlackList) - if msg != nil { + if msg := parseOwnersFile(b, c, log, labelsBlackList); msg != nil { wrongOwnersFiles[c.Filename] = *msg - continue - } - - nonTrustedUsers, err = nonTrustedUsersInOwners(ghc, log, triggerConfig, org, repo, c.Patch, c.Filename, owners, nonTrustedUsers, repoAliases) - if err != nil { - return err } } - - // React if there are files with incorrect configs or non-trusted users. - issueLabels, err := ghc.GetIssueLabels(org, repo, number) - if err != nil { - return err - } - hasInvalidOwnersLabel := github.HasLabel(labels.InvalidOwners, issueLabels) - + // React if we saw something. if len(wrongOwnersFiles) > 0 { s := "s" if len(wrongOwnersFiles) == 1 { s = "" } - if !hasInvalidOwnersLabel { - if err := ghc.AddLabel(org, repo, number, labels.InvalidOwners); err != nil { - return err - } + if err := ghc.AddLabel(org, repo, pre.Number, labels.InvalidOwners); err != nil { + return err } log.Debugf("Creating a review for %d %s file%s.", len(wrongOwnersFiles), ownersFileName, s) var comments []github.DraftReviewComment @@ -222,40 +171,17 @@ func handle(ghc githubClient, gc *git.Client, log *logrus.Entry, pre *github.Pul if err != nil { return fmt.Errorf("error creating a review for invalid %s file%s: %v", ownersFileName, s, err) } - } - - if len(nonTrustedUsers) > 0 { - if !hasInvalidOwnersLabel { - if err := ghc.AddLabel(org, repo, number, labels.InvalidOwners); err != nil { - return err - } - } - - // prune old comments before adding a new one - cp.PruneComments(func(comment github.IssueComment) bool { - return strings.Contains(comment.Body, fmt.Sprintf(nonCollaboratorResponseFormat, ownersFileName, org)) - }) - if err := ghc.CreateComment(org, repo, number, markdownFriendlyComment(org, nonTrustedUsers)); err != nil { - log.WithError(err).Errorf("Could not create comment for listing non-collaborators in %s files", ownersFileName) - } - } - - if len(wrongOwnersFiles) == 0 && len(nonTrustedUsers) == 0 { + } else { // Don't bother checking if it has the label...it's a race, and we'll have // to handle failure due to not being labeled anyway. if err := ghc.RemoveLabel(org, repo, pre.Number, labels.InvalidOwners); err != nil { return fmt.Errorf("failed removing %s label: %v", labels.InvalidOwners, err) } - cp.PruneComments(func(comment github.IssueComment) bool { - return strings.Contains(comment.Body, fmt.Sprintf(nonCollaboratorResponseFormat, ownersFileName, org)) - }) } - return nil } -func parseOwnersFile(b []byte, c github.PullRequestChange, log *logrus.Entry, labelsBlackList []string) (*messageWithLine, []string) { - var reviewers []string +func parseOwnersFile(b []byte, c github.PullRequestChange, log *logrus.Entry, labelsBlackList []string) *messageWithLine { var approvers []string var labels []string // by default we bind errors to line 1 @@ -281,17 +207,15 @@ func parseOwnersFile(b []byte, c github.PullRequestChange, log *logrus.Entry, la return &messageWithLine{ lineNumber, fmt.Sprintf("Cannot parse file: %v.", err), - }, nil + } } // it's a FullConfig for _, config := range full.Filters { - reviewers = append(reviewers, config.Reviewers...) approvers = append(approvers, config.Approvers...) labels = append(labels, config.Labels...) } } else { // it's a SimpleConfig - reviewers = simple.Config.Reviewers approvers = simple.Config.Approvers labels = simple.Config.Labels } @@ -300,106 +224,14 @@ func parseOwnersFile(b []byte, c github.PullRequestChange, log *logrus.Entry, la return &messageWithLine{ lineNumber, fmt.Sprintf("File contains blacklisted labels: %s.", sets.NewString(labels...).Intersection(sets.NewString(labelsBlackList...)).List()), - }, nil + } } // Check approvers isn't empty if filepath.Dir(c.Filename) == "." && len(approvers) == 0 { return &messageWithLine{ lineNumber, fmt.Sprintf("No approvers defined in this root directory %s file.", ownersFileName), - }, nil - } - owners := append(reviewers, approvers...) - return nil, owners -} - -func markdownFriendlyComment(org string, nonTrustedUsers map[string][]string) string { - var commentLines []string - commentLines = append(commentLines, fmt.Sprintf(nonCollaboratorResponseFormat, ownersFileName, org)) - - for user, ownersFiles := range nonTrustedUsers { - commentLines = append(commentLines, fmt.Sprintf("- @%s", user)) - for _, filename := range ownersFiles { - commentLines = append(commentLines, fmt.Sprintf(" - %s", filename)) } } - return strings.Join(commentLines, "\n") -} - -func nonTrustedUsersInOwnersAliases(ghc githubClient, log *logrus.Entry, triggerConfig plugins.Trigger, org, repo, dir, patch string, ownerAliasesModified bool) (map[string][]string, repoowners.RepoAliases, error) { - repoAliases := make(repoowners.RepoAliases) - // nonTrustedUsers is a map of non-trusted users to the list of files they are being added in - nonTrustedUsers := map[string][]string{} - var err error - - // If OWNERS_ALIASES exists, get all aliases. - path := filepath.Join(dir, ownersAliasesFileName) - if _, err := os.Stat(path); err == nil { - b, err := ioutil.ReadFile(path) - if err != nil { - return nonTrustedUsers, repoAliases, fmt.Errorf("Failed to read %s: %v", path, err) - } - repoAliases, err = repoowners.ParseAliasesConfig(b) - if err != nil { - return nonTrustedUsers, repoAliases, fmt.Errorf("error parsing aliases config for %s file: %v", ownersAliasesFileName, err) - } - } - - // If OWNERS_ALIASES file was modified, check if newly added owners are trusted. - if ownerAliasesModified { - allOwners := repoAliases.ExpandAllAliases().List() - for _, owner := range allOwners { - // cap the number of checks to avoid exhausting tokens in case of large OWNERS refactors. - if len(nonTrustedUsers) > 20 { - break - } - nonTrustedUsers, err = checkIfTrustedUser(ghc, log, triggerConfig, owner, patch, ownersAliasesFileName, org, repo, nonTrustedUsers, repoAliases) - if err != nil { - return nonTrustedUsers, repoAliases, err - } - } - } - - return nonTrustedUsers, repoAliases, nil -} - -func nonTrustedUsersInOwners(ghc githubClient, log *logrus.Entry, triggerConfig plugins.Trigger, org, repo, patch, fileName string, owners []string, nonTrustedUsers map[string][]string, repoAliases repoowners.RepoAliases) (map[string][]string, error) { - var err error - for _, owner := range owners { - // cap the number of checks to avoid exhausting tokens in case of large OWNERS refactors. - if len(nonTrustedUsers) > 20 { - break - } - - // ignore if owner is an alias - if _, ok := repoAliases[owner]; ok { - continue - } - - nonTrustedUsers, err = checkIfTrustedUser(ghc, log, triggerConfig, owner, patch, fileName, org, repo, nonTrustedUsers, repoAliases) - if err != nil { - return nonTrustedUsers, err - } - } - return nonTrustedUsers, nil -} - -// checkIfTrustedUser looks for newly addded owners by checking if they are in the patch -// and then checks if the owner is a trusted user. -func checkIfTrustedUser(ghc githubClient, log *logrus.Entry, triggerConfig plugins.Trigger, owner, patch, fileName, org, repo string, nonTrustedUsers map[string][]string, repoAliases repoowners.RepoAliases) (map[string][]string, error) { - if strings.Contains(patch, owner) { - isTrustedUser, err := trigger.TrustedUser(ghc, triggerConfig, owner, org, repo) - if err != nil { - return nonTrustedUsers, err - } - - if !isTrustedUser { - if ownersFiles, ok := nonTrustedUsers[owner]; ok { - nonTrustedUsers[owner] = append(ownersFiles, fileName) - } else { - nonTrustedUsers[owner] = []string{fileName} - } - } - } - return nonTrustedUsers, nil + return nil } diff --git a/vendor/k8s.io/test-infra/prow/pod-utils/decorate/podspec.go b/vendor/k8s.io/test-infra/prow/pod-utils/decorate/podspec.go index 4c746070..530b5ef1 100644 --- a/vendor/k8s.io/test-infra/prow/pod-utils/decorate/podspec.go +++ b/vendor/k8s.io/test-infra/prow/pod-utils/decorate/podspec.go @@ -535,7 +535,7 @@ func decorate(spec *coreapi.PodSpec, pj *prowapi.ProwJob, rawEnv map[string]stri previous = "" exitZero = false ) - wrapperOptions, err := InjectEntrypoint(&spec.Containers[0], pj.Spec.DecorationConfig.Timeout.Duration, pj.Spec.DecorationConfig.GracePeriod.Duration, prefix, previous, exitZero, logMount, toolsMount) + wrapperOptions, err := InjectEntrypoint(&spec.Containers[0], pj.Spec.DecorationConfig.Timeout.Get(), pj.Spec.DecorationConfig.GracePeriod.Get(), prefix, previous, exitZero, logMount, toolsMount) if err != nil { return fmt.Errorf("wrap container: %v", err) } diff --git a/vendor/k8s.io/test-infra/prow/pod-utils/gcs/BUILD.bazel b/vendor/k8s.io/test-infra/prow/pod-utils/gcs/BUILD.bazel index 575e6c00..a887c425 100644 --- a/vendor/k8s.io/test-infra/prow/pod-utils/gcs/BUILD.bazel +++ b/vendor/k8s.io/test-infra/prow/pod-utils/gcs/BUILD.bazel @@ -37,6 +37,7 @@ filegroup( go_test( name = "go_default_test", srcs = [ + "metadata_test.go", "target_test.go", "upload_test.go", ], diff --git a/vendor/k8s.io/test-infra/prow/pod-utils/gcs/metadata.go b/vendor/k8s.io/test-infra/prow/pod-utils/gcs/metadata.go index a837afdb..74081f4c 100644 --- a/vendor/k8s.io/test-infra/prow/pod-utils/gcs/metadata.go +++ b/vendor/k8s.io/test-infra/prow/pod-utils/gcs/metadata.go @@ -17,6 +17,11 @@ limitations under the License. package gcs import ( + "mime" + "strings" + + "cloud.google.com/go/storage" + "k8s.io/test-infra/testgrid/metadata" ) @@ -27,3 +32,49 @@ type Started = metadata.Started // Finished holds finished.json data type Finished = metadata.Finished + +// AttributesFromFileName guesses file attributes from the filename +// and returns the attributes and a simplifed filename. For example, +// build-log.txt.gz would be: +// +// Content-Type: text/plain; charset=utf-8 +// Content-Encoding: gzip +// +// and the simplified filename would be build-log.txt (excluding the +// content encoding extension). +func AttributesFromFileName(filename string) (string, *storage.ObjectAttrs) { + attrs := &storage.ObjectAttrs{} + segments := strings.Split(filename, ".") + index := len(segments) - 1 + segment := segments[index] + + // https://www.iana.org/assignments/http-parameters/http-parameters.xhtml#content-coding + switch segment { + case "gz", "gzip": + attrs.ContentEncoding = "gzip" + } + + if attrs.ContentEncoding != "" { + if index == 0 { + segment = "" + } else { + filename = filename[:len(filename)-len(segment)-1] + index -= 1 + segment = segments[index] + } + } + + if segment != "" { + mediaType := mime.TypeByExtension("." + segment) + if mediaType != "" { + attrs.ContentType = mediaType + } + } + + if attrs.ContentType == "" && attrs.ContentEncoding == "gzip" { + attrs.ContentType = "application/gzip" + attrs.ContentEncoding = "" + } + + return filename, attrs +} diff --git a/vendor/k8s.io/test-infra/prow/pod-utils/gcs/upload.go b/vendor/k8s.io/test-infra/prow/pod-utils/gcs/upload.go index 734becc2..f4962dc6 100644 --- a/vendor/k8s.io/test-infra/prow/pod-utils/gcs/upload.go +++ b/vendor/k8s.io/test-infra/prow/pod-utils/gcs/upload.go @@ -66,13 +66,27 @@ func Upload(bucket *storage.BucketHandle, uploadTargets map[string]UploadFunc) e // FileUpload returns an UploadFunc which copies all // data from the file on disk to the GCS object func FileUpload(file string) UploadFunc { + return FileUploadWithAttributes(file, nil) +} + +// FileUploadWithMetadata returns an UploadFunc which copies all +// data from the file on disk into GCS object and also sets the provided +// metadata fields on the object. +func FileUploadWithMetadata(file string, metadata map[string]string) UploadFunc { + return FileUploadWithAttributes(file, &storage.ObjectAttrs{Metadata: metadata}) +} + +// FileUploadWithAttributes returns an UploadFunc which copies all data +// from the file on disk into GCS object and also sets the provided +// attributes on the object. +func FileUploadWithAttributes(file string, attrs *storage.ObjectAttrs) UploadFunc { return func(obj *storage.ObjectHandle) error { reader, err := os.Open(file) if err != nil { return err } - uploadErr := DataUpload(reader)(obj) + uploadErr := DataUploadWithAttributes(reader, attrs)(obj) closeErr := reader.Close() return errorutil.NewAggregate(uploadErr, closeErr) @@ -80,24 +94,31 @@ func FileUpload(file string) UploadFunc { } // DataUpload returns an UploadFunc which copies all -// data from src reader into GCS +// data from src reader into GCS. func DataUpload(src io.Reader) UploadFunc { - return func(obj *storage.ObjectHandle) error { - writer := obj.NewWriter(context.Background()) - _, copyErr := io.Copy(writer, src) - closeErr := writer.Close() - - return errorutil.NewAggregate(copyErr, closeErr) - } + return DataUploadWithAttributes(src, nil) } // DataUploadWithMetadata returns an UploadFunc which copies all // data from src reader into GCS and also sets the provided metadata // fields onto the object. func DataUploadWithMetadata(src io.Reader, metadata map[string]string) UploadFunc { + return DataUploadWithAttributes(src, &storage.ObjectAttrs{Metadata: metadata}) +} + +// DataUploadWithAttributes returns an UploadFunc which copies all data +// from src reader into GCS and also sets the provided attributes on +// the object. +func DataUploadWithAttributes(src io.Reader, attrs *storage.ObjectAttrs) UploadFunc { return func(obj *storage.ObjectHandle) error { writer := obj.NewWriter(context.Background()) - writer.Metadata = metadata + + if attrs != nil { + name := writer.ObjectAttrs.Name + writer.ObjectAttrs = *attrs + writer.ObjectAttrs.Name = name + } + _, copyErr := io.Copy(writer, src) closeErr := writer.Close() diff --git a/vendor/k8s.io/test-infra/prow/repoowners/repoowners.go b/vendor/k8s.io/test-infra/prow/repoowners/repoowners.go index e281afa8..ba7e3a7c 100644 --- a/vendor/k8s.io/test-infra/prow/repoowners/repoowners.go +++ b/vendor/k8s.io/test-infra/prow/repoowners/repoowners.go @@ -319,20 +319,6 @@ func (a RepoAliases) ExpandAliases(logins sets.String) sets.String { return logins } -// ExpandAllAliases returns members of all aliases mentioned, duplicates are pruned -func (a RepoAliases) ExpandAllAliases() sets.String { - if a == nil { - return nil - } - - var result, users sets.String - for alias := range a { - users = a.ExpandAlias(alias) - result = result.Union(users) - } - return result -} - func loadAliasesFrom(baseDir string, log *logrus.Entry) RepoAliases { path := filepath.Join(baseDir, aliasesFileName) b, err := ioutil.ReadFile(path) @@ -482,24 +468,6 @@ func ParseSimpleConfig(b []byte) (SimpleConfig, error) { return *simple, err } -// ParseAliasesConfig will unmarshal an OWNERS_ALIASES file's content into RepoAliases. -// Returns an error if the content cannot be unmarshalled. -func ParseAliasesConfig(b []byte) (RepoAliases, error) { - result := make(RepoAliases) - - config := &struct { - Data map[string][]string `json:"aliases,omitempty"` - }{} - if err := yaml.Unmarshal(b, config); err != nil { - return result, err - } - - for alias, expanded := range config.Data { - result[github.NormLogin(alias)] = normLogins(expanded) - } - return result, nil -} - var mdStructuredHeaderRegex = regexp.MustCompile("^---\n(.|\n)*\n---") // decodeOwnersMdConfig will parse the yaml header if it exists and unmarshal it into a singleOwnersConfig. diff --git a/vendor/k8s.io/test-infra/prow/slack/BUILD.bazel b/vendor/k8s.io/test-infra/prow/slack/BUILD.bazel index 6bed21fb..53fc1e61 100644 --- a/vendor/k8s.io/test-infra/prow/slack/BUILD.bazel +++ b/vendor/k8s.io/test-infra/prow/slack/BUILD.bazel @@ -21,6 +21,9 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//prow/slack/reporter:all-srcs", + ], tags = ["automanaged"], ) diff --git a/vendor/k8s.io/test-infra/prow/slack/client.go b/vendor/k8s.io/test-infra/prow/slack/client.go index 22d17dd8..ad3692f2 100644 --- a/vendor/k8s.io/test-infra/prow/slack/client.go +++ b/vendor/k8s.io/test-infra/prow/slack/client.go @@ -104,6 +104,7 @@ func (sl *Client) WriteMessage(text, channel string) error { if sl.fake { return nil } + var uv = sl.urlValues() uv.Add("channel", channel) uv.Add("text", text) diff --git a/vendor/k8s.io/test-infra/testgrid/metadata/junit/junit.go b/vendor/k8s.io/test-infra/testgrid/metadata/junit/junit.go index f7e69d43..67662f31 100644 --- a/vendor/k8s.io/test-infra/testgrid/metadata/junit/junit.go +++ b/vendor/k8s.io/test-infra/testgrid/metadata/junit/junit.go @@ -44,15 +44,49 @@ type Suite struct { */ } +// Property defines the xml element that stores additional metrics about each benchmark. +type Property struct { + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +} + +// Properties defines the xml element that stores the list of properties that are associated with one benchmark. +type Properties struct { + PropertyList []Property `xml:"property"` +} + // Result holds results type Result struct { - Name string `xml:"name,attr"` - Time float64 `xml:"time,attr"` - ClassName string `xml:"classname,attr"` - Failure *string `xml:"failure,omitempty"` - Output *string `xml:"system-out,omitempty"` - Error *string `xml:"system-err,omitempty"` - Skipped *string `xml:"skipped,omitempty"` + Name string `xml:"name,attr"` + Time float64 `xml:"time,attr"` + ClassName string `xml:"classname,attr"` + Failure *string `xml:"failure,omitempty"` + Output *string `xml:"system-out,omitempty"` + Error *string `xml:"system-err,omitempty"` + Skipped *string `xml:"skipped,omitempty"` + Properties *Properties `xml:"properties,omitempty"` +} + +// SetProperty adds the specified property to the Result or replaces the +// existing value if a property with that name already exists. +func (r *Result) SetProperty(name, value string) { + if r.Properties == nil { + r.Properties = &Properties{} + } + for i, existing := range r.Properties.PropertyList { + if existing.Name == name { + r.Properties.PropertyList[i].Value = value + return + } + } + // Didn't find an existing property. Add a new one. + r.Properties.PropertyList = append( + r.Properties.PropertyList, + Property{ + Name: name, + Value: value, + }, + ) } // Message extracts the message for the junit test case. diff --git a/vendor/modules.txt b/vendor/modules.txt index 424b5e0f..0dd58532 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -442,7 +442,7 @@ k8s.io/client-go/util/jsonpath k8s.io/client-go/third_party/forked/golang/template # k8s.io/kube-openapi v0.0.0-20180711000925-0cf8f7e6ed1d k8s.io/kube-openapi/pkg/util/proto -# k8s.io/test-infra v0.0.0-20190427170421-2228dd155e83 +# k8s.io/test-infra v0.0.0-20190507120946-f699675303ae k8s.io/test-infra/prow/apis/prowjobs/v1 k8s.io/test-infra/prow/config k8s.io/test-infra/prow/hook