Skip to content

Some updates to the Prow configgen tool #3426

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
istio-testing merged 1 commit into from
Jul 8, 2021
Merged

Some updates to the Prow configgen tool #3426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions prow/config/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,13 @@ image: gcr.io/istio-testing/build-tools:master
# version
supports_release_branching: false

# A matrix can contain arbitrary number of dimensions, and can be used to easily define a combination of Prow jobs.
# Each dimension will only be respected for computation if they are referenced in the Prow job config, and the syntax
# to use the dimension is $(matrix.dimension_name)
matrix:
greet: [hey, hello, hi]
name: [foo, bar]

# Defines the actual jobs
jobs:
# A basic test requires just a name and a command to run
Expand All @@ -127,6 +134,10 @@ jobs:
- skipped # if set, the test will run only in postsubmit or by explicitly calling /test on it
- hidden # if set, the test will run but not be reported to the GitHub UI
- optional # if set, the test will not be required
- name: $(matrix.greet)-$(matrix.name)
# Prow jobs will be generated based on the combinations of each dimension.
# In this case 3*2=6 Prow jobs will be generated.
command: [echo, "${matrix.greet} $(matrix.name)"]

# Defines preset resource allocations for tests
# The map here will be intersected with the map in the global config (if there is),
Expand Down
72 changes: 68 additions & 4 deletions prow/config/generate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
prowjob "k8s.io/test-infra/prow/apis/prowjobs/v1"
"k8s.io/test-infra/prow/config"
"k8s.io/test-infra/prow/gerrit/client"
)

func exit(err error, context string) {
Expand Down Expand Up @@ -75,6 +76,9 @@ type GlobalConfig struct {

PathAliases map[string]string `json:"path_aliases,omitempty"`

GCSLogBucket string `json:"gcs_log_bucket,omitempty"`
TerminationGracePeriodSeconds int64 `json:"termination_grace_period_seconds,omitempty"`

Cluster string `json:"cluster,omitempty"`
NodeSelector map[string]string `json:"node_selector,omitempty"`

Expand Down Expand Up @@ -112,12 +116,17 @@ type JobsConfig struct {
Interval string `json:"interval,omitempty"`
Cron string `json:"cron,omitempty"`

GCSLogBucket string `json:"gcs_log_bucket,omitempty"`
TerminationGracePeriodSeconds int64 `json:"termination_grace_period_seconds,omitempty"`

Cluster string `json:"cluster,omitempty"`
NodeSelector map[string]string `json:"node_selector,omitempty"`

Annotations map[string]string `json:"annotations,omitempty"`
Labels map[string]string `json:"labels,omitempty"`

Trigger string `json:"trigger,omitempty"`

ResourcePresets map[string]v1.ResourceRequirements `json:"resources,omitempty"`
Requirements []string `json:"requirements,omitempty"`
RequirementPresets map[string]RequirementPreset `json:"requirement_presets,omitempty"`
Expand All @@ -141,11 +150,18 @@ type Job struct {
Interval string `json:"interval,omitempty"`
Cron string `json:"cron,omitempty"`

GCSLogBucket string `json:"gcs_log_bucket,omitempty"`
TerminationGracePeriodSeconds int64 `json:"termination_grace_period_seconds,omitempty"`

Cluster string `json:"cluster,omitempty"`
NodeSelector map[string]string `json:"node_selector,omitempty"`

Annotations map[string]string `json:"annotations,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
GerritPresubmitLabel string `json:"gerrit_presubmit_label,omitempty"`
GerritPostsubmitLabel string `json:"gerrit_postsubmit_label,omitempty"`

Trigger string `json:"trigger,omitempty"`

Resource string `json:"resources,omitempty"`
Modifiers []string `json:"modifiers,omitempty"`
Expand Down Expand Up @@ -231,6 +247,24 @@ func resolveOverwrites(globalConfig GlobalConfig, jobsConfig JobsConfig) JobsCon
}
job.Cluster = cluster

gcsLogBucket := globalConfig.GCSLogBucket
if jobsConfig.GCSLogBucket != "" {
gcsLogBucket = jobsConfig.GCSLogBucket
}
if job.GCSLogBucket != "" {
gcsLogBucket = job.GCSLogBucket
}
job.GCSLogBucket = gcsLogBucket

terminateGracePeriodSeconds := globalConfig.TerminationGracePeriodSeconds
if jobsConfig.TerminationGracePeriodSeconds != 0 {
terminateGracePeriodSeconds = jobsConfig.TerminationGracePeriodSeconds
}
if job.TerminationGracePeriodSeconds != 0 {
terminateGracePeriodSeconds = job.TerminationGracePeriodSeconds
}
job.TerminationGracePeriodSeconds = terminateGracePeriodSeconds

image := jobsConfig.Image
if job.Image != "" {
image = job.Image
Expand All @@ -255,6 +289,12 @@ func resolveOverwrites(globalConfig GlobalConfig, jobsConfig JobsConfig) JobsCon
}
job.Cron = cronStr

trigger := jobsConfig.Trigger
if job.Trigger != "" {
trigger = job.Trigger
}
job.Trigger = trigger

jobsConfig.Jobs[i] = job
}

Expand Down Expand Up @@ -375,6 +415,9 @@ func (cli *Client) ConvertJobConfig(jobsConfig JobsConfig, branch string) config
AlwaysRun: true,
Brancher: brancher,
}
if job.GerritPresubmitLabel != "" {
presubmit.Labels[client.GerritReportLabel] = job.GerritPresubmitLabel
}
if pa, ok := globalConfig.PathAliases[jobsConfig.Org]; ok {
presubmit.UtilityConfig.PathAlias = fmt.Sprintf("%s/%s", pa, jobsConfig.Repo)
}
Expand All @@ -384,6 +427,10 @@ func (cli *Client) ConvertJobConfig(jobsConfig JobsConfig, branch string) config
}
presubmit.AlwaysRun = false
}
if job.Trigger != "" {
presubmit.Trigger = job.Trigger
presubmit.RerunCommand = job.Trigger
}
if testgridConfig.Enabled {
presubmit.JobBase.Annotations = mergeMaps(presubmit.JobBase.Annotations, map[string]string{
TestGridDashboard: testgridJobPrefix,
Expand All @@ -405,6 +452,9 @@ func (cli *Client) ConvertJobConfig(jobsConfig JobsConfig, branch string) config
JobBase: createJobBase(globalConfig, jobsConfig, job, name, branch, jobsConfig.ResourcePresets),
Brancher: brancher,
}
if job.GerritPostsubmitLabel != "" {
postsubmit.Labels[client.GerritReportLabel] = job.GerritPostsubmitLabel
}
if pa, ok := globalConfig.PathAliases[jobsConfig.Org]; ok {
postsubmit.UtilityConfig.PathAlias = fmt.Sprintf("%s/%s", pa, jobsConfig.Repo)
}
Expand Down Expand Up @@ -675,9 +725,23 @@ func createJobBase(globalConfig GlobalConfig, jobConfig JobsConfig, job Job,
jb.Annotations = map[string]string{}
}

if job.TerminationGracePeriodSeconds != 0 {
jb.Spec.TerminationGracePeriodSeconds = &job.TerminationGracePeriodSeconds
}

if job.Timeout != nil {
jb.DecorationConfig = &prowjob.DecorationConfig{
Timeout: job.Timeout,
if jb.DecorationConfig == nil {
jb.DecorationConfig = &prowjob.DecorationConfig{}
}
jb.DecorationConfig.Timeout = job.Timeout
}
if job.GCSLogBucket != "" {
if jb.DecorationConfig == nil {
jb.DecorationConfig = &prowjob.DecorationConfig{}
}
jb.DecorationConfig.GCSConfiguration = &prowjob.GCSConfiguration{
Bucket: globalConfig.GCSLogBucket,
PathStrategy: "explicit",
}
}

Expand Down