Add architectures support to prowgen

tools/prowgen/pkg/decorator/variable.go

@@ -32,27 +32,48 @@ const (
 
 var variableSubstitutionRegex = regexp.MustCompile(`\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*\)`)
 
-func ApplyVariables(job spec.Job, params map[string]string, matrix map[string][]string) []spec.Job {
+func applyArch(arch string, job spec.Job) spec.Job {
+	// For backwards compatibility, amd64 is not suffixed
+	if arch != "amd64" {
+		job.Name += "-" + arch
+		if job.NodeSelector == nil {
+			job.NodeSelector = map[string]string{}
+		}
+		// TODO apply everywhere
+		job.NodeSelector["kubernetes.io/arch"] = arch
+	}
+	return job
+}
+
+func ApplyVariables(job spec.Job, architectures []string, params map[string]string, matrix map[string][]string) []spec.Job {
 	yamlBS, err := yaml.Marshal(job)
 	if err != nil {
 		log.Fatalf("Failed to marshal the given Job: %v", err)
 	}
 
-	subsExps := getVarSubstitutionExpressions(string(yamlBS))
-	if len(subsExps) == 0 {
-		return []spec.Job{job}
-	}
+	jobs := make([]spec.Job, 0)
 
-	resolvedYAMLStr := applyParams(string(yamlBS), subsExps, params)
-	resolvedYAMLStrs := applyMatrix(resolvedYAMLStr, subsExps, matrix)
+	for _, arch := range architectures {
+		subsExps := getVarSubstitutionExpressions(string(yamlBS))
+		if len(subsExps) == 0 {
+			jobs = append(jobs, applyArch(arch, job))
+			continue
+		}
+		if params == nil {
+			params = map[string]string{}
+		}
+		params["arch"] = arch
 
-	jobs := make([]spec.Job, 0)
-	for _, jobYaml := range resolvedYAMLStrs {
-		job := spec.Job{}
-		if err := yaml.Unmarshal([]byte(jobYaml), &job); err != nil {
-			log.Fatalf("Failed to unmarshal the yaml to Job: %v", err)
+		resolvedYAMLStr := applyParams(string(yamlBS), subsExps, params)
+		resolvedYAMLStrs := applyMatrix(resolvedYAMLStr, subsExps, matrix)
+
+		for _, jobYaml := range resolvedYAMLStrs {
+			job := spec.Job{}
+			if err := yaml.Unmarshal([]byte(jobYaml), &job); err != nil {
+				log.Fatalf("Failed to unmarshal the yaml to Job: %v", err)
+			}
+			jobs = append(jobs, applyArch(arch, job))
 		}
-		jobs = append(jobs, job)
 	}
 	return jobs
 }

tools/prowgen/pkg/generate.go

@@ -42,6 +42,9 @@ const (
 
 	DefaultAutogenHeader = "# THIS FILE IS AUTOGENERATED, DO NOT EDIT IT MANUALLY."
 
+	ArchAMD64 = "amd64"
+	ArchARM64 = "arm64"
+
 	TypePostsubmit = "postsubmit"
 	TypePresubmit  = "presubmit"
 	TypePeriodic   = "periodic"
@@ -82,7 +85,7 @@ func (cli *Client) ReadJobsConfig(file string) spec.JobsConfig {
 		log.Fatalf("Failed to read %q: %v", file, err)
 	}
 	jobsConfig := spec.JobsConfig{}
-	if err := yaml.Unmarshal(yamlFile, &jobsConfig); err != nil {
+	if err := yaml.UnmarshalStrict(yamlFile, &jobsConfig); err != nil {
 		log.Fatalf("Failed to unmarshal %q: %v", file, err)
 	}
 
@@ -184,6 +187,11 @@ func validateJobsConfig(fileName string, jobsConfig spec.JobsConfig) error {
 				err = multierror.Append(err, e)
 			}
 		}
+		for _, t := range job.Architectures {
+			if e := validate(t, sets.NewString(ArchAMD64, ArchARM64, TypePeriodic), "architectures"); e != nil {
+				err = multierror.Append(err, e)
+			}
+		}
 		for _, repo := range job.Repos {
 			if len(strings.Split(repo, "/")) != 2 {
 				err = multierror.Append(err, fmt.Errorf("%s: repo %v not valid, should take form org/repo", fileName, repo))
@@ -212,7 +220,10 @@ func (cli *Client) ConvertJobConfig(fileName string, jobsConfig spec.JobsConfig,
 	var periodics []config.Periodic
 
 	for _, parentJob := range jobsConfig.Jobs {
-		expandedJobs := decorator.ApplyVariables(parentJob, jobsConfig.Params, jobsConfig.Matrix)
+		if len(parentJob.Architectures) == 0 {
+			parentJob.Architectures = []string{ArchAMD64}
+		}
+		expandedJobs := decorator.ApplyVariables(parentJob, parentJob.Architectures, jobsConfig.Params, jobsConfig.Matrix)
 		for _, job := range expandedJobs {
 			brancher := config.Brancher{
 				Branches: []string{fmt.Sprintf("^%s$", branch)},

tools/prowgen/pkg/generate_test.go

@@ -26,7 +26,7 @@ import (
 
 func TestGenerateConfig(t *testing.T) {
 	bc := ReadBase(nil, "testdata/.base.yaml")
-	cli := &Client{BaseConfig: *bc}
+	cli := &Client{BaseConfig: bc}
 	tests := []struct {
 		name        string
 		expectError bool

tools/prowgen/pkg/spec/spec.go

@@ -76,6 +76,8 @@ type Job struct {
 	Tags    []string `json:"tags,omitempty"`
 	Types   []string `json:"types,omitempty"`
 	Repos   []string `json:"repos,omitempty"`
+	// Architectures defines architectures to build as. Defaults to amd64.
+	Architectures []string `json:"architectures,omitempty"`
 
 	GerritPresubmitLabel  string `json:"gerrit_presubmit_label,omitempty"`
 	GerritPostsubmitLabel string `json:"gerrit_postsubmit_label,omitempty"`

tools/prowgen/pkg/testdata/long-job-name.yaml

@@ -44,7 +44,6 @@ jobs:
     resources: custom
     command: [prow/command.sh]
     repos: [istio/istio]
-    modifier: [presubmit_skipped]
     trigger: "/test basic"
 
 requirements: [gocache]

tools/prowgen/pkg/testdata/simple.yaml

@@ -44,9 +44,20 @@ jobs:
     resources: custom
     command: [prow/command.sh]
     repos: [istio/istio]
-    modifier: [presubmit_skipped]
     trigger: "/test basic"
 
+  - name: multi-arch
+    types: [presubmit]
+    architectures: [amd64, arm64]
+    command: [prow/command.sh]
+    image: test
+
+  - name: multi-arch-param
+    types: [presubmit]
+    architectures: [amd64, arm64]
+    command: [prow/command.sh, $(params.arch)]
+    image: test
+
 requirements: [gocache]
 
 resources_presets: