Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 21 additions & 12 deletions cmd/ci-operator-prowgen/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,22 +36,22 @@ target=$(awk < /usr/local/e2e-targets \
--assign "r=$RANDOM" \
'BEGIN { r /= 32767 } (r -= $1) <= 0 { print $2; exit }')
case "$target" in
aws) template=e2e; CLUSTER_TYPE=aws;;
azure) template=e2e; CLUSTER_TYPE=azure4;;
aws-upi) template=upi-e2e; CLUSTER_TYPE=aws;;
vsphere) template=upi-e2e; CLUSTER_TYPE=vsphere;;
aws) template=%[1]s; CLUSTER_TYPE=aws;;
azure) template=%[1]s; CLUSTER_TYPE=azure4;;
aws-upi) template=upi-%[1]s; CLUSTER_TYPE=aws;;
vsphere) template=upi-%[1]s; CLUSTER_TYPE=vsphere;;
*) echo >&2 "invalid target $target"; exit 1 ;;
esac
ln -s "/usr/local/job-definition/cluster-launch-installer-$template.yaml" /tmp/%[1]s
ln -s "/usr/local/cluster-profiles/$CLUSTER_TYPE" /tmp/%[1]s-cluster-profile
ln -s "/usr/local/job-definition/cluster-launch-installer-$template.yaml" /tmp/%[2]s
ln -s "/usr/local/cluster-profiles/$CLUSTER_TYPE" /tmp/%[2]s-cluster-profile
export CLUSTER_TYPE
exec ci-operator \
--artifact-dir=$(ARTIFACTS) \
--give-pr-author-access-to-namespace=true \
--secret-dir=/tmp/%[1]s-cluster-profile \
--secret-dir=/tmp/%[2]s-cluster-profile \
--sentry-dsn-path=/etc/sentry-dsn/ci-operator \
--target=%[1]s \
--template=/tmp/%[1]s
--target=%[2]s \
--template=/tmp/%[2]s
`
)

Expand Down Expand Up @@ -300,11 +300,18 @@ func generatePodSpecRandom(info *config.Info, test *cioperatorapi.TestStepConfig
for _, p := range openshiftInstallerRandomProfiles {
podSpec.Volumes = append(podSpec.Volumes, generateClusterProfileVolume("cluster-profile-"+string(p), "cluster-secrets-"+string(p)))
}
podSpec.Volumes = append(podSpec.Volumes, generateConfigMapVolume("job-definition", []string{"prow-job-cluster-launch-installer-e2e", "prow-job-cluster-launch-installer-upi-e2e"}))
var template string
if test.OpenshiftInstallerRandomClusterTestConfiguration != nil {
template = "e2e"
podSpec.Volumes = append(podSpec.Volumes, generateConfigMapVolume("job-definition", []string{"prow-job-cluster-launch-installer-e2e", "prow-job-cluster-launch-installer-upi-e2e"}))
} else if test.OpenshiftInstallerSrcRandomClusterTestConfiguration != nil {
template = "src"
podSpec.Volumes = append(podSpec.Volumes, generateConfigMapVolume("job-definition", []string{"prow-job-cluster-launch-installer-src"}))
}
podSpec.Volumes = append(podSpec.Volumes, generateConfigMapVolume("e2e-targets", []string{"e2e-targets"}))
container := &podSpec.Containers[0]
container.Command = []string{"bash"}
container.Args = []string{"-c", fmt.Sprintf(openshiftInstallerRandomCmd, test.As)}
container.Args = []string{"-c", fmt.Sprintf(openshiftInstallerRandomCmd, template, test.As)}
container.Env = append(container.Env, []kubeapi.EnvVar{
{Name: "JOB_NAME_SAFE", Value: strings.Replace(test.As, "_", "-", -1)},
{Name: "TEST_COMMAND", Value: test.Commands},
Expand Down Expand Up @@ -451,7 +458,9 @@ func generateJobs(
if c := configSpec.ReleaseTagConfiguration; c != nil {
release = c.Name
}
if conf := element.OpenshiftInstallerRandomClusterTestConfiguration; conf != nil {
if element.OpenshiftInstallerRandomClusterTestConfiguration != nil {
podSpec = generatePodSpecRandom(info, &element)
} else if element.OpenshiftInstallerSrcRandomClusterTestConfiguration != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both branches are the same, is this intentional? Looks correct, but then an OR instead of a separate if may be in order.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was supposed to keep the line short(er), using those names is a bit ridiculous. I'll change to an ||.

podSpec = generatePodSpecRandom(info, &element)
} else {
podSpec = generatePodSpecTemplate(info, release, &element)
Expand Down
237 changes: 126 additions & 111 deletions cmd/ci-operator-prowgen/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -330,133 +330,148 @@ func TestGeneratePodSpecTemplate(t *testing.T) {

func TestGeneratePodSpecRandom(t *testing.T) {
info := config.Info{Org: "org", Repo: "repo", Branch: "branch"}
test := ciop.TestStepConfiguration{
As: "e2e",
Commands: "commands",
OpenshiftInstallerRandomClusterTestConfiguration: &ciop.OpenshiftInstallerRandomClusterTestConfiguration{},
}
expected := &kubeapi.PodSpec{
ServiceAccountName: "ci-operator",
Containers: []kubeapi.Container{{
Image: "ci-operator:latest",
ImagePullPolicy: kubeapi.PullAlways,
Command: []string{"bash"},
Args: []string{"-c", fmt.Sprintf(openshiftInstallerRandomCmd, "e2e")},
Env: []kubeapi.EnvVar{{
Name: "CONFIG_SPEC",
ValueFrom: &kubeapi.EnvVarSource{
ConfigMapKeyRef: &kubeapi.ConfigMapKeySelector{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "ci-operator-misc-configs",
makePodSpec := func(template string) kubeapi.PodSpec {
return kubeapi.PodSpec{
ServiceAccountName: "ci-operator",
Containers: []kubeapi.Container{{
Image: "ci-operator:latest",
ImagePullPolicy: kubeapi.PullAlways,
Command: []string{"bash"},
Args: []string{"-c", fmt.Sprintf(openshiftInstallerRandomCmd, template, "e2e")},
Env: []kubeapi.EnvVar{{
Name: "CONFIG_SPEC",
ValueFrom: &kubeapi.EnvVarSource{
ConfigMapKeyRef: &kubeapi.ConfigMapKeySelector{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "ci-operator-misc-configs",
},
Key: "org-repo-branch.yaml",
},
Key: "org-repo-branch.yaml",
},
},
},
{Name: "JOB_NAME_SAFE", Value: "e2e"},
{Name: "TEST_COMMAND", Value: "commands"},
},
Resources: kubeapi.ResourceRequirements{
Requests: kubeapi.ResourceList{"cpu": *resource.NewMilliQuantity(10, resource.DecimalSI)},
},
VolumeMounts: []kubeapi.VolumeMount{{
Name: "sentry-dsn",
MountPath: "/etc/sentry-dsn",
ReadOnly: true,
}, {
Name: "cluster-profile-aws",
MountPath: "/usr/local/cluster-profiles/aws",
}, {
Name: "cluster-profile-azure4",
MountPath: "/usr/local/cluster-profiles/azure4",
}, {
Name: "cluster-profile-vsphere",
MountPath: "/usr/local/cluster-profiles/vsphere",
}, {
Name: "e2e-targets",
MountPath: "/usr/local/e2e-targets",
SubPath: "e2e-targets",
}, {
Name: "job-definition",
MountPath: "/usr/local/job-definition",
{Name: "JOB_NAME_SAFE", Value: "e2e"},
{Name: "TEST_COMMAND", Value: "commands"},
},
Resources: kubeapi.ResourceRequirements{
Requests: kubeapi.ResourceList{"cpu": *resource.NewMilliQuantity(10, resource.DecimalSI)},
},
VolumeMounts: []kubeapi.VolumeMount{{
Name: "sentry-dsn",
MountPath: "/etc/sentry-dsn",
ReadOnly: true,
}, {
Name: "cluster-profile-aws",
MountPath: "/usr/local/cluster-profiles/aws",
}, {
Name: "cluster-profile-azure4",
MountPath: "/usr/local/cluster-profiles/azure4",
}, {
Name: "cluster-profile-vsphere",
MountPath: "/usr/local/cluster-profiles/vsphere",
}, {
Name: "e2e-targets",
MountPath: "/usr/local/e2e-targets",
SubPath: "e2e-targets",
}, {
Name: "job-definition",
MountPath: "/usr/local/job-definition",
}},
}},
}},
Volumes: []kubeapi.Volume{{
Name: "sentry-dsn",
VolumeSource: kubeapi.VolumeSource{
Secret: &kubeapi.SecretVolumeSource{SecretName: "sentry-dsn"},
},
}, {
Name: "cluster-profile-aws",
VolumeSource: kubeapi.VolumeSource{
Projected: &kubeapi.ProjectedVolumeSource{
Sources: []kubeapi.VolumeProjection{{
Secret: &kubeapi.SecretProjection{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "cluster-secrets-aws",
Volumes: []kubeapi.Volume{{
Name: "sentry-dsn",
VolumeSource: kubeapi.VolumeSource{
Secret: &kubeapi.SecretVolumeSource{SecretName: "sentry-dsn"},
},
}, {
Name: "cluster-profile-aws",
VolumeSource: kubeapi.VolumeSource{
Projected: &kubeapi.ProjectedVolumeSource{
Sources: []kubeapi.VolumeProjection{{
Secret: &kubeapi.SecretProjection{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "cluster-secrets-aws",
},
},
},
}},
}},
},
},
},
}, {
Name: "cluster-profile-azure4",
VolumeSource: kubeapi.VolumeSource{
Projected: &kubeapi.ProjectedVolumeSource{
Sources: []kubeapi.VolumeProjection{{
Secret: &kubeapi.SecretProjection{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "cluster-secrets-azure4",
}, {
Name: "cluster-profile-azure4",
VolumeSource: kubeapi.VolumeSource{
Projected: &kubeapi.ProjectedVolumeSource{
Sources: []kubeapi.VolumeProjection{{
Secret: &kubeapi.SecretProjection{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "cluster-secrets-azure4",
},
},
},
}},
}},
},
},
},
}, {
Name: "cluster-profile-vsphere",
VolumeSource: kubeapi.VolumeSource{
Projected: &kubeapi.ProjectedVolumeSource{
Sources: []kubeapi.VolumeProjection{{
Secret: &kubeapi.SecretProjection{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "cluster-secrets-vsphere",
}, {
Name: "cluster-profile-vsphere",
VolumeSource: kubeapi.VolumeSource{
Projected: &kubeapi.ProjectedVolumeSource{
Sources: []kubeapi.VolumeProjection{{
Secret: &kubeapi.SecretProjection{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "cluster-secrets-vsphere",
},
},
},
}},
}},
},
},
},
}, {
Name: "job-definition",
VolumeSource: kubeapi.VolumeSource{
Projected: &kubeapi.ProjectedVolumeSource{
Sources: []kubeapi.VolumeProjection{{
ConfigMap: &kubeapi.ConfigMapProjection{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "prow-job-cluster-launch-installer-e2e",
}, {
Name: "job-definition",
VolumeSource: kubeapi.VolumeSource{
Projected: &kubeapi.ProjectedVolumeSource{
Sources: []kubeapi.VolumeProjection{{
ConfigMap: &kubeapi.ConfigMapProjection{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "prow-job-cluster-launch-installer-" + template,
},
},
},
}, {
ConfigMap: &kubeapi.ConfigMapProjection{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "prow-job-cluster-launch-installer-upi-e2e",
}, {
ConfigMap: &kubeapi.ConfigMapProjection{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "prow-job-cluster-launch-installer-upi-" + template,
},
},
},
}},
}},
},
},
},
}, {
Name: "e2e-targets",
VolumeSource: kubeapi.VolumeSource{
ConfigMap: &kubeapi.ConfigMapVolumeSource{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "e2e-targets",
}, {
Name: "e2e-targets",
VolumeSource: kubeapi.VolumeSource{
ConfigMap: &kubeapi.ConfigMapVolumeSource{
LocalObjectReference: kubeapi.LocalObjectReference{
Name: "e2e-targets",
},
},
},
},
}},
}},
}
}
podSpec := generatePodSpecRandom(&info, &ciop.TestStepConfiguration{
As: "e2e",
Commands: "commands",
OpenshiftInstallerRandomClusterTestConfiguration: &ciop.OpenshiftInstallerRandomClusterTestConfiguration{},
})
if expected := makePodSpec("e2e"); !equality.Semantic.DeepEqual(expected, *podSpec) {
t.Fatal(diff.ObjectDiff(expected, podSpec))
}
podSpec = generatePodSpecRandom(&info, &ciop.TestStepConfiguration{
As: "e2e",
Commands: "commands",
OpenshiftInstallerSrcRandomClusterTestConfiguration: &ciop.OpenshiftInstallerSrcRandomClusterTestConfiguration{},
})
expected := makePodSpec("src")
p := &expected.Volumes[4].VolumeSource
p.ConfigMap = &kubeapi.ConfigMapVolumeSource{
LocalObjectReference: p.Projected.Sources[0].ConfigMap.LocalObjectReference,
}
podSpec := generatePodSpecRandom(&info, &test)
if !equality.Semantic.DeepEqual(expected, podSpec) {
p.Projected = nil
if !equality.Semantic.DeepEqual(expected, *podSpec) {
t.Fatal(diff.ObjectDiff(expected, podSpec))
}
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/api/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,9 @@ func validateTestConfigurationType(fieldRoot string, test TestStepConfiguration,
if test.OpenshiftInstallerRandomClusterTestConfiguration != nil {
typeCount++
}
if test.OpenshiftInstallerSrcRandomClusterTestConfiguration != nil {
typeCount++
}
if typeCount == 0 {
validationErrors = append(validationErrors, fmt.Errorf("%s has no type, you may want to specify 'container' for a container based test", fieldRoot))
} else if typeCount == 1 {
Expand Down
28 changes: 17 additions & 11 deletions pkg/api/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -293,17 +293,18 @@ type TestStepConfiguration struct {
Secret *Secret `json:"secret,omitempty"`

// Only one of the following can be not-null.
ContainerTestConfiguration *ContainerTestConfiguration `json:"container,omitempty"`
OpenshiftAnsibleClusterTestConfiguration *OpenshiftAnsibleClusterTestConfiguration `json:"openshift_ansible,omitempty"`
OpenshiftAnsibleSrcClusterTestConfiguration *OpenshiftAnsibleSrcClusterTestConfiguration `json:"openshift_ansible_src,omitempty"`
OpenshiftAnsibleCustomClusterTestConfiguration *OpenshiftAnsibleCustomClusterTestConfiguration `json:"openshift_ansible_custom,omitempty"`
OpenshiftAnsible40ClusterTestConfiguration *OpenshiftAnsible40ClusterTestConfiguration `json:"openshift_ansible_40,omitempty"`
OpenshiftAnsibleUpgradeClusterTestConfiguration *OpenshiftAnsibleUpgradeClusterTestConfiguration `json:"openshift_ansible_upgrade,omitempty"`
OpenshiftInstallerClusterTestConfiguration *OpenshiftInstallerClusterTestConfiguration `json:"openshift_installer,omitempty"`
OpenshiftInstallerSrcClusterTestConfiguration *OpenshiftInstallerSrcClusterTestConfiguration `json:"openshift_installer_src,omitempty"`
OpenshiftInstallerUPIClusterTestConfiguration *OpenshiftInstallerUPIClusterTestConfiguration `json:"openshift_installer_upi,omitempty"`
OpenshiftInstallerConsoleClusterTestConfiguration *OpenshiftInstallerConsoleClusterTestConfiguration `json:"openshift_installer_console,omitempty"`
OpenshiftInstallerRandomClusterTestConfiguration *OpenshiftInstallerRandomClusterTestConfiguration `json:"openshift_installer_random,omitempty"`
ContainerTestConfiguration *ContainerTestConfiguration `json:"container,omitempty"`
OpenshiftAnsibleClusterTestConfiguration *OpenshiftAnsibleClusterTestConfiguration `json:"openshift_ansible,omitempty"`
OpenshiftAnsibleSrcClusterTestConfiguration *OpenshiftAnsibleSrcClusterTestConfiguration `json:"openshift_ansible_src,omitempty"`
OpenshiftAnsibleCustomClusterTestConfiguration *OpenshiftAnsibleCustomClusterTestConfiguration `json:"openshift_ansible_custom,omitempty"`
OpenshiftAnsible40ClusterTestConfiguration *OpenshiftAnsible40ClusterTestConfiguration `json:"openshift_ansible_40,omitempty"`
OpenshiftAnsibleUpgradeClusterTestConfiguration *OpenshiftAnsibleUpgradeClusterTestConfiguration `json:"openshift_ansible_upgrade,omitempty"`
OpenshiftInstallerClusterTestConfiguration *OpenshiftInstallerClusterTestConfiguration `json:"openshift_installer,omitempty"`
OpenshiftInstallerSrcClusterTestConfiguration *OpenshiftInstallerSrcClusterTestConfiguration `json:"openshift_installer_src,omitempty"`
OpenshiftInstallerUPIClusterTestConfiguration *OpenshiftInstallerUPIClusterTestConfiguration `json:"openshift_installer_upi,omitempty"`
OpenshiftInstallerConsoleClusterTestConfiguration *OpenshiftInstallerConsoleClusterTestConfiguration `json:"openshift_installer_console,omitempty"`
OpenshiftInstallerRandomClusterTestConfiguration *OpenshiftInstallerRandomClusterTestConfiguration `json:"openshift_installer_random,omitempty"`
OpenshiftInstallerSrcRandomClusterTestConfiguration *OpenshiftInstallerSrcRandomClusterTestConfiguration `json:"openshift_installer_src_random,omitempty"`
}

// Secret describes a secret to be mounted inside a test
Expand Down Expand Up @@ -440,6 +441,11 @@ type OpenshiftInstallerUPIClusterTestConfiguration struct {
// chosen randomly and runs conformance tests.
type OpenshiftInstallerRandomClusterTestConfiguration struct{}

// OpenshiftInstallerSrcRandomClusterTestConfiguration describes a
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

// ... describes a <SOMETHING MISSING>
// that provisions ...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, someone actually reads those? =p

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A fluke :D

// that provisions a cluster using openshift-installer in a provider
// chosen randomly and executes a command in the `src` image.
type OpenshiftInstallerSrcRandomClusterTestConfiguration struct{}

// PipelineImageStreamTagReference is a tag on the
// ImageStream corresponding to the code under test.
// This tag will identify an image but not use any
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,6 @@ tests:
- as: e2e-random
commands: make e2e
openshift_installer_random: {}
- as: e2e-random-src
commands: make e2e
openshift_installer_src_random: {}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know this only follows what was done for the first template, but I only noticed now. If we intend ci-operator files to be usable (readable/writable) by humans, we should avoid non-intuitive clutter like this, even for a cost of slightly more complicated implementation. How hard would it be to make it:

- as: ...
  openshift_installer_src:
    cluster_profile: random

?

It's probaby not a matter of this PR, but I think we're still early enough in the IaaS-agnostic job work to make that change.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That makes sense, I didn't really think about it too long before deciding on the name. How about omitting the cluster profile altogether? I don't really like random and I think "not tied to a specific target" is a good default. This way we can remove random from the job type as well.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I liked what you proposed at first (even wrote a comment in that direction), but after more thinking I think we should keep the explicit config. I think the agnostic behavior could make a good default, but its not intuitive to whoever is trying to read and write these configs. We know every bit and quirk of it, but that's not the case for everyone.

(to be entirely clear, I did not originally like the {} part of the config)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some API Objects have a type field for this purpose when the type-specific config does not exist or is optional

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ignoring the problem of how to structure the configuration (more on that below) I think random being the default value makes the configuration more intuitive, not less.

If the test needs a cluster, use openshift_installer/openshift_installer_src. If it needs a cluster with a specific configuration, add cluster_profile: …. The former should be the default choice for test authors if the test templates do their job and all supported targets are virtually equivalent. The details of what cluster is created should not concern them.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As for the configuration, we have these test types (ignoring ansible, console, and the two random types I am proposing we remove):

  • container
  • openshift_installer
  • openshift_installer_src
  • openshift_installer_upi

container tests must have a from field. openshift_installer* tests must have a cluster_type.

I don't particularly like the empty dictionary, but there are examples even in Kubernetes (e.g.). Adding an extra field to all types seems silly in this case: unless we default the value if one of the pointers is set, which doesn't seem very ellegant either, we would end up with:

  type: container
  container:
    #

Additionally, if we ever add a field to the cluster types (openshift_installer has an optional upgrade field), that becomes entirely redundant.

Copy link
Member

@petr-muller petr-muller Jul 12, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think random being the default value makes the configuration more intuitive, not less.

I'm trying to imagine this from the PoV of a generic OpenShift engineer who does not know ci-operator configs very well. Currently the config is telling them quite well what they have set up, "smart" defaults defeat that.

The details of what cluster is created should not concern them: This IMHO makes sense only from high-level, not from individual's PoV.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not really, the point of k8s and openshift in general is to be portable cross-cloud and not worry at all, so the default idea that it matters not which infra is underneath is a good one.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good discussion. I'm not entirely convinced, but I do not need to be. I'm OK with making it that way.

Loading