Skip to content

WIP: Add VSphere IPI to e2e template. #7470

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
patrickdillon wants to merge 1 commit into from
Closed

WIP: Add VSphere IPI to e2e template. #7470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion ci-operator/config/openshift/installer/openshift-installer-master.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,11 @@ tests:
openshift_installer:
cluster_profile: gcp
upgrade: true
- as: e2e-vsphere
- as: e2e-vsphere-upi
commands: TEST_SUITE=openshift/conformance/parallel run-tests
openshift_installer_upi:
cluster_profile: vsphere
- as: e2e-vsphere-ipi
commands: TEST_SUITE=openshift/conformance/parallel run-tests
openshift_installer_upi:
cluster_profile: vsphere
Expand Down
121 changes: 109 additions & 12 deletions ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2263,19 +2263,116 @@ presubmits:
secretName: sentry-dsn
trigger: (?m)^/test( | .* )e2e-steps,?($|\s.*)
- agent: kubernetes
always_run: false
always_run: true
branches:
- master
context: ci/prow/e2e-vsphere
context: ci/prow/e2e-vsphere-ipi
decorate: true
decoration_config:
skip_cloning: true
labels:
ci-operator.openshift.io/prowgen-controlled: "true"
pj-rehearse.openshift.io/can-be-rehearsed: "true"
name: pull-ci-openshift-installer-master-e2e-vsphere
optional: true
rerun_command: /test e2e-vsphere
name: pull-ci-openshift-installer-master-e2e-vsphere-ipi
rerun_command: /test e2e-vsphere-ipi
spec:
containers:
- args:
- --artifact-dir=$(ARTIFACTS)
- --branch=master
- --give-pr-author-access-to-namespace=true
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
- --kubeconfig=/etc/apici/kubeconfig
- --lease-server-password-file=/etc/boskos/password
- --lease-server-username=ci
- --lease-server=https://boskos-ci.svc.ci.openshift.org
- --org=openshift
- --repo=installer
- --resolver-address=http://ci-operator-configresolver-ci.svc.ci.openshift.org
- --secret-dir=/usr/local/e2e-vsphere-ipi-cluster-profile
- --sentry-dsn-path=/etc/sentry-dsn/ci-operator
- --target=e2e-vsphere-ipi
- --template=/usr/local/e2e-vsphere-ipi
command:
- ci-operator
env:
- name: CLUSTER_TYPE
value: vsphere
- name: CONFIG_SPEC
valueFrom:
configMapKeyRef:
key: openshift-installer-master.yaml
name: ci-operator-master-configs
- name: JOB_NAME_SAFE
value: e2e-vsphere-ipi
- name: TEST_COMMAND
value: TEST_SUITE=openshift/conformance/parallel run-tests
image: ci-operator:latest
imagePullPolicy: Always
name: ""
resources:
requests:
cpu: 10m
volumeMounts:
- mountPath: /etc/apici
name: apici-ci-operator-credentials
readOnly: true
- mountPath: /etc/boskos
name: boskos
readOnly: true
- mountPath: /usr/local/e2e-vsphere-ipi-cluster-profile
name: cluster-profile
- mountPath: /usr/local/e2e-vsphere-ipi
name: job-definition
subPath: cluster-launch-installer-upi-e2e.yaml
- mountPath: /etc/pull-secret
name: pull-secret
readOnly: true
- mountPath: /etc/sentry-dsn
name: sentry-dsn
readOnly: true
serviceAccountName: ci-operator
volumes:
- name: apici-ci-operator-credentials
secret:
items:
- key: sa.ci-operator.apici.config
path: kubeconfig
secretName: apici-ci-operator-credentials
- name: boskos
secret:
items:
- key: password
path: password
secretName: boskos-credentials
- name: cluster-profile
projected:
sources:
- secret:
name: cluster-secrets-vsphere
- configMap:
name: prow-job-cluster-launch-installer-upi-e2e
name: job-definition
- name: pull-secret
secret:
secretName: regcred
- name: sentry-dsn
secret:
secretName: sentry-dsn
trigger: (?m)^/test( | .* )e2e-vsphere-ipi,?($|\s.*)
- agent: kubernetes
always_run: true
branches:
- master
context: ci/prow/e2e-vsphere-upi
decorate: true
decoration_config:
skip_cloning: true
labels:
ci-operator.openshift.io/prowgen-controlled: "true"
pj-rehearse.openshift.io/can-be-rehearsed: "true"
name: pull-ci-openshift-installer-master-e2e-vsphere-upi
rerun_command: /test e2e-vsphere-upi
spec:
containers:
- args:
Expand All @@ -2290,10 +2387,10 @@ presubmits:
- --org=openshift
- --repo=installer
- --resolver-address=http://ci-operator-configresolver-ci.svc.ci.openshift.org
- --secret-dir=/usr/local/e2e-vsphere-cluster-profile
- --secret-dir=/usr/local/e2e-vsphere-upi-cluster-profile
- --sentry-dsn-path=/etc/sentry-dsn/ci-operator
- --target=e2e-vsphere
- --template=/usr/local/e2e-vsphere
- --target=e2e-vsphere-upi
- --template=/usr/local/e2e-vsphere-upi
command:
- ci-operator
env:
Expand All @@ -2305,7 +2402,7 @@ presubmits:
key: openshift-installer-master.yaml
name: ci-operator-master-configs
- name: JOB_NAME_SAFE
value: e2e-vsphere
value: e2e-vsphere-upi
- name: TEST_COMMAND
value: TEST_SUITE=openshift/conformance/parallel run-tests
image: ci-operator:latest
Expand All @@ -2321,9 +2418,9 @@ presubmits:
- mountPath: /etc/boskos
name: boskos
readOnly: true
- mountPath: /usr/local/e2e-vsphere-cluster-profile
- mountPath: /usr/local/e2e-vsphere-upi-cluster-profile
name: cluster-profile
- mountPath: /usr/local/e2e-vsphere
- mountPath: /usr/local/e2e-vsphere-upi
name: job-definition
subPath: cluster-launch-installer-upi-e2e.yaml
- mountPath: /etc/pull-secret
Expand Down Expand Up @@ -2360,7 +2457,7 @@ presubmits:
- name: sentry-dsn
secret:
secretName: sentry-dsn
trigger: (?m)^/test( | .* )e2e-vsphere,?($|\s.*)
trigger: (?m)^/test( | .* )e2e-vsphere-upi,?($|\s.*)
- agent: kubernetes
always_run: true
branches:
Expand Down
74 changes: 73 additions & 1 deletion ci-operator/templates/openshift/installer/cluster-launch-installer-upi-e2e.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1133,6 +1133,78 @@ objects:
rm -f /tmp/artifacts/installer/openshift/99_openshift-cluster-api_master-machines-*.yaml
rm -f /tmp/artifacts/installer/openshift/99_openshift-cluster-api_worker-machineset-*.yaml
sed -i "s;mastersSchedulable: true;mastersSchedulable: false;g" /tmp/artifacts/installer/manifests/cluster-scheduler-02-config.yml
#DEBUG TODO Come up with better conditional
#elif [[ "${CLUSTER_TYPE}" == "vsphere" ]]; then
elif [[ "${JOB_NAME_SAFE}" == "e2e-vsphere-ipi" ]]; then
echo "DEBUG REMOVE ME installing vsphere cluster"
base_domain="${BASE_DOMAIN:-origin-ci-int-aws.dev.rhcloud.com}"
echo "DEBUG REMOVE ME base_domain ${base_domain}"
vsphere_user=$(grep -oP 'vsphere_user="\K[^"]+' ${TFVARS_PATH})
echo "DEBUG REMOVE ME vsphere_user ${vsphere_user}"
vsphere_password=$(grep -oP 'vsphere_password="\K[^"]+' ${TFVARS_PATH})
echo "DEBUG REMOVE ME vsphere_password"
#ipam=$(grep -oP 'ipam="\K[^"]+' ${TFVARS_PATH})
ipam="139.178.89.254"
echo "DEBUG REMOVE ME ipam ${ipam}"
#ipam_token=$(grep -oP 'ipam_token="\K[^"]+' ${TFVARS_PATH})
ipam_token="test"
echo "DEBUG REMOVE ME ipam_token"

# FIXME: get epel-release or otherwise add awscli to our UPI image
export PATH="${HOME}/.local/bin:${PATH}"
easy_install --user pip # our Python 2.7.5 is even too old for ensurepip
pip install --user awscli
export AWS_DEFAULT_REGION="${AWS_REGION}" # CLI prefers the former

declare -a vips
echo "DEBUG REMOVE ME about to reserve ip addresses"
for i in {0..2}
do
echo "DEBUG REMOVE ME reserving IP addresses"
args="{"network":"139.178.87.128","hostname":"$CLUSTER_NAME-$i","ipam":"$ipam","ipam_token":"$ipam_token"}"
vips[$i]=$(echo $args | bash <(curl -s https://raw.githubusercontent.com/openshift/installer/master/upi/vsphere/machine/cidr_to_ip.sh))
if [[ -z ${vips[$i]} ]]; then
echo "error: Unable to reserve virtual IP address, exiting" 1>&2
exit 1
fi
done

api_vip=${vips[0]}
ingress_vip=${vips[1]}
dns_vip=${vips[2]}

echo "DEBUG REMOVE ME writing install config"
cat > /tmp/artifacts/installer/install-config.yaml << EOF
apiVersion: v1
baseDomain: ${base_domain}
metadata:
name: ${CLUSTER_NAME}
controlPlane:
name: master
replicas: 3
compute:
- name: worker
replicas: ${workers}
networking:
machineCIDR: 139.178.87.128/25
platform:
vsphere:
vCenter: vcsa-ci.vmware.devcluster.openshift.com
username: ${vsphere_user}
password: ${vsphere_password}
cluster: devel
datacenter: dc1
defaultDatastore: nvme-ds1
network: VM Network
apiVIP: ${api_vip}
ingressVIP: ${ingress_vip}
dnsVIP: ${dns_vip}
pullSecret: >
${PULL_SECRET}
sshKey: |
${SSH_PUB_KEY}
EOF
echo "DEBUG REMOVE ME created install config"
elif [[ "${CLUSTER_TYPE}" == "vsphere" ]]; then
base_domain="${BASE_DOMAIN:-origin-ci-int-aws.dev.rhcloud.com}"
# Get user and password from TFVARS_PATH
Expand Down Expand Up @@ -1244,7 +1316,7 @@ objects:
sed -i "/publicZone/,+1d" manifests/cluster-dns-02-config.yml
sed -i "/privateZone/,+1d" manifests/cluster-dns-02-config.yml

popd
popd
else
echo "Unsupported cluster type '${CLUSTER_TYPE}'"
exit 1
Expand Down