diff --git a/ci-operator/config/cri-o/cri-o/cri-o-cri-o-main__ci.yaml b/ci-operator/config/cri-o/cri-o/cri-o-cri-o-main__ci.yaml index 47f6c5ac666a1..9095880da23ec 100644 --- a/ci-operator/config/cri-o/cri-o/cri-o-cri-o-main__ci.yaml +++ b/ci-operator/config/cri-o/cri-o/cri-o-cri-o-main__ci.yaml @@ -3,6 +3,11 @@ base_images: name: test namespace: ocp-kni tag: dev-scripts + openshift_release_golang-1.18: + name: release + namespace: openshift + tag: golang-1.18 +binary_build_commands: "echo \"no build commands\" \n" build_root: image_stream_tag: name: release @@ -10,7 +15,14 @@ build_root: tag: golang-1.18 images: - dockerfile_literal: | - FROM base + FROM registry.fedoraproject.org/fedora:36 + RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz + RUN mkdir -p /usr/local/gcloud \ + && tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \ + && /usr/local/gcloud/google-cloud-sdk/install.sh \ + && rm -f /tmp/google-cloud-sdk.tar.gz + ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin + WORKDIR /src COPY . . from: src to: crio-crio-base-src @@ -29,19 +41,19 @@ resources: requests: cpu: 100m memory: 200Mi -test_binary_build_commands: "echo \"asdf\" \n" +test_binary_build_commands: "echo \"no test commands\" \n" tests: - as: e2e steps: - cluster_profile: equinix-ocp-metal + cluster_profile: gcp workflow: cri-o-e2e - as: integration steps: - cluster_profile: equinix-ocp-metal + cluster_profile: gcp workflow: cri-o-integration - as: critest steps: - cluster_profile: equinix-ocp-metal + cluster_profile: gcp workflow: cri-o-critest zz_generated_metadata: branch: main diff --git a/ci-operator/jobs/cri-o/cri-o/cri-o-cri-o-main-presubmits.yaml b/ci-operator/jobs/cri-o/cri-o/cri-o-cri-o-main-presubmits.yaml index 637a546a82089..da9148f41feed 100644 --- a/ci-operator/jobs/cri-o/cri-o/cri-o-cri-o-main-presubmits.yaml +++ b/ci-operator/jobs/cri-o/cri-o/cri-o-cri-o-main-presubmits.yaml @@ -21,14 +21,14 @@ presubmits: branches: - ^main$ - ^main- - cluster: build04 + cluster: build02 context: ci/prow/ci-critest decorate: true decoration_config: skip_cloning: true labels: - ci-operator.openshift.io/cloud: equinix-ocp-metal - ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-metal + ci-operator.openshift.io/cloud: gcp + ci-operator.openshift.io/cloud-cluster-profile: gcp ci-operator.openshift.io/variant: ci ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -83,8 +83,12 @@ presubmits: secret: secretName: ci-pull-credentials - name: cluster-profile - secret: - secretName: cluster-secrets-equinix-ocp-metal + projected: + sources: + - secret: + name: cluster-secrets-gcp + - configMap: + name: cluster-profile-gcp - name: pull-secret secret: secretName: registry-pull-credentials @@ -97,14 +101,14 @@ presubmits: branches: - ^main$ - ^main- - cluster: build04 + cluster: build02 context: ci/prow/ci-e2e decorate: true decoration_config: skip_cloning: true labels: - ci-operator.openshift.io/cloud: equinix-ocp-metal - ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-metal + ci-operator.openshift.io/cloud: gcp + ci-operator.openshift.io/cloud-cluster-profile: gcp ci-operator.openshift.io/variant: ci ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -159,8 +163,12 @@ presubmits: secret: secretName: ci-pull-credentials - name: cluster-profile - secret: - secretName: cluster-secrets-equinix-ocp-metal + projected: + sources: + - secret: + name: cluster-secrets-gcp + - configMap: + name: cluster-profile-gcp - name: pull-secret secret: secretName: registry-pull-credentials @@ -224,14 +232,14 @@ presubmits: branches: - ^main$ - ^main- - cluster: build04 + cluster: build02 context: ci/prow/ci-integration decorate: true decoration_config: skip_cloning: true labels: - ci-operator.openshift.io/cloud: equinix-ocp-metal - ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-metal + ci-operator.openshift.io/cloud: gcp + ci-operator.openshift.io/cloud-cluster-profile: gcp ci-operator.openshift.io/variant: ci ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -286,8 +294,12 @@ presubmits: secret: secretName: ci-pull-credentials - name: cluster-profile - secret: - secretName: cluster-secrets-equinix-ocp-metal + projected: + sources: + - secret: + name: cluster-secrets-gcp + - configMap: + name: cluster-profile-gcp - name: pull-secret secret: secretName: registry-pull-credentials diff --git a/ci-operator/step-registry/cri-o/critest/cri-o-critest-workflow.yaml b/ci-operator/step-registry/cri-o/critest/cri-o-critest-workflow.yaml index fc0f34b803e74..a4c7c0b914b83 100644 --- a/ci-operator/step-registry/cri-o/critest/cri-o-critest-workflow.yaml +++ b/ci-operator/step-registry/cri-o/critest/cri-o-critest-workflow.yaml @@ -1,16 +1,17 @@ workflow: as: cri-o-critest steps: - cluster_profile: equinix-ocp-metal + cluster_profile: gcp pre: - - ref: baremetalds-packet-setup + - ref: gcp-provision-vpc + - ref: gcp-provision-buildhost test: - ref: cri-o-critest-test post: - ref: cri-o-critest-gather - - ref: baremetalds-packet-teardown + - ref: gcp-deprovision-buildhost env: - PACKET_PLAN: "c3.small.x86" - PACKET_OS: "rocky_8" + IMAGE_PROJECT: "rocky-linux-cloud" + IMAGE_FAMILY: "rocky-linux-8" documentation: |- - This workflow executes the subsystem test suite on a cluster provisioned by running assisted-installer-crio-test on a packet server. + This workflow executes the subsystem test suite on a cluster provisioned by running assisted-installer-crio-test on a GCP server. diff --git a/ci-operator/step-registry/cri-o/critest/gather/cri-o-critest-gather-commands.sh b/ci-operator/step-registry/cri-o/critest/gather/cri-o-critest-gather-commands.sh index 25dbd3f643c30..89f5c2bbc3f71 100644 --- a/ci-operator/step-registry/cri-o/critest/gather/cri-o-critest-gather-commands.sh +++ b/ci-operator/step-registry/cri-o/critest/gather/cri-o-critest-gather-commands.sh @@ -2,15 +2,39 @@ set -o nounset set -o errexit set -o pipefail -set -x echo "gathering logs" # shellcheck source=/dev/null -source "${SHARED_DIR}/packet-conf.sh" +source "${SHARED_DIR}/env" + +##################################### +###############Log In################ +##################################### + +GOOGLE_PROJECT_ID="$(< ${CLUSTER_PROFILE_DIR}/openshift_gcp_project)" +export GCP_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/gce.json" +sa_email=$(jq -r .client_email ${GCP_SHARED_CREDENTIALS_FILE}) +if ! gcloud auth list | grep -E "\*\s+${sa_email}" +then + gcloud auth activate-service-account --key-file="${GCP_SHARED_CREDENTIALS_FILE}" + gcloud config set project "${GOOGLE_PROJECT_ID}" +fi + +mkdir -p "${HOME}"/.ssh +chmod 0700 "${HOME}"/.ssh + +cp "${CLUSTER_PROFILE_DIR}"/ssh-privatekey "${HOME}"/.ssh/google_compute_engine +chmod 0600 "${HOME}"/.ssh/google_compute_engine +cp "${CLUSTER_PROFILE_DIR}"/ssh-publickey "${HOME}"/.ssh/google_compute_engine.pub + +##################################### +##################################### + +instance_name=$(<"${SHARED_DIR}/gcp-instance-ids.txt") function getlogs() { echo "### Downloading logs..." - scp -r "${SSHOPTS[@]}" "root@${IP}:/tmp/artifacts/*" "${ARTIFACT_DIR}" + gcloud compute scp --recurse --zone "${ZONE}" "${instance_name}:/tmp/artifacts/*" "${ARTIFACT_DIR}" } # Gather logs regardless of what happens after this diff --git a/ci-operator/step-registry/cri-o/critest/test/cri-o-critest-test-commands.sh b/ci-operator/step-registry/cri-o/critest/test/cri-o-critest-test-commands.sh index e2e7ee754be64..0ab6ae88669dc 100644 --- a/ci-operator/step-registry/cri-o/critest/test/cri-o-critest-test-commands.sh +++ b/ci-operator/step-registry/cri-o/critest/test/cri-o-critest-test-commands.sh @@ -2,40 +2,62 @@ set -o nounset set -o errexit set -o pipefail -set -x # shellcheck source=/dev/null -source "${SHARED_DIR}/packet-conf.sh" -echo "${IP}" -echo "${SSHOPTS[@]}" - -tar -czf - . | ssh "${SSHOPTS[@]}" "root@${IP}" "cat > /root/cri-o.tar.gz" -timeout --kill-after 10m 120m ssh "${SSHOPTS[@]}" "root@${IP}" bash - << EOF - export HOME=/root - mkdir /tmp/artifacts - mkdir /logs - mkdir /logs/artifacts - mkdir /tmp/artifacts/logs - - dnf install python39 -y +source "${SHARED_DIR}/env" + +##################################### +###############Log In################ +##################################### + +GOOGLE_PROJECT_ID="$(< ${CLUSTER_PROFILE_DIR}/openshift_gcp_project)" +export GCP_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/gce.json" +sa_email=$(jq -r .client_email ${GCP_SHARED_CREDENTIALS_FILE}) +if ! gcloud auth list | grep -E "\*\s+${sa_email}" +then + gcloud auth activate-service-account --key-file="${GCP_SHARED_CREDENTIALS_FILE}" + gcloud config set project "${GOOGLE_PROJECT_ID}" +fi + +mkdir -p "${HOME}"/.ssh +chmod 0700 "${HOME}"/.ssh + +cp "${CLUSTER_PROFILE_DIR}"/ssh-privatekey "${HOME}"/.ssh/google_compute_engine +chmod 0600 "${HOME}"/.ssh/google_compute_engine +cp "${CLUSTER_PROFILE_DIR}"/ssh-publickey "${HOME}"/.ssh/google_compute_engine.pub + +latest="v3.5.4" +if gcloud alpha storage ls gs://crio-ci | grep -q ${latest} ; then + echo "etcd is up to date" +else + echo "caching etcd" + curl https://github.com/coreos/etcd/releases/download/${latest}/etcd-${latest}-linux-amd64.tar.gz -L | gsutil cp - gs://crio-ci/etcd-${latest}.tar.gz +fi + +##################################### +##################################### + +instance_name=$(<"${SHARED_DIR}/gcp-instance-ids.txt") + +tar -czf - . | gcloud compute ssh --zone="${ZONE}" ${instance_name} -- "cat > \${HOME}/cri-o.tar.gz" +timeout --kill-after 10m 400m gcloud compute ssh --zone="${ZONE}" ${instance_name} -- bash - << EOF + export GOROOT=/usr/local/go + echo GOROOT="/usr/local/go" | sudo tee -a /etc/environment + mkdir -p \${HOME}/logs/artifacts + mkdir -p /tmp/artifacts/logs + + sudo dnf install python39 -y curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py python3.9 get-pip.py python3.9 -m pip install ansible # setup the directory where the tests will the run - REPO_DIR="/root/cri-o" + REPO_DIR="/home/deadbeef/cri-o" mkdir -p "\${REPO_DIR}" - # NVMe makes it faster - NVME_DEVICE="/dev/nvme0n1" - if [ -e "\$NVME_DEVICE" ]; - then - mkfs.xfs -f "\${NVME_DEVICE}" - mount "\${NVME_DEVICE}" "\${REPO_DIR}" - fi # copy the agent sources on the remote machine tar -xzvf cri-o.tar.gz -C "\${REPO_DIR}" - chown -R root:root "\${REPO_DIR}" cd "\${REPO_DIR}/contrib/test/ci" echo "localhost" >> hosts ansible-playbook critest-main.yml -i hosts -e "TEST_AGENT=prow" --connection=local -vvv EOF + diff --git a/ci-operator/step-registry/cri-o/e2e/cri-o-e2e-workflow.yaml b/ci-operator/step-registry/cri-o/e2e/cri-o-e2e-workflow.yaml index 9fe94dbd527ed..3a22a88ef9b82 100644 --- a/ci-operator/step-registry/cri-o/e2e/cri-o-e2e-workflow.yaml +++ b/ci-operator/step-registry/cri-o/e2e/cri-o-e2e-workflow.yaml @@ -1,16 +1,17 @@ workflow: as: cri-o-e2e steps: - cluster_profile: equinix-ocp-metal + cluster_profile: gcp pre: - - ref: baremetalds-packet-setup + - ref: gcp-provision-vpc + - ref: gcp-provision-buildhost test: - ref: cri-o-e2e-test post: - ref: cri-o-e2e-gather - - ref: baremetalds-packet-teardown + - ref: gcp-deprovision-buildhost env: - PACKET_PLAN: "c3.small.x86" - PACKET_OS: "rocky_8" + IMAGE_PROJECT: "rocky-linux-cloud" + IMAGE_FAMILY: "rocky-linux-8" documentation: |- - This workflow executes the subsystem test suite on a cluster provisioned by running assisted-installer-crio-test on a packet server. + This workflow executes the subsystem test suite on a cluster provisioned by running assisted-installer-crio-test on a GCP server. diff --git a/ci-operator/step-registry/cri-o/e2e/gather/cri-o-e2e-gather-commands.sh b/ci-operator/step-registry/cri-o/e2e/gather/cri-o-e2e-gather-commands.sh index 25dbd3f643c30..f05f33b36ffde 100644 --- a/ci-operator/step-registry/cri-o/e2e/gather/cri-o-e2e-gather-commands.sh +++ b/ci-operator/step-registry/cri-o/e2e/gather/cri-o-e2e-gather-commands.sh @@ -2,15 +2,39 @@ set -o nounset set -o errexit set -o pipefail -set -x echo "gathering logs" # shellcheck source=/dev/null -source "${SHARED_DIR}/packet-conf.sh" +source "${SHARED_DIR}/env" + +##################################### +###############Log In################ +##################################### + +GOOGLE_PROJECT_ID="$(< ${CLUSTER_PROFILE_DIR}/openshift_gcp_project)" +export GCP_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/gce.json" +sa_email=$(jq -r .client_email ${GCP_SHARED_CREDENTIALS_FILE}) +if ! gcloud auth list | grep -E "\*\s+${sa_email}" +then + gcloud auth activate-service-account --key-file="${GCP_SHARED_CREDENTIALS_FILE}" + gcloud config set project "${GOOGLE_PROJECT_ID}" +fi + +mkdir -p "${HOME}"/.ssh +chmod 0700 "${HOME}"/.ssh + +cp "${CLUSTER_PROFILE_DIR}"/ssh-privatekey "${HOME}"/.ssh/google_compute_engine +chmod 0600 "${HOME}"/.ssh/google_compute_engine +cp "${CLUSTER_PROFILE_DIR}"/ssh-publickey "${HOME}"/.ssh/google_compute_engine.pub + +##################################### +##################################### + +instance_name=$(<"${SHARED_DIR}/gcp-instance-ids.txt") function getlogs() { echo "### Downloading logs..." - scp -r "${SSHOPTS[@]}" "root@${IP}:/tmp/artifacts/*" "${ARTIFACT_DIR}" + gcloud compute scp --recurse --zone "${ZONE}" --recurse "${instance_name}:/tmp/artifacts/*" "${ARTIFACT_DIR}" } # Gather logs regardless of what happens after this diff --git a/ci-operator/step-registry/cri-o/e2e/test/cri-o-e2e-test-commands.sh b/ci-operator/step-registry/cri-o/e2e/test/cri-o-e2e-test-commands.sh index 421eea566d378..4fe9a528e73bb 100644 --- a/ci-operator/step-registry/cri-o/e2e/test/cri-o-e2e-test-commands.sh +++ b/ci-operator/step-registry/cri-o/e2e/test/cri-o-e2e-test-commands.sh @@ -2,40 +2,62 @@ set -o nounset set -o errexit set -o pipefail -set -x # shellcheck source=/dev/null -source "${SHARED_DIR}/packet-conf.sh" -echo "${IP}" -echo "${SSHOPTS[@]}" - -tar -czf - . | ssh "${SSHOPTS[@]}" "root@${IP}" "cat > /root/cri-o.tar.gz" -timeout --kill-after 10m 400m ssh "${SSHOPTS[@]}" "root@${IP}" bash - << EOF - export HOME=/root - mkdir /tmp/artifacts - mkdir /tmp/artifacts/logs - mkdir /logs - mkdir /logs/artifacts - - dnf install python39 -y +source "${SHARED_DIR}/env" + +##################################### +###############Log In################ +##################################### + +GOOGLE_PROJECT_ID="$(< ${CLUSTER_PROFILE_DIR}/openshift_gcp_project)" +export GCP_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/gce.json" +sa_email=$(jq -r .client_email ${GCP_SHARED_CREDENTIALS_FILE}) +if ! gcloud auth list | grep -E "\*\s+${sa_email}" +then + gcloud auth activate-service-account --key-file="${GCP_SHARED_CREDENTIALS_FILE}" + gcloud config set project "${GOOGLE_PROJECT_ID}" +fi + +mkdir -p "${HOME}"/.ssh +chmod 0700 "${HOME}"/.ssh + +cp "${CLUSTER_PROFILE_DIR}"/ssh-privatekey "${HOME}"/.ssh/google_compute_engine +chmod 0600 "${HOME}"/.ssh/google_compute_engine +cp "${CLUSTER_PROFILE_DIR}"/ssh-publickey "${HOME}"/.ssh/google_compute_engine.pub + +latest="v3.5.4" +if gcloud alpha storage ls gs://crio-ci | grep -q ${latest} ; then + echo "etcd is up to date" +else + echo "caching etcd" + curl https://github.com/coreos/etcd/releases/download/${latest}/etcd-${latest}-linux-amd64.tar.gz -L | gsutil cp - gs://crio-ci/etcd-${latest}.tar.gz +fi + +##################################### +##################################### + +instance_name=$(<"${SHARED_DIR}/gcp-instance-ids.txt") + +tar -czf - . | gcloud compute ssh --zone="${ZONE}" ${instance_name} -- "cat > \${HOME}/cri-o.tar.gz" +timeout --kill-after 10m 400m gcloud compute ssh --zone="${ZONE}" ${instance_name} -- bash - << EOF + export GOROOT=/usr/local/go + echo GOROOT="/usr/local/go" | sudo tee -a /etc/environment + mkdir -p \${HOME}/logs/artifacts + mkdir -p /tmp/artifacts/logs + + sudo dnf install python39 -y curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py python3.9 get-pip.py python3.9 -m pip install ansible # setup the directory where the tests will the run - REPO_DIR="/root/cri-o" + REPO_DIR="/home/deadbeef/cri-o" mkdir -p "\${REPO_DIR}" - # NVMe makes it faster - NVME_DEVICE="/dev/nvme0n1" - if [ -e "\$NVME_DEVICE" ]; - then - mkfs.xfs -f "\${NVME_DEVICE}" - mount "\${NVME_DEVICE}" "\${REPO_DIR}" - fi # copy the agent sources on the remote machine tar -xzvf cri-o.tar.gz -C "\${REPO_DIR}" - chown -R root:root "\${REPO_DIR}" cd "\${REPO_DIR}/contrib/test/ci" echo "localhost" >> hosts ansible-playbook e2e-main.yml -i hosts -e "TEST_AGENT=prow" --connection=local -vvv --tags setup,e2e EOF + diff --git a/ci-operator/step-registry/cri-o/integration/cri-o-integration-workflow.yaml b/ci-operator/step-registry/cri-o/integration/cri-o-integration-workflow.yaml index 85a60b007ce4f..db6746990c2a3 100644 --- a/ci-operator/step-registry/cri-o/integration/cri-o-integration-workflow.yaml +++ b/ci-operator/step-registry/cri-o/integration/cri-o-integration-workflow.yaml @@ -1,16 +1,17 @@ workflow: as: cri-o-integration steps: - cluster_profile: equinix-ocp-metal + cluster_profile: gcp pre: - - ref: baremetalds-packet-setup + - ref: gcp-provision-vpc + - ref: gcp-provision-buildhost test: - ref: cri-o-integration-test post: - ref: cri-o-integration-gather - - ref: baremetalds-packet-teardown + - ref: gcp-deprovision-buildhost env: - PACKET_PLAN: "c3.small.x86" - PACKET_OS: "rocky_8" + IMAGE_PROJECT: "rocky-linux-cloud" + IMAGE_FAMILY: "rocky-linux-8" documentation: |- This workflow executes the subsystem test suite on a cluster provisioned by running assisted-installer-crio-test on a packet server. diff --git a/ci-operator/step-registry/cri-o/integration/gather/cri-o-integration-gather-commands.sh b/ci-operator/step-registry/cri-o/integration/gather/cri-o-integration-gather-commands.sh index 25dbd3f643c30..f05f33b36ffde 100644 --- a/ci-operator/step-registry/cri-o/integration/gather/cri-o-integration-gather-commands.sh +++ b/ci-operator/step-registry/cri-o/integration/gather/cri-o-integration-gather-commands.sh @@ -2,15 +2,39 @@ set -o nounset set -o errexit set -o pipefail -set -x echo "gathering logs" # shellcheck source=/dev/null -source "${SHARED_DIR}/packet-conf.sh" +source "${SHARED_DIR}/env" + +##################################### +###############Log In################ +##################################### + +GOOGLE_PROJECT_ID="$(< ${CLUSTER_PROFILE_DIR}/openshift_gcp_project)" +export GCP_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/gce.json" +sa_email=$(jq -r .client_email ${GCP_SHARED_CREDENTIALS_FILE}) +if ! gcloud auth list | grep -E "\*\s+${sa_email}" +then + gcloud auth activate-service-account --key-file="${GCP_SHARED_CREDENTIALS_FILE}" + gcloud config set project "${GOOGLE_PROJECT_ID}" +fi + +mkdir -p "${HOME}"/.ssh +chmod 0700 "${HOME}"/.ssh + +cp "${CLUSTER_PROFILE_DIR}"/ssh-privatekey "${HOME}"/.ssh/google_compute_engine +chmod 0600 "${HOME}"/.ssh/google_compute_engine +cp "${CLUSTER_PROFILE_DIR}"/ssh-publickey "${HOME}"/.ssh/google_compute_engine.pub + +##################################### +##################################### + +instance_name=$(<"${SHARED_DIR}/gcp-instance-ids.txt") function getlogs() { echo "### Downloading logs..." - scp -r "${SSHOPTS[@]}" "root@${IP}:/tmp/artifacts/*" "${ARTIFACT_DIR}" + gcloud compute scp --recurse --zone "${ZONE}" --recurse "${instance_name}:/tmp/artifacts/*" "${ARTIFACT_DIR}" } # Gather logs regardless of what happens after this diff --git a/ci-operator/step-registry/cri-o/integration/test/cri-o-integration-test-commands.sh b/ci-operator/step-registry/cri-o/integration/test/cri-o-integration-test-commands.sh index f453124385e5c..d5cfb161d857f 100644 --- a/ci-operator/step-registry/cri-o/integration/test/cri-o-integration-test-commands.sh +++ b/ci-operator/step-registry/cri-o/integration/test/cri-o-integration-test-commands.sh @@ -2,39 +2,60 @@ set -o nounset set -o errexit set -o pipefail -set -x # shellcheck source=/dev/null -source "${SHARED_DIR}/packet-conf.sh" -echo "${IP}" -echo "${SSHOPTS[@]}" - -tar -czf - . | ssh "${SSHOPTS[@]}" "root@${IP}" "cat > /root/cri-o.tar.gz" -timeout --kill-after 10m 120m ssh "${SSHOPTS[@]}" "root@${IP}" bash - << EOF - export HOME=/root - mkdir /tmp/artifacts - mkdir /logs - mkdir /logs/artifacts - mkdir /tmp/artifacts/logs - - dnf install python39 -y +source "${SHARED_DIR}/env" + +##################################### +###############Log In################ +##################################### + +GOOGLE_PROJECT_ID="$(< ${CLUSTER_PROFILE_DIR}/openshift_gcp_project)" +export GCP_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/gce.json" +sa_email=$(jq -r .client_email ${GCP_SHARED_CREDENTIALS_FILE}) +if ! gcloud auth list | grep -E "\*\s+${sa_email}" +then + gcloud auth activate-service-account --key-file="${GCP_SHARED_CREDENTIALS_FILE}" + gcloud config set project "${GOOGLE_PROJECT_ID}" +fi + +mkdir -p "${HOME}"/.ssh +chmod 0700 "${HOME}"/.ssh + +cp "${CLUSTER_PROFILE_DIR}"/ssh-privatekey "${HOME}"/.ssh/google_compute_engine +chmod 0600 "${HOME}"/.ssh/google_compute_engine +cp "${CLUSTER_PROFILE_DIR}"/ssh-publickey "${HOME}"/.ssh/google_compute_engine.pub + +latest="v3.5.4" +if gcloud alpha storage ls gs://crio-ci | grep -q ${latest} ; then + echo "etcd is up to date" +else + echo "caching etcd" + curl https://github.com/coreos/etcd/releases/download/${latest}/etcd-${latest}-linux-amd64.tar.gz -L | gsutil cp - gs://crio-ci/etcd-${latest}.tar.gz +fi + +##################################### +##################################### + +instance_name=$(<"${SHARED_DIR}/gcp-instance-ids.txt") + +tar -czf - . | gcloud compute ssh --zone="${ZONE}" ${instance_name} -- "cat > \${HOME}/cri-o.tar.gz" +timeout --kill-after 10m 400m gcloud compute ssh --zone="${ZONE}" ${instance_name} -- bash - << EOF + export GOROOT=/usr/local/go + echo GOROOT="/usr/local/go" | sudo tee -a /etc/environment + mkdir -p \${HOME}/logs/artifacts + mkdir -p /tmp/artifacts/logs + + sudo dnf install python39 -y curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py python3.9 get-pip.py python3.9 -m pip install ansible # setup the directory where the tests will the run - REPO_DIR="/root/cri-o" + REPO_DIR="/home/deadbeef/cri-o" mkdir -p "\${REPO_DIR}" - # NVMe makes it faster - NVME_DEVICE="/dev/nvme0n1" - if [ -e "\$NVME_DEVICE" ]; - then - mkfs.xfs -f "\${NVME_DEVICE}" - mount "\${NVME_DEVICE}" "\${REPO_DIR}" - fi # copy the agent sources on the remote machine tar -xzvf cri-o.tar.gz -C "\${REPO_DIR}" - chown -R root:root "\${REPO_DIR}" cd "\${REPO_DIR}/contrib/test/ci" echo "localhost" >> hosts ansible-playbook integration-main.yml -i hosts -e "TEST_AGENT=prow" --connection=local -vvv diff --git a/ci-operator/step-registry/gcp/OWNERS b/ci-operator/step-registry/gcp/OWNERS index e079a72508195..c742af96b2eee 100644 --- a/ci-operator/step-registry/gcp/OWNERS +++ b/ci-operator/step-registry/gcp/OWNERS @@ -2,3 +2,4 @@ approvers: - gpei - jianlinliu - jianli-wei + diff --git a/ci-operator/step-registry/gcp/deprovision/OWNERS b/ci-operator/step-registry/gcp/deprovision/OWNERS deleted file mode 120000 index ec405d65a79df..0000000000000 --- a/ci-operator/step-registry/gcp/deprovision/OWNERS +++ /dev/null @@ -1 +0,0 @@ -../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/gcp/deprovision/OWNERS b/ci-operator/step-registry/gcp/deprovision/OWNERS new file mode 100644 index 0000000000000..edad27cb2851e --- /dev/null +++ b/ci-operator/step-registry/gcp/deprovision/OWNERS @@ -0,0 +1,6 @@ +approvers: +- wgahnagl +- gpei +- jianlinliu +- jianli-wei + diff --git a/ci-operator/step-registry/gcp/deprovision/bastionhost/gcp-deprovision-bastionhost-ref.metadata.json b/ci-operator/step-registry/gcp/deprovision/bastionhost/gcp-deprovision-bastionhost-ref.metadata.json index ebeedd9e72b33..8d4105f0ef617 100644 --- a/ci-operator/step-registry/gcp/deprovision/bastionhost/gcp-deprovision-bastionhost-ref.metadata.json +++ b/ci-operator/step-registry/gcp/deprovision/bastionhost/gcp-deprovision-bastionhost-ref.metadata.json @@ -2,6 +2,7 @@ "path": "gcp/deprovision/bastionhost/gcp-deprovision-bastionhost-ref.yaml", "owners": { "approvers": [ + "wgahnagl", "gpei", "jianlinliu", "jianli-wei" diff --git a/ci-operator/step-registry/gcp/deprovision/buildhost/OWNERS b/ci-operator/step-registry/gcp/deprovision/buildhost/OWNERS new file mode 100644 index 0000000000000..11686975a2765 --- /dev/null +++ b/ci-operator/step-registry/gcp/deprovision/buildhost/OWNERS @@ -0,0 +1,2 @@ +approvers: +- rphillips diff --git a/ci-operator/step-registry/gcp/deprovision/buildhost/gcp-deprovision-buildhost-commands.sh b/ci-operator/step-registry/gcp/deprovision/buildhost/gcp-deprovision-buildhost-commands.sh new file mode 100755 index 0000000000000..f2522befa8b95 --- /dev/null +++ b/ci-operator/step-registry/gcp/deprovision/buildhost/gcp-deprovision-buildhost-commands.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +if [ ! -f "${SHARED_DIR}/destroy.sh" ]; then + echo "No 'destroy.sh' found, aborted." && exit 0 +fi + +GOOGLE_PROJECT_ID="$(< ${CLUSTER_PROFILE_DIR}/openshift_gcp_project)" + +export GCP_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/gce.json" +sa_email=$(jq -r .client_email ${GCP_SHARED_CREDENTIALS_FILE}) +if ! gcloud auth list | grep -E "\*\s+${sa_email}" +then + gcloud auth activate-service-account --key-file="${GCP_SHARED_CREDENTIALS_FILE}" + gcloud config set project "${GOOGLE_PROJECT_ID}" +fi + +## Destroy the SSH bastion +echo "$(date -u --rfc-3339=seconds) - Destroying the server host..." +sh "${SHARED_DIR}/destroy.sh" diff --git a/ci-operator/step-registry/gcp/deprovision/buildhost/gcp-deprovision-buildhost-ref.metadata.json b/ci-operator/step-registry/gcp/deprovision/buildhost/gcp-deprovision-buildhost-ref.metadata.json new file mode 100644 index 0000000000000..e424d22513e31 --- /dev/null +++ b/ci-operator/step-registry/gcp/deprovision/buildhost/gcp-deprovision-buildhost-ref.metadata.json @@ -0,0 +1,8 @@ +{ + "path": "gcp/deprovision/buildhost/gcp-deprovision-buildhost-ref.yaml", + "owners": { + "approvers": [ + "rphillips" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/gcp/deprovision/buildhost/gcp-deprovision-buildhost-ref.yaml b/ci-operator/step-registry/gcp/deprovision/buildhost/gcp-deprovision-buildhost-ref.yaml new file mode 100644 index 0000000000000..0eddf6f0d5ae2 --- /dev/null +++ b/ci-operator/step-registry/gcp/deprovision/buildhost/gcp-deprovision-buildhost-ref.yaml @@ -0,0 +1,13 @@ +ref: + as: gcp-deprovision-buildhost + from_image: + namespace: ocp + name: "4.10" + tag: upi-installer + commands: gcp-deprovision-buildhost-commands.sh + resources: + requests: + cpu: 10m + memory: 100Mi + documentation: |- + The step destroys all resources configured by "gcp-provision-buildhost". \ No newline at end of file diff --git a/ci-operator/step-registry/gcp/deprovision/vpc/gcp-deprovision-vpc-ref.metadata.json b/ci-operator/step-registry/gcp/deprovision/vpc/gcp-deprovision-vpc-ref.metadata.json index b1668666c4418..12ef1e8bad58f 100644 --- a/ci-operator/step-registry/gcp/deprovision/vpc/gcp-deprovision-vpc-ref.metadata.json +++ b/ci-operator/step-registry/gcp/deprovision/vpc/gcp-deprovision-vpc-ref.metadata.json @@ -2,6 +2,7 @@ "path": "gcp/deprovision/vpc/gcp-deprovision-vpc-ref.yaml", "owners": { "approvers": [ + "wgahnagl", "gpei", "jianlinliu", "jianli-wei" diff --git a/ci-operator/step-registry/gcp/provision/OWNERS b/ci-operator/step-registry/gcp/provision/OWNERS deleted file mode 120000 index ec405d65a79df..0000000000000 --- a/ci-operator/step-registry/gcp/provision/OWNERS +++ /dev/null @@ -1 +0,0 @@ -../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/gcp/provision/OWNERS b/ci-operator/step-registry/gcp/provision/OWNERS new file mode 100644 index 0000000000000..edad27cb2851e --- /dev/null +++ b/ci-operator/step-registry/gcp/provision/OWNERS @@ -0,0 +1,6 @@ +approvers: +- wgahnagl +- gpei +- jianlinliu +- jianli-wei + diff --git a/ci-operator/step-registry/gcp/provision/bastionhost/gcp-provision-bastionhost-ref.metadata.json b/ci-operator/step-registry/gcp/provision/bastionhost/gcp-provision-bastionhost-ref.metadata.json index 880f5015e89b6..f3fa82b0eefc5 100644 --- a/ci-operator/step-registry/gcp/provision/bastionhost/gcp-provision-bastionhost-ref.metadata.json +++ b/ci-operator/step-registry/gcp/provision/bastionhost/gcp-provision-bastionhost-ref.metadata.json @@ -2,6 +2,7 @@ "path": "gcp/provision/bastionhost/gcp-provision-bastionhost-ref.yaml", "owners": { "approvers": [ + "wgahnagl", "gpei", "jianlinliu", "jianli-wei" diff --git a/ci-operator/step-registry/gcp/provision/buildhost/OWNERS b/ci-operator/step-registry/gcp/provision/buildhost/OWNERS new file mode 120000 index 0000000000000..ec405d65a79df --- /dev/null +++ b/ci-operator/step-registry/gcp/provision/buildhost/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/gcp/provision/buildhost/gcp-provision-buildhost-commands.sh b/ci-operator/step-registry/gcp/provision/buildhost/gcp-provision-buildhost-commands.sh new file mode 100755 index 0000000000000..b300abd4bd56a --- /dev/null +++ b/ci-operator/step-registry/gcp/provision/buildhost/gcp-provision-buildhost-commands.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM + +CLUSTER_NAME="${NAMESPACE}-${JOB_NAME_HASH}" +NETWORK=${NETWORK:-} +IMAGE_ARGS="" +if [[ -z "${IMAGE_FAMILY}" ]] && [[ ! -z "${IMAGE_NAME}" ]] ; then + IMAGE_ARGS="--image=${IMAGE_NAME}" +fi + +if [[ ! -z "${IMAGE_FAMILY}" ]] && [[ -z "${IMAGE_NAME}" ]] ; then + IMAGE_ARGS="--image-family=${IMAGE_FAMILY}" +fi + +if [[ -z "${IMAGE_ARGS}" ]]; then + echo "image info not correct" + exit 1 +fi + +workdir=`mktemp -d` + +##################################### +###############Log In################ +##################################### + +workdir=`mktemp -d` +curl -L https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_amd64 -o /tmp/yq && chmod +x /tmp/yq + +GOOGLE_PROJECT_ID="$(< ${CLUSTER_PROFILE_DIR}/openshift_gcp_project)" +export GCP_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/gce.json" +sa_email=$(jq -r .client_email ${GCP_SHARED_CREDENTIALS_FILE}) +if ! gcloud auth list | grep -E "\*\s+${sa_email}" +then + gcloud auth activate-service-account --key-file="${GCP_SHARED_CREDENTIALS_FILE}" + gcloud config set project "${GOOGLE_PROJECT_ID}" +fi + +REGION="${LEASED_RESOURCE}" +echo "Using region: ${REGION}" + +VPC_CONFIG="${SHARED_DIR}/customer_vpc_subnets.yaml" +if [[ -z "${NETWORK}" || -z "${CONTROL_PLANE_SUBNET}" ]]; then + NETWORK=$(/tmp/yq r "${VPC_CONFIG}" 'platform.gcp.network') + CONTROL_PLANE_SUBNET=$(/tmp/yq r "${VPC_CONFIG}" 'platform.gcp.controlPlaneSubnet') +fi +if [[ -z "${NETWORK}" || -z "${CONTROL_PLANE_SUBNET}" ]]; then + echo "Could not find VPC network and control-plane subnet" && exit 1 +fi +ZONE_0=$(gcloud compute regions describe ${REGION} --format=json | jq -r .zones[0] | cut -d "/" -f9) +MACHINE_TYPE="n2-standard-8" + +##################################### +##########Create server_############# +##################################### + +# we need to be able to tear down the proxy even if install fails +# cannot rely on presence of ${SHARED_DIR}/metadata.json +echo "${REGION}" >> "${SHARED_DIR}/region" + +server_name="${CLUSTER_NAME}-buildhost" +gcloud compute instances create "${server_name}" \ + ${IMAGE_ARGS} \ + --image-project=${IMAGE_PROJECT} \ + --boot-disk-type pd-ssd \ + --boot-disk-size=200GB \ + --machine-type=${MACHINE_TYPE} \ + --metadata-from-file ssh-keys="${CLUSTER_PROFILE_DIR}/ssh-publickey" \ + --network=${NETWORK} \ + --subnet=${CONTROL_PLANE_SUBNET} \ + --zone=${ZONE_0} \ + --tags="${server_name}" + +echo "Created Server instance" + +if [[ -s "${SHARED_DIR}/xpn.json" ]]; then + HOST_PROJECT="$(jq -r '.hostProject' "${SHARED_DIR}/xpn.json")" + project_option="--project=${HOST_PROJECT}" +else + project_option="" +fi +gcloud ${project_option} compute firewall-rules create "${server_name}-ingress-allow" \ + --network ${NETWORK} \ + --allow tcp:22 \ + --target-tags="${server_name}" +cat > "${SHARED_DIR}/destroy.sh" << EOF +gcloud compute instances delete -q "${server_name}" --zone=${ZONE_0} +gcloud ${project_option} compute firewall-rules delete -q "${server_name}-ingress-allow" +EOF + +##################################### +#########Save Server Info########### +##################################### +echo "Instance ${server_name}" +echo "${server_name}" >> "${SHARED_DIR}/gcp-instance-ids.txt" + +gcloud compute instances list --filter="name=${server_name}" \ + --zones "${ZONE_0}" --format json > "${workdir}/${server_name}.json" +server__private_ip="$(jq -r '.[].networkInterfaces[0].networkIP' ${workdir}/${server_name}.json)" +server__public_ip="$(jq -r '.[].networkInterfaces[0].accessConfigs[0].natIP' ${workdir}/${server_name}.json)" + +if [ X"${server__public_ip}" == X"" ] || [ X"${server__private_ip}" == X"" ] ; then + echo "Did not found public or internal IP!" + exit 1 +fi +echo "export IP=${server__public_ip}" > "${SHARED_DIR}/env" +echo "export PRIVATE_IP=${server__private_ip}" >> "${SHARED_DIR}/env" +echo "export ZONE=${ZONE_0}" >> "${SHARED_DIR}/env" +cat <> "${SHARED_DIR}/env" +export SSHOPTS=(-o 'ConnectTimeout=5' -o 'StrictHostKeyChecking=no' -o 'UserKnownHostsFile=/dev/null' -o 'ServerAliveInterval=90' -o LogLevel=ERROR -i "\${CLUSTER_PROFILE_DIR}/ssh-privatekey") +EOF diff --git a/ci-operator/step-registry/gcp/provision/buildhost/gcp-provision-buildhost-ref.metadata.json b/ci-operator/step-registry/gcp/provision/buildhost/gcp-provision-buildhost-ref.metadata.json new file mode 100644 index 0000000000000..562c3390edc96 --- /dev/null +++ b/ci-operator/step-registry/gcp/provision/buildhost/gcp-provision-buildhost-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "gcp/provision/buildhost/gcp-provision-buildhost-ref.yaml", + "owners": { + "approvers": [ + "wgahnagl", + "gpei", + "jianlinliu", + "jianli-wei" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/gcp/provision/buildhost/gcp-provision-buildhost-ref.yaml b/ci-operator/step-registry/gcp/provision/buildhost/gcp-provision-buildhost-ref.yaml new file mode 100644 index 0000000000000..170588dd322dc --- /dev/null +++ b/ci-operator/step-registry/gcp/provision/buildhost/gcp-provision-buildhost-ref.yaml @@ -0,0 +1,21 @@ +ref: + as: gcp-provision-buildhost + from_image: + namespace: ocp + name: "4.12" + tag: upi-installer + grace_period: 10m + commands: gcp-provision-buildhost-commands.sh + resources: + requests: + cpu: 10m + memory: 100Mi + env: + - name: IMAGE_NAME + default: "" + - name: IMAGE_FAMILY + default: "" + - name: IMAGE_PROJECT + default: "" + documentation: |- + The step launches Linux build host. diff --git a/ci-operator/step-registry/gcp/provision/vpc/gcp-provision-vpc-ref.metadata.json b/ci-operator/step-registry/gcp/provision/vpc/gcp-provision-vpc-ref.metadata.json index b47ed0dc0e002..ea79ab8ae2694 100644 --- a/ci-operator/step-registry/gcp/provision/vpc/gcp-provision-vpc-ref.metadata.json +++ b/ci-operator/step-registry/gcp/provision/vpc/gcp-provision-vpc-ref.metadata.json @@ -2,6 +2,7 @@ "path": "gcp/provision/vpc/gcp-provision-vpc-ref.yaml", "owners": { "approvers": [ + "wgahnagl", "gpei", "jianlinliu", "jianli-wei"