From ed7eb9e9f5e039c6529b629c5060adf8e0ce4cde Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 25 May 2022 15:06:05 -0400 Subject: [PATCH 1/4] Add namespace to kubectl-tenant command Signed-off-by: David Vossel --- kubevirtci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubevirtci b/kubevirtci index 1c8c6f221..7565ca112 100755 --- a/kubevirtci +++ b/kubevirtci @@ -145,7 +145,7 @@ function kubevirtci::kubectl_tenant { ${_default_virtctl_path} port-forward -n ${TENANT_CLUSTER_NAMESPACE} vm/${control_plane_vm_name} 64443:6443 > /dev/null 2>&1 & trap 'kill $(jobs -p) > /dev/null 2>&1' EXIT rm -f .${TENANT_CLUSTER_NAME}-kubeconfig - $CLUSTERCTL_PATH get kubeconfig ${TENANT_CLUSTER_NAME} > .${TENANT_CLUSTER_NAME}-kubeconfig + $CLUSTERCTL_PATH get kubeconfig ${TENANT_CLUSTER_NAME} -n ${TENANT_CLUSTER_NAMESPACE} > .${TENANT_CLUSTER_NAME}-kubeconfig sleep 0.1 kubectl --kubeconfig .${TENANT_CLUSTER_NAME}-kubeconfig --insecure-skip-tls-verify --server https://localhost:64443 "$@" } From 0857b5fd51fd2314922b2470db8fef9879dd4e5e Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 25 May 2022 15:43:51 -0400 Subject: [PATCH 2/4] Add ssh commands for infra and tenant nodes Signed-off-by: David Vossel --- kubevirtci | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/kubevirtci b/kubevirtci index 7565ca112..fbbdce7bb 100755 --- a/kubevirtci +++ b/kubevirtci @@ -13,6 +13,7 @@ export KUBEVIRT_DEPLOY_CDI="true" export KUBEVIRT_STORAGE="rook-ceph-default" _default_bin_path=./hack/tools/bin +_default_tmp_path=./hack/tools/bin/tmp _default_clusterctl_path=./hack/tools/bin/clusterctl _default_virtctl_path=./hack/tools/bin/virtctl @@ -21,6 +22,7 @@ export TENANT_CLUSTER_NAME=${TENANT_CLUSTER_NAME:-kvcluster} export TENANT_CLUSTER_NAMESPACE=${TENANT_CLUSTER_NAMESPACE:-kvcluster} _kubectl=cluster-up/cluster-up/kubectl.sh +_ssh_infra=cluster-up/cluster-up/ssh.sh _action=$1 shift @@ -43,6 +45,8 @@ function kubevirtci::usage() { virtctl Run virtctl commands against the cluster clusterctl Run clusterctl commands against the cluster + ssh-infra SSH into one of the infra nodes (like node01) + ssh-tenant [vmi namespace] SSH into one of the guest nodes create-cluster Create new kubernetes tenant cluster kubectl-tenant Interact with the tenant cluster @@ -100,6 +104,23 @@ function kubevirtci::build() { make docker-push } +function kubevirtci::ssh_tenant() { + vmi_name=$1 + vmi_namespace=${2:-$TENANT_CLUSTER_NAMESPACE} + + mkdir -p $_default_tmp_path + + echo "vmi $vmi_name namespace $vmi_namespace" + + ${_kubectl} get secret -n e2e-test-create-cluster-87mlkk kvcluster-ssh-keys -o jsonpath='{.data}' | grep key | awk -F '"' '{print $4}' | base64 -d > ${_default_tmp_path}/key.pem + + chmod 600 ${_default_tmp_path}/key.pem + + ssh -o IdentitiesOnly=yes -o "ProxyCommand=$_default_virtctl_path port-forward --stdio=true $vmi_name.$vmi_namespace 22" capk@$vmi_name.$vmi_namespace -i ${_default_tmp_path}/key.pem + + rm ${_default_tmp_path}/key.pem +} + function kubevirtci::refresh() { ${_kubectl} delete pods --all -n capk-system } @@ -179,6 +200,12 @@ case ${_action} in "virtctl") ${_default_virtctl_path} "$@" ;; +"ssh-infra") + $_ssh_infra "$@" + ;; +"ssh-tenant") + kubevirtci::ssh_tenant "$@" + ;; "clusterctl") $CLUSTERCTL_PATH "$@" ;; From f01885dc9eb2b58a6c26c96fd7f43478d5574cfb Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 25 May 2022 16:13:42 -0400 Subject: [PATCH 3/4] add kubevirtci destroy-cluster command Signed-off-by: David Vossel --- kubevirtci | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/kubevirtci b/kubevirtci index fbbdce7bb..a587c03fa 100755 --- a/kubevirtci +++ b/kubevirtci @@ -48,6 +48,7 @@ function kubevirtci::usage() { ssh-infra SSH into one of the infra nodes (like node01) ssh-tenant [vmi namespace] SSH into one of the guest nodes create-cluster Create new kubernetes tenant cluster + destroy-cluster Destroy the tenant cluster kubectl-tenant Interact with the tenant cluster help Print usage @@ -141,11 +142,15 @@ function kubevirtci::generate_kubeconfig() { sed -i -r 's/127.0.0.1:[0-9]+/192.168.66.101:6443/g' kubeconfig-e2e } +function kubevirtci::destroy_cluster() { + ${_kubectl} delete cluster -n ${TENANT_CLUSTER_NAMESPACE} ${TENANT_CLUSTER_NAME} +} + function kubevirtci::create_cluster() { export NODE_VM_IMAGE_TEMPLATE=quay.io/capk/ubuntu-container-disk:20.04 export IMAGE_REPO=k8s.gcr.io export CRI_PATH="/var/run/containerd/containerd.sock" - oc create secret generic external-infra-kubeconfig -n capk-system --from-file=kubeconfig=kubeconfig-e2e --from-literal=namespace=${TENANT_CLUSTER_NAMESPACE} + ${_kubectl} create secret generic external-infra-kubeconfig -n capk-system --from-file=kubeconfig=kubeconfig-e2e --from-literal=namespace=${TENANT_CLUSTER_NAMESPACE} $CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --kubernetes-version v1.21.0 --control-plane-machine-count=1 --worker-machine-count=1 --from templates/cluster-template-ext-infra.yaml | ${_kubectl} apply -f - } @@ -213,6 +218,9 @@ case ${_action} in kubevirtci::generate_kubeconfig kubevirtci::create_cluster ;; +"destroy-cluster") + kubevirtci::destroy_cluster + ;; "help") kubevirtci::usage ;; From 60a0162cc8f9a1f9dbdd17c2ab61eeb0fba3637c Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 25 May 2022 16:42:16 -0400 Subject: [PATCH 4/4] make separate kubevirtci command for simulating external clusters Signed-off-by: David Vossel --- kubevirtci | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/kubevirtci b/kubevirtci index a587c03fa..b9199bf7c 100755 --- a/kubevirtci +++ b/kubevirtci @@ -42,14 +42,15 @@ function kubevirtci::usage() { kubeconfig Return the kubeconfig of the cluster kubectl Interact with the cluster + kubectl-tenant Interact with the tenant cluster virtctl Run virtctl commands against the cluster clusterctl Run clusterctl commands against the cluster ssh-infra SSH into one of the infra nodes (like node01) ssh-tenant [vmi namespace] SSH into one of the guest nodes create-cluster Create new kubernetes tenant cluster + create-external-cluster Create new kubernetes tenant cluster simulated as running on external infra destroy-cluster Destroy the tenant cluster - kubectl-tenant Interact with the tenant cluster help Print usage " @@ -113,7 +114,7 @@ function kubevirtci::ssh_tenant() { echo "vmi $vmi_name namespace $vmi_namespace" - ${_kubectl} get secret -n e2e-test-create-cluster-87mlkk kvcluster-ssh-keys -o jsonpath='{.data}' | grep key | awk -F '"' '{print $4}' | base64 -d > ${_default_tmp_path}/key.pem + ${_kubectl} get secret -n $TENANT_CLUSTER_NAMESPACE kvcluster-ssh-keys -o jsonpath='{.data}' | grep key | awk -F '"' '{print $4}' | base64 -d > ${_default_tmp_path}/key.pem chmod 600 ${_default_tmp_path}/key.pem @@ -143,15 +144,25 @@ function kubevirtci::generate_kubeconfig() { } function kubevirtci::destroy_cluster() { - ${_kubectl} delete cluster -n ${TENANT_CLUSTER_NAMESPACE} ${TENANT_CLUSTER_NAME} + ${_kubectl} delete cluster -n ${TENANT_CLUSTER_NAMESPACE} ${TENANT_CLUSTER_NAME} --ignore-not-found } function kubevirtci::create_cluster() { export NODE_VM_IMAGE_TEMPLATE=quay.io/capk/ubuntu-container-disk:20.04 export IMAGE_REPO=k8s.gcr.io export CRI_PATH="/var/run/containerd/containerd.sock" + + $CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version v1.21.0 --control-plane-machine-count=1 --worker-machine-count=1 --from templates/cluster-template.yaml | ${_kubectl} apply -f - +} + +function kubevirtci::create_external_cluster() { + export NODE_VM_IMAGE_TEMPLATE=quay.io/capk/ubuntu-container-disk:20.04 + export IMAGE_REPO=k8s.gcr.io + export CRI_PATH="/var/run/containerd/containerd.sock" + + ${_kubectl} delete secret external-infra-kubeconfig -n capk-system --ignore-not-found ${_kubectl} create secret generic external-infra-kubeconfig -n capk-system --from-file=kubeconfig=kubeconfig-e2e --from-literal=namespace=${TENANT_CLUSTER_NAMESPACE} - $CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --kubernetes-version v1.21.0 --control-plane-machine-count=1 --worker-machine-count=1 --from templates/cluster-template-ext-infra.yaml | ${_kubectl} apply -f - + $CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version v1.21.0 --control-plane-machine-count=1 --worker-machine-count=1 --from templates/cluster-template-ext-infra.yaml | ${_kubectl} apply -f - } function kubevirtci::kubectl_tenant { @@ -215,9 +226,12 @@ case ${_action} in $CLUSTERCTL_PATH "$@" ;; "create-cluster") - kubevirtci::generate_kubeconfig kubevirtci::create_cluster ;; +"create-external-cluster") + kubevirtci::generate_kubeconfig + kubevirtci::create_external_cluster + ;; "destroy-cluster") kubevirtci::destroy_cluster ;;