Skip to content

Commit 00d80f0

Browse files
marinakogmarinakog
authored
Added shell scripts to create and configure OKE cluster and run integration test (#2113)
* fixed typo * fixed script * fixed script1 * added script * added script1 * corrected script * fixed grep * fixed err * fixed prop function * fixed typo * fixed typo * added delete cluster * cleanup * added doc * added doc1 * fixed permission * fixed typo * addressed review comments from Xian * addressed review comments from Xian1 * addressed review comments from Xian2 * addressed review comments from Xian3 * matched echo with command
1 parent cf5a1d9 commit 00d80f0

File tree

3 files changed

+299
-7
lines changed

3 files changed

+299
-7
lines changed

‎kubernetes/samples/scripts/terraform/oke.create.sh‎

Lines changed: 60 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,12 @@ function generateTFVarFile {
1111
rm -f ${tfVarsFiletfVarsFile}
1212
cp ${terraformVarDir}/template.tfvars $tfVarsFiletfVarsFile
1313
chmod 777 ${terraformVarDir}/template.tfvars $tfVarsFiletfVarsFile
14-
1514
sed -i -e "s:@TENANCYOCID@:${tenancy_ocid}:g" ${tfVarsFiletfVarsFile}
1615
sed -i -e "s:@USEROCID@:${user_ocid}:g" ${tfVarsFiletfVarsFile}
1716
sed -i -e "s:@COMPARTMENTOCID@:${compartment_ocid}:g" ${tfVarsFiletfVarsFile}
1817
sed -i -e "s:@COMPARTMENTNAME@:${compartment_name}:g" ${tfVarsFiletfVarsFile}
1918
sed -i -e "s:@OKECLUSTERNAME@:${okeclustername}:g" ${tfVarsFiletfVarsFile}
20-
sed -i -e "s:@OCIAPIPUBKEYFINGERPRINT@:"${ociapi_pubkey_fingerprint}":g" ${tfVarsFiletfVarsFile}
19+
sed -i -e "s/@OCIAPIPUBKEYFINGERPRINT@/"${ociapi_pubkey_fingerprint}"/g" ${tfVarsFiletfVarsFile}
2120
sed -i -e "s:@OCIPRIVATEKEYPATH@:${ocipk_path}:g" ${tfVarsFiletfVarsFile}
2221
sed -i -e "s:@VCNCIDRPREFIX@:${vcn_cidr_prefix}:g" ${tfVarsFiletfVarsFile}
2322
sed -i -e "s:@VCNCIDR@:${vcn_cidr_prefix}.0.0/16:g" ${tfVarsFiletfVarsFile}
@@ -27,7 +26,6 @@ function generateTFVarFile {
2726
sed -i -e "s:@NODEPOOLSSHPUBKEY@:${nodepool_ssh_pubkey}:g" ${tfVarsFiletfVarsFile}
2827
sed -i -e "s:@REGION@:${region}:g" ${tfVarsFiletfVarsFile}
2928
echo "Generated TFVars file [${tfVarsFiletfVarsFile}]"
30-
3129
}
3230

3331
function setupTerraform () {
@@ -37,14 +35,13 @@ function setupTerraform () {
3735
curl -O https://releases.hashicorp.com/terraform/0.11.10/terraform_0.11.10_darwin_amd64.zip
3836
unzip terraform_0.11.10_darwin_amd64.zip
3937
elif [[ "${OSTYPE}" == "linux"* ]]; then
40-
curl -O https://releases.hashicorp.com/terraform/0.11.8/terraform_0.11.8_linux_amd64.zip
41-
unzip terraform_0.11.8_linux_amd64.zip
38+
curl -LO --retry 3 https://releases.hashicorp.com/terraform/0.11.8/terraform_0.11.8_linux_amd64.zip
39+
unzip -o terraform_0.11.8_linux_amd64.zip -d ${terraformDir}
4240
else
4341
echo "Unsupported OS"
4442
fi
45-
chmod 777 ${terraformDir}/terraform
43+
chmod +x ${terraformDir}/terraform
4644
export PATH=${terraformDir}:${PATH}
47-
4845
}
4946

5047
function deleteOlderVersionTerraformOCIProvider() {
@@ -68,6 +65,58 @@ function createCluster () {
6865
terraform apply -auto-approve -var-file=${terraformVarDir}/${clusterTFVarsFile}.tfvars
6966
}
7067

68+
function createRoleBindings () {
69+
kubectl -n kube-system create serviceaccount $okeclustername-sa
70+
kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:$okeclustername-sa
71+
TOKENNAME=`kubectl -n kube-system get serviceaccount/$okeclustername-sa -o jsonpath='{.secrets[0].name}'`
72+
TOKEN=`kubectl -n kube-system get secret $TOKENNAME -o jsonpath='{.data.token}'| base64 --decode`
73+
kubectl config set-credentials $okeclustername-sa --token=$TOKEN
74+
kubectl config set-context --current --user=$okeclustername-sa
75+
}
76+
77+
function checkClusterRunning () {
78+
79+
echo 'Confirm we have kubectl working...'
80+
myline=`kubectl get nodes | awk '{print $2}'| tail -n+2`
81+
status="NotReady"
82+
max=50
83+
count=1
84+
85+
privateIP=${vcn_cidr_prefix//./\\.}\\.10\\.
86+
myline=`kubectl get nodes -o wide | grep "${privateIP}" | awk '{print $2}'`
87+
NODE_IP=`kubectl get nodes -o wide| grep "${privateIP}" | awk '{print $7}'`
88+
echo $myline
89+
status=$myline
90+
max=100
91+
count=1
92+
while [ "$myline" != "Ready" -a $count -le $max ] ; do
93+
echo "echo '[ERROR] Some Nodes in the Cluster are not in the Ready Status , sleep 10s more ..."
94+
sleep 10
95+
myline=`kubectl get nodes -o wide | grep "${privateIP}" | awk '{print $2}'`
96+
NODE_IP=`kubectl get nodes -o wide| grep "${privateIP}" | awk '{print $7}'`
97+
[[ ${myline} -eq "Ready" ]]
98+
echo "Status is ${myline} Iter [$count/$max]"
99+
count=`expr $count + 1`
100+
done
101+
102+
NODES=`kubectl get nodes -o wide | grep "${privateIP}" | wc -l`
103+
if [ "$NODES" == "1" ]; then
104+
echo '- looks good'
105+
else
106+
echo '- could not talk to cluster, aborting'
107+
cd ${terraformVarDir}
108+
terraform destroy -auto-approve -var-file=${terraformVarDir}/${clusterTFVarsFile}.tfvars
109+
exit 1
110+
fi
111+
112+
if [ $count -gt $max ] ; then
113+
echo "[ERROR] Unable to start the nodes in oke cluster after 200s ";
114+
cd ${terraformVarDir}
115+
terraform destroy -auto-approve -var-file=${terraformVarDir}/${clusterTFVarsFile}.tfvars
116+
exit 1
117+
fi
118+
}
119+
71120
#MAIN
72121
propsFile=${1:-$PWD/oci.props}
73122
terraformVarDir=${2:-$PWD}
@@ -106,4 +155,8 @@ chmod 600 ${ocipk_path}
106155

107156
# run terraform init,plan,apply to create OKE cluster based on the provided tfvar file ${clusterTFVarsFile).tfvar
108157
createCluster
158+
#check status of OKE cluster nodes, destroy if can not access them
109159
export KUBECONFIG=${terraformVarDir}/${okeclustername}_kubeconfig
160+
checkClusterRunning
161+
createRoleBindings
162+
echo "$okeclustername is up and running"

‎kubernetes/samples/scripts/terraform/oke.delete.sh‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/bin/bash
2+
# Copyright (c) 2020, Oracle Corporation and/or its affiliates.
3+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
4+
#
5+
# This script deletes provisioned OKE Kubernetes cluster using terraform (https://www.terraform.io/)
6+
#
7+
8+
set -o errexit
9+
set -o pipefail
10+
11+
function prop {
12+
grep "${1}" ${oci_property_file}| grep -v "#" | cut -d'=' -f2
13+
}
14+
15+
function cleanupLB {
16+
echo 'Clean up left over LB'
17+
myvcn_id=`oci network vcn list --compartment-id $compartment_ocid --display-name=${clusterName}_vcn | jq -r '.data[] | .id'`
18+
declare -a vcnidarray
19+
vcnidarray=(${myvcn_id// /})
20+
myip=`oci lb load-balancer list --compartment-id $compartment_ocid |jq -r '.data[] | .id'`
21+
mysubnets=`oci network subnet list --vcn-id=${vcnidarray[0]} --display-name=${clusterName}-LB-${1} --compartment-id $compartment_ocid | jq -r '.data[] | .id'`
22+
23+
declare -a iparray
24+
declare -a mysubnetsidarray
25+
mysubnetsidarray=(${mysubnets// /})
26+
27+
iparray=(${myip// /})
28+
vcn_cidr_prefix=$(prop 'vcn.cidr.prefix')
29+
for k in "${mysubnetsidarray[@]}"
30+
do
31+
for i in "${iparray[@]}"
32+
do
33+
lb=`oci lb load-balancer get --load-balancer-id=$i`
34+
if [[ (-z "${lb##*$vcn_cidr_prefix*}") || (-z "${lb##*$k*}") ]] ;then
35+
echo "deleting lb with id $i"
36+
oci lb load-balancer delete --load-balancer-id=$i --force || true
37+
fi
38+
done
39+
done
40+
}
41+
42+
function deleteOKE {
43+
cd ${terraform_script_dir}
44+
terraform init -var-file=${terraform_script_dir}/${clusterName}.tfvars
45+
terraform plan -var-file=${terraform_script_dir}/${clusterName}.tfvars
46+
terraform destroy -auto-approve -var-file=${terraform_script_dir}/${clusterName}.tfvars
47+
}
48+
49+
#MAIN
50+
oci_property_file=${1:-$PWD/oci.props}
51+
terraform_script_dir=${2:-$PWD}
52+
clusterName=$(prop 'okeclustername')
53+
compartment_ocid=$(prop 'compartment.ocid')
54+
vcn_cidr_prefix=$(prop 'vcn.cidr.prefix')
55+
export KUBECONFIG=${terraform_script_dir}/${clusterName}_kubeconfig
56+
export PATH=${terraform_script_dir}/terraforminstall:$PATH
57+
echo 'Deleting cluster'
58+
#check and cleanup any left over running Load Balancers
59+
cleanupLB Subnet01
60+
cleanupLB Subnet02
61+
deleteOKE

‎oketest.sh‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
#!/bin/bash
2+
# Copyright (c) 2020, Oracle Corporation and/or its affiliates.
3+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
4+
#
5+
# This script provisions a OKE Kubernetes cluster using terraform (https://www.terraform.io/) and runs the new
6+
# integration test suite against that cluster. Blog https://blogs.oracle.com/weblogicserver/easily-create-an-oci-container-engine-for-kubernetes-cluster-with-terraform-installer-to-run-weblogic-server
7+
# provides detailed explanation for OCI properties file creation.
8+
#
9+
#
10+
# As of May 6, 2020, the tests are clean on Kubernetes 1.16 with the following JDK workarounds:
11+
# 1. Maven must be run with OpenJDK 11.0.7, available here: https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.7%2B10/OpenJDK11U-jdk_x64_linux_11.0.7_10.tar.gz
12+
# This is because of a critical bug fix. Unfortunately, the Oracle JDK 11.0.7 release was based on an earlier build and doesn't have the fix.
13+
# 2. The WebLogic Image Tool will not accept an OpenJDK JDK. Set WIT_JAVA_HOME to an Oracle JDK Java Home.
14+
# For example, "export WIT_JAVA_HOME=/usr/java/jdk-11.0.7" before running this script.
15+
#
16+
set -o errexit
17+
set -o pipefail
18+
19+
script="${BASH_SOURCE[0]}"
20+
scriptDir="$( cd "$( dirname "${script}" )" && pwd )"
21+
22+
function usage {
23+
echo "usage: ${script} [-n <terraform config files directory>] [-o <directory>] [-t <tests>] [-c <name>] [-p true|false] [-x <number_of_threads>] [-d <wdt_download_url>] [-i <wit_download_url>] [-m <maven_profile_name>] [-h]"
24+
echo " -n Terraform config files directory "
25+
echo " -o Output directory (optional) "
26+
echo " (default: \${WORKSPACE}/logdir/\${BUILD_TAG}, if \${WORKSPACE} defined, else /scratch/\${USER}/kindtest) "
27+
echo " -b Availability Domain Name "
28+
echo " (for example: VPGL:PHX-AD-1 , check limits quota with OCI admin)"
29+
echo " -t Test filter (optional) "
30+
echo " (default: **/It*) "
31+
echo " -s Oracle Cloud Infra properties file "
32+
echo " -p Run It classes in parallel"
33+
echo " (default: false) "
34+
echo " -x Number of threads to run the classes in parallel"
35+
echo " (default: 2) "
36+
echo " -d WDT download URL"
37+
echo " (default: https://github.com/oracle/weblogic-deploy-tooling/releases/latest) "
38+
echo " -i WIT download URL"
39+
echo " (default: https://github.com/oracle/weblogic-image-tool/releases/latest) "
40+
echo " -m Run integration-tests or oke-cert "
41+
echo " (default: integration-tests, supported values: oke-cert) "
42+
echo " -h Help"
43+
exit $1
44+
}
45+
46+
function prop {
47+
grep "${1}" ${oci_property_file}| grep -v "#" | cut -d'=' -f2
48+
}
49+
50+
if [[ -z "${WORKSPACE}" ]]; then
51+
outdir="/scratch/${USER}/oketest"
52+
export WORKSPACE=${PWD}
53+
else
54+
outdir="${WORKSPACE}/logdir/${BUILD_TAG}"
55+
fi
56+
test_filter="**/It*"
57+
parallel_run="false"
58+
threads="2"
59+
wdt_download_url="https://github.com/oracle/weblogic-deploy-tooling/releases/latest"
60+
wit_download_url="https://github.com/oracle/weblogic-image-tool/releases/latest"
61+
maven_profile_name="integration-tests"
62+
63+
while getopts ":h:n:o:t:x:s:p:d:i:m:b:" opt; do
64+
case $opt in
65+
n) terraform_script_dir_name="${OPTARG}"
66+
;;
67+
s) oci_property_file="${OPTARG}"
68+
;;
69+
b) availability_domain="${OPTARG}"
70+
;;
71+
o) outdir="${OPTARG}"
72+
;;
73+
t) test_filter="${OPTARG}"
74+
;;
75+
x) threads="${OPTARG}"
76+
;;
77+
p) parallel_run="${OPTARG}"
78+
;;
79+
d) wdt_download_url="${OPTARG}"
80+
;;
81+
i) wit_download_url="${OPTARG}"
82+
;;
83+
m) maven_profile_name="${OPTARG}"
84+
;;
85+
h) usage 0
86+
;;
87+
d) echo "Ignoring -d=${OPTARG}"
88+
;;
89+
i) echo "Ignoring -i=${OPTARG}"
90+
;;
91+
*) usage 1
92+
;;
93+
esac
94+
done
95+
96+
k8s_version=$(prop 'k8s.version')
97+
echo "Using Kubernetes version: ${k8s_version}"
98+
99+
mkdir -m777 -p "${outdir}"
100+
export RESULT_ROOT="${outdir}/wl_k8s_test_results"
101+
if [ -d "${RESULT_ROOT}" ]; then
102+
rm -Rf "${RESULT_ROOT}/*"
103+
else
104+
mkdir -m777 "${RESULT_ROOT}"
105+
fi
106+
107+
echo "Results will be in ${RESULT_ROOT}"
108+
109+
export PV_ROOT="${outdir}/k8s-pvroot"
110+
if [ -d "${PV_ROOT}" ]; then
111+
rm -Rf "${PV_ROOT}/*"
112+
else
113+
mkdir -m777 "${PV_ROOT}"
114+
fi
115+
116+
echo "Persistent volume files, if any, will be in ${PV_ROOT}"
117+
118+
echo 'Create a OKE cluster'
119+
mkdir -p "${WORKSPACE}/terraform"
120+
cp -rf ${terraform_script_dir_name}/*.tf ${WORKSPACE}/terraform/.
121+
cp -rf ${WORKSPACE}/kubernetes/samples/scripts/terraform/template.tfvars ${WORKSPACE}/terraform/.
122+
cp -rf ${WORKSPACE}/kubernetes/samples/scripts/terraform/*.sh ${WORKSPACE}/terraform/.
123+
chmod 777 ${WORKSPACE}/terraform/*.sh
124+
mkdir -p ${WORKSPACE}/terraform/terraforminstall
125+
126+
if ! sh ${WORKSPACE}/terraform/oke.create.sh ${oci_property_file} ${WORKSPACE}/terraform ; then
127+
sh ${WORKSPACE}/terraform/oke.delete.sh ${oci_property_file} ${WORKSPACE}/terraform
128+
fi
129+
130+
clusterName=$(prop 'okeclustername')
131+
132+
export KUBECONFIG=${WORKSPACE}/terraform/${clusterName}_kubeconfig
133+
export PATH=${WORKSPACE}/terraform/terraforminstall:$PATH
134+
135+
echo "creating storage class to setup OFSS ..."
136+
137+
echo "getting MountTarget ID"
138+
compartment_ocid=$(prop 'compartment.ocid')
139+
mount_target_id=`oci fs mount-target list --compartment-id=$compartment_ocid --display-name=${clusterName}-mt --availability-domain=${availability_domain} | jq -r '.data[] | .id'`
140+
mt_privateip_id=`oci fs mount-target list --compartment-id=$compartment_ocid --display-name=${clusterName}-mt --availability-domain=${availability_domain} | jq -r '.data[] | ."private-ip-ids"[]'`
141+
mt_private_ip=`oci network private-ip get --private-ip-id $mt_privateip_id | jq -r '.data | ."ip-address"'`
142+
143+
export NFS_SERVER=$mt_private_ip
144+
echo "Using NFS Server ${NFS_SERVER}"
145+
echo "Creating Storage Class to mount OFSS"
146+
cat << EOF | kubectl apply -f -
147+
kind: StorageClass
148+
apiVersion: storage.k8s.io/v1beta1
149+
metadata:
150+
name: oci-fss
151+
provisioner: oracle.com/oci-fss
152+
parameters:
153+
# Insert mount target from the FSS here
154+
mntTargetId: ${mount_target_id}
155+
EOF
156+
157+
echo 'Set up test running ENVVARs...'
158+
NODE_IP=`kubectl get nodes -o wide| awk '{print $7}'| tail -n+3`
159+
if [ -z "$NODE_IP" ]; then
160+
echo "retry get node ip ";
161+
sleep 15;
162+
NODE_IP=`kubectl get nodes -o wide| awk '{print $7}'| tail -n+3`
163+
fi
164+
165+
export K8S_NODEPORT_HOST=$NODE_IP
166+
export JAVA_HOME="${JAVA_HOME:-`type -p java|xargs readlink -f|xargs dirname|xargs dirname`}"
167+
168+
echo 'Clean up result root...'
169+
rm -rf "${RESULT_ROOT:?}/*"
170+
cd ${WORKSPACE}
171+
echo 'Run tests...'
172+
if [ "${maven_profile_name}" = "oke-cert" ]; then
173+
echo "Running mvn -Dwdt.download.url=${wdt_download_url} -Dwit.download.url=${wit_download_url} -Djdk.tls.client.protocols=TLSv1.2 -pl integration-tests -P ${maven_profile_name} verify 2>&1 | tee ${RESULT_ROOT}/oke.log"
174+
mvn -Dwdt.download.url="${wdt_download_url}" -Dwit.download.url="${wit_download_url}" -Djdk.tls.client.protocols=TLSv1.2 -pl integration-tests -P ${maven_profile_name} verify 2>&1 | tee "${RESULT_ROOT}/oke.log"
175+
else
176+
echo "Running mvn -Dit.test=${test_filter}, !ItExternalRmiTunneling, !ItSamples, !ItMiiSample, !ItTwoDomainsLoadBalancers, !ItMonitoringExporter, !ItPodRestart -Dwdt.download.url=${wdt_download_url} -Dwit.download.url=${wit_download_url} -Djdk.tls.client.protocols=TLSv1.2 -pl integration-tests -P integration-tests verify 2>&1 | tee ${RESULT_ROOT}/oke.log"
177+
mvn -Dit.test="${test_filter}, !ItExternalRmiTunneling, !ItSamples, !ItMiiSample, !ItTwoDomainsLoadBalancers, !ItMonitoringExporter, !ItPodRestart" -Dwdt.download.url="${wdt_download_url}" -Dwit.download.url="${wit_download_url}" -Djdk.tls.client.protocols=TLSv1.2 -pl integration-tests -P ${maven_profile_name} verify 2>&1 | tee "${RESULT_ROOT}/oke.log"
178+
fi

0 commit comments

Comments
 (0)