diff --git a/cluster/bin/run.sh b/cluster/bin/run.sh new file mode 100755 index 0000000000000..7cf62cbea93c1 --- /dev/null +++ b/cluster/bin/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -euo pipefail + +ctr=gce-cluster + +docker rm $ctr &>/dev/null || true +docker create $@ --name $ctr -it openshift/origin-gce:latest /bin/bash >/dev/null +tar --mode='ug+rwX' -c -C data . | docker cp - $ctr:/usr/share/ansible/openshift-ansible-gce/playbooks/files +docker start -ai $ctr diff --git a/cluster/ci/README.md b/cluster/ci/README.md index e337f666d5b17..c7f9716b4c18f 100644 --- a/cluster/ci/README.md +++ b/cluster/ci/README.md @@ -1,11 +1,18 @@ -Deploy the OpenShift CI instance to GCE +# Deploy the OpenShift CI instance to GCE - $ ./up.sh + $ ../bin/run.sh + $ export PR_REPO_URL= + $ ansible-playbook playbooks/provision.yaml Will download the appropriate version of OpenShift and install it to -GCE. You must populate the data directory with the appropriate secret -data first (instructions pending). +GCE. You must populate the `data` directory with the appropriate secret +data first: -To get a shell into the container with the right data, run: +* ssl.crt / ssl.key: certificates for the master +* gce.json: Service account credentials for installing the master +* gce-registry.json: Service account credentials for the registry to use against GCS +* identity-providers.json: GitHub OAuth info +* ssh-privatekey / ssh-publickey: An SSH key pair for connecting to the masters (optional) - $ $(./run.sh) +The image `openshift/origin-gce:latest` is used as the environment for Ansible, and contains +a copy of the `openshift-ansible` code and `origin-gce`. \ No newline at end of file diff --git a/cluster/ci/config.sh b/cluster/ci/config.sh deleted file mode 100644 index 58e8c33f879bc..0000000000000 --- a/cluster/ci/config.sh +++ /dev/null @@ -1,135 +0,0 @@ -### CONFIG ### - -# Path to a RHEL image on local machine, downloaded from Red Hat Customer Portal -RHEL_IMAGE_PATH="${HOME}/Downloads/rhel-guest-image-7.2-20160302.0.x86_64.qcow2" - -# Username and password for Red Hat Customer Portal -RH_USERNAME='a' -RH_PASSWORD='' -# Pool ID which shall be used to register the pre-registered image -RH_POOL_ID='a' - -# Project ID and zone settings for Google Cloud -GCLOUD_PROJECT='openshift-gce-devel' -GCLOUD_ZONE='us-central1-a' -GCLOUD_SERVICE_ACCOUNT="ci-provisioner@openshift-gce-devel.iam.gserviceaccount.com" -GCLOUD_SSH_PRIVATE_KEY="/home/cloud-user/.ssh/google_compute_engine" - -RESOURCE_PREFIX='origin-ci-' - -# DNS domain which will be configured in Google Cloud DNS -DNS_DOMAIN='ci.openshift.org' -# Name of the DNS zone in the Google Cloud DNS. If empty, it will be created -DNS_DOMAIN_NAME="${RESOURCE_PREFIX:-}ocp-public-dns" -# DNS name for the Master service -MASTER_DNS_NAME="api.${DNS_DOMAIN}" -# Internal DNS name for the Master service -INTERNAL_MASTER_DNS_NAME="internal-master.${DNS_DOMAIN}" -# Domain name for the OpenShift applications -OCP_APPS_DNS_NAME="svc.${DNS_DOMAIN}" -# Paths on the local system for the certificate files. If empty, self-signed -# certificate will be generated -MASTER_HTTPS_CERT_FILE="" -MASTER_HTTPS_KEY_FILE="" - -## DEFAULT VALUES ## - -OCP_VERSION='3.3' - -CONSOLE_PORT='443' -INTERNAL_CONSOLE_PORT='8443' - -OCP_NETWORK="${RESOURCE_PREFIX:-}ocp-network" - -MASTER_MACHINE_TYPE='n1-standard-2' -NODE_MACHINE_TYPE='n1-standard-2' -INFRA_NODE_MACHINE_TYPE='n1-standard-2' -BASTION_MACHINE_TYPE='n1-standard-1' - -MASTER_INSTANCE_TEMPLATE="${RESOURCE_PREFIX:-}master-template" -NODE_INSTANCE_TEMPLATE="${RESOURCE_PREFIX:-}node-template" -INFRA_NODE_INSTANCE_TEMPLATE="${RESOURCE_PREFIX:-}infra-node-template" - -BASTION_INSTANCE="${RESOURCE_PREFIX:-}bastion" - -MASTER_INSTANCE_GROUP="${RESOURCE_PREFIX:-}ocp-master" -# How many instances should be created for this group -MASTER_INSTANCE_GROUP_SIZE='1' -MASTER_NAMED_PORT_NAME='web-console' -INFRA_NODE_INSTANCE_GROUP="${RESOURCE_PREFIX:-}ocp-infra" -INFRA_NODE_INSTANCE_GROUP_SIZE='0' -NODE_INSTANCE_GROUP="${RESOURCE_PREFIX:-}ocp-node" -NODE_INSTANCE_GROUP_SIZE='2' - -NODE_DOCKER_DISK_SIZE='25' -NODE_DOCKER_DISK_POSTFIX='-docker' -NODE_OPENSHIFT_DISK_SIZE='50' -NODE_OPENSHIFT_DISK_POSTFIX='-openshift' - -MASTER_NETWORK_LB_HEALTH_CHECK="${RESOURCE_PREFIX:-}master-network-lb-health-check" -MASTER_NETWORK_LB_POOL="${RESOURCE_PREFIX:-}master-network-lb-pool" -MASTER_NETWORK_LB_IP="${RESOURCE_PREFIX:-}master-network-lb-ip" -MASTER_NETWORK_LB_RULE="${RESOURCE_PREFIX:-}master-network-lb-rule" - -MASTER_SSL_LB_HEALTH_CHECK="${RESOURCE_PREFIX:-}master-ssl-lb-health-check" -MASTER_SSL_LB_BACKEND="${RESOURCE_PREFIX:-}master-ssl-lb-backend" -MASTER_SSL_LB_IP="${RESOURCE_PREFIX:-}master-ssl-lb-ip" -MASTER_SSL_LB_CERT="${RESOURCE_PREFIX:-}master-ssl-lb-cert" -MASTER_SSL_LB_TARGET="${RESOURCE_PREFIX:-}master-ssl-lb-target" -MASTER_SSL_LB_RULE="${RESOURCE_PREFIX:-}master-ssl-lb-rule" - -ROUTER_NETWORK_LB_HEALTH_CHECK="${RESOURCE_PREFIX:-}router-network-lb-health-check" -ROUTER_NETWORK_LB_POOL="${RESOURCE_PREFIX:-}router-network-lb-pool" -ROUTER_NETWORK_LB_IP="${RESOURCE_PREFIX:-}router-network-lb-ip" -ROUTER_NETWORK_LB_RULE="${RESOURCE_PREFIX:-}router-network-lb-rule" -# send router traffic to the master -ROUTER_NETWORK_TARGET_INSTANCE_GROUP="${MASTER_INSTANCE_GROUP}" - -REGISTRY_BUCKET="${GCLOUD_PROJECT}-${RESOURCE_PREFIX:-}registry-bucket" - -TEMP_INSTANCE="${RESOURCE_PREFIX:-}ocp-rhel-temp" - -GOOGLE_CLOUD_SDK_VERSION='130.0.0' - -STARTUP_BUCKET="${GCLOUD_PROJECT}-${RESOURCE_PREFIX:-}instance-bucket" -#STARTUP_SCRIPT_FILE="${DIR}/working/instance-startup.sh" - -# Firewall rules in a form: -# ['name']='parameters for "gcloud compute firewall-rules create"' -# For all possible parameters see: gcloud compute firewall-rules create --help -declare -A FW_RULES=( - ['icmp']='--allow icmp' - ['ssh-external']='--allow tcp:22' - ['ssh-internal']='--allow tcp:22 --source-tags bastion' - ['master-internal']="--allow tcp:2224,tcp:2379,tcp:2380,tcp:4001,udp:4789,udp:5404,udp:5405,tcp:8053,udp:8053,tcp:8444,tcp:10250,tcp:10255,udp:10255,tcp:24224,udp:24224 --source-tags ocp --target-tags ocp-master" - ['master-external']="--allow tcp:${CONSOLE_PORT},tcp:80,tcp:443,tcp:1936,tcp:${INTERNAL_CONSOLE_PORT},tcp:8080 --target-tags ocp-master" - ['node-internal']="--allow udp:4789,tcp:10250,tcp:10255,udp:10255 --source-tags ocp --target-tags ocp-node,ocp-infra-node" - ['infra-node-internal']="--allow tcp:5000 --source-tags ocp --target-tags ocp-infra-node" - ['infra-node-external']="--allow tcp:80,tcp:443,tcp:1936 --target-tags ocp-infra-node" -) - -BASTION_SSH_FW_RULE="${RESOURCE_PREFIX:-}bastion-ssh-to-external-ip" - - -### Secrets ### - -# OpenShift Identity providers -# Google default -# OCP_IDENTITY_PROVIDERS='[ {"name": "google", "kind": "GoogleIdentityProvider", "login": "true", "challenge": "false", "mapping_method": "claim", "client_id": "1043659492591-37si1gqp62olv4q6ihe5d4tgb29g79rh.apps.googleusercontent.com", "client_secret": "IWtfrF_DQEj5GRT0EAV1Biti", "hosted_domain": ""} ]' -# GitHub default - - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -OCP_IDENTITY_PROVIDERS="$( cat "${DIR}/identity-providers.json" )" - -GCE_PEM_FILE_PATH="${GCE_PEM_FILE_PATH:-${DIR}/gce.pem}" - -if [[ -f "${DIR}/ssl.crt" ]]; then - MASTER_HTTPS_CERT_FILE="${DIR}/ssl.crt" - MASTER_HTTPS_KEY_FILE="${DIR}/ssl.key" -fi - -if [[ -f "${DIR}/ansible-config.yml" ]]; then - ADDITIONAL_ANSIBLE_CONFIG="${DIR}/ansible-config.yml" -fi diff --git a/cluster/ci/data/vars.yaml b/cluster/ci/data/vars.yaml index cf29423f859b9..d575ac0adc15c 100644 --- a/cluster/ci/data/vars.yaml +++ b/cluster/ci/data/vars.yaml @@ -4,6 +4,10 @@ deployment_type: origin openshift_pkg_version: "-0.0.1" ansible_pkg_mgr: yum docker_upgrade: false +openshift_version: 1.4.0 # work around 1.5 not being a valid option yet +openshift_image_tag: v1.5.0-alpha.1 +openshift_additional_repos: [{'id': 'origin-pr', 'baseurl': '{{ lookup("env", "PR_REPO_URL") | default("https://https://storage.googleapis.com/origin-ci-test/branch-logs/master/zz_test_gcloud/latest/artifacts/rpms") }}', 'enabled': 1, 'gpgcheck': 0}] +openshift_enable_origin_repo: false # URLs and certs @@ -22,13 +26,13 @@ openshift_node_port_range: 30000-32000 # Authentication and authorization -openshift_master_identity_providers: "{{ (lookup('file', ansible_env.HOME + '/identity-providers.json' ) | default('{\"items\":[]}') | from_json).get('items') }}" +openshift_master_identity_providers: "{{ (lookup('file', 'files/identity-providers.json' ) | default('{\"items\":[]}') | from_json).get('items') }}" provision_role_mappings: [{'user': 'smarterclayton', 'role': 'cluster-admin'}] # Paths on the local system for the certificate files. If empty, self-signed # certificate will be generated -provision_master_https_cert_file: "{{ playbook_dir }}/files/ssl.crt" -provision_master_https_key_file: "{{ playbook_dir }}/files/ssl.key" +provision_master_https_cert_file: "ssl.crt" +provision_master_https_key_file: "ssl.key" # Post config setting sizes @@ -43,9 +47,9 @@ gce_project_id: openshift-gce-devel gce_region_name: us-central1 gce_zone_name: us-central1-a gce_service_account: ci-provisioner@openshift-gce-devel.iam.gserviceaccount.com -gce_service_account_keyfile: "{{ lookup('file', 'files/gce.json') }}" +gce_service_account_keyfile: "gce.json" gce_ssh_private_key: /home/cloud-user/.ssh/google_compute_engine -gcs_registry_keyfile: "{{ lookup('file', 'files/gce.json') }}" +gcs_registry_keyfile: "gcs-registry.json" inventory_ip_type: external gce_extra_tags_master: ",preserve" @@ -87,6 +91,6 @@ provision_gce_registry_gcs_bucket: openshift-gce-devel-origin-ci-registry-bucket provision_gce_router_network_instance_group: ig-m # default: ig-i # Provide a startup script file to the GCE instances -provision_gce_startup_script_file: "{{ playbook_dir }}/files/startup.sh" +provision_gce_startup_script_file: # "startup.sh" # Provide userdata to the gce instances provision_gce_user_data_file: diff --git a/cluster/ci/run.sh b/cluster/ci/run.sh deleted file mode 100755 index 2ff418aaf7ba0..0000000000000 --- a/cluster/ci/run.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -( -docker rm gce-bash || true -docker create --name gce-bash -it -e STARTUP_SCRIPT_FILE=/usr/local/install/data/startup.sh openshift/gce-cloud-installer:latest /bin/bash -docker cp data gce-bash:/usr/local/install -docker cp config.sh gce-bash:/usr/local/install/data/ -) 1>&2 -echo docker start -ai gce-bash diff --git a/cluster/ci/up.sh b/cluster/ci/up.sh deleted file mode 100755 index a9a4693e94458..0000000000000 --- a/cluster/ci/up.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -docker rm gce-ci || true -docker create --name gce-ci -e STARTUP_SCRIPT_FILE=/usr/local/install/data/startup.sh openshift/origin-gce:latest $@ -docker cp data gce-ci:/usr/local/install -docker start -a gce-ci - -# source data/config.sh -# oc login "https://${MASTER_DNS_NAME}" # has to be through UI -# grant cluster admin to user -# set up router certificate -# add self-provisioner to "self-provisioners" group -# oc project ci -# oc secrets new origin-gce data -o yaml > setup/secrets.yaml -# oc apply -f config/roles.yaml -# oc replace --force -f config/route-docker-registry.yaml -n default -# oadm policy remove-cluster-role-from-group self-provisioner system:authenticated:oauth -# oadm policy add-cluster-role-to-user cluster-reader system:serviceaccount:kube-system:heapster -# oc process -f config/heapster-standalone.yaml | oc apply -f - -n kube-system -# set env var COCKPIT_KUBE_URL=https://registry-console-default.svc.ci.openshift.org diff --git a/cluster/test-deploy/data/vars.yaml b/cluster/test-deploy/data/vars.yaml index b7c1c3bbbebc7..ff53dcabc265b 100644 --- a/cluster/test-deploy/data/vars.yaml +++ b/cluster/test-deploy/data/vars.yaml @@ -39,7 +39,7 @@ openshift_node_port_range: 30000-32000 # Authentication and authorization openshift_master_identity_providers: [{'name': 'allow_all', 'login': 'true', 'challenge': 'true', 'kind': 'AllowAllPasswordIdentityProvider'}] -provision_role_mappings: [{'user': 'admin', 'role': 'cluster-admin'}] +# provision_role_mappings: [{'user': 'admin', 'role': 'cluster-admin'}] # Paths on the local system for the certificate files. If empty, self-signed # certificate will be generated diff --git a/cluster/test-deploy/run.sh b/cluster/test-deploy/run.sh deleted file mode 100755 index b1ba42e3e169a..0000000000000 --- a/cluster/test-deploy/run.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -docker rm gce-pr &>/dev/null || true -docker create $@ --name gce-pr -it openshift/origin-gce:latest /bin/bash >/dev/null -tar -c -C data . | docker cp - gce-pr:/usr/share/ansible/openshift-ansible-gce/playbooks/files -docker start -ai gce-pr diff --git a/cluster/test-deploy/up.sh b/cluster/test-deploy/up.sh index 615c9b815eac4..7cb830656d1e1 100755 --- a/cluster/test-deploy/up.sh +++ b/cluster/test-deploy/up.sh @@ -7,23 +7,25 @@ data=$2 url=${3-} # provide simple defaulting of playbooks -playbook="${4:-playbooks/provision.yaml}" +playbook="${4:-playbooks/launch.yaml}" if [[ -z "${3-}" && -z "${4-}" ]]; then - playbook="playbooks/deprovision.yaml" + playbook="playbooks/terminate.yaml" fi +ctr=gce-pr-$build + # start a container with the custom playbook inside it -docker rm gce-pr-$build &>/dev/null || true +docker rm $ctr &>/dev/null || true args="" if [[ -n "${OPENSHIFT_ANSIBLE_REPO-}" ]]; then - docker volume rm gce-pr-$build-volume &>/dev/null || true - docker volume create --name gce-pr-$build-volume >/dev/null - args="-v gce-pr-$build-volume:/usr/share/ansible/openshift-ansible " + docker volume rm $ctr-volume &>/dev/null || true + docker volume create --name $ctr-volume >/dev/null + args="-v $ctr-volume:/usr/share/ansible/openshift-ansible " fi -docker create -e "PR_NUMBER=pr${build}" -e "PR_REPO_URL=${url}" --name gce-pr-$build $args openshift/origin-gce:latest ansible-playbook "${@:5}" "${playbook}" >/dev/null -tar -c -C "${data}" . | docker cp - gce-pr-$build:/usr/share/ansible/openshift-ansible-gce/playbooks/files +docker create -e "PR_NUMBER=pr${build}" -e "PR_REPO_URL=${url}" --name $ctr $args openshift/origin-gce:latest ansible-playbook "${@:5}" "${playbook}" >/dev/null +tar -c -C "${data}" . | docker cp - $ctr:/usr/share/ansible/openshift-ansible-gce/playbooks/files if [[ -n "${OPENSHIFT_ANSIBLE_REPO-}" ]]; then - tar -c -C "${OPENSHIFT_ANSIBLE_REPO}" . | docker cp - gce-pr-$build:/usr/share/ansible/openshift-ansible/ + tar -c -C "${OPENSHIFT_ANSIBLE_REPO}" . | docker cp - $ctr:/usr/share/ansible/openshift-ansible/ fi -docker start -a gce-pr-$build -docker cp gce-pr-$build:/tmp/admin.kubeconfig admin.kubeconfig &>/dev/null || true +docker start -a $ctr +docker cp $ctr:/tmp/admin.kubeconfig admin.kubeconfig &>/dev/null || true