diff --git a/playbooks/openstack/configuration.md b/playbooks/openstack/configuration.md index 12d3d21a2d9..1f7ab787113 100644 --- a/playbooks/openstack/configuration.md +++ b/playbooks/openstack/configuration.md @@ -291,18 +291,81 @@ follow the OpenShift documentation on the registry: https://docs.openshift.org/latest/install_config/registry/index.html -## Multi-Master Configuration +## API and Router Load Balancing + +A production deployment should contain more then one master and infra node and +have a load balancer in front of them. + +The playbooks will not create any load balancer by default. Even if you do +request multiple masters. + +You can opt into that if you want though. There are two options: a VM-based +load balancer and OpenStack's Load Balancer as a Service. + +### Load Balancer as a Service + +If your OpenStack supports Load Balancer as a Service (LBaaS) provided by the +Octavia project, our playbooks can set it up automatically. + +Put this in your `inventory/group_vars/all.yml`: + + openshift_openstack_use_lbaas_load_balancer: true + +This will create two load balancers: one for the API and UI console and the +other for the OpenShift router. Each will have its own public IP address. + +### VM-based Load Balancer + +If you can't use OpenStack's LBaaS, we can create and configure a virtual +machine running HAProxy to serve as one. + +Put this in your `inventory/group_vars/all.yml`: + + openshift_openstack_use_vm_load_balancer: true + +**WARNING** this VM will only handle the API and UI requests, *not* the +OpenShift routes. + +That means, if you have more than one infra node, you will have to balance them +externally. It is not recommended to use this option in production. + +### No Load Balancer + +If you specify neither `openshift_openstack_use_lbaas_load_balancer` nor +`openshift_openstack_use_vm_load_balancer`, the resulting OpenShift cluster +will have no load balancing configured out of the box. + +This is regardless of how many master or infra nodes you create. + +In this mode, you are expected to configure and maintain a load balancer +yourself. + +However, the cluster is usable without a load balancer as well. To talk to the +API or UI, connect to any of the master nodes. For the OpenShift routes, use +any of the infra nodes. + +### Public Cluster Endpoints + +In either of these cases (LBaaS, VM HAProxy, no LB) the public addresses to +access the cluster's API and router will be printed out at the end of the +playbook. + +If you want to get them out explicitly, run the following playbook with the +same arguments (private key, inventories, etc.) as your provision/install ones: + + playbooks/openstack/inventory.py openshift-ansible/playbooks/openstack/openshift-cluster/cluster-info.yml + +These addresses will depend on the load balancing solution. For LBaaS, they'll +be the the floating IPs of the load balancers. In the VM-based solution, +the API address will be the public IP of the load balancer VM and the router IP +will be the address of the first infra node that was created. If no load +balancer is selected, the API will be the address of the first master node and +the router will be the address of the first infra node. + +This means that regardless of the load balancing solution, you can use these +two entries to provide access to your cluster. -Please refer to the official documentation for the -[multi-master setup](https://docs.openshift.com/container-platform/3.6/install_config/install/advanced_install.html#multiple-masters) -and define the corresponding [inventory variables](https://docs.openshift.com/container-platform/3.6/install_config/install/advanced_install.html#configuring-cluster-variables) -in `inventory/group_vars/OSEv3.yml`. For example, given a load balancer node -under the ansible group named `ext_lb`: -``` -openshift_master_cluster_hostname: "{{ groups.ext_lb.0 }}" -openshift_master_cluster_public_hostname: "{{ groups.ext_lb.0 }}" -``` ## Provider Network Configuration diff --git a/playbooks/openstack/inventory.py b/playbooks/openstack/inventory.py index d39e7cda4ac..bafdcd2a30c 100755 --- a/playbooks/openstack/inventory.py +++ b/playbooks/openstack/inventory.py @@ -162,6 +162,11 @@ def build_inventory(): except KeyError: pass # Not an API load balanced deployment + inventory['localhost']['openshift_openstack_public_api_ip'] = \ + stout.get('public_api_ip') + inventory['localhost']['openshift_openstack_public_router_ip'] = \ + stout.get('public_router_ip') + try: inventory['OSEv3']['vars'] = _get_kuryr_vars(cloud, stout) except KeyError: diff --git a/playbooks/openstack/openshift-cluster/cluster-info.yml b/playbooks/openstack/openshift-cluster/cluster-info.yml new file mode 100644 index 00000000000..8793b7c41e0 --- /dev/null +++ b/playbooks/openstack/openshift-cluster/cluster-info.yml @@ -0,0 +1,11 @@ +--- +- name: Show information about the cluster + hosts: localhost + become: no + gather_facts: no + tasks: + - name: Print the API / UI Public IP Address + debug: var=openshift_openstack_public_api_ip + + - name: Print the OpenShift Router Public IP Address + debug: var=openshift_openstack_public_router_ip diff --git a/playbooks/openstack/openshift-cluster/install.yml b/playbooks/openstack/openshift-cluster/install.yml index 63181c7d617..89f961d7789 100644 --- a/playbooks/openstack/openshift-cluster/install.yml +++ b/playbooks/openstack/openshift-cluster/install.yml @@ -28,3 +28,6 @@ - name: run the cluster deploy import_playbook: ../../deploy_cluster.yml + +- name: Show information about the deployed cluster + import_playbook: cluster-info.yml diff --git a/playbooks/openstack/openshift-cluster/provision.yml b/playbooks/openstack/openshift-cluster/provision.yml index cd711053ac3..a05260c228b 100644 --- a/playbooks/openstack/openshift-cluster/provision.yml +++ b/playbooks/openstack/openshift-cluster/provision.yml @@ -49,3 +49,6 @@ - ansible_distribution == "RedHat" - rhsub_user is defined - rhsub_pass is defined + +- name: Show information about the deployed cluster + import_playbook: cluster-info.yml diff --git a/playbooks/openstack/post-install.md b/playbooks/openstack/post-install.md index 01cf2a62ebd..71d6df70aaa 100644 --- a/playbooks/openstack/post-install.md +++ b/playbooks/openstack/post-install.md @@ -19,11 +19,16 @@ DNS configured. You should add two entries to the `/etc/hosts` file on the Ansible host (where you to do a quick validation. A real deployment will however require a DNS server with the following entries set. -First, run the `openstack server list` command and note the floating IP -addresses of the *master* and *infra* nodes (we will use `10.40.128.130` for -master and `10.40.128.134` for infra here). +In either case, the IP addresses for the API and routers will be printed +out at the end of the deployment. -Then add the following entries to your `/etc/hosts`: +The first one is your API/UI address and the second one is the router address. +Depending on your load balancer configuration they may or may not be the same. + +In this example, we will use `10.40.128.130` for the `public_api_ip` and +`10.40.128.134` for `public_router_ip`. + +Add the following entries to your `/etc/hosts`: ``` 10.40.128.130 console.openshift.example.com diff --git a/playbooks/openstack/sample-inventory/group_vars/all.yml b/playbooks/openstack/sample-inventory/group_vars/all.yml index 3c1aece10a0..47dce4b5a54 100644 --- a/playbooks/openstack/sample-inventory/group_vars/all.yml +++ b/playbooks/openstack/sample-inventory/group_vars/all.yml @@ -109,6 +109,13 @@ openshift_openstack_default_flavor: "m1.medium" # # Numerical index of nodes to remove # openshift_openstack_nodes_to_remove: [] + +## Select a load balancer solution you desire. Only one of these can be +## `true` at a time. If they're both `false`, no load balancer will be deployed. +#openshift_openstack_use_lbaas_load_balancer: false +#openshift_openstack_use_vm_load_balancer: false + + # # Docker volume size # # - set specific volume size for roles by uncommenting corresponding lines # # - note: do not remove docker_default_volume_size definition diff --git a/roles/openshift_openstack/defaults/main.yml b/roles/openshift_openstack/defaults/main.yml index e553d5ecac4..9cc03c32487 100644 --- a/roles/openshift_openstack/defaults/main.yml +++ b/roles/openshift_openstack/defaults/main.yml @@ -12,6 +12,8 @@ openshift_openstack_num_cns: 0 openshift_openstack_dns_nameservers: [] openshift_openstack_nodes_to_remove: [] +openshift_openstack_use_lbaas_load_balancer: false +openshift_openstack_use_vm_load_balancer: false openshift_openstack_cluster_node_labels: app: diff --git a/roles/openshift_openstack/tasks/check-prerequisites.yml b/roles/openshift_openstack/tasks/check-prerequisites.yml index 2c5d5bdb27a..7ea06af1fd9 100644 --- a/roles/openshift_openstack/tasks/check-prerequisites.yml +++ b/roles/openshift_openstack/tasks/check-prerequisites.yml @@ -102,3 +102,12 @@ - { image: "{{ openshift_openstack_node_image }}", flavor: "{{ openshift_openstack_node_flavor }}" } - { image: "{{ openshift_openstack_lb_image }}", flavor: "{{ openshift_openstack_lb_flavor }}" } - { image: "{{ openshift_openstack_etcd_image }}", flavor: "{{ openshift_openstack_etcd_flavor }}" } + +- name: Check Load Balancer options + fail: + msg: > + Only one of `openshift_openstack_use_lbaas_load_balancer` and + `openshift_openstack_use_vm_load_balancer` can be true at a time. + when: + - openshift_openstack_use_lbaas_load_balancer + - openshift_openstack_use_vm_load_balancer diff --git a/roles/openshift_openstack/tasks/generate-dns.yml b/roles/openshift_openstack/tasks/generate-dns.yml index 8656c123c2c..dba95113c6e 100644 --- a/roles/openshift_openstack/tasks/generate-dns.yml +++ b/roles/openshift_openstack/tasks/generate-dns.yml @@ -52,25 +52,13 @@ with_items: "{{ groups['cluster_hosts'] }}" when: hostvars[item]['public_v4'] is defined -- name: "Add wildcard records to the public A records" +- name: "Add wildcard record to the public A records" set_fact: - public_records: "{{ public_records | default([]) + [ { 'type': 'A', 'hostname': '*.' + openshift_openstack_app_subdomain, 'ip': hostvars[item]['public_v4'] } ] }}" - with_items: "{{ groups['infra_hosts'] }}" - when: hostvars[item]['public_v4'] is defined - -- name: "Add public master cluster hostname records to the public A records (single master)" - set_fact: - public_records: "{{ public_records | default([]) + [ { 'type': 'A', 'hostname': (hostvars[groups.masters[0]].openshift_master_cluster_public_hostname | replace(openshift_openstack_full_dns_domain, ''))[:-1], 'ip': hostvars[groups.masters[0]].public_v4 } ] }}" - when: - - hostvars[groups.masters[0]].openshift_master_cluster_public_hostname is defined - - openshift_openstack_num_masters == 1 + public_records: "{{ public_records | default([]) + [ { 'type': 'A', 'hostname': '*.' + openshift_openstack_app_subdomain, 'ip': openshift_openstack_public_router_ip } ] }}" -- name: "Add public master cluster hostname records to the public A records (multi-master)" +- name: "Add the public API entry point record" set_fact: - public_records: "{{ public_records | default([]) + [ { 'type': 'A', 'hostname': (hostvars[groups.masters[0]].openshift_master_cluster_public_hostname | replace(openshift_openstack_full_dns_domain, ''))[:-1], 'ip': hostvars[groups.lb[0]].public_v4 } ] }}" - when: - - hostvars[groups.masters[0]].openshift_master_cluster_public_hostname is defined - - openshift_openstack_num_masters > 1 + public_records: "{{ public_records | default([]) + [ { 'type': 'A', 'hostname': (hostvars[groups.masters[0]].openshift_master_cluster_public_hostname | replace(openshift_openstack_full_dns_domain, ''))[:-1], 'ip': openshift_openstack_public_api_ip } ] }}" - name: "Set the public DNS server details to use the external value (if provided)" set_fact: diff --git a/roles/openshift_openstack/templates/heat_stack.yaml.j2 b/roles/openshift_openstack/templates/heat_stack.yaml.j2 index 7945739781e..dbd6eac4aee 100644 --- a/roles/openshift_openstack/templates/heat_stack.yaml.j2 +++ b/roles/openshift_openstack/templates/heat_stack.yaml.j2 @@ -54,6 +54,26 @@ outputs: description: Floating IPs of the nodes value: { get_attr: [ infra_nodes, floating_ip ] } + public_api_ip: + description: IP address for the API/UI endpoint +{% if openshift_openstack_use_lbaas_load_balancer %} + # TODO(shadower): Handle setups without floating IPs + value: { get_attr: [api_lb_floating_ip, floating_ip_address] } +{% elif openshift_openstack_use_vm_load_balancer %} + value: { get_attr: [loadbalancer, resource.0, floating_ip] } +{% else %} + value: { get_attr: [masters, resource.0, floating_ip] } +{% endif %} + + public_router_ip: + description: IP address of the apps/router endpoint +{% if openshift_openstack_use_lbaas_load_balancer %} + value: { get_attr: [router_lb_floating_ip, floating_ip_address] } +{% else %} + # NOTE(shadower): The VM-based loadbalancer only supports master nodes + value: { get_attr: [infra_nodes, resource.0, floating_ip] } +{% endif %} + {% if openshift_use_kuryr|default(false)|bool %} vm_subnet: description: ID of the subnet the Pods will be on @@ -89,8 +109,8 @@ conditions: resources: -{% if not openshift_openstack_provider_network_name %} -{% if openshift_use_kuryr|default(false)|bool %} +# NOTE: With Kuryr, the load balancer is necessary. +{% if openshift_openstack_use_lbaas_load_balancer or (openshift_use_kuryr|default(false)|bool and not openshift_openstack_provider_network_name) %} api_lb: type: OS::Neutron::LBaaS::LoadBalancer properties: @@ -99,8 +119,12 @@ resources: template: openshift-ansible-cluster_id-api-lb params: cluster_id: {{ openshift_openstack_full_dns_domain }} +{% if openshift_use_kuryr|default(false)|bool %} vip_address: {{ openshift_openstack_kuryr_service_subnet_cidr | ipaddr('1') | ipaddr('address') }} vip_subnet: { get_resource: service_subnet } +{% else %} + vip_subnet: { get_resource: subnet } +{% endif %} api_lb_listener: type: OS::Neutron::LBaaS::Listener @@ -112,7 +136,7 @@ resources: cluster_id: {{ openshift_openstack_full_dns_domain }} loadbalancer: { get_resource: api_lb } protocol: HTTPS - protocol_port: 443 + protocol_port: {{ openshift_master_api_port|default(8443) }} api_lb_pool: type: OS::Neutron::LBaaS::Pool @@ -123,9 +147,13 @@ resources: params: cluster_id: {{ openshift_openstack_full_dns_domain }} protocol: HTTPS + # TODO(shadower): Make this configurable? lb_algorithm: ROUND_ROBIN listener: { get_resource: api_lb_listener } +{% endif %} +{% if not openshift_openstack_provider_network_name %} +{% if openshift_use_kuryr|default(false)|bool %} pod_net: type: OS::Neutron::Net properties: @@ -505,7 +533,7 @@ resources: name: infra_server_group policies: {{ openshift_openstack_infra_server_group_policies }} {% endif %} -{% if openshift_openstack_num_masters|int > 1 %} +{% if openshift_openstack_use_vm_load_balancer %} loadbalancer: type: OS::Heat::ResourceGroup properties: @@ -594,6 +622,9 @@ resources: image: {{ openshift_openstack_master_image }} flavor: {{ openshift_openstack_master_flavor }} key_name: {{ openshift_openstack_keypair_name }} +{% if openshift_openstack_use_lbaas_load_balancer or openshift_use_kuryr|default(false)|bool %} + api_lb_pool: { get_resource: api_lb_pool } +{% endif %} {% if openshift_openstack_provider_network_name %} net: {{ openshift_openstack_provider_network_name }} net_name: {{ openshift_openstack_provider_network_name }} @@ -755,6 +786,10 @@ resources: image: {{ openshift_openstack_infra_image }} flavor: {{ openshift_openstack_infra_flavor }} key_name: {{ openshift_openstack_keypair_name }} +{% if openshift_openstack_use_lbaas_load_balancer %} + router_lb_pool_http: { get_resource: router_lb_pool_http } + router_lb_pool_https: { get_resource: router_lb_pool_https } +{% endif %} {% if openshift_openstack_provider_network_name %} net: {{ openshift_openstack_provider_network_name }} net_name: {{ openshift_openstack_provider_network_name }} @@ -873,3 +908,66 @@ resources: depends_on: - interface {% endif %} + + +{% if openshift_openstack_use_lbaas_load_balancer %} + api_lb_floating_ip: + condition: { not: no_floating } + depends_on: + - api_lb + - api_lb_listener + - api_lb_pool + type: OS::Neutron::FloatingIP + properties: + floating_network: {{ openshift_openstack_external_network_name }} + port_id: { get_attr: [api_lb, vip_port_id] } + + + router_lb: + type: OS::Neutron::LBaaS::LoadBalancer + properties: + vip_subnet: { get_resource: subnet } + + router_lb_floating_ip: + condition: { not: no_floating } + depends_on: + - router_lb + - router_lb_listener_http + - router_lb_pool_http + - router_lb_listener_https + - router_lb_pool_https + type: OS::Neutron::FloatingIP + properties: + floating_network: {{ openshift_openstack_external_network_name }} + port_id: { get_attr: [router_lb, vip_port_id] } + + router_lb_listener_http: + type: OS::Neutron::LBaaS::Listener + properties: + protocol: HTTP + protocol_port: 80 + loadbalancer: { get_resource: router_lb } + + router_lb_pool_http: + type: OS::Neutron::LBaaS::Pool + properties: + # TODO(shadower): Make this configurable? + lb_algorithm: ROUND_ROBIN + protocol: HTTP + listener: { get_resource: router_lb_listener_http } + + router_lb_listener_https: + type: OS::Neutron::LBaaS::Listener + properties: + protocol: HTTPS + protocol_port: 443 + loadbalancer: { get_resource: router_lb } + + router_lb_pool_https: + type: OS::Neutron::LBaaS::Pool + properties: + # TODO(shadower): Make this configurable? + lb_algorithm: ROUND_ROBIN + protocol: HTTPS + listener: { get_resource: router_lb_listener_https } +{% endif %} diff --git a/roles/openshift_openstack/templates/heat_stack_server.yaml.j2 b/roles/openshift_openstack/templates/heat_stack_server.yaml.j2 index 188f7008398..6f7c75fff90 100644 --- a/roles/openshift_openstack/templates/heat_stack_server.yaml.j2 +++ b/roles/openshift_openstack/templates/heat_stack_server.yaml.j2 @@ -109,13 +109,21 @@ parameters: label: Security groups description: Security group resources -{% if openshift_use_kuryr|default(false)|bool %} api_lb_pool: default: '' type: string label: API LoadBalancer pool ID description: API Loadbalancer pool resource -{% endif %} + + router_lb_pool_http: + type: string + label: Router LoadBalancer pool ID for HTTP + default: "" + + router_lb_pool_https: + type: string + label: Router LoadBalancer pool ID for HTTPS + default: "" {% if openshift_use_kuryr|default(false)|bool %} pod_secgrp: @@ -197,6 +205,7 @@ conditions: no_data_subnet: {not: { get_param: attach_data_net} } {% endif %} + resources: server: @@ -322,16 +331,32 @@ resources: {% endif %} -{% if openshift_use_kuryr|default(false)|bool %} - lb_member: + api_lb_member: type: OS::Neutron::LBaaS::PoolMember condition: - equals: - - get_param: type - - master + not: {equals: [{get_param: api_lb_pool}, ""]} properties: pool: { get_param: api_lb_pool } protocol_port: {{ openshift_master_api_port|default(8443) }} address: { get_attr: [server, first_address]} subnet: { get_param: subnet } -{% endif %} + + router_lb_pool_member_http: + condition: + not: {equals: [{get_param: router_lb_pool_http}, ""]} + type: OS::Neutron::LBaaS::PoolMember + properties: + pool: { get_param: router_lb_pool_http } + protocol_port: 80 + address: { get_attr: [server, first_address]} + subnet: { get_param: subnet } + + router_lb_pool_member_https: + condition: + not: {equals: [{get_param: router_lb_pool_https}, ""]} + type: OS::Neutron::LBaaS::PoolMember + properties: + pool: { get_param: router_lb_pool_https } + protocol_port: 443 + address: { get_attr: [server, first_address]} + subnet: { get_param: subnet }