Kuryr var generation in OSt dynamic inventory

playbooks/openstack/README.md

@@ -144,7 +144,19 @@ $ vi inventory/group_vars/all.yml
 4. Set the `openshift_openstack_default_flavor` to the flavor you want your
    OpenShift VMs to use.
    - See `openstack flavor list` for the list of available flavors.
+5. If you opt to use Kuryr for the networking, make sure that you review all
+   the kuryr options in the file. At the very least, if you use Kuryr, you
+   should uncomment:
 
+```bash
+#openshift_use_kuryr: True
+#use_trunk_ports: True
+#openshift_use_openshift_sdn: False
+#os_sdn_network_plugin_name: cni
+#openshift_node_proxy_mode: userspace
+#openshift_hosted_manage_registry: false
+#kuryr_openstack_public_subnet_id: uuid of my public subnet
+```
 
 
 #### OpenShift configuration

playbooks/openstack/inventory.py

@@ -11,6 +11,7 @@
 
 from collections import Mapping
 import json
+import os
 
 import shade
 
@@ -58,6 +59,7 @@ def base_openshift_inventory(cluster_hosts):
     inventory['glusterfs'] = {'hosts': cns}
     inventory['dns'] = {'hosts': dns}
     inventory['lb'] = {'hosts': load_balancers}
+    inventory['localhost'] = {'ansible_connection': 'local'}
 
     return inventory
 
@@ -75,6 +77,49 @@ def get_docker_storage_mountpoints(volumes):
     return docker_storage_mountpoints
 
 
+def _get_hostvars(server, docker_storage_mountpoints):
+    ssh_ip_address = server.public_v4 or server.private_v4
+    hostvars = {
+        'ansible_host': ssh_ip_address
+    }
+
+    public_v4 = server.public_v4 or server.private_v4
+    if public_v4:
+        hostvars['public_v4'] = server.public_v4
+        hostvars['openshift_public_ip'] = server.public_v4
+    # TODO(shadower): what about multiple networks?
+    if server.private_v4:
+        hostvars['private_v4'] = server.private_v4
+        hostvars['openshift_ip'] = server.private_v4
+
+        # NOTE(shadower): Yes, we set both hostname and IP to the private
+        # IP address for each node. OpenStack doesn't resolve nodes by
+        # name at all, so using a hostname here would require an internal
+        # DNS which would complicate the setup and potentially introduce
+        # performance issues.
+        hostvars['openshift_hostname'] = server.metadata.get(
+            'openshift_hostname', server.private_v4)
+    hostvars['openshift_public_hostname'] = server.name
+
+    if server.metadata['host-type'] == 'cns':
+        hostvars['glusterfs_devices'] = ['/dev/nvme0n1']
+
+    node_labels = server.metadata.get('node_labels')
+    # NOTE(shadower): the node_labels value must be a dict not string
+    if not isinstance(node_labels, Mapping):
+        node_labels = json.loads(node_labels)
+
+    if node_labels:
+        hostvars['openshift_node_labels'] = node_labels
+
+    # check for attached docker storage volumes
+    if 'os-extended-volumes:volumes_attached' in server:
+        if server.id in docker_storage_mountpoints:
+            hostvars['docker_storage_mountpoints'] = ' '.join(
+                docker_storage_mountpoints[server.id])
+    return hostvars
+
+
 def build_inventory():
     '''Build the dynamic inventory.'''
     cloud = shade.openstack_cloud()
@@ -97,51 +142,78 @@ def build_inventory():
     inventory['_meta'] = {'hostvars': {}}
 
     # cinder volumes used for docker storage
-    docker_storage_mountpoints = get_docker_storage_mountpoints(cloud.list_volumes())
-
+    docker_storage_mountpoints = get_docker_storage_mountpoints(
+        cloud.list_volumes())
     for server in cluster_hosts:
-        ssh_ip_address = server.public_v4 or server.private_v4
-        hostvars = {
-            'ansible_host': ssh_ip_address
-        }
-
-        public_v4 = server.public_v4 or server.private_v4
-        if public_v4:
-            hostvars['public_v4'] = server.public_v4
-            hostvars['openshift_public_ip'] = server.public_v4
-        # TODO(shadower): what about multiple networks?
-        if server.private_v4:
-            hostvars['private_v4'] = server.private_v4
-            hostvars['openshift_ip'] = server.private_v4
-
-            # NOTE(shadower): Yes, we set both hostname and IP to the private
-            # IP address for each node. OpenStack doesn't resolve nodes by
-            # name at all, so using a hostname here would require an internal
-            # DNS which would complicate the setup and potentially introduce
-            # performance issues.
-            hostvars['openshift_hostname'] = server.metadata.get(
-                'openshift_hostname', server.private_v4)
-        hostvars['openshift_public_hostname'] = server.name
-
-        if server.metadata['host-type'] == 'cns':
-            hostvars['glusterfs_devices'] = ['/dev/nvme0n1']
-
-        node_labels = server.metadata.get('node_labels')
-        # NOTE(shadower): the node_labels value must be a dict not string
-        if not isinstance(node_labels, Mapping):
-            node_labels = json.loads(node_labels)
-
-        if node_labels:
-            hostvars['openshift_node_labels'] = node_labels
-
-        # check for attached docker storage volumes
-        if 'os-extended-volumes:volumes_attached' in server:
-            if server.id in docker_storage_mountpoints:
-                hostvars['docker_storage_mountpoints'] = ' '.join(docker_storage_mountpoints[server.id])
-
-        inventory['_meta']['hostvars'][server.name] = hostvars
+        inventory['_meta']['hostvars'][server.name] = _get_hostvars(
+            server,
+            docker_storage_mountpoints)
+
+    stout = _get_stack_outputs(cloud)
+    if stout is not None:
+        try:
+            inventory['localhost'].update({
+                'openshift_openstack_api_lb_provider':
+                stout['api_lb_provider'],
+                'openshift_openstack_api_lb_port_id':
+                stout['api_lb_vip_port_id'],
+                'openshift_openstack_api_lb_sg_id':
+                stout['api_lb_sg_id']})
+        except KeyError:
+            pass  # Not an API load balanced deployment
+
+        try:
+            inventory['OSEv3']['vars'] = _get_kuryr_vars(cloud, stout)
+        except KeyError:
+            pass  # Not a kuryr deployment
     return inventory
 
 
+def _get_stack_outputs(cloud_client):
+    """Returns a dictionary with the stack outputs"""
+    cluster_name = os.getenv('OPENSHIFT_CLUSTER', 'openshift-cluster')
+
+    stack = cloud_client.get_stack(cluster_name)
+    if stack is None or stack['stack_status'] not in (
+            'CREATE_COMPLETE', 'UPDATE_COMPLETE'):
+        return None
+
+    data = {}
+    for output in stack['outputs']:
+        data[output['output_key']] = output['output_value']
+    return data
+
+
+def _get_kuryr_vars(cloud_client, data):
+    """Returns a dictionary of Kuryr variables resulting of heat stacking"""
+    settings = {}
+    settings['kuryr_openstack_pod_subnet_id'] = data['pod_subnet']
+    settings['kuryr_openstack_worker_nodes_subnet_id'] = data['vm_subnet']
+    settings['kuryr_openstack_service_subnet_id'] = data['service_subnet']
+    settings['kuryr_openstack_pod_sg_id'] = data['pod_access_sg_id']
+    settings['kuryr_openstack_pod_project_id'] = (
+        cloud_client.current_project_id)
+
+    settings['kuryr_openstack_auth_url'] = cloud_client.auth['auth_url']
+    settings['kuryr_openstack_username'] = cloud_client.auth['username']
+    settings['kuryr_openstack_password'] = cloud_client.auth['password']
+    if 'user_domain_id' in cloud_client.auth:
+        settings['kuryr_openstack_user_domain_name'] = (
+            cloud_client.auth['user_domain_id'])
+    else:
+        settings['kuryr_openstack_user_domain_name'] = (
+            cloud_client.auth['user_domain_name'])
+    # FIXME(apuimedo): consolidate kuryr controller credentials into the same
+    #                  vars the openstack playbook uses.
+    settings['kuryr_openstack_project_id'] = cloud_client.current_project_id
+    if 'project_domain_id' in cloud_client.auth:
+        settings['kuryr_openstack_project_domain_name'] = (
+            cloud_client.auth['project_domain_id'])
+    else:
+        settings['kuryr_openstack_project_domain_name'] = (
+            cloud_client.auth['project_domain_name'])
+    return settings
+
+
 if __name__ == '__main__':
     print(json.dumps(build_inventory(), indent=4, sort_keys=True))

playbooks/openstack/openshift-cluster/provision.yml

@@ -2,6 +2,9 @@
 - name: Create the OpenStack resources for cluster installation
   hosts: localhost
   tasks:
+  - name: retrieve cluster name from the environment if present
+    set_fact:
+      openshift_openstack_stack_name: "{{ lookup('env', 'OPENSHIFT_CLUSTER') | ternary (lookup('env', 'OPENSHIFT_CLUSTER'), omit) }}"
   - name: provision cluster
     import_role:
       name: openshift_openstack

playbooks/openstack/sample-inventory/group_vars/all.yml

@@ -20,6 +20,50 @@ openshift_openstack_external_network_name: "public"
 # # NOTE: this is only supported with Flannel SDN yet
 #openstack_private_data_network_name: "openshift-ansible-{{ openshift_openstack_stack_name }}-data-net"
 
+## Kuryr networking
+# TODO: Allow the user to specify pre-existing subnets for pod and services
+#openshift_openstack_kuryr_service_subnet_cidr: "172.30.0.0/16"
+
+#
+## You can alter the port pooling defaults here
+#kuryr_openstack_enable_pools: True
+#kuryr_openstack_pool_max: 0
+#kuryr_openstack_pool_min: 1
+#kuryr_openstack_pool_batch: 5
+#kuryr_openstack_pool_update_frequency: 20
+#
+## You should set the following if you want to use Kuryr/Neutron as your SDN
+#openshift_use_kuryr: True
+#openshift_use_openshift_sdn: False
+
+# NOTE: you must uncomment these for Kuryr to work properly as well:
+# openshift_master_open_ports:
+# - service: dns tcp
+#   port: 53/tcp
+# - service: dns udp
+#   port: 53/udp
+# openshift_node_open_ports:
+# - service: dns tcp
+#   port: 53/tcp
+# - service: dns udp
+#   port: 53/udp
+
+#use_trunk_ports: True
+#os_sdn_network_plugin_name: cni
+#openshift_node_proxy_mode: userspace
+# # Kuryr needs to have the pod based registry (if desired in the cluster)
+# deployed after kuryr is up and running. This can be done with oadm
+# #Disable management of the OpenShift Registry
+#openshift_hosted_manage_registry: false
+# # Kuryr needs to know the subnet you will be taking Floating IPs for the
+# loadbalancer services from.
+# kuryr_openstack_public_subnet_id: uuid_of_my_fip_subnet
+
+# If you VM images will name the ethernet device different than 'eth0',
+# override this
+#kuryr_cni_link_interface: eth0
+
+
 ## If you want to use a provider network, set its name here.
 ## NOTE: the `openshift_openstack_external_network_name` and
 ## `openshift_openstack_private_network_name` options will be ignored when using a

roles/kuryr/defaults/main.yaml

@@ -12,17 +12,24 @@ kuryr_openstack_user_domain_name: default
 kuryr_openstack_project_domain_name: default
 
 # Kuryr OpenShift namespace
-kuryr_namespace: kube-system
+kuryr_namespace: openshift-infra
 
 # Whether to run the cni plugin in debug mode
 kuryr_cni_debug: "false"
 
+# Default pod-in-VM link interface
+kuryr_cni_link_interface: eth0
+
 # The version of cni binaries
 cni_version: v0.5.2
 
 # Path to bin dir (where kuryr execs get installed)
 bin_dir: /usr/bin
 
+# Default controller and CNI images
+openshift_openstack_kuryr_controller_image: kuryr/controller:latest
+openshift_openstack_kuryr_cni_image: kuryr/cni:latest
+
 # Path to the cni binaries
 cni_bin_dir: /opt/cni/bin
 

roles/kuryr/templates/cni-daemonset.yaml.j2

@@ -23,7 +23,7 @@ spec:
       serviceAccountName: kuryr-controller
       containers:
       - name: kuryr-cni
-        image: kuryr/cni:latest
+        image: {{ openshift_openstack_kuryr_cni_image }}
         imagePullPolicy: IfNotPresent
         command: [ "cni_ds_init" ]
         env:

roles/kuryr/templates/configmap.yaml.j2

@@ -141,15 +141,7 @@ data:
     # Driver to use for binding and unbinding ports. (string value)
     # Deprecated group/name - [binding]/driver
     #default_driver = kuryr.lib.binding.drivers.veth
-
-    # Drivers to use for binding and unbinding ports. (list value)
-    #enabled_drivers = kuryr.lib.binding.drivers.veth
-
-    # Specifies the name of the Nova instance interface to link the virtual devices
-    # to (only applicable to some binding drivers. (string value)
-    link_iface = eth0
-
-    driver = kuryr.lib.binding.drivers.vlan
+    default_driver = kuryr.lib.binding.drivers.vlan
 
 
     [cni_daemon]
@@ -301,7 +293,7 @@ data:
     # TODO (apuimedo): Remove the duplicated line just after this one once the
     # RDO packaging contains the upstream patch
     worker_nodes_subnet = {{ kuryr_openstack_worker_nodes_subnet_id }}
-    external_svc_subnet = {{ kuryr_openstack_external_svc_subnet_id }}
+    external_svc_subnet = {{ kuryr_openstack_public_subnet_id }}
 
     [pod_vif_nested]
 
@@ -466,21 +458,9 @@ data:
     # From kuryr_kubernetes
     #
 
-    # The name prefix of the veth endpoint put inside the container. (string value)
-    #veth_dst_prefix = eth
-
-    # Driver to use for binding and unbinding ports. (string value)
-    # Deprecated group/name - [binding]/driver
-    #default_driver = kuryr.lib.binding.drivers.veth
-
-    # Drivers to use for binding and unbinding ports. (list value)
-    #enabled_drivers = kuryr.lib.binding.drivers.veth
-
     # Specifies the name of the Nova instance interface to link the virtual devices
     # to (only applicable to some binding drivers. (string value)
-    link_iface = eth0
-
-    driver = kuryr.lib.binding.drivers.vlan
+    link_iface = {{ kuryr_cni_link_interface }}
 
 
     [cni_daemon]

roles/kuryr/templates/controller-deployment.yaml.j2

@@ -19,7 +19,7 @@ spec:
       automountServiceAccountToken: true
       hostNetwork: true
       containers:
-      - image: kuryr/controller:latest
+      - image: {{ openshift_openstack_kuryr_controller_image }}
         imagePullPolicy: IfNotPresent
         name: controller
 {% if kuryr_openstack_enable_pools | default(false) %}

roles/lib_utils/action_plugins/sanity_checks.py

@@ -15,7 +15,8 @@
                    ('openshift_use_flannel', False),
                    ('openshift_use_nuage', False),
                    ('openshift_use_contiv', False),
-                   ('openshift_use_calico', False))
+                   ('openshift_use_calico', False),
+                   ('openshift_use_kuryr', False))
 
 ENTERPRISE_TAG_REGEX_ERROR = """openshift_image_tag must be in the format
 v#.#[.#[.#]]. Examples: v1.2, v3.4.1, v3.5.1.3,

roles/openshift_openstack/defaults/main.yml

@@ -53,10 +53,14 @@ openshift_openstack_app_subdomain: "apps"
 
 # heat vars
 openshift_openstack_clusterid: openshift
-openshift_openstack_stack_name: "{{ openshift_openstack_clusterid }}.{{ openshift_openstack_public_dns_domain }}"
+openshift_openstack_stack_name: "openshift-cluster"
 openshift_openstack_subnet_cidr: "192.168.99.0/24"
 openshift_openstack_pool_start: "192.168.99.3"
 openshift_openstack_pool_end: "192.168.99.254"
+openshift_openstack_kuryr_service_subnet_cidr: "172.30.0.0/16"
+openshift_openstack_kuryr_service_pool_start: "172.30.128.1"
+openshift_openstack_kuryr_service_pool_end: "172.30.255.253"
+openshift_openstack_kuryr_pod_subnet_cidr: "10.11.0.0/16"
 openshift_openstack_master_hostname: master
 openshift_openstack_infra_hostname: infra-node
 openshift_openstack_cns_hostname: cns