[WIP] [templates/openstack] Dnshacks

templates/master/00-master/openstack/files/NetworkManager-dhcp-client-conf.yaml

@@ -0,0 +1,7 @@
+filesystem: "root"
+mode: 0644
+path: "/etc/NetworkManager/conf.d/dhcp-client.conf"
+contents:
+  inline: |
+    [main]
+    dhcp=dhclient

templates/master/00-master/openstack/files/dhcp-dhclient-conf.yaml

@@ -0,0 +1,7 @@
+filesystem: "root"
+mode: 0644
+path: "/etc/dhcp/dhclient.conf"
+contents:
+  inline: |
+    send dhcp-client-identifier = hardware;
+    prepend domain-name-servers 127.0.0.1;

templates/master/00-master/openstack/files/openstack-coredns-corefile.yaml

@@ -0,0 +1,23 @@
+filesystem: "root"
+mode: 0644
+path: "/etc/kubernetes/static-pod-resources/coredns/Corefile"
+contents:
+  inline: |
+    . {
+        log
+        errors
+
+        forward . 8.8.8.8 {
+            except {{.EtcdDiscoveryDomain}}
+        }
+    }
+
+    {{.EtcdDiscoveryDomain}} {
+        log
+        mdns {{.EtcdDiscoveryDomain}} 3 {{(split "." .EtcdDiscoveryDomain)._0}}
+        cache 30
+        reload 3s
+        loadbalance round_robin
+        file /etc/coredns/coredns.db {
+        }
+    }

templates/master/00-master/openstack/files/openstack-coredns-db.yaml

@@ -0,0 +1,23 @@
+filesystem: "root"
+mode: 0644
+path: "/etc/kubernetes/static-pod-resources/coredns/coredns.db.template"
+contents:
+  inline: |
+    $ORIGIN {{.EtcdDiscoveryDomain}}.
+    @    3600 IN SOA host.{{.EtcdDiscoveryDomain}}. hostmaster (
+                                    2          ; serial
+                                    7200       ; refresh (2 hours)
+                                    3600       ; retry (1 hour)
+                                    1209600    ; expire (2 weeks)
+                                    3600       ; minimum (1 hour)
+                                    )
+
+    api IN A $FLOATING_IP
+
+    *.apps IN A $MASTER_FIXED_IPS_0
+    *.apps IN A $MASTER_FIXED_IPS_1
+    *.apps IN A $MASTER_FIXED_IPS_2
+
+    api-int IN A $MASTER_FIXED_IPS_0
+    api-int IN A $MASTER_FIXED_IPS_1
+    api-int IN A $MASTER_FIXED_IPS_2

templates/master/00-master/openstack/files/openstack-coredns.yaml

@@ -0,0 +1,116 @@
+filesystem: "root"
+mode: 0644
+path: "/etc/kubernetes/manifests/coredns.yaml"
+contents:
+  inline: |
+    kind: Pod
+    apiVersion: v1
+    metadata:
+      name: coredns
+      namespace: kube-system
+      creationTimestamp:
+      deletionGracePeriodSeconds: 65
+      labels:
+        app: kni-infra-mdns
+    spec:
+      volumes:
+      - name: conf-dir
+        hostPath:
+          path: "/etc/kubernetes/static-pod-resources/coredns"
+      - name: var-dir
+        hostPath:
+          path: "/etc/kubernetes/static-pod-resources"
+      - name: chroot-host
+        hostPath:
+          path: "/"
+      - name: kublet
+        hostPath:
+          path: "/var/lib/kubelet"
+      initContainers:
+      - name: dns-render-config
+        image: quay.io/openshift/origin-node:latest
+        command:
+        - "/bin/bash"
+        - "-c"
+        - |
+          #/bin/bash
+          set -ex
+          source /tmp/clustervars
+
+          /usr/libexec/platform-python -c "from __future__ import print_function
+          import os
+          with open('/etc/coredns/coredns.db.template', 'r') as f:
+              content = f.read()
+          with open('/etc/coredns/coredns.db', 'w') as dest:
+              print(os.path.expandvars(content), file=dest)"
+        resources: {}
+        volumeMounts:
+        - name: conf-dir
+          mountPath: "/etc/coredns"
+        - name: var-dir
+          mountPath: "/tmp"
+        imagePullPolicy: IfNotPresent
+      containers:
+      - name: coredns
+        securityContext:
+          privileged: true
+        image: quay.io/openshift-metalkube/coredns-mdns:latest
+        args:
+        - "--conf"
+        - "/etc/coredns/Corefile"
+        resources:
+          requests:
+            cpu: 150m
+            memory: 1Gi
+        volumeMounts:
+        - name: conf-dir
+          mountPath: "/etc/coredns"
+        terminationMessagePolicy: FallbackToLogsOnError
+        imagePullPolicy: IfNotPresent
+      - name: machine-watcher
+        image: quay.io/openshift/origin-node:latest
+        command:
+        - "/bin/bash"
+        - "-c"
+        - |
+          #/bin/bash
+          set -e
+
+          while true; do
+            masters=$(/host/bin/oc get machines -n openshift-machine-api --config /var/lib/kubelet/kubeconfig | grep -o "{{.EtcdDiscoveryDomain}}-master-[0-9]\+" || true)
+            workers=$(/host/bin/oc get machines -n openshift-machine-api --config /var/lib/kubelet/kubeconfig | grep -o "{{.EtcdDiscoveryDomain}}-worker-[0-9A-Za-z]\+" || true)
+            nodes="$masters $workers"
+
+            if [ -n "$nodes" ]; then
+              for node in $nodes; do
+                  ip=$(/host/bin/oc describe machine -n openshift-machine-api $node --config /var/lib/kubelet/kubeconfig | grep "address" | grep -Eo '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' || true)
+
+                  # If there is an entry for the node, replace it with what is in the machines.yaml, ortherwise create an entry
+                  if [ -n "$ip" ] && [ -z "$(cat /etc/coredns/coredns.db | grep -w $node)" ]; then
+                      echo "$node IN A $ip"
+                      echo "$node IN A $ip" >> /etc/coredns/coredns.db
+                  else
+                      sed -i "0,/$node.*/s//$node IN A $ip/" /etc/coredns/coredns.db
+                  fi
+              done
+            fi
+
+          sleep 1
+          done
+        resources: {}
+        volumeMounts:
+        - name: conf-dir
+          mountPath: "/etc/coredns"
+        - name: var-dir
+          mountPath: "/tmp"
+        - name: chroot-host
+          mountPath: "/host"
+        - name: kublet
+          mountPath: "/var/lib/kubelet"
+        imagePullPolicy: IfNotPresent
+        terminationMessagePolicy: FallbackToLogsOnError
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+      priorityClassName: system-node-critical
+    status: {}

templates/master/00-master/openstack/files/openstack-mdns-config.yaml

@@ -0,0 +1,34 @@
+filesystem: "root"
+mode: 0644
+path: "/etc/kubernetes/static-pod-resources/mdns/config.template"
+contents:
+  inline: |
+    bind_address = "$NON_VIRTUAL_IP"
+    collision_avoidance = "hostname"
+
+    service {
+        name = "$CLUSTER_NAME Etcd"
+        host_name = "$ETCD_HOSTNAME"
+        type = "_etcd-server-ssl._tcp"
+        domain = "local."
+        port = 2380
+        ttl = 3200
+    }
+
+    service {
+        name = "$CLUSTER_NAME Workstation"
+        host_name = "$MASTER_HOSTNAME"
+        type = "_workstation._tcp"
+        domain = "local."
+        port = 42424
+        ttl = 3200
+    }
+
+    service {
+        name = "$CLUSTER_NAME EtcdWorkstation"
+        host_name = "$ETCD_HOSTNAME"
+        type = "_workstation._tcp"
+        domain = "local."
+        port = 42424
+        ttl = 300
+    }

templates/master/00-master/openstack/files/openstack-mdns-publisher.yaml

@@ -0,0 +1,77 @@
+filesystem: "root"
+mode: 0644
+path: "/etc/kubernetes/manifests/mdns-publisher.yaml"
+contents:
+  inline: |
+    kind: Pod
+    apiVersion: v1
+    metadata:
+      name: mdns-publisher
+      namespace: kube-system
+      creationTimestamp:
+      deletionGracePeriodSeconds: 65
+      labels:
+        app: kni-infra-mdns
+    spec:
+      volumes:
+      - name: resource-dir
+        hostPath:
+          path: "/etc/kubernetes/static-pod-resources/mdns"
+      - name: conf-dir
+        hostPath:
+          path: "/etc/kubernetes/static-pod-resources/mdns"
+      initContainers:
+      - name: render-config
+        image: quay.io/openshift/origin-node:latest
+        command:
+        - "/bin/bash"
+        - "-c"
+        - |
+          #/bin/bash
+          set -ex
+
+          export CLUSTER_NAME={{(split "." .EtcdDiscoveryDomain)._0}}
+          export DOMAIN={{.EtcdDiscoveryDomain}}
+
+          #TODO(trown): find a better way to get the host's IP and HOSTNAME
+          API_VIP="$(dig +noall +answer "api.${DOMAIN}" | awk '{print $NF}')"
+          SUBNET_CIDR="$(ip addr show | grep -v "scope host" | grep -Po 'inet \K[\d.]+/[\d.]+' | head -n1)"
+          PREFIX="${SUBNET_CIDR#*/}"
+          export NON_VIRTUAL_IP="${SUBNET_CIDR%/*}"
+
+          HOST_NUM="$(hostname -s | grep -o '[0-9]\+$')"
+          export MASTER_HOSTNAME="master-$HOST_NUM.local."
+          export ETCD_HOSTNAME="etcd-$HOST_NUM.local."
+
+          /usr/libexec/platform-python -c "from __future__ import print_function
+          import os
+          with open('/etc/kubernetes/static-pod-resources/config.template', 'r') as f:
+              content = f.read()
+          with open('/etc/mdns/config.hcl', 'w') as dest:
+              print(os.path.expandvars(content), file=dest)"
+        resources: {}
+        volumeMounts:
+        - name: resource-dir
+          mountPath: "/etc/kubernetes/static-pod-resources"
+        - name: conf-dir
+          mountPath: "/etc/mdns"
+        imagePullPolicy: IfNotPresent
+      containers:
+      - name: mdns-publisher
+        image: quay.io/openshift-metalkube/mdns-publisher:latest
+        args:
+        - "--debug"
+        resources:
+          requests:
+            cpu: 150m
+            memory: 1Gi
+        volumeMounts:
+        - name: conf-dir
+          mountPath: "/etc/mdns"
+        terminationMessagePolicy: FallbackToLogsOnError
+        imagePullPolicy: IfNotPresent
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+      priorityClassName: system-node-critical
+    status: {}

templates/worker/00-worker/openstack/files/openstack-mdns-config.yaml

@@ -0,0 +1,17 @@
+filesystem: "root"
+mode: 0644
+path: "/etc/kubernetes/static-pod-resources/mdns/config.template"
+contents:
+  inline: |
+
+    bind_address = "$NON_VIRTUAL_IP"
+    collision_avoidance = "hostname"
+
+    service {
+        name = "$CLUSTER_NAME Workstation"
+        host_name = "$SHORT_HOSTNAME"
+        type = "_workstation._tcp"
+        domain = "local."
+        port = 42424
+        ttl = 3200
+    }