diff --git a/manifests/openstack/coredns-corefile.tmpl b/manifests/openstack/coredns-corefile.tmpl new file mode 100644 index 0000000000..28430cfcdc --- /dev/null +++ b/manifests/openstack/coredns-corefile.tmpl @@ -0,0 +1,12 @@ +. { + errors + health + mdns {{ .ControllerConfig.EtcdDiscoveryDomain }} {{`{{.Cluster.MasterAmount}}`}} {{`{{.Cluster.Name}}`}} + forward . {{`{{- range $upstream := .DNSUpstreams}} {{$upstream}}{{- end}}`}} + cache 30 + reload + hosts /etc/coredns/api-int.hosts {{ .ControllerConfig.EtcdDiscoveryDomain }} { + {{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }} api-int.{{ .ControllerConfig.EtcdDiscoveryDomain }} api.{{ .ControllerConfig.EtcdDiscoveryDomain }} + fallthrough + } +} diff --git a/manifests/openstack/coredns.yaml b/manifests/openstack/coredns.yaml new file mode 100644 index 0000000000..e95af26571 --- /dev/null +++ b/manifests/openstack/coredns.yaml @@ -0,0 +1,92 @@ +--- +kind: Pod +apiVersion: v1 +metadata: + name: coredns + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-mdns +spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/coredns" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + empty-dir: {} + - name: manifests + hostPath: + path: "/opt/openshift/manifests" + initContainers: + - name: render-config + image: {{ .Images.BaremetalRuntimeCfgBootstrap }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/coredns" + - "--cluster-config" + - "/opt/openshift/manifests/cluster-config.yaml" + resources: {} + volumeMounts: + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: resource-dir + mountPath: "/config" + - name: conf-dir + mountPath: "/etc/coredns" + - name: manifests + mountPath: "/opt/openshift/manifests" + imagePullPolicy: IfNotPresent + containers: + - name: coredns + securityContext: + privileged: true + image: {{ .Images.CorednsBootstrap }} + args: + - "--conf" + - "/etc/coredns/Corefile" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/coredns" + readinessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical +status: {} diff --git a/manifests/openstack/keepalived.conf.tmpl b/manifests/openstack/keepalived.conf.tmpl new file mode 100644 index 0000000000..7ae696c17d --- /dev/null +++ b/manifests/openstack/keepalived.conf.tmpl @@ -0,0 +1,35 @@ +# Configuration template for Keepalived, which is used to manage the DNS and +# API VIPs. +# +# For more information, see installer/data/data/bootstrap/baremetal/README.md +# in the installer repo. + +vrrp_instance {{`{{.Cluster.Name}}`}}_API { + state BACKUP + interface {{`{{.VRRPInterface}}`}} + virtual_router_id {{`{{.Cluster.APIVirtualRouterID }}`}} + priority 50 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{`{{.Cluster.Name}}`}}_api_vip + } + virtual_ipaddress { + {{`{{ .Cluster.APIVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + } +} + +vrrp_instance {{`{{.Cluster.Name}}`}}_DNS { + state MASTER + interface {{`{{.VRRPInterface}}`}} + virtual_router_id {{`{{.Cluster.DNSVirtualRouterID }}`}} + priority 140 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{`{{.Cluster.Name}}`}}_dns_vip + } + virtual_ipaddress { + {{`{{ .Cluster.DNSVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + } +} diff --git a/manifests/openstack/keepalived.yaml b/manifests/openstack/keepalived.yaml new file mode 100644 index 0000000000..281ef32e3b --- /dev/null +++ b/manifests/openstack/keepalived.yaml @@ -0,0 +1,73 @@ +--- +kind: Pod +apiVersion: v1 +metadata: + name: keepalived + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-vrrp +spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/keepalived" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + empty-dir: {} + initContainers: + - name: render-config + image: {{ .Images.BaremetalRuntimeCfgBootstrap }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/keepalived" + resources: {} + volumeMounts: + - name: resource-dir + mountPath: "/config" + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: conf-dir + mountPath: "/etc/keepalived" + imagePullPolicy: IfNotPresent + containers: + - name: keepalived + securityContext: + privileged: true + image: {{ .Images.KeepalivedBootstrap }} + command: + - /usr/sbin/keepalived + args: + - "-f" + - "/etc/keepalived/keepalived.conf" + - "--dont-fork" + - "--vrrp" + - "--log-detail" + - "--log-console" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/keepalived" + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical +status: {} diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index 8980cf170a..5048439298 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -180,6 +180,11 @@ func generateMachineConfigForName(config *RenderConfig, role, name, templateDir, platformDirs := []string{} // Loop over templates/common which applies everywhere for _, dir := range []string{platformBase, platform} { + // Bypass OpenStack template rendering until + // https://github.com/openshift/installer/pull/1959 merges + if dir == platformOpenStack && config.ControllerConfigSpec.Infra.Status.PlatformStatus.OpenStack == nil { + continue + } basePath := filepath.Join(templateDir, "common", dir) exists, err := existsDir(basePath) if err != nil { @@ -192,6 +197,11 @@ func generateMachineConfigForName(config *RenderConfig, role, name, templateDir, } // And now over the target e.g. templates/master for _, dir := range []string{platformBase, platform} { + // Bypass OpenStack template rendering until + // https://github.com/openshift/installer/pull/1959 merges + if dir == platformOpenStack && config.ControllerConfigSpec.Infra.Status.PlatformStatus.OpenStack == nil { + continue + } platformPath := filepath.Join(path, dir) exists, err := existsDir(platformPath) if err != nil { diff --git a/pkg/controller/template/test_data/controller_config_openstack.yaml b/pkg/controller/template/test_data/controller_config_openstack.yaml index 4bcc704152..22e7d9cd28 100644 --- a/pkg/controller/template/test_data/controller_config_openstack.yaml +++ b/pkg/controller/template/test_data/controller_config_openstack.yaml @@ -19,3 +19,10 @@ spec: setupEtcdEnv: image/setupEtcdEnv:1 infraImage: image/infraImage:1 kubeClientAgentImage: image/kubeClientAgentImage:1 + infra: + status: + platformStatus: + openstack: + apiServerInternalIP: 10.0.0.1 + ingressIP: 10.0.0.2 + nodeDNSIP: 10.0.0.3 diff --git a/pkg/operator/assets/bindata.go b/pkg/operator/assets/bindata.go index 72b233647a..6b7a42fc53 100644 --- a/pkg/operator/assets/bindata.go +++ b/pkg/operator/assets/bindata.go @@ -32,6 +32,10 @@ // manifests/machineconfigserver/node-bootstrapper-token.yaml // manifests/machineconfigserver/sa.yaml // manifests/master.machineconfigpool.yaml +// manifests/openstack/coredns-corefile.tmpl +// manifests/openstack/coredns.yaml +// manifests/openstack/keepalived.conf.tmpl +// manifests/openstack/keepalived.yaml // manifests/worker.machineconfigpool.yaml package assets @@ -1598,6 +1602,286 @@ func manifestsMasterMachineconfigpoolYaml() (*asset, error) { return a, nil } +var _manifestsOpenstackCorednsCorefileTmpl = []byte(`. { + errors + health + mdns {{ .ControllerConfig.EtcdDiscoveryDomain }} {{`+"`"+`{{.Cluster.MasterAmount}}`+"`"+`}} {{`+"`"+`{{.Cluster.Name}}`+"`"+`}} + forward . {{`+"`"+`{{- range $upstream := .DNSUpstreams}} {{$upstream}}{{- end}}`+"`"+`}} + cache 30 + reload + hosts /etc/coredns/api-int.hosts {{ .ControllerConfig.EtcdDiscoveryDomain }} { + {{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }} api-int.{{ .ControllerConfig.EtcdDiscoveryDomain }} api.{{ .ControllerConfig.EtcdDiscoveryDomain }} + fallthrough + } +} +`) + +func manifestsOpenstackCorednsCorefileTmplBytes() ([]byte, error) { + return _manifestsOpenstackCorednsCorefileTmpl, nil +} + +func manifestsOpenstackCorednsCorefileTmpl() (*asset, error) { + bytes, err := manifestsOpenstackCorednsCorefileTmplBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/openstack/coredns-corefile.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _manifestsOpenstackCorednsYaml = []byte(`--- +kind: Pod +apiVersion: v1 +metadata: + name: coredns + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-mdns +spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/coredns" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + empty-dir: {} + - name: manifests + hostPath: + path: "/opt/openshift/manifests" + initContainers: + - name: render-config + image: {{ .Images.BaremetalRuntimeCfgBootstrap }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/coredns" + - "--cluster-config" + - "/opt/openshift/manifests/cluster-config.yaml" + resources: {} + volumeMounts: + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: resource-dir + mountPath: "/config" + - name: conf-dir + mountPath: "/etc/coredns" + - name: manifests + mountPath: "/opt/openshift/manifests" + imagePullPolicy: IfNotPresent + containers: + - name: coredns + securityContext: + privileged: true + image: {{ .Images.CorednsBootstrap }} + args: + - "--conf" + - "/etc/coredns/Corefile" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/coredns" + readinessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical +status: {} +`) + +func manifestsOpenstackCorednsYamlBytes() ([]byte, error) { + return _manifestsOpenstackCorednsYaml, nil +} + +func manifestsOpenstackCorednsYaml() (*asset, error) { + bytes, err := manifestsOpenstackCorednsYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/openstack/coredns.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _manifestsOpenstackKeepalivedConfTmpl = []byte(`# Configuration template for Keepalived, which is used to manage the DNS and +# API VIPs. +# +# For more information, see installer/data/data/bootstrap/baremetal/README.md +# in the installer repo. + +vrrp_instance {{`+"`"+`{{.Cluster.Name}}`+"`"+`}}_API { + state BACKUP + interface {{`+"`"+`{{.VRRPInterface}}`+"`"+`}} + virtual_router_id {{`+"`"+`{{.Cluster.APIVirtualRouterID }}`+"`"+`}} + priority 50 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{`+"`"+`{{.Cluster.Name}}`+"`"+`}}_api_vip + } + virtual_ipaddress { + {{`+"`"+`{{ .Cluster.APIVIP }}`+"`"+`}}/{{`+"`"+`{{ .Cluster.VIPNetmask }}`+"`"+`}} + } +} + +vrrp_instance {{`+"`"+`{{.Cluster.Name}}`+"`"+`}}_DNS { + state MASTER + interface {{`+"`"+`{{.VRRPInterface}}`+"`"+`}} + virtual_router_id {{`+"`"+`{{.Cluster.DNSVirtualRouterID }}`+"`"+`}} + priority 140 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{`+"`"+`{{.Cluster.Name}}`+"`"+`}}_dns_vip + } + virtual_ipaddress { + {{`+"`"+`{{ .Cluster.DNSVIP }}`+"`"+`}}/{{`+"`"+`{{ .Cluster.VIPNetmask }}`+"`"+`}} + } +} +`) + +func manifestsOpenstackKeepalivedConfTmplBytes() ([]byte, error) { + return _manifestsOpenstackKeepalivedConfTmpl, nil +} + +func manifestsOpenstackKeepalivedConfTmpl() (*asset, error) { + bytes, err := manifestsOpenstackKeepalivedConfTmplBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/openstack/keepalived.conf.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _manifestsOpenstackKeepalivedYaml = []byte(`--- +kind: Pod +apiVersion: v1 +metadata: + name: keepalived + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-vrrp +spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/keepalived" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + empty-dir: {} + initContainers: + - name: render-config + image: {{ .Images.BaremetalRuntimeCfgBootstrap }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.OpenStack.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/keepalived" + resources: {} + volumeMounts: + - name: resource-dir + mountPath: "/config" + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: conf-dir + mountPath: "/etc/keepalived" + imagePullPolicy: IfNotPresent + containers: + - name: keepalived + securityContext: + privileged: true + image: {{ .Images.KeepalivedBootstrap }} + command: + - /usr/sbin/keepalived + args: + - "-f" + - "/etc/keepalived/keepalived.conf" + - "--dont-fork" + - "--vrrp" + - "--log-detail" + - "--log-console" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/keepalived" + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical +status: {} +`) + +func manifestsOpenstackKeepalivedYamlBytes() ([]byte, error) { + return _manifestsOpenstackKeepalivedYaml, nil +} + +func manifestsOpenstackKeepalivedYaml() (*asset, error) { + bytes, err := manifestsOpenstackKeepalivedYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/openstack/keepalived.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _manifestsWorkerMachineconfigpoolYaml = []byte(`apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool metadata: @@ -1711,6 +1995,10 @@ var _bindata = map[string]func() (*asset, error){ "manifests/machineconfigserver/node-bootstrapper-token.yaml": manifestsMachineconfigserverNodeBootstrapperTokenYaml, "manifests/machineconfigserver/sa.yaml": manifestsMachineconfigserverSaYaml, "manifests/master.machineconfigpool.yaml": manifestsMasterMachineconfigpoolYaml, + "manifests/openstack/coredns-corefile.tmpl": manifestsOpenstackCorednsCorefileTmpl, + "manifests/openstack/coredns.yaml": manifestsOpenstackCorednsYaml, + "manifests/openstack/keepalived.conf.tmpl": manifestsOpenstackKeepalivedConfTmpl, + "manifests/openstack/keepalived.yaml": manifestsOpenstackKeepalivedYaml, "manifests/worker.machineconfigpool.yaml": manifestsWorkerMachineconfigpoolYaml, } @@ -1796,6 +2084,12 @@ var _bintree = &bintree{nil, map[string]*bintree{ "sa.yaml": &bintree{manifestsMachineconfigserverSaYaml, map[string]*bintree{}}, }}, "master.machineconfigpool.yaml": &bintree{manifestsMasterMachineconfigpoolYaml, map[string]*bintree{}}, + "openstack": &bintree{nil, map[string]*bintree{ + "coredns-corefile.tmpl": &bintree{manifestsOpenstackCorednsCorefileTmpl, map[string]*bintree{}}, + "coredns.yaml": &bintree{manifestsOpenstackCorednsYaml, map[string]*bintree{}}, + "keepalived.conf.tmpl": &bintree{manifestsOpenstackKeepalivedConfTmpl, map[string]*bintree{}}, + "keepalived.yaml": &bintree{manifestsOpenstackKeepalivedYaml, map[string]*bintree{}}, + }}, "worker.machineconfigpool.yaml": &bintree{manifestsWorkerMachineconfigpoolYaml, map[string]*bintree{}}, }}, }} diff --git a/pkg/operator/bootstrap.go b/pkg/operator/bootstrap.go index 762a146b61..bed7398af1 100644 --- a/pkg/operator/bootstrap.go +++ b/pkg/operator/bootstrap.go @@ -189,6 +189,27 @@ func RenderBootstrap( filename: "baremetal/static-pod-resources/keepalived/keepalived.conf.tmpl", }}...) } + + if infra.Status.PlatformStatus.OpenStack != nil { + manifests = append(manifests, []struct { + name string + data []byte + filename string + }{{ + name: "manifests/openstack/coredns.yaml", + filename: "openstack/manifests/coredns.yaml", + }, { + name: "manifests/openstack/coredns-corefile.tmpl", + filename: "openstack/static-pod-resources/coredns/Corefile.tmpl", + }, { + name: "manifests/openstack/keepalived.yaml", + filename: "openstack/manifests/keepalived.yaml", + }, { + name: "manifests/openstack/keepalived.conf.tmpl", + filename: "openstack/static-pod-resources/keepalived/keepalived.conf.tmpl", + }}...) + } + for _, m := range manifests { var b []byte var err error diff --git a/pkg/operator/render.go b/pkg/operator/render.go index 8594eba65b..13cbe962ee 100644 --- a/pkg/operator/render.go +++ b/pkg/operator/render.go @@ -25,6 +25,7 @@ type renderConfig struct { APIServerURL string Images *RenderConfigImages KubeAPIServerServingCA string + Infra configv1.Infrastructure } func renderAsset(config *renderConfig, path string) ([]byte, error) { diff --git a/templates/common/openstack/files/openstack-NetworkManager-kni-conf.yaml b/templates/common/openstack/files/openstack-NetworkManager-kni-conf.yaml new file mode 100644 index 0000000000..004c551320 --- /dev/null +++ b/templates/common/openstack/files/openstack-NetworkManager-kni-conf.yaml @@ -0,0 +1,7 @@ +filesystem: "root" +mode: 0644 +path: "/etc/NetworkManager/conf.d/99-kni.conf" +contents: + inline: | + [main] + dhcp=dhclient diff --git a/templates/common/openstack/files/openstack-coredns-corefile.yaml b/templates/common/openstack/files/openstack-coredns-corefile.yaml new file mode 100644 index 0000000000..e249f54312 --- /dev/null +++ b/templates/common/openstack/files/openstack-coredns-corefile.yaml @@ -0,0 +1,14 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/coredns/Corefile.tmpl" +contents: + inline: | + . { + errors + health + mdns {{ .EtcdDiscoveryDomain }} 0 {{`{{.Cluster.Name}}`}} + forward . {{`{{- range $upstream := .DNSUpstreams}} {{$upstream}}{{- end}}`}} + cache 30 + reload + file /etc/coredns/node-dns-db {{ .EtcdDiscoveryDomain }} + } diff --git a/templates/common/openstack/files/openstack-coredns-db.yaml b/templates/common/openstack/files/openstack-coredns-db.yaml new file mode 100644 index 0000000000..8bb236060b --- /dev/null +++ b/templates/common/openstack/files/openstack-coredns-db.yaml @@ -0,0 +1,17 @@ +filesystem: "root" +mode: 0644 +path: "/etc/coredns/node-dns-db" +contents: + inline: | + $ORIGIN {{ .EtcdDiscoveryDomain }}. + @ 3600 IN SOA host.{{ .EtcdDiscoveryDomain }}. hostmaster ( + 2017042752 ; serial + 7200 ; refresh (2 hours) + 3600 ; retry (1 hour) + 1209600 ; expire (2 weeks) + 3600 ; minimum (1 hour) + ) + api-int IN A {{ .Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }} + api IN A {{ .Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }} + + *.apps IN A {{ .Infra.Status.PlatformStatus.OpenStack.IngressIP }} diff --git a/templates/common/openstack/files/openstack-coredns.yaml b/templates/common/openstack/files/openstack-coredns.yaml new file mode 100644 index 0000000000..5738a731bc --- /dev/null +++ b/templates/common/openstack/files/openstack-coredns.yaml @@ -0,0 +1,91 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/manifests/coredns.yaml" +contents: + inline: | + kind: Pod + apiVersion: v1 + metadata: + name: coredns + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-mdns + spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/coredns" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + hostPath: + path: "/etc/coredns" + initContainers: + - name: render-config + image: {{ .Images.baremetalRuntimeCfgImage }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .Infra.Status.PlatformStatus.OpenStack.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .Infra.Status.PlatformStatus.OpenStack.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/coredns" + resources: {} + volumeMounts: + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: resource-dir + mountPath: "/config" + - name: conf-dir + mountPath: "/etc/coredns" + imagePullPolicy: IfNotPresent + containers: + - name: coredns + securityContext: + privileged: true + image: {{.Images.corednsImage}} + args: + - "--conf" + - "/etc/coredns/Corefile" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/coredns" + readinessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical + status: {} \ No newline at end of file diff --git a/templates/common/openstack/files/openstack-keepalived.yaml b/templates/common/openstack/files/openstack-keepalived.yaml new file mode 100644 index 0000000000..691c2598bd --- /dev/null +++ b/templates/common/openstack/files/openstack-keepalived.yaml @@ -0,0 +1,84 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/manifests/keepalived.yaml" +contents: + inline: | + kind: Pod + apiVersion: v1 + metadata: + name: keepalived + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-vrrp + spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/keepalived" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + hostPath: + path: "/etc/keepalived" + initContainers: + - name: render-config + image: {{ .Images.baremetalRuntimeCfgImage }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .Infra.Status.PlatformStatus.OpenStack.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .Infra.Status.PlatformStatus.OpenStack.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/keepalived" + resources: {} + volumeMounts: + - name: resource-dir + mountPath: "/config" + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: conf-dir + mountPath: "/etc/keepalived" + imagePullPolicy: IfNotPresent + containers: + - name: keepalived + securityContext: + privileged: true + image: {{.Images.keepalivedImage}} + command: + - /usr/sbin/keepalived + args: + - "-f" + - "/etc/keepalived/keepalived.conf" + - "--dont-fork" + - "--vrrp" + - "--log-detail" + - "--log-console" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/keepalived" + livenessProbe: + exec: + command: + - pgrep + - keepalived + initialDelaySeconds: 10 + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical + status: {} \ No newline at end of file diff --git a/templates/common/openstack/files/openstack-mdns-publisher.yaml b/templates/common/openstack/files/openstack-mdns-publisher.yaml new file mode 100644 index 0000000000..a836577c00 --- /dev/null +++ b/templates/common/openstack/files/openstack-mdns-publisher.yaml @@ -0,0 +1,75 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/manifests/mdns-publisher.yaml" +contents: + inline: | + kind: Pod + apiVersion: v1 + metadata: + name: mdns-publisher + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-mdns + spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/mdns" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: conf-dir + hostPath: + path: "/etc/mdns" + initContainers: + - name: render-config + image: {{ .Images.baremetalRuntimeCfgImage }} + command: + - runtimecfg + - render + - "/etc/kubernetes/kubeconfig" + - "--api-vip" + - "{{ .Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }}" + - "--dns-vip" + - "{{ .Infra.Status.PlatformStatus.OpenStack.NodeDNSIP }}" + - "--ingress-vip" + - "{{ .Infra.Status.PlatformStatus.OpenStack.IngressIP }}" + - "/config" + - "--out-dir" + - "/etc/mdns" + resources: {} + volumeMounts: + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: resource-dir + mountPath: "/config" + - name: conf-dir + mountPath: "/etc/mdns" + imagePullPolicy: IfNotPresent + containers: + - name: mdns-publisher + image: {{.Images.mdnsPublisherImage}} + args: + - "--debug" + resources: + requests: + cpu: 150m + memory: 1Gi + volumeMounts: + - name: conf-dir + mountPath: "/etc/mdns" + livenessProbe: + exec: + command: + - pgrep + - mdns-publisher + initialDelaySeconds: 10 + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical + status: {} \ No newline at end of file diff --git a/templates/master/00-master/openstack/files/dhcp-dhclient-conf.yaml b/templates/master/00-master/openstack/files/dhcp-dhclient-conf.yaml new file mode 100644 index 0000000000..aa84b764c0 --- /dev/null +++ b/templates/master/00-master/openstack/files/dhcp-dhclient-conf.yaml @@ -0,0 +1,7 @@ +filesystem: "root" +mode: 0644 +path: "/etc/dhcp/dhclient.conf" +contents: + inline: | + supersede domain-search "{{ .EtcdDiscoveryDomain }}"; + prepend domain-name-servers {{ .Infra.Status.PlatformStatus.OpenStack.NodeDNSIP }}; diff --git a/templates/master/00-master/openstack/files/openstack-haproxy-haproxy.yaml b/templates/master/00-master/openstack/files/openstack-haproxy-haproxy.yaml new file mode 100644 index 0000000000..07f3e9176b --- /dev/null +++ b/templates/master/00-master/openstack/files/openstack-haproxy-haproxy.yaml @@ -0,0 +1,39 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/haproxy/haproxy.cfg.tmpl" +contents: + inline: | + defaults + mode tcp + log global + option dontlognull + retries 3 + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 86400s + timeout server 86400s + timeout tunnel 86400s + frontend main + bind :{{`{{ .LBConfig.LbPort }}`}} + default_backend masters + listen health_check_http_url + bind :50936 + mode http + monitor-uri /healthz + option dontlognull + listen stats + bind 127.0.0.1:{{`{{ .LBConfig.StatPort }}`}} + mode http + stats enable + stats hide-version + stats uri /haproxy_stats + stats refresh 30s + stats auth Username:Password + backend masters + option httpchk GET /healthz HTTP/1.0 + option log-health-checks + balance roundrobin + {{`{{- range .LBConfig.Backends }} + server {{ .Host }} {{ .Address }}:{{ .Port }} weight 1 verify none check check-ssl inter 3s fall 3 rise 3 + {{- end }}`}} diff --git a/templates/master/00-master/openstack/files/openstack-haproxy.yaml b/templates/master/00-master/openstack/files/openstack-haproxy.yaml new file mode 100644 index 0000000000..d767539c80 --- /dev/null +++ b/templates/master/00-master/openstack/files/openstack-haproxy.yaml @@ -0,0 +1,107 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/manifests/haproxy.yaml" +contents: + inline: | + kind: Pod + apiVersion: v1 + metadata: + name: haproxy + namespace: openshift-kni-infra + creationTimestamp: + deletionGracePeriodSeconds: 65 + labels: + app: kni-infra-api-lb + spec: + volumes: + - name: resource-dir + hostPath: + path: "/etc/kubernetes/static-pod-resources/haproxy" + - name: kubeconfig + hostPath: + path: "/etc/kubernetes/kubeconfig" + - name: run-dir + empty-dir: {} + - name: conf-dir + hostPath: + path: "/etc/haproxy" + - name: chroot-host + hostPath: + path: "/" + containers: + - name: haproxy + image: {{.Images.haproxyImage}} + command: + - "/bin/bash" + - "-c" + - | + #/bin/bash + reload_haproxy() + { + old_pids=$(pidof haproxy) + if [ -n "$old_pids" ]; then + /usr/sbin/haproxy -W -db -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/run/haproxy.pid -x /var/lib/haproxy/run/haproxy.sock -sf $old_pids & + else + /usr/sbin/haproxy -W -db -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/run/haproxy.pid & + fi + } + + msg_handler() + { + while read -r line; do + echo "The client send: $line" >&2 + # currently only 'reload' msg is supported + if [ "$line" = reload ]; then + reload_haproxy + fi + done + } + set -ex + declare -r haproxy_sock="/var/run/haproxy/haproxy-master.sock" + export -f msg_handler + export -f reload_haproxy + if [ -S "$haproxy_sock" ]; then + rm "$haproxy_sock" + fi + socat UNIX-LISTEN:${haproxy_sock},fork system:'bash -c msg_handler' + volumeMounts: + - name: conf-dir + mountPath: "/etc/haproxy" + - name: run-dir + mountPath: "/var/run/haproxy" + livenessProbe: + initialDelaySeconds: 10 + httpGet: + path: /healthz + port: 50936 + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + - name: haproxy-monitor + securityContext: + privileged: true + image: {{ .Images.baremetalRuntimeCfgImage }} + command: + - monitor + - "/etc/kubernetes/kubeconfig" + - "/config/haproxy.cfg.tmpl" + - "/etc/haproxy/haproxy.cfg" + - "--api-vip" + - "{{ .Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }}" + volumeMounts: + - name: conf-dir + mountPath: "/etc/haproxy" + - name: run-dir + mountPath: "/var/run/haproxy" + - name: resource-dir + mountPath: "/config" + - name: chroot-host + mountPath: "/host" + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + terminationMessagePolicy: FallbackToLogsOnError + imagePullPolicy: IfNotPresent + hostNetwork: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical + status: {} diff --git a/templates/master/00-master/openstack/files/openstack-keepalived-keepalived.yaml b/templates/master/00-master/openstack/files/openstack-keepalived-keepalived.yaml new file mode 100644 index 0000000000..55644d6493 --- /dev/null +++ b/templates/master/00-master/openstack/files/openstack-keepalived-keepalived.yaml @@ -0,0 +1,73 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/keepalived/keepalived.conf.tmpl" +contents: + inline: | + vrrp_script chk_ocp { + script "/usr/bin/curl -o /dev/null -kLs https://0:6443/readyz" + interval 1 + weight 50 + } + vrrp_script chk_dns { + script "/usr/bin/host -t SRV _etcd-server-ssl._tcp.{{ .EtcdDiscoveryDomain }} localhost" + interval 1 + weight 50 + } + # TODO: Improve this check. The port is assumed to be alive. + # Need to assess what is the ramification if the port is not there. + vrrp_script chk_ingress { + script "/usr/bin/curl -o /dev/null -kLs http://0:1936/healthz" + interval 1 + weight 50 + } + vrrp_instance {{`{{ .Cluster.Name }}`}}_API { + state BACKUP + interface {{`{{ .VRRPInterface }}`}} + virtual_router_id {{`{{ .Cluster.APIVirtualRouterID }}`}} + priority 40 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{`{{ .Cluster.Name }}`}}_api_vip + } + virtual_ipaddress { + {{`{{ .Cluster.APIVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + } + track_script { + chk_ocp + } + } + vrrp_instance {{`{{ .Cluster.Name }}`}}_DNS { + state BACKUP + interface {{`{{ .VRRPInterface }}`}} + virtual_router_id {{`{{ .Cluster.DNSVirtualRouterID }}`}} + priority 40 + advert_int 1 + authentication { + auth_type PASS + auth_pass {{`{{ .Cluster.Name }}`}}_dns_vip + } + virtual_ipaddress { + {{`{{ .Cluster.DNSVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + } + track_script { + chk_dns + } + } + vrrp_instance {{`{{ .Cluster.Name }}`}}_INGRESS { + state BACKUP + interface {{`{{ .VRRPInterface }}`}} + virtual_router_id {{`{{ .Cluster.IngressVirtualRouterID }}`}} + priority 40 + advert_int 1 + authentication { + auth_type PASS + auth_pass cluster_uuid_ingress_vip + } + virtual_ipaddress { + {{`{{ .Cluster.IngressVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + } + track_script { + chk_ingress + } + } diff --git a/templates/master/00-master/openstack/files/openstack-mdns-config.yaml b/templates/master/00-master/openstack/files/openstack-mdns-config.yaml new file mode 100644 index 0000000000..3628bde9a6 --- /dev/null +++ b/templates/master/00-master/openstack/files/openstack-mdns-config.yaml @@ -0,0 +1,31 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/mdns/config.hcl.tmpl" +contents: + inline: | + bind_address = "{{`{{ .NonVirtualIP }}`}}" + collision_avoidance = "hostname" + service { + name = "{{`{{ .Cluster.Name }}`}} Etcd" + host_name = "{{`{{ .EtcdShortHostname }}`}}.local." + type = "_etcd-server-ssl._tcp" + domain = "local." + port = 2380 + ttl = 3200 + } + service { + name = "{{`{{ .Cluster.Name }}`}} Workstation" + host_name = "{{`{{ .ShortHostname }}`}}.local." + type = "_workstation._tcp" + domain = "local." + port = 42424 + ttl = 3200 + } + service { + name = "{{`{{ .Cluster.Name }}`}} EtcdWorkstation" + host_name = "{{`{{ .EtcdShortHostname }}`}}.local." + type = "_workstation._tcp" + domain = "local." + port = 42424 + ttl = 300 + } diff --git a/templates/worker/00-worker/openstack/files/NetworkManager-non-virtual-ip-prepender.yaml b/templates/worker/00-worker/openstack/files/NetworkManager-non-virtual-ip-prepender.yaml new file mode 100644 index 0000000000..9607da4138 --- /dev/null +++ b/templates/worker/00-worker/openstack/files/NetworkManager-non-virtual-ip-prepender.yaml @@ -0,0 +1,41 @@ +filesystem: "root" +mode: 0755 +path: "/etc/NetworkManager/dispatcher.d/pre-up.d/non-virtual-ip-prepender" +contents: + inline: | + #!/bin/bash + IFACE=$1 + STATUS=$2 + case "$STATUS" in + pre-up) + logger -s "NM non-virtual-ip-prepender triggered by pre-upping ${1}." + NON_VIRTUAL_IP=$(/usr/local/bin/non_virtual_ip \ + "{{.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP}}" \ + "{{.Infra.Status.PlatformStatus.OpenStack.NodeDNSIP}}" \ + "{{.Infra.Status.PlatformStatus.OpenStack.IngressIP}}") + set +e + if [[ -n $NON_VIRTUAL_IP ]]; then + logger -s "NM non-virtual-ip-prepender: Checking if worker non virtual IP is the first entry in resolv.conf" + if grep nameserver /etc/resolv.conf | head -n 1 | grep -q "$NON_VIRTUAL_IP" ; then + logger -s "NM non-virtual-ip-prepender: worker node non virtual IP already is the first entry in resolv.conf" + exit 0 + else + logger -s "NM non-virtual-ip-prepender: Setting dhclient to prepend non virtual IP in resolv.conf" + sed "s/{{`{{ .NonVirtualIP }}`}}/$NON_VIRTUAL_IP/" /etc/dhcp/dhclient.conf.template > /etc/dhcp/dhclient.conf + logger -s "NM non-virtual-ip-prepender: Looking for 'search' in /etc/resolv.conf to place 'nameserver $NON_VIRTUAL_IP'" + sed -i "/^search .*$/a nameserver $NON_VIRTUAL_IP" /etc/resolv.conf + fi + fi + ;; + down) + logger -s "NM non-virtual-ip-prepender triggered by downing $IFACE" + ;; + up) + logger -s "NM non-virtual-ip-prepender triggered by upping $IFACE" + ;; + post-down) + logger -s "NM non-virtual-ip-prepender triggered by post-downing $IFACE" + ;; + *) + ;; + esac diff --git a/templates/worker/00-worker/openstack/files/dhcp-dhclient-conf.yaml b/templates/worker/00-worker/openstack/files/dhcp-dhclient-conf.yaml new file mode 100644 index 0000000000..ee141be526 --- /dev/null +++ b/templates/worker/00-worker/openstack/files/dhcp-dhclient-conf.yaml @@ -0,0 +1,7 @@ +filesystem: "root" +mode: 0644 +path: "/etc/dhcp/dhclient.conf.template" +contents: + inline: | + supersede domain-search "{{ .EtcdDiscoveryDomain }}"; + prepend domain-name-servers {{`{{ .NonVirtualIP }}`}}; diff --git a/templates/worker/00-worker/openstack/files/openstack-keepalived-keepalived.yaml b/templates/worker/00-worker/openstack/files/openstack-keepalived-keepalived.yaml new file mode 100644 index 0000000000..b039c56fc0 --- /dev/null +++ b/templates/worker/00-worker/openstack/files/openstack-keepalived-keepalived.yaml @@ -0,0 +1,29 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/keepalived/keepalived.conf.tmpl" +contents: + inline: | + # TODO: Improve this check. The port is assumed to be alive. + # Need to assess what is the ramification if the port is not there. + vrrp_script chk_ingress { + script "/usr/bin/curl -o /dev/null -kLs http://0:1936/healthz" + interval 1 + weight 50 + } + vrrp_instance {{`{{ .Cluster.Name }}`}}_INGRESS { + state BACKUP + interface {{`{{ .VRRPInterface }}`}} + virtual_router_id {{`{{ .Cluster.IngressVirtualRouterID }}`}} + priority 40 + advert_int 1 + authentication { + auth_type PASS + auth_pass cluster_uuid_ingress_vip + } + virtual_ipaddress { + {{`{{ .Cluster.IngressVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + } + track_script { + chk_ingress + } + } diff --git a/templates/worker/00-worker/openstack/files/openstack-mdns-config.yaml b/templates/worker/00-worker/openstack/files/openstack-mdns-config.yaml new file mode 100644 index 0000000000..ddde1e4060 --- /dev/null +++ b/templates/worker/00-worker/openstack/files/openstack-mdns-config.yaml @@ -0,0 +1,15 @@ +filesystem: "root" +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/mdns/config.hcl.tmpl" +contents: + inline: | + bind_address = "{{`{{ .NonVirtualIP }}`}}" + collision_avoidance = "hostname" + service { + name = "{{`{{ .Cluster.Name }}`}} Workstation" + host_name = "{{`{{ .ShortHostname }}`}}.local." + type = "_workstation._tcp" + domain = "local." + port = 42424 + ttl = 3200 + } diff --git a/templates/worker/00-worker/openstack/files/openstack-non-virtual-ip.yaml b/templates/worker/00-worker/openstack/files/openstack-non-virtual-ip.yaml new file mode 100644 index 0000000000..c66cd2c5ba --- /dev/null +++ b/templates/worker/00-worker/openstack/files/openstack-non-virtual-ip.yaml @@ -0,0 +1,138 @@ +filesystem: "root" +mode: 0755 +path: "/usr/local/bin/non_virtual_ip" +contents: + inline: | + #!/usr/libexec/platform-python + import collections + import socket + import struct + import subprocess + import sys + + + class SubnetNotFoundException(Exception): + """ + Exception raised when no subnet in the systems ifaces is on the VIP subnet + """ + pass + + + Address = collections.namedtuple('Address', 'index name family cidr scope') + + SUBNET_MASK_LEN = { + 'inet': 32, + 'inet6': 128 + } + + + def ntoa(family, num): + if family == 'inet': + result = socket.inet_ntoa(struct.pack("!I", num)) + else: + lo_half = num & 0xFFFFFFFFFFFFFFFF + hi_half = num >> 64 + result = socket.inet_ntop(socket.AF_INET6, + struct.pack(">QQ", hi_half, lo_half)) + return result + + + def aton(family, rep): + if family == 'inet': + result = struct.unpack("!I", socket.inet_aton(rep))[0] + else: + hi_half, lo_half = struct.unpack(">QQ", socket.inet_pton(socket.AF_INET6, rep)) + result = (hi_half << 64) | lo_half + return result + + + def addr_subnet_int_min_max(addr): + ip, prefix = addr.cidr.split('/') + ip_int = aton(addr.family, ip) + + prefix_int = int(prefix) + mask = int('1' * prefix_int + + '0' * (SUBNET_MASK_LEN[addr.family] - prefix_int), 2) + + subnet_ip_int_min = ip_int & mask + + remainder = '1' * (SUBNET_MASK_LEN[addr.family] - prefix_int) + subnet_ip_int_max = subnet_ip_int_min | ( + 0 if remainder == '' else int(remainder, 2)) + return subnet_ip_int_min, subnet_ip_int_max + + + def vip_subnet_cidr(vip, addrs): + try: + vip_int = aton('inet', vip) + except Exception: + vip_int = aton('inet6', vip) + for addr in addrs: + subnet_ip_int_min, subnet_ip_int_max = addr_subnet_int_min_max(addr) + subnet_ip = ntoa(addr.family, subnet_ip_int_min) + subnet_ip_max = ntoa(addr.family, subnet_ip_int_max) + + sys.stderr.write('Is %s between %s and %s\n' % + (vip, subnet_ip, subnet_ip_max)) + if subnet_ip_int_min < vip_int < subnet_ip_int_max: + subnet_ip = socket.inet_ntoa(struct.pack("!I", subnet_ip_int_min)) + return Address(index="-1", + name="subnet", + cidr='%s/%s' % (subnet_ip, addr.cidr.split('/')[1]), + family=addr.family, + scope='') + raise SubnetNotFoundException() + + + def line_to_address(line): + spl = line.split() + return Address(index=spl[0][:-1], + name=spl[1], + family=spl[2], + cidr=spl[3], + scope=spl[5]) + + + def interface_cidrs(filter_func=None): + try: + out = subprocess.check_output(["ip", "-o", "addr", "show"], encoding=sys.stdout.encoding) + except TypeError: # python2 + out = subprocess.check_output(["ip", "-o", "addr", "show"]) + for addr in (line_to_address(line) for line in out.splitlines()): + if filter_func(addr): + yield addr + + + def non_host_scope(addr): + return addr.scope != 'host' + + + def in_subnet(subnet): + subnet_ip_int_min, subnet_ip_int_max = addr_subnet_int_min_max(subnet) + + def filt(addr): + ip, _ = addr.cidr.split('/') + ip_int = aton(addr.family, ip) + return subnet_ip_int_min < ip_int < subnet_ip_int_max + return filt + + + def main(): + api_vip, dns_vip, ingress_vip = sys.argv[1:4] + vips = set(sys.argv[1:4]) + iface_cidrs = list(interface_cidrs(non_host_scope)) + try: + subnet = vip_subnet_cidr(api_vip, iface_cidrs) + + sys.stderr.write('VIP Subnet %s\n' % subnet.cidr) + + for addr in interface_cidrs(in_subnet(subnet)): + ip, prefix = addr.cidr.split('/') + if ip not in vips: + print(addr.cidr.split('/')[0]) + sys.exit(0) + except SubnetNotFoundException: + sys.exit(1) + + if __name__ == '__main__': + main()