diff --git a/cmd/aro/operator.go b/cmd/aro/operator.go index 78eba80d284..f241a0fe2c7 100644 --- a/cmd/aro/operator.go +++ b/cmd/aro/operator.go @@ -23,6 +23,7 @@ import ( "github.com/Azure/ARO-RP/pkg/operator/controllers" "github.com/Azure/ARO-RP/pkg/operator/controllers/alertwebhook" "github.com/Azure/ARO-RP/pkg/operator/controllers/checker" + "github.com/Azure/ARO-RP/pkg/operator/controllers/dnsmasq" "github.com/Azure/ARO-RP/pkg/operator/controllers/genevalogging" "github.com/Azure/ARO-RP/pkg/operator/controllers/monitoring" "github.com/Azure/ARO-RP/pkg/operator/controllers/pullsecret" @@ -126,6 +127,21 @@ func operator(ctx context.Context, log *logrus.Entry) error { arocli, dh)).SetupWithManager(mgr); err != nil { return fmt.Errorf("unable to create controller RBAC: %v", err) } + if err = (dnsmasq.NewClusterReconciler( + log.WithField("controller", controllers.DnsmasqClusterControllerName), + arocli, mcocli, dh)).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create controller DnsmasqCluster: %v", err) + } + if err = (dnsmasq.NewMachineConfigReconciler( + log.WithField("controller", controllers.DnsmasqMachineConfigControllerName), + arocli, mcocli, dh)).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create controller DnsmasqMachineConfig: %v", err) + } + if err = (dnsmasq.NewMachineConfigPoolReconciler( + log.WithField("controller", controllers.DnsmasqMachineConfigPoolControllerName), + arocli, mcocli, dh)).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create controller DnsmasqMachineConfigPool: %v", err) + } } if err = (checker.NewReconciler( diff --git a/go.mod b/go.mod index 30deaa03a93..a507abd462a 100644 --- a/go.mod +++ b/go.mod @@ -126,7 +126,7 @@ replace ( github.com/openshift/cluster-api-provider-libvirt => github.com/openshift/cluster-api-provider-libvirt v0.2.1-0.20200919090150-1ca52adab176 github.com/openshift/cluster-api-provider-ovirt => github.com/openshift/cluster-api-provider-ovirt v0.1.1-0.20210210114935-91f12f3f7dee github.com/openshift/console-operator => github.com/openshift/console-operator v0.0.0-20210116095614-7fd78a283616 - github.com/openshift/installer => github.com/jim-minter/installer v0.9.0-master.0.20210128115533-6feac498cb32 + github.com/openshift/installer => github.com/jim-minter/installer v0.9.0-master.0.20210221211908-aaebddb9dcf1 github.com/openshift/machine-api-operator => github.com/openshift/machine-api-operator v0.2.1-0.20210212025836-cb508cd8777d github.com/openshift/machine-config-operator => github.com/openshift/machine-config-operator v0.0.1-0.20210211205336-14a2b82d9f4c github.com/operator-framework/operator-sdk => github.com/operator-framework/operator-sdk v0.19.4 diff --git a/go.sum b/go.sum index 59fb0b33de7..2c63bc4f605 100644 --- a/go.sum +++ b/go.sum @@ -23,7 +23,6 @@ cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZ cloud.google.com/go v0.58.0/go.mod h1:W+9FnSUw6nhVwXlFcp1eL+krq5+HQUJeUogSeJZZiWg= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0 h1:eWRCuwubtDrCJG0oSUMgnsbD4CmPFQF2ei4OFbXvwww= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.77.0 h1:qA5V5+uQf6Mgr+tmFI8UT3D/ELyhIYkPwNGao/3Y+sQ= @@ -92,7 +91,6 @@ github.com/Azure/go-autorest/autorest v0.9.3-0.20191028180845-3492b2aff503/go.mo github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest v0.10.0/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest v0.11.17 h1:2zCdHwNgRH+St1J+ZMf66xI8aLr/5KMy+wWLH97zwYM= github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= @@ -103,7 +101,6 @@ github.com/Azure/go-autorest/autorest/adal v0.8.1-0.20191028180845-3492b2aff503/ github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= @@ -135,7 +132,6 @@ github.com/Azure/go-autorest/autorest/validation v0.2.1-0.20191028180845-3492b2a github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= @@ -353,7 +349,6 @@ github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmE github.com/cilium/ebpf v0.0.0-20200507155900-a9f01edf17e3/go.mod h1:XT+cAw5wfvsodedcijoh1l9cf7v1x9FlFB/3VmF/O8s= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= -github.com/clarketm/json v1.14.1 h1:43bkbTTKKdDx7crs3WHzkrnH6S1EvAF1VZrdFGMmmz4= github.com/clarketm/json v1.14.1/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= github.com/clarketm/json v1.15.7 h1:zWsOtfj736/nP76KiS0HpcyO6W50ojEodx7T4LW4NMc= github.com/clarketm/json v1.15.7/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= @@ -397,12 +392,10 @@ github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a h1:spAGlqziZjCJL25C6F1zsQY05tfCKE9F5YwtEWWe6hU= github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.0.2/go.mod h1:nsOhbP19flrX6rE7ieGFvBlr7modwmNjsqWarIUce4M= -github.com/containers/ocicrypt v1.0.3 h1:vYgl+RZ9Q3DPMuTfxmN+qp0X2Bj52uuY2vnt6GzVe1c= github.com/containers/ocicrypt v1.0.3/go.mod h1:CUBa+8MRNL/VkpxYIpaMtgn1WgXGyvPQj8jcy0EVG6g= github.com/containers/ocicrypt v1.1.0 h1:A6UzSUFMla92uxO43O6lm86i7evMGjTY7wTKB2DyGPY= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/containers/storage v1.20.2/go.mod h1:oOB9Ie8OVPojvoaKWEGSEtHbXUAs+tSyr7RO7ZGteMc= -github.com/containers/storage v1.24.5 h1:BusfdU0rCS2/Daa/DPw+0iLfGRlYA7UVF7D0el3N7Vk= github.com/containers/storage v1.24.5/go.mod h1:YC+2pY8SkfEAcZkwycxYbpK8EiRbx5soPPwz9dxe4IQ= github.com/containers/storage v1.25.0 h1:p0PLlQcWmtE+7XLfOCR0WuYyMTby1yozpI4DaKOtWTA= github.com/containers/storage v1.25.0/go.mod h1:UxTYd5F4mPVqmDRcRL0PBS8+HP74aBn96eahnhEvPtk= @@ -418,7 +411,6 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-json v0.0.0-20170920214419-6a2fe990e083/go.mod h1:FmxyHfvrCFfCsXRylD4QQRlQmvzl+DG6iTHyEEykPfU= github.com/coreos/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:FmxyHfvrCFfCsXRylD4QQRlQmvzl+DG6iTHyEEykPfU= github.com/coreos/go-oidc v2.0.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= @@ -432,7 +424,6 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQamW5YV28= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0 h1:kq/SbG2BCKLkDKkjQf5OWwKWUKj1lgs3lFI4PxnR5lg= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= @@ -442,7 +433,6 @@ github.com/coreos/ignition v0.34.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/Pkr github.com/coreos/ignition v0.35.0 h1:UFodoYq1mOPrbEjtxIsZbThcDyQwAI1owczRDqWmKkQ= github.com/coreos/ignition v0.35.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= github.com/coreos/ignition/v2 v2.1.1/go.mod h1:RqmqU64zxarUJa3l4cHtbhcSwfQLpUhv0WVziZwoXvE= -github.com/coreos/ignition/v2 v2.3.0 h1:TK+STbzVe6KZp4tQ2IaNSRMiWX4/diNngep1F7tP7Zk= github.com/coreos/ignition/v2 v2.3.0/go.mod h1:85dmM/CERMZXNrJsXqtNLIxR/dn8G9qlL1CmEjCugp0= github.com/coreos/ignition/v2 v2.9.0 h1:Zl5N08OyqlECB8BrBlMDp3Jf1ShwVTtREPcUq/YO034= github.com/coreos/ignition/v2 v2.9.0/go.mod h1:A5lFFzA2/zvZQPVEvI1lR5WPLWRb7KZ7Q1QOeUMtcAc= @@ -452,7 +442,6 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/coreos/prometheus-operator v0.38.1-0.20200424145508-7e176fda06cc/go.mod h1:erio69w1R/aC14D5nfvAXSlE8FT8jt2Hnavc50Dp33A= github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= -github.com/coreos/vcontext v0.0.0-20200225161404-ee043618d38d h1:Nu473BdYOxcnhFfPrl1ihpCtxI/VZr2IfhVIHDGP43Y= github.com/coreos/vcontext v0.0.0-20200225161404-ee043618d38d/go.mod h1:z4pMVvaUrxs98RROlIYdAQCKhEicjnTirOaVyDRH5h8= github.com/coreos/vcontext v0.0.0-20201120045928-b0e13dab675c h1:jA28WeORitsxGFVWhyWB06sAG2HbLHPQuHwDydhU2CQ= github.com/coreos/vcontext v0.0.0-20201120045928-b0e13dab675c/go.mod h1:z4pMVvaUrxs98RROlIYdAQCKhEicjnTirOaVyDRH5h8= @@ -510,7 +499,6 @@ github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v0.7.3-0.20190817195342-4760db040282/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v1.4.2-0.20191219165747-a9416c67da9f/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.4.2-0.20200203170920-46ec8731fbce h1:KXS1Jg+ddGcWA8e1N7cupxaHHZhit5rB9tfDU+mfjyY= github.com/docker/docker v1.4.2-0.20200203170920-46ec8731fbce/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.3+incompatible h1:+HS4XO73J41FpA260ztGujJ+0WibrA2TPJEnWNSyGNE= github.com/docker/docker v20.10.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= @@ -554,7 +542,6 @@ github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/elazarl/goproxy v0.0.0-20190911111923-ecfe977594f1 h1:yY9rWGoXv1U5pl4gxqlULARMQD7x0QG85lqEXTWysik= github.com/elazarl/goproxy v0.0.0-20190911111923-ecfe977594f1/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/elazarl/goproxy v0.0.0-20200426045556-49ad98f6dac1 h1:TEmChtx8+IeOghiySC8kQIr0JZOdKUmRmmkuRDuYs3E= github.com/elazarl/goproxy v0.0.0-20200426045556-49ad98f6dac1/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= @@ -648,7 +635,6 @@ github.com/go-logr/logr v0.2.1/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= -github.com/go-logr/zapr v0.1.1 h1:qXBXPDdNncunGs7XeEpsJt8wCjYBygluzfdLO0G5baE= github.com/go-logr/zapr v0.1.1/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= @@ -793,7 +779,6 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -930,7 +915,6 @@ github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4Mgqvf github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -1210,8 +1194,8 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jim-minter/go-cosmosdb v0.0.0-20201119201311-b37af9b82812 h1:il0jxCpyWRQ5klfw8ey8yg+WCUdsZGjziYEk5rIDkuc= github.com/jim-minter/go-cosmosdb v0.0.0-20201119201311-b37af9b82812/go.mod h1:n4wXKwl/rXS49qkPRFf3vovG0V6nkwAO4SbRwjGYibM= -github.com/jim-minter/installer v0.9.0-master.0.20210128115533-6feac498cb32 h1:okMO7pHJHbdwTsoq0UUFCXHYBa+BV8g43gAvkAOw09k= -github.com/jim-minter/installer v0.9.0-master.0.20210128115533-6feac498cb32/go.mod h1:vx7AMiiqU8UPAgvDGDwirwneDd8quvkB59KVDKw+Qfw= +github.com/jim-minter/installer v0.9.0-master.0.20210221211908-aaebddb9dcf1 h1:zlcSnLKB97lVo9tX6vgmg2jHHoribX2kglr0u8y4z50= +github.com/jim-minter/installer v0.9.0-master.0.20210221211908-aaebddb9dcf1/go.mod h1:vx7AMiiqU8UPAgvDGDwirwneDd8quvkB59KVDKw+Qfw= github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a h1:GmsqmapfzSJkm28dhRoHz2tLRbJmqhU86IPgBtN3mmk= github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 h1:jNYPNLe3d8smommaoQlK7LOA5ESyUJJ+Wf79ZtA7Vp4= @@ -1307,7 +1291,6 @@ github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/leanovate/gopter v0.2.4/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= @@ -1339,7 +1322,6 @@ github.com/lovoo/gcloud-opentracing v0.3.0/go.mod h1:ZFqk2y38kMDDikZPAK7ynTTGuyt github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lusis/go-artifactory v0.0.0-20160115162124-7e4ce345df82/go.mod h1:y54tfGmO3NKssKveTEFFzH8C/akrSOy/iW9qEAUDV84= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magefile/mage v1.10.0 h1:3HiXzCUY12kh9bIuyXShaVe529fJfyqoVM42o/uom2g= github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magefile/mage v1.11.0 h1:C/55Ywp9BpgVVclD3lRnSYCwXTYxmSppIgLeDYlNuls= github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= @@ -1353,7 +1335,6 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -1392,7 +1373,6 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= @@ -1405,7 +1385,6 @@ github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -1451,7 +1430,6 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.0 h1:iDwIio/3gk2QtLLEsqU5lInaMzos0hDTz8a6lazSFVw= github.com/mitchellh/mapstructure v1.3.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -1642,7 +1620,6 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/ovirt/go-ovirt v0.0.0-20200313072907-d30f754823a6/go.mod h1:fLDxPk1Sf64DBYtwIYxrnx3gPZ1q0xPdWdI1y9vxUaw= github.com/ovirt/go-ovirt v0.0.0-20200613023950-320a86f9df27/go.mod h1:fLDxPk1Sf64DBYtwIYxrnx3gPZ1q0xPdWdI1y9vxUaw= -github.com/ovirt/go-ovirt v0.0.0-20201023070830-77e357c438d5 h1:xU3p8lGWaKEsqdRNw3wkmfDypCKezVSYC2Yx0rvgViw= github.com/ovirt/go-ovirt v0.0.0-20201023070830-77e357c438d5/go.mod h1:fLDxPk1Sf64DBYtwIYxrnx3gPZ1q0xPdWdI1y9vxUaw= github.com/ovirt/go-ovirt v0.0.0-20210112072624-e4d3b104de71 h1:rMPlu5YNQomOQ9hXQDcYTfcmFy8rlYgeDJPPl1qgqz8= github.com/ovirt/go-ovirt v0.0.0-20210112072624-e4d3b104de71/go.mod h1:fLDxPk1Sf64DBYtwIYxrnx3gPZ1q0xPdWdI1y9vxUaw= @@ -1692,7 +1669,6 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/posener/complete v1.2.1/go.mod h1:6gapUrK/U1TAN7ciCoNRIdVC5sbdBTUh1DKN0g6uH7E= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU= github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/pquerna/cachecontrol v0.0.0-20201205024021-ac21108117ac h1:jWKYCNlX4J5s8M0nHYkh7Y7c9gRVDEb3mq51j5J0F5M= github.com/pquerna/cachecontrol v0.0.0-20201205024021-ac21108117ac/go.mod h1:hoLfEwdY11HjRfKFH6KqnPsfxlo3BP6bJehpDv8t6sQ= @@ -1757,7 +1733,6 @@ github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= @@ -1773,7 +1748,6 @@ github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:r github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rickb777/date v1.12.5-0.20200422084442-6300e543c4d9/go.mod h1:L8WrssTzvgYw34/Ppa0JpJfI7KKXZ2cVGI6Djt0brUU= github.com/rickb777/plural v1.2.0/go.mod h1:UdpyWFCGbo3mvK3f/PfZOAOrkjzJlYN/sD46XNWJ+Es= -github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -1814,7 +1788,6 @@ github.com/satori/uuid v1.2.0/go.mod h1:B8HLsPLik/YNn6KKWVMDJ8nzCL8RP5WyfsnmvnAE github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8= github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do= github.com/securego/gosec v0.0.0-20200316084457-7da9f46445fd h1:qB+l4fYZsH78xORC1aqVS0zNmgkQp4rkj2rvfxQMtzc= github.com/securego/gosec v0.0.0-20200316084457-7da9f46445fd/go.mod h1:NurAFZsWJAEZjogSwdVPlHkOZB3DOAU7gsPP8VFZCHc= @@ -1843,7 +1816,6 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.7.1 h1:rsizeFmZP+GYwyb4V6t6qpG7ZNWzA2bvgW/yC2xHCcg= github.com/sirupsen/logrus v1.7.1/go.mod h1:4GuYW9TZmE769R5STWrRakJc4UqQ3+QQ95fyz7ENv1A= @@ -1922,7 +1894,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -1973,20 +1944,17 @@ github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6 github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go v1.2.0 h1:6eXlzYLLwZwXroJx9NyqbYcbv/d93twiOzQLDewE6qM= github.com/ugorji/go v1.2.0/go.mod h1:1ny++pKMXhLWrwWV5Nf+CbOuZJhMoaFD+0GMFfd8fEc= github.com/ugorji/go v1.2.4 h1:cTciPbZ/VSOzCLKclmssnfQ/jyoVyOcJ3aoJyUV1Urc= github.com/ugorji/go v1.2.4/go.mod h1:EuaSCk8iZMdIspsu6HXH7X2UGKw1ezO4wCfGszGmmo4= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= -github.com/ugorji/go/codec v1.2.0 h1:As6RccOIlbm9wHuWYMlB30dErcI+4WiKWsYsmPkyrUw= github.com/ugorji/go/codec v1.2.0/go.mod h1:dXvG35r7zTX6QImXOSFhGMmKtX+wJ7VTWzGvYQGIjBs= github.com/ugorji/go/codec v1.2.4 h1:C5VurWRRCKjuENsbM6GYVw8W++WVW9rSxoACKIvxzz8= github.com/ugorji/go/codec v1.2.4/go.mod h1:bWBu1+kIRWcF8uMklKaJrR6fTWQOwAlrIzX22pHwryA= github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= @@ -2100,7 +2068,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.22.6 h1:BdkrbWrzDlV9dnbzoP7sfN+dHheJ4J9JOaYxcUDL+ok= go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= @@ -2196,7 +2163,6 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -2207,9 +2173,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -2277,9 +2241,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= @@ -2310,7 +2272,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2412,7 +2373,6 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48 h1:AYCWBZhgIw6XobZ5CibNJr0Rc4ZofGGKvWa1vcx2IGk= golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2421,12 +2381,10 @@ golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210217105451-b926d437f341 h1:2/QtM1mL37YmcsT8HaDNHDgTqqFVw+zr8UzMiBVLzYU= golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2438,7 +2396,6 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -2446,7 +2403,6 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2580,7 +2536,6 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k= gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= @@ -2615,9 +2570,7 @@ google.golang.org/api v0.26.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0 h1:TBCmTTxUrRDA1iTctnK/fIeitxIZ+TQuaf0j29fmCGo= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0 h1:l2Nfbl2GPXdWorv+dT2XfinX2jOOw4zv1VhLstx+6rE= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0 h1:uWrpz12dpVPn7cojP82mk02XDgTJLDPc2KbVTxrWb4A= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= @@ -2719,9 +2672,7 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= @@ -2787,7 +2738,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -2795,7 +2745,6 @@ gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -2855,7 +2804,6 @@ k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.5.0 h1:8mOnjf1RmUPW6KRqQCfYSZq/K20Unmp3IhuZUhxl8KI= k8s.io/klog/v2 v2.5.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= @@ -2909,7 +2857,6 @@ sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5 sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06 h1:zD2IemQ4LmOcAumeiyDWXKUI2SO0NYDe3H6QGvPOVgU= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.1-0.20200706213357-43c19bbb7fba h1:AAbnc5KQuTWKuh2QSnyghKIOTFzB0Jayv7/OFDn3Cy4= sigs.k8s.io/structured-merge-diff/v3 v3.0.1-0.20200706213357-43c19bbb7fba/go.mod h1:V06abazjHneE37ZdSY/UUwPVgcJMKI/jU5XGUjgIKoc= diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go index 43442263555..52a12bc22a8 100644 --- a/pkg/api/openshiftcluster.go +++ b/pkg/api/openshiftcluster.go @@ -224,6 +224,7 @@ type APIServerProfile struct { Visibility Visibility `json:"visibility,omitempty"` URL string `json:"url,omitempty"` IP string `json:"ip,omitempty"` + IntIP string `json:"intIp,omitempty"` } // Visibility represents visibility. diff --git a/pkg/api/openshiftclusterdocument.go b/pkg/api/openshiftclusterdocument.go index 2ee56154e38..1536d66db08 100644 --- a/pkg/api/openshiftclusterdocument.go +++ b/pkg/api/openshiftclusterdocument.go @@ -36,8 +36,8 @@ type OpenShiftClusterDocument struct { Bucket int `json:"bucket,omitempty"` - LeaseOwner string `json:"leaseOwner,omitempty"` - LeaseExpires int `json:"leaseExpires,omitempty"` + LeaseOwner string `json:"leaseOwner,omitempty" deep:"-"` + LeaseExpires int `json:"leaseExpires,omitempty" deep:"-"` Dequeues int `json:"dequeues,omitempty"` AsyncOperationID string `json:"asyncOperationId,omitempty" deep:"-"` diff --git a/pkg/api/validate/dynamic.go b/pkg/api/validate/dynamic.go index 78fe4573d21..740aca9c082 100644 --- a/pkg/api/validate/dynamic.go +++ b/pkg/api/validate/dynamic.go @@ -31,7 +31,6 @@ import ( type SlimDynamic interface { ValidateVnetPermissions(ctx context.Context) error ValidateRouteTablesPermissions(ctx context.Context) error - ValidateVnetDns(ctx context.Context) error // etc // does Quota code go in here too? } @@ -167,23 +166,6 @@ func (dv *dynamic) validateRouteTablePermissions(ctx context.Context, rtID strin return err } -func (dv *dynamic) ValidateVnetDNS(ctx context.Context) error { - dv.log.Print("validateVnetDns") - - vnet, err := dv.virtualNetworks.Get(ctx, dv.vnetr.ResourceGroup, dv.vnetr.ResourceName, "") - if err != nil { - return err - } - - if vnet.DhcpOptions != nil && - vnet.DhcpOptions.DNSServers != nil && - len(*vnet.DhcpOptions.DNSServers) > 0 { - return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidLinkedVNet, "", "The provided vnet '%s' is invalid: custom DNS servers are not supported.", *vnet.ID) - } - - return nil -} - func (dv *dynamic) validateActions(ctx context.Context, r *azure.Resource, actions []string) error { timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() diff --git a/pkg/api/validate/dynamic_test.go b/pkg/api/validate/dynamic_test.go index 5489a0b9200..4a7c892c286 100644 --- a/pkg/api/validate/dynamic_test.go +++ b/pkg/api/validate/dynamic_test.go @@ -185,80 +185,6 @@ func TestGetRouteTableID(t *testing.T) { } } -func TestValidateVnetDNS(t *testing.T) { - ctx := context.Background() - - controller := gomock.NewController(t) - defer controller.Finish() - - resourceGroupName := "testGroup" - vnetName := "testVnet" - vnetID := "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/" + resourceGroupName + "/providers/Microsoft.Network/virtualNetworks/" + vnetName - - for _, tt := range []struct { - name string - vnetMocks func(*mock_network.MockVirtualNetworksClient, mgmtnetwork.VirtualNetwork) - wantErr string - }{ - { - name: "pass", - vnetMocks: func(vnetClient *mock_network.MockVirtualNetworksClient, vnet mgmtnetwork.VirtualNetwork) { - vnetClient.EXPECT(). - Get(gomock.Any(), resourceGroupName, vnetName, ""). - Return(vnet, nil) - }, - }, - { - name: "fail: dhcp options set", - vnetMocks: func(vnetClient *mock_network.MockVirtualNetworksClient, vnet mgmtnetwork.VirtualNetwork) { - vnet.DhcpOptions = &mgmtnetwork.DhcpOptions{ - DNSServers: &[]string{ - "8.8.8.8", - }, - } - vnetClient.EXPECT(). - Get(gomock.Any(), resourceGroupName, vnetName, ""). - Return(vnet, nil) - }, - wantErr: "400: InvalidLinkedVNet: : The provided vnet '" + vnetID + "' is invalid: custom DNS servers are not supported.", - }, - { - name: "fail: failed to get vnet", - vnetMocks: func(vnetClient *mock_network.MockVirtualNetworksClient, vnet mgmtnetwork.VirtualNetwork) { - vnetClient.EXPECT(). - Get(gomock.Any(), resourceGroupName, vnetName, ""). - Return(vnet, errors.New("failed to get vnet")) - }, - wantErr: "failed to get vnet", - }, - } { - vnet := mgmtnetwork.VirtualNetwork{ - ID: to.StringPtr(vnetID), - VirtualNetworkPropertiesFormat: &mgmtnetwork.VirtualNetworkPropertiesFormat{ - DhcpOptions: nil, - }, - } - - vnetClient := mock_network.NewMockVirtualNetworksClient(controller) - tt.vnetMocks(vnetClient, vnet) - - dv := &dynamic{ - log: logrus.NewEntry(logrus.StandardLogger()), - virtualNetworks: vnetClient, - vnetr: &azure.Resource{ - ResourceGroup: resourceGroupName, - ResourceName: vnetName, - }, - } - - err := dv.ValidateVnetDNS(ctx) - if err != nil && err.Error() != tt.wantErr || - err == nil && tt.wantErr != "" { - t.Error(err) - } - } -} - func TestValidateRouteTablePermissions(t *testing.T) { ctx := context.Background() diff --git a/pkg/api/validate/openshiftcluster_validatedynamic.go b/pkg/api/validate/openshiftcluster_validatedynamic.go index a5d1d27f876..06d0d8b24b0 100644 --- a/pkg/api/validate/openshiftcluster_validatedynamic.go +++ b/pkg/api/validate/openshiftcluster_validatedynamic.go @@ -116,11 +116,6 @@ func (dv *openShiftClusterDynamicValidator) Dynamic(ctx context.Context) error { } // Additional checks - use any dynamic because they both have the correct permissions - err = spDynamic.ValidateVnetDNS(ctx) - if err != nil { - return err - } - vnet, err := spDynamic.virtualNetworks.Get(ctx, spDynamic.vnetr.ResourceGroup, spDynamic.vnetr.ResourceName, "") if err != nil { return err diff --git a/pkg/cluster/blobservice.go b/pkg/cluster/blobservice.go deleted file mode 100644 index bd83c79f5df..00000000000 --- a/pkg/cluster/blobservice.go +++ /dev/null @@ -1,42 +0,0 @@ -package cluster - -// Copyright (c) Microsoft Corporation. -// Licensed under the Apache License 2.0. - -import ( - "context" - "net/url" - "time" - - mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" - azstorage "github.com/Azure/azure-sdk-for-go/storage" - "github.com/Azure/go-autorest/autorest/date" - - "github.com/Azure/ARO-RP/pkg/util/stringutils" -) - -func (m *manager) getBlobService(ctx context.Context, p mgmtstorage.Permissions, r mgmtstorage.SignedResourceTypes) (*azstorage.BlobStorageClient, error) { - resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') - - t := time.Now().UTC().Truncate(time.Second) - res, err := m.storageAccounts.ListAccountSAS(ctx, resourceGroup, "cluster"+m.doc.OpenShiftCluster.Properties.StorageSuffix, mgmtstorage.AccountSasParameters{ - Services: mgmtstorage.B, - ResourceTypes: r, - Permissions: p, - Protocols: mgmtstorage.HTTPS, - SharedAccessStartTime: &date.Time{Time: t}, - SharedAccessExpiryTime: &date.Time{Time: t.Add(24 * time.Hour)}, - }) - if err != nil { - return nil, err - } - - v, err := url.ParseQuery(*res.AccountSasToken) - if err != nil { - return nil, err - } - - blobcli := azstorage.NewAccountSASClient("cluster"+m.doc.OpenShiftCluster.Properties.StorageSuffix, v, *m.env.Environment()).GetBlobService() - - return &blobcli, nil -} diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 8202589c40d..1663b9889c4 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -11,11 +11,14 @@ import ( operatorclient "github.com/openshift/client-go/operator/clientset/versioned" samplesclient "github.com/openshift/client-go/samples/clientset/versioned" securityclient "github.com/openshift/client-go/security/clientset/versioned" + maoclient "github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned" + mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned" "github.com/sirupsen/logrus" extensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/kubernetes" "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/cluster/graph" "github.com/Azure/ARO-RP/pkg/database" "github.com/Azure/ARO-RP/pkg/env" aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned" @@ -24,12 +27,12 @@ import ( "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/privatedns" - "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage" "github.com/Azure/ARO-RP/pkg/util/billing" "github.com/Azure/ARO-RP/pkg/util/dns" "github.com/Azure/ARO-RP/pkg/util/encryption" "github.com/Azure/ARO-RP/pkg/util/privateendpoint" "github.com/Azure/ARO-RP/pkg/util/refreshable" + "github.com/Azure/ARO-RP/pkg/util/storage" "github.com/Azure/ARO-RP/pkg/util/subnet" ) @@ -48,7 +51,6 @@ type manager struct { billing billing.Manager doc *api.OpenShiftClusterDocument subscriptionDoc *api.SubscriptionDocument - aead encryption.AEAD fpAuthorizer refreshable.Authorizer localFpAuthorizer refreshable.Authorizer @@ -62,15 +64,19 @@ type manager struct { deployments features.DeploymentsClient resourceGroups features.ResourceGroupsClient resources features.ResourcesClient + privateZones privatedns.PrivateZonesClient virtualNetworkLinks privatedns.VirtualNetworkLinksClient - storageAccounts storage.AccountsClient dns dns.Manager privateendpoint privateendpoint.Manager + storage storage.Manager subnet subnet.Manager + graph graph.Manager kubernetescli kubernetes.Interface extensionscli extensionsclient.Interface + maocli maoclient.Interface + mcocli mcoclient.Interface operatorcli operatorclient.Interface configcli configclient.Interface samplescli samplesclient.Interface @@ -98,6 +104,8 @@ func New(ctx context.Context, log *logrus.Entry, env env.Interface, db database. return nil, err } + storage := storage.NewManager(env, r.SubscriptionID, fpAuthorizer) + return &manager{ log: log, env: env, @@ -105,7 +113,6 @@ func New(ctx context.Context, log *logrus.Entry, env env.Interface, db database. billing: billing, doc: doc, subscriptionDoc: subscriptionDoc, - aead: aead, fpAuthorizer: fpAuthorizer, localFpAuthorizer: localFPAuthorizer, @@ -118,11 +125,13 @@ func New(ctx context.Context, log *logrus.Entry, env env.Interface, db database. deployments: features.NewDeploymentsClient(env.Environment(), r.SubscriptionID, fpAuthorizer), resourceGroups: features.NewResourceGroupsClient(env.Environment(), r.SubscriptionID, fpAuthorizer), resources: features.NewResourcesClient(env.Environment(), r.SubscriptionID, fpAuthorizer), + privateZones: privatedns.NewPrivateZonesClient(env.Environment(), r.SubscriptionID, fpAuthorizer), virtualNetworkLinks: privatedns.NewVirtualNetworkLinksClient(env.Environment(), r.SubscriptionID, fpAuthorizer), - storageAccounts: storage.NewAccountsClient(env.Environment(), r.SubscriptionID, fpAuthorizer), dns: dns.NewManager(env, localFPAuthorizer), privateendpoint: privateendpoint.NewManager(env, localFPAuthorizer), + storage: storage, subnet: subnet.NewManager(env, r.SubscriptionID, fpAuthorizer), + graph: graph.NewManager(log, aead, storage), }, nil } diff --git a/pkg/cluster/deployresources.go b/pkg/cluster/deployresources.go index 6fb6fe05392..f2b2556eef4 100644 --- a/pkg/cluster/deployresources.go +++ b/pkg/cluster/deployresources.go @@ -14,25 +14,20 @@ import ( "github.com/Azure/ARO-RP/pkg/util/arm" "github.com/Azure/ARO-RP/pkg/util/stringutils" - "github.com/Azure/ARO-RP/pkg/util/subnet" ) func (m *manager) deployResourceTemplate(ctx context.Context) error { - pg, err := m.loadPersistedGraph(ctx) + resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') + account := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix + + pg, err := m.graph.LoadPersisted(ctx, resourceGroup, account) if err != nil { return err } var installConfig *installconfig.InstallConfig var machineMaster *machine.Master - err = pg.get(&installConfig, &machineMaster) - if err != nil { - return err - } - - resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') - - vnetID, _, err := subnet.Split(m.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID) + err = pg.Get(&installConfig, &machineMaster) if err != nil { return err } @@ -51,14 +46,6 @@ func (m *manager) deployResourceTemplate(ctx context.Context) error { }, }, Resources: []*arm.Resource{ - m.dnsPrivateZone(installConfig), - m.dnsPrivateRecordAPIINT(installConfig), - m.dnsPrivateRecordAPI(installConfig), - m.dnsVirtualNetworkLink(installConfig, vnetID), - m.networkPrivateLinkService(installConfig), - m.networkPublicIPAddress(installConfig), - m.networkInternalLoadBalancer(installConfig), - m.networkPublicLoadBalancer(installConfig), m.networkBootstrapNIC(installConfig), m.networkMasterNICs(installConfig), m.computeBootstrapVM(installConfig), diff --git a/pkg/cluster/deployresources_resources.go b/pkg/cluster/deployresources_resources.go index 39bd8714f5b..3817dfa2ea4 100644 --- a/pkg/cluster/deployresources_resources.go +++ b/pkg/cluster/deployresources_resources.go @@ -9,411 +9,14 @@ import ( mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute" mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network" - mgmtprivatedns "github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns" "github.com/Azure/go-autorest/autorest/to" "github.com/openshift/installer/pkg/asset/ignition/machine" "github.com/openshift/installer/pkg/asset/installconfig" - "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/util/arm" "github.com/Azure/ARO-RP/pkg/util/azureclient" ) -func (m *manager) dnsPrivateZone(installConfig *installconfig.InstallConfig) *arm.Resource { - return &arm.Resource{ - Resource: &mgmtprivatedns.PrivateZone{ - Name: to.StringPtr(installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain), - Type: to.StringPtr("Microsoft.Network/privateDnsZones"), - Location: to.StringPtr("global"), - }, - APIVersion: azureclient.APIVersion("Microsoft.Network/privateDnsZones"), - } -} - -func (m *manager) dnsPrivateRecordAPIINT(installConfig *installconfig.InstallConfig) *arm.Resource { - return &arm.Resource{ - Resource: &mgmtprivatedns.RecordSet{ - Name: to.StringPtr(installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain + "/api-int"), - Type: to.StringPtr("Microsoft.Network/privateDnsZones/A"), - RecordSetProperties: &mgmtprivatedns.RecordSetProperties{ - TTL: to.Int64Ptr(300), - ARecords: &[]mgmtprivatedns.ARecord{ - { - Ipv4Address: to.StringPtr(fmt.Sprintf("[reference('Microsoft.Network/loadBalancers/%s-internal', '%s').frontendIpConfigurations[0].properties.privateIPAddress]", m.doc.OpenShiftCluster.Properties.InfraID, azureclient.APIVersion("Microsoft.Network"))), - }, - }, - }, - }, - APIVersion: azureclient.APIVersion("Microsoft.Network/privateDnsZones"), - DependsOn: []string{ - "Microsoft.Network/loadBalancers/" + m.doc.OpenShiftCluster.Properties.InfraID + "-internal", - "Microsoft.Network/privateDnsZones/" + installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain, - }, - } -} - -func (m *manager) dnsPrivateRecordAPI(installConfig *installconfig.InstallConfig) *arm.Resource { - return &arm.Resource{ - Resource: &mgmtprivatedns.RecordSet{ - Name: to.StringPtr(installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain + "/api"), - Type: to.StringPtr("Microsoft.Network/privateDnsZones/A"), - RecordSetProperties: &mgmtprivatedns.RecordSetProperties{ - TTL: to.Int64Ptr(300), - ARecords: &[]mgmtprivatedns.ARecord{ - { - Ipv4Address: to.StringPtr(fmt.Sprintf("[reference('Microsoft.Network/loadBalancers/%s-internal', '%s').frontendIpConfigurations[0].properties.privateIPAddress]", m.doc.OpenShiftCluster.Properties.InfraID, azureclient.APIVersion("Microsoft.Network"))), - }, - }, - }, - }, - APIVersion: azureclient.APIVersion("Microsoft.Network/privateDnsZones"), - DependsOn: []string{ - "Microsoft.Network/loadBalancers/" + m.doc.OpenShiftCluster.Properties.InfraID + "-internal", - "Microsoft.Network/privateDnsZones/" + installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain, - }, - } -} - -func (m *manager) dnsVirtualNetworkLink(installConfig *installconfig.InstallConfig, vnetID string) *arm.Resource { - return &arm.Resource{ - Resource: &mgmtprivatedns.VirtualNetworkLink{ - VirtualNetworkLinkProperties: &mgmtprivatedns.VirtualNetworkLinkProperties{ - VirtualNetwork: &mgmtprivatedns.SubResource{ - ID: to.StringPtr(vnetID), - }, - RegistrationEnabled: to.BoolPtr(false), - }, - Name: to.StringPtr(installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain + "/" + installConfig.Config.ObjectMeta.Name + "-network-link"), - Type: to.StringPtr("Microsoft.Network/privateDnsZones/virtualNetworkLinks"), - Location: to.StringPtr("global"), - }, - APIVersion: azureclient.APIVersion("Microsoft.Network/privateDnsZones"), - DependsOn: []string{ - "Microsoft.Network/privateDnsZones/" + installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain, - }, - } -} - -func (m *manager) networkPrivateLinkService(installConfig *installconfig.InstallConfig) *arm.Resource { - return &arm.Resource{ - Resource: &mgmtnetwork.PrivateLinkService{ - PrivateLinkServiceProperties: &mgmtnetwork.PrivateLinkServiceProperties{ - LoadBalancerFrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ - { - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - }, - IPConfigurations: &[]mgmtnetwork.PrivateLinkServiceIPConfiguration{ - { - PrivateLinkServiceIPConfigurationProperties: &mgmtnetwork.PrivateLinkServiceIPConfigurationProperties{ - Subnet: &mgmtnetwork.Subnet{ - ID: to.StringPtr(m.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID), - }, - }, - Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID + "-pls-nic"), - }, - }, - Visibility: &mgmtnetwork.PrivateLinkServicePropertiesVisibility{ - Subscriptions: &[]string{ - m.env.SubscriptionID(), - }, - }, - AutoApproval: &mgmtnetwork.PrivateLinkServicePropertiesAutoApproval{ - Subscriptions: &[]string{ - m.env.SubscriptionID(), - }, - }, - }, - Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID + "-pls"), - Type: to.StringPtr("Microsoft.Network/privateLinkServices"), - Location: &installConfig.Config.Azure.Region, - }, - APIVersion: azureclient.APIVersion("Microsoft.Network"), - DependsOn: []string{ - "Microsoft.Network/loadBalancers/" + m.doc.OpenShiftCluster.Properties.InfraID + "-internal", - }, - } -} - -func (m *manager) networkPublicIPAddress(installConfig *installconfig.InstallConfig) *arm.Resource { - return &arm.Resource{ - Resource: &mgmtnetwork.PublicIPAddress{ - Sku: &mgmtnetwork.PublicIPAddressSku{ - Name: mgmtnetwork.PublicIPAddressSkuNameStandard, - }, - PublicIPAddressPropertiesFormat: &mgmtnetwork.PublicIPAddressPropertiesFormat{ - PublicIPAllocationMethod: mgmtnetwork.Static, - }, - Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID + "-pip-v4"), - Type: to.StringPtr("Microsoft.Network/publicIPAddresses"), - Location: &installConfig.Config.Azure.Region, - }, - APIVersion: azureclient.APIVersion("Microsoft.Network"), - } -} - -func (m *manager) networkInternalLoadBalancer(installConfig *installconfig.InstallConfig) *arm.Resource { - return &arm.Resource{ - Resource: &mgmtnetwork.LoadBalancer{ - Sku: &mgmtnetwork.LoadBalancerSku{ - Name: mgmtnetwork.LoadBalancerSkuNameStandard, - }, - LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ - { - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: mgmtnetwork.Dynamic, - Subnet: &mgmtnetwork.Subnet{ - ID: to.StringPtr(m.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID), - }, - }, - Name: to.StringPtr("internal-lb-ip-v4"), - }, - }, - BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{ - { - Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID), - }, - { - Name: to.StringPtr("ssh-0"), - }, - { - Name: to.StringPtr("ssh-1"), - }, - { - Name: to.StringPtr("ssh-2"), - }, - }, - LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{ - { - LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{ - FrontendIPConfiguration: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - BackendAddressPool: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', '%[1]s')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Probe: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'api-internal-probe')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Protocol: mgmtnetwork.TransportProtocolTCP, - LoadDistribution: mgmtnetwork.LoadDistributionDefault, - FrontendPort: to.Int32Ptr(6443), - BackendPort: to.Int32Ptr(6443), - IdleTimeoutInMinutes: to.Int32Ptr(30), - DisableOutboundSnat: to.BoolPtr(true), - }, - Name: to.StringPtr("api-internal-v4"), - }, - { - LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{ - FrontendIPConfiguration: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - BackendAddressPool: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', '%[1]s')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Probe: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'sint-probe')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Protocol: mgmtnetwork.TransportProtocolTCP, - LoadDistribution: mgmtnetwork.LoadDistributionDefault, - FrontendPort: to.Int32Ptr(22623), - BackendPort: to.Int32Ptr(22623), - IdleTimeoutInMinutes: to.Int32Ptr(30), - }, - Name: to.StringPtr("sint-v4"), - }, - { - LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{ - FrontendIPConfiguration: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - BackendAddressPool: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', 'ssh-0')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Probe: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'ssh')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Protocol: mgmtnetwork.TransportProtocolTCP, - LoadDistribution: mgmtnetwork.LoadDistributionDefault, - FrontendPort: to.Int32Ptr(2200), - BackendPort: to.Int32Ptr(22), - IdleTimeoutInMinutes: to.Int32Ptr(30), - DisableOutboundSnat: to.BoolPtr(true), - }, - Name: to.StringPtr("ssh-0"), - }, - { - LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{ - FrontendIPConfiguration: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - BackendAddressPool: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', 'ssh-1')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Probe: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'ssh')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Protocol: mgmtnetwork.TransportProtocolTCP, - LoadDistribution: mgmtnetwork.LoadDistributionDefault, - FrontendPort: to.Int32Ptr(2201), - BackendPort: to.Int32Ptr(22), - IdleTimeoutInMinutes: to.Int32Ptr(30), - DisableOutboundSnat: to.BoolPtr(true), - }, - Name: to.StringPtr("ssh-1"), - }, - { - LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{ - FrontendIPConfiguration: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - BackendAddressPool: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', 'ssh-2')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Probe: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'ssh')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Protocol: mgmtnetwork.TransportProtocolTCP, - LoadDistribution: mgmtnetwork.LoadDistributionDefault, - FrontendPort: to.Int32Ptr(2202), - BackendPort: to.Int32Ptr(22), - IdleTimeoutInMinutes: to.Int32Ptr(30), - DisableOutboundSnat: to.BoolPtr(true), - }, - Name: to.StringPtr("ssh-2"), - }, - }, - Probes: &[]mgmtnetwork.Probe{ - { - ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{ - Protocol: mgmtnetwork.ProbeProtocolHTTPS, - Port: to.Int32Ptr(6443), - IntervalInSeconds: to.Int32Ptr(5), - NumberOfProbes: to.Int32Ptr(2), - RequestPath: to.StringPtr("/readyz"), - }, - Name: to.StringPtr("api-internal-probe"), - }, - { - ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{ - Protocol: mgmtnetwork.ProbeProtocolHTTPS, - Port: to.Int32Ptr(22623), - IntervalInSeconds: to.Int32Ptr(5), - NumberOfProbes: to.Int32Ptr(2), - RequestPath: to.StringPtr("/healthz"), - }, - Name: to.StringPtr("sint-probe"), - }, - { - ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{ - Protocol: mgmtnetwork.ProbeProtocolTCP, - Port: to.Int32Ptr(22), - IntervalInSeconds: to.Int32Ptr(5), - NumberOfProbes: to.Int32Ptr(2), - }, - Name: to.StringPtr("ssh"), - }, - }, - }, - Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID + "-internal"), - Type: to.StringPtr("Microsoft.Network/loadBalancers"), - Location: &installConfig.Config.Azure.Region, - }, - APIVersion: azureclient.APIVersion("Microsoft.Network"), - } -} - -func (m *manager) networkPublicLoadBalancer(installConfig *installconfig.InstallConfig) *arm.Resource { - lb := &mgmtnetwork.LoadBalancer{ - Sku: &mgmtnetwork.LoadBalancerSku{ - Name: mgmtnetwork.LoadBalancerSkuNameStandard, - }, - LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ - { - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: to.StringPtr("[resourceId('Microsoft.Network/publicIPAddresses', '" + m.doc.OpenShiftCluster.Properties.InfraID + "-pip-v4')]"), - }, - }, - Name: to.StringPtr("public-lb-ip-v4"), - }, - }, - BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{ - { - Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID), - }, - }, - LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{}, //required to override default LB rules for port 80 and 443 - Probes: &[]mgmtnetwork.Probe{}, //required to override default LB rules for port 80 and 443 - OutboundRules: &[]mgmtnetwork.OutboundRule{ - { - OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '" + m.doc.OpenShiftCluster.Properties.InfraID + "', 'public-lb-ip-v4')]"), - }, - }, - BackendAddressPool: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s', '%[1]s')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Protocol: mgmtnetwork.LoadBalancerOutboundRuleProtocolAll, - IdleTimeoutInMinutes: to.Int32Ptr(30), - }, - Name: to.StringPtr("outbound-rule-v4"), - }, - }, - }, - Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID), - Type: to.StringPtr("Microsoft.Network/loadBalancers"), - Location: &installConfig.Config.Azure.Region, - } - - if m.doc.OpenShiftCluster.Properties.APIServerProfile.Visibility == api.VisibilityPublic { - *lb.LoadBalancingRules = append(*lb.LoadBalancingRules, mgmtnetwork.LoadBalancingRule{ - LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{ - FrontendIPConfiguration: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s', 'public-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - BackendAddressPool: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s', '%[1]s')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Probe: &mgmtnetwork.SubResource{ - ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s', 'api-internal-probe')]", m.doc.OpenShiftCluster.Properties.InfraID)), - }, - Protocol: mgmtnetwork.TransportProtocolTCP, - LoadDistribution: mgmtnetwork.LoadDistributionDefault, - FrontendPort: to.Int32Ptr(6443), - BackendPort: to.Int32Ptr(6443), - IdleTimeoutInMinutes: to.Int32Ptr(30), - DisableOutboundSnat: to.BoolPtr(true), - }, - Name: to.StringPtr("api-internal-v4"), - }) - - *lb.Probes = append(*lb.Probes, mgmtnetwork.Probe{ - ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{ - Protocol: mgmtnetwork.ProbeProtocolHTTPS, - Port: to.Int32Ptr(6443), - IntervalInSeconds: to.Int32Ptr(5), - NumberOfProbes: to.Int32Ptr(2), - RequestPath: to.StringPtr("/readyz"), - }, - Name: to.StringPtr("api-internal-probe"), - }) - } - - return &arm.Resource{ - Resource: lb, - APIVersion: azureclient.APIVersion("Microsoft.Network"), - DependsOn: []string{ - "Microsoft.Network/publicIPAddresses/" + m.doc.OpenShiftCluster.Properties.InfraID + "-pip-v4", - }, - } -} - func (m *manager) networkBootstrapNIC(installConfig *installconfig.InstallConfig) *arm.Resource { return &arm.Resource{ Resource: &mgmtnetwork.Interface{ @@ -442,10 +45,6 @@ func (m *manager) networkBootstrapNIC(installConfig *installconfig.InstallConfig Location: &installConfig.Config.Azure.Region, }, APIVersion: azureclient.APIVersion("Microsoft.Network"), - DependsOn: []string{ - "Microsoft.Network/loadBalancers/" + m.doc.OpenShiftCluster.Properties.InfraID + "-internal", - "Microsoft.Network/loadBalancers/" + m.doc.OpenShiftCluster.Properties.InfraID, - }, } } @@ -484,10 +83,6 @@ func (m *manager) networkMasterNICs(installConfig *installconfig.InstallConfig) Name: "networkcopy", Count: int(*installConfig.Config.ControlPlane.Replicas), }, - DependsOn: []string{ - "Microsoft.Network/loadBalancers/" + m.doc.OpenShiftCluster.Properties.InfraID + "-internal", - "Microsoft.Network/loadBalancers/" + m.doc.OpenShiftCluster.Properties.InfraID, - }, } } @@ -545,7 +140,6 @@ func (m *manager) computeBootstrapVM(installConfig *installconfig.InstallConfig) APIVersion: azureclient.APIVersion("Microsoft.Compute"), DependsOn: []string{ "Microsoft.Network/networkInterfaces/" + m.doc.OpenShiftCluster.Properties.InfraID + "-bootstrap-nic", - "Microsoft.Network/privateDnsZones/" + installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain + "/virtualNetworkLinks/" + installConfig.Config.ObjectMeta.Name + "-network-link", }, } } @@ -609,7 +203,6 @@ func (m *manager) computeMasterVMs(installConfig *installconfig.InstallConfig, z }, DependsOn: []string{ "[concat('Microsoft.Network/networkInterfaces/" + m.doc.OpenShiftCluster.Properties.InfraID + "-master', copyIndex(), '-nic')]", - "Microsoft.Network/privateDnsZones/" + installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain + "/virtualNetworkLinks/" + installConfig.Config.ObjectMeta.Name + "-network-link", }, } } diff --git a/pkg/cluster/deploystorage.go b/pkg/cluster/deploystorage.go index ff9e2f6a3db..5f98c12a9e7 100644 --- a/pkg/cluster/deploystorage.go +++ b/pkg/cluster/deploystorage.go @@ -18,11 +18,13 @@ import ( "github.com/openshift/installer/pkg/asset/kubeconfig" "github.com/openshift/installer/pkg/asset/releaseimage" "github.com/openshift/installer/pkg/asset/targets" + "github.com/openshift/installer/pkg/asset/templates/content/bootkube" "github.com/openshift/installer/pkg/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/bootstraplogging" + "github.com/Azure/ARO-RP/pkg/cluster/graph" "github.com/Azure/ARO-RP/pkg/util/arm" "github.com/Azure/ARO-RP/pkg/util/deployment" "github.com/Azure/ARO-RP/pkg/util/stringutils" @@ -35,8 +37,8 @@ func (m *manager) createDNS(ctx context.Context) error { func (m *manager) deployStorageTemplate(ctx context.Context, installConfig *installconfig.InstallConfig, image *releaseimage.Image) error { if m.doc.OpenShiftCluster.Properties.InfraID == "" { - g := graph{} - g.set(&installconfig.InstallConfig{ + g := graph.Graph{} + g.Set(&installconfig.InstallConfig{ Config: &types.InstallConfig{ ObjectMeta: metav1.ObjectMeta{ Name: strings.ToLower(m.doc.OpenShiftCluster.Name), @@ -44,12 +46,12 @@ func (m *manager) deployStorageTemplate(ctx context.Context, installConfig *inst }, }) - err := g.resolve(&installconfig.ClusterID{}) + err := g.Resolve(&installconfig.ClusterID{}) if err != nil { return err } - clusterID := g.get(&installconfig.ClusterID{}).(*installconfig.ClusterID) + clusterID := g.Get(&installconfig.ClusterID{}).(*installconfig.ClusterID) m.doc, err = m.db.PatchWithLease(ctx, m.doc.Key, func(doc *api.OpenShiftClusterDocument) error { doc.OpenShiftCluster.Properties.InfraID = clusterID.InfraID @@ -99,32 +101,48 @@ func (m *manager) deployStorageTemplate(ctx context.Context, installConfig *inst return err } + resources := []*arm.Resource{ + m.clusterStorageAccount(installConfig.Config.Azure.Region), + m.clusterStorageAccountBlob("ignition"), + m.clusterStorageAccountBlob("aro"), + m.clusterNSG(infraID, installConfig.Config.Azure.Region), + m.clusterServicePrincipalRBAC(), + m.networkPrivateLinkService(installConfig), + m.networkPublicIPAddress(installConfig, m.doc.OpenShiftCluster.Properties.InfraID+"-pip-v4"), + m.networkInternalLoadBalancer(installConfig), + m.networkPublicLoadBalancer(installConfig), + } + + if m.doc.OpenShiftCluster.Properties.IngressProfiles[0].Visibility == api.VisibilityPublic { + resources = append(resources, + m.networkPublicIPAddress(installConfig, m.doc.OpenShiftCluster.Properties.InfraID+"-default-v4"), + ) + } + t := &arm.Template{ Schema: "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", ContentVersion: "1.0.0.0", - Resources: []*arm.Resource{ - m.clusterStorageAccount(installConfig.Config.Azure.Region), - m.clusterStorageAccountBlob("ignition"), - m.clusterStorageAccountBlob("aro"), - m.clusterNSG(infraID, installConfig.Config.Azure.Region), - m.clusterServicePrincipalRBAC(), - }, + Resources: resources, } if m.env.DeploymentMode() == deployment.Production { t.Resources = append(t.Resources, m.denyAssignments()) } - err = m.deployARMTemplate(ctx, resourceGroup, "storage", t, nil) - if err != nil { - return err - } + return m.deployARMTemplate(ctx, resourceGroup, "storage", t, nil) +} - exists, err := m.graphExists(ctx) +func (m *manager) ensureGraph(ctx context.Context, installConfig *installconfig.InstallConfig, image *releaseimage.Image) error { + resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') + account := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix + + exists, err := m.graph.Exists(ctx, resourceGroup, account) if err != nil || exists { return err } + infraID := m.doc.OpenShiftCluster.Properties.InfraID + clusterID := &installconfig.ClusterID{ UUID: m.doc.ID, InfraID: infraID, @@ -135,19 +153,24 @@ func (m *manager) deployStorageTemplate(ctx context.Context, installConfig *inst return err } - g := graph{} - g.set(installConfig, image, clusterID, bootstrapLoggingConfig) + dnsConfig := &bootkube.ARODNSConfig{ + APIIntIP: m.doc.OpenShiftCluster.Properties.APIServerProfile.IntIP, + IngressIP: m.doc.OpenShiftCluster.Properties.IngressProfiles[0].IP, + } + + g := graph.Graph{} + g.Set(installConfig, image, clusterID, bootstrapLoggingConfig, dnsConfig) m.log.Print("resolving graph") for _, a := range targets.Cluster { - err = g.resolve(a) + err = g.Resolve(a) if err != nil { return err } } // the graph is quite big so we store it in a storage account instead of in cosmosdb - return m.saveGraph(ctx, g) + return m.graph.Save(ctx, resourceGroup, account, g) } func (m *manager) deploySnapshotUpgradeTemplate(ctx context.Context) error { @@ -173,7 +196,10 @@ func (m *manager) deploySnapshotUpgradeTemplate(ctx context.Context) error { } func (m *manager) attachNSGsAndPatch(ctx context.Context) error { - pg, err := m.loadPersistedGraph(ctx) + resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') + account := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix + + pg, err := m.graph.LoadPersisted(ctx, resourceGroup, account) if err != nil { return err } @@ -223,7 +249,7 @@ func (m *manager) attachNSGsAndPatch(ctx context.Context) error { } var adminInternalClient *kubeconfig.AdminInternalClient - err = pg.get(&adminInternalClient) + err = pg.Get(&adminInternalClient) if err != nil { return err } diff --git a/pkg/cluster/deploystorage_resources.go b/pkg/cluster/deploystorage_resources.go index d3e6645436c..66dff54261c 100644 --- a/pkg/cluster/deploystorage_resources.go +++ b/pkg/cluster/deploystorage_resources.go @@ -4,10 +4,15 @@ package cluster // Licensed under the Apache License 2.0. import ( + "fmt" + + mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network" mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2018-09-01-preview/authorization" mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" "github.com/Azure/go-autorest/autorest/to" + "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/util/arm" "github.com/Azure/ARO-RP/pkg/util/azureclient" "github.com/Azure/ARO-RP/pkg/util/feature" @@ -110,3 +115,323 @@ func (m *manager) clusterStorageAccountBlob(name string) *arm.Resource { }, } } + +func (m *manager) networkPrivateLinkService(installConfig *installconfig.InstallConfig) *arm.Resource { + return &arm.Resource{ + Resource: &mgmtnetwork.PrivateLinkService{ + PrivateLinkServiceProperties: &mgmtnetwork.PrivateLinkServiceProperties{ + LoadBalancerFrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + }, + IPConfigurations: &[]mgmtnetwork.PrivateLinkServiceIPConfiguration{ + { + PrivateLinkServiceIPConfigurationProperties: &mgmtnetwork.PrivateLinkServiceIPConfigurationProperties{ + Subnet: &mgmtnetwork.Subnet{ + ID: to.StringPtr(m.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID), + }, + }, + Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID + "-pls-nic"), + }, + }, + Visibility: &mgmtnetwork.PrivateLinkServicePropertiesVisibility{ + Subscriptions: &[]string{ + m.env.SubscriptionID(), + }, + }, + AutoApproval: &mgmtnetwork.PrivateLinkServicePropertiesAutoApproval{ + Subscriptions: &[]string{ + m.env.SubscriptionID(), + }, + }, + }, + Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID + "-pls"), + Type: to.StringPtr("Microsoft.Network/privateLinkServices"), + Location: &installConfig.Config.Azure.Region, + }, + APIVersion: azureclient.APIVersion("Microsoft.Network"), + DependsOn: []string{ + "Microsoft.Network/loadBalancers/" + m.doc.OpenShiftCluster.Properties.InfraID + "-internal", + }, + } +} + +func (m *manager) networkPublicIPAddress(installConfig *installconfig.InstallConfig, name string) *arm.Resource { + return &arm.Resource{ + Resource: &mgmtnetwork.PublicIPAddress{ + Sku: &mgmtnetwork.PublicIPAddressSku{ + Name: mgmtnetwork.PublicIPAddressSkuNameStandard, + }, + PublicIPAddressPropertiesFormat: &mgmtnetwork.PublicIPAddressPropertiesFormat{ + PublicIPAllocationMethod: mgmtnetwork.Static, + }, + Name: &name, + Type: to.StringPtr("Microsoft.Network/publicIPAddresses"), + Location: &installConfig.Config.Azure.Region, + }, + APIVersion: azureclient.APIVersion("Microsoft.Network"), + } +} + +func (m *manager) networkInternalLoadBalancer(installConfig *installconfig.InstallConfig) *arm.Resource { + return &arm.Resource{ + Resource: &mgmtnetwork.LoadBalancer{ + Sku: &mgmtnetwork.LoadBalancerSku{ + Name: mgmtnetwork.LoadBalancerSkuNameStandard, + }, + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PrivateIPAllocationMethod: mgmtnetwork.Dynamic, + Subnet: &mgmtnetwork.Subnet{ + ID: to.StringPtr(m.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID), + }, + }, + Name: to.StringPtr("internal-lb-ip-v4"), + }, + }, + BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{ + { + Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID), + }, + { + Name: to.StringPtr("ssh-0"), + }, + { + Name: to.StringPtr("ssh-1"), + }, + { + Name: to.StringPtr("ssh-2"), + }, + }, + LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{ + { + LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{ + FrontendIPConfiguration: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + BackendAddressPool: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', '%[1]s')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Probe: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'api-internal-probe')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Protocol: mgmtnetwork.TransportProtocolTCP, + LoadDistribution: mgmtnetwork.LoadDistributionDefault, + FrontendPort: to.Int32Ptr(6443), + BackendPort: to.Int32Ptr(6443), + IdleTimeoutInMinutes: to.Int32Ptr(30), + DisableOutboundSnat: to.BoolPtr(true), + }, + Name: to.StringPtr("api-internal-v4"), + }, + { + LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{ + FrontendIPConfiguration: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + BackendAddressPool: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', '%[1]s')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Probe: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'sint-probe')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Protocol: mgmtnetwork.TransportProtocolTCP, + LoadDistribution: mgmtnetwork.LoadDistributionDefault, + FrontendPort: to.Int32Ptr(22623), + BackendPort: to.Int32Ptr(22623), + IdleTimeoutInMinutes: to.Int32Ptr(30), + }, + Name: to.StringPtr("sint-v4"), + }, + { + LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{ + FrontendIPConfiguration: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + BackendAddressPool: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', 'ssh-0')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Probe: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'ssh')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Protocol: mgmtnetwork.TransportProtocolTCP, + LoadDistribution: mgmtnetwork.LoadDistributionDefault, + FrontendPort: to.Int32Ptr(2200), + BackendPort: to.Int32Ptr(22), + IdleTimeoutInMinutes: to.Int32Ptr(30), + DisableOutboundSnat: to.BoolPtr(true), + }, + Name: to.StringPtr("ssh-0"), + }, + { + LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{ + FrontendIPConfiguration: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + BackendAddressPool: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', 'ssh-1')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Probe: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'ssh')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Protocol: mgmtnetwork.TransportProtocolTCP, + LoadDistribution: mgmtnetwork.LoadDistributionDefault, + FrontendPort: to.Int32Ptr(2201), + BackendPort: to.Int32Ptr(22), + IdleTimeoutInMinutes: to.Int32Ptr(30), + DisableOutboundSnat: to.BoolPtr(true), + }, + Name: to.StringPtr("ssh-1"), + }, + { + LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{ + FrontendIPConfiguration: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + BackendAddressPool: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', 'ssh-2')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Probe: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'ssh')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Protocol: mgmtnetwork.TransportProtocolTCP, + LoadDistribution: mgmtnetwork.LoadDistributionDefault, + FrontendPort: to.Int32Ptr(2202), + BackendPort: to.Int32Ptr(22), + IdleTimeoutInMinutes: to.Int32Ptr(30), + DisableOutboundSnat: to.BoolPtr(true), + }, + Name: to.StringPtr("ssh-2"), + }, + }, + Probes: &[]mgmtnetwork.Probe{ + { + ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{ + Protocol: mgmtnetwork.ProbeProtocolHTTPS, + Port: to.Int32Ptr(6443), + IntervalInSeconds: to.Int32Ptr(5), + NumberOfProbes: to.Int32Ptr(2), + RequestPath: to.StringPtr("/readyz"), + }, + Name: to.StringPtr("api-internal-probe"), + }, + { + ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{ + Protocol: mgmtnetwork.ProbeProtocolHTTPS, + Port: to.Int32Ptr(22623), + IntervalInSeconds: to.Int32Ptr(5), + NumberOfProbes: to.Int32Ptr(2), + RequestPath: to.StringPtr("/healthz"), + }, + Name: to.StringPtr("sint-probe"), + }, + { + ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{ + Protocol: mgmtnetwork.ProbeProtocolTCP, + Port: to.Int32Ptr(22), + IntervalInSeconds: to.Int32Ptr(5), + NumberOfProbes: to.Int32Ptr(2), + }, + Name: to.StringPtr("ssh"), + }, + }, + }, + Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID + "-internal"), + Type: to.StringPtr("Microsoft.Network/loadBalancers"), + Location: &installConfig.Config.Azure.Region, + }, + APIVersion: azureclient.APIVersion("Microsoft.Network"), + } +} + +func (m *manager) networkPublicLoadBalancer(installConfig *installconfig.InstallConfig) *arm.Resource { + lb := &mgmtnetwork.LoadBalancer{ + Sku: &mgmtnetwork.LoadBalancerSku{ + Name: mgmtnetwork.LoadBalancerSkuNameStandard, + }, + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("[resourceId('Microsoft.Network/publicIPAddresses', '" + m.doc.OpenShiftCluster.Properties.InfraID + "-pip-v4')]"), + }, + }, + Name: to.StringPtr("public-lb-ip-v4"), + }, + }, + BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{ + { + Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID), + }, + }, + LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{}, //required to override default LB rules for port 80 and 443 + Probes: &[]mgmtnetwork.Probe{}, //required to override default LB rules for port 80 and 443 + OutboundRules: &[]mgmtnetwork.OutboundRule{ + { + OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '" + m.doc.OpenShiftCluster.Properties.InfraID + "', 'public-lb-ip-v4')]"), + }, + }, + BackendAddressPool: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s', '%[1]s')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Protocol: mgmtnetwork.LoadBalancerOutboundRuleProtocolAll, + IdleTimeoutInMinutes: to.Int32Ptr(30), + }, + Name: to.StringPtr("outbound-rule-v4"), + }, + }, + }, + Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID), + Type: to.StringPtr("Microsoft.Network/loadBalancers"), + Location: &installConfig.Config.Azure.Region, + } + + if m.doc.OpenShiftCluster.Properties.APIServerProfile.Visibility == api.VisibilityPublic { + *lb.LoadBalancingRules = append(*lb.LoadBalancingRules, mgmtnetwork.LoadBalancingRule{ + LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{ + FrontendIPConfiguration: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s', 'public-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + BackendAddressPool: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s', '%[1]s')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Probe: &mgmtnetwork.SubResource{ + ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s', 'api-internal-probe')]", m.doc.OpenShiftCluster.Properties.InfraID)), + }, + Protocol: mgmtnetwork.TransportProtocolTCP, + LoadDistribution: mgmtnetwork.LoadDistributionDefault, + FrontendPort: to.Int32Ptr(6443), + BackendPort: to.Int32Ptr(6443), + IdleTimeoutInMinutes: to.Int32Ptr(30), + DisableOutboundSnat: to.BoolPtr(true), + }, + Name: to.StringPtr("api-internal-v4"), + }) + + *lb.Probes = append(*lb.Probes, mgmtnetwork.Probe{ + ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{ + Protocol: mgmtnetwork.ProbeProtocolHTTPS, + Port: to.Int32Ptr(6443), + IntervalInSeconds: to.Int32Ptr(5), + NumberOfProbes: to.Int32Ptr(2), + RequestPath: to.StringPtr("/readyz"), + }, + Name: to.StringPtr("api-internal-probe"), + }) + } + + return &arm.Resource{ + Resource: lb, + APIVersion: azureclient.APIVersion("Microsoft.Network"), + DependsOn: []string{ + "Microsoft.Network/publicIPAddresses/" + m.doc.OpenShiftCluster.Properties.InfraID + "-pip-v4", + }, + } +} diff --git a/pkg/cluster/fixmcscert.go b/pkg/cluster/fixmcscert.go new file mode 100644 index 00000000000..810d40ea6b8 --- /dev/null +++ b/pkg/cluster/fixmcscert.go @@ -0,0 +1,100 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "net" + "strings" + + "github.com/openshift/installer/pkg/asset/tls" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + + "github.com/Azure/ARO-RP/pkg/util/pem" + "github.com/Azure/ARO-RP/pkg/util/stringutils" +) + +func (m *manager) fixMCSCert(ctx context.Context) error { + resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') + account := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix + + intIP := net.ParseIP(m.doc.OpenShiftCluster.Properties.APIServerProfile.IntIP) + + domain := m.doc.OpenShiftCluster.Properties.ClusterProfile.Domain + if !strings.ContainsRune(domain, '.') { + domain += "." + m.env.Domain() + } + + var rootCA *tls.RootCA + var certChanged bool + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + s, err := m.kubernetescli.CoreV1().Secrets("openshift-machine-config-operator").Get(ctx, "machine-config-server-tls", metav1.GetOptions{}) + if err != nil { + return err + } + + _, certs, err := pem.Parse(s.Data[v1.TLSCertKey]) + if err != nil { + return err + } + + if len(certs) != 1 { + return fmt.Errorf("expected 1 certificate, got %d", len(certs)) + } + + if len(certs[0].IPAddresses) == 1 && certs[0].IPAddresses[0].Equal(intIP) { + return nil + } + + certChanged = true + + if rootCA == nil { + pg, err := m.graph.LoadPersisted(ctx, resourceGroup, account) + if err != nil { + return err + } + + err = pg.Get(&rootCA) + if err != nil { + return err + } + } + + cfg := &tls.CertCfg{ + Subject: pkix.Name{CommonName: "system:machine-config-server"}, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + Validity: tls.ValidityTenYears, + IPAddresses: []net.IP{intIP}, + DNSNames: []string{"api-int." + domain, intIP.String()}, + } + + var mcsCertKey tls.AdminKubeConfigClientCertKey + + err = mcsCertKey.SignedCertKey.Generate(cfg, rootCA, "machine-config-server", tls.DoNotAppendParent) + if err != nil { + return err + } + + s.Data[v1.TLSCertKey] = mcsCertKey.CertRaw + s.Data[v1.TLSPrivateKeyKey] = mcsCertKey.KeyRaw + + _, err = m.kubernetescli.CoreV1().Secrets("openshift-machine-config-operator").Update(ctx, s, metav1.UpdateOptions{}) + return err + }) + if err != nil || !certChanged { + return err + } + + /* don't crash */ + + return m.kubernetescli.CoreV1().Pods("openshift-machine-config-operator").DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ + LabelSelector: "k8s-app=machine-config-server", + }) +} diff --git a/pkg/cluster/fixmcscert_test.go b/pkg/cluster/fixmcscert_test.go new file mode 100644 index 00000000000..0108dafe55f --- /dev/null +++ b/pkg/cluster/fixmcscert_test.go @@ -0,0 +1,207 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "net" + "testing" + + "github.com/golang/mock/gomock" + "github.com/openshift/installer/pkg/asset/tls" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/cluster/graph" + mock_graph "github.com/Azure/ARO-RP/pkg/util/mocks/graph" + utilpem "github.com/Azure/ARO-RP/pkg/util/pem" + utiltls "github.com/Azure/ARO-RP/pkg/util/tls" +) + +func TestFixMCSCert(t *testing.T) { + ctx := context.Background() + + validCaKey, validCaCerts, err := utiltls.GenerateKeyAndCertificate("validca", nil, nil, true, false) + if err != nil { + t.Fatal(err) + } + + pool := x509.NewCertPool() + pool.AddCert(validCaCerts[0]) + + for _, tt := range []struct { + name string + manager func(*gomock.Controller, *bool) (*manager, error) + wantDeleteCalled bool + }{ + { + name: "basic", + manager: func(controller *gomock.Controller, deleteCalled *bool) (*manager, error) { + b := x509.MarshalPKCS1PrivateKey(validCaKey) + + _, validCerts, err := utiltls.GenerateKeyAndCertificate("cert", validCaKey, validCaCerts[0], false, false) + if err != nil { + t.Fatal(err) + } + + pg := graph.PersistedGraph{} + err = pg.Set(&tls.RootCA{ + SelfSignedCertKey: tls.SelfSignedCertKey{ + CertKey: tls.CertKey{ + CertRaw: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: validCaCerts[0].Raw}), + KeyRaw: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: b}), + }, + }, + }) + if err != nil { + return nil, err + } + + graph := mock_graph.NewMockManager(controller) + graph.EXPECT().LoadPersisted(ctx, "", "cluster").Return(pg, nil) + + kubernetescli := fake.NewSimpleClientset(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machine-config-server-tls", + Namespace: "openshift-machine-config-operator", + }, + Data: map[string][]byte{ + v1.TLSCertKey: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: validCerts[0].Raw}), + }, + }) + kubernetescli.AddReactor("delete-collection", "pods", func(action ktesting.Action) (handled bool, ret runtime.Object, err error) { + if action, ok := action.(ktesting.DeleteCollectionAction); ok { + if action.GetListRestrictions().Labels.String() == "k8s-app=machine-config-server" { + *deleteCalled = true + } + } + return false, nil, nil + }) + + return &manager{ + doc: &api.OpenShiftClusterDocument{ + OpenShiftCluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + Domain: "foo.bar", + }, + APIServerProfile: api.APIServerProfile{ + IntIP: "10.0.0.1", + }, + }, + }, + }, + graph: graph, + kubernetescli: kubernetescli, + }, nil + }, + wantDeleteCalled: true, + }, + { + name: "noop", + manager: func(controller *gomock.Controller, deleteCalled *bool) (*manager, error) { + validKey, validCerts, err := utiltls.GenerateTestKeyAndCertificate("system:machine-config-server", validCaKey, validCaCerts[0], false, false, func(template *x509.Certificate) { + template.IPAddresses = []net.IP{net.ParseIP("10.0.0.1")} + }) + if err != nil { + t.Fatal(err) + } + + b := x509.MarshalPKCS1PrivateKey(validKey) + + return &manager{ + doc: &api.OpenShiftClusterDocument{ + OpenShiftCluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + Domain: "foo.bar", + }, + APIServerProfile: api.APIServerProfile{ + IntIP: "10.0.0.1", + }, + }, + }, + }, + kubernetescli: fake.NewSimpleClientset(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machine-config-server-tls", + Namespace: "openshift-machine-config-operator", + }, + Data: map[string][]byte{ + v1.TLSCertKey: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: validCerts[0].Raw}), + v1.TLSPrivateKeyKey: pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: b}), + }, + }), + }, nil + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + var deleteCalled bool + m, err := tt.manager(controller, &deleteCalled) + if err != nil { + t.Error(err) + } + + err = m.fixMCSCert(ctx) + if err != nil { + t.Error(err) + } + + if deleteCalled != tt.wantDeleteCalled { + t.Error(deleteCalled) + } + + s, err := m.kubernetescli.CoreV1().Secrets("openshift-machine-config-operator").Get(ctx, "machine-config-server-tls", metav1.GetOptions{}) + if err != nil { + t.Error(err) + } + + var pemdata []byte + pemdata = append(pemdata, s.Data[v1.TLSCertKey]...) + pemdata = append(pemdata, s.Data[v1.TLSPrivateKeyKey]...) + + key, certs, err := utilpem.Parse(pemdata) + if err != nil { + t.Error(err) + } + + cert := certs[0] + + _, err = cert.Verify(x509.VerifyOptions{ + Roots: pool, + }) + if err != nil { + t.Error(err) + } + + if !publicKeysEqual(cert.PublicKey.(*rsa.PublicKey), &key.PublicKey) { + t.Error("key mismatch") + } + + if cert.Subject.String() != "CN=system:machine-config-server" { + t.Error(cert.Subject) + } + + if !cert.IPAddresses[0].Equal(net.ParseIP("10.0.0.1")) { + t.Error(cert.IPAddresses[0]) + } + }) + } +} + +// TODO: at Go >= 1.15, use (*rsa.PublicKey) Equal() +func publicKeysEqual(a, b *rsa.PublicKey) bool { + return a.N.Cmp(b.N) == 0 && a.E == b.E +} diff --git a/pkg/cluster/fixmcsuserdata.go b/pkg/cluster/fixmcsuserdata.go new file mode 100644 index 00000000000..24c3b16cac4 --- /dev/null +++ b/pkg/cluster/fixmcsuserdata.go @@ -0,0 +1,199 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net" + "net/url" + "strings" + + machinev1beta1 "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1" + "github.com/ugorji/go/codec" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/util/retry" + azureproviderv1beta1 "sigs.k8s.io/cluster-api-provider-azure/pkg/apis/azureprovider/v1beta1" + + "github.com/Azure/ARO-RP/pkg/api" + _ "github.com/Azure/ARO-RP/pkg/util/scheme" +) + +type userData struct { + api.MissingFields + Ignition struct { + api.MissingFields + Config struct { + api.MissingFields + Merge []struct { // ignition 3.x + api.MissingFields + Source string `json:"source,omitempty"` + } `json:"merge,omitempty"` + Append []struct { // ignition 2.x + api.MissingFields + Source string `json:"source,omitempty"` + } `json:"append,omitempty"` + } `json:"config,omitempty"` + } `json:"ignition,omitempty"` +} + +func (m *manager) enumerateUserDataSecrets(ctx context.Context) map[corev1.SecretReference]struct{} { + secretRefs := map[corev1.SecretReference]struct{}{} + + machinesets, err := m.maocli.MachineV1beta1().MachineSets("").List(ctx, metav1.ListOptions{}) + if err != nil { + m.log.Print(err) + + } else { + for _, machineset := range machinesets.Items { + ref, err := getUserDataSecretReference(&machineset.ObjectMeta, &machineset.Spec.Template.Spec) + if err != nil { + m.log.Printf("%s/%s: %s", machineset.Namespace, machineset.Name, err) + continue + } + if ref != nil { + secretRefs[*ref] = struct{}{} + } + } + } + + machines, err := m.maocli.MachineV1beta1().Machines("").List(ctx, metav1.ListOptions{}) + if err != nil { + m.log.Print(err) + + } else { + for _, machine := range machines.Items { + ref, err := getUserDataSecretReference(&machine.ObjectMeta, &machine.Spec) + if err != nil { + m.log.Printf("%s/%s: %s", machine.Namespace, machine.Name, err) + continue + } + if ref != nil { + secretRefs[*ref] = struct{}{} + } + } + } + + return secretRefs +} + +func getUserDataSecretReference(objMeta *metav1.ObjectMeta, spec *machinev1beta1.MachineSpec) (*corev1.SecretReference, error) { + if spec.ProviderSpec.Value == nil { + return nil, nil + } + + o, _, err := scheme.Codecs.UniversalDeserializer().Decode(spec.ProviderSpec.Value.Raw, nil, nil) + if err != nil { + return nil, err + } + + machineProviderSpec, ok := o.(*azureproviderv1beta1.AzureMachineProviderSpec) + if !ok { + return nil, fmt.Errorf("failed to read provider spec: %T", o) + } + + if machineProviderSpec.UserDataSecret == nil { + return nil, nil + } + + if machineProviderSpec.UserDataSecret.Namespace == "" { + machineProviderSpec.UserDataSecret.Namespace = objMeta.Namespace + } + + return machineProviderSpec.UserDataSecret, nil +} + +func (m *manager) fixMCSUserData(ctx context.Context) error { + h := codec.JsonHandle{ + BasicHandle: codec.BasicHandle{ + EncodeOptions: codec.EncodeOptions{ + Canonical: true, + }, + }, + } + + for secretRef := range m.enumerateUserDataSecrets(ctx) { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + s, err := m.kubernetescli.CoreV1().Secrets(secretRef.Namespace).Get(ctx, secretRef.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + var userData *userData + err = codec.NewDecoderBytes(s.Data["userData"], &h).Decode(&userData) + if err != nil { + return err + } + + var changed bool + for i, a := range userData.Ignition.Config.Merge { + var _changed bool + a.Source, _changed, err = m.fixSource(a.Source) + if err != nil { + return err + } + + changed = changed || _changed + + userData.Ignition.Config.Merge[i] = a + } + + for i, a := range userData.Ignition.Config.Append { + var _changed bool + a.Source, _changed, err = m.fixSource(a.Source) + if err != nil { + return err + } + + changed = changed || _changed + + userData.Ignition.Config.Append[i] = a + } + + if !changed { + return nil + } + + var b []byte + err = codec.NewEncoderBytes(&b, &h).Encode(userData) + if err != nil { + return err + } + + s.Data["userData"] = b + + _, err = m.kubernetescli.CoreV1().Secrets(secretRef.Namespace).Update(ctx, s, metav1.UpdateOptions{}) + return err + }) + if err != nil { + m.log.Printf("%s/%s: %s", secretRef.Namespace, secretRef.Name, err) + } + } + + return nil +} + +func (m *manager) fixSource(source string) (string, bool, error) { + intIP := net.ParseIP(m.doc.OpenShiftCluster.Properties.APIServerProfile.IntIP) + + domain := m.doc.OpenShiftCluster.Properties.ClusterProfile.Domain + if !strings.ContainsRune(domain, '.') { + domain += "." + m.env.Domain() + } + + u, err := url.Parse(source) + if err != nil { + return "", false, err + } + + var changed bool + if u.Hostname() == "api-int."+domain { + u.Host = intIP.String() + ":" + u.Port() + changed = true + } + + return u.String(), changed, nil +} diff --git a/pkg/cluster/fixmcsuserdata_test.go b/pkg/cluster/fixmcsuserdata_test.go new file mode 100644 index 00000000000..d6418cb4161 --- /dev/null +++ b/pkg/cluster/fixmcsuserdata_test.go @@ -0,0 +1,206 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "bytes" + "context" + "encoding/json" + "reflect" + "testing" + + machinev1beta1 "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1" + maofake "github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned/fake" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + kjson "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/kubernetes/scheme" + azureproviderv1beta1 "sigs.k8s.io/cluster-api-provider-azure/pkg/apis/azureprovider/v1beta1" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/cmp" + _ "github.com/Azure/ARO-RP/pkg/util/scheme" +) + +func marshalAzureMachineProviderSpec(t *testing.T, spec *azureproviderv1beta1.AzureMachineProviderSpec) []byte { + serializer := kjson.NewSerializerWithOptions( + kjson.DefaultMetaFactory, scheme.Scheme, scheme.Scheme, + kjson.SerializerOptions{Yaml: true}, + ) + + yaml := scheme.Codecs.CodecForVersions(serializer, nil, schema.GroupVersions(scheme.Scheme.PrioritizedVersionsAllGroups()), nil) + + buf := &bytes.Buffer{} + err := yaml.Encode(spec, buf) + if err != nil { + t.Fatal(err) + } + + return buf.Bytes() +} + +func marshal(t *testing.T, i interface{}) []byte { + b, err := json.Marshal(i) + if err != nil { + t.Fatal(i) + } + if err != nil { + t.Fatal(err) + } + + return b +} + +func canonicalise(t *testing.T, b []byte) []byte { + var i interface{} + + err := json.Unmarshal(b, &i) + if err != nil { + t.Fatal(i) + } + b, err = json.Marshal(i) + if err != nil { + t.Fatal(i) + } + + return b +} + +func userDataSecret(t *testing.T, namespace, name, appendSource, mergeSource string) *v1.Secret { + config := map[string]interface{}{ + "extrakey": true, + } + + if appendSource != "" { + config["append"] = []interface{}{ + map[string]interface{}{ + "extrakey": []interface{}{}, + "source": appendSource, + }, + } + } + + if mergeSource != "" { + config["merge"] = []interface{}{ + map[string]interface{}{ + "extrakey": map[string]interface{}{}, + "source": appendSource, + }, + } + } + + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: map[string][]byte{ + "userData": marshal(t, map[string]interface{}{ + "extrakey": 1, + "ignition": map[string]interface{}{ + "extrakey": "2", + "config": config, + }, + }), + }, + } +} + +func testMachine(t *testing.T, namespace, name string, spec *azureproviderv1beta1.AzureMachineProviderSpec) *machinev1beta1.Machine { + return &machinev1beta1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: machinev1beta1.MachineSpec{ + ProviderSpec: machinev1beta1.ProviderSpec{ + Value: &runtime.RawExtension{ + Raw: marshalAzureMachineProviderSpec(t, spec), + }, + }, + }, + } +} + +func testMachineSet(t *testing.T, namespace, name string, spec *azureproviderv1beta1.AzureMachineProviderSpec) *machinev1beta1.MachineSet { + return &machinev1beta1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: machinev1beta1.MachineSetSpec{ + Template: machinev1beta1.MachineTemplateSpec{ + Spec: machinev1beta1.MachineSpec{ + ProviderSpec: machinev1beta1.ProviderSpec{ + Value: &runtime.RawExtension{ + Raw: marshalAzureMachineProviderSpec(t, spec), + }, + }, + }, + }, + }, + } +} + +func TestFixMCSUserData(t *testing.T) { + ctx := context.Background() + + m := &manager{ + doc: &api.OpenShiftClusterDocument{ + OpenShiftCluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + Domain: "example.com", + }, + APIServerProfile: api.APIServerProfile{ + IntIP: "1.2.3.4", + }, + }, + }, + }, + kubernetescli: fake.NewSimpleClientset( + userDataSecret(t, "openshift-machine-api", "master-user-data", "https://api-int.example.com:22623/config/master", ""), + userDataSecret(t, "openshift-machine-api", "worker-user-data", "", "https://api-int.example.com:22623/config/worker"), + ), + maocli: maofake.NewSimpleClientset( + testMachineSet(t, "openshift-machine-api", "worker", &azureproviderv1beta1.AzureMachineProviderSpec{ + UserDataSecret: &v1.SecretReference{ + Name: "worker-user-data", + }, + }), + testMachine(t, "openshift-machine-api", "master", &azureproviderv1beta1.AzureMachineProviderSpec{ + UserDataSecret: &v1.SecretReference{ + Name: "master-user-data", + }, + }), + ), + } + + wantSecrets := []*v1.Secret{ + userDataSecret(t, "openshift-machine-api", "master-user-data", "https://1.2.3.4:22623/config/master", ""), + userDataSecret(t, "openshift-machine-api", "worker-user-data", "", "https://1.2.3.4:22623/config/worker"), + } + + err := m.fixMCSUserData(ctx) + if err != nil { + t.Fatal(err) + } + + for _, wantSecret := range wantSecrets { + s, err := m.kubernetescli.CoreV1().Secrets(wantSecret.Namespace).Get(ctx, wantSecret.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + + // needed because of https://github.com/ugorji/go/issues/354 + s.Data["userData"] = canonicalise(t, s.Data["userData"]) + + if !reflect.DeepEqual(s, wantSecret) { + t.Error(cmp.Diff(s, wantSecret)) + } + } +} diff --git a/pkg/cluster/fixsrekubeconfig.go b/pkg/cluster/fixsrekubeconfig.go index 80ee1561e60..8f65d66e568 100644 --- a/pkg/cluster/fixsrekubeconfig.go +++ b/pkg/cluster/fixsrekubeconfig.go @@ -7,6 +7,7 @@ import ( "context" "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/stringutils" ) func (m *manager) fixSREKubeconfig(ctx context.Context) error { @@ -14,7 +15,10 @@ func (m *manager) fixSREKubeconfig(ctx context.Context) error { return nil } - pg, err := m.loadPersistedGraph(ctx) + resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') + account := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix + + pg, err := m.graph.LoadPersisted(ctx, resourceGroup, account) if err != nil { return err } diff --git a/pkg/cluster/graph.go b/pkg/cluster/graph.go deleted file mode 100644 index dac9abb440d..00000000000 --- a/pkg/cluster/graph.go +++ /dev/null @@ -1,100 +0,0 @@ -package cluster - -// Copyright (c) Microsoft Corporation. -// Licensed under the Apache License 2.0. - -import ( - "bytes" - "context" - "encoding/json" - "reflect" - - mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" - "github.com/openshift/installer/pkg/asset" - "github.com/openshift/installer/pkg/asset/ignition/bootstrap" -) - -// graph is used to generate and persist the graph as a one-off. For subsequent -// uses, use persistedGraph. - -type graph map[string]asset.Asset - -func (g graph) get(a asset.Asset) asset.Asset { - return g[reflect.TypeOf(a).String()] -} - -func (g graph) set(as ...asset.Asset) { - for _, a := range as { - g[reflect.TypeOf(a).String()] = a - } -} - -func (g graph) resolve(a asset.Asset) error { - if g.get(a) != nil { - return nil - } - - for _, dep := range a.Dependencies() { - err := g.resolve(dep) - if err != nil { - return err - } - } - - parents := asset.Parents{} - for _, v := range g { - parents[reflect.TypeOf(v)] = v - } - - err := a.Generate(parents) - if err != nil { - return err - } - - g.set(a) - - return nil -} - -func (m *manager) graphExists(ctx context.Context) (bool, error) { - m.log.Print("checking if graph exists") - - blobService, err := m.getBlobService(ctx, mgmtstorage.Permissions("r"), mgmtstorage.SignedResourceTypesO) - if err != nil { - return false, err - } - - aro := blobService.GetContainerReference("aro") - return aro.GetBlobReference("graph").Exists() -} - -// loadGraph() should not be implemented: use loadPersistedGraph - -func (m *manager) saveGraph(ctx context.Context, g graph) error { - m.log.Print("save graph") - - blobService, err := m.getBlobService(ctx, mgmtstorage.Permissions("cw"), mgmtstorage.SignedResourceTypesO) - if err != nil { - return err - } - - bootstrap := g.get(&bootstrap.Bootstrap{}).(*bootstrap.Bootstrap) - bootstrapIgn := blobService.GetContainerReference("ignition").GetBlobReference("bootstrap.ign") - err = bootstrapIgn.CreateBlockBlobFromReader(bytes.NewReader(bootstrap.File.Data), nil) - if err != nil { - return err - } - - graph := blobService.GetContainerReference("aro").GetBlobReference("graph") - b, err := json.MarshalIndent(g, "", " ") - if err != nil { - return err - } - - b, err = m.aead.Seal(b) - if err != nil { - return err - } - - return graph.CreateBlockBlobFromReader(bytes.NewReader(b), nil) -} diff --git a/pkg/cluster/graph/generate.go b/pkg/cluster/graph/generate.go new file mode 100644 index 00000000000..5fd91b14c4a --- /dev/null +++ b/pkg/cluster/graph/generate.go @@ -0,0 +1,8 @@ +package graph + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +//go:generate rm -rf ../../util/mocks/$GOPACKAGE +//go:generate go run ../../../vendor/github.com/golang/mock/mockgen -destination=../../util/mocks/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/cluster/$GOPACKAGE Manager +//go:generate go run ../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../util/mocks/$GOPACKAGE/$GOPACKAGE.go diff --git a/pkg/cluster/graph/graph.go b/pkg/cluster/graph/graph.go new file mode 100644 index 00000000000..d5b98837b98 --- /dev/null +++ b/pkg/cluster/graph/graph.go @@ -0,0 +1,52 @@ +package graph + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "reflect" + + "github.com/openshift/installer/pkg/asset" +) + +// Graph is used to generate and persist the graph as a one-off. For subsequent +// uses, use PersistedGraph. + +type Graph map[string]asset.Asset + +func (g Graph) Get(a asset.Asset) asset.Asset { + return g[reflect.TypeOf(a).String()] +} + +func (g Graph) Set(as ...asset.Asset) { + for _, a := range as { + g[reflect.TypeOf(a).String()] = a + } +} + +func (g Graph) Resolve(a asset.Asset) error { + if g.Get(a) != nil { + return nil + } + + for _, dep := range a.Dependencies() { + err := g.Resolve(dep) + if err != nil { + return err + } + } + + parents := asset.Parents{} + for _, v := range g { + parents[reflect.TypeOf(v)] = v + } + + err := a.Generate(parents) + if err != nil { + return err + } + + g.Set(a) + + return nil +} diff --git a/pkg/cluster/graph/manager.go b/pkg/cluster/graph/manager.go new file mode 100644 index 00000000000..c6baa638685 --- /dev/null +++ b/pkg/cluster/graph/manager.go @@ -0,0 +1,121 @@ +package graph + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "bytes" + "context" + "encoding/json" + "io/ioutil" + + mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" + "github.com/openshift/installer/pkg/asset/ignition/bootstrap" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/util/encryption" + "github.com/Azure/ARO-RP/pkg/util/storage" +) + +type Manager interface { + Exists(ctx context.Context, resourceGroup, account string) (bool, error) + Save(ctx context.Context, resourceGroup, account string, g Graph) error + LoadPersisted(ctx context.Context, resourceGroup, account string) (PersistedGraph, error) +} + +type manager struct { + log *logrus.Entry + + aead encryption.AEAD + storage storage.Manager +} + +func NewManager(log *logrus.Entry, aead encryption.AEAD, storage storage.Manager) Manager { + return &manager{ + log: log, + + aead: aead, + storage: storage, + } +} + +func (m *manager) Exists(ctx context.Context, resourceGroup, account string) (bool, error) { + m.log.Print("checking if graph exists") + + blobService, err := m.storage.BlobService(ctx, resourceGroup, account, mgmtstorage.Permissions("r"), mgmtstorage.SignedResourceTypesO) + if err != nil { + return false, err + } + + aro := blobService.GetContainerReference("aro") + return aro.GetBlobReference("graph").Exists() +} + +// Load() should not be implemented: use LoadPersisted + +func (m *manager) Save(ctx context.Context, resourceGroup, account string, g Graph) error { + m.log.Print("save graph") + + blobService, err := m.storage.BlobService(ctx, resourceGroup, account, mgmtstorage.Permissions("cw"), mgmtstorage.SignedResourceTypesO) + if err != nil { + return err + } + + bootstrap := g.Get(&bootstrap.Bootstrap{}).(*bootstrap.Bootstrap) + bootstrapIgn := blobService.GetContainerReference("ignition").GetBlobReference("bootstrap.ign") + err = bootstrapIgn.CreateBlockBlobFromReader(bytes.NewReader(bootstrap.File.Data), nil) + if err != nil { + return err + } + + graph := blobService.GetContainerReference("aro").GetBlobReference("graph") + b, err := json.MarshalIndent(g, "", " ") + if err != nil { + return err + } + + b, err = m.aead.Seal(b) + if err != nil { + return err + } + + return graph.CreateBlockBlobFromReader(bytes.NewReader(b), nil) +} + +func (m *manager) LoadPersisted(ctx context.Context, resourceGroup, account string) (PersistedGraph, error) { + m.log.Print("load persisted graph") + + blobService, err := m.storage.BlobService(ctx, resourceGroup, account, mgmtstorage.Permissions("r"), mgmtstorage.SignedResourceTypesO) + if err != nil { + return nil, err + } + + aro := blobService.GetContainerReference("aro") + cluster := aro.GetBlobReference("graph") + rc, err := cluster.Get(nil) + if err != nil { + return nil, err + } + defer rc.Close() + + b, err := ioutil.ReadAll(rc) + if err != nil { + return nil, err + } + + b, err = m.aead.Open(b) + if err != nil { + return nil, err + } + + var pg PersistedGraph + err = json.Unmarshal(b, &pg) + if err != nil { + return nil, err + } + + return pg, nil +} + +// SavePersistedGraph could be implemented and used with care if needed, but +// currently we don't need it (and it's better that way) diff --git a/pkg/cluster/graph/persistedgraph.go b/pkg/cluster/graph/persistedgraph.go new file mode 100644 index 00000000000..4d2d0bb0863 --- /dev/null +++ b/pkg/cluster/graph/persistedgraph.go @@ -0,0 +1,46 @@ +package graph + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "bytes" + "encoding/json" + "reflect" +) + +// PersistedGraph is a graph read from the cluster storage account. +// Unfortunately as the object schema changes over time, there are no guarantees +// that we can easily parse the objects in the graph, so we leave them as json +// RawMessages. You can expect Get() to work in the context of cluster creation, +// but not necessarily subsequently. + +type PersistedGraph map[string]json.RawMessage + +func (pg PersistedGraph) Get(is ...interface{}) error { + for _, i := range is { + d := json.NewDecoder(bytes.NewReader(pg[reflect.TypeOf(i).Elem().String()])) + d.DisallowUnknownFields() + + err := d.Decode(i) + if err != nil { + return err + } + } + + return nil +} + +// Set is currently only used in unit test context. If you want to use this in +// production, you will want to be very sure that you are not losing state that +// you may need later +func (pg PersistedGraph) Set(is ...interface{}) (err error) { + for _, i := range is { + pg[reflect.TypeOf(i).String()], err = json.Marshal(i) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go index 7e55888a1f3..44d2d7e9b9e 100644 --- a/pkg/cluster/install.go +++ b/pkg/cluster/install.go @@ -14,6 +14,8 @@ import ( securityclient "github.com/openshift/client-go/security/clientset/versioned" "github.com/openshift/installer/pkg/asset/installconfig" "github.com/openshift/installer/pkg/asset/releaseimage" + maoclient "github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned" + mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned" extensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/kubernetes" @@ -37,11 +39,15 @@ func (m *manager) AdminUpdate(ctx context.Context) error { steps.Action(m.fixSSH), steps.Action(m.populateCreatedAt), // TODO(mikalai): Remove after a round of admin updates steps.Action(m.fixSREKubeconfig), + steps.Action(m.createOrUpdateRouterIPFromCluster), + steps.Action(m.populateDatabaseIntIP), + steps.Action(m.fixMCSCert), + steps.Action(m.fixMCSUserData), steps.Action(m.ensureAROOperator), steps.Condition(m.aroDeploymentReady, 20*time.Minute), steps.Action(m.configureAPIServerCertificate), steps.Action(m.configureIngressCertificate), - steps.Action(m.createOrUpdateRouterIP), + steps.Action(m.removePrivateDNSZone), steps.Action(m.updateProvisionedBy), // Run this last so we capture the resource provider only once the upgrade has been fully performed } @@ -82,11 +88,15 @@ func (m *manager) Install(ctx context.Context) error { steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(func(ctx context.Context) error { return m.deployStorageTemplate(ctx, installConfig, image) })), + steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.updateAPIIPEarly)), + steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.createOrUpdateRouterIPEarly)), + steps.Action(func(ctx context.Context) error { + return m.ensureGraph(ctx, installConfig, image) + }), steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.attachNSGsAndPatch)), steps.Action(m.ensureBillingRecord), steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.deployResourceTemplate)), steps.Action(m.createPrivateEndpoint), - steps.Action(m.updateAPIIP), steps.Action(m.createCertificates), steps.Action(m.initializeKubernetesClients), steps.Condition(m.bootstrapConfigMapReady, 30*time.Minute), @@ -108,7 +118,6 @@ func (m *manager) Install(ctx context.Context) error { steps.Action(m.disableUpdates), steps.Action(m.disableSamples), steps.Action(m.disableOperatorHubSources), - steps.Action(m.createOrUpdateRouterIP), steps.Action(m.updateClusterData), steps.Action(m.configureIngressCertificate), steps.Condition(m.ingressControllerReady, 30*time.Minute), @@ -183,6 +192,16 @@ func (m *manager) initializeKubernetesClients(ctx context.Context) error { return err } + m.maocli, err = maoclient.NewForConfig(restConfig) + if err != nil { + return err + } + + m.mcocli, err = mcoclient.NewForConfig(restConfig) + if err != nil { + return err + } + m.operatorcli, err = operatorclient.NewForConfig(restConfig) if err != nil { return err diff --git a/pkg/cluster/ipaddresses.go b/pkg/cluster/ipaddresses.go index 263d1d16ddb..01cd19567d7 100644 --- a/pkg/cluster/ipaddresses.go +++ b/pkg/cluster/ipaddresses.go @@ -6,6 +6,7 @@ package cluster import ( "context" "fmt" + "net/http" "github.com/openshift/installer/pkg/asset/installconfig" "github.com/openshift/installer/pkg/asset/password" @@ -16,14 +17,17 @@ import ( ) func (m *manager) updateClusterData(ctx context.Context) error { - pg, err := m.loadPersistedGraph(ctx) + resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') + account := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix + + pg, err := m.graph.LoadPersisted(ctx, resourceGroup, account) if err != nil { return err } var installConfig *installconfig.InstallConfig var kubeadminPassword *password.KubeadminPassword - err = pg.get(&installConfig, &kubeadminPassword) + err = pg.Get(&installConfig, &kubeadminPassword) if err != nil { return err } @@ -37,7 +41,7 @@ func (m *manager) updateClusterData(ctx context.Context) error { return err } -func (m *manager) createOrUpdateRouterIP(ctx context.Context) error { +func (m *manager) createOrUpdateRouterIPFromCluster(ctx context.Context) error { svc, err := m.kubernetescli.CoreV1().Services("openshift-ingress").Get(ctx, "router-default", metav1.GetOptions{}) // default ingress must be present in the cluster if err != nil { @@ -49,57 +53,140 @@ func (m *manager) createOrUpdateRouterIP(ctx context.Context) error { return fmt.Errorf("routerIP not found") } - routerIP := svc.Status.LoadBalancer.Ingress[0].IP + ipAddress := svc.Status.LoadBalancer.Ingress[0].IP - err = m.dns.CreateOrUpdateRouter(ctx, m.doc.OpenShiftCluster, routerIP) + err = m.dns.CreateOrUpdateRouter(ctx, m.doc.OpenShiftCluster, ipAddress) if err != nil { return err } m.doc, err = m.db.PatchWithLease(ctx, m.doc.Key, func(doc *api.OpenShiftClusterDocument) error { - doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = routerIP + doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = ipAddress return nil }) return err } -func (m *manager) updateAPIIP(ctx context.Context) error { +func (m *manager) createOrUpdateRouterIPEarly(ctx context.Context) error { infraID := m.doc.OpenShiftCluster.Properties.InfraID resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') var ipAddress string - if m.doc.OpenShiftCluster.Properties.APIServerProfile.Visibility == api.VisibilityPublic { - ip, err := m.publicIPAddresses.Get(ctx, resourceGroup, infraID+"-pip-v4", "") + if m.doc.OpenShiftCluster.Properties.IngressProfiles[0].Visibility == api.VisibilityPublic { + ip, err := m.publicIPAddresses.Get(ctx, resourceGroup, infraID+"-default-v4", "") if err != nil { return err } ipAddress = *ip.IPAddress } else { - lb, err := m.loadBalancers.Get(ctx, resourceGroup, infraID+"-internal", "") + // there's no way to reserve private IPs in Azure, so we pick the + // highest free address in the subnet (i.e., there's a race here). Azure + // specifically documents that dynamic allocation proceeds from the + // bottom of the subnet, so there's a good chance that we'll get away + // with this. + // https://docs.microsoft.com/en-us/azure/virtual-network/private-ip-addresses#allocation-method + var err error + ipAddress, err = m.subnet.GetHighestFreeIP(ctx, m.doc.OpenShiftCluster.Properties.WorkerProfiles[0].SubnetID) if err != nil { return err } - ipAddress = *((*lb.FrontendIPConfigurations)[0].PrivateIPAddress) + if ipAddress == "" { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidLinkedVNet, "", "The subnet '%s' has no remaining IP addresses.", m.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID) + } } - err := m.dns.Update(ctx, m.doc.OpenShiftCluster, ipAddress) + err := m.dns.CreateOrUpdateRouter(ctx, m.doc.OpenShiftCluster, ipAddress) if err != nil { return err } - privateEndpointIP, err := m.privateendpoint.GetIP(ctx, m.doc) + m.doc, err = m.db.PatchWithLease(ctx, m.doc.Key, func(doc *api.OpenShiftClusterDocument) error { + doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = ipAddress + return nil + }) + return err +} + +func (m *manager) populateDatabaseIntIP(ctx context.Context) error { + if m.doc.OpenShiftCluster.Properties.APIServerProfile.IntIP != "" { + return nil + } + + resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') + + infraID := m.doc.OpenShiftCluster.Properties.InfraID + + var lbName string + switch m.doc.OpenShiftCluster.Properties.ArchitectureVersion { + case api.ArchitectureVersionV1: + lbName = infraID + "-internal-lb" + case api.ArchitectureVersionV2: + lbName = infraID + "-internal" + default: + return fmt.Errorf("unknown architecture version %d", m.doc.OpenShiftCluster.Properties.ArchitectureVersion) + } + + lb, err := m.loadBalancers.Get(ctx, resourceGroup, lbName, "") + if err != nil { + return err + } + + m.doc, err = m.db.PatchWithLease(ctx, m.doc.Key, func(doc *api.OpenShiftClusterDocument) error { + doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = *((*lb.FrontendIPConfigurations)[0].PrivateIPAddress) + return nil + }) + return err +} + +// this function can only be called on create - not on update - because it +// refers to -pip-v4, which doesn't exist on pre-DNS change clusters. +func (m *manager) updateAPIIPEarly(ctx context.Context) error { + infraID := m.doc.OpenShiftCluster.Properties.InfraID + + resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') + + lb, err := m.loadBalancers.Get(ctx, resourceGroup, infraID+"-internal", "") + if err != nil { + return err + } + intIPAddress := *((*lb.FrontendIPConfigurations)[0].PrivateIPAddress) + + ipAddress := intIPAddress + if m.doc.OpenShiftCluster.Properties.APIServerProfile.Visibility == api.VisibilityPublic { + ip, err := m.publicIPAddresses.Get(ctx, resourceGroup, infraID+"-pip-v4", "") + if err != nil { + return err + } + ipAddress = *ip.IPAddress + } + + err = m.dns.Update(ctx, m.doc.OpenShiftCluster, ipAddress) if err != nil { return err } m.doc, err = m.db.PatchWithLease(ctx, m.doc.Key, func(doc *api.OpenShiftClusterDocument) error { - doc.OpenShiftCluster.Properties.NetworkProfile.PrivateEndpointIP = privateEndpointIP doc.OpenShiftCluster.Properties.APIServerProfile.IP = ipAddress + doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = intIPAddress return nil }) return err } func (m *manager) createPrivateEndpoint(ctx context.Context) error { - return m.privateendpoint.Create(ctx, m.doc) + err := m.privateendpoint.Create(ctx, m.doc) + if err != nil { + return err + } + + privateEndpointIP, err := m.privateendpoint.GetIP(ctx, m.doc) + if err != nil { + return err + } + + m.doc, err = m.db.PatchWithLease(ctx, m.doc.Key, func(doc *api.OpenShiftClusterDocument) error { + doc.OpenShiftCluster.Properties.NetworkProfile.PrivateEndpointIP = privateEndpointIP + return nil + }) + return err } diff --git a/pkg/cluster/ipaddresses_test.go b/pkg/cluster/ipaddresses_test.go index 10a01b0bb16..b71ef33a581 100644 --- a/pkg/cluster/ipaddresses_test.go +++ b/pkg/cluster/ipaddresses_test.go @@ -8,31 +8,60 @@ import ( "strings" "testing" + mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network" + "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" "github.com/Azure/ARO-RP/pkg/api" - mock_dnsmanager "github.com/Azure/ARO-RP/pkg/util/mocks/dns" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + mock_network "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/network" + mock_dns "github.com/Azure/ARO-RP/pkg/util/mocks/dns" + mock_subnet "github.com/Azure/ARO-RP/pkg/util/mocks/subnet" testdatabase "github.com/Azure/ARO-RP/test/database" ) -func TestUpdateOrCreateRouterIP(t *testing.T) { +func TestCreateOrUpdateRouterIPFromCluster(t *testing.T) { ctx := context.Background() - type test struct { - name string - kubernetescli *fake.Clientset - mocks func(*mock_dnsmanager.MockManager) - wantErr string - } + const ( + key = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName1" + ) - for _, tt := range []*test{ + for _, tt := range []struct { + name string + kubernetescli *fake.Clientset + fixtureChecker func(*testdatabase.Fixture, *testdatabase.Checker, *cosmosdb.FakeOpenShiftClusterDocumentClient) + mocks func(*mock_dns.MockManager) + wantErr string + }{ { name: "create/update success", - mocks: func(dm *mock_dnsmanager.MockManager) { - dm.EXPECT(). + fixtureChecker: func(fixture *testdatabase.Fixture, checker *testdatabase.Checker, dbClient *cosmosdb.FakeOpenShiftClusterDocumentClient) { + doc := &api.OpenShiftClusterDocument{ + Key: strings.ToLower(key), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: key, + Properties: api.OpenShiftClusterProperties{ + IngressProfiles: []api.IngressProfile{ + { + Visibility: api.VisibilityPublic, + }, + }, + ProvisioningState: api.ProvisioningStateCreating, + }, + }, + } + fixture.AddOpenShiftClusterDocuments(doc) + + doc.Dequeues = 1 + doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = "1.2.3.4" + checker.AddOpenShiftClusterDocuments(doc) + }, + mocks: func(dns *mock_dns.MockManager) { + dns.EXPECT(). CreateOrUpdateRouter(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) }, @@ -65,51 +94,519 @@ func TestUpdateOrCreateRouterIP(t *testing.T) { controller := gomock.NewController(t) defer controller.Finish() - dm := mock_dnsmanager.NewMockManager(controller) + dns := mock_dns.NewMockManager(controller) if tt.mocks != nil { - tt.mocks(dm) + tt.mocks(dns) + } + + dbOpenShiftClusters, dbClient := testdatabase.NewFakeOpenShiftClusters() + fixture := testdatabase.NewFixture().WithOpenShiftClusters(dbOpenShiftClusters) + checker := testdatabase.NewChecker() + + if tt.fixtureChecker != nil { + tt.fixtureChecker(fixture, checker, dbClient) + } + + err := fixture.Create() + if err != nil { + t.Fatal(err) } - key := "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName1" + doc, err := dbOpenShiftClusters.Dequeue(ctx) + if err != nil { + t.Fatal(err) + } + + m := &manager{ + doc: doc, + db: dbOpenShiftClusters, + dns: dns, + kubernetescli: tt.kubernetescli, + } + + err = m.createOrUpdateRouterIPFromCluster(ctx) + if err != nil && err.Error() != tt.wantErr || + err == nil && tt.wantErr != "" { + t.Error(err) + } - openShiftClustersDatabase, _ := testdatabase.NewFakeOpenShiftClusters() - fixture := testdatabase.NewFixture().WithOpenShiftClusters(openShiftClustersDatabase) - fixture.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ - Key: strings.ToLower(key), - OpenShiftCluster: &api.OpenShiftCluster{ - ID: key, - Properties: api.OpenShiftClusterProperties{ - IngressProfiles: []api.IngressProfile{ - { + for _, err = range checker.CheckOpenShiftClusters(dbClient) { + t.Error(err) + } + }) + } +} + +func TestCreateOrUpdateRouterIPEarly(t *testing.T) { + ctx := context.Background() + + const ( + key = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName1" + resourceGroupID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterResourceGroup" + ) + + for _, tt := range []struct { + name string + fixtureChecker func(*testdatabase.Fixture, *testdatabase.Checker, *cosmosdb.FakeOpenShiftClusterDocumentClient) + mocks func(*mock_network.MockPublicIPAddressesClient, *mock_dns.MockManager, *mock_subnet.MockManager) + wantErr string + }{ + { + name: "public", + fixtureChecker: func(fixture *testdatabase.Fixture, checker *testdatabase.Checker, dbClient *cosmosdb.FakeOpenShiftClusterDocumentClient) { + doc := &api.OpenShiftClusterDocument{ + Key: strings.ToLower(key), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: key, + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: resourceGroupID, + }, + IngressProfiles: []api.IngressProfile{ + { + Visibility: api.VisibilityPublic, + }, + }, + ProvisioningState: api.ProvisioningStateCreating, + InfraID: "infra", + }, + }, + } + fixture.AddOpenShiftClusterDocuments(doc) + + doc.Dequeues = 1 + doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = "1.2.3.4" + checker.AddOpenShiftClusterDocuments(doc) + + }, + mocks: func(publicIPAddresses *mock_network.MockPublicIPAddressesClient, dns *mock_dns.MockManager, subnet *mock_subnet.MockManager) { + publicIPAddresses.EXPECT(). + Get(gomock.Any(), "clusterResourceGroup", "infra-default-v4", ""). + Return(mgmtnetwork.PublicIPAddress{ + PublicIPAddressPropertiesFormat: &mgmtnetwork.PublicIPAddressPropertiesFormat{ + IPAddress: to.StringPtr("1.2.3.4"), + }, + }, nil) + dns.EXPECT(). + CreateOrUpdateRouter(gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil) + }, + }, + { + name: "private", + fixtureChecker: func(fixture *testdatabase.Fixture, checker *testdatabase.Checker, dbClient *cosmosdb.FakeOpenShiftClusterDocumentClient) { + doc := &api.OpenShiftClusterDocument{ + Key: strings.ToLower(key), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: key, + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: resourceGroupID, + }, + WorkerProfiles: []api.WorkerProfile{ + { + SubnetID: "subnetid", + }, + }, + IngressProfiles: []api.IngressProfile{ + { + Visibility: api.VisibilityPrivate, + }, + }, + ProvisioningState: api.ProvisioningStateCreating, + InfraID: "infra", + }, + }, + } + fixture.AddOpenShiftClusterDocuments(doc) + + doc.Dequeues = 1 + doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = "1.2.3.4" + checker.AddOpenShiftClusterDocuments(doc) + + }, + mocks: func(publicIPAddresses *mock_network.MockPublicIPAddressesClient, dns *mock_dns.MockManager, subnet *mock_subnet.MockManager) { + subnet.EXPECT(). + GetHighestFreeIP(gomock.Any(), "subnetid"). + Return("1.2.3.4", nil) + dns.EXPECT(). + CreateOrUpdateRouter(gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil) + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + publicIPAddresses := mock_network.NewMockPublicIPAddressesClient(controller) + dns := mock_dns.NewMockManager(controller) + subnet := mock_subnet.NewMockManager(controller) + if tt.mocks != nil { + tt.mocks(publicIPAddresses, dns, subnet) + } + + dbOpenShiftClusters, dbClient := testdatabase.NewFakeOpenShiftClusters() + fixture := testdatabase.NewFixture().WithOpenShiftClusters(dbOpenShiftClusters) + checker := testdatabase.NewChecker() + + if tt.fixtureChecker != nil { + tt.fixtureChecker(fixture, checker, dbClient) + } + + err := fixture.Create() + if err != nil { + t.Fatal(err) + } + + doc, err := dbOpenShiftClusters.Dequeue(ctx) + if err != nil { + t.Fatal(err) + } + + m := &manager{ + doc: doc, + db: dbOpenShiftClusters, + publicIPAddresses: publicIPAddresses, + dns: dns, + subnet: subnet, + } + + err = m.createOrUpdateRouterIPEarly(ctx) + if err != nil && err.Error() != tt.wantErr || + err == nil && tt.wantErr != "" { + t.Error(err) + } + + for _, err = range checker.CheckOpenShiftClusters(dbClient) { + t.Error(err) + } + }) + } +} + +func TestPopulateDatabaseIntIP(t *testing.T) { + ctx := context.Background() + + const ( + key = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName1" + resourceGroupID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterResourceGroup" + ) + + for _, tt := range []struct { + name string + fixtureChecker func(*testdatabase.Fixture, *testdatabase.Checker, *cosmosdb.FakeOpenShiftClusterDocumentClient) + mocks func(*mock_network.MockLoadBalancersClient) + wantErr string + }{ + { + name: "v1", + fixtureChecker: func(fixture *testdatabase.Fixture, checker *testdatabase.Checker, dbClient *cosmosdb.FakeOpenShiftClusterDocumentClient) { + doc := &api.OpenShiftClusterDocument{ + Key: strings.ToLower(key), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: key, + Properties: api.OpenShiftClusterProperties{ + ArchitectureVersion: api.ArchitectureVersionV1, + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: resourceGroupID, + }, + ProvisioningState: api.ProvisioningStateCreating, + InfraID: "infra", + }, + }, + } + fixture.AddOpenShiftClusterDocuments(doc) + + doc.Dequeues = 1 + doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = "10.0.0.1" + checker.AddOpenShiftClusterDocuments(doc) + }, + mocks: func(loadBalancers *mock_network.MockLoadBalancersClient) { + loadBalancers.EXPECT(). + Get(gomock.Any(), "clusterResourceGroup", "infra-internal-lb", ""). + Return(mgmtnetwork.LoadBalancer{ + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PrivateIPAddress: to.StringPtr("10.0.0.1"), + }, + }, + }, + }, + }, nil) + }, + }, + { + name: "v2", + fixtureChecker: func(fixture *testdatabase.Fixture, checker *testdatabase.Checker, dbClient *cosmosdb.FakeOpenShiftClusterDocumentClient) { + doc := &api.OpenShiftClusterDocument{ + Key: strings.ToLower(key), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: key, + Properties: api.OpenShiftClusterProperties{ + ArchitectureVersion: api.ArchitectureVersionV2, + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: resourceGroupID, + }, + ProvisioningState: api.ProvisioningStateCreating, + InfraID: "infra", + }, + }, + } + fixture.AddOpenShiftClusterDocuments(doc) + + doc.Dequeues = 1 + doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = "10.0.0.1" + checker.AddOpenShiftClusterDocuments(doc) + }, + mocks: func(loadBalancers *mock_network.MockLoadBalancersClient) { + loadBalancers.EXPECT(). + Get(gomock.Any(), "clusterResourceGroup", "infra-internal", ""). + Return(mgmtnetwork.LoadBalancer{ + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PrivateIPAddress: to.StringPtr("10.0.0.1"), + }, + }, + }, + }, + }, nil) + }, + }, + { + name: "noop", + fixtureChecker: func(fixture *testdatabase.Fixture, checker *testdatabase.Checker, dbClient *cosmosdb.FakeOpenShiftClusterDocumentClient) { + doc := &api.OpenShiftClusterDocument{ + Key: strings.ToLower(key), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: key, + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: resourceGroupID, + }, + APIServerProfile: api.APIServerProfile{ + IntIP: "10.0.0.1", + }, + ProvisioningState: api.ProvisioningStateCreating, + InfraID: "infra", + }, + }, + } + fixture.AddOpenShiftClusterDocuments(doc) + + doc.Dequeues = 1 + checker.AddOpenShiftClusterDocuments(doc) + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + loadBalancers := mock_network.NewMockLoadBalancersClient(controller) + if tt.mocks != nil { + tt.mocks(loadBalancers) + } + + dbOpenShiftClusters, dbClient := testdatabase.NewFakeOpenShiftClusters() + fixture := testdatabase.NewFixture().WithOpenShiftClusters(dbOpenShiftClusters) + checker := testdatabase.NewChecker() + + if tt.fixtureChecker != nil { + tt.fixtureChecker(fixture, checker, dbClient) + } + + err := fixture.Create() + if err != nil { + t.Fatal(err) + } + + doc, err := dbOpenShiftClusters.Dequeue(ctx) + if err != nil { + t.Fatal(err) + } + + m := &manager{ + doc: doc, + db: dbOpenShiftClusters, + loadBalancers: loadBalancers, + } + + err = m.populateDatabaseIntIP(ctx) + if err != nil && err.Error() != tt.wantErr || + err == nil && tt.wantErr != "" { + t.Error(err) + } + + for _, err = range checker.CheckOpenShiftClusters(dbClient) { + t.Error(err) + } + }) + } +} + +func TestUpdateAPIIPEarly(t *testing.T) { + ctx := context.Background() + + const ( + key = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName1" + resourceGroupID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterResourceGroup" + ) + + for _, tt := range []struct { + name string + fixtureChecker func(*testdatabase.Fixture, *testdatabase.Checker, *cosmosdb.FakeOpenShiftClusterDocumentClient) + mocks func(*mock_network.MockLoadBalancersClient, *mock_network.MockPublicIPAddressesClient, *mock_dns.MockManager) + wantErr string + }{ + { + name: "public", + fixtureChecker: func(fixture *testdatabase.Fixture, checker *testdatabase.Checker, dbClient *cosmosdb.FakeOpenShiftClusterDocumentClient) { + doc := &api.OpenShiftClusterDocument{ + Key: strings.ToLower(key), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: key, + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: resourceGroupID, + }, + APIServerProfile: api.APIServerProfile{ Visibility: api.VisibilityPublic, }, + ProvisioningState: api.ProvisioningStateCreating, + InfraID: "infra", }, - ProvisioningState: api.ProvisioningStateCreating, }, - }, - }) + } + fixture.AddOpenShiftClusterDocuments(doc) + + doc.Dequeues = 1 + doc.OpenShiftCluster.Properties.APIServerProfile.IP = "1.2.3.4" + doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = "10.0.0.1" + checker.AddOpenShiftClusterDocuments(doc) + }, + mocks: func(loadBalancers *mock_network.MockLoadBalancersClient, publicIPAddresses *mock_network.MockPublicIPAddressesClient, dns *mock_dns.MockManager) { + loadBalancers.EXPECT(). + Get(gomock.Any(), "clusterResourceGroup", "infra-internal", ""). + Return(mgmtnetwork.LoadBalancer{ + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PrivateIPAddress: to.StringPtr("10.0.0.1"), + }, + }, + }, + }, + }, nil) + publicIPAddresses.EXPECT(). + Get(gomock.Any(), "clusterResourceGroup", "infra-pip-v4", ""). + Return(mgmtnetwork.PublicIPAddress{ + PublicIPAddressPropertiesFormat: &mgmtnetwork.PublicIPAddressPropertiesFormat{ + IPAddress: to.StringPtr("1.2.3.4"), + }, + }, nil) + dns.EXPECT(). + Update(gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil) + }, + }, + { + name: "private", + fixtureChecker: func(fixture *testdatabase.Fixture, checker *testdatabase.Checker, dbClient *cosmosdb.FakeOpenShiftClusterDocumentClient) { + doc := &api.OpenShiftClusterDocument{ + Key: strings.ToLower(key), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: key, + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: resourceGroupID, + }, + APIServerProfile: api.APIServerProfile{ + Visibility: api.VisibilityPrivate, + }, + ProvisioningState: api.ProvisioningStateCreating, + InfraID: "infra", + }, + }, + } + fixture.AddOpenShiftClusterDocuments(doc) + + doc.Dequeues = 1 + doc.OpenShiftCluster.Properties.APIServerProfile.IP = "10.0.0.1" + doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = "10.0.0.1" + checker.AddOpenShiftClusterDocuments(doc) + }, + mocks: func(loadBalancers *mock_network.MockLoadBalancersClient, publicIPAddresses *mock_network.MockPublicIPAddressesClient, dns *mock_dns.MockManager) { + loadBalancers.EXPECT(). + Get(gomock.Any(), "clusterResourceGroup", "infra-internal", ""). + Return(mgmtnetwork.LoadBalancer{ + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PrivateIPAddress: to.StringPtr("10.0.0.1"), + }, + }, + }, + }, + }, nil) + dns.EXPECT(). + Update(gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil) + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + loadBalancers := mock_network.NewMockLoadBalancersClient(controller) + publicIPAddresses := mock_network.NewMockPublicIPAddressesClient(controller) + dns := mock_dns.NewMockManager(controller) + if tt.mocks != nil { + tt.mocks(loadBalancers, publicIPAddresses, dns) + } + + dbOpenShiftClusters, dbClient := testdatabase.NewFakeOpenShiftClusters() + fixture := testdatabase.NewFixture().WithOpenShiftClusters(dbOpenShiftClusters) + checker := testdatabase.NewChecker() + + if tt.fixtureChecker != nil { + tt.fixtureChecker(fixture, checker, dbClient) + } + err := fixture.Create() if err != nil { t.Fatal(err) } - clusterdoc, err := openShiftClustersDatabase.Dequeue(ctx) + doc, err := dbOpenShiftClusters.Dequeue(ctx) if err != nil { t.Fatal(err) } m := &manager{ - kubernetescli: tt.kubernetescli, - dns: dm, - db: openShiftClustersDatabase, - doc: clusterdoc, + doc: doc, + db: dbOpenShiftClusters, + publicIPAddresses: publicIPAddresses, + loadBalancers: loadBalancers, + dns: dns, } - err = m.createOrUpdateRouterIP(ctx) + err = m.updateAPIIPEarly(ctx) if err != nil && err.Error() != tt.wantErr || err == nil && tt.wantErr != "" { t.Error(err) } + + for _, err = range checker.CheckOpenShiftClusters(dbClient) { + t.Error(err) + } }) } } diff --git a/pkg/cluster/kubeconfig.go b/pkg/cluster/kubeconfig.go index 0d74ee4c4db..c49af532ab1 100644 --- a/pkg/cluster/kubeconfig.go +++ b/pkg/cluster/kubeconfig.go @@ -13,25 +13,27 @@ import ( "github.com/openshift/installer/pkg/asset/kubeconfig" "github.com/openshift/installer/pkg/asset/tls" clientcmd "k8s.io/client-go/tools/clientcmd/api/v1" + + "github.com/Azure/ARO-RP/pkg/cluster/graph" ) // generateAROServiceKubeconfig generates additional admin credentials and a // kubeconfig for the ARO service, based on the admin kubeconfig found in the // graph. -func (m *manager) generateAROServiceKubeconfig(pg persistedGraph) (*kubeconfig.AdminInternalClient, error) { +func (m *manager) generateAROServiceKubeconfig(pg graph.PersistedGraph) (*kubeconfig.AdminInternalClient, error) { return generateKubeconfig(pg, "system:aro-service", []string{"system:masters"}) } // generateAROSREKubeconfig generates additional admin credentials and a // kubeconfig for ARO SREs, based on the admin kubeconfig found in the graph. -func (m *manager) generateAROSREKubeconfig(pg persistedGraph) (*kubeconfig.AdminInternalClient, error) { +func (m *manager) generateAROSREKubeconfig(pg graph.PersistedGraph) (*kubeconfig.AdminInternalClient, error) { return generateKubeconfig(pg, "system:aro-sre", nil) } -func generateKubeconfig(pg persistedGraph, commonName string, organization []string) (*kubeconfig.AdminInternalClient, error) { +func generateKubeconfig(pg graph.PersistedGraph, commonName string, organization []string) (*kubeconfig.AdminInternalClient, error) { var ca *tls.AdminKubeConfigSignerCertKey var adminInternalClient *kubeconfig.AdminInternalClient - err := pg.get(&ca, &adminInternalClient) + err := pg.Get(&ca, &adminInternalClient) if err != nil { return nil, err } diff --git a/pkg/cluster/kubeconfig_test.go b/pkg/cluster/kubeconfig_test.go index 8aa63052c77..7b7b03f5d84 100644 --- a/pkg/cluster/kubeconfig_test.go +++ b/pkg/cluster/kubeconfig_test.go @@ -14,6 +14,7 @@ import ( "github.com/openshift/installer/pkg/asset/tls" clientcmd "k8s.io/client-go/tools/clientcmd/api/v1" + "github.com/Azure/ARO-RP/pkg/cluster/graph" utilpem "github.com/Azure/ARO-RP/pkg/util/pem" utiltls "github.com/Azure/ARO-RP/pkg/util/tls" ) @@ -62,9 +63,9 @@ func TestGenerateAROServiceKubeconfig(t *testing.T) { CurrentContext: serviceName, } - pg := persistedGraph{} + pg := graph.PersistedGraph{} - err = pg.set(ca, adminInternalClient) + err = pg.Set(ca, adminInternalClient) if err != nil { t.Fatal(err) } diff --git a/pkg/cluster/persistedgraph.go b/pkg/cluster/persistedgraph.go deleted file mode 100644 index 25112b54140..00000000000 --- a/pkg/cluster/persistedgraph.go +++ /dev/null @@ -1,88 +0,0 @@ -package cluster - -// Copyright (c) Microsoft Corporation. -// Licensed under the Apache License 2.0. - -import ( - "bytes" - "context" - "encoding/json" - "io/ioutil" - "reflect" - - mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" -) - -// persistedGraph is a graph read from the cluster storage account. -// Unfortunately as the object schema changes over time, there are no guarantees -// that we can easily parse the objects in the graph, so we leave them as json -// RawMessages. You can expect get() to work in the context of cluster creation, -// but not necessarily subsequently. - -type persistedGraph map[string]json.RawMessage - -func (pg persistedGraph) get(is ...interface{}) error { - for _, i := range is { - d := json.NewDecoder(bytes.NewReader(pg[reflect.TypeOf(i).Elem().String()])) - d.DisallowUnknownFields() - - err := d.Decode(i) - if err != nil { - return err - } - } - - return nil -} - -// set is currently only used in unit test context. If you want to use this in -// production, you will want to be very sure that you are not losing state that -// you may need later -func (pg persistedGraph) set(is ...interface{}) (err error) { - for _, i := range is { - pg[reflect.TypeOf(i).String()], err = json.Marshal(i) - if err != nil { - return err - } - } - - return nil -} - -func (m *manager) loadPersistedGraph(ctx context.Context) (persistedGraph, error) { - m.log.Print("load persisted graph") - - blobService, err := m.getBlobService(ctx, mgmtstorage.Permissions("r"), mgmtstorage.SignedResourceTypesO) - if err != nil { - return nil, err - } - - aro := blobService.GetContainerReference("aro") - cluster := aro.GetBlobReference("graph") - rc, err := cluster.Get(nil) - if err != nil { - return nil, err - } - defer rc.Close() - - b, err := ioutil.ReadAll(rc) - if err != nil { - return nil, err - } - - b, err = m.aead.Open(b) - if err != nil { - return nil, err - } - - var pg persistedGraph - err = json.Unmarshal(b, &pg) - if err != nil { - return nil, err - } - - return pg, nil -} - -// savePersistedGraph could be implemented and used with care if needed, but -// currently we don't need it (and it's better that way) diff --git a/pkg/cluster/removebootstrap.go b/pkg/cluster/removebootstrap.go index bf65ce5d94c..aafaa975d3a 100644 --- a/pkg/cluster/removebootstrap.go +++ b/pkg/cluster/removebootstrap.go @@ -35,7 +35,10 @@ func (m *manager) removeBootstrap(ctx context.Context) error { func (m *manager) removeBootstrapIgnition(ctx context.Context) error { m.log.Print("remove ignition config") - blobService, err := m.getBlobService(ctx, mgmtstorage.Permissions("d"), mgmtstorage.SignedResourceTypesC) + resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') + account := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix + + blobService, err := m.storage.BlobService(ctx, resourceGroup, account, mgmtstorage.Permissions("d"), mgmtstorage.SignedResourceTypesC) if err != nil { return err } diff --git a/pkg/cluster/removeprivatednszone.go b/pkg/cluster/removeprivatednszone.go new file mode 100644 index 00000000000..5b7a7ba14a4 --- /dev/null +++ b/pkg/cluster/removeprivatednszone.go @@ -0,0 +1,76 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + "github.com/Azure/go-autorest/autorest/azure" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/Azure/ARO-RP/pkg/util/ready" + "github.com/Azure/ARO-RP/pkg/util/stringutils" +) + +func (m *manager) removePrivateDNSZone(ctx context.Context) error { + resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') + + zones, err := m.privateZones.ListByResourceGroup(ctx, resourceGroup, nil) + if err != nil { + m.log.Print(err) + return nil + } + + if len(zones) == 0 { + return nil + } + + mcps, err := m.mcocli.MachineconfigurationV1().MachineConfigPools().List(ctx, metav1.ListOptions{}) + if err != nil { + m.log.Print(err) + return nil + } + + for _, mcp := range mcps.Items { + var found bool + for _, source := range mcp.Status.Configuration.Source { + if source.Name == "99-"+mcp.Name+"-aro-dns" { + found = true + break + } + } + + if !found { + m.log.Printf("ARO DNS config not found in MCP %s", mcp.Name) + return nil + } + + if !ready.MachineConfigPoolIsReady(&mcp) { + m.log.Printf("MCP %s not ready", mcp.Name) + return nil + } + } + + for _, zone := range zones { + err = m.deletePrivateDNSVirtualNetworkLinks(ctx, *zone.ID) + if err != nil { + m.log.Print(err) + return nil + } + + r, err := azure.ParseResourceID(*zone.ID) + if err != nil { + m.log.Print(err) + return nil + } + + err = m.privateZones.DeleteAndWait(ctx, resourceGroup, r.ResourceName, "") + if err != nil { + m.log.Print(err) + return nil + } + } + + return nil +} diff --git a/pkg/cluster/removeprivatednszone_test.go b/pkg/cluster/removeprivatednszone_test.go new file mode 100644 index 00000000000..78e61ddab00 --- /dev/null +++ b/pkg/cluster/removeprivatednszone_test.go @@ -0,0 +1,189 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" + + mgmtprivatedns "github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns" + "github.com/Azure/go-autorest/autorest/to" + "github.com/golang/mock/gomock" + mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned" + fakemcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned/fake" + "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/Azure/ARO-RP/pkg/api" + mock_privatedns "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/privatedns" +) + +func TestRemovePrivateDNSZone(t *testing.T) { + ctx := context.Background() + const resourceGroupID = "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup" + + for _, tt := range []struct { + name string + doc *api.OpenShiftClusterDocument + mocks func(*mock_privatedns.MockPrivateZonesClient, *mock_privatedns.MockVirtualNetworkLinksClient) + mcocli mcoclient.Interface + }{ + { + name: "no private zones", + doc: &api.OpenShiftClusterDocument{ + OpenShiftCluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: resourceGroupID, + }, + }, + }, + }, + mocks: func(privateZones *mock_privatedns.MockPrivateZonesClient, virtualNetworkLinks *mock_privatedns.MockVirtualNetworkLinksClient) { + privateZones.EXPECT(). + ListByResourceGroup(ctx, "testGroup", nil). + Return(nil, nil) + }, + }, + { + name: "has private zone, dnsmasq config not yet reconciled", + doc: &api.OpenShiftClusterDocument{ + OpenShiftCluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: resourceGroupID, + }, + }, + }, + }, + mocks: func(privateZones *mock_privatedns.MockPrivateZonesClient, virtualNetworkLinks *mock_privatedns.MockVirtualNetworkLinksClient) { + privateZones.EXPECT(). + ListByResourceGroup(ctx, "testGroup", nil). + Return([]mgmtprivatedns.PrivateZone{ + { + ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateZones/zone1"), + }, + }, nil) + }, + mcocli: fakemcoclient.NewSimpleClientset( + &mcv1.MachineConfigPool{}, + ), + }, + { + name: "has private zone, pool not yet ready", + doc: &api.OpenShiftClusterDocument{ + OpenShiftCluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: resourceGroupID, + }, + }, + }, + }, + mocks: func(privateZones *mock_privatedns.MockPrivateZonesClient, virtualNetworkLinks *mock_privatedns.MockVirtualNetworkLinksClient) { + privateZones.EXPECT(). + ListByResourceGroup(ctx, "testGroup", nil). + Return([]mgmtprivatedns.PrivateZone{ + { + ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateZones/zone1"), + }, + }, nil) + }, + mcocli: fakemcoclient.NewSimpleClientset( + &mcv1.MachineConfigPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master", + }, + Status: mcv1.MachineConfigPoolStatus{ + Configuration: mcv1.MachineConfigPoolStatusConfiguration{ + Source: []v1.ObjectReference{ + { + Name: "99-master-aro-dns", + }, + }, + }, + MachineCount: 1, + }, + }, + ), + }, + { + name: "has private zone, dnsmasq rolled out", + doc: &api.OpenShiftClusterDocument{ + OpenShiftCluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: resourceGroupID, + }, + }, + }, + }, + mocks: func(privateZones *mock_privatedns.MockPrivateZonesClient, virtualNetworkLinks *mock_privatedns.MockVirtualNetworkLinksClient) { + privateZones.EXPECT(). + ListByResourceGroup(ctx, "testGroup", nil). + Return([]mgmtprivatedns.PrivateZone{ + { + ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateZones/zone1"), + }, + }, nil) + + virtualNetworkLinks.EXPECT(). + List(ctx, "testGroup", "zone1", nil). + Return([]mgmtprivatedns.VirtualNetworkLink{ + { + Name: to.StringPtr("link1"), + }, + }, nil) + + virtualNetworkLinks.EXPECT(). + DeleteAndWait(ctx, "testGroup", "zone1", "link1", ""). + Return(nil) + + privateZones.EXPECT(). + DeleteAndWait(ctx, "testGroup", "zone1", ""). + Return(nil) + }, + mcocli: fakemcoclient.NewSimpleClientset( + &mcv1.MachineConfigPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master", + }, + Status: mcv1.MachineConfigPoolStatus{ + Configuration: mcv1.MachineConfigPoolStatusConfiguration{ + Source: []v1.ObjectReference{ + { + Name: "99-master-aro-dns", + }, + }, + }, + }, + }, + ), + }, + } { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + privateZones := mock_privatedns.NewMockPrivateZonesClient(controller) + virtualNetworkLinks := mock_privatedns.NewMockVirtualNetworkLinksClient(controller) + tt.mocks(privateZones, virtualNetworkLinks) + + m := &manager{ + log: logrus.NewEntry(logrus.StandardLogger()), + doc: tt.doc, + privateZones: privateZones, + virtualNetworkLinks: virtualNetworkLinks, + mcocli: tt.mcocli, + } + + err := m.removePrivateDNSZone(ctx) + if err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/pkg/cluster/samples.go b/pkg/cluster/samples.go index 0a92f2cb8a0..cd318da7bf3 100644 --- a/pkg/cluster/samples.go +++ b/pkg/cluster/samples.go @@ -61,6 +61,14 @@ func (m *manager) disableOperatorHubSources(ctx context.Context) error { Name: "certified-operators", Disabled: true, }, + { + Name: "community-operators", + Disabled: true, + }, + { + Name: "redhat-marketplace", + Disabled: true, + }, { Name: "redhat-operators", Disabled: true, @@ -68,7 +76,8 @@ func (m *manager) disableOperatorHubSources(ctx context.Context) error { } for _, s := range c.Spec.Sources { switch s.Name { - case "certified-operators", "redhat-operators": + case "certified-operators", "community-operators", + "redhat-marketplace", "redhat-operators": default: sources = append(sources, s) } diff --git a/pkg/frontend/admin_openshiftcluster_kubernetesobjects.go b/pkg/frontend/admin_openshiftcluster_kubernetesobjects.go index 006181760ac..abad145c8fd 100644 --- a/pkg/frontend/admin_openshiftcluster_kubernetesobjects.go +++ b/pkg/frontend/admin_openshiftcluster_kubernetesobjects.go @@ -49,12 +49,7 @@ func (f *frontend) _getAdminKubernetesObjects(ctx context.Context, r *http.Reque return nil, err } - subscriptionDoc, err := f.getSubscriptionDocument(ctx, doc.Key) - if err != nil { - return nil, err - } - - k, err := f.kubeActionsFactory(log, f.env, doc.OpenShiftCluster, subscriptionDoc) + k, err := f.kubeActionsFactory(log, f.env, doc.OpenShiftCluster) if err != nil { return nil, err } @@ -95,12 +90,7 @@ func (f *frontend) _deleteAdminKubernetesObjects(ctx context.Context, r *http.Re return err } - subscriptionDoc, err := f.getSubscriptionDocument(ctx, doc.Key) - if err != nil { - return err - } - - k, err := f.kubeActionsFactory(log, f.env, doc.OpenShiftCluster, subscriptionDoc) + k, err := f.kubeActionsFactory(log, f.env, doc.OpenShiftCluster) if err != nil { return err } @@ -149,12 +139,7 @@ func (f *frontend) _postAdminKubernetesObjects(ctx context.Context, r *http.Requ return err } - subscriptionDoc, err := f.getSubscriptionDocument(ctx, doc.Key) - if err != nil { - return err - } - - k, err := f.kubeActionsFactory(log, f.env, doc.OpenShiftCluster, subscriptionDoc) + k, err := f.kubeActionsFactory(log, f.env, doc.OpenShiftCluster) if err != nil { return err } diff --git a/pkg/frontend/admin_openshiftcluster_kubernetesobjects_test.go b/pkg/frontend/admin_openshiftcluster_kubernetesobjects_test.go index 9b9e3419de9..99092920c25 100644 --- a/pkg/frontend/admin_openshiftcluster_kubernetesobjects_test.go +++ b/pkg/frontend/admin_openshiftcluster_kubernetesobjects_test.go @@ -174,7 +174,7 @@ func TestAdminKubernetesObjectsGetAndDelete(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, api.APIs, &noop.Noop{}, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.KubeActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, api.APIs, &noop.Noop{}, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { return k, nil }, nil, nil) if err != nil { @@ -368,7 +368,7 @@ func TestAdminPostKubernetesObjects(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, api.APIs, &noop.Noop{}, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.KubeActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, api.APIs, &noop.Noop{}, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { return k, nil }, nil, nil) if err != nil { diff --git a/pkg/frontend/admin_openshiftcluster_upgrade.go b/pkg/frontend/admin_openshiftcluster_upgrade.go index ed995c2f3f1..d43558d0459 100644 --- a/pkg/frontend/admin_openshiftcluster_upgrade.go +++ b/pkg/frontend/admin_openshiftcluster_upgrade.go @@ -42,12 +42,7 @@ func (f *frontend) _postAdminOpenShiftUpgrade(ctx context.Context, r *http.Reque return err } - subscriptionDoc, err := f.getSubscriptionDocument(ctx, doc.Key) - if err != nil { - return err - } - - k, err := f.kubeActionsFactory(log, f.env, doc.OpenShiftCluster, subscriptionDoc) + k, err := f.kubeActionsFactory(log, f.env, doc.OpenShiftCluster) if err != nil { return err } diff --git a/pkg/frontend/adminactions/kubeactions.go b/pkg/frontend/adminactions/kubeactions.go index cfe9ca893bf..ffcd3bc0a86 100644 --- a/pkg/frontend/adminactions/kubeactions.go +++ b/pkg/frontend/adminactions/kubeactions.go @@ -12,7 +12,6 @@ import ( "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/env" - "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network" "github.com/Azure/ARO-RP/pkg/util/dynamichelper" "github.com/Azure/ARO-RP/pkg/util/restconfig" ) @@ -31,14 +30,10 @@ type kubeActions struct { oc *api.OpenShiftCluster dh dynamichelper.Interface configcli configclient.Interface - - virtualNetworks network.VirtualNetworksClient } // NewKubeActions returns a kubeActions -func NewKubeActions(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, - subscriptionDoc *api.SubscriptionDocument) (KubeActions, error) { - +func NewKubeActions(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster) (KubeActions, error) { restConfig, err := restconfig.RestConfig(env, oc) if err != nil { return nil, err @@ -54,19 +49,11 @@ func NewKubeActions(log *logrus.Entry, env env.Interface, oc *api.OpenShiftClust return nil, err } - fpAuth, err := env.FPAuthorizer(subscriptionDoc.Subscription.Properties.TenantID, - env.Environment().ResourceManagerEndpoint) - if err != nil { - return nil, err - } - return &kubeActions{ log: log, oc: oc, dh: dh, configcli: configcli, - - virtualNetworks: network.NewVirtualNetworksClient(env.Environment(), subscriptionDoc.ID, fpAuth), }, nil } diff --git a/pkg/frontend/adminactions/upgrade.go b/pkg/frontend/adminactions/upgrade.go index e71b53a88f6..b7f6f7a9637 100644 --- a/pkg/frontend/adminactions/upgrade.go +++ b/pkg/frontend/adminactions/upgrade.go @@ -7,7 +7,6 @@ import ( "context" "net/http" - "github.com/Azure/go-autorest/autorest/azure" configv1 "github.com/openshift/api/config/v1" configclient "github.com/openshift/client-go/config/clientset/versioned" "github.com/sirupsen/logrus" @@ -15,18 +14,11 @@ import ( "k8s.io/client-go/util/retry" "github.com/Azure/ARO-RP/pkg/api" - "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network" "github.com/Azure/ARO-RP/pkg/util/status" - "github.com/Azure/ARO-RP/pkg/util/subnet" "github.com/Azure/ARO-RP/pkg/util/version" ) func (k *kubeActions) Upgrade(ctx context.Context, upgradeY bool) error { - err := preUpgradeChecks(ctx, k.oc, k.virtualNetworks) - if err != nil { - return err - } - return upgrade(ctx, k.log, k.configcli, version.Streams, upgradeY) } @@ -65,34 +57,3 @@ func upgrade(ctx context.Context, log *logrus.Entry, configcli configclient.Inte return err }) } - -func preUpgradeChecks(ctx context.Context, oc *api.OpenShiftCluster, virtualNetworks network.VirtualNetworksClient) error { - return checkCustomDNS(ctx, oc, virtualNetworks) -} - -// checkCustomDNS checks if customer has custom DNS configured on VNET. -// This would cause nodes to rotate and render cluster inoperable -func checkCustomDNS(ctx context.Context, oc *api.OpenShiftCluster, vnet network.VirtualNetworksClient) error { - vnetID, _, err := subnet.Split(oc.Properties.MasterProfile.SubnetID) - if err != nil { - return err - } - - r, err := azure.ParseResourceID(vnetID) - if err != nil { - return err - } - - v, err := vnet.Get(ctx, r.ResourceGroup, r.ResourceName, "") - if err != nil { - return err - } - - if v.VirtualNetworkPropertiesFormat.DhcpOptions != nil && - v.VirtualNetworkPropertiesFormat.DhcpOptions.DNSServers != nil && - len(*v.VirtualNetworkPropertiesFormat.DhcpOptions.DNSServers) > 0 { - return api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "Not upgrading: custom DNS is set.") - } - - return nil -} diff --git a/pkg/frontend/adminactions/upgrade_test.go b/pkg/frontend/adminactions/upgrade_test.go index cca3c8a3044..15fa582507a 100644 --- a/pkg/frontend/adminactions/upgrade_test.go +++ b/pkg/frontend/adminactions/upgrade_test.go @@ -5,11 +5,8 @@ package adminactions import ( "context" - "fmt" "testing" - mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network" - "github.com/golang/mock/gomock" configv1 "github.com/openshift/api/config/v1" "github.com/openshift/client-go/config/clientset/versioned/fake" "github.com/sirupsen/logrus" @@ -17,8 +14,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" ktesting "k8s.io/client-go/testing" - "github.com/Azure/ARO-RP/pkg/api" - mock_network "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/network" "github.com/Azure/ARO-RP/pkg/util/version" ) @@ -204,72 +199,3 @@ func TestUpgradeCluster(t *testing.T) { }) } } - -func TestCheckCustomDNS(t *testing.T) { - ctx := context.Background() - subscriptionID := "af848f0a-dbe3-449f-9ccd-6f23ac6ef9f1" - - tests := []struct { - name string - mocks func(*mock_network.MockVirtualNetworksClient) - wantErr string - }{ - { - name: "default dns", - mocks: func(vnetc *mock_network.MockVirtualNetworksClient) { - vnetc.EXPECT().Get(gomock.Any(), "test-cluster", "test-vnet", "").Return( - mgmtnetwork.VirtualNetwork{ - VirtualNetworkPropertiesFormat: &mgmtnetwork.VirtualNetworkPropertiesFormat{ - DhcpOptions: &mgmtnetwork.DhcpOptions{ - DNSServers: &[]string{}, - }, - }, - }, nil) - }, - }, - { - name: "custom dns", - mocks: func(vnetc *mock_network.MockVirtualNetworksClient) { - vnetc.EXPECT().Get(gomock.Any(), "test-cluster", "test-vnet", "").Return( - mgmtnetwork.VirtualNetwork{ - VirtualNetworkPropertiesFormat: &mgmtnetwork.VirtualNetworkPropertiesFormat{ - DhcpOptions: &mgmtnetwork.DhcpOptions{ - DNSServers: &[]string{"1.1.1.1"}, - }, - }, - }, nil) - }, - wantErr: "500: InternalServerError: : Not upgrading: custom DNS is set.", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - controller := gomock.NewController(t) - defer controller.Finish() - - virtualNetworks := mock_network.NewMockVirtualNetworksClient(controller) - if tt.mocks != nil { - tt.mocks(virtualNetworks) - } - - k := &kubeActions{ - log: logrus.NewEntry(logrus.StandardLogger()), - virtualNetworks: virtualNetworks, - } - - oc := &api.OpenShiftCluster{ - Properties: api.OpenShiftClusterProperties{ - MasterProfile: api.MasterProfile{ - SubnetID: fmt.Sprintf("/subscriptions/%s/resourceGroups/test-cluster/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/master", subscriptionID), - }, - }, - } - - err := checkCustomDNS(ctx, oc, k.virtualNetworks) - if err != nil && err.Error() != tt.wantErr || - err == nil && tt.wantErr != "" { - t.Error(err) - } - }) - } -} diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index 40f4912ad6f..91dbd3391ba 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -38,7 +38,7 @@ func (err statusCodeError) Error() string { return fmt.Sprintf("%d", err) } -type kubeActionsFactory func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.KubeActions, error) +type kubeActionsFactory func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) type azureActionsFactory func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) diff --git a/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go b/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go index 1be22e4b221..389f1213549 100644 --- a/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go +++ b/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go @@ -34,12 +34,15 @@ type InternetCheckerSpec struct { type ClusterSpec struct { // ResourceID is the Azure resourceId of the cluster ResourceID string `json:"resourceId,omitempty"` + Domain string `json:"domain,omitempty"` ACRDomain string `json:"acrDomain,omitempty"` AZEnvironment string `json:"azEnvironment,omitempty"` Location string `json:"location,omitempty"` GenevaLogging GenevaLoggingSpec `json:"genevaLogging,omitempty"` InternetChecker InternetCheckerSpec `json:"internetChecker,omitempty"` VnetID string `json:"vnetId,omitempty"` + APIIntIP string `json:"apiIntIP,omitempty"` + IngressIP string `json:"ingressIP,omitempty"` Features FeaturesSpec `json:"features,omitempty"` } diff --git a/pkg/operator/controllers/const.go b/pkg/operator/controllers/const.go index 830f394b09f..d7a31b7ccea 100644 --- a/pkg/operator/controllers/const.go +++ b/pkg/operator/controllers/const.go @@ -4,12 +4,15 @@ package controllers // Licensed under the Apache License 2.0. const ( - AlertwebhookControllerName = "Alertwebhook" - GenevaLoggingControllerName = "GenevaLogging" - PullSecretControllerName = "PullSecret" - WorkaroundControllerName = "Workaround" - CheckerControllerName = "Checker" - RouteFixControllerName = "RouteFix" - MonitoringControllerName = "Monitoring" - RBACControllerName = "RBAC" + AlertwebhookControllerName = "Alertwebhook" + GenevaLoggingControllerName = "GenevaLogging" + PullSecretControllerName = "PullSecret" + WorkaroundControllerName = "Workaround" + CheckerControllerName = "Checker" + RouteFixControllerName = "RouteFix" + MonitoringControllerName = "Monitoring" + RBACControllerName = "RBAC" + DnsmasqClusterControllerName = "DnsmasqCluster" + DnsmasqMachineConfigControllerName = "DnsmasqMachineConfig" + DnsmasqMachineConfigPoolControllerName = "DnsmasqMachineConfigPool" ) diff --git a/pkg/operator/controllers/dnsmasq/cluster_controller.go b/pkg/operator/controllers/dnsmasq/cluster_controller.go new file mode 100644 index 00000000000..8d1dbb6bc9a --- /dev/null +++ b/pkg/operator/controllers/dnsmasq/cluster_controller.go @@ -0,0 +1,104 @@ +package dnsmasq + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + "github.com/openshift/installer/pkg/aro/dnsmasq" + mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned" + "github.com/Azure/ARO-RP/pkg/operator/controllers" + "github.com/Azure/ARO-RP/pkg/util/dynamichelper" +) + +type ClusterReconciler struct { + log *logrus.Entry + + arocli aroclient.Interface + mcocli mcoclient.Interface + dh dynamichelper.Interface +} + +func NewClusterReconciler(log *logrus.Entry, arocli aroclient.Interface, mcocli mcoclient.Interface, dh dynamichelper.Interface) *ClusterReconciler { + return &ClusterReconciler{ + log: log, + arocli: arocli, + mcocli: mcocli, + dh: dh, + } +} + +// Reconcile watches the ARO object, and if it changes, reconciles all the +// 99-%s-aro-dns machineconfigs +func (r *ClusterReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) { + // TODO(mj): controller-runtime master fixes the need for this (https://github.com/kubernetes-sigs/controller-runtime/blob/master/pkg/reconcile/reconcile.go#L93) but it's not yet released. + ctx := context.Background() + if request.Name != arov1alpha1.SingletonClusterName { + return reconcile.Result{}, nil + } + + mcps, err := r.mcocli.MachineconfigurationV1().MachineConfigPools().List(ctx, metav1.ListOptions{}) + if err != nil { + r.log.Error(err) + return reconcile.Result{}, err + } + + roles := make([]string, 0, len(mcps.Items)) + for _, mcp := range mcps.Items { + roles = append(roles, mcp.Name) + } + + err = reconcileMachineConfigs(ctx, r.arocli, r.dh, roles...) + if err != nil { + r.log.Error(err) + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +// SetupWithManager setup our mananger +func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&arov1alpha1.Cluster{}). + Named(controllers.DnsmasqClusterControllerName). + Complete(r) +} + +func reconcileMachineConfigs(ctx context.Context, arocli aroclient.Interface, dh dynamichelper.Interface, roles ...string) error { + instance, err := arocli.AroV1alpha1().Clusters().Get(ctx, arov1alpha1.SingletonClusterName, metav1.GetOptions{}) + if err != nil { + return err + } + + var resources []runtime.Object + for _, role := range roles { + resource, err := dnsmasq.MachineConfig(instance.Spec.Domain, instance.Spec.APIIntIP, instance.Spec.IngressIP, role) + if err != nil { + return err + } + + resources = append(resources, resource) + } + + err = dynamichelper.SetControllerReferences(resources, instance) + if err != nil { + return err + } + + uns, err := dynamichelper.Prepare(resources) + if err != nil { + return err + } + + return dh.Ensure(ctx, uns...) +} diff --git a/pkg/operator/controllers/dnsmasq/doc.go b/pkg/operator/controllers/dnsmasq/doc.go new file mode 100644 index 00000000000..54dcd6fa241 --- /dev/null +++ b/pkg/operator/controllers/dnsmasq/doc.go @@ -0,0 +1,32 @@ +package dnsmasq + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +// The controllers in this package aim to ensure that MachineConfig objects +// exist and are correctly configured to lay down dnsmasq configuration on +// cluster VMs. The data path is: +// +// * RP sets Domain, APIIntIP, IngressIP fields on the ARO Cluster object at +// admin upgrade time. +// +// * ClusterReconciler controller ensures MachineConfigs exist. +// +// * MachineConfigs cause dnsmasq.service and dnsmasq.conf files to be laid down +// to ensure that DNS resolution of the appropriate names succeeds locally to +// each VM. +// +// In a little more detail, the controllers work as follows: +// +// func (*ClusterReconciler) Reconcile watches the ARO object for changes and +// reconciles all the 99-%s-aro-dns machineconfigs. +// +// func (*MachineConfigReconciler) Reconcile watches the 99-%s-aro-dns +// machineconfigs in case someone tries to change them, and reconciles them. +// +// func (*MachineConfigPoolReconciler) Reconcile watches MachineConfigPool +// objects and reconciles all the 99-%s-aro-dns machineconfigs. This covers +// automatically adding dnsmasq configuration to newly created pools. +// +// Internally, all of the above functions call the same reconciler +// (reconcileMachineConfigs). diff --git a/pkg/operator/controllers/dnsmasq/machineconfig_controller.go b/pkg/operator/controllers/dnsmasq/machineconfig_controller.go new file mode 100644 index 00000000000..815863eba47 --- /dev/null +++ b/pkg/operator/controllers/dnsmasq/machineconfig_controller.go @@ -0,0 +1,78 @@ +package dnsmasq + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "regexp" + + mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned" + "github.com/Azure/ARO-RP/pkg/operator/controllers" + "github.com/Azure/ARO-RP/pkg/util/dynamichelper" +) + +type MachineConfigReconciler struct { + log *logrus.Entry + + arocli aroclient.Interface + mcocli mcoclient.Interface + dh dynamichelper.Interface +} + +var rxARODNS = regexp.MustCompile("^99-(.*)-aro-dns$") + +func NewMachineConfigReconciler(log *logrus.Entry, arocli aroclient.Interface, mcocli mcoclient.Interface, dh dynamichelper.Interface) *MachineConfigReconciler { + return &MachineConfigReconciler{ + log: log, + arocli: arocli, + mcocli: mcocli, + dh: dh, + } +} + +// Reconcile watches ARO DNS MachineConfig objects, and if any changes, +// reconciles it +func (r *MachineConfigReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) { + // TODO(mj): controller-runtime master fixes the need for this (https://github.com/kubernetes-sigs/controller-runtime/blob/master/pkg/reconcile/reconcile.go#L93) but it's not yet released. + ctx := context.Background() + + m := rxARODNS.FindStringSubmatch(request.Name) + if m == nil { + return reconcile.Result{}, nil + } + role := m[1] + + _, err := r.mcocli.MachineconfigurationV1().MachineConfigPools().Get(ctx, role, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + if err != nil { + r.log.Error(err) + return reconcile.Result{}, err + } + + err = reconcileMachineConfigs(ctx, r.arocli, r.dh, role) + if err != nil { + r.log.Error(err) + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +// SetupWithManager setup our mananger +func (r *MachineConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&mcv1.MachineConfig{}). + Named(controllers.DnsmasqMachineConfigControllerName). + Complete(r) +} diff --git a/pkg/operator/controllers/dnsmasq/machineconfigpool_controller.go b/pkg/operator/controllers/dnsmasq/machineconfigpool_controller.go new file mode 100644 index 00000000000..fd396938ce5 --- /dev/null +++ b/pkg/operator/controllers/dnsmasq/machineconfigpool_controller.go @@ -0,0 +1,69 @@ +package dnsmasq + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned" + "github.com/Azure/ARO-RP/pkg/operator/controllers" + "github.com/Azure/ARO-RP/pkg/util/dynamichelper" +) + +type MachineConfigPoolReconciler struct { + log *logrus.Entry + + arocli aroclient.Interface + mcocli mcoclient.Interface + dh dynamichelper.Interface +} + +func NewMachineConfigPoolReconciler(log *logrus.Entry, arocli aroclient.Interface, mcocli mcoclient.Interface, dh dynamichelper.Interface) *MachineConfigPoolReconciler { + return &MachineConfigPoolReconciler{ + log: log, + arocli: arocli, + mcocli: mcocli, + dh: dh, + } +} + +// Reconcile watches MachineConfigPool objects, and if any changes, +// reconciles the associated ARO DNS MachineConfig object +func (r *MachineConfigPoolReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) { + // TODO(mj): controller-runtime master fixes the need for this (https://github.com/kubernetes-sigs/controller-runtime/blob/master/pkg/reconcile/reconcile.go#L93) but it's not yet released. + ctx := context.Background() + + _, err := r.mcocli.MachineconfigurationV1().MachineConfigPools().Get(ctx, request.Name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + if err != nil { + r.log.Error(err) + return reconcile.Result{}, err + } + + err = reconcileMachineConfigs(ctx, r.arocli, r.dh, request.Name) + if err != nil { + r.log.Error(err) + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +// SetupWithManager setup our mananger +func (r *MachineConfigPoolReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&mcv1.MachineConfigPool{}). + Named(controllers.DnsmasqMachineConfigPoolControllerName). + Complete(r) +} diff --git a/pkg/operator/deploy/bindata.go b/pkg/operator/deploy/bindata.go index 6d1289d4dab..3a337de0754 100644 --- a/pkg/operator/deploy/bindata.go +++ b/pkg/operator/deploy/bindata.go @@ -86,7 +86,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _aroOpenshiftIo_clustersYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x58\xcd\x72\xe3\xb8\x11\xbe\xeb\x29\xba\x9c\x83\x0f\xb1\x68\x6b\x26\x33\xb3\xab\x9b\xcb\xb3\x9b\x72\x65\x93\x75\x8d\x5d\x73\x59\xef\xa1\x09\xb6\xc8\x8e\x41\x80\x01\x9a\xf6\x68\x52\x79\xf7\x54\x83\xa4\x44\xc9\xa4\xed\xa4\x6a\x79\x51\xb1\x01\xf4\x7f\x7f\xf8\xc4\xc5\x72\xb9\x5c\x60\xc3\x5f\x29\x44\xf6\x6e\x0d\xd8\x30\x7d\x13\x72\xfa\x16\xb3\x87\x1f\x62\xc6\xfe\xfc\x71\x95\x93\xe0\x6a\xf1\xc0\xae\x58\xc3\x55\x1b\xc5\xd7\x5f\x28\xfa\x36\x18\xfa\x4c\x1b\x76\x2c\xec\xdd\xa2\x26\xc1\x02\x05\xd7\x0b\x00\x74\xce\x0b\xaa\x38\xea\x2b\x80\xf1\x4e\x82\xb7\x96\xc2\xb2\x24\x97\x3d\xb4\x39\xe5\x2d\xdb\x82\x42\xb2\x30\xd8\x7f\xbc\xc8\xde\x67\xab\xe5\x45\xf6\xee\xe2\xdd\xc5\xc5\xc7\xd5\xa7\x77\xab\xd5\xc7\x8b\x0f\xcb\x8f\x1f\x56\x3f\x5e\xbc\xff\xcb\xa7\x4f\xab\x1f\x3e\x2c\x00\x4c\xa0\xa4\xfc\x8e\x6b\x8a\x82\x75\xb3\x06\xd7\x5a\xbb\x00\x70\x58\xd3\x1a\x8c\x6d\xa3\x50\x88\x19\x06\x9f\xf9\x86\x5c\xac\x78\x23\x19\xfb\x45\x6c\xc8\xa8\x47\x65\xf0\x6d\xb3\x86\x67\xeb\x9d\x86\xde\xe9\x3e\xe0\x4e\x59\x92\x58\x8e\xf2\xb7\xb1\xf4\x17\x8e\x92\x56\x1a\xdb\x06\xb4\x7b\xd3\x49\x18\xd9\x95\xad\xc5\xb0\x13\x2f\x00\xa2\xf1\x0d\x8d\xb5\xc6\x36\x0f\x7d\x36\x7b\xbb\x51\x50\xda\xb8\x86\x7f\xff\x67\x01\xf0\x88\x96\x8b\x14\x6d\xb7\xa8\xee\x5e\xde\x5c\x7f\x7d\x7f\x6b\x2a\xaa\xb1\x13\x02\x14\x14\x4d\xe0\x26\xed\x1b\x94\x03\x47\x90\x8a\xa0\xdb\x09\x1b\x1f\xd2\xeb\xe0\x22\x5c\xde\x5c\xf7\xa7\x9b\xe0\x1b\x0a\xc2\x83\x07\xfa\x8c\xfa\x62\x27\x3b\xb2\x73\xaa\x8e\x74\x7b\xa0\xd0\x4e\xa0\xce\x60\x5f\x4f\x2a\x20\x76\xa6\xfd\x06\xa4\xe2\x08\x81\x9a\x40\x91\x5c\xd7\x1b\x23\xb5\xa0\x5b\xd0\x81\xcf\xff\x49\x46\x32\xb8\xa5\xa0\x4a\x20\x56\xbe\xb5\x85\xb6\xcf\x23\x05\x81\x40\xc6\x97\x8e\xbf\xef\x34\x47\x10\x9f\x4c\x5a\x14\xea\x4b\x31\x3c\xec\x84\x82\x43\xab\x29\x6c\xe9\x0c\xd0\x15\x50\xe3\x16\x02\xa9\x0d\x68\xdd\x48\x5b\xda\x12\x33\xf8\xbb\x0f\x04\xec\x36\x7e\x0d\x95\x48\x13\xd7\xe7\xe7\x25\xcb\x30\x09\xc6\xd7\x75\xeb\x58\xb6\xe7\xa9\x9f\x39\x6f\xc5\x87\x78\x5e\xd0\x23\xd9\xf3\xc8\xe5\x12\x83\xa9\x58\xc8\x48\x1b\xe8\x1c\x1b\x5e\x26\xc7\x5d\x1a\x84\xac\x2e\xfe\xb4\x2b\xf4\xe9\xc8\x53\xd9\x6a\x43\x44\x09\xec\xca\x9d\x38\xf5\xde\x6c\xde\xb5\x07\xb5\xba\xd8\x1f\xeb\xfc\xdf\xa7\x57\x45\x9a\x95\x2f\x3f\xdd\xde\xc1\x60\x34\x95\xe0\x30\xe7\x29\xdb\xfb\x63\x71\x9f\x78\x4d\x14\xbb\x0d\x85\xae\x70\x9b\xe0\xeb\xa4\x91\x5c\xd1\x78\x76\xd2\x77\x12\x93\x3b\x4c\x7a\x6c\xf3\x9a\x45\x2b\xfd\xaf\x96\xa2\x68\x7d\x32\xb8\x4a\x78\x00\x39\x41\xdb\x14\x28\x54\x64\x70\xed\xe0\x0a\x6b\xb2\x57\x18\xe9\x0f\x4f\xbb\x66\x38\x2e\x35\xa5\xaf\x27\x7e\x0c\x63\x87\x1b\xbb\x6c\xed\xc4\x03\x94\x4c\x56\xa8\x9f\xc0\xdb\x86\xcc\xc1\x64\x14\x14\x39\x68\xf7\x0a\x0a\x69\xcf\x8f\xd1\x65\x7e\x16\xd3\x3c\x9a\xf0\xd9\xd7\xc8\xee\x50\x3c\x13\x46\x3a\xf1\xfd\x27\xf7\xc8\xc1\xbb\x9a\x9c\xbc\xf9\xd4\x86\x50\xf3\x18\x8f\x0f\x1c\xc4\xf7\x73\xbf\xe9\x20\xc0\xcb\x2f\xbf\x2a\x3c\x05\x14\x1f\x06\x35\x50\xea\x60\x1e\xa9\x9a\x0b\x31\xad\x29\x76\x44\x21\x27\x37\xc1\xd7\x24\x15\xb5\x13\xbb\x06\xff\x73\xef\x2d\xa1\x9b\x8c\xed\xa8\x5e\xfa\x94\xe4\xe8\x11\x7f\xf1\x65\xc9\xae\x3c\xd6\xfa\x92\x57\xc6\xbb\x0d\x97\x13\x78\xb8\x3b\x8c\xa2\x68\xb3\x86\xd3\xdf\x2e\x96\x3f\xfe\xfe\xe7\xac\xfb\x39\x9d\xf5\x7c\x22\xf3\xfa\xd4\xde\xb1\x78\x5d\xfa\xeb\xd5\xed\x0b\xe5\xd3\x87\x5c\x5b\x4f\xc9\x97\xf0\x99\xb1\x74\x3e\x0a\x9b\x78\x13\x7c\x31\xb9\xe7\xee\x18\x2f\x5f\xf5\x6e\x36\xad\x1d\xd2\x92\x5c\x55\x64\x1e\x28\xfc\x2f\x89\x6d\x83\x9d\x2c\x2f\x0b\xd5\x93\x0b\xaf\xe4\x6f\x58\xc6\x10\x70\xfb\x56\xff\xad\x37\xa3\x1b\xf6\x0d\x96\x06\x48\xbd\x2e\x5e\x9c\x92\x81\x25\x5d\x7f\x1e\xae\xe2\xcb\xef\x3a\x13\xfb\xe3\xdd\xcd\x48\x23\x7e\xf0\x26\xfb\x8f\x8e\xe4\xb9\xed\x99\xed\xd3\xe8\xd5\x71\x8c\xd7\xf0\x2b\xed\x3a\x40\x30\x9f\x47\xbd\x22\xfe\x2f\x08\x33\xde\x15\x3c\x62\x85\x73\xc6\x77\xdb\xfa\x3b\x8e\x24\xd9\x19\xc4\xc0\x2e\x0a\x3a\x43\x31\x3b\x52\x33\xd3\x35\x07\xda\x4f\xf6\x7a\xf6\x17\x5f\xc7\x3d\x34\xb2\xd4\x08\x07\x6c\xe4\x34\x76\xb1\x1e\x1b\xd3\x67\xe4\x2a\x06\xd2\x33\x3b\xfe\x0c\x35\x99\x0a\x1d\xc7\x3a\xcd\x87\x2b\xa8\x50\xb2\xa2\x97\x60\xa4\x02\x9e\x2a\x72\x9a\xd0\x09\xa5\x05\x09\xb2\x8d\x3b\x27\xf6\x6e\xa9\x0d\xbd\x49\x11\x9a\xc0\x3e\x30\x3c\x38\xff\xe4\xc0\x07\x78\x4a\x3c\x29\xad\x35\x8d\x3d\xee\xfc\xd4\x06\x1e\xd0\xda\x7d\xee\x92\x7a\x28\xf9\x91\x1c\x28\x9f\xc8\xe0\xde\x8d\xe3\xe9\xa9\x57\x4e\x80\x45\x41\x53\x18\x22\x1e\xe8\x5b\x63\xd9\xb0\xd8\x6d\xc7\xd1\xb6\xa3\xda\x83\x54\x28\x1a\x6c\x88\x89\x7b\x19\x5f\x37\xde\xa5\x6c\x9b\x94\xac\xdc\xb7\x53\x08\x14\x50\xaa\xc4\x3b\xd0\x25\x1a\xc1\xa1\xa3\x33\x3e\xd2\x81\xf6\x94\xcb\xc4\x51\xf4\x46\x4d\x0c\xc5\xeb\xc9\x09\x95\xa3\x1c\xc6\x0c\x7e\x75\x86\xfa\x9e\x2e\xce\x52\x53\xd7\x84\x4e\x8d\xa4\x94\xec\xfb\xc3\xa0\x83\x8e\xb8\x4c\xe8\xd4\xe2\x96\x54\x00\x86\x9c\x25\x60\x60\xbb\x85\x25\xb0\xee\x36\xbe\xa6\x08\x0d\x06\x19\xe6\xfb\xf2\xe6\xba\x23\xa0\x15\x76\x63\x14\xb1\x9e\x52\x9a\xa3\x79\x78\xc2\x50\xc4\x65\xda\xbd\xf1\xa1\x7b\xd3\xdc\xa1\x70\xce\x96\x25\xa5\xda\x50\x70\x7d\x87\x6c\xbb\xb0\x93\xbd\xa9\xd8\x77\x1e\x64\x27\xcf\x6f\xdb\x17\xa0\x19\xc0\x62\x94\xbb\x80\x2e\xf2\xf0\x6f\x6b\x1a\x91\x37\x3e\xd4\x28\x6b\x50\x6e\xb7\x14\x9e\x8c\xec\x55\xdc\xae\x29\x46\x2c\x67\x2c\xbc\x72\x36\x10\xc6\xe9\x7b\x79\x0e\x5a\xbe\xa4\x13\x8a\x2f\x47\xc3\x89\xe0\x1d\x2d\x9f\x7c\x28\xce\xf6\x0c\x75\x52\x31\x1c\xfd\x9d\xd9\x61\x39\x0a\x95\x3e\x6c\xf5\xdd\x60\x1b\x69\xb7\xd0\x86\x40\x4e\x7a\xec\x9d\x82\x13\x7d\xae\x65\xc2\xab\x04\x19\xec\x52\xe5\x59\x35\xb6\xd2\xb4\x72\x06\xb1\x35\x15\x60\x4c\x3e\x5b\x76\x73\x8e\xea\xff\x6d\x23\x16\x4a\x45\xd2\xfe\xa8\xf6\x17\x3b\x88\x6d\x5d\x63\xe0\xef\xa9\xfd\x4d\xe7\x62\x8f\x0e\xc9\xf9\x19\x3f\x5f\x29\xc8\xf3\xeb\xe5\xcd\x47\xd3\xf2\xeb\x95\xdc\xc3\xf8\xdd\xb6\xa1\xe1\x7e\xd5\xc3\xbb\x74\xef\xe6\x38\x85\x7a\x4c\x42\x47\xf6\xd8\xa0\xb5\x5b\x1d\xfd\xa1\xe0\x05\x68\x07\x28\xb0\xc6\xca\x07\x81\xa6\x0a\xe9\x8f\xca\x18\x22\x93\xb1\x39\xad\x3d\x7a\xb2\x2b\x58\xfb\xa1\xbf\x2d\x39\x41\x3e\xdc\x9f\x60\xee\x74\x66\xec\x52\x42\x4b\xf7\x27\xd0\x78\x8b\x81\x65\x9b\xc1\xcf\x7e\x0a\xc0\xf4\xa1\x6f\x58\x37\x96\xce\x80\x8f\xe3\x1b\xac\xc4\xee\x56\x41\x55\xc7\x66\xdb\xf5\x51\xfa\x80\x70\x36\x17\x7c\xf2\x86\x63\xf7\x99\xe1\xfe\x04\x0c\xc6\x94\xcc\x26\xf8\x1c\x73\xbb\x4d\x3b\xd4\xd7\x33\x88\xfe\xd0\xec\xcb\x91\xe7\x3a\x08\xd6\x52\x01\xf7\x27\xd7\xae\x57\x3f\x81\x40\xf0\x5a\x47\x74\x57\x00\x3d\x63\x3b\xca\x61\xbb\x36\x9b\x58\x50\x8d\xcf\xc4\xb3\xfc\x6f\x9e\x34\x0e\x7f\x67\x66\x98\xff\xdb\x19\xd7\x91\x68\xff\xc5\x6b\x85\xb6\xa9\x70\xb5\x97\xa5\xa9\x59\xf6\x5f\xb2\x46\xcb\x00\x1d\xf1\x5a\x83\xf6\x4c\xff\xa1\xc8\x07\x85\xcc\x4e\xb2\x9f\x39\x34\x86\x1a\xa1\xe2\x1f\xc7\xdf\xb2\x4e\x4e\x0e\x3e\x56\xa5\xd7\x11\x29\x83\xdf\x7e\x5f\x74\x5a\xa9\xf8\x3a\x78\xa3\xc2\xff\x06\x00\x00\xff\xff\x54\x72\xf0\xa3\x29\x14\x00\x00") +var _aroOpenshiftIo_clustersYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x58\xcd\x72\xe3\xc8\x0d\xbe\xeb\x29\x50\xce\xc1\x87\x58\xb4\xb5\x93\x99\xd9\xd5\xcd\xe5\xd9\x4d\xa9\xb2\xc9\xba\xc6\xae\xb9\xac\xf7\x00\x36\x21\x0a\x71\xb3\x9b\xe9\x06\xed\xd1\xa4\xf2\xee\x29\x34\x49\x89\x92\x49\xcb\x93\xaa\xf0\xa2\x22\xba\x1b\x3f\x1f\x80\xaf\x21\xce\xe6\xf3\xf9\x0c\x6b\xfe\x42\x21\xb2\x77\x4b\xc0\x9a\xe9\xab\x90\xd3\xb7\x98\x3d\xfe\x18\x33\xf6\x97\x4f\x8b\x9c\x04\x17\xb3\x47\x76\xc5\x12\x6e\x9a\x28\xbe\xfa\x4c\xd1\x37\xc1\xd0\x27\x5a\xb3\x63\x61\xef\x66\x15\x09\x16\x28\xb8\x9c\x01\xa0\x73\x5e\x50\xc5\x51\x5f\x01\x8c\x77\x12\xbc\xb5\x14\xe6\x25\xb9\xec\xb1\xc9\x29\x6f\xd8\x16\x14\x92\x85\xde\xfe\xd3\x55\xf6\x2e\x5b\xcc\xaf\xb2\x1f\xae\x7e\xb8\xba\xfa\xb0\xf8\xf8\xc3\x62\xf1\xe1\xea\xfd\xfc\xc3\xfb\xc5\x4f\x57\xef\xfe\xf2\xf1\xe3\xe2\xc7\xf7\x33\x00\x13\x28\x29\xbf\xe7\x8a\xa2\x60\x55\x2f\xc1\x35\xd6\xce\x00\x1c\x56\xb4\x04\x63\x9b\x28\x14\x62\x86\xc1\x67\xbe\x26\x17\x37\xbc\x96\x8c\xfd\x2c\xd6\x64\xd4\xa3\x32\xf8\xa6\x5e\xc2\x8b\xf5\x56\x43\xe7\x74\x17\x70\xab\x2c\x49\x2c\x47\xf9\xdb\x50\xfa\x2b\x47\x49\x2b\xb5\x6d\x02\xda\xbd\xe9\x24\x8c\xec\xca\xc6\x62\xd8\x89\x67\x00\xd1\xf8\x9a\x86\x5a\x63\x93\x87\x0e\xcd\xce\x6e\x14\x94\x26\x2e\xe1\xdf\xff\x99\x01\x3c\xa1\xe5\x22\x45\xdb\x2e\xaa\xbb\xd7\xb7\xab\x2f\xef\xee\xcc\x86\x2a\x6c\x85\x00\x05\x45\x13\xb8\x4e\xfb\x7a\xe5\xc0\x11\x64\x43\xd0\xee\x84\xb5\x0f\xe9\xb5\x77\x11\xae\x6f\x57\xdd\xe9\x3a\xf8\x9a\x82\x70\xef\x81\x3e\x83\xba\xd8\xc9\x8e\xec\x9c\xab\x23\xed\x1e\x28\xb4\x12\xa8\x35\xd8\xe5\x93\x0a\x88\xad\x69\xbf\x06\xd9\x70\x84\x40\x75\xa0\x48\xae\xad\x8d\x81\x5a\xd0\x2d\xe8\xc0\xe7\xff\x24\x23\x19\xdc\x51\x50\x25\x10\x37\xbe\xb1\x85\x96\xcf\x13\x05\x81\x40\xc6\x97\x8e\xbf\xed\x34\x47\x10\x9f\x4c\x5a\x14\xea\x52\xd1\x3f\xec\x84\x82\x43\xab\x10\x36\x74\x01\xe8\x0a\xa8\x70\x0b\x81\xd4\x06\x34\x6e\xa0\x2d\x6d\x89\x19\xfc\xdd\x07\x02\x76\x6b\xbf\x84\x8d\x48\x1d\x97\x97\x97\x25\x4b\xdf\x09\xc6\x57\x55\xe3\x58\xb6\x97\xa9\x9e\x39\x6f\xc4\x87\x78\x59\xd0\x13\xd9\xcb\xc8\xe5\x1c\x83\xd9\xb0\x90\x91\x26\xd0\x25\xd6\x3c\x4f\x8e\xbb\xd4\x08\x59\x55\xfc\x69\x97\xe8\xf3\x81\xa7\xb2\xd5\x82\x88\x12\xd8\x95\x3b\x71\xaa\xbd\x49\xdc\xb5\x06\x35\xbb\xd8\x1d\x6b\xfd\xdf\xc3\xab\x22\x45\xe5\xf3\xcf\x77\xf7\xd0\x1b\x4d\x29\x38\xc4\x3c\xa1\xbd\x3f\x16\xf7\xc0\x2b\x50\xec\xd6\x14\xda\xc4\xad\x83\xaf\x92\x46\x72\x45\xed\xd9\x49\x57\x49\x4c\xee\x10\xf4\xd8\xe4\x15\x8b\x66\xfa\x5f\x0d\x45\xd1\xfc\x64\x70\x93\xf8\x00\x72\x82\xa6\x2e\x50\xa8\xc8\x60\xe5\xe0\x06\x2b\xb2\x37\x18\xe9\xff\x0e\xbb\x22\x1c\xe7\x0a\xe9\x69\xe0\x87\x34\x76\xb8\xb1\x45\x6b\x27\xee\xa9\x64\x34\x43\x5d\x07\xde\xd5\x64\x0e\x3a\xa3\xa0\xc8\x41\xab\x57\x50\x48\x6b\x7e\xc8\x2e\xd3\xbd\x98\xfa\xd1\x84\x4f\xbe\x42\x76\x87\xe2\x89\x30\xa0\xed\xe0\x95\x93\xd5\xed\xdb\x0f\x7c\xfb\xd9\x3d\x71\xf0\xae\x22\x27\x6f\x3e\x55\x7c\x9f\x57\x6b\x42\xcd\x53\x3c\x3e\x70\x80\xdf\x2f\xdd\xa6\x03\x00\xaf\x3f\xff\xa6\xf4\x17\x50\x7c\xe8\xd5\x40\xa9\x8d\x7f\xa4\x6a\x0a\xc2\xb4\xa6\xdc\x14\x85\x9c\xdc\x06\x5f\x91\x6c\xa8\x19\xd9\xd5\xfb\x9f\x7b\x6f\x09\xdd\x68\x6c\x47\xf5\xa0\x4f\x49\x8e\x9e\xf0\x57\x5f\x96\xec\xca\x63\xad\xaf\x79\x65\xbc\x5b\x73\x39\xc2\xb7\xbb\xc3\x28\xca\x66\x4b\x38\xff\xfd\x6a\xfe\xd3\x1f\x7f\xce\xda\x9f\xf3\x49\xcf\x47\x90\xd7\xa7\xf2\x8e\xc5\xeb\xd2\x5f\x6f\xee\x5e\xc9\xb6\x3e\xe4\x9a\x6a\x4c\x3e\x87\x4f\x8c\xa5\xf3\x51\xd8\xc4\xdb\xe0\x8b\xd1\x3d\xf7\xc7\x7c\x7c\xd2\xbb\x49\x58\xd9\x95\x81\x62\xfc\x8e\x4a\x6e\xb9\x9f\xe4\x66\x43\xe6\x91\xc2\xf7\xa4\xa2\x09\x76\xb4\x20\x58\xa8\x1a\x5d\x38\x81\x78\xbf\x8c\x21\xe0\xf6\xad\x11\x5b\x6f\x06\x77\xfe\x1b\x2c\xf5\x24\xbf\x2a\x5e\xed\xab\x7e\x6e\x5b\x7d\xea\x87\x83\xeb\x6f\xda\x45\xfb\xe3\xed\x5d\x4d\x83\x89\xe5\x4d\xf6\x9f\x1c\xc9\x4b\xdb\x13\xdb\xc7\xf9\xb4\x9d\x7a\x4e\x31\x6a\xda\x75\xc0\xa9\x3e\x8f\x7a\x69\xfd\x4f\xa4\x6a\xbc\x2b\x78\x30\xa7\x4e\x19\xdf\x6d\xeb\x6e\x5d\x92\x64\xa7\x17\x03\xbb\x28\xe8\x0c\xc5\xec\x48\xcd\x44\xd5\x1c\x68\x3f\xdb\xeb\xd9\x5f\xc5\xed\x34\xa4\x91\xa5\x42\x38\x98\x8f\xce\x63\x1b\xeb\xb1\x31\x7d\x06\xae\x62\x20\x3d\xb3\x9b\xe8\xa1\x22\xb3\x41\xc7\xb1\x4a\xfd\xe1\x0a\x2a\x74\x7c\xd2\x6b\x39\x52\x01\xcf\x1b\x72\x0a\xe8\x88\xd2\x82\x04\xd9\xc6\x9d\x13\x7b\xb7\xd4\x86\xde\xed\x08\x75\x60\x1f\x18\x1e\x9d\x7f\x76\xe0\x03\x3c\xa7\xc9\x2d\xad\xd5\xb5\x3d\xae\xfc\x54\x06\x1e\xd0\xda\x3d\x76\x49\x3d\x94\xfc\x44\x0e\x74\xc2\xc9\xe0\xc1\x0d\xe3\xe9\x86\xc1\x9c\x00\x8b\x82\xc6\x58\x47\x3c\xd0\xd7\xda\xb2\x61\xb1\xdb\x76\x6a\xdc\x0e\x72\x0f\xb2\x41\xd1\x60\x43\x4c\xd3\xa0\xf1\x55\xed\x5d\x42\xdb\x24\xb0\x72\xdf\x8c\x71\x56\x40\xd9\xa4\x49\x08\x5d\x1a\x6c\x38\xb4\x03\x96\x8f\x74\xa0\x3d\x61\x99\xa6\x26\xbd\xe3\xd3\xcc\xe4\xf5\xe4\x88\xca\x01\x86\x31\x83\xdf\x9c\xa1\xae\xa6\x8b\x8b\x54\xd4\x15\xa1\x53\x23\x09\x92\x7d\x7d\x18\x74\xd0\x8e\x52\x23\x3a\x35\xb9\x25\x15\x80\x21\x67\x09\x18\xd8\x6e\x61\x0e\xac\xbb\x8d\xaf\x28\x42\x8d\x41\xfa\xfe\xbe\xbe\x5d\xb5\x23\xf1\x06\xdb\x36\x8a\x58\x8d\x29\xcd\xd1\x3c\x3e\x63\x28\xe2\x3c\xed\x5e\xfb\xd0\xbe\x29\x76\x28\x9c\xb3\x65\x49\x50\x1b\x0a\xae\xab\x90\x6d\x1b\x76\xb2\x37\x16\xfb\xce\x83\xec\xec\xe5\xfd\xfc\x0a\x35\x03\x58\x8c\x72\x1f\xd0\x45\xee\xff\xff\x8d\x33\xf2\xda\x87\x0a\x65\x09\x3a\x6d\xce\x85\x47\x23\x3b\xc9\xdb\x15\xc5\x88\xe5\x84\x85\x13\x67\x03\x61\x1c\xbf\xc9\xa7\xa8\xe5\x73\x3a\xa1\xfc\x72\xd4\x9c\x08\xde\xd1\xfc\xd9\x87\xe2\x62\x3f\x33\x8f\x2a\x86\xa3\x3f\x58\x3b\x2e\x47\xa1\xd2\x87\xad\xbe\x1b\x6c\x22\xed\x16\x9a\x10\xc8\x49\xc7\xbd\x63\x74\xa2\xcf\x4a\x46\xbc\x4a\x94\xc1\x2e\x65\x9e\x55\x63\x23\x75\x23\x17\x10\x1b\xb3\x01\x8c\xc9\x67\xcb\x6e\xca\xd1\xc7\x26\x27\x23\x16\x4a\x65\xd2\xee\xa8\xd6\x17\x3b\x88\x4d\x55\x61\xe0\x6f\xa9\xfc\x4d\xeb\x62\xc7\x0e\xc9\xf9\x09\x3f\x4f\x24\xe4\xe5\xf5\xf2\xe6\xa3\x69\xf9\x74\x26\xf7\x34\x7e\xbf\xad\xa9\xbf\x5f\xf5\xf0\x0e\xee\x5d\x1f\xa7\x50\x8f\xc7\xd6\x81\x3d\x36\x68\xed\x56\x5b\xbf\x4f\x78\x01\x5a\x01\x4a\xac\x71\xe3\x83\x40\xbd\x09\xe9\xaf\xd3\x90\x22\x93\xb1\x29\xad\x1d\x7b\xb2\x2b\x58\xeb\xa1\xbb\x2d\x39\x51\x3e\x3c\x9c\x61\xee\xb4\x67\xec\x5c\x42\x43\x0f\x67\x50\x7b\x8b\x81\x65\x9b\xc1\x2f\x7e\x8c\xc0\xf4\xa1\xaf\x58\xd5\x96\x2e\x80\x8f\xe3\xeb\xad\xc4\xf6\x56\x41\x55\xc7\x66\xdb\xd6\x51\xfa\xa4\x71\x31\x15\x7c\xf2\x86\x63\xfb\xe1\xe3\xe1\x0c\x0c\xc6\x04\x66\x1d\x7c\x8e\xb9\xdd\xa6\x1d\xea\xeb\x05\x44\x7f\x68\xf6\xf5\xc8\x73\x6d\x04\x6b\xa9\x80\x87\xb3\x95\xeb\xd4\x8f\x30\x10\x9c\xaa\x88\xf6\x0a\xa0\x17\xd3\x8e\x4e\xbd\x6d\x99\x8d\x2c\xa8\xc6\x17\xe2\xc9\xf9\x6f\x7a\x68\xec\xff\x00\x4d\xfc\x57\x78\xfb\xc4\x75\x24\xda\x7f\x83\x5b\xa0\xad\x37\xb8\xd8\xcb\x52\xd7\xcc\xbb\x6f\x6b\x83\x65\x80\x76\xf0\x5a\x82\xd6\x4c\xf7\xe9\xca\x07\xa5\xcc\x56\xb2\xef\x39\x34\x86\x6a\xa1\xe2\x1f\xc7\x5f\xd7\xce\xce\x0e\x3e\x9f\xa5\xd7\xc1\x50\x06\xbf\xff\x31\x6b\xb5\x52\xf1\xa5\xf7\x46\x85\xff\x0d\x00\x00\xff\xff\x72\x8a\x49\x5c\xbb\x14\x00\x00") func aroOpenshiftIo_clustersYamlBytes() ([]byte, error) { return bindataRead( diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go index 859f404a376..c192d149c15 100644 --- a/pkg/operator/deploy/deploy.go +++ b/pkg/operator/deploy/deploy.go @@ -6,6 +6,7 @@ package deploy import ( "context" "fmt" + "strings" "time" "github.com/Azure/go-autorest/autorest/azure" @@ -144,6 +145,11 @@ func (o *operator) resources() ([]runtime.Object, error) { return nil, err } + domain := o.oc.Properties.ClusterProfile.Domain + if !strings.ContainsRune(domain, '.') { + domain += "." + o.env.Domain() + } + // create a secret here for genevalogging, later we will copy it to // the genevalogging namespace. return append(results, @@ -164,6 +170,7 @@ func (o *operator) resources() ([]runtime.Object, error) { }, Spec: arov1alpha1.ClusterSpec{ ResourceID: o.oc.ID, + Domain: domain, ACRDomain: o.env.ACRDomain(), AZEnvironment: o.env.Environment().Name, Location: o.env.Location(), @@ -180,6 +187,8 @@ func (o *operator) resources() ([]runtime.Object, error) { monitoringEndpoint, }, }, + APIIntIP: o.oc.Properties.APIServerProfile.IntIP, + IngressIP: o.oc.Properties.IngressProfiles[0].IP, Features: arov1alpha1.FeaturesSpec{ PersistentPrometheus: false, }, diff --git a/pkg/operator/deploy/staticresources/aro.openshift.io_clusters.yaml b/pkg/operator/deploy/staticresources/aro.openshift.io_clusters.yaml index f0f7b051d7c..de535f29a99 100644 --- a/pkg/operator/deploy/staticresources/aro.openshift.io_clusters.yaml +++ b/pkg/operator/deploy/staticresources/aro.openshift.io_clusters.yaml @@ -38,8 +38,12 @@ spec: properties: acrDomain: type: string + apiIntIP: + type: string azEnvironment: type: string + domain: + type: string features: description: FeaturesSpec defines ARO operator feature gates properties: @@ -57,6 +61,8 @@ spec: - Test type: string type: object + ingressIP: + type: string internetChecker: properties: urls: diff --git a/pkg/util/azureclient/mgmt/privatedns/generate.go b/pkg/util/azureclient/mgmt/privatedns/generate.go new file mode 100644 index 00000000000..62999e60227 --- /dev/null +++ b/pkg/util/azureclient/mgmt/privatedns/generate.go @@ -0,0 +1,8 @@ +package privatedns + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +//go:generate rm -rf ../../../../util/mocks/$GOPACKAGE +//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE PrivateZonesClient,VirtualNetworkLinksClient +//go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go diff --git a/pkg/util/azureclient/mgmt/privatedns/privatezones.go b/pkg/util/azureclient/mgmt/privatedns/privatezones.go new file mode 100644 index 00000000000..452c2e7dd5f --- /dev/null +++ b/pkg/util/azureclient/mgmt/privatedns/privatezones.go @@ -0,0 +1,31 @@ +package privatedns + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + mgmtprivatedns "github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns" + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" +) + +// PrivateZonesClient is a minimal interface for azure PrivateZonesClient +type PrivateZonesClient interface { + PrivateZonesClientAddons +} + +type privateZonesClient struct { + mgmtprivatedns.PrivateZonesClient +} + +var _ PrivateZonesClient = &privateZonesClient{} + +// NewPrivateZonesClient creates a new PrivateZonesClient +func NewPrivateZonesClient(environment *azure.Environment, subscriptionID string, authorizer autorest.Authorizer) PrivateZonesClient { + client := mgmtprivatedns.NewPrivateZonesClientWithBaseURI(environment.ResourceManagerEndpoint, subscriptionID) + client.Authorizer = authorizer + + return &privateZonesClient{ + PrivateZonesClient: client, + } +} diff --git a/pkg/util/azureclient/mgmt/privatedns/privatezones_addons.go b/pkg/util/azureclient/mgmt/privatedns/privatezones_addons.go new file mode 100644 index 00000000000..731eca99545 --- /dev/null +++ b/pkg/util/azureclient/mgmt/privatedns/privatezones_addons.go @@ -0,0 +1,43 @@ +package privatedns + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + mgmtprivatedns "github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns" +) + +// PrivateZonesClientAddons contains addons for PrivateZonesClient +type PrivateZonesClientAddons interface { + DeleteAndWait(ctx context.Context, resourceGroupName string, privateZoneName string, ifMatch string) error + ListByResourceGroup(ctx context.Context, resourceGroupName string, top *int32) ([]mgmtprivatedns.PrivateZone, error) +} + +func (c *privateZonesClient) DeleteAndWait(ctx context.Context, resourceGroupName string, privateZoneName string, ifMatch string) error { + future, err := c.PrivateZonesClient.Delete(ctx, resourceGroupName, privateZoneName, ifMatch) + if err != nil { + return err + } + + return future.WaitForCompletionRef(ctx, c.Client) +} + +func (c *privateZonesClient) ListByResourceGroup(ctx context.Context, resourceGroupName string, top *int32) (privateZones []mgmtprivatedns.PrivateZone, err error) { + page, err := c.PrivateZonesClient.ListByResourceGroup(ctx, resourceGroupName, top) + if err != nil { + return nil, err + } + + for page.NotDone() { + privateZones = append(privateZones, page.Values()...) + + err = page.NextWithContext(ctx) + if err != nil { + return nil, err + } + } + + return privateZones, nil +} diff --git a/pkg/util/mocks/azureclient/mgmt/privatedns/privatedns.go b/pkg/util/mocks/azureclient/mgmt/privatedns/privatedns.go new file mode 100644 index 00000000000..c7b65a85ef7 --- /dev/null +++ b/pkg/util/mocks/azureclient/mgmt/privatedns/privatedns.go @@ -0,0 +1,117 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/privatedns (interfaces: PrivateZonesClient,VirtualNetworkLinksClient) + +// Package mock_privatedns is a generated GoMock package. +package mock_privatedns + +import ( + context "context" + reflect "reflect" + + privatedns "github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns" + gomock "github.com/golang/mock/gomock" +) + +// MockPrivateZonesClient is a mock of PrivateZonesClient interface +type MockPrivateZonesClient struct { + ctrl *gomock.Controller + recorder *MockPrivateZonesClientMockRecorder +} + +// MockPrivateZonesClientMockRecorder is the mock recorder for MockPrivateZonesClient +type MockPrivateZonesClientMockRecorder struct { + mock *MockPrivateZonesClient +} + +// NewMockPrivateZonesClient creates a new mock instance +func NewMockPrivateZonesClient(ctrl *gomock.Controller) *MockPrivateZonesClient { + mock := &MockPrivateZonesClient{ctrl: ctrl} + mock.recorder = &MockPrivateZonesClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockPrivateZonesClient) EXPECT() *MockPrivateZonesClientMockRecorder { + return m.recorder +} + +// DeleteAndWait mocks base method +func (m *MockPrivateZonesClient) DeleteAndWait(arg0 context.Context, arg1, arg2, arg3 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAndWait", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAndWait indicates an expected call of DeleteAndWait +func (mr *MockPrivateZonesClientMockRecorder) DeleteAndWait(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAndWait", reflect.TypeOf((*MockPrivateZonesClient)(nil).DeleteAndWait), arg0, arg1, arg2, arg3) +} + +// ListByResourceGroup mocks base method +func (m *MockPrivateZonesClient) ListByResourceGroup(arg0 context.Context, arg1 string, arg2 *int32) ([]privatedns.PrivateZone, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListByResourceGroup", arg0, arg1, arg2) + ret0, _ := ret[0].([]privatedns.PrivateZone) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListByResourceGroup indicates an expected call of ListByResourceGroup +func (mr *MockPrivateZonesClientMockRecorder) ListByResourceGroup(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListByResourceGroup", reflect.TypeOf((*MockPrivateZonesClient)(nil).ListByResourceGroup), arg0, arg1, arg2) +} + +// MockVirtualNetworkLinksClient is a mock of VirtualNetworkLinksClient interface +type MockVirtualNetworkLinksClient struct { + ctrl *gomock.Controller + recorder *MockVirtualNetworkLinksClientMockRecorder +} + +// MockVirtualNetworkLinksClientMockRecorder is the mock recorder for MockVirtualNetworkLinksClient +type MockVirtualNetworkLinksClientMockRecorder struct { + mock *MockVirtualNetworkLinksClient +} + +// NewMockVirtualNetworkLinksClient creates a new mock instance +func NewMockVirtualNetworkLinksClient(ctrl *gomock.Controller) *MockVirtualNetworkLinksClient { + mock := &MockVirtualNetworkLinksClient{ctrl: ctrl} + mock.recorder = &MockVirtualNetworkLinksClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockVirtualNetworkLinksClient) EXPECT() *MockVirtualNetworkLinksClientMockRecorder { + return m.recorder +} + +// DeleteAndWait mocks base method +func (m *MockVirtualNetworkLinksClient) DeleteAndWait(arg0 context.Context, arg1, arg2, arg3, arg4 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAndWait", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAndWait indicates an expected call of DeleteAndWait +func (mr *MockVirtualNetworkLinksClientMockRecorder) DeleteAndWait(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAndWait", reflect.TypeOf((*MockVirtualNetworkLinksClient)(nil).DeleteAndWait), arg0, arg1, arg2, arg3, arg4) +} + +// List mocks base method +func (m *MockVirtualNetworkLinksClient) List(arg0 context.Context, arg1, arg2 string, arg3 *int32) ([]privatedns.VirtualNetworkLink, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].([]privatedns.VirtualNetworkLink) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List +func (mr *MockVirtualNetworkLinksClientMockRecorder) List(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockVirtualNetworkLinksClient)(nil).List), arg0, arg1, arg2, arg3) +} diff --git a/pkg/util/mocks/graph/graph.go b/pkg/util/mocks/graph/graph.go new file mode 100644 index 00000000000..051a7476230 --- /dev/null +++ b/pkg/util/mocks/graph/graph.go @@ -0,0 +1,81 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/Azure/ARO-RP/pkg/cluster/graph (interfaces: Manager) + +// Package mock_graph is a generated GoMock package. +package mock_graph + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + + graph "github.com/Azure/ARO-RP/pkg/cluster/graph" +) + +// MockManager is a mock of Manager interface +type MockManager struct { + ctrl *gomock.Controller + recorder *MockManagerMockRecorder +} + +// MockManagerMockRecorder is the mock recorder for MockManager +type MockManagerMockRecorder struct { + mock *MockManager +} + +// NewMockManager creates a new mock instance +func NewMockManager(ctrl *gomock.Controller) *MockManager { + mock := &MockManager{ctrl: ctrl} + mock.recorder = &MockManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockManager) EXPECT() *MockManagerMockRecorder { + return m.recorder +} + +// Exists mocks base method +func (m *MockManager) Exists(arg0 context.Context, arg1, arg2 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", arg0, arg1, arg2) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists +func (mr *MockManagerMockRecorder) Exists(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockManager)(nil).Exists), arg0, arg1, arg2) +} + +// LoadPersisted mocks base method +func (m *MockManager) LoadPersisted(arg0 context.Context, arg1, arg2 string) (graph.PersistedGraph, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadPersisted", arg0, arg1, arg2) + ret0, _ := ret[0].(graph.PersistedGraph) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LoadPersisted indicates an expected call of LoadPersisted +func (mr *MockManagerMockRecorder) LoadPersisted(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadPersisted", reflect.TypeOf((*MockManager)(nil).LoadPersisted), arg0, arg1, arg2) +} + +// Save mocks base method +func (m *MockManager) Save(arg0 context.Context, arg1, arg2 string, arg3 graph.Graph) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Save", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// Save indicates an expected call of Save +func (mr *MockManagerMockRecorder) Save(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockManager)(nil).Save), arg0, arg1, arg2, arg3) +} diff --git a/pkg/util/mocks/storage/storage.go b/pkg/util/mocks/storage/storage.go new file mode 100644 index 00000000000..0d23ec071db --- /dev/null +++ b/pkg/util/mocks/storage/storage.go @@ -0,0 +1,52 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/Azure/ARO-RP/pkg/util/storage (interfaces: Manager) + +// Package mock_storage is a generated GoMock package. +package mock_storage + +import ( + context "context" + reflect "reflect" + + storage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" + storage0 "github.com/Azure/azure-sdk-for-go/storage" + gomock "github.com/golang/mock/gomock" +) + +// MockManager is a mock of Manager interface +type MockManager struct { + ctrl *gomock.Controller + recorder *MockManagerMockRecorder +} + +// MockManagerMockRecorder is the mock recorder for MockManager +type MockManagerMockRecorder struct { + mock *MockManager +} + +// NewMockManager creates a new mock instance +func NewMockManager(ctrl *gomock.Controller) *MockManager { + mock := &MockManager{ctrl: ctrl} + mock.recorder = &MockManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockManager) EXPECT() *MockManagerMockRecorder { + return m.recorder +} + +// BlobService mocks base method +func (m *MockManager) BlobService(arg0 context.Context, arg1, arg2 string, arg3 storage.Permissions, arg4 storage.SignedResourceTypes) (*storage0.BlobStorageClient, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BlobService", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(*storage0.BlobStorageClient) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BlobService indicates an expected call of BlobService +func (mr *MockManagerMockRecorder) BlobService(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BlobService", reflect.TypeOf((*MockManager)(nil).BlobService), arg0, arg1, arg2, arg3, arg4) +} diff --git a/pkg/util/mocks/subnet/subnet.go b/pkg/util/mocks/subnet/subnet.go index f4f776edff7..1fd16eb1761 100644 --- a/pkg/util/mocks/subnet/subnet.go +++ b/pkg/util/mocks/subnet/subnet.go @@ -63,3 +63,18 @@ func (mr *MockManagerMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockManager)(nil).Get), arg0, arg1) } + +// GetHighestFreeIP mocks base method +func (m *MockManager) GetHighestFreeIP(arg0 context.Context, arg1 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetHighestFreeIP", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetHighestFreeIP indicates an expected call of GetHighestFreeIP +func (mr *MockManagerMockRecorder) GetHighestFreeIP(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHighestFreeIP", reflect.TypeOf((*MockManager)(nil).GetHighestFreeIP), arg0, arg1) +} diff --git a/pkg/util/ready/ready.go b/pkg/util/ready/ready.go index 01b595a43b9..1de4c612a15 100644 --- a/pkg/util/ready/ready.go +++ b/pkg/util/ready/ready.go @@ -7,6 +7,8 @@ import ( "context" "net" + mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + machineconfigurationv1 "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned/typed/machineconfiguration.openshift.io/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -76,6 +78,7 @@ func DeploymentIsReady(d *appsv1.Deployment) bool { return specReplicas == d.Status.AvailableReplicas && specReplicas == d.Status.UpdatedReplicas && + specReplicas == d.Status.Replicas && d.Generation == d.Status.ObservedGeneration } @@ -143,3 +146,27 @@ func CheckStatefulSetIsReady(ctx context.Context, cli appsv1client.StatefulSetIn return StatefulSetIsReady(s), nil } } + +// MachineConfigPoolIsReady returns true if a MachineConfigPool is considered +// ready +func MachineConfigPoolIsReady(s *mcv1.MachineConfigPool) bool { + return s.Status.MachineCount == s.Status.UpdatedMachineCount && + s.Status.MachineCount == s.Status.ReadyMachineCount && + s.Generation == s.Status.ObservedGeneration +} + +// CheckMachineConfigPoolIsReady returns a function which polls a +// MachineConfigPool and returns its readiness +func CheckMachineConfigPoolIsReady(ctx context.Context, cli machineconfigurationv1.MachineConfigPoolInterface, name string) func() (bool, error) { + return func() (bool, error) { + s, err := cli.Get(ctx, name, metav1.GetOptions{}) + switch { + case errors.IsNotFound(err): + return false, nil + case err != nil: + return false, err + } + + return MachineConfigPoolIsReady(s), nil + } +} diff --git a/pkg/util/storage/generate.go b/pkg/util/storage/generate.go new file mode 100644 index 00000000000..ceb54ea09a4 --- /dev/null +++ b/pkg/util/storage/generate.go @@ -0,0 +1,8 @@ +package storage + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +//go:generate rm -rf ../mocks/$GOPACKAGE +//go:generate go run ../../../vendor/github.com/golang/mock/mockgen -destination=../mocks/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/$GOPACKAGE Manager +//go:generate go run ../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../mocks/$GOPACKAGE/$GOPACKAGE.go diff --git a/pkg/util/storage/manager.go b/pkg/util/storage/manager.go new file mode 100644 index 00000000000..066367b18e0 --- /dev/null +++ b/pkg/util/storage/manager.go @@ -0,0 +1,58 @@ +package storage + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "net/url" + "time" + + mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" + azstorage "github.com/Azure/azure-sdk-for-go/storage" + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/date" + + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage" +) + +type Manager interface { + BlobService(ctx context.Context, resourceGroup, account string, p mgmtstorage.Permissions, r mgmtstorage.SignedResourceTypes) (*azstorage.BlobStorageClient, error) +} + +type manager struct { + env env.Core + storageAccounts storage.AccountsClient +} + +func NewManager(env env.Core, subscriptionID string, authorizer autorest.Authorizer) Manager { + return &manager{ + env: env, + storageAccounts: storage.NewAccountsClient(env.Environment(), subscriptionID, authorizer), + } +} + +func (m *manager) BlobService(ctx context.Context, resourceGroup, account string, p mgmtstorage.Permissions, r mgmtstorage.SignedResourceTypes) (*azstorage.BlobStorageClient, error) { + t := time.Now().UTC().Truncate(time.Second) + res, err := m.storageAccounts.ListAccountSAS(ctx, resourceGroup, account, mgmtstorage.AccountSasParameters{ + Services: mgmtstorage.B, + ResourceTypes: r, + Permissions: p, + Protocols: mgmtstorage.HTTPS, + SharedAccessStartTime: &date.Time{Time: t}, + SharedAccessExpiryTime: &date.Time{Time: t.Add(24 * time.Hour)}, + }) + if err != nil { + return nil, err + } + + v, err := url.ParseQuery(*res.AccountSasToken) + if err != nil { + return nil, err + } + + blobcli := azstorage.NewAccountSASClient(account, v, *m.env.Environment()).GetBlobService() + + return &blobcli, nil +} diff --git a/pkg/util/subnet/subnet.go b/pkg/util/subnet/subnet.go index 58509826336..643e3cdd6b9 100644 --- a/pkg/util/subnet/subnet.go +++ b/pkg/util/subnet/subnet.go @@ -6,11 +6,13 @@ package subnet import ( "context" "fmt" + "net" "strings" mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" + "github.com/apparentlymart/go-cidr/cidr" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/env" @@ -19,6 +21,7 @@ import ( type Manager interface { Get(ctx context.Context, subnetID string) (*mgmtnetwork.Subnet, error) + GetHighestFreeIP(ctx context.Context, subnetID string) (string, error) CreateOrUpdate(ctx context.Context, subnetID string, subnet *mgmtnetwork.Subnet) error } @@ -34,6 +37,10 @@ func NewManager(env env.Core, subscriptionID string, spAuthorizer autorest.Autho // Get retrieves the linked subnet func (m *manager) Get(ctx context.Context, subnetID string) (*mgmtnetwork.Subnet, error) { + return m.get(ctx, subnetID, "") +} + +func (m *manager) get(ctx context.Context, subnetID, expand string) (*mgmtnetwork.Subnet, error) { vnetID, subnetName, err := Split(subnetID) if err != nil { return nil, err @@ -44,7 +51,7 @@ func (m *manager) Get(ctx context.Context, subnetID string) (*mgmtnetwork.Subnet return nil, err } - subnet, err := m.subnets.Get(ctx, r.ResourceGroup, r.ResourceName, subnetName, "") + subnet, err := m.subnets.Get(ctx, r.ResourceGroup, r.ResourceName, subnetName, expand) if err != nil { return nil, err } @@ -52,6 +59,47 @@ func (m *manager) Get(ctx context.Context, subnetID string) (*mgmtnetwork.Subnet return &subnet, nil } +func (m *manager) GetHighestFreeIP(ctx context.Context, subnetID string) (string, error) { + // Probably anyone who calls this function has a race condition. + + subnet, err := m.get(ctx, subnetID, "ipConfigurations") + if err != nil { + return "", err + } + + _, subnetCIDR, err := net.ParseCIDR(*subnet.AddressPrefix) + if err != nil { + return "", err + } + + bottom, top := cidr.AddressRange(subnetCIDR) + + allocated := map[string]struct{}{} + + // first four addresses and the broadcast address are reserved: + // https://docs.microsoft.com/en-us/azure/virtual-network/private-ip-addresses#allocation-method + for i, ip := 0, bottom; i < 4 && !ip.Equal(top); i, ip = i+1, cidr.Inc(ip) { + allocated[ip.String()] = struct{}{} + } + allocated[top.String()] = struct{}{} + + if subnet.IPConfigurations != nil { + for _, ipconfig := range *subnet.IPConfigurations { + if ipconfig.PrivateIPAddress != nil { + allocated[*ipconfig.PrivateIPAddress] = struct{}{} + } + } + } + + for ip := top; !ip.Equal(cidr.Dec(bottom)); ip = cidr.Dec(ip) { + if _, ok := allocated[ip.String()]; !ok { + return ip.String(), nil + } + } + + return "", nil +} + // CreateOrUpdate updates the linked subnet func (m *manager) CreateOrUpdate(ctx context.Context, subnetID string, subnet *mgmtnetwork.Subnet) error { vnetID, subnetName, err := Split(subnetID) diff --git a/pkg/util/subnet/subnet_test.go b/pkg/util/subnet/subnet_test.go index c332ffe02b1..5440486f0b0 100644 --- a/pkg/util/subnet/subnet_test.go +++ b/pkg/util/subnet/subnet_test.go @@ -88,6 +88,118 @@ func TestGet(t *testing.T) { } } +func TestGetGetHighestFreeIP(t *testing.T) { + ctx := context.Background() + + type test struct { + name string + mocks func(*test, *mock_network.MockSubnetsClient) + wantIP string + wantErr string + } + + for _, tt := range []*test{ + { + name: "valid", + mocks: func(tt *test, subnets *mock_network.MockSubnetsClient) { + subnets.EXPECT(). + Get(ctx, "vnetResourceGroup", "vnet", "subnet", "ipConfigurations"). + Return(mgmtnetwork.Subnet{ + SubnetPropertiesFormat: &mgmtnetwork.SubnetPropertiesFormat{ + AddressPrefix: to.StringPtr("10.0.0.0/29"), + }, + }, nil) + }, + wantIP: "10.0.0.6", + }, + { + name: "valid, top address used", + mocks: func(tt *test, subnets *mock_network.MockSubnetsClient) { + subnets.EXPECT(). + Get(ctx, "vnetResourceGroup", "vnet", "subnet", "ipConfigurations"). + Return(mgmtnetwork.Subnet{ + SubnetPropertiesFormat: &mgmtnetwork.SubnetPropertiesFormat{ + AddressPrefix: to.StringPtr("10.0.0.0/29"), + IPConfigurations: &[]mgmtnetwork.IPConfiguration{ + { + IPConfigurationPropertiesFormat: &mgmtnetwork.IPConfigurationPropertiesFormat{ + PrivateIPAddress: to.StringPtr("10.0.0.6"), + }, + }, + { + IPConfigurationPropertiesFormat: &mgmtnetwork.IPConfigurationPropertiesFormat{}, + }, + }, + }, + }, nil) + }, + wantIP: "10.0.0.5", + }, + { + name: "exhausted", + mocks: func(tt *test, subnets *mock_network.MockSubnetsClient) { + subnets.EXPECT(). + Get(ctx, "vnetResourceGroup", "vnet", "subnet", "ipConfigurations"). + Return(mgmtnetwork.Subnet{ + SubnetPropertiesFormat: &mgmtnetwork.SubnetPropertiesFormat{ + AddressPrefix: to.StringPtr("10.0.0.0/29"), + IPConfigurations: &[]mgmtnetwork.IPConfiguration{ + { + IPConfigurationPropertiesFormat: &mgmtnetwork.IPConfigurationPropertiesFormat{ + PrivateIPAddress: to.StringPtr("10.0.0.4"), + }, + }, + { + IPConfigurationPropertiesFormat: &mgmtnetwork.IPConfigurationPropertiesFormat{ + PrivateIPAddress: to.StringPtr("10.0.0.5"), + }, + }, + { + IPConfigurationPropertiesFormat: &mgmtnetwork.IPConfigurationPropertiesFormat{ + PrivateIPAddress: to.StringPtr("10.0.0.6"), + }, + }, + }, + }, + }, nil) + }, + }, + { + name: "broken", + mocks: func(tt *test, subnets *mock_network.MockSubnetsClient) { + subnets.EXPECT(). + Get(ctx, "vnetResourceGroup", "vnet", "subnet", "ipConfigurations"). + Return(mgmtnetwork.Subnet{}, fmt.Errorf("broken")) + }, + wantErr: "broken", + }, + } { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + subnets := mock_network.NewMockSubnetsClient(controller) + if tt.mocks != nil { + tt.mocks(tt, subnets) + } + + m := &manager{ + subnets: subnets, + } + + ip, err := m.GetHighestFreeIP(ctx, "/subscriptions/subscriptionId/resourceGroups/vnetResourceGroup/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet") + if err != nil && err.Error() != tt.wantErr || + err == nil && tt.wantErr != "" { + t.Error(err) + } + + if ip != tt.wantIP { + t.Error(ip) + } + }) + } +} + func TestCreateOrUpdate(t *testing.T) { ctx := context.Background() diff --git a/pkg/util/tls/tls.go b/pkg/util/tls/tls.go index e390ab4b78c..b88c746ce3b 100644 --- a/pkg/util/tls/tls.go +++ b/pkg/util/tls/tls.go @@ -14,6 +14,14 @@ import ( ) func GenerateKeyAndCertificate(commonName string, parentKey *rsa.PrivateKey, parentCert *x509.Certificate, isCA bool, isClient bool) (*rsa.PrivateKey, []*x509.Certificate, error) { + return generateKeyAndCertificate(commonName, parentKey, parentCert, isCA, isClient, nil) +} + +func GenerateTestKeyAndCertificate(commonName string, parentKey *rsa.PrivateKey, parentCert *x509.Certificate, isCA bool, isClient bool, tweakTemplate func(*x509.Certificate)) (*rsa.PrivateKey, []*x509.Certificate, error) { + return generateKeyAndCertificate(commonName, parentKey, parentCert, isCA, isClient, tweakTemplate) +} + +func generateKeyAndCertificate(commonName string, parentKey *rsa.PrivateKey, parentCert *x509.Certificate, isCA bool, isClient bool, tweakTemplate func(*x509.Certificate)) (*rsa.PrivateKey, []*x509.Certificate, error) { if isCA && isClient { return nil, nil, fmt.Errorf("cannot generate CA client certificate") } @@ -46,6 +54,10 @@ func GenerateKeyAndCertificate(commonName string, parentKey *rsa.PrivateKey, par DNSNames: []string{commonName}, } + if tweakTemplate != nil { + tweakTemplate(template) + } + if isCA { template.KeyUsage |= x509.KeyUsageCertSign } else { diff --git a/test/e2e/get.go b/test/e2e/get.go index 40d466c3a99..4b6bd3ca012 100644 --- a/test/e2e/get.go +++ b/test/e2e/get.go @@ -24,6 +24,7 @@ var _ = Describe("Get cluster", func() { ingressProfile := (*oc.IngressProfiles)[0] Expect(*ingressProfile.Name).To(Equal("default")) Expect(ingressProfile.Visibility).To(Equal(redhatopenshift.Visibility1Private)) + Expect(ingressProfile.IP).NotTo(BeNil()) // Check we retrieve Cluster version clusterProfile := oc.ClusterProfile diff --git a/vendor/github.com/openshift/installer/data/assets_vfsdata.go b/vendor/github.com/openshift/installer/data/assets_vfsdata.go index 81ff63ee1c8..84805708e88 100644 --- a/vendor/github.com/openshift/installer/data/assets_vfsdata.go +++ b/vendor/github.com/openshift/installer/data/assets_vfsdata.go @@ -1261,6 +1261,18 @@ var Assets = func() http.FileSystem { compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\xcb\x31\x12\xc2\x20\x10\x05\xd0\x9e\x53\xec\xa4\x47\x27\x2d\x87\xb0\xb4\xff\x92\x1f\xb3\x23\xec\x32\x80\x39\xbf\x63\x65\x6d\xff\x1e\x9a\xde\xd9\x87\xba\x25\x39\xd7\xf0\x52\xdb\x92\xdc\x50\x39\x1a\x32\x43\xe5\xc4\x86\x89\x14\x44\x0c\x95\x49\xbc\xd1\xc6\xa1\xfb\x8c\x15\xf9\x50\x63\xcc\x6e\xbb\x3e\xa3\x37\x76\x4c\xef\x41\xa4\xe0\xc1\x32\xbe\xe7\x9f\x25\x3f\x75\x51\xbf\xf6\xb7\xc5\xc2\x93\x25\xc9\xb2\x2e\xe1\x13\x00\x00\xff\xff\x52\xeb\x08\xe7\xa9\x00\x00\x00"), }, + "/manifests/bootkube/aro-ingress-namespace.yaml": &vfsgen۰FileInfo{ + name: "aro-ingress-namespace.yaml", + modTime: time.Time{}, + content: []byte("\x61\x70\x69\x56\x65\x72\x73\x69\x6f\x6e\x3a\x20\x76\x31\x0a\x6b\x69\x6e\x64\x3a\x20\x4e\x61\x6d\x65\x73\x70\x61\x63\x65\x0a\x6d\x65\x74\x61\x64\x61\x74\x61\x3a\x0a\x20\x20\x6e\x61\x6d\x65\x3a\x20\x6f\x70\x65\x6e\x73\x68\x69\x66\x74\x2d\x69\x6e\x67\x72\x65\x73\x73\x0a"), + }, + "/manifests/bootkube/aro-ingress-service.yaml.template": &vfsgen۰CompressedFileInfo{ + name: "aro-ingress-service.yaml.template", + modTime: time.Time{}, + uncompressedSize: 655, + + compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\x90\x4f\xcb\xd3\x40\x10\xc6\xef\xf9\x14\x43\xef\x89\x8a\x3d\xc8\xde\xd4\x53\x41\x30\x68\xf1\x3e\xd9\x4c\xda\xa5\xdb\x99\x65\x76\x52\xad\x21\xdf\x5d\x92\x6c\xb1\x20\xc8\xfb\xde\x76\xe7\xcf\x33\xbf\xe7\xc1\x14\x7e\x90\xe6\x20\xec\xe0\xf6\xae\xba\x04\xee\x1d\x7c\x27\xbd\x05\x4f\xd5\x95\x0c\x7b\x34\x74\x15\x00\xe3\x95\x1c\xa8\x8c\x46\x5a\xf7\x34\xe0\x18\xad\x94\x73\x42\x4f\x0e\x24\x11\xe7\x73\x18\xac\x0e\x7c\x52\xca\xb9\x02\x40\x66\x31\xb4\x20\x9c\x17\x11\x80\xbc\x49\x37\x1d\x19\x36\x97\xb1\x23\x65\x32\xca\x4d\x90\x37\xf8\x7b\x54\xaa\xa3\x60\x5f\x77\x18\x91\x3d\x69\x1d\xd8\x48\x19\xa3\x83\xdd\x34\x41\x18\xa0\xf9\xf8\xed\xeb\x61\x93\x3f\x94\x1e\xcc\xb3\xe9\x48\xd3\x04\xc4\x3d\xcc\xf3\xae\x02\x88\xd8\x51\x2c\x27\x31\xa5\x07\xf8\xfa\x2f\x74\x5e\xd8\x54\x62\x24\x6d\x24\x91\xa2\xc9\xfa\xd8\x2c\x2c\x3c\xf2\x93\x03\x9f\xea\x7f\xc6\x1d\x3c\xec\xe7\x44\x7e\xb9\x41\xbf\x36\x92\xa3\xe2\x30\x04\xdf\x4a\x0c\xfe\xee\xe0\x8b\x78\x8c\x0b\x8c\x60\xff\xa9\x38\x3a\xb4\x9b\x95\x67\x1f\x6d\x81\x4e\xa2\xb6\x32\xd7\x25\xed\xb3\x59\x5a\x91\x97\x8e\x83\x0f\x6f\xb7\x8f\x8a\x89\x97\xe8\xe0\xf8\xb9\x5d\x2b\x86\x7a\x22\x6b\xd7\xa1\xb2\xf3\x2c\x91\x9f\x34\xf6\xfb\xf7\x2f\x14\x59\xb6\x32\x45\xf2\x26\xea\x5e\x15\x5c\x4f\x29\xca\xfd\x4a\x6c\xff\x0b\x0f\xc0\xee\x89\x96\x94\xfe\xa6\x53\xfd\x09\x00\x00\xff\xff\xda\x16\xbb\x3c\x8f\x02\x00\x00"), + }, "/manifests/bootkube/aro-worker-registries.yaml.template": &vfsgen۰CompressedFileInfo{ name: "aro-worker-registries.yaml.template", modTime: time.Time{}, @@ -1433,23 +1445,23 @@ var Assets = func() http.FileSystem { "/openstack/bootstrap/main.tf": &vfsgen۰CompressedFileInfo{ name: "main.tf", modTime: time.Time{}, - uncompressedSize: 2191, + uncompressedSize: 2305, - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x55\xcf\x6e\xfb\x36\x0c\xbe\xfb\x29\x38\x63\xd7\x1a\xed\x76\x0b\x60\xf4\xb0\x5d\x76\xd9\x0b\x04\x81\xa0\x58\x4c\x42\xd4\x96\x04\xfd\x71\xd3\x16\x79\xf7\x81\x92\xdc\xd8\x49\x9b\xac\x3f\x9f\x12\x8a\xfc\x44\x7e\xfc\x48\x39\xf4\x26\xba\x0e\xa1\x36\x16\xb5\x0f\xb2\x7b\x11\x1a\xc3\xab\x71\x2f\xa4\xf7\xc2\x1a\x17\xc4\xf8\x47\x0d\xf5\xd6\x98\xe0\x83\x93\x36\xd9\x6a\xf8\xa8\x00\xb4\x1c\x10\x5a\xa8\x7f\xff\x18\xa5\x6b\xba\x3e\xfa\x80\x4e\x90\x3a\x3d\x7c\x7a\x3f\x24\xef\xaa\x02\x90\x6a\x20\x2d\x7c\x90\x01\x45\xb4\xc0\x5f\x0b\x75\x70\x11\x6b\x86\xca\x77\x0a\x52\x30\x7d\x2d\x30\xaa\x75\x34\x72\xc8\xd9\xa1\x02\xf0\xd8\x45\x47\xe1\x4d\xec\x9d\x89\x56\x90\xf2\xd0\xc2\x9a\xdd\x07\x99\x72\xf0\x7b\x41\x6a\x53\x01\x04\xb9\xf7\xb0\xfc\x5a\x58\xe7\x62\x0f\xb4\x0b\x7f\xe5\xa4\xff\xf9\xbb\xbd\x2a\xa2\xde\x70\xda\x78\x0c\x4e\x0a\x75\xe8\xac\x30\x36\x90\xd1\xa9\xf2\x52\x3b\x57\xa0\xcc\x20\x49\x3f\x78\x94\xae\x3b\xd4\xe9\x6c\x94\x7d\xc4\x92\xff\x04\x98\xdd\x2a\x80\x13\xa3\xee\xe8\x88\x4a\x90\x2d\x60\x3e\x6e\x35\x06\x2e\x3e\x07\x69\xa3\xd0\x8b\x4f\xeb\x14\x25\xfb\xde\xbc\xa2\x12\x52\x29\x87\xde\x0b\x2b\xc9\xf9\x02\x41\x76\x32\x17\x0c\x69\x49\x90\x0e\x82\xec\x14\xae\xd0\xa2\x56\x5e\x18\x7d\xc1\x56\xea\x32\x29\xbf\xa9\x4e\x55\xa5\x64\x90\x73\x35\x74\x66\xb0\x31\xa0\xd8\xf5\x72\x34\xee\x52\x0c\xd9\xba\x90\x03\x03\x17\x67\xb6\x30\xe6\x57\x2a\xdb\xf6\xa6\x7b\xf1\xc1\x38\xb9\x47\x31\x9a\x3e\x0e\x28\xc6\x3f\x17\xe0\xd9\x3a\x03\xbf\x23\x36\x66\xbf\x33\x51\x87\x92\x85\x33\x26\x4c\xd0\x9e\xde\x11\xda\x16\x74\xec\x7b\x78\x86\x47\x58\xc1\x13\x93\x92\xec\x0b\xc5\x5d\x46\x55\x00\xe5\x5f\x78\xb3\xf8\x85\x13\x9b\x2b\x00\x1a\xb8\x92\x2c\xe0\xec\xb4\x95\x1e\xc5\x64\xfe\x8e\x87\x89\x5f\xe2\xff\xba\xc3\x0b\x86\xe7\xd5\x2f\x55\x7c\x8f\x89\xd2\x83\xc5\x44\x71\x73\x9b\x1b\xbd\x6d\x2e\x3b\xdb\x24\xf5\xcd\x4b\xbb\x45\xd5\x8c\xe0\xab\xfa\x61\x95\x8e\x58\xc6\xa3\xa4\x5e\x6e\xa9\xe7\x09\x7e\x37\x7a\xe2\x94\x7f\x72\x4b\xa2\xe7\x79\x61\x19\x16\x1a\x3f\x73\xf2\x07\x1a\x04\xed\x35\xf1\x20\x26\x49\xbf\x69\x39\x50\x07\x49\x4c\x42\xe1\x48\x1d\x96\x81\xd8\x19\x27\x50\x76\x87\xfb\xa9\xae\x37\xb0\x82\xf5\x3d\x65\x36\x97\xba\x5c\x3f\x6e\x9a\xbc\x64\x58\x75\x3a\xa0\x0e\xe5\x6a\x80\x18\xe7\x64\x9d\x49\x9b\xe7\xd9\xa4\x35\x51\x02\xb2\x2e\xb2\xc2\x16\x4d\x2e\x33\x50\xdc\x38\x07\x41\x5a\xe1\x71\x89\xfb\x58\xce\x15\xfa\x40\x5a\x32\x3d\x33\xac\x2b\x18\x85\x3d\x06\x14\xec\x84\x6e\x28\x01\xd0\x02\x6f\xe3\xe4\x72\x9a\x76\x46\x59\xbb\xa5\x30\x5e\x14\xd0\xc2\xad\xa7\xa2\x59\x3e\x14\xcd\x79\x7b\xa5\x55\xfc\xb3\xe5\x3b\x60\x90\x45\x08\x39\x81\x7f\xaf\xe6\xe0\x7f\x0c\x02\xc0\xf5\x8d\x17\xdb\xb9\x64\xf9\xcd\x88\xce\xaa\xdc\xf5\x46\x06\xd2\x7b\xb2\x57\x9b\x90\xca\xac\xe6\x0d\x34\x1f\x13\x3c\x06\x74\x5a\xf6\x13\x10\xfc\xd6\x42\x5d\xc3\x33\x3c\xc1\x2a\xb5\x4e\xa1\xef\x1c\xd9\xd2\x85\x9b\xef\x29\x5f\x53\x71\x27\x4c\x0f\x37\x2f\xa9\x72\xbb\xa6\xa9\xfd\x85\xae\xcd\xdf\xce\x9f\xf5\x6d\xf9\xd2\xdc\xdc\x75\xe7\x9b\xf9\xf5\xf9\x2f\x00\x00\xff\xff\xdd\xdc\x09\x13\x8f\x08\x00\x00"), + compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x55\xc1\x6e\xf3\x36\x0c\xbe\xfb\x29\x38\x63\xd7\x1a\xfd\xb7\x5b\x00\xe3\x3f\x6c\x97\x5d\xf6\x02\x41\x20\x30\x16\x93\x10\xb5\x25\x41\x92\xdd\xb4\x45\xde\x7d\xa0\x2c\x27\x76\xd2\x26\xeb\xef\x53\x42\x51\x1f\xc9\x8f\x1f\x29\x4f\xc1\xf6\xbe\x21\x28\xad\x23\x13\x22\x36\x2f\xca\x50\x7c\xb5\xfe\x85\xcd\x5e\x39\xeb\xa3\x1a\xfe\x28\xa1\xdc\x5a\x1b\x43\xf4\xe8\x92\xad\x84\x8f\x02\xc0\x60\x47\x50\x43\xf9\xfb\xc7\x80\xbe\x6a\xda\x3e\x44\xf2\x8a\xf5\xe9\xe9\xec\xfd\x94\xbc\x8b\x02\x00\x75\xc7\x46\x85\x88\x91\x54\xef\x40\xbe\x1a\xca\xe8\x7b\x2a\x05\x6a\x8c\xa9\x58\xc3\xf4\xd5\x20\xa8\xce\xf3\x20\x57\x2e\x0e\x05\x40\xa0\xa6\xf7\x1c\xdf\xd4\xde\xdb\xde\x29\xd6\x21\x7b\x77\x98\x52\x08\x7b\xb1\x15\x00\x11\xf7\x01\x96\x5f\x0d\xeb\xb1\xd6\x03\xef\xe2\x5f\x63\xce\xff\xfc\x5d\xdf\xd4\x50\x6e\x24\x6b\x3a\x46\x8f\x4a\x1f\x1a\xa7\xac\x8b\x6c\x4d\x2a\x3c\x97\x2e\x05\x68\xdb\x21\x9b\xa7\x40\xe8\x9b\x43\x99\xce\x06\x6c\x7b\xca\x09\x4d\x80\xa3\x5b\x01\x70\x12\xd4\x1d\x1f\x49\x2b\x76\x19\x2c\xf4\x5b\x43\x51\x6a\x1f\x2f\x19\xab\x29\xa8\xb3\x75\xba\x85\x6d\x6b\x5f\x49\x2b\xd4\xda\x53\x08\xca\x21\xfb\x90\x21\xd8\x4d\xe6\x8c\x81\x8e\x15\x9b\xa8\xd8\x4d\xd7\x35\x39\x32\x3a\x28\x6b\x84\x83\x19\x5b\xa9\xc9\xac\xc3\xa6\x38\x15\x85\xc6\x88\x73\x31\x34\xb6\x73\x7d\x24\xb5\x6b\x71\xb0\xfe\x5a\x0b\xa3\x75\xa1\x06\x01\xce\xce\x62\x11\xcc\xcf\x44\xb6\x6d\x6d\xf3\x12\xa2\xf5\xb8\x27\x35\xd8\xb6\xef\x48\x0d\x7f\x2e\xc0\x47\xeb\x0c\xfc\x81\xd6\x84\xfd\xc6\xf6\x26\xe6\x2c\xbc\xb5\x71\x82\x0e\xfc\x4e\x50\xd7\x60\xfa\xb6\x85\x9f\xf0\x0c\x2b\xf8\x21\xa4\x24\xfb\x42\x70\xd7\xb7\x0a\x80\xfc\x2f\xbe\x39\xfa\xc4\x49\xcc\x05\x00\x77\x52\xc9\xa8\xdf\xd1\x69\x8b\x81\xd4\x64\xfe\x8a\x87\x89\x5f\x96\xff\xa6\xa1\x2b\x86\xe7\xd5\x2f\x55\xfc\x88\x89\xdc\x83\xc5\x40\x49\x73\xab\x3b\xbd\xad\xae\x3b\x5b\x25\xf5\xcd\x4b\xbb\x47\xd5\x8c\xe0\x9b\xfa\x61\x95\x8e\x44\xc6\x03\x72\x8b\x5b\x6e\x65\x80\xdf\xad\x99\x38\x95\x9f\xd2\x92\x3e\xc8\xbc\x88\x0c\x33\x8d\xe7\x9c\xc2\x81\x3b\xc5\x7b\xc3\x32\x88\x49\xd2\x6f\x06\x3b\x6e\x20\x89\x49\x69\x1a\xb8\xa1\x3c\x10\x3b\xeb\x15\x61\x73\x78\x9c\xea\x7a\x03\x2b\x58\x3f\x52\x66\x75\xad\xcb\xf5\xf3\xa6\x62\xbd\x49\xc1\x1a\x6b\x22\x99\x98\x43\x03\xf4\xfd\x9c\xac\x0b\x69\xf3\x3c\xab\xb4\x26\xf2\x85\x51\x17\xa3\xc2\x16\x4d\xce\x33\x90\xdd\x24\x07\xc5\x46\xd3\x71\x89\xfb\x9c\xcf\x35\x85\xc8\x06\x85\x9e\x19\xd6\x0d\x8c\xa6\x96\x22\x29\x71\x22\xdf\xe5\x0b\x50\x83\x2c\xe3\xe4\x72\x9a\x76\x46\xde\xba\xb9\x30\x59\x14\x50\xc3\xbd\x97\xa2\x5a\xbe\x13\xd5\x65\x7b\x4d\xbd\x5a\x42\x5e\xb5\x09\xb5\x4e\xcd\xc5\x76\xb6\xf0\x43\xf1\x35\xc9\xf5\x84\x37\xa3\xf3\x9c\x7d\xda\xfe\xdf\xdb\xf7\x1d\x45\xcc\xda\x1b\xe3\xfc\x7b\x33\x7a\xff\x63\xf6\x00\x6e\x23\x5e\x3d\x08\x99\x98\x2f\xb6\xc2\x8c\xd8\x5d\x6b\x31\xb2\xd9\xb3\xbb\x59\xbe\x9c\xd7\xc3\xb8\xf4\xe6\x93\x49\xc7\x48\x7e\xc6\x22\xfc\x56\x43\x59\xc2\x4f\xf8\x01\xab\xa4\x16\x4d\xa1\xf1\xec\x72\xe3\xef\xbe\xe0\x12\xa6\x90\xe6\xdb\x16\xee\x06\x29\x46\x85\x4c\x8b\xe2\x17\x84\x32\x7f\xae\xbf\xd7\xb7\xe5\xe3\x76\x77\xbd\x5e\x22\xcb\x83\xf7\x5f\x00\x00\x00\xff\xff\x2e\xea\xf8\x14\x01\x09\x00\x00"), }, "/openstack/bootstrap/variables.tf": &vfsgen۰CompressedFileInfo{ name: "variables.tf", modTime: time.Time{}, - uncompressedSize: 1458, + uncompressedSize: 1595, - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x94\xc1\x6e\xdb\x30\x0c\x86\xef\x7e\x0a\xc2\xa7\x16\x18\xfa\x00\x43\x73\xd8\xa1\xdb\x6d\xbb\xec\xb4\x8b\x4a\xd9\xb4\x43\x44\x16\x05\x89\x72\x9b\x06\x79\xf7\x41\x4e\x9d\xce\x49\x0a\x18\x58\x6e\x12\xc8\xff\xa3\xf8\xff\xf1\x88\x91\xd1\x3a\x82\xda\x62\x22\xc3\x03\xf6\x64\xb8\xad\xe1\x50\x01\xe8\x3e\x10\xbc\xff\x36\x90\x34\xb2\xef\x2b\x80\x96\x52\x13\x39\x28\x8b\x87\x0d\xd4\xbf\xb7\x04\xdc\x92\x57\xee\x98\x22\x48\x07\xba\x25\xf8\xe1\xd0\x37\x04\x93\x20\x74\x12\xa7\x4b\x2b\xa2\x49\x23\x06\xf0\xd2\xd2\x43\x5d\x1d\xab\xea\x63\x02\x7a\xd5\x88\x46\xb1\x4f\x4b\xfc\x06\x06\x0c\x77\x27\xfc\xfd\xc4\xef\x30\x3b\x85\x0d\x1c\x8e\xd5\xd5\x3c\x8f\x8f\x4f\xbf\xbe\x57\x77\x32\x9d\xd1\xdd\xc3\x53\x91\x85\x22\x0b\x2a\x60\x09\x30\x04\xc7\xd4\x96\x53\x13\x09\x95\x5a\x88\x94\x24\xc7\x86\xd2\x43\x55\x3d\xbd\xe2\x10\x1c\x7d\x85\xe7\x03\xd4\x3b\xda\xd7\xe5\x91\x23\xba\x4c\xf5\x17\xa8\x3b\x91\xe9\xc2\x62\xac\xe1\xf8\x5c\x15\xd8\xf2\x19\x8d\xcb\x49\x29\x2e\xb7\xb8\x7a\x7d\xf3\xaa\xde\x55\x2e\x77\xc4\xbd\xe7\xd2\xb9\x5a\xba\x11\xaf\xe4\x75\xb6\xe5\xc3\x81\x59\x09\x3a\x76\x57\x56\x9c\xcb\x4c\xda\xf2\x60\xfe\x97\xba\x60\xc1\x0b\xeb\x16\x72\xa2\x08\x0d\x82\xcd\xbe\xbd\xe6\x77\x0e\x47\x89\xc6\xe3\x40\xab\x99\x3f\x65\x44\x38\x35\xae\x0c\x1c\x06\x36\xec\xd5\x70\xb8\x05\xb9\xcc\x26\x45\x8f\xce\x78\xd2\x17\x89\xbb\xcf\xa6\x9a\x93\x59\x5f\xa0\x42\xe4\x11\x95\xe6\xf6\x4f\xc2\xb1\x68\x19\x70\x8a\x51\xea\xd7\x14\x97\xe7\x25\x93\xb2\xf5\xa4\x6b\xea\xe7\x90\xb6\x32\x20\xdf\xf4\xf5\xd6\x2c\x41\x62\x51\x4f\x8b\x7a\xc7\x49\xcf\xff\xce\x45\x57\x14\x51\x33\x8a\xcb\x03\x99\xc4\x6f\x4b\x2b\x7d\x1e\x2c\xc5\xdb\x56\x96\xe2\x39\x3b\xa7\x7e\x60\x0f\x3d\xf7\x68\xf7\x4a\xe9\x6c\x6f\x01\x80\x75\xd2\xec\xa0\xa5\x91\x9b\x2b\x87\xff\x9d\xa0\x70\x57\x87\x69\x2a\x91\x6e\xa6\xaf\x06\xbe\x89\x5f\x07\xf9\x36\x22\x3b\xb4\xec\x58\xf7\xf0\x47\x3c\x95\xcf\x51\x6a\xb6\xd4\x66\x47\x37\xb2\x0b\xe2\x55\x0a\xec\x6f\x00\x00\x00\xff\xff\x70\x30\xc1\x7f\xb2\x05\x00\x00"), + compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x94\xc1\x6e\xdb\x30\x0c\x86\xef\x7e\x0a\xc2\xa7\x16\x18\xfa\x00\x43\x73\x18\xb0\x6e\xd8\x65\xbb\xec\xb4\x8b\x4a\x5b\xb4\x43\x54\x16\x05\x89\x76\x9b\x06\x79\xf7\x41\x4e\x9c\xd4\x49\x5a\x78\x98\x6f\x36\xc4\xff\x23\xf9\xff\xd6\x80\x91\xb1\x72\x04\x65\x85\x89\x0c\x77\xd8\x92\x61\x5b\xc2\xb6\x00\xd0\x4d\x20\x38\x3c\x2b\x48\x1a\xd9\xb7\x05\x80\xa5\x54\x47\x0e\xca\xe2\x61\x05\xe5\xef\x35\x01\x5b\xf2\xca\x0d\x53\x04\x69\x40\xd7\x04\xdf\x1d\xfa\x9a\x60\x14\x84\x46\xe2\xf8\xb1\x12\xd1\xa4\x11\x03\x78\xb1\x74\x57\x16\xbb\xa2\x38\x75\x40\x2f\x1a\xd1\x28\xb6\x69\x8e\x5f\x41\x87\xe1\x66\x8f\xbf\x1d\xf9\x0d\xf6\x4e\x61\x05\xdb\x5d\x71\xd1\xcf\xfd\xfd\xc3\xaf\x6f\xc5\x8d\x8c\xef\xe8\x6e\xe1\x21\xcb\x42\x96\x05\x15\xa8\x08\x30\x04\xc7\x64\xf3\x5b\x1d\x09\x95\x2c\x44\x4a\xd2\xc7\x9a\xd2\x5d\x51\x3c\xbc\x60\x17\x1c\x7d\x86\xc7\x2d\x94\x4f\xb4\x29\xf3\x90\x03\xba\x9e\xca\x4f\x50\x36\x22\xe3\x87\x0a\x63\x09\xbb\xc7\x22\xc3\xe6\x63\xd4\xae\x4f\x4a\x71\xbe\xc5\xc5\xeb\x9b\x56\x75\x50\x39\xdf\x11\xb7\x9e\x73\xe5\x62\xe9\x5a\xbc\x92\xd7\xc9\x96\x93\x03\x93\x12\x34\xec\x2e\xac\x38\x1e\x33\x69\xcd\x9d\xf9\x5f\xea\x8c\x05\xcf\xac\x6b\xe8\x13\x45\xa8\x11\xaa\xde\xdb\x4b\x7e\xe3\x70\x90\x68\x3c\x76\xb4\x98\xf9\x53\x06\x84\x7d\xe1\xc2\xc0\x61\x60\xc3\x5e\x0d\x87\x6b\x90\xf3\x6c\x52\xf4\xe8\x8c\x27\x7d\x96\xf8\xf4\x5e\x57\x53\x32\xcb\x33\x54\x88\x3c\xa0\xd2\x54\xfe\x4e\x38\x66\x25\x1d\x8e\x31\x4a\xad\x61\x9b\x66\xa7\x1d\x27\x3d\xfe\x0f\xb3\x9a\x3c\x65\x32\xa9\xaf\x3c\xe9\x12\xc6\x94\x55\x2b\x1d\xf2\x55\x7b\xaf\xb5\x14\x24\xea\x3f\x34\x15\x45\xd4\x0c\xe2\xfa\x8e\x4c\xe2\xd7\xb9\xa3\xbe\xef\x2a\x8a\xd7\x1d\xcd\x87\xa7\x08\xed\xeb\x81\x3d\xb4\xdc\x62\xb5\x51\x4a\x47\x97\x33\x00\x2a\x27\xf5\x13\x58\x1a\xb8\xbe\x30\xfa\x6d\x07\x99\xbb\x38\x53\xe3\x11\x69\x26\xfa\x62\xe0\xab\xf8\x65\x90\x2f\x03\xb2\xc3\x8a\x1d\xeb\x06\xfe\x88\xa7\x7c\x2b\xa5\x7a\x4d\xb6\x77\x74\x25\xc2\x20\x5e\xe5\x3c\xc6\xd6\xf2\xfe\xb2\x7b\x13\xaf\x0f\xcc\xb9\x68\xe2\xc7\xd7\x94\x87\x3c\xe9\xc0\x41\x27\x7d\xf4\x23\xfd\x0d\x00\x00\xff\xff\xff\x49\x48\xc8\x3b\x06\x00\x00"), }, "/openstack/main.tf": &vfsgen۰CompressedFileInfo{ name: "main.tf", modTime: time.Time{}, - uncompressedSize: 4015, + uncompressedSize: 4078, - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x97\xdf\x8e\xeb\x26\x10\xc6\xef\x79\x8a\x51\xae\x5a\xa9\x4a\xab\xaa\xb7\xfb\x24\xab\x15\x22\x66\xd6\x4b\x63\x33\x16\x60\xef\xc9\x1e\x9d\x77\x3f\x02\x9b\x18\xe3\x3f\x21\xda\x5c\x9a\xdf\x07\x66\xe6\x9b\x19\xa7\x33\x34\x28\x89\x06\x4e\xd4\xa1\xb6\x4e\x54\xd7\x13\xfc\x64\x00\xa2\x77\x1f\xbc\x37\x0d\x24\xbf\x17\x18\x84\x39\xdf\x41\x5e\x19\x94\xa8\x9d\x12\x8d\xe5\x91\x67\x00\x15\x1a\x07\xd9\xef\x48\xea\x79\x2f\x6b\xa8\x97\xcf\xc8\x3c\xcf\x00\x24\xb5\x42\x69\xae\x64\xa1\xee\xce\xcf\x5a\x2d\x5a\x7c\x46\xeb\x79\x06\x80\x5a\x76\xa4\xb4\xe3\xee\xd6\x61\x81\x7a\xc1\x33\x00\xa5\x2d\x56\xbd\xc1\xd2\x1b\x47\x9e\x01\x5c\xf1\x96\x87\xf8\x50\x7a\xc5\x1b\x03\xe8\x84\xb5\x9f\x64\x64\xa9\x2a\xf2\x5e\x6a\xe8\x7f\xac\x1c\x4f\xa3\x7d\x28\xcd\xf9\xf5\x1e\x21\xea\x4f\xec\x31\x45\xdd\x60\xad\x48\x97\x5f\x7d\xe4\x19\x80\xfd\xf4\x26\x2d\x17\x8e\x3c\x03\x70\xa8\x85\x76\xe5\x0e\xbb\xf3\xb3\xb6\xd8\x61\x09\xef\xd5\x74\x45\xfd\x44\x9a\x03\xcf\x00\x7a\x8b\x9c\x2a\x27\x06\x25\x8a\x74\x09\x3f\xaa\x0d\xcf\xca\xea\x81\xda\x2c\x12\x9d\x3e\x88\x37\x2f\xdd\x60\xba\x79\x78\xa4\x64\xf1\xcd\x27\x3e\x2a\x17\xf1\x2e\x50\x86\x53\x7f\x31\xd6\x92\xec\x1b\x84\xd3\x85\xc8\x59\x67\x44\x37\x76\x43\x4b\xbd\xa9\xbc\x59\x4f\xe7\xbf\xe7\x25\x16\xba\x56\x6f\xdd\xfa\x55\xa7\x13\xe7\x55\xdf\x2e\x7e\x38\x23\xb8\x13\xb5\x3d\x4e\xe8\xcc\x31\x80\x8b\xb0\xc8\x55\x2b\x6a\xcc\xfd\x27\x85\x13\x89\x2a\x30\x76\x42\x87\x7f\xcf\xb3\xf0\x1c\x4e\x7f\x6f\xc4\x40\xab\xc0\xac\x4f\x6f\x45\x78\xe3\x04\xf7\x9d\xaa\xd6\xca\xad\xaa\x2e\x8a\xe3\x2a\xbf\x87\xc6\x0f\x90\x4e\x71\xdf\xeb\x54\x77\x7c\xdb\x99\x1b\x43\x84\x46\x8b\x86\x6b\x74\x9f\x64\xae\x07\x21\x5a\x70\x49\x22\x46\x1b\xed\x25\x62\x5c\x65\x00\x9a\x24\x5a\x6e\xfb\x8b\xc6\x45\x65\xbf\xc0\x68\x81\xb3\xa3\x8e\x1a\xaa\x6f\xe7\x8c\x0c\x7d\x4c\x0d\xc2\x61\x3c\x3c\xa9\x91\x5c\xbb\x26\x19\xc0\x14\x61\x5b\x67\xa6\x59\xcb\x53\xd2\x7b\x21\x86\x97\xdb\x0f\xd5\xf2\x7b\x52\xf2\xe8\xec\x70\xf3\xd1\x1d\x19\x7f\x15\xfb\xf8\xe8\x48\xfa\xbe\x4b\xe4\xf8\x40\x4d\xdf\x22\xb7\xea\x0b\x1f\x18\x28\xc7\xb3\x1d\x92\x91\x59\xb2\xc3\x34\x31\xbf\x48\x63\xee\xc1\x83\x1d\xc4\x20\x54\x23\x2e\xaa\x51\xee\xc6\xbd\xd6\xbe\xfe\xf3\x96\x96\xf9\xc8\xd9\x75\x91\xc7\x05\xb6\x51\x82\x4f\x97\x5e\xde\x23\x36\x7a\x43\x5e\x9d\x65\x55\xe9\x17\x75\x85\xbc\xa2\x5e\xbb\x28\x9a\xd0\xf0\x6c\x23\xe5\x25\xa9\x1e\xfb\xb1\x70\xc2\x7b\x67\x5d\xe6\xa3\x20\x73\xb2\xdf\xb9\x22\x5d\x09\xf7\x07\xf3\x37\xd8\x7c\xff\xb1\xb5\x8d\xfc\x5f\x01\x7b\x3d\xf2\xfc\x9b\x67\xfe\xdc\xf7\xde\xb7\xad\x57\xee\x3c\x8b\x66\x40\xc3\x6b\x43\x7d\x97\x34\xd1\x9d\x0d\x56\xb4\x6f\x88\x52\x86\xf0\xcd\x5d\x6b\x8a\x5a\xd6\x0f\x37\xb1\xc9\xfb\xf6\x1b\xd6\x4f\x7d\x1f\xa3\xbd\x36\xfe\x7d\x25\x0c\x37\x25\x0d\xbf\x34\x54\x5d\xf3\xf3\x5a\x51\x7d\x28\x8d\x7c\xf8\x8f\x7b\x28\xd4\xd5\xf6\x30\xdc\x30\xfb\x56\xab\xde\x69\xd3\x5b\x03\xa1\x60\x18\xe4\x8f\xfc\x1b\x3d\x92\xa5\xcd\xd9\xc6\x92\x5a\x5c\x79\x51\x57\xcd\x85\xbf\x37\x24\x9c\xd2\xf5\x7d\xc6\xe5\x47\x2c\x99\x50\xb1\xb5\x41\x6b\x17\xca\x5c\xb4\xc1\xec\x8c\xd3\xc3\x51\x1a\xb7\x39\x12\xcc\x4c\x1a\x33\xa9\xed\x9e\x20\x65\xfc\xe7\xa9\xe9\xf5\x95\xdb\xbe\xf3\xbd\x63\x47\xb2\x60\x18\xc0\xf4\x99\xb9\x50\xe5\x9a\x8c\x09\x69\x09\x7e\x5b\x4c\xec\xb5\xf3\x73\x26\x15\x1e\x38\x61\x83\x99\x9b\x5b\xda\xb2\xf6\x8a\x2d\x65\x7c\x99\xf9\xd6\x99\xfc\xa3\xce\x07\xc4\x09\x4e\xf3\x88\x18\x4b\x70\xf3\xbf\x50\x32\x78\xa6\xef\xd3\xdf\x01\x00\x00\xff\xff\x59\x3f\x5b\xe6\xaf\x0f\x00\x00"), + compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xbc\x97\xdf\x6e\xec\x26\x10\xc6\xef\x79\x8a\xd1\x5e\xb5\x52\xb5\xad\xaa\xde\xe6\x49\x8e\x22\xc4\x9a\x89\x43\xd7\x66\x2c\xc0\x4e\x37\x55\xdf\xbd\x02\xec\x35\xc6\x78\xc3\x2a\xd2\xf1\xa5\xf9\x7d\xfc\x19\xbe\x99\xb1\x07\x43\x93\x92\x68\xe0\x44\x03\x6a\xeb\x44\x73\x3d\xc1\xbf\x0c\x40\x8c\xee\x9d\x8f\xa6\x83\xe4\x79\x81\x49\x98\xf3\x1d\xe4\x8d\x41\x89\xda\x29\xd1\x59\xbe\xf0\x0c\xa0\x41\xe3\x20\x7b\x1e\x49\x3d\xef\x65\x1d\x8d\xf2\x19\x99\xe7\x19\x80\xa4\x5e\x28\xcd\x95\xac\xd4\xdd\xf9\x55\xab\x45\x8f\xcf\x68\x3d\xcf\x00\x50\xcb\x81\x94\x76\xdc\xdd\x06\xac\x50\x6f\x78\x06\xa0\xb4\xc5\x66\x34\x58\x7b\xe2\x85\x67\x00\x57\xbc\xe5\x21\x7e\x28\xbd\xe2\x8d\x01\x0c\xc2\xda\x0f\x32\xb2\x56\xb5\xf0\x5e\x6a\xe8\x6f\x6c\x1c\x4f\xa3\xfd\x50\x9a\xf3\xfb\x39\x42\xd4\x9f\x98\x63\x8e\xba\xc1\x56\x91\xae\x3f\x7a\xe4\x19\x80\xfd\xf0\x26\xad\x17\x46\x9e\x01\x38\xd4\x42\xbb\x7a\x87\xdd\xf9\x55\x5b\xed\xb0\x84\xf7\x6a\xba\xa2\x7e\xe2\x9a\x03\xcf\x00\x46\x8b\x9c\x1a\x27\x26\x25\xaa\x74\x09\x1f\xd5\x86\x67\x69\xf5\x85\xda\x6c\x2e\x3a\x7d\xb1\x9c\xbc\x76\x82\xf9\xe4\xe1\x95\x92\xd5\x27\x9f\xf9\x45\xb9\x89\x77\x85\x32\xac\xfa\x1f\x63\x3d\xc9\xb1\x43\x38\x5d\x88\x9c\x75\x46\x0c\xb1\x1a\x5a\x1a\x4d\xe3\xcd\x7a\x3a\xff\xbe\x0e\xb1\x50\xb5\x46\xeb\xb6\x5b\x8d\x8b\xad\x03\xbe\x52\xfc\xe3\x8c\xe0\x4e\xb4\xf6\x70\x4b\x2b\xc2\x00\x2e\xc2\x22\x57\xbd\x68\x31\xb9\x00\x29\x9c\x48\x04\x61\xd8\xce\xd4\xf4\xe7\x79\xd5\x9c\xc3\x9a\x6f\x9d\x98\x68\x1b\x89\x7c\xcd\x5e\x84\x2d\x26\xa4\xaf\x4a\xad\x56\x6e\x9b\x61\x51\xb7\x0c\xf0\x7b\x04\x7c\x9f\x18\x14\xf7\x25\x4d\x0d\x87\x27\x5b\x91\x18\x09\x34\x5a\x74\x5c\xa3\xfb\x20\x73\x2d\x47\x62\x83\x24\x51\x8e\x1e\x29\x44\x39\x0e\x30\x00\x4d\x12\x2d\xb7\xe3\x45\xe3\x92\xb1\x2f\x10\x6f\xf5\xec\x68\xa0\x8e\xda\xdb\x39\x83\x42\x69\x52\x93\x70\xb8\x2c\xe9\x95\x7b\xd9\x1e\x62\x00\x73\x0c\x6d\xcb\x95\xb4\xf0\x02\x0d\xe9\x46\xb8\x5f\x98\x5f\xb9\x18\xed\x78\xd1\x91\xff\x2d\x60\x3f\xf2\x85\xd2\x39\x5f\x3d\xf3\xab\xf7\xc4\x12\x75\x6e\xdf\x55\xcf\xef\xd7\x94\x87\xef\x80\x5b\xb7\x3a\x90\x71\x61\xb3\xeb\x85\x1d\x6c\x60\x21\x7d\xd5\x25\x72\x7c\xa2\x6e\xec\x91\x5b\xf5\xf9\x95\xa5\x72\x3c\x9b\x21\x69\x98\x35\x33\xcc\xfd\xf2\x93\x34\x42\xe1\x39\x98\x41\x4c\x42\x75\xe2\xa2\x3a\xe5\x6e\xdc\x6b\xed\x8f\x3f\x5e\xbd\x65\xa5\x0c\x11\x59\x1d\x16\xa3\xb1\xf3\x6d\x91\x4b\xab\x44\x5c\xc8\xee\x6b\xc4\x32\xc0\x0a\xb9\xfc\x74\x22\xe7\x25\xa6\x50\x5f\xf2\x5c\xaf\x4b\x74\x3f\xa8\x1b\xe4\x0d\x8d\xda\x2d\xa2\x19\x0d\xef\x0a\x9e\xa9\xf1\x4a\x2c\xe7\xc2\x09\x6f\xbe\x7d\xf9\x88\x82\x9f\x99\x3a\x07\xe6\xfd\xb6\x77\xeb\xad\x6b\xd1\x4c\x68\x78\x6b\x68\x1c\x92\x92\x7c\x30\xc1\x8e\x3e\x76\x6d\xa5\x69\x63\xf2\xd8\x6f\xe4\x4e\xea\xfb\x25\xda\x7b\xe3\xdf\x47\x42\x6f\x54\xd2\xf0\x4b\x47\xcd\x35\x5f\xaf\x17\xcd\xbb\xd2\xc8\xa7\xbf\xb8\x87\xe6\xc4\x2c\xf4\xd2\x92\xd9\x0b\xcd\xe0\xa8\x1b\x14\xba\x4d\x4d\xbb\xc9\x5f\xc5\x76\xf0\x58\x96\x76\x03\xbb\xa4\xd4\xe6\xc8\x9b\xbc\xea\x2e\xfc\xad\x23\xe1\x94\x6e\xef\xbd\x33\x5f\x62\xcb\x84\x8c\x6d\x0d\x5a\xbb\x51\xe6\xa2\x02\x53\x6e\xd3\x8f\xfb\xf4\x32\xcd\x23\xc1\xca\xa4\x31\x93\xda\x1e\x09\x52\xc6\x7f\xdd\x9a\x51\x5f\xb9\x1d\x07\x5f\x3b\x0e\x24\x1b\x86\x01\xcc\x5f\xa9\x1b\x55\xae\xc9\x98\x70\x2d\xc1\x6f\x9b\x0f\x83\xbd\xf3\x73\x26\x15\x3e\x70\x42\x81\x59\x8b\x5b\x5a\xb2\x8e\x92\x2d\x65\x7c\x9a\xf9\xd2\x99\xfc\x90\xe7\x0d\xe2\x04\xa7\xb5\x45\xc4\x14\x2c\xfe\x4a\x25\x8d\x67\xfe\xbc\xfd\x3f\x00\x00\xff\xff\x60\x4b\x43\x8d\xee\x0f\x00\x00"), }, "/openstack/masters": &vfsgen۰DirInfo{ name: "masters", @@ -1458,9 +1470,9 @@ var Assets = func() http.FileSystem { "/openstack/masters/main.tf": &vfsgen۰CompressedFileInfo{ name: "main.tf", modTime: time.Time{}, - uncompressedSize: 2164, + uncompressedSize: 5357, - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x55\xdb\x6e\xe3\x36\x10\x7d\xd7\x57\x0c\x88\x14\x48\x8a\xb5\x1c\x6c\x83\x45\x91\x5d\xa1\x0f\xbd\x00\x7d\x69\x3f\x20\x30\x08\x86\x1c\xd9\x44\x28\x52\x20\x47\xea\x66\x03\xff\x7b\xc1\x8b\x14\x69\xed\xc4\x7d\xe8\x9b\x3d\x9c\xeb\x39\x73\x46\x4a\x90\x00\xe6\x7a\xb4\x81\x84\x7c\xe2\xd2\x75\xfd\x40\xc8\x5b\x23\x46\xe7\xf9\xf8\x91\x01\xeb\x44\x20\xf4\xa1\xd8\x18\xbc\x54\x00\x56\x74\x08\x0d\x8c\xc2\xd7\xc5\x35\x5a\xaa\x63\x55\xe5\x8c\x7a\x6f\x35\x69\x67\x79\xab\x0d\x32\x60\x07\x17\x28\x7a\xe4\x68\xe9\x06\x4b\x25\x5c\xc7\xca\x56\x22\x4f\xc6\x0a\xa0\x73\x0a\x01\x1a\x60\x77\x1f\x6f\x19\x6c\xb7\x70\xfb\xe9\xee\xae\x02\xe8\x05\x1d\x92\x7d\x8b\x24\xb7\x73\xc2\x2a\xe5\xb3\x84\x96\x52\xee\xd7\x7f\x0d\x7c\xf9\xf2\xfb\xdf\x7f\x54\x57\x2f\xb1\x8e\x34\x43\x1c\x83\x6b\x75\xdc\xe4\x89\x36\x57\x2f\xa9\x66\xad\xad\xc2\xaf\xc7\x2a\xfa\x02\x1c\xcf\x0c\x21\x9d\x6d\xf5\x7e\x86\x82\x9f\x3c\x5c\x18\x2a\x4e\x85\x7e\x8f\xa5\xc1\xe0\x06\x2f\x23\x7c\x2c\xd6\xb9\x27\xfc\x4a\xdb\xde\x08\x6d\x3f\xcb\x83\xf0\x01\xa9\x19\xa8\xdd\xfc\xfc\xf9\x51\x04\xfc\x74\xf7\xe1\xea\x25\xff\x40\x2b\x9d\xc2\xeb\x98\x7f\x08\xe8\x79\x0c\x8e\xad\xdc\x1c\x59\xea\xbb\x02\x88\x68\x07\x68\xe0\x21\xd5\x41\x83\x1d\x5a\xba\x8e\x8e\xf5\x8a\x91\x7a\x82\xaf\xfe\xb1\xf6\x68\x15\x7a\x54\x1f\x60\x81\xc6\x4d\x05\xb0\x8b\x48\x78\x2c\xdd\x2e\x96\xe4\xd1\x38\xf9\x14\xc8\x79\xb1\x47\x3e\x3a\x33\x74\xc8\xc7\x9f\x5e\xe1\xc9\xa6\xd5\xa2\xb0\xff\x48\x02\xfb\x0e\x48\xef\x1c\x4d\x25\x82\xfe\x86\xd0\x34\x60\x07\x63\xe0\x17\xb8\x85\xfb\x37\xb0\xce\x8e\x67\xe3\x2b\x80\xf2\x8f\x9e\xfb\x73\x4e\xd1\x5c\x01\xe8\x2e\xce\xa6\x55\xf1\x88\x04\xf0\xc9\xf6\x16\x2c\x93\x76\x02\xfa\x11\xfd\xde\xbb\xa1\x5f\x0a\x88\x27\xcb\x89\x7c\xb2\x77\x7e\xcc\x22\x02\xe8\x9d\xd1\x52\x67\x26\x59\x70\x2d\x6d\x84\x25\xbd\x11\x6d\xab\xad\xa6\x67\xf6\x26\x33\x53\x0b\x33\x26\xcb\xfa\x71\x59\xff\x17\x52\x4e\x11\x2f\x37\x20\xe1\x95\xb6\xed\x9d\x8b\x52\xaf\xef\x49\xad\xd5\x29\xde\xef\xd0\x7e\x42\x07\xdc\xa7\xa7\xc8\x3b\xca\xc1\x6b\x7a\xce\x60\x86\x92\xab\x4c\x1f\xf6\x5c\xab\x50\x01\x88\x51\x68\x23\x1e\xb5\x89\x9e\xdf\x9c\x9d\x88\x88\x3f\xc3\xc3\x62\x6e\xf8\x01\x0c\xda\x3d\x1d\xae\xe7\xe7\x9b\x5d\x05\x30\x8b\x0f\x9a\x59\x62\x49\x6f\x6b\x9d\xe5\xdb\x50\x9f\x3f\x19\x4b\xdd\x95\xa3\x35\xd7\x8d\x86\x9b\x88\xaa\x7a\xb6\xa2\xd3\x12\x92\xde\xb8\xc2\x51\xcb\xe9\x84\xb4\xce\x73\x14\xf2\x70\x19\xaf\x87\x1d\xdc\xc3\xc3\x25\xf1\xd6\x2b\xe9\x2e\x51\xd8\xd5\x5a\xed\x56\x77\x35\x77\x00\x30\x0c\x89\xaf\x65\x77\xf5\x28\xcc\x80\xe5\x3d\xaf\xe7\x24\x34\x56\xae\x42\x79\x7c\x8c\x1d\x67\x98\x1b\xb8\x2d\x46\x85\x81\xb4\x15\x09\xa7\xf3\x61\x0a\x0d\x12\xf2\xf8\x8e\xbe\x2b\xbe\xd0\x00\xf9\x52\xf6\x38\x9d\x42\x8b\xf4\x8f\xf3\x4f\xa5\xdb\xde\x79\x5a\xef\x43\xb4\xc4\x8d\x58\xcd\x3a\x05\x07\x79\x40\x35\x18\xf4\xfc\xa0\x2d\x85\x92\x24\xad\x15\x34\x70\x49\xf2\xf5\x52\xf0\x79\xbf\x8f\x4b\x3e\xd7\xad\x7d\x47\xa5\x50\x2a\x2d\x8a\x30\xbc\xf8\xa5\xbd\x7d\x8f\x81\xe2\xb7\x00\x7f\x46\x81\xc4\x3e\x5f\x91\xd4\xf3\x41\xb7\xf4\x6b\x56\xfb\x9f\xbf\x35\x27\xea\x67\xbb\xfc\xa5\x22\x51\xb6\x3b\xd7\xf9\xeb\xc2\xb5\xc8\xdc\x9c\x16\x28\x03\xbd\x06\x94\xaf\xeb\xbf\x01\x00\x00\xff\xff\x8d\x78\x4c\x9b\x74\x08\x00\x00"), + compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x57\x5b\x8b\x1b\xb9\x12\x7e\xef\x5f\x51\xb4\x73\x38\x99\xc3\xb8\xdb\xf6\x19\xc2\x32\x89\xd9\x87\xec\x2e\xec\xcb\xe6\x61\x03\xfb\x60\x4c\x23\x4b\xd5\x6e\x31\x6a\xa9\x23\x55\x7b\xe2\x0c\xf3\xdf\x97\x92\xda\xb7\xcc\x95\x85\x0d\x61\xc8\x93\xed\x52\xa9\x2e\x5f\x55\x7d\x25\x2b\x41\x02\x72\xd7\xa1\x0d\x24\xe4\x55\x25\x5d\xdb\xf5\x84\x55\x6d\xc4\xc6\xf9\x6a\x33\xcb\x21\x6f\x45\x20\xf4\x61\x90\xe5\x70\x93\x01\x58\xd1\x22\xcc\x61\x23\x7c\x31\xa8\xb2\x24\xbb\xcd\xb2\x64\x51\xaf\xad\x26\xed\x6c\x55\x6b\x83\x39\xe4\x8d\x0b\xc4\x1a\xe9\xb6\x74\xbd\xa5\xe1\xba\x66\xcf\x56\x62\x15\x85\x19\x40\xeb\x14\x02\xcc\x21\xbf\x98\x4d\x72\x28\x4b\x98\xbc\xb9\xb8\xc8\x00\x3a\x41\x4d\x94\x97\x48\xb2\xdc\x1b\xcc\xa2\x3d\x4b\x68\x29\xda\x3e\xfc\x9a\xc3\xbb\x77\xbf\x7e\xf8\x2d\x7b\x75\xc3\x7e\xa4\xe9\x39\x8d\x4a\xab\xdb\x71\xca\x68\xfc\xea\x26\xfa\x2c\xb4\x55\xf8\xf9\x36\x63\x5d\x80\xdb\x7b\x92\x90\xce\xd6\x7a\xbd\x87\xa2\xba\x73\xf0\x44\x52\x9c\x15\xfa\x35\x0e\x01\x06\xd7\x7b\xc9\xf0\xe5\xec\xe7\x92\xf0\x33\x95\x9d\x11\xda\xbe\x95\x8d\xf0\x01\x69\xde\x53\x3d\xfe\xe9\xed\x4a\x04\x7c\x73\x71\xfe\xea\x26\x7d\x41\x2b\x9d\xc2\xd7\x6c\xbf\x0f\xe8\x2b\xbe\xcc\xa1\x9c\xdd\xe6\x31\xee\x0c\x80\xd1\x0e\x30\x87\x45\xf4\x83\x06\x5b\xb4\xf4\x9a\x15\x8b\x93\x8a\x14\x3b\xf8\x8a\xff\x15\x1e\xad\x42\x8f\xea\x1c\x8e\xd0\x38\xcb\x00\x96\x8c\x84\xc7\x21\xda\xa3\x26\x59\x19\x27\xaf\x02\x39\x2f\xd6\x58\x6d\x9c\xe9\x5b\xac\x36\xff\x3f\xc0\x93\x44\x27\x8d\x92\x3f\xb3\x08\xf9\x57\x40\x7a\xe7\x68\xe7\x22\xe8\x2f\x08\xf3\x39\xd8\xde\x18\xf8\x19\x26\x70\xf9\x00\xd6\x49\xf1\xde\xfb\x19\xc0\xf0\x8b\xb6\xdd\x7d\x4a\x2c\xce\x00\x74\xcb\xb9\x69\x35\x68\x70\x01\xaa\x9d\xec\x21\x58\x76\xb3\x13\xd0\x6f\xd0\xaf\xbd\xeb\xbb\xe3\x01\xaa\xa2\xe4\xce\xf8\x24\xed\x74\x98\x86\x08\xa0\x73\x46\x4b\x9d\x2a\x99\x07\x57\xd3\x58\x58\xd2\x63\x51\xd7\xda\x6a\xda\xe6\xb1\x32\x23\xf8\xd8\x20\x24\xdb\x90\xac\x04\x10\x1e\x41\x7a\x14\x84\x0a\xb4\x05\x6a\x3c\x22\x04\xec\x84\x17\x84\xb0\x8f\x5a\x61\xb4\xa4\x9d\x85\x54\xcc\xf3\x6c\x04\x5e\x50\x83\x1e\xa8\x11\x16\x04\x04\x6d\xd7\x06\xd3\x31\x5c\x6b\x6a\x40\x40\x1e\x21\xce\xa1\x45\x12\xe3\xce\xbb\x0e\x3d\x6d\xcf\x61\x85\x52\xf4\x01\x81\x1a\xdc\x82\x45\x54\x40\x2e\x1b\xc1\xea\x10\x4a\xc0\x4f\x3d\x5a\xd2\xc2\x98\xed\x89\x1f\xe9\xac\xec\xbd\x47\x4b\x66\x0b\xab\x2d\x7c\x44\xef\x45\xed\x7c\x5b\x64\xa3\x21\x43\x8f\x22\x38\x0b\xd7\xcd\xf6\xc4\xc1\xb1\x79\x67\x11\x04\x81\x00\xd2\x2d\x82\x0e\x6c\x9b\xe0\x43\x87\xf6\x4f\xae\xcc\x7f\x43\x36\x82\xf7\xa9\x3a\x4c\x2f\xbd\x89\x4a\x07\xc7\xbd\x15\x2b\x83\x6c\xb4\x71\xd6\xf5\x1c\x1b\xc2\x7d\xc0\xa7\xca\x6c\xb3\x11\x5c\x37\x68\xa3\xda\x7d\xc8\x1f\x67\x95\x12\xf9\x0b\x41\x36\x2e\x44\x1f\xbd\xf5\xce\x98\x78\xd9\x38\xd7\x81\xb6\xe4\x86\x4a\xed\x7a\x39\x1c\x83\x1a\x0f\x38\x27\xcc\x46\xd0\x6a\xab\xdb\xbe\x05\xdb\xb7\x2b\xf4\xe0\x6a\xf0\xf8\xa9\xd7\x1e\x15\xbc\x77\x96\xbc\x33\xd0\x19\x61\x11\xac\x53\x6c\x46\x04\x08\x74\x68\x07\x36\xa1\x9c\xec\x99\x17\x04\xd7\x7f\x31\x5d\x1e\xa0\xc6\xcf\x1d\xca\x24\xdf\xa3\xd8\x0a\xd9\x68\x8b\x63\xd1\xe9\x31\x17\x5c\x90\xf3\x70\xad\x39\x01\x71\x85\x20\x39\x6f\x57\xa7\xd4\xb5\x5d\x83\xb0\x0c\x8f\x8b\x25\xe6\xd0\x30\xb0\xf3\x18\x4d\x0c\xc6\x39\xcb\x9f\x9c\xbd\xc2\xce\xb8\x2d\x87\xc2\xde\xb0\xae\x51\x92\xde\xe0\x39\x08\xab\x92\x73\x91\x8d\x20\xe8\x56\x1b\xe1\xe1\xda\xf9\x2b\xe1\x5d\x6f\x15\x6b\x8b\xae\x33\x1a\x15\xd4\xce\x83\x12\xdb\xf1\x0c\x52\x70\xda\xd9\x90\x12\x5a\x4c\x97\x97\xd0\x10\x75\xe1\xb2\x2c\xd7\x9a\x9a\x7e\x55\x48\xd7\x96\x71\x60\x1b\x5d\x53\x19\xd1\x36\x06\x7d\x49\x1e\xb1\x4c\xa3\x54\x2a\x27\x43\xc9\x04\x5b\xee\x47\x7b\x34\x70\x55\xcc\xe2\xd1\xd9\xdf\x93\xd1\xf1\xe0\xf3\x96\xa8\x26\xcf\xe4\xc3\x49\xdc\x68\xc3\x52\x8d\x04\x14\xe9\xfb\x91\x15\x5d\x9c\x2e\xe8\x42\xab\xbb\x04\xf6\x08\x8f\xde\xe1\x37\xb8\x8c\x47\x4c\xa4\x28\x7b\xaf\x69\x9b\xd8\x29\x0c\xb6\x86\xac\xc2\xba\xd2\x2a\x64\x00\x62\x23\xb4\x11\x2b\x6d\x58\xf3\x0b\xcf\x62\xd2\xe3\xaf\x61\x31\x81\xff\x80\x41\xbb\xa6\xe6\xf5\x5e\x78\xb6\xcc\x00\xf6\x3b\x0c\xe6\xfb\x4d\x15\xd7\xd6\xe9\xba\x4a\x2b\xb6\xb8\x7f\xf3\x1e\xaf\xaf\x78\x77\xc2\x1f\x67\x8c\xa0\xda\x5a\xd1\x6a\x99\x08\xac\x52\xb8\xd1\x72\xb7\x7f\x6b\xe7\x2b\x14\xb2\x79\x1a\x9b\xc5\x12\x2e\x61\xf1\xd4\xe6\x2b\x4e\xf6\xde\x62\xb2\x2c\xb4\x5a\x9e\x3c\x45\x92\x5f\x80\xbe\x8f\x15\x39\x8e\xa9\xd8\x08\xd3\xe3\x70\x9e\x1a\x6b\xb7\x9b\xf2\x61\x91\x0e\x87\x2b\x8e\x33\xae\x49\x98\xc3\x64\x10\x2a\x0c\xa4\x6d\xec\xfb\x07\xae\x29\x34\x48\x58\xf1\x39\xfa\x76\xd0\x85\x39\x90\x1f\xdc\xde\xee\x5e\x0f\x16\x89\x87\x6c\x88\xb6\x73\x9e\x4e\x2b\xce\x12\xae\xf9\x62\xb2\xdc\x5d\x09\xb2\x41\xe6\x53\x5f\x35\xda\x52\x18\xae\xc6\x76\x81\x39\x3c\xb5\x1b\x8b\xe3\xcd\x98\xfa\xf6\xf6\xb8\x76\xa7\x01\x7d\x55\x36\xa1\x54\x6c\x05\x61\xaa\x41\x2f\xf6\xe3\x63\xb8\x0f\x7a\x47\x90\xef\x73\x27\xb1\x4e\xeb\x76\x4f\x0f\xef\xd3\x6c\xfe\xfe\xcb\xfc\xce\xac\xe6\xcb\xf4\xa4\x23\x31\xf4\x6f\xf2\xf3\xc7\x13\xb3\x9d\x2a\x72\xd7\xc1\x90\xd0\xe1\xc2\xee\x19\xfa\xcf\x98\x66\xfa\x4c\xa6\x99\xbe\x30\xa6\x99\x7e\x53\xa6\x99\x7e\x07\x4c\x33\x7d\xf1\x4c\x33\xfd\xc1\x34\xff\x3e\xd3\x30\x0e\xd8\xa1\x55\xa1\x8a\xf5\x5a\x3c\xca\x36\xc5\xc9\xab\xe6\xc1\x3f\x89\x4f\xf3\xd4\xec\x99\x3c\x35\x7b\x61\x3c\x35\xfb\xa6\x3c\x35\xfb\x0e\x78\x6a\xf6\xe2\x79\x6a\xf6\x83\xa7\xbe\x6b\x9e\x9a\x32\x4f\xfd\x1d\x00\x00\xff\xff\x1a\xaa\xc4\x1d\xed\x14\x00\x00"), }, "/openstack/masters/variables.tf": &vfsgen۰CompressedFileInfo{ name: "variables.tf", @@ -2097,6 +2109,8 @@ var Assets = func() http.FileSystem { } fs["/manifests/bootkube"].(*vfsgen۰DirInfo).entries = []os.FileInfo{ fs["/manifests/bootkube/04-openshift-machine-config-operator.yaml"].(os.FileInfo), + fs["/manifests/bootkube/aro-ingress-namespace.yaml"].(os.FileInfo), + fs["/manifests/bootkube/aro-ingress-service.yaml.template"].(os.FileInfo), fs["/manifests/bootkube/aro-worker-registries.yaml.template"].(os.FileInfo), fs["/manifests/bootkube/cvo-overrides.yaml.template"].(os.FileInfo), fs["/manifests/bootkube/etcd-ca-bundle-configmap.yaml.template"].(os.FileInfo), diff --git a/vendor/github.com/openshift/installer/pkg/aro/dnsmasq/dnsmasq.go b/vendor/github.com/openshift/installer/pkg/aro/dnsmasq/dnsmasq.go new file mode 100644 index 00000000000..2f920a0962f --- /dev/null +++ b/vendor/github.com/openshift/installer/pkg/aro/dnsmasq/dnsmasq.go @@ -0,0 +1,136 @@ +package dnsmasq + +import ( + "bytes" + "fmt" + "text/template" + + ignutil "github.com/coreos/ignition/v2/config/util" + igntypes "github.com/coreos/ignition/v2/config/v3_1/types" + "github.com/openshift/installer/pkg/asset/ignition" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var t = template.Must(template.New("").Parse(` + +{{ define "dnsmasq.conf" }} +resolv-file=/etc/resolv.conf.dnsmasq +strict-order +address=/api.{{ .ClusterDomain }}/{{ .APIIntIP }} +address=/api-int.{{ .ClusterDomain }}/{{ .APIIntIP }} +address=/.apps.{{ .ClusterDomain }}/{{ .IngressIP }} +user=dnsmasq +group=dnsmasq +no-hosts +cache-size=0 +{{ end }} + +{{ define "dnsmasq.service" }} +[Unit] +Description=DNS caching server. +After=network-online.target +Before=bootkube.service + +[Service] +ExecStartPre=/bin/cp /etc/resolv.conf /etc/resolv.conf.dnsmasq +ExecStartPre=/bin/bash -c '/bin/sed -ni -e "/^nameserver /!p; \\$$a nameserver $$(hostname -I)" /etc/resolv.conf' +ExecStart=/usr/sbin/dnsmasq -k +ExecStop=/bin/mv /etc/resolv.conf.dnsmasq /etc/resolv.conf +Restart=always + +[Install] +WantedBy=multi-user.target +{{ end }} + +`)) + +func config(clusterDomain, apiIntIP, ingressIP string) ([]byte, error) { + buf := &bytes.Buffer{} + + err := t.ExecuteTemplate(buf, "dnsmasq.conf", &struct { + ClusterDomain string + APIIntIP string + IngressIP string + }{ + ClusterDomain: clusterDomain, + APIIntIP: apiIntIP, + IngressIP: ingressIP, + }) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func service() (string, error) { + buf := &bytes.Buffer{} + + err := t.ExecuteTemplate(buf, "dnsmasq.service", nil) + if err != nil { + return "", err + } + + return buf.String(), nil +} + +func IgnitionConfig(clusterDomain, apiIntIP, ingressIP string) (*igntypes.Config, error) { + service, err := service() + if err != nil { + return nil, err + } + + config, err := config(clusterDomain, apiIntIP, ingressIP) + if err != nil { + return nil, err + } + + return &igntypes.Config{ + Ignition: igntypes.Ignition{ + Version: igntypes.MaxVersion.String(), + }, + Storage: igntypes.Storage{ + Files: []igntypes.File{ + ignition.FileFromBytes("/etc/dnsmasq.conf", "root", 0644, config), + }, + }, + Systemd: igntypes.Systemd{ + Units: []igntypes.Unit{ + { + Contents: &service, + Enabled: ignutil.BoolToPtr(true), + Name: "dnsmasq.service", + }, + }, + }, + }, nil +} + +func MachineConfig(clusterDomain, apiIntIP, ingressIP, role string) (*mcfgv1.MachineConfig, error) { + ignConfig, err := IgnitionConfig(clusterDomain, apiIntIP, ingressIP) + if err != nil { + return nil, err + } + + rawExt, err := ignition.ConvertToRawExtension(*ignConfig) + if err != nil { + return nil, err + } + + return &mcfgv1.MachineConfig{ + TypeMeta: metav1.TypeMeta{ + APIVersion: mcfgv1.SchemeGroupVersion.String(), + Kind: "MachineConfig", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("99-%s-aro-dns", role), + Labels: map[string]string{ + "machineconfiguration.openshift.io/role": role, + }, + }, + Spec: mcfgv1.MachineConfigSpec{ + Config: rawExt, + }, + }, nil +} diff --git a/vendor/github.com/openshift/installer/pkg/asset/ignition/bootstrap/bootstrap.go b/vendor/github.com/openshift/installer/pkg/asset/ignition/bootstrap/bootstrap.go index 9d61a20df10..6d2bdbafd23 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/vendor/github.com/openshift/installer/pkg/asset/ignition/bootstrap/bootstrap.go @@ -20,6 +20,7 @@ import ( "github.com/sirupsen/logrus" "github.com/openshift/installer/data" + "github.com/openshift/installer/pkg/aro/dnsmasq" "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/bootstraplogging" "github.com/openshift/installer/pkg/asset/ignition" @@ -31,6 +32,7 @@ import ( "github.com/openshift/installer/pkg/asset/manifests" "github.com/openshift/installer/pkg/asset/releaseimage" "github.com/openshift/installer/pkg/asset/rhcos" + "github.com/openshift/installer/pkg/asset/templates/content/bootkube" "github.com/openshift/installer/pkg/asset/tls" "github.com/openshift/installer/pkg/types" baremetaltypes "github.com/openshift/installer/pkg/types/baremetal" @@ -130,6 +132,7 @@ func (a *Bootstrap) Dependencies() []asset.Asset { &tls.ServiceAccountKeyPair{}, &releaseimage.Image{}, new(rhcos.Image), + &bootkube.ARODNSConfig{}, } } @@ -141,7 +144,8 @@ func (a *Bootstrap) Generate(dependencies asset.Parents) error { releaseImage := &releaseimage.Image{} rhcosImage := new(rhcos.Image) bootstrapSSHKeyPair := &tls.BootstrapSSHKeyPair{} - dependencies.Get(installConfig, proxy, releaseImage, rhcosImage, bootstrapSSHKeyPair, loggingConfig) + aroDNSConfig := &bootkube.ARODNSConfig{} + dependencies.Get(installConfig, proxy, releaseImage, rhcosImage, bootstrapSSHKeyPair, loggingConfig, aroDNSConfig) templateData, err := a.getTemplateData(installConfig.Config, releaseImage.PullSpec, installConfig.Config.ImageContentSources, proxy.Config, rhcosImage, loggingConfig) @@ -196,6 +200,14 @@ func (a *Bootstrap) Generate(dependencies asset.Parents) error { }}, ) + dnsmasqIgnConfig, err := dnsmasq.IgnitionConfig(installConfig.Config.ClusterDomain(), aroDNSConfig.APIIntIP, aroDNSConfig.IngressIP) + if err != nil { + return err + } + + a.Config.Storage.Files = append(a.Config.Storage.Files, dnsmasqIgnConfig.Storage.Files...) + a.Config.Systemd.Units = append(a.Config.Systemd.Units, dnsmasqIgnConfig.Systemd.Units...) + data, err := ignition.Marshal(a.Config) if err != nil { return errors.Wrap(err, "failed to Marshal Ignition config") diff --git a/vendor/github.com/openshift/installer/pkg/asset/ignition/machine/master.go b/vendor/github.com/openshift/installer/pkg/asset/ignition/machine/master.go index f6156e843ff..63c37dc4d8d 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/ignition/machine/master.go +++ b/vendor/github.com/openshift/installer/pkg/asset/ignition/machine/master.go @@ -10,6 +10,7 @@ import ( "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/ignition" "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/asset/templates/content/bootkube" "github.com/openshift/installer/pkg/asset/tls" ) @@ -30,6 +31,7 @@ func (a *Master) Dependencies() []asset.Asset { return []asset.Asset{ &installconfig.InstallConfig{}, &tls.RootCA{}, + &bootkube.ARODNSConfig{}, } } @@ -37,9 +39,10 @@ func (a *Master) Dependencies() []asset.Asset { func (a *Master) Generate(dependencies asset.Parents) error { installConfig := &installconfig.InstallConfig{} rootCA := &tls.RootCA{} - dependencies.Get(installConfig, rootCA) + aroDNSConfig := &bootkube.ARODNSConfig{} + dependencies.Get(installConfig, rootCA, aroDNSConfig) - a.Config = pointerIgnitionConfig(installConfig.Config, rootCA.Cert(), "master") + a.Config = pointerIgnitionConfig(installConfig.Config, aroDNSConfig, rootCA.Cert(), "master") data, err := ignition.Marshal(a.Config) if err != nil { diff --git a/vendor/github.com/openshift/installer/pkg/asset/ignition/machine/node.go b/vendor/github.com/openshift/installer/pkg/asset/ignition/machine/node.go index 1ebda003e28..686264f1628 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/ignition/machine/node.go +++ b/vendor/github.com/openshift/installer/pkg/asset/ignition/machine/node.go @@ -9,7 +9,9 @@ import ( igntypes "github.com/coreos/ignition/v2/config/v3_1/types" "github.com/vincent-petithory/dataurl" + "github.com/openshift/installer/pkg/asset/templates/content/bootkube" "github.com/openshift/installer/pkg/types" + azuretypes "github.com/openshift/installer/pkg/types/azure" baremetaltypes "github.com/openshift/installer/pkg/types/baremetal" openstacktypes "github.com/openshift/installer/pkg/types/openstack" ovirttypes "github.com/openshift/installer/pkg/types/ovirt" @@ -18,12 +20,16 @@ import ( // pointerIgnitionConfig generates a config which references the remote config // served by the machine config server. -func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, role string) *igntypes.Config { +func pointerIgnitionConfig(installConfig *types.InstallConfig, aroDNSConfig *bootkube.ARODNSConfig, rootCA []byte, role string) *igntypes.Config { var ignitionHost string // Default platform independent ignitionHost ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain()) // Update ignitionHost as necessary for platform switch installConfig.Platform.Name() { + case azuretypes.Name: + if installConfig.Azure.ARO { + ignitionHost = net.JoinHostPort(aroDNSConfig.APIIntIP, "22623") + } case baremetaltypes.Name: // Baremetal needs to point directly at the VIP because we don't have a // way to configure DNS before Ignition runs. diff --git a/vendor/github.com/openshift/installer/pkg/asset/ignition/machine/worker.go b/vendor/github.com/openshift/installer/pkg/asset/ignition/machine/worker.go index a1904aa038f..1919be3430c 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/ignition/machine/worker.go +++ b/vendor/github.com/openshift/installer/pkg/asset/ignition/machine/worker.go @@ -10,6 +10,7 @@ import ( "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/ignition" "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/asset/templates/content/bootkube" "github.com/openshift/installer/pkg/asset/tls" ) @@ -30,6 +31,7 @@ func (a *Worker) Dependencies() []asset.Asset { return []asset.Asset{ &installconfig.InstallConfig{}, &tls.RootCA{}, + &bootkube.ARODNSConfig{}, } } @@ -37,9 +39,10 @@ func (a *Worker) Dependencies() []asset.Asset { func (a *Worker) Generate(dependencies asset.Parents) error { installConfig := &installconfig.InstallConfig{} rootCA := &tls.RootCA{} - dependencies.Get(installConfig, rootCA) + aroDNSConfig := &bootkube.ARODNSConfig{} + dependencies.Get(installConfig, rootCA, aroDNSConfig) - a.Config = pointerIgnitionConfig(installConfig.Config, rootCA.Cert(), "worker") + a.Config = pointerIgnitionConfig(installConfig.Config, aroDNSConfig, rootCA.Cert(), "worker") data, err := ignition.Marshal(a.Config) if err != nil { diff --git a/vendor/github.com/openshift/installer/pkg/asset/installconfig/openstack/validate.go b/vendor/github.com/openshift/installer/pkg/asset/installconfig/openstack/validate.go index ced64f34ae9..c907e3e57b5 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/installconfig/openstack/validate.go +++ b/vendor/github.com/openshift/installer/pkg/asset/installconfig/openstack/validate.go @@ -31,3 +31,11 @@ func Validate(ic *types.InstallConfig) error { return allErrs.ToAggregate() } + +// ValidateForProvisioning validates that the install config is valid for provisioning the cluster. +func ValidateForProvisioning(ic *types.InstallConfig) error { + if ic.ControlPlane.Replicas != nil && *ic.ControlPlane.Replicas != 3 { + return field.Invalid(field.NewPath("controlPlane", "replicas"), ic.ControlPlane.Replicas, "control plane must be exactly three nodes when provisioning on OpenStack") + } + return nil +} diff --git a/vendor/github.com/openshift/installer/pkg/asset/installconfig/platformprovisioncheck.go b/vendor/github.com/openshift/installer/pkg/asset/installconfig/platformprovisioncheck.go index 0f099312ec5..25baec63f63 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/installconfig/platformprovisioncheck.go +++ b/vendor/github.com/openshift/installer/pkg/asset/installconfig/platformprovisioncheck.go @@ -8,6 +8,7 @@ import ( azconfig "github.com/openshift/installer/pkg/asset/installconfig/azure" bmconfig "github.com/openshift/installer/pkg/asset/installconfig/baremetal" gcpconfig "github.com/openshift/installer/pkg/asset/installconfig/gcp" + osconfig "github.com/openshift/installer/pkg/asset/installconfig/openstack" vsconfig "github.com/openshift/installer/pkg/asset/installconfig/vsphere" "github.com/openshift/installer/pkg/types/aws" "github.com/openshift/installer/pkg/types/azure" @@ -70,12 +71,17 @@ func (a *PlatformProvisionCheck) Generate(dependencies asset.Parents) error { if err != nil { return err } + case openstack.Name: + err = osconfig.ValidateForProvisioning(ic.Config) + if err != nil { + return err + } case vsphere.Name: err = vsconfig.ValidateForProvisioning(ic.Config) if err != nil { return err } - case aws.Name, libvirt.Name, none.Name, openstack.Name, ovirt.Name: + case aws.Name, libvirt.Name, none.Name, ovirt.Name: // no special provisioning requirements to check default: err = fmt.Errorf("unknown platform type %q", platform) diff --git a/vendor/github.com/openshift/installer/pkg/asset/installconfig/vsphere/vsphere.go b/vendor/github.com/openshift/installer/pkg/asset/installconfig/vsphere/vsphere.go index 537c6d27a56..9975ad81636 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/installconfig/vsphere/vsphere.go +++ b/vendor/github.com/openshift/installer/pkg/asset/installconfig/vsphere/vsphere.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "sort" + "strings" "time" "github.com/pkg/errors" @@ -172,16 +173,17 @@ func getDataCenter(ctx context.Context, finder *find.Finder, client *vim25.Clien return "", "", errors.New("did not find any datacenters") } if len(dataCenters) == 1 { - logrus.Infof("Defaulting to only available datacenter: %s", dataCenters[0].Name()) - dc := dataCenters[0] - return dc.Name(), formatPath(dc.InventoryPath), nil + name := strings.TrimPrefix(dataCenters[0].InventoryPath, "/") + logrus.Infof("Defaulting to only available datacenter: %s", name) + return name, dataCenters[0].InventoryPath, nil } dataCenterPaths := make(map[string]string) var dataCenterChoices []string for _, dc := range dataCenters { - dataCenterPaths[dc.Name()] = dc.InventoryPath - dataCenterChoices = append(dataCenterChoices, dc.Name()) + name := strings.TrimPrefix(dc.InventoryPath, "/") + dataCenterPaths[name] = dc.InventoryPath + dataCenterChoices = append(dataCenterChoices, name) } sort.Strings(dataCenterChoices) @@ -199,15 +201,14 @@ func getDataCenter(ctx context.Context, finder *find.Finder, client *vim25.Clien return "", "", err } - selectedDataCenterPath := formatPath(dataCenterPaths[selectedDataCenter]) - return selectedDataCenter, selectedDataCenterPath, nil + return selectedDataCenter, dataCenterPaths[selectedDataCenter], nil } func getCluster(ctx context.Context, path string, finder *find.Finder, client *vim25.Client) (string, error) { ctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() - clusters, err := finder.ClusterComputeResourceList(ctx, path) + clusters, err := finder.ClusterComputeResourceList(ctx, formatPath(path)) if err != nil { return "", errors.Wrap(err, "unable to list clusters") } @@ -217,13 +218,15 @@ func getCluster(ctx context.Context, path string, finder *find.Finder, client *v return "", errors.New("did not find any clusters") } if len(clusters) == 1 { - logrus.Infof("Defaulting to only available cluster: %s", clusters[0].Name()) - return clusters[0].Name(), nil + name := strings.TrimPrefix(clusters[0].InventoryPath, path+"/host/") + logrus.Infof("Defaulting to only available cluster: %s", name) + return name, nil } var clusterChoices []string for _, c := range clusters { - clusterChoices = append(clusterChoices, c.Name()) + name := strings.TrimPrefix(c.InventoryPath, path+"/host/") + clusterChoices = append(clusterChoices, name) } sort.Strings(clusterChoices) @@ -248,7 +251,7 @@ func getDataStore(ctx context.Context, path string, finder *find.Finder, client ctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() - dataStores, err := finder.DatastoreList(ctx, path) + dataStores, err := finder.DatastoreList(ctx, formatPath(path)) if err != nil { return "", errors.Wrap(err, "unable to list datastores") } @@ -289,7 +292,7 @@ func getNetwork(ctx context.Context, path string, finder *find.Finder, client *v ctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() - networks, err := finder.NetworkList(ctx, path) + networks, err := finder.NetworkList(ctx, formatPath(path)) if err != nil { return "", errors.Wrap(err, "unable to list networks") } diff --git a/vendor/github.com/openshift/installer/pkg/asset/machines/master.go b/vendor/github.com/openshift/installer/pkg/asset/machines/master.go index e4fbda0ee63..c2315d2b81a 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/machines/master.go +++ b/vendor/github.com/openshift/installer/pkg/asset/machines/master.go @@ -31,6 +31,7 @@ import ( openstackapi "sigs.k8s.io/cluster-api-provider-openstack/pkg/apis" openstackprovider "sigs.k8s.io/cluster-api-provider-openstack/pkg/apis/openstackproviderconfig/v1alpha1" + "github.com/openshift/installer/pkg/aro/dnsmasq" "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/ignition/machine" "github.com/openshift/installer/pkg/asset/installconfig" @@ -44,6 +45,7 @@ import ( "github.com/openshift/installer/pkg/asset/machines/ovirt" "github.com/openshift/installer/pkg/asset/machines/vsphere" "github.com/openshift/installer/pkg/asset/rhcos" + "github.com/openshift/installer/pkg/asset/templates/content/bootkube" rhcosutils "github.com/openshift/installer/pkg/rhcos" "github.com/openshift/installer/pkg/types" awstypes "github.com/openshift/installer/pkg/types/aws" @@ -120,6 +122,7 @@ func (m *Master) Dependencies() []asset.Asset { &installconfig.InstallConfig{}, new(rhcos.Image), &machine.Master{}, + &bootkube.ARODNSConfig{}, } } @@ -139,7 +142,8 @@ func (m *Master) Generate(dependencies asset.Parents) error { installConfig := &installconfig.InstallConfig{} rhcosImage := new(rhcos.Image) mign := &machine.Master{} - dependencies.Get(clusterID, installConfig, rhcosImage, mign) + aroDNSConfig := &bootkube.ARODNSConfig{} + dependencies.Get(clusterID, installConfig, rhcosImage, mign, aroDNSConfig) ic := installConfig.Config @@ -386,6 +390,11 @@ func (m *Master) Generate(dependencies asset.Parents) error { } machineConfigs = append(machineConfigs, ignFIPS) } + ignARODNS, err := dnsmasq.MachineConfig(installConfig.Config.ClusterDomain(), aroDNSConfig.APIIntIP, aroDNSConfig.IngressIP, "master") + if err != nil { + return errors.Wrap(err, "failed to create ignition for ARO DNS for master machines") + } + machineConfigs = append(machineConfigs, ignARODNS) m.MachineConfigFiles, err = machineconfig.Manifests(machineConfigs, "master", directory) if err != nil { diff --git a/vendor/github.com/openshift/installer/pkg/asset/machines/worker.go b/vendor/github.com/openshift/installer/pkg/asset/machines/worker.go index a210413ace6..46875bf54c4 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/machines/worker.go +++ b/vendor/github.com/openshift/installer/pkg/asset/machines/worker.go @@ -31,6 +31,7 @@ import ( openstackapi "sigs.k8s.io/cluster-api-provider-openstack/pkg/apis" openstackprovider "sigs.k8s.io/cluster-api-provider-openstack/pkg/apis/openstackproviderconfig/v1alpha1" + "github.com/openshift/installer/pkg/aro/dnsmasq" "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/ignition/machine" "github.com/openshift/installer/pkg/asset/installconfig" @@ -44,6 +45,7 @@ import ( "github.com/openshift/installer/pkg/asset/machines/ovirt" "github.com/openshift/installer/pkg/asset/machines/vsphere" "github.com/openshift/installer/pkg/asset/rhcos" + "github.com/openshift/installer/pkg/asset/templates/content/bootkube" rhcosutils "github.com/openshift/installer/pkg/rhcos" "github.com/openshift/installer/pkg/types" awstypes "github.com/openshift/installer/pkg/types/aws" @@ -174,6 +176,7 @@ func (w *Worker) Dependencies() []asset.Asset { &installconfig.InstallConfig{}, new(rhcos.Image), &machine.Worker{}, + &bootkube.ARODNSConfig{}, } } @@ -184,7 +187,8 @@ func (w *Worker) Generate(dependencies asset.Parents) error { installConfig := &installconfig.InstallConfig{} rhcosImage := new(rhcos.Image) wign := &machine.Worker{} - dependencies.Get(clusterID, installConfig, rhcosImage, wign) + aroDNSConfig := &bootkube.ARODNSConfig{} + dependencies.Get(clusterID, installConfig, rhcosImage, wign, aroDNSConfig) machineConfigs := []*mcfgv1.MachineConfig{} machineSets := []runtime.Object{} @@ -212,6 +216,11 @@ func (w *Worker) Generate(dependencies asset.Parents) error { } machineConfigs = append(machineConfigs, ignFIPS) } + ignARODNS, err := dnsmasq.MachineConfig(installConfig.Config.ClusterDomain(), aroDNSConfig.APIIntIP, aroDNSConfig.IngressIP, "worker") + if err != nil { + return errors.Wrap(err, "failed to create ignition for ARO DNS for worker machines") + } + machineConfigs = append(machineConfigs, ignARODNS) switch ic.Platform.Name() { case awstypes.Name: subnets := map[string]string{} diff --git a/vendor/github.com/openshift/installer/pkg/asset/manifests/dns.go b/vendor/github.com/openshift/installer/pkg/asset/manifests/dns.go index 90288885b8a..63ba8d309df 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/manifests/dns.go +++ b/vendor/github.com/openshift/installer/pkg/asset/manifests/dns.go @@ -100,15 +100,16 @@ func (d *DNS) Generate(dependencies asset.Parents) error { return err } - if !installConfig.Config.Azure.ARO && - installConfig.Config.Publish == types.ExternalPublishingStrategy { - //currently, this guesses the azure resource IDs from known parameter. - config.Spec.PublicZone = &configv1.DNSZone{ - ID: dnsConfig.GetDNSZoneID(installConfig.Config.Azure.BaseDomainResourceGroupName, installConfig.Config.BaseDomain), + if !installConfig.Config.Azure.ARO { + if installConfig.Config.Publish == types.ExternalPublishingStrategy { + //currently, this guesses the azure resource IDs from known parameter. + config.Spec.PublicZone = &configv1.DNSZone{ + ID: dnsConfig.GetDNSZoneID(installConfig.Config.Azure.BaseDomainResourceGroupName, installConfig.Config.BaseDomain), + } + } + config.Spec.PrivateZone = &configv1.DNSZone{ + ID: dnsConfig.GetPrivateDNSZoneID(installConfig.Config.Azure.ClusterResourceGroupName(clusterID.InfraID), installConfig.Config.ClusterDomain()), } - } - config.Spec.PrivateZone = &configv1.DNSZone{ - ID: dnsConfig.GetPrivateDNSZoneID(installConfig.Config.Azure.ClusterResourceGroupName(clusterID.InfraID), installConfig.Config.ClusterDomain()), } case gcptypes.Name: if installConfig.Config.Publish == types.ExternalPublishingStrategy { diff --git a/vendor/github.com/openshift/installer/pkg/asset/manifests/operators.go b/vendor/github.com/openshift/installer/pkg/asset/manifests/operators.go index ca1bb9ff78c..5987422589e 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/manifests/operators.go +++ b/vendor/github.com/openshift/installer/pkg/asset/manifests/operators.go @@ -88,6 +88,8 @@ func (m *Manifests) Dependencies() []asset.Asset { &bootkube.OpenshiftConfigSecretPullSecret{}, &bootkube.OpenshiftMachineConfigOperator{}, &bootkube.AROWorkerRegistries{}, + &bootkube.AROIngressService{}, + &bootkube.ARODNSConfig{}, } } @@ -153,6 +155,7 @@ func (m *Manifests) generateBootKubeManifests(dependencies asset.Parents) []*ass etcdSignerCertKey := &tls.EtcdSignerCertKey{} etcdCABundle := &tls.EtcdCABundle{} etcdSignerClientCertKey := &tls.EtcdSignerClientCertKey{} + aroDNSConfig := &bootkube.ARODNSConfig{} dependencies.Get( clusterID, installConfig, @@ -164,6 +167,7 @@ func (m *Manifests) generateBootKubeManifests(dependencies asset.Parents) []*ass etcdMetricSignerCertKey, mcsCertKey, rootCA, + aroDNSConfig, ) templateData := &bootkubeTemplateData{ @@ -183,6 +187,8 @@ func (m *Manifests) generateBootKubeManifests(dependencies asset.Parents) []*ass PullSecretBase64: base64.StdEncoding.EncodeToString([]byte(installConfig.Config.PullSecret)), RootCaCert: string(rootCA.Cert()), AROWorkerRegistries: aroWorkerRegistries(installConfig.Config.ImageContentSources), + AROIngressIP: aroDNSConfig.IngressIP, + AROIngressInternal: installConfig.Config.Publish == types.InternalPublishingStrategy, } files := []*asset.File{} @@ -203,6 +209,7 @@ func (m *Manifests) generateBootKubeManifests(dependencies asset.Parents) []*ass &bootkube.OpenshiftConfigSecretPullSecret{}, &bootkube.OpenshiftMachineConfigOperator{}, &bootkube.AROWorkerRegistries{}, + &bootkube.AROIngressService{}, } { dependencies.Get(a) for _, f := range a.Files() { diff --git a/vendor/github.com/openshift/installer/pkg/asset/manifests/template.go b/vendor/github.com/openshift/installer/pkg/asset/manifests/template.go index ddfa47cf912..0a0397c55bf 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/manifests/template.go +++ b/vendor/github.com/openshift/installer/pkg/asset/manifests/template.go @@ -74,6 +74,8 @@ type bootkubeTemplateData struct { RootCaCert string WorkerIgnConfig string AROWorkerRegistries string + AROIngressIP string + AROIngressInternal bool } type baremetalTemplateData struct { diff --git a/vendor/github.com/openshift/installer/pkg/asset/targets/targets.go b/vendor/github.com/openshift/installer/pkg/asset/targets/targets.go index 82220a98fe3..03090acf117 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/targets/targets.go +++ b/vendor/github.com/openshift/installer/pkg/asset/targets/targets.go @@ -45,6 +45,7 @@ var ( &bootkube.EtcdMetricServingCAConfigMap{}, &bootkube.OpenshiftConfigSecretPullSecret{}, &bootkube.AROWorkerRegistries{}, + &bootkube.AROIngressService{}, &openshift.CloudCredsSecret{}, &openshift.KubeadminPasswordSecret{}, &openshift.RoleCloudCredsSecretReader{}, diff --git a/vendor/github.com/openshift/installer/pkg/asset/templates/content/bootkube/aro-dns-config.go b/vendor/github.com/openshift/installer/pkg/asset/templates/content/bootkube/aro-dns-config.go new file mode 100644 index 00000000000..abb60b9419d --- /dev/null +++ b/vendor/github.com/openshift/installer/pkg/asset/templates/content/bootkube/aro-dns-config.go @@ -0,0 +1,38 @@ +package bootkube + +import ( + "github.com/openshift/installer/pkg/asset" +) + +var _ asset.WritableAsset = (*ARODNSConfig)(nil) + +// ARODNSConfig is an asset for the openshift-apiserver namespace +type ARODNSConfig struct { + APIIntIP string + IngressIP string +} + +// Dependencies returns all of the dependencies directly needed by the asset +func (t *ARODNSConfig) Dependencies() []asset.Asset { + return nil +} + +// Name returns the human-friendly name of the asset. +func (t *ARODNSConfig) Name() string { + return "ARODNSConfig" +} + +// Generate generates the actual files by this asset +func (t *ARODNSConfig) Generate(parents asset.Parents) error { + return nil +} + +// Files returns the files generated by the asset. +func (t *ARODNSConfig) Files() []*asset.File { + return nil +} + +// Load returns the asset from disk. +func (t *ARODNSConfig) Load(f asset.FileFetcher) (bool, error) { + return true, nil +} diff --git a/vendor/github.com/openshift/installer/pkg/asset/templates/content/bootkube/aro-ingress-service.go b/vendor/github.com/openshift/installer/pkg/asset/templates/content/bootkube/aro-ingress-service.go new file mode 100644 index 00000000000..a0c85561119 --- /dev/null +++ b/vendor/github.com/openshift/installer/pkg/asset/templates/content/bootkube/aro-ingress-service.go @@ -0,0 +1,75 @@ +package bootkube + +import ( + "os" + "path/filepath" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/templates/content" +) + +const ( + aroIngressNamespaceFileName = "aro-ingress-namespace.yaml" + aroIngressServiceFileName = "aro-ingress-service.yaml.template" +) + +var _ asset.WritableAsset = (*AROIngressService)(nil) + +// AROIngressService is an asset for the openshift-apiserver namespace +type AROIngressService struct { + FileList []*asset.File +} + +// Dependencies returns all of the dependencies directly needed by the asset +func (t *AROIngressService) Dependencies() []asset.Asset { + return []asset.Asset{} +} + +// Name returns the human-friendly name of the asset. +func (t *AROIngressService) Name() string { + return "AROIngressService" +} + +// Generate generates the actual files by this asset +func (t *AROIngressService) Generate(parents asset.Parents) error { + t.FileList = nil + + for _, filename := range []string{aroIngressNamespaceFileName, aroIngressServiceFileName} { + data, err := content.GetBootkubeTemplate(filename) + if err != nil { + return err + } + + t.FileList = append(t.FileList, &asset.File{ + Filename: filepath.Join(content.TemplateDir, filename), + Data: []byte(data), + }) + } + + return nil +} + +// Files returns the files generated by the asset. +func (t *AROIngressService) Files() []*asset.File { + return t.FileList +} + +// Load returns the asset from disk. +func (t *AROIngressService) Load(f asset.FileFetcher) (bool, error) { + ingressNamespaceData, err := f.FetchByName(filepath.Join(content.TemplateDir, aroIngressNamespaceFileName)) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + ingressServiceData, err := f.FetchByName(filepath.Join(content.TemplateDir, aroIngressServiceFileName)) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + t.FileList = []*asset.File{ingressNamespaceData, ingressServiceData} + return true, nil +} diff --git a/vendor/github.com/openshift/installer/pkg/asset/tls/mcscertkey.go b/vendor/github.com/openshift/installer/pkg/asset/tls/mcscertkey.go index ac15113c0f7..2914cc2cbeb 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/tls/mcscertkey.go +++ b/vendor/github.com/openshift/installer/pkg/asset/tls/mcscertkey.go @@ -7,6 +7,8 @@ import ( "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/asset/templates/content/bootkube" + azuretypes "github.com/openshift/installer/pkg/types/azure" baremetaltypes "github.com/openshift/installer/pkg/types/baremetal" openstacktypes "github.com/openshift/installer/pkg/types/openstack" ovirttypes "github.com/openshift/installer/pkg/types/ovirt" @@ -27,6 +29,7 @@ func (a *MCSCertKey) Dependencies() []asset.Asset { return []asset.Asset{ &RootCA{}, &installconfig.InstallConfig{}, + &bootkube.ARODNSConfig{}, } } @@ -34,7 +37,8 @@ func (a *MCSCertKey) Dependencies() []asset.Asset { func (a *MCSCertKey) Generate(dependencies asset.Parents) error { ca := &RootCA{} installConfig := &installconfig.InstallConfig{} - dependencies.Get(ca, installConfig) + aroDNSConfig := &bootkube.ARODNSConfig{} + dependencies.Get(ca, installConfig, aroDNSConfig) hostname := internalAPIAddress(installConfig.Config) @@ -45,6 +49,13 @@ func (a *MCSCertKey) Generate(dependencies asset.Parents) error { } switch installConfig.Config.Platform.Name() { + case azuretypes.Name: + if installConfig.Config.Azure.ARO { + cfg.IPAddresses = []net.IP{net.ParseIP(aroDNSConfig.APIIntIP)} + cfg.DNSNames = []string{hostname, aroDNSConfig.APIIntIP} + } else { + cfg.DNSNames = []string{hostname} + } case baremetaltypes.Name: cfg.IPAddresses = []net.IP{net.ParseIP(installConfig.Config.BareMetal.APIVIP)} cfg.DNSNames = []string{hostname, installConfig.Config.BareMetal.APIVIP} diff --git a/vendor/modules.txt b/vendor/modules.txt index 3a44c5953fe..ec76a69da78 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -752,9 +752,10 @@ github.com/openshift/cluster-api-provider-ovirt/pkg/apis/ovirtprovider/v1beta1 # github.com/openshift/console-operator v0.0.0-20210216151626-6e1cbc849915 => github.com/openshift/console-operator v0.0.0-20210116095614-7fd78a283616 ## explicit github.com/openshift/console-operator/pkg/api -# github.com/openshift/installer v0.16.1 => github.com/jim-minter/installer v0.9.0-master.0.20210128115533-6feac498cb32 +# github.com/openshift/installer v0.16.1 => github.com/jim-minter/installer v0.9.0-master.0.20210221211908-aaebddb9dcf1 ## explicit github.com/openshift/installer/data +github.com/openshift/installer/pkg/aro/dnsmasq github.com/openshift/installer/pkg/asset github.com/openshift/installer/pkg/asset/bootstraplogging github.com/openshift/installer/pkg/asset/cluster @@ -1724,7 +1725,7 @@ sigs.k8s.io/yaml # github.com/openshift/cluster-api-provider-libvirt => github.com/openshift/cluster-api-provider-libvirt v0.2.1-0.20200919090150-1ca52adab176 # github.com/openshift/cluster-api-provider-ovirt => github.com/openshift/cluster-api-provider-ovirt v0.1.1-0.20210210114935-91f12f3f7dee # github.com/openshift/console-operator => github.com/openshift/console-operator v0.0.0-20210116095614-7fd78a283616 -# github.com/openshift/installer => github.com/jim-minter/installer v0.9.0-master.0.20210128115533-6feac498cb32 +# github.com/openshift/installer => github.com/jim-minter/installer v0.9.0-master.0.20210221211908-aaebddb9dcf1 # github.com/openshift/machine-api-operator => github.com/openshift/machine-api-operator v0.2.1-0.20210212025836-cb508cd8777d # github.com/openshift/machine-config-operator => github.com/openshift/machine-config-operator v0.0.1-0.20210211205336-14a2b82d9f4c # github.com/operator-framework/operator-sdk => github.com/operator-framework/operator-sdk v0.19.4