Remove private DNS zones from architecture

cmd/aro/operator.go

@@ -23,6 +23,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/operator/controllers"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/alertwebhook"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/checker"
+	"github.com/Azure/ARO-RP/pkg/operator/controllers/dnsmasq"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/genevalogging"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/monitoring"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/pullsecret"
@@ -126,6 +127,21 @@ func operator(ctx context.Context, log *logrus.Entry) error {
 			arocli, dh)).SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("unable to create controller RBAC: %v", err)
 		}
+		if err = (dnsmasq.NewClusterReconciler(
+			log.WithField("controller", controllers.DnsmasqClusterControllerName),
+			arocli, mcocli, dh)).SetupWithManager(mgr); err != nil {
+			return fmt.Errorf("unable to create controller DnsmasqCluster: %v", err)
+		}
+		if err = (dnsmasq.NewMachineConfigReconciler(
+			log.WithField("controller", controllers.DnsmasqMachineConfigControllerName),
+			arocli, mcocli, dh)).SetupWithManager(mgr); err != nil {
+			return fmt.Errorf("unable to create controller DnsmasqMachineConfig: %v", err)
+		}
+		if err = (dnsmasq.NewMachineConfigPoolReconciler(
+			log.WithField("controller", controllers.DnsmasqMachineConfigPoolControllerName),
+			arocli, mcocli, dh)).SetupWithManager(mgr); err != nil {
+			return fmt.Errorf("unable to create controller DnsmasqMachineConfigPool: %v", err)
+		}
 	}
 
 	if err = (checker.NewReconciler(

go.mod

@@ -126,7 +126,7 @@ replace (
 	github.com/openshift/cluster-api-provider-libvirt => github.com/openshift/cluster-api-provider-libvirt v0.2.1-0.20200919090150-1ca52adab176
 	github.com/openshift/cluster-api-provider-ovirt => github.com/openshift/cluster-api-provider-ovirt v0.1.1-0.20210210114935-91f12f3f7dee
 	github.com/openshift/console-operator => github.com/openshift/console-operator v0.0.0-20210116095614-7fd78a283616
-	github.com/openshift/installer => github.com/jim-minter/installer v0.9.0-master.0.20210128115533-6feac498cb32
+	github.com/openshift/installer => github.com/jim-minter/installer v0.9.0-master.0.20210221211908-aaebddb9dcf1
 	github.com/openshift/machine-api-operator => github.com/openshift/machine-api-operator v0.2.1-0.20210212025836-cb508cd8777d
 	github.com/openshift/machine-config-operator => github.com/openshift/machine-config-operator v0.0.1-0.20210211205336-14a2b82d9f4c
 	github.com/operator-framework/operator-sdk => github.com/operator-framework/operator-sdk v0.19.4

pkg/api/openshiftcluster.go

@@ -224,6 +224,7 @@ type APIServerProfile struct {
 	Visibility Visibility `json:"visibility,omitempty"`
 	URL        string     `json:"url,omitempty"`
 	IP         string     `json:"ip,omitempty"`
+	IntIP      string     `json:"intIp,omitempty"`
 }
 
 // Visibility represents visibility.

pkg/api/openshiftclusterdocument.go

@@ -36,8 +36,8 @@ type OpenShiftClusterDocument struct {
 
 	Bucket int `json:"bucket,omitempty"`
 
-	LeaseOwner   string `json:"leaseOwner,omitempty"`
-	LeaseExpires int    `json:"leaseExpires,omitempty"`
+	LeaseOwner   string `json:"leaseOwner,omitempty" deep:"-"`
+	LeaseExpires int    `json:"leaseExpires,omitempty" deep:"-"`
 	Dequeues     int    `json:"dequeues,omitempty"`
 
 	AsyncOperationID string `json:"asyncOperationId,omitempty" deep:"-"`

pkg/api/validate/dynamic.go

@@ -31,7 +31,6 @@ import (
 type SlimDynamic interface {
 	ValidateVnetPermissions(ctx context.Context) error
 	ValidateRouteTablesPermissions(ctx context.Context) error
-	ValidateVnetDns(ctx context.Context) error
 	// etc
 	// does Quota code go in here too?
 }
@@ -167,23 +166,6 @@ func (dv *dynamic) validateRouteTablePermissions(ctx context.Context, rtID strin
 	return err
 }
 
-func (dv *dynamic) ValidateVnetDNS(ctx context.Context) error {
-	dv.log.Print("validateVnetDns")
-
-	vnet, err := dv.virtualNetworks.Get(ctx, dv.vnetr.ResourceGroup, dv.vnetr.ResourceName, "")
-	if err != nil {
-		return err
-	}
-
-	if vnet.DhcpOptions != nil &&
-		vnet.DhcpOptions.DNSServers != nil &&
-		len(*vnet.DhcpOptions.DNSServers) > 0 {
-		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidLinkedVNet, "", "The provided vnet '%s' is invalid: custom DNS servers are not supported.", *vnet.ID)
-	}
-
-	return nil
-}
-
 func (dv *dynamic) validateActions(ctx context.Context, r *azure.Resource, actions []string) error {
 	timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
 	defer cancel()

pkg/api/validate/dynamic_test.go

@@ -185,80 +185,6 @@ func TestGetRouteTableID(t *testing.T) {
 	}
 }
 
-func TestValidateVnetDNS(t *testing.T) {
-	ctx := context.Background()
-
-	controller := gomock.NewController(t)
-	defer controller.Finish()
-
-	resourceGroupName := "testGroup"
-	vnetName := "testVnet"
-	vnetID := "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/" + resourceGroupName + "/providers/Microsoft.Network/virtualNetworks/" + vnetName
-
-	for _, tt := range []struct {
-		name      string
-		vnetMocks func(*mock_network.MockVirtualNetworksClient, mgmtnetwork.VirtualNetwork)
-		wantErr   string
-	}{
-		{
-			name: "pass",
-			vnetMocks: func(vnetClient *mock_network.MockVirtualNetworksClient, vnet mgmtnetwork.VirtualNetwork) {
-				vnetClient.EXPECT().
-					Get(gomock.Any(), resourceGroupName, vnetName, "").
-					Return(vnet, nil)
-			},
-		},
-		{
-			name: "fail: dhcp options set",
-			vnetMocks: func(vnetClient *mock_network.MockVirtualNetworksClient, vnet mgmtnetwork.VirtualNetwork) {
-				vnet.DhcpOptions = &mgmtnetwork.DhcpOptions{
-					DNSServers: &[]string{
-						"8.8.8.8",
-					},
-				}
-				vnetClient.EXPECT().
-					Get(gomock.Any(), resourceGroupName, vnetName, "").
-					Return(vnet, nil)
-			},
-			wantErr: "400: InvalidLinkedVNet: : The provided vnet '" + vnetID + "' is invalid: custom DNS servers are not supported.",
-		},
-		{
-			name: "fail: failed to get vnet",
-			vnetMocks: func(vnetClient *mock_network.MockVirtualNetworksClient, vnet mgmtnetwork.VirtualNetwork) {
-				vnetClient.EXPECT().
-					Get(gomock.Any(), resourceGroupName, vnetName, "").
-					Return(vnet, errors.New("failed to get vnet"))
-			},
-			wantErr: "failed to get vnet",
-		},
-	} {
-		vnet := mgmtnetwork.VirtualNetwork{
-			ID: to.StringPtr(vnetID),
-			VirtualNetworkPropertiesFormat: &mgmtnetwork.VirtualNetworkPropertiesFormat{
-				DhcpOptions: nil,
-			},
-		}
-
-		vnetClient := mock_network.NewMockVirtualNetworksClient(controller)
-		tt.vnetMocks(vnetClient, vnet)
-
-		dv := &dynamic{
-			log:             logrus.NewEntry(logrus.StandardLogger()),
-			virtualNetworks: vnetClient,
-			vnetr: &azure.Resource{
-				ResourceGroup: resourceGroupName,
-				ResourceName:  vnetName,
-			},
-		}
-
-		err := dv.ValidateVnetDNS(ctx)
-		if err != nil && err.Error() != tt.wantErr ||
-			err == nil && tt.wantErr != "" {
-			t.Error(err)
-		}
-	}
-}
-
 func TestValidateRouteTablePermissions(t *testing.T) {
 	ctx := context.Background()
 

pkg/api/validate/openshiftcluster_validatedynamic.go

@@ -116,11 +116,6 @@ func (dv *openShiftClusterDynamicValidator) Dynamic(ctx context.Context) error {
 	}
 
 	// Additional checks - use any dynamic because they both have the correct permissions
-	err = spDynamic.ValidateVnetDNS(ctx)
-	if err != nil {
-		return err
-	}
-
 	vnet, err := spDynamic.virtualNetworks.Get(ctx, spDynamic.vnetr.ResourceGroup, spDynamic.vnetr.ResourceName, "")
 	if err != nil {
 		return err

pkg/cluster/cluster.go

@@ -11,11 +11,14 @@ import (
 	operatorclient "github.com/openshift/client-go/operator/clientset/versioned"
 	samplesclient "github.com/openshift/client-go/samples/clientset/versioned"
 	securityclient "github.com/openshift/client-go/security/clientset/versioned"
+	maoclient "github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned"
+	mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
 	"github.com/sirupsen/logrus"
 	extensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/cluster/graph"
 	"github.com/Azure/ARO-RP/pkg/database"
 	"github.com/Azure/ARO-RP/pkg/env"
 	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
@@ -24,12 +27,12 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/privatedns"
-	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage"
 	"github.com/Azure/ARO-RP/pkg/util/billing"
 	"github.com/Azure/ARO-RP/pkg/util/dns"
 	"github.com/Azure/ARO-RP/pkg/util/encryption"
 	"github.com/Azure/ARO-RP/pkg/util/privateendpoint"
 	"github.com/Azure/ARO-RP/pkg/util/refreshable"
+	"github.com/Azure/ARO-RP/pkg/util/storage"
 	"github.com/Azure/ARO-RP/pkg/util/subnet"
 )
 
@@ -48,7 +51,6 @@ type manager struct {
 	billing           billing.Manager
 	doc               *api.OpenShiftClusterDocument
 	subscriptionDoc   *api.SubscriptionDocument
-	aead              encryption.AEAD
 	fpAuthorizer      refreshable.Authorizer
 	localFpAuthorizer refreshable.Authorizer
 
@@ -62,15 +64,19 @@ type manager struct {
 	deployments         features.DeploymentsClient
 	resourceGroups      features.ResourceGroupsClient
 	resources           features.ResourcesClient
+	privateZones        privatedns.PrivateZonesClient
 	virtualNetworkLinks privatedns.VirtualNetworkLinksClient
-	storageAccounts     storage.AccountsClient
 
 	dns             dns.Manager
 	privateendpoint privateendpoint.Manager
+	storage         storage.Manager
 	subnet          subnet.Manager
+	graph           graph.Manager
 
 	kubernetescli kubernetes.Interface
 	extensionscli extensionsclient.Interface
+	maocli        maoclient.Interface
+	mcocli        mcoclient.Interface
 	operatorcli   operatorclient.Interface
 	configcli     configclient.Interface
 	samplescli    samplesclient.Interface
@@ -98,14 +104,15 @@ func New(ctx context.Context, log *logrus.Entry, env env.Interface, db database.
 		return nil, err
 	}
 
+	storage := storage.NewManager(env, r.SubscriptionID, fpAuthorizer)
+
 	return &manager{
 		log:               log,
 		env:               env,
 		db:                db,
 		billing:           billing,
 		doc:               doc,
 		subscriptionDoc:   subscriptionDoc,
-		aead:              aead,
 		fpAuthorizer:      fpAuthorizer,
 		localFpAuthorizer: localFPAuthorizer,
 
@@ -118,11 +125,13 @@ func New(ctx context.Context, log *logrus.Entry, env env.Interface, db database.
 		deployments:         features.NewDeploymentsClient(env.Environment(), r.SubscriptionID, fpAuthorizer),
 		resourceGroups:      features.NewResourceGroupsClient(env.Environment(), r.SubscriptionID, fpAuthorizer),
 		resources:           features.NewResourcesClient(env.Environment(), r.SubscriptionID, fpAuthorizer),
+		privateZones:        privatedns.NewPrivateZonesClient(env.Environment(), r.SubscriptionID, fpAuthorizer),
 		virtualNetworkLinks: privatedns.NewVirtualNetworkLinksClient(env.Environment(), r.SubscriptionID, fpAuthorizer),
-		storageAccounts:     storage.NewAccountsClient(env.Environment(), r.SubscriptionID, fpAuthorizer),
 
 		dns:             dns.NewManager(env, localFPAuthorizer),
 		privateendpoint: privateendpoint.NewManager(env, localFPAuthorizer),
+		storage:         storage,
 		subnet:          subnet.NewManager(env, r.SubscriptionID, fpAuthorizer),
+		graph:           graph.NewManager(log, aead, storage),
 	}, nil
 }

pkg/cluster/deployresources.go

@@ -14,25 +14,20 @@ import (
 
 	"github.com/Azure/ARO-RP/pkg/util/arm"
 	"github.com/Azure/ARO-RP/pkg/util/stringutils"
-	"github.com/Azure/ARO-RP/pkg/util/subnet"
 )
 
 func (m *manager) deployResourceTemplate(ctx context.Context) error {
-	pg, err := m.loadPersistedGraph(ctx)
+	resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/')
+	account := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix
+
+	pg, err := m.graph.LoadPersisted(ctx, resourceGroup, account)
 	if err != nil {
 		return err
 	}
 
 	var installConfig *installconfig.InstallConfig
 	var machineMaster *machine.Master
-	err = pg.get(&installConfig, &machineMaster)
-	if err != nil {
-		return err
-	}
-
-	resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/')
-
-	vnetID, _, err := subnet.Split(m.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID)
+	err = pg.Get(&installConfig, &machineMaster)
 	if err != nil {
 		return err
 	}
@@ -51,14 +46,6 @@ func (m *manager) deployResourceTemplate(ctx context.Context) error {
 			},
 		},
 		Resources: []*arm.Resource{
-			m.dnsPrivateZone(installConfig),
-			m.dnsPrivateRecordAPIINT(installConfig),
-			m.dnsPrivateRecordAPI(installConfig),
-			m.dnsVirtualNetworkLink(installConfig, vnetID),
-			m.networkPrivateLinkService(installConfig),
-			m.networkPublicIPAddress(installConfig),
-			m.networkInternalLoadBalancer(installConfig),
-			m.networkPublicLoadBalancer(installConfig),
 			m.networkBootstrapNIC(installConfig),
 			m.networkMasterNICs(installConfig),
 			m.computeBootstrapVM(installConfig),