Enable metrics

cmd/manager/main.go

@@ -24,6 +24,7 @@ import (
 	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1"
 	"github.com/openshift/machine-api-operator/pkg/controller/machine"
+	"github.com/openshift/machine-api-operator/pkg/metrics"
 	"k8s.io/klog"
 	"k8s.io/klog/klogr"
 	"sigs.k8s.io/cluster-api-provider-azure/pkg/apis"
@@ -50,6 +51,12 @@ func main() {
 		"The address for health checking.",
 	)
 
+	metricsAddress := flag.String(
+		"metrics-bind-address",
+		metrics.DefaultMachineMetricsAddress,
+		"Address for hosting metrics",
+	)
+
 	leaderElectResourceNamespace := flag.String(
 		"leader-elect-resource-namespace",
 		"",
@@ -80,6 +87,7 @@ func main() {
 		LeaderElectionNamespace: *leaderElectResourceNamespace,
 		LeaderElectionID:        "cluster-api-provider-azure-leader",
 		LeaseDuration:           leaderElectLeaseDuration,
+		MetricsBindAddress:      *metricsAddress,
 	}
 
 	if *watchNamespace != "" {

go.mod

@@ -7,14 +7,13 @@ require (
 	github.com/Azure/go-autorest/autorest v0.10.0
 	github.com/Azure/go-autorest/autorest/adal v0.8.2
 	github.com/Azure/go-autorest/autorest/to v0.3.0
-	github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-logr/logr v0.1.0
 	github.com/golang/mock v1.3.1
 	github.com/onsi/ginkgo v1.12.0
 	github.com/onsi/gomega v1.8.1
 	github.com/openshift/api v0.0.0-20200424083944-0422dc17083e
-	github.com/openshift/machine-api-operator v0.2.1-0.20200611014855-9a69f85c32dd
+	github.com/openshift/machine-api-operator v0.2.1-0.20200701225707-950912b03628
 	github.com/spf13/cobra v0.0.5
 	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
 
@@ -30,6 +29,9 @@ require (
 	sigs.k8s.io/yaml v1.2.0
 )
 
-replace sigs.k8s.io/cluster-api-provider-aws => github.com/openshift/cluster-api-provider-aws v0.2.1-0.20200612013036-1a97c62077cb
-
 replace github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.0.1+incompatible
+
+replace (
+	sigs.k8s.io/cluster-api-provider-aws => github.com/openshift/cluster-api-provider-aws v0.2.1-0.20200618031251-e16dd65fdd85
+	sigs.k8s.io/cluster-api-provider-azure => github.com/openshift/cluster-api-provider-azure v0.1.0-alpha.3.0.20200618001858-af08a66b92de
+)

go.sum

@@ -1,14 +1,17 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+github.com/Azure/azure-sdk-for-go v32.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go v42.0.0+incompatible h1:yz6sFf5bHZ+gEOQVuK5JhPqTTAmv+OvSLSaqgzqaCwY=
 github.com/Azure/azure-sdk-for-go v42.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest v0.10.0 h1:mvdtztBqcL8se7MdrUweNieTNi4kfNG6GOJuurQJpuY=
 github.com/Azure/go-autorest/autorest v0.10.0/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/adal v0.7.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
 github.com/Azure/go-autorest/autorest/adal v0.8.2 h1:O1X4oexUxnZCaEUGsvMnr8ZGj8HI37tNezwY4npRqA0=
 github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
@@ -344,13 +347,16 @@ github.com/openshift/build-machinery-go v0.0.0-20200424080330-082bf86082cc h1:Bu
 github.com/openshift/build-machinery-go v0.0.0-20200424080330-082bf86082cc/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc=
 github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0 h1:kMiuiZXH1GdfbiMwsuAQOqGaMxlo9NCUk0wT4XAdfNM=
 github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0/go.mod h1:uUQ4LClRO+fg5MF/P6QxjMCb1C9f7Oh4RKepftDnEJE=
-github.com/openshift/cluster-api-provider-aws v0.2.1-0.20200612013036-1a97c62077cb h1:DoTy8JE3gI99Km1Qe2mn0mVRZRrsfr9tc8TJxfzXPL4=
-github.com/openshift/cluster-api-provider-aws v0.2.1-0.20200612013036-1a97c62077cb/go.mod h1:4560PS0EgzfODrXC01k3f1vYfbCDjEQwhrlRrsiDhO0=
+github.com/openshift/cluster-api-provider-aws v0.2.1-0.20200618031251-e16dd65fdd85 h1:f+ly4ppDKaxS5JTdZJjBhuG7EnNrsn4JaI0VW2QQztA=
+github.com/openshift/cluster-api-provider-aws v0.2.1-0.20200618031251-e16dd65fdd85/go.mod h1:FxIU+vzZ4b5A7kGKootyJeTI3xafRTljVJF7/+4Iz4s=
+github.com/openshift/cluster-api-provider-azure v0.1.0-alpha.3.0.20200618001858-af08a66b92de/go.mod h1:XTKOAoCAzFtgljQ9HUs4KeTyczeFUoXa0S9+0R2WCmk=
+github.com/openshift/cluster-api-provider-gcp v0.0.1-0.20200701112720-3a7d727c9a10 h1:AZczulAAkBuMm058BGtk5id9lhupszVQXvZFdKmSHEE=
+github.com/openshift/cluster-api-provider-gcp v0.0.1-0.20200701112720-3a7d727c9a10/go.mod h1:wgkZrOlcIMWTzo8khB4Js2PoDJDlIUUdzCBm7BuDdqw=
 github.com/openshift/library-go v0.0.0-20200512120242-21a1ff978534/go.mod h1:2kWwXTkpoQJUN3jZ3QW88EIY1hdRMqxgRs2hheEW/pg=
-github.com/openshift/machine-api-operator v0.2.1-0.20200527204437-14e5e0c7d862 h1:uI/6lz/E7ngKe0Fy0wy/wOpLuoS8eJlmoyuLLYfK1bs=
 github.com/openshift/machine-api-operator v0.2.1-0.20200527204437-14e5e0c7d862/go.mod h1:YKEQMHjXzrzm4fQGTyHBafFfQ/Yq/FrV+1YcGdPCp+0=
-github.com/openshift/machine-api-operator v0.2.1-0.20200611014855-9a69f85c32dd h1:DSKbDT7nkLuoidsw0MAXjVaSZULy/alBARdN0/VV4OU=
 github.com/openshift/machine-api-operator v0.2.1-0.20200611014855-9a69f85c32dd/go.mod h1:6vMi+R3xqznBdq5rgeal9N3ak3sOpy50t0fdRCcQXjE=
+github.com/openshift/machine-api-operator v0.2.1-0.20200701225707-950912b03628 h1:8ErZADoEg3XJBqY8tM9eUEJ9szb7jQHNHSlw2j9Gr5Y=
+github.com/openshift/machine-api-operator v0.2.1-0.20200701225707-950912b03628/go.mod h1:cxjy/RUzv5C2T5FNl1KKXUgtakWsezWQ642B/CD9VQA=
 github.com/operator-framework/operator-sdk v0.5.1-0.20190301204940-c2efe6f74e7b/go.mod h1:iVyukRkam5JZa8AnjYf+/G3rk7JI1+M6GsU0sq0B9NA=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -671,7 +677,6 @@ modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs
 modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0=
 sigs.k8s.io/controller-runtime v0.3.1-0.20191016212439-2df793d02076/go.mod h1:p2vzQ3RuSVv9YR4AcM0y8TKHQA+0oLXazKFt6Z0OdS8=
-sigs.k8s.io/controller-runtime v0.5.1-0.20200330174416-a11a908d91e0 h1:3aPkagGN7ZrvaDFAMJoZXyDesY0Otz+S4aXvmHGkQ2s=
 sigs.k8s.io/controller-runtime v0.5.1-0.20200330174416-a11a908d91e0/go.mod h1:j4echH3Y/UPHRpXS65rxGXujda8iWOheMQvDh1uNgaY=
 sigs.k8s.io/controller-runtime v0.6.0 h1:Fzna3DY7c4BIP6KwfSlrfnj20DJ+SeMBK8HSFvOk9NM=
 sigs.k8s.io/controller-runtime v0.6.0/go.mod h1:CpYf5pdNY/B352A1TFLAS2JVSlnGQ5O2cftPHndTroo=

pkg/cloud/azure/actuators/machine/reconciler.go

@@ -31,6 +31,7 @@ import (
 	"github.com/Azure/go-autorest/autorest/to"
 	machinev1 "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1"
 	machinecontroller "github.com/openshift/machine-api-operator/pkg/controller/machine"
+	"github.com/openshift/machine-api-operator/pkg/metrics"
 	apicorev1 "k8s.io/api/core/v1"
 	"k8s.io/klog"
 	"sigs.k8s.io/cluster-api-provider-azure/pkg/apis/azureprovider/v1beta1"
@@ -410,6 +411,11 @@ func (s *Reconciler) Delete(ctx context.Context) error {
 	}
 	err = s.disksSvc.Delete(ctx, osDiskSpec)
 	if err != nil {
+		metrics.RegisterFailedInstanceDelete(&metrics.MachineLabels{
+			Name:      s.scope.Machine.Name,
+			Namespace: s.scope.Machine.Namespace,
+			Reason:    err.Error(),
+		})
 		return fmt.Errorf("failed to delete OS disk: %w", err)
 	}
 
@@ -424,6 +430,11 @@ func (s *Reconciler) Delete(ctx context.Context) error {
 
 	err = s.networkInterfacesSvc.Delete(ctx, networkInterfaceSpec)
 	if err != nil {
+		metrics.RegisterFailedInstanceDelete(&metrics.MachineLabels{
+			Name:      s.scope.Machine.Name,
+			Namespace: s.scope.Machine.Namespace,
+			Reason:    err.Error(),
+		})
 		return fmt.Errorf("Unable to delete network interface: %w", err)
 	}
 
@@ -443,6 +454,11 @@ func (s *Reconciler) Delete(ctx context.Context) error {
 			Name: publicIPName,
 		})
 		if err != nil {
+			metrics.RegisterFailedInstanceDelete(&metrics.MachineLabels{
+				Name:      s.scope.Machine.Name,
+				Namespace: s.scope.Machine.Namespace,
+				Reason:    err.Error(),
+			})
 			return fmt.Errorf("unable to delete Public IP: %w", err)
 		}
 	}
@@ -495,13 +511,23 @@ func (s *Reconciler) createNetworkInterface(ctx context.Context, nicName string)
 		}
 		err = s.publicIPSvc.CreateOrUpdate(ctx, &publicips.Spec{Name: publicIPName})
 		if err != nil {
+			metrics.RegisterFailedInstanceCreate(&metrics.MachineLabels{
+				Name:      s.scope.Machine.Name,
+				Namespace: s.scope.Machine.Namespace,
+				Reason:    err.Error(),
+			})
 			return fmt.Errorf("unable to create Public IP: %w", err)
 		}
 		networkInterfaceSpec.PublicIP = publicIPName
 	}
 
 	err := s.networkInterfacesSvc.CreateOrUpdate(ctx, networkInterfaceSpec)
 	if err != nil {
+		metrics.RegisterFailedInstanceCreate(&metrics.MachineLabels{
+			Name:      s.scope.Machine.Name,
+			Namespace: s.scope.Machine.Namespace,
+			Reason:    err.Error(),
+		})
 		return fmt.Errorf("unable to create VM network interface: %w", err)
 	}
 
@@ -569,6 +595,11 @@ func (s *Reconciler) createVirtualMachine(ctx context.Context, nicName string) e
 
 		err = s.virtualMachinesSvc.CreateOrUpdate(ctx, vmSpec)
 		if err != nil {
+			metrics.RegisterFailedInstanceCreate(&metrics.MachineLabels{
+				Name:      s.scope.Machine.Name,
+				Namespace: s.scope.Machine.Namespace,
+				Reason:    err.Error(),
+			})
 			return fmt.Errorf("failed to create or get machine: %w", err)
 		}
 	} else if err != nil {