[receiver/iis] Emit metrics per-site and per-application-pool (#14545)

Emit metrics per-site and per-application-pool

receiver/iisreceiver/documentation.md

@@ -30,6 +30,13 @@ metrics:
     enabled: <true|false>
 ```
 
+## Resource attributes
+
+| Name | Description | Type |
+| ---- | ----------- | ---- |
+| iis.application_pool | The application pool, which is associated with worker processes of one or more applications. | String |
+| iis.site | The site of the web server. | String |
+
 ## Metric attributes
 
 | Name | Description | Values |

receiver/iisreceiver/metadata.yaml

@@ -1,5 +1,13 @@
 name: iisreceiver
 
+resource_attributes:
+  iis.site:
+    description: The site of the web server.
+    type: string
+  iis.application_pool:
+    description: The application pool, which is associated with worker processes of one or more applications.
+    type: string
+
 attributes:
   direction:
     value: direction

receiver/iisreceiver/recorder.go

@@ -23,18 +23,30 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/iisreceiver/internal/metadata"
 )
 
-type recordFunc = func(*metadata.MetricsBuilder, pcommon.Timestamp, float64)
+type recordFunc = func(md *metadata.MetricsBuilder, ts pcommon.Timestamp, val float64)
 
 type perfCounterRecorderConf struct {
 	object    string
 	instance  string
 	recorders map[string]recordFunc
 }
 
-var perfCounterRecorders = []perfCounterRecorderConf{
+var totalPerfCounterRecorders = []perfCounterRecorderConf{
 	{
-		object:   "Web Service",
+		object:   "Process",
 		instance: "_Total",
+		recorders: map[string]recordFunc{
+			"Thread Count": func(mb *metadata.MetricsBuilder, ts pcommon.Timestamp, val float64) {
+				mb.RecordIisThreadActiveDataPoint(ts, int64(val))
+			},
+		},
+	},
+}
+
+var sitePerfCounterRecorders = []perfCounterRecorderConf{
+	{
+		object:   "Web Service",
+		instance: "*",
 		recorders: map[string]recordFunc{
 			"Current Connections": func(mb *metadata.MetricsBuilder, ts pcommon.Timestamp, val float64) {
 				mb.RecordIisConnectionActiveDataPoint(ts, int64(val))
@@ -87,6 +99,9 @@ var perfCounterRecorders = []perfCounterRecorderConf{
 			},
 		},
 	},
+}
+
+var appPoolPerfCounterRecorders = []perfCounterRecorderConf{
 	{
 		object:   "HTTP Service Request Queues",
 		instance: "*",
@@ -102,13 +117,4 @@ var perfCounterRecorders = []perfCounterRecorderConf{
 			},
 		},
 	},
-	{
-		object:   "Process",
-		instance: "_Total",
-		recorders: map[string]recordFunc{
-			"Thread Count": func(mb *metadata.MetricsBuilder, ts pcommon.Timestamp, val float64) {
-				mb.RecordIisThreadActiveDataPoint(ts, int64(val))
-			},
-		},
-	},
 }

receiver/iisreceiver/scraper.go

@@ -34,11 +34,13 @@ import (
 )
 
 type iisReceiver struct {
-	params           component.TelemetrySettings
-	config           *Config
-	consumer         consumer.Metrics
-	watcherRecorders []watcherRecorder
-	metricBuilder    *metadata.MetricsBuilder
+	params                  component.TelemetrySettings
+	config                  *Config
+	consumer                consumer.Metrics
+	totalWatcherRecorders   []watcherRecorder
+	siteWatcherRecorders    []watcherRecorder
+	appPoolWatcherRecorders []watcherRecorder
+	metricBuilder           *metadata.MetricsBuilder
 
 	// for mocking
 	newWatcher func(string, string, string) (winperfcounters.PerfCounterWatcher, error)
@@ -63,29 +65,29 @@ func newIisReceiver(settings component.ReceiverCreateSettings, cfg *Config, cons
 
 // start builds the paths to the watchers
 func (rcvr *iisReceiver) start(ctx context.Context, host component.Host) error {
-	rcvr.watcherRecorders = []watcherRecorder{}
+	errs := &scrapererror.ScrapeErrors{}
 
-	var errors scrapererror.ScrapeErrors
-	for _, pcr := range perfCounterRecorders {
-		for perfCounterName, recorder := range pcr.recorders {
-			w, err := rcvr.newWatcher(pcr.object, pcr.instance, perfCounterName)
-			if err != nil {
-				errors.AddPartial(1, err)
-				continue
-			}
-			rcvr.watcherRecorders = append(rcvr.watcherRecorders, watcherRecorder{w, recorder})
-		}
-	}
+	rcvr.totalWatcherRecorders = rcvr.buildWatcherRecorders(totalPerfCounterRecorders, errs)
+	rcvr.siteWatcherRecorders = rcvr.buildWatcherRecorders(sitePerfCounterRecorders, errs)
+	rcvr.appPoolWatcherRecorders = rcvr.buildWatcherRecorders(appPoolPerfCounterRecorders, errs)
 
-	return errors.Combine()
+	return errs.Combine()
 }
 
 // scrape pulls counter values from the watchers
 func (rcvr *iisReceiver) scrape(ctx context.Context) (pmetric.Metrics, error) {
 	var errs error
 	now := pcommon.NewTimestampFromTime(time.Now())
 
-	for _, wr := range rcvr.watcherRecorders {
+	rcvr.scrapeInstanceMetrics(now, rcvr.siteWatcherRecorders, metadata.WithIisSite)
+	rcvr.scrapeInstanceMetrics(now, rcvr.appPoolWatcherRecorders, metadata.WithIisApplicationPool)
+	rcvr.scrapeTotalMetrics(now)
+
+	return rcvr.metricBuilder.Emit(), errs
+}
+
+func (rcvr *iisReceiver) scrapeTotalMetrics(now pcommon.Timestamp) {
+	for _, wr := range rcvr.totalWatcherRecorders {
 		counterValues, err := wr.watcher.ScrapeData()
 		if err != nil {
 			rcvr.params.Logger.Warn("some performance counters could not be scraped; ", zap.Error(err))
@@ -98,13 +100,85 @@ func (rcvr *iisReceiver) scrape(ctx context.Context) (pmetric.Metrics, error) {
 		wr.recorder(rcvr.metricBuilder, now, value)
 	}
 
-	return rcvr.metricBuilder.Emit(), errs
+	// resource for total metrics is empty
+	// this makes it so that the order that the scrape functions are called doesn't matter
+	rcvr.metricBuilder.EmitForResource()
+}
+
+type valRecorder struct {
+	val    float64
+	record recordFunc
+}
+
+func (rcvr *iisReceiver) scrapeInstanceMetrics(now pcommon.Timestamp, wrs []watcherRecorder, resourceOption func(string) metadata.ResourceMetricsOption) {
+	// Maintain a map of instance -> {val, recordFunc}
+	// so that we can emit all metrics for a particular instance (site, app_pool) at once,
+	// keeping them in a single resource metric.
+	instanceToRecorders := map[string][]valRecorder{}
+
+	for _, wr := range wrs {
+		counterValues, err := wr.watcher.ScrapeData()
+		if err != nil {
+			rcvr.params.Logger.Warn("some performance counters could not be scraped; ", zap.Error(err))
+			continue
+		}
+
+		// This avoids recording the _Total instance.
+		// The _Total instance may be the only instance, because some instances require elevated permissions
+		// to list and scrape. In these cases, the per-instance metric is not available, and should not be recorded.
+		if len(counterValues) == 1 && counterValues[0].InstanceName == "" {
+			rcvr.params.Logger.Warn("Performance counter was scraped, but only the _Total instance was available, skipping metric...", zap.String("path", wr.watcher.Path()))
+			continue
+		}
+
+		for _, cv := range counterValues {
+			instanceToRecorders[cv.InstanceName] = append(instanceToRecorders[cv.InstanceName],
+				valRecorder{
+					val:    cv.Value,
+					record: wr.recorder,
+				})
+		}
+	}
+
+	// record all metrics for each instance, then emit them all as a single resource metric
+	for instanceName, recorders := range instanceToRecorders {
+		for _, recorder := range recorders {
+			recorder.record(rcvr.metricBuilder, now, recorder.val)
+		}
+
+		rcvr.metricBuilder.EmitForResource(resourceOption(instanceName))
+	}
 }
 
 // shutdown closes the watchers
 func (rcvr iisReceiver) shutdown(ctx context.Context) error {
 	var errs error
-	for _, wr := range rcvr.watcherRecorders {
+	errs = multierr.Append(errs, closeWatcherRecorders(rcvr.totalWatcherRecorders))
+	errs = multierr.Append(errs, closeWatcherRecorders(rcvr.siteWatcherRecorders))
+	errs = multierr.Append(errs, closeWatcherRecorders(rcvr.appPoolWatcherRecorders))
+	return errs
+}
+
+func (rcvr *iisReceiver) buildWatcherRecorders(confs []perfCounterRecorderConf, scrapeErrors *scrapererror.ScrapeErrors) []watcherRecorder {
+	wrs := []watcherRecorder{}
+
+	for _, pcr := range confs {
+		for perfCounterName, recorder := range pcr.recorders {
+			w, err := rcvr.newWatcher(pcr.object, pcr.instance, perfCounterName)
+			if err != nil {
+				scrapeErrors.AddPartial(1, err)
+				continue
+			}
+			wrs = append(wrs, watcherRecorder{w, recorder})
+		}
+	}
+
+	return wrs
+}
+
+func closeWatcherRecorders(wrs []watcherRecorder) error {
+	var errs error
+	for _, wr := range wrs {
 		err := wr.watcher.Close()
 		if err != nil {
 			errs = multierr.Append(errs, err)

receiver/iisreceiver/scraper_test.go

@@ -78,7 +78,7 @@ func TestScrapeFailure(t *testing.T) {
 	expectedError := "failure to collect metric"
 	mockWatcher, err := newMockWatcherFactory(fmt.Errorf(expectedError), 1)("", "", "")
 	require.NoError(t, err)
-	scraper.watcherRecorders = []watcherRecorder{
+	scraper.totalWatcherRecorders = []watcherRecorder{
 		{
 			watcher: mockWatcher,
 			recorder: func(mb *metadata.MetricsBuilder, ts pcommon.Timestamp, val float64) {
@@ -115,7 +115,7 @@ func (mpc *mockPerfCounter) Path() string {
 
 // ScrapeData
 func (mpc *mockPerfCounter) ScrapeData() ([]winperfcounters.CounterValue, error) {
-	return []winperfcounters.CounterValue{{Value: 1}}, mpc.watchErr
+	return []winperfcounters.CounterValue{{InstanceName: "Instance", Value: 1}}, mpc.watchErr
 }
 
 // Close