Httpcheckreceiver log response

receiver/httpcheckreceiver/README.md

@@ -3,7 +3,7 @@
 <!-- status autogenerated section -->
 | Status        |           |
 | ------------- |-----------|
-| Stability     | [development]: metrics   |
+| Stability     | [development]: metrics, logs   |
 | Distributions | [contrib] |
 | Issues        | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Areceiver%2Fhttpcheck%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Areceiver%2Fhttpcheck) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Areceiver%2Fhttpcheck%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Areceiver%2Fhttpcheck) |
 | [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner)    | [@codeboten](https://www.github.com/codeboten) |
@@ -60,4 +60,23 @@ receivers:
 
 Details about the metrics produced by this receiver can be found in [documentation.md](./documentation.md)
 
-[confighttp]: https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/confighttp#client-configuration
+## Logs
+
+Additionally, the responses from the checks can be reported as logs, including
+response body, duration (nanoseconds), endpoint and status code. `httpcheck` MUST
+be enabled for both metrics and logs, e.g.
+
+```yaml
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp, httpcheck]
+      processors: [batch]
+      exporters: [debug]
+    logs:
+      receivers: [otlp, httpcheck]
+      processors: [batch]
+      exporters: [debug]
+```
+
+[confighttp]: https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/confighttp#client-configuration

receiver/httpcheckreceiver/factory.go

@@ -20,10 +20,19 @@ var errConfigNotHTTPCheck = errors.New("config was not a HTTP check receiver con
 
 // NewFactory creates a new receiver factory
 func NewFactory() receiver.Factory {
+	f := &httpcheckReceiverFactory{
+		httpScrapers: make(map[*Config]*httpcheckScraper),
+	}
 	return receiver.NewFactory(
 		metadata.Type,
 		createDefaultConfig,
-		receiver.WithMetrics(createMetricsReceiver, metadata.MetricsStability))
+		receiver.WithMetrics(f.createMetricsReceiver, metadata.MetricsStability),
+		receiver.WithLogs(f.createLogsReceiver, metadata.LogsStability),
+	)
+}
+
+type httpcheckReceiverFactory struct {
+	httpScrapers map[*Config]*httpcheckScraper
 }
 
 func createDefaultConfig() component.Config {
@@ -37,17 +46,67 @@ func createDefaultConfig() component.Config {
 	}
 }
 
-func createMetricsReceiver(_ context.Context, params receiver.CreateSettings, rConf component.Config, consumer consumer.Metrics) (receiver.Metrics, error) {
-	cfg, ok := rConf.(*Config)
+func (factory *httpcheckReceiverFactory) ensureScraper(
+	params receiver.CreateSettings,
+	config component.Config) (*httpcheckScraper, error) {
+
+	rconfig, ok := config.(*Config)
 	if !ok {
 		return nil, errConfigNotHTTPCheck
 	}
 
-	httpcheckScraper := newScraper(cfg, params)
+	httpcheckScraper := factory.httpScrapers[rconfig]
+	if httpcheckScraper != nil {
+		return httpcheckScraper, nil
+	}
+
+	httpcheckScraper = newScraper(rconfig, params)
+	factory.httpScrapers[rconfig] = httpcheckScraper
+	return httpcheckScraper, nil
+}
+
+func (factory *httpcheckReceiverFactory) createMetricsReceiver(
+	_ context.Context,
+	params receiver.CreateSettings,
+	rConf component.Config,
+	consumer consumer.Metrics) (receiver.Metrics, error) {
+
+	httpcheckScraper, err := factory.ensureScraper(params, rConf)
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := rConf.(*Config)
 	scraper, err := scraperhelper.NewScraper(metadata.Type.String(), httpcheckScraper.scrape, scraperhelper.WithStart(httpcheckScraper.start))
 	if err != nil {
 		return nil, err
 	}
 
 	return scraperhelper.NewScraperControllerReceiver(&cfg.ControllerConfig, params, consumer, scraperhelper.AddScraper(scraper))
 }
+
+func (factory *httpcheckReceiverFactory) createLogsReceiver(
+	_ context.Context,
+	params receiver.CreateSettings,
+	rConf component.Config,
+	consumer consumer.Logs) (receiver.Logs, error) {
+
+	httpcheckScraper, err := factory.ensureScraper(params, rConf)
+	if err != nil {
+		return nil, err
+	}
+
+	httpcheckScraper.logs = consumer
+	return &nopReceiver{}, nil
+}
+
+type nopReceiver struct {
+}
+
+func (receiver *nopReceiver) Start(_ context.Context, _ component.Host) error {
+	return nil
+}
+
+func (receiver *nopReceiver) Shutdown(_ context.Context) error {
+	return nil
+}

receiver/httpcheckreceiver/logsbuilder.go

@@ -0,0 +1,62 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package httpcheckreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/httpcheckreceiver"
+
+import (
+	"time"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/pdata/pcommon"
+	"go.opentelemetry.io/collector/pdata/plog"
+	"go.opentelemetry.io/collector/receiver"
+)
+
+type LogsBuilder struct {
+	logBuffer  plog.Logs
+	logRecords plog.LogRecordSlice
+	buildInfo  component.BuildInfo
+}
+
+func NewLogsBuilder(settings receiver.CreateSettings) *LogsBuilder {
+	lb := &LogsBuilder{
+		buildInfo: settings.BuildInfo,
+	}
+	lb.reset()
+	return lb
+}
+
+func (lb *LogsBuilder) reset() {
+	out := plog.NewLogs()
+	logs := out.ResourceLogs()
+	rls := logs.AppendEmpty()
+
+	ills := rls.ScopeLogs().AppendEmpty()
+	ills.Scope().SetName("otelcol/httpcheckreceiver")
+	ills.Scope().SetVersion(lb.buildInfo.Version)
+
+	lb.logBuffer = out
+	lb.logRecords = ills.LogRecords()
+}
+
+func (lb *LogsBuilder) RecordResponse(
+	endpoint string,
+	body string,
+	statusCode int,
+	timestamp pcommon.Timestamp,
+	elapsed time.Duration) {
+
+	lr := lb.logRecords.AppendEmpty()
+	lr.SetTimestamp(timestamp)
+	lr.Body().SetStr(body)
+	attrs := lr.Attributes()
+	attrs.PutStr("http.url", endpoint)
+	attrs.PutInt("http.status_code", int64(statusCode))
+	attrs.PutInt("http.client.request.duration", elapsed.Milliseconds())
+}
+
+func (lb *LogsBuilder) Emit() plog.Logs {
+	logs := lb.logBuffer
+	lb.reset()
+	return logs
+}

receiver/httpcheckreceiver/metadata.yaml

@@ -4,7 +4,7 @@ scope_name: otelcol/httpcheckreceiver
 status:
   class: receiver
   stability:
-    development: [metrics]
+    development: [metrics, logs]
   distributions: [contrib]
   warnings: []
   codeowners:

receiver/httpcheckreceiver/scraper.go

@@ -6,11 +6,13 @@ package httpcheckreceiver // import "github.com/open-telemetry/opentelemetry-col
 import (
 	"context"
 	"errors"
+	"io"
 	"net/http"
 	"sync"
 	"time"
 
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/consumer"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/pmetric"
 	"go.opentelemetry.io/collector/receiver"
@@ -30,6 +32,8 @@ type httpcheckScraper struct {
 	cfg      *Config
 	settings component.TelemetrySettings
 	mb       *metadata.MetricsBuilder
+	lb       *LogsBuilder
+	logs     consumer.Logs
 }
 
 // start starts the scraper by creating a new HTTP Client on the scraper
@@ -68,8 +72,9 @@ func (h *httpcheckScraper) scrape(ctx context.Context) (pmetric.Metrics, error)
 
 			start := time.Now()
 			resp, err := targetClient.Do(req)
+			duration := time.Since(start)
 			mux.Lock()
-			h.mb.RecordHttpcheckDurationDataPoint(now, time.Since(start).Milliseconds(), h.cfg.Targets[targetIndex].Endpoint)
+			h.mb.RecordHttpcheckDurationDataPoint(now, duration.Milliseconds(), h.cfg.Targets[targetIndex].Endpoint)
 
 			statusCode := 0
 			if err != nil {
@@ -85,19 +90,51 @@ func (h *httpcheckScraper) scrape(ctx context.Context) (pmetric.Metrics, error)
 					h.mb.RecordHttpcheckStatusDataPoint(now, int64(0), h.cfg.Targets[targetIndex].Endpoint, int64(statusCode), req.Method, class)
 				}
 			}
+
+			if h.logs != nil && statusCode != 0 {
+				err = h.logResponse(ctx, h.cfg.Targets[targetIndex].Endpoint, resp, now, duration)
+				if err != nil {
+					h.settings.Logger.Error("failed to log response", zap.Error(err))
+				}
+			}
+
 			mux.Unlock()
 		}(client, idx)
 	}
 
 	wg.Wait()
 
+	if h.logs != nil {
+		err := h.logs.ConsumeLogs(ctx, h.lb.Emit())
+		if err != nil {
+			h.settings.Logger.Error("failed to consume logs", zap.Error(err))
+		}
+	}
 	return h.mb.Emit(), nil
 }
 
+func (h *httpcheckScraper) logResponse(
+	_ context.Context,
+	endpoint string,
+	resp *http.Response,
+	timestamp pcommon.Timestamp,
+	elapsed time.Duration) error {
+
+	defer resp.Body.Close()
+	bodyBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	h.lb.RecordResponse(endpoint, string(bodyBytes), resp.StatusCode, timestamp, elapsed)
+	return nil
+}
+
 func newScraper(conf *Config, settings receiver.CreateSettings) *httpcheckScraper {
 	return &httpcheckScraper{
 		cfg:      conf,
 		settings: settings.TelemetrySettings,
 		mb:       metadata.NewMetricsBuilder(conf.MetricsBuilderConfig, settings),
+		lb:       NewLogsBuilder(settings),
 	}
 }

receiver/httpcheckreceiver/scraper_test.go

@@ -9,9 +9,12 @@ import (
 	"net/http/httptest"
 	"path/filepath"
 	"testing"
+	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/consumer/consumertest"
 	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/config/configtls"
 	"go.opentelemetry.io/collector/pdata/pmetric"
@@ -189,6 +192,30 @@ func TestScaperScrape(t *testing.T) {
 	}
 }
 
+func TestScraperScrapeLogs(t *testing.T) {
+	cfg := createDefaultConfig().(*Config)
+	ms := newMockServer(t, 200)
+	defer ms.Close()
+
+	cfg.Targets = append(cfg.Targets, &targetConfig{
+		ClientConfig: confighttp.ClientConfig{
+			Endpoint: ms.URL,
+		},
+	})
+
+	logsSink := new(consumertest.LogsSink)
+	scraper := newScraper(cfg, receivertest.NewNopCreateSettings())
+	scraper.logs = logsSink
+
+	require.NoError(t, scraper.start(context.Background(), componenttest.NewNopHost()))
+	_, err := scraper.scrape(context.Background())
+	require.NoError(t, err)
+
+	assert.Eventually(t, func() bool {
+		return logsSink.LogRecordCount() == 1
+	}, time.Second, 10*time.Millisecond)
+}
+
 func TestNilClient(t *testing.T) {
 	scraper := newScraper(createDefaultConfig().(*Config), receivertest.NewNopCreateSettings())
 	actualMetrics, err := scraper.scrape(context.Background())