Pick timestamp from metric

based on #97 and #80 this provides the posibility to use a metric that has a unix style timestamp as the timestamp of the scraped metric

When deserializing objects we need to take the key json path into account as well like we would do for all the values as well. This allows collections to be defined still with each entry having a separate timestamp (e.g. list of time-stamped log messages).

Update examples for timestamp

Update Readme about staleness for custom timestamps

Signed-off-by: Jan Phillip Kretzschmar <[email protected]>

Author: janphkre

README.md

@@ -83,6 +83,10 @@ $ docker run --rm -it -p 9090:9090 -v $PWD/examples/prometheus.yml:/etc/promethe
 ```
 Then head over to http://localhost:9090/graph?g0.range_input=1h&g0.expr=example_value_active&g0.tab=1 or http://localhost:9090/targets to check the scraped metrics or the targets.
 
+## Using custom timestamps
+
+This exporter allows you to use a field of the metric as the (unix/epoch) timestamp for the data as an int64. However, this may lead to unexpected behaviour, as the prometheus implements a [Staleness](https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness) mechanism. Effectively, this means that samples older than 5 minutes can not be scrapped and will be ignored by the prometheus instance.
+
 ## Exposing metrics through HTTPS
 
 TLS configuration supported by this exporter can be found at [exporter-toolkit/web](https://github.com/prometheus/exporter-toolkit/blob/v0.5.1/docs/web-configuration.md)

config/config.go

@@ -14,82 +14,83 @@
 package config
 
 import (
-	"io/ioutil"
+    "io/ioutil"
 
-	pconfig "github.com/prometheus/common/config"
-	"gopkg.in/yaml.v2"
+    pconfig "github.com/prometheus/common/config"
+    "gopkg.in/yaml.v2"
 )
 
 // Metric contains values that define a metric
 type Metric struct {
-	Name      string
-	Path      string
-	Labels    map[string]string
-	Type      ScrapeType
-	ValueType ValueType
-	Help      string
-	Values    map[string]string
+    Name           string
+    Path           string
+    Labels         map[string]string
+    EpochTimestamp string
+    Type           ScrapeType
+    ValueType      ValueType
+    Help           string
+    Values         map[string]string
 }
 
 type ScrapeType string
 
 const (
-	ValueScrape  ScrapeType = "value" // default
-	ObjectScrape ScrapeType = "object"
+    ValueScrape  ScrapeType = "value" // default
+    ObjectScrape ScrapeType = "object"
 )
 
 type ValueType string
 
 const (
-	ValueTypeGauge   ValueType = "gauge"
-	ValueTypeCounter ValueType = "counter"
-	ValueTypeUntyped ValueType = "untyped"
+    ValueTypeGauge   ValueType = "gauge"
+    ValueTypeCounter ValueType = "counter"
+    ValueTypeUntyped ValueType = "untyped"
 )
 
 // Config contains multiple modules.
 type Config struct {
-	Modules map[string]Module `yaml:"modules"`
+    Modules map[string]Module `yaml:"modules"`
 }
 
 // Module contains metrics and headers defining a configuration
 type Module struct {
-	Headers          map[string]string        `yaml:"headers,omitempty"`
-	Metrics          []Metric                 `yaml:"metrics"`
-	HTTPClientConfig pconfig.HTTPClientConfig `yaml:"http_client_config,omitempty"`
-	Body             Body                     `yaml:"body,omitempty"`
-	ValidStatusCodes []int                    `yaml:"valid_status_codes,omitempty"`
+    Headers          map[string]string        `yaml:"headers,omitempty"`
+    Metrics          []Metric                 `yaml:"metrics"`
+    HTTPClientConfig pconfig.HTTPClientConfig `yaml:"http_client_config,omitempty"`
+    Body             Body                     `yaml:"body,omitempty"`
+    ValidStatusCodes []int                    `yaml:"valid_status_codes,omitempty"`
 }
 
 type Body struct {
-	Content    string `yaml:"content"`
-	Templatize bool   `yaml:"templatize,omitempty"`
+    Content    string `yaml:"content"`
+    Templatize bool   `yaml:"templatize,omitempty"`
 }
 
 func LoadConfig(configPath string) (Config, error) {
-	var config Config
-	data, err := ioutil.ReadFile(configPath)
-	if err != nil {
-		return config, err
-	}
+    var config Config
+    data, err := ioutil.ReadFile(configPath)
+    if err != nil {
+        return config, err
+    }
 
-	if err := yaml.Unmarshal(data, &config); err != nil {
-		return config, err
-	}
+    if err := yaml.Unmarshal(data, &config); err != nil {
+        return config, err
+    }
 
-	// Complete Defaults
-	for _, module := range config.Modules {
-		for i := 0; i < len(module.Metrics); i++ {
-			if module.Metrics[i].Type == "" {
-				module.Metrics[i].Type = ValueScrape
-			}
-			if module.Metrics[i].Help == "" {
-				module.Metrics[i].Help = module.Metrics[i].Name
-			}
-			if module.Metrics[i].ValueType == "" {
-				module.Metrics[i].ValueType = ValueTypeUntyped
-			}
-		}
-	}
+    // Complete Defaults
+    for _, module := range config.Modules {
+        for i := 0; i < len(module.Metrics); i++ {
+            if module.Metrics[i].Type == "" {
+                module.Metrics[i].Type = ValueScrape
+            }
+            if module.Metrics[i].Help == "" {
+                module.Metrics[i].Help = module.Metrics[i].Name
+            }
+            if module.Metrics[i].ValueType == "" {
+                module.Metrics[i].ValueType = ValueTypeUntyped
+            }
+        }
+    }
 
-	return config, nil
-}
+    return config, nil
+}

examples/config.yml

@@ -8,7 +8,12 @@ modules:
       labels:
         environment: beta # static label
         location: "planet-{.location}"          # dynamic label
-
+    - name: example_timestamped_value
+      path: "{ .values[?(@.state == "INACTIVE")] }"
+      epochTimestamp: "{ .timestamp }"
+      help: Example of a timestamped value scrape in the json
+      labels:
+        environment: beta # static label
     - name: example_value
       type: object
       help: Example of sub-level value scrapes from a json

examples/data.json

@@ -1,5 +1,6 @@
 {
     "counter": 1234,
+    "timestamp": 1657568506,
     "values": [
         {
             "id": "id-A",

exporter/collector.go

@@ -14,6 +14,7 @@
 package exporter
 
 import (
+	"time"
 	"bytes"
 	"encoding/json"
 
@@ -31,12 +32,13 @@ type JSONMetricCollector struct {
 }
 
 type JSONMetric struct {
-	Desc            *prometheus.Desc
-	Type            config.ScrapeType
-	KeyJSONPath     string
-	ValueJSONPath   string
-	LabelsJSONPaths []string
-	ValueType       prometheus.ValueType
+	Desc                    *prometheus.Desc
+	Type                    config.ScrapeType
+	KeyJSONPath             string
+	ValueJSONPath           string
+	LabelsJSONPaths         []string
+	ValueType               prometheus.ValueType
+	EpochTimestampJSONPath  string
 }
 
 func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) {
@@ -56,13 +58,13 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 			}
 
 			if floatValue, err := SanitizeValue(value); err == nil {
-
-				ch <- prometheus.MustNewConstMetric(
+				metric := prometheus.MustNewConstMetric(
 					m.Desc,
 					m.ValueType,
 					floatValue,
 					extractLabels(mc.Logger, mc.Data, m.LabelsJSONPaths)...,
 				)
+				ch <- timestampMetric(mc.Logger, m, mc.Data, metric)
 			} else {
 				level.Error(mc.Logger).Log("msg", "Failed to convert extracted value to float64", "path", m.KeyJSONPath, "value", value, "err", err, "metric", m.Desc)
 				continue
@@ -90,12 +92,13 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 					}
 
 					if floatValue, err := SanitizeValue(value); err == nil {
-						ch <- prometheus.MustNewConstMetric(
+						metric := prometheus.MustNewConstMetric(
 							m.Desc,
 							m.ValueType,
 							floatValue,
 							extractLabels(mc.Logger, jdata, m.LabelsJSONPaths)...,
 						)
+						ch <- timestampMetric(mc.Logger, m, jdata, metric)
 					} else {
 						level.Error(mc.Logger).Log("msg", "Failed to convert extracted value to float64", "path", m.ValueJSONPath, "value", value, "err", err, "metric", m.Desc)
 						continue
@@ -157,3 +160,21 @@ func extractLabels(logger log.Logger, data []byte, paths []string) []string {
 	}
 	return labels
 }
+
+func timestampMetric(logger log.Logger, m JSONMetric, data []byte, pm prometheus.Metric) prometheus.Metric {
+	if m.EpochTimestampJSONPath == "" {
+		return pm
+	}
+	ts, err := extractValue(logger, data, m.EpochTimestampJSONPath, false)
+	if err != nil {
+		level.Error(logger).Log("msg", "Failed to extract timestamp for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
+		return pm
+	}
+	epochTime, err := SanitizeIntValue(ts)
+	if err != nil {
+		level.Error(logger).Log("msg", "Failed to parse timestamp for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
+		return pm
+	}
+	timestamp := time.UnixMilli(epochTime)
+	return prometheus.NewMetricWithTimestamp(timestamp, pm)
+}

exporter/util.go

@@ -62,6 +62,19 @@ func SanitizeValue(s string) (float64, error) {
 	return value, fmt.Errorf(resultErr)
 }
 
+func SanitizeIntValue(s string) (int64, error) {
+	var err error
+	var value int64
+	var resultErr string
+
+	if value, err = strconv.ParseInt(s, 10, 64); err == nil {
+		return value, nil
+	}
+	resultErr = fmt.Sprintf("%s", err)
+
+	return value, fmt.Errorf(resultErr)
+}
+
 func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 	var (
 		metrics   []JSONMetric
@@ -91,9 +104,10 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 					variableLabels,
 					nil,
 				),
-				KeyJSONPath:     metric.Path,
-				LabelsJSONPaths: variableLabelsValues,
-				ValueType:       valueType,
+				KeyJSONPath:             metric.Path,
+				LabelsJSONPaths:         variableLabelsValues,
+				ValueType:               valueType,
+				EpochTimestampJSONPath:  metric.EpochTimestamp,
 			}
 			metrics = append(metrics, jsonMetric)
 		case config.ObjectScrape:
@@ -112,10 +126,11 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 						variableLabels,
 						nil,
 					),
-					KeyJSONPath:     metric.Path,
-					ValueJSONPath:   valuePath,
-					LabelsJSONPaths: variableLabelsValues,
-					ValueType:       valueType,
+					KeyJSONPath:             metric.Path,
+					ValueJSONPath:           valuePath,
+					LabelsJSONPaths:         variableLabelsValues,
+					ValueType:               valueType,
+					EpochTimestampJSONPath:  metric.EpochTimestamp,
 				}
 				metrics = append(metrics, jsonMetric)
 			}