[Accurate counter PoC_v2] Aggregate counter downsampling using first value and reset docs.

x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml

@@ -76,18 +76,21 @@ setup:
           - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "multi-counter" : [10, 11, 12], "scaled-counter": 10.0, "multi-gauge": [100, 200, 150], "scaled-gauge": 100.0,  "network": {"tx": 2001818691, "rx": 802133794}, "created_at": "2021-04-28T19:34:00.000Z", "running": false, "number_of_containers": 2, "tags": ["backend", "prod"], "values": [2, 3, 6]}}}'
           - '{"index": {}}'
           - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.26", "multi-counter" : [21, 22, 23], "scaled-counter": 20.0, "multi-gauge": [90, 91, 95], "scaled-gauge": 90.0, "network": {"tx": 2005177954, "rx": 801479970}, "created_at": "2021-04-28T19:35:00.000Z", "running": true, "number_of_containers": 2, "tags": ["backend", "prod", "us-west1"], "values": [1, 1, 3]}}}'
+
           - '{"index": {}}'
           - '{"@timestamp": "2021-04-28T20:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.41", "multi-counter" : [1, 5, 10], "scaled-counter": 1.0, "multi-gauge": [103, 110, 109], "scaled-gauge": 104.0, "network": {"tx": 2006223737, "rx": 802337279}, "created_at": "2021-04-28T19:36:00.000Z", "running": true, "number_of_containers": 2, "tags": ["backend", "prod", "us-west2"], "values": [4, 1, 2]}}}'
           - '{"index": {}}'
           - '{"@timestamp": "2021-04-28T20:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.22", "multi-counter" : [101, 102, 105], "scaled-counter": 100.0, "multi-gauge": [100, 100, 100], "scaled-gauge": 102.0, "network": {"tx": 2012916202, "rx": 803685721}, "created_at": "2021-04-28T19:37:00.000Z", "running": true, "number_of_containers": 2, "tags": ["backend", "prod"], "values": [2, 3, 1]}}}'
           - '{"index": {}}'
+
           - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.33", "multi-counter" : [7, 11, 44], "scaled-counter": 7.0, "multi-gauge": [100, 100, 102], "scaled-gauge": 100.0, "network": {"tx": 1434521831, "rx": 530575198}, "created_at": "2021-04-28T19:42:00.000Z", "running": false, "number_of_containers": 1, "tags": ["backend", "test"], "values": [2, 3, 4]}}}'
           - '{"index": {}}'
           - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.56", "multi-counter" : [0, 0, 1], "scaled-counter": 0.0, "multi-gauge": [101, 102, 102], "scaled-gauge": 101.0, "network": {"tx": 1434577921, "rx": 530600088}, "created_at": "2021-04-28T19:43:00.000Z", "running": false, "number_of_containers": 1, "tags": ["backend", "test", "us-west2"], "values": [2, 1, 1]}}}'
           - '{"index": {}}'
+
           - '{"@timestamp": "2021-04-28T19:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.37", "multi-counter" : [1000, 1001, 1002], "scaled-counter": 1000.0, "multi-gauge": [99, 100, 110], "scaled-gauge": 99.0, "network": {"tx": 1434587694, "rx": 530604797}, "created_at": "2021-04-28T19:44:00.000Z", "running": true, "number_of_containers": 1, "tags": ["backend", "test", "us-west1"], "values": [4, 5, 2]}}}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T19:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.120", "multi-counter" : [76, 77, 78], "scaled-counter": 70.0, "multi-gauge": [95, 98, 100], "scaled-gauge": 95.0, "network": {"tx": 1434595272, "rx": 530605511}, "created_at": "2021-04-28T19:45:00.000Z", "running": true, "number_of_containers": 1, "tags": ["backend", "test", "us-west1"], "values": [3, 2, 1]}}}'
+          - '{"@timestamp": "2021-04-28T19:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.120", "multi-counter" : [76, 77, 78], "scaled-counter": 1070.0, "multi-gauge": [95, 98, 100], "scaled-gauge": 95.0, "network": {"tx": 1434595272, "rx": 530605511}, "created_at": "2021-04-28T19:45:00.000Z", "running": true, "number_of_containers": 1, "tags": ["backend", "test", "us-west1"], "values": [3, 2, 1]}}}'
 
   - do:
       indices.put_settings:
@@ -302,8 +305,12 @@ setup:
 ---
 "Downsample index":
   - requires:
-      cluster_features: ["gte_v8.13.0"]
-      reason: _tsid hashing introduced in 8.13
+      capabilities:
+        - method: POST
+          path: /{index}/_downsample/{target_index}
+          capabilities: [ "downsampling.store_reset_counters" ]
+      test_runner_features: [ "capabilities" ]
+      reason: Storing counter resets when downsampling was added in 9.4
 
   - do:
       allowed_warnings:
@@ -323,14 +330,17 @@ setup:
         body:
           sort: [ "_tsid", "@timestamp" ]
 
-  - length: { hits.hits: 4 }
+  - length: { hits.hits: 7 }
+
+  # Downsampled doc
   - match:  { hits.hits.0._source._doc_count: 2 }
+  - match:  { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
+  # Dimensions
   - match:  { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match:  { hits.hits.0._source.metricset: pod }
-
-  - match:  { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
-  - match:  { hits.hits.0._source.k8s\.pod\.multi-counter: 0 }
-  - match:  { hits.hits.0._source.k8s\.pod\.scaled-counter: 0.00 }
+  # Metrics
+  - match:  { hits.hits.0._source.k8s\.pod\.multi-counter: 7 }
+  - match:  { hits.hits.0._source.k8s\.pod\.scaled-counter: 7.0 }
   - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.min: 100 }
   - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.max: 102 }
   - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.sum: 607 }
@@ -342,13 +352,49 @@ setup:
   - match:  { hits.hits.0._source.k8s\.pod\.network\.tx.min: 1434521831 }
   - match:  { hits.hits.0._source.k8s\.pod\.network\.tx.max: 1434577921 }
   - match:  { hits.hits.0._source.k8s\.pod\.network\.tx.value_count: 2 }
+  # Labels
   - match:  { hits.hits.0._source.k8s\.pod\.ip: "10.10.55.56" }
   - match:  { hits.hits.0._source.k8s\.pod\.created_at: "2021-04-28T19:43:00.000Z" }
   - match:  { hits.hits.0._source.k8s\.pod\.number_of_containers: 1 }
   - match:  { hits.hits.0._source.k8s\.pod\.tags: ["backend", "test", "us-west2"] }
   - match:  { hits.hits.0._source.k8s\.pod\.values: [1, 1, 2] }
   - is_false: hits.hits.0._source.k8s\.pod\.running
 
+  # Reset doc
+  - is_false: hits.hits.1._source._doc_count
+  - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z }
+  # Dimensions
+  - match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.1._source.metricset: pod }
+  # Metrics
+  - match: { hits.hits.1._source.k8s\.pod\.multi-counter: 0 }
+  - match: { hits.hits.1._source.k8s\.pod\.scaled-counter: 0.0 }
+  # Only dimensions and counters that have been reset are in this doc
+  - is_false: hits.hits.1._source.k8s\.pod\.multi-gauge
+
+  # Next downsampled doc
+  - match:  { hits.hits.2._source._doc_count: 2 }
+  - match:  { hits.hits.2._source.@timestamp: 2021-04-28T19:00:00.000Z }
+  # Dimensions
+  - match:  { hits.hits.2._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match:  { hits.hits.2._source.metricset: pod }
+  # Metrics
+  - match:  { hits.hits.2._source.k8s\.pod\.multi-counter: 1000 }
+  - match:  { hits.hits.2._source.k8s\.pod\.scaled-counter: 1000.0 }
+  - match:  { hits.hits.2._source.k8s\.pod\.multi-gauge.min: 95 }
+
+  # Reset doc
+  - is_false: hits.hits.3._source._doc_count
+  - match: { hits.hits.3._source.@timestamp: 2021-04-28T19:51:03.142Z }
+  # Dimensions
+  - match: { hits.hits.3._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.3._source.metricset: pod }
+  # Metrics
+  - match: { hits.hits.3._source.k8s\.pod\.multi-counter: 76 }
+  # Only dimensions and counters that have been reset are in this doc
+  - is_false: hits.hits.3._source.k8s\.pod\.scaled-counter
+  - is_false: hits.hits.3._source.k8s\.pod\.multi-gauge
+
   # Assert downsample index settings
   - do:
       indices.get_settings:
@@ -381,6 +427,7 @@ setup:
   - match: { test-downsample.mappings.properties.k8s\.pod\.scaled-gauge.metrics: [ "min", "max", "sum", "value_count" ] }
   - match: { test-downsample.mappings.properties.k8s\.pod\.scaled-gauge.time_series_metric: gauge }
   - match: { test-downsample.mappings.properties.k8s\.pod\.uid.type: keyword }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.uid.type: keyword }
   - match: { test-downsample.mappings.properties.k8s\.pod\.uid.time_series_dimension: true }
 
 
@@ -815,8 +862,12 @@ setup:
 ---
 "Downsample a downsampled index":
   - requires:
-      cluster_features: ["gte_v8.13.0"]
-      reason: _tsid hashing introduced in 8.13
+      capabilities:
+        - method: POST
+          path: /{index}/_downsample/{target_index}
+          capabilities: [ "downsampling.store_reset_counters" ]
+      test_runner_features: [ "capabilities" ]
+      reason: Storing counter resets when downsampling was added in 9.4
 
   - do:
       allowed_warnings:
@@ -868,12 +919,13 @@ setup:
         body:
           sort: [ "_tsid", "@timestamp" ]
 
-  - length: { hits.hits: 3 }
-  - match: { hits.hits.0._source._doc_count: 4 }
+  - length: { hits.hits: 7 }
+  # Unfortunately, the doc count counts the reset documents as separate docs
+  - match: { hits.hits.0._source._doc_count: 6 }
   - match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match: { hits.hits.0._source.metricset: pod }
   - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
-  - match: { hits.hits.0._source.k8s\.pod\.multi-counter: 76 }
+  - match: { hits.hits.0._source.k8s\.pod\.multi-counter: 7 }
   - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.min: 95.0 }
   - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.max: 110.0 }
   - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.sum: 1209.0 }
@@ -887,15 +939,34 @@ setup:
   - match: { hits.hits.0._source.k8s\.pod\.tags: [ "backend", "test", "us-west1" ] }
   - match: { hits.hits.0._source.k8s\.pod\.values: [ 1, 2, 3 ] }
 
-  - match: { hits.hits.1._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match: { hits.hits.1._source.metricset: pod }
-  - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:00:00.000Z }
-  - match: { hits.hits.1._source._doc_count: 2 }
+  - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z }
+  - match: { hits.hits.1._source.k8s\.pod\.multi-counter: 0 }
 
-  - match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.2._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match: { hits.hits.2._source.metricset: pod }
-  - match: { hits.hits.2._source.@timestamp: 2021-04-28T20:00:00.000Z }
-  - match: { hits.hits.2._source._doc_count: 2 }
+  - match: { hits.hits.2._source.@timestamp: 2021-04-28T19:00:00.000Z }
+  - match: { hits.hits.2._source.k8s\.pod\.multi-counter: 1000 }
+
+  - match: { hits.hits.3._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.3._source.metricset: pod }
+  - match: { hits.hits.3._source.@timestamp: 2021-04-28T19:51:03.142Z }
+  - match: { hits.hits.3._source.k8s\.pod\.multi-counter: 76 }
+
+  - match: { hits.hits.4._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.4._source.metricset: pod }
+  - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:00:00.000Z }
+  - match: { hits.hits.4._source._doc_count: 3 } # 2 + 1 reset doc
+
+  - match: { hits.hits.5._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.5._source.metricset: pod }
+  - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:50:24.467Z }
+
+  - match: { hits.hits.6._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.6._source.metricset: pod }
+  - match: { hits.hits.6._source.@timestamp: 2021-04-28T20:00:00.000Z }
+  - match: { hits.hits.6._source._doc_count: 2 }
 
   - do:
       allowed_warnings:

x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml

@@ -1,9 +1,12 @@
 ---
 "Runtime fields accessing metric fields in downsample target index":
   - requires:
-      cluster_features: ["gte_v8.13.0"]
-      reason: _tsid hashing introduced in 8.13
-      test_runner_features: [close_to, allowed_warnings]
+      capabilities:
+        - method: POST
+          path: /{index}/_downsample/{target_index}
+          capabilities: [ "downsampling.store_reset_counters" ]
+      test_runner_features: [ capabilities, close_to, allowed_warnings]
+      reason: Storing counter resets when downsampling was added in 9.4
 
   - do:
       indices.create:
@@ -142,23 +145,23 @@
 
   - length: { hits.hits: 4 }
 
-  - close_to: { hits.hits.0.fields.received_kb.0: { value: 518142.3935, error: 0.0001 } }
-  - close_to: { hits.hits.0.fields.sent_kb.0: { value: 1400935.4472, error: 0.0001 } }
+  - close_to: { hits.hits.0.fields.received_kb.0: { value: 518139.8417, error: 0.0001 } }
+  - close_to: { hits.hits.0.fields.sent_kb.0: { value: 1400900.2255, error: 0.0001 } }
   - close_to: { hits.hits.0.fields.tx_kb.0: { value: 1400955.0009, error: 0.0001 } }
   - close_to: { hits.hits.0.fields.rx_kb.0: { value: 518164.1484, error: 0.0001 } }
 
-  - close_to: { hits.hits.1.fields.received_kb.0: { value: 518186.5039, error: 0.0001 } }
-  - close_to: { hits.hits.1.fields.sent_kb.0: { value: 1400988.2822, error: 0.0001 } }
+  - close_to: { hits.hits.1.fields.received_kb.0: { value: 518164.1699, error: 0.0001 } }
+  - close_to: { hits.hits.1.fields.sent_kb.0: { value: 1400966.6992, error: 0.0001 } }
   - close_to: { hits.hits.1.fields.tx_kb.0: { value: 1400971.9453, error: 0.0001 } }
   - close_to: { hits.hits.1.fields.rx_kb.0: { value: 518169.4443, error: 0.0001 } }
 
-  - close_to: { hits.hits.2.fields.received_kb.0: { value: 783343.5488, error: 0.0001 } }
-  - close_to: { hits.hits.2.fields.sent_kb.0: { value: 1954908.8779, error: 0.0001 } }
+  - close_to: { hits.hits.2.fields.received_kb.0: { value: 783333.7832, error: 0.0001 } }
+  - close_to: { hits.hits.2.fields.sent_kb.0: { value: 1954901.0654, error: 0.0001 } }
   - close_to: { hits.hits.2.fields.tx_kb.0: { value: 1958181.5957, error: 0.0001 } }
   - close_to: { hits.hits.2.fields.rx_kb.0: { value: 783333.7832, error: 0.0001 } }
 
-  - close_to: { hits.hits.3.fields.received_kb.0: { value: 783377.7343, error: 0.0001 } }
-  - close_to: { hits.hits.3.fields.sent_kb.0: { value: 1955339.7343, error: 0.0001 } }
+  - close_to: { hits.hits.3.fields.received_kb.0: { value: 783372.8505, error: 0.0001 } }
+  - close_to: { hits.hits.3.fields.sent_kb.0: { value: 1955096.3671, error: 0.0001 } }
   - close_to: { hits.hits.3.fields.tx_kb.0: { value: 1965738.4785, error: 0.0001 } }
   - close_to: { hits.hits.3.fields.rx_kb.0: { value: 784849.3369, error: 0.0001 } }