diff --git a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java index fdd70280bb817..ca0b2083c414b 100644 --- a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java +++ b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java @@ -38,6 +38,7 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .feature(FeatureFlag.SYNTHETIC_VECTORS) .feature(FeatureFlag.RANDOM_SAMPLING) .feature(FeatureFlag.EXTENDED_DOC_VALUES_PARAMS) + .feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG) .build(); public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 5a5ae03ab938f..2560dd74b7292 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -4,6 +4,10 @@ setup: cluster_features: "gte_v8.14.0" reason: _tsid hashing introduced in 8.13 and tsid routing changed in 8.14 + - skip: + cluster_features: "index.time_series_synthetic_id" + reason: when cluster has synthetic_id feature use 26_id_generation_synthetic_id_false instead + - do: indices.create: index: id_generation_test diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/26_id_generation_synthetic_id_false.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/26_id_generation_synthetic_id_false.yml new file mode 100644 index 0000000000000..e73c88b0da551 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/26_id_generation_synthetic_id_false.yml @@ -0,0 +1,510 @@ +--- +setup: + - requires: + cluster_features: "index.time_series_synthetic_id" + reason: explicit synthetic_id false only applies when cluster has the feature + - requires: + cluster_features: "gte_v8.14.0" + reason: _tsid hashing introduced in 8.13 and tsid routing changed in 8.14 + + - do: + indices.create: + index: id_generation_test_synthetic_id_false + body: + settings: + index: + mode: time_series + routing_path: [metricset, k8s.pod.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + number_of_shards: 2 + mapping: + synthetic_id: false + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + ip: + type: ip + network: + properties: + tx: + type: long + rx: + type: long + + - do: + bulk: + refresh: true + index: id_generation_test_synthetic_id_false + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}' + +--- +generates a consistent id: + - do: + bulk: + refresh: true + index: id_generation_test_synthetic_id_false + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} + + - do: + bulk: + refresh: true + index: id_generation_test_synthetic_id_false + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} + + - do: + get: + index: id_generation_test_synthetic_id_false + id: cn4excfoxSs_KdA5AAABeRnRFAY + - match: {_index: id_generation_test_synthetic_id_false} + - match: {_id: cn4excfoxSs_KdA5AAABeRnRFAY} + - match: + _source: + "@timestamp": "2021-04-28T18:50:03.142Z" + metricset: pod + k8s: + pod: + name: dog + uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 + ip: 10.10.55.3 + network: + tx: 1434521831 + rx: 530575198 + + + - do: + search: + index: id_generation_test_synthetic_id_false + body: + query: + match_all: {} + sort: ["@timestamp"] + _source: true + docvalue_fields: [_ts_routing_hash] + + - match: {hits.total.value: 9} + + - match: { hits.hits.0._id: cn4excfoxSs_KdA5AAABeRnRFAY } + - match: { hits.hits.0.fields._ts_routing_hash: [ cn4exQ ] } + - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.1._id: cZZNs7B9sSWsyrL5AAABeRnRGTM } + - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:04.467Z } + - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.2._id: cn4excfoxSs_KdA5AAABeRnRYiY } + - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:23.142Z } + - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.3._id: cZZNs7B9sSWsyrL5AAABeRnRZ1M } + - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:24.467Z } + - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.4._id: cZZNs7B9sSWsyrL5AAABeRnRtXM } + - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:44.467Z } + - match: { hits.hits.4._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.5._id: cn4excfoxSs_KdA5AAABeRnR11Y } + - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:50:53.142Z } + - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.6._id: cn4excfoxSs_KdA5AAABeRnR_mY } + - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:03.142Z } + - match: { hits.hits.6._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.7._id: cZZNs7B9sSWsyrL5AAABeRnSA5M } + - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:51:04.467Z } + - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.8._id: cZZNs7B9sSWsyrL5AAABeRnS7fM } + - match: { hits.hits.8._source.@timestamp: 2021-04-28T18:52:04.467Z } + - match: { hits.hits.8._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - do: + search: + index: id_generation_test_synthetic_id_false + body: + query: + match_all: {} + sort: ["@timestamp", "_ts_routing_hash"] + _source: true + search_after: [ "2021-04-28T18:50:03.142Z", "cn4exQ" ] + docvalue_fields: [_ts_routing_hash] + + - match: {hits.total.value: 9} + + - match: { hits.hits.0._id: cZZNs7B9sSWsyrL5AAABeRnRGTM } + - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:04.467Z } + - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.1._id: cn4excfoxSs_KdA5AAABeRnRYiY } + - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z } + - match: { hits.hits.1._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.2._id: cZZNs7B9sSWsyrL5AAABeRnRZ1M } + - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:24.467Z } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.3._id: cZZNs7B9sSWsyrL5AAABeRnRtXM } + - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:44.467Z } + - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.4._id: cn4excfoxSs_KdA5AAABeRnR11Y } + - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:53.142Z } + - match: { hits.hits.4._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.5._id: cn4excfoxSs_KdA5AAABeRnR_mY } + - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:51:03.142Z } + - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.6._id: cZZNs7B9sSWsyrL5AAABeRnSA5M } + - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:04.467Z } + - match: { hits.hits.6._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.7._id: cZZNs7B9sSWsyrL5AAABeRnS7fM } + - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:52:04.467Z } + - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } +--- +index a new document on top of an old one: + - do: + search: + index: id_generation_test_synthetic_id_false + body: + size: 0 + aggs: + max_tx: + max: + field: k8s.pod.network.tx + max_rx: + min: + field: k8s.pod.network.rx + - match: {aggregations.max_tx.value: 2.012916202E9} + - match: {aggregations.max_rx.value: 5.30575198E8} + + - do: + index: + refresh: true + index: id_generation_test_synthetic_id_false + op_type: index + body: + "@timestamp": "2021-04-28T18:51:03.142Z" + metricset: pod + k8s: + pod: + name: dog + uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 + ip: 10.10.55.3 + network: + tx: 111434595272 + rx: 430605511 + - match: {_id: cn4excfoxSs_KdA5AAABeRnR_mY} + + - do: + search: + index: id_generation_test_synthetic_id_false + body: + size: 0 + aggs: + max_tx: + max: + field: k8s.pod.network.tx + min_rx: + min: + field: k8s.pod.network.rx + - match: {aggregations.max_tx.value: 1.11434595272E11} + - match: {aggregations.min_rx.value: 4.30605511E8} + +--- +index a new document on top of an old one over bulk: + - do: + search: + index: id_generation_test_synthetic_id_false + body: + size: 0 + aggs: + max_tx: + max: + field: k8s.pod.network.tx + max_rx: + min: + field: k8s.pod.network.rx + - match: {aggregations.max_tx.value: 2.012916202E9} + - match: {aggregations.max_rx.value: 5.30575198E8} + + - do: + bulk: + refresh: true + index: id_generation_test_synthetic_id_false + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' + - match: {items.0.index._id: cn4excfoxSs_KdA5AAABeRnR_mY} + + - do: + search: + index: id_generation_test_synthetic_id_false + body: + size: 0 + aggs: + max_tx: + max: + field: k8s.pod.network.tx + max_rx: + min: + field: k8s.pod.network.rx + - match: {aggregations.max_tx.value: 1.11434595272E11} + - match: {aggregations.max_rx.value: 4.30605511E8} + +--- +create operation on top of old document fails: + - do: + catch: "/\\[cn4excfoxSs_KdA5AAABeRnR_mY\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" + index: + refresh: true + index: id_generation_test_synthetic_id_false + body: + "@timestamp": "2021-04-28T18:51:03.142Z" + metricset: pod + k8s: + pod: + name: dog + uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 + ip: 10.10.55.3 + network: + tx: 111434595272 + rx: 430605511 + +--- +create operation on top of old document fails over bulk: + - do: + bulk: + refresh: true + index: id_generation_test_synthetic_id_false + body: + - '{"create": {}}' + - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' + - match: { items.0.create.error.reason: "[cn4excfoxSs_KdA5AAABeRnR_mY][KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + +--- +ids query: + - do: + search: + index: id_generation_test_synthetic_id_false + body: + fields: + - field: k8s.pod.network.tx + query: + ids: + values: ["cn4excfoxSs_KdA5AAABeRnR11Y", "cn4excfoxSs_KdA5AAABeRnR_mY"] + sort: ["@timestamp"] + - match: {hits.total.value: 2} + - match: {hits.hits.0._id: "cn4excfoxSs_KdA5AAABeRnR11Y"} + - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434587694]} + - match: {hits.hits.1._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } + - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [1434595272]} + +--- +get: + - do: + get: + index: id_generation_test_synthetic_id_false + id: cZZNs7B9sSWsyrL5AAABeRnSA5M + - match: {_index: id_generation_test_synthetic_id_false} + - match: {_id: cZZNs7B9sSWsyrL5AAABeRnSA5M} + - match: + _source: + "@timestamp": "2021-04-28T18:51:04.467Z" + metricset: pod + k8s: + pod: + name: cat + uid: 947e4ced-1786-4e53-9e0c-5c447e959507 + ip: 10.10.55.2 + network: + tx: 2012916202 + rx: 803685721 + +--- +get not found: + - do: + catch: missing + get: + index: id_generation_test_synthetic_id_false + id: not found + +--- +get with routing: + - do: + catch: bad_request + get: + index: id_generation_test_synthetic_id_false + id: cZZNs-xII2fZweptAAABeRnSA5M + routing: routing + +--- +delete: + - do: + delete: + index: id_generation_test_synthetic_id_false + id: cn4excfoxSs_KdA5AAABeRnR_mY + - match: {result: deleted} + +--- +delete not found: + - do: + catch: missing + delete: + index: id_generation_test_synthetic_id_false + id: not found + +--- +delete with routing: + - do: + catch: bad_request + delete: + index: id_generation_test_synthetic_id_false + id: not found + routing: routing + +--- +delete over _bulk: + # mget call added to investigate test failure: https://github.com/elastic/elasticsearch/issues/93852 + # (should be removed when test issue is resolved) + - do: + mget: + index: id_generation_test_synthetic_id_false + body: + ids: [ cn4excfoxSs_KdA5AAABeRnR_mY, cn4excfoxSs_KdA5AAABeRnR11Y ] + - match: { docs.0._index: "id_generation_test_synthetic_id_false" } + - match: { docs.0._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } + - match: { docs.0.found: true } + - match: { docs.1._index: "id_generation_test_synthetic_id_false" } + - match: { docs.1._id: "cn4excfoxSs_KdA5AAABeRnR11Y" } + - match: { docs.1.found: true } + + - do: + bulk: + index: id_generation_test_synthetic_id_false + body: + - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR_mY"}}' + - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR11Y"}}' + - '{"delete": {"_id": "not found ++ not found"}}' + - match: {items.0.delete.result: deleted} + - match: {items.1.delete.result: deleted} + - match: {items.2.delete.status: 404} + - match: {items.2.delete.error.reason: '/invalid\ id\ \[not\ found\ \+\+\ not\ found\]\ for\ index\ \[id_generation_test_synthetic_id_false\]\ in\ time.series\ mode/'} + +--- +routing_path matches deep object: + - do: + indices.create: + index: routing_path_test + body: + settings: + index: + mode: time_series + routing_path: [dim.**.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + dim: + properties: + foo: + properties: + bar: + properties: + baz: + properties: + uid: + type: keyword + time_series_dimension: true + + - do: + bulk: + refresh: true + index: routing_path_test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"bar": {"baz": {"uid": "uid1"}}}}}' + - match: {items.0.index.result: created} + - match: {items.0.index._id: OcEOGchJrjH1fFX8AAABeRnRGTM} + +--- +routing_path matches object: + - do: + indices.create: + index: routing_path_test_2 + body: + settings: + index: + mode: time_series + routing_path: [dim.*.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + dim: + properties: + foo: + properties: + uid: + type: keyword + time_series_dimension: true + + - do: + bulk: + refresh: true + index: routing_path_test_2 + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"uid": "uid1"}}}' + - match: {items.0.index.result: created} + - match: {items.0.index._id: 8bgiqW9JKwAyp1bZAAABeRnRGTM} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/27_id_generation_synthetic_id_true.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/27_id_generation_synthetic_id_true.yml new file mode 100644 index 0000000000000..2f5f058a69c01 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/27_id_generation_synthetic_id_true.yml @@ -0,0 +1,514 @@ +--- +setup: + - requires: + cluster_features: "index.time_series_synthetic_id" + reason: index.mapping.synthetic_id requires the tsdb_synthetic_id feature flag + - requires: + cluster_features: "gte_v8.14.0" + reason: _tsid hashing introduced in 8.13 and tsid routing changed in 8.14 + + - do: + indices.create: + index: id_generation_test_synthetic_id_true + body: + settings: + index: + mode: time_series + routing_path: [metricset, k8s.pod.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + number_of_shards: 2 + mapping: + synthetic_id: true + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + ip: + type: ip + network: + properties: + tx: + type: long + rx: + type: long + + - do: + bulk: + refresh: true + index: id_generation_test_synthetic_id_true + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}' + +--- +generates a consistent id: + - do: + bulk: + refresh: true + index: id_generation_test_synthetic_id_true + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - match: {items.0.index._id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i0SDLNNlnE} + + - do: + bulk: + refresh: true + index: id_generation_test_synthetic_id_true + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - match: {items.0.index._id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i0SDLNNlnE} + + - do: + get: + index: id_generation_test_synthetic_id_true + id: KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i7r-cUefnI + - match: {_index: id_generation_test_synthetic_id_true} + - match: {_id: KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i7r-cUefnI} + - match: + _source: + "@timestamp": "2021-04-28T18:50:03.142Z" + metricset: pod + k8s: + pod: + name: dog + uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 + ip: 10.10.55.3 + network: + tx: 1434521831 + rx: 530575198 + + + - do: + search: + index: id_generation_test_synthetic_id_true + body: + query: + match_all: {} + sort: ["@timestamp"] + _source: true + docvalue_fields: [_ts_routing_hash] + + - match: {hits.total.value: 9} + + - match: { hits.hits.0._id: KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i7r-cUefnI } + - match: { hits.hits.0.fields._ts_routing_hash: [ cn4exQ ] } + - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.1._id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i7mzLNNlnE } + - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:04.467Z } + - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.2._id: KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i6d2cUefnI } + - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:23.142Z } + - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.3._id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i6YrLNNlnE } + - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:24.467Z } + - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.4._id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i5KjLNNlnE } + - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:44.467Z } + - match: { hits.hits.4._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.5._id: KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4oqcUefnI } + - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:50:53.142Z } + - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.6._id: KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4BmcUefnI } + - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:03.142Z } + - match: { hits.hits.6._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.7._id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i38bLNNlnE } + - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:51:04.467Z } + - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.8._id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i0SDLNNlnE } + - match: { hits.hits.8._source.@timestamp: 2021-04-28T18:52:04.467Z } + - match: { hits.hits.8._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - do: + search: + index: id_generation_test_synthetic_id_true + body: + query: + match_all: {} + sort: ["@timestamp", "_ts_routing_hash"] + _source: true + search_after: [ "2021-04-28T18:50:03.142Z", "cn4exQ" ] + docvalue_fields: [_ts_routing_hash] + + - match: {hits.total.value: 9} + + - match: { hits.hits.0._id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i7mzLNNlnE } + - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:04.467Z } + - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.1._id: KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i6d2cUefnI } + - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z } + - match: { hits.hits.1._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.2._id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i6YrLNNlnE } + - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:24.467Z } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.3._id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i5KjLNNlnE } + - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:44.467Z } + - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.4._id: KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4oqcUefnI } + - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:53.142Z } + - match: { hits.hits.4._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.5._id: KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4BmcUefnI } + - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:51:03.142Z } + - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.6._id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i38bLNNlnE } + - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:04.467Z } + - match: { hits.hits.6._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.7._id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i0SDLNNlnE } + - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:52:04.467Z } + - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } +--- +index a new document on top of an old one: + - do: + search: + index: id_generation_test_synthetic_id_true + body: + size: 0 + aggs: + max_tx: + max: + field: k8s.pod.network.tx + max_rx: + min: + field: k8s.pod.network.rx + - match: {aggregations.max_tx.value: 2.012916202E9} + - match: {aggregations.max_rx.value: 5.30575198E8} + + - do: + index: + refresh: true + index: id_generation_test_synthetic_id_true + op_type: index + body: + "@timestamp": "2021-04-28T18:51:03.142Z" + metricset: pod + k8s: + pod: + name: dog + uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 + ip: 10.10.55.3 + network: + tx: 111434595272 + rx: 430605511 + - match: {_id: KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4BmcUefnI} + + - do: + search: + index: id_generation_test_synthetic_id_true + body: + size: 0 + aggs: + max_tx: + max: + field: k8s.pod.network.tx + min_rx: + min: + field: k8s.pod.network.rx + - match: {aggregations.max_tx.value: 1.11434595272E11} + - match: {aggregations.min_rx.value: 4.30605511E8} + +--- +index a new document on top of an old one over bulk: + - do: + search: + index: id_generation_test_synthetic_id_true + body: + size: 0 + aggs: + max_tx: + max: + field: k8s.pod.network.tx + max_rx: + min: + field: k8s.pod.network.rx + - match: {aggregations.max_tx.value: 2.012916202E9} + - match: {aggregations.max_rx.value: 5.30575198E8} + + - do: + bulk: + refresh: true + index: id_generation_test_synthetic_id_true + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' + - match: {items.0.index._id: KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4BmcUefnI} + + - do: + search: + index: id_generation_test_synthetic_id_true + body: + size: 0 + aggs: + max_tx: + max: + field: k8s.pod.network.tx + max_rx: + min: + field: k8s.pod.network.rx + - match: {aggregations.max_tx.value: 1.11434595272E11} + - match: {aggregations.max_rx.value: 4.30605511E8} + +--- +create operation on top of old document fails: + - do: + catch: "/\\[KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4BmcUefnI\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" + index: + refresh: true + index: id_generation_test_synthetic_id_true + body: + "@timestamp": "2021-04-28T18:51:03.142Z" + metricset: pod + k8s: + pod: + name: dog + uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 + ip: 10.10.55.3 + network: + tx: 111434595272 + rx: 430605511 + +--- +create operation on top of old document fails over bulk: + - do: + bulk: + refresh: true + index: id_generation_test_synthetic_id_true + body: + - '{"create": {}}' + - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' + - match: { items.0.create.error.reason: "[KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4BmcUefnI][KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + +--- +ids query: + - do: + search: + index: id_generation_test_synthetic_id_true + body: + fields: + - field: k8s.pod.network.tx + query: + ids: + values: ["KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4oqcUefnI", "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4BmcUefnI"] + sort: ["@timestamp"] + - match: {hits.total.value: 2} + - match: {hits.hits.0._id: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4oqcUefnI"} + - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434587694]} + - match: {hits.hits.1._id: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4BmcUefnI" } + - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [1434595272]} + +--- +get: + - do: + get: + index: id_generation_test_synthetic_id_true + id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i38bLNNlnE + - match: {_index: id_generation_test_synthetic_id_true} + - match: {_id: KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0p___6G5i38bLNNlnE} + - match: + _source: + "@timestamp": "2021-04-28T18:51:04.467Z" + metricset: pod + k8s: + pod: + name: cat + uid: 947e4ced-1786-4e53-9e0c-5c447e959507 + ip: 10.10.55.2 + network: + tx: 2012916202 + rx: 803685721 + +--- +get not found: + - do: + catch: missing + get: + index: id_generation_test_synthetic_id_true + id: not found + +--- +get with routing: + - do: + catch: bad_request + get: + index: id_generation_test_synthetic_id_true + id: cZZNs-xII2fZweptAAABeRnSA5M + routing: routing + +--- +delete: + - do: + delete: + index: id_generation_test_synthetic_id_true + id: KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4BmcUefnI + - match: {result: deleted} + +--- +delete not found: + - do: + catch: missing + delete: + index: id_generation_test_synthetic_id_true + id: not found + +--- +delete with routing: + - do: + catch: bad_request + delete: + index: id_generation_test_synthetic_id_true + id: not found + routing: routing + +--- +delete over _bulk: + # mget call added to investigate test failure: https://github.com/elastic/elasticsearch/issues/93852 + # (should be removed when test issue is resolved) + - do: + mget: + index: id_generation_test_synthetic_id_true + body: + ids: [ KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4BmcUefnI, KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4oqcUefnI ] + - match: { docs.0._index: "id_generation_test_synthetic_id_true" } + - match: { docs.0._id: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4BmcUefnI" } + - match: { docs.0.found: true } + - match: { docs.1._index: "id_generation_test_synthetic_id_true" } + - match: { docs.1._id: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4oqcUefnI" } + - match: { docs.1.found: true } + + - do: + bulk: + index: id_generation_test_synthetic_id_true + body: + - '{"delete": {"_id": "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4BmcUefnI"}}' + - '{"delete": {"_id": "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKR___6G5i4oqcUefnI"}}' + - '{"delete": {"_id": "not found ++ not found"}}' + - match: {items.0.delete.result: deleted} + - match: {items.1.delete.result: deleted} + - match: {items.2.delete.status: 404} + - match: {items.2.delete.error.reason: '/invalid\ id\ \[not\ found\ \+\+\ not\ found\]\ for\ index\ \[id_generation_test_synthetic_id_true\]\ in\ time.series\ mode/'} + +--- +routing_path matches deep object: + - do: + indices.create: + index: routing_path_test_synthetic_id + body: + settings: + index: + mode: time_series + routing_path: [dim.**.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + number_of_replicas: 0 + number_of_shards: 2 + mapping: + synthetic_id: true + mappings: + properties: + "@timestamp": + type: date + dim: + properties: + foo: + properties: + bar: + properties: + baz: + properties: + uid: + type: keyword + time_series_dimension: true + + - do: + bulk: + refresh: true + index: routing_path_test_synthetic_id + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"bar": {"baz": {"uid": "uid1"}}}}}' + - match: {items.0.index.result: created} + - match: {items.0.index._id: JM470g16EHdYU4GgBL87HDcWrvVxXB-xBpIYlWsMx6RFR8fHaX___obmLubMGQ7BOQ} + +--- +routing_path matches object: + - do: + indices.create: + index: routing_path_test_2_synthetic_id + body: + settings: + index: + mode: time_series + routing_path: [dim.*.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + number_of_replicas: 0 + number_of_shards: 2 + mapping: + synthetic_id: true + mappings: + properties: + "@timestamp": + type: date + dim: + properties: + foo: + properties: + uid: + type: keyword + time_series_dimension: true + + - do: + bulk: + refresh: true + index: routing_path_test_2_synthetic_id + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"uid": "uid1"}}}' + - match: {items.0.index.result: created} + - match: {items.0.index._id: JFvylKRG_nnAwbD-SKLruAAWrvVxXB-xBpIYlWsMx6RFR8fHaX___obmLubMqSK48Q}