[Logs Data Telemetry] Add logs-dsns `logs-*-*` pattern to obs telemetry patterns by awahab07 · Pull Request #192874 · elastic/kibana

awahab07 · 2024-09-13T14:23:44Z

Summary

Adds the logs-*-* logs pattern in observability telemetry patterns to separate out logs with indices named per the Data Stream Naming Scheme (ref) vs. the generic logs indices not conforming to DSNS naming.

For a typical scenario (test data), the addition of logs-*-* affects the collection as (notice the addition of "dsns-logs" object):

Before

After

[
  {
    "pattern_name": "heartbeat",
    "shipper": "heartbeat",
    "doc_count": 9530,
    "structure_level": {
      "5": 9530
    },
    "index_count": 1,
    "failure_store_doc_count": 9530,
    "failure_store_index_count": 1,
    "namespace_count": 0,
    "field_count": 1508,
    "field_existence": {
      "container.id": 9530,
      "log.level": 9530,
      "container.name": 9530,
      "host.name": 9530,
      "host.hostname": 9530,
      "kubernetes.pod.name": 9530,
      "kubernetes.pod.uid": 9530,
      "cloud.provider": 9530,
      "agent.type": 9530,
      "event.dataset": 9530,
      "event.category": 9530,
      "event.module": 9530,
      "service.name": 9530,
      "service.type": 9530,
      "service.version": 9530,
      "message": 9530,
      "event.original": 9530,
      "error.message": 9530,
      "@timestamp": 9530,
      "data_stream.dataset": 9530,
      "data_stream.namespace": 9530,
      "data_stream.type": 9530
    },
    "size_in_bytes": 13090458,
    "managed_by": [],
    "package_name": [],
    "beat": [
      "heartbeat"
    ]
  },
  {
    "pattern_name": "nginx",
    "doc_count": 10080,
    "structure_level": {
      "6": 10080
    },
    "index_count": 1,
    "failure_store_doc_count": 0,
    "failure_store_index_count": 0,
    "namespace_count": 1,
    "field_count": 1562,
    "field_existence": {
      "container.id": 10080,
      "log.level": 10080,
      "host.name": 10080,
      "kubernetes.pod.uid": 10080,
      "cloud.provider": 10080,
      "event.dataset": 10080,
      "service.name": 10080,
      "message": 10080,
      "@timestamp": 10080,
      "data_stream.dataset": 10080,
      "data_stream.namespace": 10080,
      "data_stream.type": 10080
    },
    "size_in_bytes": 12098071,
    "managed_by": [],
    "package_name": [],
    "beat": []
  },
  {
    "pattern_name": "apache",
    "doc_count": 1643,
    "structure_level": {
      "6": 1643
    },
    "index_count": 2,
    "failure_store_doc_count": 0,
    "failure_store_index_count": 0,
    "namespace_count": 2,
    "field_count": 1562,
    "field_existence": {
      "container.id": 1643,
      "log.level": 1643,
      "host.name": 1643,
      "kubernetes.pod.uid": 1643,
      "cloud.provider": 1643,
      "event.dataset": 1643,
      "service.name": 1643,
      "message": 1643,
      "@timestamp": 1643,
      "data_stream.dataset": 1643,
      "data_stream.namespace": 1643,
      "data_stream.type": 1643
    },
    "size_in_bytes": 5593675,
    "managed_by": [],
    "package_name": [],
    "beat": []
  },
  {
    "pattern_name": "generic-logs",
    "doc_count": 123979,
    "structure_level": {
      "2": 112925,
      "3": 11054
    },
    "index_count": 18,
    "failure_store_doc_count": 2,
    "failure_store_index_count": 1,
    "namespace_count": 3,
    "field_count": 1582,
    "field_existence": {
      "container.id": 11054,
      "log.level": 123979,
      "host.name": 123979,
      "kubernetes.pod.uid": 11046,
      "cloud.provider": 11046,
      "event.dataset": 11046,
      "service.name": 123971,
      "message": 11054,
      "@timestamp": 123979,
      "data_stream.dataset": 123979,
      "data_stream.namespace": 123979,
      "data_stream.type": 123979
    },
    "size_in_bytes": 60270084,
    "managed_by": [],
    "package_name": [],
    "beat": []
  }
]

[
  {
    "pattern_name": "heartbeat",
    "shipper": "heartbeat",
    "doc_count": 9530,
    "structure_level": {
      "5": 9530
    },
    "index_count": 1,
    "failure_store_doc_count": 9530,
    "failure_store_index_count": 1,
    "namespace_count": 0,
    "field_count": 1508,
    "field_existence": {
      "container.id": 9530,
      "log.level": 9530,
      "container.name": 9530,
      "host.name": 9530,
      "host.hostname": 9530,
      "kubernetes.pod.name": 9530,
      "kubernetes.pod.uid": 9530,
      "cloud.provider": 9530,
      "agent.type": 9530,
      "event.dataset": 9530,
      "event.category": 9530,
      "event.module": 9530,
      "service.name": 9530,
      "service.type": 9530,
      "service.version": 9530,
      "message": 9530,
      "event.original": 9530,
      "error.message": 9530,
      "@timestamp": 9530,
      "data_stream.dataset": 9530,
      "data_stream.namespace": 9530,
      "data_stream.type": 9530
    },
    "size_in_bytes": 13090458,
    "managed_by": [],
    "package_name": [],
    "beat": [
      "heartbeat"
    ]
  },
  {
    "pattern_name": "nginx",
    "doc_count": 10080,
    "structure_level": {
      "6": 10080
    },
    "index_count": 1,
    "failure_store_doc_count": 0,
    "failure_store_index_count": 0,
    "namespace_count": 1,
    "field_count": 1562,
    "field_existence": {
      "container.id": 10080,
      "log.level": 10080,
      "host.name": 10080,
      "kubernetes.pod.uid": 10080,
      "cloud.provider": 10080,
      "event.dataset": 10080,
      "service.name": 10080,
      "message": 10080,
      "@timestamp": 10080,
      "data_stream.dataset": 10080,
      "data_stream.namespace": 10080,
      "data_stream.type": 10080
    },
    "size_in_bytes": 12098071,
    "managed_by": [],
    "package_name": [],
    "beat": []
  },
  {
    "pattern_name": "apache",
    "doc_count": 1643,
    "structure_level": {
      "6": 1643
    },
    "index_count": 2,
    "failure_store_doc_count": 0,
    "failure_store_index_count": 0,
    "namespace_count": 2,
    "field_count": 1562,
    "field_existence": {
      "container.id": 1643,
      "log.level": 1643,
      "host.name": 1643,
      "kubernetes.pod.uid": 1643,
      "cloud.provider": 1643,
      "event.dataset": 1643,
      "service.name": 1643,
      "message": 1643,
      "@timestamp": 1643,
      "data_stream.dataset": 1643,
      "data_stream.namespace": 1643,
      "data_stream.type": 1643
    },
    "size_in_bytes": 5593675,
    "managed_by": [],
    "package_name": [],
    "beat": []
  },
  {
    "pattern_name": "dsns-logs",
    "doc_count": 123971,
    "structure_level": {
      "2": 112925,
      "6": 11046
    },
    "index_count": 17,
    "failure_store_doc_count": 0,
    "failure_store_index_count": 0,
    "namespace_count": 2,
    "field_count": 1581,
    "field_existence": {
      "container.id": 11046,
      "log.level": 123971,
      "host.name": 123971,
      "kubernetes.pod.uid": 11046,
      "cloud.provider": 11046,
      "event.dataset": 11046,
      "service.name": 123971,
      "message": 11046,
      "@timestamp": 123971,
      "data_stream.dataset": 123971,
      "data_stream.namespace": 123971,
      "data_stream.type": 123971
    },
    "size_in_bytes": 60245641,
    "managed_by": [],
    "package_name": [],
    "beat": []
  },
  {
    "pattern_name": "generic-logs",
    "doc_count": 8,
    "structure_level": {
      "3": 8
    },
    "index_count": 1,
    "failure_store_doc_count": 2,
    "failure_store_index_count": 1,
    "namespace_count": 3,
    "field_count": 1582,
    "field_existence": {
      "container.id": 8,
      "log.level": 8,
      "host.name": 8,
      "message": 8,
      "@timestamp": 8,
      "data_stream.dataset": 8,
      "data_stream.namespace": 8,
      "data_stream.type": 8
    },
    "size_in_bytes": 24826,
    "managed_by": [],
    "package_name": [],
    "beat": []
  }
]

obltmachine · 2024-09-13T14:23:57Z

🤖 GitHub comments

Expand to view the GitHub comments

Just comment with:

/oblt-deploy : Deploy a Kibana instance using the Observability test environments.
run docs-build : Re-trigger the docs validation. (use unformatted text in the comment!)

awahab07 · 2024-09-14T07:32:53Z

/ci

… telemetry log patterns.

afharo

LGTM! Thank you for this!

cc @elastic/platform-analytics in case folks highlight a jump in the data :)

awahab07 · 2024-09-23T09:30:27Z

@elasticmachine merge upstream

awahab07 · 2024-09-25T05:09:33Z

@elasticmachine merge upstream

kibana-ci · 2024-09-25T06:07:01Z

💚 Build Succeeded

Buildkite Build
Commit: 80a0fd6
Kibana Serverless Image: docker.elastic.co/kibana-ci/kibana-serverless:pr-192874-80a0fd626783

Metrics [docs]

✅ unchanged

History

💛 Build #236261 was flaky 3e7507a
💛 Build #235125 was flaky 24c89e9
💛 Build #234216 was flaky 9a6535722c1ea7c0a854e0340b071ac63d3d9a6d

To update your PR or re-run it, just comment with:
@elasticmachine merge upstream

tonyghiani

LGTM 👌

@timestamp

…ry patterns (elastic#192874) ## Summary Adds the `logs-*-*` logs pattern in observability telemetry patterns to separate out logs with indices named per the Data Stream Naming Scheme ([ref](https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme)) vs. the generic logs indices not conforming to DSNS naming. For a typical scenario (test data), the addition of `logs-*-*` affects the collection as (notice the addition of "dsns-logs" object): <table> <thead> <tr> <th>Before</th> <th>After</th> </tr> </thead> <tbody> <tr> <td> ```yml [ { "pattern_name": "heartbeat", "shipper": "heartbeat", "doc_count": 9530, "structure_level": { "5": 9530 }, "index_count": 1, "failure_store_doc_count": 9530, "failure_store_index_count": 1, "namespace_count": 0, "field_count": 1508, "field_existence": { "container.id": 9530, "log.level": 9530, "container.name": 9530, "host.name": 9530, "host.hostname": 9530, "kubernetes.pod.name": 9530, "kubernetes.pod.uid": 9530, "cloud.provider": 9530, "agent.type": 9530, "event.dataset": 9530, "event.category": 9530, "event.module": 9530, "service.name": 9530, "service.type": 9530, "service.version": 9530, "message": 9530, "event.original": 9530, "error.message": 9530, "@timestamp": 9530, "data_stream.dataset": 9530, "data_stream.namespace": 9530, "data_stream.type": 9530 }, "size_in_bytes": 13090458, "managed_by": [], "package_name": [], "beat": [ "heartbeat" ] }, { "pattern_name": "nginx", "doc_count": 10080, "structure_level": { "6": 10080 }, "index_count": 1, "failure_store_doc_count": 0, "failure_store_index_count": 0, "namespace_count": 1, "field_count": 1562, "field_existence": { "container.id": 10080, "log.level": 10080, "host.name": 10080, "kubernetes.pod.uid": 10080, "cloud.provider": 10080, "event.dataset": 10080, "service.name": 10080, "message": 10080, "@timestamp": 10080, "data_stream.dataset": 10080, "data_stream.namespace": 10080, "data_stream.type": 10080 }, "size_in_bytes": 12098071, "managed_by": [], "package_name": [], "beat": [] }, { "pattern_name": "apache", "doc_count": 1643, "structure_level": { "6": 1643 }, "index_count": 2, "failure_store_doc_count": 0, "failure_store_index_count": 0, "namespace_count": 2, "field_count": 1562, "field_existence": { "container.id": 1643, "log.level": 1643, "host.name": 1643, "kubernetes.pod.uid": 1643, "cloud.provider": 1643, "event.dataset": 1643, "service.name": 1643, "message": 1643, "@timestamp": 1643, "data_stream.dataset": 1643, "data_stream.namespace": 1643, "data_stream.type": 1643 }, "size_in_bytes": 5593675, "managed_by": [], "package_name": [], "beat": [] }, { "pattern_name": "generic-logs", "doc_count": 123979, "structure_level": { "2": 112925, "3": 11054 }, "index_count": 18, "failure_store_doc_count": 2, "failure_store_index_count": 1, "namespace_count": 3, "field_count": 1582, "field_existence": { "container.id": 11054, "log.level": 123979, "host.name": 123979, "kubernetes.pod.uid": 11046, "cloud.provider": 11046, "event.dataset": 11046, "service.name": 123971, "message": 11054, "@timestamp": 123979, "data_stream.dataset": 123979, "data_stream.namespace": 123979, "data_stream.type": 123979 }, "size_in_bytes": 60270084, "managed_by": [], "package_name": [], "beat": [] } ] ``` </td> <td> ```yml [ { "pattern_name": "heartbeat", "shipper": "heartbeat", "doc_count": 9530, "structure_level": { "5": 9530 }, "index_count": 1, "failure_store_doc_count": 9530, "failure_store_index_count": 1, "namespace_count": 0, "field_count": 1508, "field_existence": { "container.id": 9530, "log.level": 9530, "container.name": 9530, "host.name": 9530, "host.hostname": 9530, "kubernetes.pod.name": 9530, "kubernetes.pod.uid": 9530, "cloud.provider": 9530, "agent.type": 9530, "event.dataset": 9530, "event.category": 9530, "event.module": 9530, "service.name": 9530, "service.type": 9530, "service.version": 9530, "message": 9530, "event.original": 9530, "error.message": 9530, "@timestamp": 9530, "data_stream.dataset": 9530, "data_stream.namespace": 9530, "data_stream.type": 9530 }, "size_in_bytes": 13090458, "managed_by": [], "package_name": [], "beat": [ "heartbeat" ] }, { "pattern_name": "nginx", "doc_count": 10080, "structure_level": { "6": 10080 }, "index_count": 1, "failure_store_doc_count": 0, "failure_store_index_count": 0, "namespace_count": 1, "field_count": 1562, "field_existence": { "container.id": 10080, "log.level": 10080, "host.name": 10080, "kubernetes.pod.uid": 10080, "cloud.provider": 10080, "event.dataset": 10080, "service.name": 10080, "message": 10080, "@timestamp": 10080, "data_stream.dataset": 10080, "data_stream.namespace": 10080, "data_stream.type": 10080 }, "size_in_bytes": 12098071, "managed_by": [], "package_name": [], "beat": [] }, { "pattern_name": "apache", "doc_count": 1643, "structure_level": { "6": 1643 }, "index_count": 2, "failure_store_doc_count": 0, "failure_store_index_count": 0, "namespace_count": 2, "field_count": 1562, "field_existence": { "container.id": 1643, "log.level": 1643, "host.name": 1643, "kubernetes.pod.uid": 1643, "cloud.provider": 1643, "event.dataset": 1643, "service.name": 1643, "message": 1643, "@timestamp": 1643, "data_stream.dataset": 1643, "data_stream.namespace": 1643, "data_stream.type": 1643 }, "size_in_bytes": 5593675, "managed_by": [], "package_name": [], "beat": [] }, { "pattern_name": "dsns-logs", "doc_count": 123971, "structure_level": { "2": 112925, "6": 11046 }, "index_count": 17, "failure_store_doc_count": 0, "failure_store_index_count": 0, "namespace_count": 2, "field_count": 1581, "field_existence": { "container.id": 11046, "log.level": 123971, "host.name": 123971, "kubernetes.pod.uid": 11046, "cloud.provider": 11046, "event.dataset": 11046, "service.name": 123971, "message": 11046, "@timestamp": 123971, "data_stream.dataset": 123971, "data_stream.namespace": 123971, "data_stream.type": 123971 }, "size_in_bytes": 60245641, "managed_by": [], "package_name": [], "beat": [] }, { "pattern_name": "generic-logs", "doc_count": 8, "structure_level": { "3": 8 }, "index_count": 1, "failure_store_doc_count": 2, "failure_store_index_count": 1, "namespace_count": 3, "field_count": 1582, "field_existence": { "container.id": 8, "log.level": 8, "host.name": 8, "message": 8, "@timestamp": 8, "data_stream.dataset": 8, "data_stream.namespace": 8, "data_stream.type": 8 }, "size_in_bytes": 24826, "managed_by": [], "package_name": [], "beat": [] } ] ``` </td> </tr> </tbody> </table> Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> (cherry picked from commit 8d7dad2)

kibanamachine · 2024-09-25T11:02:25Z

💚 All backports created successfully

Status	Branch	Result
✅	8.x

Note: Successful backport PRs will be merged automatically after passing CI.

Questions ?

Please refer to the Backport tool documentation

@timestamp

…n to obs telemetry patterns (#192874) (#193969) # Backport This will backport the following commits from `main` to `8.x`: - [[Logs Data Telemetry] Add logs-dsns `logs-*-*` pattern to obs telemetry patterns (#192874)](#192874)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Abdul Wahab Zahid <awahab07@yahoo.com>

elasticmachine · 2024-10-01T13:08:26Z

Pinging @elastic/obs-ux-logs-team (Team:obs-ux-logs)

Add dsns-logs logs pattern representing logs-*-* in observability…

24c89e9

… telemetry log patterns.

awahab07 force-pushed the observability-dev-3950-add-logs-dsns-pattern branch from 9a65357 to 24c89e9 Compare September 17, 2024 15:18

awahab07 added Team:obs-onboarding Observability Onboarding Team and removed Team:obs-onboarding Observability Onboarding Team labels Sep 18, 2024

awahab07 marked this pull request as ready for review September 18, 2024 09:19

awahab07 requested review from a team as code owners September 18, 2024 09:19

awahab07 added the release_note:skip Skip the PR/issue when compiling release notes label Sep 18, 2024

afharo approved these changes Sep 18, 2024

View reviewed changes

botelastic Bot added the ci:project-deploy-observability Create an Observability project label Sep 18, 2024

Merge branch 'main' into observability-dev-3950-add-logs-dsns-pattern

3e7507a

awahab07 added backport:skip This PR does not require backporting backport:prev-minor and removed backport:skip This PR does not require backporting labels Sep 23, 2024

Merge branch 'main' into observability-dev-3950-add-logs-dsns-pattern

80a0fd6

tonyghiani approved these changes Sep 25, 2024

View reviewed changes

awahab07 merged commit 8d7dad2 into elastic:main Sep 25, 2024

kibanamachine added the v9.0.0 label Sep 25, 2024

kibanamachine mentioned this pull request Sep 25, 2024

[8.x] [Logs Data Telemetry] Add logs-dsns `logs-*-*` pattern to obs telemetry patterns (#192874) #193969

Merged

kibanamachine added the v8.16.0 label Sep 25, 2024

gbamparop added the Team:obs-onboarding Observability Onboarding Team label Oct 1, 2024

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[Logs Data Telemetry] Add logs-dsns `logs--` pattern to obs telemetry patterns#192874

[Logs Data Telemetry] Add logs-dsns `logs--` pattern to obs telemetry patterns#192874
awahab07 merged 3 commits intoelastic:mainfrom
awahab07:observability-dev-3950-add-logs-dsns-pattern

awahab07 commented Sep 13, 2024 •

edited by kibanamachine

Loading

Uh oh!

obltmachine commented Sep 13, 2024

Uh oh!

awahab07 commented Sep 14, 2024

Uh oh!

afharo left a comment

Uh oh!

awahab07 commented Sep 23, 2024

Uh oh!

awahab07 commented Sep 25, 2024

Uh oh!

kibana-ci commented Sep 25, 2024 •

edited

Loading

Uh oh!

tonyghiani left a comment

Uh oh!

kibanamachine commented Sep 25, 2024

Uh oh!

elasticmachine commented Oct 1, 2024

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

8 participants

Conversation

awahab07 commented Sep 13, 2024 • edited by kibanamachine Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Summary

Uh oh!

obltmachine commented Sep 13, 2024

🤖 GitHub comments

Uh oh!

awahab07 commented Sep 14, 2024

Uh oh!

afharo left a comment

Choose a reason for hiding this comment

Uh oh!

awahab07 commented Sep 23, 2024

Uh oh!

awahab07 commented Sep 25, 2024

Uh oh!

kibana-ci commented Sep 25, 2024 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

💚 Build Succeeded

Metrics [docs]

History

Uh oh!

tonyghiani left a comment

Choose a reason for hiding this comment

Uh oh!

kibanamachine commented Sep 25, 2024

💚 All backports created successfully

Questions ?

Uh oh!

elasticmachine commented Oct 1, 2024

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

8 participants

awahab07 commented Sep 13, 2024 •

edited by kibanamachine

Loading

kibana-ci commented Sep 25, 2024 •

edited

Loading