[Meta] Add ECS Dataset fields

docs/field-details.asciidoc

@@ -5616,6 +5616,76 @@ example: `co.uk`
 // ===============================================================
 
 
+|=====
+
+[[ecs-stream]]
+=== Stream Fields
+
+The stream fields are part of the new [indexing strategy](https://github.com/elastic/kibana/blob/master/docs/ingest_manager/index.asciidoc#indexing-strategy-1).
+
+These fields are used to determine into which index the data is shipped in Elasticsearch and allow efficient querying of data. Initially these fields are mainly used by data shipped by the Elastic Agent but any time series data shipper should switch to using data streams and the new indexing strategy with these fields.
+
+All three fields are `constant_keyword` fields.
+
+==== Stream Field Details
+
+[options="header"]
+|=====
+| Field  | Description | Level
+
+// ===============================================================
+
+| stream.dataset
+| Dataset describes the structure of the data.
+
+The dataset describes the structure of the data. All data shipped into a single dataset should have the same or very similar data structure. For example `system.cpu` and `system.disk` are two different datasets as they have very different fields.
+
+The name of the dataset should be descriptive of the data and it is encourage to use `.` to combine multiple words. All characters which are allowed in index names can be used for the dataset except `-`.
+
+The default for dataset is `generic`.
+
+type: keyword
+
+
+
+example: `nginx.access`
+
+| extended
+
+// ===============================================================
+
+| stream.namespace
+| Namespace of your stream.
+
+This is the namespace used in your index. The namespace is used to separate the same structure into different Data Streams. For example if nginx logs are shipped for testing and production into the same cluster, two different namespaces can be used. This allows to assign different ILM policies as an example.
+
+The default value for a namespace is `default`.
+
+type: constant_keyword
+
+
+
+example: `production`
+
+| extended
+
+// ===============================================================
+
+| stream.type
+| Type of the stream.
+
+The type of the stream can be `logs` or `metrics`. More types can be added in the future but no other types then the one describe here should be used.
+
+type: constant_keyword
+
+
+
+example: `logs`
+
+| extended
+
+// ===============================================================
+
 |=====
 
 [[ecs-threat]]

docs/fields.asciidoc

@@ -86,6 +86,8 @@ all fields are defined.
 
 | <<ecs-source,Source>> | Fields about the source side of a network connection, used with destination.
 
+| <<ecs-stream,Stream>> | Fields about the monitoring agent.
+
 | <<ecs-threat,Threat>> | Fields to classify events and alerts according to a threat taxonomy.
 
 | <<ecs-tls,TLS>> | Fields describing a TLS connection.

generated/beats/fields.ecs.yml

@@ -4380,6 +4380,61 @@
         default_field: false
       description: Short name or login of the user.
       example: albert
+  - name: stream
+    title: Stream
+    group: 2
+    description: 'The stream fields are part of the new [indexing strategy](https://github.com/elastic/kibana/blob/master/docs/ingest_manager/index.asciidoc#indexing-strategy-1).
+
+      These fields are used to determine into which index the data is shipped in Elasticsearch
+      and allow efficient querying of data. Initially these fields are mainly used
+      by data shipped by the Elastic Agent but any time series data shipper should
+      switch to using data streams and the new indexing strategy with these fields.
+
+      All three fields are `constant_keyword` fields.'
+    footnote: 'Examples: The new indexing strategy is {stream.type}-{stream.dataset}-{stream.namespace}`.`
+      As an example, nginx access logs are shipped into `logs-nginx.access-default`.'
+    type: group
+    fields:
+    - name: dataset
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'Dataset describes the structure of the data.
+
+        The dataset describes the structure of the data. All data shipped into a single
+        dataset should have the same or very similar data structure. For example `system.cpu`
+        and `system.disk` are two different datasets as they have very different fields.
+
+        The name of the dataset should be descriptive of the data and it is encourage
+        to use `.` to combine multiple words. All characters which are allowed in
+        index names can be used for the dataset except `-`.
+
+        The default for dataset is `generic`.'
+      example: nginx.access
+      default_field: false
+    - name: namespace
+      level: extended
+      type: constant_keyword
+      description: 'Namespace of your stream.
+
+        This is the namespace used in your index. The namespace is used to separate
+        the same structure into different Data Streams. For example if nginx logs
+        are shipped for testing and production into the same cluster, two different
+        namespaces can be used. This allows to assign different ILM policies as an
+        example.
+
+        The default value for a namespace is `default`.'
+      example: production
+      default_field: false
+    - name: type
+      level: extended
+      type: constant_keyword
+      description: 'Type of the stream.
+
+        The type of the stream can be `logs` or `metrics`. More types can be added
+        in the future but no other types then the one describe here should be used.'
+      example: logs
+      default_field: false
   - name: threat
     title: Threat
     group: 2

generated/csv/fields.csv

@@ -516,6 +516,9 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.6.0-dev,true,source,source.user.id,keyword,core,,,Unique identifier of the user.
 1.6.0-dev,true,source,source.user.name,keyword,core,,albert,Short name or login of the user.
 1.6.0-dev,true,source,source.user.name.text,text,core,,albert,Short name or login of the user.
+1.6.0-dev,true,stream,stream.dataset,keyword,extended,,nginx.access,Dataset describes the structure of the data.
+1.6.0-dev,true,stream,stream.namespace,constant_keyword,extended,,production,Namespace of your stream.
+1.6.0-dev,true,stream,stream.type,constant_keyword,extended,,logs,Type of the stream.
 1.6.0-dev,true,threat,threat.framework,keyword,extended,,MITRE ATT&CK,Threat classification framework.
 1.6.0-dev,true,threat,threat.tactic.id,keyword,extended,array,TA0040,Threat tactic id.
 1.6.0-dev,true,threat,threat.tactic.name,keyword,extended,array,impact,Threat tactic.

generated/ecs/ecs_flat.yml

@@ -6669,6 +6669,57 @@ source.user.name:
   original_fieldset: user
   short: Short name or login of the user.
   type: keyword
+stream.dataset:
+  dashed_name: stream-dataset
+  description: 'Dataset describes the structure of the data.
+
+    The dataset describes the structure of the data. All data shipped into a single
+    dataset should have the same or very similar data structure. For example `system.cpu`
+    and `system.disk` are two different datasets as they have very different fields.
+
+    The name of the dataset should be descriptive of the data and it is encourage
+    to use `.` to combine multiple words. All characters which are allowed in index
+    names can be used for the dataset except `-`.
+
+    The default for dataset is `generic`.'
+  example: nginx.access
+  flat_name: stream.dataset
+  ignore_above: 1024
+  level: extended
+  name: dataset
+  normalize: []
+  short: Dataset describes the structure of the data.
+  type: keyword
+stream.namespace:
+  dashed_name: stream-namespace
+  description: 'Namespace of your stream.
+
+    This is the namespace used in your index. The namespace is used to separate the
+    same structure into different Data Streams. For example if nginx logs are shipped
+    for testing and production into the same cluster, two different namespaces can
+    be used. This allows to assign different ILM policies as an example.
+
+    The default value for a namespace is `default`.'
+  example: production
+  flat_name: stream.namespace
+  level: extended
+  name: namespace
+  normalize: []
+  short: Namespace of your stream.
+  type: constant_keyword
+stream.type:
+  dashed_name: stream-type
+  description: 'Type of the stream.
+
+    The type of the stream can be `logs` or `metrics`. More types can be added in
+    the future but no other types then the one describe here should be used.'
+  example: logs
+  flat_name: stream.type
+  level: extended
+  name: type
+  normalize: []
+  short: Type of the stream.
+  type: constant_keyword
 tags:
   dashed_name: tags
   description: List of keywords used to tag each event.

generated/ecs/ecs_nested.yml

@@ -7698,6 +7698,76 @@ source:
   short: Fields about the source side of a network connection, used with destination.
   title: Source
   type: group
+stream:
+  description: 'The stream fields are part of the new [indexing strategy](https://github.com/elastic/kibana/blob/master/docs/ingest_manager/index.asciidoc#indexing-strategy-1).
+
+    These fields are used to determine into which index the data is shipped in Elasticsearch
+    and allow efficient querying of data. Initially these fields are mainly used by
+    data shipped by the Elastic Agent but any time series data shipper should switch
+    to using data streams and the new indexing strategy with these fields.
+
+    All three fields are `constant_keyword` fields.'
+  fields:
+    dataset:
+      dashed_name: stream-dataset
+      description: 'Dataset describes the structure of the data.
+
+        The dataset describes the structure of the data. All data shipped into a single
+        dataset should have the same or very similar data structure. For example `system.cpu`
+        and `system.disk` are two different datasets as they have very different fields.
+
+        The name of the dataset should be descriptive of the data and it is encourage
+        to use `.` to combine multiple words. All characters which are allowed in
+        index names can be used for the dataset except `-`.
+
+        The default for dataset is `generic`.'
+      example: nginx.access
+      flat_name: stream.dataset
+      ignore_above: 1024
+      level: extended
+      name: dataset
+      normalize: []
+      short: Dataset describes the structure of the data.
+      type: keyword
+    namespace:
+      dashed_name: stream-namespace
+      description: 'Namespace of your stream.
+
+        This is the namespace used in your index. The namespace is used to separate
+        the same structure into different Data Streams. For example if nginx logs
+        are shipped for testing and production into the same cluster, two different
+        namespaces can be used. This allows to assign different ILM policies as an
+        example.
+
+        The default value for a namespace is `default`.'
+      example: production
+      flat_name: stream.namespace
+      level: extended
+      name: namespace
+      normalize: []
+      short: Namespace of your stream.
+      type: constant_keyword
+    type:
+      dashed_name: stream-type
+      description: 'Type of the stream.
+
+        The type of the stream can be `logs` or `metrics`. More types can be added
+        in the future but no other types then the one describe here should be used.'
+      example: logs
+      flat_name: stream.type
+      level: extended
+      name: type
+      normalize: []
+      short: Type of the stream.
+      type: constant_keyword
+  footnote: 'Examples: The new indexing strategy is {stream.type}-{stream.dataset}-{stream.namespace}`.`
+    As an example, nginx access logs are shipped into `logs-nginx.access-default`.'
+  group: 2
+  name: stream
+  prefix: stream.
+  short: Fields about the monitoring agent.
+  title: Stream
+  type: group
 threat:
   description: 'Fields to classify events and alerts according to a threat taxonomy
     such as the Mitre ATT&CK framework.

generated/elasticsearch/6/template.json

@@ -2450,6 +2450,20 @@
             }
           }
         },
+        "stream": {
+          "properties": {
+            "dataset": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "namespace": {
+              "type": "constant_keyword"
+            },
+            "type": {
+              "type": "constant_keyword"
+            }
+          }
+        },
         "tags": {
           "ignore_above": 1024,
           "type": "keyword"

generated/elasticsearch/7/template.json

@@ -2449,6 +2449,20 @@
           }
         }
       },
+      "stream": {
+        "properties": {
+          "dataset": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "namespace": {
+            "type": "constant_keyword"
+          },
+          "type": {
+            "type": "constant_keyword"
+          }
+        }
+      },
       "tags": {
         "ignore_above": 1024,
         "type": "keyword"