From 5813057fec3898b6bc80a67678459d279d675075 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 12 Nov 2020 09:27:04 -0600 Subject: [PATCH 1/6] strip index param from wildcard fields --- scripts/schema/cleaner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/schema/cleaner.py b/scripts/schema/cleaner.py index ab3acfcaeb..185d0abedc 100644 --- a/scripts/schema/cleaner.py +++ b/scripts/schema/cleaner.py @@ -144,6 +144,9 @@ def field_or_multi_field_datatype_defaults(field_details): field_details.setdefault('ignore_above', 1024) if field_details['type'] == 'text': field_details.setdefault('norms', False) + # wildcard needs the index param stripped + if field_details['type'] == 'wildcard': + field_details.pop('index', None) if 'index' in field_details and not field_details['index']: field_details.setdefault('doc_values', False) From 5dace8e47bd31be559112cce8efd19f0a1c2f35b Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 12 Nov 2020 09:31:48 -0600 Subject: [PATCH 2/6] add unit test for wildcard index param stripping --- scripts/tests/unit/test_schema_cleaner.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/tests/unit/test_schema_cleaner.py b/scripts/tests/unit/test_schema_cleaner.py index 13f78c4e91..56f31bab55 100644 --- a/scripts/tests/unit/test_schema_cleaner.py +++ b/scripts/tests/unit/test_schema_cleaner.py @@ -223,6 +223,10 @@ def test_field_defaults(self): cleaner.field_defaults({'field_details': field_details}) self.assertEqual(field_details['doc_values'], False) + field_details = {**field_min_details, **{'type': 'wildcard'}} + cleaner.field_defaults({'field_details': field_details}) + self.assertNotIn('index', field_details) + def test_field_defaults_dont_override(self): field_details = { 'description': 'description', From ed6928326da9811c64f16793b5dea3c61fdb12ce Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 12 Nov 2020 09:33:23 -0600 Subject: [PATCH 3/6] update experimental artifacts --- experimental/generated/beats/fields.ecs.yml | 1 - experimental/generated/ecs/ecs_flat.yml | 1 - experimental/generated/ecs/ecs_nested.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 5352e2bb18..5b7158538e 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -1160,7 +1160,6 @@ norms: false default_field: false description: The stack trace of this error in plain text. - index: true - name: type level: extended type: wildcard diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index 7a92b47716..3c14cf04a3 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -1599,7 +1599,6 @@ error.stack_trace: dashed_name: error-stack-trace description: The stack trace of this error in plain text. flat_name: error.stack_trace - index: true level: extended multi_fields: - flat_name: error.stack_trace.text diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index da428dae70..f7a2bc93ae 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -1971,7 +1971,6 @@ error: dashed_name: error-stack-trace description: The stack trace of this error in plain text. flat_name: error.stack_trace - index: true level: extended multi_fields: - flat_name: error.stack_trace.text From a5a06d95deea314039621610f81fc2f64fc42484 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 12 Nov 2020 10:09:19 -0600 Subject: [PATCH 4/6] document index has no effect with wildcard --- schemas/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/schemas/README.md b/schemas/README.md index 88440c0354..39b18f4bd7 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -129,7 +129,8 @@ Supported keys to describe fields Example values that are composite types (array, object) should be quoted to avoid YAML interpretation in ECS-generated artifacts and other downstream projects depending on the schema. - multi\_fields (optional): Specify additional ways to index the field. -- index (optional): If `False`, means field is not indexed (overrides type) +- index (optional): If `False`, means field is not indexed (overrides type). This parameter has no effect + on a `wildcard` field. - format: Field format that can be used in a Kibana index template. - normalize: Normalization steps that should be applied at ingestion time. Supported values: - array: the content of the field should be an array (even when there's only one value). @@ -151,7 +152,7 @@ Supported keys to describe expected values for a field Optionally, entries in this list can specify 'expected\_event\_types'. - expected\_event\_types: list of expected "event.type" values to use in association with that category. - + Supported keys when using the [alias field type](https://www.elastic.co/guide/en/elasticsearch/reference/current/alias.html) ```YAML From 02d568b223c0cdbc873d80bd8d732b56fc41e753 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 12 Nov 2020 10:09:45 -0600 Subject: [PATCH 5/6] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index edd614a6b0..c06a627c99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ All notable changes to this project will be documented in this file based on the * Added `threat.technique.subtechnique` to capture MITRE ATT&CK® subtechniques. #951 * Added `configuration` as an allowed `event.category`. #963 * Added a new directory with experimental artifacts, which includes all changes - from RFCs that have reached stage 2. #993, #1053 + from RFCs that have reached stage 2. #993, #1053, #1115 #### Improvements From c56c79b8f25ffc566830ef926b6690a25e988ad6 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 12 Nov 2020 10:46:27 -0600 Subject: [PATCH 6/6] field_min_details needs index --- scripts/tests/unit/test_schema_cleaner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/unit/test_schema_cleaner.py b/scripts/tests/unit/test_schema_cleaner.py index 56f31bab55..bc3dbdc621 100644 --- a/scripts/tests/unit/test_schema_cleaner.py +++ b/scripts/tests/unit/test_schema_cleaner.py @@ -223,7 +223,7 @@ def test_field_defaults(self): cleaner.field_defaults({'field_details': field_details}) self.assertEqual(field_details['doc_values'], False) - field_details = {**field_min_details, **{'type': 'wildcard'}} + field_details = {**field_min_details, **{'type': 'wildcard', 'index': True}} cleaner.field_defaults({'field_details': field_details}) self.assertNotIn('index', field_details)