From c2e2eb9f5029cd15f4055474a2393bb98ac48401 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 29 Sep 2020 16:21:39 -0500 Subject: [PATCH 01/13] add usage doc file helper --- scripts/generators/ecs_helpers.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py index 275c0569ac..951dfe4103 100644 --- a/scripts/generators/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -2,6 +2,7 @@ import os import yaml import git +import pathlib import warnings from collections import OrderedDict @@ -113,12 +114,19 @@ def get_tree_by_ref(ref): return commit.tree +def usage_doc_files(): + usage_docs_dir = os.path.join(os.path.dirname(__file__), '../../docs/usage') + usage_docs_path = pathlib.Path(usage_docs_dir) + if usage_docs_path.is_dir(): + return [x.name for x in usage_docs_path.glob('*.asciidoc') if x.is_file()] + return [] + + def ecs_files(): """Return the schema file list to load""" schema_glob = os.path.join(os.path.dirname(__file__), '../../schemas/*.yml') return sorted(glob.glob(schema_glob)) - def make_dirs(path): try: os.makedirs(path, exist_ok=True) From e103bd31fb46c02187db597ae47de26a3ffe3951 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 29 Sep 2020 16:21:54 -0500 Subject: [PATCH 02/13] add usage file check against fieldset name --- scripts/generators/asciidoc_fields.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 2aa6f4a8cd..17404982ad 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -71,6 +71,13 @@ def sort_fields(fieldset): field['allowed_value_names'] = extract_allowed_values_key_names(field) return sorted(fields_list, key=lambda field: field['name']) +def check_for_usage_doc(fieldset_name, usage_file_list=ecs_helpers.usage_doc_files()): + """Checks if a usage doc exists for the specified + fieldset. + + :param fieldset_name: The name of the target fieldset + """ + return f"{fieldset_name}.asciidoc" in usage_file_list def templated(template_name): """Decorator function to simplify rendering a template. @@ -138,10 +145,12 @@ def generate_field_details_page(fieldset): sorted_reuse_fields = render_fieldset_reuse_text(fieldset) render_nestings_reuse_fields = render_nestings_reuse_section(fieldset) sorted_fields = sort_fields(fieldset) + usage_doc = check_for_usage_doc(fieldset.get('name')) return dict(fieldset=fieldset, sorted_reuse_fields=sorted_reuse_fields, render_nestings_reuse_section=render_nestings_reuse_fields, - sorted_fields=sorted_fields) + sorted_fields=sorted_fields, + usage_doc=usage_doc) # Allowed values section From f97c9a0008bedbf14a4e2310cb10e1c3594e4ba0 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 29 Sep 2020 16:22:13 -0500 Subject: [PATCH 03/13] add condiitonal check for usage_doc --- scripts/templates/field_details.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/templates/field_details.j2 b/scripts/templates/field_details.j2 index 1ceedf55e0..16092f4028 100644 --- a/scripts/templates/field_details.j2 +++ b/scripts/templates/field_details.j2 @@ -114,3 +114,9 @@ Note also that the `{{ fieldset['name'] }}` fields are not expected to be used d {% endif %}{# if 'nestings' #} {%- endif %}{# if 'nestings' or 'reusable' in fieldset #} + +{% if usage_doc -%} + +include::usage/{{ fieldset['name'] }}.asciidoc[] + +{% endif %} From 1e033e072ea66e7b6239beac9002b23280dc6a2f Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 29 Sep 2020 16:22:34 -0500 Subject: [PATCH 04/13] add coverage for check_for_usage_doc --- scripts/tests/test_asciidoc_fields.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py index 1a099a9958..4737977807 100644 --- a/scripts/tests/test_asciidoc_fields.py +++ b/scripts/tests/test_asciidoc_fields.py @@ -127,6 +127,15 @@ def test_rendering_fieldset_nesting(self): self.assertEqual('as', foo_nesting_fields[0]['name']) self.assertEqual('Fields describing an AS', foo_nesting_fields[0]['short']) + def test_check_for_usage_doc_true(self): + usage_files = ["foo.asciidoc"] + foo_name = self.foo_fieldset.get('name') + self.assertTrue(asciidoc_fields.check_for_usage_doc(foo_name, usage_file_list=usage_files)) + + def test_check_for_usage_doc_false(self): + usage_files = ["notfoo.asciidoc"] + foo_name = self.foo_fieldset.get('name') + self.assertFalse(asciidoc_fields.check_for_usage_doc(foo_name, usage_file_list=usage_files)) if __name__ == '__main__': unittest.main() From c69a7909d1e078873934459e700a9b16a6f89da4 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 30 Sep 2020 12:34:32 -0500 Subject: [PATCH 05/13] increase chunking level --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4261504635..07f0442421 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ docs: if [ ! -d $(PWD)/build/docs ]; then \ git clone --depth=1 https://github.com/elastic/docs.git ./build/docs ; \ fi - ./build/docs/build_docs --asciidoctor --doc ./docs/index.asciidoc --chunk=1 $(OPEN_DOCS) --out ./build/html_docs + ./build/docs/build_docs --asciidoctor --doc ./docs/index.asciidoc --chunk=2 $(OPEN_DOCS) --out ./build/html_docs # Alias to generate experimental artifacts .PHONY: experimental From 5792f9b6913e4ba917ed8820c7cc4ce6edbdf621 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 30 Sep 2020 12:49:10 -0500 Subject: [PATCH 06/13] adjust for whitespace --- scripts/templates/field_details.j2 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/templates/field_details.j2 b/scripts/templates/field_details.j2 index 16092f4028..48d28f0902 100644 --- a/scripts/templates/field_details.j2 +++ b/scripts/templates/field_details.j2 @@ -113,9 +113,8 @@ Note also that the `{{ fieldset['name'] }}` fields are not expected to be used d |===== {% endif %}{# if 'nestings' #} -{%- endif %}{# if 'nestings' or 'reusable' in fieldset #} - -{% if usage_doc -%} +{%- endif -%}{# if 'nestings' or 'reusable' in fieldset #} +{%- if usage_doc %} include::usage/{{ fieldset['name'] }}.asciidoc[] From a85d9312c20d7b14cefde085dbebbce262b75698 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 30 Sep 2020 13:11:52 -0500 Subject: [PATCH 07/13] adding usage directory and readme --- docs/usage/README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 docs/usage/README.md diff --git a/docs/usage/README.md b/docs/usage/README.md new file mode 100644 index 0000000000..67e4ea0bde --- /dev/null +++ b/docs/usage/README.md @@ -0,0 +1,39 @@ +# Usage Docs + +ECS fields can benefit from additional context and examples which describe their real-world usage. This directory provides a place in the documentation to capture these usage details. AsciiDoc markdown files can be added for any fieldset defined in ECS. + +## Adding a Usage Doc + +1. Create an AsciiDoc formatted file with the `.asciidoc` file extension. +2. Save the file in this directory (`docs/usage`), naming it after the its associated fieldset (e.g. a usage document for the `base.*` fields would be named `base.asciidoc`). +3. Run `make`. The asciidoc generator will generate the ECS field reference, including the present usage docs. + +If the filename doesn't match a currently defined fieldset, the usage document will not appear on the ECS docs site. This logic is handled in the AsciiDoc generator scripts, `scripts/generators/asciidoc_fields.py`. + +## Template + +The following is a simple AsciiDoc template as a starting point: + +```asciidoc + +[ecs-usage-template] +==== Fieldset Usage + +Add relevant text here. + +[discrete] +===== New Section header + +Text for the new section. + +[discrete] +===== Examples + +[source,sh] +----------- +{ + "key": "value" +} +----------- + +``` From 176b5e9f24d2ddfec25837b02aba592f9e9a91c2 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 30 Sep 2020 13:39:24 -0500 Subject: [PATCH 08/13] fix linting errors --- scripts/generators/asciidoc_fields.py | 2 ++ scripts/generators/ecs_helpers.py | 1 + scripts/tests/test_asciidoc_fields.py | 1 + 3 files changed, 4 insertions(+) diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 17404982ad..e5e2262bd0 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -71,6 +71,7 @@ def sort_fields(fieldset): field['allowed_value_names'] = extract_allowed_values_key_names(field) return sorted(fields_list, key=lambda field: field['name']) + def check_for_usage_doc(fieldset_name, usage_file_list=ecs_helpers.usage_doc_files()): """Checks if a usage doc exists for the specified fieldset. @@ -79,6 +80,7 @@ def check_for_usage_doc(fieldset_name, usage_file_list=ecs_helpers.usage_doc_fil """ return f"{fieldset_name}.asciidoc" in usage_file_list + def templated(template_name): """Decorator function to simplify rendering a template. diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py index 951dfe4103..2da446f3e3 100644 --- a/scripts/generators/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -127,6 +127,7 @@ def ecs_files(): schema_glob = os.path.join(os.path.dirname(__file__), '../../schemas/*.yml') return sorted(glob.glob(schema_glob)) + def make_dirs(path): try: os.makedirs(path, exist_ok=True) diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py index 4737977807..2fbec15e84 100644 --- a/scripts/tests/test_asciidoc_fields.py +++ b/scripts/tests/test_asciidoc_fields.py @@ -137,5 +137,6 @@ def test_check_for_usage_doc_false(self): foo_name = self.foo_fieldset.get('name') self.assertFalse(asciidoc_fields.check_for_usage_doc(foo_name, usage_file_list=usage_files)) + if __name__ == '__main__': unittest.main() From 8231f0ac946fd779a8aca04ff25a49088214abc6 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 30 Sep 2020 14:00:11 -0500 Subject: [PATCH 09/13] changelog --- CHANGELOG.next.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index ef52884095..944f9c6346 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -28,6 +28,8 @@ Thanks, you're awesome :-) --> #### Added +* Added ability to supply free-form usage documentation per fieldset. #988 + #### Improvements #### Deprecated From cfb76833da8d6b7a581c161828757a0a6c569a3d Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 1 Oct 2020 11:39:52 -0500 Subject: [PATCH 10/13] Update docs/usage/README.md Co-authored-by: Mathieu Martin --- docs/usage/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/README.md b/docs/usage/README.md index 67e4ea0bde..e538577b29 100644 --- a/docs/usage/README.md +++ b/docs/usage/README.md @@ -5,7 +5,7 @@ ECS fields can benefit from additional context and examples which describe their ## Adding a Usage Doc 1. Create an AsciiDoc formatted file with the `.asciidoc` file extension. -2. Save the file in this directory (`docs/usage`), naming it after the its associated fieldset (e.g. a usage document for the `base.*` fields would be named `base.asciidoc`). +2. Save the file in this directory (`docs/usage`), naming it after its associated field set (e.g. a usage document for the fields defined in `schemas/base.yml` fields would be named `docs/usage/base.asciidoc`). 3. Run `make`. The asciidoc generator will generate the ECS field reference, including the present usage docs. If the filename doesn't match a currently defined fieldset, the usage document will not appear on the ECS docs site. This logic is handled in the AsciiDoc generator scripts, `scripts/generators/asciidoc_fields.py`. From a51de956256403e51e212ae93a9de227a055856a Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 1 Oct 2020 12:22:04 -0500 Subject: [PATCH 11/13] incorrect markdown in asciidoc template --- docs/usage/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/README.md b/docs/usage/README.md index e538577b29..36148464d7 100644 --- a/docs/usage/README.md +++ b/docs/usage/README.md @@ -16,7 +16,7 @@ The following is a simple AsciiDoc template as a starting point: ```asciidoc -[ecs-usage-template] +[[ecs-usage-template]] ==== Fieldset Usage Add relevant text here. From 2b9325c183f1a30c0a5c56aff750f2fb5b240d38 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 1 Oct 2020 12:22:43 -0500 Subject: [PATCH 12/13] add conditional sentence if usage doc is present --- scripts/templates/field_details.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/templates/field_details.j2 b/scripts/templates/field_details.j2 index 48d28f0902..3eef363fa8 100644 --- a/scripts/templates/field_details.j2 +++ b/scripts/templates/field_details.j2 @@ -4,6 +4,12 @@ {{ fieldset['description']|replace("\n", "\n\n") }} +{%- if usage_doc %} + +Find additional usage and examples in the {{ fieldset['name'] }} fields <> section. + +{% endif %} + {# Field Details Table Header -#} [discrete] ==== {{ fieldset['title'] }} Field Details From 066945829325450604775376613fa9c10ddc4d22 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 8 Oct 2020 11:17:50 -0500 Subject: [PATCH 13/13] note expected anchor convention --- docs/usage/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/usage/README.md b/docs/usage/README.md index 36148464d7..fce7219a2b 100644 --- a/docs/usage/README.md +++ b/docs/usage/README.md @@ -6,7 +6,8 @@ ECS fields can benefit from additional context and examples which describe their 1. Create an AsciiDoc formatted file with the `.asciidoc` file extension. 2. Save the file in this directory (`docs/usage`), naming it after its associated field set (e.g. a usage document for the fields defined in `schemas/base.yml` fields would be named `docs/usage/base.asciidoc`). -3. Run `make`. The asciidoc generator will generate the ECS field reference, including the present usage docs. +3. The anchor at the top of the file (e.g. `[[ecs-base-usage]]`) must use the following convention for valid link references in the generated docs: `[[ecs-<>-usage]]`. +4. Run `make`. The asciidoc generator will generate the ECS field reference, including the present usage docs. If the filename doesn't match a currently defined fieldset, the usage document will not appear on the ECS docs site. This logic is handled in the AsciiDoc generator scripts, `scripts/generators/asciidoc_fields.py`. @@ -16,7 +17,7 @@ The following is a simple AsciiDoc template as a starting point: ```asciidoc -[[ecs-usage-template]] +[[ecs-fieldset-usage]] ==== Fieldset Usage Add relevant text here.