From 97e003bf6e90a461ae2da277c595b67ecd7a0079 Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Mon, 1 Jun 2020 14:51:44 -0700 Subject: [PATCH 01/14] Validation of configuration examples in docs Signed-off-by: Dmitri Dolguikh --- docs/_ext/validating_code_block.py | 46 +++++++++++++++++++ docs/build.sh | 2 +- docs/conf.py | 5 +- .../root/configuration/operations/runtime.rst | 3 +- tools/config_validation/validate_fragment.py | 7 +-- 5 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 docs/_ext/validating_code_block.py diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py new file mode 100644 index 0000000000000..6289373b54728 --- /dev/null +++ b/docs/_ext/validating_code_block.py @@ -0,0 +1,46 @@ +from typing import List +from docutils import nodes +from docutils.parsers.rst import Directive +from docutils.parsers.rst import directives +from sphinx.application import Sphinx +from sphinx.util.docutils import SphinxDirective +from sphinx.directives.code import CodeBlock +from sphinx.errors import ExtensionError + +import subprocess + +class ValidatingCodeBlock(CodeBlock): + + has_content = True + required_arguments = CodeBlock.required_arguments + optional_arguments = CodeBlock.optional_arguments + final_argument_whitespace = CodeBlock.final_argument_whitespace + option_spec = { + 'type-name': directives.unchanged, + } + option_spec.update(CodeBlock.option_spec) + + def run(self): + source, line = self.state_machine.get_source_and_line(self.lineno) + # built-in directives.unchanged_required option validator produces a confusing error message + if (self.options.get('type-name') == None): + raise ExtensionError("Expected type name in: {0} line: {1}".format(source, line)) + + process = subprocess.Popen(['bazel', 'run', '//tools/config_validation:validate_fragment', '--', self.options.get('type-name'), "-s", "\n".join(self.content)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if (process.poll()): + raise ExtensionError("Failed config validation for type: '{0}' in: {1} line: {2}".format(self.options.get('type-name'), source, line)) + + self.options.pop('type-name', None) + return list(CodeBlock.run(self)) + +def setup(app): + app.add_directive("validated-code-block", ValidatingCodeBlock) + + return { + 'version': '0.1', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/docs/build.sh b/docs/build.sh index 0ebb4d0854971..b6d20626a86ce 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -121,6 +121,6 @@ cp -f $API_DIR/xds_protocol.rst "${GENERATED_RST_DIR}/api-docs/xds_protocol.rst" rsync -rav $API_DIR/diagrams "${GENERATED_RST_DIR}/api-docs" -rsync -av "${SCRIPT_DIR}"/root/ "${SCRIPT_DIR}"/conf.py "${GENERATED_RST_DIR}" +rsync -av "${SCRIPT_DIR}"/root/ "${SCRIPT_DIR}"/conf.py "${SCRIPT_DIR}"/_ext "${GENERATED_RST_DIR}" sphinx-build -W --keep-going -b html "${GENERATED_RST_DIR}" "${DOCS_OUTPUT_DIR}" diff --git a/docs/conf.py b/docs/conf.py index a2f4d250d939b..98286eec82655 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -67,7 +67,10 @@ def setup(app): # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinxcontrib.httpdomain', 'sphinx.ext.extlinks', 'sphinx.ext.ifconfig'] + +sys.path.append(os.path.abspath("./_ext")) + +extensions = ['sphinxcontrib.httpdomain', 'sphinx.ext.extlinks', 'sphinx.ext.ifconfig', 'validating_code_block'] extlinks = { 'repo': ('https://github.com/envoyproxy/envoy/blob/{}/%s'.format(blob_sha), ''), 'api': ('https://github.com/envoyproxy/envoy/blob/{}/api/%s'.format(blob_sha), ''), diff --git a/docs/root/configuration/operations/runtime.rst b/docs/root/configuration/operations/runtime.rst index 45b1f72634803..2e72e52bb953b 100644 --- a/docs/root/configuration/operations/runtime.rst +++ b/docs/root/configuration/operations/runtime.rst @@ -24,7 +24,8 @@ runtime ` bootstrap configu layering. Runtime settings in later layers override earlier layers. A typical configuration might be: -.. code-block:: yaml +.. validated-code-block:: yaml + :type-name: envoy.config.bootstrap.v3.LayeredRuntime layers: - name: static_layer_0 diff --git a/tools/config_validation/validate_fragment.py b/tools/config_validation/validate_fragment.py index 403b5540418f1..17ed31775a6c3 100644 --- a/tools/config_validation/validate_fragment.py +++ b/tools/config_validation/validate_fragment.py @@ -51,6 +51,7 @@ def ValidateFragment(type_name, fragment): if __name__ == '__main__': - type_name, yaml_path = sys.argv[1:] - ValidateFragment(type_name, yaml.load(pathlib.Path(yaml_path).read_text(), - Loader=yaml.FullLoader)) + type_name = sys.argv[1] + yaml_path = sys.argv[2] + content = sys.argv[3] if (yaml_path == "-s") else pathlib.Path(yaml_path).read_text() + ValidateFragment(type_name, yaml.load(content, Loader=yaml.FullLoader)) From 906ff635db866ac1f26520b5b24684fc14cfb2a0 Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Tue, 2 Jun 2020 16:01:50 -0700 Subject: [PATCH 02/14] Responded to feedback Signed-off-by: Dmitri Dolguikh --- docs/_ext/validating_code_block.py | 77 ++++++++++++-------- docs/build.sh | 2 + docs/conf.py | 5 +- tools/config_validation/validate_fragment.py | 2 +- 4 files changed, 54 insertions(+), 32 deletions(-) diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py index 6289373b54728..e4c13b064fcee 100644 --- a/docs/_ext/validating_code_block.py +++ b/docs/_ext/validating_code_block.py @@ -7,40 +7,57 @@ from sphinx.directives.code import CodeBlock from sphinx.errors import ExtensionError +import os import subprocess + class ValidatingCodeBlock(CodeBlock): + """A directive that provides protobuf yaml formatting and validation. + + 'type-name' option is required and expected to conain full Envoy API type. + An ExtensionError is raised on validation failure. + Validation will be skipped if SPHINX_SKIP_CONFIG_VALIDATION environment variable is set. + """ + has_content = True + required_arguments = CodeBlock.required_arguments + optional_arguments = CodeBlock.optional_arguments + final_argument_whitespace = CodeBlock.final_argument_whitespace + option_spec = { + 'type-name': directives.unchanged, + } + option_spec.update(CodeBlock.option_spec) + skip_validation = os.getenv('SPHINX_SKIP_CONFIG_VALIDATION') or 'false' + bazel_build_options = os.getenv('BAZEL_BUILD_OPTIONS') + + def run(self): + source, line = self.state_machine.get_source_and_line(self.lineno) + # built-in directives.unchanged_required option validator produces a confusing error message + if self.options.get('type-name') == None: + raise ExtensionError("Expected type name in: {0} line: {1}".format(source, line)) + + if ValidatingCodeBlock.skip_validation.lower() != 'true': + args = [ + arg for arg in [ + 'bazel', 'run', ValidatingCodeBlock.bazel_build_options, + '//tools/config_validation:validate_fragment', '--', + self.options.get('type-name'), '-s', '\n'.join(self.content) + ] if arg != None + ] + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if process.poll(): + raise ExtensionError("Failed config validation for type: '{0}' in: {1} line: {2}".format( + self.options.get('type-name'), source, line)) + + self.options.pop('type-name', None) + return list(CodeBlock.run(self)) - has_content = True - required_arguments = CodeBlock.required_arguments - optional_arguments = CodeBlock.optional_arguments - final_argument_whitespace = CodeBlock.final_argument_whitespace - option_spec = { - 'type-name': directives.unchanged, - } - option_spec.update(CodeBlock.option_spec) - - def run(self): - source, line = self.state_machine.get_source_and_line(self.lineno) - # built-in directives.unchanged_required option validator produces a confusing error message - if (self.options.get('type-name') == None): - raise ExtensionError("Expected type name in: {0} line: {1}".format(source, line)) - - process = subprocess.Popen(['bazel', 'run', '//tools/config_validation:validate_fragment', '--', self.options.get('type-name'), "-s", "\n".join(self.content)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - if (process.poll()): - raise ExtensionError("Failed config validation for type: '{0}' in: {1} line: {2}".format(self.options.get('type-name'), source, line)) - - self.options.pop('type-name', None) - return list(CodeBlock.run(self)) def setup(app): - app.add_directive("validated-code-block", ValidatingCodeBlock) + app.add_directive("validated-code-block", ValidatingCodeBlock) - return { - 'version': '0.1', - 'parallel_read_safe': True, - 'parallel_write_safe': True, - } + return { + 'version': '0.1', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/docs/build.sh b/docs/build.sh index 58a9947373344..78e067f723057 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -128,4 +128,6 @@ rsync -rav $API_DIR/diagrams "${GENERATED_RST_DIR}/api-docs" rsync -av "${SCRIPT_DIR}"/root/ "${SCRIPT_DIR}"/conf.py "${SCRIPT_DIR}"/_ext "${GENERATED_RST_DIR}" +bazel build ${BAZEL_BUILD_OPTIONS} //tools/config_validation:validate_fragment + sphinx-build -W --keep-going -b html "${GENERATED_RST_DIR}" "${DOCS_OUTPUT_DIR}" diff --git a/docs/conf.py b/docs/conf.py index 98286eec82655..1eb5725b689b8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -70,7 +70,10 @@ def setup(app): sys.path.append(os.path.abspath("./_ext")) -extensions = ['sphinxcontrib.httpdomain', 'sphinx.ext.extlinks', 'sphinx.ext.ifconfig', 'validating_code_block'] +extensions = [ + 'sphinxcontrib.httpdomain', 'sphinx.ext.extlinks', 'sphinx.ext.ifconfig', + 'validating_code_block' +] extlinks = { 'repo': ('https://github.com/envoyproxy/envoy/blob/{}/%s'.format(blob_sha), ''), 'api': ('https://github.com/envoyproxy/envoy/blob/{}/api/%s'.format(blob_sha), ''), diff --git a/tools/config_validation/validate_fragment.py b/tools/config_validation/validate_fragment.py index 17ed31775a6c3..a6dbbc0a3aaa1 100644 --- a/tools/config_validation/validate_fragment.py +++ b/tools/config_validation/validate_fragment.py @@ -51,7 +51,7 @@ def ValidateFragment(type_name, fragment): if __name__ == '__main__': - type_name = sys.argv[1] + type_name = sys.argv[1] yaml_path = sys.argv[2] content = sys.argv[3] if (yaml_path == "-s") else pathlib.Path(yaml_path).read_text() ValidateFragment(type_name, yaml.load(content, Loader=yaml.FullLoader)) From 6b92ccb2dc400b1881d1fe1120cb416922b310f1 Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Wed, 3 Jun 2020 11:00:03 -0700 Subject: [PATCH 03/14] Improved error reporting Signed-off-by: Dmitri Dolguikh --- docs/_ext/validating_code_block.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py index e4c13b064fcee..cc570b858ca29 100644 --- a/docs/_ext/validating_code_block.py +++ b/docs/_ext/validating_code_block.py @@ -46,8 +46,8 @@ def run(self): process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.poll(): - raise ExtensionError("Failed config validation for type: '{0}' in: {1} line: {2}".format( - self.options.get('type-name'), source, line)) + raise ExtensionError("Failed config validation for type: '{0}' in: {1} line: {2}:\n {3}".format( + self.options.get('type-name'), source, line, stderr.decode(encoding='utf-8'))) self.options.pop('type-name', None) return list(CodeBlock.run(self)) From 204428d42eb5210e286d094463b6610a1dc758ec Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Wed, 3 Jun 2020 12:38:37 -0700 Subject: [PATCH 04/14] Fixed rendering of BAZEL_BUILD_OPTIONS during bazel command generation Signed-off-by: Dmitri Dolguikh --- docs/_ext/validating_code_block.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py index cc570b858ca29..e37e52e2541f4 100644 --- a/docs/_ext/validating_code_block.py +++ b/docs/_ext/validating_code_block.py @@ -27,7 +27,7 @@ class ValidatingCodeBlock(CodeBlock): } option_spec.update(CodeBlock.option_spec) skip_validation = os.getenv('SPHINX_SKIP_CONFIG_VALIDATION') or 'false' - bazel_build_options = os.getenv('BAZEL_BUILD_OPTIONS') + bazel_build_options = os.getenv('BAZEL_BUILD_OPTIONS') or '' def run(self): source, line = self.state_machine.get_source_and_line(self.lineno) @@ -37,8 +37,7 @@ def run(self): if ValidatingCodeBlock.skip_validation.lower() != 'true': args = [ - arg for arg in [ - 'bazel', 'run', ValidatingCodeBlock.bazel_build_options, + arg for arg in ['bazel', 'run'] + ValidatingCodeBlock.bazel_build_options.split() + [ '//tools/config_validation:validate_fragment', '--', self.options.get('type-name'), '-s', '\n'.join(self.content) ] if arg != None @@ -46,8 +45,9 @@ def run(self): process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.poll(): - raise ExtensionError("Failed config validation for type: '{0}' in: {1} line: {2}:\n {3}".format( - self.options.get('type-name'), source, line, stderr.decode(encoding='utf-8'))) + raise ExtensionError( + "Failed config validation for type: '{0}' in: {1} line: {2}:\n {3}".format( + self.options.get('type-name'), source, line, stderr.decode(encoding='utf-8'))) self.options.pop('type-name', None) return list(CodeBlock.run(self)) From 382c36d3c2c51109d89a1c4c03399cf89ef9fb43 Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Thu, 4 Jun 2020 11:23:06 -0700 Subject: [PATCH 05/14] Responded to feedback Signed-off-by: Dmitri Dolguikh --- docs/_ext/validating_code_block.py | 6 ++---- docs/build.sh | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py index e37e52e2541f4..239c8d1d85737 100644 --- a/docs/_ext/validating_code_block.py +++ b/docs/_ext/validating_code_block.py @@ -10,7 +10,6 @@ import os import subprocess - class ValidatingCodeBlock(CodeBlock): """A directive that provides protobuf yaml formatting and validation. @@ -26,7 +25,7 @@ class ValidatingCodeBlock(CodeBlock): 'type-name': directives.unchanged, } option_spec.update(CodeBlock.option_spec) - skip_validation = os.getenv('SPHINX_SKIP_CONFIG_VALIDATION') or 'false' + skip_validation = (os.getenv('SPHINX_SKIP_CONFIG_VALIDATION') or 'false').lower() == 'true' bazel_build_options = os.getenv('BAZEL_BUILD_OPTIONS') or '' def run(self): @@ -35,7 +34,7 @@ def run(self): if self.options.get('type-name') == None: raise ExtensionError("Expected type name in: {0} line: {1}".format(source, line)) - if ValidatingCodeBlock.skip_validation.lower() != 'true': + if not ValidatingCodeBlock.skip_validation: args = [ arg for arg in ['bazel', 'run'] + ValidatingCodeBlock.bazel_build_options.split() + [ '//tools/config_validation:validate_fragment', '--', @@ -52,7 +51,6 @@ def run(self): self.options.pop('type-name', None) return list(CodeBlock.run(self)) - def setup(app): app.add_directive("validated-code-block", ValidatingCodeBlock) diff --git a/docs/build.sh b/docs/build.sh index 78e067f723057..cde2b626470ac 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -128,6 +128,7 @@ rsync -rav $API_DIR/diagrams "${GENERATED_RST_DIR}/api-docs" rsync -av "${SCRIPT_DIR}"/root/ "${SCRIPT_DIR}"/conf.py "${SCRIPT_DIR}"/_ext "${GENERATED_RST_DIR}" +# To speed up validate_fragment invocations in validating_code_block bazel build ${BAZEL_BUILD_OPTIONS} //tools/config_validation:validate_fragment sphinx-build -W --keep-going -b html "${GENERATED_RST_DIR}" "${DOCS_OUTPUT_DIR}" From c7bdb31612227d6fa79ff890ed59c0385a63dde6 Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Thu, 4 Jun 2020 12:11:37 -0700 Subject: [PATCH 06/14] Fixed formatting Signed-off-by: Dmitri Dolguikh --- docs/_ext/validating_code_block.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py index 239c8d1d85737..6ef6eed0e00bc 100644 --- a/docs/_ext/validating_code_block.py +++ b/docs/_ext/validating_code_block.py @@ -10,6 +10,7 @@ import os import subprocess + class ValidatingCodeBlock(CodeBlock): """A directive that provides protobuf yaml formatting and validation. @@ -51,6 +52,7 @@ def run(self): self.options.pop('type-name', None) return list(CodeBlock.run(self)) + def setup(app): app.add_directive("validated-code-block", ValidatingCodeBlock) From 881e654eebd1fe35b2d8c074f6fddc42845a94f3 Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Wed, 17 Jun 2020 13:50:58 -0700 Subject: [PATCH 07/14] Dcoumented SPHINX_SKIP_CONFIG_VALIDATION env var Signed-off-by: Dmitri Dolguikh --- api/CONTRIBUTING.md | 6 ++++ docs/README.md | 8 ++++- docs/_ext/validating_code_block.py | 7 ++--- docs/build.sh | 3 ++ docs/root/configuration/overview/examples.rst | 29 ++++++++++--------- 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/api/CONTRIBUTING.md b/api/CONTRIBUTING.md index dc77573c683b1..773248f2e2ea6 100644 --- a/api/CONTRIBUTING.md +++ b/api/CONTRIBUTING.md @@ -26,6 +26,12 @@ The documentation can be built locally in the root of https://github.com/envoypr docs/build.sh ``` +To skip configuration examples validation: + +``` +SPHINX_SKIP_CONFIG_VALIDATION=true docs/build.sh +``` + Or to use a hermetic Docker container: ``` diff --git a/docs/README.md b/docs/README.md index 119596fec9805..b672f51c8a4f8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,7 +4,13 @@ ./docs/build.sh ``` -The output can be found in `generated/docs`. +The output can be found in `generated/docs`. By default configuration examples are going to be validated during build. +To disable validation, set `SPHINX_SKIP_CONFIG_VALIDATION` environment variable to `true`: + +```bash +SPHINX_SKIP_CONFIG_VALIDATION=true docs/build.sh +``` + # How the Envoy website and docs are updated diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py index 6ef6eed0e00bc..04bf96912f93b 100644 --- a/docs/_ext/validating_code_block.py +++ b/docs/_ext/validating_code_block.py @@ -37,10 +37,9 @@ def run(self): if not ValidatingCodeBlock.skip_validation: args = [ - arg for arg in ['bazel', 'run'] + ValidatingCodeBlock.bazel_build_options.split() + [ - '//tools/config_validation:validate_fragment', '--', - self.options.get('type-name'), '-s', '\n'.join(self.content) - ] if arg != None + arg for arg in ['bazel-bin/tools/config_validation/validate_fragment'] + + ValidatingCodeBlock.bazel_build_options.split() + + [self.options.get('type-name'), '-s', '\n'.join(self.content)] if arg != None ] process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() diff --git a/docs/build.sh b/docs/build.sh index cde2b626470ac..9ca1bec440ebc 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# set SPHINX_SKIP_CONFIG_VALIDATION environment variable to true to skip +# validation of configuration examples + . tools/shell_utils.sh set -e diff --git a/docs/root/configuration/overview/examples.rst b/docs/root/configuration/overview/examples.rst index a4758cb151040..bc8124c488823 100644 --- a/docs/root/configuration/overview/examples.rst +++ b/docs/root/configuration/overview/examples.rst @@ -9,7 +9,8 @@ Static A minimal fully static bootstrap config is provided below: -.. code-block:: yaml +.. validated-code-block:: yaml + :type-name: envoy.config.bootstrap.v3.Bootstrap admin: access_log_path: /tmp/admin_access.log @@ -61,7 +62,8 @@ discovery ` via an :ref:`EDS` gRPC management server listening on 127.0.0.1:5678 is provided below: -.. code-block:: yaml +.. validated-code-block:: yaml + :type-name: envoy.config.bootstrap.v3.Bootstrap admin: access_log_path: /tmp/admin_access.log @@ -100,8 +102,8 @@ on 127.0.0.1:5678 is provided below: api_config_source: api_type: GRPC grpc_services: - envoy_grpc: - cluster_name: xds_cluster + - envoy_grpc: + cluster_name: xds_cluster - name: xds_cluster connect_timeout: 0.25s type: STATIC @@ -159,7 +161,8 @@ A fully dynamic bootstrap configuration, in which all resources other than those belonging to the management server are discovered via xDS is provided below: -.. code-block:: yaml +.. validated-code-block:: yaml + :type-name: envoy.config.bootstrap.v3.Bootstrap admin: access_log_path: /tmp/admin_access.log @@ -171,14 +174,14 @@ below: api_config_source: api_type: GRPC grpc_services: - envoy_grpc: - cluster_name: xds_cluster + - envoy_grpc: + cluster_name: xds_cluster cds_config: api_config_source: api_type: GRPC grpc_services: - envoy_grpc: - cluster_name: xds_cluster + - envoy_grpc: + cluster_name: xds_cluster static_resources: clusters: @@ -226,8 +229,8 @@ The management server could respond to LDS requests with: api_config_source: api_type: GRPC grpc_services: - envoy_grpc: - cluster_name: xds_cluster + - envoy_grpc: + cluster_name: xds_cluster http_filters: - name: envoy.filters.http.router @@ -262,8 +265,8 @@ The management server could respond to CDS requests with: api_config_source: api_type: GRPC grpc_services: - envoy_grpc: - cluster_name: xds_cluster + - envoy_grpc: + cluster_name: xds_cluster The management server could respond to EDS requests with: From 0f0446b19a82e36c37f4637f2d0e93f875a94cc8 Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Wed, 17 Jun 2020 14:33:23 -0700 Subject: [PATCH 08/14] Fixed build failure Signed-off-by: Dmitri Dolguikh --- docs/_ext/validating_code_block.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py index 04bf96912f93b..3c610f24a47d5 100644 --- a/docs/_ext/validating_code_block.py +++ b/docs/_ext/validating_code_block.py @@ -27,7 +27,6 @@ class ValidatingCodeBlock(CodeBlock): } option_spec.update(CodeBlock.option_spec) skip_validation = (os.getenv('SPHINX_SKIP_CONFIG_VALIDATION') or 'false').lower() == 'true' - bazel_build_options = os.getenv('BAZEL_BUILD_OPTIONS') or '' def run(self): source, line = self.state_machine.get_source_and_line(self.lineno) @@ -36,11 +35,7 @@ def run(self): raise ExtensionError("Expected type name in: {0} line: {1}".format(source, line)) if not ValidatingCodeBlock.skip_validation: - args = [ - arg for arg in ['bazel-bin/tools/config_validation/validate_fragment'] + - ValidatingCodeBlock.bazel_build_options.split() + - [self.options.get('type-name'), '-s', '\n'.join(self.content)] if arg != None - ] + args = ['bazel-bin/tools/config_validation/validate_fragment', self.options.get('type-name'), '-s', '\n'.join(self.content)] process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.poll(): From 071de202da5b2497d99d7f811675b91abe148df0 Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Wed, 17 Jun 2020 15:35:43 -0700 Subject: [PATCH 09/14] Fixed formatting issues Signed-off-by: Dmitri Dolguikh --- docs/_ext/validating_code_block.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py index 3c610f24a47d5..0ee708d29f5f0 100644 --- a/docs/_ext/validating_code_block.py +++ b/docs/_ext/validating_code_block.py @@ -35,7 +35,10 @@ def run(self): raise ExtensionError("Expected type name in: {0} line: {1}".format(source, line)) if not ValidatingCodeBlock.skip_validation: - args = ['bazel-bin/tools/config_validation/validate_fragment', self.options.get('type-name'), '-s', '\n'.join(self.content)] + args = [ + 'bazel-bin/tools/config_validation/validate_fragment', + self.options.get('type-name'), '-s', '\n'.join(self.content) + ] process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.poll(): From ee0ef1d1500bdc7b43ade4d06da9f1c9fc4b2895 Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Wed, 17 Jun 2020 19:14:21 -0700 Subject: [PATCH 10/14] Fixed format Signed-off-by: Dmitri Dolguikh --- docs/_ext/validating_code_block.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py index 0ee708d29f5f0..e2a2d9bc33779 100644 --- a/docs/_ext/validating_code_block.py +++ b/docs/_ext/validating_code_block.py @@ -38,7 +38,7 @@ def run(self): args = [ 'bazel-bin/tools/config_validation/validate_fragment', self.options.get('type-name'), '-s', '\n'.join(self.content) - ] + ] process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.poll(): From 4f0aecde5e2f432bcecd0e73883d103df74a0dfb Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Mon, 6 Jul 2020 13:27:13 -0700 Subject: [PATCH 11/14] Switched to argparse in validate_fragment.py Signed-off-by: Dmitri Dolguikh --- tools/config_validation/validate_fragment.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tools/config_validation/validate_fragment.py b/tools/config_validation/validate_fragment.py index a6dbbc0a3aaa1..e1bee4c6ec3b0 100644 --- a/tools/config_validation/validate_fragment.py +++ b/tools/config_validation/validate_fragment.py @@ -19,6 +19,7 @@ from bazel_tools.tools.python.runfiles import runfiles +import argparse def ValidateFragment(type_name, fragment): """Validate a dictionary representing a JSON/YAML fragment against an Envoy API proto3 type. @@ -49,9 +50,20 @@ def ValidateFragment(type_name, fragment): msg = message_factory.MessageFactory(pool=pool).GetPrototype(desc)() json_format.Parse(json_fragment, msg, descriptor_pool=pool) +def ParseArgs(): + parser = argparse.ArgumentParser(description='Validate a YAML fragment against an Envoy API proto3 type.') + parser.add_argument('Type', + help='a string providing the type name, e.g. envoy.config.bootstrap.v3.Bootstrap.') + parser.add_argument('Fragment', nargs='?', + help='Path to a YAML configuration fragment.') + parser.add_argument('-s', required=False, + help='YAML configuration fragment.') + + return parser.parse_args() + if __name__ == '__main__': - type_name = sys.argv[1] - yaml_path = sys.argv[2] - content = sys.argv[3] if (yaml_path == "-s") else pathlib.Path(yaml_path).read_text() + parsed_args = ParseArgs() + type_name = parsed_args.Type + content = parsed_args.s if (parsed_args.Fragment is None) else pathlib.Path(parsed_args.Fragment).read_text() ValidateFragment(type_name, yaml.load(content, Loader=yaml.FullLoader)) From 1fb328238a6f6bce83ab79fa10378c0589478b6f Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Mon, 6 Jul 2020 15:55:23 -0700 Subject: [PATCH 12/14] Fixed format Signed-off-by: Dmitri Dolguikh --- tools/config_validation/validate_fragment.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tools/config_validation/validate_fragment.py b/tools/config_validation/validate_fragment.py index e1bee4c6ec3b0..6f366b439e657 100644 --- a/tools/config_validation/validate_fragment.py +++ b/tools/config_validation/validate_fragment.py @@ -21,6 +21,7 @@ import argparse + def ValidateFragment(type_name, fragment): """Validate a dictionary representing a JSON/YAML fragment against an Envoy API proto3 type. @@ -50,14 +51,14 @@ def ValidateFragment(type_name, fragment): msg = message_factory.MessageFactory(pool=pool).GetPrototype(desc)() json_format.Parse(json_fragment, msg, descriptor_pool=pool) + def ParseArgs(): - parser = argparse.ArgumentParser(description='Validate a YAML fragment against an Envoy API proto3 type.') - parser.add_argument('Type', - help='a string providing the type name, e.g. envoy.config.bootstrap.v3.Bootstrap.') - parser.add_argument('Fragment', nargs='?', - help='Path to a YAML configuration fragment.') - parser.add_argument('-s', required=False, - help='YAML configuration fragment.') + parser = argparse.ArgumentParser( + description='Validate a YAML fragment against an Envoy API proto3 type.') + parser.add_argument( + 'Type', help='a string providing the type name, e.g. envoy.config.bootstrap.v3.Bootstrap.') + parser.add_argument('Fragment', nargs='?', help='Path to a YAML configuration fragment.') + parser.add_argument('-s', required=False, help='YAML configuration fragment.') return parser.parse_args() @@ -65,5 +66,6 @@ def ParseArgs(): if __name__ == '__main__': parsed_args = ParseArgs() type_name = parsed_args.Type - content = parsed_args.s if (parsed_args.Fragment is None) else pathlib.Path(parsed_args.Fragment).read_text() + content = parsed_args.s if (parsed_args.Fragment is None) else pathlib.Path( + parsed_args.Fragment).read_text() ValidateFragment(type_name, yaml.load(content, Loader=yaml.FullLoader)) From 6c13087742b9421957e4801641dc45b2441e60ab Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Wed, 8 Jul 2020 10:49:39 -0700 Subject: [PATCH 13/14] Responded to feedback Signed-off-by: Dmitri Dolguikh --- docs/_ext/validating_code_block.py | 7 +++---- tools/config_validation/validate_fragment.py | 12 ++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py index e2a2d9bc33779..517e4244d49c2 100644 --- a/docs/_ext/validating_code_block.py +++ b/docs/_ext/validating_code_block.py @@ -39,12 +39,11 @@ def run(self): 'bazel-bin/tools/config_validation/validate_fragment', self.options.get('type-name'), '-s', '\n'.join(self.content) ] - process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - if process.poll(): + completed = subprocess.run(args, capture_output=True, encoding='utf-8') + if completed.returncode != 0: raise ExtensionError( "Failed config validation for type: '{0}' in: {1} line: {2}:\n {3}".format( - self.options.get('type-name'), source, line, stderr.decode(encoding='utf-8'))) + self.options.get('type-name'), source, line, completed.stderr)) self.options.pop('type-name', None) return list(CodeBlock.run(self)) diff --git a/tools/config_validation/validate_fragment.py b/tools/config_validation/validate_fragment.py index 6f366b439e657..e36c688c716c8 100644 --- a/tools/config_validation/validate_fragment.py +++ b/tools/config_validation/validate_fragment.py @@ -56,8 +56,8 @@ def ParseArgs(): parser = argparse.ArgumentParser( description='Validate a YAML fragment against an Envoy API proto3 type.') parser.add_argument( - 'Type', help='a string providing the type name, e.g. envoy.config.bootstrap.v3.Bootstrap.') - parser.add_argument('Fragment', nargs='?', help='Path to a YAML configuration fragment.') + 'message_type', help='a string providing the type name, e.g. envoy.config.bootstrap.v3.Bootstrap.') + parser.add_argument('fragment_path', nargs='?', help='Path to a YAML configuration fragment.') parser.add_argument('-s', required=False, help='YAML configuration fragment.') return parser.parse_args() @@ -65,7 +65,7 @@ def ParseArgs(): if __name__ == '__main__': parsed_args = ParseArgs() - type_name = parsed_args.Type - content = parsed_args.s if (parsed_args.Fragment is None) else pathlib.Path( - parsed_args.Fragment).read_text() - ValidateFragment(type_name, yaml.load(content, Loader=yaml.FullLoader)) + message_type = parsed_args.message_type + content = parsed_args.s if (parsed_args.fragment_path is None) else pathlib.Path( + parsed_args.fragment_path).read_text() + ValidateFragment(message_type, yaml.load(content, Loader=yaml.FullLoader)) From a60d3ff9f753cc13a53b6137aae8b776a1318f91 Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Wed, 8 Jul 2020 11:29:48 -0700 Subject: [PATCH 14/14] Fixed build failure Signed-off-by: Dmitri Dolguikh --- docs/_ext/validating_code_block.py | 5 ++++- tools/config_validation/validate_fragment.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py index 517e4244d49c2..6220ae98618bb 100644 --- a/docs/_ext/validating_code_block.py +++ b/docs/_ext/validating_code_block.py @@ -39,7 +39,10 @@ def run(self): 'bazel-bin/tools/config_validation/validate_fragment', self.options.get('type-name'), '-s', '\n'.join(self.content) ] - completed = subprocess.run(args, capture_output=True, encoding='utf-8') + completed = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding='utf-8') if completed.returncode != 0: raise ExtensionError( "Failed config validation for type: '{0}' in: {1} line: {2}:\n {3}".format( diff --git a/tools/config_validation/validate_fragment.py b/tools/config_validation/validate_fragment.py index e36c688c716c8..faa9951114a87 100644 --- a/tools/config_validation/validate_fragment.py +++ b/tools/config_validation/validate_fragment.py @@ -56,7 +56,8 @@ def ParseArgs(): parser = argparse.ArgumentParser( description='Validate a YAML fragment against an Envoy API proto3 type.') parser.add_argument( - 'message_type', help='a string providing the type name, e.g. envoy.config.bootstrap.v3.Bootstrap.') + 'message_type', + help='a string providing the type name, e.g. envoy.config.bootstrap.v3.Bootstrap.') parser.add_argument('fragment_path', nargs='?', help='Path to a YAML configuration fragment.') parser.add_argument('-s', required=False, help='YAML configuration fragment.')