diff --git a/api/CONTRIBUTING.md b/api/CONTRIBUTING.md index dc77573c683b1..773248f2e2ea6 100644 --- a/api/CONTRIBUTING.md +++ b/api/CONTRIBUTING.md @@ -26,6 +26,12 @@ The documentation can be built locally in the root of https://github.com/envoypr docs/build.sh ``` +To skip configuration examples validation: + +``` +SPHINX_SKIP_CONFIG_VALIDATION=true docs/build.sh +``` + Or to use a hermetic Docker container: ``` diff --git a/docs/README.md b/docs/README.md index 119596fec9805..b672f51c8a4f8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,7 +4,13 @@ ./docs/build.sh ``` -The output can be found in `generated/docs`. +The output can be found in `generated/docs`. By default configuration examples are going to be validated during build. +To disable validation, set `SPHINX_SKIP_CONFIG_VALIDATION` environment variable to `true`: + +```bash +SPHINX_SKIP_CONFIG_VALIDATION=true docs/build.sh +``` + # How the Envoy website and docs are updated diff --git a/docs/_ext/validating_code_block.py b/docs/_ext/validating_code_block.py new file mode 100644 index 0000000000000..6220ae98618bb --- /dev/null +++ b/docs/_ext/validating_code_block.py @@ -0,0 +1,62 @@ +from typing import List +from docutils import nodes +from docutils.parsers.rst import Directive +from docutils.parsers.rst import directives +from sphinx.application import Sphinx +from sphinx.util.docutils import SphinxDirective +from sphinx.directives.code import CodeBlock +from sphinx.errors import ExtensionError + +import os +import subprocess + + +class ValidatingCodeBlock(CodeBlock): + """A directive that provides protobuf yaml formatting and validation. + + 'type-name' option is required and expected to conain full Envoy API type. + An ExtensionError is raised on validation failure. + Validation will be skipped if SPHINX_SKIP_CONFIG_VALIDATION environment variable is set. + """ + has_content = True + required_arguments = CodeBlock.required_arguments + optional_arguments = CodeBlock.optional_arguments + final_argument_whitespace = CodeBlock.final_argument_whitespace + option_spec = { + 'type-name': directives.unchanged, + } + option_spec.update(CodeBlock.option_spec) + skip_validation = (os.getenv('SPHINX_SKIP_CONFIG_VALIDATION') or 'false').lower() == 'true' + + def run(self): + source, line = self.state_machine.get_source_and_line(self.lineno) + # built-in directives.unchanged_required option validator produces a confusing error message + if self.options.get('type-name') == None: + raise ExtensionError("Expected type name in: {0} line: {1}".format(source, line)) + + if not ValidatingCodeBlock.skip_validation: + args = [ + 'bazel-bin/tools/config_validation/validate_fragment', + self.options.get('type-name'), '-s', '\n'.join(self.content) + ] + completed = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding='utf-8') + if completed.returncode != 0: + raise ExtensionError( + "Failed config validation for type: '{0}' in: {1} line: {2}:\n {3}".format( + self.options.get('type-name'), source, line, completed.stderr)) + + self.options.pop('type-name', None) + return list(CodeBlock.run(self)) + + +def setup(app): + app.add_directive("validated-code-block", ValidatingCodeBlock) + + return { + 'version': '0.1', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/docs/build.sh b/docs/build.sh index bc0c302414a16..9ca1bec440ebc 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# set SPHINX_SKIP_CONFIG_VALIDATION environment variable to true to skip +# validation of configuration examples + . tools/shell_utils.sh set -e @@ -126,6 +129,9 @@ cp -f "${CONFIGS_DIR}"/google-vrp/envoy-edge.yaml "${GENERATED_RST_DIR}"/configu rsync -rav $API_DIR/diagrams "${GENERATED_RST_DIR}/api-docs" -rsync -av "${SCRIPT_DIR}"/root/ "${SCRIPT_DIR}"/conf.py "${GENERATED_RST_DIR}" +rsync -av "${SCRIPT_DIR}"/root/ "${SCRIPT_DIR}"/conf.py "${SCRIPT_DIR}"/_ext "${GENERATED_RST_DIR}" + +# To speed up validate_fragment invocations in validating_code_block +bazel build ${BAZEL_BUILD_OPTIONS} //tools/config_validation:validate_fragment sphinx-build -W --keep-going -b html "${GENERATED_RST_DIR}" "${DOCS_OUTPUT_DIR}" diff --git a/docs/conf.py b/docs/conf.py index a2f4d250d939b..1eb5725b689b8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -67,7 +67,13 @@ def setup(app): # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinxcontrib.httpdomain', 'sphinx.ext.extlinks', 'sphinx.ext.ifconfig'] + +sys.path.append(os.path.abspath("./_ext")) + +extensions = [ + 'sphinxcontrib.httpdomain', 'sphinx.ext.extlinks', 'sphinx.ext.ifconfig', + 'validating_code_block' +] extlinks = { 'repo': ('https://github.com/envoyproxy/envoy/blob/{}/%s'.format(blob_sha), ''), 'api': ('https://github.com/envoyproxy/envoy/blob/{}/api/%s'.format(blob_sha), ''), diff --git a/docs/root/configuration/operations/runtime.rst b/docs/root/configuration/operations/runtime.rst index 45b1f72634803..2e72e52bb953b 100644 --- a/docs/root/configuration/operations/runtime.rst +++ b/docs/root/configuration/operations/runtime.rst @@ -24,7 +24,8 @@ runtime ` bootstrap configu layering. Runtime settings in later layers override earlier layers. A typical configuration might be: -.. code-block:: yaml +.. validated-code-block:: yaml + :type-name: envoy.config.bootstrap.v3.LayeredRuntime layers: - name: static_layer_0 diff --git a/docs/root/configuration/overview/examples.rst b/docs/root/configuration/overview/examples.rst index a4758cb151040..bc8124c488823 100644 --- a/docs/root/configuration/overview/examples.rst +++ b/docs/root/configuration/overview/examples.rst @@ -9,7 +9,8 @@ Static A minimal fully static bootstrap config is provided below: -.. code-block:: yaml +.. validated-code-block:: yaml + :type-name: envoy.config.bootstrap.v3.Bootstrap admin: access_log_path: /tmp/admin_access.log @@ -61,7 +62,8 @@ discovery ` via an :ref:`EDS` gRPC management server listening on 127.0.0.1:5678 is provided below: -.. code-block:: yaml +.. validated-code-block:: yaml + :type-name: envoy.config.bootstrap.v3.Bootstrap admin: access_log_path: /tmp/admin_access.log @@ -100,8 +102,8 @@ on 127.0.0.1:5678 is provided below: api_config_source: api_type: GRPC grpc_services: - envoy_grpc: - cluster_name: xds_cluster + - envoy_grpc: + cluster_name: xds_cluster - name: xds_cluster connect_timeout: 0.25s type: STATIC @@ -159,7 +161,8 @@ A fully dynamic bootstrap configuration, in which all resources other than those belonging to the management server are discovered via xDS is provided below: -.. code-block:: yaml +.. validated-code-block:: yaml + :type-name: envoy.config.bootstrap.v3.Bootstrap admin: access_log_path: /tmp/admin_access.log @@ -171,14 +174,14 @@ below: api_config_source: api_type: GRPC grpc_services: - envoy_grpc: - cluster_name: xds_cluster + - envoy_grpc: + cluster_name: xds_cluster cds_config: api_config_source: api_type: GRPC grpc_services: - envoy_grpc: - cluster_name: xds_cluster + - envoy_grpc: + cluster_name: xds_cluster static_resources: clusters: @@ -226,8 +229,8 @@ The management server could respond to LDS requests with: api_config_source: api_type: GRPC grpc_services: - envoy_grpc: - cluster_name: xds_cluster + - envoy_grpc: + cluster_name: xds_cluster http_filters: - name: envoy.filters.http.router @@ -262,8 +265,8 @@ The management server could respond to CDS requests with: api_config_source: api_type: GRPC grpc_services: - envoy_grpc: - cluster_name: xds_cluster + - envoy_grpc: + cluster_name: xds_cluster The management server could respond to EDS requests with: diff --git a/tools/config_validation/validate_fragment.py b/tools/config_validation/validate_fragment.py index 403b5540418f1..faa9951114a87 100644 --- a/tools/config_validation/validate_fragment.py +++ b/tools/config_validation/validate_fragment.py @@ -19,6 +19,8 @@ from bazel_tools.tools.python.runfiles import runfiles +import argparse + def ValidateFragment(type_name, fragment): """Validate a dictionary representing a JSON/YAML fragment against an Envoy API proto3 type. @@ -50,7 +52,21 @@ def ValidateFragment(type_name, fragment): json_format.Parse(json_fragment, msg, descriptor_pool=pool) +def ParseArgs(): + parser = argparse.ArgumentParser( + description='Validate a YAML fragment against an Envoy API proto3 type.') + parser.add_argument( + 'message_type', + help='a string providing the type name, e.g. envoy.config.bootstrap.v3.Bootstrap.') + parser.add_argument('fragment_path', nargs='?', help='Path to a YAML configuration fragment.') + parser.add_argument('-s', required=False, help='YAML configuration fragment.') + + return parser.parse_args() + + if __name__ == '__main__': - type_name, yaml_path = sys.argv[1:] - ValidateFragment(type_name, yaml.load(pathlib.Path(yaml_path).read_text(), - Loader=yaml.FullLoader)) + parsed_args = ParseArgs() + message_type = parsed_args.message_type + content = parsed_args.s if (parsed_args.fragment_path is None) else pathlib.Path( + parsed_args.fragment_path).read_text() + ValidateFragment(message_type, yaml.load(content, Loader=yaml.FullLoader))