diff --git a/api/BUILD b/api/BUILD index 0a3d7c74b2943..7efa53f9f5ad5 100644 --- a/api/BUILD +++ b/api/BUILD @@ -266,3 +266,10 @@ filegroup( ], visibility = ["//visibility:public"], ) + +genquery( + name = "v3_proto_srcs", + expression = "labels(srcs, labels(deps, @envoy_api//:v3_protos))", + scope = ["@envoy_api//:v3_protos"], + visibility = ["//visibility:public"], +) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index cb8967cd6faee..4ffcab400090b 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -77,10 +77,14 @@ def _envoy_repo_impl(repository_ctx): constraints of a `genquery`. """ + repo_path = repository_ctx.path(repository_ctx.attr.envoy_root).dirname version = repository_ctx.read(repo_path.get_child("VERSION")).strip() repository_ctx.file("version.bzl", "VERSION = '%s'" % version) repository_ctx.file("__init__.py", "PATH = '%s'\nVERSION = '%s'" % (repo_path, version)) + data_root = repo_path.get_child("tools").get_child("data") + for child in data_root.readdir(): + repository_ctx.symlink(child, child.basename) repository_ctx.file("WORKSPACE", "") repository_ctx.file("BUILD", """ load("@rules_python//python:defs.bzl", "py_library") diff --git a/source/extensions/BUILD b/source/extensions/BUILD index 7105032fd6d61..072058e38df8a 100644 --- a/source/extensions/BUILD +++ b/source/extensions/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 exports_files([ "extensions_metadata.yaml", "extensions_build_config.bzl", + "extension_status_categories.yaml", ]) json_data( diff --git a/source/extensions/extension_status_categories.yaml b/source/extensions/extension_status_categories.yaml new file mode 100644 index 0000000000000..e9b77875816bb --- /dev/null +++ b/source/extensions/extension_status_categories.yaml @@ -0,0 +1,25 @@ +security_postures: + robust_to_untrusted_downstream: > + This extension is intended to be robust against untrusted downstream + traffic. It assumes that the upstream is trusted. + robust_to_untrusted_downstream_and_upstream: > + This extension is intended to be robust against both untrusted + downstream and upstream traffic. + requires_trusted_downstream_and_upstream: > + This extension is not hardened and should only be used in + deployments where both the downstream and upstream are trusted. + unknown: > + This extension has an unknown security posture and should only be + used in deployments where both the downstream and upstream are + trusted. + data_plane_agnostic: > + This extension does not operate on the data plane and hence is + intended to be robust against untrusted traffic. + +status_types: + alpha: > + This extension is functional but has not had substantial production + burn time, use only with this caveat. + wip: > + This extension is work-in-progress. Functionality is incomplete and + it is not intended for production use. diff --git a/tools/base/envoy_python.bzl b/tools/base/envoy_python.bzl index 550ff901768de..0a461615129e2 100644 --- a/tools/base/envoy_python.bzl +++ b/tools/base/envoy_python.bzl @@ -33,6 +33,7 @@ def envoy_py_library( name = None, deps = [], data = [], + srcs = [], visibility = ["//visibility:public"], envoy_prefix = "", test = True): @@ -42,7 +43,7 @@ def envoy_py_library( py_library( name = name, - srcs = ["%s.py" % name], + srcs = srcs or ["%s.py" % name], deps = deps, data = data, visibility = visibility, @@ -54,6 +55,8 @@ def envoy_py_binary( name = None, deps = [], data = [], + args = [], + srcs = [], visibility = ["//visibility:public"], envoy_prefix = "@envoy", test = True): @@ -63,9 +66,10 @@ def envoy_py_binary( py_binary( name = name, - srcs = ["%s.py" % name], + srcs = ["%s.py" % name] + srcs, deps = deps, data = data, + args = args, visibility = visibility, ) diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index d01897fc338a0..8042dc68a889d 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -280,6 +280,7 @@ def __init__(self, args): self.operation_type = args.operation_type self.target_path = args.target_path self.api_prefix = args.api_prefix + self.data_prefix = args.data_prefix self.envoy_build_rule_check = not args.skip_envoy_build_rule_check self.namespace_check = args.namespace_check self.namespace_check_excluded_paths = args.namespace_check_excluded_paths + [ @@ -491,6 +492,9 @@ def is_build_file(self, file_path): return True return False + def is_data_file(self, file_path): + return file_path.startswith(self.data_prefix) + def is_external_build_file(self, file_path): return self.is_build_file(file_path) and ( file_path.startswith("./bazel/external/") @@ -884,6 +888,7 @@ def check_build_line(self, line, file_path, report_error): "//source/common/protobuf instead.") if (self.envoy_build_rule_check and not self.is_starlark_file(file_path) and not self.is_workspace_file(file_path) + and not self.is_data_file(file_path) and not self.is_external_build_file(file_path) and "@envoy//" in line): report_error("Superfluous '@envoy//' prefix") if not self.allow_listed_for_build_urls(file_path) and (" urls = " in line @@ -893,6 +898,7 @@ def check_build_line(self, line, file_path, report_error): def fix_build_line(self, file_path, line, line_number): if (self.envoy_build_rule_check and not self.is_starlark_file(file_path) and not self.is_workspace_file(file_path) + and not self.is_data_file(file_path) and not self.is_external_build_file(file_path)): line = line.replace("@envoy//", "//") return line @@ -1123,6 +1129,7 @@ def whitelisted_for_memcpy(self, file_path): default=multiprocessing.cpu_count(), help="number of worker processes to use; defaults to one per core.") parser.add_argument("--api-prefix", type=str, default="./api/", help="path of the API tree.") + parser.add_argument("--data-prefix", type=str, default="./tools/data", help="path of the tools data tree.") parser.add_argument( "--skip_envoy_build_rule_check", action="store_true", diff --git a/tools/data/api/BUILD b/tools/data/api/BUILD new file mode 100644 index 0000000000000..c7e536647ef45 --- /dev/null +++ b/tools/data/api/BUILD @@ -0,0 +1,9 @@ +load("@envoy_repo//:utils.bzl", "py_data") + +licenses(["notice"]) # Apache 2 + +py_data( + name = "v3_proto_rst", + filters = ["//filters:proto_rst_srcs"], + source = "@envoy_api//:v3_proto_srcs", +) diff --git a/tools/data/api/bazel/BUILD b/tools/data/api/bazel/BUILD new file mode 100644 index 0000000000000..2210bbe849393 --- /dev/null +++ b/tools/data/api/bazel/BUILD @@ -0,0 +1,9 @@ +load("@envoy_repo//:utils.bzl", "py_data") + +licenses(["notice"]) # Apache 2 + +py_data( + name = "repository_locations", + filters = ["//filters:interpolate_repository_locations"], + source = "@envoy_api//bazel:repository_locations.json", +) diff --git a/tools/data/bazel/BUILD b/tools/data/bazel/BUILD new file mode 100644 index 0000000000000..671f7081bfe4f --- /dev/null +++ b/tools/data/bazel/BUILD @@ -0,0 +1,15 @@ +load("@envoy_repo//:utils.bzl", "py_data") + +licenses(["notice"]) # Apache 2 + +py_data( + name = "repository_locations", + filters = ["//filters:interpolate_repository_locations"], + source = "@envoy//bazel:repository_locations.json", +) + +py_data( + name = "all_repository_locations", + filters = ["//filters:add_api_repository_locations"], + source = "@envoy//bazel:repository_locations.json", +) diff --git a/tools/data/contrib/BUILD b/tools/data/contrib/BUILD new file mode 100644 index 0000000000000..6f83448f071f8 --- /dev/null +++ b/tools/data/contrib/BUILD @@ -0,0 +1,19 @@ +load("@envoy_repo//:utils.bzl", "py_data") + +licenses(["notice"]) # Apache 2 + +py_data( + name = "extensions_metadata", + source = "@envoy//contrib:extensions_metadata.yaml", +) + +py_data( + name = "extensions_categories", + filters = ["//filters:build_categories"], + source = "@envoy//contrib:extensions_metadata.yaml", +) + +py_data( + name = "extensions_build_config", + source = "@envoy//contrib:contrib_extensions_build_config.json", +) diff --git a/tools/data/docs/BUILD b/tools/data/docs/BUILD new file mode 100644 index 0000000000000..7aa46a6aa7401 --- /dev/null +++ b/tools/data/docs/BUILD @@ -0,0 +1,19 @@ +load("@envoy_repo//:utils.bzl", "py_data") + +licenses(["notice"]) # Apache 2 + +py_data( + name = "protodoc_manifest", + source = "@envoy//docs:protodoc_manifest.yaml", +) + +py_data( + name = "v2_mapping", + source = "@envoy//docs:v2_mapping.json", +) + +py_data( + name = "empty_extensions", + filters = ["//filters:empty_extensions"], + source = "@envoy//docs:empty_extensions.json", +) diff --git a/tools/data/filters/BUILD b/tools/data/filters/BUILD new file mode 100644 index 0000000000000..0a5f4a038ddc4 --- /dev/null +++ b/tools/data/filters/BUILD @@ -0,0 +1,47 @@ +load("@dev_pip3//:requirements.bzl", dev_requirement = "requirement") +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "proto_rst_srcs", + srcs = ["proto_rst_srcs.py"], + visibility = ["//visibility:public"], +) + +py_library( + name = "empty_extensions", + srcs = ["empty_extensions.py"], + visibility = ["//visibility:public"], +) + +py_library( + name = "interpolate_repository_locations", + srcs = ["interpolate_repository_locations.py"], + visibility = ["//visibility:public"], +) + +py_library( + name = "add_api_repository_locations", + srcs = ["add_api_repository_locations.py"], + deps = ["//api/bazel:repository_locations_py"], + visibility = ["//visibility:public"], +) + +py_library( + name = "build_categories", + srcs = ["build_categories.py"], + visibility = ["//visibility:public"], +) + +py_library( + name = "add_contrib_extensions_metadata", + srcs = ["add_contrib_extensions_metadata.py"], + deps = ["//contrib:extensions_metadata_py"], + visibility = ["//visibility:public"], +) + +py_library( + name = "extension_security_postures", + srcs = ["extension_security_postures.py"], + visibility = ["//visibility:public"], + deps = [dev_requirement("envoy.docs.abstract")], +) diff --git a/tools/data/filters/add_api_repository_locations.py b/tools/data/filters/add_api_repository_locations.py new file mode 100644 index 0000000000000..61d0fbe90d28b --- /dev/null +++ b/tools/data/filters/add_api_repository_locations.py @@ -0,0 +1,6 @@ +from envoy_repo.api.bazel import repository_locations + + +def main(data): + data.update(repository_locations.data.copy()) + return data diff --git a/tools/data/filters/add_contrib_extensions_metadata.py b/tools/data/filters/add_contrib_extensions_metadata.py new file mode 100644 index 0000000000000..9b709c7312d6d --- /dev/null +++ b/tools/data/filters/add_contrib_extensions_metadata.py @@ -0,0 +1,9 @@ +from envoy_repo.contrib import extensions_metadata + + +def main(data): + contrib_extensions = extensions_metadata.data.copy() + for v in contrib_extensions.values(): + v['contrib'] = True + data.update(contrib_extensions) + return data diff --git a/tools/data/filters/build_categories.py b/tools/data/filters/build_categories.py new file mode 100644 index 0000000000000..c206bc3b6604d --- /dev/null +++ b/tools/data/filters/build_categories.py @@ -0,0 +1,7 @@ +# create an index of extension categories from extension dbs +def main(data): + ret = {} + for _k, _v in data.items(): + for _cat in _v['categories']: + ret.setdefault(_cat, []).append(_k) + return ret diff --git a/tools/data/filters/empty_extensions.py b/tools/data/filters/empty_extensions.py new file mode 100644 index 0000000000000..0d8c629b259bb --- /dev/null +++ b/tools/data/filters/empty_extensions.py @@ -0,0 +1,7 @@ +import pathlib + + +def main(data): + for k, v in data.items(): + v["docs_path"] = str(pathlib.Path(v['path'], 'empty', f"{v['path'].split('/').pop()}.rst")) + return data diff --git a/tools/data/filters/extension_security_postures.py b/tools/data/filters/extension_security_postures.py new file mode 100644 index 0000000000000..892c7514091a4 --- /dev/null +++ b/tools/data/filters/extension_security_postures.py @@ -0,0 +1,12 @@ +"""Transform the extensions metadata dict into the security postures dict""" + +from collections import defaultdict + +from envoy.docs import abstract + + +def main(data: abstract.ExtensionsMetadataDict) -> abstract.ExtensionSecurityPosturesDict: + security_postures = defaultdict(list) + for extension, metadata in data.items(): + security_postures[metadata['security_posture']].append(extension) + return security_postures diff --git a/tools/data/filters/interpolate_repository_locations.py b/tools/data/filters/interpolate_repository_locations.py new file mode 100644 index 0000000000000..aef59ab1a0c60 --- /dev/null +++ b/tools/data/filters/interpolate_repository_locations.py @@ -0,0 +1,17 @@ +def format_data(s, data): + return s.format( + version=data["version"], + underscore_version=data["version"].replace(".", "_"), + dash_version=data["version"].replace(".", "-")) + + +def main(data): + for k, v in data.items(): + # this should reflect any transformations in `api/bazel/repository_locations_utils.bzl` + if not v.get("version"): + data[k] = v + continue + v["strip_prefix"] = format_data(v.get("strip_prefix", ""), v) + v["urls"] = [format_data(url, v) for url in v.get("urls", [])] + data[k] = v + return data diff --git a/tools/data/filters/proto_rst_srcs.py b/tools/data/filters/proto_rst_srcs.py new file mode 100644 index 0000000000000..742bfd8d03b18 --- /dev/null +++ b/tools/data/filters/proto_rst_srcs.py @@ -0,0 +1,18 @@ +"""Transform bazel api labels into rst paths in the docs""" + +from typing import Tuple + + +def format_proto_src(src: str) -> str: + """Transform api bazel Label -> rst path in docs + + eg: + @envoy_api//envoy/watchdog/v3alpha:abort_action.proto + + -> envoy/watchdog/v3alpha/abort_action.proto.rst + """ + return f"{src.replace(':', '/').strip('@').replace('//', '/')[10:]}.rst" + + +def main(data) -> Tuple[str]: + return tuple(format_proto_src(src) for src in data if src) diff --git a/tools/data/loader/BUILD b/tools/data/loader/BUILD new file mode 100644 index 0000000000000..d66c1b0c39ba1 --- /dev/null +++ b/tools/data/loader/BUILD @@ -0,0 +1,8 @@ +load("@rules_python//python:defs.bzl", "py_binary") + +licenses(["notice"]) # Apache 2 + +exports_files([ + "load_data.py", + "print_data.py", +]) diff --git a/tools/data/loader/load_data.py b/tools/data/loader/load_data.py new file mode 100644 index 0000000000000..eec9a125af34d --- /dev/null +++ b/tools/data/loader/load_data.py @@ -0,0 +1,38 @@ +import os +import pathlib +from rules_python.python.runfiles import runfiles + +__IMPORTS__ # noqa: F821 + +_loader = __LOADER__ # noqa: F821 + + +def _resolve(provided_path): + # Resolve the path, to the data file + # Adapts the filepath to work in different invokations + # ie - run, build, genrules, etc + path = pathlib.Path(provided_path) + if path.exists(): + return path + run = runfiles.Create() + location = run.Rlocation(str(path).strip(".").strip("/")) + if location: + path = pathlib.Path(location) + if not path.exists(): + # If the build is invoked from the envoy workspace it has no prefix, + # so search in the runfiles with `envoy/` prefix to path. + path = pathlib.Path(run.Rlocation(os.path.join("envoy", provided_path))) + return path + raise Exception(f"Unable to find data file {provided_path}") + + +data = _loader(_resolve("__DATA_FILE__").read_text()) + +_filters = __FILTERS__ # noqa: F821 + +_filters = (_filters,) if not isinstance(_filters, tuple) else _filters + +for _filter in _filters: + data = _filter(data) + +__all__ = ("data",) diff --git a/tools/data/loader/print_data.py b/tools/data/loader/print_data.py new file mode 100644 index 0000000000000..bb008dd55b17a --- /dev/null +++ b/tools/data/loader/print_data.py @@ -0,0 +1,12 @@ +import json +import sys + +__IMPORT__ # noqa: F821 + + +def main(): + print(json.dumps(data)) # noqa: F821 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/data/source/extensions/BUILD b/tools/data/source/extensions/BUILD new file mode 100644 index 0000000000000..37a0cb97fdae4 --- /dev/null +++ b/tools/data/source/extensions/BUILD @@ -0,0 +1,39 @@ +load("@envoy_repo//:utils.bzl", "py_data") + +licenses(["notice"]) # Apache 2 + +py_data( + name = "extensions_metadata", + source = "@envoy//source/extensions:extensions_metadata.yaml", +) + +py_data( + name = "extensions_categories", + filters = ["//filters:build_categories"], + source = "@envoy//source/extensions:extensions_metadata.yaml", +) + +py_data( + name = "extensions_build_config", + source = "@envoy//source/extensions:extensions_build_config.json", +) + +py_data( + name = "all_extensions_metadata", + filters = ["//filters:add_contrib_extensions_metadata"], + source = "@envoy//source/extensions:extensions_metadata.yaml", +) + +py_data( + name = "security_postures", + filters = [ + "//filters:add_contrib_extensions_metadata", + "//filters:extension_security_postures", + ], + source = "@envoy//source/extensions:extensions_metadata.yaml", +) + +py_data( + name = "extension_status_categories", + source = "@envoy//source/extensions:extension_status_categories.yaml", +) diff --git a/tools/data/utils.bzl b/tools/data/utils.bzl new file mode 100644 index 0000000000000..5cf0a3708dc2a --- /dev/null +++ b/tools/data/utils.bzl @@ -0,0 +1,194 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_binary") + + +def _py_filter_imports(filters): + # Python imports for filters + # Transforms python deps labels for workspace rules into python import statements + filter_imports = [] + for i, filter in enumerate(filters): + python_path = filter.path.split(".")[0].replace("/", ".") + if python_path.startswith("external"): + python_path = python_path[9:] + filter_imports.append("from %s import main as _filter%s" % (python_path, i)) + return "\n".join(filter_imports) + +def _py_imports(imports, filters): + # Dynamic python imports for loader and filters + return "\n".join([("import %s" % imp) for imp in imports] + [""]) + _py_filter_imports(filters) + +def _py_data_impl(ctx): + out = ctx.actions.declare_file(ctx.label.name + ".py") + ctx.actions.expand_template( + output = out, + template = ctx.file._template, + substitutions = { + "__DATA_FILE__": ctx.file.source.short_path, + "__LOADER__": ctx.attr.loader, + "__IMPORTS__": _py_imports(ctx.attr.imports, ctx.files.filters), + "__FILTERS__": ",".join(["_filter%s" % i for i in range(0, len(ctx.files.filters))]) or "()", + }, + ) + return [DefaultInfo(files = depset([out]))] + +_py_data = rule( + implementation = _py_data_impl, + attrs = { + "source": attr.label(allow_single_file = True), + "filters": attr.label_list(), + "loader": attr.string(default="str.splitlines"), + "imports": attr.string_list(default=[]), + "_template": attr.label( + allow_single_file = True, + default="//loader:load_data.py"), + }, +) + +def _py_printer_impl(ctx): + out = ctx.actions.declare_file(ctx.label.name + ".py") + python_path = ctx.file.source.short_path.lstrip("./").split(".")[0].replace("/", ".") + if python_path.startswith("external"): + python_path = python_path[9:] + ctx.actions.expand_template( + output = out, + template = ctx.file._template, + substitutions = { + "__IMPORT__": "from %s import data" % python_path, + }, + ) + return [DefaultInfo(files = depset([out]))] + +_py_printer = rule( + implementation = _py_printer_impl, + attrs = { + "source": attr.label(allow_single_file = True), + "_template": attr.label( + allow_single_file = True, + default="//loader:print_data.py"), + }, +) + +def py_data( + name, + source, + cache = True, + format = "", + filters = [], + visibility = ["//visibility:public"], + **kwargs): + """Provide a json source as an importable python module + + Default format is `json`, and can be ommitted from the rule, but `yaml` + should also work. + + For example with a rule such as: + + ``` + py_data( + name = "some_bazel_data", + source = ":some_bazel_data_source.json", + format = "json", + ) + ``` + + ...in `/tools/foo`, and the following py library set up: + + ``` + py_library( + name = "some_lib", + srcs = ["some_lib.py"], + deps = [ + "//tools/foo:some_bazel_data_py", + ], + ) + ``` + + Note the `_py` suffix. + + The library can import the data as follows (assuming the data source + provides a `dict`): + + ``` + from tools.foo import some_bazel_data + + assert isinstance(some_bazel_data.data, dict) + ``` + + You can also specify a list of labels for `filters`, which are python libs + containing a `main` function that is called with the data, and should + return it after making any mutations. + """ + if not format: + if source.endswith(".yaml"): + format = "yaml" + elif source.endswith(".json"): + format = "json" + imports = [] + loader = "str.splitlines" + if format == "yaml": + loader = "yaml.safe_load" + imports = ["yaml"] + elif format == "json": + loader = "json.loads" + imports = ["json"] + + cache = cache and filters + + if cache: + cached_name = "cached_%s" % name + cached_lib_name = "cached_lib_%s" % name + json_name = "%s.json" % name + printer_name = "%s_printer_py" % name + bin_name = "%s_bin_py" % name + + # create the filtered data + _py_data( + name = cached_lib_name, + source = source, + loader = loader, + imports = imports, + filters = filters) + # create a lib exposing filtered data + py_library( + name = cached_name, + srcs = [cached_lib_name], + data = [source], + deps = filters + ["@rules_python//python/runfiles"], + visibility = visibility, + ) + # create a printer to print out the filtered data + _py_printer( + name = printer_name, + source = cached_name) + py_binary( + name = bin_name, + srcs = [printer_name, cached_name], + main = "%s.py" % printer_name, + deps = filters + ["@rules_python//python/runfiles"], + visibility = visibility, + ) + # serialize the filtered data to json + native.genrule( + name = "%s_json" % name, + cmd = "$(location %s) > $@" % bin_name, + tools = [bin_name], + outs = [json_name], + ) + source = json_name + loader = "json.loads" + filters = [] + imports = ["json"] + + _py_data( + name = name, + source = source, + loader = loader, + imports = imports, + filters = filters) + + py_library( + name = "%s_py" % name, + srcs = [name], + data = [source], + deps = filters + ["@rules_python//python/runfiles"], + visibility = visibility, + ) diff --git a/tools/extensions/BUILD b/tools/extensions/BUILD index 52147a0e6b446..c895270fc2100 100644 --- a/tools/extensions/BUILD +++ b/tools/extensions/BUILD @@ -6,6 +6,10 @@ licenses(["notice"]) # Apache 2 envoy_package() +exports_files([ + "extension_security_postures_metadata.yaml", +]) + envoy_py_binary( name = "tools.extensions.extensions_check", data = [