[WIP] bazel: Add python data providers

api/BUILD

@@ -266,3 +266,10 @@ filegroup(
     ],
     visibility = ["//visibility:public"],
 )
+
+genquery(
+    name = "v3_proto_srcs",
+    expression = "labels(srcs, labels(deps, @envoy_api//:v3_protos))",
+    scope = ["@envoy_api//:v3_protos"],
+    visibility = ["//visibility:public"],
+)

bazel/repositories.bzl

@@ -77,10 +77,14 @@ def _envoy_repo_impl(repository_ctx):
     constraints of a `genquery`.
 
     """
+
     repo_path = repository_ctx.path(repository_ctx.attr.envoy_root).dirname
     version = repository_ctx.read(repo_path.get_child("VERSION")).strip()
     repository_ctx.file("version.bzl", "VERSION = '%s'" % version)
     repository_ctx.file("__init__.py", "PATH = '%s'\nVERSION = '%s'" % (repo_path, version))
+    data_root = repo_path.get_child("tools").get_child("data")
+    for child in data_root.readdir():
+        repository_ctx.symlink(child, child.basename)
     repository_ctx.file("WORKSPACE", "")
     repository_ctx.file("BUILD", """
 load("@rules_python//python:defs.bzl", "py_library")

source/extensions/BUILD

@@ -6,6 +6,7 @@ licenses(["notice"])  # Apache 2
 exports_files([
     "extensions_metadata.yaml",
     "extensions_build_config.bzl",
+    "extension_status_categories.yaml",
 ])
 
 json_data(

source/extensions/extension_status_categories.yaml

@@ -0,0 +1,25 @@
+security_postures:
+  robust_to_untrusted_downstream: >
+    This extension is intended to be robust against untrusted downstream
+    traffic. It assumes that the upstream is trusted.
+  robust_to_untrusted_downstream_and_upstream: >
+    This extension is intended to be robust against both untrusted
+    downstream and upstream traffic.
+  requires_trusted_downstream_and_upstream: >
+    This extension is not hardened and should only be used in
+    deployments where both the downstream and upstream are trusted.
+  unknown: >
+    This extension has an unknown security posture and should only be
+    used in deployments where both the downstream and upstream are
+    trusted.
+  data_plane_agnostic: >
+    This extension does not operate on the data plane and hence is
+    intended to be robust against untrusted traffic.
+
+status_types:
+  alpha: >
+    This extension is functional but has not had substantial production
+    burn time, use only with this caveat.
+  wip: >
+    This extension is work-in-progress. Functionality is incomplete and
+    it is not intended for production use.

tools/base/envoy_python.bzl

@@ -33,6 +33,7 @@ def envoy_py_library(
         name = None,
         deps = [],
         data = [],
+        srcs = [],
         visibility = ["//visibility:public"],
         envoy_prefix = "",
         test = True):
@@ -42,7 +43,7 @@ def envoy_py_library(
 
     py_library(
         name = name,
-        srcs = ["%s.py" % name],
+        srcs = srcs or ["%s.py" % name],
         deps = deps,
         data = data,
         visibility = visibility,
@@ -54,6 +55,8 @@ def envoy_py_binary(
         name = None,
         deps = [],
         data = [],
+        args = [],
+        srcs = [],
         visibility = ["//visibility:public"],
         envoy_prefix = "@envoy",
         test = True):
@@ -63,9 +66,10 @@ def envoy_py_binary(
 
     py_binary(
         name = name,
-        srcs = ["%s.py" % name],
+        srcs = ["%s.py" % name] + srcs,
         deps = deps,
         data = data,
+        args = args,
         visibility = visibility,
     )
 

tools/code_format/check_format.py

@@ -280,6 +280,7 @@ def __init__(self, args):
         self.operation_type = args.operation_type
         self.target_path = args.target_path
         self.api_prefix = args.api_prefix
+        self.data_prefix = args.data_prefix
         self.envoy_build_rule_check = not args.skip_envoy_build_rule_check
         self.namespace_check = args.namespace_check
         self.namespace_check_excluded_paths = args.namespace_check_excluded_paths + [
@@ -491,6 +492,9 @@ def is_build_file(self, file_path):
             return True
         return False
 
+    def is_data_file(self, file_path):
+        return file_path.startswith(self.data_prefix)
+
     def is_external_build_file(self, file_path):
         return self.is_build_file(file_path) and (
             file_path.startswith("./bazel/external/")
@@ -884,6 +888,7 @@ def check_build_line(self, line, file_path, report_error):
                 "//source/common/protobuf instead.")
         if (self.envoy_build_rule_check and not self.is_starlark_file(file_path)
                 and not self.is_workspace_file(file_path)
+                and not self.is_data_file(file_path)
                 and not self.is_external_build_file(file_path) and "@envoy//" in line):
             report_error("Superfluous '@envoy//' prefix")
         if not self.allow_listed_for_build_urls(file_path) and (" urls = " in line
@@ -893,6 +898,7 @@ def check_build_line(self, line, file_path, report_error):
     def fix_build_line(self, file_path, line, line_number):
         if (self.envoy_build_rule_check and not self.is_starlark_file(file_path)
                 and not self.is_workspace_file(file_path)
+                and not self.is_data_file(file_path)
                 and not self.is_external_build_file(file_path)):
             line = line.replace("@envoy//", "//")
         return line
@@ -1123,6 +1129,7 @@ def whitelisted_for_memcpy(self, file_path):
         default=multiprocessing.cpu_count(),
         help="number of worker processes to use; defaults to one per core.")
     parser.add_argument("--api-prefix", type=str, default="./api/", help="path of the API tree.")
+    parser.add_argument("--data-prefix", type=str, default="./tools/data", help="path of the tools data tree.")
     parser.add_argument(
         "--skip_envoy_build_rule_check",
         action="store_true",

tools/data/api/BUILD

@@ -0,0 +1,9 @@
+load("@envoy_repo//:utils.bzl", "py_data")
+
+licenses(["notice"])  # Apache 2
+
+py_data(
+    name = "v3_proto_rst",
+    filters = ["//filters:proto_rst_srcs"],
+    source = "@envoy_api//:v3_proto_srcs",
+)

tools/data/api/bazel/BUILD

@@ -0,0 +1,9 @@
+load("@envoy_repo//:utils.bzl", "py_data")
+
+licenses(["notice"])  # Apache 2
+
+py_data(
+    name = "repository_locations",
+    filters = ["//filters:interpolate_repository_locations"],
+    source = "@envoy_api//bazel:repository_locations.json",
+)

tools/data/bazel/BUILD

@@ -0,0 +1,15 @@
+load("@envoy_repo//:utils.bzl", "py_data")
+
+licenses(["notice"])  # Apache 2
+
+py_data(
+    name = "repository_locations",
+    filters = ["//filters:interpolate_repository_locations"],
+    source = "@envoy//bazel:repository_locations.json",
+)
+
+py_data(
+    name = "all_repository_locations",
+    filters = ["//filters:add_api_repository_locations"],
+    source = "@envoy//bazel:repository_locations.json",
+)

tools/data/contrib/BUILD

@@ -0,0 +1,19 @@
+load("@envoy_repo//:utils.bzl", "py_data")
+
+licenses(["notice"])  # Apache 2
+
+py_data(
+    name = "extensions_metadata",
+    source = "@envoy//contrib:extensions_metadata.yaml",
+)
+
+py_data(
+    name = "extensions_categories",
+    filters = ["//filters:build_categories"],
+    source = "@envoy//contrib:extensions_metadata.yaml",
+)
+
+py_data(
+    name = "extensions_build_config",
+    source = "@envoy//contrib:contrib_extensions_build_config.json",
+)

tools/data/docs/BUILD

@@ -0,0 +1,19 @@
+load("@envoy_repo//:utils.bzl", "py_data")
+
+licenses(["notice"])  # Apache 2
+
+py_data(
+    name = "protodoc_manifest",
+    source = "@envoy//docs:protodoc_manifest.yaml",
+)
+
+py_data(
+    name = "v2_mapping",
+    source = "@envoy//docs:v2_mapping.json",
+)
+
+py_data(
+    name = "empty_extensions",
+    filters = ["//filters:empty_extensions"],
+    source = "@envoy//docs:empty_extensions.json",
+)

tools/data/filters/BUILD

@@ -0,0 +1,47 @@
+load("@dev_pip3//:requirements.bzl", dev_requirement = "requirement")
+load("@rules_python//python:defs.bzl", "py_library")
+
+py_library(
+    name = "proto_rst_srcs",
+    srcs = ["proto_rst_srcs.py"],
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "empty_extensions",
+    srcs = ["empty_extensions.py"],
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "interpolate_repository_locations",
+    srcs = ["interpolate_repository_locations.py"],
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "add_api_repository_locations",
+    srcs = ["add_api_repository_locations.py"],
+    deps = ["//api/bazel:repository_locations_py"],
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "build_categories",
+    srcs = ["build_categories.py"],
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "add_contrib_extensions_metadata",
+    srcs = ["add_contrib_extensions_metadata.py"],
+    deps = ["//contrib:extensions_metadata_py"],
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "extension_security_postures",
+    srcs = ["extension_security_postures.py"],
+    visibility = ["//visibility:public"],
+    deps = [dev_requirement("envoy.docs.abstract")],
+)

tools/data/filters/add_api_repository_locations.py

@@ -0,0 +1,6 @@
+from envoy_repo.api.bazel import repository_locations
+
+
+def main(data):
+    data.update(repository_locations.data.copy())
+    return data

tools/data/filters/add_contrib_extensions_metadata.py

@@ -0,0 +1,9 @@
+from envoy_repo.contrib import extensions_metadata
+
+
+def main(data):
+    contrib_extensions = extensions_metadata.data.copy()
+    for v in contrib_extensions.values():
+        v['contrib'] = True
+    data.update(contrib_extensions)
+    return data

tools/data/filters/build_categories.py

@@ -0,0 +1,7 @@
+# create an index of extension categories from extension dbs
+def main(data):
+    ret = {}
+    for _k, _v in data.items():
+        for _cat in _v['categories']:
+            ret.setdefault(_cat, []).append(_k)
+    return ret

tools/data/filters/empty_extensions.py

@@ -0,0 +1,7 @@
+import pathlib
+
+
+def main(data):
+    for k, v in data.items():
+        v["docs_path"] = str(pathlib.Path(v['path'], 'empty', f"{v['path'].split('/').pop()}.rst"))
+    return data

tools/data/filters/extension_security_postures.py

@@ -0,0 +1,12 @@
+"""Transform the extensions metadata dict into the security postures dict"""
+
+from collections import defaultdict
+
+from envoy.docs import abstract
+
+
+def main(data: abstract.ExtensionsMetadataDict) -> abstract.ExtensionSecurityPosturesDict:
+    security_postures = defaultdict(list)
+    for extension, metadata in data.items():
+        security_postures[metadata['security_posture']].append(extension)
+    return security_postures

tools/data/filters/interpolate_repository_locations.py

@@ -0,0 +1,17 @@
+def format_data(s, data):
+    return s.format(
+        version=data["version"],
+        underscore_version=data["version"].replace(".", "_"),
+        dash_version=data["version"].replace(".", "-"))
+
+
+def main(data):
+    for k, v in data.items():
+        # this should reflect any transformations in `api/bazel/repository_locations_utils.bzl`
+        if not v.get("version"):
+            data[k] = v
+            continue
+        v["strip_prefix"] = format_data(v.get("strip_prefix", ""), v)
+        v["urls"] = [format_data(url, v) for url in v.get("urls", [])]
+        data[k] = v
+    return data

tools/data/filters/proto_rst_srcs.py

@@ -0,0 +1,18 @@
+"""Transform bazel api labels into rst paths in the docs"""
+
+from typing import Tuple
+
+
+def format_proto_src(src: str) -> str:
+    """Transform api bazel Label -> rst path in docs
+
+    eg:
+       @envoy_api//envoy/watchdog/v3alpha:abort_action.proto
+
+       ->  envoy/watchdog/v3alpha/abort_action.proto.rst
+    """
+    return f"{src.replace(':', '/').strip('@').replace('//', '/')[10:]}.rst"
+
+
+def main(data) -> Tuple[str]:
+    return tuple(format_proto_src(src) for src in data if src)

tools/data/loader/BUILD

@@ -0,0 +1,8 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+
+licenses(["notice"])  # Apache 2
+
+exports_files([
+    "load_data.py",
+    "print_data.py",
+])

tools/data/loader/load_data.py

@@ -0,0 +1,38 @@
+import os
+import pathlib
+from rules_python.python.runfiles import runfiles
+
+__IMPORTS__  # noqa: F821
+
+_loader = __LOADER__  # noqa: F821
+
+
+def _resolve(provided_path):
+    # Resolve the path, to the data file
+    # Adapts the filepath to work in different invokations
+    #   ie - run, build, genrules, etc
+    path = pathlib.Path(provided_path)
+    if path.exists():
+        return path
+    run = runfiles.Create()
+    location = run.Rlocation(str(path).strip(".").strip("/"))
+    if location:
+        path = pathlib.Path(location)
+        if not path.exists():
+            # If the build is invoked from the envoy workspace it has no prefix,
+            # so search in the runfiles with `envoy/` prefix to path.
+            path = pathlib.Path(run.Rlocation(os.path.join("envoy", provided_path)))
+        return path
+    raise Exception(f"Unable to find data file {provided_path}")
+
+
+data = _loader(_resolve("__DATA_FILE__").read_text())
+
+_filters = __FILTERS__  # noqa: F821
+
+_filters = (_filters,) if not isinstance(_filters, tuple) else _filters
+
+for _filter in _filters:
+    data = _filter(data)
+
+__all__ = ("data",)