extensions: Separate generation of extension db from docs in CI

api/bazel/repositories.bzl

@@ -22,6 +22,9 @@ def api_dependencies():
     external_http_archive(
         name = "com_google_googleapis",
     )
+    external_http_archive(
+        name = "com_github_bazelbuild_buildtools",
+    )
     external_http_archive(
         name = "com_github_cncf_udpa",
     )

api/bazel/repository_locations.bzl

@@ -28,6 +28,17 @@ REPOSITORY_LOCATIONS_SPEC = dict(
             "org_golang_x_text",
         ],
     ),
+    com_github_bazelbuild_buildtools = dict(
+        project_name = "Bazel build tools",
+        project_desc = "Developer tools for working with Google's bazel buildtool.",
+        project_url = "https://github.com/bazelbuild/buildtools",
+        version = "4.0.0",
+        sha256 = "0d3ca4ed434958dda241fb129f77bd5ef0ce246250feed2d5a5470c6f29a77fa",
+        strip_prefix = "buildtools-4.0.0",
+        urls = ["https://github.com/bazelbuild/buildtools/archive/4.0.0.tar.gz"],
+        release_date = "2021-02-03",
+        use_category = ["api"],
+    ),
     com_github_cncf_udpa = dict(
         project_name = "xDS API",
         project_desc = "xDS API Working Group (xDS-WG)",

docs/build.sh

@@ -73,7 +73,8 @@ BAZEL_BUILD_OPTIONS+=(
 
 # Generate extension database. This maps from extension name to extension
 # metadata, based on the envoy_cc_extension() Bazel target attributes.
-./docs/generate_extension_db.py "${EXTENSION_DB_PATH}"
+# bazel build //tools/extensions:generate_extension_db
+bazel run //tools/extensions:generate_extension_db
 
 # Generate RST for the lists of trusted/untrusted extensions in
 # intro/arch_overview/security docs.

source/extensions/extensions_build_config.bzl

@@ -243,15 +243,13 @@ EXTENSIONS = {
     #
 
     "envoy.rate_limit_descriptors.expr":                "//source/extensions/rate_limit_descriptors/expr:config",
-    
+
     #
     # IO socket
     #
 
     "envoy.io_socket.user_space":                       "//source/extensions/io_socket/user_space:config",
 }
 
-# These can be changed to ["//visibility:public"], for  downstream builds which
-# need to directly reference Envoy extensions.
-EXTENSION_CONFIG_VISIBILITY = ["//:extension_config"]
-EXTENSION_PACKAGE_VISIBILITY = ["//:extension_library"]
+EXTENSION_CONFIG_VISIBILITY = ["//visibility:public"]
+EXTENSION_PACKAGE_VISIBILITY = ["//visibility:public"]

tools/extensions/BUILD

@@ -0,0 +1,18 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+load("//bazel:envoy_build_system.bzl", "envoy_package")
+load("//source/extensions:all_extensions.bzl", "envoy_all_extensions")
+
+licenses(["notice"])  # Apache 2
+
+envoy_package()
+
+py_binary(
+    name = "generate_extension_db",
+    srcs = ["generate_extension_db.py"],
+    data = [
+        "@com_github_bazelbuild_buildtools//buildozer:buildozer",
+    ] + envoy_all_extensions(),
+    python_version = "PY3",
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+)

docs/generate_extension_db.py → tools/extensions/generate_extension_db.py

@@ -15,14 +15,21 @@
 from importlib.util import spec_from_loader, module_from_spec
 from importlib.machinery import SourceFileLoader
 
-BUILDOZER_PATH = os.getenv("BUILDOZER_BIN") or (os.path.expandvars("$GOPATH/bin/buildozer") if
-                                                os.getenv("GOPATH") else shutil.which("buildozer"))
+BUILDOZER_PATH = os.path.abspath(
+    "external/com_github_bazelbuild_buildtools/buildozer/buildozer_/buildozer")
+
+ENVOY_SOURCE = os.getenv('ENVOY_SOURCE', '/source')
+
+if not os.path.exists(ENVOY_SOURCE):
+  raise SystemExit(
+    "Envoy source must either be located at /source, or ENVOY_SOURCE env var must be set")
 
 # source/extensions/extensions_build_config.bzl must have a .bzl suffix for Starlark
 # import, so we are forced to do this workaround.
 _extensions_build_config_spec = spec_from_loader(
     'extensions_build_config',
-    SourceFileLoader('extensions_build_config', 'source/extensions/extensions_build_config.bzl'))
+    SourceFileLoader('extensions_build_config',
+                     os.path.join(ENVOY_SOURCE, 'source/extensions/extensions_build_config.bzl')))
 extensions_build_config = module_from_spec(_extensions_build_config_spec)
 _extensions_build_config_spec.loader.exec_module(extensions_build_config)
 
@@ -37,7 +44,7 @@ def IsMissing(value):
 
 def NumReadFiltersFuzzed():
   data = pathlib.Path(
-      'test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc').read_text()
+    os.path.join(ENVOY_SOURCE, 'test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc')).read_text()
   # Hack-ish! We only search the first 50 lines to capture the filters in filterNames().
   return len(re.findall('NetworkFilterNames::get()', ''.join(data.splitlines()[:50])))
 
@@ -81,7 +88,11 @@ def GetExtensionMetadata(target):
 
 
 if __name__ == '__main__':
-  output_path = sys.argv[1]
+  try:
+    output_path = (os.environ["EXTENSION_DB_PATH"] if os.getenv("EXTENSION_DB_PATH") else sys.argv[1])
+  except IndexError:
+    raise SystemExit("Output path must be either specified as arg or with EXTENSION_DB_PATH env var")
+
   extension_db = {}
   # Include all extensions from source/extensions/extensions_build_config.bzl
   all_extensions = {}
@@ -91,7 +102,7 @@ def GetExtensionMetadata(target):
   if NumRobustToDownstreamNetworkFilters(extension_db) != NumReadFiltersFuzzed():
     raise ExtensionDbError('Check that all network filters robust against untrusted'
                            'downstreams are fuzzed by adding them to filterNames() in'
-                           'test/extensions/filters/network/common/uber_per_readfilter.cc')
+                           '/source/test/extensions/filters/network/common/uber_per_readfilter.cc')
   # The TLS and generic upstream extensions are hard-coded into the build, so
   # not in source/extensions/extensions_build_config.bzl
   extension_db['envoy.transport_sockets.tls'] = GetExtensionMetadata(
@@ -103,4 +114,5 @@ def GetExtensionMetadata(target):
   extension_db['envoy.upstreams.http.http_protocol_options'] = GetExtensionMetadata(
       '//source/extensions/upstreams/http:config')
 
+  pathlib.Path(os.path.dirname(output_path)).mkdir(parents=True, exist_ok=True)
   pathlib.Path(output_path).write_text(json.dumps(extension_db))