diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0190c14a4eb32..4f73d3353d7af 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -259,6 +259,7 @@ API can be found [here](api/STYLE.md#adding-an-extension-configuration-to-the-ap Other changes will likely include * Editing [source/extensions/extensions_build_config.bzl](source/extensions/extensions_build_config.bzl) to include the new extensions + * Editing [source/extensions/extensions_metadata.yaml](source/extensions/extensions_metadata.yaml) to include metadata for the new extensions * Editing [docs/root/api-v3/config/config.rst](docs/root/api-v3/config/config.rst) to add area/area * Adding `docs/root/api-v3/config/area/area.rst` to add a table of contents for the API docs * Adding `source/extensions/area/well_known_names.h` for registered plugins diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index 0c4ae3a53a361..c6458f3b442bf 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -44,76 +44,6 @@ def envoy_basic_cc_library(name, deps = [], external_deps = [], **kargs): **kargs ) -# All Envoy extensions must be tagged with their security hardening stance with -# respect to downstream and upstream data plane threats. These are verbose -# labels intended to make clear the trust that operators may place in -# extensions. -EXTENSION_SECURITY_POSTURES = [ - # This extension is hardened against untrusted downstream traffic. It - # assumes that the upstream is trusted. - "robust_to_untrusted_downstream", - # This extension is hardened against both untrusted downstream and upstream - # traffic. - "robust_to_untrusted_downstream_and_upstream", - # This extension is not hardened and should only be used in deployments - # where both the downstream and upstream are trusted. - "requires_trusted_downstream_and_upstream", - # This is functionally equivalent to - # requires_trusted_downstream_and_upstream, but acts as a placeholder to - # allow us to identify extensions that need classifying. - "unknown", - # Not relevant to data plane threats, e.g. stats sinks. - "data_plane_agnostic", -] - -# Extension categories as defined by factories -EXTENSION_CATEGORIES = [ - "envoy.access_loggers", - "envoy.bootstrap", - "envoy.clusters", - "envoy.compression.compressor", - "envoy.compression.decompressor", - "envoy.filters.http", - "envoy.filters.http.cache", - "envoy.filters.listener", - "envoy.filters.network", - "envoy.filters.udp_listener", - "envoy.grpc_credentials", - "envoy.guarddog_actions", - "envoy.health_checkers", - "envoy.http.stateful_header_formatters", - "envoy.internal_redirect_predicates", - "envoy.io_socket", - "envoy.http.original_ip_detection", - "envoy.matching.common_inputs", - "envoy.matching.input_matchers", - "envoy.rate_limit_descriptors", - "envoy.request_id", - "envoy.resource_monitors", - "envoy.retry_host_predicates", - "envoy.retry_priorities", - "envoy.stats_sinks", - "envoy.thrift_proxy.filters", - "envoy.tracers", - "envoy.transport_sockets.downstream", - "envoy.transport_sockets.upstream", - "envoy.tls.cert_validator", - "envoy.upstreams", - "envoy.wasm.runtime", - "DELIBERATELY_OMITTED", -] - -EXTENSION_STATUS_VALUES = [ - # This extension is stable and is expected to be production usable. - "stable", - # This extension is functional but has not had substantial production burn - # time, use only with this caveat. - "alpha", - # This extension is work-in-progress. Functionality is incomplete and it is - # not intended for production use. - "wip", -] - def envoy_cc_extension( name, security_posture, @@ -125,18 +55,6 @@ def envoy_cc_extension( extra_visibility = [], visibility = EXTENSION_CONFIG_VISIBILITY, **kwargs): - if not category: - fail("Category not set for %s" % name) - if type(category) == "string": - category = (category,) - for cat in category: - if cat not in EXTENSION_CATEGORIES: - fail("Unknown extension category for %s: %s" % - (name, cat)) - if security_posture not in EXTENSION_SECURITY_POSTURES: - fail("Unknown extension security posture: " + security_posture) - if status not in EXTENSION_STATUS_VALUES: - fail("Unknown extension status: " + status) if "//visibility:public" not in visibility: visibility = visibility + extra_visibility diff --git a/docs/build.sh b/docs/build.sh index 6e0b83731ba28..c774d8f09cd0e 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -77,6 +77,9 @@ BAZEL_BUILD_OPTIONS+=( "--action_env=ENVOY_BLOB_SHA" "--action_env=EXTENSION_DB_PATH") +# TODO(phlax): move this to format_pre checks +bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/extensions:validate_extensions + # Generate RST for the lists of trusted/untrusted extensions in # intro/arch_overview/security docs. bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/extensions:generate_extension_rst diff --git a/docs/requirements.txt b/docs/requirements.txt index 6a3f7c3cc01e7..e95299cbdb0f7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -35,6 +35,7 @@ docutils==0.16 \ # -r docs/requirements.txt # sphinx # sphinx-rtd-theme + # sphinx-tabs gitdb==4.0.7 \ --hash=sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0 \ --hash=sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005 @@ -101,6 +102,7 @@ markupsafe==2.0.1 \ # via # -r docs/requirements.txt # jinja2 + # sphinx packaging==20.9 \ --hash=sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5 \ --hash=sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a @@ -108,8 +110,8 @@ packaging==20.9 \ # -r docs/requirements.txt # sphinx pygments==2.9.0 \ - --hash=sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e \ - --hash=sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f + --hash=sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f \ + --hash=sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e # via # -r docs/requirements.txt # sphinx @@ -126,6 +128,37 @@ pytz==2021.1 \ # via # -r docs/requirements.txt # babel +pyyaml==5.4.1 \ + --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ + --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \ + --hash=sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393 \ + --hash=sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77 \ + --hash=sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922 \ + --hash=sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5 \ + --hash=sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8 \ + --hash=sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10 \ + --hash=sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc \ + --hash=sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018 \ + --hash=sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e \ + --hash=sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253 \ + --hash=sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347 \ + --hash=sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183 \ + --hash=sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541 \ + --hash=sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb \ + --hash=sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185 \ + --hash=sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc \ + --hash=sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db \ + --hash=sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa \ + --hash=sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46 \ + --hash=sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122 \ + --hash=sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b \ + --hash=sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63 \ + --hash=sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df \ + --hash=sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc \ + --hash=sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247 \ + --hash=sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6 \ + --hash=sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0 + # via -r docs/requirements.txt requests==2.25.1 \ --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e @@ -133,8 +166,8 @@ requests==2.25.1 \ # -r docs/requirements.txt # sphinx six==1.16.0 \ - --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \ - --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via # -r docs/requirements.txt # sphinxcontrib-httpdomain diff --git a/source/extensions/BUILD b/source/extensions/BUILD index 779d1695d3b7c..5e24d5c07b6ad 100644 --- a/source/extensions/BUILD +++ b/source/extensions/BUILD @@ -1 +1,5 @@ licenses(["notice"]) # Apache 2 + +exports_files([ + "extensions_metadata.yaml", +]) diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml new file mode 100644 index 0000000000000..56570a9404282 --- /dev/null +++ b/source/extensions/extensions_metadata.yaml @@ -0,0 +1,681 @@ +envoy.access_loggers.file: + categories: + - envoy.access_loggers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.access_loggers.http_grpc: + categories: + - envoy.access_loggers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.access_loggers.open_telemetry: + categories: + - envoy.access_loggers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.access_loggers.stream: + categories: + - envoy.access_loggers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.access_loggers.tcp_grpc: + categories: + - envoy.access_loggers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.access_loggers.wasm: + categories: + - envoy.access_loggers + security_posture: unknown + status: alpha +envoy.bootstrap.wasm: + categories: + - envoy.bootstrap + security_posture: unknown + status: alpha +envoy.cache.simple_http_cache: + categories: + - envoy.filters.http.cache + security_posture: robust_to_untrusted_downstream_and_upstream + status: wip +envoy.clusters.aggregate: + categories: + - envoy.clusters + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.clusters.dynamic_forward_proxy: + categories: + - envoy.clusters + security_posture: robust_to_untrusted_downstream + status: stable +envoy.clusters.redis: + categories: + - envoy.clusters + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.compression.brotli.compressor: + categories: + - envoy.compression.compressor + security_posture: robust_to_untrusted_downstream + status: stable +envoy.compression.brotli.decompressor: + categories: + - envoy.compression.decompressor + security_posture: robust_to_untrusted_downstream + status: stable +envoy.compression.gzip.compressor: + categories: + - envoy.compression.compressor + security_posture: robust_to_untrusted_downstream + status: stable +envoy.compression.gzip.decompressor: + categories: + - envoy.compression.decompressor + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.adaptive_concurrency: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.http.admission_control: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.http.aws_lambda: + categories: + - envoy.filters.http + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.http.aws_request_signing: + categories: + - envoy.filters.http + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.http.bandwidth_limit: + categories: + - envoy.filters.http + security_posture: unknown + status: stable +envoy.filters.http.buffer: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.cache: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream_and_upstream + status: wip +envoy.filters.http.cdn_loop: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.http.composite: + categories: + - envoy.filters.http + security_posture: unknown + status: stable +envoy.filters.http.compressor: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.cors: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.csrf: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.decompressor: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.filters.http.dynamic_forward_proxy: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.dynamo: + categories: + - envoy.filters.http + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.http.ext_authz: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.ext_proc: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.http.fault: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.grpc_http1_bridge: + categories: + - envoy.filters.http + security_posture: unknown + status: stable +envoy.filters.http.grpc_http1_reverse_bridge: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.http.grpc_json_transcoder: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.grpc_stats: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.http.grpc_web: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.gzip: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.header_to_metadata: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.health_check: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.ip_tagging: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.jwt_authn: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: alpha +envoy.filters.http.kill_request: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.local_ratelimit: + categories: + - envoy.filters.http + security_posture: unknown + status: stable +envoy.filters.http.lua: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.oauth2: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: alpha +envoy.filters.http.on_demand: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.original_src: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: alpha +envoy.filters.http.ratelimit: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.rbac: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.router: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.http.set_metadata: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.filters.http.squash: + categories: + - envoy.filters.http + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.http.tap: + categories: + - envoy.filters.http + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.http.wasm: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha +envoy.filters.listener.http_inspector: + categories: + - envoy.filters.listener + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.listener.original_dst: + categories: + - envoy.filters.listener + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.listener.original_src: + categories: + - envoy.filters.listener + security_posture: robust_to_untrusted_downstream + status: alpha +envoy.filters.listener.proxy_protocol: + categories: + - envoy.filters.listener + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.listener.tls_inspector: + categories: + - envoy.filters.listener + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.client_ssl_auth: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.direct_response: + categories: + - envoy.filters.network + security_posture: unknown + status: stable +envoy.filters.network.dubbo_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.network.echo: + categories: + - envoy.filters.network + security_posture: unknown + status: stable +envoy.filters.network.ext_authz: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.http_connection_manager: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.kafka_broker: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: wip +envoy.filters.network.local_ratelimit: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.mongo_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.network.mysql_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.network.postgres_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.network.ratelimit: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.rbac: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.redis_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.network.rocketmq_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.network.sni_cluster: + categories: + - envoy.filters.network + security_posture: unknown + status: stable +envoy.filters.network.sni_dynamic_forward_proxy: + categories: + - envoy.filters.network + security_posture: unknown + status: alpha +envoy.filters.network.tcp_proxy: + categories: + - envoy.filters.network + security_posture: robust_to_untrusted_downstream + status: stable +envoy.filters.network.thrift_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.network.wasm: + categories: + - envoy.filters.network + security_posture: unknown + status: alpha +envoy.filters.network.zookeeper_proxy: + categories: + - envoy.filters.network + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.thrift.ratelimit: + categories: + - envoy.thrift_proxy.filters + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.filters.thrift.router: + categories: + - envoy.thrift_proxy.filters + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.filters.udp_listener.dns_filter: + categories: + - envoy.filters.udp_listener + security_posture: robust_to_untrusted_downstream + status: alpha +envoy.filters.udp_listener.udp_proxy: + categories: + - envoy.filters.udp_listener + security_posture: robust_to_untrusted_downstream + status: stable +envoy.grpc_credentials.aws_iam: + categories: + - envoy.grpc_credentials + security_posture: data_plane_agnostic + status: alpha +envoy.grpc_credentials.file_based_metadata: + categories: + - envoy.grpc_credentials + security_posture: data_plane_agnostic + status: alpha +envoy.health_checkers.redis: + categories: + - envoy.health_checkers + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.http.original_ip_detection.custom_header: + categories: + - envoy.http.original_ip_detection + security_posture: robust_to_untrusted_downstream + status: stable +envoy.http.original_ip_detection.xff: + categories: + - envoy.http.original_ip_detection + security_posture: robust_to_untrusted_downstream + status: stable +envoy.http.stateful_header_formatters.preserve_case: + categories: + - envoy.http.stateful_header_formatters + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.internal_redirect_predicates.allow_listed_routes: + categories: + - envoy.internal_redirect_predicates + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.internal_redirect_predicates.previous_routes: + categories: + - envoy.internal_redirect_predicates + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.internal_redirect_predicates.safe_cross_scheme: + categories: + - envoy.internal_redirect_predicates + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.io_socket.user_space: + categories: + - envoy.io_socket + security_posture: unknown + status: wip + undocumented: true +envoy.matching.common_inputs.environment_variable: + categories: + - envoy.matching.common_inputs + security_posture: robust_to_untrusted_downstream + status: stable +envoy.matching.input_matchers.consistent_hashing: + categories: + - envoy.matching.input_matchers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.rate_limit_descriptors.expr: + categories: + - envoy.rate_limit_descriptors + security_posture: unknown + status: stable +envoy.request_id.uuid: + categories: + - envoy.request_id + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.resource_monitors.fixed_heap: + categories: + - envoy.resource_monitors + security_posture: data_plane_agnostic + status: alpha +envoy.resource_monitors.injected_resource: + categories: + - envoy.resource_monitors + security_posture: data_plane_agnostic + status: alpha +envoy.retry_host_predicates.omit_canary_hosts: + categories: + - envoy.retry_host_predicates + security_posture: robust_to_untrusted_downstream + status: stable +envoy.retry_host_predicates.omit_host_metadata: + categories: + - envoy.retry_host_predicates + security_posture: robust_to_untrusted_downstream + status: stable +envoy.retry_host_predicates.previous_hosts: + categories: + - envoy.retry_host_predicates + security_posture: robust_to_untrusted_downstream + status: stable +envoy.retry_priorities.previous_priorities: + categories: + - envoy.retry_priorities + security_posture: robust_to_untrusted_downstream + status: stable +envoy.stat_sinks.dog_statsd: + categories: + - envoy.stats_sinks + security_posture: data_plane_agnostic + status: stable +envoy.stat_sinks.hystrix: + categories: + - envoy.stats_sinks + security_posture: data_plane_agnostic + status: stable +envoy.stat_sinks.metrics_service: + categories: + - envoy.stats_sinks + security_posture: data_plane_agnostic + status: stable +envoy.stat_sinks.statsd: + categories: + - envoy.stats_sinks + security_posture: data_plane_agnostic + status: stable +envoy.stat_sinks.wasm: + categories: + - envoy.stats_sinks + security_posture: data_plane_agnostic + status: alpha +envoy.tls.cert_validator.spiffe: + categories: + - envoy.tls.cert_validator + security_posture: unknown + status: wip +envoy.tracers.datadog: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.tracers.dynamic_ot: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.tracers.lightstep: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.tracers.opencensus: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.tracers.skywalking: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: wip +envoy.tracers.xray: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.tracers.zipkin: + categories: + - envoy.tracers + security_posture: robust_to_untrusted_downstream + status: stable +envoy.transport_sockets.alts: + categories: + - envoy.transport_sockets.downstream + - envoy.transport_sockets.upstream + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.transport_sockets.raw_buffer: + categories: + - envoy.transport_sockets.downstream + - envoy.transport_sockets.upstream + security_posture: requires_trusted_downstream_and_upstream + status: stable +envoy.transport_sockets.starttls: + categories: + - envoy.transport_sockets.downstream + - envoy.transport_sockets.upstream + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.transport_sockets.tap: + categories: + - envoy.transport_sockets.downstream + - envoy.transport_sockets.upstream + security_posture: requires_trusted_downstream_and_upstream + status: alpha +envoy.transport_sockets.tls: + categories: + - envoy.transport_sockets.downstream + - envoy.transport_sockets.upstream + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.transport_sockets.upstream_proxy_protocol: + categories: + - envoy.transport_sockets.upstream + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable +envoy.upstreams.http.generic: + categories: + - envoy.upstreams + security_posture: robust_to_untrusted_downstream + status: stable +envoy.upstreams.http.http: + categories: + - envoy.upstreams + security_posture: robust_to_untrusted_downstream + status: stable +envoy.upstreams.http.http_protocol_options: + categories: + - envoy.upstreams + security_posture: robust_to_untrusted_downstream + status: stable +envoy.upstreams.http.tcp: + categories: + - envoy.upstreams + security_posture: robust_to_untrusted_downstream + status: stable +envoy.upstreams.tcp.generic: + categories: + - envoy.upstreams + security_posture: robust_to_untrusted_downstream + status: stable +envoy.wasm.runtime.null: + categories: + - envoy.wasm.runtime + security_posture: unknown + status: alpha +envoy.wasm.runtime.v8: + categories: + - envoy.wasm.runtime + security_posture: unknown + status: alpha +envoy.wasm.runtime.wasmtime: + categories: + - envoy.wasm.runtime + security_posture: unknown + status: alpha +envoy.wasm.runtime.wavm: + categories: + - envoy.wasm.runtime + security_posture: unknown + status: alpha +envoy.watchdog.profile_action: + categories: + - envoy.guarddog_actions + security_posture: data_plane_agnostic + status: alpha diff --git a/tools/extensions/BUILD b/tools/extensions/BUILD index 43f21c21d2075..1c246dd8306e0 100644 --- a/tools/extensions/BUILD +++ b/tools/extensions/BUILD @@ -7,19 +7,16 @@ licenses(["notice"]) # Apache 2 envoy_package() py_binary( - name = "generate_extension_db", - srcs = ["generate_extension_db.py"], + name = "validate_extensions", + srcs = ["validate_extensions.py"], data = [ "@com_github_bazelbuild_buildtools//buildozer:buildozer", + "//source/extensions:extensions_metadata.yaml", ] + envoy_all_extensions(), - python_version = "PY3", - srcs_version = "PY3", - visibility = ["//visibility:public"], ) py_binary( name = "generate_extension_rst", srcs = ["generate_extension_rst.py"], - data = [":generate_extension_db"], - visibility = ["//visibility:public"], + data = ["//source/extensions:extensions_metadata.yaml"], ) diff --git a/tools/extensions/generate_extension_db.py b/tools/extensions/generate_extension_db.py deleted file mode 100644 index ec689e701e010..0000000000000 --- a/tools/extensions/generate_extension_db.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 - -# Generate an extension database, a JSON file mapping from qualified well known -# extension name to metadata derived from the envoy_cc_extension target. - -# This script expects a copy of the envoy source to be located at /source -# Alternatively, you can specify a path to the source dir with `ENVOY_SRCDIR` - -# You must specify the target file to save the generated json db to. -# You can do this either as an arg to this script/target or with the env var -# `EXTENSION_DB_PATH` - -import ast -import json -import os -import pathlib -import re -import subprocess -import sys - -from importlib.util import spec_from_loader, module_from_spec -from importlib.machinery import SourceFileLoader - -BUILDOZER_PATH = os.path.abspath( - "external/com_github_bazelbuild_buildtools/buildozer/buildozer_/buildozer") - -ENVOY_SRCDIR = os.getenv('ENVOY_SRCDIR', '/source') - -if not os.path.exists(ENVOY_SRCDIR): - raise SystemExit( - "Envoy source must either be located at /source, or ENVOY_SRCDIR env var must be set") - -# source/extensions/extensions_build_config.bzl must have a .bzl suffix for Starlark -# import, so we are forced to do this workaround. -_extensions_build_config_spec = spec_from_loader( - 'extensions_build_config', - SourceFileLoader( - 'extensions_build_config', - os.path.join(ENVOY_SRCDIR, 'source/extensions/extensions_build_config.bzl'))) -extensions_build_config = module_from_spec(_extensions_build_config_spec) -_extensions_build_config_spec.loader.exec_module(extensions_build_config) - - -class ExtensionDbError(Exception): - pass - - -def is_missing(value): - return value == '(missing)' - - -def num_read_filters_fuzzed(): - data = pathlib.Path( - os.path.join( - ENVOY_SRCDIR, - 'test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc')).read_text() - # Hack-ish! We only search the first 50 lines to capture the filters in filterNames(). - return len(re.findall('NetworkFilterNames::get()', ''.join(data.splitlines()[:50]))) - - -def num_robust_to_downstream_network_filters(db): - # Count number of network filters robust to untrusted downstreams. - return len([ - ext for ext, data in db.items() - if 'network' in ext and data['security_posture'] == 'robust_to_untrusted_downstream' - ]) - - -def get_extension_metadata(target): - if not BUILDOZER_PATH: - raise ExtensionDbError('Buildozer not found!') - r = subprocess.run( - [BUILDOZER_PATH, '-stdout', 'print security_posture status undocumented category', target], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - rout = r.stdout.decode('utf-8').strip().split(' ') - security_posture, status, undocumented = rout[:3] - categories = ' '.join(rout[3:]) - if is_missing(security_posture): - raise ExtensionDbError( - 'Missing security posture for %s. Please make sure the target is an envoy_cc_extension and security_posture is set' - % target) - if is_missing(categories): - raise ExtensionDbError( - 'Missing extension category for %s. Please make sure the target is an envoy_cc_extension and category is set' - % target) - # evaluate tuples/lists - # wrap strings in a list - categories = ( - ast.literal_eval(categories) if ('[' in categories or '(' in categories) else [categories]) - return { - 'security_posture': security_posture, - 'undocumented': False if is_missing(undocumented) else bool(undocumented), - 'status': 'stable' if is_missing(status) else status, - 'categories': categories, - } - - -if __name__ == '__main__': - try: - output_path = os.getenv("EXTENSION_DB_PATH") or sys.argv[1] - except IndexError: - raise SystemExit( - "Output path must be either specified as arg or with EXTENSION_DB_PATH env var") - - extension_db = {} - # Include all extensions from source/extensions/extensions_build_config.bzl - all_extensions = {} - all_extensions.update(extensions_build_config.EXTENSIONS) - for extension, target in all_extensions.items(): - extension_db[extension] = get_extension_metadata(target) - if num_robust_to_downstream_network_filters(extension_db) != num_read_filters_fuzzed(): - raise ExtensionDbError( - 'Check that all network filters robust against untrusted' - 'downstreams are fuzzed by adding them to filterNames() in' - 'test/extensions/filters/network/common/uber_per_readfilter.cc') - # The TLS and generic upstream extensions are hard-coded into the build, so - # not in source/extensions/extensions_build_config.bzl - # TODO(mattklein123): Read these special keys from all_extensions.bzl or a shared location to - # avoid duplicate logic. - extension_db['envoy.transport_sockets.tls'] = get_extension_metadata( - '//source/extensions/transport_sockets/tls:config') - extension_db['envoy.upstreams.http.generic'] = get_extension_metadata( - '//source/extensions/upstreams/http/generic:config') - extension_db['envoy.upstreams.tcp.generic'] = get_extension_metadata( - '//source/extensions/upstreams/tcp/generic:config') - extension_db['envoy.upstreams.http.http_protocol_options'] = get_extension_metadata( - '//source/extensions/upstreams/http:config') - extension_db['envoy.request_id.uuid'] = get_extension_metadata( - '//source/extensions/request_id/uuid:config') - - pathlib.Path(os.path.dirname(output_path)).mkdir(parents=True, exist_ok=True) - pathlib.Path(output_path).write_text(json.dumps(extension_db)) diff --git a/tools/extensions/generate_extension_rst.py b/tools/extensions/generate_extension_rst.py index 9199873f7079c..62e4b6250d90d 100644 --- a/tools/extensions/generate_extension_rst.py +++ b/tools/extensions/generate_extension_rst.py @@ -3,18 +3,18 @@ # Generate RST lists of extensions grouped by their security posture. from collections import defaultdict -import json import os import pathlib -import subprocess + +import yaml def format_item(extension, metadata): - if metadata['undocumented']: + if metadata.get('undocumented'): item = '* %s' % extension else: item = '* :ref:`%s `' % (extension, extension) - if metadata['status'] == 'alpha': + if metadata.get('status') == 'alpha': item += ' (alpha)' return item @@ -27,14 +27,8 @@ def format_item(extension, metadata): "Path to an output directory must be specified with GENERATED_RST_DIR env var") security_rst_root = os.path.join(generated_rst_dir, "intro/arch_overview/security") - try: - extension_db_path = os.environ["EXTENSION_DB_PATH"] - except KeyError: - raise SystemExit( - "Path to a json extension db must be specified with EXTENSION_DB_PATH env var") - if not os.path.exists(extension_db_path): - subprocess.run("tools/extensions/generate_extension_db".split(), check=True) - extension_db = json.loads(pathlib.Path(extension_db_path).read_text()) + with open("source/extensions/extensions_metadata.yaml") as f: + extension_db = yaml.safe_load(f.read()) pathlib.Path(security_rst_root).mkdir(parents=True, exist_ok=True) @@ -47,5 +41,5 @@ def format_item(extension, metadata): content = '\n'.join( format_item(extension, extension_db[extension]) for extension in sorted(extensions) - if extension_db[extension]['status'] != 'wip') + if extension_db[extension].get('status') != 'wip') output_path.write_text(content) diff --git a/tools/extensions/validate_extensions.py b/tools/extensions/validate_extensions.py new file mode 100644 index 0000000000000..d349b13d33f3e --- /dev/null +++ b/tools/extensions/validate_extensions.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 + +# Validate extension metadata + +# This script expects a copy of the envoy source to be located at /source +# Alternatively, you can specify a path to the source dir with `ENVOY_SRCDIR` + +import ast +import os +import pathlib +import re +import subprocess +import sys +from importlib.util import spec_from_loader, module_from_spec +from importlib.machinery import SourceFileLoader + +import yaml + +# All Envoy extensions must be tagged with their security hardening stance with +# respect to downstream and upstream data plane threats. These are verbose +# labels intended to make clear the trust that operators may place in +# extensions. +EXTENSION_SECURITY_POSTURES = [ + # This extension is hardened against untrusted downstream traffic. It + # assumes that the upstream is trusted. + "robust_to_untrusted_downstream", + # This extension is hardened against both untrusted downstream and upstream + # traffic. + "robust_to_untrusted_downstream_and_upstream", + # This extension is not hardened and should only be used in deployments + # where both the downstream and upstream are trusted. + "requires_trusted_downstream_and_upstream", + # This is functionally equivalent to + # requires_trusted_downstream_and_upstream, but acts as a placeholder to + # allow us to identify extensions that need classifying. + "unknown", + # Not relevant to data plane threats, e.g. stats sinks. + "data_plane_agnostic", +] + +# Extension categories as defined by factories +EXTENSION_CATEGORIES = [ + "envoy.access_loggers", + "envoy.bootstrap", + "envoy.clusters", + "envoy.compression.compressor", + "envoy.compression.decompressor", + "envoy.filters.http", + "envoy.filters.http.cache", + "envoy.filters.listener", + "envoy.filters.network", + "envoy.filters.udp_listener", + "envoy.grpc_credentials", + "envoy.guarddog_actions", + "envoy.health_checkers", + "envoy.http.stateful_header_formatters", + "envoy.internal_redirect_predicates", + "envoy.io_socket", + "envoy.http.original_ip_detection", + "envoy.matching.common_inputs", + "envoy.matching.input_matchers", + "envoy.rate_limit_descriptors", + "envoy.request_id", + "envoy.resource_monitors", + "envoy.retry_host_predicates", + "envoy.retry_priorities", + "envoy.stats_sinks", + "envoy.thrift_proxy.filters", + "envoy.tracers", + "envoy.transport_sockets.downstream", + "envoy.transport_sockets.upstream", + "envoy.tls.cert_validator", + "envoy.upstreams", + "envoy.wasm.runtime", + "DELIBERATELY_OMITTED", +] + +EXTENSION_STATUS_VALUES = [ + # This extension is stable and is expected to be production usable. + "stable", + # This extension is functional but has not had substantial production burn + # time, use only with this caveat. + "alpha", + # This extension is work-in-progress. Functionality is incomplete and it is + # not intended for production use. + "wip", +] + +# TODO(phlax): remove this +BUILDOZER_PATH = os.path.abspath( + "external/com_github_bazelbuild_buildtools/buildozer/buildozer_/buildozer") + +# TODO(phlax): remove this +ENVOY_SRCDIR = os.getenv('ENVOY_SRCDIR', '/source') + +# TODO(phlax): remove this +if not os.path.exists(ENVOY_SRCDIR): + raise SystemExit( + "Envoy source must either be located at /source, or ENVOY_SRCDIR env var must be set") + +# source/extensions/extensions_build_config.bzl must have a .bzl suffix for Starlark +# import, so we are forced to do this workaround. +_extensions_build_config_spec = spec_from_loader( + 'extensions_build_config', + SourceFileLoader( + 'extensions_build_config', + os.path.join(ENVOY_SRCDIR, 'source/extensions/extensions_build_config.bzl'))) +extensions_build_config = module_from_spec(_extensions_build_config_spec) +_extensions_build_config_spec.loader.exec_module(extensions_build_config) + + +class ExtensionDbError(Exception): + pass + + +# TODO(phlax): remove this +def is_missing(value): + return value == '(missing)' + + +def num_read_filters_fuzzed(): + data = pathlib.Path( + os.path.join( + ENVOY_SRCDIR, + 'test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc')).read_text() + # Hack-ish! We only search the first 50 lines to capture the filters in filterNames(). + return len(re.findall('NetworkFilterNames::get()', ''.join(data.splitlines()[:50]))) + + +def num_robust_to_downstream_network_filters(db): + # Count number of network filters robust to untrusted downstreams. + return len([ + ext for ext, data in db.items() + if 'network' in ext and data['security_posture'] == 'robust_to_untrusted_downstream' + ]) + + +# TODO(phlax): remove this +def get_extension_metadata(target): + if not BUILDOZER_PATH: + raise ExtensionDbError('Buildozer not found!') + r = subprocess.run( + [BUILDOZER_PATH, '-stdout', 'print security_posture status undocumented category', target], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + rout = r.stdout.decode('utf-8').strip().split(' ') + security_posture, status, undocumented = rout[:3] + categories = ' '.join(rout[3:]) + # evaluate tuples/lists + # wrap strings in a list + categories = list( + ast.literal_eval(categories) if ('[' in categories or '(' in categories) else [categories]) + return { + 'security_posture': security_posture, + 'undocumented': False if is_missing(undocumented) else bool(undocumented), + 'status': 'stable' if is_missing(status) else status, + 'categories': categories, + } + + +# TODO(phlax): remove this +def compare_old_and_new(old_db, new_db): + returns = 0 + + if sorted(old_db.keys()) != sorted(new_db.keys()): + old_only = set(old_db.keys()) - set(new_db.keys()) + new_only = set(new_db.keys()) - set(old_db.keys()) + extra_old = (f"only old {old_only}" if old_only else "") + extra_new = (f"only new {new_only}" if new_only else "") + raise ExtensionDbError(f"Extensions list does not match - {extra_old} {extra_new}") + + for k in new_db: + new_db[k]["undocumented"] = new_db[k].get("undocumented", False) + if old_db[k] != new_db[k]: + returns = 1 + print( + f"ERROR: extension metadata in `source/extensions/extensions_metadata.yaml` does not match `BUILD` for {k}" + ) + print(old_db[k]) + print(new_db[k]) + return returns + + +# TODO(phlax): remove this +def generate_old_extension_db(): + extension_db = {} + # Include all extensions from source/extensions/extensions_build_config.bzl + all_extensions = {} + all_extensions.update(extensions_build_config.EXTENSIONS) + for extension, target in all_extensions.items(): + extension_db[extension] = get_extension_metadata(target) + # The TLS and generic upstream extensions are hard-coded into the build, so + # not in source/extensions/extensions_build_config.bzl + # TODO(mattklein123): Read these special keys from all_extensions.bzl or a shared location to + # avoid duplicate logic. + extension_db['envoy.transport_sockets.tls'] = get_extension_metadata( + '//source/extensions/transport_sockets/tls:config') + extension_db['envoy.upstreams.http.generic'] = get_extension_metadata( + '//source/extensions/upstreams/http/generic:config') + extension_db['envoy.upstreams.tcp.generic'] = get_extension_metadata( + '//source/extensions/upstreams/tcp/generic:config') + extension_db['envoy.upstreams.http.http_protocol_options'] = get_extension_metadata( + '//source/extensions/upstreams/http:config') + extension_db['envoy.request_id.uuid'] = get_extension_metadata( + '//source/extensions/request_id/uuid:config') + return extension_db + + +# TODO(phlax): move this to a checker class, remove `compare_old_and_new` and add pytests +def validate_extensions(): + returns = 0 + with open("source/extensions/extensions_metadata.yaml") as f: + metadata = yaml.safe_load(f.read()) + returns = compare_old_and_new(generate_old_extension_db(), metadata) + + if num_robust_to_downstream_network_filters(metadata) != num_read_filters_fuzzed(): + returns = 1 + print( + 'Check that all network filters robust against untrusted' + 'downstreams are fuzzed by adding them to filterNames() in' + 'test/extensions/filters/network/common/uber_per_readfilter.cc') + + for k, v in metadata.items(): + if not v["security_posture"]: + returns = 1 + print( + f"Missing security posture for {k}. " + "Please make sure the target is an envoy_cc_extension and security_posture is set") + elif v["security_posture"] not in EXTENSION_SECURITY_POSTURES: + print("Unknown extension security posture: {v['security_posture']}") + returns = 1 + if not v["categories"]: + returns = 1 + print( + f"Missing extension category for {k}. " + "Please make sure the target is an envoy_cc_extension and category is set") + else: + for cat in v["categories"]: + if cat not in EXTENSION_CATEGORIES: + returns = 1 + print(f"Unknown extension category for {k}: {cat}") + if v["status"] not in EXTENSION_STATUS_VALUES: + returns = 1 + print(f"Unknown extension status: {v['status']}") + + return returns + + +if __name__ == '__main__': + sys.exit(validate_extensions()) diff --git a/tools/protodoc/BUILD b/tools/protodoc/BUILD index d9eeb6ac203fb..d7df1f49ac10f 100644 --- a/tools/protodoc/BUILD +++ b/tools/protodoc/BUILD @@ -24,6 +24,7 @@ py_binary( data = [ "//docs:protodoc_manifest.yaml", "//docs:v2_mapping.json", + "//source/extensions:extensions_metadata.yaml", ], visibility = ["//visibility:public"], deps = [ diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index a66d6cf021047..2edcd91031491 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -6,8 +6,6 @@ from collections import defaultdict import json import functools -import os -import pathlib import sys from google.protobuf import json_format @@ -115,7 +113,10 @@ 'This extension is work-in-progress. Functionality is incomplete and it is not intended for production use.', } -EXTENSION_DB = json.loads(pathlib.Path(os.getenv('EXTENSION_DB_PATH')).read_text()) +r = runfiles.Create() + +with open(r.Rlocation("envoy/source/extensions/extensions_metadata.yaml")) as f: + EXTENSION_DB = yaml.safe_load(f.read()) # create an index of extension categories from extension db EXTENSION_CATEGORIES = {} @@ -241,7 +242,7 @@ def format_extension(extension): """ try: extension_metadata = EXTENSION_DB[extension] - status = EXTENSION_STATUS_VALUES.get(extension_metadata['status'], '') + status = EXTENSION_STATUS_VALUES.get(extension_metadata.get('status'), '') security_posture = EXTENSION_SECURITY_POSTURES[extension_metadata['security_posture']] categories = extension_metadata["categories"] except KeyError as e: @@ -662,8 +663,6 @@ class RstFormatVisitor(visitor.Visitor): """ def __init__(self): - r = runfiles.Create() - with open(r.Rlocation('envoy/docs/v2_mapping.json'), 'r') as f: self.v2_mapping = json.load(f)