diff --git a/.changelog/15979.txt b/.changelog/15979.txt new file mode 100644 index 00000000000..d06c39be24f --- /dev/null +++ b/.changelog/15979.txt @@ -0,0 +1,3 @@ +```release-note:improvement +envoy: add `MaxEjectionPercent` and `BaseEjectionTime` to passive health check configs. +``` \ No newline at end of file diff --git a/.changelog/16257.txt b/.changelog/16257.txt new file mode 100644 index 00000000000..8e98530c421 --- /dev/null +++ b/.changelog/16257.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fix issue where mesh gateways would use the wrong address when contacting a remote peer with the same datacenter name. +``` diff --git a/.changelog/16263.txt b/.changelog/16263.txt new file mode 100644 index 00000000000..a8cd3f9043a --- /dev/null +++ b/.changelog/16263.txt @@ -0,0 +1,4 @@ +```release-note:security +Upgrade to use Go 1.20.1. +This resolves vulnerabilities [CVE-2022-41724](https://go.dev/issue/58001) in `crypto/tls` and [CVE-2022-41723](https://go.dev/issue/57855) in `net/http`. +``` diff --git a/.changelog/16274.txt b/.changelog/16274.txt new file mode 100644 index 00000000000..983d33b1959 --- /dev/null +++ b/.changelog/16274.txt @@ -0,0 +1,3 @@ +```release-note:improvement +connect: Bump Envoy 1.22.5 to 1.22.7, 1.23.2 to 1.23.4, 1.24.0 to 1.24.2, add 1.25.1, remove 1.21.5 +``` diff --git a/.changelog/16284.txt b/.changelog/16284.txt new file mode 100644 index 00000000000..23dd2aa6fef --- /dev/null +++ b/.changelog/16284.txt @@ -0,0 +1,3 @@ +```release-note:feature +cli: adds new CLI commands `consul troubleshoot upstreams` and `consul troubleshoot proxy` to troubleshoot Consul's service mesh configuration and network issues. +``` \ No newline at end of file diff --git a/.changelog/16288.txt b/.changelog/16288.txt new file mode 100644 index 00000000000..5e2820ec27d --- /dev/null +++ b/.changelog/16288.txt @@ -0,0 +1,8 @@ +```release-note:deprecation +cli: Deprecate the `-merge-policies` and `-merge-roles` flags from the `consul token update` command in favor of: `-append-policy-id`, `-append-policy-name`, `-append-role-name`, and `-append-role-id`. +``` + +```release-note:improvement +cli: added `-append-policy-id`, `-append-policy-name`, `-append-role-name`, and `-append-role-id` flags to the `consul token update` command. +These flags allow updates to a token's policies/roles without having to override them completely. +``` \ No newline at end of file diff --git a/.changelog/16301.txt b/.changelog/16301.txt new file mode 100644 index 00000000000..e1dc5deb1c5 --- /dev/null +++ b/.changelog/16301.txt @@ -0,0 +1,3 @@ +```release-note:bug +agent configuration: Fix issue of using unix socket when https is used. +``` diff --git a/.changelog/16339.txt b/.changelog/16339.txt new file mode 100644 index 00000000000..cf44f010aff --- /dev/null +++ b/.changelog/16339.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fix bug where services were incorrectly imported as connect-enabled. +``` diff --git a/.changelog/16358.txt b/.changelog/16358.txt new file mode 100644 index 00000000000..91fcfe4505c --- /dev/null +++ b/.changelog/16358.txt @@ -0,0 +1,3 @@ +```release-note:improvement +container: Upgrade container image to use to Alpine 3.17. +``` diff --git a/.changelog/16369.txt b/.changelog/16369.txt new file mode 100644 index 00000000000..1ae86968c40 --- /dev/null +++ b/.changelog/16369.txt @@ -0,0 +1,3 @@ +```release-note:feature +**API Gateway (Beta)** This version adds support for API gateway on VMs. API gateway provides a highly-configurable ingress for requests coming into a Consul network. For more information, refer to the [API gateway](https://developer.hashicorp.com/consul/docs/connect/gateways/api-gateway) documentation. +``` diff --git a/.changelog/16444.txt b/.changelog/16444.txt new file mode 100644 index 00000000000..542f0560fec --- /dev/null +++ b/.changelog/16444.txt @@ -0,0 +1,3 @@ +```release-note:bug +ui: Fix issue with lists and filters not rendering properly +``` diff --git a/.changelog/16445.txt b/.changelog/16445.txt new file mode 100644 index 00000000000..19745c6df99 --- /dev/null +++ b/.changelog/16445.txt @@ -0,0 +1,3 @@ +```release-note:bug +cli: ensure acl token read -self works +``` diff --git a/.changelog/16485.txt b/.changelog/16485.txt new file mode 100644 index 00000000000..7e1938b00ea --- /dev/null +++ b/.changelog/16485.txt @@ -0,0 +1,3 @@ +```release-note:bug +cli: fix panic read non-existent acl policy +``` diff --git a/.changelog/16495.txt b/.changelog/16495.txt new file mode 100644 index 00000000000..4b8ee933ed0 --- /dev/null +++ b/.changelog/16495.txt @@ -0,0 +1,3 @@ +```release-note:improvement +mesh: Add ServiceResolver RequestTimeout for route timeouts to make request timeouts configurable +``` diff --git a/.changelog/16497.txt b/.changelog/16497.txt new file mode 100644 index 00000000000..3aa3633ac3a --- /dev/null +++ b/.changelog/16497.txt @@ -0,0 +1,3 @@ +```release-note:bug +proxycfg: ensure that an irrecoverable error in proxycfg closes the xds session and triggers a replacement proxycfg watcher +``` diff --git a/.changelog/16498.txt b/.changelog/16498.txt new file mode 100644 index 00000000000..cdb045d67c9 --- /dev/null +++ b/.changelog/16498.txt @@ -0,0 +1,3 @@ +```release-note:bug +proxycfg: fix a bug where terminating gateways were not cleaning up deleted service resolvers for their referenced services +``` diff --git a/.changelog/16499.txt b/.changelog/16499.txt new file mode 100644 index 00000000000..4bd50db47e8 --- /dev/null +++ b/.changelog/16499.txt @@ -0,0 +1,3 @@ +```release-note:bug +mesh: Fix resolution of service resolvers with subsets for external upstreams +``` diff --git a/.changelog/16506.txt b/.changelog/16506.txt new file mode 100644 index 00000000000..2560c247466 --- /dev/null +++ b/.changelog/16506.txt @@ -0,0 +1,8 @@ +```release-note:deprecation +cli: Deprecate the `-merge-node-identites` and `-merge-service-identities` flags from the `consul token update` command in favor of: `-append-node-identity` and `-append-service-identity`. +``` + +```release-note:improvement +cli: added `-append-service-identity` and `-append-node-identity` flags to the `consul token update` command. +These flags allow updates to a token's node identities/service identities without having to override them. +``` \ No newline at end of file diff --git a/.changelog/16508.txt b/.changelog/16508.txt new file mode 100644 index 00000000000..4732ed553c6 --- /dev/null +++ b/.changelog/16508.txt @@ -0,0 +1,3 @@ +```release-note:improvement +ui: support filtering API gateways in the ui and displaying their documentation links +``` diff --git a/.changelog/16512.txt b/.changelog/16512.txt new file mode 100644 index 00000000000..288ff8aa45e --- /dev/null +++ b/.changelog/16512.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateways: fix HTTPRoute bug where service weights could be less than or equal to 0 and result in a downstream envoy protocol error +``` \ No newline at end of file diff --git a/.changelog/16530.txt b/.changelog/16530.txt new file mode 100644 index 00000000000..38d98036ab9 --- /dev/null +++ b/.changelog/16530.txt @@ -0,0 +1,7 @@ +```release-note:bug +cli: Fixes an issue with `consul connect envoy` where a log to STDOUT could malform JSON when used with `-bootstrap`. +``` + +```release-note:bug +cli: Fixes an issue with `consul connect envoy` where grpc-disabled agents were not error-handled correctly. +``` diff --git a/.changelog/16531.txt b/.changelog/16531.txt new file mode 100644 index 00000000000..71f83ad2acc --- /dev/null +++ b/.changelog/16531.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateways: fix HTTPRoute bug where services with a weight not divisible by 10000 are never registered properly +``` \ No newline at end of file diff --git a/.changelog/16552.txt b/.changelog/16552.txt new file mode 100644 index 00000000000..40633be1730 --- /dev/null +++ b/.changelog/16552.txt @@ -0,0 +1,3 @@ +```release-note:improvement +raft: Remove expensive reflection from raft/mesh hot path +``` diff --git a/.changelog/16570.txt b/.changelog/16570.txt new file mode 100644 index 00000000000..ad07cda81c0 --- /dev/null +++ b/.changelog/16570.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fixes a bug that can lead to peering service deletes impacting the state of local services +``` diff --git a/.changelog/16574.txt b/.changelog/16574.txt new file mode 100644 index 00000000000..78bfc334984 --- /dev/null +++ b/.changelog/16574.txt @@ -0,0 +1,3 @@ +```release-note:bug +ui: fix rendering issues on Overview and empty-states by addressing isHTMLSafe errors +``` diff --git a/.changelog/16585.txt b/.changelog/16585.txt new file mode 100644 index 00000000000..11e2959cfbd --- /dev/null +++ b/.changelog/16585.txt @@ -0,0 +1,3 @@ +```release-note:feature +xds: Allow for configuring connect proxies to send service mesh telemetry to an HCP metrics collection service. +``` \ No newline at end of file diff --git a/.changelog/16592.txt b/.changelog/16592.txt new file mode 100644 index 00000000000..ba37d69015f --- /dev/null +++ b/.changelog/16592.txt @@ -0,0 +1,3 @@ +```release-note:bug +ca: Fixes a bug where updating Vault CA Provider config would cause TLS issues in the service mesh +``` diff --git a/.changelog/16647.txt b/.changelog/16647.txt new file mode 100644 index 00000000000..cbe38b3ed28 --- /dev/null +++ b/.changelog/16647.txt @@ -0,0 +1,3 @@ +```release-note:bug +raft_logstore: Fixes a bug where restoring a snapshot when using the experimental WAL storage backend causes a panic. +``` diff --git a/.changelog/16649.txt b/.changelog/16649.txt new file mode 100644 index 00000000000..e510558ff90 --- /dev/null +++ b/.changelog/16649.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateways: Adds validation to ensure the API Gateway has a listener defined when created +``` \ No newline at end of file diff --git a/.changelog/16651.txt b/.changelog/16651.txt new file mode 100644 index 00000000000..c297cca489d --- /dev/null +++ b/.changelog/16651.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateway: **(Enterprise only)** Fix bug where namespace/partition would fail to unmarshal. +``` diff --git a/.changelog/16660.txt b/.changelog/16660.txt new file mode 100644 index 00000000000..f8971862165 --- /dev/null +++ b/.changelog/16660.txt @@ -0,0 +1,3 @@ +```release-note:bug +ui: fix PUT token request with adding missed AccessorID property to requestBody +``` \ No newline at end of file diff --git a/.changelog/16661.txt b/.changelog/16661.txt new file mode 100644 index 00000000000..41336502116 --- /dev/null +++ b/.changelog/16661.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateways: Fixes a bug API gateways using HTTP listeners were taking upwards of 15 seconds to get configured over xDS. +``` diff --git a/.changelog/16675.txt b/.changelog/16675.txt new file mode 100644 index 00000000000..f72eedc61c8 --- /dev/null +++ b/.changelog/16675.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fixes a bug where the importing partition was not added to peered failover targets, which causes issues when the importing partition is a non-default partition. +``` diff --git a/.changelog/16700.txt b/.changelog/16700.txt new file mode 100644 index 00000000000..82da5936ddb --- /dev/null +++ b/.changelog/16700.txt @@ -0,0 +1,3 @@ +```release-note:bug +audit-logging: (Enterprise only) Fix a bug where `/agent/monitor` and `/agent/metrics` endpoints return a `Streaming not supported` error when audit logs are enabled. This also fixes the delay receiving logs when running `consul monitor` against an agent with audit logs enabled. +``` diff --git a/.changelog/16729.txt b/.changelog/16729.txt new file mode 100644 index 00000000000..36c6e1aeabb --- /dev/null +++ b/.changelog/16729.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fix issue resulting in prepared query failover to cluster peers never un-failing over. +``` diff --git a/.changelog/16754.txt b/.changelog/16754.txt new file mode 100644 index 00000000000..fc2abc9ebfd --- /dev/null +++ b/.changelog/16754.txt @@ -0,0 +1,3 @@ +```release-note:security +Upgrade golang.org/x/net to address [CVE-2022-41723](https://nvd.nist.gov/vuln/detail/CVE-2022-41723) +``` diff --git a/.changelog/16776.txt b/.changelog/16776.txt new file mode 100644 index 00000000000..0159aee8589 --- /dev/null +++ b/.changelog/16776.txt @@ -0,0 +1,3 @@ +```release-note:improvement +peering: allow re-establishing terminated peering from new token without deleting existing peering first. +``` \ No newline at end of file diff --git a/.changelog/16781.txt b/.changelog/16781.txt new file mode 100644 index 00000000000..708a91d40c8 --- /dev/null +++ b/.changelog/16781.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateway: **(Enterprise only)** Fix bug where namespace/partition would fail to unmarshal for TCPServices. +``` diff --git a/.changelog/16789.txt b/.changelog/16789.txt new file mode 100644 index 00000000000..ed25e11bbb6 --- /dev/null +++ b/.changelog/16789.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateway: **(Enterprise only)** Fix bug where parent refs and service refs for a route in the same namespace as the route would fallback to the default namespace if the namespace was not specified in the configuration rather than falling back to the routes namespace. +``` diff --git a/.changelog/16818.txt b/.changelog/16818.txt new file mode 100644 index 00000000000..665c11034cc --- /dev/null +++ b/.changelog/16818.txt @@ -0,0 +1,3 @@ +```release-note:bug +cache: revert cache refactor which could cause blocking queries to never return +``` diff --git a/.changelog/16845.txt b/.changelog/16845.txt new file mode 100644 index 00000000000..7181e319e3a --- /dev/null +++ b/.changelog/16845.txt @@ -0,0 +1,3 @@ +```release-note:improvement +systemd: set service type to notify. +``` diff --git a/.changelog/16889.txt b/.changelog/16889.txt new file mode 100644 index 00000000000..67a859a430a --- /dev/null +++ b/.changelog/16889.txt @@ -0,0 +1,3 @@ +```release-note:improvement +connect: update supported envoy versions to 1.22.11, 1.23.8, 1.24.6, 1.25.4 +``` diff --git a/.changelog/16916.txt b/.changelog/16916.txt new file mode 100644 index 00000000000..a911e14d7e0 --- /dev/null +++ b/.changelog/16916.txt @@ -0,0 +1,3 @@ +```release-note:improvement +hcp: Add support for linking existing Consul clusters to HCP management plane. +``` \ No newline at end of file diff --git a/.changelog/17038.txt b/.changelog/17038.txt new file mode 100644 index 00000000000..b3a47f98a77 --- /dev/null +++ b/.changelog/17038.txt @@ -0,0 +1,3 @@ +```release-note:improvement +agent: add new metrics to track cpu disk and memory usage for server hosts (defaults to: enabled) +``` diff --git a/.changelog/17048.txt b/.changelog/17048.txt new file mode 100644 index 00000000000..74f31c7ce27 --- /dev/null +++ b/.changelog/17048.txt @@ -0,0 +1,3 @@ +```release-note:bug +Fix an bug where decoding some Config structs with unset pointer fields could fail with `reflect: call of reflect.Value.Type on zero Value`. +``` diff --git a/.changelog/17055.txt b/.changelog/17055.txt new file mode 100644 index 00000000000..9300c411219 --- /dev/null +++ b/.changelog/17055.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateways: Fix an bug where targeting a virtual service defined by a service-resolver was broken for HTTPRoutes. +``` diff --git a/.changelog/17081.txt b/.changelog/17081.txt new file mode 100644 index 00000000000..5d17a304847 --- /dev/null +++ b/.changelog/17081.txt @@ -0,0 +1,3 @@ +```release-note:improvement +Fixes a performance issue in Raft where commit latency can increase by 100x or more when under heavy load. For more details see https://github.com/hashicorp/raft/pull/541. +``` diff --git a/.changelog/17115.txt b/.changelog/17115.txt new file mode 100644 index 00000000000..8b6a9090db9 --- /dev/null +++ b/.changelog/17115.txt @@ -0,0 +1,3 @@ +```release-note:improvement +gateway: Change status condition reason for invalid certificate on a listener from "Accepted" to "ResolvedRefs". +``` diff --git a/.changelog/17160.txt b/.changelog/17160.txt new file mode 100644 index 00000000000..666a6e8f252 --- /dev/null +++ b/.changelog/17160.txt @@ -0,0 +1,3 @@ +```release-note:bug +Fix a bug that wrongly trims domains when there is an overlap with DC name. +``` diff --git a/.changelog/17171.txt b/.changelog/17171.txt new file mode 100644 index 00000000000..882b6358793 --- /dev/null +++ b/.changelog/17171.txt @@ -0,0 +1,3 @@ +```release-note:improvement +agent: add a configurable maximimum age (default: 7 days) to prevent servers re-joining a cluster with stale data +``` diff --git a/.changelog/17179.txt b/.changelog/17179.txt new file mode 100644 index 00000000000..efcfba70927 --- /dev/null +++ b/.changelog/17179.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: ensure that merged central configs of peered upstreams for partitioned downstreams work +``` diff --git a/.changelog/17185.txt b/.changelog/17185.txt new file mode 100644 index 00000000000..cde123e2deb --- /dev/null +++ b/.changelog/17185.txt @@ -0,0 +1,3 @@ +```release-note:bug +xds: Fix possible panic that can when generating clusters before the root certificates have been fetched. +``` diff --git a/.changelog/17231.txt b/.changelog/17231.txt new file mode 100644 index 00000000000..fd25d07398c --- /dev/null +++ b/.changelog/17231.txt @@ -0,0 +1,3 @@ +```release-note:bug +acl: Fix an issue where the anonymous token was synthesized in non-primary datacenters which could cause permission errors when federating clusters with ACL replication enabled. +``` diff --git a/.changelog/17235.txt b/.changelog/17235.txt new file mode 100644 index 00000000000..3356b715ef3 --- /dev/null +++ b/.changelog/17235.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fix issue where peer streams could incorrectly deregister services in various scenarios. +``` diff --git a/.changelog/17236.txt b/.changelog/17236.txt new file mode 100644 index 00000000000..c824bb7ed78 --- /dev/null +++ b/.changelog/17236.txt @@ -0,0 +1,3 @@ +```release-note:improvement +logging: change snapshot log header from `agent.server.snapshot` to `agent.server.raft.snapshot` +``` diff --git a/.changelog/17240.txt b/.changelog/17240.txt new file mode 100644 index 00000000000..59d120f747b --- /dev/null +++ b/.changelog/17240.txt @@ -0,0 +1,12 @@ +```release-note:security +Upgrade to use Go 1.20.4. +This resolves vulnerabilities [CVE-2023-24537](https://github.com/advisories/GHSA-9f7g-gqwh-jpf5)(`go/scanner`), +[CVE-2023-24538](https://github.com/advisories/GHSA-v4m2-x4rp-hv22)(`html/template`), +[CVE-2023-24534](https://github.com/advisories/GHSA-8v5j-pwr7-w5f8)(`net/textproto`) and +[CVE-2023-24536](https://github.com/advisories/GHSA-9f7g-gqwh-jpf5)(`mime/multipart`). +Also, `golang.org/x/net` has been updated to v0.7.0 to resolve CVEs [CVE-2022-41721 +](https://github.com/advisories/GHSA-fxg5-wq6x-vr4w +), [CVE-2022-27664](https://github.com/advisories/GHSA-69cg-p879-7622) and [CVE-2022-41723 +](https://github.com/advisories/GHSA-vvpx-j8f3-3w6h +.) +``` diff --git a/.changelog/17241.txt b/.changelog/17241.txt new file mode 100644 index 00000000000..0369710928e --- /dev/null +++ b/.changelog/17241.txt @@ -0,0 +1,3 @@ +```release-note:bug +connect: Fix multiple inefficient behaviors when querying service health. +``` diff --git a/.changelog/17270.txt b/.changelog/17270.txt new file mode 100644 index 00000000000..b9bd52888e4 --- /dev/null +++ b/.changelog/17270.txt @@ -0,0 +1,3 @@ +```release-note:bug +grpc: ensure grpc resolver correctly uses lan/wan addresses on servers +``` diff --git a/.changelog/17327.txt b/.changelog/17327.txt new file mode 100644 index 00000000000..24b1c28c1df --- /dev/null +++ b/.changelog/17327.txt @@ -0,0 +1,3 @@ +```release-note:improvement + xds: rename envoy_hcp_metrics_bind_socket_dir to envoy_telemetry_collector_bind_socket_dir to remove HCP naming references. + ``` \ No newline at end of file diff --git a/.changelog/17415.txt b/.changelog/17415.txt new file mode 100644 index 00000000000..3f5b1e11cf7 --- /dev/null +++ b/.changelog/17415.txt @@ -0,0 +1,7 @@ +```release-note:security +extensions: Disable remote downstream proxy patching by Envoy Extensions other than AWS Lambda. Previously, an operator with service:write ACL permissions for an upstream service could modify Envoy proxy config for downstream services without equivalent permissions for those services. This issue only impacts the Lua extension. [[CVE-2023-2816](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-2816)] +``` + +```release-note:breaking-change +extensions: The Lua extension now targets local proxy listeners for the configured service's upstreams, rather than remote downstream listeners for the configured service, when ListenerType is set to outbound in extension configuration. See [CVE-2023-2816](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-2816) changelog entry for more details. +``` diff --git a/.changelog/17426.txt b/.changelog/17426.txt new file mode 100644 index 00000000000..d8fbd2ae2c4 --- /dev/null +++ b/.changelog/17426.txt @@ -0,0 +1,5 @@ +```release-note:improvement + peering: gRPC queries for TrustBundleList, TrustBundleRead, PeeringList, and PeeringRead now support blocking semantics, + reducing network and CPU demand. + The HTTP APIs for Peering List and Read have been updated to support blocking. + ``` \ No newline at end of file diff --git a/.changelog/17456.txt b/.changelog/17456.txt new file mode 100644 index 00000000000..7b81d53543c --- /dev/null +++ b/.changelog/17456.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fix issue where modifying the list of exported services did not correctly replicate changes for services that exist in a non-default namespace. +``` diff --git a/.changelog/17460.txt b/.changelog/17460.txt new file mode 100644 index 00000000000..8e9c55517f6 --- /dev/null +++ b/.changelog/17460.txt @@ -0,0 +1,3 @@ +```release-note:feature +hcp: Add new metrics sink to collect, aggregate and export server metrics to HCP in OTEL format. +``` \ No newline at end of file diff --git a/.changelog/17483.txt b/.changelog/17483.txt new file mode 100644 index 00000000000..26c81dbe4cd --- /dev/null +++ b/.changelog/17483.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: Fix a bug that caused server agents to continue cleaning up peering resources even after loss of leadership. +``` diff --git a/.changelog/17513.txt b/.changelog/17513.txt new file mode 100644 index 00000000000..a87557d08ca --- /dev/null +++ b/.changelog/17513.txt @@ -0,0 +1,3 @@ +```release-note:security +Update to UBI base image to 9.2. +``` diff --git a/.changelog/17545.txt b/.changelog/17545.txt new file mode 100644 index 00000000000..a605f21bb3a --- /dev/null +++ b/.changelog/17545.txt @@ -0,0 +1,3 @@ +```release-note:improvement +connect: update supported envoy versions to 1.22.11, 1.23.9, 1.24.7, 1.25.6 +``` diff --git a/.changelog/17565.txt b/.changelog/17565.txt new file mode 100644 index 00000000000..f7cf46c3895 --- /dev/null +++ b/.changelog/17565.txt @@ -0,0 +1,3 @@ +```release-note:feature +reloadable config: Made enable_debug config reloadable and enable pprof command to work when config toggles to true +``` \ No newline at end of file diff --git a/.changelog/17566.txt b/.changelog/17566.txt new file mode 100644 index 00000000000..f15718bd760 --- /dev/null +++ b/.changelog/17566.txt @@ -0,0 +1,3 @@ +```release-note:bug +xds: Fixed a bug where modifying ACLs on a token being actively used for an xDS connection caused all xDS updates to fail. +``` diff --git a/.changelog/17577.txt b/.changelog/17577.txt new file mode 100644 index 00000000000..3699d526112 --- /dev/null +++ b/.changelog/17577.txt @@ -0,0 +1,3 @@ +```release-note:improvement +fix metric names in /docs/agent/telemetry +``` \ No newline at end of file diff --git a/.changelog/17581.txt b/.changelog/17581.txt new file mode 100644 index 00000000000..9277dbcd3ef --- /dev/null +++ b/.changelog/17581.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateways: **(Enterprise only)** Fixed a bug in API gateways where gateway configuration objects in non-default partitions did not reconcile properly. +``` diff --git a/.changelog/17582.txt b/.changelog/17582.txt new file mode 100644 index 00000000000..122b9df9811 --- /dev/null +++ b/.changelog/17582.txt @@ -0,0 +1,3 @@ +```release-note:feature +cli: `consul operator raft list-peers` command shows the number of commits each follower is trailing the leader by to aid in troubleshooting. +``` diff --git a/.changelog/17593.txt b/.changelog/17593.txt new file mode 100644 index 00000000000..1f84e75f574 --- /dev/null +++ b/.changelog/17593.txt @@ -0,0 +1,3 @@ +```release-note:bug +docs: fix list of telemetry metrics +``` diff --git a/.changelog/17596.txt b/.changelog/17596.txt new file mode 100644 index 00000000000..1058df1ea3a --- /dev/null +++ b/.changelog/17596.txt @@ -0,0 +1,3 @@ +```release-note:improvement + debug: change default setting of consul debug command. now default duration is 5ms and default log level is 'TRACE' + ``` \ No newline at end of file diff --git a/.changelog/17609.txt b/.changelog/17609.txt new file mode 100644 index 00000000000..cbace1f8c7d --- /dev/null +++ b/.changelog/17609.txt @@ -0,0 +1,4 @@ +```release-note:bug +gateways: Fixed a bug in API gateways where binding a route that only targets a service imported from a peer results +in the programmed gateway having no routes. +``` diff --git a/.changelog/17631.txt b/.changelog/17631.txt new file mode 100644 index 00000000000..b24b7461ec6 --- /dev/null +++ b/.changelog/17631.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateways: Fixed a bug where API gateways were not being taken into account in determining xDS rate limits. +``` diff --git a/.changelog/17636.txt b/.changelog/17636.txt new file mode 100644 index 00000000000..aa06f9191b9 --- /dev/null +++ b/.changelog/17636.txt @@ -0,0 +1,3 @@ +```release-note:bug +cache: fix a few minor goroutine leaks in leaf certs and the agent cache +``` diff --git a/.changelog/17739.txt b/.changelog/17739.txt new file mode 100644 index 00000000000..14bbceeaa08 --- /dev/null +++ b/.changelog/17739.txt @@ -0,0 +1,3 @@ +```release-note:bug +http: fixed API endpoint `PUT /acl/token/:AccessorID` (update token), no longer requires `AccessorID` in the request body. Web UI can now update tokens. + ``` diff --git a/.changelog/17780.txt b/.changelog/17780.txt new file mode 100644 index 00000000000..b90925a8b9f --- /dev/null +++ b/.changelog/17780.txt @@ -0,0 +1,3 @@ +```release-note:feature +cli: `consul watch` command uses `-filter` expression to filter response from checks, services, nodes, and service. +``` diff --git a/.changelog/17846.txt b/.changelog/17846.txt new file mode 100644 index 00000000000..bd5a052f851 --- /dev/null +++ b/.changelog/17846.txt @@ -0,0 +1,3 @@ +```release-note:bug +connect/ca: Fixes a bug preventing CA configuration updates in secondary datacenters +``` diff --git a/.changelog/17885.txt b/.changelog/17885.txt new file mode 100644 index 00000000000..2cd690488d9 --- /dev/null +++ b/.changelog/17885.txt @@ -0,0 +1,2 @@ +```release-note:bug +ca: Fixed a bug where the Vault provider was not passing the configured role param for AWS auth diff --git a/.changelog/17888.txt b/.changelog/17888.txt new file mode 100644 index 00000000000..f50fcae09b0 --- /dev/null +++ b/.changelog/17888.txt @@ -0,0 +1,3 @@ +```release-note:improvement +connect: Add capture group labels from Envoy cluster FQDNs to Envoy exported metric labels +``` \ No newline at end of file diff --git a/.changelog/17894.txt b/.changelog/17894.txt new file mode 100644 index 00000000000..5749f995f71 --- /dev/null +++ b/.changelog/17894.txt @@ -0,0 +1,3 @@ +```release-note:bug +connect: Fix incorrect protocol config merging for transparent proxy implicit upstreams. +``` diff --git a/.changelog/18011.txt b/.changelog/18011.txt new file mode 100644 index 00000000000..d6c989f00e9 --- /dev/null +++ b/.changelog/18011.txt @@ -0,0 +1,4 @@ +```release-note:bug +connect: Removes the default health check from the `consul connect envoy` command when starting an API Gateway. +This health check would always fail. +``` diff --git a/.changelog/18024.txt b/.changelog/18024.txt new file mode 100644 index 00000000000..a661e7304c6 --- /dev/null +++ b/.changelog/18024.txt @@ -0,0 +1,3 @@ +```release-note:bug +connect: fix a bug with Envoy potentially starting with incomplete configuration by not waiting enough for initial xDS configuration. +``` \ No newline at end of file diff --git a/.changelog/18080.txt b/.changelog/18080.txt new file mode 100644 index 00000000000..9826b249eb3 --- /dev/null +++ b/.changelog/18080.txt @@ -0,0 +1,3 @@ +```release-note:improvement +Fix some typos in metrics docs +``` \ No newline at end of file diff --git a/.changelog/18140.txt b/.changelog/18140.txt new file mode 100644 index 00000000000..fabd9fc2916 --- /dev/null +++ b/.changelog/18140.txt @@ -0,0 +1,3 @@ +```release-note:improvement +hcp: Removes requirement for HCP to provide a management token +``` diff --git a/.changelog/18150.txt b/.changelog/18150.txt new file mode 100644 index 00000000000..492e7ad1b9f --- /dev/null +++ b/.changelog/18150.txt @@ -0,0 +1,3 @@ +```release-note:improvement +xds: Explicitly enable WebSocket connection upgrades in HTTP connection manager +``` diff --git a/.changelog/18168.txt b/.changelog/18168.txt new file mode 100644 index 00000000000..a68483527e1 --- /dev/null +++ b/.changelog/18168.txt @@ -0,0 +1,3 @@ +```release-note:improvement +hcp: Add dynamic configuration support for the export of server metrics to HCP. +``` \ No newline at end of file diff --git a/.changelog/18186.txt b/.changelog/18186.txt new file mode 100644 index 00000000000..dcc75b57653 --- /dev/null +++ b/.changelog/18186.txt @@ -0,0 +1,3 @@ +```release-note:security +Upgrade golang.org/x/net to address [CVE-2023-29406](https://nvd.nist.gov/vuln/detail/CVE-2023-29406) +``` diff --git a/.changelog/18190.txt b/.changelog/18190.txt new file mode 100644 index 00000000000..3468442e216 --- /dev/null +++ b/.changelog/18190.txt @@ -0,0 +1,5 @@ +```release-note:security +Upgrade to use Go 1.20.6. +This resolves [CVE-2023-29406](https://github.com/advisories/GHSA-f8f7-69v5-w4vx)(`net/http`) for uses of the standard library. +A separate change updates dependencies on `golang.org/x/net` to use `0.12.0`. +``` diff --git a/.changelog/18223.txt b/.changelog/18223.txt new file mode 100644 index 00000000000..067ca64f48e --- /dev/null +++ b/.changelog/18223.txt @@ -0,0 +1,3 @@ +```release-note:feature +cli: `consul members` command uses `-filter` expression to filter members based on bexpr. +``` diff --git a/.changelog/18291.txt b/.changelog/18291.txt new file mode 100644 index 00000000000..bb0ec6f8929 --- /dev/null +++ b/.changelog/18291.txt @@ -0,0 +1,3 @@ +```release-note:bug +api-gateway: fix race condition in proxy config generation when Consul is notified of the bound-api-gateway config entry before it is notified of the api-gateway config entry. +``` diff --git a/.changelog/18302.txt b/.changelog/18302.txt new file mode 100644 index 00000000000..c77e7106be9 --- /dev/null +++ b/.changelog/18302.txt @@ -0,0 +1,4 @@ +```release-note:bug +snapshot: fix access denied and handle is invalid when we call snapshot save on windows - skip sync() for folders in windows in +https://github.com/rboyer/safeio/pull/3 +``` diff --git a/.changelog/18304.txt b/.changelog/18304.txt new file mode 100644 index 00000000000..b5f8fbc1410 --- /dev/null +++ b/.changelog/18304.txt @@ -0,0 +1,3 @@ +```release-note:improvement +connect: update supported envoy versions to 1.22.11, 1.23.12, 1.24.10, 1.25.9 +``` diff --git a/.changelog/18319.txt b/.changelog/18319.txt new file mode 100644 index 00000000000..bb9c8cdf2c7 --- /dev/null +++ b/.changelog/18319.txt @@ -0,0 +1,6 @@ +```release-note:improvement +acl: added builtin ACL policy that provides global read-only access (builtin/global-read-only) +``` +```release-note:improvement +acl: allow for a single slash character in policy names +``` diff --git a/.changelog/18358.txt b/.changelog/18358.txt new file mode 100644 index 00000000000..e29d258c6c9 --- /dev/null +++ b/.changelog/18358.txt @@ -0,0 +1,7 @@ +```release-note:security +Upgrade to use Go 1.20.7. +This resolves vulnerability [CVE-2023-29409](https://nvd.nist.gov/vuln/detail/CVE-2023-29409)(`crypto/tls`). +``` +```release-note:security +Update `golang.org/x/net` to v0.13.0 to address [CVE-2023-3978](https://nvd.nist.gov/vuln/detail/CVE-2023-3978). +``` diff --git a/.changelog/18558.txt b/.changelog/18558.txt new file mode 100644 index 00000000000..9c2b9b44bb3 --- /dev/null +++ b/.changelog/18558.txt @@ -0,0 +1,3 @@ +```release-note:bug +check: prevent go routine leakage when existing Defercheck of same check id is not nil +``` diff --git a/.changelog/18584.txt b/.changelog/18584.txt new file mode 100644 index 00000000000..e7329655ba6 --- /dev/null +++ b/.changelog/18584.txt @@ -0,0 +1,3 @@ +```release-note:improvement +Reduce the frequency of metric exports from Consul to HCP from every 10s to every 1m +``` \ No newline at end of file diff --git a/.changelog/18617.txt b/.changelog/18617.txt new file mode 100644 index 00000000000..1f840d836de --- /dev/null +++ b/.changelog/18617.txt @@ -0,0 +1,4 @@ +```release-note:improvement +log: Currently consul logs files like this consul-{timestamp}.log. This change makes sure that there is always +consul.log file with the latest logs in it. +``` \ No newline at end of file diff --git a/.changelog/18625.txt b/.changelog/18625.txt new file mode 100644 index 00000000000..8474cac8dc1 --- /dev/null +++ b/.changelog/18625.txt @@ -0,0 +1,5 @@ +```release-note:improvement +Adds flag -append-filename (which works on values version, dc, node and status) to consul snapshot save command. +Adding the flag -append-filename version,dc,node,status will add consul version, consul datacenter, node name and leader/follower +(status) in the file name given in the snapshot save command before the file extension. +``` diff --git a/.changelog/18667.txt b/.changelog/18667.txt new file mode 100644 index 00000000000..c9ef7b45512 --- /dev/null +++ b/.changelog/18667.txt @@ -0,0 +1,3 @@ +```release-note:improvement +api: Add support for listing ACL tokens by service name. +``` diff --git a/.changelog/18681.txt b/.changelog/18681.txt new file mode 100644 index 00000000000..971e9ef8163 --- /dev/null +++ b/.changelog/18681.txt @@ -0,0 +1,3 @@ +```release-note:bug +api: Fix `/v1/agent/self` not returning latest configuration +``` diff --git a/.changelog/18724.txt b/.changelog/18724.txt new file mode 100644 index 00000000000..7fa289eba19 --- /dev/null +++ b/.changelog/18724.txt @@ -0,0 +1,3 @@ +```release-note:bug +telemetry: emit consul version metric on a regular interval. +``` diff --git a/.changelog/18742.txt b/.changelog/18742.txt new file mode 100644 index 00000000000..2d31e526675 --- /dev/null +++ b/.changelog/18742.txt @@ -0,0 +1,8 @@ +```release-note:security +Upgrade to use Go 1.20.8. This resolves CVEs +[CVE-2023-39320](https://github.com/advisories/GHSA-rxv8-v965-v333) (`cmd/go`), +[CVE-2023-39318](https://github.com/advisories/GHSA-vq7j-gx56-rxjh) (`html/template`), +[CVE-2023-39319](https://github.com/advisories/GHSA-vv9m-32rr-3g55) (`html/template`), +[CVE-2023-39321](https://github.com/advisories/GHSA-9v7r-x7cv-v437) (`crypto/tls`), and +[CVE-2023-39322](https://github.com/advisories/GHSA-892h-r6cr-53g4) (`crypto/tls`) +``` \ No newline at end of file diff --git a/.changelog/18773.txt b/.changelog/18773.txt new file mode 100644 index 00000000000..1d59fe98f0d --- /dev/null +++ b/.changelog/18773.txt @@ -0,0 +1,3 @@ +```release-note:bug +ca: Vault provider now cleans up the previous Vault issuer and key when generating a new leaf signing certificate [[GH-18779](https://github.com/hashicorp/consul/issues/18779)] +``` diff --git a/.changelog/5102.txt b/.changelog/5102.txt new file mode 100644 index 00000000000..97d8c7bf8bf --- /dev/null +++ b/.changelog/5102.txt @@ -0,0 +1,3 @@ +```release-note:feature +server: **(Enterprise Only)** allow automatic license utilization reporting. +``` \ No newline at end of file diff --git a/.changelog/_16677.txt b/.changelog/_16677.txt new file mode 100644 index 00000000000..0bf621f09ac --- /dev/null +++ b/.changelog/_16677.txt @@ -0,0 +1,3 @@ +```release-note:bug +gateway: **(Enterprise only)** Fix bug where routes defined in a different namespace than a gateway would fail to register. [[GH-16677](https://github.com/hashicorp/consul/pull/16677)]. +``` diff --git a/.changelog/_4696.txt b/.changelog/_4696.txt new file mode 100644 index 00000000000..951896fd66f --- /dev/null +++ b/.changelog/_4696.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: **(Consul Enterprise only)** Fix issue where connect-enabled services with peer upstreams incorrectly required `service:write` access in the `default` namespace to query data, which was too restrictive. Now having `service:write` to any namespace is sufficient to query the peering data. +``` diff --git a/.changelog/_4832.txt b/.changelog/_4832.txt new file mode 100644 index 00000000000..b4576871554 --- /dev/null +++ b/.changelog/_4832.txt @@ -0,0 +1,3 @@ +```release-note:bug +peering: **(Consul Enterprise only)** Fix issue where resolvers, routers, and splitters referencing peer targets may not work correctly for non-default partitions and namespaces. Enterprise customers leveraging peering are encouraged to upgrade both servers and agents to avoid this problem. +``` diff --git a/.changelog/_5517.txt b/.changelog/_5517.txt new file mode 100644 index 00000000000..5152a6ff78f --- /dev/null +++ b/.changelog/_5517.txt @@ -0,0 +1,3 @@ +```release-note:bug +namespaces: **(Enterprise only)** fixes a bug where agent health checks stop syncing for all services on a node if the namespace of any service has been removed from the server. +``` diff --git a/.changelog/_5614.txt b/.changelog/_5614.txt new file mode 100644 index 00000000000..9951b911187 --- /dev/null +++ b/.changelog/_5614.txt @@ -0,0 +1,4 @@ +```release-note:bug +namespaces: **(Enterprise only)** fixes a bug where namespaces are stuck in a deferred deletion state indefinitely under some conditions. +Also fixes the Consul query metadata present in the HTTP headers of the namespace read and list endpoints. +``` diff --git a/.circleci/bash_env.sh b/.circleci/bash_env.sh deleted file mode 100644 index 38bcfd5bd36..00000000000 --- a/.circleci/bash_env.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -export GIT_COMMIT=$(git rev-parse --short HEAD) -export GIT_COMMIT_YEAR=$(git show -s --format=%cd --date=format:%Y HEAD) -export GIT_DIRTY=$(test -n "`git status --porcelain`" && echo "+CHANGES" || true) -export GIT_IMPORT=github.com/hashicorp/consul/version -# we're using this for build date because it's stable across platform builds -# the env -i and -noprofile are used to ensure we don't try to recursively call this profile when starting bash -export GIT_DATE=$(env -i /bin/bash --noprofile -norc /home/circleci/project/build-support/scripts/build-date.sh) -export GOLDFLAGS="-X ${GIT_IMPORT}.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -X ${GIT_IMPORT}.BuildDate=${GIT_DATE}" diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index dae806a625b..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,1263 +0,0 @@ ---- -version: 2.1 - -parameters: - commit: - type: string - default: "" - description: "Commit to run load tests against" - trigger-load-test: - type: boolean - default: false - description: "Boolean whether to run the load test workflow" - -references: - paths: - test-results: &TEST_RESULTS_DIR /tmp/test-results - environment: &ENVIRONMENT - TEST_RESULTS_DIR: *TEST_RESULTS_DIR - EMAIL: noreply@hashicorp.com - GIT_AUTHOR_NAME: circleci-consul - GIT_COMMITTER_NAME: circleci-consul - S3_ARTIFACT_BUCKET: consul-dev-artifacts-v2 - BASH_ENV: .circleci/bash_env.sh - GO_VERSION: 1.19.4 - envoy-versions: &supported_envoy_versions - - &default_envoy_version "1.21.5" - - "1.22.5" - - "1.23.2" - - "1.24.0" - nomad-versions: &supported_nomad_versions - - &default_nomad_version "1.3.3" - - "1.2.10" - - "1.1.16" - vault-versions: &supported_vault_versions - - &default_vault_version "1.12.2" - - "1.11.6" - - "1.10.9" - - "1.9.10" - images: - # When updating the Go version, remember to also update the versions in the - # workflows section for go-test-lib jobs. - go: &GOLANG_IMAGE docker.mirror.hashicorp.services/cimg/go:1.19.4 - ember: &EMBER_IMAGE docker.mirror.hashicorp.services/circleci/node:14-browsers - ubuntu: &UBUNTU_CI_IMAGE ubuntu-2004:202201-02 - cache: - yarn: &YARN_CACHE_KEY consul-ui-v9-{{ checksum "ui/yarn.lock" }} - -steps: - install-gotestsum: &install-gotestsum - name: install gotestsum - environment: - GOTESTSUM_RELEASE: 1.9.0 - command: | - ARCH=`uname -m` - if [[ "$ARCH" == "aarch64" ]]; then - ARCH="arm64" - else - ARCH="amd64" - fi - url=https://github.com/gotestyourself/gotestsum/releases/download - curl -sSL "${url}/v${GOTESTSUM_RELEASE}/gotestsum_${GOTESTSUM_RELEASE}_linux_${ARCH}.tar.gz" | \ - sudo tar -xz --overwrite -C /usr/local/bin gotestsum - - get-aws-cli: &get-aws-cli - run: - name: download and install AWS CLI - command: | - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - echo -e "${AWS_CLI_GPG_KEY}" | gpg --import - curl -o awscliv2.sig https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip.sig - gpg --verify awscliv2.sig awscliv2.zip - unzip awscliv2.zip - sudo ./aws/install - - # This step MUST be at the end of any set of steps due to the 'when' condition - notify-slack-failure: ¬ify-slack-failure - name: notify-slack-failure - when: on_fail - command: | - if [[ $CIRCLE_BRANCH == "main" ]]; then - CIRCLE_ENDPOINT="https://app.circleci.com/pipelines/github/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}?branch=${CIRCLE_BRANCH}" - GITHUB_ENDPOINT="https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/commit/${CIRCLE_SHA1}" - COMMIT_MESSAGE=$(git log -1 --pretty=%B | head -n1) - SHORT_REF=$(git rev-parse --short "${CIRCLE_SHA1}") - curl -X POST -H 'Content-type: application/json' \ - --data \ - "{ \ - \"attachments\": [ \ - { \ - \"fallback\": \"CircleCI job failed!\", \ - \"text\": \"❌ Failed: \`${CIRCLE_USERNAME}\`'s <${CIRCLE_BUILD_URL}|${CIRCLE_STAGE}> job failed for commit <${GITHUB_ENDPOINT}|${SHORT_REF}> on \`${CIRCLE_BRANCH}\`!\n\n- <${COMMIT_MESSAGE}\", \ - \"footer\": \"${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}\", \ - \"ts\": \"$(date +%s)\", \ - \"color\": \"danger\" \ - } \ - ] \ - }" "${FEED_CONSUL_GH_URL}" - else - echo "Not posting slack failure notifications for non-main branch" - fi - -commands: - assume-role: - description: "Assume role to an ARN" - parameters: - access-key: - type: env_var_name - default: AWS_ACCESS_KEY_ID - secret-key: - type: env_var_name - default: AWS_SECRET_ACCESS_KEY - role-arn: - type: env_var_name - default: ROLE_ARN - steps: - # Only run the assume-role command for the main repo. The AWS credentials aren't available for forks. - - run: | - if [[ "${CIRCLE_BRANCH%%/*}/" != "pull/" ]]; then - export AWS_ACCESS_KEY_ID="${<< parameters.access-key >>}" - export AWS_SECRET_ACCESS_KEY="${<< parameters.secret-key >>}" - export ROLE_ARN="${<< parameters.role-arn >>}" - # assume role has duration of 15 min (the minimum allowed) - CREDENTIALS="$(aws sts assume-role --duration-seconds 900 --role-arn ${ROLE_ARN} --role-session-name build-${CIRCLE_SHA1} | jq '.Credentials')" - echo "export AWS_ACCESS_KEY_ID=$(echo $CREDENTIALS | jq -r '.AccessKeyId')" >> $BASH_ENV - echo "export AWS_SECRET_ACCESS_KEY=$(echo $CREDENTIALS | jq -r '.SecretAccessKey')" >> $BASH_ENV - echo "export AWS_SESSION_TOKEN=$(echo $CREDENTIALS | jq -r '.SessionToken')" >> $BASH_ENV - fi - - run-go-test-full: - parameters: - go_test_flags: - type: string - default: "" - steps: - - attach_workspace: - at: /home/circleci/go/bin - - run: go mod download - - run: - name: go test - command: | - mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile - PACKAGE_NAMES=$(go list -tags "$GOTAGS" ./... | circleci tests split --split-by=timings --timings-type=classname) - echo "Running $(echo $PACKAGE_NAMES | wc -w) packages" - echo $PACKAGE_NAMES - # some tests expect this umask, and arm images have a different default - umask 0022 - - << parameters.go_test_flags >> - - gotestsum \ - --format=short-verbose \ - --jsonfile /tmp/jsonfile/go-test-${CIRCLE_NODE_INDEX}.log \ - --debug \ - --rerun-fails=3 \ - --rerun-fails-max-failures=40 \ - --rerun-fails-report=/tmp/gotestsum-rerun-fails \ - --packages="$PACKAGE_NAMES" \ - --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \ - -tags="$GOTAGS" -p 2 \ - ${GO_TEST_FLAGS-} \ - -cover -coverprofile=coverage.txt - - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: /tmp/jsonfile - - run: &rerun-fails-report - name: "Re-run fails report" - command: | - .circleci/scripts/rerun-fails-report.sh /tmp/gotestsum-rerun-fails - - run: *notify-slack-failure - -jobs: - # lint consul tests - lint-consul-retry: - docker: - - image: *GOLANG_IMAGE - steps: - - checkout - - run: go install github.com/hashicorp/lint-consul-retry@master && lint-consul-retry - - run: *notify-slack-failure - - lint-enums: - docker: - - image: *GOLANG_IMAGE - steps: - - checkout - - run: go install github.com/reillywatson/enumcover/cmd/enumcover@master && enumcover ./... - - run: *notify-slack-failure - - lint-container-test-deps: - docker: - - image: *GOLANG_IMAGE - steps: - - checkout - - run: make lint-container-test-deps - - run: *notify-slack-failure - - lint: - description: "Run golangci-lint" - parameters: - go-arch: - type: string - default: "" - docker: - - image: *GOLANG_IMAGE - resource_class: xlarge - environment: - GOTAGS: "" # No tags for OSS but there are for enterprise - GOARCH: "<>" - steps: - - checkout - - run: go env - - run: - name: Install golangci-lint - command: make lint-tools - - run: go mod download - - run: - name: lint - command: &lintcmd | - golangci-lint run --build-tags="$GOTAGS" -v - - run: - name: lint api - working_directory: api - command: *lintcmd - - run: - name: lint sdk - working_directory: sdk - command: *lintcmd - - run: - name: lint envoyextensions - working_directory: envoyextensions - command: *lintcmd - - run: - name: lint troubleshoot - working_directory: troubleshoot - command: *lintcmd - - run: - name: lint container tests - working_directory: test/integration/consul-container - command: *lintcmd - - run: *notify-slack-failure - - check-go-mod: - docker: - - image: *GOLANG_IMAGE - environment: - <<: *ENVIRONMENT - steps: - - checkout - - run: go mod tidy - - run: | - if [[ -n $(git status -s) ]]; then - echo "Git directory has changes" - git status -s - exit 1 - fi - - run: *notify-slack-failure - - check-generated-protobuf: - docker: - - image: *GOLANG_IMAGE - environment: - <<: *ENVIRONMENT - # tput complains if this isn't set to something. - TERM: ansi - steps: - - checkout - - run: - name: Install protobuf - command: make proto-tools - - run: - name: "Protobuf Format" - command: make proto-format - - run: - command: make --always-make proto - - run: | - if ! git diff --exit-code; then - echo "Generated code was not updated correctly" - exit 1 - fi - - run: - name: "Protobuf Lint" - command: make proto-lint - - check-generated-deep-copy: - docker: - - image: *GOLANG_IMAGE - environment: - <<: *ENVIRONMENT - # tput complains if this isn't set to something. - TERM: ansi - steps: - - checkout - - run: - name: Install deep-copy - command: make codegen-tools - - run: - command: make --always-make deep-copy - - run: | - if ! git diff --exit-code; then - echo "Generated code was not updated correctly" - exit 1 - fi - - go-test-arm64: - machine: - image: *UBUNTU_CI_IMAGE - resource_class: arm.large - parallelism: 4 - environment: - <<: *ENVIRONMENT - GOTAGS: "" # No tags for OSS but there are for enterprise - # GOMAXPROCS defaults to number of cores on underlying hardware, set - # explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues - GOMAXPROCS: 4 - steps: - - checkout - - run: - command: | - sudo rm -rf /usr/local/go - wget https://dl.google.com/go/go${GO_VERSION}.linux-arm64.tar.gz - sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-arm64.tar.gz - - run: *install-gotestsum - - run: go mod download - - run: - name: make dev - command: | - if [[ "$CIRCLE_BRANCH" =~ ^main$|^release/ ]]; then - make dev - mkdir -p /home/circleci/bin - cp ./bin/consul /home/circleci/bin/consul - fi - - run-go-test-full: - go_test_flags: 'if ! [[ "$CIRCLE_BRANCH" =~ ^main$|^release/ ]]; then export GO_TEST_FLAGS="-short"; fi' - - go-test: - docker: - - image: *GOLANG_IMAGE - resource_class: large - parallelism: 4 - environment: - <<: *ENVIRONMENT - GOTAGS: "" # No tags for OSS but there are for enterprise - # GOMAXPROCS defaults to number of cores on underlying hardware, set - # explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues - GOMAXPROCS: 4 - steps: - - checkout - - run-go-test-full - - go-test-race: - docker: - - image: *GOLANG_IMAGE - environment: - <<: *ENVIRONMENT - GOTAGS: "" # No tags for OSS but there are for enterprise - # GOMAXPROCS defaults to number of cores on underlying hardware, set - # explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues - GOMAXPROCS: 4 - # The medium resource class (default) boxes are 2 vCPUs, 4GB RAM - # https://circleci.com/docs/2.0/configuration-reference/#docker-executor - # but we can run a little over that limit. - steps: - - checkout - - run: go mod download - - run: - name: go test -race - command: | - mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile - pkgs="$(go list ./... | \ - grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | \ - grep -E -v '^github.com/hashicorp/consul/command/')" - gotestsum \ - --jsonfile /tmp/jsonfile/go-test-race.log \ - --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \ - -tags="$GOTAGS" -p 2 \ - -race -gcflags=all=-d=checkptr=0 \ - $pkgs - - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: /tmp/jsonfile - - run: *notify-slack-failure - - # go-test-32bit is to catch problems where 64-bit ints must be 64-bit aligned - # to use them with sync/atomic. See https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - # Running tests with GOARCH=386 seems to be the best way to detect this - # problem. Only runs tests that are -short to limit the time we spend checking - # for these bugs. - go-test-32bit: - docker: - - image: *GOLANG_IMAGE - resource_class: large - environment: - <<: *ENVIRONMENT - GOTAGS: "" # No tags for OSS but there are for enterprise - steps: - - checkout - - run: go mod download - - run: - name: go test 32-bit - environment: - GOARCH: 386 - command: | - mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile - go env - PACKAGE_NAMES=$(go list -tags "$GOTAGS" ./...) - gotestsum \ - --jsonfile /tmp/jsonfile/go-test-32bit.log \ - --rerun-fails=3 \ - --rerun-fails-max-failures=40 \ - --rerun-fails-report=/tmp/gotestsum-rerun-fails \ - --packages="$PACKAGE_NAMES" \ - --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \ - -tags="$GOTAGS" -p 2 \ - -short - - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - run: *notify-slack-failure - - go-test-lib: - description: "test a library against a specific Go version" - parameters: - go-version: - type: string - path: - type: string - docker: - - image: "docker.mirror.hashicorp.services/cimg/go:<>" - environment: - <<: *ENVIRONMENT - GOTAGS: "" # No tags for OSS but there are for enterprise - steps: - - checkout - - attach_workspace: - at: /home/circleci/go/bin - - run: - working_directory: <> - command: go mod download - - run: - working_directory: <> - name: go test - command: | - mkdir -p $TEST_RESULTS_DIR /tmp/jsonfile - gotestsum \ - --format=short-verbose \ - --jsonfile /tmp/jsonfile/go-test-<>.log \ - --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \ - -tags="$GOTAGS" -cover -coverprofile=coverage.txt \ - ./... - - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: /tmp/jsonfile - - run: *notify-slack-failure - - # build is a templated job for build-x - build-distros: &build-distros - docker: - - image: *GOLANG_IMAGE - resource_class: large - environment: &build-env - <<: *ENVIRONMENT - steps: - - checkout - - run: - name: Build - command: | - for os in $XC_OS; do - target="./pkg/bin/${GOOS}_${GOARCH}/" - GOOS="$os" CGO_ENABLED=0 go build -o "${target}" -ldflags "${GOLDFLAGS}" -tags "${GOTAGS}" - done - - # save dev build to CircleCI - - store_artifacts: - path: ./pkg/bin - - run: *notify-slack-failure - - # build all 386 architecture supported OS binaries - build-386: - <<: *build-distros - environment: - <<: *build-env - XC_OS: "freebsd linux windows" - GOARCH: "386" - - # build all amd64 architecture supported OS binaries - build-amd64: - <<: *build-distros - environment: - <<: *build-env - XC_OS: "darwin freebsd linux solaris windows" - GOARCH: "amd64" - - # build all arm/arm64 architecture supported OS binaries - build-arm: - docker: - - image: *GOLANG_IMAGE - resource_class: large - environment: - <<: *ENVIRONMENT - CGO_ENABLED: 1 - GOOS: linux - steps: - - checkout - - run: - command: | - sudo rm -fv /etc/apt/sources.list.d/github_git-lfs.list # workaround for https://github.com/actions/runner-images/issues/1983 - sudo apt-get update --allow-releaseinfo-change-suite --allow-releaseinfo-change-version && sudo apt-get install -y gcc-arm-linux-gnueabi gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu - - run: - environment: - GOARM: 5 - CC: arm-linux-gnueabi-gcc - GOARCH: arm - command: go build -o ./pkg/bin/linux_armel/consul -ldflags="-linkmode=external ${GOLDFLAGS}" - - run: - environment: - GOARM: 6 - CC: arm-linux-gnueabihf-gcc - GOARCH: arm - command: go build -o ./pkg/bin/linux_armhf/consul -ldflags="-linkmode=external ${GOLDFLAGS}" - - run: - environment: - CC: aarch64-linux-gnu-gcc - GOARCH: arm64 - command: go build -o ./pkg/bin/linux_aarch64/consul -ldflags="-linkmode=external ${GOLDFLAGS}" - - store_artifacts: - path: ./pkg/bin - - run: *notify-slack-failure - - # create a development build - dev-build: - docker: - - image: *GOLANG_IMAGE - resource_class: large - environment: - <<: *ENVIRONMENT - steps: - - checkout - - attach_workspace: # this normally runs as the first job and has nothing to attach; only used in main branch after rebuilding UI - at: . - - run: - name: Build - command: | - make dev - mkdir -p /home/circleci/go/bin - cp ./bin/consul /home/circleci/go/bin/consul - - # save dev build to pass to downstream jobs - - persist_to_workspace: - root: /home/circleci/go/bin - paths: - - consul - - run: *notify-slack-failure - - # upload development build to s3 - dev-upload-s3: - docker: - - image: *GOLANG_IMAGE - environment: - <<: *ENVIRONMENT - steps: - - checkout - - *get-aws-cli - - assume-role: - access-key: AWS_ACCESS_KEY_ID_S3_UPLOAD - secret-key: AWS_SECRET_ACCESS_KEY_S3_UPLOAD - role-arn: ROLE_ARN_S3_UPLOAD - # get consul binary - - attach_workspace: - at: bin/ - - run: - name: package binary - command: zip -j consul.zip bin/consul - - run: - name: Upload to s3 - command: | - if [ -n "${S3_ARTIFACT_PATH}" ]; then - aws s3 cp \ - --metadata "CIRCLECI=${CIRCLECI},CIRCLE_BUILD_URL=${CIRCLE_BUILD_URL},CIRCLE_BRANCH=${CIRCLE_BRANCH}" \ - "consul.zip" "s3://${S3_ARTIFACT_BUCKET}/${S3_ARTIFACT_PATH}/${CIRCLE_SHA1}.zip" --acl public-read - else - echo "CircleCI - S3_ARTIFACT_PATH was not set" - exit 1 - fi - - run: *notify-slack-failure - - # upload dev docker image - dev-upload-docker: - docker: - - image: *GOLANG_IMAGE # use a circleci image so the attach_workspace step works (has ca-certs installed) - environment: - <<: *ENVIRONMENT - steps: - - checkout - # get consul binary - - attach_workspace: - at: bin/ - - setup_remote_docker - - run: make ci.dev-docker - - run: *notify-slack-failure - nomad-integration-test: &NOMAD_TESTS - docker: - - image: docker.mirror.hashicorp.services/cimg/go:1.19 - parameters: - nomad-version: - type: enum - enum: *supported_nomad_versions - default: *default_nomad_version - environment: - <<: *ENVIRONMENT - NOMAD_WORKING_DIR: &NOMAD_WORKING_DIR /home/circleci/go/src/github.com/hashicorp/nomad - NOMAD_VERSION: << parameters.nomad-version >> - steps: &NOMAD_INTEGRATION_TEST_STEPS - - run: git clone https://github.com/hashicorp/nomad.git --branch v${NOMAD_VERSION} ${NOMAD_WORKING_DIR} - - # get consul binary - - attach_workspace: - at: /home/circleci/go/bin - - # make dev build of nomad - - run: - command: make pkg/linux_amd64/nomad - working_directory: *NOMAD_WORKING_DIR - - - run: *install-gotestsum - - # run integration tests - - run: - name: go test - command: | - mkdir -p $TEST_RESULTS_DIR - gotestsum \ - --format=short-verbose \ - --junitfile $TEST_RESULTS_DIR/results.xml -- \ - ./command/agent/consul -run TestConsul - working_directory: *NOMAD_WORKING_DIR - - # store test results for CircleCI - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - run: *notify-slack-failure - - # build frontend yarn cache - frontend-cache: - docker: - - image: *EMBER_IMAGE - steps: - - checkout - - # cache yarn deps - - restore_cache: - key: *YARN_CACHE_KEY - - - run: - name: install yarn packages - command: cd ui && make deps - - - save_cache: - key: *YARN_CACHE_KEY - paths: - - ui/node_modules - - ui/packages/consul-ui/node_modules - - run: *notify-slack-failure - - # build ember so frontend tests run faster - ember-build-oss: &ember-build-oss - docker: - - image: *EMBER_IMAGE - environment: - JOBS: 2 # limit parallelism for broccoli-babel-transpiler - CONSUL_NSPACES_ENABLED: 0 - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - run: cd ui/packages/consul-ui && make build-ci - - # saves the build to a workspace to be passed to a downstream job - - persist_to_workspace: - root: ui - paths: - - packages/consul-ui/dist - - run: *notify-slack-failure - - # build ember so frontend tests run faster - ember-build-ent: - <<: *ember-build-oss - environment: - JOBS: 2 # limit parallelism for broccoli-babel-transpiler - CONSUL_NSPACES_ENABLED: 1 - - # rebuild UI for packaging - ember-build-prod: - docker: - - image: *EMBER_IMAGE - environment: - JOBS: 2 # limit parallelism for broccoli-babel-transpiler - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - run: cd ui && make - - # saves the build to a workspace to be passed to a downstream job - - persist_to_workspace: - root: ui - paths: - - packages/consul-ui/dist - - run: *notify-slack-failure - - # commits static assets to git - publish-static-assets: - docker: - - image: *GOLANG_IMAGE - steps: - - checkout - - add_ssh_keys: # needs a key to push updated static asset commit back to github - fingerprints: - - "94:03:9e:8b:24:7f:36:60:00:30:b8:32:ed:e7:59:10" - - attach_workspace: - at: . - - run: - name: move compiled ui files to agent/uiserver - command: | - rm -rf agent/uiserver/dist - mv packages/consul-ui/dist agent/uiserver - - run: - name: commit agent/uiserver/dist/ if there are UI changes - command: | - # check if there are any changes in ui/ - # if there are, we commit the ui static asset file - # HEAD^! is shorthand for HEAD^..HEAD (parent of HEAD and HEAD) - if ! git diff --quiet --exit-code HEAD^! ui/; then - git config --local user.email "github-team-consul-core@hashicorp.com" - git config --local user.name "hc-github-team-consul-core" - - # -B resets the CI branch to main which may diverge history - # but we will force push anyways. - git checkout -B ci/main-assetfs-build main - - short_sha=$(git rev-parse --short HEAD) - git add agent/uiserver/dist/ - git commit -m "auto-updated agent/uiserver/dist/ from commit ${short_sha}" - git push --force origin ci/main-assetfs-build - else - echo "no UI changes so no static assets to publish" - fi - - run: *notify-slack-failure - - # run node tests - node-tests: - docker: - - image: *EMBER_IMAGE - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - attach_workspace: - at: ui - - run: - working_directory: ui/packages/consul-ui - command: make test-node - - run: *notify-slack-failure - # run yarn workspace wide checks/tests - workspace-tests: - docker: - - image: *EMBER_IMAGE - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - attach_workspace: - at: ui - - run: - working_directory: ui - command: make test-workspace - - run: *notify-slack-failure - - # run ember frontend tests - ember-test-oss: - docker: - - image: *EMBER_IMAGE - environment: - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary - EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam - CONSUL_NSPACES_ENABLED: 0 - parallelism: 4 - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - attach_workspace: - at: ui - - run: - working_directory: ui/packages/consul-ui - command: node_modules/.bin/ember exam --split=$CIRCLE_NODE_TOTAL --partition=`expr $CIRCLE_NODE_INDEX + 1` --path dist --silent -r xunit - - store_test_results: - path: ui/packages/consul-ui/test-results - - run: *notify-slack-failure - - # run ember frontend tests - ember-test-ent: - docker: - - image: *EMBER_IMAGE - environment: - EMBER_TEST_REPORT: test-results/report-ent.xml #outputs test report for CircleCI test summary - EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam - CONSUL_NSPACES_ENABLED: 1 - parallelism: 4 - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - attach_workspace: - at: ui - - run: - working_directory: ui/packages/consul-ui - command: node_modules/.bin/ember exam --split=$CIRCLE_NODE_TOTAL --partition=`expr $CIRCLE_NODE_INDEX + 1` --path dist --silent -r xunit - - store_test_results: - path: ui/packages/consul-ui/test-results - - run: *notify-slack-failure - - # run ember frontend unit tests to produce coverage report - ember-coverage: - docker: - - image: *EMBER_IMAGE - steps: - - checkout - - restore_cache: - key: *YARN_CACHE_KEY - - attach_workspace: - at: ui - - run: - working_directory: ui/packages/consul-ui - command: make test-coverage-ci - - run: *notify-slack-failure - - compatibility-integration-test: - machine: - image: *UBUNTU_CI_IMAGE - docker_layer_caching: true - parallelism: 1 - steps: - - checkout - # Get go binary from workspace - - attach_workspace: - at: . - # Build the consul:local image from the already built binary - - run: - command: | - sudo rm -rf /usr/local/go - wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz - sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz - environment: - <<: *ENVIRONMENT - - run: *install-gotestsum - - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . - - run: - name: Compatibility Integration Tests - command: | - mkdir -p /tmp/test-results/ - cd ./test/integration/consul-container - docker run --rm consul:local consul version - gotestsum \ - --raw-command \ - --format=short-verbose \ - --debug \ - --rerun-fails=3 \ - --packages="./..." \ - -- \ - go test \ - -p=4 \ - -timeout=30m \ - -json \ - ./... \ - --target-image consul \ - --target-version local \ - --latest-image consul \ - --latest-version latest - ls -lrt - environment: - # this is needed because of incompatibility between RYUK container and circleci - GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml - GOTESTSUM_FORMAT: standard-verbose - COMPOSE_INTERACTIVE_NO_CLI: 1 - # tput complains if this isn't set to something. - TERM: ansi - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - run: *notify-slack-failure - - envoy-integration-test: &ENVOY_TESTS - machine: - image: *UBUNTU_CI_IMAGE - parallelism: 4 - resource_class: medium - parameters: - envoy-version: - type: enum - enum: *supported_envoy_versions - default: *default_envoy_version - xds-target: - type: enum - enum: ["server", "client"] - default: "server" - environment: - ENVOY_VERSION: << parameters.envoy-version >> - XDS_TARGET: << parameters.xds-target >> - AWS_LAMBDA_REGION: us-west-2 - steps: &ENVOY_INTEGRATION_TEST_STEPS - - checkout - - assume-role: - access-key: AWS_ACCESS_KEY_ID_LAMBDA - secret-key: AWS_SECRET_ACCESS_KEY_LAMBDA - role-arn: ROLE_ARN_LAMBDA - # Get go binary from workspace - - attach_workspace: - at: . - - run: *install-gotestsum - # Build the consul:local image from the already built binary - - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . - - run: - name: Envoy Integration Tests - command: | - subtests=$(ls -d test/integration/connect/envoy/*/ | xargs -n 1 basename | circleci tests split) - echo "Running $(echo $subtests | wc -w) subtests" - echo "$subtests" - subtests_pipe_sepr=$(echo "$subtests" | xargs | sed 's/ /|/g') - mkdir -p /tmp/test-results/ - gotestsum -- -timeout=30m -tags integration ./test/integration/connect/envoy -run="TestEnvoy/($subtests_pipe_sepr)" - environment: - GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml - GOTESTSUM_FORMAT: standard-verbose - COMPOSE_INTERACTIVE_NO_CLI: 1 - LAMBDA_TESTS_ENABLED: "true" - # tput complains if this isn't set to something. - TERM: ansi - - store_artifacts: - path: ./test/integration/connect/envoy/workdir/logs - destination: container-logs - - store_test_results: - path: *TEST_RESULTS_DIR - - store_artifacts: - path: *TEST_RESULTS_DIR - - run: *notify-slack-failure - - # run integration tests for the connect ca providers with vault - vault-integration-test: - docker: - - image: *GOLANG_IMAGE - parameters: - vault-version: - type: enum - enum: *supported_vault_versions - default: *default_vault_version - environment: - <<: *ENVIRONMENT - VAULT_BINARY_VERSION: << parameters.vault-version >> - steps: &VAULT_INTEGRATION_TEST_STEPS - - run: - name: Install vault - command: | - wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/${VAULT_BINARY_VERSION}/vault_${VAULT_BINARY_VERSION}_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - vault version - - checkout - - run: go mod download - - run: - name: go test - command: | - mkdir -p $TEST_RESULTS_DIR - make test-connect-ca-providers - - store_test_results: - path: *TEST_RESULTS_DIR - - run: *notify-slack-failure - - # Run load tests against a commit - load-test: - docker: - - image: hashicorp/terraform:latest - environment: - AWS_DEFAULT_REGION: us-east-2 - BUCKET: consul-ci-load-tests - BASH_ENV: /etc/profile - shell: /bin/sh -leo pipefail - steps: - - checkout - - run: apk add jq curl bash - - run: - name: export load-test credentials - command: | - echo "export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_LOAD_TEST" >> $BASH_ENV - echo "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_LOAD_TEST" >> $BASH_ENV - - run: - name: export role arn - command: | - echo "export TF_VAR_role_arn=$ROLE_ARN_LOAD_TEST" >> $BASH_ENV - - run: - name: setup TF_VARs - command: | - # if pipeline.parameters.commit="" it was not triggered/set through the API - # so we use the latest commit from _this_ branch. This is the case for nightly tests. - if [ "<< pipeline.parameters.commit >>" = "" ]; then - LOCAL_COMMIT_SHA=$(git rev-parse HEAD) - else - LOCAL_COMMIT_SHA="<< pipeline.parameters.commit >>" - fi - echo "export LOCAL_COMMIT_SHA=${LOCAL_COMMIT_SHA}" >> $BASH_ENV - git checkout ${LOCAL_COMMIT_SHA} - - short_ref=$(git rev-parse --short ${LOCAL_COMMIT_SHA}) - echo "export TF_VAR_ami_owners=$LOAD_TEST_AMI_OWNERS" >> $BASH_ENV - echo "export TF_VAR_vpc_name=$short_ref" >> $BASH_ENV - echo "export TF_VAR_cluster_name=$short_ref" >> $BASH_ENV - echo "export TF_VAR_consul_download_url=https://${S3_ARTIFACT_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/${S3_ARTIFACT_PATH}/${LOCAL_COMMIT_SHA}.zip" >> $BASH_ENV - - run: - name: wait for dev build from test-integrations workflow - command: | - echo "curl-ing https://${S3_ARTIFACT_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/${S3_ARTIFACT_PATH}/${LOCAL_COMMIT_SHA}.zip" - until [ $SECONDS -ge 300 ] && exit 1; do - curl -o /dev/null --fail --silent "https://${S3_ARTIFACT_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/${S3_ARTIFACT_PATH}/${LOCAL_COMMIT_SHA}.zip" && exit - echo -n "." - sleep 2 - done - - run: - working_directory: .circleci/terraform/load-test - name: terraform init - command: | - short_ref=$(git rev-parse --short HEAD) - echo "Testing commit id: $short_ref" - terraform init \ - -backend-config="bucket=${BUCKET}" \ - -backend-config="key=${LOCAL_COMMIT_SHA}" \ - -backend-config="region=${AWS_DEFAULT_REGION}" \ - -backend-config="role_arn=${ROLE_ARN_LOAD_TEST}" - - run: - working_directory: .circleci/terraform/load-test - name: run terraform apply - command: | - terraform apply -auto-approve - - run: - working_directory: .circleci/terraform/load-test - when: always - name: terraform destroy - command: | - for i in $(seq 1 5); do terraform destroy -auto-approve && s=0 && break || s=$? && sleep 20; done; (exit $s) - - run: *notify-slack-failure - - # The noop job is a used as a very fast job in the verify-ci workflow because every workflow - # requires at least one job. It does nothing. - noop: - docker: - - image: docker.mirror.hashicorp.services/alpine:latest - steps: - - run: "echo ok" - -workflows: - version: 2 - # verify-ci is a no-op workflow that must run on every PR. It is used in a - # branch protection rule to detect when CI workflows are not running. - verify-ci: - jobs: [noop] - - go-tests: - unless: << pipeline.parameters.trigger-load-test >> - jobs: - - check-go-mod: &filter-ignore-non-go-branches - filters: - branches: - ignore: - - stable-website - - /^docs\/.*/ - - /^ui\/.*/ - - /^mktg-.*/ # Digital Team Terraform-generated branches' prefix - - /^backport\/docs\/.*/ - - /^backport\/ui\/.*/ - - /^backport\/mktg-.*/ - - check-generated-protobuf: *filter-ignore-non-go-branches - - check-generated-deep-copy: *filter-ignore-non-go-branches - - lint-enums: *filter-ignore-non-go-branches - - lint-container-test-deps: *filter-ignore-non-go-branches - - lint-consul-retry: *filter-ignore-non-go-branches - - lint: *filter-ignore-non-go-branches - - lint: - name: "lint-32bit" - go-arch: "386" - <<: *filter-ignore-non-go-branches - - go-test-arm64: *filter-ignore-non-go-branches - - dev-build: *filter-ignore-non-go-branches - - go-test: - requires: [dev-build] - - go-test-lib: - name: "go-test-envoyextensions" - path: envoyextensions - go-version: "1.19" - requires: [dev-build] - <<: *filter-ignore-non-go-branches - - go-test-lib: - name: "go-test-troubleshoot" - path: troubleshoot - go-version: "1.19" - requires: [dev-build] - <<: *filter-ignore-non-go-branches - - go-test-lib: - name: "go-test-api go1.18" - path: api - go-version: "1.18" - requires: [dev-build] - - go-test-lib: - name: "go-test-api go1.19" - path: api - go-version: "1.19" - requires: [dev-build] - - go-test-lib: - name: "go-test-sdk go1.18" - path: sdk - go-version: "1.18" - <<: *filter-ignore-non-go-branches - - go-test-lib: - name: "go-test-sdk go1.19" - path: sdk - go-version: "1.19" - <<: *filter-ignore-non-go-branches - - go-test-race: *filter-ignore-non-go-branches - - go-test-32bit: *filter-ignore-non-go-branches - - noop - build-distros: - unless: << pipeline.parameters.trigger-load-test >> - jobs: - - check-go-mod: *filter-ignore-non-go-branches - - build-386: &require-check-go-mod - requires: - - check-go-mod - - build-amd64: *require-check-go-mod - - build-arm: *require-check-go-mod - # every commit on main will have a rebuilt UI - - frontend-cache: - filters: - branches: - only: - - main - - ember-build-prod: - requires: - - frontend-cache - - publish-static-assets: - requires: - - ember-build-prod - - dev-build: - requires: - - ember-build-prod - - dev-upload-s3: - requires: - - dev-build - - dev-upload-docker: - requires: - - dev-build - context: consul-ci - - noop - test-integrations: - unless: << pipeline.parameters.trigger-load-test >> - jobs: - - dev-build: *filter-ignore-non-go-branches - - dev-upload-s3: &dev-upload - requires: - - dev-build - filters: - branches: - ignore: - - /^pull\/.*$/ # only push dev builds from non forks - - main # all main dev uploads will include a UI rebuild in build-distros - - dev-upload-docker: - <<: *dev-upload - context: consul-ci - - nomad-integration-test: - requires: - - dev-build - matrix: - parameters: - nomad-version: *supported_nomad_versions - - vault-integration-test: - matrix: - parameters: - vault-version: *supported_vault_versions - <<: *filter-ignore-non-go-branches - - envoy-integration-test: - requires: - - dev-build - matrix: - parameters: - envoy-version: *supported_envoy_versions - xds-target: ["server", "client"] - - compatibility-integration-test: - requires: - - dev-build - - noop - frontend: - unless: << pipeline.parameters.trigger-load-test >> - jobs: - - frontend-cache: - filters: - branches: - only: - - main - - /^ui\/.*/ - - /^backport\/ui\/.*/ - - workspace-tests: - requires: - - frontend-cache - - node-tests: - requires: - - frontend-cache - - ember-build-oss: - requires: - - frontend-cache - - ember-build-ent: - requires: - - frontend-cache - - ember-test-oss: - requires: - - ember-build-oss - - ember-test-ent: - requires: - - ember-build-ent - # ember-coverage in CI uses the dist/ folder to run tests so it requires - # either/or ent/oss to be built first - - ember-coverage: - requires: - - ember-build-ent - - noop - - load-test: - when: << pipeline.parameters.trigger-load-test >> - jobs: - - load-test - - nightly-jobs: - triggers: - - schedule: - cron: "0 4 * * *" # 4AM UTC <> 12AM EST <> 9PM PST should have no impact - filters: - branches: - only: - - main - jobs: - - load-test diff --git a/.circleci/terraform/load-test/.terraform.lock.hcl b/.circleci/terraform/load-test/.terraform.lock.hcl deleted file mode 100755 index 9f4d960dcd4..00000000000 --- a/.circleci/terraform/load-test/.terraform.lock.hcl +++ /dev/null @@ -1,110 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "3.41.0" - constraints = "~> 3.0, >= 3.27.0" - hashes = [ - "h1:YLbsjPt/oZdEhV+KJzMVBwGDViw14Ih5bYr+EOudIVw=", - "zh:01449ed390710428c92dcd3c6b8ba7e06cc1581b927e96eabe9ebc2653d1e3e0", - "zh:259c1267ab5798e90c8edb4b9c3b17c1dd98e5265c121eaf025a5836e88f4d1d", - "zh:2671ec766eb63d642b8b3d847d67db83d578a44d4945bc45ddd7fbb6d09298ca", - "zh:36082943070c8f4f9a1e557a6b18d279db079b19210cd5249ba03c87de44e5d4", - "zh:49a52c679a14c7755db34e0b98ef062f5e42b7beec1decec2511ecef16690b3f", - "zh:82cf0db34865d8844139a6db35436a6b4664995972dc53e298c93a7133101b0f", - "zh:9082239ae74e4f8b9763087bf454dcfb1019e1a65c4d9ab8057f8425b9da550b", - "zh:a9b51d299b3ffe07684e86d8ea11513411f53375439be5aa87fdfef59cbe5dfa", - "zh:b33fb3990c9bb2a1337725651a98d9563a3b91b50ddeb7c7b655c463faa81dda", - "zh:bd759da1e0c18a2c17bfe607660d52d8981aa51460d70d2e338ddbcef1b50183", - "zh:eebb98f9ba764dd09b059c5865ce7e8bace49fe470980f813a767cbe833a933e", - ] -} - -provider "registry.terraform.io/hashicorp/local" { - version = "2.1.0" - hashes = [ - "h1:KfieWtVyGWwplSoLIB5usKAUnrIkDQBkWaR5TI+4WYg=", - "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", - "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", - "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", - "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", - "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", - "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", - "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", - "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", - "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", - "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", - "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.1.0" - hashes = [ - "h1:xhbHC6in3nQryvTQBWKxebi3inG5OCgHgc4fRxL0ymc=", - "zh:02a1675fd8de126a00460942aaae242e65ca3380b5bb192e8773ef3da9073fd2", - "zh:53e30545ff8926a8e30ad30648991ca8b93b6fa496272cd23b26763c8ee84515", - "zh:5f9200bf708913621d0f6514179d89700e9aa3097c77dac730e8ba6e5901d521", - "zh:9ebf4d9704faba06b3ec7242c773c0fbfe12d62db7d00356d4f55385fc69bfb2", - "zh:a6576c81adc70326e4e1c999c04ad9ca37113a6e925aefab4765e5a5198efa7e", - "zh:a8a42d13346347aff6c63a37cda9b2c6aa5cc384a55b2fe6d6adfa390e609c53", - "zh:c797744d08a5307d50210e0454f91ca4d1c7621c68740441cf4579390452321d", - "zh:cecb6a304046df34c11229f20a80b24b1603960b794d68361a67c5efe58e62b8", - "zh:e1371aa1e502000d9974cfaff5be4cfa02f47b17400005a16f14d2ef30dc2a70", - "zh:fc39cc1fe71234a0b0369d5c5c7f876c71b956d23d7d6f518289737a001ba69b", - "zh:fea4227271ebf7d9e2b61b89ce2328c7262acd9fd190e1fd6d15a591abfa848e", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.1.0" - hashes = [ - "h1:rKYu5ZUbXwrLG1w81k7H3nce/Ys6yAxXhWcbtk36HjY=", - "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", - "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", - "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", - "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", - "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", - "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", - "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", - "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", - "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", - "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", - "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", - ] -} - -provider "registry.terraform.io/hashicorp/template" { - version = "2.2.0" - hashes = [ - "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", - "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", - "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", - "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", - "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", - "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", - "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", - "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", - "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", - "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", - "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", - ] -} - -provider "registry.terraform.io/hashicorp/tls" { - version = "3.1.0" - hashes = [ - "h1:XTU9f6sGMZHOT8r/+LWCz2BZOPH127FBTPjMMEAAu1U=", - "zh:3d46616b41fea215566f4a957b6d3a1aa43f1f75c26776d72a98bdba79439db6", - "zh:623a203817a6dafa86f1b4141b645159e07ec418c82fe40acd4d2a27543cbaa2", - "zh:668217e78b210a6572e7b0ecb4134a6781cc4d738f4f5d09eb756085b082592e", - "zh:95354df03710691773c8f50a32e31fca25f124b7f3d6078265fdf3c4e1384dca", - "zh:9f97ab190380430d57392303e3f36f4f7835c74ea83276baa98d6b9a997c3698", - "zh:a16f0bab665f8d933e95ca055b9c8d5707f1a0dd8c8ecca6c13091f40dc1e99d", - "zh:be274d5008c24dc0d6540c19e22dbb31ee6bfdd0b2cddd4d97f3cd8a8d657841", - "zh:d5faa9dce0a5fc9d26b2463cea5be35f8586ab75030e7fa4d4920cd73ee26989", - "zh:e9b672210b7fb410780e7b429975adcc76dd557738ecc7c890ea18942eb321a5", - "zh:eb1f8368573d2370605d6dbf60f9aaa5b64e55741d96b5fb026dbfe91de67c0d", - "zh:fc1e12b713837b85daf6c3bb703d7795eaf1c5177aebae1afcf811dd7009f4b0", - ] -} diff --git a/.circleci/terraform/load-test/main.tf b/.circleci/terraform/load-test/main.tf deleted file mode 100644 index 1a8865c065d..00000000000 --- a/.circleci/terraform/load-test/main.tf +++ /dev/null @@ -1,26 +0,0 @@ -terraform { - backend "s3" { - } -} - -provider "aws" { - assume_role { - role_arn = var.role_arn - } -} - -module "load-test" { - source = "../../../test/load/terraform" - - vpc_az = ["us-east-2a", "us-east-2b"] - vpc_name = var.vpc_name - vpc_cidr = "10.0.0.0/16" - vpc_allwed_ssh_cidr = "0.0.0.0/0" - public_subnet_cidrs = ["10.0.1.0/24", "10.0.2.0/24"] - private_subnet_cidrs = ["10.0.3.0/24"] - test_public_ip = true - ami_owners = var.ami_owners - consul_download_url = var.consul_download_url - cluster_name = var.cluster_name - cluster_tag_key = var.cluster_tag_key -} diff --git a/.circleci/terraform/load-test/variables.tf b/.circleci/terraform/load-test/variables.tf deleted file mode 100644 index 414cfa84e7f..00000000000 --- a/.circleci/terraform/load-test/variables.tf +++ /dev/null @@ -1,30 +0,0 @@ -variable "vpc_name" { - description = "Name of the VPC" -} - -variable "ami_owners" { - type = list(string) - description = "The account owner number which the desired AMI is in" -} - -variable "role_arn" { - type = string - description = "Role ARN for assume role" -} - -variable "consul_download_url" { - type = string - description = "URL to download the Consul binary from" - default = "" -} -variable "cluster_name" { - description = "What to name the Consul cluster and all of its associated resources" - type = string - default = "consul-example" -} - -variable "cluster_tag_key" { - description = "The tag the EC2 Instances will look for to automatically discover each other and form a cluster." - type = string - default = "consul-ci-load-test" -} diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 31495d06491..d9af3f042a7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,3 +8,34 @@ # release configuration /.release/ @hashicorp/release-engineering @hashicorp/github-consul-core /.github/workflows/build.yml @hashicorp/release-engineering @hashicorp/github-consul-core + + +# Staff Engineer Review (protocol buffer definitions) +/proto-public/ @hashicorp/consul-core-staff +/proto/ @hashicorp/consul-core-staff + +# Staff Engineer Review (v1 architecture shared components) +/agent/cache/ @hashicorp/consul-core-staff +/agent/consul/fsm/ @hashicorp/consul-core-staff +/agent/consul/leader*.go @hashicorp/consul-core-staff +/agent/consul/server*.go @hashicorp/consul-core-staff +/agent/consul/state/ @hashicorp/consul-core-staff +/agent/consul/stream/ @hashicorp/consul-core-staff +/agent/submatview/ @hashicorp/consul-core-staff +/agent/blockingquery/ @hashicorp/consul-core-staff + +# Staff Engineer Review (raft/autopilot) +/agent/consul/autopilotevents/ @hashicorp/consul-core-staff +/agent/consul/autopilot*.go @hashicorp/consul-core-staff + +# Staff Engineer Review (v2 architecture shared components) +/internal/controller/ @hashicorp/consul-core-staff +/internal/resource/ @hashicorp/consul-core-staff +/internal/storage/ @hashicorp/consul-core-staff +/agent/consul/controller/ @hashicorp/consul-core-staff +/agent/grpc-external/services/resource/ @hashicorp/consul-core-staff + +# Staff Engineer Review (v1 security) +/acl/ @hashicorp/consul-core-staff +/agent/xds/rbac*.go @hashicorp/consul-core-staff +/agent/xds/jwt*.go @hashicorp/consul-core-staff diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 131d9057d2c..c1b965235c9 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -67,8 +67,7 @@ issue. Stale issues will be closed. ### Prerequisites If you wish to work on Consul itself, you'll first need to: -- install [Go](https://golang.org) (the version should match that of our - [CI config's](https://github.com/hashicorp/consul/blob/main/.circleci/config.yml) Go image). +- install [Go](https://golang.org) - [fork the Consul repo](../docs/contributing/fork-the-project.md) ### Building Consul diff --git a/.github/pr-labeler.yml b/.github/pr-labeler.yml index 1e371d5f901..759130301bf 100644 --- a/.github/pr-labeler.yml +++ b/.github/pr-labeler.yml @@ -57,7 +57,7 @@ theme/ui: # thinking: # type/bug: type/ci: - - .circleci/* + - .github/workflows/* # type/crash: type/docs: - website/**/* diff --git a/.github/scripts/changelog_checker.sh b/.github/scripts/changelog_checker.sh new file mode 100755 index 00000000000..e46030da1e1 --- /dev/null +++ b/.github/scripts/changelog_checker.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -euo pipefail + +# check if there is a diff in the .changelog directory +# for PRs against the main branch, the changelog file name should match the PR number +if [ "$GITHUB_BASE_REF" = "$GITHUB_DEFAULT_BRANCH" ]; then + enforce_matching_pull_request_number="matching this PR number " + changelog_file_path=".changelog/(_)?$PR_NUMBER.txt" +else + changelog_file_path=".changelog/[_0-9]*.txt" +fi + +changelog_files=$(git --no-pager diff --name-only HEAD "$(git merge-base HEAD "origin/main")" | egrep "${changelog_file_path}") + +# If we do not find a file in .changelog/, we fail the check +if [ -z "$changelog_files" ]; then + # Fail status check when no .changelog entry was found on the PR + echo "Did not find a .changelog entry ${enforce_matching_pull_request_number}and the 'pr/no-changelog' label was not applied. Reference - https://github.com/hashicorp/consul/pull/8387" + exit 1 +else + echo "Found .changelog entry in PR!" +fi diff --git a/.github/scripts/filter_changed_files_go_test.sh b/.github/scripts/filter_changed_files_go_test.sh new file mode 100755 index 00000000000..4db9e7a8f2b --- /dev/null +++ b/.github/scripts/filter_changed_files_go_test.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Get the list of changed files +files_to_check=$(git diff --name-only origin/$GITHUB_BASE_REF) + +# Define the directories to check +skipped_directories=("docs/" "ui/" "website/" "grafana/") + +# Initialize a variable to track directories outside the skipped ones +other_directories="" +trigger_ci=true + +# # Loop through the changed files and find directories/files outside the skipped ones +# for file_to_check in $files_to_check; do +# file_is_skipped=false +# for dir in "${skipped_directories[@]}"; do +# if [[ "$file_to_check" == "$dir"* ]] || [[ "$file_to_check" == *.md && "$dir" == *"/" ]]; then +# file_is_skipped=true +# break +# fi +# done +# if [ "$file_is_skipped" = "false" ]; then +# other_directories+="$(dirname "$file_to_check")\n" +# trigger_ci=true +# echo "Non doc file(s) changed - triggered ci: $trigger_ci" +# echo -e $other_directories +# echo "trigger-ci=$trigger_ci" >>"$GITHUB_OUTPUT" +# exit 0 ## if file is outside of the skipped_directory exit script +# fi +# done + +# echo "Only doc file(s) changed - triggered ci: $trigger_ci" +echo "Doc file(s) change detection is currently disabled - triggering ci" +echo "trigger-ci=$trigger_ci" >>"$GITHUB_OUTPUT" diff --git a/.github/scripts/get_runner_classes.sh b/.github/scripts/get_runner_classes.sh new file mode 100755 index 00000000000..5eb56ff172b --- /dev/null +++ b/.github/scripts/get_runner_classes.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# This script generates tag-sets that can be used as runs-on: values to select runners. + +set -euo pipefail + +case "$GITHUB_REPOSITORY" in +*-enterprise) + # shellcheck disable=SC2129 + echo "compute-small=['self-hosted', 'linux', 'small']" >>"$GITHUB_OUTPUT" + echo "compute-medium=['self-hosted', 'linux', 'medium']" >>"$GITHUB_OUTPUT" + echo "compute-large=['self-hosted', 'linux', 'large']" >>"$GITHUB_OUTPUT" + # m5d.8xlarge is equivalent to our xl custom runner in CE + echo "compute-xl=['self-hosted', 'ondemand', 'linux', 'type=m6a.2xlarge']" >>"$GITHUB_OUTPUT" + ;; +*) + # shellcheck disable=SC2129 + echo "compute-small=['custom-linux-s-consul-latest']" >>"$GITHUB_OUTPUT" + echo "compute-medium=['custom-linux-m-consul-latest']" >>"$GITHUB_OUTPUT" + echo "compute-large=['custom-linux-l-consul-latest']" >>"$GITHUB_OUTPUT" + echo "compute-xl=['custom-linux-xl-consul-latest']" >>"$GITHUB_OUTPUT" + ;; +esac diff --git a/.github/scripts/license_checker.sh b/.github/scripts/license_checker.sh new file mode 100755 index 00000000000..6ba026f04fd --- /dev/null +++ b/.github/scripts/license_checker.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + + +busl_files=$(grep -r 'SPDX-License-Identifier: BUSL' . --exclude-dir .github) + +# If we do not find a file in .changelog/, we fail the check +if [ -n "$busl_files" ]; then + echo "Found BUSL occurrences in the PR branch! (See NET-5258 for details)" + echo -n "$busl_files" + exit 1 +else + echo "Did not find any occurrences of BUSL in the PR branch" + exit 0 +fi diff --git a/.github/scripts/notify_slack.sh b/.github/scripts/notify_slack.sh new file mode 100755 index 00000000000..b3dcdb210dc --- /dev/null +++ b/.github/scripts/notify_slack.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -uo pipefail + +# This script is used in GitHub Actions pipelines to notify Slack of a job failure. + +if [[ $GITHUB_REF_NAME == "main" ]]; then + GITHUB_ENDPOINT="https://github.com/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}" + GITHUB_ACTIONS_ENDPOINT="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + COMMIT_MESSAGE=$(git log -1 --pretty=%B | head -n1) + SHORT_REF=$(git rev-parse --short "${GITHUB_SHA}") + curl -X POST -H 'Content-type: application/json' \ + --data \ + "{ \ + \"attachments\": [ \ + { \ + \"fallback\": \"GitHub Actions workflow failed!\", \ + \"text\": \"❌ Failed: \`${GITHUB_ACTOR}\`'s <${GITHUB_ACTIONS_ENDPOINT}|${GITHUB_JOB}> job failed for commit <${GITHUB_ENDPOINT}|${SHORT_REF}> on \`${GITHUB_REF_NAME}\`!\n\n- <${COMMIT_MESSAGE}\", \ + \"footer\": \"${GITHUB_REPOSITORY}\", \ + \"ts\": \"$(date +%s)\", \ + \"color\": \"danger\" \ + } \ + ] \ + }" "${FEED_CONSUL_GH_URL}" +else + echo "Not posting slack failure notifications for non-main branch" +fi diff --git a/.circleci/scripts/rerun-fails-report.sh b/.github/scripts/rerun_fails_report.sh similarity index 65% rename from .circleci/scripts/rerun-fails-report.sh rename to .github/scripts/rerun_fails_report.sh index 959f0708e68..ac6b7cf2ff9 100755 --- a/.circleci/scripts/rerun-fails-report.sh +++ b/.github/scripts/rerun_fails_report.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + # # Add a comment on the github PR if there were any rerun tests. # @@ -11,11 +14,11 @@ if [ ! -s "$report_filename" ]; then fi function report { - echo ":repeat: gotestsum re-ran some tests in $CIRCLE_BUILD_URL" + echo ":repeat: gotestsum re-ran some tests in https://github.com/hashicorp/consul/actions/run/$GITHUB_RUN_ID" echo echo '```' cat "$report_filename" echo '```' } -report \ No newline at end of file +report diff --git a/.github/scripts/set_test_package_matrix.sh b/.github/scripts/set_test_package_matrix.sh new file mode 100755 index 00000000000..a9a21c747e5 --- /dev/null +++ b/.github/scripts/set_test_package_matrix.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +export RUNNER_COUNT=$1 + +# set matrix var to list of unique packages containing tests +matrix="$(go list -json="ImportPath,TestGoFiles" ./... | jq --compact-output '. | select(.TestGoFiles != null) | .ImportPath' | jq --slurp --compact-output '.' | jq --argjson runnercount $RUNNER_COUNT -cM '[_nwise(length / $runnercount | floor)]'))" + +echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}" diff --git a/.github/scripts/verify_envoy_version.sh b/.github/scripts/verify_envoy_version.sh new file mode 100755 index 00000000000..773431f50bc --- /dev/null +++ b/.github/scripts/verify_envoy_version.sh @@ -0,0 +1,178 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +set -euo pipefail + +current_branch=$GITHUB_REF +GITHUB_DEFAULT_BRANCH='main' + +if [ -z "$GITHUB_TOKEN" ]; then + echo "GITHUB_TOKEN must be set" + exit 1 +fi + +if [ -z "$current_branch" ]; then + echo "GITHUB_REF must be set" + exit 1 +fi + +# Get Consul and Envoy version +SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +pushd $SCRIPT_DIR/../.. # repository root +consul_envoy_data_json=$(echo go run ./test/integration/consul-container/test/consul_envoy_version/consul_envoy_version.go) +# go back to where you started when finished +popd + +if [ -z "$consul_envoy_data_json" ]; then + echo "Error! Consul and Envoy versions not returned: $consul_envoy_data_json" + exit 1 +fi + +# sanitize_consul_envoy_version removes characters from result that may contain new lines, spaces, and [...] +# example envoyVersions:[1.25.4 1.24.6 1.23.8 1.22.11] => 1.25.4 1.24.6 1.23.8 1.22.11 +sanitize_consul_envoy_version() { + local _consul_version=$(eval "$consul_envoy_data_json" | jq -r '.ConsulVersion') + local _envoy_version=$(eval "$consul_envoy_data_json" | jq -r '.EnvoyVersions' | tr -d '"' | tr -d '\n' | tr -d ' '| tr -d '[]') + echo "${_consul_version}" "${_envoy_version}" +} + +# get major version for Consul and Envoy +get_major_version(){ + local _verison="$1" + local _abbrVersion="$(cut -d "." -f1-2 <<< $_verison)" + echo "${_abbrVersion}" +} + +get_latest_envoy_version() { + OUTPUT_FILE=$(mktemp) + HTTP_CODE=$(curl -L --silent --output "$OUTPUT_FILE" -w "%{http_code}" \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}"\ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/envoyproxy/envoy/releases/latest) + if [[ ${HTTP_CODE} -lt 200 || ${HTTP_CODE} -gt 299 ]]; then + cat >&2 "$OUTPUT_FILE" + rm "$OUTPUT_FILE" + exit 1 + fi + _latest_envoy_version=$(jq -r '.tag_name' "$OUTPUT_FILE") + echo "$_latest_envoy_version" + rm "$OUTPUT_FILE" +} + +# major_envoy_versions takes multiple arguments +major_envoy_versions(){ + version=("$@") + for i in "${version[@]}"; + do + envoy_versions_array+="$(cut -d "." -f1-2 <<< $i)" + done + echo "${envoy_versions_array}" +} + +# Get latest Envoy version from envoyproxy repo +released_envoy_version=$(get_latest_envoy_version) +major_released_envoy_version="${released_envoy_version[@]:1:4}" + +validate_envoy_version_main(){ + echo "verify "main" GitHub branch has latest envoy version" + # Get envoy version for current branch + ENVOY_VERSIONS=$(sanitize_consul_envoy_version | awk '{print $2}' | tr ',' ' ') + envoy_version_main_branch=$(get_major_version ${ENVOY_VERSIONS}) + + if [[ "$envoy_version_main_branch" != "$major_released_envoy_version" ]]; then + echo + echo "Latest released Envoy version is: "$released_envoy_version"" + echo "ERROR! Branch $current_branch; Envoy versions: "$ENVOY_VERSIONS" needs to be updated." + exit 1 + else + echo "#### SUCCESS! ##### Compatible Envoy versions found: ${ENVOY_VERSIONS}" + exit 0 + fi +} + +if [[ "$current_branch" == *"$GITHUB_DEFAULT_BRANCH"* ]]; then + validate_envoy_version_main +fi + +# filter consul and envoy version +CONSUL_VERSION=$(sanitize_consul_envoy_version | awk '{print $1}') +ENVOY_VERSIONS=$(sanitize_consul_envoy_version | awk '{print $2}' | tr ',' ' ') + +# Get Consul and Envoy version from default branch +echo checking out "${GITHUB_DEFAULT_BRANCH}" branch +git checkout "${GITHUB_DEFAULT_BRANCH}" + +# filter consul and envoy version from default branch +CONSUL_VERSION_DEFAULT_BRANCH=$(sanitize_consul_envoy_version | awk '{print $1}') +ENVOY_VERSIONS_DEFAULT_BRANCH=$(sanitize_consul_envoy_version | awk '{print $2}' | tr ',' ' ') + +# Ensure required values are not empty +if [ -z "$CONSUL_VERSION" ] || [ -z "$CONSUL_VERSION_DEFAULT_BRANCH" ] || [ -z "$ENVOY_VERSIONS" ] || [ -z "$ENVOY_VERSIONS_DEFAULT_BRANCH" ]; then + echo "Error! Consul version: $CONSUL_VERSION | Consul version default branch: $CONSUL_VERSION_DEFAULT_BRANCH | Envoy version: $ENVOY_VERSIONS | Envoy version default branch: $ENVOY_VERSIONS_DEFAULT_BRANCH cannot be empty" + exit 1 +fi + +echo checking out branch: "${current_branch}" +git checkout "${current_branch}" + +echo +echo "Branch ${current_branch} =>Consul version: ${CONSUL_VERSION}; Envoy Version: ${ENVOY_VERSIONS}" +echo "Branch ${GITHUB_DEFAULT_BRANCH} =>Consul version: ${CONSUL_VERSION_DEFAULT_BRANCH}; Envoy Version: ${ENVOY_VERSIONS_DEFAULT_BRANCH}" + +## Get major Consul and Envoy versions on release and default branch +MAJOR_CONSUL_VERSION=$(get_major_version ${CONSUL_VERSION}) +MAJOR_CONSUL_VERSION_DEFAULT_BRANCH=$(get_major_version ${CONSUL_VERSION_DEFAULT_BRANCH}) +MAJOR_ENVOY_VERSION_DEFAULT_BRANCH=$(get_major_version ${ENVOY_VERSIONS_DEFAULT_BRANCH}) + +_envoy_versions=($ENVOY_VERSIONS) +_envoy_versions_default=($ENVOY_VERSIONS_DEFAULT_BRANCH) + +## Validate supported envoy versions available - should be 4 +echo +echo "Validating supported envoy versions available on branches: $current_branch and $GITHUB_DEFAULT_BRANCH" +if [ "${#_envoy_versions_default[@]}" != 4 ] || [ "${#_envoy_versions[@]}" != 4 ]; then + echo "Branch $GITHUB_DEFAULT_BRANCH =>Consul version: ${CONSUL_VERSION_DEFAULT_BRANCH}; Envoy versions: $ENVOY_VERSIONS_DEFAULT_BRANCH" + echo "Branch $current_branch =>Consul version: ${CONSUL_VERSION}; Envoy versions: $_envoy_versions" + echo "ERROR! Envoy should have 4 compatible versions." + exit 1 +fi + +echo "Checking if branch $GITHUB_DEFAULT_BRANCH has latest Envoy version" +## 1. Check "main" GitHub branch has latest envoy version +if [[ "$MAJOR_ENVOY_VERSION_DEFAULT_BRANCH" != "$major_released_envoy_version" ]]; then + echo + echo "Latest released Envoy version is: "$released_envoy_version"" + echo "ERROR! Branch $GITHUB_DEFAULT_BRANCH; Envoy versions: "$ENVOY_VERSIONS_DEFAULT_BRANCH" needs to be updated." + exit 1 + else + echo "#### SUCCESS! #####. Compatible Envoy versions found: ${ENVOY_VERSIONS_DEFAULT_BRANCH}" + echo + + ## 2. Check main branch and release branch support the same Envoy major versions + ## Get the major Consul version on the main and release branch. If both branches have + ## the same major Consul version, verify both branches have the same major Envoy versions. + ## Return error if major envoy versions are not the same. + echo "Checking branch $current_branch and $GITHUB_DEFAULT_BRANCH have the same compatible major Envoy versions." + consul_version_diff=$(echo "$MAJOR_CONSUL_VERSION_DEFAULT_BRANCH $MAJOR_CONSUL_VERSION" | awk '{print $1 - $2}') + check=$(echo "$consul_version_diff == 0" | bc -l) + + if (( $check )); then + echo "Branch $current_branch and $GITHUB_DEFAULT_BRANCH have the same major Consul version "$MAJOR_CONSUL_VERSION"" + echo "Validating branches have the same Envoy major versions..." + _major_envoy_versions=$(major_envoy_versions $ENVOY_VERSIONS) + _major_envoy_versions_default=$(major_envoy_versions $ENVOY_VERSIONS_DEFAULT_BRANCH) + + if [[ "$_major_envoy_versions_default" != "$_major_envoy_versions" ]]; then + echo "Branch $GITHUB_DEFAULT_BRANCH =>Envoy versions: $_major_envoy_versions" + echo "Branch $current_branch =>Envoy versions: $_major_envoy_versions_default" + echo "ERROR! Branches should support the same major versions for envoy." + exit 1 + else + echo "#### SUCCESS! #####. Compatible Envoy major versions found: $ENVOY_VERSIONS_DEFAULT_BRANCH" + fi + else + echo "No validation needed. Branches have different Consul versions" + fi +fi \ No newline at end of file diff --git a/.github/workflows/backport-assistant.yml b/.github/workflows/backport-assistant.yml index 7eac100546c..d842f1b1673 100644 --- a/.github/workflows/backport-assistant.yml +++ b/.github/workflows/backport-assistant.yml @@ -16,7 +16,7 @@ jobs: backport: if: github.event.pull_request.merged runs-on: ubuntu-latest - container: hashicorpdev/backport-assistant:0.3.0 + container: hashicorpdev/backport-assistant:0.3.4 steps: - name: Run Backport Assistant for stable-website run: | @@ -52,3 +52,16 @@ jobs: BACKPORT_LABEL_REGEXP: "backport/(?P\\d+\\.\\d+)" BACKPORT_TARGET_TEMPLATE: "release/{{.target}}.x" GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + handle-failure: + needs: + - backport + if: always() && needs.backport.result == 'failure' + runs-on: ubuntu-latest + steps: + - name: Comment on PR + run: | + github_message="Backport failed @${{ github.event.sender.login }}. Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + curl -s -H "Authorization: token ${{ secrets.PR_COMMENT_TOKEN }}" \ + -X POST \ + -d "{ \"body\": \"${github_message}\"}" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${{ github.event.pull_request.number }}/comments" diff --git a/.github/workflows/backport-checker.yml b/.github/workflows/backport-checker.yml new file mode 100644 index 00000000000..c9af23ea971 --- /dev/null +++ b/.github/workflows/backport-checker.yml @@ -0,0 +1,32 @@ +# This workflow checks that there is either a 'pr/no-backport' label applied to a PR +# or there is a backport/* label indicating a backport has been set + +name: Backport Checker + +on: + pull_request: + types: [opened, synchronize, labeled] + # Runs on PRs to main and all release branches + branches: + - main + - release/* + +jobs: + # checks that a backport label is present for a PR + backport-check: + # If there's a `pr/no-backport` label we ignore this check. Also, we ignore PRs created by the bot assigned to `backport-assistant` + if: "! ( contains(github.event.pull_request.labels.*.name, 'pr/no-backport') || github.event.pull_request.user.login == 'hc-github-team-consul-core' )" + runs-on: ubuntu-latest + + steps: + - name: Check for Backport Label + run: | + labels="${{join(github.event.pull_request.labels.*.name, ', ') }}" + if [[ "$labels" =~ .*"backport/".* ]]; then + echo "Found backport label!" + exit 0 + fi + # Fail status check when no backport label was found on the PR + echo "Did not find a backport label matching the pattern 'backport/*' and the 'pr/no-backport' label was not applied. Reference - https://github.com/hashicorp/consul/pull/16567" + exit 1 + diff --git a/.github/workflows/backport-reminder.yml b/.github/workflows/backport-reminder.yml deleted file mode 100644 index 359451269d7..00000000000 --- a/.github/workflows/backport-reminder.yml +++ /dev/null @@ -1,26 +0,0 @@ -# This workflow sends a reminder comment to PRs that have labels starting with -# `backport/` to check that the backport has run successfully. - -name: Backport Assistant Reminder - -on: - pull_request: - types: [ labeled ] - # Runs on PRs to main and all release branches - branches: - - main - - release/* - -jobs: - backport-label-check: - if: "startsWith(github.event.label.name, 'backport/')" - runs-on: ubuntu-latest - - steps: - - name: Comment on PR - run: | - github_message="After merging, confirm that you see linked PRs AND check them for CI errors." - curl -s -H "Authorization: token ${{ secrets.PR_COMMENT_TOKEN }}" \ - -X POST \ - -d "{ \"body\": \"${github_message}\"}" \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${{ github.event.pull_request.number }}/comments" diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 00000000000..f57ea3527d4 --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,134 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# This workflow builds a dev binary and distributes a Docker image on every push to the main branch. +name: build-artifacts + +on: + push: + branches: + - main + +permissions: + contents: read + +env: + GOPRIVATE: github.com/hashicorp + +jobs: + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + compute-small: ${{ steps.setup-outputs.outputs.compute-small }} + compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }} + compute-large: ${{ steps.setup-outputs.outputs.compute-large }} + compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - id: setup-outputs + name: Setup outputs + run: ./.github/scripts/get_runner_classes.sh + + dev-build-push: + needs: setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + steps: + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/dockerhub username | DOCKERHUB_USERNAME; + kv/data/github/${{ github.repository }}/dockerhub token | DOCKERHUB_TOKEN; + + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + # NOTE: ENT specific step as we need to set elevated GitHub permissions. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + + - run: go env + + - name: Build dev binary + run: make dev + + - name: Set env vars + run: | + echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + echo "GITHUB_BUILD_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # pin@v2.4.1 + + # NOTE: conditional specific logic as we store secrets in Vault in ENT and use GHA secrets in CE. + - name: Login to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # pin@v2.1.0 + with: + username: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }} + password: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_TOKEN || secrets.DOCKERHUB_TOKEN }} + + - name: Docker build and push + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # pin@v4.0.0 + with: + context: ./bin + file: ./build-support/docker/Consul-Dev.dockerfile + labels: COMMIT_SHA=${{ github.sha }},GITHUB_BUILD_URL=${{ env.GITHUB_BUILD_URL }} + push: true + # This is required or else the image is not pullable. + # See https://github.com/docker/build-push-action/issues/820 for further + # details. + # TODO - investigate further and see if we can find a solution where we + # we don't have to know to set this. + provenance: false + tags: | + hashicorpdev/${{ github.event.repository.name }}:${{ env.SHORT_SHA }} + hashicorpdev/${{ github.event.repository.name }}:latest + + # This is job is required for branch protection as a required gihub check + # because GitHub actions show up as checks at the job level and not the + # workflow level. This is currently a feature request: + # https://github.com/orgs/community/discussions/12395 + # + # This job must: + # - be placed after the fanout of a workflow so that everything fans back in + # to this job. + # - "need" any job that is part of the fan out / fan in + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make + # sure it does not get skipped. + + build-artifacts-success: + needs: + - setup + - dev-build-push + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: ${{ always() }} + steps: + - name: evaluate upstream job results + run: | + # exit 1 if failure or cancelled result for any upstream job + if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then + printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}" + exit 1 + fi diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml new file mode 100644 index 00000000000..10c52089334 --- /dev/null +++ b/.github/workflows/build-distros.yml @@ -0,0 +1,170 @@ +# NOTE: this workflow builds Consul binaries on multiple architectures for PRs. +# It is aimed at checking new commits don't introduce any breaking build changes. +name: build-distros + +on: + pull_request: + push: + branches: + # Push events on the main branch + - main + - release/** + +permissions: + contents: read + +env: + GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }} + GOPRIVATE: github.com/hashicorp # Required for enterprise deps + +jobs: + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + compute-small: ${{ steps.setup-outputs.outputs.compute-small }} + compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }} + compute-large: ${{ steps.setup-outputs.outputs.compute-large }} + compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + - id: setup-outputs + name: Setup outputs + run: ./.github/scripts/get_runner_classes.sh + + check-go-mod: + needs: + - setup + uses: ./.github/workflows/reusable-check-go-mod.yml + with: + runs-on: ${{ needs.setup.outputs.compute-medium }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + build-386: + needs: + - setup + - check-go-mod + env: + XC_OS: "freebsd linux windows" + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - name: Build + run: | + for os in $XC_OS; do + GOOS="$os" GOARCH=386 CGO_ENABLED=0 go build -tags "${{ env.GOTAGS }}" + done + + build-amd64: + needs: + - setup + - check-go-mod + env: + XC_OS: "darwin freebsd linux solaris windows" + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - name: Build + run: | + for os in $XC_OS; do + GOOS="$os" GOARCH=amd64 CGO_ENABLED=0 go build -tags "${{ env.GOTAGS }}" + done + + build-arm: + needs: + - setup + - check-go-mod + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + env: + CGO_ENABLED: 1 + GOOS: linux + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - run: | + sudo apt-get update --allow-releaseinfo-change-suite --allow-releaseinfo-change-version && sudo apt-get install -y gcc-arm-linux-gnueabi gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu + + - run: CC=arm-linux-gnueabi-gcc GOARCH=arm GOARM=5 go build -tags "${{ env.GOTAGS }}" + - run: CC=arm-linux-gnueabihf-gcc GOARCH=arm GOARM=6 go build -tags "${{ env.GOTAGS }}" + - run: CC=aarch64-linux-gnu-gcc GOARCH=arm64 go build -tags "${{ env.GOTAGS }}" + + + build-s390x: + if: ${{ endsWith(github.repository, '-enterprise') }} + needs: + - setup + - check-go-mod + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - name: Build + run: GOOS=linux GOARCH=s390x CGO_ENABLED=0 go build -tags "${{ env.GOTAGS }}" + + # This is job is required for branch protection as a required gihub check + # because GitHub actions show up as checks at the job level and not the + # workflow level. This is currently a feature request: + # https://github.com/orgs/community/discussions/12395 + # + # This job must: + # - be placed after the fanout of a workflow so that everything fans back in + # to this job. + # - "need" any job that is part of the fan out / fan in + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make + # sure it does not get skipped. + build-distros-success: + needs: + - setup + - check-go-mod + - build-386 + - build-amd64 + - build-arm + - build-s390x + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: ${{ always() }} + steps: + - name: evaluate upstream job results + run: | + # exit 1 if failure or cancelled result for any upstream job + if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then + printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}" + exit 1 + fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20f316c12e3..48b73391738 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,3 +1,6 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + name: build on: @@ -10,7 +13,9 @@ on: env: PKG_NAME: consul + # TODO(spatel): CE refactor METADATA: oss + GOPRIVATE: github.com/hashicorp # Required for enterprise deps jobs: set-product-version: @@ -22,7 +27,7 @@ jobs: pre-version: ${{ steps.set-product-version.outputs.prerelease-product-version }} shared-ldflags: ${{ steps.shared-ldflags.outputs.shared-ldflags }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: set product version id: set-product-version uses: hashicorp/actions-set-product-version@v1 @@ -31,16 +36,16 @@ jobs: run: | CONSUL_DATE=$(build-support/scripts/build-date.sh) ## TODO: This assumes `make version` outputs 1.1.1+ent-prerel - echo "::set-output name=product-date::${CONSUL_DATE}" + echo "product-date=${CONSUL_DATE}" >> "$GITHUB_OUTPUT" - name: Set shared -ldflags id: shared-ldflags run: | T="github.com/hashicorp/consul/version" - echo "::set-output name=shared-ldflags::-X ${T}.GitCommit=${GITHUB_SHA::8} \ + echo "shared-ldflags=-X ${T}.GitCommit=${GITHUB_SHA::8} \ -X ${T}.GitDescribe=${{ steps.set-product-version.outputs.product-version }} \ -X ${T}.BuildDate=${{ steps.get-product-version.outputs.product-date }} \ - " + " >> "$GITHUB_OUTPUT" validate-outputs: needs: set-product-version runs-on: ubuntu-latest @@ -53,6 +58,7 @@ jobs: echo "Product Date: ${{ needs.set-product-version.outputs.product-date }}" echo "Prerelease Version: ${{ needs.set-product-version.outputs.pre-version }}" echo "Ldflags: ${{ needs.set-product-version.outputs.shared-ldflags }}" + generate-metadata-file: needs: set-product-version runs-on: ubuntu-latest @@ -60,7 +66,7 @@ jobs: filepath: ${{ steps.generate-metadata-file.outputs.filepath }} steps: - name: 'Checkout directory' - uses: actions/checkout@v3 + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Generate metadata file id: generate-metadata-file uses: hashicorp/actions-generate-metadata@v1 @@ -68,7 +74,7 @@ jobs: version: ${{ needs.set-product-version.outputs.product-version }} product: ${{ env.PKG_NAME }} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: name: metadata.json path: ${{ steps.generate-metadata-file.outputs.filepath }} @@ -79,23 +85,23 @@ jobs: strategy: matrix: include: - - {go: "1.19.4", goos: "linux", goarch: "386"} - - {go: "1.19.4", goos: "linux", goarch: "amd64"} - - {go: "1.19.4", goos: "linux", goarch: "arm"} - - {go: "1.19.4", goos: "linux", goarch: "arm64"} - - {go: "1.19.4", goos: "freebsd", goarch: "386"} - - {go: "1.19.4", goos: "freebsd", goarch: "amd64"} - - {go: "1.19.4", goos: "windows", goarch: "386"} - - {go: "1.19.4", goos: "windows", goarch: "amd64"} - - {go: "1.19.4", goos: "solaris", goarch: "amd64"} + - {go: "1.20.8", goos: "linux", goarch: "386"} + - {go: "1.20.8", goos: "linux", goarch: "amd64"} + - {go: "1.20.8", goos: "linux", goarch: "arm"} + - {go: "1.20.8", goos: "linux", goarch: "arm64"} + - {go: "1.20.8", goos: "freebsd", goarch: "386"} + - {go: "1.20.8", goos: "freebsd", goarch: "amd64"} + - {go: "1.20.8", goos: "windows", goarch: "386"} + - {go: "1.20.8", goos: "windows", goarch: "amd64"} + - {go: "1.20.8", goos: "solaris", goarch: "amd64"} fail-fast: true name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Setup with node and yarn - uses: actions/setup-node@v3 + uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 with: node-version: '14' cache: 'yarn' @@ -157,18 +163,69 @@ jobs: echo "RPM_PACKAGE=$(basename out/*.rpm)" >> $GITHUB_ENV echo "DEB_PACKAGE=$(basename out/*.deb)" >> $GITHUB_ENV - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 if: ${{ matrix.goos == 'linux' }} with: name: ${{ env.RPM_PACKAGE }} path: out/${{ env.RPM_PACKAGE }} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 if: ${{ matrix.goos == 'linux' }} with: name: ${{ env.DEB_PACKAGE }} path: out/${{ env.DEB_PACKAGE }} + build-s390x: + needs: set-product-version + if: ${{ endsWith(github.repository, '-enterprise') }} + runs-on: ubuntu-latest + strategy: + matrix: + include: + - {go: "1.20.8", goos: "linux", goarch: "s390x"} + fail-fast: true + + name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + + - name: Setup with node and yarn + uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 + with: + node-version: '14' + cache: 'yarn' + cache-dependency-path: 'ui/yarn.lock' + + - name: Build UI + run: | + CONSUL_VERSION=${{ needs.set-product-version.outputs.product-version }} + CONSUL_DATE=${{ needs.set-product-version.outputs.product-date }} + CONSUL_BINARY_TYPE=${CONSUL_BINARY_TYPE} + CONSUL_COPYRIGHT_YEAR=$(git show -s --format=%cd --date=format:%Y HEAD) + echo "consul_version is ${CONSUL_VERSION}" + echo "consul_date is ${CONSUL_DATE}" + echo "consul binary type is ${CONSUL_BINARY_TYPE}" + echo "consul copyright year is ${CONSUL_COPYRIGHT_YEAR}" + cd ui && make && cd .. + rm -rf agent/uiserver/dist + mv ui/packages/consul-ui/dist agent/uiserver/ + - name: Go Build + env: + PRODUCT_VERSION: ${{ needs.set-product-version.outputs.product-version }} + PRERELEASE_VERSION: ${{ needs.set-product-version.outputs.pre-version }} + CGO_ENABLED: "0" + GOLDFLAGS: "${{needs.set-product-version.outputs.shared-ldflags}}" + uses: hashicorp/actions-go-build@v0.1.7 + with: + product_name: ${{ env.PKG_NAME }} + product_version: ${{ needs.set-product-version.outputs.product-version }} + go_version: ${{ matrix.go }} + os: ${{ matrix.goos }} + arch: ${{ matrix.goarch }} + reproducible: report + instructions: |- + go build -ldflags="$GOLDFLAGS" -o "$BIN_PATH" -trimpath -buildvcs=false + build-darwin: needs: set-product-version runs-on: macos-latest @@ -176,15 +233,15 @@ jobs: matrix: goos: [ darwin ] goarch: [ "amd64", "arm64" ] - go: [ "1.19.4" ] + go: [ "1.20.8" ] fail-fast: true name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build steps: - - uses: actions/checkout@v3 - + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + - name: Setup with node and yarn - uses: actions/setup-node@v3 + uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 with: node-version: '14' cache: 'yarn' @@ -232,7 +289,7 @@ jobs: version: ${{needs.set-product-version.outputs.product-version}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 # Strip everything but MAJOR.MINOR from the version string and add a `-dev` suffix # This naming convention will be used ONLY for per-commit dev images @@ -266,7 +323,7 @@ jobs: version: ${{needs.set-product-version.outputs.product-version}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - uses: hashicorp/actions-docker-build@v1 with: version: ${{env.version}} @@ -286,7 +343,7 @@ jobs: version: ${{needs.set-product-version.outputs.product-version}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 # Strip everything but MAJOR.MINOR from the version string and add a `-dev` suffix # This naming convention will be used ONLY for per-commit dev images @@ -323,21 +380,24 @@ jobs: name: Verify ${{ matrix.arch }} linux binary steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + if: ${{ endsWith(github.repository, '-enterprise') || matrix.arch != 's390x' }} - name: Download ${{ matrix.arch }} zip - uses: actions/download-artifact@v3 + if: ${{ endsWith(github.repository, '-enterprise') || matrix.arch != 's390x' }} + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: ${{ env.zip_name }} - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 if: ${{ matrix.arch == 'arm' || matrix.arch == 'arm64' }} with: # this should be a comma-separated string as opposed to an array platforms: arm,arm64 - name: Run verification for ${{ matrix.arch }} binary + if: ${{ endsWith(github.repository, '-enterprise') || matrix.arch != 's390x' }} run: .github/scripts/verify_artifact.sh ${{ env.zip_name }} v${{ env.version }} verify-darwin: @@ -353,10 +413,10 @@ jobs: name: Verify amd64 darwin binary steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Download amd64 darwin zip - uses: actions/download-artifact@v3 + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: ${{ env.zip_name }} @@ -380,7 +440,7 @@ jobs: name: Verify ${{ matrix.arch }} debian package steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Set package version run: | @@ -391,12 +451,12 @@ jobs: echo "pkg_name=consul_${{ env.pkg_version }}-1_${{ matrix.arch }}.deb" >> $GITHUB_ENV - name: Download workflow artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: ${{ env.pkg_name }} - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 with: platforms: all @@ -417,7 +477,7 @@ jobs: name: Verify ${{ matrix.arch }} rpm steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Set package version run: | @@ -428,12 +488,12 @@ jobs: echo "pkg_name=consul-${{ env.pkg_version }}-1.${{ matrix.arch }}.rpm" >> $GITHUB_ENV - name: Download workflow artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: ${{ env.pkg_name }} - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 with: platforms: all diff --git a/.github/workflows/oss-merge-trigger.yml b/.github/workflows/ce-merge-trigger.yml similarity index 83% rename from .github/workflows/oss-merge-trigger.yml rename to .github/workflows/ce-merge-trigger.yml index 4d08442d90f..3a62146631a 100644 --- a/.github/workflows/oss-merge-trigger.yml +++ b/.github/workflows/ce-merge-trigger.yml @@ -1,4 +1,4 @@ -name: Trigger OSS to Enterprise Merge +name: Trigger Community Edition to Enterprise Merge on: pull_request_target: types: @@ -8,8 +8,8 @@ on: - 'release/*.*.x' jobs: - trigger-oss-merge: - # run this only on merge events in OSS repo + trigger-ce-merge: + # run this only on merge events in CE repo if: ${{ github.event.pull_request.merged && github.repository == 'hashicorp/consul' }} runs-on: ubuntu-latest steps: @@ -19,6 +19,7 @@ jobs: GIT_SHA: ${{ github.sha }} GH_PAT: ${{ secrets.ELEVATED_GITHUB_TOKEN }} GIT_ACTOR: ${{ github.actor }} + # TODO(spatel): CE refactor run: | curl -H "Authorization: token $GH_PAT" \ -H 'Accept: application/json' \ diff --git a/.github/workflows/changelog-checker.yml b/.github/workflows/changelog-checker.yml index bb13255d297..c81eb8a7a04 100644 --- a/.github/workflows/changelog-checker.yml +++ b/.github/workflows/changelog-checker.yml @@ -15,7 +15,7 @@ jobs: # checks that a .changelog entry is present for a PR changelog-check: # If there a `pr/no-changelog` label we ignore this check. Also, we ignore PRs created by the bot assigned to `backport-assistant` - if: "! ( contains(github.event.pull_request.labels.*.name, 'pr/no-changelog') || github.event.pull_request.user.login == 'hc-github-team-consul-core' )" + if: "! ( contains(github.event.pull_request.labels.*.name, 'pr/no-changelog') || github.event.pull_request.user.login == 'hc-github-team-consul-core' )" runs-on: ubuntu-latest steps: @@ -24,23 +24,8 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 # by default the checkout action doesn't checkout all branches - name: Check for changelog entry in diff - run: | - # check if there is a diff in the .changelog directory - # for PRs against the main branch, the changelog file name should match the PR number - if [ "${{ github.event.pull_request.base.ref }}" = "${{ github.event.repository.default_branch }}" ]; then - enforce_matching_pull_request_number="matching this PR number " - changelog_file_path=".changelog/(_)?${{ github.event.pull_request.number }}.txt" - else - changelog_file_path=".changelog/[_0-9]*.txt" - fi - - changelog_files=$(git --no-pager diff --name-only HEAD "$(git merge-base HEAD "origin/main")" | egrep ${changelog_file_path}) - - # If we do not find a file in .changelog/, we fail the check - if [ -z "$changelog_files" ]; then - # Fail status check when no .changelog entry was found on the PR - echo "Did not find a .changelog entry ${enforce_matching_pull_request_number}and the 'pr/no-changelog' label was not applied. Reference - https://github.com/hashicorp/consul/pull/8387" - exit 1 - else - echo "Found .changelog entry in PR!" - fi + run: ./.github/scripts/changelog_checker.sh + env: + GITHUB_BASE_REF: ${{ github.event.pull_request.base.ref }} + GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + PR_NUMBER: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/check-legacy-links-format.yml b/.github/workflows/check-legacy-links-format.yml deleted file mode 100644 index f067bd231a4..00000000000 --- a/.github/workflows/check-legacy-links-format.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Legacy Link Format Checker - -on: - push: - paths: - - "website/content/**/*.mdx" - - "website/data/*-nav-data.json" - -jobs: - check-links: - uses: hashicorp/dev-portal/.github/workflows/docs-content-check-legacy-links-format.yml@475289345d312552b745224b46895f51cc5fc490 - with: - repo-owner: "hashicorp" - repo-name: "consul" - commit-sha: ${{ github.sha }} - mdx-directory: "website/content" - nav-data-directory: "website/data" diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml new file mode 100644 index 00000000000..defd0b22e38 --- /dev/null +++ b/.github/workflows/frontend.yml @@ -0,0 +1,144 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +name: frontend + +on: + push: + branches: + - main + - ui/** + - backport/ui/** + +permissions: + contents: read + +jobs: + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + compute-small: ${{ steps.setup-outputs.outputs.compute-small }} + compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }} + compute-large: ${{ steps.setup-outputs.outputs.compute-large }} + compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - id: setup-outputs + name: Setup outputs + run: ./.github/scripts/get_runner_classes.sh + + workspace-tests: + needs: setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + defaults: + run: + working-directory: ui + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # pin@v3.6.0 + with: + node-version: '16' + + - name: Install Yarn + run: npm install -g yarn + + # Install dependencies. + - name: install yarn packages + working-directory: ui + run: make deps + + - run: make test-workspace + + node-tests: + needs: setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # pin@v3.6.0 + with: + node-version: '16' + + - name: Install Yarn + run: npm install -g yarn + + # Install dependencies. + - name: install yarn packages + working-directory: ui + run: make deps + + - run: make test-node + working-directory: ui/packages/consul-ui + + ember-build-test: + needs: setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + strategy: + matrix: + partition: [1, 2, 3, 4] + env: + EMBER_TEST_REPORT: test-results/report-ce.xml # outputs test report for CI test summary + EMBER_TEST_PARALLEL: true # enables test parallelization with ember-exam + CONSUL_NSPACES_ENABLED: ${{ endsWith(github.repository, '-enterprise') && 1 || 0 }} # NOTE: this should be 1 in ENT. + JOBS: 2 # limit parallelism for broccoli-babel-transpiler + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # pin@v3.6.0 + with: + node-version: '16' + + - name: Install Yarn + run: npm install -g yarn + + - name: Install Chrome + uses: browser-actions/setup-chrome@29abc1a83d1d71557708563b4bc962d0f983a376 # pin@v1.2.1 + + - name: Install dependencies + working-directory: ui + run: make deps + + - name: Build CI + working-directory: ui/packages/consul-ui + run: make build-ci + + - name: Ember exam + working-directory: ui/packages/consul-ui + run: node_modules/.bin/ember exam --split=4 --partition=${{ matrix.partition }} --path dist --silent -r xunit + + - name: Test Coverage CI + working-directory: ui/packages/consul-ui + run: make test-coverage-ci + + # This is job is required for branch protection as a required gihub check + # because GitHub actions show up as checks at the job level and not the + # workflow level. This is currently a feature request: + # https://github.com/orgs/community/discussions/12395 + # + # This job must: + # - be placed after the fanout of a workflow so that everything fans back in + # to this job. + # - "need" any job that is part of the fan out / fan in + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make + # sure it does not get skipped. + + frontend-success: + needs: + - setup + - workspace-tests + - node-tests + - ember-build-test + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: ${{ always() }} + steps: + - name: evaluate upstream job results + run: | + # exit 1 if failure or cancelled result for any upstream job + if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then + printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}" + exit 1 + fi diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml new file mode 100644 index 00000000000..3fb2d66661f --- /dev/null +++ b/.github/workflows/go-tests.yml @@ -0,0 +1,517 @@ +name: go-tests + +on: + pull_request: + branches-ignore: + - stable-website + - 'docs/**' + - 'ui/**' + - 'mktg-**' # Digital Team Terraform-generated branches' prefix + - 'backport/docs/**' + - 'backport/ui/**' + - 'backport/mktg-**' + push: + branches: + # Push events on the main branch + - main + - release/** + +permissions: + contents: read + +env: + TEST_RESULTS: /tmp/test-results + GOPRIVATE: github.com/hashicorp # Required for enterprise deps + +# concurrency +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + conditional-skip: + runs-on: ubuntu-latest + name: Get files changed and conditionally skip CI + outputs: + trigger-ci: ${{ steps.read-files.outputs.trigger-ci }} + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + fetch-depth: 0 + - name: Get changed files + id: read-files + run: ./.github/scripts/filter_changed_files_go_test.sh + + setup: + needs: [conditional-skip] + name: Setup + if: needs.conditional-skip.outputs.trigger-ci == 'true' + runs-on: ubuntu-latest + outputs: + compute-small: ${{ steps.setup-outputs.outputs.compute-small }} + compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }} + compute-large: ${{ steps.setup-outputs.outputs.compute-large }} + compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + - id: setup-outputs + name: Setup outputs + run: ./.github/scripts/get_runner_classes.sh + + check-go-mod: + needs: + - setup + uses: ./.github/workflows/reusable-check-go-mod.yml + with: + runs-on: ${{ needs.setup.outputs.compute-small }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + check-generated-protobuf: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - run: make proto-tools + name: Install protobuf + - run: make proto-format + name: "Protobuf Format" + - run: make --always-make proto + - run: | + if ! git diff --exit-code; then + echo "Generated code was not updated correctly" + exit 1 + fi + - run: make proto-lint + name: "Protobuf Lint" + - name: Notify Slack + if: ${{ failure() }} + run: .github/scripts/notify_slack.sh + check-generated-deep-copy: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - run: make --always-make deep-copy + - run: | + if ! git diff --exit-code; then + echo "Generated code was not updated correctly" + exit 1 + fi + - name: Notify Slack + if: ${{ failure() }} + run: .github/scripts/notify_slack.sh + + lint-enums: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - run: go install github.com/reillywatson/enumcover/cmd/enumcover@master && enumcover ./... + - name: Notify Slack + if: ${{ failure() }} + run: .github/scripts/notify_slack.sh + + lint-container-test-deps: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - run: make lint-container-test-deps + - name: Notify Slack + if: ${{ failure() }} + run: .github/scripts/notify_slack.sh + + lint-consul-retry: + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - run: go install github.com/hashicorp/lint-consul-retry@master && lint-consul-retry + - name: Notify Slack + if: ${{ failure() }} + run: .github/scripts/notify_slack.sh + + lint: + needs: + - setup + uses: ./.github/workflows/reusable-lint.yml + with: + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + lint-32bit: + needs: + - setup + uses: ./.github/workflows/reusable-lint.yml + with: + go-arch: "386" + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + # create a development build + dev-build: + needs: + - setup + uses: ./.github/workflows/reusable-dev-build.yml + with: + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + # dev-build-s390x: + # if: ${{ endsWith(github.repository, '-enterprise') }} + # needs: + # - setup + # uses: ./.github/workflows/reusable-dev-build.yml + # with: + # uploaded-binary-name: 'consul-bin-s390x' + # runs-on: ${{ needs.setup.outputs.compute-large }} + # go-arch: "s390x" + # repository-name: ${{ github.repository }} + # secrets: + # elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + # dev-build-arm64: + # # only run on enterprise because GHA does not have arm64 runners in CE + # if: ${{ endsWith(github.repository, '-enterprise') }} + # needs: + # - setup + # uses: ./.github/workflows/reusable-dev-build.yml + # with: + # uploaded-binary-name: 'consul-bin-arm64' + # runs-on: ${{ needs.setup.outputs.compute-large }} + # go-arch: "arm64" + # repository-name: ${{ github.repository }} + # secrets: + # elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + # go-test-arm64: + # # only run on enterprise because GHA does not have arm64 runners in CE + # if: ${{ endsWith(github.repository, '-enterprise') }} + # needs: + # - setup + # - dev-build-arm64 + # uses: ./.github/workflows/reusable-unit-split.yml + # with: + # directory: . + # uploaded-binary-name: 'consul-bin-arm64' + # runner-count: 12 + # runs-on: "['self-hosted', 'ondemand', 'os=macos-arm', 'arm64']" + # go-test-flags: 'if ! [[ "$GITHUB_REF_NAME" =~ ^main$|^release/ ]]; then export GO_TEST_FLAGS="-short"; fi' + # repository-name: ${{ github.repository }} + # secrets: + # elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + # consul-license: ${{secrets.CONSUL_LICENSE}} + # datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + go-test-ce: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit-split.yml + with: + directory: . + runner-count: 12 + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + go-tags: "" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + go-test-enterprise: + if: ${{ endsWith(github.repository, '-enterprise') }} + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit-split.yml + with: + directory: . + runner-count: 12 + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + go-test-race: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: . + go-test-flags: 'GO_TEST_FLAGS="-race -gcflags=all=-d=checkptr=0"' + package-names-command: "go list ./... | grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | grep -E -v '^github.com/hashicorp/consul(/command|/connect|/snapshot)'" + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + go-test-32bit: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: . + go-arch: "386" + go-test-flags: 'export GO_TEST_FLAGS="-short"' + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + # go-test-s390x: + # if: ${{ endsWith(github.repository, '-enterprise') }} + # needs: + # - setup + # - dev-build-s390x + # uses: ./.github/workflows/reusable-unit.yml + # with: + # uploaded-binary-name: 'consul-bin-s390x' + # directory: . + # go-test-flags: 'export GO_TEST_FLAGS="-short"' + # runs-on: ${{ needs.setup.outputs.compute-large }} + # repository-name: ${{ github.repository }} + # go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + # permissions: + # id-token: write # NOTE: this permission is explicitly required for Vault auth. + # contents: read + # secrets: + # elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + # consul-license: ${{secrets.CONSUL_LICENSE}} + # datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + go-test-envoyextensions: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: envoyextensions + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + go-test-troubleshoot: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: troubleshoot + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + go-test-api-1-19: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: api + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + go-version: "1.19" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + go-test-api-1-20: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: api + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + go-version: "1.20" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + go-test-sdk-1-19: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: sdk + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + go-version: "1.19" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + go-test-sdk-1-20: + needs: + - setup + - dev-build + uses: ./.github/workflows/reusable-unit.yml + with: + directory: sdk + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + go-version: "1.20" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + noop: + runs-on: ubuntu-latest + steps: + - run: "echo ok" + + # This is job is required for branch protection as a required gihub check + # because GitHub actions show up as checks at the job level and not the + # workflow level. This is currently a feature request: + # https://github.com/orgs/community/discussions/12395 + # + # This job must: + # - be placed after the fanout of a workflow so that everything fans back in + # to this job. + # - "need" any job that is part of the fan out / fan in + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make + # sure it does not get skipped. + + go-tests-success: + needs: + - conditional-skip + - setup + - check-generated-deep-copy + - check-generated-protobuf + - check-go-mod + - lint-consul-retry + - lint-container-test-deps + - lint-enums + - lint + - lint-32bit + # - go-test-arm64 + - go-test-enterprise + - go-test-ce + - go-test-race + - go-test-envoyextensions + - go-test-troubleshoot + - go-test-api-1-19 + - go-test-api-1-20 + - go-test-sdk-1-19 + - go-test-sdk-1-20 + - go-test-32bit + # - go-test-s390x + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: always() && needs.conditional-skip.outputs.trigger-ci == 'true' + steps: + - name: evaluate upstream job results + run: | + # exit 1 if failure or cancelled result for any upstream job + if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then + printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}" + exit 1 + fi diff --git a/.github/workflows/jira-pr.yaml b/.github/workflows/jira-pr.yaml index 17b7d26b1ff..be8eb77865b 100644 --- a/.github/workflows/jira-pr.yaml +++ b/.github/workflows/jira-pr.yaml @@ -41,6 +41,12 @@ jobs: if [[ -n ${ROLE} ]]; then echo "Actor ${{ github.actor }} is a ${TEAM} team member" echo "MESSAGE=true" >> $GITHUB_OUTPUT + elif [[ "${{ contains(github.actor, 'hc-github-team-consul-core') }}" == "true" ]]; then + echo "Actor ${{ github.actor }} is a ${TEAM} team member" + echo "MESSAGE=true" >> $GITHUB_OUTPUT + elif [[ "${{ contains(github.actor, 'dependabot') }}" == "true" ]]; then + echo "Actor ${{ github.actor }} is a ${TEAM} team member" + echo "MESSAGE=true" >> $GITHUB_OUTPUT else echo "Actor ${{ github.actor }} is NOT a ${TEAM} team member" echo "MESSAGE=false" >> $GITHUB_OUTPUT diff --git a/.github/workflows/license-checker.yml b/.github/workflows/license-checker.yml new file mode 100644 index 00000000000..747f81490e6 --- /dev/null +++ b/.github/workflows/license-checker.yml @@ -0,0 +1,27 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# This workflow checks that the BUSL license is not mentioned anywhere in +# a PR targeting a release that should maintain the MPL-2.0 license. +name: License Checker + +on: + pull_request: + types: [opened, synchronize] + branches: + - release/1.14.* + - release/1.15.* + - release/1.16.* + +jobs: + # checks that the diff does not contain any reference to + # the BUSL license and thus retains the MPL-2.0 license + license-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 # by default the checkout action doesn't checkout all branches + - name: Check for BUSL text in diff + run: ./.github/scripts/license_checker.sh diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml deleted file mode 100644 index ab7d793e794..00000000000 --- a/.github/workflows/load-test.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Load Test - -on: - pull_request: - branches: - - main - types: [labeled] - workflow_dispatch: {} - -jobs: - trigger-load-test: - if: ${{ github.event.label.name == 'pr/load-test' }} - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Trigger CircleCI Load Test Pipeline - run: | - # build json payload to trigger CircleCI Load Test Pipeline - # This only runs the load test pipeline on the 'main' branch - jsonData=$(jq --null-input -r --arg commit ${{ github.event.pull_request.head.sha }} \ - '{branch:"main", parameters: {"commit": $commit, "trigger-load-test": true}}') - echo "Passing JSON data to CircleCI API: $jsonData" - load_test_pipeline_id=$(curl -X POST \ - -H "Circle-Token: ${{ secrets.CIRCLE_TOKEN }}" \ - -H "Content-Type: application/json" \ - -d "$jsonData" \ - "https://circleci.com/api/v2/project/gh/${GITHUB_REPOSITORY}/pipeline" | jq -r '.id') - echo "LOAD_TEST_PIPELINE_ID=$load_test_pipeline_id" >> $GITHUB_ENV - - name: Post Load Test URL to PR - env: - PR_COMMENT_URL: ${{ github.event.pull_request.comments_url }} - run: | - echo "LOAD_TEST_PIPELINE_ID is: $LOAD_TEST_PIPELINE_ID" - # get load-test workflow - workflow= - # wait up to a minute for load-test workflow to start - until [ $SECONDS -ge 60 ] && exit 1; do - workflow=$(curl -s -X GET \ - -H "Circle-Token: ${{ secrets.CIRCLE_TOKEN }}" \ - -H "Content-Type: application/json" \ - "https://circleci.com/api/v2/pipeline/${LOAD_TEST_PIPELINE_ID}/workflow" | jq '.items[] | select(.name=="load-test")') - # if we found a workflow we exit - if [ -n "$workflow" ]; then - break - fi - echo -n "." - sleep 5 - done - echo "$workflow" - # get pipeline number - pipeline_number=$(echo "$workflow" | jq -r '.pipeline_number') - # get workflow id - workflow_id=$(echo "$workflow" | jq -r '.id') - # get project slug - project_slug=$(echo "$workflow" | jq -r '.project_slug') - # build load test URL - load_test_url="https://app.circleci.com/pipelines/${project_slug}/${pipeline_number}/workflows/${workflow_id}" - # comment URL to pull request - curl -X POST \ - -H "Authorization: token ${{ secrets.PR_COMMENT_TOKEN }}" \ - -H "Content-Type: application/json" \ - -d "{\"body\": \"Load Test Pipeline Started at: $load_test_url\"}" \ - "$PR_COMMENT_URL" diff --git a/.github/workflows/nightly-test-1.11.x.yaml b/.github/workflows/nightly-test-1.11.x.yaml index cd913d4eca4..e913dd0aaf5 100644 --- a/.github/workflows/nightly-test-1.11.x.yaml +++ b/.github/workflows/nightly-test-1.11.x.yaml @@ -39,7 +39,7 @@ jobs: working-directory: ./ui/packages/consul-ui run: make test-node - frontend-build-oss: + frontend-build-ce: runs-on: ubuntu-latest env: JOBS: 2 @@ -61,27 +61,27 @@ jobs: working-directory: ./ui run: make deps - - name: Ember Build OSS - id: build-oss + - name: Ember Build CE + id: build-ce working-directory: ./ui/packages/consul-ui run: make build-ci - - name: Upload OSS Frontend + - name: Upload CE Frontend uses: actions/upload-artifact@v3 with: - name: frontend-oss-${{ env.BRANCH_NAME }} + name: frontend-ce-${{ env.BRANCH_NAME }} path: ./ui/packages/consul-ui/dist if-no-files-found: error - frontend-test-oss: + frontend-test-ce: runs-on: ubuntu-latest - needs: [frontend-build-oss] + needs: [frontend-build-ce] strategy: matrix: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 0 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-ce.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -100,13 +100,13 @@ jobs: working-directory: ./ui run: make deps - - name: Download OSS Frontend + - name: Download CE Frontend uses: actions/download-artifact@v3 with: - name: frontend-oss-${{ env.BRANCH_NAME }} + name: frontend-ce-${{ env.BRANCH_NAME }} path: ./ui/packages/consul-ui/dist - - name: Ember Test OSS + - name: Ember Test CE id: cache working-directory: ./ui/packages/consul-ui run: node_modules/.bin/ember exam --split=$EMBER_PARTITION_TOTAL --partition=${{ matrix.partition }} --path dist --silent -r xunit @@ -134,7 +134,7 @@ jobs: run: make deps - name: Ember Build ENT - id: build-oss + id: build-ce working-directory: ./ui/packages/consul-ui run: make build-ci @@ -153,7 +153,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 1 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-ce.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -215,7 +215,7 @@ jobs: slack-failure-notification: runs-on: ubuntu-latest - needs: [frontend-test-oss, frontend-test-ent] + needs: [frontend-test-ce, frontend-test-ent] if: ${{ failure() }} steps: - name: Slack Notification diff --git a/.github/workflows/nightly-test-1.12.x.yaml b/.github/workflows/nightly-test-1.12.x.yaml index 906f2ba8fb3..4d4fbde750a 100644 --- a/.github/workflows/nightly-test-1.12.x.yaml +++ b/.github/workflows/nightly-test-1.12.x.yaml @@ -39,7 +39,7 @@ jobs: working-directory: ./ui/packages/consul-ui run: make test-node - frontend-build-oss: + frontend-build-ce: runs-on: ubuntu-latest env: JOBS: 2 @@ -61,27 +61,27 @@ jobs: working-directory: ./ui run: make deps - - name: Ember Build OSS - id: build-oss + - name: Ember Build CE + id: build-ce working-directory: ./ui/packages/consul-ui run: make build-ci - - name: Upload OSS Frontend + - name: Upload CE Frontend uses: actions/upload-artifact@v3 with: - name: frontend-oss-${{ env.BRANCH_NAME }} + name: frontend-ce-${{ env.BRANCH_NAME }} path: ./ui/packages/consul-ui/dist if-no-files-found: error - frontend-test-oss: + frontend-test-ce: runs-on: ubuntu-latest - needs: [frontend-build-oss] + needs: [frontend-build-ce] strategy: matrix: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 0 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-ce.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -100,13 +100,13 @@ jobs: working-directory: ./ui run: make deps - - name: Download OSS Frontend + - name: Download CE Frontend uses: actions/download-artifact@v3 with: - name: frontend-oss-${{ env.BRANCH_NAME }} + name: frontend-ce-${{ env.BRANCH_NAME }} path: ./ui/packages/consul-ui/dist - - name: Ember Test OSS + - name: Ember Test CE id: cache working-directory: ./ui/packages/consul-ui run: node_modules/.bin/ember exam --split=$EMBER_PARTITION_TOTAL --partition=${{ matrix.partition }} --path dist --silent -r xunit @@ -134,7 +134,7 @@ jobs: run: make deps - name: Ember Build ENT - id: build-oss + id: build-ce working-directory: ./ui/packages/consul-ui run: make build-ci @@ -153,7 +153,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 1 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-ce.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -215,7 +215,7 @@ jobs: slack-failure-notification: runs-on: ubuntu-latest - needs: [frontend-test-oss, frontend-test-ent] + needs: [frontend-test-ce, frontend-test-ent] if: ${{ failure() }} steps: - name: Slack Notification diff --git a/.github/workflows/nightly-test-1.13.x.yaml b/.github/workflows/nightly-test-1.13.x.yaml index 2df70ff6880..c1e9d035be0 100644 --- a/.github/workflows/nightly-test-1.13.x.yaml +++ b/.github/workflows/nightly-test-1.13.x.yaml @@ -39,7 +39,7 @@ jobs: working-directory: ./ui/packages/consul-ui run: make test-node - frontend-build-oss: + frontend-build-ce: runs-on: ubuntu-latest env: JOBS: 2 @@ -61,27 +61,27 @@ jobs: working-directory: ./ui run: make deps - - name: Ember Build OSS - id: build-oss + - name: Ember Build CE + id: build-ce working-directory: ./ui/packages/consul-ui run: make build-ci - - name: Upload OSS Frontend + - name: Upload CE Frontend uses: actions/upload-artifact@v3 with: - name: frontend-oss-${{ env.BRANCH_NAME }} + name: frontend-ce-${{ env.BRANCH_NAME }} path: ./ui/packages/consul-ui/dist if-no-files-found: error - frontend-test-oss: + frontend-test-ce: runs-on: ubuntu-latest - needs: [frontend-build-oss] + needs: [frontend-build-ce] strategy: matrix: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 0 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-ce.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -100,13 +100,13 @@ jobs: working-directory: ./ui run: make deps - - name: Download OSS Frontend + - name: Download CE Frontend uses: actions/download-artifact@v3 with: - name: frontend-oss-${{ env.BRANCH_NAME }} + name: frontend-ce-${{ env.BRANCH_NAME }} path: ./ui/packages/consul-ui/dist - - name: Ember Test OSS + - name: Ember Test CE id: cache working-directory: ./ui/packages/consul-ui run: node_modules/.bin/ember exam --split=$EMBER_PARTITION_TOTAL --partition=${{ matrix.partition }} --path dist --silent -r xunit @@ -134,7 +134,7 @@ jobs: run: make deps - name: Ember Build ENT - id: build-oss + id: build-ce working-directory: ./ui/packages/consul-ui run: make build-ci @@ -153,7 +153,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 1 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-ce.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -215,7 +215,7 @@ jobs: slack-failure-notification: runs-on: ubuntu-latest - needs: [frontend-test-oss, frontend-test-ent] + needs: [frontend-test-ce, frontend-test-ent] if: ${{ failure() }} steps: - name: Slack Notification diff --git a/.github/workflows/nightly-test-1.14.x.yaml b/.github/workflows/nightly-test-1.14.x.yaml index 745ad7608ee..ee9fbd117c2 100644 --- a/.github/workflows/nightly-test-1.14.x.yaml +++ b/.github/workflows/nightly-test-1.14.x.yaml @@ -39,7 +39,7 @@ jobs: working-directory: ./ui/packages/consul-ui run: make test-node - frontend-build-oss: + frontend-build-ce: runs-on: ubuntu-latest env: JOBS: 2 @@ -61,27 +61,27 @@ jobs: working-directory: ./ui run: make deps - - name: Ember Build OSS - id: build-oss + - name: Ember Build CE + id: build-ce working-directory: ./ui/packages/consul-ui run: make build-ci - - name: Upload OSS Frontend + - name: Upload CE Frontend uses: actions/upload-artifact@v3 with: - name: frontend-oss-${{ env.BRANCH_NAME }} + name: frontend-ce-${{ env.BRANCH_NAME }} path: ./ui/packages/consul-ui/dist if-no-files-found: error - frontend-test-oss: + frontend-test-ce: runs-on: ubuntu-latest - needs: [frontend-build-oss] + needs: [frontend-build-ce] strategy: matrix: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 0 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-ce.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -100,13 +100,13 @@ jobs: working-directory: ./ui run: make deps - - name: Download OSS Frontend + - name: Download CE Frontend uses: actions/download-artifact@v3 with: - name: frontend-oss-${{ env.BRANCH_NAME }} + name: frontend-ce-${{ env.BRANCH_NAME }} path: ./ui/packages/consul-ui/dist - - name: Ember Test OSS + - name: Ember Test CE id: cache working-directory: ./ui/packages/consul-ui run: node_modules/.bin/ember exam --split=$EMBER_PARTITION_TOTAL --partition=${{ matrix.partition }} --path dist --silent -r xunit @@ -134,7 +134,7 @@ jobs: run: make deps - name: Ember Build ENT - id: build-oss + id: build-ce working-directory: ./ui/packages/consul-ui run: make build-ci @@ -153,7 +153,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 1 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-ce.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -215,7 +215,7 @@ jobs: slack-failure-notification: runs-on: ubuntu-latest - needs: [frontend-test-oss, frontend-test-ent] + needs: [frontend-test-ce, frontend-test-ent] if: ${{ failure() }} steps: - name: Slack Notification diff --git a/.github/workflows/nightly-test-integrations.yml b/.github/workflows/nightly-test-integrations.yml new file mode 100644 index 00000000000..ab42dbc25d0 --- /dev/null +++ b/.github/workflows/nightly-test-integrations.yml @@ -0,0 +1,326 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +name: Nightly test-integrations + +on: + schedule: + # Run nightly at 12AM UTC/8PM EST/5PM PST + - cron: '* 0 * * *' + workflow_dispatch: {} + +env: + TEST_RESULTS_DIR: /tmp/test-results + TEST_RESULTS_ARTIFACT_NAME: test-results + CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }} + GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }} + GOTESTSUM_VERSION: "1.10.1" + CONSUL_BINARY_UPLOAD_NAME: consul-bin + # strip the hashicorp/ off the front of github.repository for consul + CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'hashicorp/consul' }} + GOPRIVATE: github.com/hashicorp # Required for enterprise deps + +jobs: + setup: + runs-on: ubuntu-latest + name: Setup + outputs: + compute-small: ${{ steps.runners.outputs.compute-small }} + compute-medium: ${{ steps.runners.outputs.compute-medium }} + compute-large: ${{ steps.runners.outputs.compute-large }} + compute-xl: ${{ steps.runners.outputs.compute-xl }} + enterprise: ${{ steps.runners.outputs.enterprise }} + steps: + - name: Checkout code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + ref: ${{ inputs.branch }} + - id: runners + run: .github/scripts/get_runner_classes.sh + + dev-build: + needs: [setup] + uses: ./.github/workflows/reusable-dev-build.yml + with: + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + uploaded-binary-name: 'consul-bin' + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + generate-envoy-job-matrices: + needs: [setup] + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + name: Generate Envoy Job Matrices + outputs: + envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }} + steps: + - name: Checkout code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + ref: ${{ inputs.branch }} + - name: Generate Envoy Job Matrix + id: set-matrix + env: + # this is further going to multiplied in envoy-integration tests by the + # other dimensions in the matrix. Currently TOTAL_RUNNERS would be + # multiplied by 8 based on these values: + # envoy-version: ["1.24.10", "1.25.9", "1.26.4", "1.27.0"] + # xds-target: ["server", "client"] + TOTAL_RUNNERS: 4 + JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' + run: | + NUM_RUNNERS=$TOTAL_RUNNERS + NUM_DIRS=$(find ./test/integration/connect/envoy -mindepth 1 -maxdepth 1 -type d | wc -l) + + if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then + echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." + NUM_RUNNERS=$((NUM_DIRS-1)) + fi + # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. + NUM_RUNNERS=$((NUM_RUNNERS-1)) + { + echo -n "envoy-matrix=" + find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \ + | xargs -0 -n 1 basename \ + | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ + | jq --compact-output 'map(join("|"))' + } >> "$GITHUB_OUTPUT" + + envoy-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + needs: + - setup + - generate-envoy-job-matrices + - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + strategy: + fail-fast: false + matrix: + envoy-version: ["1.22.11", "1.23.12", "1.24.10", "1.25.9"] + xds-target: ["server", "client"] + test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }} + env: + ENVOY_VERSION: ${{ matrix.envoy-version }} + XDS_TARGET: ${{ matrix.xds-target }} + AWS_LAMBDA_REGION: us-west-2 + steps: + - name: Checkout code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + ref: ${{ inputs.branch }} + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: ./bin + - name: restore mode+x + run: chmod +x ./bin/consul + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@2a1a44ac4aa01993040736bd95bb470da1a38365 # v2.9.0 + + - name: Docker build + run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin + + - name: Envoy Integration Tests + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" + # shellcheck disable=SC2001 + sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --debug \ + --rerun-fails \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --jsonfile /tmp/jsonfile/go-test.log \ + --packages=./test/integration/connect/envoy \ + -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + # do not run on forks + if: github.event.pull_request.head.repo.full_name == github.repository + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml + + upgrade-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + needs: + - setup + - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + strategy: + fail-fast: false + matrix: + consul-version: [ "1.14", "1.15"] + env: + CONSUL_LATEST_VERSION: ${{ matrix.consul-version }} + ENVOY_VERSION: "1.24.6" + steps: + - name: Checkout code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + ref: ${{ inputs.branch }} + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - run: go env + + # Get go binary from workspace + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: . + - name: restore mode+x + run: chmod +x consul + - name: Build consul:local image + run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . + - name: Build consul-envoy:latest-version image + id: buildConsulEnvoyLatestImage + continue-on-error: true + run: docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }} --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets + - name: Retry Build consul-envoy:latest-version image + if: steps.buildConsulEnvoyLatestImage.outcome == 'failure' + run: docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }} --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets + - name: Build consul-envoy:target-version image + id: buildConsulEnvoyTargetImage + continue-on-error: true + run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets + - name: Retry Build consul-envoy:target-version image + if: steps.buildConsulEnvoyTargetImage.outcome == 'failure' + run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets + - name: Build sds image + run: docker build -t consul-sds-server ./test/integration/connect/envoy/test-sds-server/ + - name: Configure GH workaround for ipv6 loopback + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + cat /etc/hosts && echo "-----------" + sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts + cat /etc/hosts + - name: Upgrade Integration Tests + run: | + mkdir -p "${{ env.TEST_RESULTS_DIR }}" + cd ./test/integration/consul-container/test/upgrade + docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --raw-command \ + --format=short-verbose \ + --debug \ + --rerun-fails=2 \ + --packages="./..." \ + -- \ + go test \ + -p=4 \ + -tags "${{ env.GOTAGS }}" \ + -timeout=30m \ + -json \ + ./... \ + --follow-log=false \ + --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --target-version local \ + --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --latest-version "${{ env.CONSUL_LATEST_VERSION }}" + ls -lrt + env: + # this is needed because of incompatibility between RYUK container and GHA + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + # tput complains if this isn't set to something. + TERM: ansi + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + # do not run on forks + if: github.event.pull_request.head.repo.full_name == github.repository + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml + + + test-integrations-success: + needs: + - setup + - dev-build + - generate-envoy-job-matrices + - envoy-integration-test + - upgrade-integration-test + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: ${{ always() }} + steps: + - name: evaluate upstream job results + run: | + # exit 1 if failure or cancelled result for any upstream job + if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then + printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}" + exit 1 + fi diff --git a/.github/workflows/nightly-test-main.yaml b/.github/workflows/nightly-test-main.yaml index e823bac7b56..33b2a8b6221 100644 --- a/.github/workflows/nightly-test-main.yaml +++ b/.github/workflows/nightly-test-main.yaml @@ -39,7 +39,7 @@ jobs: working-directory: ./ui/packages/consul-ui run: make test-node - frontend-build-oss: + frontend-build-ce: runs-on: ubuntu-latest env: JOBS: 2 @@ -61,27 +61,27 @@ jobs: working-directory: ./ui run: make deps - - name: Ember Build OSS - id: build-oss + - name: Ember Build CE + id: build-ce working-directory: ./ui/packages/consul-ui run: make build-ci - - name: Upload OSS Frontend + - name: Upload CE Frontend uses: actions/upload-artifact@v3 with: - name: frontend-oss-${{ env.BRANCH_NAME }} + name: frontend-ce-${{ env.BRANCH_NAME }} path: ./ui/packages/consul-ui/dist if-no-files-found: error - frontend-test-oss: + frontend-test-ce: runs-on: ubuntu-latest - needs: [frontend-build-oss] + needs: [frontend-build-ce] strategy: matrix: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 0 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-ce.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -100,13 +100,13 @@ jobs: working-directory: ./ui run: make deps - - name: Download OSS Frontend + - name: Download CE Frontend uses: actions/download-artifact@v3 with: - name: frontend-oss-${{ env.BRANCH_NAME }} + name: frontend-ce-${{ env.BRANCH_NAME }} path: ./ui/packages/consul-ui/dist - - name: Ember Test OSS + - name: Ember Test CE id: cache working-directory: ./ui/packages/consul-ui run: node_modules/.bin/ember exam --split=$EMBER_PARTITION_TOTAL --partition=${{ matrix.partition }} --path dist --silent -r xunit @@ -134,7 +134,7 @@ jobs: run: make deps - name: Ember Build ENT - id: build-oss + id: build-ce working-directory: ./ui/packages/consul-ui run: make build-ci @@ -153,7 +153,7 @@ jobs: partition: [ 1, 2, 3, 4 ] env: CONSUL_NSPACES_ENABLED: 1 - EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary + EMBER_TEST_REPORT: test-results/report-ce.xml #outputs test report for CI test summary EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam steps: - uses: actions/checkout@v2 @@ -215,7 +215,7 @@ jobs: slack-failure-notification: runs-on: ubuntu-latest - needs: [frontend-test-oss, frontend-test-ent] + needs: [frontend-test-ce, frontend-test-ent] if: ${{ failure() }} steps: - name: Slack Notification diff --git a/.github/workflows/reusable-check-go-mod.yml b/.github/workflows/reusable-check-go-mod.yml new file mode 100644 index 00000000000..2078b0c3217 --- /dev/null +++ b/.github/workflows/reusable-check-go-mod.yml @@ -0,0 +1,38 @@ +name: check-go-mod + +on: + workflow_call: + inputs: + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string + repository-name: + required: true + type: string + secrets: + elevated-github-token: + required: true +jobs: + check-go-mod: + runs-on: ${{ fromJSON(inputs.runs-on) }} + + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(inputs.repository-name, '-enterprise') }} + run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: go mod tidy + - run: | + if [[ -n $(git status -s) ]]; then + echo "Git directory has changes" + git status -s + exit 1 + fi + - name: Notify Slack + if: ${{ failure() }} + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-dev-build.yml b/.github/workflows/reusable-dev-build.yml new file mode 100644 index 00000000000..2db9670655e --- /dev/null +++ b/.github/workflows/reusable-dev-build.yml @@ -0,0 +1,47 @@ +name: reusable-dev-build + +on: + workflow_call: + inputs: + uploaded-binary-name: + required: false + type: string + default: "consul-bin" + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string + repository-name: + required: true + type: string + go-arch: + required: false + type: string + default: "" + secrets: + elevated-github-token: + required: true +jobs: + build: + runs-on: ${{ fromJSON(inputs.runs-on) }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(inputs.repository-name, '-enterprise') }} + run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - name: Build + env: + GOARCH: ${{ inputs.goarch }} + run: make dev + # save dev build to pass to downstream jobs + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{inputs.uploaded-binary-name}} + path: ./bin/consul + - name: Notify Slack + if: ${{ failure() }} + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-lint.yml b/.github/workflows/reusable-lint.yml new file mode 100644 index 00000000000..f7032f98666 --- /dev/null +++ b/.github/workflows/reusable-lint.yml @@ -0,0 +1,57 @@ +name: reusable-lint + +on: + workflow_call: + inputs: + go-arch: + required: false + type: string + default: "" + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string + repository-name: + required: true + type: string + secrets: + elevated-github-token: + required: true +env: + GOTAGS: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + GOARCH: ${{inputs.go-arch}} + +jobs: + lint: + runs-on: ${{ fromJSON(inputs.runs-on) }} + strategy: + matrix: + directory: + - "" + - "api" + - "sdk" + - "envoyextensions" + - "troubleshoot" + - "test/integration/consul-container" + fail-fast: true + name: lint ${{ matrix.directory }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(inputs.repository-name, '-enterprise') }} + run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - run: go env + - name: lint-${{ matrix.directory }} + uses: golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5 # pin@v3.4.0 + with: + working-directory: ${{ matrix.directory }} + version: v1.51.1 + args: --build-tags="${{ env.GOTAGS }}" -v + skip-cache: true + - name: Notify Slack + if: ${{ failure() }} + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-unit-split.yml b/.github/workflows/reusable-unit-split.yml new file mode 100644 index 00000000000..6c13670e742 --- /dev/null +++ b/.github/workflows/reusable-unit-split.yml @@ -0,0 +1,179 @@ +name: reusable-unit-split + +on: + workflow_call: + inputs: + directory: + required: true + type: string + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string + go-arch: + required: false + type: string + default: "" + uploaded-binary-name: + required: false + type: string + default: "consul-bin" + args: + required: false + type: string + default: "" + runner-count: + required: false + type: number + default: 1 + go-test-flags: + required: false + type: string + default: "" + repository-name: + required: true + type: string + go-tags: + required: false + type: string + default: "" + secrets: + elevated-github-token: + required: true + consul-license: + required: true + datadog-api-key: + required: true +env: + TEST_RESULTS: /tmp/test-results + GOTESTSUM_VERSION: "1.10.1" + GOARCH: ${{inputs.go-arch}} + TOTAL_RUNNERS: ${{inputs.runner-count}} + CONSUL_LICENSE: ${{secrets.consul-license}} + GOTAGS: ${{ inputs.go-tags}} + DATADOG_API_KEY: ${{secrets.datadog-api-key}} + +jobs: + set-test-package-matrix: + runs-on: ubuntu-latest + outputs: + package-matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + - id: set-matrix + run: ./.github/scripts/set_test_package_matrix.sh ${{env.TOTAL_RUNNERS}} + + go-test: + runs-on: ${{ fromJSON(inputs.runs-on) }} + name: "go-test" + needs: + - set-test-package-matrix + strategy: + fail-fast: false + matrix: + package: ${{ fromJson(needs.set-test-package-matrix.outputs.package-matrix) }} + steps: + - name: ulimit + run: | + echo "Soft limits" + ulimit -Sa + echo "Hard limits" + ulimit -Ha + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(inputs.repository-name, '-enterprise') }} + run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + cache: true + - run: mkdir -p ${{env.TEST_RESULTS}} + - name: go mod download + working-directory: ${{inputs.directory}} + run: go mod download + - name: Download consul + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # pin@v3.0.2 + with: + name: ${{inputs.uploaded-binary-name}} + path: ${{inputs.directory}} + - name: Display downloaded file + run: ls -ld consul + working-directory: ${{inputs.directory}} + - run: echo "$GITHUB_WORKSPACE/${{inputs.directory}}" >> $GITHUB_PATH + - name: Make sure consul is executable + run: chmod +x $GITHUB_WORKSPACE/${{inputs.directory}}/consul + - run: go env + - name: Run tests + working-directory: ${{inputs.directory}} + run: | + # separate the list + PACKAGE_NAMES="${{ join(matrix.package, ' ') }}" + # PACKAGE_NAMES="${{ matrix.package }}" + + ${{inputs.go-test-flags}} + + # some tests expect this umask, and arm images have a different default + umask 0022 + + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --format=short-verbose \ + --jsonfile /tmp/jsonfile/go-test.log \ + --debug \ + --rerun-fails=3 \ + --rerun-fails-max-failures=40 \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --packages="$PACKAGE_NAMES" \ + --junitfile ${{env.TEST_RESULTS}}/gotestsum-report.xml -- \ + -tags="${{env.GOTAGS}}" -p 2 \ + ${GO_TEST_FLAGS-} \ + -cover -coverprofile=coverage.txt + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + # do not run on forks + if: ${{ env.DATADOG_API_KEY}} + env: + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" ${{env.TEST_RESULTS}}/gotestsum-report.xml + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: test-results + path: ${{env.TEST_RESULTS}} + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: jsonfile + path: /tmp/jsonfile + - name: "Re-run fails report" + run: | + .github/scripts/rerun_fails_report.sh /tmp/gotestsum-rerun-fails + - name: Notify Slack + if: ${{ failure() }} + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/reusable-unit.yml b/.github/workflows/reusable-unit.yml new file mode 100644 index 00000000000..c49a6291fa2 --- /dev/null +++ b/.github/workflows/reusable-unit.yml @@ -0,0 +1,157 @@ +name: reusable-unit + +on: + workflow_call: + inputs: + directory: + required: true + type: string + runs-on: + description: An expression indicating which kind of runners to use. + required: true + type: string + go-arch: + required: false + type: string + default: "" + uploaded-binary-name: + required: false + type: string + default: "consul-bin" + package-names-command: + required: false + type: string + default: 'go list -tags "$GOTAGS" ./...' + go-test-flags: + required: false + type: string + default: "" + repository-name: + required: true + type: string + go-tags: + required: false + type: string + default: "" + go-version: + required: false + type: string + default: "" + secrets: + elevated-github-token: + required: true + consul-license: + required: true + datadog-api-key: + required: true +env: + TEST_RESULTS: /tmp/test-results + GOTESTSUM_VERSION: "1.10.1" + GOARCH: ${{inputs.go-arch}} + CONSUL_LICENSE: ${{secrets.consul-license}} + GOTAGS: ${{ inputs.go-tags}} + DATADOG_API_KEY: ${{secrets.datadog-api-key}} + +jobs: + go-test: + runs-on: ${{ fromJSON(inputs.runs-on) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(inputs.repository-name, '-enterprise') }} + run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com" + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + if: ${{ inputs.go-version != '' }} + with: + go-version: ${{ inputs.go-version }} + cache: true + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + if: ${{ inputs.go-version == '' }} + with: + go-version-file: 'go.mod' + cache: true + - run: mkdir -p ${{env.TEST_RESULTS}} + - name: go mod download + working-directory: ${{inputs.directory}} + run: go mod download + - name: Download consul + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # pin@v3.0.2 + with: + name: ${{inputs.uploaded-binary-name}} + path: ${{inputs.directory}} + - name: Display downloaded file + run: ls -ld consul + working-directory: ${{inputs.directory}} + - run: echo "$GITHUB_WORKSPACE/${{inputs.directory}}" >> $GITHUB_PATH + - name: Make sure consul is executable + run: chmod +x $GITHUB_WORKSPACE/${{inputs.directory}}/consul + - run: go env + - name: Run tests + working-directory: ${{inputs.directory}} + run: | + PACKAGE_NAMES=$(${{inputs.package-names-command}}) + + # some tests expect this umask, and arm images have a different default + umask 0022 + + ${{inputs.go-test-flags}} + + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --format=short-verbose \ + --jsonfile /tmp/jsonfile/go-test.log \ + --debug \ + --rerun-fails=3 \ + --rerun-fails-max-failures=40 \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --packages="$PACKAGE_NAMES" \ + --junitfile ${{env.TEST_RESULTS}}/gotestsum-report.xml -- \ + -tags="${{env.GOTAGS}}" \ + ${GO_TEST_FLAGS-} \ + -cover -coverprofile=coverage.txt + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + # do not run on forks + if: ${{ env.DATADOG_API_KEY}} + env: + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" ${{env.TEST_RESULTS}}/gotestsum-report.xml + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: test-results + path: ${{env.TEST_RESULTS}} + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 + with: + name: jsonfile + path: /tmp/jsonfile + - name: "Re-run fails report" + run: | + .github/scripts/rerun_fails_report.sh /tmp/gotestsum-rerun-fails + - name: Notify Slack + if: ${{ failure() }} + run: .github/scripts/notify_slack.sh diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml new file mode 100644 index 00000000000..16b88210d74 --- /dev/null +++ b/.github/workflows/test-integrations.yml @@ -0,0 +1,494 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +name: test-integrations + +on: + pull_request: + branches-ignore: + - stable-website + - 'docs/**' + - 'ui/**' + - 'mktg-**' # Digital Team Terraform-generated branch prefix + - 'backport/docs/**' + - 'backport/ui/**' + - 'backport/mktg-**' + +env: + TEST_RESULTS_DIR: /tmp/test-results + TEST_RESULTS_ARTIFACT_NAME: test-results + CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }} + GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }} + GOTESTSUM_VERSION: "1.10.1" + CONSUL_BINARY_UPLOAD_NAME: consul-bin + # strip the hashicorp/ off the front of github.repository for consul + CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'hashicorp/consul' }} + GOPRIVATE: github.com/hashicorp # Required for enterprise deps + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + conditional-skip: + runs-on: ubuntu-latest + name: Get files changed and conditionally skip CI + outputs: + trigger-ci: ${{ steps.read-files.outputs.trigger-ci }} + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + fetch-depth: 0 + - name: Get changed files + id: read-files + run: ./.github/scripts/filter_changed_files_go_test.sh + + setup: + needs: [conditional-skip] + runs-on: ubuntu-latest + name: Setup + if: needs.conditional-skip.outputs.trigger-ci == 'true' + outputs: + compute-small: ${{ steps.runners.outputs.compute-small }} + compute-medium: ${{ steps.runners.outputs.compute-medium }} + compute-large: ${{ steps.runners.outputs.compute-large }} + compute-xl: ${{ steps.runners.outputs.compute-xl }} + enterprise: ${{ steps.runners.outputs.enterprise }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - id: runners + run: .github/scripts/get_runner_classes.sh + + dev-build: + needs: [setup] + uses: ./.github/workflows/reusable-dev-build.yml + with: + runs-on: ${{ needs.setup.outputs.compute-large }} + repository-name: ${{ github.repository }} + uploaded-binary-name: 'consul-bin' + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + nomad-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + needs: + - setup + - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + strategy: + matrix: + nomad-version: ['v1.3.3', 'v1.2.10', 'v1.1.16'] + steps: + - name: Checkout Nomad + uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + with: + repository: hashicorp/nomad + ref: ${{ matrix.nomad-version }} + + - name: Install Go + uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + + - name: Fetch Consul binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: ./bin + - name: Restore Consul permissions + run: | + chmod +x ./bin/consul + echo "$(pwd)/bin" >> $GITHUB_PATH + + - name: Make Nomad dev build + run: make pkg/linux_amd64/nomad + + - name: Run integration tests + run: | + go install gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} && \ + gotestsum \ + --format=short-verbose \ + --rerun-fails \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --packages="./command/agent/consul" \ + --junitfile $TEST_RESULTS_DIR/results.xml -- \ + -run TestConsul + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + # do not run on forks + if: github.event.pull_request.head.repo.full_name == github.repository + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml + + vault-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + needs: + - setup + - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + strategy: + matrix: + vault-version: ["1.13.1", "1.12.5", "1.11.9", "1.10.11"] + env: + VAULT_BINARY_VERSION: ${{ matrix.vault-version }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + + - name: Install Vault + run: | + wget -q -O /tmp/vault.zip "https://releases.hashicorp.com/vault/${{ env.VAULT_BINARY_VERSION }}/vault_${{ env.VAULT_BINARY_VERSION }}_linux_amd64.zip" + unzip -d /tmp /tmp/vault.zip + echo "/tmp" >> $GITHUB_PATH + + - name: Run Connect CA Provider Tests + run: | + mkdir -p "${{ env.TEST_RESULTS_DIR }}" + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --format=short-verbose \ + --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report.xml" \ + -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage.txt ./agent/connect/ca + # Run leader tests that require Vault + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --format=short-verbose \ + --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-leader.xml" \ + -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-leader.txt -run Vault ./agent/consul + # Run agent tests that require Vault + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --format=short-verbose \ + --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" \ + -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-agent.txt -run Vault ./agent + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + # do not run on forks + if: github.event.pull_request.head.repo.full_name == github.repository + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" "${{ env.TEST_RESULTS_DIR }}/gotestsum-report.xml" + + - name: upload leader coverage + # do not run on forks + if: github.event.pull_request.head.repo.full_name == github.repository + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-leader.xml" + + - name: upload agent coverage + # do not run on forks + if: github.event.pull_request.head.repo.full_name == github.repository + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" + + generate-envoy-job-matrices: + needs: [setup] + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + name: Generate Envoy Job Matrices + outputs: + envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - name: Generate Envoy Job Matrix + id: set-matrix + env: + # this is further going to multiplied in envoy-integration tests by the + # other dimensions in the matrix. Currently TOTAL_RUNNERS would be + # multiplied by 8 based on these values: + # envoy-version: ["1.22.11", "1.23.12", "1.24.10", "1.25.9"] + # xds-target: ["server", "client"] + TOTAL_RUNNERS: 4 + JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' + run: | + NUM_RUNNERS=$TOTAL_RUNNERS + NUM_DIRS=$(find ./test/integration/connect/envoy -mindepth 1 -maxdepth 1 -type d | wc -l) + + if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then + echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." + NUM_RUNNERS=$((NUM_DIRS-1)) + fi + # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. + NUM_RUNNERS=$((NUM_RUNNERS-1)) + { + echo -n "envoy-matrix=" + find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \ + | xargs -0 -n 1 basename \ + | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ + | jq --compact-output 'map(join("|"))' + } >> "$GITHUB_OUTPUT" + + envoy-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + needs: + - setup + - generate-envoy-job-matrices + - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + strategy: + fail-fast: false + matrix: + envoy-version: ["1.25.9"] + xds-target: ["server", "client"] + test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }} + env: + ENVOY_VERSION: ${{ matrix.envoy-version }} + XDS_TARGET: ${{ matrix.xds-target }} + AWS_LAMBDA_REGION: us-west-2 + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: ./bin + - name: restore mode+x + run: chmod +x ./bin/consul + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2.4.1 + + - name: Docker build + run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin + + - name: Envoy Integration Tests + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" + # shellcheck disable=SC2001 + sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --debug \ + --rerun-fails \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --jsonfile /tmp/jsonfile/go-test.log \ + --packages=./test/integration/connect/envoy \ + -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + # do not run on forks + if: github.event.pull_request.head.repo.full_name == github.repository + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml + + compatibility-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + needs: + - setup + - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + env: + ENVOY_VERSION: "1.25.4" + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + - run: go env + - name: docker env + run: | + docker version + docker info + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: . + - name: restore mode+x + run: chmod +x consul + # Build the consul:local image from the already built binary + - name: Build consul:local image + run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . + - name: Configure GH workaround for ipv6 loopback + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + cat /etc/hosts && echo "-----------" + sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts + cat /etc/hosts + - name: Compatibility Integration Tests + run: | + mkdir -p "/tmp/test-results" + cd ./test/integration/consul-container + docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --raw-command \ + --format=standard-verbose \ + --debug \ + --rerun-fails=3 \ + -- \ + go test \ + -p=6 \ + -parallel=4 \ + -tags "${{ env.GOTAGS }}" \ + -timeout=30m \ + -json \ + `go list ./... | grep -v upgrade` \ + --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --target-version local \ + --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --latest-version latest + ls -lrt + env: + # this is needed because of incompatibility between RYUK container and GHA + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + # tput complains if this isn't set to something. + TERM: ansi + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + # do not run on forks + if: github.event.pull_request.head.repo.full_name == github.repository + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml + + test-integrations-success: + needs: + - conditional-skip + - setup + - dev-build + - nomad-integration-test + - vault-integration-test + - generate-envoy-job-matrices + - envoy-integration-test + - compatibility-integration-test + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: always() && needs.conditional-skip.outputs.trigger-ci == 'true' + steps: + - name: evaluate upstream job results + run: | + # exit 1 if failure or cancelled result for any upstream job + if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then + printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}" + exit 1 + fi diff --git a/.github/workflows/verify-ci.yml b/.github/workflows/verify-ci.yml new file mode 100644 index 00000000000..4bd4536add1 --- /dev/null +++ b/.github/workflows/verify-ci.yml @@ -0,0 +1,23 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# verify-ci is a no-op workflow that must run on every PR. It is used in a +# branch protection rule to detect when CI workflows are not running. +name: verify-ci + +permissions: + contents: read + +on: + pull_request: + push: + branches: + # Push events on the main branch + - main + - release/** + +jobs: + verify-ci-success: + runs-on: ubuntu-latest + steps: + - run: echo "verify-ci succeeded" diff --git a/.github/workflows/verify-envoy-version.yml b/.github/workflows/verify-envoy-version.yml new file mode 100644 index 00000000000..d097e335d37 --- /dev/null +++ b/.github/workflows/verify-envoy-version.yml @@ -0,0 +1,28 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# This action ensures that Envoy is up to date on main and release branches. +# This workflow is only triggered on the main and release branches and will +# only perform a version check when a new release branch is created +# Contact Consul team for any questions + +name: Verify Envoy Version + +on: + push: + branches: + - main + - release/** + +jobs: + verify-envoy-version: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 # by default the checkout action doesn't checkout all branches + - name: Run Envoy Version Verification for main and release branches + run: ./.github/scripts/verify_envoy_version.sh + env: + GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }} \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index 60cfc505950..69e8f43a1ce 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,3 +1,6 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + linters: disable-all: true enable: @@ -20,29 +23,14 @@ issues: # Temp Ignore SA9004: only the first constant in this group has an explicit type # https://staticcheck.io/docs/checks#SA9004 - linters: [staticcheck] - text: 'SA9004:' - - - linters: [staticcheck] - text: 'SA1019: "github.com/golang/protobuf/jsonpb" is deprecated: Use the "google.golang.org/protobuf/encoding/protojson" package instead.' - - - linters: [staticcheck] - text: 'SA1019: "github.com/golang/protobuf/proto" is deprecated: Use the "google.golang.org/protobuf/proto" package instead.' - - - linters: [staticcheck] - text: 'SA1019: ptypes.MarshalAny is deprecated' - - - linters: [staticcheck] - text: 'SA1019: ptypes.UnmarshalAny is deprecated' - - - linters: [staticcheck] - text: 'SA1019: "github.com/golang/protobuf/ptypes" is deprecated: Well-known types have specialized functionality directly injected into the generated packages for each message type. See the deprecation notice for each function for the suggested alternative.' + text: "SA9004:" - linters: [staticcheck] text: 'SA1019: "io/ioutil" has been deprecated since Go 1.16' # An argument that always receives the same value is often not a problem. - linters: [unparam] - text: 'always receives' + text: "always receives" # Often functions will implement an interface that returns an error without # needing to return an error. Sometimes the error return value is unnecessary @@ -56,18 +44,18 @@ issues: # self-documentation than a single underscore by itself. Underscore arguments # should generally only be used when a function is implementing an interface. - linters: [unparam] - text: '`_[^`]*` is unused' + text: "`_[^`]*` is unused" # Temp ignore some common unused parameters so that unparam can be added # incrementally. - linters: [unparam] - text: '`(t|resp|req|entMeta)` is unused' + text: "`(t|resp|req|entMeta)` is unused" - # Temp ignore everything in _oss(_test).go and _ent(_test).go. Many of these + # Temp ignore everything in _ce(_test).go and _ent(_test).go. Many of these # could use underscore to ignore the unused arguments, but the "always returns" - # issue will likely remain in oss, and will need to be excluded. + # issue will likely remain in CE, and will need to be excluded. - linters: [unparam] - path: '(_oss.go|_oss_test.go|_ent.go|_ent_test.go)' + path: "(_ce.go|_ce_test.go|_ent.go|_ent_test.go)" linters-settings: govet: @@ -112,7 +100,7 @@ linters-settings: # Specify an error message to output when a denied package is used. # Default: [] packages-with-error-message: - - net/rpc: 'only use forked copy in github.com/hashicorp/consul-net-rpc/net/rpc' + - net/rpc: "only use forked copy in github.com/hashicorp/consul-net-rpc/net/rpc" run: timeout: 10m diff --git a/.release/ci.hcl b/.release/ci.hcl index 084450dd4cd..25f64e4c6b7 100644 --- a/.release/ci.hcl +++ b/.release/ci.hcl @@ -38,6 +38,41 @@ event "prepare" { } } +## These are promotion and post-publish events +## they should be added to the end of the file after the verify event stanza. + +event "trigger-staging" { +// This event is dispatched by the bob trigger-promotion command +// and is required - do not delete. +} + +event "promote-staging" { + depends = ["trigger-staging"] + action "promote-staging" { + organization = "hashicorp" + repository = "crt-workflows-common" + workflow = "promote-staging" + config = "release-metadata.hcl" + } + + notification { + on = "always" + } +} + +event "promote-staging-docker" { + depends = ["promote-staging"] + action "promote-staging-docker" { + organization = "hashicorp" + repository = "crt-workflows-common" + workflow = "promote-staging-docker" + } + + notification { + on = "always" + } +} + event "trigger-production" { // This event is dispatched by the bob trigger-promotion command // and is required - do not delete. diff --git a/.release/linux/package/usr/lib/systemd/system/consul.service b/.release/linux/package/usr/lib/systemd/system/consul.service index 1bbf51a7a10..65eca696e1a 100644 --- a/.release/linux/package/usr/lib/systemd/system/consul.service +++ b/.release/linux/package/usr/lib/systemd/system/consul.service @@ -6,6 +6,7 @@ After=network-online.target ConditionFileNotEmpty=/etc/consul.d/consul.hcl [Service] +Type=notify EnvironmentFile=-/etc/consul.d/consul.env User=consul Group=consul diff --git a/.release/security-scan.hcl b/.release/security-scan.hcl index 490ddd27463..97124dd4eb3 100644 --- a/.release/security-scan.hcl +++ b/.release/security-scan.hcl @@ -8,6 +8,7 @@ binary { secrets = false go_modules = false osv = true + # TODO(spatel): CE refactor oss_index = true nvd = true } diff --git a/CHANGELOG.md b/CHANGELOG.md index 975050ecf3e..271214de30b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,273 @@ +## 1.15.5 (August 8, 2023) + +SECURITY: + +* Update `golang.org/x/net` to v0.13.0 to address [CVE-2023-3978](https://nvd.nist.gov/vuln/detail/CVE-2023-3978). [[GH-18358](https://github.com/hashicorp/consul/issues/18358)] +* Upgrade golang.org/x/net to address [CVE-2023-29406](https://nvd.nist.gov/vuln/detail/CVE-2023-29406) [[GH-18186](https://github.com/hashicorp/consul/issues/18186)] +* Upgrade to use Go 1.20.6. +This resolves [CVE-2023-29406](https://github.com/advisories/GHSA-f8f7-69v5-w4vx)(`net/http`) for uses of the standard library. +A separate change updates dependencies on `golang.org/x/net` to use `0.12.0`. [[GH-18190](https://github.com/hashicorp/consul/issues/18190)] +* Upgrade to use Go 1.20.7. +This resolves vulnerability [CVE-2023-29409](https://nvd.nist.gov/vuln/detail/CVE-2023-29409)(`crypto/tls`). [[GH-18358](https://github.com/hashicorp/consul/issues/18358)] + +FEATURES: + +* cli: `consul members` command uses `-filter` expression to filter members based on bexpr. [[GH-18223](https://github.com/hashicorp/consul/issues/18223)] +* cli: `consul watch` command uses `-filter` expression to filter response from checks, services, nodes, and service. [[GH-17780](https://github.com/hashicorp/consul/issues/17780)] +* reloadable config: Made enable_debug config reloadable and enable pprof command to work when config toggles to true [[GH-17565](https://github.com/hashicorp/consul/issues/17565)] + +IMPROVEMENTS: + +* Fix some typos in metrics docs [[GH-18080](https://github.com/hashicorp/consul/issues/18080)] +* acl: added builtin ACL policy that provides global read-only access (builtin/global-read-only) [[GH-18319](https://github.com/hashicorp/consul/issues/18319)] +* acl: allow for a single slash character in policy names [[GH-18319](https://github.com/hashicorp/consul/issues/18319)] +* connect: Add capture group labels from Envoy cluster FQDNs to Envoy exported metric labels [[GH-17888](https://github.com/hashicorp/consul/issues/17888)] +* connect: update supported envoy versions to 1.22.11, 1.23.12, 1.24.10, 1.25.9 [[GH-18304](https://github.com/hashicorp/consul/issues/18304)] +* hcp: Add dynamic configuration support for the export of server metrics to HCP. [[GH-18168](https://github.com/hashicorp/consul/issues/18168)] +* hcp: Removes requirement for HCP to provide a management token [[GH-18140](https://github.com/hashicorp/consul/issues/18140)] +* xds: Explicitly enable WebSocket connection upgrades in HTTP connection manager [[GH-18150](https://github.com/hashicorp/consul/issues/18150)] + +BUG FIXES: + +* Fix a bug that wrongly trims domains when there is an overlap with DC name. [[GH-17160](https://github.com/hashicorp/consul/issues/17160)] +* api-gateway: fix race condition in proxy config generation when Consul is notified of the bound-api-gateway config entry before it is notified of the api-gateway config entry. [[GH-18291](https://github.com/hashicorp/consul/issues/18291)] +* connect/ca: Fixes a bug preventing CA configuration updates in secondary datacenters [[GH-17846](https://github.com/hashicorp/consul/issues/17846)] +* connect: Fix incorrect protocol config merging for transparent proxy implicit upstreams. [[GH-17894](https://github.com/hashicorp/consul/issues/17894)] +* connect: Removes the default health check from the `consul connect envoy` command when starting an API Gateway. +This health check would always fail. [[GH-18011](https://github.com/hashicorp/consul/issues/18011)] +* connect: fix a bug with Envoy potentially starting with incomplete configuration by not waiting enough for initial xDS configuration. [[GH-18024](https://github.com/hashicorp/consul/issues/18024)] +* snapshot: fix access denied and handle is invalid when we call snapshot save on windows - skip sync() for folders in windows in +https://github.com/rboyer/safeio/pull/3 [[GH-18302](https://github.com/hashicorp/consul/issues/18302)] + +## 1.15.4 (June 26, 2023) +FEATURES: + +* cli: `consul operator raft list-peers` command shows the number of commits each follower is trailing the leader by to aid in troubleshooting. [[GH-17582](https://github.com/hashicorp/consul/issues/17582)] +* server: **(Enterprise Only)** allow automatic license utilization reporting. [[GH-5102](https://github.com/hashicorp/consul/issues/5102)] + +IMPROVEMENTS: + +* connect: update supported envoy versions to 1.22.11, 1.23.9, 1.24.7, 1.25.6 [[GH-17545](https://github.com/hashicorp/consul/issues/17545)] +* debug: change default setting of consul debug command. now default duration is 5ms and default log level is 'TRACE' [[GH-17596](https://github.com/hashicorp/consul/issues/17596)] +* fix metric names in /docs/agent/telemetry [[GH-17577](https://github.com/hashicorp/consul/issues/17577)] +* gateway: Change status condition reason for invalid certificate on a listener from "Accepted" to "ResolvedRefs". [[GH-17115](https://github.com/hashicorp/consul/issues/17115)] +* systemd: set service type to notify. [[GH-16845](https://github.com/hashicorp/consul/issues/16845)] + +BUG FIXES: + +* cache: fix a few minor goroutine leaks in leaf certs and the agent cache [[GH-17636](https://github.com/hashicorp/consul/issues/17636)] +* docs: fix list of telemetry metrics [[GH-17593](https://github.com/hashicorp/consul/issues/17593)] +* gateways: **(Enterprise only)** Fixed a bug in API gateways where gateway configuration objects in non-default partitions did not reconcile properly. [[GH-17581](https://github.com/hashicorp/consul/issues/17581)] +* gateways: Fixed a bug in API gateways where binding a route that only targets a service imported from a peer results + in the programmed gateway having no routes. [[GH-17609](https://github.com/hashicorp/consul/issues/17609)] +* gateways: Fixed a bug where API gateways were not being taken into account in determining xDS rate limits. [[GH-17631](https://github.com/hashicorp/consul/issues/17631)] +* http: fixed API endpoint `PUT /acl/token/:AccessorID` (update token), no longer requires `AccessorID` in the request body. Web UI can now update tokens. [[GH-17739](https://github.com/hashicorp/consul/issues/17739)] +* namespaces: **(Enterprise only)** fixes a bug where agent health checks stop syncing for all services on a node if the namespace of any service has been removed from the server. +* namespaces: **(Enterprise only)** fixes a bug where namespaces are stuck in a deferred deletion state indefinitely under some conditions. + Also fixes the Consul query metadata present in the HTTP headers of the namespace read and list endpoints. +* peering: Fix a bug that caused server agents to continue cleaning up peering resources even after loss of leadership. [[GH-17483](https://github.com/hashicorp/consul/issues/17483)] +* xds: Fixed a bug where modifying ACLs on a token being actively used for an xDS connection caused all xDS updates to fail. [[GH-17566](https://github.com/hashicorp/consul/issues/17566)] + +## 1.15.3 (June 1, 2023) + +BREAKING CHANGES: + +* extensions: The Lua extension now targets local proxy listeners for the configured service's upstreams, rather than remote downstream listeners for the configured service, when ListenerType is set to outbound in extension configuration. See [CVE-2023-2816](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-2816) changelog entry for more details. [[GH-17415](https://github.com/hashicorp/consul/issues/17415)] + +SECURITY: + +* Update to UBI base image to 9.2. [[GH-17513](https://github.com/hashicorp/consul/issues/17513)] +* Upgrade golang.org/x/net to address [CVE-2022-41723](https://nvd.nist.gov/vuln/detail/CVE-2022-41723) [[GH-16754](https://github.com/hashicorp/consul/issues/16754)] +* Upgrade to use Go 1.20.4. +This resolves vulnerabilities [CVE-2023-24537](https://github.com/advisories/GHSA-9f7g-gqwh-jpf5)(`go/scanner`), +[CVE-2023-24538](https://github.com/advisories/GHSA-v4m2-x4rp-hv22)(`html/template`), +[CVE-2023-24534](https://github.com/advisories/GHSA-8v5j-pwr7-w5f8)(`net/textproto`) and +[CVE-2023-24536](https://github.com/advisories/GHSA-9f7g-gqwh-jpf5)(`mime/multipart`). +Also, `golang.org/x/net` has been updated to v0.7.0 to resolve CVEs [CVE-2022-41721 +](https://github.com/advisories/GHSA-fxg5-wq6x-vr4w +), [CVE-2022-27664](https://github.com/advisories/GHSA-69cg-p879-7622) and [CVE-2022-41723 +](https://github.com/advisories/GHSA-vvpx-j8f3-3w6h +.) [[GH-17240](https://github.com/hashicorp/consul/issues/17240)] +* extensions: Disable remote downstream proxy patching by Envoy Extensions other than AWS Lambda. Previously, an operator with service:write ACL permissions for an upstream service could modify Envoy proxy config for downstream services without equivalent permissions for those services. This issue only impacts the Lua extension. [[CVE-2023-2816](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-2816)] [[GH-17415](https://github.com/hashicorp/consul/issues/17415)] + +FEATURES: + +* hcp: Add new metrics sink to collect, aggregate and export server metrics to HCP in OTEL format. [[GH-17460](https://github.com/hashicorp/consul/issues/17460)] + +IMPROVEMENTS: + +* Fixes a performance issue in Raft where commit latency can increase by 100x or more when under heavy load. For more details see https://github.com/hashicorp/raft/pull/541. [[GH-17081](https://github.com/hashicorp/consul/issues/17081)] +* agent: add a configurable maximimum age (default: 7 days) to prevent servers re-joining a cluster with stale data [[GH-17171](https://github.com/hashicorp/consul/issues/17171)] +* agent: add new metrics to track cpu disk and memory usage for server hosts (defaults to: enabled) [[GH-17038](https://github.com/hashicorp/consul/issues/17038)] +* connect: update supported envoy versions to 1.22.11, 1.23.8, 1.24.6, 1.25.4 [[GH-16889](https://github.com/hashicorp/consul/issues/16889)] +* envoy: add `MaxEjectionPercent` and `BaseEjectionTime` to passive health check configs. [[GH-15979](https://github.com/hashicorp/consul/issues/15979)] +* hcp: Add support for linking existing Consul clusters to HCP management plane. [[GH-16916](https://github.com/hashicorp/consul/issues/16916)] +* logging: change snapshot log header from `agent.server.snapshot` to `agent.server.raft.snapshot` [[GH-17236](https://github.com/hashicorp/consul/issues/17236)] +* peering: allow re-establishing terminated peering from new token without deleting existing peering first. [[GH-16776](https://github.com/hashicorp/consul/issues/16776)] +* peering: gRPC queries for TrustBundleList, TrustBundleRead, PeeringList, and PeeringRead now support blocking semantics, + reducing network and CPU demand. + The HTTP APIs for Peering List and Read have been updated to support blocking. [[GH-17426](https://github.com/hashicorp/consul/issues/17426)] +* raft: Remove expensive reflection from raft/mesh hot path [[GH-16552](https://github.com/hashicorp/consul/issues/16552)] +* xds: rename envoy_hcp_metrics_bind_socket_dir to envoy_telemetry_collector_bind_socket_dir to remove HCP naming references. [[GH-17327](https://github.com/hashicorp/consul/issues/17327)] + +BUG FIXES: + +* Fix an bug where decoding some Config structs with unset pointer fields could fail with `reflect: call of reflect.Value.Type on zero Value`. [[GH-17048](https://github.com/hashicorp/consul/issues/17048)] +* acl: **(Enterprise only)** Check permissions in correct partition/namespace when resolving service in non-default partition/namespace +* acl: Fix an issue where the anonymous token was synthesized in non-primary datacenters which could cause permission errors when federating clusters with ACL replication enabled. [[GH-17231](https://github.com/hashicorp/consul/issues/17231)] +* acls: Fix ACL bug that can result in sidecar proxies having incorrect endpoints. +* connect: Fix multiple inefficient behaviors when querying service health. [[GH-17241](https://github.com/hashicorp/consul/issues/17241)] +* gateways: Fix an bug where targeting a virtual service defined by a service-resolver was broken for HTTPRoutes. [[GH-17055](https://github.com/hashicorp/consul/issues/17055)] +* grpc: ensure grpc resolver correctly uses lan/wan addresses on servers [[GH-17270](https://github.com/hashicorp/consul/issues/17270)] +* namespaces: adjusts the return type from HTTP list API to return the `api` module representation of a namespace. +This fixes an error with the `consul namespace list` command when a namespace has a deferred deletion timestamp. +* peering: Fix issue where modifying the list of exported services did not correctly replicate changes for services that exist in a non-default namespace. [[GH-17456](https://github.com/hashicorp/consul/issues/17456)] +* peering: Fix issue where peer streams could incorrectly deregister services in various scenarios. [[GH-17235](https://github.com/hashicorp/consul/issues/17235)] +* peering: ensure that merged central configs of peered upstreams for partitioned downstreams work [[GH-17179](https://github.com/hashicorp/consul/issues/17179)] +* xds: Fix possible panic that can when generating clusters before the root certificates have been fetched. [[GH-17185](https://github.com/hashicorp/consul/issues/17185)] + +## 1.15.2 (March 30, 2023) + +FEATURES: + +* xds: Allow for configuring connect proxies to send service mesh telemetry to an HCP metrics collection service. [[GH-16585](https://github.com/hashicorp/consul/issues/16585)] + +BUG FIXES: + +* audit-logging: (Enterprise only) Fix a bug where `/agent/monitor` and `/agent/metrics` endpoints return a `Streaming not supported` error when audit logs are enabled. This also fixes the delay receiving logs when running `consul monitor` against an agent with audit logs enabled. [[GH-16700](https://github.com/hashicorp/consul/issues/16700)] +* ca: Fixes a bug where updating Vault CA Provider config would cause TLS issues in the service mesh [[GH-16592](https://github.com/hashicorp/consul/issues/16592)] +* cache: revert cache refactor which could cause blocking queries to never return [[GH-16818](https://github.com/hashicorp/consul/issues/16818)] +* gateway: **(Enterprise only)** Fix bug where namespace/partition would fail to unmarshal for TCPServices. [[GH-16781](https://github.com/hashicorp/consul/issues/16781)] +* gateway: **(Enterprise only)** Fix bug where namespace/partition would fail to unmarshal. [[GH-16651](https://github.com/hashicorp/consul/issues/16651)] +* gateway: **(Enterprise only)** Fix bug where parent refs and service refs for a route in the same namespace as the route would fallback to the default namespace if the namespace was not specified in the configuration rather than falling back to the routes namespace. [[GH-16789](https://github.com/hashicorp/consul/issues/16789)] +* gateway: **(Enterprise only)** Fix bug where routes defined in a different namespace than a gateway would fail to register. [[GH-16677](https://github.com/hashicorp/consul/pull/16677)]. +* gateways: Adds validation to ensure the API Gateway has a listener defined when created [[GH-16649](https://github.com/hashicorp/consul/issues/16649)] +* gateways: Fixes a bug API gateways using HTTP listeners were taking upwards of 15 seconds to get configured over xDS. [[GH-16661](https://github.com/hashicorp/consul/issues/16661)] +* peering: **(Consul Enterprise only)** Fix issue where connect-enabled services with peer upstreams incorrectly required `service:write` access in the `default` namespace to query data, which was too restrictive. Now having `service:write` to any namespace is sufficient to query the peering data. +* peering: **(Consul Enterprise only)** Fix issue where resolvers, routers, and splitters referencing peer targets may not work correctly for non-default partitions and namespaces. Enterprise customers leveraging peering are encouraged to upgrade both servers and agents to avoid this problem. +* peering: Fix issue resulting in prepared query failover to cluster peers never un-failing over. [[GH-16729](https://github.com/hashicorp/consul/issues/16729)] +* peering: Fixes a bug that can lead to peering service deletes impacting the state of local services [[GH-16570](https://github.com/hashicorp/consul/issues/16570)] +* peering: Fixes a bug where the importing partition was not added to peered failover targets, which causes issues when the importing partition is a non-default partition. [[GH-16675](https://github.com/hashicorp/consul/issues/16675)] +* raft_logstore: Fixes a bug where restoring a snapshot when using the experimental WAL storage backend causes a panic. [[GH-16647](https://github.com/hashicorp/consul/issues/16647)] +* ui: fix PUT token request with adding missed AccessorID property to requestBody [[GH-16660](https://github.com/hashicorp/consul/issues/16660)] +* ui: fix rendering issues on Overview and empty-states by addressing isHTMLSafe errors [[GH-16574](https://github.com/hashicorp/consul/issues/16574)] + +## 1.15.1 (March 7, 2023) + +IMPROVEMENTS: + +* cli: added `-append-policy-id`, `-append-policy-name`, `-append-role-name`, and `-append-role-id` flags to the `consul token update` command. +These flags allow updates to a token's policies/roles without having to override them completely. [[GH-16288](https://github.com/hashicorp/consul/issues/16288)] +* cli: added `-append-service-identity` and `-append-node-identity` flags to the `consul token update` command. +These flags allow updates to a token's node identities/service identities without having to override them. [[GH-16506](https://github.com/hashicorp/consul/issues/16506)] +* connect: Bump Envoy 1.22.5 to 1.22.7, 1.23.2 to 1.23.4, 1.24.0 to 1.24.2, add 1.25.1, remove 1.21.5 [[GH-16274](https://github.com/hashicorp/consul/issues/16274)] +* mesh: Add ServiceResolver RequestTimeout for route timeouts to make request timeouts configurable [[GH-16495](https://github.com/hashicorp/consul/issues/16495)] +* ui: support filtering API gateways in the ui and displaying their documentation links [[GH-16508](https://github.com/hashicorp/consul/issues/16508)] + +DEPRECATIONS: + +* cli: Deprecate the `-merge-node-identites` and `-merge-service-identities` flags from the `consul token update` command in favor of: `-append-node-identity` and `-append-service-identity`. [[GH-16506](https://github.com/hashicorp/consul/issues/16506)] +* cli: Deprecate the `-merge-policies` and `-merge-roles` flags from the `consul token update` command in favor of: `-append-policy-id`, `-append-policy-name`, `-append-role-name`, and `-append-role-id`. [[GH-16288](https://github.com/hashicorp/consul/issues/16288)] + +BUG FIXES: + +* cli: Fixes an issue with `consul connect envoy` where a log to STDOUT could malform JSON when used with `-bootstrap`. [[GH-16530](https://github.com/hashicorp/consul/issues/16530)] +* cli: Fixes an issue with `consul connect envoy` where grpc-disabled agents were not error-handled correctly. [[GH-16530](https://github.com/hashicorp/consul/issues/16530)] +* cli: ensure acl token read -self works [[GH-16445](https://github.com/hashicorp/consul/issues/16445)] +* cli: fix panic read non-existent acl policy [[GH-16485](https://github.com/hashicorp/consul/issues/16485)] +* gateways: fix HTTPRoute bug where service weights could be less than or equal to 0 and result in a downstream envoy protocol error [[GH-16512](https://github.com/hashicorp/consul/issues/16512)] +* gateways: fix HTTPRoute bug where services with a weight not divisible by 10000 are never registered properly [[GH-16531](https://github.com/hashicorp/consul/issues/16531)] +* mesh: Fix resolution of service resolvers with subsets for external upstreams [[GH-16499](https://github.com/hashicorp/consul/issues/16499)] +* proxycfg: ensure that an irrecoverable error in proxycfg closes the xds session and triggers a replacement proxycfg watcher [[GH-16497](https://github.com/hashicorp/consul/issues/16497)] +* proxycfg: fix a bug where terminating gateways were not cleaning up deleted service resolvers for their referenced services [[GH-16498](https://github.com/hashicorp/consul/issues/16498)] +* ui: Fix issue with lists and filters not rendering properly [[GH-16444](https://github.com/hashicorp/consul/issues/16444)] + +## 1.15.0 (February 23, 2023) + +KNOWN ISSUES: + +* connect: A race condition can cause some service instances to lose their ability to communicate in the mesh after 72 hours (LeafCertTTL) due to a problem with leaf certificate rotation. This bug is fixed in Consul v1.15.2 by [GH-16818](https://github.com/hashicorp/consul/issues/16818). + +BREAKING CHANGES: + +* acl errors: Delete and get requests now return descriptive errors when the specified resource cannot be found. Other ACL request errors provide more information about when a resource is missing. Add error for when the ACL system has not been bootstrapped. + + Delete Token/Policy/AuthMethod/Role/BindingRule endpoints now return 404 when the resource cannot be found. + - New error formats: "Requested * does not exist: ACL not found", "* not found in namespace $NAMESPACE: ACL not found" + + Read Token/Policy/Role endpoints now return 404 when the resource cannot be found. + - New error format: "Cannot find * to delete" + + Logout now returns a 401 error when the supplied token cannot be found + - New error format: "Supplied token does not exist" + + Token Self endpoint now returns 404 when the token cannot be found. + - New error format: "Supplied token does not exist" [[GH-16105](https://github.com/hashicorp/consul/issues/16105)] +* acl: remove all acl migration functionality and references to the legacy acl system. [[GH-15947](https://github.com/hashicorp/consul/issues/15947)] +* acl: remove all functionality and references for legacy acl policies. [[GH-15922](https://github.com/hashicorp/consul/issues/15922)] +* config: Deprecate `-join`, `-join-wan`, `start_join`, and `start_join_wan`. +These options are now aliases of `-retry-join`, `-retry-join-wan`, `retry_join`, and `retry_join_wan`, respectively. [[GH-15598](https://github.com/hashicorp/consul/issues/15598)] +* connect: Add `peer` field to service-defaults upstream overrides. The addition of this field makes it possible to apply upstream overrides only to peer services. Prior to this change, overrides would be applied based on matching the `namespace` and `name` fields only, which means users could not have different configuration for local versus peer services. With this change, peer upstreams are only affected if the `peer` field matches the destination peer name. [[GH-15956](https://github.com/hashicorp/consul/issues/15956)] +* connect: Consul will now error and exit when using the `consul connect envoy` command if the Envoy version is incompatible. To ignore this check use flag `--ignore-envoy-compatibility` [[GH-15818](https://github.com/hashicorp/consul/issues/15818)] +* extensions: Refactor Lambda integration to get configured with the Envoy extensions field on service-defaults configuration entries. [[GH-15817](https://github.com/hashicorp/consul/issues/15817)] +* ingress-gateway: upstream cluster will have empty outlier_detection if passive health check is unspecified [[GH-15614](https://github.com/hashicorp/consul/issues/15614)] +* xds: Remove the `connect.enable_serverless_plugin` agent configuration option. Now +Lambda integration is enabled by default. [[GH-15710](https://github.com/hashicorp/consul/issues/15710)] + +SECURITY: + +* Upgrade to use Go 1.20.1. +This resolves vulnerabilities [CVE-2022-41724](https://go.dev/issue/58001) in `crypto/tls` and [CVE-2022-41723](https://go.dev/issue/57855) in `net/http`. [[GH-16263](https://github.com/hashicorp/consul/issues/16263)] + +FEATURES: + +* **API Gateway (Beta)** This version adds support for API gateway on VMs. API gateway provides a highly-configurable ingress for requests coming into a Consul network. For more information, refer to the [API gateway](https://developer.hashicorp.com/consul/docs/connect/gateways/api-gateway) documentation. [[GH-16369](https://github.com/hashicorp/consul/issues/16369)] +* acl: Add new `acl.tokens.config_file_registration` config field which specifies the token used +to register services and checks that are defined in config files. [[GH-15828](https://github.com/hashicorp/consul/issues/15828)] +* acl: anonymous token is logged as 'anonymous token' instead of its accessor ID [[GH-15884](https://github.com/hashicorp/consul/issues/15884)] +* cli: adds new CLI commands `consul troubleshoot upstreams` and `consul troubleshoot proxy` to troubleshoot Consul's service mesh configuration and network issues. [[GH-16284](https://github.com/hashicorp/consul/issues/16284)] +* command: Adds the `operator usage instances` subcommand for displaying total services, connect service instances and billable service instances in the local datacenter or globally. [[GH-16205](https://github.com/hashicorp/consul/issues/16205)] +* config-entry(ingress-gateway): support outlier detection (passive health check) for upstream cluster [[GH-15614](https://github.com/hashicorp/consul/issues/15614)] +* connect: adds support for Envoy [access logging](https://developer.hashicorp.com/consul/docs/connect/observability/access-logs). Access logging can be enabled using the [`proxy-defaults`](https://developer.hashicorp.com/consul/docs/connect/config-entries/proxy-defaults#accesslogs) config entry. [[GH-15864](https://github.com/hashicorp/consul/issues/15864)] +* xds: Add a built-in Envoy extension that inserts Lua HTTP filters. [[GH-15906](https://github.com/hashicorp/consul/issues/15906)] +* xds: Insert originator service identity into Envoy's dynamic metadata under the `consul` namespace. [[GH-15906](https://github.com/hashicorp/consul/issues/15906)] + +IMPROVEMENTS: + +* connect: for early awareness of Envoy incompatibilities, when using the `consul connect envoy` command the Envoy version will now be checked for compatibility. If incompatible Consul will error and exit. [[GH-15818](https://github.com/hashicorp/consul/issues/15818)] +* grpc: client agents will switch server on error, and automatically retry on `RESOURCE_EXHAUSTED` responses [[GH-15892](https://github.com/hashicorp/consul/issues/15892)] +* raft: add an operator api endpoint and a command to initiate raft leadership transfer. [[GH-14132](https://github.com/hashicorp/consul/issues/14132)] +* acl: Added option to allow for an operator-generated bootstrap token to be passed to the `acl bootstrap` command. [[GH-14437](https://github.com/hashicorp/consul/issues/14437)] +* agent: Give better error when client specifies wrong datacenter when auto-encrypt is enabled. [[GH-14832](https://github.com/hashicorp/consul/issues/14832)] +* api: updated the go module directive to 1.18. [[GH-15297](https://github.com/hashicorp/consul/issues/15297)] +* ca: support Vault agent auto-auth config for Vault CA provider using AWS/GCP authentication. [[GH-15970](https://github.com/hashicorp/consul/issues/15970)] +* cli: always use name "global" for proxy-defaults config entries [[GH-14833](https://github.com/hashicorp/consul/issues/14833)] +* cli: connect envoy command errors if grpc ports are not open [[GH-15794](https://github.com/hashicorp/consul/issues/15794)] +* client: add support for RemoveEmptyTags in Prepared Queries templates. [[GH-14244](https://github.com/hashicorp/consul/issues/14244)] +* connect: Warn if ACLs are enabled but a token is not provided to envoy [[GH-15967](https://github.com/hashicorp/consul/issues/15967)] +* container: Upgrade container image to use to Alpine 3.17. [[GH-16358](https://github.com/hashicorp/consul/issues/16358)] +* dns: support RFC 2782 SRV lookups for prepared queries using format `_._tcp.query[.].`. [[GH-14465](https://github.com/hashicorp/consul/issues/14465)] +* ingress-gateways: Don't log error when gateway is registered without a config entry [[GH-15001](https://github.com/hashicorp/consul/issues/15001)] +* licensing: **(Enterprise Only)** Consul Enterprise non-terminating production licenses do not degrade or terminate Consul upon expiration. They will only fail when trying to upgrade to a newer version of Consul. Evaluation licenses still terminate. +* raft: Added experimental `wal` backend for log storage. [[GH-16176](https://github.com/hashicorp/consul/issues/16176)] +* sdk: updated the go module directive to 1.18. [[GH-15297](https://github.com/hashicorp/consul/issues/15297)] +* telemetry: Added a `consul.xds.server.streamsUnauthenticated` metric to track +the number of active xDS streams handled by the server that are unauthenticated +because ACLs are not enabled or ACL tokens were missing. [[GH-15967](https://github.com/hashicorp/consul/issues/15967)] +* ui: Update sidebar width to 280px [[GH-16204](https://github.com/hashicorp/consul/issues/16204)] +* ui: update Ember version to 3.27; [[GH-16227](https://github.com/hashicorp/consul/issues/16227)] + +DEPRECATIONS: + +* acl: Deprecate the `token` query parameter and warn when it is used for authentication. [[GH-16009](https://github.com/hashicorp/consul/issues/16009)] +* cli: The `-id` flag on acl token operations has been changed to `-accessor-id` for clarity in documentation. The `-id` flag will continue to work, but operators should use `-accessor-id` in the future. [[GH-16044](https://github.com/hashicorp/consul/issues/16044)] + +BUG FIXES: + +* agent configuration: Fix issue of using unix socket when https is used. [[GH-16301](https://github.com/hashicorp/consul/issues/16301)] +* cache: refactor agent cache fetching to prevent unnecessary fetches on error [[GH-14956](https://github.com/hashicorp/consul/issues/14956)] +* cli: fatal error if config file does not have HCL or JSON extension, instead of warn and skip [[GH-15107](https://github.com/hashicorp/consul/issues/15107)] +* cli: fix ACL token processing unexpected precedence [[GH-15274](https://github.com/hashicorp/consul/issues/15274)] +* peering: Fix bug where services were incorrectly imported as connect-enabled. [[GH-16339](https://github.com/hashicorp/consul/issues/16339)] +* peering: Fix issue where mesh gateways would use the wrong address when contacting a remote peer with the same datacenter name. [[GH-16257](https://github.com/hashicorp/consul/issues/16257)] +* peering: Fix issue where secondary wan-federated datacenters could not be used as peering acceptors. [[GH-16230](https://github.com/hashicorp/consul/issues/16230)] + ## 1.14.4 (January 26, 2023) BREAKING CHANGES: diff --git a/Dockerfile b/Dockerfile index 4882f1b46d9..f5550f8f14d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ # Official docker image that includes binaries from releases.hashicorp.com. This # downloads the release from releases.hashicorp.com and therefore requires that # the release is published before building the Docker image. -FROM docker.mirror.hashicorp.services/alpine:3.15 as official +FROM docker.mirror.hashicorp.services/alpine:3.17 as official # This is the release of Consul to pull in. ARG VERSION @@ -109,7 +109,7 @@ CMD ["agent", "-dev", "-client", "0.0.0.0"] # Production docker image that uses CI built binaries. # Remember, this image cannot be built locally. -FROM docker.mirror.hashicorp.services/alpine:3.15 as default +FROM docker.mirror.hashicorp.services/alpine:3.17 as default ARG PRODUCT_VERSION ARG BIN_NAME @@ -198,7 +198,7 @@ CMD ["agent", "-dev", "-client", "0.0.0.0"] # Red Hat UBI-based image # This target is used to build a Consul image for use on OpenShift. -FROM registry.access.redhat.com/ubi9-minimal:9.1.0 as ubi +FROM registry.access.redhat.com/ubi9-minimal:9.2 as ubi ARG PRODUCT_NAME ARG PRODUCT_VERSION diff --git a/GNUmakefile b/GNUmakefile index f1cebb68955..3541fc5d10a 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -7,11 +7,11 @@ SHELL = bash # These version variables can either be a valid string for "go install @" # or the string @DEV to imply use what is currently installed locally. ### -GOLANGCI_LINT_VERSION='v1.50.1' -MOCKERY_VERSION='v2.15.0' +GOLANGCI_LINT_VERSION='v1.51.1' +MOCKERY_VERSION='v2.20.0' BUF_VERSION='v1.4.0' PROTOC_GEN_GO_GRPC_VERSION="v1.2.0" -MOG_VERSION='v0.3.0' +MOG_VERSION='v0.4.0' PROTOC_GO_INJECT_TAG_VERSION='v1.3.0' PROTOC_GEN_GO_BINARY_VERSION="v0.1.0" DEEP_COPY_VERSION='bc3f5aa5735d8a54961580a3a24422c308c831c2' @@ -164,7 +164,7 @@ dev-build: dev-docker: linux dev-build @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" - @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null + @docker pull hashicorp/consul:$(CONSUL_IMAGE_VERSION) >/dev/null @echo "Building Consul Development container - $(CONSUL_DEV_IMAGE)" @# 'consul:local' tag is needed to run the integration tests @# 'consul-dev:latest' is needed by older workflows @@ -183,7 +183,7 @@ remote-docker: check-remote-dev-image-env $(MAKE) GOARCH=amd64 linux $(MAKE) GOARCH=arm64 linux @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" - @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null + @docker pull hashicorp/consul:$(CONSUL_IMAGE_VERSION) >/dev/null @echo "Building and Pushing Consul Development container - $(REMOTE_DEV_IMAGE)" @if ! docker buildx inspect consul-builder; then \ docker buildx create --name consul-builder --driver docker-container --bootstrap; \ @@ -194,11 +194,11 @@ remote-docker: check-remote-dev-image-env --push \ -f $(CURDIR)/build-support/docker/Consul-Dev-Multiarch.dockerfile $(CURDIR)/pkg/bin/ -# In CircleCI, the linux binary will be attached from a previous step at bin/. This make target +# In CI, the linux binary will be attached from a previous step at bin/. This make target # should only run in CI and not locally. ci.dev-docker: @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" - @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null + @docker pull hashicorp/consul:$(CONSUL_IMAGE_VERSION) >/dev/null @echo "Building Consul Development container - $(CI_DEV_DOCKER_IMAGE_NAME)" @docker build $(NOCACHE) $(QUIET) -t '$(CI_DEV_DOCKER_NAMESPACE)/$(CI_DEV_DOCKER_IMAGE_NAME):$(GIT_COMMIT)' \ --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \ @@ -369,6 +369,8 @@ codegen-tools: deep-copy: codegen-tools @$(SHELL) $(CURDIR)/agent/structs/deep-copy.sh @$(SHELL) $(CURDIR)/agent/proxycfg/deep-copy.sh + @$(SHELL) $(CURDIR)/agent/consul/state/deep-copy.sh + @$(SHELL) $(CURDIR)/agent/config/deep-copy.sh version: @echo -n "Version: " @@ -391,6 +393,7 @@ ui-build-image: @echo "Building UI build container" @docker build $(NOCACHE) $(QUIET) -t $(UI_BUILD_TAG) - < build-support/docker/Build-UI.dockerfile +# Builds consul in a docker container and then dumps executable into ./pkg/bin/... consul-docker: go-build-image @$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh consul @@ -459,20 +462,10 @@ test-metrics-integ: test-compat-integ-setup --latest-version latest test-connect-ca-providers: -ifeq ("$(CIRCLECI)","true") -# Run in CI - gotestsum --format=short-verbose --junitfile "$(TEST_RESULTS_DIR)/gotestsum-report.xml" -- -cover -coverprofile=coverage.txt ./agent/connect/ca -# Run leader tests that require Vault - gotestsum --format=short-verbose --junitfile "$(TEST_RESULTS_DIR)/gotestsum-report-leader.xml" -- -cover -coverprofile=coverage-leader.txt -run Vault ./agent/consul -# Run agent tests that require Vault - gotestsum --format=short-verbose --junitfile "$(TEST_RESULTS_DIR)/gotestsum-report-agent.xml" -- -cover -coverprofile=coverage-agent.txt -run Vault ./agent -else -# Run locally @echo "Running /agent/connect/ca tests in verbose mode" @go test -v ./agent/connect/ca @go test -v ./agent/consul -run Vault @go test -v ./agent -run Vault -endif .PHONY: proto proto: proto-tools proto-gen proto-mocks @@ -535,6 +528,11 @@ envoy-regen: @find "command/connect/envoy/testdata" -name '*.golden' -delete @go test -tags '$(GOTAGS)' ./command/connect/envoy -update +# Point your web browser to http://localhost:3000/consul to live render docs from ./website/ +.PHONY: docs +docs: + make -C website + .PHONY: help help: $(info available make targets) diff --git a/acl/acl.go b/acl/acl.go index c18ba0b0781..29413974c40 100644 --- a/acl/acl.go +++ b/acl/acl.go @@ -9,6 +9,8 @@ const ( AnonymousTokenID = "00000000-0000-0000-0000-000000000002" AnonymousTokenAlias = "anonymous token" AnonymousTokenSecret = "anonymous" + + ReservedBuiltinPrefix = "builtin/" ) // Config encapsulates all of the generic configuration parameters used for diff --git a/acl/acl_oss.go b/acl/acl_ce.go similarity index 95% rename from acl/acl_oss.go rename to acl/acl_ce.go index 48f671ac7ae..26485503564 100644 --- a/acl/acl_oss.go +++ b/acl/acl_ce.go @@ -14,7 +14,7 @@ const ( const DefaultNamespaceName = "default" type EnterpriseConfig struct { - // no fields in OSS + // no fields in CE } func (_ *EnterpriseConfig) Close() { diff --git a/acl/authorizer_oss.go b/acl/authorizer_ce.go similarity index 100% rename from acl/authorizer_oss.go rename to acl/authorizer_ce.go diff --git a/acl/enterprisemeta_oss.go b/acl/enterprisemeta_ce.go similarity index 100% rename from acl/enterprisemeta_oss.go rename to acl/enterprisemeta_ce.go diff --git a/acl/errors_oss.go b/acl/errors_ce.go similarity index 100% rename from acl/errors_oss.go rename to acl/errors_ce.go diff --git a/acl/policy_authorizer_oss.go b/acl/policy_authorizer_ce.go similarity index 100% rename from acl/policy_authorizer_oss.go rename to acl/policy_authorizer_ce.go diff --git a/acl/policy_oss.go b/acl/policy_ce.go similarity index 100% rename from acl/policy_oss.go rename to acl/policy_ce.go diff --git a/acl/policy_merger_oss.go b/acl/policy_merger_ce.go similarity index 100% rename from acl/policy_merger_oss.go rename to acl/policy_merger_ce.go diff --git a/acl/validation.go b/acl/validation.go index 816ec0cae1f..a8151e121d0 100644 --- a/acl/validation.go +++ b/acl/validation.go @@ -1,16 +1,21 @@ package acl -import "regexp" +import ( + "fmt" + "regexp" + "strings" +) const ( ServiceIdentityNameMaxLength = 256 NodeIdentityNameMaxLength = 256 + PolicyNameMaxLength = 128 ) var ( validServiceIdentityName = regexp.MustCompile(`^[a-z0-9]([a-z0-9\-_]*[a-z0-9])?$`) validNodeIdentityName = regexp.MustCompile(`^[a-z0-9]([a-z0-9\-_]*[a-z0-9])?$`) - validPolicyName = regexp.MustCompile(`^[A-Za-z0-9\-_]{1,128}$`) + validPolicyName = regexp.MustCompile(`^[A-Za-z0-9\-_]+\/?[A-Za-z0-9\-_]*$`) validRoleName = regexp.MustCompile(`^[A-Za-z0-9\-_]{1,256}$`) validAuthMethodName = regexp.MustCompile(`^[A-Za-z0-9\-_]{1,128}$`) ) @@ -37,10 +42,21 @@ func IsValidNodeIdentityName(name string) bool { return validNodeIdentityName.MatchString(name) } -// IsValidPolicyName returns true if the provided name can be used as an -// ACLPolicy Name. -func IsValidPolicyName(name string) bool { - return validPolicyName.MatchString(name) +// ValidatePolicyName returns nil if the provided name can be used as an +// ACLPolicy Name otherwise a useful error is returned. +func ValidatePolicyName(name string) error { + if len(name) < 1 || len(name) > PolicyNameMaxLength { + return fmt.Errorf("Invalid Policy: invalid Name. Length must be greater than 0 and less than %d", PolicyNameMaxLength) + } + + if strings.HasPrefix(name, "/") || strings.HasPrefix(name, ReservedBuiltinPrefix) { + return fmt.Errorf("Invalid Policy: invalid Name. Names cannot be prefixed with '/' or '%s'", ReservedBuiltinPrefix) + } + + if !validPolicyName.MatchString(name) { + return fmt.Errorf("Invalid Policy: invalid Name. Only alphanumeric characters, a single '/', '-' and '_' are allowed") + } + return nil } // IsValidRoleName returns true if the provided name can be used as an diff --git a/acl/validation_test.go b/acl/validation_test.go new file mode 100644 index 00000000000..d5d01e0e905 --- /dev/null +++ b/acl/validation_test.go @@ -0,0 +1,78 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package acl + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_ValidatePolicyName(t *testing.T) { + for _, tc := range []struct { + description string + name string + valid bool + }{ + { + description: "valid policy", + name: "this-is-valid", + valid: true, + }, + { + description: "empty policy", + name: "", + valid: false, + }, + { + description: "with slash", + name: "policy/with-slash", + valid: true, + }, + { + description: "leading slash", + name: "/no-leading-slash", + valid: false, + }, + { + description: "too many slashes", + name: "too/many/slashes", + valid: false, + }, + { + description: "no double-slash", + name: "no//double-slash", + valid: false, + }, + { + description: "builtin prefix", + name: "builtin/prefix-cannot-be-used", + valid: false, + }, + { + description: "long", + name: "this-policy-name-is-very-very-long-but-it-is-okay-because-it-is-the-max-length-that-we-allow-here-in-a-policy-name-which-is-good", + valid: true, + }, + { + description: "too long", + name: "this-is-a-policy-that-has-one-character-too-many-it-is-way-too-long-for-a-policy-we-do-not-want-a-policy-of-this-length-because-1", + valid: false, + }, + { + description: "invalid start character", + name: "!foo", + valid: false, + }, + { + description: "invalid character", + name: "this%is%bad", + valid: false, + }, + } { + t.Run(tc.description, func(t *testing.T) { + require.Equal(t, tc.valid, ValidatePolicyName(tc.name) == nil) + }) + } +} diff --git a/agent/acl_oss.go b/agent/acl_ce.go similarity index 100% rename from agent/acl_oss.go rename to agent/acl_ce.go diff --git a/agent/acl_endpoint.go b/agent/acl_endpoint.go index af9f3a15d90..8e952392a86 100644 --- a/agent/acl_endpoint.go +++ b/agent/acl_endpoint.go @@ -3,6 +3,7 @@ package agent import ( "fmt" "net/http" + "net/url" "strings" "github.com/hashicorp/consul/acl" @@ -142,6 +143,12 @@ func (s *HTTPHandlers) ACLPolicyCRUD(resp http.ResponseWriter, req *http.Request } func (s *HTTPHandlers) ACLPolicyRead(resp http.ResponseWriter, req *http.Request, policyID, policyName string) (interface{}, error) { + // policy name needs to be unescaped in case there were `/` characters + policyName, err := url.QueryUnescape(policyName) + if err != nil { + return nil, err + } + args := structs.ACLPolicyGetRequest{ Datacenter: s.agent.config.Datacenter, PolicyID: policyID, @@ -284,6 +291,7 @@ func (s *HTTPHandlers) ACLTokenList(resp http.ResponseWriter, req *http.Request) args.Policy = req.URL.Query().Get("policy") args.Role = req.URL.Query().Get("role") args.AuthMethod = req.URL.Query().Get("authmethod") + args.ServiceName = req.URL.Query().Get("servicename") if err := parseACLAuthMethodEnterpriseMeta(req, &args.ACLAuthMethodEnterpriseMeta); err != nil { return nil, err } @@ -438,8 +446,16 @@ func (s *HTTPHandlers) aclTokenSetInternal(req *http.Request, tokenAccessorID st return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: fmt.Sprintf("Token decoding failed: %v", err)} } - if !create && args.ACLToken.AccessorID != tokenAccessorID { - return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: "Token Accessor ID in URL and payload do not match"} + if !create { + // NOTE: AccessorID in the request body is optional when not creating a new token. + // If not present in the body and only in the URL then it will be filled in by Consul. + if args.ACLToken.AccessorID == "" { + args.ACLToken.AccessorID = tokenAccessorID + } + + if args.ACLToken.AccessorID != tokenAccessorID { + return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: "Token Accessor ID in URL and payload do not match"} + } } var out structs.ACLToken diff --git a/agent/acl_endpoint_test.go b/agent/acl_endpoint_test.go index d2822e39e9a..4ff21fe4723 100644 --- a/agent/acl_endpoint_test.go +++ b/agent/acl_endpoint_test.go @@ -435,8 +435,8 @@ func TestACL_HTTP(t *testing.T) { policies, ok := raw.(structs.ACLPolicyListStubs) require.True(t, ok) - // 2 we just created + global management - require.Len(t, policies, 3) + // 2 we just created + builtin policies + require.Len(t, policies, 2+len(structs.ACLBuiltinPolicies)) for policyID, expected := range policyMap { found := false @@ -904,6 +904,48 @@ func TestACL_HTTP(t *testing.T) { tokenMap[token.AccessorID] = token }) + t.Run("Update without AccessorID in request body", func(t *testing.T) { + originalToken := tokenMap[idMap["token-cloned"]] + + // Secret will be filled in + tokenInput := &structs.ACLToken{ + Description: "Even Better description for this cloned token", + Policies: []structs.ACLTokenPolicyLink{ + { + ID: idMap["policy-read-all-nodes"], + Name: policyMap[idMap["policy-read-all-nodes"]].Name, + }, + }, + NodeIdentities: []*structs.ACLNodeIdentity{ + { + NodeName: "foo", + Datacenter: "bar", + }, + }, + } + + req, _ := http.NewRequest("PUT", "/v1/acl/token/"+originalToken.AccessorID, jsonBody(tokenInput)) + req.Header.Add("X-Consul-Token", "root") + resp := httptest.NewRecorder() + obj, err := a.srv.ACLTokenCRUD(resp, req) + require.NoError(t, err) + token, ok := obj.(*structs.ACLToken) + require.True(t, ok) + + require.Equal(t, originalToken.AccessorID, token.AccessorID) + require.Equal(t, originalToken.SecretID, token.SecretID) + require.Equal(t, tokenInput.Description, token.Description) + require.Equal(t, tokenInput.Policies, token.Policies) + require.Equal(t, tokenInput.NodeIdentities, token.NodeIdentities) + require.True(t, token.CreateIndex > 0) + require.True(t, token.CreateIndex < token.ModifyIndex) + require.NotNil(t, token.Hash) + require.NotEqual(t, token.Hash, []byte{}) + require.NotEqual(t, token.Hash, originalToken.Hash) + + tokenMap[token.AccessorID] = token + }) + t.Run("CRUD Missing Token Accessor ID", func(t *testing.T) { req, _ := http.NewRequest("GET", "/v1/acl/token/", nil) req.Header.Add("X-Consul-Token", "root") @@ -1283,6 +1325,38 @@ func TestACL_HTTP(t *testing.T) { require.Error(t, err) testutil.RequireErrorContains(t, err, "Only lowercase alphanumeric") }) + + t.Run("Create with valid service identity", func(t *testing.T) { + tokenInput := &structs.ACLToken{ + Description: "token for service identity sn1", + ServiceIdentities: []*structs.ACLServiceIdentity{ + { + ServiceName: "sn1", + }, + }, + } + + req, _ := http.NewRequest("PUT", "/v1/acl/token", jsonBody(tokenInput)) + req.Header.Add("X-Consul-Token", "root") + resp := httptest.NewRecorder() + _, err := a.srv.ACLTokenCreate(resp, req) + require.NoError(t, err) + }) + + t.Run("List by ServiceName", func(t *testing.T) { + req, _ := http.NewRequest("GET", "/v1/acl/tokens?servicename=sn1", nil) + req.Header.Add("X-Consul-Token", "root") + resp := httptest.NewRecorder() + raw, err := a.srv.ACLTokenList(resp, req) + require.NoError(t, err) + tokens, ok := raw.(structs.ACLTokenListStubs) + require.True(t, ok) + require.Len(t, tokens, 1) + token := tokens[0] + require.Equal(t, "token for service identity sn1", token.Description) + require.Len(t, token.ServiceIdentities, 1) + require.Equal(t, "sn1", token.ServiceIdentities[0].ServiceName) + }) }) } diff --git a/agent/agent.go b/agent/agent.go index bff47a0fcf2..64282e3200a 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "encoding/json" + "errors" "fmt" "io" "net" @@ -15,6 +16,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/armon/go-metrics" @@ -45,7 +47,6 @@ import ( grpcDNS "github.com/hashicorp/consul/agent/grpc-external/services/dns" middleware "github.com/hashicorp/consul/agent/grpc-middleware" "github.com/hashicorp/consul/agent/hcp/scada" - libscada "github.com/hashicorp/consul/agent/hcp/scada" "github.com/hashicorp/consul/agent/local" "github.com/hashicorp/consul/agent/proxycfg" proxycfgglue "github.com/hashicorp/consul/agent/proxycfg-glue" @@ -404,6 +405,8 @@ type Agent struct { // enterpriseAgent embeds fields that we only access in consul-enterprise builds enterpriseAgent + + enableDebug atomic.Bool } // New process the desired options and creates a new Agent. @@ -553,13 +556,15 @@ func (a *Agent) Start(ctx context.Context) error { return err } - // copy over the existing node id, this cannot be - // changed while running anyways but this prevents - // breaking some existing behavior. then overwrite - // the configuration + // Copy over the existing node id. This cannot be + // changed while running, but this prevents + // breaking some existing behavior. c.NodeID = a.config.NodeID + // Overwrite the configuration. a.config = c + a.enableDebug.Store(c.EnableDebug) + if err := a.tlsConfigurator.Update(a.config.TLS); err != nil { return fmt.Errorf("Failed to load TLS configurations after applying auto-config settings: %w", err) } @@ -603,6 +608,22 @@ func (a *Agent) Start(ctx context.Context) error { if c.ServerMode { serverLogger := a.baseDeps.Logger.NamedIntercept(logging.ConsulServer) + // Check for a last seen timestamp and exit if deemed stale before attempting to join + // Serf/Raft or listen for requests. + if err := a.checkServerLastSeen(consul.ReadServerMetadata); err != nil { + deadline := time.Now().Add(time.Minute) + for time.Now().Before(deadline) { + a.logger.Error("startup error", "error", err) + time.Sleep(10 * time.Second) + } + return err + } + + // Periodically write server metadata to disk. + if !consulCfg.DevMode { + go a.persistServerMetadata() + } + incomingRPCLimiter := consul.ConfiguredIncomingRPCLimiter( &lib.StopChannelContext{StopCh: a.shutdownCh}, serverLogger, @@ -639,7 +660,6 @@ func (a *Agent) Start(ctx context.Context) error { return fmt.Errorf("failed to start server cert manager: %w", err) } } - } else { a.externalGRPCServer = external.NewServer( a.logger.Named("grpc.external"), @@ -721,11 +741,12 @@ func (a *Agent) Start(ctx context.Context) error { go localproxycfg.Sync( &lib.StopChannelContext{StopCh: a.shutdownCh}, localproxycfg.SyncConfig{ - Manager: a.proxyConfig, - State: a.State, - Logger: a.proxyConfig.Logger.Named("agent-state"), - Tokens: a.baseDeps.Tokens, - NodeName: a.config.NodeName, + Manager: a.proxyConfig, + State: a.State, + Logger: a.proxyConfig.Logger.Named("agent-state"), + Tokens: a.baseDeps.Tokens, + NodeName: a.config.NodeName, + ResyncFrequency: a.config.LocalProxyConfigResyncInterval, }, ) @@ -792,12 +813,6 @@ func (a *Agent) Start(ctx context.Context) error { go m.Monitor(&lib.StopChannelContext{StopCh: a.shutdownCh}) } - // consul version metric with labels - metrics.SetGaugeWithLabels([]string{"version"}, 1, []metrics.Label{ - {Name: "version", Value: a.config.VersionWithMetadata()}, - {Name: "pre_release", Value: a.config.VersionPrerelease}, - }) - // start a go routine to reload config based on file watcher events if a.configFileWatcher != nil { a.baseDeps.Logger.Debug("starting file watcher") @@ -1051,7 +1066,8 @@ func (a *Agent) listenHTTP() ([]apiServer, error) { for _, l := range listeners { var tlscfg *tls.Config _, isTCP := l.(*tcpKeepAliveListener) - if isTCP && proto == "https" { + isUnix := l.Addr().Network() == "unix" + if (isTCP || isUnix) && proto == "https" { tlscfg = a.tlsConfigurator.IncomingHTTPSConfig() l = tls.NewListener(l, tlscfg) } @@ -1066,13 +1082,13 @@ func (a *Agent) listenHTTP() ([]apiServer, error) { httpServer := &http.Server{ Addr: l.Addr().String(), TLSConfig: tlscfg, - Handler: srv.handler(a.config.EnableDebug), + Handler: srv.handler(), MaxHeaderBytes: a.config.HTTPMaxHeaderBytes, } - if libscada.IsCapability(l.Addr()) { + if scada.IsCapability(l.Addr()) { // wrap in http2 server handler - httpServer.Handler = h2c.NewHandler(srv.handler(a.config.EnableDebug), &http2.Server{}) + httpServer.Handler = h2c.NewHandler(srv.handler(), &http2.Server{}) } // Load the connlimit helper into the server @@ -1492,7 +1508,14 @@ func newConsulConfig(runtimeCfg *config.RuntimeConfig, logger hclog.Logger) (*co cfg.RequestLimitsReadRate = runtimeCfg.RequestLimitsReadRate cfg.RequestLimitsWriteRate = runtimeCfg.RequestLimitsWriteRate + cfg.Cloud.ManagementToken = runtimeCfg.Cloud.ManagementToken + + cfg.Reporting.License.Enabled = runtimeCfg.Reporting.License.Enabled + + cfg.ServerRejoinAgeMax = runtimeCfg.ServerRejoinAgeMax + enterpriseConsulConfig(cfg, runtimeCfg) + return cfg, nil } @@ -1598,10 +1621,7 @@ func (a *Agent) ShutdownAgent() error { a.stopLicenseManager() - // this would be cancelled anyways (by the closing of the shutdown ch) but - // this should help them to be stopped more quickly - a.baseDeps.AutoConfig.Stop() - a.baseDeps.MetricsConfig.Cancel() + a.baseDeps.Close() a.stateLock.Lock() defer a.stateLock.Unlock() @@ -3669,7 +3689,7 @@ func (a *Agent) loadServices(conf *config.RuntimeConfig, snap map[structs.CheckI } if acl.EqualPartitions("", p.Service.PartitionOrEmpty()) { - // NOTE: in case loading a service with empty partition (e.g., OSS -> ENT), + // NOTE: in case loading a service with empty partition (e.g., CE -> ENT), // we always default the service partition to the agent's partition. p.Service.OverridePartition(a.AgentEnterpriseMeta().PartitionOrDefault()) } else if !acl.EqualPartitions(a.AgentEnterpriseMeta().PartitionOrDefault(), p.Service.PartitionOrDefault()) { @@ -4183,6 +4203,11 @@ func (a *Agent) reloadConfigInternal(newCfg *config.RuntimeConfig) error { HeartbeatTimeout: newCfg.ConsulRaftHeartbeatTimeout, ElectionTimeout: newCfg.ConsulRaftElectionTimeout, RaftTrailingLogs: newCfg.RaftTrailingLogs, + Reporting: consul.Reporting{ + License: consul.License{ + Enabled: newCfg.Reporting.License.Enabled, + }, + }, } if err := a.delegate.ReloadConfig(cc); err != nil { return err @@ -4208,6 +4233,12 @@ func (a *Agent) reloadConfigInternal(newCfg *config.RuntimeConfig) error { a.proxyConfig.SetUpdateRateLimit(newCfg.XDSUpdateRateLimit) + a.enableDebug.Store(newCfg.EnableDebug) + a.config.EnableDebug = newCfg.EnableDebug + + // update Agent config with new config + a.config = newCfg.DeepCopy() + return nil } @@ -4499,6 +4530,70 @@ func (a *Agent) proxyDataSources() proxycfg.DataSources { } +// persistServerMetadata periodically writes a server's metadata to a file +// in the configured data directory. +func (a *Agent) persistServerMetadata() { + file := filepath.Join(a.config.DataDir, consul.ServerMetadataFile) + + // Create a timer with no initial tick to allow metadata to be written immediately. + t := time.NewTimer(0) + defer t.Stop() + + for { + select { + case <-t.C: + // Reset the timer to the larger periodic interval. + t.Reset(1 * time.Hour) + + f, err := consul.OpenServerMetadata(file) + if err != nil { + a.logger.Error("failed to open existing server metadata", "error", err) + continue + } + + if err := consul.WriteServerMetadata(f); err != nil { + f.Close() + a.logger.Error("failed to write server metadata", "error", err) + continue + } + + f.Close() + case <-a.shutdownCh: + return + } + } +} + +// checkServerLastSeen is a safety check that only occurs once of startup to prevent old servers +// with stale data from rejoining an existing cluster. +// +// It attempts to read a server's metadata file and check the last seen Unix timestamp against a +// configurable max age. If the metadata file does not exist, we treat this as an initial startup +// and return no error. +// +// Example: if the server recorded a last seen timestamp of now-7d, and we configure a max age +// of 3d, then we should prevent the server from rejoining. +func (a *Agent) checkServerLastSeen(readFn consul.ServerMetadataReadFunc) error { + filename := filepath.Join(a.config.DataDir, consul.ServerMetadataFile) + + // Read server metadata file. + md, err := readFn(filename) + if err != nil { + // Return early if it doesn't exist as this likely indicates the server is starting for the first time. + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("error reading server metadata: %w", err) + } + + maxAge := a.config.ServerRejoinAgeMax + if md.IsLastSeenStale(maxAge) { + return fmt.Errorf("refusing to rejoin cluster because server has been offline for more than the configured server_rejoin_age_max (%s) - consider wiping your data dir", maxAge) + } + + return nil +} + func listenerPortKey(svcID structs.ServiceID, checkID structs.CheckID) string { return fmt.Sprintf("%s:%s", svcID, checkID) } diff --git a/agent/agent_oss.go b/agent/agent_ce.go similarity index 100% rename from agent/agent_oss.go rename to agent/agent_ce.go diff --git a/agent/agent_ce_test.go b/agent/agent_ce_test.go new file mode 100644 index 00000000000..ceb90beb063 --- /dev/null +++ b/agent/agent_ce_test.go @@ -0,0 +1,46 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build !consulent +// +build !consulent + +package agent + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAgent_consulConfig_Reporting(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + hcl := ` + reporting { + license { + enabled = true + } + } + ` + a := NewTestAgent(t, hcl) + defer a.Shutdown() + require.Equal(t, false, a.consulConfig().Reporting.License.Enabled) +} + +func TestAgent_consulConfig_Reporting_Default(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + hcl := ` + reporting { + } + ` + a := NewTestAgent(t, hcl) + defer a.Shutdown() + require.Equal(t, false, a.consulConfig().Reporting.License.Enabled) +} diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index c8c78f7d79d..d33e6ccc0a2 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -614,6 +614,21 @@ func (s *HTTPHandlers) AgentMembers(resp http.ResponseWriter, req *http.Request) } } + // filter the members by parsed filter expression + var filterExpression string + s.parseFilter(req, &filterExpression) + if filterExpression != "" { + filter, err := bexpr.CreateFilter(filterExpression, nil, members) + if err != nil { + return nil, err + } + raw, err := filter.Execute(members) + if err != nil { + return nil, err + } + members = raw.([]serf.Member) + } + total := len(members) if err := s.agent.filterMembers(token, &members); err != nil { return nil, err diff --git a/agent/agent_endpoint_oss.go b/agent/agent_endpoint_ce.go similarity index 100% rename from agent/agent_endpoint_oss.go rename to agent/agent_endpoint_ce.go diff --git a/agent/agent_endpoint_oss_test.go b/agent/agent_endpoint_ce_test.go similarity index 100% rename from agent/agent_endpoint_oss_test.go rename to agent/agent_endpoint_ce_test.go diff --git a/agent/agent_endpoint_test.go b/agent/agent_endpoint_test.go index c9cfbee45cb..9f89afb77d0 100644 --- a/agent/agent_endpoint_test.go +++ b/agent/agent_endpoint_test.go @@ -186,7 +186,7 @@ func TestAgent_Services_ExternalConnectProxy(t *testing.T) { Port: 5000, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "db", - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), }, } a.State.AddServiceWithChecks(srv1, nil, "", false) @@ -226,7 +226,7 @@ func TestAgent_Services_Sidecar(t *testing.T) { LocallyRegisteredAsSidecar: true, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "db", - Upstreams: structs.TestUpstreams(t), + Upstreams: structs.TestUpstreams(t, false), Mode: structs.ProxyModeTransparent, TransparentProxy: structs.TransparentProxyConfig{ OutboundListenerPort: 10101, @@ -1619,7 +1619,7 @@ func TestHTTPHandlers_AgentMetricsStream_ACLDeny(t *testing.T) { resp := httptest.NewRecorder() req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/v1/agent/metrics/stream", nil) require.NoError(t, err) - handle := h.handler(false) + handle := h.handler() handle.ServeHTTP(resp, req) require.Equal(t, http.StatusForbidden, resp.Code) require.Contains(t, resp.Body.String(), "Permission denied") @@ -1656,7 +1656,7 @@ func TestHTTPHandlers_AgentMetricsStream(t *testing.T) { resp := httptest.NewRecorder() req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/v1/agent/metrics/stream", nil) require.NoError(t, err) - handle := h.handler(false) + handle := h.handler() handle.ServeHTTP(resp, req) require.Equal(t, http.StatusOK, resp.Code) @@ -1816,7 +1816,7 @@ func TestAgent_ReloadDoesNotTriggerWatch(t *testing.T) { for i := 1; i < 7; i++ { contents, err := os.ReadFile(tmpFile) if err != nil { - t.Fatalf("should be able to read file, but had: %#v", err) + r.Fatalf("should be able to read file, but had: %#v", err) } contentsStr = string(contents) if contentsStr != "" { @@ -1903,14 +1903,14 @@ func TestAgent_ReloadDoesNotTriggerWatch(t *testing.T) { ensureNothingCritical(r, "red-is-dead") if err := a.reloadConfigInternal(cfg2); err != nil { - t.Fatalf("got error %v want nil", err) + r.Fatalf("got error %v want nil", err) } // We check that reload does not go to critical ensureNothingCritical(r, "red-is-dead") ensureNothingCritical(r, "testing-agent-reload-001") - require.NoError(t, a.updateTTLCheck(checkID, api.HealthPassing, "testing-agent-reload-002")) + require.NoError(r, a.updateTTLCheck(checkID, api.HealthPassing, "testing-agent-reload-002")) ensureNothingCritical(r, "red-is-dead") }) @@ -2920,7 +2920,7 @@ func TestAgent_RegisterCheck_ACLDeny(t *testing.T) { req, _ := http.NewRequest("PUT", "/v1/agent/check/register", jsonReader(nodeCheck)) resp := httptest.NewRecorder() a.srv.h.ServeHTTP(resp, req) - require.Equal(t, http.StatusForbidden, resp.Code) + require.Equal(r, http.StatusForbidden, resp.Code) }) }) @@ -2930,7 +2930,7 @@ func TestAgent_RegisterCheck_ACLDeny(t *testing.T) { req.Header.Add("X-Consul-Token", svcToken.SecretID) resp := httptest.NewRecorder() a.srv.h.ServeHTTP(resp, req) - require.Equal(t, http.StatusForbidden, resp.Code) + require.Equal(r, http.StatusForbidden, resp.Code) }) }) @@ -2940,7 +2940,7 @@ func TestAgent_RegisterCheck_ACLDeny(t *testing.T) { req.Header.Add("X-Consul-Token", nodeToken.SecretID) resp := httptest.NewRecorder() a.srv.h.ServeHTTP(resp, req) - require.Equal(t, http.StatusOK, resp.Code) + require.Equal(r, http.StatusOK, resp.Code) }) }) @@ -2949,7 +2949,7 @@ func TestAgent_RegisterCheck_ACLDeny(t *testing.T) { req, _ := http.NewRequest("PUT", "/v1/agent/check/register", jsonReader(svcCheck)) resp := httptest.NewRecorder() a.srv.h.ServeHTTP(resp, req) - require.Equal(t, http.StatusForbidden, resp.Code) + require.Equal(r, http.StatusForbidden, resp.Code) }) }) @@ -2959,7 +2959,7 @@ func TestAgent_RegisterCheck_ACLDeny(t *testing.T) { req.Header.Add("X-Consul-Token", nodeToken.SecretID) resp := httptest.NewRecorder() a.srv.h.ServeHTTP(resp, req) - require.Equal(t, http.StatusForbidden, resp.Code) + require.Equal(r, http.StatusForbidden, resp.Code) }) }) @@ -2969,7 +2969,7 @@ func TestAgent_RegisterCheck_ACLDeny(t *testing.T) { req.Header.Add("X-Consul-Token", svcToken.SecretID) resp := httptest.NewRecorder() a.srv.h.ServeHTTP(resp, req) - require.Equal(t, http.StatusOK, resp.Code) + require.Equal(r, http.StatusOK, resp.Code) }) }) } @@ -5970,17 +5970,17 @@ func TestAgent_Monitor(t *testing.T) { res := httptest.NewRecorder() a.srv.h.ServeHTTP(res, registerReq) if http.StatusOK != res.Code { - t.Fatalf("expected 200 but got %v", res.Code) + r.Fatalf("expected 200 but got %v", res.Code) } // Wait until we have received some type of logging output - require.Eventually(t, func() bool { + require.Eventually(r, func() bool { return len(resp.Body.Bytes()) > 0 }, 3*time.Second, 100*time.Millisecond) cancelFunc() code := <-codeCh - require.Equal(t, http.StatusOK, code) + require.Equal(r, http.StatusOK, code) got := resp.Body.String() // Only check a substring that we are highly confident in finding @@ -6004,8 +6004,10 @@ func TestAgent_Monitor(t *testing.T) { cancelCtx, cancelFunc := context.WithCancel(context.Background()) req = req.WithContext(cancelCtx) + a.enableDebug.Store(true) + resp := httptest.NewRecorder() - handler := a.srv.handler(true) + handler := a.srv.handler() go handler.ServeHTTP(resp, req) args := &structs.ServiceDefinition{ @@ -6020,11 +6022,11 @@ func TestAgent_Monitor(t *testing.T) { res := httptest.NewRecorder() a.srv.h.ServeHTTP(res, registerReq) if http.StatusOK != res.Code { - t.Fatalf("expected 200 but got %v", res.Code) + r.Fatalf("expected 200 but got %v", res.Code) } // Wait until we have received some type of logging output - require.Eventually(t, func() bool { + require.Eventually(r, func() bool { return len(resp.Body.Bytes()) > 0 }, 3*time.Second, 100*time.Millisecond) cancelFunc() @@ -6057,24 +6059,24 @@ func TestAgent_Monitor(t *testing.T) { res := httptest.NewRecorder() a.srv.h.ServeHTTP(res, registerReq) if http.StatusOK != res.Code { - t.Fatalf("expected 200 but got %v", res.Code) + r.Fatalf("expected 200 but got %v", res.Code) } // Wait until we have received some type of logging output - require.Eventually(t, func() bool { + require.Eventually(r, func() bool { return len(resp.Body.Bytes()) > 0 }, 3*time.Second, 100*time.Millisecond) cancelFunc() code := <-codeCh - require.Equal(t, http.StatusOK, code) + require.Equal(r, http.StatusOK, code) // Each line is output as a separate JSON object, we grab the first and // make sure it can be unmarshalled. firstLine := bytes.Split(resp.Body.Bytes(), []byte("\n"))[0] var output map[string]interface{} if err := json.Unmarshal(firstLine, &output); err != nil { - t.Fatalf("err: %v", err) + r.Fatalf("err: %v", err) } }) }) @@ -6666,7 +6668,7 @@ func TestAgentConnectCARoots_list(t *testing.T) { dec := json.NewDecoder(resp.Body) value := &structs.IndexedCARoots{} - require.NoError(t, dec.Decode(value)) + require.NoError(r, dec.Decode(value)) if ca.ID != value.ActiveRootID { r.Fatalf("%s != %s", ca.ID, value.ActiveRootID) } @@ -7074,7 +7076,7 @@ func TestAgentConnectCALeafCert_goodNotLocal(t *testing.T) { dec := json.NewDecoder(resp.Body) issued2 := &structs.IssuedCert{} - require.NoError(t, dec.Decode(issued2)) + require.NoError(r, dec.Decode(issued2)) if issued.CertPEM == issued2.CertPEM { r.Fatalf("leaf has not updated") } @@ -7086,9 +7088,9 @@ func TestAgentConnectCALeafCert_goodNotLocal(t *testing.T) { } // Verify that the cert is signed by the new CA - requireLeafValidUnderCA(t, issued2, ca) + requireLeafValidUnderCA(r, issued2, ca) - require.NotEqual(t, issued, issued2) + require.NotEqual(r, issued, issued2) }) } } @@ -7465,11 +7467,11 @@ func TestAgentConnectCALeafCert_secondaryDC_good(t *testing.T) { // Try and sign again (note no index/wait arg since cache should update in // background even if we aren't actively blocking) a2.srv.h.ServeHTTP(resp, req) - require.Equal(t, http.StatusOK, resp.Code) + require.Equal(r, http.StatusOK, resp.Code) dec := json.NewDecoder(resp.Body) issued2 := &structs.IssuedCert{} - require.NoError(t, dec.Decode(issued2)) + require.NoError(r, dec.Decode(issued2)) if issued.CertPEM == issued2.CertPEM { r.Fatalf("leaf has not updated") } @@ -7481,9 +7483,9 @@ func TestAgentConnectCALeafCert_secondaryDC_good(t *testing.T) { } // Verify that the cert is signed by the new CA - requireLeafValidUnderCA(t, issued2, dc1_ca2) + requireLeafValidUnderCA(r, issued2, dc1_ca2) - require.NotEqual(t, issued, issued2) + require.NotEqual(r, issued, issued2) }) } @@ -7493,12 +7495,12 @@ func waitForActiveCARoot(t *testing.T, srv *HTTPHandlers, expect *structs.CARoot resp := httptest.NewRecorder() srv.h.ServeHTTP(resp, req) if http.StatusOK != resp.Code { - t.Fatalf("expected 200 but got %v", resp.Code) + r.Fatalf("expected 200 but got %v", resp.Code) } dec := json.NewDecoder(resp.Body) roots := &structs.IndexedCARoots{} - require.NoError(t, dec.Decode(roots)) + require.NoError(r, dec.Decode(roots)) var root *structs.CARoot for _, r := range roots.Roots { @@ -8128,3 +8130,59 @@ func TestAgent_Services_ExposeConfig(t *testing.T) { } require.Equal(t, srv1.Proxy.ToAPI(), actual.Proxy) } + +func TestAgent_Self_Reload(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + + // create new test agent + a := NewTestAgent(t, ` + log_level = "info" + raft_snapshot_threshold = 100 + `) + defer a.Shutdown() + + testrpc.WaitForTestAgent(t, a.RPC, "dc1") + req, _ := http.NewRequest("GET", "/v1/agent/self", nil) + resp := httptest.NewRecorder() + a.srv.h.ServeHTTP(resp, req) + + dec := json.NewDecoder(resp.Body) + val := &Self{} + require.NoError(t, dec.Decode(val)) + + require.Equal(t, "info", val.DebugConfig["Logging"].(map[string]interface{})["LogLevel"]) + require.Equal(t, float64(100), val.DebugConfig["RaftSnapshotThreshold"].(float64)) + + // reload with new config + shim := &delegateConfigReloadShim{delegate: a.delegate} + a.delegate = shim + newCfg := TestConfig(testutil.Logger(t), config.FileSource{ + Name: "Reload", + Format: "hcl", + Data: ` + data_dir = "` + a.Config.DataDir + `" + log_level = "debug" + raft_snapshot_threshold = 200 + `, + }) + if err := a.reloadConfigInternal(newCfg); err != nil { + t.Fatalf("got error %v want nil", err) + } + require.Equal(t, 200, shim.newCfg.RaftSnapshotThreshold) + + // validate new config is reflected in API response + req, _ = http.NewRequest("GET", "/v1/agent/self", nil) + resp = httptest.NewRecorder() + a.srv.h.ServeHTTP(resp, req) + + dec = json.NewDecoder(resp.Body) + val = &Self{} + require.NoError(t, dec.Decode(val)) + require.Equal(t, "debug", val.DebugConfig["Logging"].(map[string]interface{})["LogLevel"]) + require.Equal(t, float64(200), val.DebugConfig["RaftSnapshotThreshold"].(float64)) + +} diff --git a/agent/agent_test.go b/agent/agent_test.go index d6dd2cc5fcf..7f5de5b69a1 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "crypto/md5" + "crypto/rand" "crypto/tls" "crypto/x509" "encoding/base64" "encoding/json" + "errors" "fmt" - "math/rand" + mathrand "math/rand" "net" "net/http" "net/http/httptest" @@ -752,7 +754,7 @@ func testAgent_AddServices_AliasUpdateCheckNotReverted(t *testing.T, extraHCL st func test_createAlias(t *testing.T, agent *TestAgent, chk *structs.CheckType, expectedResult string) func(r *retry.R) { t.Helper() - serviceNum := rand.Int() + serviceNum := mathrand.Int() srv := &structs.NodeService{ Service: fmt.Sprintf("serviceAlias-%d", serviceNum), Tags: []string{"tag1"}, @@ -4179,6 +4181,42 @@ func TestAgent_ReloadConfig_XDSUpdateRateLimit(t *testing.T) { require.Equal(t, rate.Limit(1000), a.proxyConfig.UpdateRateLimit()) } +func TestAgent_ReloadConfig_EnableDebug(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + cfg := fmt.Sprintf(`data_dir = %q`, testutil.TempDir(t, "agent")) + + a := NewTestAgent(t, cfg) + defer a.Shutdown() + + c := TestConfig( + testutil.Logger(t), + config.FileSource{ + Name: t.Name(), + Format: "hcl", + Data: cfg + ` enable_debug = true`, + }, + ) + + require.NoError(t, a.reloadConfigInternal(c)) + + require.Equal(t, true, a.enableDebug.Load()) + + c = TestConfig( + testutil.Logger(t), + config.FileSource{ + Name: t.Name(), + Format: "hcl", + Data: cfg + ` enable_debug = false`, + }, + ) + require.NoError(t, a.reloadConfigInternal(c)) + + require.Equal(t, false, a.enableDebug.Load()) +} + func TestAgent_consulConfig_AutoEncryptAllowTLS(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") @@ -4818,19 +4856,19 @@ services { deadlineCh := time.After(10 * time.Second) start := time.Now() +LOOP: for { select { case evt := <-ch: // We may receive several notifications of an error until we get the // first successful reply. require.Equal(t, "foo", evt.CorrelationID) - if evt.Err == nil { - require.NoError(t, evt.Err) - require.NotNil(t, evt.Result) - t.Logf("took %s to get first success", time.Since(start)) - return + if evt.Err != nil { + break LOOP } - t.Logf("saw error: %v", evt.Err) + require.NoError(t, evt.Err) + require.NotNil(t, evt.Result) + t.Logf("took %s to get first success", time.Since(start)) case <-deadlineCh: t.Fatal("did not get notified successfully") } @@ -6199,6 +6237,70 @@ cloud { require.NoError(t, err) } +func TestAgent_checkServerLastSeen(t *testing.T) { + bd := BaseDeps{ + Deps: consul.Deps{ + Logger: hclog.NewInterceptLogger(nil), + Tokens: new(token.Store), + GRPCConnPool: &fakeGRPCConnPool{}, + }, + RuntimeConfig: &config.RuntimeConfig{}, + Cache: cache.New(cache.Options{}), + } + agent, err := New(bd) + require.NoError(t, err) + + // Test that an ErrNotExist OS error is treated as ok. + t.Run("TestReadErrNotExist", func(t *testing.T) { + readFn := func(filename string) (*consul.ServerMetadata, error) { + return nil, os.ErrNotExist + } + + err := agent.checkServerLastSeen(readFn) + require.NoError(t, err) + }) + + // Test that an error reading server metadata is treated as an error. + t.Run("TestReadErr", func(t *testing.T) { + expected := errors.New("read error") + readFn := func(filename string) (*consul.ServerMetadata, error) { + return nil, expected + } + + err := agent.checkServerLastSeen(readFn) + require.ErrorIs(t, err, expected) + }) + + // Test that a server with a 7d old last seen timestamp is treated as an error. + t.Run("TestIsLastSeenStaleErr", func(t *testing.T) { + agent.config.ServerRejoinAgeMax = time.Hour + + readFn := func(filename string) (*consul.ServerMetadata, error) { + return &consul.ServerMetadata{ + LastSeenUnix: time.Now().Add(-24 * 7 * time.Hour).Unix(), + }, nil + } + + err := agent.checkServerLastSeen(readFn) + require.Error(t, err) + require.ErrorContains(t, err, "refusing to rejoin cluster because server has been offline for more than the configured server_rejoin_age_max") + }) + + // Test that a server with a 6h old last seen timestamp is not treated as an error. + t.Run("TestNoErr", func(t *testing.T) { + agent.config.ServerRejoinAgeMax = 24 * 7 * time.Hour + + readFn := func(filename string) (*consul.ServerMetadata, error) { + return &consul.ServerMetadata{ + LastSeenUnix: time.Now().Add(-6 * time.Hour).Unix(), + }, nil + } + + err := agent.checkServerLastSeen(readFn) + require.NoError(t, err) + }) +} + func getExpectedCaPoolByFile(t *testing.T) *x509.CertPool { pool := x509.NewCertPool() data, err := os.ReadFile("../test/ca/root.cer") diff --git a/agent/auto-config/auto_config_oss.go b/agent/auto-config/auto_config_ce.go similarity index 85% rename from agent/auto-config/auto_config_oss.go rename to agent/auto-config/auto_config_ce.go index 95b38aa056f..29f626568fd 100644 --- a/agent/auto-config/auto_config_oss.go +++ b/agent/auto-config/auto_config_ce.go @@ -3,7 +3,7 @@ package autoconf -// AutoConfigEnterprise has no fields in OSS +// AutoConfigEnterprise has no fields in CE type AutoConfigEnterprise struct{} // newAutoConfigEnterprise initializes the enterprise AutoConfig struct diff --git a/agent/auto-config/auto_config_oss_test.go b/agent/auto-config/auto_config_ce_test.go similarity index 100% rename from agent/auto-config/auto_config_oss_test.go rename to agent/auto-config/auto_config_ce_test.go diff --git a/agent/auto-config/config_oss.go b/agent/auto-config/config_ce.go similarity index 88% rename from agent/auto-config/config_oss.go rename to agent/auto-config/config_ce.go index a8048954a18..dbd7d90eb3c 100644 --- a/agent/auto-config/config_oss.go +++ b/agent/auto-config/config_ce.go @@ -6,7 +6,7 @@ package autoconf // EnterpriseConfig stub - only populated in Consul Enterprise type EnterpriseConfig struct{} -// finalize is a noop for OSS +// finalize is a noop for CE func (_ *EnterpriseConfig) validateAndFinalize() error { return nil } diff --git a/agent/auto-config/mock_oss_test.go b/agent/auto-config/mock_ce_test.go similarity index 85% rename from agent/auto-config/mock_oss_test.go rename to agent/auto-config/mock_ce_test.go index 0518753bbbe..1a7acaa4837 100644 --- a/agent/auto-config/mock_oss_test.go +++ b/agent/auto-config/mock_ce_test.go @@ -7,7 +7,7 @@ import ( "testing" ) -// mockedEnterpriseConfig is pretty much just a stub in OSS +// mockedEnterpriseConfig is pretty much just a stub in CE. // It does contain an enterprise config for compatibility // purposes but that in and of itself is just a stub. type mockedEnterpriseConfig struct { diff --git a/agent/blockingquery/blockingquery.go b/agent/blockingquery/blockingquery.go new file mode 100644 index 00000000000..cb46110222d --- /dev/null +++ b/agent/blockingquery/blockingquery.go @@ -0,0 +1,208 @@ +package blockingquery + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/armon/go-metrics" + "github.com/hashicorp/go-memdb" + + "github.com/hashicorp/consul/agent/consul/state" + "github.com/hashicorp/consul/lib" +) + +// Sentinel errors that must be used with blockingQuery +var ( + ErrNotFound = fmt.Errorf("no data found for query") + ErrNotChanged = fmt.Errorf("data did not change for query") +) + +// QueryFn is used to perform a query operation. See Server.blockingQuery for +// the requirements of this function. +type QueryFn func(memdb.WatchSet, *state.Store) error + +// RequestOptions are options used by Server.blockingQuery to modify the +// behaviour of the query operation, or to populate response metadata. +type RequestOptions interface { + GetToken() string + GetMinQueryIndex() uint64 + GetMaxQueryTime() (time.Duration, error) + GetRequireConsistent() bool +} + +// ResponseMeta is an interface used to populate the response struct +// with metadata about the query and the state of the server. +type ResponseMeta interface { + SetLastContact(time.Duration) + SetKnownLeader(bool) + GetIndex() uint64 + SetIndex(uint64) + SetResultsFilteredByACLs(bool) +} + +// FSMServer is interface into the stateful components of a Consul server, such +// as memdb or raft leadership. +type FSMServer interface { + ConsistentRead() error + DecrementBlockingQueries() uint64 + GetShutdownChannel() chan struct{} + GetState() *state.Store + IncrementBlockingQueries() uint64 + RPCQueryTimeout(time.Duration) time.Duration + SetQueryMeta(ResponseMeta, string) +} + +// Query performs a blocking query if opts.GetMinQueryIndex is +// greater than 0, otherwise performs a non-blocking query. Blocking queries will +// block until responseMeta.Index is greater than opts.GetMinQueryIndex, +// or opts.GetMaxQueryTime is reached. Non-blocking queries return immediately +// after performing the query. +// +// If opts.GetRequireConsistent is true, blockingQuery will first verify it is +// still the cluster leader before performing the query. +// +// The query function is expected to be a closure that has access to responseMeta +// so that it can set the Index. The actual result of the query is opaque to blockingQuery. +// +// The query function can return ErrNotFound, which is a sentinel error. Returning +// ErrNotFound indicates that the query found no results, which allows +// blockingQuery to keep blocking until the query returns a non-nil error. +// The query function must take care to set the actual result of the query to +// nil in these cases, otherwise when blockingQuery times out it may return +// a previous result. ErrNotFound will never be returned to the caller, it is +// converted to nil before returning. +// +// The query function can return ErrNotChanged, which is a sentinel error. This +// can only be returned on calls AFTER the first call, as it would not be +// possible to detect the absence of a change on the first call. Returning +// ErrNotChanged indicates that the query results are identical to the prior +// results which allows blockingQuery to keep blocking until the query returns +// a real changed result. +// +// The query function must take care to ensure the actual result of the query +// is either left unmodified or explicitly left in a good state before +// returning, otherwise when blockingQuery times out it may return an +// incomplete or unexpected result. ErrNotChanged will never be returned to the +// caller, it is converted to nil before returning. +// +// If query function returns any other error, the error is returned to the caller +// immediately. +// +// The query function must follow these rules: +// +// 1. to access data it must use the passed in state.Store. +// 2. it must set the responseMeta.Index to an index greater than +// opts.GetMinQueryIndex if the results return by the query have changed. +// 3. any channels added to the memdb.WatchSet must unblock when the results +// returned by the query have changed. +// +// To ensure optimal performance of the query, the query function should make a +// best-effort attempt to follow these guidelines: +// +// 1. only set responseMeta.Index to an index greater than +// opts.GetMinQueryIndex when the results returned by the query have changed. +// 2. any channels added to the memdb.WatchSet should only unblock when the +// results returned by the query have changed. +func Query( + fsmServer FSMServer, + requestOpts RequestOptions, + responseMeta ResponseMeta, + query QueryFn, +) error { + var ctx context.Context = &lib.StopChannelContext{StopCh: fsmServer.GetShutdownChannel()} + + metrics.IncrCounter([]string{"rpc", "query"}, 1) + + minQueryIndex := requestOpts.GetMinQueryIndex() + // Perform a non-blocking query + if minQueryIndex == 0 { + if requestOpts.GetRequireConsistent() { + if err := fsmServer.ConsistentRead(); err != nil { + return err + } + } + + var ws memdb.WatchSet + err := query(ws, fsmServer.GetState()) + fsmServer.SetQueryMeta(responseMeta, requestOpts.GetToken()) + if errors.Is(err, ErrNotFound) || errors.Is(err, ErrNotChanged) { + return nil + } + return err + } + + maxQueryTimeout, err := requestOpts.GetMaxQueryTime() + if err != nil { + return err + } + timeout := fsmServer.RPCQueryTimeout(maxQueryTimeout) + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + count := fsmServer.IncrementBlockingQueries() + metrics.SetGauge([]string{"rpc", "queries_blocking"}, float32(count)) + // decrement the count when the function returns. + defer fsmServer.DecrementBlockingQueries() + + var ( + notFound bool + ranOnce bool + ) + + for { + if requestOpts.GetRequireConsistent() { + if err := fsmServer.ConsistentRead(); err != nil { + return err + } + } + + // Operate on a consistent set of state. This makes sure that the + // abandon channel goes with the state that the caller is using to + // build watches. + store := fsmServer.GetState() + + ws := memdb.NewWatchSet() + // This channel will be closed if a snapshot is restored and the + // whole state store is abandoned. + ws.Add(store.AbandonCh()) + + err := query(ws, store) + fsmServer.SetQueryMeta(responseMeta, requestOpts.GetToken()) + + switch { + case errors.Is(err, ErrNotFound): + if notFound { + // query result has not changed + minQueryIndex = responseMeta.GetIndex() + } + notFound = true + case errors.Is(err, ErrNotChanged): + if ranOnce { + // query result has not changed + minQueryIndex = responseMeta.GetIndex() + } + case err != nil: + return err + } + ranOnce = true + + if responseMeta.GetIndex() > minQueryIndex { + return nil + } + + // block until something changes, or the timeout + if err := ws.WatchCtx(ctx); err != nil { + // exit if we've reached the timeout, or other cancellation + return nil + } + + // exit if the state store has been abandoned + select { + case <-store.AbandonCh(): + return nil + default: + } + } +} diff --git a/agent/blockingquery/blockingquery_test.go b/agent/blockingquery/blockingquery_test.go new file mode 100644 index 00000000000..6cfc07c114a --- /dev/null +++ b/agent/blockingquery/blockingquery_test.go @@ -0,0 +1,4 @@ +package blockingquery + +// TODO: move tests from the consul package, rpc_test.go, TestServer_blockingQuery +// here using mock for FSMServer w/ structs.QueryOptions and structs.QueryOptions diff --git a/agent/cache-types/connect_ca_leaf.go b/agent/cache-types/connect_ca_leaf.go index 9bee39af7d7..06b06c206c3 100644 --- a/agent/cache-types/connect_ca_leaf.go +++ b/agent/cache-types/connect_ca_leaf.go @@ -12,12 +12,11 @@ import ( "github.com/mitchellh/hashstructure" "github.com/hashicorp/consul/acl" - "github.com/hashicorp/consul/lib" - "github.com/hashicorp/consul/agent/cache" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/consul" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/lib" ) // Recommended name for registration. @@ -424,20 +423,25 @@ func (c *ConnectCALeaf) Fetch(opts cache.FetchOptions, req cache.Request) (cache // Setup the timeout chan outside the loop so we don't keep bumping the timeout // later if we loop around. - timeoutCh := time.After(opts.Timeout) + timeoutTimer := time.NewTimer(opts.Timeout) + defer timeoutTimer.Stop() // Setup initial expiry chan. We may change this if root update occurs in the // loop below. - expiresCh := time.After(expiresAt.Sub(now)) + expiresTimer := time.NewTimer(expiresAt.Sub(now)) + defer func() { + // Resolve the timer reference at defer time, so we use the latest one each time. + expiresTimer.Stop() + }() // Current cert is valid so just wait until it expires or we time out. for { select { - case <-timeoutCh: + case <-timeoutTimer.C: // We timed out the request with same cert. return lastResultWithNewState(), nil - case <-expiresCh: + case <-expiresTimer.C: // Cert expired or was force-expired by a root change. return c.generateNewLeaf(reqReal, lastResultWithNewState()) @@ -478,7 +482,9 @@ func (c *ConnectCALeaf) Fetch(opts cache.FetchOptions, req cache.Request) (cache // loop back around, we'll wait at most delay until generating a new cert. if state.forceExpireAfter.Before(expiresAt) { expiresAt = state.forceExpireAfter - expiresCh = time.After(delay) + // Stop the former one and create a new one. + expiresTimer.Stop() + expiresTimer = time.NewTimer(delay) } continue } diff --git a/agent/cache-types/connect_ca_leaf_oss.go b/agent/cache-types/connect_ca_leaf_ce.go similarity index 100% rename from agent/cache-types/connect_ca_leaf_oss.go rename to agent/cache-types/connect_ca_leaf_ce.go diff --git a/agent/cache-types/peerings.go b/agent/cache-types/peerings.go index 7ecbb183e87..9f587598844 100644 --- a/agent/cache-types/peerings.go +++ b/agent/cache-types/peerings.go @@ -4,22 +4,24 @@ import ( "context" "fmt" "strconv" - "time" - external "github.com/hashicorp/consul/agent/grpc-external" - "github.com/hashicorp/consul/proto/pbpeering" "github.com/mitchellh/hashstructure" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" "github.com/hashicorp/consul/agent/cache" + external "github.com/hashicorp/consul/agent/grpc-external" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/proto/pbpeering" ) // PeeringListName is the recommended name for registration. const PeeringListName = "peers" +// PeeringListRequest represents the combination of request payload +// and options that would normally be sent over headers. type PeeringListRequest struct { Request *pbpeering.PeeringListRequest structs.QueryOptions @@ -29,13 +31,10 @@ func (r *PeeringListRequest) CacheInfo() cache.RequestInfo { info := cache.RequestInfo{ Token: r.Token, Datacenter: "", - MinIndex: 0, - Timeout: 0, - MustRevalidate: false, - - // OPTIMIZE(peering): Cache.notifyPollingQuery polls at this interval. We need to revisit how that polling works. - // Using an exponential backoff when the result hasn't changed may be preferable. - MaxAge: 1 * time.Second, + MinIndex: r.MinQueryIndex, + Timeout: r.MaxQueryTime, + MaxAge: r.MaxAge, + MustRevalidate: r.MustRevalidate, } v, err := hashstructure.Hash([]interface{}{ @@ -53,7 +52,7 @@ func (r *PeeringListRequest) CacheInfo() cache.RequestInfo { // Peerings supports fetching the list of peers for a given partition or wildcard-specifier. type Peerings struct { - RegisterOptionsNoRefresh + RegisterOptionsBlockingRefresh Client PeeringLister } @@ -64,7 +63,7 @@ type PeeringLister interface { ) (*pbpeering.PeeringListResponse, error) } -func (t *Peerings) Fetch(_ cache.FetchOptions, req cache.Request) (cache.FetchResult, error) { +func (t *Peerings) Fetch(opts cache.FetchOptions, req cache.Request) (cache.FetchResult, error) { var result cache.FetchResult // The request should be a PeeringListRequest. @@ -76,10 +75,17 @@ func (t *Peerings) Fetch(_ cache.FetchOptions, req cache.Request) (cache.FetchRe "Internal cache failure: request wrong type: %T", req) } - // Always allow stale - there's no point in hitting leader if the request is - // going to be served from cache and end up arbitrarily stale anyway. This - // allows cached service-discover to automatically read scale across all - // servers too. + // Lightweight copy this object so that manipulating QueryOptions doesn't race. + dup := *reqReal + reqReal = &dup + + // Set the minimum query index to our current index, so we block + reqReal.QueryOptions.MinQueryIndex = opts.MinIndex + reqReal.QueryOptions.MaxQueryTime = opts.Timeout + + // We allow stale queries here to spread out the RPC load, but peerstream information, including the STATUS, + // will not be returned. Right now this is fine for the watch in proxycfg/mesh_gateway.go, + // but it could be a problem for a future consumer. reqReal.QueryOptions.SetAllowStale(true) ctx, err := external.ContextWithQueryOptions(context.Background(), reqReal.QueryOptions) @@ -88,7 +94,8 @@ func (t *Peerings) Fetch(_ cache.FetchOptions, req cache.Request) (cache.FetchRe } // Fetch - reply, err := t.Client.PeeringList(ctx, reqReal.Request) + var header metadata.MD + reply, err := t.Client.PeeringList(ctx, reqReal.Request, grpc.Header(&header)) if err != nil { // Return an empty result if the error is due to peering being disabled. // This allows mesh gateways to receive an update and confirm that the watch is set. @@ -100,8 +107,19 @@ func (t *Peerings) Fetch(_ cache.FetchOptions, req cache.Request) (cache.FetchRe return result, err } + // This first case is using the legacy index field + // It should be removed in a future version in favor of the index from QueryMeta + if reply.OBSOLETE_Index != 0 { + result.Index = reply.OBSOLETE_Index + } else { + meta, err := external.QueryMetaFromGRPCMeta(header) + if err != nil { + return result, fmt.Errorf("could not convert gRPC metadata to query meta: %w", err) + } + result.Index = meta.GetIndex() + } + result.Value = reply - result.Index = reply.Index return result, nil } diff --git a/agent/cache-types/peerings_test.go b/agent/cache-types/peerings_test.go index e96e6256e93..525de96ed88 100644 --- a/agent/cache-types/peerings_test.go +++ b/agent/cache-types/peerings_test.go @@ -5,14 +5,16 @@ import ( "testing" "time" - "github.com/mitchellh/copystructure" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" grpcstatus "google.golang.org/grpc/status" "github.com/hashicorp/consul/agent/cache" + external "github.com/hashicorp/consul/agent/grpc-external" + "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/proto/pbpeering" ) @@ -21,7 +23,6 @@ func TestPeerings(t *testing.T) { typ := &Peerings{Client: client} resp := &pbpeering.PeeringListResponse{ - Index: 48, Peerings: []*pbpeering.Peering{ { Name: "peer1", @@ -33,12 +34,38 @@ func TestPeerings(t *testing.T) { } // Expect the proper call. - // This also returns the canned response above. - client.On("PeeringList", mock.Anything, mock.Anything). - Return(resp, nil) + // This also set the gRPC metadata returned by pointer. + client.On("PeeringList", mock.Anything, mock.Anything, mock.Anything). + Return(resp, nil). + Run(func(args mock.Arguments) { + // Validate Query Options + ctx := args.Get(0).(context.Context) + out, ok := metadata.FromOutgoingContext(ctx) + require.True(t, ok) + ctx = metadata.NewIncomingContext(ctx, out) + + options, err := external.QueryOptionsFromContext(ctx) + require.NoError(t, err) + require.Equal(t, uint64(28), options.MinQueryIndex) + require.Equal(t, time.Duration(1100), options.MaxQueryTime) + require.True(t, options.AllowStale) + + // Send back Query Meta on pointer of header + header := args.Get(2).(grpc.HeaderCallOption) + qm := structs.QueryMeta{ + Index: 48, + } + + md, err := external.GRPCMetadataFromQueryMeta(qm) + require.NoError(t, err) + *header.HeaderAddr = md + }) // Fetch and assert against the result. - result, err := typ.Fetch(cache.FetchOptions{}, &PeeringListRequest{ + result, err := typ.Fetch(cache.FetchOptions{ + MinIndex: 28, + Timeout: time.Duration(1100), + }, &PeeringListRequest{ Request: &pbpeering.PeeringListRequest{}, }) require.NoError(t, err) @@ -55,7 +82,7 @@ func TestPeerings_PeeringDisabled(t *testing.T) { var resp *pbpeering.PeeringListResponse // Expect the proper call, but return the peering disabled error - client.On("PeeringList", mock.Anything, mock.Anything). + client.On("PeeringList", mock.Anything, mock.Anything, mock.Anything). Return(resp, grpcstatus.Error(codes.FailedPrecondition, "peering must be enabled to use this endpoint")) // Fetch and assert against the result. @@ -78,54 +105,3 @@ func TestPeerings_badReqType(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "wrong type") } - -// This test asserts that we can continuously poll this cache type, given that it doesn't support blocking. -func TestPeerings_MultipleUpdates(t *testing.T) { - c := cache.New(cache.Options{}) - - client := NewMockPeeringLister(t) - - // On each mock client call to PeeringList we will increment the index by 1 - // to simulate new data arriving. - resp := &pbpeering.PeeringListResponse{ - Index: uint64(0), - } - - client.On("PeeringList", mock.Anything, mock.Anything). - Return(func(ctx context.Context, in *pbpeering.PeeringListRequest, opts ...grpc.CallOption) *pbpeering.PeeringListResponse { - resp.Index++ - // Avoids triggering the race detection by copying the output - copyResp, err := copystructure.Copy(resp) - require.NoError(t, err) - output := copyResp.(*pbpeering.PeeringListResponse) - return output - }, nil) - - c.RegisterType(PeeringListName, &Peerings{Client: client}) - - ch := make(chan cache.UpdateEvent) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - t.Cleanup(cancel) - - require.NoError(t, c.Notify(ctx, PeeringListName, &PeeringListRequest{ - Request: &pbpeering.PeeringListRequest{}, - }, "updates", ch)) - - i := uint64(1) - for { - select { - case <-ctx.Done(): - t.Fatal("context deadline exceeded") - return - case update := <-ch: - // Expect to receive updates for increasing indexes serially. - actual := update.Result.(*pbpeering.PeeringListResponse) - require.Equal(t, i, actual.Index) - i++ - - if i > 3 { - return - } - } - } -} diff --git a/agent/cache-types/trust_bundle.go b/agent/cache-types/trust_bundle.go index addc65ac94c..f9015d9d259 100644 --- a/agent/cache-types/trust_bundle.go +++ b/agent/cache-types/trust_bundle.go @@ -4,10 +4,10 @@ import ( "context" "fmt" "strconv" - "time" "github.com/mitchellh/hashstructure" "google.golang.org/grpc" + "google.golang.org/grpc/metadata" "github.com/hashicorp/consul/agent/cache" external "github.com/hashicorp/consul/agent/grpc-external" @@ -18,6 +18,8 @@ import ( // Recommended name for registration. const TrustBundleReadName = "peer-trust-bundle" +// TrustBundleReadRequest represents the combination of request payload +// and options that would normally be sent over headers. type TrustBundleReadRequest struct { Request *pbpeering.TrustBundleReadRequest structs.QueryOptions @@ -27,13 +29,10 @@ func (r *TrustBundleReadRequest) CacheInfo() cache.RequestInfo { info := cache.RequestInfo{ Token: r.Token, Datacenter: "", - MinIndex: 0, - Timeout: 0, - MustRevalidate: false, - - // OPTIMIZE(peering): Cache.notifyPollingQuery polls at this interval. We need to revisit how that polling works. - // Using an exponential backoff when the result hasn't changed may be preferable. - MaxAge: 1 * time.Second, + MinIndex: r.MinQueryIndex, + Timeout: r.MaxQueryTime, + MaxAge: r.MaxAge, + MustRevalidate: r.MustRevalidate, } v, err := hashstructure.Hash([]interface{}{ @@ -53,7 +52,7 @@ func (r *TrustBundleReadRequest) CacheInfo() cache.RequestInfo { // TrustBundle supports fetching discovering service instances via prepared // queries. type TrustBundle struct { - RegisterOptionsNoRefresh + RegisterOptionsBlockingRefresh Client TrustBundleReader } @@ -64,7 +63,7 @@ type TrustBundleReader interface { ) (*pbpeering.TrustBundleReadResponse, error) } -func (t *TrustBundle) Fetch(_ cache.FetchOptions, req cache.Request) (cache.FetchResult, error) { +func (t *TrustBundle) Fetch(opts cache.FetchOptions, req cache.Request) (cache.FetchResult, error) { var result cache.FetchResult // The request should be a TrustBundleReadRequest. @@ -76,6 +75,14 @@ func (t *TrustBundle) Fetch(_ cache.FetchOptions, req cache.Request) (cache.Fetc "Internal cache failure: request wrong type: %T", req) } + // Lightweight copy this object so that manipulating QueryOptions doesn't race. + dup := *reqReal + reqReal = &dup + + // Set the minimum query index to our current index, so we block + reqReal.QueryOptions.MinQueryIndex = opts.MinIndex + reqReal.QueryOptions.MaxQueryTime = opts.Timeout + // Always allow stale - there's no point in hitting leader if the request is // going to be served from cache and end up arbitrarily stale anyway. This // allows cached service-discover to automatically read scale across all @@ -88,13 +95,25 @@ func (t *TrustBundle) Fetch(_ cache.FetchOptions, req cache.Request) (cache.Fetc return result, err } - reply, err := t.Client.TrustBundleRead(ctx, reqReal.Request) + var header metadata.MD + reply, err := t.Client.TrustBundleRead(ctx, reqReal.Request, grpc.Header(&header)) if err != nil { return result, err } + // This first case is using the legacy index field + // It should be removed in a future version in favor of the index from QueryMeta + if reply.OBSOLETE_Index != 0 { + result.Index = reply.OBSOLETE_Index + } else { + meta, err := external.QueryMetaFromGRPCMeta(header) + if err != nil { + return result, fmt.Errorf("could not convert gRPC metadata to query meta: %w", err) + } + result.Index = meta.GetIndex() + } + result.Value = reply - result.Index = reply.Index return result, nil } diff --git a/agent/cache-types/trust_bundle_test.go b/agent/cache-types/trust_bundle_test.go index ea36e8d8a7c..80347132e72 100644 --- a/agent/cache-types/trust_bundle_test.go +++ b/agent/cache-types/trust_bundle_test.go @@ -7,8 +7,12 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" "github.com/hashicorp/consul/agent/cache" + external "github.com/hashicorp/consul/agent/grpc-external" + "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/proto/pbpeering" ) @@ -17,7 +21,6 @@ func TestTrustBundle(t *testing.T) { typ := &TrustBundle{Client: client} resp := &pbpeering.TrustBundleReadResponse{ - Index: 48, Bundle: &pbpeering.PeeringTrustBundle{ PeerName: "peer1", RootPEMs: []string{"peer1-roots"}, @@ -26,15 +29,41 @@ func TestTrustBundle(t *testing.T) { // Expect the proper call. // This also returns the canned response above. - client.On("TrustBundleRead", mock.Anything, mock.Anything). + client.On("TrustBundleRead", mock.Anything, mock.Anything, mock.Anything). Run(func(args mock.Arguments) { + // Validate Query Options + ctx := args.Get(0).(context.Context) + out, ok := metadata.FromOutgoingContext(ctx) + require.True(t, ok) + ctx = metadata.NewIncomingContext(ctx, out) + + options, err := external.QueryOptionsFromContext(ctx) + require.NoError(t, err) + require.Equal(t, uint64(28), options.MinQueryIndex) + require.Equal(t, time.Duration(1100), options.MaxQueryTime) + require.True(t, options.AllowStale) + + // Validate Request req := args.Get(1).(*pbpeering.TrustBundleReadRequest) require.Equal(t, "foo", req.Name) + + // Send back Query Meta on pointer of header + header := args.Get(2).(grpc.HeaderCallOption) + qm := structs.QueryMeta{ + Index: 48, + } + + md, err := external.GRPCMetadataFromQueryMeta(qm) + require.NoError(t, err) + *header.HeaderAddr = md }). Return(resp, nil) // Fetch and assert against the result. - result, err := typ.Fetch(cache.FetchOptions{}, &TrustBundleReadRequest{ + result, err := typ.Fetch(cache.FetchOptions{ + MinIndex: 28, + Timeout: time.Duration(1100), + }, &TrustBundleReadRequest{ Request: &pbpeering.TrustBundleReadRequest{ Name: "foo", }, @@ -56,55 +85,3 @@ func TestTrustBundle_badReqType(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "wrong type") } - -// This test asserts that we can continuously poll this cache type, given that it doesn't support blocking. -func TestTrustBundle_MultipleUpdates(t *testing.T) { - c := cache.New(cache.Options{}) - - client := NewMockTrustBundleReader(t) - - // On each mock client call to TrustBundleList by service we will increment the index by 1 - // to simulate new data arriving. - resp := &pbpeering.TrustBundleReadResponse{ - Index: uint64(0), - } - - client.On("TrustBundleRead", mock.Anything, mock.Anything). - Run(func(args mock.Arguments) { - req := args.Get(1).(*pbpeering.TrustBundleReadRequest) - require.Equal(t, "foo", req.Name) - - // Increment on each call. - resp.Index++ - }). - Return(resp, nil) - - c.RegisterType(TrustBundleReadName, &TrustBundle{Client: client}) - - ch := make(chan cache.UpdateEvent) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - t.Cleanup(cancel) - - err := c.Notify(ctx, TrustBundleReadName, &TrustBundleReadRequest{ - Request: &pbpeering.TrustBundleReadRequest{Name: "foo"}, - }, "updates", ch) - require.NoError(t, err) - - i := uint64(1) - for { - select { - case <-ctx.Done(): - t.Fatal("context deadline exceeded") - return - case update := <-ch: - // Expect to receive updates for increasing indexes serially. - actual := update.Result.(*pbpeering.TrustBundleReadResponse) - require.Equal(t, i, actual.Index) - i++ - - if i > 3 { - return - } - } - } -} diff --git a/agent/cache-types/trust_bundles.go b/agent/cache-types/trust_bundles.go index 47f411f02fe..010848f5b86 100644 --- a/agent/cache-types/trust_bundles.go +++ b/agent/cache-types/trust_bundles.go @@ -4,11 +4,11 @@ import ( "context" "fmt" "strconv" - "time" "github.com/mitchellh/hashstructure" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" "github.com/hashicorp/consul/agent/cache" @@ -20,6 +20,8 @@ import ( // Recommended name for registration. const TrustBundleListName = "trust-bundles" +// TrustBundleListRequest represents the combination of request payload +// and options that would normally be sent over headers. type TrustBundleListRequest struct { Request *pbpeering.TrustBundleListByServiceRequest structs.QueryOptions @@ -29,13 +31,10 @@ func (r *TrustBundleListRequest) CacheInfo() cache.RequestInfo { info := cache.RequestInfo{ Token: r.Token, Datacenter: "", - MinIndex: 0, - Timeout: 0, - MustRevalidate: false, - - // OPTIMIZE(peering): Cache.notifyPollingQuery polls at this interval. We need to revisit how that polling works. - // Using an exponential backoff when the result hasn't changed may be preferable. - MaxAge: 1 * time.Second, + MinIndex: r.MinQueryIndex, + Timeout: r.MaxQueryTime, + MaxAge: r.MaxAge, + MustRevalidate: r.MustRevalidate, } v, err := hashstructure.Hash([]interface{}{ @@ -57,7 +56,7 @@ func (r *TrustBundleListRequest) CacheInfo() cache.RequestInfo { // TrustBundles supports fetching discovering service instances via prepared // queries. type TrustBundles struct { - RegisterOptionsNoRefresh + RegisterOptionsBlockingRefresh Client TrustBundleLister } @@ -68,7 +67,7 @@ type TrustBundleLister interface { ) (*pbpeering.TrustBundleListByServiceResponse, error) } -func (t *TrustBundles) Fetch(_ cache.FetchOptions, req cache.Request) (cache.FetchResult, error) { +func (t *TrustBundles) Fetch(opts cache.FetchOptions, req cache.Request) (cache.FetchResult, error) { var result cache.FetchResult // The request should be a TrustBundleListRequest. @@ -80,6 +79,14 @@ func (t *TrustBundles) Fetch(_ cache.FetchOptions, req cache.Request) (cache.Fet "Internal cache failure: request wrong type: %T", req) } + // Lightweight copy this object so that manipulating QueryOptions doesn't race. + dup := *reqReal + reqReal = &dup + + // Set the minimum query index to our current index, so we block + reqReal.QueryOptions.MinQueryIndex = opts.MinIndex + reqReal.QueryOptions.MaxQueryTime = opts.Timeout + // Always allow stale - there's no point in hitting leader if the request is // going to be served from cache and end up arbitrarily stale anyway. This // allows cached service-discover to automatically read scale across all @@ -92,20 +99,32 @@ func (t *TrustBundles) Fetch(_ cache.FetchOptions, req cache.Request) (cache.Fet return result, err } - reply, err := t.Client.TrustBundleListByService(ctx, reqReal.Request) + var header metadata.MD + reply, err := t.Client.TrustBundleListByService(ctx, reqReal.Request, grpc.Header(&header)) if err != nil { // Return an empty result if the error is due to peering being disabled. // This allows mesh gateways to receive an update and confirm that the watch is set. if e, ok := status.FromError(err); ok && e.Code() == codes.FailedPrecondition { result.Index = 1 - result.Value = &pbpeering.TrustBundleListByServiceResponse{Index: 1} + result.Value = &pbpeering.TrustBundleListByServiceResponse{OBSOLETE_Index: 1} return result, nil } return result, err } + // This first case is using the legacy index field + // It should be removed in a future version in favor of the index from QueryMeta + if reply.OBSOLETE_Index != 0 { + result.Index = reply.OBSOLETE_Index + } else { + meta, err := external.QueryMetaFromGRPCMeta(header) + if err != nil { + return result, fmt.Errorf("could not convert gRPC metadata to query meta: %w", err) + } + result.Index = meta.GetIndex() + } + result.Value = reply - result.Index = reply.Index return result, nil } diff --git a/agent/cache-types/trust_bundles_test.go b/agent/cache-types/trust_bundles_test.go index 733becdd60b..0f104253ef5 100644 --- a/agent/cache-types/trust_bundles_test.go +++ b/agent/cache-types/trust_bundles_test.go @@ -7,10 +7,14 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" grpcstatus "google.golang.org/grpc/status" "github.com/hashicorp/consul/agent/cache" + external "github.com/hashicorp/consul/agent/grpc-external" + "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/proto/pbpeering" ) @@ -19,7 +23,6 @@ func TestTrustBundles(t *testing.T) { typ := &TrustBundles{Client: client} resp := &pbpeering.TrustBundleListByServiceResponse{ - Index: 48, Bundles: []*pbpeering.PeeringTrustBundle{ { PeerName: "peer1", @@ -30,15 +33,41 @@ func TestTrustBundles(t *testing.T) { // Expect the proper call. // This also returns the canned response above. - client.On("TrustBundleListByService", mock.Anything, mock.Anything). + client.On("TrustBundleListByService", mock.Anything, mock.Anything, mock.Anything). Run(func(args mock.Arguments) { + // Validate Query Options + ctx := args.Get(0).(context.Context) + out, ok := metadata.FromOutgoingContext(ctx) + require.True(t, ok) + ctx = metadata.NewIncomingContext(ctx, out) + + options, err := external.QueryOptionsFromContext(ctx) + require.NoError(t, err) + require.Equal(t, uint64(28), options.MinQueryIndex) + require.Equal(t, time.Duration(1100), options.MaxQueryTime) + require.True(t, options.AllowStale) + + // Validate Request req := args.Get(1).(*pbpeering.TrustBundleListByServiceRequest) require.Equal(t, "foo", req.ServiceName) + + // Send back Query Meta on pointer of header + header := args.Get(2).(grpc.HeaderCallOption) + qm := structs.QueryMeta{ + Index: 48, + } + + md, err := external.GRPCMetadataFromQueryMeta(qm) + require.NoError(t, err) + *header.HeaderAddr = md }). Return(resp, nil) // Fetch and assert against the result. - result, err := typ.Fetch(cache.FetchOptions{}, &TrustBundleListRequest{ + result, err := typ.Fetch(cache.FetchOptions{ + MinIndex: 28, + Timeout: time.Duration(1100), + }, &TrustBundleListRequest{ Request: &pbpeering.TrustBundleListByServiceRequest{ ServiceName: "foo", }, @@ -58,7 +87,7 @@ func TestTrustBundles_PeeringDisabled(t *testing.T) { // Expect the proper call. // This also returns the canned response above. - client.On("TrustBundleListByService", mock.Anything, mock.Anything). + client.On("TrustBundleListByService", mock.Anything, mock.Anything, mock.Anything). Return(resp, grpcstatus.Error(codes.FailedPrecondition, "peering must be enabled to use this endpoint")) // Fetch and assert against the result. @@ -83,55 +112,3 @@ func TestTrustBundles_badReqType(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "wrong type") } - -// This test asserts that we can continuously poll this cache type, given that it doesn't support blocking. -func TestTrustBundles_MultipleUpdates(t *testing.T) { - c := cache.New(cache.Options{}) - - client := NewMockTrustBundleLister(t) - - // On each mock client call to TrustBundleList by service we will increment the index by 1 - // to simulate new data arriving. - resp := &pbpeering.TrustBundleListByServiceResponse{ - Index: uint64(0), - } - - client.On("TrustBundleListByService", mock.Anything, mock.Anything). - Run(func(args mock.Arguments) { - req := args.Get(1).(*pbpeering.TrustBundleListByServiceRequest) - require.Equal(t, "foo", req.ServiceName) - - // Increment on each call. - resp.Index++ - }). - Return(resp, nil) - - c.RegisterType(TrustBundleListName, &TrustBundles{Client: client}) - - ch := make(chan cache.UpdateEvent) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - t.Cleanup(cancel) - - err := c.Notify(ctx, TrustBundleListName, &TrustBundleListRequest{ - Request: &pbpeering.TrustBundleListByServiceRequest{ServiceName: "foo"}, - }, "updates", ch) - require.NoError(t, err) - - i := uint64(1) - for { - select { - case <-ctx.Done(): - t.Fatal("context deadline exceeded") - return - case update := <-ch: - // Expect to receive updates for increasing indexes serially. - resp := update.Result.(*pbpeering.TrustBundleListByServiceResponse) - require.Equal(t, i, resp.Index) - i++ - - if i > 3 { - return - } - } - } -} diff --git a/agent/cache/cache.go b/agent/cache/cache.go index 55b1654af26..5871d826fde 100644 --- a/agent/cache/cache.go +++ b/agent/cache/cache.go @@ -84,14 +84,14 @@ var Counters = []prometheus.CounterDefinition{ // Constants related to refresh backoff. We probably don't ever need to // make these configurable knobs since they primarily exist to lower load. const ( - DefaultCacheRefreshBackoffMin = 3 // 3 attempts before backing off - DefaultCacheRefreshMaxWait = 1 * time.Minute // maximum backoff wait time + CacheRefreshBackoffMin = 3 // 3 attempts before backing off + CacheRefreshMaxWait = 1 * time.Minute // maximum backoff wait time // The following constants are default values for the cache entry // rate limiter settings. // DefaultEntryFetchRate is the default rate at which cache entries can - // be fetch. This defaults to not being unlimited + // be fetch. This defaults to not being limited DefaultEntryFetchRate = rate.Inf // DefaultEntryFetchMaxBurst is the number of cache entry fetches that can @@ -138,7 +138,10 @@ type Cache struct { entriesLock sync.RWMutex entries map[string]cacheEntry entriesExpiryHeap *ttlcache.ExpiryHeap - lastGoroutineID uint64 + + fetchLock sync.Mutex + lastFetchID uint64 + fetchHandles map[string]fetchHandle // stopped is used as an atomic flag to signal that the Cache has been // discarded so background fetches and expiry processing should stop. @@ -151,6 +154,11 @@ type Cache struct { rateLimitCancel context.CancelFunc } +type fetchHandle struct { + id uint64 + stopCh chan struct{} +} + // typeEntry is a single type that is registered with a Cache. type typeEntry struct { // Name that was used to register the Type @@ -196,13 +204,6 @@ type Options struct { EntryFetchMaxBurst int // EntryFetchRate represents the max calls/sec for a single cache entry EntryFetchRate rate.Limit - - // CacheRefreshBackoffMin is the number of attempts to wait before backing off. - // Mostly configurable just for testing. - CacheRefreshBackoffMin uint - // CacheRefreshMaxWait is the maximum backoff wait time. - // Mostly configurable just for testing. - CacheRefreshMaxWait time.Duration } // Equal return true if both options are equivalent @@ -218,12 +219,6 @@ func applyDefaultValuesOnOptions(options Options) Options { if options.EntryFetchMaxBurst == 0 { options.EntryFetchMaxBurst = DefaultEntryFetchMaxBurst } - if options.CacheRefreshBackoffMin == 0 { - options.CacheRefreshBackoffMin = DefaultCacheRefreshBackoffMin - } - if options.CacheRefreshMaxWait == 0 { - options.CacheRefreshMaxWait = DefaultCacheRefreshMaxWait - } if options.Logger == nil { options.Logger = hclog.New(nil) } @@ -239,6 +234,7 @@ func New(options Options) *Cache { types: make(map[string]typeEntry), entries: make(map[string]cacheEntry), entriesExpiryHeap: ttlcache.NewExpiryHeap(), + fetchHandles: make(map[string]fetchHandle), stopCh: make(chan struct{}), options: options, rateLimitContext: ctx, @@ -408,23 +404,11 @@ func (c *Cache) getEntryLocked( // Check if re-validate is requested. If so the first time round the // loop is not a hit but subsequent ones should be treated normally. if !tEntry.Opts.Refresh && info.MustRevalidate { - // It is important to note that this block ONLY applies when we are not - // in indefinite refresh mode (where the underlying goroutine will - // continue to re-query for data). - // - // In this mode goroutines have a 1:1 relationship to RPCs that get - // executed, and importantly they DO NOT SLEEP after executing. - // - // This means that a running goroutine for this cache entry extremely - // strongly implies that the RPC has not yet completed, which is why - // this check works for the revalidation-avoidance optimization here. - if entry.GoroutineID != 0 { - // There is an active goroutine performing a blocking query for - // this data, which has not returned. - // - // We can logically deduce that the contents of the cache are - // actually current, and we can simply return this while leaving - // the blocking query alone. + if entry.Fetching { + // There is an active blocking query for this data, which has not + // returned. We can logically deduce that the contents of the cache + // are actually current, and we can simply return this while + // leaving the blocking query alone. return true, true, entry } return true, false, entry @@ -549,12 +533,15 @@ RETRY_GET: // Set our timeout channel if we must if r.Info.Timeout > 0 && timeoutCh == nil { - timeoutCh = time.After(r.Info.Timeout) + timeoutTimer := time.NewTimer(r.Info.Timeout) + defer timeoutTimer.Stop() + + timeoutCh = timeoutTimer.C } // At this point, we know we either don't have a value at all or the // value we have is too old. We need to wait for new data. - waiterCh := c.fetch(key, r) + waiterCh := c.fetch(key, r, true, 0, false) // No longer our first time through first = false @@ -581,36 +568,46 @@ func makeEntryKey(t, dc, peerName, token, key string) string { return fmt.Sprintf("%s/%s/%s/%s", t, dc, token, key) } -// fetch triggers a new background fetch for the given Request. If a background -// fetch is already running or a goroutine to manage that still exists for a -// matching Request, the waiter channel for that request is returned. The -// effect of this is that there is only ever one blocking query and goroutine -// for any matching requests. -func (c *Cache) fetch(key string, r getOptions) <-chan struct{} { +// fetch triggers a new background fetch for the given Request. If a +// background fetch is already running for a matching Request, the waiter +// channel for that request is returned. The effect of this is that there +// is only ever one blocking query for any matching requests. +// +// If allowNew is true then the fetch should create the cache entry +// if it doesn't exist. If this is false, then fetch will do nothing +// if the entry doesn't exist. This latter case is to support refreshing. +func (c *Cache) fetch(key string, r getOptions, allowNew bool, attempt uint, ignoreExisting bool) <-chan struct{} { + // We acquire a write lock because we may have to set Fetching to true. c.entriesLock.Lock() defer c.entriesLock.Unlock() - ok, entryValid, entry := c.getEntryLocked(r.TypeEntry, key, r.Info) - switch { - case ok && entryValid: - // This handles the case where a fetch succeeded after checking for its - // existence in getWithIndex. This ensures that we don't miss updates. + // This handles the case where a fetch succeeded after checking for its existence in + // getWithIndex. This ensures that we don't miss updates. + if ok && entryValid && !ignoreExisting { ch := make(chan struct{}) close(ch) return ch + } - case ok && entry.GoroutineID != 0: - // If we already have an entry and there's a goroutine to keep it - // refreshed then don't spawn another one to do the same work. - // - // Return the currently active waiter. + // If we aren't allowing new values and we don't have an existing value, + // return immediately. We return an immediately-closed channel so nothing + // blocks. + if !ok && !allowNew { + ch := make(chan struct{}) + close(ch) + return ch + } + + // If we already have an entry and it is actively fetching, then return + // the currently active waiter. + if ok && entry.Fetching { return entry.Waiter + } - case !ok: - // If we don't have an entry, then create it. The entry must be marked - // as invalid so that it isn't returned as a valid value for a zero - // index. + // If we don't have an entry, then create it. The entry must be marked + // as invalid so that it isn't returned as a valid value for a zero index. + if !ok { entry = cacheEntry{ Valid: false, Waiter: make(chan struct{}), @@ -621,100 +618,27 @@ func (c *Cache) fetch(key string, r getOptions) <-chan struct{} { } } - // Assign each background fetching goroutine a unique ID and fingerprint - // the cache entry with the same ID. This way if the cache entry is ever - // cleaned up due to expiry and later recreated the old goroutine can - // detect that and terminate rather than leak and do double work. - c.lastGoroutineID++ - entry.GoroutineID = c.lastGoroutineID + // Set that we're fetching to true, which makes it so that future + // identical calls to fetch will return the same waiter rather than + // perform multiple fetches. + entry.Fetching = true c.entries[key] = entry metrics.SetGauge([]string{"consul", "cache", "entries_count"}, float32(len(c.entries))) metrics.SetGauge([]string{"cache", "entries_count"}, float32(len(c.entries))) - // The actual Fetch must be performed in a goroutine. - go c.launchBackgroundFetcher(entry.GoroutineID, key, r) - - return entry.Waiter -} - -func (c *Cache) launchBackgroundFetcher(goroutineID uint64, key string, r getOptions) { - defer func() { - c.entriesLock.Lock() - defer c.entriesLock.Unlock() - entry, ok := c.entries[key] - if ok && entry.GoroutineID == goroutineID { - entry.GoroutineID = 0 - c.entries[key] = entry - } - }() - - var attempt uint - for { - shouldStop, shouldBackoff := c.runBackgroundFetcherOnce(goroutineID, key, r) - if shouldStop { - return - } - - if shouldBackoff { - attempt++ - } else { - attempt = 0 - } - // If we're over the attempt minimum, start an exponential backoff. - wait := backOffWait(c.options, attempt) - - // If we have a timer, wait for it - wait += r.TypeEntry.Opts.RefreshTimer - - select { - case <-time.After(wait): - case <-c.stopCh: - return // Check if cache was stopped - } - - // Trigger. - r.Info.MustRevalidate = false - r.Info.MinIndex = 0 - - // We acquire a write lock because we may have to set Fetching to true. - c.entriesLock.Lock() - - entry, ok := c.entries[key] - if !ok || entry.GoroutineID != goroutineID { - // If we don't have an existing entry, return immediately. - // - // Also if we already have an entry and it is actively fetching, then - // return immediately. - // - // If we've somehow lost control of the entry, also return. - c.entriesLock.Unlock() - return - } - - c.entries[key] = entry - metrics.SetGauge([]string{"consul", "cache", "entries_count"}, float32(len(c.entries))) - metrics.SetGauge([]string{"cache", "entries_count"}, float32(len(c.entries))) - c.entriesLock.Unlock() - } -} - -func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOptions) (shouldStop, shouldBackoff bool) { - // Freshly re-read this, rather than relying upon the caller to fetch it - // and pass it in. - c.entriesLock.RLock() - entry, ok := c.entries[key] - c.entriesLock.RUnlock() + tEntry := r.TypeEntry - if !ok || entry.GoroutineID != goroutineID { - // If we don't have an existing entry, return immediately. - // - // Also if something weird has happened to orphan this goroutine, also - // return immediately. - return true, false - } + // The actual Fetch must be performed in a goroutine. Ensure that we only + // have one in-flight at a time, but don't use a deferred + // context.WithCancel style termination so that these things outlive their + // requester. + // + // By the time we get here the system WANTS to make a replacement fetcher, so + // we terminate the prior one and replace it. + handle := c.getOrReplaceFetchHandle(key) + go func(handle fetchHandle) { + defer c.deleteFetchHandle(key, handle.id) - tEntry := r.TypeEntry - { // NOTE: this indentation is here to facilitate the PR review diff only // If we have background refresh and currently are in "disconnected" state, // waiting for a response might mean we mark our results as stale for up to // 10 minutes (max blocking timeout) after connection is restored. To reduce @@ -728,7 +652,7 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp c.entriesLock.Lock() defer c.entriesLock.Unlock() entry, ok := c.entries[key] - if !ok || entry.RefreshLostContact.IsZero() || entry.GoroutineID != goroutineID { + if !ok || entry.RefreshLostContact.IsZero() { return } entry.RefreshLostContact = time.Time{} @@ -752,15 +676,12 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp Index: entry.Index, } } - if err := entry.FetchRateLimiter.Wait(c.rateLimitContext); err != nil { if connectedTimer != nil { connectedTimer.Stop() } entry.Error = fmt.Errorf("rateLimitContext canceled: %s", err.Error()) - // NOTE: this can only happen when the entire cache is being - // shutdown and isn't something that can happen normally. - return true, false + return } // Start building the new entry by blocking on the fetch. result, err := r.Fetch(fOpts) @@ -768,8 +689,17 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp connectedTimer.Stop() } + // If we were stopped while waiting on a blocking query now would be a + // good time to detect that. + select { + case <-handle.stopCh: + return + default: + } + // Copy the existing entry to start. newEntry := entry + newEntry.Fetching = false // Importantly, always reset the Error. Having both Error and a Value that // are non-nil is allowed in the cache entry but it indicates that the Error @@ -825,7 +755,7 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp if result.Index > 0 { // Reset the attempts counter so we don't have any backoff - shouldBackoff = false + attempt = 0 } else { // Result having a zero index is an implicit error case. There was no // actual error but it implies the RPC found in index (nothing written @@ -840,7 +770,7 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp // state it can be considered a bug in the RPC implementation (to ever // return a zero index) however since it can happen this is a safety net // for the future. - shouldBackoff = true + attempt++ } // If we have refresh active, this successful response means cache is now @@ -860,7 +790,7 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp metrics.IncrCounterWithLabels([]string{"cache", tEntry.Name, "fetch_error"}, 1, labels) // Increment attempt counter - shouldBackoff = true + attempt++ // If we are refreshing and just failed, updated the lost contact time as // our cache will be stale until we get successfully reconnected. We only @@ -877,7 +807,7 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp // Set our entry c.entriesLock.Lock() - if currEntry, ok := c.entries[key]; !ok || currEntry.GoroutineID != goroutineID { + if _, ok := c.entries[key]; !ok { // This entry was evicted during our fetch. DON'T re-insert it or fall // through to the refresh loop below otherwise it will live forever! In // theory there should not be any Get calls waiting on entry.Waiter since @@ -890,7 +820,7 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp // Trigger any waiters that are around. close(entry.Waiter) - return true, false + return } // If this is a new entry (not in the heap yet), then setup the @@ -915,22 +845,79 @@ func (c *Cache) runBackgroundFetcherOnce(goroutineID uint64, key string, r getOp // request back up again shortly but in the general case this prevents // spamming the logs with tons of ACL not found errors for days. if tEntry.Opts.Refresh && !preventRefresh { - return false, shouldBackoff + // Check if cache was stopped + if atomic.LoadUint32(&c.stopped) == 1 { + return + } + + // If we're over the attempt minimum, start an exponential backoff. + wait := backOffWait(attempt) + + // If we have a timer, wait for it + wait += tEntry.Opts.RefreshTimer + + select { + case <-time.After(wait): + case <-handle.stopCh: + return + } + + // Trigger. The "allowNew" field is false because in the time we were + // waiting to refresh we may have expired and got evicted. If that + // happened, we don't want to create a new entry. + r.Info.MustRevalidate = false + r.Info.MinIndex = 0 + c.fetch(key, r, false, attempt, true) } + }(handle) + + return entry.Waiter +} + +func (c *Cache) getOrReplaceFetchHandle(key string) fetchHandle { + c.fetchLock.Lock() + defer c.fetchLock.Unlock() + + if prevHandle, ok := c.fetchHandles[key]; ok { + close(prevHandle.stopCh) } - return true, false + c.lastFetchID++ + + handle := fetchHandle{ + id: c.lastFetchID, + stopCh: make(chan struct{}), + } + + c.fetchHandles[key] = handle + + return handle +} + +func (c *Cache) deleteFetchHandle(key string, fetchID uint64) { + c.fetchLock.Lock() + defer c.fetchLock.Unlock() + + // Only remove a fetchHandle if it's YOUR fetchHandle. + handle, ok := c.fetchHandles[key] + if !ok { + return + } + + if handle.id == fetchID { + delete(c.fetchHandles, key) + } } -func backOffWait(opts Options, failures uint) time.Duration { - if failures > opts.CacheRefreshBackoffMin { - shift := failures - opts.CacheRefreshBackoffMin - waitTime := opts.CacheRefreshMaxWait +func backOffWait(failures uint) time.Duration { + if failures > CacheRefreshBackoffMin { + shift := failures - CacheRefreshBackoffMin + waitTime := CacheRefreshMaxWait if shift < 31 { waitTime = (1 << shift) * time.Second } - if waitTime > opts.CacheRefreshMaxWait { - waitTime = opts.CacheRefreshMaxWait + if waitTime > CacheRefreshMaxWait { + waitTime = CacheRefreshMaxWait } return waitTime + lib.RandomStagger(waitTime) } diff --git a/agent/cache/cache_test.go b/agent/cache/cache_test.go index 98b04ee9a48..6f8805be06d 100644 --- a/agent/cache/cache_test.go +++ b/agent/cache/cache_test.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/lib/ttlcache" "github.com/hashicorp/consul/sdk/testutil" + "github.com/hashicorp/consul/sdk/testutil/retry" ) // Test a basic Get with no indexes (and therefore no blocking queries). @@ -1750,12 +1751,22 @@ func TestCache_RefreshLifeCycle(t *testing.T) { require.NoError(t, err) require.Equal(t, true, result) + waitUntilFetching := func(expectValue bool) { + retry.Run(t, func(t *retry.R) { + c.entriesLock.Lock() + defer c.entriesLock.Unlock() + entry, ok := c.entries[key] + require.True(t, ok) + if expectValue { + require.True(t, entry.Fetching) + } else { + require.False(t, entry.Fetching) + } + }) + } + // ensure that the entry is fetching again - c.entriesLock.Lock() - entry, ok := c.entries[key] - require.True(t, ok) - require.True(t, entry.GoroutineID > 0) - c.entriesLock.Unlock() + waitUntilFetching(true) requestChan := make(chan error) @@ -1789,11 +1800,7 @@ func TestCache_RefreshLifeCycle(t *testing.T) { } // ensure that the entry is fetching again - c.entriesLock.Lock() - entry, ok = c.entries[key] - require.True(t, ok) - require.True(t, entry.GoroutineID > 0) - c.entriesLock.Unlock() + waitUntilFetching(true) // background a call that will wait for a newer version - will result in an acl not found error go getError(5) @@ -1814,11 +1821,7 @@ func TestCache_RefreshLifeCycle(t *testing.T) { // ensure that the ACL not found error killed off the background refresh // but didn't remove it from the cache - c.entriesLock.Lock() - entry, ok = c.entries[key] - require.True(t, ok) - require.False(t, entry.GoroutineID > 0) - c.entriesLock.Unlock() + waitUntilFetching(false) } type fakeType struct { diff --git a/agent/cache/entry.go b/agent/cache/entry.go index 7130381dea4..0c71e944371 100644 --- a/agent/cache/entry.go +++ b/agent/cache/entry.go @@ -26,9 +26,9 @@ type cacheEntry struct { Index uint64 // Metadata that is used for internal accounting - Valid bool // True if the Value is set - GoroutineID uint64 // Nonzero if a fetch goroutine is running. - Waiter chan struct{} // Closed when this entry is invalidated + Valid bool // True if the Value is set + Fetching bool // True if a fetch is already active + Waiter chan struct{} // Closed when this entry is invalidated // Expiry contains information about the expiration of this // entry. This is a pointer as its shared as a value in the diff --git a/agent/cache/watch.go b/agent/cache/watch.go index d87bca38a54..abd247c4f1c 100644 --- a/agent/cache/watch.go +++ b/agent/cache/watch.go @@ -137,7 +137,7 @@ func (c *Cache) notifyBlockingQuery(ctx context.Context, r getOptions, correlati failures = 0 } else { failures++ - wait = backOffWait(c.options, failures) + wait = backOffWait(failures) c.options.Logger. With("error", err). @@ -224,7 +224,7 @@ func (c *Cache) notifyPollingQuery(ctx context.Context, r getOptions, correlatio // as this would eliminate the single-flighting of these requests in the cache and // the efficiencies gained by it. if failures > 0 { - wait = backOffWait(c.options, failures) + wait = backOffWait(failures) } else { // Calculate when the cached data's Age will get too stale and // need to be re-queried. When the data's Age already exceeds the diff --git a/agent/catalog_endpoint_oss.go b/agent/catalog_endpoint_ce.go similarity index 100% rename from agent/catalog_endpoint_oss.go rename to agent/catalog_endpoint_ce.go diff --git a/agent/checks/check_test.go b/agent/checks/check_test.go index 495fc1472bb..a4bdf2ca3ed 100644 --- a/agent/checks/check_test.go +++ b/agent/checks/check_test.go @@ -1147,9 +1147,6 @@ func TestCheckTCPPassing(t *testing.T) { if os.Getenv("TRAVIS") == "true" { t.Skip("IPV6 not supported on travis-ci") } - if os.Getenv("CIRCLECI") == "true" { - t.Skip("IPV6 not supported on CircleCI") - } tcpServer = mockTCPServer(`tcp6`) expectTCPStatus(t, tcpServer.Addr().String(), api.HealthPassing) tcpServer.Close() diff --git a/agent/config/builder.go b/agent/config/builder.go index f682bf7b142..33fcdfe1d34 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -25,8 +25,6 @@ import ( "github.com/hashicorp/memberlist" "golang.org/x/time/rate" - hcpconfig "github.com/hashicorp/consul/agent/hcp/config" - "github.com/hashicorp/consul/agent/cache" "github.com/hashicorp/consul/agent/checks" "github.com/hashicorp/consul/agent/connect/ca" @@ -34,6 +32,7 @@ import ( "github.com/hashicorp/consul/agent/consul/authmethod/ssoauth" consulrate "github.com/hashicorp/consul/agent/consul/rate" "github.com/hashicorp/consul/agent/dns" + hcpconfig "github.com/hashicorp/consul/agent/hcp/config" "github.com/hashicorp/consul/agent/rpc/middleware" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/token" @@ -972,7 +971,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) { AutoEncryptIPSAN: autoEncryptIPSAN, AutoEncryptAllowTLS: autoEncryptAllowTLS, AutoConfig: autoConfig, - Cloud: b.cloudConfigVal(c.Cloud), + Cloud: b.cloudConfigVal(c), ConnectEnabled: connectEnabled, ConnectCAProvider: connectCAProvider, ConnectCAConfig: connectCAConfig, @@ -1078,6 +1077,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) { ServerMode: serverMode, ServerName: stringVal(c.ServerName), ServerPort: serverPort, + ServerRejoinAgeMax: b.durationValWithDefaultMin("server_rejoin_age_max", c.ServerRejoinAgeMax, 24*7*time.Hour, 6*time.Hour), Services: services, SessionTTLMin: b.durationVal("session_ttl_min", c.SessionTTLMin), SkipLeaveOnInt: skipLeaveOnInt, @@ -1091,8 +1091,12 @@ func (b *builder) build() (rt RuntimeConfig, err error) { Watches: c.Watches, XDSUpdateRateLimit: limitVal(c.XDS.UpdateMaxPerSecond), AutoReloadConfigCoalesceInterval: 1 * time.Second, + LocalProxyConfigResyncInterval: 30 * time.Second, } + // host metrics are enabled by default if consul is configured with HashiCorp Cloud Platform integration + rt.Telemetry.EnableHostMetrics = boolValWithDefault(c.Telemetry.EnableHostMetrics, rt.IsCloudEnabled()) + rt.TLS, err = b.buildTLSConfig(rt, c.TLS) if err != nil { return RuntimeConfig{}, err @@ -1939,6 +1943,16 @@ func (b *builder) durationValWithDefault(name string, v *string, defaultVal time return d } +// durationValWithDefaultMin is equivalent to durationValWithDefault, but enforces a minimum duration. +func (b *builder) durationValWithDefaultMin(name string, v *string, defaultVal, minVal time.Duration) (d time.Duration) { + d = b.durationValWithDefault(name, v, defaultVal) + if d < minVal { + b.err = multierror.Append(b.err, fmt.Errorf("%s: duration '%s' cannot be less than: %s", name, *v, minVal)) + } + + return d +} + func (b *builder) durationVal(name string, v *string) (d time.Duration) { return b.durationValWithDefault(name, v, 0) } @@ -2514,18 +2528,28 @@ func validateAutoConfigAuthorizer(rt RuntimeConfig) error { return nil } -func (b *builder) cloudConfigVal(v *CloudConfigRaw) (val hcpconfig.CloudConfig) { - if v == nil { +func (b *builder) cloudConfigVal(v Config) hcpconfig.CloudConfig { + val := hcpconfig.CloudConfig{ + ResourceID: os.Getenv("HCP_RESOURCE_ID"), + } + // Node id might get overriden in setup.go:142 + nodeID := stringVal(v.NodeID) + val.NodeID = types.NodeID(nodeID) + val.NodeName = b.nodeName(v.NodeName) + + if v.Cloud == nil { return val } - val.ResourceID = stringVal(v.ResourceID) - val.ClientID = stringVal(v.ClientID) - val.ClientSecret = stringVal(v.ClientSecret) - val.AuthURL = stringVal(v.AuthURL) - val.Hostname = stringVal(v.Hostname) - val.ScadaAddress = stringVal(v.ScadaAddress) + val.ClientID = stringVal(v.Cloud.ClientID) + val.ClientSecret = stringVal(v.Cloud.ClientSecret) + val.AuthURL = stringVal(v.Cloud.AuthURL) + val.Hostname = stringVal(v.Cloud.Hostname) + val.ScadaAddress = stringVal(v.Cloud.ScadaAddress) + if resourceID := stringVal(v.Cloud.ResourceID); resourceID != "" { + val.ResourceID = resourceID + } return val } diff --git a/agent/config/builder_oss.go b/agent/config/builder_ce.go similarity index 92% rename from agent/config/builder_oss.go rename to agent/config/builder_ce.go index ce6e8d44ce0..cfe05b5fd9d 100644 --- a/agent/config/builder_oss.go +++ b/agent/config/builder_ce.go @@ -10,7 +10,7 @@ import ( // validateEnterpriseConfig is a function to validate the enterprise specific // configuration items after Parsing but before merging into the overall // configuration. The original intent is to use it to ensure that we warn -// for enterprise configurations used in OSS. +// for enterprise configurations used in CE. func validateEnterpriseConfigKeys(config *Config) []error { var result []error add := func(k string) { @@ -57,6 +57,10 @@ func validateEnterpriseConfigKeys(config *Config) []error { add("license_path") config.LicensePath = nil } + if config.Reporting.License.Enabled != nil { + add("reporting.license.enabled") + config.Reporting.License.Enabled = nil + } return result } diff --git a/agent/config/builder_oss_test.go b/agent/config/builder_ce_test.go similarity index 91% rename from agent/config/builder_oss_test.go rename to agent/config/builder_ce_test.go index 2fd5f50ad7c..1fdba09e602 100644 --- a/agent/config/builder_oss_test.go +++ b/agent/config/builder_ce_test.go @@ -107,6 +107,19 @@ func TestValidateEnterpriseConfigKeys(t *testing.T) { require.Empty(t, c.LicensePath) }, }, + "reporting.license.enabled": { + config: Config{ + Reporting: Reporting{ + License: License{ + Enabled: &boolVal, + }, + }, + }, + badKeys: []string{"reporting.license.enabled"}, + check: func(t *testing.T, c *Config) { + require.Nil(t, c.Reporting.License.Enabled) + }, + }, "multi": { config: Config{ ReadReplica: &boolVal, diff --git a/agent/config/builder_test.go b/agent/config/builder_test.go index 9fdd085b1d8..16bacbedffe 100644 --- a/agent/config/builder_test.go +++ b/agent/config/builder_test.go @@ -308,6 +308,21 @@ func TestBuilder_DurationVal_InvalidDuration(t *testing.T) { require.Contains(t, b.err.Error(), badDuration2) } +func TestBuilder_DurationValWithDefaultMin(t *testing.T) { + b := builder{} + + // Attempt to validate that a duration of 10 hours will not error when the min val is 1 hour. + dur := "10h0m0s" + b.durationValWithDefaultMin("field2", &dur, 24*7*time.Hour, time.Hour) + require.NoError(t, b.err) + + // Attempt to validate that a duration of 1 min will error when the min val is 1 hour. + dur = "0h1m0s" + b.durationValWithDefaultMin("field1", &dur, 24*7*time.Hour, time.Hour) + require.Error(t, b.err) + require.Contains(t, b.err.Error(), "1 error") +} + func TestBuilder_ServiceVal_MultiError(t *testing.T) { b := builder{} b.serviceVal(&ServiceDefinition{ @@ -538,3 +553,22 @@ func TestBuilder_parsePrefixFilter(t *testing.T) { } }) } + +func TestBuidler_hostMetricsWithCloud(t *testing.T) { + devMode := true + builderOpts := LoadOpts{ + DevMode: &devMode, + DefaultConfig: FileSource{ + Name: "test", + Format: "hcl", + Data: `cloud{ resource_id = "abc" client_id = "abc" client_secret = "abc"}`, + }, + } + + result, err := Load(builderOpts) + require.NoError(t, err) + require.Empty(t, result.Warnings) + cfg := result.RuntimeConfig + require.NotNil(t, cfg) + require.True(t, cfg.Telemetry.EnableHostMetrics) +} diff --git a/agent/config/config.deepcopy.go b/agent/config/config.deepcopy.go new file mode 100644 index 00000000000..447277fd562 --- /dev/null +++ b/agent/config/config.deepcopy.go @@ -0,0 +1,1125 @@ +// generated by deep-copy -pointer-receiver -o ./config.deepcopy.go -type RuntimeConfig ./; DO NOT EDIT. + +package config + +import ( + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "github.com/armon/go-metrics" + "github.com/armon/go-metrics/prometheus" + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/types" + "math/big" + "net" + "net/url" + "time" +) + +// DeepCopy generates a deep copy of *RuntimeConfig +func (o *RuntimeConfig) DeepCopy() *RuntimeConfig { + var cp RuntimeConfig = *o + if o.Cloud.TLSConfig != nil { + cp.Cloud.TLSConfig = new(tls.Config) + *cp.Cloud.TLSConfig = *o.Cloud.TLSConfig + if o.Cloud.TLSConfig.Certificates != nil { + cp.Cloud.TLSConfig.Certificates = make([]tls.Certificate, len(o.Cloud.TLSConfig.Certificates)) + copy(cp.Cloud.TLSConfig.Certificates, o.Cloud.TLSConfig.Certificates) + for i5 := range o.Cloud.TLSConfig.Certificates { + if o.Cloud.TLSConfig.Certificates[i5].Certificate != nil { + cp.Cloud.TLSConfig.Certificates[i5].Certificate = make([][]byte, len(o.Cloud.TLSConfig.Certificates[i5].Certificate)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Certificate, o.Cloud.TLSConfig.Certificates[i5].Certificate) + for i7 := range o.Cloud.TLSConfig.Certificates[i5].Certificate { + if o.Cloud.TLSConfig.Certificates[i5].Certificate[i7] != nil { + cp.Cloud.TLSConfig.Certificates[i5].Certificate[i7] = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Certificate[i7])) + copy(cp.Cloud.TLSConfig.Certificates[i5].Certificate[i7], o.Cloud.TLSConfig.Certificates[i5].Certificate[i7]) + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].SupportedSignatureAlgorithms != nil { + cp.Cloud.TLSConfig.Certificates[i5].SupportedSignatureAlgorithms = make([]tls.SignatureScheme, len(o.Cloud.TLSConfig.Certificates[i5].SupportedSignatureAlgorithms)) + copy(cp.Cloud.TLSConfig.Certificates[i5].SupportedSignatureAlgorithms, o.Cloud.TLSConfig.Certificates[i5].SupportedSignatureAlgorithms) + } + if o.Cloud.TLSConfig.Certificates[i5].OCSPStaple != nil { + cp.Cloud.TLSConfig.Certificates[i5].OCSPStaple = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].OCSPStaple)) + copy(cp.Cloud.TLSConfig.Certificates[i5].OCSPStaple, o.Cloud.TLSConfig.Certificates[i5].OCSPStaple) + } + if o.Cloud.TLSConfig.Certificates[i5].SignedCertificateTimestamps != nil { + cp.Cloud.TLSConfig.Certificates[i5].SignedCertificateTimestamps = make([][]byte, len(o.Cloud.TLSConfig.Certificates[i5].SignedCertificateTimestamps)) + copy(cp.Cloud.TLSConfig.Certificates[i5].SignedCertificateTimestamps, o.Cloud.TLSConfig.Certificates[i5].SignedCertificateTimestamps) + for i7 := range o.Cloud.TLSConfig.Certificates[i5].SignedCertificateTimestamps { + if o.Cloud.TLSConfig.Certificates[i5].SignedCertificateTimestamps[i7] != nil { + cp.Cloud.TLSConfig.Certificates[i5].SignedCertificateTimestamps[i7] = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].SignedCertificateTimestamps[i7])) + copy(cp.Cloud.TLSConfig.Certificates[i5].SignedCertificateTimestamps[i7], o.Cloud.TLSConfig.Certificates[i5].SignedCertificateTimestamps[i7]) + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf = new(x509.Certificate) + *cp.Cloud.TLSConfig.Certificates[i5].Leaf = *o.Cloud.TLSConfig.Certificates[i5].Leaf + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Raw != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Raw = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Raw)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Raw, o.Cloud.TLSConfig.Certificates[i5].Leaf.Raw) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.RawTBSCertificate != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.RawTBSCertificate = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.RawTBSCertificate)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.RawTBSCertificate, o.Cloud.TLSConfig.Certificates[i5].Leaf.RawTBSCertificate) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.RawSubjectPublicKeyInfo != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.RawSubjectPublicKeyInfo = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.RawSubjectPublicKeyInfo)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.RawSubjectPublicKeyInfo, o.Cloud.TLSConfig.Certificates[i5].Leaf.RawSubjectPublicKeyInfo) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.RawSubject != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.RawSubject = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.RawSubject)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.RawSubject, o.Cloud.TLSConfig.Certificates[i5].Leaf.RawSubject) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.RawIssuer != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.RawIssuer = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.RawIssuer)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.RawIssuer, o.Cloud.TLSConfig.Certificates[i5].Leaf.RawIssuer) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Signature != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Signature = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Signature)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Signature, o.Cloud.TLSConfig.Certificates[i5].Leaf.Signature) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.SerialNumber != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.SerialNumber = new(big.Int) + *cp.Cloud.TLSConfig.Certificates[i5].Leaf.SerialNumber = *o.Cloud.TLSConfig.Certificates[i5].Leaf.SerialNumber + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Country != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Country = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Country)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Country, o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Country) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Organization != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Organization = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Organization)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Organization, o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Organization) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.OrganizationalUnit != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.OrganizationalUnit = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.OrganizationalUnit)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.OrganizationalUnit, o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.OrganizationalUnit) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Locality != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Locality = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Locality)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Locality, o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Locality) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Province != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Province = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Province)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Province, o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Province) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.StreetAddress != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.StreetAddress = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.StreetAddress)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.StreetAddress, o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.StreetAddress) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.PostalCode != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.PostalCode = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.PostalCode)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.PostalCode, o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.PostalCode) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Names != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Names = make([]pkix.AttributeTypeAndValue, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Names)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Names, o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Names) + for i10 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Names { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Names[i10].Type != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Names[i10].Type = make([]int, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Names[i10].Type)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Names[i10].Type, o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.Names[i10].Type) + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.ExtraNames != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.ExtraNames = make([]pkix.AttributeTypeAndValue, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.ExtraNames)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.ExtraNames, o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.ExtraNames) + for i10 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.ExtraNames { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.ExtraNames[i10].Type != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.ExtraNames[i10].Type = make([]int, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.ExtraNames[i10].Type)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.ExtraNames[i10].Type, o.Cloud.TLSConfig.Certificates[i5].Leaf.Issuer.ExtraNames[i10].Type) + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Country != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Country = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Country)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Country, o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Country) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Organization != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Organization = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Organization)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Organization, o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Organization) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.OrganizationalUnit != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.OrganizationalUnit = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.OrganizationalUnit)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.OrganizationalUnit, o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.OrganizationalUnit) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Locality != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Locality = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Locality)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Locality, o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Locality) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Province != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Province = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Province)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Province, o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Province) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.StreetAddress != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.StreetAddress = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.StreetAddress)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.StreetAddress, o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.StreetAddress) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.PostalCode != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.PostalCode = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.PostalCode)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.PostalCode, o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.PostalCode) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Names != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Names = make([]pkix.AttributeTypeAndValue, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Names)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Names, o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Names) + for i10 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Names { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Names[i10].Type != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Names[i10].Type = make([]int, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Names[i10].Type)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Names[i10].Type, o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.Names[i10].Type) + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.ExtraNames != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.ExtraNames = make([]pkix.AttributeTypeAndValue, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.ExtraNames)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.ExtraNames, o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.ExtraNames) + for i10 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.ExtraNames { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.ExtraNames[i10].Type != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.ExtraNames[i10].Type = make([]int, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.ExtraNames[i10].Type)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.ExtraNames[i10].Type, o.Cloud.TLSConfig.Certificates[i5].Leaf.Subject.ExtraNames[i10].Type) + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions = make([]pkix.Extension, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions, o.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions) + for i9 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions[i9].Id != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions[i9].Id = make([]int, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions[i9].Id)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions[i9].Id, o.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions[i9].Id) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions[i9].Value != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions[i9].Value = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions[i9].Value)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions[i9].Value, o.Cloud.TLSConfig.Certificates[i5].Leaf.Extensions[i9].Value) + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions = make([]pkix.Extension, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions, o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions) + for i9 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions[i9].Id != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions[i9].Id = make([]int, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions[i9].Id)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions[i9].Id, o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions[i9].Id) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions[i9].Value != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions[i9].Value = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions[i9].Value)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions[i9].Value, o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtraExtensions[i9].Value) + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.UnhandledCriticalExtensions != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.UnhandledCriticalExtensions = make([]asn1.ObjectIdentifier, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.UnhandledCriticalExtensions)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.UnhandledCriticalExtensions, o.Cloud.TLSConfig.Certificates[i5].Leaf.UnhandledCriticalExtensions) + for i9 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.UnhandledCriticalExtensions { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.UnhandledCriticalExtensions[i9] != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.UnhandledCriticalExtensions[i9] = make([]int, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.UnhandledCriticalExtensions[i9])) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.UnhandledCriticalExtensions[i9], o.Cloud.TLSConfig.Certificates[i5].Leaf.UnhandledCriticalExtensions[i9]) + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtKeyUsage != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExtKeyUsage = make([]x509.ExtKeyUsage, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtKeyUsage)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExtKeyUsage, o.Cloud.TLSConfig.Certificates[i5].Leaf.ExtKeyUsage) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.UnknownExtKeyUsage != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.UnknownExtKeyUsage = make([]asn1.ObjectIdentifier, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.UnknownExtKeyUsage)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.UnknownExtKeyUsage, o.Cloud.TLSConfig.Certificates[i5].Leaf.UnknownExtKeyUsage) + for i9 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.UnknownExtKeyUsage { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.UnknownExtKeyUsage[i9] != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.UnknownExtKeyUsage[i9] = make([]int, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.UnknownExtKeyUsage[i9])) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.UnknownExtKeyUsage[i9], o.Cloud.TLSConfig.Certificates[i5].Leaf.UnknownExtKeyUsage[i9]) + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.SubjectKeyId != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.SubjectKeyId = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.SubjectKeyId)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.SubjectKeyId, o.Cloud.TLSConfig.Certificates[i5].Leaf.SubjectKeyId) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.AuthorityKeyId != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.AuthorityKeyId = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.AuthorityKeyId)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.AuthorityKeyId, o.Cloud.TLSConfig.Certificates[i5].Leaf.AuthorityKeyId) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.OCSPServer != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.OCSPServer = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.OCSPServer)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.OCSPServer, o.Cloud.TLSConfig.Certificates[i5].Leaf.OCSPServer) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.IssuingCertificateURL != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.IssuingCertificateURL = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.IssuingCertificateURL)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.IssuingCertificateURL, o.Cloud.TLSConfig.Certificates[i5].Leaf.IssuingCertificateURL) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.DNSNames != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.DNSNames = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.DNSNames)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.DNSNames, o.Cloud.TLSConfig.Certificates[i5].Leaf.DNSNames) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.EmailAddresses != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.EmailAddresses = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.EmailAddresses)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.EmailAddresses, o.Cloud.TLSConfig.Certificates[i5].Leaf.EmailAddresses) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.IPAddresses != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.IPAddresses = make([]net.IP, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.IPAddresses)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.IPAddresses, o.Cloud.TLSConfig.Certificates[i5].Leaf.IPAddresses) + for i9 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.IPAddresses { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.IPAddresses[i9] != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.IPAddresses[i9] = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.IPAddresses[i9])) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.IPAddresses[i9], o.Cloud.TLSConfig.Certificates[i5].Leaf.IPAddresses[i9]) + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.URIs != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.URIs = make([]*url.URL, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.URIs)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.URIs, o.Cloud.TLSConfig.Certificates[i5].Leaf.URIs) + for i9 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.URIs { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.URIs[i9] != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.URIs[i9] = new(url.URL) + *cp.Cloud.TLSConfig.Certificates[i5].Leaf.URIs[i9] = *o.Cloud.TLSConfig.Certificates[i5].Leaf.URIs[i9] + if o.Cloud.TLSConfig.Certificates[i5].Leaf.URIs[i9].User != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.URIs[i9].User = new(url.Userinfo) + *cp.Cloud.TLSConfig.Certificates[i5].Leaf.URIs[i9].User = *o.Cloud.TLSConfig.Certificates[i5].Leaf.URIs[i9].User + } + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedDNSDomains != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedDNSDomains = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedDNSDomains)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedDNSDomains, o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedDNSDomains) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedDNSDomains != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedDNSDomains = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedDNSDomains)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedDNSDomains, o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedDNSDomains) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges = make([]*net.IPNet, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges, o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges) + for i9 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9] != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9] = new(net.IPNet) + *cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9] = *o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9] + if o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9].IP != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9].IP = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9].IP)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9].IP, o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9].IP) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9].Mask != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9].Mask = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9].Mask)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9].Mask, o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedIPRanges[i9].Mask) + } + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges = make([]*net.IPNet, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges, o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges) + for i9 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9] != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9] = new(net.IPNet) + *cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9] = *o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9] + if o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9].IP != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9].IP = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9].IP)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9].IP, o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9].IP) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9].Mask != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9].Mask = make([]byte, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9].Mask)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9].Mask, o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedIPRanges[i9].Mask) + } + } + } + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedEmailAddresses != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedEmailAddresses = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedEmailAddresses)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedEmailAddresses, o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedEmailAddresses) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedEmailAddresses != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedEmailAddresses = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedEmailAddresses)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedEmailAddresses, o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedEmailAddresses) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedURIDomains != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedURIDomains = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedURIDomains)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedURIDomains, o.Cloud.TLSConfig.Certificates[i5].Leaf.PermittedURIDomains) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedURIDomains != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedURIDomains = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedURIDomains)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedURIDomains, o.Cloud.TLSConfig.Certificates[i5].Leaf.ExcludedURIDomains) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.CRLDistributionPoints != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.CRLDistributionPoints = make([]string, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.CRLDistributionPoints)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.CRLDistributionPoints, o.Cloud.TLSConfig.Certificates[i5].Leaf.CRLDistributionPoints) + } + if o.Cloud.TLSConfig.Certificates[i5].Leaf.PolicyIdentifiers != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.PolicyIdentifiers = make([]asn1.ObjectIdentifier, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.PolicyIdentifiers)) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.PolicyIdentifiers, o.Cloud.TLSConfig.Certificates[i5].Leaf.PolicyIdentifiers) + for i9 := range o.Cloud.TLSConfig.Certificates[i5].Leaf.PolicyIdentifiers { + if o.Cloud.TLSConfig.Certificates[i5].Leaf.PolicyIdentifiers[i9] != nil { + cp.Cloud.TLSConfig.Certificates[i5].Leaf.PolicyIdentifiers[i9] = make([]int, len(o.Cloud.TLSConfig.Certificates[i5].Leaf.PolicyIdentifiers[i9])) + copy(cp.Cloud.TLSConfig.Certificates[i5].Leaf.PolicyIdentifiers[i9], o.Cloud.TLSConfig.Certificates[i5].Leaf.PolicyIdentifiers[i9]) + } + } + } + } + } + } + if o.Cloud.TLSConfig.NameToCertificate != nil { + cp.Cloud.TLSConfig.NameToCertificate = make(map[string]*tls.Certificate, len(o.Cloud.TLSConfig.NameToCertificate)) + for k5, v5 := range o.Cloud.TLSConfig.NameToCertificate { + var cp_Cloud_TLSConfig_NameToCertificate_v5 *tls.Certificate + if v5 != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5 = new(tls.Certificate) + *cp_Cloud_TLSConfig_NameToCertificate_v5 = *v5 + if v5.Certificate != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Certificate = make([][]byte, len(v5.Certificate)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Certificate, v5.Certificate) + for i8 := range v5.Certificate { + if v5.Certificate[i8] != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Certificate[i8] = make([]byte, len(v5.Certificate[i8])) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Certificate[i8], v5.Certificate[i8]) + } + } + } + if v5.SupportedSignatureAlgorithms != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.SupportedSignatureAlgorithms = make([]tls.SignatureScheme, len(v5.SupportedSignatureAlgorithms)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.SupportedSignatureAlgorithms, v5.SupportedSignatureAlgorithms) + } + if v5.OCSPStaple != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.OCSPStaple = make([]byte, len(v5.OCSPStaple)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.OCSPStaple, v5.OCSPStaple) + } + if v5.SignedCertificateTimestamps != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.SignedCertificateTimestamps = make([][]byte, len(v5.SignedCertificateTimestamps)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.SignedCertificateTimestamps, v5.SignedCertificateTimestamps) + for i8 := range v5.SignedCertificateTimestamps { + if v5.SignedCertificateTimestamps[i8] != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.SignedCertificateTimestamps[i8] = make([]byte, len(v5.SignedCertificateTimestamps[i8])) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.SignedCertificateTimestamps[i8], v5.SignedCertificateTimestamps[i8]) + } + } + } + if v5.Leaf != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf = new(x509.Certificate) + *cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf = *v5.Leaf + if v5.Leaf.Raw != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Raw = make([]byte, len(v5.Leaf.Raw)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Raw, v5.Leaf.Raw) + } + if v5.Leaf.RawTBSCertificate != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.RawTBSCertificate = make([]byte, len(v5.Leaf.RawTBSCertificate)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.RawTBSCertificate, v5.Leaf.RawTBSCertificate) + } + if v5.Leaf.RawSubjectPublicKeyInfo != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.RawSubjectPublicKeyInfo = make([]byte, len(v5.Leaf.RawSubjectPublicKeyInfo)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.RawSubjectPublicKeyInfo, v5.Leaf.RawSubjectPublicKeyInfo) + } + if v5.Leaf.RawSubject != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.RawSubject = make([]byte, len(v5.Leaf.RawSubject)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.RawSubject, v5.Leaf.RawSubject) + } + if v5.Leaf.RawIssuer != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.RawIssuer = make([]byte, len(v5.Leaf.RawIssuer)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.RawIssuer, v5.Leaf.RawIssuer) + } + if v5.Leaf.Signature != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Signature = make([]byte, len(v5.Leaf.Signature)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Signature, v5.Leaf.Signature) + } + if v5.Leaf.SerialNumber != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.SerialNumber = new(big.Int) + *cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.SerialNumber = *v5.Leaf.SerialNumber + } + if v5.Leaf.Issuer.Country != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.Country = make([]string, len(v5.Leaf.Issuer.Country)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.Country, v5.Leaf.Issuer.Country) + } + if v5.Leaf.Issuer.Organization != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.Organization = make([]string, len(v5.Leaf.Issuer.Organization)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.Organization, v5.Leaf.Issuer.Organization) + } + if v5.Leaf.Issuer.OrganizationalUnit != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.OrganizationalUnit = make([]string, len(v5.Leaf.Issuer.OrganizationalUnit)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.OrganizationalUnit, v5.Leaf.Issuer.OrganizationalUnit) + } + if v5.Leaf.Issuer.Locality != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.Locality = make([]string, len(v5.Leaf.Issuer.Locality)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.Locality, v5.Leaf.Issuer.Locality) + } + if v5.Leaf.Issuer.Province != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.Province = make([]string, len(v5.Leaf.Issuer.Province)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.Province, v5.Leaf.Issuer.Province) + } + if v5.Leaf.Issuer.StreetAddress != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.StreetAddress = make([]string, len(v5.Leaf.Issuer.StreetAddress)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.StreetAddress, v5.Leaf.Issuer.StreetAddress) + } + if v5.Leaf.Issuer.PostalCode != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.PostalCode = make([]string, len(v5.Leaf.Issuer.PostalCode)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.PostalCode, v5.Leaf.Issuer.PostalCode) + } + if v5.Leaf.Issuer.Names != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.Names = make([]pkix.AttributeTypeAndValue, len(v5.Leaf.Issuer.Names)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.Names, v5.Leaf.Issuer.Names) + for i11 := range v5.Leaf.Issuer.Names { + if v5.Leaf.Issuer.Names[i11].Type != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.Names[i11].Type = make([]int, len(v5.Leaf.Issuer.Names[i11].Type)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.Names[i11].Type, v5.Leaf.Issuer.Names[i11].Type) + } + } + } + if v5.Leaf.Issuer.ExtraNames != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.ExtraNames = make([]pkix.AttributeTypeAndValue, len(v5.Leaf.Issuer.ExtraNames)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.ExtraNames, v5.Leaf.Issuer.ExtraNames) + for i11 := range v5.Leaf.Issuer.ExtraNames { + if v5.Leaf.Issuer.ExtraNames[i11].Type != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.ExtraNames[i11].Type = make([]int, len(v5.Leaf.Issuer.ExtraNames[i11].Type)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Issuer.ExtraNames[i11].Type, v5.Leaf.Issuer.ExtraNames[i11].Type) + } + } + } + if v5.Leaf.Subject.Country != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.Country = make([]string, len(v5.Leaf.Subject.Country)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.Country, v5.Leaf.Subject.Country) + } + if v5.Leaf.Subject.Organization != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.Organization = make([]string, len(v5.Leaf.Subject.Organization)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.Organization, v5.Leaf.Subject.Organization) + } + if v5.Leaf.Subject.OrganizationalUnit != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.OrganizationalUnit = make([]string, len(v5.Leaf.Subject.OrganizationalUnit)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.OrganizationalUnit, v5.Leaf.Subject.OrganizationalUnit) + } + if v5.Leaf.Subject.Locality != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.Locality = make([]string, len(v5.Leaf.Subject.Locality)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.Locality, v5.Leaf.Subject.Locality) + } + if v5.Leaf.Subject.Province != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.Province = make([]string, len(v5.Leaf.Subject.Province)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.Province, v5.Leaf.Subject.Province) + } + if v5.Leaf.Subject.StreetAddress != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.StreetAddress = make([]string, len(v5.Leaf.Subject.StreetAddress)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.StreetAddress, v5.Leaf.Subject.StreetAddress) + } + if v5.Leaf.Subject.PostalCode != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.PostalCode = make([]string, len(v5.Leaf.Subject.PostalCode)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.PostalCode, v5.Leaf.Subject.PostalCode) + } + if v5.Leaf.Subject.Names != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.Names = make([]pkix.AttributeTypeAndValue, len(v5.Leaf.Subject.Names)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.Names, v5.Leaf.Subject.Names) + for i11 := range v5.Leaf.Subject.Names { + if v5.Leaf.Subject.Names[i11].Type != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.Names[i11].Type = make([]int, len(v5.Leaf.Subject.Names[i11].Type)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.Names[i11].Type, v5.Leaf.Subject.Names[i11].Type) + } + } + } + if v5.Leaf.Subject.ExtraNames != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.ExtraNames = make([]pkix.AttributeTypeAndValue, len(v5.Leaf.Subject.ExtraNames)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.ExtraNames, v5.Leaf.Subject.ExtraNames) + for i11 := range v5.Leaf.Subject.ExtraNames { + if v5.Leaf.Subject.ExtraNames[i11].Type != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.ExtraNames[i11].Type = make([]int, len(v5.Leaf.Subject.ExtraNames[i11].Type)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Subject.ExtraNames[i11].Type, v5.Leaf.Subject.ExtraNames[i11].Type) + } + } + } + if v5.Leaf.Extensions != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Extensions = make([]pkix.Extension, len(v5.Leaf.Extensions)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Extensions, v5.Leaf.Extensions) + for i10 := range v5.Leaf.Extensions { + if v5.Leaf.Extensions[i10].Id != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Extensions[i10].Id = make([]int, len(v5.Leaf.Extensions[i10].Id)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Extensions[i10].Id, v5.Leaf.Extensions[i10].Id) + } + if v5.Leaf.Extensions[i10].Value != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Extensions[i10].Value = make([]byte, len(v5.Leaf.Extensions[i10].Value)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.Extensions[i10].Value, v5.Leaf.Extensions[i10].Value) + } + } + } + if v5.Leaf.ExtraExtensions != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExtraExtensions = make([]pkix.Extension, len(v5.Leaf.ExtraExtensions)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExtraExtensions, v5.Leaf.ExtraExtensions) + for i10 := range v5.Leaf.ExtraExtensions { + if v5.Leaf.ExtraExtensions[i10].Id != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExtraExtensions[i10].Id = make([]int, len(v5.Leaf.ExtraExtensions[i10].Id)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExtraExtensions[i10].Id, v5.Leaf.ExtraExtensions[i10].Id) + } + if v5.Leaf.ExtraExtensions[i10].Value != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExtraExtensions[i10].Value = make([]byte, len(v5.Leaf.ExtraExtensions[i10].Value)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExtraExtensions[i10].Value, v5.Leaf.ExtraExtensions[i10].Value) + } + } + } + if v5.Leaf.UnhandledCriticalExtensions != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.UnhandledCriticalExtensions = make([]asn1.ObjectIdentifier, len(v5.Leaf.UnhandledCriticalExtensions)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.UnhandledCriticalExtensions, v5.Leaf.UnhandledCriticalExtensions) + for i10 := range v5.Leaf.UnhandledCriticalExtensions { + if v5.Leaf.UnhandledCriticalExtensions[i10] != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.UnhandledCriticalExtensions[i10] = make([]int, len(v5.Leaf.UnhandledCriticalExtensions[i10])) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.UnhandledCriticalExtensions[i10], v5.Leaf.UnhandledCriticalExtensions[i10]) + } + } + } + if v5.Leaf.ExtKeyUsage != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExtKeyUsage = make([]x509.ExtKeyUsage, len(v5.Leaf.ExtKeyUsage)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExtKeyUsage, v5.Leaf.ExtKeyUsage) + } + if v5.Leaf.UnknownExtKeyUsage != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.UnknownExtKeyUsage = make([]asn1.ObjectIdentifier, len(v5.Leaf.UnknownExtKeyUsage)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.UnknownExtKeyUsage, v5.Leaf.UnknownExtKeyUsage) + for i10 := range v5.Leaf.UnknownExtKeyUsage { + if v5.Leaf.UnknownExtKeyUsage[i10] != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.UnknownExtKeyUsage[i10] = make([]int, len(v5.Leaf.UnknownExtKeyUsage[i10])) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.UnknownExtKeyUsage[i10], v5.Leaf.UnknownExtKeyUsage[i10]) + } + } + } + if v5.Leaf.SubjectKeyId != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.SubjectKeyId = make([]byte, len(v5.Leaf.SubjectKeyId)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.SubjectKeyId, v5.Leaf.SubjectKeyId) + } + if v5.Leaf.AuthorityKeyId != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.AuthorityKeyId = make([]byte, len(v5.Leaf.AuthorityKeyId)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.AuthorityKeyId, v5.Leaf.AuthorityKeyId) + } + if v5.Leaf.OCSPServer != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.OCSPServer = make([]string, len(v5.Leaf.OCSPServer)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.OCSPServer, v5.Leaf.OCSPServer) + } + if v5.Leaf.IssuingCertificateURL != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.IssuingCertificateURL = make([]string, len(v5.Leaf.IssuingCertificateURL)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.IssuingCertificateURL, v5.Leaf.IssuingCertificateURL) + } + if v5.Leaf.DNSNames != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.DNSNames = make([]string, len(v5.Leaf.DNSNames)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.DNSNames, v5.Leaf.DNSNames) + } + if v5.Leaf.EmailAddresses != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.EmailAddresses = make([]string, len(v5.Leaf.EmailAddresses)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.EmailAddresses, v5.Leaf.EmailAddresses) + } + if v5.Leaf.IPAddresses != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.IPAddresses = make([]net.IP, len(v5.Leaf.IPAddresses)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.IPAddresses, v5.Leaf.IPAddresses) + for i10 := range v5.Leaf.IPAddresses { + if v5.Leaf.IPAddresses[i10] != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.IPAddresses[i10] = make([]byte, len(v5.Leaf.IPAddresses[i10])) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.IPAddresses[i10], v5.Leaf.IPAddresses[i10]) + } + } + } + if v5.Leaf.URIs != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.URIs = make([]*url.URL, len(v5.Leaf.URIs)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.URIs, v5.Leaf.URIs) + for i10 := range v5.Leaf.URIs { + if v5.Leaf.URIs[i10] != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.URIs[i10] = new(url.URL) + *cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.URIs[i10] = *v5.Leaf.URIs[i10] + if v5.Leaf.URIs[i10].User != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.URIs[i10].User = new(url.Userinfo) + *cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.URIs[i10].User = *v5.Leaf.URIs[i10].User + } + } + } + } + if v5.Leaf.PermittedDNSDomains != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedDNSDomains = make([]string, len(v5.Leaf.PermittedDNSDomains)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedDNSDomains, v5.Leaf.PermittedDNSDomains) + } + if v5.Leaf.ExcludedDNSDomains != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedDNSDomains = make([]string, len(v5.Leaf.ExcludedDNSDomains)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedDNSDomains, v5.Leaf.ExcludedDNSDomains) + } + if v5.Leaf.PermittedIPRanges != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedIPRanges = make([]*net.IPNet, len(v5.Leaf.PermittedIPRanges)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedIPRanges, v5.Leaf.PermittedIPRanges) + for i10 := range v5.Leaf.PermittedIPRanges { + if v5.Leaf.PermittedIPRanges[i10] != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedIPRanges[i10] = new(net.IPNet) + *cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedIPRanges[i10] = *v5.Leaf.PermittedIPRanges[i10] + if v5.Leaf.PermittedIPRanges[i10].IP != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedIPRanges[i10].IP = make([]byte, len(v5.Leaf.PermittedIPRanges[i10].IP)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedIPRanges[i10].IP, v5.Leaf.PermittedIPRanges[i10].IP) + } + if v5.Leaf.PermittedIPRanges[i10].Mask != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedIPRanges[i10].Mask = make([]byte, len(v5.Leaf.PermittedIPRanges[i10].Mask)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedIPRanges[i10].Mask, v5.Leaf.PermittedIPRanges[i10].Mask) + } + } + } + } + if v5.Leaf.ExcludedIPRanges != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedIPRanges = make([]*net.IPNet, len(v5.Leaf.ExcludedIPRanges)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedIPRanges, v5.Leaf.ExcludedIPRanges) + for i10 := range v5.Leaf.ExcludedIPRanges { + if v5.Leaf.ExcludedIPRanges[i10] != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedIPRanges[i10] = new(net.IPNet) + *cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedIPRanges[i10] = *v5.Leaf.ExcludedIPRanges[i10] + if v5.Leaf.ExcludedIPRanges[i10].IP != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedIPRanges[i10].IP = make([]byte, len(v5.Leaf.ExcludedIPRanges[i10].IP)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedIPRanges[i10].IP, v5.Leaf.ExcludedIPRanges[i10].IP) + } + if v5.Leaf.ExcludedIPRanges[i10].Mask != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedIPRanges[i10].Mask = make([]byte, len(v5.Leaf.ExcludedIPRanges[i10].Mask)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedIPRanges[i10].Mask, v5.Leaf.ExcludedIPRanges[i10].Mask) + } + } + } + } + if v5.Leaf.PermittedEmailAddresses != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedEmailAddresses = make([]string, len(v5.Leaf.PermittedEmailAddresses)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedEmailAddresses, v5.Leaf.PermittedEmailAddresses) + } + if v5.Leaf.ExcludedEmailAddresses != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedEmailAddresses = make([]string, len(v5.Leaf.ExcludedEmailAddresses)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedEmailAddresses, v5.Leaf.ExcludedEmailAddresses) + } + if v5.Leaf.PermittedURIDomains != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedURIDomains = make([]string, len(v5.Leaf.PermittedURIDomains)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PermittedURIDomains, v5.Leaf.PermittedURIDomains) + } + if v5.Leaf.ExcludedURIDomains != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedURIDomains = make([]string, len(v5.Leaf.ExcludedURIDomains)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.ExcludedURIDomains, v5.Leaf.ExcludedURIDomains) + } + if v5.Leaf.CRLDistributionPoints != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.CRLDistributionPoints = make([]string, len(v5.Leaf.CRLDistributionPoints)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.CRLDistributionPoints, v5.Leaf.CRLDistributionPoints) + } + if v5.Leaf.PolicyIdentifiers != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PolicyIdentifiers = make([]asn1.ObjectIdentifier, len(v5.Leaf.PolicyIdentifiers)) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PolicyIdentifiers, v5.Leaf.PolicyIdentifiers) + for i10 := range v5.Leaf.PolicyIdentifiers { + if v5.Leaf.PolicyIdentifiers[i10] != nil { + cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PolicyIdentifiers[i10] = make([]int, len(v5.Leaf.PolicyIdentifiers[i10])) + copy(cp_Cloud_TLSConfig_NameToCertificate_v5.Leaf.PolicyIdentifiers[i10], v5.Leaf.PolicyIdentifiers[i10]) + } + } + } + } + } + cp.Cloud.TLSConfig.NameToCertificate[k5] = cp_Cloud_TLSConfig_NameToCertificate_v5 + } + } + if o.Cloud.TLSConfig.RootCAs != nil { + cp.Cloud.TLSConfig.RootCAs = new(x509.CertPool) + *cp.Cloud.TLSConfig.RootCAs = *o.Cloud.TLSConfig.RootCAs + } + if o.Cloud.TLSConfig.NextProtos != nil { + cp.Cloud.TLSConfig.NextProtos = make([]string, len(o.Cloud.TLSConfig.NextProtos)) + copy(cp.Cloud.TLSConfig.NextProtos, o.Cloud.TLSConfig.NextProtos) + } + if o.Cloud.TLSConfig.ClientCAs != nil { + cp.Cloud.TLSConfig.ClientCAs = new(x509.CertPool) + *cp.Cloud.TLSConfig.ClientCAs = *o.Cloud.TLSConfig.ClientCAs + } + if o.Cloud.TLSConfig.CipherSuites != nil { + cp.Cloud.TLSConfig.CipherSuites = make([]uint16, len(o.Cloud.TLSConfig.CipherSuites)) + copy(cp.Cloud.TLSConfig.CipherSuites, o.Cloud.TLSConfig.CipherSuites) + } + if o.Cloud.TLSConfig.CurvePreferences != nil { + cp.Cloud.TLSConfig.CurvePreferences = make([]tls.CurveID, len(o.Cloud.TLSConfig.CurvePreferences)) + copy(cp.Cloud.TLSConfig.CurvePreferences, o.Cloud.TLSConfig.CurvePreferences) + } + } + if o.DNSServiceTTL != nil { + cp.DNSServiceTTL = make(map[string]time.Duration, len(o.DNSServiceTTL)) + for k2, v2 := range o.DNSServiceTTL { + cp.DNSServiceTTL[k2] = v2 + } + } + if o.DNSRecursors != nil { + cp.DNSRecursors = make([]string, len(o.DNSRecursors)) + copy(cp.DNSRecursors, o.DNSRecursors) + } + if o.HTTPBlockEndpoints != nil { + cp.HTTPBlockEndpoints = make([]string, len(o.HTTPBlockEndpoints)) + copy(cp.HTTPBlockEndpoints, o.HTTPBlockEndpoints) + } + if o.AllowWriteHTTPFrom != nil { + cp.AllowWriteHTTPFrom = make([]*net.IPNet, len(o.AllowWriteHTTPFrom)) + copy(cp.AllowWriteHTTPFrom, o.AllowWriteHTTPFrom) + for i2 := range o.AllowWriteHTTPFrom { + if o.AllowWriteHTTPFrom[i2] != nil { + cp.AllowWriteHTTPFrom[i2] = new(net.IPNet) + *cp.AllowWriteHTTPFrom[i2] = *o.AllowWriteHTTPFrom[i2] + if o.AllowWriteHTTPFrom[i2].IP != nil { + cp.AllowWriteHTTPFrom[i2].IP = make([]byte, len(o.AllowWriteHTTPFrom[i2].IP)) + copy(cp.AllowWriteHTTPFrom[i2].IP, o.AllowWriteHTTPFrom[i2].IP) + } + if o.AllowWriteHTTPFrom[i2].Mask != nil { + cp.AllowWriteHTTPFrom[i2].Mask = make([]byte, len(o.AllowWriteHTTPFrom[i2].Mask)) + copy(cp.AllowWriteHTTPFrom[i2].Mask, o.AllowWriteHTTPFrom[i2].Mask) + } + } + } + } + if o.HTTPResponseHeaders != nil { + cp.HTTPResponseHeaders = make(map[string]string, len(o.HTTPResponseHeaders)) + for k2, v2 := range o.HTTPResponseHeaders { + cp.HTTPResponseHeaders[k2] = v2 + } + } + if o.Telemetry.DogstatsdTags != nil { + cp.Telemetry.DogstatsdTags = make([]string, len(o.Telemetry.DogstatsdTags)) + copy(cp.Telemetry.DogstatsdTags, o.Telemetry.DogstatsdTags) + } + if o.Telemetry.AllowedPrefixes != nil { + cp.Telemetry.AllowedPrefixes = make([]string, len(o.Telemetry.AllowedPrefixes)) + copy(cp.Telemetry.AllowedPrefixes, o.Telemetry.AllowedPrefixes) + } + if o.Telemetry.BlockedPrefixes != nil { + cp.Telemetry.BlockedPrefixes = make([]string, len(o.Telemetry.BlockedPrefixes)) + copy(cp.Telemetry.BlockedPrefixes, o.Telemetry.BlockedPrefixes) + } + if o.Telemetry.PrometheusOpts.GaugeDefinitions != nil { + cp.Telemetry.PrometheusOpts.GaugeDefinitions = make([]prometheus.GaugeDefinition, len(o.Telemetry.PrometheusOpts.GaugeDefinitions)) + copy(cp.Telemetry.PrometheusOpts.GaugeDefinitions, o.Telemetry.PrometheusOpts.GaugeDefinitions) + for i4 := range o.Telemetry.PrometheusOpts.GaugeDefinitions { + if o.Telemetry.PrometheusOpts.GaugeDefinitions[i4].Name != nil { + cp.Telemetry.PrometheusOpts.GaugeDefinitions[i4].Name = make([]string, len(o.Telemetry.PrometheusOpts.GaugeDefinitions[i4].Name)) + copy(cp.Telemetry.PrometheusOpts.GaugeDefinitions[i4].Name, o.Telemetry.PrometheusOpts.GaugeDefinitions[i4].Name) + } + if o.Telemetry.PrometheusOpts.GaugeDefinitions[i4].ConstLabels != nil { + cp.Telemetry.PrometheusOpts.GaugeDefinitions[i4].ConstLabels = make([]metrics.Label, len(o.Telemetry.PrometheusOpts.GaugeDefinitions[i4].ConstLabels)) + copy(cp.Telemetry.PrometheusOpts.GaugeDefinitions[i4].ConstLabels, o.Telemetry.PrometheusOpts.GaugeDefinitions[i4].ConstLabels) + } + } + } + if o.Telemetry.PrometheusOpts.SummaryDefinitions != nil { + cp.Telemetry.PrometheusOpts.SummaryDefinitions = make([]prometheus.SummaryDefinition, len(o.Telemetry.PrometheusOpts.SummaryDefinitions)) + copy(cp.Telemetry.PrometheusOpts.SummaryDefinitions, o.Telemetry.PrometheusOpts.SummaryDefinitions) + for i4 := range o.Telemetry.PrometheusOpts.SummaryDefinitions { + if o.Telemetry.PrometheusOpts.SummaryDefinitions[i4].Name != nil { + cp.Telemetry.PrometheusOpts.SummaryDefinitions[i4].Name = make([]string, len(o.Telemetry.PrometheusOpts.SummaryDefinitions[i4].Name)) + copy(cp.Telemetry.PrometheusOpts.SummaryDefinitions[i4].Name, o.Telemetry.PrometheusOpts.SummaryDefinitions[i4].Name) + } + if o.Telemetry.PrometheusOpts.SummaryDefinitions[i4].ConstLabels != nil { + cp.Telemetry.PrometheusOpts.SummaryDefinitions[i4].ConstLabels = make([]metrics.Label, len(o.Telemetry.PrometheusOpts.SummaryDefinitions[i4].ConstLabels)) + copy(cp.Telemetry.PrometheusOpts.SummaryDefinitions[i4].ConstLabels, o.Telemetry.PrometheusOpts.SummaryDefinitions[i4].ConstLabels) + } + } + } + if o.Telemetry.PrometheusOpts.CounterDefinitions != nil { + cp.Telemetry.PrometheusOpts.CounterDefinitions = make([]prometheus.CounterDefinition, len(o.Telemetry.PrometheusOpts.CounterDefinitions)) + copy(cp.Telemetry.PrometheusOpts.CounterDefinitions, o.Telemetry.PrometheusOpts.CounterDefinitions) + for i4 := range o.Telemetry.PrometheusOpts.CounterDefinitions { + if o.Telemetry.PrometheusOpts.CounterDefinitions[i4].Name != nil { + cp.Telemetry.PrometheusOpts.CounterDefinitions[i4].Name = make([]string, len(o.Telemetry.PrometheusOpts.CounterDefinitions[i4].Name)) + copy(cp.Telemetry.PrometheusOpts.CounterDefinitions[i4].Name, o.Telemetry.PrometheusOpts.CounterDefinitions[i4].Name) + } + if o.Telemetry.PrometheusOpts.CounterDefinitions[i4].ConstLabels != nil { + cp.Telemetry.PrometheusOpts.CounterDefinitions[i4].ConstLabels = make([]metrics.Label, len(o.Telemetry.PrometheusOpts.CounterDefinitions[i4].ConstLabels)) + copy(cp.Telemetry.PrometheusOpts.CounterDefinitions[i4].ConstLabels, o.Telemetry.PrometheusOpts.CounterDefinitions[i4].ConstLabels) + } + } + } + if o.AdvertiseAddrLAN != nil { + cp.AdvertiseAddrLAN = new(net.IPAddr) + *cp.AdvertiseAddrLAN = *o.AdvertiseAddrLAN + if o.AdvertiseAddrLAN.IP != nil { + cp.AdvertiseAddrLAN.IP = make([]byte, len(o.AdvertiseAddrLAN.IP)) + copy(cp.AdvertiseAddrLAN.IP, o.AdvertiseAddrLAN.IP) + } + } + if o.AdvertiseAddrWAN != nil { + cp.AdvertiseAddrWAN = new(net.IPAddr) + *cp.AdvertiseAddrWAN = *o.AdvertiseAddrWAN + if o.AdvertiseAddrWAN.IP != nil { + cp.AdvertiseAddrWAN.IP = make([]byte, len(o.AdvertiseAddrWAN.IP)) + copy(cp.AdvertiseAddrWAN.IP, o.AdvertiseAddrWAN.IP) + } + } + if o.BindAddr != nil { + cp.BindAddr = new(net.IPAddr) + *cp.BindAddr = *o.BindAddr + if o.BindAddr.IP != nil { + cp.BindAddr.IP = make([]byte, len(o.BindAddr.IP)) + copy(cp.BindAddr.IP, o.BindAddr.IP) + } + } + if o.Checks != nil { + cp.Checks = make([]*structs.CheckDefinition, len(o.Checks)) + copy(cp.Checks, o.Checks) + for i2 := range o.Checks { + if o.Checks[i2] != nil { + cp.Checks[i2] = new(structs.CheckDefinition) + *cp.Checks[i2] = *o.Checks[i2] + if o.Checks[i2].ScriptArgs != nil { + cp.Checks[i2].ScriptArgs = make([]string, len(o.Checks[i2].ScriptArgs)) + copy(cp.Checks[i2].ScriptArgs, o.Checks[i2].ScriptArgs) + } + if o.Checks[i2].Header != nil { + cp.Checks[i2].Header = make(map[string][]string, len(o.Checks[i2].Header)) + for k5, v5 := range o.Checks[i2].Header { + var cp_Checks_i2_Header_v5 []string + if v5 != nil { + cp_Checks_i2_Header_v5 = make([]string, len(v5)) + copy(cp_Checks_i2_Header_v5, v5) + } + cp.Checks[i2].Header[k5] = cp_Checks_i2_Header_v5 + } + } + } + } + } + if o.ClientAddrs != nil { + cp.ClientAddrs = make([]*net.IPAddr, len(o.ClientAddrs)) + copy(cp.ClientAddrs, o.ClientAddrs) + for i2 := range o.ClientAddrs { + if o.ClientAddrs[i2] != nil { + cp.ClientAddrs[i2] = new(net.IPAddr) + *cp.ClientAddrs[i2] = *o.ClientAddrs[i2] + if o.ClientAddrs[i2].IP != nil { + cp.ClientAddrs[i2].IP = make([]byte, len(o.ClientAddrs[i2].IP)) + copy(cp.ClientAddrs[i2].IP, o.ClientAddrs[i2].IP) + } + } + } + } + if o.ConfigEntryBootstrap != nil { + cp.ConfigEntryBootstrap = make([]structs.ConfigEntry, len(o.ConfigEntryBootstrap)) + copy(cp.ConfigEntryBootstrap, o.ConfigEntryBootstrap) + } + if o.AutoEncryptDNSSAN != nil { + cp.AutoEncryptDNSSAN = make([]string, len(o.AutoEncryptDNSSAN)) + copy(cp.AutoEncryptDNSSAN, o.AutoEncryptDNSSAN) + } + if o.AutoEncryptIPSAN != nil { + cp.AutoEncryptIPSAN = make([]net.IP, len(o.AutoEncryptIPSAN)) + copy(cp.AutoEncryptIPSAN, o.AutoEncryptIPSAN) + for i2 := range o.AutoEncryptIPSAN { + if o.AutoEncryptIPSAN[i2] != nil { + cp.AutoEncryptIPSAN[i2] = make([]byte, len(o.AutoEncryptIPSAN[i2])) + copy(cp.AutoEncryptIPSAN[i2], o.AutoEncryptIPSAN[i2]) + } + } + } + if o.AutoConfig.ServerAddresses != nil { + cp.AutoConfig.ServerAddresses = make([]string, len(o.AutoConfig.ServerAddresses)) + copy(cp.AutoConfig.ServerAddresses, o.AutoConfig.ServerAddresses) + } + if o.AutoConfig.DNSSANs != nil { + cp.AutoConfig.DNSSANs = make([]string, len(o.AutoConfig.DNSSANs)) + copy(cp.AutoConfig.DNSSANs, o.AutoConfig.DNSSANs) + } + if o.AutoConfig.IPSANs != nil { + cp.AutoConfig.IPSANs = make([]net.IP, len(o.AutoConfig.IPSANs)) + copy(cp.AutoConfig.IPSANs, o.AutoConfig.IPSANs) + for i3 := range o.AutoConfig.IPSANs { + if o.AutoConfig.IPSANs[i3] != nil { + cp.AutoConfig.IPSANs[i3] = make([]byte, len(o.AutoConfig.IPSANs[i3])) + copy(cp.AutoConfig.IPSANs[i3], o.AutoConfig.IPSANs[i3]) + } + } + } + if o.AutoConfig.Authorizer.AuthMethod.Config != nil { + cp.AutoConfig.Authorizer.AuthMethod.Config = make(map[string]interface{}, len(o.AutoConfig.Authorizer.AuthMethod.Config)) + for k5, v5 := range o.AutoConfig.Authorizer.AuthMethod.Config { + cp.AutoConfig.Authorizer.AuthMethod.Config[k5] = v5 + } + } + if o.AutoConfig.Authorizer.ClaimAssertions != nil { + cp.AutoConfig.Authorizer.ClaimAssertions = make([]string, len(o.AutoConfig.Authorizer.ClaimAssertions)) + copy(cp.AutoConfig.Authorizer.ClaimAssertions, o.AutoConfig.Authorizer.ClaimAssertions) + } + if o.ConnectCAConfig != nil { + cp.ConnectCAConfig = make(map[string]interface{}, len(o.ConnectCAConfig)) + for k2, v2 := range o.ConnectCAConfig { + cp.ConnectCAConfig[k2] = v2 + } + } + if o.DNSAddrs != nil { + cp.DNSAddrs = make([]net.Addr, len(o.DNSAddrs)) + copy(cp.DNSAddrs, o.DNSAddrs) + } + if o.GRPCAddrs != nil { + cp.GRPCAddrs = make([]net.Addr, len(o.GRPCAddrs)) + copy(cp.GRPCAddrs, o.GRPCAddrs) + } + if o.GRPCTLSAddrs != nil { + cp.GRPCTLSAddrs = make([]net.Addr, len(o.GRPCTLSAddrs)) + copy(cp.GRPCTLSAddrs, o.GRPCTLSAddrs) + } + if o.HTTPAddrs != nil { + cp.HTTPAddrs = make([]net.Addr, len(o.HTTPAddrs)) + copy(cp.HTTPAddrs, o.HTTPAddrs) + } + if o.HTTPSAddrs != nil { + cp.HTTPSAddrs = make([]net.Addr, len(o.HTTPSAddrs)) + copy(cp.HTTPSAddrs, o.HTTPSAddrs) + } + if o.NodeMeta != nil { + cp.NodeMeta = make(map[string]string, len(o.NodeMeta)) + for k2, v2 := range o.NodeMeta { + cp.NodeMeta[k2] = v2 + } + } + if o.PrimaryGateways != nil { + cp.PrimaryGateways = make([]string, len(o.PrimaryGateways)) + copy(cp.PrimaryGateways, o.PrimaryGateways) + } + if o.RPCAdvertiseAddr != nil { + cp.RPCAdvertiseAddr = new(net.TCPAddr) + *cp.RPCAdvertiseAddr = *o.RPCAdvertiseAddr + if o.RPCAdvertiseAddr.IP != nil { + cp.RPCAdvertiseAddr.IP = make([]byte, len(o.RPCAdvertiseAddr.IP)) + copy(cp.RPCAdvertiseAddr.IP, o.RPCAdvertiseAddr.IP) + } + } + if o.RPCBindAddr != nil { + cp.RPCBindAddr = new(net.TCPAddr) + *cp.RPCBindAddr = *o.RPCBindAddr + if o.RPCBindAddr.IP != nil { + cp.RPCBindAddr.IP = make([]byte, len(o.RPCBindAddr.IP)) + copy(cp.RPCBindAddr.IP, o.RPCBindAddr.IP) + } + } + if o.RetryJoinLAN != nil { + cp.RetryJoinLAN = make([]string, len(o.RetryJoinLAN)) + copy(cp.RetryJoinLAN, o.RetryJoinLAN) + } + if o.RetryJoinWAN != nil { + cp.RetryJoinWAN = make([]string, len(o.RetryJoinWAN)) + copy(cp.RetryJoinWAN, o.RetryJoinWAN) + } + if o.Segments != nil { + cp.Segments = make([]structs.NetworkSegment, len(o.Segments)) + copy(cp.Segments, o.Segments) + for i2 := range o.Segments { + if o.Segments[i2].Bind != nil { + cp.Segments[i2].Bind = new(net.TCPAddr) + *cp.Segments[i2].Bind = *o.Segments[i2].Bind + if o.Segments[i2].Bind.IP != nil { + cp.Segments[i2].Bind.IP = make([]byte, len(o.Segments[i2].Bind.IP)) + copy(cp.Segments[i2].Bind.IP, o.Segments[i2].Bind.IP) + } + } + if o.Segments[i2].Advertise != nil { + cp.Segments[i2].Advertise = new(net.TCPAddr) + *cp.Segments[i2].Advertise = *o.Segments[i2].Advertise + if o.Segments[i2].Advertise.IP != nil { + cp.Segments[i2].Advertise.IP = make([]byte, len(o.Segments[i2].Advertise.IP)) + copy(cp.Segments[i2].Advertise.IP, o.Segments[i2].Advertise.IP) + } + } + } + } + if o.SerfAdvertiseAddrLAN != nil { + cp.SerfAdvertiseAddrLAN = new(net.TCPAddr) + *cp.SerfAdvertiseAddrLAN = *o.SerfAdvertiseAddrLAN + if o.SerfAdvertiseAddrLAN.IP != nil { + cp.SerfAdvertiseAddrLAN.IP = make([]byte, len(o.SerfAdvertiseAddrLAN.IP)) + copy(cp.SerfAdvertiseAddrLAN.IP, o.SerfAdvertiseAddrLAN.IP) + } + } + if o.SerfAdvertiseAddrWAN != nil { + cp.SerfAdvertiseAddrWAN = new(net.TCPAddr) + *cp.SerfAdvertiseAddrWAN = *o.SerfAdvertiseAddrWAN + if o.SerfAdvertiseAddrWAN.IP != nil { + cp.SerfAdvertiseAddrWAN.IP = make([]byte, len(o.SerfAdvertiseAddrWAN.IP)) + copy(cp.SerfAdvertiseAddrWAN.IP, o.SerfAdvertiseAddrWAN.IP) + } + } + if o.SerfAllowedCIDRsLAN != nil { + cp.SerfAllowedCIDRsLAN = make([]net.IPNet, len(o.SerfAllowedCIDRsLAN)) + copy(cp.SerfAllowedCIDRsLAN, o.SerfAllowedCIDRsLAN) + for i2 := range o.SerfAllowedCIDRsLAN { + if o.SerfAllowedCIDRsLAN[i2].IP != nil { + cp.SerfAllowedCIDRsLAN[i2].IP = make([]byte, len(o.SerfAllowedCIDRsLAN[i2].IP)) + copy(cp.SerfAllowedCIDRsLAN[i2].IP, o.SerfAllowedCIDRsLAN[i2].IP) + } + if o.SerfAllowedCIDRsLAN[i2].Mask != nil { + cp.SerfAllowedCIDRsLAN[i2].Mask = make([]byte, len(o.SerfAllowedCIDRsLAN[i2].Mask)) + copy(cp.SerfAllowedCIDRsLAN[i2].Mask, o.SerfAllowedCIDRsLAN[i2].Mask) + } + } + } + if o.SerfAllowedCIDRsWAN != nil { + cp.SerfAllowedCIDRsWAN = make([]net.IPNet, len(o.SerfAllowedCIDRsWAN)) + copy(cp.SerfAllowedCIDRsWAN, o.SerfAllowedCIDRsWAN) + for i2 := range o.SerfAllowedCIDRsWAN { + if o.SerfAllowedCIDRsWAN[i2].IP != nil { + cp.SerfAllowedCIDRsWAN[i2].IP = make([]byte, len(o.SerfAllowedCIDRsWAN[i2].IP)) + copy(cp.SerfAllowedCIDRsWAN[i2].IP, o.SerfAllowedCIDRsWAN[i2].IP) + } + if o.SerfAllowedCIDRsWAN[i2].Mask != nil { + cp.SerfAllowedCIDRsWAN[i2].Mask = make([]byte, len(o.SerfAllowedCIDRsWAN[i2].Mask)) + copy(cp.SerfAllowedCIDRsWAN[i2].Mask, o.SerfAllowedCIDRsWAN[i2].Mask) + } + } + } + if o.SerfBindAddrLAN != nil { + cp.SerfBindAddrLAN = new(net.TCPAddr) + *cp.SerfBindAddrLAN = *o.SerfBindAddrLAN + if o.SerfBindAddrLAN.IP != nil { + cp.SerfBindAddrLAN.IP = make([]byte, len(o.SerfBindAddrLAN.IP)) + copy(cp.SerfBindAddrLAN.IP, o.SerfBindAddrLAN.IP) + } + } + if o.SerfBindAddrWAN != nil { + cp.SerfBindAddrWAN = new(net.TCPAddr) + *cp.SerfBindAddrWAN = *o.SerfBindAddrWAN + if o.SerfBindAddrWAN.IP != nil { + cp.SerfBindAddrWAN.IP = make([]byte, len(o.SerfBindAddrWAN.IP)) + copy(cp.SerfBindAddrWAN.IP, o.SerfBindAddrWAN.IP) + } + } + if o.Services != nil { + cp.Services = make([]*structs.ServiceDefinition, len(o.Services)) + copy(cp.Services, o.Services) + for i2 := range o.Services { + if o.Services[i2] != nil { + cp.Services[i2] = o.Services[i2].DeepCopy() + } + } + } + if o.TLS.InternalRPC.CipherSuites != nil { + cp.TLS.InternalRPC.CipherSuites = make([]types.TLSCipherSuite, len(o.TLS.InternalRPC.CipherSuites)) + copy(cp.TLS.InternalRPC.CipherSuites, o.TLS.InternalRPC.CipherSuites) + } + if o.TLS.GRPC.CipherSuites != nil { + cp.TLS.GRPC.CipherSuites = make([]types.TLSCipherSuite, len(o.TLS.GRPC.CipherSuites)) + copy(cp.TLS.GRPC.CipherSuites, o.TLS.GRPC.CipherSuites) + } + if o.TLS.HTTPS.CipherSuites != nil { + cp.TLS.HTTPS.CipherSuites = make([]types.TLSCipherSuite, len(o.TLS.HTTPS.CipherSuites)) + copy(cp.TLS.HTTPS.CipherSuites, o.TLS.HTTPS.CipherSuites) + } + if o.TaggedAddresses != nil { + cp.TaggedAddresses = make(map[string]string, len(o.TaggedAddresses)) + for k2, v2 := range o.TaggedAddresses { + cp.TaggedAddresses[k2] = v2 + } + } + if o.UIConfig.MetricsProviderFiles != nil { + cp.UIConfig.MetricsProviderFiles = make([]string, len(o.UIConfig.MetricsProviderFiles)) + copy(cp.UIConfig.MetricsProviderFiles, o.UIConfig.MetricsProviderFiles) + } + if o.UIConfig.MetricsProxy.AddHeaders != nil { + cp.UIConfig.MetricsProxy.AddHeaders = make([]UIMetricsProxyAddHeader, len(o.UIConfig.MetricsProxy.AddHeaders)) + copy(cp.UIConfig.MetricsProxy.AddHeaders, o.UIConfig.MetricsProxy.AddHeaders) + } + if o.UIConfig.MetricsProxy.PathAllowlist != nil { + cp.UIConfig.MetricsProxy.PathAllowlist = make([]string, len(o.UIConfig.MetricsProxy.PathAllowlist)) + copy(cp.UIConfig.MetricsProxy.PathAllowlist, o.UIConfig.MetricsProxy.PathAllowlist) + } + if o.UIConfig.DashboardURLTemplates != nil { + cp.UIConfig.DashboardURLTemplates = make(map[string]string, len(o.UIConfig.DashboardURLTemplates)) + for k3, v3 := range o.UIConfig.DashboardURLTemplates { + cp.UIConfig.DashboardURLTemplates[k3] = v3 + } + } + if o.Watches != nil { + cp.Watches = make([]map[string]interface{}, len(o.Watches)) + copy(cp.Watches, o.Watches) + for i2 := range o.Watches { + if o.Watches[i2] != nil { + cp.Watches[i2] = make(map[string]interface{}, len(o.Watches[i2])) + for k3, v3 := range o.Watches[i2] { + cp.Watches[i2][k3] = v3 + } + } + } + } + return &cp +} diff --git a/agent/config/config.go b/agent/config/config.go index 6ed4e9616f0..8e7e7330e6c 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -224,6 +224,7 @@ type Config struct { SerfBindAddrWAN *string `mapstructure:"serf_wan" json:"serf_wan,omitempty"` ServerMode *bool `mapstructure:"server" json:"server,omitempty"` ServerName *string `mapstructure:"server_name" json:"server_name,omitempty"` + ServerRejoinAgeMax *string `mapstructure:"server_rejoin_age_max" json:"server_rejoin_age_max,omitempty"` Service *ServiceDefinition `mapstructure:"service" json:"-"` Services []ServiceDefinition `mapstructure:"services" json:"-"` SessionTTLMin *string `mapstructure:"session_ttl_min" json:"session_ttl_min,omitempty"` @@ -291,6 +292,9 @@ type Config struct { LicensePollMaxTime *string `mapstructure:"license_poll_max_time" json:"-"` LicenseUpdateBaseTime *string `mapstructure:"license_update_base_time" json:"-"` LicenseUpdateMaxTime *string `mapstructure:"license_update_max_time" json:"-"` + + // license reporting + Reporting Reporting `mapstructure:"reporting" json:"-"` } type GossipLANConfig struct { @@ -674,6 +678,7 @@ type Telemetry struct { CirconusSubmissionInterval *string `mapstructure:"circonus_submission_interval" json:"circonus_submission_interval,omitempty"` CirconusSubmissionURL *string `mapstructure:"circonus_submission_url" json:"circonus_submission_url,omitempty"` DisableHostname *bool `mapstructure:"disable_hostname" json:"disable_hostname,omitempty"` + EnableHostMetrics *bool `mapstructure:"enable_host_metrics" json:"enable_host_metrics,omitempty"` DogstatsdAddr *string `mapstructure:"dogstatsd_addr" json:"dogstatsd_addr,omitempty"` DogstatsdTags []string `mapstructure:"dogstatsd_tags" json:"dogstatsd_tags,omitempty"` RetryFailedConfiguration *bool `mapstructure:"retry_failed_connection" json:"retry_failed_connection,omitempty"` @@ -943,3 +948,11 @@ type RaftBoltDBConfigRaw struct { type RaftWALConfigRaw struct { SegmentSizeMB *int `mapstructure:"segment_size_mb" json:"segment_size_mb,omitempty"` } + +type License struct { + Enabled *bool `mapstructure:"enabled"` +} + +type Reporting struct { + License License `mapstructure:"license"` +} diff --git a/agent/config/config_oss.go b/agent/config/config_ce.go similarity index 100% rename from agent/config/config_oss.go rename to agent/config/config_ce.go diff --git a/agent/config/deep-copy.sh b/agent/config/deep-copy.sh new file mode 100644 index 00000000000..f69b95da931 --- /dev/null +++ b/agent/config/deep-copy.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +readonly PACKAGE_DIR="$(dirname "${BASH_SOURCE[0]}")" +cd $PACKAGE_DIR + +# Uses: https://github.com/globusdigital/deep-copy +deep-copy \ + -pointer-receiver \ + -o ./config.deepcopy.go \ + -type RuntimeConfig \ + ./ diff --git a/agent/config/default.go b/agent/config/default.go index bff8ddf8380..8ef20942efd 100644 --- a/agent/config/default.go +++ b/agent/config/default.go @@ -55,6 +55,7 @@ func DefaultSource() Source { segment_limit = 64 server = false + server_rejoin_age_max = "168h" syslog_facility = "LOCAL0" tls = { diff --git a/agent/config/default_oss.go b/agent/config/default_ce.go similarity index 100% rename from agent/config/default_oss.go rename to agent/config/default_ce.go diff --git a/agent/config/runtime.go b/agent/config/runtime.go index 627c1e56440..d5159ee6e59 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -1353,6 +1353,18 @@ type RuntimeConfig struct { // hcl: ports { server = int } ServerPort int + // ServerRejoinAgeMax is used to specify the duration of time a server + // is allowed to be down/offline before a startup operation is refused. + // + // For example: if a server has been offline for 5 days, and this option + // is configured to 3 days, then any subsequent startup operation will fail + // and require an operator to manually intervene. + // + // The default is: 7 days + // + // hcl: server_rejoin_age_max = "duration" + ServerRejoinAgeMax time.Duration + // Services contains the provided service definitions: // // hcl: services = [ @@ -1475,9 +1487,23 @@ type RuntimeConfig struct { // AutoReloadConfigCoalesceInterval Coalesce Interval for auto reload config AutoReloadConfigCoalesceInterval time.Duration + // LocalProxyConfigResyncInterval is not a user-configurable value and exists + // here so that tests can use a smaller value. + LocalProxyConfigResyncInterval time.Duration + + Reporting ReportingConfig + EnterpriseRuntimeConfig } +type LicenseConfig struct { + Enabled bool +} + +type ReportingConfig struct { + License LicenseConfig +} + type AutoConfig struct { Enabled bool IntroToken string @@ -1719,6 +1745,9 @@ func (c *RuntimeConfig) Sanitized() map[string]interface{} { // IsCloudEnabled returns true if a cloud.resource_id is set and the server mode is enabled func (c *RuntimeConfig) IsCloudEnabled() bool { + if c == nil { + return false + } return c.ServerMode && c.Cloud.ResourceID != "" } diff --git a/agent/config/runtime_oss.go b/agent/config/runtime_ce.go similarity index 100% rename from agent/config/runtime_oss.go rename to agent/config/runtime_ce.go diff --git a/agent/config/runtime_oss_test.go b/agent/config/runtime_ce_test.go similarity index 54% rename from agent/config/runtime_oss_test.go rename to agent/config/runtime_ce_test.go index 6274511bc03..c95e2d3add4 100644 --- a/agent/config/runtime_oss_test.go +++ b/agent/config/runtime_ce_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/hashicorp/consul/sdk/testutil" + "github.com/stretchr/testify/require" ) var testRuntimeConfigSanitizeExpectedFilename = "TestRuntimeConfig_Sanitize.golden" @@ -28,11 +29,12 @@ var enterpriseConfigKeyWarnings = []string{ enterpriseConfigKeyError{key: "acl.msp_disable_bootstrap"}.Error(), enterpriseConfigKeyError{key: "acl.tokens.managed_service_provider"}.Error(), enterpriseConfigKeyError{key: "audit"}.Error(), + enterpriseConfigKeyError{key: "reporting.license.enabled"}.Error(), } -// OSS-only equivalent of TestConfigFlagsAndEdgecases +// CE-only equivalent of TestConfigFlagsAndEdgecases // used for flags validated in ent-only code -func TestLoad_IntegrationWithFlags_OSS(t *testing.T) { +func TestLoad_IntegrationWithFlags_CE(t *testing.T) { dataDir := testutil.TempDir(t, "consul") defer os.RemoveAll(dataDir) @@ -84,3 +86,82 @@ func TestLoad_IntegrationWithFlags_OSS(t *testing.T) { } } } + +func TestLoad_ReportingConfig(t *testing.T) { + dir := testutil.TempDir(t, t.Name()) + + t.Run("load from JSON defaults to false", func(t *testing.T) { + content := `{ + "reporting": {} + }` + + opts := LoadOpts{ + FlagValues: FlagValuesTarget{Config: Config{ + DataDir: &dir, + }}, + Overrides: []Source{ + FileSource{ + Name: "reporting.json", + Format: "json", + Data: content, + }, + }, + } + patchLoadOptsShims(&opts) + result, err := Load(opts) + require.NoError(t, err) + require.Len(t, result.Warnings, 0) + require.Equal(t, false, result.RuntimeConfig.Reporting.License.Enabled) + }) + + t.Run("load from HCL defaults to false", func(t *testing.T) { + content := ` + reporting {} + ` + + opts := LoadOpts{ + FlagValues: FlagValuesTarget{Config: Config{ + DataDir: &dir, + }}, + Overrides: []Source{ + FileSource{ + Name: "reporting.hcl", + Format: "hcl", + Data: content, + }, + }, + } + patchLoadOptsShims(&opts) + result, err := Load(opts) + require.NoError(t, err) + require.Len(t, result.Warnings, 0) + require.Equal(t, false, result.RuntimeConfig.Reporting.License.Enabled) + }) + + t.Run("with value set returns warning and defaults to false", func(t *testing.T) { + content := `reporting { + license { + enabled = true + } + }` + + opts := LoadOpts{ + FlagValues: FlagValuesTarget{Config: Config{ + DataDir: &dir, + }}, + Overrides: []Source{ + FileSource{ + Name: "reporting.hcl", + Format: "hcl", + Data: content, + }, + }, + } + patchLoadOptsShims(&opts) + result, err := Load(opts) + require.NoError(t, err) + require.Len(t, result.Warnings, 1) + require.Contains(t, result.Warnings[0], "\"reporting.license.enabled\" is a Consul Enterprise configuration and will have no effect") + require.Equal(t, false, result.RuntimeConfig.Reporting.License.Enabled) + }) +} diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 2954dee6700..32863218817 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -22,13 +22,12 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/time/rate" - hcpconfig "github.com/hashicorp/consul/agent/hcp/config" - "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/cache" "github.com/hashicorp/consul/agent/checks" "github.com/hashicorp/consul/agent/consul" consulrate "github.com/hashicorp/consul/agent/consul/rate" + hcpconfig "github.com/hashicorp/consul/agent/hcp/config" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/token" "github.com/hashicorp/consul/lib" @@ -44,6 +43,7 @@ type testCase struct { desc string args []string setup func() // TODO: accept a testing.T instead of panic + cleanup func() expected func(rt *RuntimeConfig) expectedErr string expectedWarnings []string @@ -615,6 +615,7 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { rt.NodeName = "a" rt.TLS.NodeName = "a" rt.DataDir = dataDir + rt.Cloud.NodeName = "a" }, }) run(t, testCase{ @@ -626,6 +627,7 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { expected: func(rt *RuntimeConfig) { rt.NodeID = "a" rt.DataDir = dataDir + rt.Cloud.NodeID = "a" }, }) run(t, testCase{ @@ -2298,6 +2300,79 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { rt.HTTPUseCache = false }, }) + run(t, testCase{ + desc: "cloud resource id from env", + args: []string{ + `-server`, + `-data-dir=` + dataDir, + }, + setup: func() { + os.Setenv("HCP_RESOURCE_ID", "env-id") + }, + cleanup: func() { + os.Unsetenv("HCP_RESOURCE_ID") + }, + expected: func(rt *RuntimeConfig) { + rt.DataDir = dataDir + rt.Cloud = hcpconfig.CloudConfig{ + // ID is only populated from env if not populated from other sources. + ResourceID: "env-id", + NodeName: "thehostname", + NodeID: "", + } + + // server things + rt.ServerMode = true + rt.Telemetry.EnableHostMetrics = true + rt.TLS.ServerMode = true + rt.LeaveOnTerm = false + rt.SkipLeaveOnInt = true + rt.RPCConfig.EnableStreaming = true + rt.GRPCTLSPort = 8503 + rt.GRPCTLSAddrs = []net.Addr{defaultGrpcTlsAddr} + }, + }) + run(t, testCase{ + desc: "cloud resource id from file", + args: []string{ + `-server`, + `-data-dir=` + dataDir, + }, + setup: func() { + os.Setenv("HCP_RESOURCE_ID", "env-id") + }, + cleanup: func() { + os.Unsetenv("HCP_RESOURCE_ID") + }, + json: []string{`{ + "cloud": { + "resource_id": "file-id" + } + }`}, + hcl: []string{` + cloud = { + resource_id = "file-id" + } + `}, + expected: func(rt *RuntimeConfig) { + rt.DataDir = dataDir + rt.Cloud = hcpconfig.CloudConfig{ + // ID is only populated from env if not populated from other sources. + ResourceID: "file-id", + NodeName: "thehostname", + } + + // server things + rt.ServerMode = true + rt.Telemetry.EnableHostMetrics = true + rt.TLS.ServerMode = true + rt.LeaveOnTerm = false + rt.SkipLeaveOnInt = true + rt.RPCConfig.EnableStreaming = true + rt.GRPCTLSPort = 8503 + rt.GRPCTLSAddrs = []net.Addr{defaultGrpcTlsAddr} + }, + }) run(t, testCase{ desc: "sidecar_service can't have ID", args: []string{ @@ -5962,6 +6037,9 @@ func (tc testCase) run(format string, dataDir string) func(t *testing.T) { expected.ACLResolverSettings.EnterpriseMeta = *structs.NodeEnterpriseMetaInPartition(expected.PartitionOrDefault()) prototest.AssertDeepEqual(t, expected, actual, cmpopts.EquateEmpty()) + if tc.cleanup != nil { + tc.cleanup() + } } } @@ -5995,12 +6073,13 @@ func TestLoad_FullConfig(t *testing.T) { nodeEntMeta := structs.NodeEnterpriseMetaInDefaultPartition() expected := &RuntimeConfig{ // non-user configurable values - AEInterval: time.Minute, - CheckDeregisterIntervalMin: time.Minute, - CheckReapInterval: 30 * time.Second, - SegmentNameLimit: 64, - SyncCoordinateIntervalMin: 15 * time.Second, - SyncCoordinateRateTarget: 64, + AEInterval: time.Minute, + CheckDeregisterIntervalMin: time.Minute, + CheckReapInterval: 30 * time.Second, + SegmentNameLimit: 64, + SyncCoordinateIntervalMin: 15 * time.Second, + SyncCoordinateRateTarget: 64, + LocalProxyConfigResyncInterval: 30 * time.Second, Revision: "JNtPSav3", Version: "R909Hblt", @@ -6239,6 +6318,8 @@ func TestLoad_FullConfig(t *testing.T) { Hostname: "DH4bh7aC", AuthURL: "332nCdR2", ScadaAddress: "aoeusth232", + NodeID: types.NodeID("AsUIlw99"), + NodeName: "otlLxGaI", }, DNSAddrs: []net.Addr{tcpAddr("93.95.95.81:7001"), udpAddr("93.95.95.81:7001")}, DNSARecordLimit: 29907, @@ -6347,6 +6428,7 @@ func TestLoad_FullConfig(t *testing.T) { SerfPortWAN: 8302, ServerMode: true, ServerName: "Oerr9n1G", + ServerRejoinAgeMax: 604800 * time.Second, ServerPort: 3757, Services: []*structs.ServiceDefinition{ { @@ -6682,6 +6764,7 @@ func TestLoad_FullConfig(t *testing.T) { Expiration: 15 * time.Second, Name: "ftO6DySn", // notice this is the same as the metrics prefix }, + EnableHostMetrics: true, }, TLS: tlsutil.Config{ InternalRPC: tlsutil.ProtocolConfig{ @@ -7091,6 +7174,7 @@ func TestRuntimeConfig_Sanitize(t *testing.T) { }, }, }, + ServerRejoinAgeMax: 24 * 7 * time.Hour, } b, err := json.MarshalIndent(rt.Sanitized(), "", " ") diff --git a/agent/config/segment_oss.go b/agent/config/segment_ce.go similarity index 100% rename from agent/config/segment_oss.go rename to agent/config/segment_ce.go diff --git a/agent/config/segment_oss_test.go b/agent/config/segment_ce_test.go similarity index 95% rename from agent/config/segment_oss_test.go rename to agent/config/segment_ce_test.go index 52b4a0964cd..e31a9602b41 100644 --- a/agent/config/segment_oss_test.go +++ b/agent/config/segment_ce_test.go @@ -15,7 +15,7 @@ func TestSegments(t *testing.T) { tests := []testCase{ { - desc: "segment name not in OSS", + desc: "segment name not in CE", args: []string{ `-data-dir=` + dataDir, }, @@ -39,7 +39,7 @@ func TestSegments(t *testing.T) { }, }, { - desc: "segments not in OSS", + desc: "segments not in CE", args: []string{ `-data-dir=` + dataDir, }, diff --git a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden index 75d216fabdb..57217f6eaa9 100644 --- a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden +++ b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden @@ -79,8 +79,6 @@ "BootstrapExpect": 0, "BuildDate": "2019-11-20 05:00:00 +0000 UTC", "Cache": { - "CacheRefreshBackoffMin": 0, - "CacheRefreshMaxWait": "0s", "EntryFetchMaxBurst": 42, "EntryFetchRate": 0.334, "Logger": null @@ -133,8 +131,12 @@ "ClientID": "id", "ClientSecret": "hidden", "Hostname": "", + "ManagementToken": "hidden", "ResourceID": "cluster1", - "ScadaAddress": "" + "ScadaAddress": "", + "TLSConfig": null, + "NodeID": "", + "NodeName": "" }, "ConfigEntryBootstrap": [], "ConnectCAConfig": {}, @@ -233,6 +235,7 @@ "KVMaxValueSize": 1234567800000000, "LeaveDrainTime": "0s", "LeaveOnTerm": false, + "LocalProxyConfigResyncInterval": "0s", "Logging": { "EnableSyslog": false, "LogFilePath": "", @@ -290,6 +293,11 @@ "ReconnectTimeoutLAN": "0s", "ReconnectTimeoutWAN": "0s", "RejoinAfterLeave": false, + "Reporting": { + "License": { + "Enabled": false + } + }, "RequestLimitsMode": 0, "RequestLimitsReadRate": 0, "RequestLimitsWriteRate": 0, @@ -322,6 +330,7 @@ "ServerMode": false, "ServerName": "", "ServerPort": 0, + "ServerRejoinAgeMax": "168h0m0s", "Services": [ { "Address": "", @@ -453,6 +462,7 @@ "DisableHostname": false, "DogstatsdAddr": "", "DogstatsdTags": [], + "EnableHostMetrics": false, "FilterDefault": false, "MetricsPrefix": "", "PrometheusOpts": { diff --git a/agent/config/testdata/full-config.hcl b/agent/config/testdata/full-config.hcl index dced4781c91..7d43254a982 100644 --- a/agent/config/testdata/full-config.hcl +++ b/agent/config/testdata/full-config.hcl @@ -372,6 +372,11 @@ reconnect_timeout = "23739s" reconnect_timeout_wan = "26694s" recursors = [ "63.38.39.58", "92.49.18.18" ] rejoin_after_leave = true +reporting = { + license = { + enabled = false + } +} retry_interval = "8067s" retry_interval_wan = "28866s" retry_join = [ "pbsSFY7U", "l0qLtWij" ] @@ -386,6 +391,7 @@ serf_lan = "99.43.63.15" serf_wan = "67.88.33.19" server = true server_name = "Oerr9n1G" +server_rejoin_age_max = "604800s" service = { id = "dLOXpSCI" name = "o1ynPkp0" @@ -681,6 +687,7 @@ telemetry { circonus_check_tags = "prvO4uBl" circonus_submission_interval = "DolzaflP" circonus_submission_url = "gTcbS93G" + enable_host_metrics = true disable_hostname = true dogstatsd_addr = "0wSndumK" dogstatsd_tags = [ "3N81zSUB","Xtj8AnXZ" ] diff --git a/agent/config/testdata/full-config.json b/agent/config/testdata/full-config.json index 7c36981561a..52dab37bfa5 100644 --- a/agent/config/testdata/full-config.json +++ b/agent/config/testdata/full-config.json @@ -428,6 +428,11 @@ "92.49.18.18" ], "rejoin_after_leave": true, + "reporting": { + "license": { + "enabled": false + } + }, "retry_interval": "8067s", "retry_interval_wan": "28866s", "retry_join": [ @@ -448,6 +453,7 @@ "serf_wan": "67.88.33.19", "server": true, "server_name": "Oerr9n1G", + "server_rejoin_age_max": "604800s", "service": { "id": "dLOXpSCI", "name": "o1ynPkp0", @@ -802,6 +808,7 @@ "circonus_check_tags": "prvO4uBl", "circonus_submission_interval": "DolzaflP", "circonus_submission_url": "gTcbS93G", + "enable_host_metrics": true, "disable_hostname": true, "dogstatsd_addr": "0wSndumK", "dogstatsd_tags": [ diff --git a/agent/configentry/merge_service_config.go b/agent/configentry/merge_service_config.go index 458cab012fe..c11a3112ed9 100644 --- a/agent/configentry/merge_service_config.go +++ b/agent/configentry/merge_service_config.go @@ -154,6 +154,10 @@ func MergeServiceConfig(defaults *structs.ServiceConfigResponse, service *struct // remoteUpstreams contains synthetic Upstreams generated from central config (service-defaults.UpstreamConfigs). remoteUpstreams := make(map[structs.PeeredServiceName]structs.Upstream) + // If the arguments did not fully normalize tenancy stuff, take care of that now. + entMeta := ns.EnterpriseMeta + entMeta.Normalize() + if len(defaults.UpstreamIDConfigs) > 0 { // Handle legacy upstreams. This should be removed in Consul 1.16. for _, us := range defaults.UpstreamIDConfigs { @@ -183,11 +187,21 @@ func MergeServiceConfig(defaults *structs.ServiceConfigResponse, service *struct return nil, fmt.Errorf("failed to parse upstream config map for %s: %v", us.Upstream.String(), err) } + // If the defaults did not fully normalize tenancy stuff, take care of + // that now too. + psn := us.Upstream // only normalize the copy + psn.ServiceName.EnterpriseMeta.Normalize() + + // Normalize the partition field specially. + if psn.Peer != "" { + psn.ServiceName.OverridePartition(entMeta.PartitionOrDefault()) + } + remoteUpstreams[us.Upstream] = structs.Upstream{ - DestinationNamespace: us.Upstream.ServiceName.NamespaceOrDefault(), - DestinationPartition: us.Upstream.ServiceName.PartitionOrDefault(), - DestinationName: us.Upstream.ServiceName.Name, - DestinationPeer: us.Upstream.Peer, + DestinationNamespace: psn.ServiceName.NamespaceOrDefault(), + DestinationPartition: psn.ServiceName.PartitionOrDefault(), + DestinationName: psn.ServiceName.Name, + DestinationPeer: psn.Peer, Config: us.Config, MeshGateway: parsed.MeshGateway, CentrallyConfigured: true, @@ -209,6 +223,12 @@ func MergeServiceConfig(defaults *structs.ServiceConfigResponse, service *struct } uid := us.DestinationID() + + // Normalize the partition field specially. + if uid.Peer != "" { + uid.ServiceName.OverridePartition(entMeta.PartitionOrDefault()) + } + localUpstreams[uid] = struct{}{} remoteCfg, ok := remoteUpstreams[uid] if !ok { diff --git a/agent/configentry/merge_service_config_test.go b/agent/configentry/merge_service_config_test.go index fa29bbdfa99..fbe136244c6 100644 --- a/agent/configentry/merge_service_config_test.go +++ b/agent/configentry/merge_service_config_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/structs" ) @@ -270,6 +271,214 @@ func Test_MergeServiceConfig_Extensions(t *testing.T) { } } +func isEnterprise() bool { + return acl.PartitionOrDefault("") == "default" +} + +func Test_MergeServiceConfig_peeredCentralDefaultsMerging(t *testing.T) { + partitions := []string{"default"} + if isEnterprise() { + partitions = append(partitions, "part1") + } + + const peerName = "my-peer" + + newDefaults := func(partition string) *structs.ServiceConfigResponse { + // client agents + return &structs.ServiceConfigResponse{ + ProxyConfig: map[string]any{ + "protocol": "http", + }, + UpstreamConfigs: []structs.OpaqueUpstreamConfig{ + { + Upstream: structs.PeeredServiceName{ + ServiceName: structs.ServiceName{ + Name: "*", + EnterpriseMeta: acl.NewEnterpriseMetaWithPartition(partition, "*"), + }, + }, + Config: map[string]any{ + "mesh_gateway": map[string]any{ + "Mode": "local", + }, + "protocol": "http", + }, + }, + { + Upstream: structs.PeeredServiceName{ + ServiceName: structs.ServiceName{ + Name: "static-server", + EnterpriseMeta: acl.NewEnterpriseMetaWithPartition(partition, "default"), + }, + Peer: peerName, + }, + Config: map[string]any{ + "mesh_gateway": map[string]any{ + "Mode": "local", + }, + "protocol": "http", + }, + }, + }, + MeshGateway: structs.MeshGatewayConfig{ + Mode: "local", + }, + } + } + + for _, partition := range partitions { + t.Run("partition="+partition, func(t *testing.T) { + t.Run("clients", func(t *testing.T) { + defaults := newDefaults(partition) + + service := &structs.NodeService{ + Kind: "connect-proxy", + ID: "static-client-sidecar-proxy", + Service: "static-client-sidecar-proxy", + Address: "", + Port: 21000, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "static-client", + DestinationServiceID: "static-client", + LocalServiceAddress: "127.0.0.1", + LocalServicePort: 8080, + Upstreams: []structs.Upstream{ + { + DestinationType: "service", + DestinationNamespace: "default", + DestinationPartition: partition, + DestinationPeer: peerName, + DestinationName: "static-server", + LocalBindAddress: "0.0.0.0", + LocalBindPort: 5000, + }, + }, + }, + EnterpriseMeta: acl.NewEnterpriseMetaWithPartition(partition, "default"), + } + + expect := &structs.NodeService{ + Kind: "connect-proxy", + ID: "static-client-sidecar-proxy", + Service: "static-client-sidecar-proxy", + Address: "", + Port: 21000, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "static-client", + DestinationServiceID: "static-client", + LocalServiceAddress: "127.0.0.1", + LocalServicePort: 8080, + Config: map[string]any{ + "protocol": "http", + }, + Upstreams: []structs.Upstream{ + { + DestinationType: "service", + DestinationNamespace: "default", + DestinationPartition: partition, + DestinationPeer: peerName, + DestinationName: "static-server", + LocalBindAddress: "0.0.0.0", + LocalBindPort: 5000, + MeshGateway: structs.MeshGatewayConfig{ + Mode: "local", + }, + Config: map[string]any{}, + }, + }, + MeshGateway: structs.MeshGatewayConfig{ + Mode: "local", + }, + }, + EnterpriseMeta: acl.NewEnterpriseMetaWithPartition(partition, "default"), + } + + got, err := MergeServiceConfig(defaults, service) + require.NoError(t, err) + require.Equal(t, expect, got) + }) + + t.Run("dataplanes", func(t *testing.T) { + defaults := newDefaults(partition) + + service := &structs.NodeService{ + Kind: "connect-proxy", + ID: "static-client-sidecar-proxy", + Service: "static-client-sidecar-proxy", + Address: "10.61.57.9", + TaggedAddresses: map[string]structs.ServiceAddress{ + "consul-virtual": { + Address: "240.0.0.2", + Port: 20000, + }, + }, + Port: 20000, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "static-client", + DestinationServiceID: "static-client", + LocalServicePort: 8080, + Upstreams: []structs.Upstream{ + { + DestinationType: "", + DestinationNamespace: "default", + DestinationPeer: peerName, + DestinationName: "static-server", + LocalBindAddress: "0.0.0.0", + LocalBindPort: 5000, + }, + }, + }, + EnterpriseMeta: acl.NewEnterpriseMetaWithPartition(partition, "default"), + } + + expect := &structs.NodeService{ + Kind: "connect-proxy", + ID: "static-client-sidecar-proxy", + Service: "static-client-sidecar-proxy", + Address: "10.61.57.9", + TaggedAddresses: map[string]structs.ServiceAddress{ + "consul-virtual": { + Address: "240.0.0.2", + Port: 20000, + }, + }, + Port: 20000, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "static-client", + DestinationServiceID: "static-client", + LocalServicePort: 8080, + Config: map[string]any{ + "protocol": "http", + }, + Upstreams: []structs.Upstream{ + { + DestinationType: "", + DestinationNamespace: "default", + DestinationPeer: peerName, + DestinationName: "static-server", + LocalBindAddress: "0.0.0.0", + LocalBindPort: 5000, + MeshGateway: structs.MeshGatewayConfig{ + Mode: "local", // This field vanishes if the merging does not work for dataplanes. + }, + Config: map[string]any{}, + }, + }, + MeshGateway: structs.MeshGatewayConfig{ + Mode: "local", + }, + }, + EnterpriseMeta: acl.NewEnterpriseMetaWithPartition(partition, "default"), + } + + got, err := MergeServiceConfig(defaults, service) + require.NoError(t, err) + require.Equal(t, expect, got) + }) + }) + } +} + func Test_MergeServiceConfig_UpstreamOverrides(t *testing.T) { type args struct { defaults *structs.ServiceConfigResponse diff --git a/agent/configentry/resolve.go b/agent/configentry/resolve.go index 4484fa42e0f..9daad801882 100644 --- a/agent/configentry/resolve.go +++ b/agent/configentry/resolve.go @@ -33,6 +33,7 @@ func ComputeResolvedServiceConfig( // blocking query, this function will be rerun and these state store lookups will both be current. // We use the default enterprise meta to look up the global proxy defaults because they are not namespaced. + var proxyConfGlobalProtocol string proxyConf := entries.GetProxyDefaults(args.PartitionOrDefault()) if proxyConf != nil { // Apply the proxy defaults to the sidecar's proxy config @@ -59,9 +60,30 @@ func ComputeResolvedServiceConfig( if !proxyConf.MeshGateway.IsZero() { wildcardUpstreamDefaults["mesh_gateway"] = proxyConf.MeshGateway } - if protocol, ok := thisReply.ProxyConfig["protocol"]; ok { - wildcardUpstreamDefaults["protocol"] = protocol + + // We explicitly DO NOT merge the protocol from proxy-defaults into the wildcard upstream here. + // TProxy will try to use the data from the `wildcardUpstreamDefaults` as a source of truth, which is + // normally correct to inherit from proxy-defaults. However, it is NOT correct for protocol. + // + // This edge-case is different for `protocol` from other fields, since the protocol can be + // set on both the local `ServiceDefaults.UpstreamOverrides` and upstream `ServiceDefaults.Protocol`. + // This means that when proxy-defaults is set, it would always be treated as an explicit override, + // and take precedence over the protocol that is set on the discovery chain (which comes from the + // service's preference in its service-defaults), which is wrong. + // + // When the upstream is not explicitly defined, we should only get the protocol from one of these locations: + // 1. For tproxy non-peering services, it can be fetched via the discovery chain. + // The chain compiler merges the proxy-defaults protocol with the upstream's preferred service-defaults protocol. + // 2. For tproxy non-peering services with default upstream overrides, it will come from the wildcard upstream overrides. + // 3. For tproxy non-peering services with specific upstream overrides, it will come from the specific upstream override defined. + // 4. For tproxy peering services, they do not honor the proxy-defaults, since they reside in a different cluster. + // The data will come from a separate peerMeta field. + // In all of these cases, it is not necessary for the proxy-defaults to exist in the wildcard upstream. + parsed, err := structs.ParseUpstreamConfigNoDefaults(mapCopy.(map[string]interface{})) + if err != nil { + return nil, fmt.Errorf("failed to parse upstream config map for proxy-defaults: %v", err) } + proxyConfGlobalProtocol = parsed.Protocol } serviceConf := entries.GetServiceDefaults( @@ -227,6 +249,10 @@ func ComputeResolvedServiceConfig( // 2. Protocol for upstream service defined in its service-defaults (how the upstream wants to be addressed) // 3. Protocol defined for the upstream in the service-defaults.(upstream_config.defaults|upstream_config.overrides) of the downstream // (how the downstream wants to address it) + if proxyConfGlobalProtocol != "" { + resolvedCfg["protocol"] = proxyConfGlobalProtocol + } + if err := mergo.MergeWithOverwrite(&resolvedCfg, wildcardUpstreamDefaults); err != nil { return nil, fmt.Errorf("failed to merge wildcard defaults into upstream: %v", err) } diff --git a/agent/connect/ca/provider_vault.go b/agent/connect/ca/provider_vault.go index c069e5aa9b3..b61bd95d3f1 100644 --- a/agent/connect/ca/provider_vault.go +++ b/agent/connect/ca/provider_vault.go @@ -10,7 +10,6 @@ import ( "net/http" "os" "strings" - "sync" "time" "github.com/hashicorp/go-hclog" @@ -56,9 +55,7 @@ type VaultProvider struct { config *structs.VaultCAProviderConfig client *vaultapi.Client - // We modify the namespace on the fly to override default namespace for rootCertificate and intermediateCertificate. Can't guarantee - // all operations (specifically Sign) are not called re-entrantly, so we add this for safety. - clientMutex sync.Mutex + baseNamespace string stopWatcher func() @@ -516,9 +513,7 @@ func (v *VaultProvider) ActiveIntermediate() (string, error) { // because the endpoint only returns the raw PEM contents of the CA cert // and not the typical format of the secrets endpoints. func (v *VaultProvider) getCA(namespace, path string) (string, error) { - defer v.setNamespace(namespace)() - - resp, err := v.client.Logical().ReadRaw(path + "/ca/pem") + resp, err := v.client.WithNamespace(namespace).Logical().ReadRaw(path + "/ca/pem") if resp != nil { defer resp.Body.Close() } @@ -544,9 +539,7 @@ func (v *VaultProvider) getCA(namespace, path string) (string, error) { // TODO: refactor to remove duplication with getCA func (v *VaultProvider) getCAChain(namespace, path string) (string, error) { - defer v.setNamespace(namespace)() - - resp, err := v.client.Logical().ReadRaw(path + "/ca_chain") + resp, err := v.client.WithNamespace(namespace).Logical().ReadRaw(path + "/ca_chain") if resp != nil { defer resp.Body.Close() } @@ -617,6 +610,9 @@ func (v *VaultProvider) GenerateIntermediate() (string, error) { // should contain a "mapping" data field we can use to cross-reference // with the keyId returned when calling [/intermediate/generate/internal]. // +// After a new default issuer is written, this function also cleans up +// the previous default issuer along with its associated key. +// // [/intermediate/set-signed]: https://developer.hashicorp.com/vault/api-docs/secret/pki#import-ca-certificates-and-keys // [/intermediate/generate/internal]: https://developer.hashicorp.com/vault/api-docs/secret/pki#generate-intermediate-csr func (v *VaultProvider) setDefaultIntermediateIssuer(vaultResp *vaultapi.Secret, keyId string) error { @@ -654,14 +650,50 @@ func (v *VaultProvider) setDefaultIntermediateIssuer(vaultResp *vaultapi.Secret, return fmt.Errorf("could not read from /config/issuers: %w", err) } issuersConf := resp.Data + prevIssuer, ok := issuersConf["default"].(string) + if !ok { + return fmt.Errorf("unexpected type for 'default' value in Vault response from /pki/config/issuers") + } + + if prevIssuer == intermediateId { + return nil + } + // Overwrite the default issuer issuersConf["default"] = intermediateId - _, err = v.writeNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"config/issuers", issuersConf) if err != nil { return fmt.Errorf("could not write default issuer to /config/issuers: %w", err) } + // Find the key_id of the previous issuer. In Consul, issuers have 1:1 relationship with + // keys so we can delete issuer first then the key. + resp, err = v.readNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"issuer/"+prevIssuer) + if err != nil { + return fmt.Errorf("could not read issuer %q: %w", prevIssuer, err) + } + prevKeyId, ok := resp.Data["key_id"].(string) + if !ok { + return fmt.Errorf("unexpected type for 'key_id' value in Vault response") + } + + // Delete the previously known default issuer to prevent the number of unused + // issuers from increasing too much. + _, err = v.deleteNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"issuer/"+prevIssuer) + if err != nil { + v.logger.Warn("Could not delete previous issuer. Manually delete from Vault to prevent the list of issuers from growing too large.", + "prev_issuer_id", prevIssuer, + "error", err) + } + + // Keys can only be deleted if there are no more issuers referencing them. + _, err = v.deleteNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"key/"+prevKeyId) + if err != nil { + v.logger.Warn("Could not delete previous key. Manually delete from Vault to prevent the list of keys from growing too large.", + "prev_key_id", prevKeyId, + "error", err) + } + return nil } @@ -816,41 +848,27 @@ func (v *VaultProvider) PrimaryUsesIntermediate() {} // We use raw path here func (v *VaultProvider) mountNamespaced(namespace, path string, mountInfo *vaultapi.MountInput) error { - defer v.setNamespace(namespace)() - return v.client.Sys().Mount(path, mountInfo) + return v.client.WithNamespace(namespace).Sys().Mount(path, mountInfo) } func (v *VaultProvider) tuneMountNamespaced(namespace, path string, mountConfig *vaultapi.MountConfigInput) error { - defer v.setNamespace(namespace)() - return v.client.Sys().TuneMount(path, *mountConfig) + return v.client.WithNamespace(namespace).Sys().TuneMount(path, *mountConfig) } func (v *VaultProvider) unmountNamespaced(namespace, path string) error { - defer v.setNamespace(namespace)() - return v.client.Sys().Unmount(path) + return v.client.WithNamespace(namespace).Sys().Unmount(path) } func (v *VaultProvider) readNamespaced(namespace string, resource string) (*vaultapi.Secret, error) { - defer v.setNamespace(namespace)() - return v.client.Logical().Read(resource) + return v.client.WithNamespace(namespace).Logical().Read(resource) } func (v *VaultProvider) writeNamespaced(namespace string, resource string, data map[string]interface{}) (*vaultapi.Secret, error) { - defer v.setNamespace(namespace)() - return v.client.Logical().Write(resource, data) + return v.client.WithNamespace(namespace).Logical().Write(resource, data) } -func (v *VaultProvider) setNamespace(namespace string) func() { - if namespace != "" { - v.clientMutex.Lock() - v.client.SetNamespace(namespace) - return func() { - v.client.SetNamespace(v.baseNamespace) - v.clientMutex.Unlock() - } - } else { - return func() {} - } +func (v *VaultProvider) deleteNamespaced(namespace string, resource string) (*vaultapi.Secret, error) { + return v.client.WithNamespace(namespace).Logical().Delete(resource) } func ParseVaultCAConfig(raw map[string]interface{}) (*structs.VaultCAProviderConfig, error) { diff --git a/agent/connect/ca/provider_vault_auth_aws.go b/agent/connect/ca/provider_vault_auth_aws.go index 6188b2cf2e2..0bb3c7d55d2 100644 --- a/agent/connect/ca/provider_vault_auth_aws.go +++ b/agent/connect/ca/provider_vault_auth_aws.go @@ -69,6 +69,13 @@ func (g *AWSLoginDataGenerator) GenerateLoginData(authMethod *structs.VaultAuthM if err != nil { return nil, fmt.Errorf("aws auth failed to generate login data: %w", err) } + + // If a Vault role name is specified, we need to manually add this + role, ok := authMethod.Params["role"] + if ok { + loginData["role"] = role + } + return loginData, nil } diff --git a/agent/connect/ca/provider_vault_auth_test.go b/agent/connect/ca/provider_vault_auth_test.go index e1398eeb3bb..d9971c6930b 100644 --- a/agent/connect/ca/provider_vault_auth_test.go +++ b/agent/connect/ca/provider_vault_auth_test.go @@ -269,15 +269,22 @@ func TestVaultCAProvider_AWSCredentialsConfig(t *testing.T) { func TestVaultCAProvider_AWSLoginDataGenerator(t *testing.T) { cases := map[string]struct { - expErr error + expErr error + authMethod structs.VaultAuthMethod }{ - "valid login data": {}, + "valid login data": { + authMethod: structs.VaultAuthMethod{}, + }, + "with role": { + expErr: nil, + authMethod: structs.VaultAuthMethod{Type: "aws", MountPath: "", Params: map[string]interface{}{"role": "test-role"}}, + }, } for name, c := range cases { t.Run(name, func(t *testing.T) { ldg := &AWSLoginDataGenerator{credentials: credentials.AnonymousCredentials} - loginData, err := ldg.GenerateLoginData(&structs.VaultAuthMethod{}) + loginData, err := ldg.GenerateLoginData(&c.authMethod) if c.expErr != nil { require.Error(t, err) require.Contains(t, err.Error(), c.expErr.Error()) @@ -298,6 +305,10 @@ func TestVaultCAProvider_AWSLoginDataGenerator(t *testing.T) { require.True(t, exists, "missing expected key: %s", key) require.NotEmpty(t, val, "expected non-empty value for key: %s", key) } + + if c.authMethod.Params["role"] != nil { + require.Equal(t, c.authMethod.Params["role"], loginData["role"]) + } }) } } diff --git a/agent/connect/ca/provider_vault_test.go b/agent/connect/ca/provider_vault_test.go index f7375a01760..52c971a6086 100644 --- a/agent/connect/ca/provider_vault_test.go +++ b/agent/connect/ca/provider_vault_test.go @@ -13,6 +13,7 @@ import ( vaultapi "github.com/hashicorp/vault/api" "github.com/hashicorp/vault/api/auth/gcp" vaultconst "github.com/hashicorp/vault/sdk/helper/consts" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/hashicorp/consul/agent/connect" @@ -558,12 +559,6 @@ func TestVaultCAProvider_CrossSignCA(t *testing.T) { run := func(t *testing.T, tc CASigningKeyTypes, withSudo, expectFailure bool) { t.Parallel() - if tc.SigningKeyType != tc.CSRKeyType { - // TODO: uncomment since the bug is closed - // See https://github.com/hashicorp/vault/issues/7709 - t.Skip("Vault doesn't support cross-signing different key types yet.") - } - testVault1 := NewTestVaultServer(t) attr1 := &VaultTokenAttributes{ @@ -1140,6 +1135,51 @@ func TestVaultCAProvider_GenerateIntermediate(t *testing.T) { require.NotEqual(t, orig, new) } +func TestVaultCAProvider_DeletePreviousIssuerAndKey(t *testing.T) { + SkipIfVaultNotPresent(t) + t.Parallel() + + testVault := NewTestVaultServer(t) + attr := &VaultTokenAttributes{ + RootPath: "pki-root", + IntermediatePath: "pki-intermediate", + ConsulManaged: true, + } + token := CreateVaultTokenWithAttrs(t, testVault.client, attr) + provider := createVaultProvider(t, true, testVault.Addr, token, + map[string]any{ + "RootPKIPath": "pki-root/", + "IntermediatePKIPath": "pki-intermediate/", + }) + res, err := testVault.Client().Logical().List("pki-intermediate/issuers") + require.NoError(t, err) + // Why 2 issuers? There is always an initial issuer that + // gets created before we manage the lifecycle of issuers. + // Since we're asserting that the number doesn't grow + // this isn't too much of a concern. + // + // Note the key "keys" refers to the IDs of the resource based + // on the endpoint the response is from. + require.Len(t, res.Data["keys"], 2) + + res, err = testVault.Client().Logical().List("pki-intermediate/keys") + require.NoError(t, err) + require.Len(t, res.Data["keys"], 1) + + for i := 0; i < 3; i++ { + _, err := provider.GenerateLeafSigningCert() + require.NoError(t, err) + + res, err := testVault.Client().Logical().List("pki-intermediate/issuers") + require.NoError(t, err) + require.Len(t, res.Data["keys"], 2) + + res, err = testVault.Client().Logical().List("pki-intermediate/keys") + require.NoError(t, err) + assert.Len(t, res.Data["keys"], 1) + } +} + func TestVaultCAProvider_GenerateIntermediate_inSecondary(t *testing.T) { SkipIfVaultNotPresent(t) diff --git a/agent/connect/ca/testing.go b/agent/connect/ca/testing.go index 70c297732b8..450c13d6dd8 100644 --- a/agent/connect/ca/testing.go +++ b/agent/connect/ca/testing.go @@ -146,7 +146,7 @@ func NewTestVaultServer(t testing.T) *TestVaultServer { // We pass '-dev-no-store-token' to avoid having multiple vaults oddly // interact and fail like this: // - // Error initializing Dev mode: rename /home/circleci/.vault-token.tmp /home/circleci/.vault-token: no such file or directory + // Error initializing Dev mode: rename /.vault-token.tmp /.vault-token: no such file or directory // "-dev-no-store-token", } diff --git a/agent/connect/sni.go b/agent/connect/sni.go index e60740e4b12..221d7c61c15 100644 --- a/agent/connect/sni.go +++ b/agent/connect/sni.go @@ -28,7 +28,7 @@ func UpstreamSNI(u *structs.Upstream, subset string, dc string, trustDomain stri func GatewaySNI(dc string, partition, trustDomain string) string { if partition == "" { - // TODO(partitions) Make default available in OSS as a constant for uses like this one + // TODO(partitions) Make default available in CE as a constant for uses like this one partition = "default" } @@ -45,7 +45,7 @@ func ServiceSNI(service string, subset string, namespace string, partition strin namespace = structs.IntentionDefaultNamespace } if partition == "" { - // TODO(partitions) Make default available in OSS as a constant for uses like this one + // TODO(partitions) Make default available in CE as a constant for uses like this one partition = "default" } @@ -106,7 +106,7 @@ func PeeredServiceSNI(service, namespace, partition, peerName, trustDomain strin namespace = structs.IntentionDefaultNamespace } if partition == "" { - // TODO(partitions) Make default available in OSS as a constant for uses like this one + // TODO(partitions) Make default available in CE as a constant for uses like this one partition = "default" } diff --git a/agent/connect/uri_agent_oss.go b/agent/connect/uri_agent_ce.go similarity index 86% rename from agent/connect/uri_agent_oss.go rename to agent/connect/uri_agent_ce.go index e24f9b56098..fdfb075a8d4 100644 --- a/agent/connect/uri_agent_oss.go +++ b/agent/connect/uri_agent_ce.go @@ -10,7 +10,7 @@ import ( ) // GetEnterpriseMeta will synthesize an EnterpriseMeta struct from the SpiffeIDAgent. -// in OSS this just returns an empty (but never nil) struct pointer +// in CE this just returns an empty (but never nil) struct pointer func (id SpiffeIDAgent) GetEnterpriseMeta() *acl.EnterpriseMeta { return &acl.EnterpriseMeta{} } diff --git a/agent/connect/uri_agent_oss_test.go b/agent/connect/uri_agent_ce_test.go similarity index 100% rename from agent/connect/uri_agent_oss_test.go rename to agent/connect/uri_agent_ce_test.go diff --git a/agent/connect/uri_mesh_gateway_oss.go b/agent/connect/uri_mesh_gateway_ce.go similarity index 85% rename from agent/connect/uri_mesh_gateway_oss.go rename to agent/connect/uri_mesh_gateway_ce.go index 8865b97f94f..a92d0fc7ffc 100644 --- a/agent/connect/uri_mesh_gateway_oss.go +++ b/agent/connect/uri_mesh_gateway_ce.go @@ -10,7 +10,7 @@ import ( ) // GetEnterpriseMeta will synthesize an EnterpriseMeta struct from the SpiffeIDAgent. -// in OSS this just returns an empty (but never nil) struct pointer +// in CE this just returns an empty (but never nil) struct pointer func (id SpiffeIDMeshGateway) GetEnterpriseMeta() *acl.EnterpriseMeta { return &acl.EnterpriseMeta{} } diff --git a/agent/connect/uri_mesh_gateway_oss_test.go b/agent/connect/uri_mesh_gateway_ce_test.go similarity index 100% rename from agent/connect/uri_mesh_gateway_oss_test.go rename to agent/connect/uri_mesh_gateway_ce_test.go diff --git a/agent/connect/uri_service.go b/agent/connect/uri_service.go index 685498b1afb..e3ed1b28895 100644 --- a/agent/connect/uri_service.go +++ b/agent/connect/uri_service.go @@ -40,10 +40,10 @@ func (id SpiffeIDService) uriPath() string { id.Service, ) - // Although OSS has no support for partitions, it still needs to be able to + // Although CE has no support for partitions, it still needs to be able to // handle exportedPartition from peered Consul Enterprise clusters in order // to generate the correct SpiffeID. - // We intentionally avoid using pbpartition.DefaultName here to be OSS friendly. + // We intentionally avoid using pbpartition.DefaultName here to be CE friendly. if ap := id.PartitionOrDefault(); ap != "" && ap != "default" { return "/ap/" + ap + path } diff --git a/agent/connect/uri_service_oss.go b/agent/connect/uri_service_ce.go similarity index 72% rename from agent/connect/uri_service_oss.go rename to agent/connect/uri_service_ce.go index 63a51bf7003..7eedecade67 100644 --- a/agent/connect/uri_service_oss.go +++ b/agent/connect/uri_service_ce.go @@ -10,13 +10,13 @@ import ( ) // GetEnterpriseMeta will synthesize an EnterpriseMeta struct from the SpiffeIDService. -// in OSS this just returns an empty (but never nil) struct pointer +// in CE this just returns an empty (but never nil) struct pointer func (id SpiffeIDService) GetEnterpriseMeta() *acl.EnterpriseMeta { return &acl.EnterpriseMeta{} } -// PartitionOrDefault breaks from OSS's pattern of returning empty strings. -// Although OSS has no support for partitions, it still needs to be able to +// PartitionOrDefault breaks from CE's pattern of returning empty strings. +// Although CE has no support for partitions, it still needs to be able to // handle exportedPartition from peered Consul Enterprise clusters in order // to generate the correct SpiffeID. func (id SpiffeIDService) PartitionOrDefault() string { diff --git a/agent/connect/uri_service_oss_test.go b/agent/connect/uri_service_ce_test.go similarity index 100% rename from agent/connect/uri_service_oss_test.go rename to agent/connect/uri_service_ce_test.go diff --git a/agent/consul/acl_authmethod_oss.go b/agent/consul/acl_authmethod_ce.go similarity index 100% rename from agent/consul/acl_authmethod_oss.go rename to agent/consul/acl_authmethod_ce.go diff --git a/agent/consul/acl_oss.go b/agent/consul/acl_ce.go similarity index 95% rename from agent/consul/acl_oss.go rename to agent/consul/acl_ce.go index 1fe4fbbf817..85555c57fc3 100644 --- a/agent/consul/acl_oss.go +++ b/agent/consul/acl_ce.go @@ -33,13 +33,13 @@ func (r *ACLResolver) resolveEnterpriseDefaultsForIdentity(identity structs.ACLI // resolveEnterpriseIdentityAndRoles will resolve an enterprise identity to an additional set of roles func (_ *ACLResolver) resolveEnterpriseIdentityAndRoles(_ structs.ACLIdentity) (structs.ACLIdentity, structs.ACLRoles, error) { - // this function does nothing in OSS + // this function does nothing in CE return nil, nil, nil } // resolveEnterpriseIdentityAndPolicies will resolve an enterprise identity to an additional set of policies func (_ *ACLResolver) resolveEnterpriseIdentityAndPolicies(_ structs.ACLIdentity) (structs.ACLIdentity, structs.ACLPolicies, error) { - // this function does nothing in OSS + // this function does nothing in CE return nil, nil, nil } diff --git a/agent/consul/acl_oss_test.go b/agent/consul/acl_ce_test.go similarity index 100% rename from agent/consul/acl_oss_test.go rename to agent/consul/acl_ce_test.go diff --git a/agent/consul/acl_endpoint.go b/agent/consul/acl_endpoint.go index 790bd71a5b5..0072445caff 100644 --- a/agent/consul/acl_endpoint.go +++ b/agent/consul/acl_endpoint.go @@ -677,8 +677,18 @@ func (a *ACL) TokenList(args *structs.ACLTokenListRequest, reply *structs.ACLTok } return a.srv.blockingQuery(&args.QueryOptions, &reply.QueryMeta, - func(ws memdb.WatchSet, state *state.Store) error { - index, tokens, err := state.ACLTokenList(ws, args.IncludeLocal, args.IncludeGlobal, args.Policy, args.Role, args.AuthMethod, methodMeta, &args.EnterpriseMeta) + func(ws memdb.WatchSet, s *state.Store) error { + index, tokens, err := s.ACLTokenListWithParameters(ws, state.ACLTokenListParameters{ + Local: args.IncludeLocal, + Global: args.IncludeGlobal, + Policy: args.Policy, + Role: args.Role, + MethodName: args.AuthMethod, + ServiceName: args.ServiceName, + MethodMeta: methodMeta, + EnterpriseMeta: &args.EnterpriseMeta, + }) + if err != nil { return err } @@ -866,8 +876,8 @@ func (a *ACL) PolicySet(args *structs.ACLPolicySetRequest, reply *structs.ACLPol return fmt.Errorf("Invalid Policy: no Name is set") } - if !acl.IsValidPolicyName(policy.Name) { - return fmt.Errorf("Invalid Policy: invalid Name. Only alphanumeric characters, '-' and '_' are allowed") + if err := acl.ValidatePolicyName(policy.Name); err != nil { + return err } var idMatch *structs.ACLPolicy @@ -912,13 +922,13 @@ func (a *ACL) PolicySet(args *structs.ACLPolicySetRequest, reply *structs.ACLPol return fmt.Errorf("Invalid Policy: A policy with name %q already exists", policy.Name) } - if policy.ID == structs.ACLPolicyGlobalManagementID { + if builtinPolicy, ok := structs.ACLBuiltinPolicies[policy.ID]; ok { if policy.Datacenters != nil || len(policy.Datacenters) > 0 { - return fmt.Errorf("Changing the Datacenters of the builtin global-management policy is not permitted") + return fmt.Errorf("Changing the Datacenters of the %s policy is not permitted", builtinPolicy.Name) } if policy.Rules != idMatch.Rules { - return fmt.Errorf("Changing the Rules for the builtin global-management policy is not permitted") + return fmt.Errorf("Changing the Rules for the builtin %s policy is not permitted", builtinPolicy.Name) } } } @@ -996,8 +1006,8 @@ func (a *ACL) PolicyDelete(args *structs.ACLPolicyDeleteRequest, reply *string) return fmt.Errorf("policy does not exist: %w", acl.ErrNotFound) } - if policy.ID == structs.ACLPolicyGlobalManagementID { - return fmt.Errorf("Delete operation not permitted on the builtin global-management policy") + if builtinPolicy, ok := structs.ACLBuiltinPolicies[policy.ID]; ok { + return fmt.Errorf("Delete operation not permitted on the builtin %s policy", builtinPolicy.Name) } req := structs.ACLPolicyBatchDeleteRequest{ @@ -1110,7 +1120,7 @@ func (a *ACL) PolicyResolve(args *structs.ACLPolicyBatchGetRequest, reply *struc } } - a.srv.setQueryMeta(&reply.QueryMeta, args.Token) + a.srv.SetQueryMeta(&reply.QueryMeta, args.Token) return nil } @@ -1517,7 +1527,7 @@ func (a *ACL) RoleResolve(args *structs.ACLRoleBatchGetRequest, reply *structs.A } } - a.srv.setQueryMeta(&reply.QueryMeta, args.Token) + a.srv.SetQueryMeta(&reply.QueryMeta, args.Token) return nil } diff --git a/agent/consul/acl_endpoint_oss.go b/agent/consul/acl_endpoint_ce.go similarity index 100% rename from agent/consul/acl_endpoint_oss.go rename to agent/consul/acl_endpoint_ce.go diff --git a/agent/consul/acl_endpoint_test.go b/agent/consul/acl_endpoint_test.go index 8bbf8379b53..c010d16a8ed 100644 --- a/agent/consul/acl_endpoint_test.go +++ b/agent/consul/acl_endpoint_test.go @@ -140,7 +140,7 @@ func TestACLEndpoint_ReplicationStatus(t *testing.T) { retry.Run(t, func(r *retry.R) { var status structs.ACLReplicationStatus err := msgpackrpc.CallWithCodec(codec, "ACL.ReplicationStatus", &getR, &status) - require.NoError(t, err) + require.NoError(r, err) require.True(r, status.Enabled) require.True(r, status.Running) @@ -217,7 +217,7 @@ func TestACLEndpoint_TokenRead(t *testing.T) { time.Sleep(200 * time.Millisecond) err := aclEp.TokenRead(&req, &resp) require.Error(r, err) - require.ErrorContains(t, err, "ACL not found") + require.ErrorContains(r, err, "ACL not found") require.Nil(r, resp.Token) }) }) @@ -2180,7 +2180,7 @@ func TestACLEndpoint_PolicySet_CustomID(t *testing.T) { require.Error(t, err) } -func TestACLEndpoint_PolicySet_globalManagement(t *testing.T) { +func TestACLEndpoint_PolicySet_builtins(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } @@ -2192,47 +2192,50 @@ func TestACLEndpoint_PolicySet_globalManagement(t *testing.T) { aclEp := ACL{srv: srv} - // Can't change the rules - { - req := structs.ACLPolicySetRequest{ - Datacenter: "dc1", - Policy: structs.ACLPolicy{ - ID: structs.ACLPolicyGlobalManagementID, - Name: "foobar", // This is required to get past validation - Rules: "service \"\" { policy = \"write\" }", - }, - WriteRequest: structs.WriteRequest{Token: TestDefaultInitialManagementToken}, - } - resp := structs.ACLPolicy{} + for _, builtinPolicy := range structs.ACLBuiltinPolicies { + name := fmt.Sprintf("foobar-%s", builtinPolicy.Name) // This is required to get past validation - err := aclEp.PolicySet(&req, &resp) - require.EqualError(t, err, "Changing the Rules for the builtin global-management policy is not permitted") - } + // Can't change the rules + { + req := structs.ACLPolicySetRequest{ + Datacenter: "dc1", + Policy: structs.ACLPolicy{ + ID: builtinPolicy.ID, + Name: name, + Rules: "service \"\" { policy = \"write\" }", + }, + WriteRequest: structs.WriteRequest{Token: TestDefaultInitialManagementToken}, + } + resp := structs.ACLPolicy{} - // Can rename it - { - req := structs.ACLPolicySetRequest{ - Datacenter: "dc1", - Policy: structs.ACLPolicy{ - ID: structs.ACLPolicyGlobalManagementID, - Name: "foobar", - Rules: structs.ACLPolicyGlobalManagement, - }, - WriteRequest: structs.WriteRequest{Token: TestDefaultInitialManagementToken}, + err := aclEp.PolicySet(&req, &resp) + require.EqualError(t, err, fmt.Sprintf("Changing the Rules for the builtin %s policy is not permitted", builtinPolicy.Name)) } - resp := structs.ACLPolicy{} - err := aclEp.PolicySet(&req, &resp) - require.NoError(t, err) + // Can rename it + { + req := structs.ACLPolicySetRequest{ + Datacenter: "dc1", + Policy: structs.ACLPolicy{ + ID: builtinPolicy.ID, + Name: name, + Rules: builtinPolicy.Rules, + }, + WriteRequest: structs.WriteRequest{Token: TestDefaultInitialManagementToken}, + } + resp := structs.ACLPolicy{} - // Get the policy again - policyResp, err := retrieveTestPolicy(codec, TestDefaultInitialManagementToken, "dc1", structs.ACLPolicyGlobalManagementID) - require.NoError(t, err) - policy := policyResp.Policy + err := aclEp.PolicySet(&req, &resp) + require.NoError(t, err) - require.Equal(t, policy.ID, structs.ACLPolicyGlobalManagementID) - require.Equal(t, policy.Name, "foobar") + // Get the policy again + policyResp, err := retrieveTestPolicy(codec, TestDefaultInitialManagementToken, "dc1", builtinPolicy.ID) + require.NoError(t, err) + policy := policyResp.Policy + require.Equal(t, policy.ID, builtinPolicy.ID) + require.Equal(t, policy.Name, name) + } } } @@ -2268,7 +2271,7 @@ func TestACLEndpoint_PolicyDelete(t *testing.T) { require.Nil(t, tokenResp.Policy) } -func TestACLEndpoint_PolicyDelete_globalManagement(t *testing.T) { +func TestACLEndpoint_PolicyDelete_builtins(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } @@ -2279,16 +2282,17 @@ func TestACLEndpoint_PolicyDelete_globalManagement(t *testing.T) { waitForLeaderEstablishment(t, srv) aclEp := ACL{srv: srv} - req := structs.ACLPolicyDeleteRequest{ - Datacenter: "dc1", - PolicyID: structs.ACLPolicyGlobalManagementID, - WriteRequest: structs.WriteRequest{Token: TestDefaultInitialManagementToken}, - } - var resp string - - err := aclEp.PolicyDelete(&req, &resp) + for _, builtinPolicy := range structs.ACLBuiltinPolicies { + req := structs.ACLPolicyDeleteRequest{ + Datacenter: "dc1", + PolicyID: builtinPolicy.ID, + WriteRequest: structs.WriteRequest{Token: TestDefaultInitialManagementToken}, + } + var resp string - require.EqualError(t, err, "Delete operation not permitted on the builtin global-management policy") + err := aclEp.PolicyDelete(&req, &resp) + require.EqualError(t, err, fmt.Sprintf("Delete operation not permitted on the builtin %s policy", builtinPolicy.Name)) + } } func TestACLEndpoint_PolicyList(t *testing.T) { @@ -2321,6 +2325,7 @@ func TestACLEndpoint_PolicyList(t *testing.T) { policies := []string{ structs.ACLPolicyGlobalManagementID, + structs.ACLPolicyGlobalReadOnlyID, p1.ID, p2.ID, } diff --git a/agent/consul/acl_replication_test.go b/agent/consul/acl_replication_test.go index c3c564fedd2..bbb6120f35c 100644 --- a/agent/consul/acl_replication_test.go +++ b/agent/consul/acl_replication_test.go @@ -375,8 +375,10 @@ func TestACLReplication_Tokens(t *testing.T) { checkSame := func(t *retry.R) { // only account for global tokens - local tokens shouldn't be replicated + // nolint:staticcheck index, remote, err := s1.fsm.State().ACLTokenList(nil, false, true, "", "", "", nil, nil) require.NoError(t, err) + // nolint:staticcheck _, local, err := s2.fsm.State().ACLTokenList(nil, false, true, "", "", "", nil, nil) require.NoError(t, err) @@ -480,6 +482,7 @@ func TestACLReplication_Tokens(t *testing.T) { }) // verify dc2 local tokens didn't get blown away + // nolint:staticcheck _, local, err := s2.fsm.State().ACLTokenList(nil, true, false, "", "", "", nil, nil) require.NoError(t, err) require.Len(t, local, 50) @@ -818,9 +821,11 @@ func TestACLReplication_AllTypes(t *testing.T) { checkSameTokens := func(t *retry.R) { // only account for global tokens - local tokens shouldn't be replicated + // nolint:staticcheck index, remote, err := s1.fsm.State().ACLTokenList(nil, false, true, "", "", "", nil, nil) require.NoError(t, err) // Query for all of them, so that we can prove that no globals snuck in. + // nolint:staticcheck _, local, err := s2.fsm.State().ACLTokenList(nil, true, true, "", "", "", nil, nil) require.NoError(t, err) diff --git a/agent/consul/acl_replication_types.go b/agent/consul/acl_replication_types.go index a395270d6e1..20e4456988e 100644 --- a/agent/consul/acl_replication_types.go +++ b/agent/consul/acl_replication_types.go @@ -35,6 +35,7 @@ func (r *aclTokenReplicator) FetchRemote(srv *Server, lastRemoteIndex uint64) (i func (r *aclTokenReplicator) FetchLocal(srv *Server) (int, uint64, error) { r.local = nil + // nolint:staticcheck idx, local, err := srv.fsm.State().ACLTokenList(nil, false, true, "", "", "", nil, srv.replicationEnterpriseMeta()) if err != nil { return 0, 0, err diff --git a/agent/consul/acl_server.go b/agent/consul/acl_server.go index 48e80191e38..299d7392dbf 100644 --- a/agent/consul/acl_server.go +++ b/agent/consul/acl_server.go @@ -110,7 +110,7 @@ type serverACLResolverBackend struct { } func (s *serverACLResolverBackend) IsServerManagementToken(token string) bool { - mgmt, err := s.getSystemMetadata(structs.ServerManagementTokenAccessorID) + mgmt, err := s.GetSystemMetadata(structs.ServerManagementTokenAccessorID) if err != nil { s.logger.Debug("failed to fetch server management token: %w", err) return false @@ -148,9 +148,9 @@ func (s *Server) ResolveIdentityFromToken(token string) (bool, structs.ACLIdenti } else if aclToken != nil && !aclToken.IsExpired(time.Now()) { return true, aclToken, nil } - if aclToken == nil && token == acl.AnonymousTokenSecret { + if aclToken == nil && token == acl.AnonymousTokenSecret && s.InPrimaryDatacenter() { // synthesize the anonymous token for early use, bootstrapping has not completed - s.InsertAnonymousToken() + s.insertAnonymousToken() fallbackId := structs.ACLToken{ AccessorID: acl.AnonymousTokenID, SecretID: acl.AnonymousTokenSecret, diff --git a/agent/consul/acl_server_oss.go b/agent/consul/acl_server_ce.go similarity index 100% rename from agent/consul/acl_server_oss.go rename to agent/consul/acl_server_ce.go diff --git a/agent/consul/auth/binder_oss.go b/agent/consul/auth/binder_ce.go similarity index 100% rename from agent/consul/auth/binder_oss.go rename to agent/consul/auth/binder_ce.go diff --git a/agent/consul/auth/token_writer.go b/agent/consul/auth/token_writer.go index 957d34fada8..2b90e257b5d 100644 --- a/agent/consul/auth/token_writer.go +++ b/agent/consul/auth/token_writer.go @@ -241,7 +241,7 @@ func (w *TokenWriter) Delete(secretID string, fromLogout bool) error { func validateTokenID(id string) error { if structs.ACLIDReserved(id) { - return fmt.Errorf("UUIDs with the prefix %q are reserved", structs.ACLReservedPrefix) + return fmt.Errorf("UUIDs with the prefix %q are reserved", structs.ACLReservedIDPrefix) } if _, err := uuid.ParseUUID(id); err != nil { return errors.New("not a valid UUID") diff --git a/agent/consul/auth/token_writer_oss.go b/agent/consul/auth/token_writer_ce.go similarity index 100% rename from agent/consul/auth/token_writer_oss.go rename to agent/consul/auth/token_writer_ce.go diff --git a/agent/consul/auth/token_writer_test.go b/agent/consul/auth/token_writer_test.go index 499c10ab573..86c34f90a8c 100644 --- a/agent/consul/auth/token_writer_test.go +++ b/agent/consul/auth/token_writer_test.go @@ -38,7 +38,7 @@ func TestTokenWriter_Create_Validation(t *testing.T) { errorContains: "not a valid UUID", }, "AccessorID is reserved": { - token: structs.ACLToken{AccessorID: structs.ACLReservedPrefix + generateID(t)}, + token: structs.ACLToken{AccessorID: structs.ACLReservedIDPrefix + generateID(t)}, errorContains: "reserved", }, "AccessorID already in use (as AccessorID)": { @@ -54,7 +54,7 @@ func TestTokenWriter_Create_Validation(t *testing.T) { errorContains: "not a valid UUID", }, "SecretID is reserved": { - token: structs.ACLToken{SecretID: structs.ACLReservedPrefix + generateID(t)}, + token: structs.ACLToken{SecretID: structs.ACLReservedIDPrefix + generateID(t)}, errorContains: "reserved", }, "SecretID already in use (as AccessorID)": { diff --git a/agent/consul/authmethod/authmethods_oss.go b/agent/consul/authmethod/authmethods_ce.go similarity index 100% rename from agent/consul/authmethod/authmethods_oss.go rename to agent/consul/authmethod/authmethods_ce.go diff --git a/agent/consul/authmethod/kubeauth/k8s_oss.go b/agent/consul/authmethod/kubeauth/k8s_ce.go similarity index 100% rename from agent/consul/authmethod/kubeauth/k8s_oss.go rename to agent/consul/authmethod/kubeauth/k8s_ce.go diff --git a/agent/consul/authmethod/ssoauth/sso_oss.go b/agent/consul/authmethod/ssoauth/sso_ce.go similarity index 100% rename from agent/consul/authmethod/ssoauth/sso_oss.go rename to agent/consul/authmethod/ssoauth/sso_ce.go diff --git a/agent/consul/authmethod/testauth/testing_oss.go b/agent/consul/authmethod/testauth/testing_ce.go similarity index 100% rename from agent/consul/authmethod/testauth/testing_oss.go rename to agent/consul/authmethod/testauth/testing_ce.go diff --git a/agent/consul/auto_config_endpoint_test.go b/agent/consul/auto_config_endpoint_test.go index ac9ea4128dd..1f0f8e18a1c 100644 --- a/agent/consul/auto_config_endpoint_test.go +++ b/agent/consul/auto_config_endpoint_test.go @@ -3,12 +3,11 @@ package consul import ( "bytes" "crypto" - crand "crypto/rand" + "crypto/rand" "crypto/x509" "encoding/base64" "encoding/pem" "fmt" - "math/rand" "net" "net/url" "os" @@ -884,7 +883,7 @@ func TestAutoConfig_parseAutoConfigCSR(t *testing.T) { // customizations to allow for better unit testing. createCSR := func(tmpl *x509.CertificateRequest, privateKey crypto.Signer) (string, error) { connect.HackSANExtensionForCSR(tmpl) - bs, err := x509.CreateCertificateRequest(crand.Reader, tmpl, privateKey) + bs, err := x509.CreateCertificateRequest(rand.Reader, tmpl, privateKey) require.NoError(t, err) var csrBuf bytes.Buffer err = pem.Encode(&csrBuf, &pem.Block{Type: "CERTIFICATE REQUEST", Bytes: bs}) diff --git a/agent/consul/autopilot_oss.go b/agent/consul/autopilot_ce.go similarity index 100% rename from agent/consul/autopilot_oss.go rename to agent/consul/autopilot_ce.go diff --git a/agent/consul/catalog_endpoint.go b/agent/consul/catalog_endpoint.go index bacafa688ae..98399dffb6a 100644 --- a/agent/consul/catalog_endpoint.go +++ b/agent/consul/catalog_endpoint.go @@ -447,7 +447,7 @@ func vetDeregisterWithACL( } // This order must match the code in applyDeregister() in - // fsm/commands_oss.go since it also evaluates things in this order, + // fsm/commands_ce.go since it also evaluates things in this order, // and will ignore fields based on this precedence. This lets us also // ignore them from an ACL perspective. if subj.ServiceID != "" { diff --git a/agent/consul/client.go b/agent/consul/client.go index c1ea41f8996..7f6ed803405 100644 --- a/agent/consul/client.go +++ b/agent/consul/client.go @@ -29,15 +29,15 @@ import ( var ClientCounters = []prometheus.CounterDefinition{ { Name: []string{"client", "rpc"}, - Help: "Increments whenever a Consul agent in client mode makes an RPC request to a Consul server.", + Help: "Increments whenever a Consul agent makes an RPC request to a Consul server.", }, { Name: []string{"client", "rpc", "exceeded"}, - Help: "Increments whenever a Consul agent in client mode makes an RPC request to a Consul server gets rate limited by that agent's limits configuration.", + Help: "Increments whenever a Consul agent makes an RPC request to a Consul server gets rate limited by that agent's limits configuration.", }, { Name: []string{"client", "rpc", "failed"}, - Help: "Increments whenever a Consul agent in client mode makes an RPC request to a Consul server and fails.", + Help: "Increments whenever a Consul agent makes an RPC request to a Consul server and fails.", }, } diff --git a/agent/consul/client_test.go b/agent/consul/client_test.go index 15af5550946..e8b79f04dc6 100644 --- a/agent/consul/client_test.go +++ b/agent/consul/client_test.go @@ -176,7 +176,7 @@ func TestClient_LANReap(t *testing.T) { retry.Run(t, func(r *retry.R) { require.Len(r, c1.LANMembersInAgentPartition(), 1) server := c1.router.FindLANServer() - require.Nil(t, server) + require.Nil(r, server) }) } @@ -501,11 +501,15 @@ func newClient(t *testing.T, config *Config) *Client { return client } -func newTestResolverConfig(t *testing.T, suffix string) resolver.Config { +func newTestResolverConfig(t *testing.T, suffix string, dc, agentType string) resolver.Config { n := t.Name() s := strings.Replace(n, "/", "", -1) s = strings.Replace(s, "_", "", -1) - return resolver.Config{Authority: strings.ToLower(s) + "-" + suffix} + return resolver.Config{ + Datacenter: dc, + AgentType: agentType, + Authority: strings.ToLower(s) + "-" + suffix, + } } func newDefaultDeps(t *testing.T, c *Config) Deps { @@ -520,11 +524,15 @@ func newDefaultDeps(t *testing.T, c *Config) Deps { tls, err := tlsutil.NewConfigurator(c.TLSConfig, logger) require.NoError(t, err, "failed to create tls configuration") - resolverBuilder := resolver.NewServerResolverBuilder(newTestResolverConfig(t, c.NodeName+"-"+c.Datacenter)) + resolverBuilder := resolver.NewServerResolverBuilder(newTestResolverConfig(t, c.NodeName+"-"+c.Datacenter, c.Datacenter, "server")) resolver.Register(resolverBuilder) + t.Cleanup(func() { + resolver.Deregister(resolverBuilder.Authority()) + }) balancerBuilder := balancer.NewBuilder(resolverBuilder.Authority(), testutil.Logger(t)) balancerBuilder.Register() + t.Cleanup(balancerBuilder.Deregister) r := router.NewRouter( logger, @@ -559,7 +567,6 @@ func newDefaultDeps(t *testing.T, c *Config) Deps { UseTLSForDC: tls.UseTLS, DialingFromServer: true, DialingFromDatacenter: c.Datacenter, - BalancerBuilder: balancerBuilder, }), LeaderForwarder: resolverBuilder, NewRequestRecorderFunc: middleware.NewRequestRecorder, diff --git a/agent/consul/config.go b/agent/consul/config.go index f114dcecfc8..db3cc5d19d4 100644 --- a/agent/consul/config.go +++ b/agent/consul/config.go @@ -436,8 +436,16 @@ type Config struct { PeeringTestAllowPeerRegistrations bool + Cloud CloudConfig + + Reporting Reporting + // Embedded Consul Enterprise specific configuration *EnterpriseConfig + + // ServerRejoinAgeMax is used to specify the duration of time a server + // is allowed to be down/offline before a startup operation is refused. + ServerRejoinAgeMax time.Duration } func (c *Config) InPrimaryDatacenter() bool { @@ -563,6 +571,8 @@ func DefaultConfig() *Config { PeeringTestAllowPeerRegistrations: false, EnterpriseConfig: DefaultEnterpriseConfig(), + + ServerRejoinAgeMax: 24 * 7 * time.Hour, } // Increase our reap interval to 3 days instead of 24h. @@ -671,6 +681,7 @@ type ReloadableConfig struct { RaftTrailingLogs int HeartbeatTimeout time.Duration ElectionTimeout time.Duration + Reporting Reporting } type RaftLogStoreConfig struct { @@ -693,3 +704,11 @@ type RaftBoltDBConfig struct { type WALConfig struct { SegmentSize int } + +type License struct { + Enabled bool +} + +type Reporting struct { + License License +} diff --git a/agent/consul/config_oss.go b/agent/consul/config_ce.go similarity index 100% rename from agent/consul/config_oss.go rename to agent/consul/config_ce.go diff --git a/agent/consul/config_cloud.go b/agent/consul/config_cloud.go new file mode 100644 index 00000000000..b0780052f66 --- /dev/null +++ b/agent/consul/config_cloud.go @@ -0,0 +1,5 @@ +package consul + +type CloudConfig struct { + ManagementToken string +} diff --git a/agent/consul/config_endpoint_test.go b/agent/consul/config_endpoint_test.go index e836db9977e..569ec4f51c1 100644 --- a/agent/consul/config_endpoint_test.go +++ b/agent/consul/config_endpoint_test.go @@ -1316,16 +1316,6 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { "protocol": "grpc", }, UpstreamConfigs: structs.OpaqueUpstreamConfigs{ - { - Upstream: structs.PeeredServiceName{ - ServiceName: structs.NewServiceName( - structs.WildcardSpecifier, - acl.DefaultEnterpriseMeta().WithWildcardNamespace()), - }, - Config: map[string]interface{}{ - "protocol": "grpc", - }, - }, { Upstream: cache, Config: map[string]interface{}{ @@ -1377,15 +1367,6 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { "protocol": "grpc", }, UpstreamIDConfigs: structs.OpaqueUpstreamConfigsDeprecated{ - { - Upstream: structs.NewServiceID( - structs.WildcardSpecifier, - acl.DefaultEnterpriseMeta().WithWildcardNamespace(), - ), - Config: map[string]interface{}{ - "protocol": "grpc", - }, - }, { Upstream: structs.NewServiceID("cache", nil), Config: map[string]interface{}{ @@ -1442,12 +1423,6 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { "protocol": "grpc", }, UpstreamConfigs: structs.OpaqueUpstreamConfigs{ - { - Upstream: wildcard, - Config: map[string]interface{}{ - "protocol": "grpc", - }, - }, { Upstream: cache, Config: map[string]interface{}{ @@ -1541,6 +1516,8 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { Interval: 10, MaxFailures: 2, EnforcingConsecutive5xx: uintPointer(60), + MaxEjectionPercent: uintPointer(61), + BaseEjectionTime: durationPointer(62 * time.Second), }, }, Overrides: []*structs.UpstreamConfig{ @@ -1575,6 +1552,8 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { "Interval": int64(10), "MaxFailures": int64(2), "EnforcingConsecutive5xx": int64(60), + "MaxEjectionPercent": int64(61), + "BaseEjectionTime": uint64(62 * time.Second), }, "mesh_gateway": map[string]interface{}{ "Mode": "none", @@ -1589,6 +1568,8 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { "Interval": int64(10), "MaxFailures": int64(2), "EnforcingConsecutive5xx": int64(60), + "MaxEjectionPercent": int64(61), + "BaseEjectionTime": uint64(62 * time.Second), }, "mesh_gateway": map[string]interface{}{ "Mode": "local", @@ -2196,17 +2177,6 @@ func TestConfigEntry_ResolveServiceConfig_UpstreamProxyDefaultsProtocol(t *testi require.NoError(t, msgpackrpc.CallWithCodec(codec, "ConfigEntry.ResolveServiceConfig", &args, &out)) expected := structs.OpaqueUpstreamConfigs{ - { - Upstream: structs.PeeredServiceName{ - ServiceName: structs.NewServiceName( - structs.WildcardSpecifier, - acl.DefaultEnterpriseMeta().WithWildcardNamespace(), - ), - }, - Config: map[string]interface{}{ - "protocol": "http", - }, - }, { Upstream: id("bar"), Config: map[string]interface{}{ @@ -2275,16 +2245,6 @@ func TestConfigEntry_ResolveServiceConfig_ProxyDefaultsProtocol_UsedForAllUpstre "protocol": "http", }, UpstreamConfigs: structs.OpaqueUpstreamConfigs{ - { - Upstream: structs.PeeredServiceName{ - ServiceName: structs.NewServiceName( - structs.WildcardSpecifier, - acl.DefaultEnterpriseMeta().WithWildcardNamespace()), - }, - Config: map[string]interface{}{ - "protocol": "http", - }, - }, { Upstream: psn, Config: map[string]interface{}{ @@ -2699,3 +2659,7 @@ func Test_gateWriteToSecondary_AllowedKinds(t *testing.T) { func uintPointer(v uint32) *uint32 { return &v } + +func durationPointer(d time.Duration) *time.Duration { + return &d +} diff --git a/agent/consul/discoverychain/compile.go b/agent/consul/discoverychain/compile.go index 7af98bc06a1..212c3d8fbd3 100644 --- a/agent/consul/discoverychain/compile.go +++ b/agent/consul/discoverychain/compile.go @@ -719,10 +719,21 @@ func (c *compiler) newTarget(opts structs.DiscoveryTargetOpts) *structs.Discover } else { // Don't allow Peer and Datacenter. opts.Datacenter = "" - // Peer and Partition cannot both be set. - opts.Partition = acl.PartitionOrDefault("") + // Since discovery targets (for peering) are ONLY used to query the catalog, and + // not to generate the SNI it is more correct to switch this to the calling-side + // of the peering's partition as that matches where the replicated data is stored + // in the catalog. This is done to simplify the usage of peer targets in both + // the xds and proxycfg packages. + // + // The peer info data attached to service instances will have the embedded opaque + // SNI/SAN information generated by the remote side and that will have the + // OTHER partition properly specified. + opts.Partition = acl.PartitionOrDefault(c.evaluateInPartition) // Default to "default" rather than c.evaluateInNamespace. - opts.Namespace = acl.PartitionOrDefault(opts.Namespace) + // Note that the namespace is not swapped out, because it should + // always match the value in the remote cluster (and shouldn't + // have been changed anywhere). + opts.Namespace = acl.NamespaceOrDefault(opts.Namespace) } t := structs.NewDiscoveryTarget(opts) @@ -978,6 +989,7 @@ RESOLVE_AGAIN: Default: resolver.IsDefault(), Target: target.ID, ConnectTimeout: connectTimeout, + RequestTimeout: resolver.RequestTimeout, }, LoadBalancer: resolver.LoadBalancer, } diff --git a/agent/consul/discoverychain/compile_oss.go b/agent/consul/discoverychain/compile_ce.go similarity index 100% rename from agent/consul/discoverychain/compile_oss.go rename to agent/consul/discoverychain/compile_ce.go diff --git a/agent/consul/discoverychain/gateway.go b/agent/consul/discoverychain/gateway.go index cd582f1ec02..f711a4915ca 100644 --- a/agent/consul/discoverychain/gateway.go +++ b/agent/consul/discoverychain/gateway.go @@ -5,6 +5,7 @@ import ( "hash/crc32" "sort" "strconv" + "strings" "github.com/hashicorp/consul/agent/configentry" "github.com/hashicorp/consul/agent/structs" @@ -17,6 +18,7 @@ type GatewayChainSynthesizer struct { trustDomain string suffix string gateway *structs.APIGatewayConfigEntry + hostname string matchesByHostname map[string][]hostnameMatch tcpRoutes []structs.TCPRouteConfigEntry } @@ -44,17 +46,17 @@ func (l *GatewayChainSynthesizer) AddTCPRoute(route structs.TCPRouteConfigEntry) l.tcpRoutes = append(l.tcpRoutes, route) } +// SetHostname sets the base hostname for a listener that this is being synthesized for +func (l *GatewayChainSynthesizer) SetHostname(hostname string) { + l.hostname = hostname +} + // AddHTTPRoute takes a new route and flattens its rule matches out per hostname. // This is required since a single route can specify multiple hostnames, and a // single hostname can be specified in multiple routes. Routing for a given // hostname must behave based on the aggregate of all rules that apply to it. func (l *GatewayChainSynthesizer) AddHTTPRoute(route structs.HTTPRouteConfigEntry) { - hostnames := route.Hostnames - if len(route.Hostnames) == 0 { - // add a wildcard if there are no explicit hostnames set - hostnames = append(hostnames, "*") - } - + hostnames := route.FilteredHostnames(l.hostname) for _, host := range hostnames { matches, ok := l.matchesByHostname[host] if !ok { @@ -125,6 +127,46 @@ func (l *GatewayChainSynthesizer) Synthesize(chains ...*structs.CompiledDiscover if err != nil { return nil, nil, err } + + node := compiled.Nodes[compiled.StartNode] + if node.IsRouter() { + resolverPrefix := structs.DiscoveryGraphNodeTypeResolver + ":" + node.Name + + // clean out the clusters that will get added for the router + for name := range compiled.Nodes { + if strings.HasPrefix(name, resolverPrefix) { + delete(compiled.Nodes, name) + } + } + + // clean out the route rules that'll get added for the router + filtered := []*structs.DiscoveryRoute{} + for _, route := range node.Routes { + if strings.HasPrefix(route.NextNode, resolverPrefix) { + continue + } + filtered = append(filtered, route) + } + node.Routes = filtered + } + compiled.Nodes[compiled.StartNode] = node + + // fix up the nodes for the terminal targets to either be a splitter or resolver if there is no splitter present + for name, node := range compiled.Nodes { + switch node.Type { + // we should only have these two types + case structs.DiscoveryGraphNodeTypeRouter: + for i, route := range node.Routes { + node.Routes[i].NextNode = targetForResolverNode(route.NextNode, chains) + } + case structs.DiscoveryGraphNodeTypeSplitter: + for i, split := range node.Splits { + node.Splits[i].NextNode = targetForResolverNode(split.NextNode, chains) + } + } + compiled.Nodes[name] = node + } + for _, c := range chains { for id, target := range c.Targets { compiled.Targets[id] = target @@ -176,6 +218,34 @@ func (l *GatewayChainSynthesizer) consolidateHTTPRoutes() []structs.HTTPRouteCon return routes } +func targetForResolverNode(nodeName string, chains []*structs.CompiledDiscoveryChain) string { + resolverPrefix := structs.DiscoveryGraphNodeTypeResolver + ":" + splitterPrefix := structs.DiscoveryGraphNodeTypeSplitter + ":" + + if !strings.HasPrefix(nodeName, resolverPrefix) { + return nodeName + } + + splitterName := splitterPrefix + strings.TrimPrefix(nodeName, resolverPrefix) + + for _, c := range chains { + targetChainPrefix := resolverPrefix + c.ServiceName + "." + if strings.HasPrefix(nodeName, targetChainPrefix) && len(c.Nodes) == 1 { + // we have a virtual resolver that just maps to another resolver, return + // the given node name + return c.StartNode + } + + for name, node := range c.Nodes { + if node.IsSplitter() && strings.HasPrefix(splitterName, name) { + return name + } + } + } + + return nodeName +} + func hostsKey(hosts ...string) string { sort.Strings(hosts) hostsHash := crc32.NewIEEE() diff --git a/agent/consul/discoverychain/gateway_httproute.go b/agent/consul/discoverychain/gateway_httproute.go index aaaec12e6b1..30968c9a4d5 100644 --- a/agent/consul/discoverychain/gateway_httproute.go +++ b/agent/consul/discoverychain/gateway_httproute.go @@ -77,15 +77,14 @@ func httpRouteToDiscoveryChain(route structs.HTTPRouteConfigEntry) (*structs.Ser for idx, rule := range route.Rules { modifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.Filters.Headers) - prefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(rule.Filters.URLRewrites) + prefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(rule.Filters.URLRewrite) var destination structs.ServiceRouteDestination if len(rule.Services) == 1 { - // TODO open question: is there a use case where someone might want to set the rewrite to ""? service := rule.Services[0] - servicePrefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(service.Filters.URLRewrites) - if servicePrefixRewrite == "" { + servicePrefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(service.Filters.URLRewrite) + if service.Filters.URLRewrite == nil { servicePrefixRewrite = prefixRewrite } serviceModifier := httpRouteFiltersToServiceRouteHeaderModifier(service.Filters.Headers) @@ -176,13 +175,11 @@ func httpRouteToDiscoveryChain(route structs.HTTPRouteConfigEntry) (*structs.Ser return router, splitters, defaults } -func httpRouteFiltersToDestinationPrefixRewrite(rewrites []structs.URLRewrite) string { - for _, rewrite := range rewrites { - if rewrite.Path != "" { - return rewrite.Path - } +func httpRouteFiltersToDestinationPrefixRewrite(rewrite *structs.URLRewrite) string { + if rewrite == nil { + return "" } - return "" + return rewrite.Path } // httpRouteFiltersToServiceRouteHeaderModifier will consolidate a list of HTTP filters diff --git a/agent/consul/discoverychain/gateway_test.go b/agent/consul/discoverychain/gateway_test.go index 1d6ec78d24c..17fc539c3c5 100644 --- a/agent/consul/discoverychain/gateway_test.go +++ b/agent/consul/discoverychain/gateway_test.go @@ -3,6 +3,7 @@ package discoverychain import ( "testing" + "github.com/hashicorp/consul/agent/configentry" "github.com/hashicorp/consul/agent/structs" "github.com/stretchr/testify/require" ) @@ -46,7 +47,7 @@ func TestGatewayChainSynthesizer_AddHTTPRoute(t *testing.T) { route structs.HTTPRouteConfigEntry expectedMatchesByHostname map[string][]hostnameMatch }{ - "no hostanames": { + "no hostnames": { route: structs.HTTPRouteConfigEntry{ Kind: structs.HTTPRoute, Name: "route", @@ -459,6 +460,7 @@ func TestGatewayChainSynthesizer_AddHTTPRoute(t *testing.T) { gatewayChainSynthesizer := NewGatewayChainSynthesizer(datacenter, "domain", "suffix", gateway) + gatewayChainSynthesizer.SetHostname("*") gatewayChainSynthesizer.AddHTTPRoute(tc.route) require.Equal(t, tc.expectedMatchesByHostname, gatewayChainSynthesizer.matchesByHostname) @@ -537,15 +539,6 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) { Protocol: "http", StartNode: "router:gateway-suffix-9b9265b.default.default", Nodes: map[string]*structs.DiscoveryGraphNode{ - "resolver:gateway-suffix-9b9265b.default.default.dc1": { - Type: "resolver", - Name: "gateway-suffix-9b9265b.default.default.dc1", - Resolver: &structs.DiscoveryResolver{ - Target: "gateway-suffix-9b9265b.default.default.dc1", - Default: true, - ConnectTimeout: 5000000000, - }, - }, "router:gateway-suffix-9b9265b.default.default": { Type: "router", Name: "gateway-suffix-9b9265b.default.default", @@ -567,7 +560,92 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) { }, }, NextNode: "resolver:foo.default.default.dc1", - }, { + }}, + }, + "resolver:foo.default.default.dc1": { + Type: "resolver", + Name: "foo.default.default.dc1", + Resolver: &structs.DiscoveryResolver{ + Target: "foo.default.default.dc1", + Default: true, + ConnectTimeout: 5000000000, + }, + }, + }, + Targets: map[string]*structs.DiscoveryTarget{ + "gateway-suffix-9b9265b.default.default.dc1": { + ID: "gateway-suffix-9b9265b.default.default.dc1", + Service: "gateway-suffix-9b9265b", + Datacenter: "dc1", + Partition: "default", + Namespace: "default", + ConnectTimeout: 5000000000, + SNI: "gateway-suffix-9b9265b.default.dc1.internal.domain", + Name: "gateway-suffix-9b9265b.default.dc1.internal.domain", + }, + "foo.default.default.dc1": { + ID: "foo.default.default.dc1", + Service: "foo", + Datacenter: "dc1", + Partition: "default", + Namespace: "default", + ConnectTimeout: 5000000000, + SNI: "foo.default.dc1.internal.domain", + Name: "foo.default.dc1.internal.domain", + }, + }, + }}, + }, + "HTTPRoute with virtual resolver": { + synthesizer: NewGatewayChainSynthesizer("dc1", "domain", "suffix", &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "gateway", + }), + httpRoutes: []*structs.HTTPRouteConfigEntry{ + { + Kind: structs.HTTPRoute, + Name: "http-route", + Rules: []structs.HTTPRouteRule{{ + Services: []structs.HTTPService{{ + Name: "foo", + }}, + }}, + }, + }, + chain: &structs.CompiledDiscoveryChain{ + ServiceName: "foo", + Namespace: "default", + Datacenter: "dc1", + StartNode: "resolver:foo-2.default.default.dc2", + Nodes: map[string]*structs.DiscoveryGraphNode{ + "resolver:foo-2.default.default.dc2": { + Type: "resolver", + Name: "foo-2.default.default.dc2", + Resolver: &structs.DiscoveryResolver{ + Target: "foo-2.default.default.dc2", + Default: true, + ConnectTimeout: 5000000000, + }, + }, + }, + }, + extra: []*structs.CompiledDiscoveryChain{}, + expectedIngressServices: []structs.IngressService{{ + Name: "gateway-suffix-9b9265b", + Hosts: []string{"*"}, + }}, + expectedDiscoveryChains: []*structs.CompiledDiscoveryChain{{ + ServiceName: "gateway-suffix-9b9265b", + Partition: "default", + Namespace: "default", + Datacenter: "dc1", + Protocol: "http", + StartNode: "router:gateway-suffix-9b9265b.default.default", + Nodes: map[string]*structs.DiscoveryGraphNode{ + "router:gateway-suffix-9b9265b.default.default": { + Type: "router", + Name: "gateway-suffix-9b9265b.default.default", + Routes: []*structs.DiscoveryRoute{{ Definition: &structs.ServiceRoute{ Match: &structs.ServiceRouteMatch{ HTTP: &structs.ServiceRouteHTTPMatch{ @@ -575,12 +653,16 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) { }, }, Destination: &structs.ServiceRouteDestination{ - Service: "gateway-suffix-9b9265b", + Service: "foo", Partition: "default", Namespace: "default", + RequestHeaders: &structs.HTTPHeaderModifiers{ + Add: make(map[string]string), + Set: make(map[string]string), + }, }, }, - NextNode: "resolver:gateway-suffix-9b9265b.default.default.dc1", + NextNode: "resolver:foo-2.default.default.dc2", }}, }, "resolver:foo.default.default.dc1": { @@ -592,6 +674,15 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) { ConnectTimeout: 5000000000, }, }, + "resolver:foo-2.default.default.dc2": { + Type: "resolver", + Name: "foo-2.default.default.dc2", + Resolver: &structs.DiscoveryResolver{ + Target: "foo-2.default.default.dc2", + Default: true, + ConnectTimeout: 5000000000, + }, + }, }, Targets: map[string]*structs.DiscoveryTarget{ "gateway-suffix-9b9265b.default.default.dc1": { @@ -621,6 +712,8 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) { for name, tc := range cases { t.Run(name, func(t *testing.T) { + tc.synthesizer.SetHostname("*") + for _, tcpRoute := range tc.tcpRoutes { tc.synthesizer.AddTCPRoute(*tcpRoute) } @@ -637,3 +730,233 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) { }) } } + +func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) { + t.Parallel() + + cases := map[string]struct { + synthesizer *GatewayChainSynthesizer + route *structs.HTTPRouteConfigEntry + entries []structs.ConfigEntry + expectedDiscoveryChain *structs.CompiledDiscoveryChain + }{ + "HTTP-Route with nested splitters": { + synthesizer: NewGatewayChainSynthesizer("dc1", "domain", "suffix", &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "gateway", + }), + route: &structs.HTTPRouteConfigEntry{ + Kind: structs.HTTPRoute, + Name: "test", + Rules: []structs.HTTPRouteRule{{ + Services: []structs.HTTPService{{ + Name: "splitter-one", + }}, + }}, + }, + entries: []structs.ConfigEntry{ + &structs.ServiceSplitterConfigEntry{ + Kind: structs.ServiceSplitter, + Name: "splitter-one", + Splits: []structs.ServiceSplit{{ + Service: "service-one", + Weight: 50, + }, { + Service: "splitter-two", + Weight: 50, + }}, + }, + &structs.ServiceSplitterConfigEntry{ + Kind: structs.ServiceSplitter, + Name: "splitter-two", + Splits: []structs.ServiceSplit{{ + Service: "service-two", + Weight: 50, + }, { + Service: "service-three", + Weight: 50, + }}, + }, + &structs.ProxyConfigEntry{ + Kind: structs.ProxyConfigGlobal, + Name: "global", + Config: map[string]interface{}{ + "protocol": "http", + }, + }, + }, + expectedDiscoveryChain: &structs.CompiledDiscoveryChain{ + ServiceName: "gateway-suffix-9b9265b", + Namespace: "default", + Partition: "default", + Datacenter: "dc1", + Protocol: "http", + StartNode: "router:gateway-suffix-9b9265b.default.default", + Nodes: map[string]*structs.DiscoveryGraphNode{ + "resolver:service-one.default.default.dc1": { + Type: "resolver", + Name: "service-one.default.default.dc1", + Resolver: &structs.DiscoveryResolver{ + Target: "service-one.default.default.dc1", + Default: true, + ConnectTimeout: 5000000000, + }, + }, + "resolver:service-three.default.default.dc1": { + Type: "resolver", + Name: "service-three.default.default.dc1", + Resolver: &structs.DiscoveryResolver{ + Target: "service-three.default.default.dc1", + Default: true, + ConnectTimeout: 5000000000, + }, + }, + "resolver:service-two.default.default.dc1": { + Type: "resolver", + Name: "service-two.default.default.dc1", + Resolver: &structs.DiscoveryResolver{ + Target: "service-two.default.default.dc1", + Default: true, + ConnectTimeout: 5000000000, + }, + }, + "resolver:splitter-one.default.default.dc1": { + Type: "resolver", + Name: "splitter-one.default.default.dc1", + Resolver: &structs.DiscoveryResolver{ + Target: "splitter-one.default.default.dc1", + Default: true, + ConnectTimeout: 5000000000, + }, + }, + "router:gateway-suffix-9b9265b.default.default": { + Type: "router", + Name: "gateway-suffix-9b9265b.default.default", + Routes: []*structs.DiscoveryRoute{{ + Definition: &structs.ServiceRoute{ + Match: &structs.ServiceRouteMatch{ + HTTP: &structs.ServiceRouteHTTPMatch{ + PathPrefix: "/", + }, + }, + Destination: &structs.ServiceRouteDestination{ + Service: "splitter-one", + Partition: "default", + Namespace: "default", + RequestHeaders: &structs.HTTPHeaderModifiers{ + Add: make(map[string]string), + Set: make(map[string]string), + }, + }, + }, + NextNode: "splitter:splitter-one.default.default", + }}, + }, + "splitter:splitter-one.default.default": { + Type: structs.DiscoveryGraphNodeTypeSplitter, + Name: "splitter-one.default.default", + Splits: []*structs.DiscoverySplit{{ + Definition: &structs.ServiceSplit{ + Weight: 50, + Service: "service-one", + }, + Weight: 50, + NextNode: "resolver:service-one.default.default.dc1", + }, { + Definition: &structs.ServiceSplit{ + Weight: 50, + Service: "service-two", + }, + Weight: 25, + NextNode: "resolver:service-two.default.default.dc1", + }, { + Definition: &structs.ServiceSplit{ + Weight: 50, + Service: "service-three", + }, + Weight: 25, + NextNode: "resolver:service-three.default.default.dc1", + }}, + }, + }, Targets: map[string]*structs.DiscoveryTarget{ + "gateway-suffix-9b9265b.default.default.dc1": { + ID: "gateway-suffix-9b9265b.default.default.dc1", + Service: "gateway-suffix-9b9265b", + Datacenter: "dc1", + Partition: "default", + Namespace: "default", + ConnectTimeout: 5000000000, + SNI: "gateway-suffix-9b9265b.default.dc1.internal.domain", + Name: "gateway-suffix-9b9265b.default.dc1.internal.domain", + }, + "service-one.default.default.dc1": { + ID: "service-one.default.default.dc1", + Service: "service-one", + Datacenter: "dc1", + Partition: "default", + Namespace: "default", + ConnectTimeout: 5000000000, + SNI: "service-one.default.dc1.internal.domain", + Name: "service-one.default.dc1.internal.domain", + }, + "service-three.default.default.dc1": { + ID: "service-three.default.default.dc1", + Service: "service-three", + Datacenter: "dc1", + Partition: "default", + Namespace: "default", + ConnectTimeout: 5000000000, + SNI: "service-three.default.dc1.internal.domain", + Name: "service-three.default.dc1.internal.domain", + }, + "service-two.default.default.dc1": { + ID: "service-two.default.default.dc1", + Service: "service-two", + Datacenter: "dc1", + Partition: "default", + Namespace: "default", + ConnectTimeout: 5000000000, + SNI: "service-two.default.dc1.internal.domain", + Name: "service-two.default.dc1.internal.domain", + }, + "splitter-one.default.default.dc1": { + ID: "splitter-one.default.default.dc1", + Service: "splitter-one", + Datacenter: "dc1", + Partition: "default", + Namespace: "default", + ConnectTimeout: 5000000000, + SNI: "splitter-one.default.dc1.internal.domain", + Name: "splitter-one.default.dc1.internal.domain", + }, + }}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + service := tc.entries[0] + entries := configentry.NewDiscoveryChainSet() + entries.AddEntries(tc.entries...) + compiled, err := Compile(CompileRequest{ + ServiceName: service.GetName(), + EvaluateInNamespace: service.GetEnterpriseMeta().NamespaceOrDefault(), + EvaluateInPartition: service.GetEnterpriseMeta().PartitionOrDefault(), + EvaluateInDatacenter: "dc1", + EvaluateInTrustDomain: "domain", + Entries: entries, + }) + require.NoError(t, err) + + tc.synthesizer.SetHostname("*") + tc.synthesizer.AddHTTPRoute(*tc.route) + + chains := []*structs.CompiledDiscoveryChain{compiled} + _, discoveryChains, err := tc.synthesizer.Synthesize(chains...) + + require.NoError(t, err) + require.Len(t, discoveryChains, 1) + require.Equal(t, tc.expectedDiscoveryChain, discoveryChains[0]) + }) + } +} diff --git a/agent/consul/enterprise_client_oss.go b/agent/consul/enterprise_client_ce.go similarity index 100% rename from agent/consul/enterprise_client_oss.go rename to agent/consul/enterprise_client_ce.go diff --git a/agent/consul/enterprise_config_oss.go b/agent/consul/enterprise_config_ce.go similarity index 100% rename from agent/consul/enterprise_config_oss.go rename to agent/consul/enterprise_config_ce.go diff --git a/agent/consul/enterprise_server_oss.go b/agent/consul/enterprise_server_ce.go similarity index 98% rename from agent/consul/enterprise_server_oss.go rename to agent/consul/enterprise_server_ce.go index d6e07ddd8ce..b0a8129ad5f 100644 --- a/agent/consul/enterprise_server_oss.go +++ b/agent/consul/enterprise_server_ce.go @@ -74,7 +74,7 @@ func (s *Server) validateEnterpriseIntentionPartition(partition string) error { return nil } - // No special handling for wildcard partitions as they are pointless in OSS. + // No special handling for wildcard partitions as they are pointless in CE. return errors.New("Partitions is a Consul Enterprise feature") } @@ -86,7 +86,7 @@ func (s *Server) validateEnterpriseIntentionNamespace(ns string, _ bool) error { return nil } - // No special handling for wildcard namespaces as they are pointless in OSS. + // No special handling for wildcard namespaces as they are pointless in CE. return errors.New("Namespaces is a Consul Enterprise feature") } diff --git a/agent/consul/enterprise_server_oss_test.go b/agent/consul/enterprise_server_ce_test.go similarity index 100% rename from agent/consul/enterprise_server_oss_test.go rename to agent/consul/enterprise_server_ce_test.go diff --git a/agent/consul/fsm/commands_oss.go b/agent/consul/fsm/commands_ce.go similarity index 100% rename from agent/consul/fsm/commands_oss.go rename to agent/consul/fsm/commands_ce.go diff --git a/agent/consul/fsm/commands_oss_test.go b/agent/consul/fsm/commands_ce_test.go similarity index 99% rename from agent/consul/fsm/commands_oss_test.go rename to agent/consul/fsm/commands_ce_test.go index 58608fe1491..ea3196e55ac 100644 --- a/agent/consul/fsm/commands_oss_test.go +++ b/agent/consul/fsm/commands_ce_test.go @@ -8,11 +8,6 @@ import ( "testing" "time" - "github.com/hashicorp/go-raftchunking" - raftchunkingtypes "github.com/hashicorp/go-raftchunking/types" - "github.com/hashicorp/go-uuid" - "github.com/hashicorp/raft" - "github.com/hashicorp/serf/coordinate" "github.com/mitchellh/mapstructure" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -23,6 +18,11 @@ import ( "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/consul/types" + "github.com/hashicorp/go-raftchunking" + raftchunkingtypes "github.com/hashicorp/go-raftchunking/types" + "github.com/hashicorp/go-uuid" + "github.com/hashicorp/raft" + "github.com/hashicorp/serf/coordinate" ) func generateUUID() (ret string) { @@ -1361,9 +1361,10 @@ func TestFSM_ConfigEntry_StatusCAS(t *testing.T) { EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), Status: structs.Status{ Conditions: []structs.Condition{{ - Status: "Foo", + Status: string(api.ConditionStatusTrue), }}, - }} + }, + } // Create a new request. req := &structs.ConfigEntryRequest{ @@ -1392,7 +1393,7 @@ func TestFSM_ConfigEntry_StatusCAS(t *testing.T) { // do a status update entry.Status = structs.Status{ Conditions: []structs.Condition{{ - Status: "Foo", + Status: string(api.ConditionStatusTrue), }}, } req = &structs.ConfigEntryRequest{ @@ -1416,7 +1417,7 @@ func TestFSM_ConfigEntry_StatusCAS(t *testing.T) { entry.RaftIndex.ModifyIndex = 2 conditions := config.(*structs.APIGatewayConfigEntry).Status.Conditions require.Len(t, conditions, 1) - require.Equal(t, "Foo", conditions[0].Status) + require.Equal(t, string(api.ConditionStatusTrue), conditions[0].Status) } // attempt to change the status with a regular update and make sure it's ignored @@ -1445,7 +1446,7 @@ func TestFSM_ConfigEntry_StatusCAS(t *testing.T) { require.NoError(t, err) conditions := config.(*structs.APIGatewayConfigEntry).Status.Conditions require.Len(t, conditions, 1) - require.Equal(t, "Foo", conditions[0].Status) + require.Equal(t, string(api.ConditionStatusTrue), conditions[0].Status) } } diff --git a/agent/consul/fsm/fsm.go b/agent/consul/fsm/fsm.go index c6f40e65af5..afa922143a8 100644 --- a/agent/consul/fsm/fsm.go +++ b/agent/consul/fsm/fsm.go @@ -205,7 +205,7 @@ func (c *FSM) Restore(old io.ReadCloser) error { } default: if msg >= 64 { - return fmt.Errorf("msg type <%d> is a Consul Enterprise log entry. Consul OSS cannot restore it", msg) + return fmt.Errorf("msg type <%d> is a Consul Enterprise log entry. Consul CE cannot restore it", msg) } else { return fmt.Errorf("Unrecognized msg type %d", msg) } diff --git a/agent/consul/fsm/snapshot_oss.go b/agent/consul/fsm/snapshot_ce.go similarity index 99% rename from agent/consul/fsm/snapshot_oss.go rename to agent/consul/fsm/snapshot_ce.go index 3d0cd83439b..8d3d4fe78a6 100644 --- a/agent/consul/fsm/snapshot_oss.go +++ b/agent/consul/fsm/snapshot_ce.go @@ -14,7 +14,7 @@ import ( ) func init() { - registerPersister(persistOSS) + registerPersister(persistCE) registerRestorer(structs.RegisterRequestType, restoreRegistration) registerRestorer(structs.KVSRequestType, restoreKV) @@ -43,7 +43,7 @@ func init() { registerRestorer(structs.PeeringSecretsWriteType, restorePeeringSecrets) } -func persistOSS(s *snapshot, sink raft.SnapshotSink, encoder *codec.Encoder) error { +func persistCE(s *snapshot, sink raft.SnapshotSink, encoder *codec.Encoder) error { if err := s.persistVirtualIPs(sink, encoder); err != nil { return err } diff --git a/agent/consul/fsm/snapshot_oss_test.go b/agent/consul/fsm/snapshot_ce_test.go similarity index 88% rename from agent/consul/fsm/snapshot_oss_test.go rename to agent/consul/fsm/snapshot_ce_test.go index 301db3c2aea..13c5b1dcc40 100644 --- a/agent/consul/fsm/snapshot_oss_test.go +++ b/agent/consul/fsm/snapshot_ce_test.go @@ -19,7 +19,7 @@ func TestRestoreFromEnterprise(t *testing.T) { logger := testutil.Logger(t) fsm, err := New(nil, logger) require.NoError(t, err) - // To verify if a proper message is displayed when Consul OSS tries to + // To verify if a proper message is displayed when Consul CE tries to // unsuccessfully restore entries from a Consul Ent snapshot. buf := bytes.NewBuffer(nil) sink := &MockSink{buf, false} @@ -43,6 +43,6 @@ func TestRestoreFromEnterprise(t *testing.T) { sink.Write([]byte{byte(structs.MessageType(entMockEntry.ID))}) encoder.Encode(entMockEntry) - require.EqualError(t, fsm.Restore(sink), "msg type <65> is a Consul Enterprise log entry. Consul OSS cannot restore it") + require.EqualError(t, fsm.Restore(sink), "msg type <65> is a Consul Enterprise log entry. Consul CE cannot restore it") sink.Cancel() } diff --git a/agent/consul/fsm/snapshot_test.go b/agent/consul/fsm/snapshot_test.go index 97fa87e4348..9ba3ba32f1c 100644 --- a/agent/consul/fsm/snapshot_test.go +++ b/agent/consul/fsm/snapshot_test.go @@ -21,7 +21,7 @@ import ( "github.com/hashicorp/consul/sdk/testutil" ) -func TestFSM_SnapshotRestore_OSS(t *testing.T) { +func TestFSM_SnapshotRestore_CE(t *testing.T) { t.Parallel() logger := testutil.Logger(t) @@ -83,8 +83,8 @@ func TestFSM_SnapshotRestore_OSS(t *testing.T) { policy := &structs.ACLPolicy{ ID: structs.ACLPolicyGlobalManagementID, Name: "global-management", - Description: "Builtin Policy that grants unlimited access", - Rules: structs.ACLPolicyGlobalManagement, + Description: structs.ACLPolicyGlobalManagementDesc, + Rules: structs.ACLPolicyGlobalManagementRules, } policy.SetHash(true) require.NoError(t, fsm.state.ACLPolicySet(1, policy)) @@ -860,7 +860,7 @@ func TestFSM_SnapshotRestore_OSS(t *testing.T) { } } -func TestFSM_BadRestore_OSS(t *testing.T) { +func TestFSM_BadRestore_CE(t *testing.T) { t.Parallel() // Create an FSM with some state. logger := testutil.Logger(t) diff --git a/agent/consul/fsm_data_store.go b/agent/consul/fsm_data_store.go deleted file mode 100644 index 4bbdf6f24d7..00000000000 --- a/agent/consul/fsm_data_store.go +++ /dev/null @@ -1,77 +0,0 @@ -package consul - -import ( - "github.com/hashicorp/consul/acl" - "github.com/hashicorp/consul/agent/consul/fsm" - "github.com/hashicorp/consul/agent/structs" -) - -// FSMDataStore implements the DataStore interface using the Consul server and finite state manager. -type FSMDataStore struct { - server *Server - fsm *fsm.FSM -} - -func NewFSMDataStore(server *Server, fsm *fsm.FSM) *FSMDataStore { - return &FSMDataStore{ - server: server, - fsm: fsm, - } -} - -// GetConfigEntry takes in a kind, name, and meta and returns a configentry and an error from the FSM state -func (f *FSMDataStore) GetConfigEntry(kind string, name string, meta *acl.EnterpriseMeta) (structs.ConfigEntry, error) { - store := f.fsm.State() - - _, entry, err := store.ConfigEntry(nil, kind, name, meta) - if err != nil { - return nil, err - } - return entry, nil -} - -// GetConfigEntriesByKind takes in a kind and returns all instances of that kind of config entry from the FSM state -func (f *FSMDataStore) GetConfigEntriesByKind(kind string) ([]structs.ConfigEntry, error) { - store := f.fsm.State() - - _, entries, err := store.ConfigEntriesByKind(nil, kind, acl.WildcardEnterpriseMeta()) - if err != nil { - return nil, err - } - return entries, nil -} - -// Update takes a config entry and upserts it in the FSM state -func (f *FSMDataStore) Update(entry structs.ConfigEntry) error { - _, err := f.server.leaderRaftApply("ConfigEntry.Apply", structs.ConfigEntryRequestType, &structs.ConfigEntryRequest{ - Op: structs.ConfigEntryUpsertCAS, - Entry: entry, - }) - return err -} - -// UpdateStatus takes a config entry, an error, and updates the status field as needed in the FSM state -func (f *FSMDataStore) UpdateStatus(entry structs.ControlledConfigEntry, err error) error { - if err == nil { - //TODO additional status messages for success? - return nil - } - status := structs.Status{ - Conditions: []structs.Condition{{ - - Status: err.Error() + ": Accepted == false", - }, - }, - } - entry.SetStatus(status) - return f.Update(entry) -} - -// Delete takes a config entry and deletes it from the FSM state -func (f *FSMDataStore) Delete(entry structs.ConfigEntry) error { - _, err := f.server.leaderRaftApply("ConfigEntry.Delete", structs.ConfigEntryRequestType, &structs.ConfigEntryRequest{ - Op: structs.ConfigEntryDelete, - Entry: entry, - }) - return err -} diff --git a/agent/consul/gateway_locator.go b/agent/consul/gateway_locator.go index ab72fdce3d7..756592bce71 100644 --- a/agent/consul/gateway_locator.go +++ b/agent/consul/gateway_locator.go @@ -11,6 +11,7 @@ import ( "github.com/hashicorp/go-hclog" memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/consul/agent/blockingquery" "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/api" @@ -270,7 +271,7 @@ func getRandomItem(items []string) string { } type serverDelegate interface { - blockingQuery(queryOpts blockingQueryOptions, queryMeta blockingQueryResponseMeta, fn queryFn) error + blockingQuery(requestOpts blockingquery.RequestOptions, responseMeta blockingquery.ResponseMeta, query blockingquery.QueryFn) error IsLeader() bool LeaderLastContact() time.Time setDatacenterSupportsFederationStates() diff --git a/agent/consul/gateway_locator_test.go b/agent/consul/gateway_locator_test.go index 8f32acb86af..622445e2261 100644 --- a/agent/consul/gateway_locator_test.go +++ b/agent/consul/gateway_locator_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/hashicorp/consul/agent/blockingquery" "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/api" @@ -472,6 +473,8 @@ func TestGatewayLocator(t *testing.T) { }) } +var _ serverDelegate = (*testServerDelegate)(nil) + type testServerDelegate struct { dcSupportsFederationStates int32 // atomically accessed, at start to prevent alignment issues @@ -493,9 +496,9 @@ func (d *testServerDelegate) datacenterSupportsFederationStates() bool { // This is just enough to exercise the logic. func (d *testServerDelegate) blockingQuery( - queryOpts blockingQueryOptions, - queryMeta blockingQueryResponseMeta, - fn queryFn, + queryOpts blockingquery.RequestOptions, + queryMeta blockingquery.ResponseMeta, + fn blockingquery.QueryFn, ) error { minQueryIndex := queryOpts.GetMinQueryIndex() diff --git a/agent/consul/gateways/controller_gateways.go b/agent/consul/gateways/controller_gateways.go index 53d8c9a888c..da595d915a8 100644 --- a/agent/consul/gateways/controller_gateways.go +++ b/agent/consul/gateways/controller_gateways.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "sync" - "time" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-memdb" @@ -17,6 +16,7 @@ import ( "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/consul/stream" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/api" ) var ( @@ -71,7 +71,7 @@ func (r *apiGatewayReconciler) Reconcile(ctx context.Context, req controller.Req func reconcileEntry[T structs.ControlledConfigEntry](store *state.Store, logger hclog.Logger, ctx context.Context, req controller.Request, reconciler func(ctx context.Context, req controller.Request, store *state.Store, entry T) error, cleaner func(ctx context.Context, req controller.Request, store *state.Store) error) error { _, entry, err := store.ConfigEntry(nil, req.Kind, req.Name, req.Meta) if err != nil { - requestLogger(logger, req).Error("error fetching config entry for reconciliation request", "error", err) + requestLogger(logger, req).Warn("error fetching config entry for reconciliation request", "error", err) return err } @@ -87,12 +87,12 @@ func reconcileEntry[T structs.ControlledConfigEntry](store *state.Store, logger func (r *apiGatewayReconciler) enqueueCertificateReferencedGateways(store *state.Store, _ context.Context, req controller.Request) error { logger := certificateRequestLogger(r.logger, req) - logger.Debug("certificate changed, enqueueing dependent gateways") - defer logger.Debug("finished enqueuing gateways") + logger.Trace("certificate changed, enqueueing dependent gateways") + defer logger.Trace("finished enqueuing gateways") - _, entries, err := store.ConfigEntriesByKind(nil, structs.APIGateway, acl.WildcardEnterpriseMeta()) + _, entries, err := store.ConfigEntriesByKind(nil, structs.APIGateway, wildcardMeta()) if err != nil { - logger.Error("error retrieving api gateways", "error", err) + logger.Warn("error retrieving api gateways", "error", err) return err } @@ -127,12 +127,12 @@ func (r *apiGatewayReconciler) enqueueCertificateReferencedGateways(store *state func (r *apiGatewayReconciler) cleanupBoundGateway(_ context.Context, req controller.Request, store *state.Store) error { logger := gatewayRequestLogger(r.logger, req) - logger.Debug("cleaning up bound gateway") - defer logger.Debug("finished cleaning up bound gateway") + logger.Trace("cleaning up bound gateway") + defer logger.Trace("finished cleaning up bound gateway") routes, err := retrieveAllRoutesFromStore(store) if err != nil { - logger.Error("error retrieving routes", "error", err) + logger.Warn("error retrieving routes", "error", err) return err } @@ -141,9 +141,9 @@ func (r *apiGatewayReconciler) cleanupBoundGateway(_ context.Context, req contro for _, modifiedRoute := range removeGateway(resource, routes...) { routeLogger := routeLogger(logger, modifiedRoute) - routeLogger.Debug("persisting route status") + routeLogger.Trace("persisting route status") if err := r.updater.Update(modifiedRoute); err != nil { - routeLogger.Error("error removing gateway from route", "error", err) + routeLogger.Warn("error removing gateway from route", "error", err) return err } } @@ -156,20 +156,20 @@ func (r *apiGatewayReconciler) cleanupBoundGateway(_ context.Context, req contro func (r *apiGatewayReconciler) reconcileBoundGateway(_ context.Context, req controller.Request, store *state.Store, bound *structs.BoundAPIGatewayConfigEntry) error { logger := gatewayRequestLogger(r.logger, req) - logger.Debug("reconciling bound gateway") - defer logger.Debug("finished reconciling bound gateway") + logger.Trace("reconciling bound gateway") + defer logger.Trace("finished reconciling bound gateway") _, gateway, err := store.ConfigEntry(nil, structs.APIGateway, req.Name, req.Meta) if err != nil { - logger.Error("error retrieving api gateway", "error", err) + logger.Warn("error retrieving api gateway", "error", err) return err } if gateway == nil { // delete the bound gateway - logger.Debug("deleting bound api gateway") + logger.Trace("deleting bound api gateway") if err := r.updater.Delete(bound); err != nil { - logger.Error("error deleting bound api gateway", "error", err) + logger.Warn("error deleting bound api gateway", "error", err) return err } } @@ -183,18 +183,18 @@ func (r *apiGatewayReconciler) reconcileBoundGateway(_ context.Context, req cont func (r *apiGatewayReconciler) cleanupGateway(_ context.Context, req controller.Request, store *state.Store) error { logger := gatewayRequestLogger(r.logger, req) - logger.Debug("cleaning up deleted gateway") - defer logger.Debug("finished cleaning up deleted gateway") + logger.Trace("cleaning up deleted gateway") + defer logger.Trace("finished cleaning up deleted gateway") _, bound, err := store.ConfigEntry(nil, structs.BoundAPIGateway, req.Name, req.Meta) if err != nil { - logger.Error("error retrieving bound api gateway", "error", err) + logger.Warn("error retrieving bound api gateway", "error", err) return err } - logger.Debug("deleting bound api gateway") + logger.Trace("deleting bound api gateway") if err := r.updater.Delete(bound); err != nil { - logger.Error("error deleting bound api gateway", "error", err) + logger.Warn("error deleting bound api gateway", "error", err) return err } @@ -208,12 +208,10 @@ func (r *apiGatewayReconciler) cleanupGateway(_ context.Context, req controller. // referenced this gateway. It then persists any status updates for the gateway, // the modified routes, and updates the bound gateway. func (r *apiGatewayReconciler) reconcileGateway(_ context.Context, req controller.Request, store *state.Store, gateway *structs.APIGatewayConfigEntry) error { - conditions := newGatewayConditionGenerator() - logger := gatewayRequestLogger(r.logger, req) - logger.Debug("started reconciling gateway") - defer logger.Debug("finished reconciling gateway") + logger.Trace("started reconciling gateway") + defer logger.Trace("finished reconciling gateway") updater := structs.NewStatusUpdater(gateway) // we clear out the initial status conditions since we're doing a full update @@ -222,32 +220,45 @@ func (r *apiGatewayReconciler) reconcileGateway(_ context.Context, req controlle routes, err := retrieveAllRoutesFromStore(store) if err != nil { - logger.Error("error retrieving routes", "error", err) + logger.Warn("error retrieving routes", "error", err) return err } // construct the tuple we'll be working on to update state _, bound, err := store.ConfigEntry(nil, structs.BoundAPIGateway, req.Name, req.Meta) if err != nil { - logger.Error("error retrieving bound api gateway", "error", err) + logger.Warn("error retrieving bound api gateway", "error", err) return err } meta := newGatewayMeta(gateway, bound) certificateErrors, err := meta.checkCertificates(store) if err != nil { - logger.Error("error checking gateway certificates", "error", err) + logger.Warn("error checking gateway certificates", "error", err) return err } + // set each listener as having valid certs, then overwrite that status condition + // if there are any certificate errors + meta.eachListener(func(listener *structs.APIGatewayListener, bound *structs.BoundAPIGatewayListener) error { + listenerRef := structs.ResourceReference{ + Kind: structs.APIGateway, + Name: meta.BoundGateway.Name, + SectionName: bound.Name, + EnterpriseMeta: meta.BoundGateway.EnterpriseMeta, + } + updater.SetCondition(validCertificate(listenerRef)) + return nil + }) + for ref, err := range certificateErrors { - updater.SetCondition(conditions.invalidCertificate(ref, err)) + updater.SetCondition(invalidCertificate(ref, err)) } if len(certificateErrors) > 0 { - updater.SetCondition(conditions.invalidCertificates()) + updater.SetCondition(invalidCertificates()) } else { - updater.SetCondition(conditions.gatewayAccepted()) + updater.SetCondition(gatewayAccepted()) } // now we bind all of the routes we can @@ -259,19 +270,19 @@ func (r *apiGatewayReconciler) reconcileGateway(_ context.Context, req controlle // unset the old gateway binding in case it's stale for _, parent := range route.GetParents() { if parent.Kind == gateway.Kind && parent.Name == gateway.Name && parent.EnterpriseMeta.IsSame(&gateway.EnterpriseMeta) { - routeUpdater.RemoveCondition(conditions.routeBound(parent)) + routeUpdater.RemoveCondition(routeBound(parent)) } } // set the status for parents that have bound successfully for _, ref := range boundRefs { - routeUpdater.SetCondition(conditions.routeBound(ref)) + routeUpdater.SetCondition(routeBound(ref)) } // set the status for any parents that have errored trying to // bind for ref, err := range bindErrors { - routeUpdater.SetCondition(conditions.routeUnbound(ref, err)) + routeUpdater.SetCondition(routeUnbound(ref, err)) } // if we've updated any statuses, then store them as needing @@ -286,9 +297,9 @@ func (r *apiGatewayReconciler) reconcileGateway(_ context.Context, req controlle // now check if we need to update the gateway status if modifiedGateway, shouldUpdate := updater.UpdateEntry(); shouldUpdate { - logger.Debug("persisting gateway status") + logger.Trace("persisting gateway status") if err := r.updater.UpdateWithStatus(modifiedGateway); err != nil { - logger.Error("error persisting gateway status", "error", err) + logger.Warn("error persisting gateway status", "error", err) return err } } @@ -296,18 +307,18 @@ func (r *apiGatewayReconciler) reconcileGateway(_ context.Context, req controlle // next update route statuses for _, modifiedRoute := range updatedRoutes { routeLogger := routeLogger(logger, modifiedRoute) - routeLogger.Debug("persisting route status") + routeLogger.Trace("persisting route status") if err := r.updater.UpdateWithStatus(modifiedRoute); err != nil { - routeLogger.Error("error persisting route status", "error", err) + routeLogger.Warn("error persisting route status", "error", err) return err } } // now update the bound state if it changed if bound == nil || !bound.(*structs.BoundAPIGatewayConfigEntry).IsSame(meta.BoundGateway) { - logger.Debug("persisting bound api gateway") + logger.Trace("persisting bound api gateway") if err := r.updater.Update(meta.BoundGateway); err != nil { - logger.Error("error persisting bound api gateway", "error", err) + logger.Warn("error persisting bound api gateway", "error", err) return err } } @@ -320,20 +331,20 @@ func (r *apiGatewayReconciler) reconcileGateway(_ context.Context, req controlle func (r *apiGatewayReconciler) cleanupRoute(_ context.Context, req controller.Request, store *state.Store) error { logger := routeRequestLogger(r.logger, req) - logger.Debug("cleaning up route") - defer logger.Debug("finished cleaning up route") + logger.Trace("cleaning up route") + defer logger.Trace("finished cleaning up route") meta, err := getAllGatewayMeta(store) if err != nil { - logger.Error("error retrieving gateways", "error", err) + logger.Warn("error retrieving gateways", "error", err) return err } for _, modifiedGateway := range removeRoute(requestToResourceRef(req), meta...) { gatewayLogger := gatewayLogger(logger, modifiedGateway.BoundGateway) - gatewayLogger.Debug("persisting bound gateway state") + gatewayLogger.Trace("persisting bound gateway state") if err := r.updater.Update(modifiedGateway.BoundGateway); err != nil { - gatewayLogger.Error("error updating bound api gateway", "error", err) + gatewayLogger.Warn("error updating bound api gateway", "error", err) return err } } @@ -351,16 +362,14 @@ func (r *apiGatewayReconciler) cleanupRoute(_ context.Context, req controller.Re // gateways that now have route conflicts, and updates all statuses and states // as necessary. func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller.Request, store *state.Store, route structs.BoundRoute) error { - conditions := newGatewayConditionGenerator() - logger := routeRequestLogger(r.logger, req) - logger.Debug("reconciling route") - defer logger.Debug("finished reconciling route") + logger.Trace("reconciling route") + defer logger.Trace("finished reconciling route") meta, err := getAllGatewayMeta(store) if err != nil { - logger.Error("error retrieving gateways", "error", err) + logger.Warn("error retrieving gateways", "error", err) return err } @@ -378,9 +387,9 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. modifiedGateway, shouldUpdate := gateway.checkConflicts() if shouldUpdate { gatewayLogger := gatewayLogger(logger, modifiedGateway) - gatewayLogger.Debug("persisting gateway status") + gatewayLogger.Trace("persisting gateway status") if err := r.updater.UpdateWithStatus(modifiedGateway); err != nil { - gatewayLogger.Error("error persisting gateway", "error", err) + gatewayLogger.Warn("error persisting gateway", "error", err) return err } } @@ -388,9 +397,9 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. // next update the route status if modifiedRoute, shouldUpdate := updater.UpdateEntry(); shouldUpdate { - r.logger.Debug("persisting route status") + r.logger.Trace("persisting route status") if err := r.updater.UpdateWithStatus(modifiedRoute); err != nil { - r.logger.Error("error persisting route", "error", err) + r.logger.Warn("error persisting route", "error", err) return err } } @@ -398,9 +407,9 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. // now update all of the bound gateways that have been modified for _, bound := range modifiedGateways { gatewayLogger := gatewayLogger(logger, bound) - gatewayLogger.Debug("persisting bound api gateway") + gatewayLogger.Trace("persisting bound api gateway") if err := r.updater.Update(bound); err != nil { - gatewayLogger.Error("error persisting bound api gateway", "error", err) + gatewayLogger.Warn("error persisting bound api gateway", "error", err) return err } } @@ -409,11 +418,10 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. } var triggerOnce sync.Once - validTargets := true for _, service := range route.GetServiceNames() { _, chainSet, err := store.ReadDiscoveryChainConfigEntries(ws, service.Name, pointerTo(service.EnterpriseMeta)) if err != nil { - logger.Error("error reading discovery chain", "error", err) + logger.Warn("error reading discovery chain", "error", err) return err } @@ -422,11 +430,6 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. r.controller.AddTrigger(req, ws.WatchCtx) }) - if chainSet.IsEmpty() { - updater.SetCondition(conditions.routeInvalidDiscoveryChain(errServiceDoesNotExist)) - continue - } - // make sure that we can actually compile a discovery chain based on this route // the main check is to make sure that all of the protocols align chain, err := discoverychain.Compile(discoverychain.CompileRequest{ @@ -438,60 +441,37 @@ func (r *apiGatewayReconciler) reconcileRoute(_ context.Context, req controller. Entries: chainSet, }) if err != nil { - // we only really need to return the first error for an invalid - // discovery chain, but we still want to set watches on everything in the - // store - if validTargets { - updater.SetCondition(conditions.routeInvalidDiscoveryChain(err)) - validTargets = false - } + updater.SetCondition(routeInvalidDiscoveryChain(err)) continue } if chain.Protocol != string(route.GetProtocol()) { - if validTargets { - updater.SetCondition(conditions.routeInvalidDiscoveryChain(errInvalidProtocol)) - validTargets = false - } + updater.SetCondition(routeInvalidDiscoveryChain(errInvalidProtocol)) continue } - // this makes sure we don't override an already set status - if validTargets { - updater.SetCondition(conditions.routeAccepted()) - } + updater.SetCondition(routeAccepted()) } // if we have no upstream targets, then set the route as invalid // this should already happen in the validation check on write, but // we'll do it here too just in case if len(route.GetServiceNames()) == 0 { - updater.SetCondition(conditions.routeNoUpstreams()) - validTargets = false - } - - if !validTargets { - // we return early, but need to make sure we're removed from all referencing - // gateways and our status is updated properly - updated := []*structs.BoundAPIGatewayConfigEntry{} - for _, modifiedGateway := range removeRoute(requestToResourceRef(req), meta...) { - updated = append(updated, modifiedGateway.BoundGateway) - } - return finalize(updated) + updater.SetCondition(routeNoUpstreams()) } // the route is valid, attempt to bind it to all gateways - r.logger.Debug("binding routes to gateway") + r.logger.Trace("binding routes to gateway") modifiedGateways, boundRefs, bindErrors := bindRoutesToGateways(route, meta...) // set the status of the references that are bound for _, ref := range boundRefs { - updater.SetCondition(conditions.routeBound(ref)) + updater.SetCondition(routeBound(ref)) } // set any binding errors for ref, err := range bindErrors { - updater.SetCondition(conditions.routeUnbound(ref, err)) + updater.SetCondition(routeUnbound(ref, err)) } // set any refs that haven't been bound or explicitly errored @@ -505,7 +485,7 @@ PARENT_LOOP: if _, ok := bindErrors[ref]; ok { continue PARENT_LOOP } - updater.SetCondition(conditions.gatewayNotFound(ref)) + updater.SetCondition(gatewayNotFound(ref)) } return finalize(modifiedGateways) @@ -581,11 +561,11 @@ type gatewayMeta struct { // tuples based on the state coming from the store. Any gateway that does not have // a corresponding bound-api-gateway config entry will be filtered out. func getAllGatewayMeta(store *state.Store) ([]*gatewayMeta, error) { - _, gateways, err := store.ConfigEntriesByKind(nil, structs.APIGateway, acl.WildcardEnterpriseMeta()) + _, gateways, err := store.ConfigEntriesByKind(nil, structs.APIGateway, wildcardMeta()) if err != nil { return nil, err } - _, boundGateways, err := store.ConfigEntriesByKind(nil, structs.BoundAPIGateway, acl.WildcardEnterpriseMeta()) + _, boundGateways, err := store.ConfigEntriesByKind(nil, structs.BoundAPIGateway, wildcardMeta()) if err != nil { return nil, err } @@ -701,6 +681,25 @@ func (g *gatewayMeta) bindRoute(listener *structs.APIGatewayListener, bound *str return false, nil } + // check to make sure we're not binding to an invalid gateway + if !g.Gateway.Status.MatchesConditionStatus(gatewayAccepted()) { + return false, fmt.Errorf("failed to bind route to gateway %s: gateway has not been accepted", g.Gateway.Name) + } + + // check to make sure we're not binding to an invalid route + status := route.GetStatus() + if !status.MatchesConditionStatus(routeAccepted()) { + return false, fmt.Errorf("failed to bind route to gateway %s: route has not been accepted", g.Gateway.Name) + } + + if route, ok := route.(*structs.HTTPRouteConfigEntry); ok { + // check our hostnames + hostnames := route.FilteredHostnames(listener.GetHostname()) + if len(hostnames) == 0 { + return false, fmt.Errorf("failed to bind route to gateway %s: listener %s is does not have any hostnames that match the route", g.Gateway.Name, listener.Name) + } + } + if listener.Protocol == route.GetProtocol() && bound.BindRoute(structs.ResourceReference{ Kind: route.GetKind(), Name: route.GetName(), @@ -752,15 +751,20 @@ func (g *gatewayMeta) checkCertificates(store *state.Store) (map[structs.Resourc if err != nil { return err } + listenerRef := structs.ResourceReference{ + Kind: structs.APIGateway, + Name: g.BoundGateway.Name, + SectionName: bound.Name, + EnterpriseMeta: g.BoundGateway.EnterpriseMeta, + } if certificate == nil { - certificateErrors[ref] = errors.New("certificate not found") + certificateErrors[listenerRef] = fmt.Errorf("certificate %q not found", ref.Name) } else { bound.Certificates = append(bound.Certificates, ref) } } return nil }) - if err != nil { return nil, err } @@ -778,8 +782,6 @@ func (g *gatewayMeta) checkConflicts() (structs.ControlledConfigEntry, bool) { // setConflicts ensures that no TCP listener has more than the one allowed route and // assigns an appropriate status func (g *gatewayMeta) setConflicts(updater *structs.StatusUpdater) { - conditions := newGatewayConditionGenerator() - g.eachListener(func(listener *structs.APIGatewayListener, bound *structs.BoundAPIGatewayListener) error { ref := structs.ResourceReference{ Kind: structs.APIGateway, @@ -790,11 +792,11 @@ func (g *gatewayMeta) setConflicts(updater *structs.StatusUpdater) { switch listener.Protocol { case structs.ListenerProtocolTCP: if len(bound.Routes) > 1 { - updater.SetCondition(conditions.gatewayListenerConflicts(ref)) + updater.SetCondition(gatewayListenerConflicts(ref)) return nil } } - updater.SetCondition(conditions.gatewayListenerNoConflicts(ref)) + updater.SetCondition(gatewayListenerNoConflicts(ref)) return nil }) } @@ -843,151 +845,145 @@ func newGatewayMeta(gateway *structs.APIGatewayConfigEntry, bound structs.Config }).initialize() } -// gatewayConditionGenerator is a simple struct used for isolating -// the status conditions that we generate for our components -type gatewayConditionGenerator struct { - now *time.Time +// gatewayAccepted marks the APIGateway as valid. +func gatewayAccepted() structs.Condition { + return structs.NewGatewayCondition( + api.GatewayConditionAccepted, + api.ConditionStatusTrue, + api.GatewayReasonAccepted, + "gateway is valid", + structs.ResourceReference{}, + ) } -// newGatewayConditionGenerator initializes a status conditions generator -func newGatewayConditionGenerator() *gatewayConditionGenerator { - return &gatewayConditionGenerator{ - now: pointerTo(time.Now().UTC()), - } +// validCertificate returns a condition used when a gateway references a +// certificate that does exist. It takes a ref used to scope the condition +// to a given APIGateway listener. +func validCertificate(ref structs.ResourceReference) structs.Condition { + return structs.NewGatewayCondition( + api.GatewayConditionResolvedRefs, + api.ConditionStatusTrue, + api.GatewayReasonResolvedRefs, + "resolved refs", + ref, + ) } // invalidCertificate returns a condition used when a gateway references a // certificate that does not exist. It takes a ref used to scope the condition // to a given APIGateway listener. -func (g *gatewayConditionGenerator) invalidCertificate(ref structs.ResourceReference, err error) structs.Condition { - return structs.Condition{ - Type: "Accepted", - Status: "False", - Reason: "InvalidCertificate", - Message: err.Error(), - Resource: pointerTo(ref), - LastTransitionTime: g.now, - } +func invalidCertificate(ref structs.ResourceReference, err error) structs.Condition { + return structs.NewGatewayCondition( + api.GatewayConditionResolvedRefs, + api.ConditionStatusFalse, + api.GatewayListenerReasonInvalidCertificateRef, + err.Error(), + ref, + ) } // invalidCertificates is used to set the overall condition of the APIGateway // to invalid due to missing certificates that it references. -func (g *gatewayConditionGenerator) invalidCertificates() structs.Condition { - return structs.Condition{ - Type: "Accepted", - Status: "False", - Reason: "InvalidCertificates", - Message: "gateway references invalid certificates", - LastTransitionTime: g.now, - } +func invalidCertificates() structs.Condition { + return structs.NewGatewayCondition( + api.GatewayConditionAccepted, + api.ConditionStatusFalse, + api.GatewayReasonInvalidCertificates, + "gateway references invalid certificates", + structs.ResourceReference{}, + ) } -// gatewayAccepted marks the APIGateway as valid. -func (g *gatewayConditionGenerator) gatewayAccepted() structs.Condition { - return structs.Condition{ - Type: "Accepted", - Status: "True", - Reason: "Accepted", - Message: "gateway is valid", - LastTransitionTime: g.now, - } +// gatewayListenerNoConflicts marks an APIGateway listener as having no conflicts within its +// bound routes +func gatewayListenerNoConflicts(ref structs.ResourceReference) structs.Condition { + return structs.NewGatewayCondition( + api.GatewayConditionConflicted, + api.ConditionStatusFalse, + api.GatewayReasonNoConflict, + "listener has no route conflicts", + ref, + ) +} + +// gatewayListenerConflicts marks an APIGateway listener as having bound routes that conflict with each other +// and make the listener, therefore invalid +func gatewayListenerConflicts(ref structs.ResourceReference) structs.Condition { + return structs.NewGatewayCondition( + api.GatewayConditionConflicted, + api.ConditionStatusTrue, + api.GatewayReasonRouteConflict, + "TCP-based listeners currently only support binding a single route", + ref, + ) } // routeBound marks a Route as bound to the referenced APIGateway -func (g *gatewayConditionGenerator) routeBound(ref structs.ResourceReference) structs.Condition { - return structs.Condition{ - Type: "Bound", - Status: "True", - Reason: "Bound", - Resource: pointerTo(ref), - Message: "successfully bound route", - LastTransitionTime: g.now, - } +func routeBound(ref structs.ResourceReference) structs.Condition { + return structs.NewRouteCondition( + api.RouteConditionBound, + api.ConditionStatusTrue, + api.RouteReasonBound, + "successfully bound route", + ref, + ) } -// routeAccepted marks the Route as valid -func (g *gatewayConditionGenerator) routeAccepted() structs.Condition { - return structs.Condition{ - Type: "Accepted", - Status: "True", - Reason: "Accepted", - Message: "route is valid", - LastTransitionTime: g.now, - } +// gatewayNotFound marks a Route as having failed to bind to a referenced APIGateway due to +// the Gateway not existing (or having not been reconciled yet) +func gatewayNotFound(ref structs.ResourceReference) structs.Condition { + return structs.NewRouteCondition( + api.RouteConditionBound, + api.ConditionStatusFalse, + api.RouteReasonGatewayNotFound, + "gateway was not found", + ref, + ) } // routeUnbound marks the route as having failed to bind to the referenced APIGateway -func (g *gatewayConditionGenerator) routeUnbound(ref structs.ResourceReference, err error) structs.Condition { - return structs.Condition{ - Type: "Bound", - Status: "False", - Reason: "FailedToBind", - Resource: pointerTo(ref), - Message: err.Error(), - LastTransitionTime: g.now, - } +func routeUnbound(ref structs.ResourceReference, err error) structs.Condition { + return structs.NewRouteCondition( + api.RouteConditionBound, + api.ConditionStatusFalse, + api.RouteReasonFailedToBind, + err.Error(), + ref, + ) +} + +// routeAccepted marks the Route as valid +func routeAccepted() structs.Condition { + return structs.NewRouteCondition( + api.RouteConditionAccepted, + api.ConditionStatusTrue, + api.RouteReasonAccepted, + "route is valid", + structs.ResourceReference{}, + ) } // routeInvalidDiscoveryChain marks the route as invalid due to an error while validating its referenced // discovery chian -func (g *gatewayConditionGenerator) routeInvalidDiscoveryChain(err error) structs.Condition { - return structs.Condition{ - Type: "Accepted", - Status: "False", - Reason: "InvalidDiscoveryChain", - Message: err.Error(), - LastTransitionTime: g.now, - } +func routeInvalidDiscoveryChain(err error) structs.Condition { + return structs.NewRouteCondition( + api.RouteConditionAccepted, + api.ConditionStatusFalse, + api.RouteReasonInvalidDiscoveryChain, + err.Error(), + structs.ResourceReference{}, + ) } // routeNoUpstreams marks the route as invalid because it has no upstreams that it targets -func (g *gatewayConditionGenerator) routeNoUpstreams() structs.Condition { - return structs.Condition{ - Type: "Accepted", - Status: "False", - Reason: "NoUpstreamServicesTargeted", - Message: "route must target at least one upstream service", - LastTransitionTime: g.now, - } -} - -// gatewayListenerConflicts marks an APIGateway listener as having bound routes that conflict with each other -// and make the listener, therefore invalid -func (g *gatewayConditionGenerator) gatewayListenerConflicts(ref structs.ResourceReference) structs.Condition { - return structs.Condition{ - Type: "Conflicted", - Status: "True", - Reason: "RouteConflict", - Resource: pointerTo(ref), - Message: "TCP-based listeners currently only support binding a single route", - LastTransitionTime: g.now, - } -} - -// gatewayListenerNoConflicts marks an APIGateway listener as having no conflicts within its -// bound routes -func (g *gatewayConditionGenerator) gatewayListenerNoConflicts(ref structs.ResourceReference) structs.Condition { - return structs.Condition{ - Type: "Conflicted", - Status: "False", - Reason: "NoConflict", - Resource: pointerTo(ref), - Message: "listener has no route conflicts", - LastTransitionTime: g.now, - } -} - -// gatewayNotFound marks a Route as having failed to bind to a referenced APIGateway due to -// the Gateway not existing (or having not been reconciled yet) -func (g *gatewayConditionGenerator) gatewayNotFound(ref structs.ResourceReference) structs.Condition { - return structs.Condition{ - Type: "Bound", - Status: "False", - Reason: "GatewayNotFound", - Resource: pointerTo(ref), - Message: "gateway was not found", - LastTransitionTime: g.now, - } +func routeNoUpstreams() structs.Condition { + return structs.NewRouteCondition( + api.RouteConditionAccepted, + api.ConditionStatusFalse, + api.RouteReasonNoUpstreamServicesTargeted, + "route must target at least one upstream service", + structs.ResourceReference{}, + ) } // bindRoutesToGateways takes a route variadic number of gateways. @@ -1027,7 +1023,6 @@ func bindRoutesToGateways(route structs.BoundRoute, gateways ...*gatewayMeta) ([ // removeGateway sets the route's status appropriately when the gateway that it's // attempting to bind to does not exist func removeGateway(gateway structs.ResourceReference, entries ...structs.BoundRoute) []structs.ControlledConfigEntry { - conditions := newGatewayConditionGenerator() modified := []structs.ControlledConfigEntry{} for _, route := range entries { @@ -1035,7 +1030,7 @@ func removeGateway(gateway structs.ResourceReference, entries ...structs.BoundRo for _, parent := range route.GetParents() { if parent.Kind == gateway.Kind && parent.Name == gateway.Name && parent.EnterpriseMeta.IsSame(&gateway.EnterpriseMeta) { - updater.SetCondition(conditions.gatewayNotFound(parent)) + updater.SetCondition(gatewayNotFound(parent)) } } @@ -1076,12 +1071,12 @@ func requestToResourceRef(req controller.Request) structs.ResourceReference { // retrieveAllRoutesFromStore retrieves all HTTP and TCP routes from the given store func retrieveAllRoutesFromStore(store *state.Store) ([]structs.BoundRoute, error) { - _, httpRoutes, err := store.ConfigEntriesByKind(nil, structs.HTTPRoute, acl.WildcardEnterpriseMeta()) + _, httpRoutes, err := store.ConfigEntriesByKind(nil, structs.HTTPRoute, wildcardMeta()) if err != nil { return nil, err } - _, tcpRoutes, err := store.ConfigEntriesByKind(nil, structs.TCPRoute, acl.WildcardEnterpriseMeta()) + _, tcpRoutes, err := store.ConfigEntriesByKind(nil, structs.TCPRoute, wildcardMeta()) if err != nil { return nil, err } @@ -1143,3 +1138,9 @@ func routeLogger(logger hclog.Logger, route structs.ConfigEntry) hclog.Logger { meta := route.GetEnterpriseMeta() return logger.With("route.kind", route.GetKind(), "route.name", route.GetName(), "route.namespace", meta.NamespaceOrDefault(), "route.partition", meta.PartitionOrDefault()) } + +func wildcardMeta() *acl.EnterpriseMeta { + meta := acl.WildcardEnterpriseMeta() + meta.OverridePartition(acl.WildcardPartitionName) + return meta +} diff --git a/agent/consul/gateways/controller_gateways_test.go b/agent/consul/gateways/controller_gateways_test.go index 0c47809c763..973cc4cb889 100644 --- a/agent/consul/gateways/controller_gateways_test.go +++ b/agent/consul/gateways/controller_gateways_test.go @@ -8,14 +8,15 @@ import ( "testing" "time" + "github.com/hashicorp/go-hclog" + "github.com/stretchr/testify/require" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/consul/controller" "github.com/hashicorp/consul/agent/consul/fsm" "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/consul/stream" "github.com/hashicorp/consul/agent/structs" - "github.com/hashicorp/go-hclog" - "github.com/stretchr/testify/require" ) func TestBoundAPIGatewayBindRoute(t *testing.T) { @@ -49,6 +50,11 @@ func TestBoundAPIGatewayBindRoute(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, route: &structs.TCPRouteConfigEntry{ @@ -61,6 +67,11 @@ func TestBoundAPIGatewayBindRoute(t *testing.T) { SectionName: "Listener", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, expectedBoundGateway: structs.BoundAPIGatewayConfigEntry{ Kind: structs.BoundAPIGateway, @@ -116,6 +127,11 @@ func TestBoundAPIGatewayBindRoute(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, route: &structs.TCPRouteConfigEntry{ @@ -127,6 +143,11 @@ func TestBoundAPIGatewayBindRoute(t *testing.T) { Name: "Gateway", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, expectedBoundGateway: structs.BoundAPIGatewayConfigEntry{ Kind: structs.BoundAPIGateway, @@ -417,6 +438,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, }, @@ -431,6 +457,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -521,6 +552,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, { @@ -541,6 +577,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, }, @@ -560,6 +601,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -624,6 +670,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, }, @@ -638,6 +689,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener 2", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -691,6 +747,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, }, @@ -705,6 +766,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -770,6 +836,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, }, @@ -784,6 +855,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -843,6 +919,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, { @@ -871,6 +952,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, }, @@ -890,6 +976,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener 2", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -968,6 +1059,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, }, @@ -982,6 +1078,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener 2", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -1027,6 +1128,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, { @@ -1047,6 +1153,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, }, @@ -1061,6 +1172,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener 1", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, &structs.TCPRouteConfigEntry{ Name: "TCP Route 2", @@ -1072,6 +1188,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener 2", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ @@ -1128,6 +1249,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolHTTP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, }, @@ -1142,6 +1268,11 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "Listener", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{}, @@ -1181,6 +1312,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, }, @@ -1195,11 +1331,15 @@ func TestBindRoutesToGateways(t *testing.T) { SectionName: "", }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{ { - Name: "Gateway", Listeners: []structs.BoundAPIGatewayListener{ { @@ -1289,6 +1429,11 @@ func TestBindRoutesToGateways(t *testing.T) { Protocol: structs.ListenerProtocolTCP, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, }, }, @@ -1302,6 +1447,11 @@ func TestBindRoutesToGateways(t *testing.T) { Kind: structs.APIGateway, }, }, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, }, expectedBoundAPIGateways: []*structs.BoundAPIGatewayConfigEntry{}, @@ -1339,7 +1489,6 @@ func TestBindRoutesToGateways(t *testing.T) { } func TestAPIGatewayController(t *testing.T) { - conditions := newGatewayConditionGenerator() defaultMeta := acl.DefaultEnterpriseMeta() for name, tc := range map[string]struct { requests []controller.Request @@ -1367,7 +1516,7 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), + gatewayAccepted(), }, }, }, @@ -1398,7 +1547,7 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeNoUpstreams(), + routeNoUpstreams(), }, }, }, @@ -1424,13 +1573,13 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeNoUpstreams(), + routeNoUpstreams(), }, }, }, }, }, - "tcp-route-no-gateways-invalid-targets": { + "tcp-route-not-accepted-bind": { requests: []controller.Request{{ Kind: structs.TCPRoute, Name: "tcp-route", @@ -1444,6 +1593,27 @@ func TestAPIGatewayController(t *testing.T) { Services: []structs.TCPService{{ Name: "tcp-upstream", }}, + Parents: []structs.ResourceReference{{ + Name: "api-gateway", + EnterpriseMeta: *defaultMeta, + }}, + }, + &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "api-gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.APIGatewayListener{{ + Name: "listener", + Port: 80, + }}, + }, + &structs.BoundAPIGatewayConfigEntry{ + Kind: structs.BoundAPIGateway, + Name: "api-gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.BoundAPIGatewayListener{{ + Name: "listener", + }}, }, }, finalEntries: []structs.ConfigEntry{ @@ -1453,10 +1623,41 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeInvalidDiscoveryChain(errServiceDoesNotExist), + routeAccepted(), + routeUnbound(structs.ResourceReference{ + Name: "api-gateway", + EnterpriseMeta: *defaultMeta, + }, errors.New("failed to bind route to gateway api-gateway: gateway has not been accepted")), + }, + }, + }, + &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "api-gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.APIGatewayListener{{ + Name: "listener", + Port: 80, + }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "api-gateway", + SectionName: "listener", + EnterpriseMeta: *defaultMeta, + }), }, }, }, + &structs.BoundAPIGatewayConfigEntry{ + Kind: structs.BoundAPIGateway, + Name: "api-gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.BoundAPIGatewayListener{{ + Name: "listener", + }}, + }, }, }, "tcp-route-no-gateways-invalid-targets-bad-protocol": { @@ -1488,7 +1689,7 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeInvalidDiscoveryChain(errInvalidProtocol), + routeInvalidDiscoveryChain(errInvalidProtocol), }, }, }, @@ -1527,8 +1728,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.gatewayNotFound(structs.ResourceReference{ + routeAccepted(), + gatewayNotFound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -1581,8 +1782,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.gatewayNotFound(structs.ResourceReference{ + routeAccepted(), + gatewayNotFound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -1645,8 +1846,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeUnbound(structs.ResourceReference{ + routeAccepted(), + routeUnbound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -1714,8 +1915,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.gatewayNotFound(structs.ResourceReference{ + routeAccepted(), + gatewayNotFound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -1748,6 +1949,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, @@ -1763,6 +1969,11 @@ func TestAPIGatewayController(t *testing.T) { Protocol: structs.ListenerProtocolTCP, Port: 22, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + gatewayAccepted(), + }, + }, }, &structs.BoundAPIGatewayConfigEntry{ Kind: structs.BoundAPIGateway, @@ -1790,8 +2001,14 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayAccepted(), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + SectionName: "listener", + }), + validCertificate(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -1806,8 +2023,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -1840,6 +2057,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, @@ -1877,8 +2099,14 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayAccepted(), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + SectionName: "listener", + }), + validCertificate(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -1893,8 +2121,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeUnbound(structs.ResourceReference{ + routeAccepted(), + routeUnbound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -1931,6 +2159,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, &structs.TCPRouteConfigEntry{ Kind: structs.TCPRoute, @@ -1944,6 +2177,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, @@ -1990,8 +2228,14 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), - conditions.gatewayListenerConflicts(structs.ResourceReference{ + gatewayAccepted(), + gatewayListenerConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + SectionName: "listener", + }), + validCertificate(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2006,8 +2250,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2021,8 +2265,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2061,6 +2305,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, &structs.HTTPRouteConfigEntry{ Kind: structs.HTTPRoute, @@ -2076,6 +2325,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, @@ -2123,8 +2377,14 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayAccepted(), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + SectionName: "listener", + }), + validCertificate(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2139,8 +2399,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2154,8 +2414,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2174,6 +2434,14 @@ func TestAPIGatewayController(t *testing.T) { Kind: structs.TCPRoute, Name: "tcp-route", Meta: acl.DefaultEnterpriseMeta(), + }, { + Kind: structs.TCPRoute, + Name: "tcp-route", + Meta: acl.DefaultEnterpriseMeta(), + }, { + Kind: structs.HTTPRoute, + Name: "http-route", + Meta: acl.DefaultEnterpriseMeta(), }, { Kind: structs.HTTPRoute, Name: "http-route", @@ -2256,8 +2524,14 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayAccepted(), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + SectionName: "listener", + }), + validCertificate(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2272,8 +2546,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2287,8 +2561,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeUnbound(structs.ResourceReference{ + routeAccepted(), + routeUnbound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2327,6 +2601,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, &structs.TCPRouteConfigEntry{ Kind: structs.TCPRoute, @@ -2340,6 +2619,11 @@ func TestAPIGatewayController(t *testing.T) { Name: "gateway", EnterpriseMeta: *defaultMeta, }}, + Status: structs.Status{ + Conditions: []structs.Condition{ + routeAccepted(), + }, + }, }, &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, @@ -2400,13 +2684,23 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayAccepted(), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "http-listener", + }), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "tcp-listener", + }), + validCertificate(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", SectionName: "http-listener", }), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + validCertificate(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", SectionName: "tcp-listener", @@ -2420,8 +2714,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2435,8 +2729,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2462,8 +2756,8 @@ func TestAPIGatewayController(t *testing.T) { }}, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2517,7 +2811,7 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayListenerNoConflicts(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", SectionName: "tcp-listener", @@ -2532,7 +2826,7 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), + routeAccepted(), }, }, }, @@ -2551,8 +2845,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2606,7 +2900,7 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayListenerNoConflicts(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", SectionName: "tcp-listener", @@ -2621,7 +2915,7 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeNoUpstreams(), + routeNoUpstreams(), }, }, }, @@ -2649,8 +2943,8 @@ func TestAPIGatewayController(t *testing.T) { }}, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2675,8 +2969,8 @@ func TestAPIGatewayController(t *testing.T) { }}, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeUnbound(structs.ResourceReference{ + routeAccepted(), + routeUnbound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", }, errors.New("foo")), @@ -2706,8 +3000,8 @@ func TestAPIGatewayController(t *testing.T) { }}, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayAccepted(), + gatewayListenerNoConflicts(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", SectionName: "tcp-listener", @@ -2749,8 +3043,13 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayAccepted(), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "http-listener", + }), + validCertificate(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", SectionName: "http-listener", @@ -2764,8 +3063,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeUnbound(structs.ResourceReference{ + routeAccepted(), + routeUnbound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", }, errors.New("failed to bind route tcp-route to gateway gateway with listener ''")), @@ -2778,8 +3077,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", }), @@ -2814,8 +3113,8 @@ func TestAPIGatewayController(t *testing.T) { }}, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", EnterpriseMeta: *defaultMeta, @@ -2850,8 +3149,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.gatewayNotFound(structs.ResourceReference{ + routeAccepted(), + gatewayNotFound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", }), @@ -2891,8 +3190,8 @@ func TestAPIGatewayController(t *testing.T) { }}, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayAccepted(), + gatewayListenerNoConflicts(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", SectionName: "tcp-listener", @@ -2917,8 +3216,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayAccepted(), + gatewayListenerNoConflicts(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", SectionName: "tcp-listener", @@ -2962,8 +3261,8 @@ func TestAPIGatewayController(t *testing.T) { }}, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.routeBound(structs.ResourceReference{ + routeAccepted(), + routeBound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", }), @@ -2978,8 +3277,8 @@ func TestAPIGatewayController(t *testing.T) { EnterpriseMeta: *defaultMeta, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.routeAccepted(), - conditions.gatewayNotFound(structs.ResourceReference{ + routeAccepted(), + gatewayNotFound(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", }), @@ -3028,12 +3327,13 @@ func TestAPIGatewayController(t *testing.T) { }}, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.invalidCertificate(structs.ResourceReference{ - Kind: structs.InlineCertificate, - Name: "certificate", - }, errors.New("certificate not found")), - conditions.invalidCertificates(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + invalidCertificate(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "http-listener", + }, errors.New("certificate \"certificate\" not found")), + invalidCertificates(), + gatewayListenerNoConflicts(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", SectionName: "http-listener", @@ -3096,8 +3396,13 @@ func TestAPIGatewayController(t *testing.T) { }}, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayAccepted(), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "http-listener", + }), + validCertificate(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", SectionName: "http-listener", @@ -3120,6 +3425,345 @@ func TestAPIGatewayController(t *testing.T) { }, }, }, + "all-listeners-valid-certificate-refs": { + requests: []controller.Request{{ + Kind: structs.APIGateway, + Name: "gateway", + Meta: acl.DefaultEnterpriseMeta(), + }}, + initialEntries: []structs.ConfigEntry{ + &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.APIGatewayListener{ + { + Name: "listener-1", + Port: 80, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "cert-1", + }}, + }, + }, + { + Name: "listener-2", + Port: 80, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "cert-2", + }}, + }, + }, + }, + }, + &structs.InlineCertificateConfigEntry{ + Kind: structs.InlineCertificate, + Name: "cert-1", + EnterpriseMeta: *defaultMeta, + }, + &structs.InlineCertificateConfigEntry{ + Kind: structs.InlineCertificate, + Name: "cert-2", + EnterpriseMeta: *defaultMeta, + }, + }, + finalEntries: []structs.ConfigEntry{ + &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.APIGatewayListener{ + { + Name: "listener-1", + Port: 80, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "cert-1", + }}, + }, + }, + { + Name: "listener-2", + Port: 80, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "cert-2", + }}, + }, + }, + }, + Status: structs.Status{ + Conditions: []structs.Condition{ + validCertificate(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "listener-1", + }), + validCertificate(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "listener-2", + }), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "listener-1", + }), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "listener-2", + }), + gatewayAccepted(), + }, + }, + }, + &structs.BoundAPIGatewayConfigEntry{ + Kind: structs.BoundAPIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.BoundAPIGatewayListener{ + { + Name: "listener-1", + Certificates: []structs.ResourceReference{ + { + Kind: structs.InlineCertificate, + Name: "cert-1", + }, + }, + }, + { + Name: "listener-2", + Certificates: []structs.ResourceReference{ + { + Kind: structs.InlineCertificate, + Name: "cert-2", + }, + }, + }, + }, + }, + }, + }, + "all-listeners-invalid-certificates": { + requests: []controller.Request{{ + Kind: structs.APIGateway, + Name: "gateway", + Meta: acl.DefaultEnterpriseMeta(), + }}, + initialEntries: []structs.ConfigEntry{ + &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.APIGatewayListener{ + { + Name: "listener-1", + Port: 80, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "missing certificate", + }}, + }, + }, + { + Name: "listener-2", + Port: 80, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "another missing certificate", + }}, + }, + }, + }, + }, + }, + finalEntries: []structs.ConfigEntry{ + &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.APIGatewayListener{ + { + Name: "listener-1", + Port: 80, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "missing certificate", + }}, + }, + }, + { + Name: "listener-2", + Port: 80, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "another missing certificate", + }}, + }, + }, + }, + Status: structs.Status{ + Conditions: []structs.Condition{ + invalidCertificate(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "listener-1", + }, errors.New("certificate \"missing certificate\" not found")), + invalidCertificate(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "listener-2", + }, errors.New("certificate \"another missing certificate\" not found")), + invalidCertificates(), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "listener-1", + }), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "listener-2", + }), + }, + }, + }, + &structs.BoundAPIGatewayConfigEntry{ + Kind: structs.BoundAPIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.BoundAPIGatewayListener{ + {Name: "listener-1"}, + {Name: "listener-2"}, + }, + }, + }, + }, + "mixed-valid-and-invalid-certificate-refs-for-listeners": { + requests: []controller.Request{{ + Kind: structs.APIGateway, + Name: "gateway", + Meta: acl.DefaultEnterpriseMeta(), + }}, + initialEntries: []structs.ConfigEntry{ + &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.APIGatewayListener{ + { + Name: "invalid-listener", + Port: 80, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "missing certificate", + }}, + }, + }, + { + Name: "valid-listener", + Port: 80, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "certificate", + }}, + }, + }, + }, + }, + &structs.InlineCertificateConfigEntry{ + Kind: structs.InlineCertificate, + Name: "certificate", + EnterpriseMeta: *defaultMeta, + }, + }, + finalEntries: []structs.ConfigEntry{ + &structs.APIGatewayConfigEntry{ + Kind: structs.APIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.APIGatewayListener{ + { + Name: "invalid-listener", + Port: 80, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "missing certificate", + }}, + }, + }, + { + Name: "valid-listener", + Port: 80, + TLS: structs.APIGatewayTLSConfiguration{ + Certificates: []structs.ResourceReference{{ + Kind: structs.InlineCertificate, + Name: "certificate", + }}, + }, + }, + }, + Status: structs.Status{ + Conditions: []structs.Condition{ + invalidCertificate(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "invalid-listener", + }, errors.New("certificate \"missing certificate\" not found")), + validCertificate(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "valid-listener", + }), + + invalidCertificates(), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "valid-listener", + }), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "invalid-listener", + }), + }, + }, + }, + &structs.BoundAPIGatewayConfigEntry{ + Kind: structs.BoundAPIGateway, + Name: "gateway", + EnterpriseMeta: *defaultMeta, + Listeners: []structs.BoundAPIGatewayListener{ + { + Name: "valid-listener", + Certificates: []structs.ResourceReference{ + { + Kind: structs.InlineCertificate, + Name: "certificate", + }, + }, + }, + { + Name: "invalid-listener", + }, + }, + }, + }, + }, "updated-gateway-certificates": { requests: []controller.Request{{ Kind: structs.APIGateway, @@ -3143,12 +3787,12 @@ func TestAPIGatewayController(t *testing.T) { }}, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.invalidCertificate(structs.ResourceReference{ + invalidCertificate(structs.ResourceReference{ Kind: structs.InlineCertificate, Name: "certificate", }, errors.New("certificate not found")), - conditions.invalidCertificates(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + invalidCertificates(), + gatewayListenerNoConflicts(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", SectionName: "http-listener", @@ -3179,8 +3823,13 @@ func TestAPIGatewayController(t *testing.T) { }}, Status: structs.Status{ Conditions: []structs.Condition{ - conditions.gatewayAccepted(), - conditions.gatewayListenerNoConflicts(structs.ResourceReference{ + gatewayAccepted(), + gatewayListenerNoConflicts(structs.ResourceReference{ + Kind: structs.APIGateway, + Name: "gateway", + SectionName: "http-listener", + }), + validCertificate(structs.ResourceReference{ Kind: structs.APIGateway, Name: "gateway", SectionName: "http-listener", @@ -3366,7 +4015,7 @@ func TestAPIGatewayController(t *testing.T) { require.NoError(t, err) ppExpected, err := json.MarshalIndent(expectedStatus, "", " ") require.NoError(t, err) - require.True(t, statusEqual, fmt.Sprintf("statuses are unequal: %+v != %+v", string(ppActual), string(ppExpected))) + require.True(t, statusEqual, fmt.Sprintf("statuses are unequal (actual != expected): %+v != %+v", string(ppActual), string(ppExpected))) if bound, ok := controlled.(*structs.BoundAPIGatewayConfigEntry); ok { ppActual, err := json.MarshalIndent(bound, "", " ") require.NoError(t, err) @@ -3444,12 +4093,15 @@ func (n *noopController) WithWorkers(i int) controller.Controller func (n *noopController) WithQueueFactory(fn func(ctx context.Context, baseBackoff time.Duration, maxBackoff time.Duration) controller.WorkQueue) controller.Controller { return n } + func (n *noopController) AddTrigger(request controller.Request, trigger func(ctx context.Context) error) { n.triggers[request] = struct{}{} } + func (n *noopController) RemoveTrigger(request controller.Request) { delete(n.triggers, request) } + func (n *noopController) Enqueue(requests ...controller.Request) { n.enqueued = append(n.enqueued, requests...) } diff --git a/agent/consul/health_endpoint.go b/agent/consul/health_endpoint.go index 858f3394176..82d130946ba 100644 --- a/agent/consul/health_endpoint.go +++ b/agent/consul/health_endpoint.go @@ -223,16 +223,6 @@ func (h *Health) ServiceNodes(args *structs.ServiceSpecificRequest, reply *struc return err } - // If we're doing a connect or ingress query, we need read access to the service - // we're trying to find proxies for, so check that. - if args.Connect || args.Ingress { - // TODO(acl-error-enhancements) Look for ways to percolate this information up to give any feedback to the user. - if authz.ServiceRead(args.ServiceName, &authzContext) != acl.Allow { - // Just return nil, which will return an empty response (tested) - return nil - } - } - filter, err := bexpr.CreateFilter(args.Filter, nil, reply.Nodes) if err != nil { return err @@ -254,6 +244,17 @@ func (h *Health) ServiceNodes(args *structs.ServiceSpecificRequest, reply *struc return err } + // If we're doing a connect or ingress query, we need read access to the service + // we're trying to find proxies for, so check that. + if args.Connect || args.Ingress { + // TODO(acl-error-enhancements) Look for ways to percolate this information up to give any feedback to the user. + if authz.ServiceRead(args.ServiceName, &authzContext) != acl.Allow { + // Return the index here so that the agent cache does not infinitely loop. + reply.Index = index + return nil + } + } + resolvedNodes := nodes if args.MergeCentralConfig { for _, node := range resolvedNodes { diff --git a/agent/consul/health_endpoint_test.go b/agent/consul/health_endpoint_test.go index 77fd64f4e61..e31263a9871 100644 --- a/agent/consul/health_endpoint_test.go +++ b/agent/consul/health_endpoint_test.go @@ -1124,6 +1124,7 @@ node "foo" { var resp structs.IndexedCheckServiceNodes assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Health.ServiceNodes", &req, &resp)) assert.Len(t, resp.Nodes, 0) + assert.Greater(t, resp.Index, uint64(0)) // List w/ token. This should work since we're requesting "foo", but should // also only contain the proxies with names that adhere to our ACL. @@ -1764,5 +1765,11 @@ func TestHealth_RPC_Filter(t *testing.T) { out = new(structs.IndexedHealthChecks) require.NoError(t, msgpackrpc.CallWithCodec(codec, "Health.ChecksInState", &args, out)) require.Len(t, out.HealthChecks, 1) + + args.State = api.HealthAny + args.Filter = "connect in ServiceTags and v2 in ServiceTags" + out = new(structs.IndexedHealthChecks) + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Health.ChecksInState", &args, out)) + require.Len(t, out.HealthChecks, 1) }) } diff --git a/agent/consul/internal_endpoint.go b/agent/consul/internal_endpoint.go index 2b5e8a1942a..1ede41881ab 100644 --- a/agent/consul/internal_endpoint.go +++ b/agent/consul/internal_endpoint.go @@ -760,7 +760,7 @@ func (m *Internal) EventFire(args *structs.EventFireRequest, } // Set the query meta data - m.srv.setQueryMeta(&reply.QueryMeta, args.Token) + m.srv.SetQueryMeta(&reply.QueryMeta, args.Token) // Add the consul prefix to the event name eventName := userEventName(args.Name) diff --git a/agent/consul/internal_endpoint_test.go b/agent/consul/internal_endpoint_test.go index e0aa941b90e..181de4ed82c 100644 --- a/agent/consul/internal_endpoint_test.go +++ b/agent/consul/internal_endpoint_test.go @@ -1,9 +1,9 @@ package consul import ( + "crypto/rand" "encoding/base64" "fmt" - "math/rand" "os" "strings" "testing" diff --git a/agent/consul/leader.go b/agent/consul/leader.go index d5eb00fbbfe..8c3090f6922 100644 --- a/agent/consul/leader.go +++ b/agent/consul/leader.go @@ -338,6 +338,10 @@ func (s *Server) establishLeadership(ctx context.Context) error { s.startLogVerification(ctx) } + if s.config.Reporting.License.Enabled && s.reportingManager != nil { + s.reportingManager.StartReportingAgent() + } + s.logger.Debug("successfully established leadership", "duration", time.Since(start)) return nil } @@ -357,6 +361,8 @@ func (s *Server) revokeLeadership() { s.revokeEnterpriseLeadership() + s.stopDeferredDeletion() + s.stopFederationStateAntiEntropy() s.stopFederationStateReplication() @@ -374,6 +380,8 @@ func (s *Server) revokeLeadership() { s.resetConsistentReadReady() s.autopilot.DisableReconciliation() + + s.reportingManager.StopReportingAgent() } // initializeACLs is used to setup the ACLs if we are the leader @@ -409,104 +417,31 @@ func (s *Server) initializeACLs(ctx context.Context) error { if s.InPrimaryDatacenter() { s.logger.Info("initializing acls") - // Create/Upgrade the builtin global-management policy - _, policy, err := s.fsm.State().ACLPolicyGetByID(nil, structs.ACLPolicyGlobalManagementID, structs.DefaultEnterpriseMetaInDefaultPartition()) - if err != nil { - return fmt.Errorf("failed to get the builtin global-management policy") - } - if policy == nil || policy.Rules != structs.ACLPolicyGlobalManagement { - newPolicy := structs.ACLPolicy{ - ID: structs.ACLPolicyGlobalManagementID, - Name: "global-management", - Description: "Builtin Policy that grants unlimited access", - Rules: structs.ACLPolicyGlobalManagement, - EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), - } - if policy != nil { - newPolicy.Name = policy.Name - newPolicy.Description = policy.Description - } - - newPolicy.SetHash(true) - - req := structs.ACLPolicyBatchSetRequest{ - Policies: structs.ACLPolicies{&newPolicy}, - } - _, err := s.raftApply(structs.ACLPolicySetRequestType, &req) - if err != nil { - return fmt.Errorf("failed to create global-management policy: %v", err) + // Create/Upgrade the builtin policies + for _, policy := range structs.ACLBuiltinPolicies { + if err := s.writeBuiltinACLPolicy(policy); err != nil { + return err } - s.logger.Info("Created ACL 'global-management' policy") } // Check for configured initial management token. if initialManagement := s.config.ACLInitialManagementToken; len(initialManagement) > 0 { - state := s.fsm.State() - if _, err := uuid.ParseUUID(initialManagement); err != nil { - s.logger.Warn("Configuring a non-UUID initial management token is deprecated") - } - - _, token, err := state.ACLTokenGetBySecret(nil, initialManagement, nil) + err := s.initializeManagementToken("Initial Management Token", initialManagement) if err != nil { - return fmt.Errorf("failed to get initial management token: %v", err) + return fmt.Errorf("failed to initialize initial management token: %w", err) } - // Ignoring expiration times to avoid an insertion collision. - if token == nil { - accessor, err := lib.GenerateUUID(s.checkTokenUUID) - if err != nil { - return fmt.Errorf("failed to generate the accessor ID for the initial management token: %v", err) - } - - token := structs.ACLToken{ - AccessorID: accessor, - SecretID: initialManagement, - Description: "Initial Management Token", - Policies: []structs.ACLTokenPolicyLink{ - { - ID: structs.ACLPolicyGlobalManagementID, - }, - }, - CreateTime: time.Now(), - Local: false, - EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), - } - - token.SetHash(true) - - done := false - if canBootstrap, _, err := state.CanBootstrapACLToken(); err == nil && canBootstrap { - req := structs.ACLTokenBootstrapRequest{ - Token: token, - ResetIndex: 0, - } - if _, err := s.raftApply(structs.ACLBootstrapRequestType, &req); err == nil { - s.logger.Info("Bootstrapped ACL initial management token from configuration") - done = true - } else { - if err.Error() != structs.ACLBootstrapNotAllowedErr.Error() && - err.Error() != structs.ACLBootstrapInvalidResetIndexErr.Error() { - return fmt.Errorf("failed to bootstrap initial management token: %v", err) - } - } - } - - if !done { - // either we didn't attempt to or setting the token with a bootstrap request failed. - req := structs.ACLTokenBatchSetRequest{ - Tokens: structs.ACLTokens{&token}, - CAS: false, - } - if _, err := s.raftApply(structs.ACLTokenSetRequestType, &req); err != nil { - return fmt.Errorf("failed to create initial management token: %v", err) - } + } - s.logger.Info("Created ACL initial management token from configuration") - } + // Check for configured management token from HCP. It MUST NOT override the user-provided initial management token. + if hcpManagement := s.config.Cloud.ManagementToken; len(hcpManagement) > 0 { + err := s.initializeManagementToken("HCP Management Token", hcpManagement) + if err != nil { + return fmt.Errorf("failed to initialize HCP management token: %w", err) } } // Insert the anonymous token if it does not exist. - if err := s.InsertAnonymousToken(); err != nil { + if err := s.insertAnonymousToken(); err != nil { return err } } else { @@ -522,7 +457,7 @@ func (s *Server) initializeACLs(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to generate the secret ID for the server management token: %w", err) } - if err := s.setSystemMetadataKey(structs.ServerManagementTokenAccessorID, secretID); err != nil { + if err := s.SetSystemMetadataKey(structs.ServerManagementTokenAccessorID, secretID); err != nil { return fmt.Errorf("failed to persist server management token: %w", err) } @@ -531,7 +466,104 @@ func (s *Server) initializeACLs(ctx context.Context) error { return nil } -func (s *Server) InsertAnonymousToken() error { +// writeBuiltinACLPolicy writes the given built-in policy to Raft if the policy +// is not found or if the policy rules have been changed. The name and +// description of a built-in policy are user-editable and must be preserved +// during updates. This function must only be called in a primary datacenter. +func (s *Server) writeBuiltinACLPolicy(newPolicy structs.ACLPolicy) error { + _, policy, err := s.fsm.State().ACLPolicyGetByID(nil, newPolicy.ID, structs.DefaultEnterpriseMetaInDefaultPartition()) + if err != nil { + return fmt.Errorf("failed to get the builtin %s policy", newPolicy.Name) + } + if policy == nil || policy.Rules != newPolicy.Rules { + if policy != nil { + newPolicy.Name = policy.Name + newPolicy.Description = policy.Description + } + + newPolicy.EnterpriseMeta = *structs.DefaultEnterpriseMetaInDefaultPartition() + newPolicy.SetHash(true) + + req := structs.ACLPolicyBatchSetRequest{ + Policies: structs.ACLPolicies{&newPolicy}, + } + _, err := s.raftApply(structs.ACLPolicySetRequestType, &req) + if err != nil { + return fmt.Errorf("failed to create %s policy: %v", newPolicy.Name, err) + } + s.logger.Info(fmt.Sprintf("Created ACL '%s' policy", newPolicy.Name)) + } + return nil +} + +func (s *Server) initializeManagementToken(name, secretID string) error { + state := s.fsm.State() + if _, err := uuid.ParseUUID(secretID); err != nil { + s.logger.Warn("Configuring a non-UUID management token is deprecated") + } + + _, token, err := state.ACLTokenGetBySecret(nil, secretID, nil) + if err != nil { + return fmt.Errorf("failed to get %s: %v", name, err) + } + // Ignoring expiration times to avoid an insertion collision. + if token == nil { + accessor, err := lib.GenerateUUID(s.checkTokenUUID) + if err != nil { + return fmt.Errorf("failed to generate the accessor ID for %s: %v", name, err) + } + + token := structs.ACLToken{ + AccessorID: accessor, + SecretID: secretID, + Description: name, + Policies: []structs.ACLTokenPolicyLink{ + { + ID: structs.ACLPolicyGlobalManagementID, + }, + }, + CreateTime: time.Now(), + Local: false, + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + } + + token.SetHash(true) + + done := false + if canBootstrap, _, err := state.CanBootstrapACLToken(); err == nil && canBootstrap { + req := structs.ACLTokenBootstrapRequest{ + Token: token, + ResetIndex: 0, + } + if _, err := s.raftApply(structs.ACLBootstrapRequestType, &req); err == nil { + s.logger.Info("Bootstrapped ACL token from configuration", "description", name) + done = true + } else { + if err.Error() != structs.ACLBootstrapNotAllowedErr.Error() && + err.Error() != structs.ACLBootstrapInvalidResetIndexErr.Error() { + return fmt.Errorf("failed to bootstrap with %s: %v", name, err) + } + } + } + + if !done { + // either we didn't attempt to or setting the token with a bootstrap request failed. + req := structs.ACLTokenBatchSetRequest{ + Tokens: structs.ACLTokens{&token}, + CAS: false, + } + if _, err := s.raftApply(structs.ACLTokenSetRequestType, &req); err != nil { + return fmt.Errorf("failed to create %s: %v", name, err) + } + + s.logger.Info("Created ACL token from configuration", "description", name) + } + } + + return nil +} + +func (s *Server) insertAnonymousToken() error { state := s.fsm.State() _, token, err := state.ACLTokenGetBySecret(nil, anonymousToken, nil) if err != nil { diff --git a/agent/consul/leader_oss_test.go b/agent/consul/leader_ce_test.go similarity index 100% rename from agent/consul/leader_oss_test.go rename to agent/consul/leader_ce_test.go diff --git a/agent/consul/leader_connect.go b/agent/consul/leader_connect.go index 0c57bc13a7f..7166ad75cd6 100644 --- a/agent/consul/leader_connect.go +++ b/agent/consul/leader_connect.go @@ -204,7 +204,7 @@ func (s *Server) setVirtualIPFlags() (bool, error) { } func (s *Server) setVirtualIPVersionFlag() (bool, error) { - val, err := s.getSystemMetadata(structs.SystemMetadataVirtualIPsEnabled) + val, err := s.GetSystemMetadata(structs.SystemMetadataVirtualIPsEnabled) if err != nil { return false, err } @@ -217,7 +217,7 @@ func (s *Server) setVirtualIPVersionFlag() (bool, error) { minVirtualIPVersion.String()) } - if err := s.setSystemMetadataKey(structs.SystemMetadataVirtualIPsEnabled, "true"); err != nil { + if err := s.SetSystemMetadataKey(structs.SystemMetadataVirtualIPsEnabled, "true"); err != nil { return false, nil } @@ -225,7 +225,7 @@ func (s *Server) setVirtualIPVersionFlag() (bool, error) { } func (s *Server) setVirtualIPTerminatingGatewayVersionFlag() (bool, error) { - val, err := s.getSystemMetadata(structs.SystemMetadataTermGatewayVirtualIPsEnabled) + val, err := s.GetSystemMetadata(structs.SystemMetadataTermGatewayVirtualIPsEnabled) if err != nil { return false, err } @@ -238,7 +238,7 @@ func (s *Server) setVirtualIPTerminatingGatewayVersionFlag() (bool, error) { minVirtualIPTerminatingGatewayVersion.String()) } - if err := s.setSystemMetadataKey(structs.SystemMetadataTermGatewayVirtualIPsEnabled, "true"); err != nil { + if err := s.SetSystemMetadataKey(structs.SystemMetadataTermGatewayVirtualIPsEnabled, "true"); err != nil { return false, nil } diff --git a/agent/consul/leader_connect_ca.go b/agent/consul/leader_connect_ca.go index 008289c947e..39defb752b0 100644 --- a/agent/consul/leader_connect_ca.go +++ b/agent/consul/leader_connect_ca.go @@ -12,7 +12,7 @@ import ( "time" "github.com/hashicorp/go-hclog" - uuid "github.com/hashicorp/go-uuid" + "github.com/hashicorp/go-uuid" "golang.org/x/time/rate" "github.com/hashicorp/consul/acl" @@ -272,7 +272,7 @@ func newCARoot(pemValue, provider, clusterID string) (*structs.CARoot, error) { ExternalTrustDomain: clusterID, NotBefore: primaryCert.NotBefore, NotAfter: primaryCert.NotAfter, - RootCert: pemValue, + RootCert: lib.EnsureTrailingNewline(pemValue), PrivateKeyType: keyType, PrivateKeyBits: keyBits, Active: true, @@ -754,7 +754,9 @@ func shouldPersistNewRootAndConfig(newActiveRoot *structs.CARoot, oldConfig, new if newConfig == nil { return false } - return newConfig.Provider == oldConfig.Provider && reflect.DeepEqual(newConfig.Config, oldConfig.Config) + + // Do not persist if the new provider and config are the same as the old + return !(newConfig.Provider == oldConfig.Provider && reflect.DeepEqual(newConfig.Config, oldConfig.Config)) } func (c *CAManager) UpdateConfiguration(args *structs.CARequest) (reterr error) { @@ -887,6 +889,23 @@ func (c *CAManager) primaryUpdateRootCA(newProvider ca.Provider, args *structs.C return err } + // TODO: https://github.com/hashicorp/consul/issues/12386 + intermediate, err := newProvider.ActiveIntermediate() + if err != nil { + return fmt.Errorf("error fetching active intermediate: %w", err) + } + if intermediate == "" { + intermediate, err = newProvider.GenerateIntermediate() + if err != nil { + return fmt.Errorf("error generating intermediate: %w", err) + } + } + if intermediate != newRootPEM { + if err := setLeafSigningCert(newActiveRoot, intermediate); err != nil { + return err + } + } + // See if the provider needs to persist any state along with the config pState, err := newProvider.State() if err != nil { @@ -970,19 +989,9 @@ func (c *CAManager) primaryUpdateRootCA(newProvider ca.Provider, args *structs.C } // Add the cross signed cert to the new CA's intermediates (to be attached - // to leaf certs). - newActiveRoot.IntermediateCerts = []string{xcCert} - } - } - - // TODO: https://github.com/hashicorp/consul/issues/12386 - intermediate, err := newProvider.GenerateIntermediate() - if err != nil { - return err - } - if intermediate != newRootPEM { - if err := setLeafSigningCert(newActiveRoot, intermediate); err != nil { - return err + // to leaf certs). We do not want it to be the last cert if there are any + // existing intermediate certs so we push to the front. + newActiveRoot.IntermediateCerts = append([]string{xcCert}, newActiveRoot.IntermediateCerts...) } } diff --git a/agent/consul/leader_connect_ca_test.go b/agent/consul/leader_connect_ca_test.go index 7e84a87b19b..1e4e4d2af96 100644 --- a/agent/consul/leader_connect_ca_test.go +++ b/agent/consul/leader_connect_ca_test.go @@ -25,7 +25,7 @@ import ( "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" - ca "github.com/hashicorp/consul/agent/connect/ca" + "github.com/hashicorp/consul/agent/connect/ca" "github.com/hashicorp/consul/agent/consul/fsm" "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/structs" @@ -612,39 +612,72 @@ func TestCAManager_UpdateConfiguration_Vault_Primary(t *testing.T) { _, origRoot, err := s1.fsm.State().CARootActive(nil) require.NoError(t, err) require.Len(t, origRoot.IntermediateCerts, 1) + origRoot.CreateIndex = s1.caManager.providerRoot.CreateIndex + origRoot.ModifyIndex = s1.caManager.providerRoot.ModifyIndex + require.Equal(t, s1.caManager.providerRoot, origRoot) cert, err := connect.ParseCert(s1.caManager.getLeafSigningCertFromRoot(origRoot)) require.NoError(t, err) require.Equal(t, connect.HexString(cert.SubjectKeyId), origRoot.SigningKeyID) - vaultToken2 := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{ - RootPath: "pki-root-2", - IntermediatePath: "pki-intermediate-2", - ConsulManaged: true, + t.Run("update config without changing root", func(t *testing.T) { + err = s1.caManager.UpdateConfiguration(&structs.CARequest{ + Config: &structs.CAConfiguration{ + Provider: "vault", + Config: map[string]interface{}{ + "Address": vault.Addr, + "Token": vaultToken, + "RootPKIPath": "pki-root/", + "IntermediatePKIPath": "pki-intermediate/", + "CSRMaxPerSecond": 100, + }, + }, + }) + require.NoError(t, err) + _, sameRoot, err := s1.fsm.State().CARootActive(nil) + require.NoError(t, err) + require.Len(t, sameRoot.IntermediateCerts, 1) + sameRoot.CreateIndex = s1.caManager.providerRoot.CreateIndex + sameRoot.ModifyIndex = s1.caManager.providerRoot.ModifyIndex + + cert, err := connect.ParseCert(s1.caManager.getLeafSigningCertFromRoot(sameRoot)) + require.NoError(t, err) + require.Equal(t, connect.HexString(cert.SubjectKeyId), sameRoot.SigningKeyID) + + require.Equal(t, origRoot, sameRoot) + require.Equal(t, sameRoot, s1.caManager.providerRoot) }) - err = s1.caManager.UpdateConfiguration(&structs.CARequest{ - Config: &structs.CAConfiguration{ - Provider: "vault", - Config: map[string]interface{}{ - "Address": vault.Addr, - "Token": vaultToken2, - "RootPKIPath": "pki-root-2/", - "IntermediatePKIPath": "pki-intermediate-2/", + t.Run("update config and change root", func(t *testing.T) { + vaultToken2 := ca.CreateVaultTokenWithAttrs(t, vault.Client(), &ca.VaultTokenAttributes{ + RootPath: "pki-root-2", + IntermediatePath: "pki-intermediate-2", + ConsulManaged: true, + }) + + err = s1.caManager.UpdateConfiguration(&structs.CARequest{ + Config: &structs.CAConfiguration{ + Provider: "vault", + Config: map[string]interface{}{ + "Address": vault.Addr, + "Token": vaultToken2, + "RootPKIPath": "pki-root-2/", + "IntermediatePKIPath": "pki-intermediate-2/", + }, }, - }, - }) - require.NoError(t, err) + }) + require.NoError(t, err) - _, newRoot, err := s1.fsm.State().CARootActive(nil) - require.NoError(t, err) - require.Len(t, newRoot.IntermediateCerts, 2, - "expected one cross-sign cert and one local leaf sign cert") - require.NotEqual(t, origRoot.ID, newRoot.ID) + _, newRoot, err := s1.fsm.State().CARootActive(nil) + require.NoError(t, err) + require.Len(t, newRoot.IntermediateCerts, 2, + "expected one cross-sign cert and one local leaf sign cert") + require.NotEqual(t, origRoot.ID, newRoot.ID) - cert, err = connect.ParseCert(s1.caManager.getLeafSigningCertFromRoot(newRoot)) - require.NoError(t, err) - require.Equal(t, connect.HexString(cert.SubjectKeyId), newRoot.SigningKeyID) + cert, err = connect.ParseCert(s1.caManager.getLeafSigningCertFromRoot(newRoot)) + require.NoError(t, err) + require.Equal(t, connect.HexString(cert.SubjectKeyId), newRoot.SigningKeyID) + }) } func TestCAManager_Initialize_Vault_WithIntermediateAsPrimaryCA(t *testing.T) { diff --git a/agent/consul/leader_intentions.go b/agent/consul/leader_intentions.go index 9adc26795de..2894acd5846 100644 --- a/agent/consul/leader_intentions.go +++ b/agent/consul/leader_intentions.go @@ -22,7 +22,7 @@ func (s *Server) startIntentionConfigEntryMigration(ctx context.Context) error { // Check for the system metadata first, as that's the most trustworthy in // both the primary and secondaries. - intentionFormat, err := s.getSystemMetadata(structs.SystemMetadataIntentionFormatKey) + intentionFormat, err := s.GetSystemMetadata(structs.SystemMetadataIntentionFormatKey) if err != nil { return err } @@ -240,7 +240,7 @@ func (s *Server) legacyIntentionMigrationInSecondaryDC(ctx context.Context) erro // error. for { // Check for the system metadata first, as that's the most trustworthy. - intentionFormat, err := s.getSystemMetadata(structs.SystemMetadataIntentionFormatKey) + intentionFormat, err := s.GetSystemMetadata(structs.SystemMetadataIntentionFormatKey) if err != nil { return err } diff --git a/agent/consul/leader_intentions_oss.go b/agent/consul/leader_intentions_ce.go similarity index 96% rename from agent/consul/leader_intentions_oss.go rename to agent/consul/leader_intentions_ce.go index db9c742bd1d..6913b6a90a3 100644 --- a/agent/consul/leader_intentions_oss.go +++ b/agent/consul/leader_intentions_ce.go @@ -10,7 +10,7 @@ import ( ) func migrateIntentionsToConfigEntries(ixns structs.Intentions) []*structs.ServiceIntentionsConfigEntry { - // Remove any intention in OSS that happened to have used a non-default + // Remove any intention in CE that happened to have used a non-default // namespace. // // The one exception is that if we find wildcards namespaces we "upgrade" @@ -53,7 +53,7 @@ func migrateIntentionsToConfigEntries(ixns structs.Intentions) []*structs.Servic } retained[name] = struct{}{} output = append(output, ixn) - continue // a-ok for OSS + continue // a-ok for CE } // If anything is wildcarded, attempt to reify it as "default". diff --git a/agent/consul/leader_intentions_oss_test.go b/agent/consul/leader_intentions_ce_test.go similarity index 100% rename from agent/consul/leader_intentions_oss_test.go rename to agent/consul/leader_intentions_ce_test.go diff --git a/agent/consul/leader_intentions_test.go b/agent/consul/leader_intentions_test.go index e0dcb8b3d10..89dbc9d6256 100644 --- a/agent/consul/leader_intentions_test.go +++ b/agent/consul/leader_intentions_test.go @@ -520,7 +520,7 @@ func TestLeader_LegacyIntentionMigration(t *testing.T) { // Wait until the migration routine is complete. retry.Run(t, func(r *retry.R) { - intentionFormat, err := s1.getSystemMetadata(structs.SystemMetadataIntentionFormatKey) + intentionFormat, err := s1.GetSystemMetadata(structs.SystemMetadataIntentionFormatKey) require.NoError(r, err) if intentionFormat != structs.SystemMetadataIntentionFormatConfigValue { r.Fatal("intention migration is not yet complete") diff --git a/agent/consul/leader_peering_test.go b/agent/consul/leader_peering_test.go index 4be6326c095..757fc87eebc 100644 --- a/agent/consul/leader_peering_test.go +++ b/agent/consul/leader_peering_test.go @@ -468,6 +468,30 @@ func TestLeader_PeeringSync_Lifecycle_ServerDeletion(t *testing.T) { require.NoError(r, err) require.Equal(r, pbpeering.PeeringState_TERMINATED, peering.State) }) + + // Re-establishing a peering terminated by the acceptor should be possible + // without needing to delete the terminated peering first. + ctx, cancel = context.WithTimeout(context.Background(), 3*time.Second) + t.Cleanup(cancel) + + req = pbpeering.GenerateTokenRequest{ + PeerName: "my-peer-dialer", + } + resp, err = peeringClient.GenerateToken(ctx, &req) + require.NoError(t, err) + + tokenJSON, err = base64.StdEncoding.DecodeString(resp.PeeringToken) + require.NoError(t, err) + + token = structs.PeeringToken{} + require.NoError(t, json.Unmarshal(tokenJSON, &token)) + + establishReq = pbpeering.EstablishRequest{ + PeerName: "my-peer-acceptor", + PeeringToken: resp.PeeringToken, + } + _, err = dialerClient.Establish(ctx, &establishReq) + require.NoError(t, err) } func TestLeader_PeeringSync_FailsForTLSError(t *testing.T) { @@ -478,7 +502,7 @@ func TestLeader_PeeringSync_FailsForTLSError(t *testing.T) { t.Run("server-name-validation", func(t *testing.T) { testLeader_PeeringSync_failsForTLSError(t, func(token *structs.PeeringToken) { token.ServerName = "wrong.name" - }, `transport: authentication handshake failed: x509: certificate is valid for server.dc1.peering.11111111-2222-3333-4444-555555555555.consul, not wrong.name`) + }, `transport: authentication handshake failed: tls: failed to verify certificate: x509: certificate is valid for server.dc1.peering.11111111-2222-3333-4444-555555555555.consul, not wrong.name`) }) t.Run("bad-ca-roots", func(t *testing.T) { wrongRoot, err := os.ReadFile("../../test/client_certs/rootca.crt") @@ -486,7 +510,7 @@ func TestLeader_PeeringSync_FailsForTLSError(t *testing.T) { testLeader_PeeringSync_failsForTLSError(t, func(token *structs.PeeringToken) { token.CA = []string{string(wrongRoot)} - }, `transport: authentication handshake failed: x509: certificate signed by unknown authority`) + }, `transport: authentication handshake failed: tls: failed to verify certificate: x509: certificate signed by unknown authority`) }) } diff --git a/agent/consul/leader_test.go b/agent/consul/leader_test.go index e8bcb39a653..80af470e4f3 100644 --- a/agent/consul/leader_test.go +++ b/agent/consul/leader_test.go @@ -1258,49 +1258,82 @@ func TestLeader_ACL_Initialization(t *testing.T) { tests := []struct { name string - build string initialManagement string - bootstrap bool + hcpManagement string + + // canBootstrap tracks whether the ACL system can be bootstrapped + // after the leader initializes ACLs. Bootstrapping is the act + // of persisting a token with the Global Management policy. + canBootstrap bool }{ - {"old version, no initial management", "0.8.0", "", true}, - {"old version, initial management", "0.8.0", "root", false}, - {"new version, no initial management", "0.9.1", "", true}, - {"new version, initial management", "0.9.1", "root", false}, + { + name: "bootstrap from initial management", + initialManagement: "c9ad785a-420d-470d-9b4d-6d9f084bfa87", + hcpManagement: "", + canBootstrap: false, + }, + { + name: "bootstrap from hcp management", + initialManagement: "", + hcpManagement: "924bc0e1-a41b-4f3a-b5e8-0899502fc50e", + canBootstrap: false, + }, + { + name: "bootstrap with both", + initialManagement: "c9ad785a-420d-470d-9b4d-6d9f084bfa87", + hcpManagement: "924bc0e1-a41b-4f3a-b5e8-0899502fc50e", + canBootstrap: false, + }, + { + name: "did not bootstrap", + initialManagement: "", + hcpManagement: "", + canBootstrap: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { conf := func(c *Config) { - c.Build = tt.build c.Bootstrap = true c.Datacenter = "dc1" c.PrimaryDatacenter = "dc1" c.ACLsEnabled = true c.ACLInitialManagementToken = tt.initialManagement + c.Cloud.ManagementToken = tt.hcpManagement } - dir1, s1 := testServerWithConfig(t, conf) - defer os.RemoveAll(dir1) - defer s1.Shutdown() + _, s1 := testServerWithConfig(t, conf) testrpc.WaitForTestAgent(t, s1.RPC, "dc1") + // check that the builtin policies were created + for _, builtinPolicy := range structs.ACLBuiltinPolicies { + _, policy, err := s1.fsm.State().ACLPolicyGetByID(nil, builtinPolicy.ID, nil) + require.NoError(t, err) + require.NotNil(t, policy) + } + if tt.initialManagement != "" { _, initialManagement, err := s1.fsm.State().ACLTokenGetBySecret(nil, tt.initialManagement, nil) require.NoError(t, err) require.NotNil(t, initialManagement) + require.Equal(t, tt.initialManagement, initialManagement.SecretID) } - _, anon, err := s1.fsm.State().ACLTokenGetBySecret(nil, anonymousToken, nil) - require.NoError(t, err) - require.NotNil(t, anon) + if tt.hcpManagement != "" { + _, hcpManagement, err := s1.fsm.State().ACLTokenGetBySecret(nil, tt.hcpManagement, nil) + require.NoError(t, err) + require.NotNil(t, hcpManagement) + require.Equal(t, tt.hcpManagement, hcpManagement.SecretID) + } canBootstrap, _, err := s1.fsm.State().CanBootstrapACLToken() require.NoError(t, err) - require.Equal(t, tt.bootstrap, canBootstrap) + require.Equal(t, tt.canBootstrap, canBootstrap) - _, policy, err := s1.fsm.State().ACLPolicyGetByID(nil, structs.ACLPolicyGlobalManagementID, nil) + _, anon, err := s1.fsm.State().ACLTokenGetBySecret(nil, anonymousToken, nil) require.NoError(t, err) - require.NotNil(t, policy) + require.NotNil(t, anon) - serverToken, err := s1.getSystemMetadata(structs.ServerManagementTokenAccessorID) + serverToken, err := s1.GetSystemMetadata(structs.ServerManagementTokenAccessorID) require.NoError(t, err) require.NotEmpty(t, serverToken) @@ -1338,14 +1371,14 @@ func TestLeader_ACL_Initialization_SecondaryDC(t *testing.T) { testrpc.WaitForTestAgent(t, s2.RPC, "dc2") // Check dc1's management token - serverToken1, err := s1.getSystemMetadata(structs.ServerManagementTokenAccessorID) + serverToken1, err := s1.GetSystemMetadata(structs.ServerManagementTokenAccessorID) require.NoError(t, err) require.NotEmpty(t, serverToken1) _, err = uuid.ParseUUID(serverToken1) require.NoError(t, err) // Check dc2's management token - serverToken2, err := s2.getSystemMetadata(structs.ServerManagementTokenAccessorID) + serverToken2, err := s2.GetSystemMetadata(structs.ServerManagementTokenAccessorID) require.NoError(t, err) require.NotEmpty(t, serverToken2) _, err = uuid.ParseUUID(serverToken2) @@ -1406,15 +1439,17 @@ func TestLeader_ACLUpgrade_IsStickyEvenIfSerfTagsRegress(t *testing.T) { waitForLeaderEstablishment(t, s2) waitForNewACLReplication(t, s2, structs.ACLReplicatePolicies, 1, 0, 0) - // Everybody has the management policy. + // Everybody has the builtin policies. retry.Run(t, func(r *retry.R) { - _, policy1, err := s1.fsm.State().ACLPolicyGetByID(nil, structs.ACLPolicyGlobalManagementID, structs.DefaultEnterpriseMetaInDefaultPartition()) - require.NoError(r, err) - require.NotNil(r, policy1) + for _, builtinPolicy := range structs.ACLBuiltinPolicies { + _, policy1, err := s1.fsm.State().ACLPolicyGetByID(nil, builtinPolicy.ID, structs.DefaultEnterpriseMetaInDefaultPartition()) + require.NoError(r, err) + require.NotNil(r, policy1) - _, policy2, err := s2.fsm.State().ACLPolicyGetByID(nil, structs.ACLPolicyGlobalManagementID, structs.DefaultEnterpriseMetaInDefaultPartition()) - require.NoError(r, err) - require.NotNil(r, policy2) + _, policy2, err := s2.fsm.State().ACLPolicyGetByID(nil, builtinPolicy.ID, structs.DefaultEnterpriseMetaInDefaultPartition()) + require.NoError(r, err) + require.NotNil(r, policy2) + } }) // Shutdown s1 and s2. diff --git a/agent/consul/merge_oss.go b/agent/consul/merge_ce.go similarity index 100% rename from agent/consul/merge_oss.go rename to agent/consul/merge_ce.go diff --git a/agent/consul/merge_oss_test.go b/agent/consul/merge_ce_test.go similarity index 97% rename from agent/consul/merge_oss_test.go rename to agent/consul/merge_ce_test.go index 99333c7dd84..f046300de67 100644 --- a/agent/consul/merge_oss_test.go +++ b/agent/consul/merge_ce_test.go @@ -13,7 +13,7 @@ import ( "github.com/hashicorp/consul/types" ) -func TestMerge_OSS_LAN(t *testing.T) { +func TestMerge_CE_LAN(t *testing.T) { type testcase struct { segment string server bool diff --git a/agent/consul/multilimiter/multilimiter_test.go b/agent/consul/multilimiter/multilimiter_test.go index b64f95febdc..ceb38a2b850 100644 --- a/agent/consul/multilimiter/multilimiter_test.go +++ b/agent/consul/multilimiter/multilimiter_test.go @@ -88,7 +88,7 @@ func TestRateLimiterCleanup(t *testing.T) { retry.RunWith(&retry.Timer{Wait: 100 * time.Millisecond, Timeout: 2 * time.Second}, t, func(r *retry.R) { v, ok := limiters.Get(key) require.True(r, ok) - require.NotNil(t, v) + require.NotNil(r, v) }) time.Sleep(c.ReconcileCheckInterval) diff --git a/agent/consul/operator_raft_endpoint.go b/agent/consul/operator_raft_endpoint.go index 328f8ff964e..f5678fffdde 100644 --- a/agent/consul/operator_raft_endpoint.go +++ b/agent/consul/operator_raft_endpoint.go @@ -45,6 +45,12 @@ func (op *Operator) RaftGetConfiguration(args *structs.DCSpecificRequest, reply serverMap[raft.ServerAddress(addr)] = member } + serverIDLastIndexMap := make(map[raft.ServerID]uint64) + + for _, serverState := range op.srv.autopilot.GetState().Servers { + serverIDLastIndexMap[serverState.Server.ID] = serverState.Stats.LastIndex + } + // Fill out the reply. leader := op.srv.raft.Leader() reply.Index = future.Index() @@ -63,6 +69,7 @@ func (op *Operator) RaftGetConfiguration(args *structs.DCSpecificRequest, reply Leader: server.Address == leader, Voter: server.Suffrage == raft.Voter, ProtocolVersion: raftProtocolVersion, + LastIndex: serverIDLastIndexMap[server.ID], } reply.Servers = append(reply.Servers, entry) } diff --git a/agent/consul/operator_raft_endpoint_test.go b/agent/consul/operator_raft_endpoint_test.go index be60ec66a31..e4f322130a7 100644 --- a/agent/consul/operator_raft_endpoint_test.go +++ b/agent/consul/operator_raft_endpoint_test.go @@ -47,6 +47,13 @@ func TestOperator_RaftGetConfiguration(t *testing.T) { if len(future.Configuration().Servers) != 1 { t.Fatalf("bad: %v", future.Configuration().Servers) } + + serverIDLastIndexMap := make(map[raft.ServerID]uint64) + + for _, serverState := range s1.autopilot.GetState().Servers { + serverIDLastIndexMap[serverState.Server.ID] = serverState.Stats.LastIndex + } + me := future.Configuration().Servers[0] expected := structs.RaftConfigurationResponse{ Servers: []*structs.RaftServer{ @@ -57,6 +64,7 @@ func TestOperator_RaftGetConfiguration(t *testing.T) { Leader: true, Voter: true, ProtocolVersion: "3", + LastIndex: serverIDLastIndexMap[me.ID], }, }, Index: future.Index(), @@ -110,6 +118,10 @@ func TestOperator_RaftGetConfiguration_ACLDeny(t *testing.T) { if len(future.Configuration().Servers) != 1 { t.Fatalf("bad: %v", future.Configuration().Servers) } + serverIDLastIndexMap := make(map[raft.ServerID]uint64) + for _, serverState := range s1.autopilot.GetState().Servers { + serverIDLastIndexMap[serverState.Server.ID] = serverState.Stats.LastIndex + } me := future.Configuration().Servers[0] expected := structs.RaftConfigurationResponse{ Servers: []*structs.RaftServer{ @@ -120,6 +132,7 @@ func TestOperator_RaftGetConfiguration_ACLDeny(t *testing.T) { Leader: true, Voter: true, ProtocolVersion: "3", + LastIndex: serverIDLastIndexMap[me.ID], }, }, Index: future.Index(), diff --git a/agent/consul/options_oss.go b/agent/consul/options_ce.go similarity index 100% rename from agent/consul/options_oss.go rename to agent/consul/options_ce.go diff --git a/agent/consul/peering_backend.go b/agent/consul/peering_backend.go index 7c0b7c2e542..aa18cfd04ae 100644 --- a/agent/consul/peering_backend.go +++ b/agent/consul/peering_backend.go @@ -147,8 +147,11 @@ func (b *PeeringBackend) fetchPeerServerAddresses(ws memdb.WatchSet, peerID stri if err != nil { return nil, fmt.Errorf("failed to fetch peer %q: %w", peerID, err) } - if !peering.IsActive() { - return nil, fmt.Errorf("there is no active peering for %q", peerID) + if peering == nil { + return nil, fmt.Errorf("unknown peering %q", peerID) + } + if peering.DeletedAt != nil && !structs.IsZeroProtoTime(peering.DeletedAt) { + return nil, fmt.Errorf("peering %q was deleted", peerID) } return bufferFromAddresses(peering.GetAddressesToDial()) } diff --git a/agent/consul/peering_backend_oss.go b/agent/consul/peering_backend_ce.go similarity index 100% rename from agent/consul/peering_backend_oss.go rename to agent/consul/peering_backend_ce.go diff --git a/agent/consul/peering_backend_oss_test.go b/agent/consul/peering_backend_ce_test.go similarity index 100% rename from agent/consul/peering_backend_oss_test.go rename to agent/consul/peering_backend_ce_test.go diff --git a/agent/consul/peering_backend_test.go b/agent/consul/peering_backend_test.go index b7a725409bc..48580663189 100644 --- a/agent/consul/peering_backend_test.go +++ b/agent/consul/peering_backend_test.go @@ -253,7 +253,7 @@ func TestPeeringBackend_GetDialAddresses(t *testing.T) { }, peerID: acceptorPeerID, expect: expectation{ - err: fmt.Sprintf(`there is no active peering for %q`, acceptorPeerID), + err: fmt.Sprintf(`unknown peering %q`, acceptorPeerID), }, }, { @@ -384,6 +384,25 @@ func TestPeeringBackend_GetDialAddresses(t *testing.T) { gatewayAddrs: []string{"5.6.7.8:8443", "6.7.8.9:8443"}, }, }, + { + name: "addresses are returned if the peering is marked as terminated", + setup: func(store *state.Store) { + require.NoError(t, store.PeeringWrite(5, &pbpeering.PeeringWriteRequest{ + Peering: &pbpeering.Peering{ + Name: "dialer", + ID: dialerPeerID, + PeerServerAddresses: []string{"1.2.3.4:8502", "2.3.4.5:8503"}, + State: pbpeering.PeeringState_TERMINATED, + }, + })) + }, + peerID: dialerPeerID, + expect: expectation{ + // Gateways come first, and we use their LAN addresses since this is for outbound communication. + addrs: []string{"5.6.7.8:8443", "6.7.8.9:8443", "1.2.3.4:8502", "2.3.4.5:8503"}, + gatewayAddrs: []string{"5.6.7.8:8443", "6.7.8.9:8443"}, + }, + }, { name: "addresses are not returned if the peering is deleted", setup: func(store *state.Store) { @@ -401,7 +420,7 @@ func TestPeeringBackend_GetDialAddresses(t *testing.T) { }, peerID: dialerPeerID, expect: expectation{ - err: fmt.Sprintf(`there is no active peering for %q`, dialerPeerID), + err: fmt.Sprintf(`peering %q was deleted`, dialerPeerID), }, }, } diff --git a/agent/consul/prepared_query/walk_oss_test.go b/agent/consul/prepared_query/walk_ce_test.go similarity index 100% rename from agent/consul/prepared_query/walk_oss_test.go rename to agent/consul/prepared_query/walk_ce_test.go diff --git a/agent/consul/prepared_query_endpoint.go b/agent/consul/prepared_query_endpoint.go index ffa4b5e5091..2d6384c39d2 100644 --- a/agent/consul/prepared_query_endpoint.go +++ b/agent/consul/prepared_query_endpoint.go @@ -305,9 +305,9 @@ func (p *PreparedQuery) Explain(args *structs.PreparedQueryExecuteRequest, defer metrics.MeasureSince([]string{"prepared-query", "explain"}, time.Now()) // We have to do this ourselves since we are not doing a blocking RPC. - p.srv.setQueryMeta(&reply.QueryMeta, args.Token) + p.srv.SetQueryMeta(&reply.QueryMeta, args.Token) if args.RequireConsistent { - if err := p.srv.consistentRead(); err != nil { + if err := p.srv.ConsistentRead(); err != nil { return err } } @@ -353,7 +353,7 @@ func (p *PreparedQuery) Execute(args *structs.PreparedQueryExecuteRequest, // We have to do this ourselves since we are not doing a blocking RPC. if args.RequireConsistent { - if err := p.srv.consistentRead(); err != nil { + if err := p.srv.ConsistentRead(); err != nil { return err } } @@ -389,7 +389,7 @@ func (p *PreparedQuery) Execute(args *structs.PreparedQueryExecuteRequest, // though, since this is essentially a misconfiguration. // We have to do this ourselves since we are not doing a blocking RPC. - p.srv.setQueryMeta(&reply.QueryMeta, token) + p.srv.SetQueryMeta(&reply.QueryMeta, token) // Shuffle the results in case coordinates are not available if they // requested an RTT sort. @@ -468,7 +468,7 @@ func (p *PreparedQuery) Execute(args *structs.PreparedQueryExecuteRequest, // by the query setup. if len(reply.Nodes) == 0 { wrapper := &queryServerWrapper{srv: p.srv, executeRemote: p.ExecuteRemote} - if err := queryFailover(wrapper, query, args, reply); err != nil { + if err := queryFailover(wrapper, *query, args, reply); err != nil { return err } } @@ -490,7 +490,7 @@ func (p *PreparedQuery) ExecuteRemote(args *structs.PreparedQueryExecuteRemoteRe // We have to do this ourselves since we are not doing a blocking RPC. if args.RequireConsistent { - if err := p.srv.consistentRead(); err != nil { + if err := p.srv.ConsistentRead(); err != nil { return err } } @@ -511,7 +511,7 @@ func (p *PreparedQuery) ExecuteRemote(args *structs.PreparedQueryExecuteRemoteRe } // We have to do this ourselves since we are not doing a blocking RPC. - p.srv.setQueryMeta(&reply.QueryMeta, token) + p.srv.SetQueryMeta(&reply.QueryMeta, token) // We don't bother trying to do an RTT sort here since we are by // definition in another DC. We just shuffle to make sure that we @@ -707,7 +707,7 @@ func (q *queryServerWrapper) GetOtherDatacentersByDistance() ([]string, error) { // queryFailover runs an algorithm to determine which DCs to try and then calls // them to try to locate alternative services. -func queryFailover(q queryServer, query *structs.PreparedQuery, +func queryFailover(q queryServer, query structs.PreparedQuery, args *structs.PreparedQueryExecuteRequest, reply *structs.PreparedQueryExecuteResponse) error { @@ -789,7 +789,7 @@ func queryFailover(q queryServer, query *structs.PreparedQuery, // the remote query as well. remote := &structs.PreparedQueryExecuteRemoteRequest{ Datacenter: dc, - Query: *query, + Query: query, Limit: args.Limit, QueryOptions: args.QueryOptions, Connect: args.Connect, diff --git a/agent/consul/prepared_query_endpoint_test.go b/agent/consul/prepared_query_endpoint_test.go index 418710b8b50..5e20d6976bf 100644 --- a/agent/consul/prepared_query_endpoint_test.go +++ b/agent/consul/prepared_query_endpoint_test.go @@ -19,7 +19,6 @@ import ( msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc" "github.com/hashicorp/consul-net-rpc/net/rpc" - "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" grpcexternal "github.com/hashicorp/consul/agent/grpc-external" @@ -1676,8 +1675,7 @@ func TestPreparedQuery_Execute(t *testing.T) { assert.Len(t, reply.Nodes, 0) }) - expectNodes := func(t *testing.T, query *structs.PreparedQueryRequest, reply *structs.PreparedQueryExecuteResponse, n int) { - t.Helper() + expectNodes := func(t require.TestingT, query *structs.PreparedQueryRequest, reply *structs.PreparedQueryExecuteResponse, n int) { assert.Len(t, reply.Nodes, n) assert.Equal(t, "dc1", reply.Datacenter) assert.Equal(t, 0, reply.Failovers) @@ -1685,8 +1683,7 @@ func TestPreparedQuery_Execute(t *testing.T) { assert.Equal(t, query.Query.DNS, reply.DNS) assert.True(t, reply.QueryMeta.KnownLeader) } - expectFailoverNodes := func(t *testing.T, query *structs.PreparedQueryRequest, reply *structs.PreparedQueryExecuteResponse, n int) { - t.Helper() + expectFailoverNodes := func(t require.TestingT, query *structs.PreparedQueryRequest, reply *structs.PreparedQueryExecuteResponse, n int) { assert.Len(t, reply.Nodes, n) assert.Equal(t, "dc2", reply.Datacenter) assert.Equal(t, 1, reply.Failovers) @@ -1695,8 +1692,7 @@ func TestPreparedQuery_Execute(t *testing.T) { assert.True(t, reply.QueryMeta.KnownLeader) } - expectFailoverPeerNodes := func(t *testing.T, query *structs.PreparedQueryRequest, reply *structs.PreparedQueryExecuteResponse, n int) { - t.Helper() + expectFailoverPeerNodes := func(t require.TestingT, query *structs.PreparedQueryRequest, reply *structs.PreparedQueryExecuteResponse, n int) { assert.Len(t, reply.Nodes, n) assert.Equal(t, "", reply.Datacenter) assert.Equal(t, acceptingPeerName, reply.PeerName) @@ -2092,16 +2088,16 @@ func TestPreparedQuery_Execute(t *testing.T) { require.NoError(t, msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Apply", &query, &query.Query.ID)) // Update the health of a node to mark it critical. - setHealth := func(t *testing.T, codec rpc.ClientCodec, dc string, node string, health string) { + setHealth := func(t *testing.T, codec rpc.ClientCodec, dc string, i int, health string) { t.Helper() req := structs.RegisterRequest{ Datacenter: dc, - Node: node, + Node: fmt.Sprintf("node%d", i), Address: "127.0.0.1", Service: &structs.NodeService{ Service: "foo", Port: 8000, - Tags: []string{"dc1", "tag1"}, + Tags: []string{dc, fmt.Sprintf("tag%d", i)}, }, Check: &structs.HealthCheck{ Name: "failing", @@ -2113,7 +2109,7 @@ func TestPreparedQuery_Execute(t *testing.T) { var reply struct{} require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &req, &reply)) } - setHealth(t, codec1, "dc1", "node1", api.HealthCritical) + setHealth(t, codec1, "dc1", 1, api.HealthCritical) // The failing node should be filtered. t.Run("failing node filtered", func(t *testing.T) { @@ -2133,7 +2129,7 @@ func TestPreparedQuery_Execute(t *testing.T) { }) // Upgrade it to a warning and re-query, should be 10 nodes again. - setHealth(t, codec1, "dc1", "node1", api.HealthWarning) + setHealth(t, codec1, "dc1", 1, api.HealthWarning) t.Run("warning nodes are included", func(t *testing.T) { req := structs.PreparedQueryExecuteRequest{ Datacenter: "dc1", @@ -2303,7 +2299,7 @@ func TestPreparedQuery_Execute(t *testing.T) { // Now fail everything in dc1 and we should get an empty list back. for i := 0; i < 10; i++ { - setHealth(t, codec1, "dc1", fmt.Sprintf("node%d", i+1), api.HealthCritical) + setHealth(t, codec1, "dc1", i+1, api.HealthCritical) } t.Run("everything is failing so should get empty list", func(t *testing.T) { req := structs.PreparedQueryExecuteRequest{ @@ -2474,7 +2470,7 @@ func TestPreparedQuery_Execute(t *testing.T) { // Set all checks in dc2 as critical for i := 0; i < 10; i++ { - setHealth(t, codec2, "dc2", fmt.Sprintf("node%d", i+1), api.HealthCritical) + setHealth(t, codec2, "dc2", i+1, api.HealthCritical) } // Now we should see 9 nodes from dc3 (we have the tag filter still) @@ -2493,6 +2489,31 @@ func TestPreparedQuery_Execute(t *testing.T) { } expectFailoverPeerNodes(t, &query, &reply, 9) }) + + // Set all checks in dc1 as passing + for i := 0; i < 10; i++ { + setHealth(t, codec1, "dc1", i+1, api.HealthPassing) + } + + // Nothing is healthy so nothing is returned + t.Run("un-failing over", func(t *testing.T) { + retry.Run(t, func(r *retry.R) { + req := structs.PreparedQueryExecuteRequest{ + Datacenter: "dc1", + QueryIDOrName: query.Query.ID, + QueryOptions: structs.QueryOptions{Token: execToken}, + } + + var reply structs.PreparedQueryExecuteResponse + require.NoError(r, msgpackrpc.CallWithCodec(codec1, "PreparedQuery.Execute", &req, &reply)) + + for _, node := range reply.Nodes { + assert.NotEqual(r, "node3", node.Node.Node) + } + + expectNodes(r, &query, &reply, 9) + }) + }) } func TestPreparedQuery_Execute_ForwardLeader(t *testing.T) { @@ -2982,7 +3003,7 @@ func (m *mockQueryServer) ExecuteRemote(args *structs.PreparedQueryExecuteRemote func TestPreparedQuery_queryFailover(t *testing.T) { t.Parallel() - query := &structs.PreparedQuery{ + query := structs.PreparedQuery{ Name: "test", Service: structs.ServiceQuery{ Failover: structs.QueryFailoverOptions{ diff --git a/agent/consul/reporting/reporting.go b/agent/consul/reporting/reporting.go new file mode 100644 index 00000000000..94f780ec060 --- /dev/null +++ b/agent/consul/reporting/reporting.go @@ -0,0 +1,52 @@ +package reporting + +import ( + "sync" + "time" + + "github.com/hashicorp/consul/agent/consul/state" + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-memdb" +) + +type ReportingManager struct { + logger hclog.Logger + server ServerDelegate + stateProvider StateDelegate + tickerInterval time.Duration + EntDeps + sync.RWMutex +} + +const ( + SystemMetadataReportingProcessID = "reporting-process-id" + ReportingInterval = 1 * time.Hour +) + +//go:generate mockery --name ServerDelegate --inpackage +type ServerDelegate interface { + GetSystemMetadata(key string) (string, error) + SetSystemMetadataKey(key, val string) error + IsLeader() bool +} + +type StateDelegate interface { + NodeUsage() (uint64, state.NodeUsage, error) + ServiceUsage(ws memdb.WatchSet) (uint64, structs.ServiceUsage, error) +} + +func NewReportingManager(logger hclog.Logger, deps EntDeps, server ServerDelegate, stateProvider StateDelegate) *ReportingManager { + rm := &ReportingManager{ + logger: logger.Named("reporting"), + server: server, + stateProvider: stateProvider, + tickerInterval: ReportingInterval, + } + err := rm.initEnterpriseReporting(deps) + if err != nil { + rm.logger.Error("Error initializing reporting manager", "error", err) + return nil + } + return rm +} diff --git a/agent/consul/reporting/reporting_ce.go b/agent/consul/reporting/reporting_ce.go new file mode 100644 index 00000000000..240500d6d50 --- /dev/null +++ b/agent/consul/reporting/reporting_ce.go @@ -0,0 +1,29 @@ +//go:build !consulent +// +build !consulent + +package reporting + +import ( + "context" +) + +type EntDeps struct{} + +func (rm *ReportingManager) initEnterpriseReporting(entDeps EntDeps) error { + // no op + return nil +} + +func (rm *ReportingManager) StartReportingAgent() error { + // no op + return nil +} + +func (rm *ReportingManager) StopReportingAgent() error { + // no op + return nil +} + +func (m *ReportingManager) Run(ctx context.Context) { + // no op +} diff --git a/agent/consul/rpc.go b/agent/consul/rpc.go index 2bad1a8b501..f608fc30723 100644 --- a/agent/consul/rpc.go +++ b/agent/consul/rpc.go @@ -1,7 +1,6 @@ package consul import ( - "context" "crypto/tls" "encoding/binary" "errors" @@ -10,7 +9,6 @@ import ( "math" "net" "strings" - "sync/atomic" "time" "github.com/armon/go-metrics" @@ -27,6 +25,7 @@ import ( msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc" "github.com/hashicorp/consul/acl" + "github.com/hashicorp/consul/agent/blockingquery" "github.com/hashicorp/consul/agent/consul/rate" "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/consul/wanfed" @@ -992,167 +991,26 @@ type blockingQueryResponseMeta interface { SetResultsFilteredByACLs(bool) } -// blockingQuery performs a blocking query if opts.GetMinQueryIndex is -// greater than 0, otherwise performs a non-blocking query. Blocking queries will -// block until responseMeta.Index is greater than opts.GetMinQueryIndex, -// or opts.GetMaxQueryTime is reached. Non-blocking queries return immediately -// after performing the query. -// -// If opts.GetRequireConsistent is true, blockingQuery will first verify it is -// still the cluster leader before performing the query. -// -// The query function is expected to be a closure that has access to responseMeta -// so that it can set the Index. The actual result of the query is opaque to blockingQuery. -// -// The query function can return errNotFound, which is a sentinel error. Returning -// errNotFound indicates that the query found no results, which allows -// blockingQuery to keep blocking until the query returns a non-nil error. -// The query function must take care to set the actual result of the query to -// nil in these cases, otherwise when blockingQuery times out it may return -// a previous result. errNotFound will never be returned to the caller, it is -// converted to nil before returning. -// -// The query function can return errNotChanged, which is a sentinel error. This -// can only be returned on calls AFTER the first call, as it would not be -// possible to detect the absence of a change on the first call. Returning -// errNotChanged indicates that the query results are identical to the prior -// results which allows blockingQuery to keep blocking until the query returns -// a real changed result. -// -// The query function must take care to ensure the actual result of the query -// is either left unmodified or explicitly left in a good state before -// returning, otherwise when blockingQuery times out it may return an -// incomplete or unexpected result. errNotChanged will never be returned to the -// caller, it is converted to nil before returning. -// -// If query function returns any other error, the error is returned to the caller -// immediately. -// -// The query function must follow these rules: -// -// 1. to access data it must use the passed in state.Store. -// 2. it must set the responseMeta.Index to an index greater than -// opts.GetMinQueryIndex if the results return by the query have changed. -// 3. any channels added to the memdb.WatchSet must unblock when the results -// returned by the query have changed. -// -// To ensure optimal performance of the query, the query function should make a -// best-effort attempt to follow these guidelines: -// -// 1. only set responseMeta.Index to an index greater than -// opts.GetMinQueryIndex when the results returned by the query have changed. -// 2. any channels added to the memdb.WatchSet should only unblock when the -// results returned by the query have changed. +// blockingQuery is a passthrough to blockingquery.Query that keeps API +// compatibility with Server. That has RPC and FSM machinery mixed in the same consul +// package. func (s *Server) blockingQuery( - opts blockingQueryOptions, - responseMeta blockingQueryResponseMeta, - query queryFn, + requestOpts blockingquery.RequestOptions, + responseMeta blockingquery.ResponseMeta, + query blockingquery.QueryFn, ) error { - var ctx context.Context = &lib.StopChannelContext{StopCh: s.shutdownCh} - - metrics.IncrCounter([]string{"rpc", "query"}, 1) - - minQueryIndex := opts.GetMinQueryIndex() - // Perform a non-blocking query - if minQueryIndex == 0 { - if opts.GetRequireConsistent() { - if err := s.consistentRead(); err != nil { - return err - } - } - - var ws memdb.WatchSet - err := query(ws, s.fsm.State()) - s.setQueryMeta(responseMeta, opts.GetToken()) - if errors.Is(err, errNotFound) || errors.Is(err, errNotChanged) { - return nil - } - return err - } - - maxQueryTimeout, err := opts.GetMaxQueryTime() - if err != nil { - return err - } - timeout := s.rpcQueryTimeout(maxQueryTimeout) - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - count := atomic.AddUint64(&s.queriesBlocking, 1) - metrics.SetGauge([]string{"rpc", "queries_blocking"}, float32(count)) - // decrement the count when the function returns. - defer atomic.AddUint64(&s.queriesBlocking, ^uint64(0)) - - var ( - notFound bool - ranOnce bool - ) - - for { - if opts.GetRequireConsistent() { - if err := s.consistentRead(); err != nil { - return err - } - } - - // Operate on a consistent set of state. This makes sure that the - // abandon channel goes with the state that the caller is using to - // build watches. - state := s.fsm.State() - - ws := memdb.NewWatchSet() - // This channel will be closed if a snapshot is restored and the - // whole state store is abandoned. - ws.Add(state.AbandonCh()) - - err := query(ws, state) - s.setQueryMeta(responseMeta, opts.GetToken()) - - switch { - case errors.Is(err, errNotFound): - if notFound { - // query result has not changed - minQueryIndex = responseMeta.GetIndex() - } - notFound = true - case errors.Is(err, errNotChanged): - if ranOnce { - // query result has not changed - minQueryIndex = responseMeta.GetIndex() - } - case err != nil: - return err - } - ranOnce = true - - if responseMeta.GetIndex() > minQueryIndex { - return nil - } - - // block until something changes, or the timeout - if err := ws.WatchCtx(ctx); err != nil { - // exit if we've reached the timeout, or other cancellation - return nil - } - - // exit if the state store has been abandoned - select { - case <-state.AbandonCh(): - return nil - default: - } - } + return blockingquery.Query(s, requestOpts, responseMeta, query) } var ( - errNotFound = fmt.Errorf("no data found for query") - errNotChanged = fmt.Errorf("data did not change for query") + errNotFound = blockingquery.ErrNotFound + errNotChanged = blockingquery.ErrNotChanged ) -// setQueryMeta is used to populate the QueryMeta data for an RPC call +// SetQueryMeta is used to populate the QueryMeta data for an RPC call // // Note: This method must be called *after* filtering query results with ACLs. -func (s *Server) setQueryMeta(m blockingQueryResponseMeta, token string) { +func (s *Server) SetQueryMeta(m blockingquery.ResponseMeta, token string) { if s.IsLeader() { m.SetLastContact(0) m.SetKnownLeader(true) @@ -1176,7 +1034,7 @@ func (s *Server) setQueryMeta(m blockingQueryResponseMeta, token string) { // consistentRead is used to ensure we do not perform a stale // read. This is done by verifying leadership before the read. -func (s *Server) consistentRead() error { +func (s *Server) ConsistentRead() error { defer metrics.MeasureSince([]string{"rpc", "consistentRead"}, time.Now()) future := s.raft.VerifyLeader() if err := future.Error(); err != nil { @@ -1207,10 +1065,10 @@ func (s *Server) consistentRead() error { return structs.ErrNotReadyForConsistentReads } -// rpcQueryTimeout calculates the timeout for the query, ensures it is +// RPCQueryTimeout calculates the timeout for the query, ensures it is // constrained to the configured limit, and adds jitter to prevent multiple // blocking queries from all timing out at the same time. -func (s *Server) rpcQueryTimeout(queryTimeout time.Duration) time.Duration { +func (s *Server) RPCQueryTimeout(queryTimeout time.Duration) time.Duration { // Restrict the max query time, and ensure there is always one. if queryTimeout > s.config.MaxQueryTime { queryTimeout = s.config.MaxQueryTime diff --git a/agent/consul/rpc_test.go b/agent/consul/rpc_test.go index 843dd4c1f49..263689f625b 100644 --- a/agent/consul/rpc_test.go +++ b/agent/consul/rpc_test.go @@ -499,7 +499,7 @@ func TestRPC_ReadyForConsistentReads(t *testing.T) { } s.resetConsistentReadReady() - err := s.consistentRead() + err := s.ConsistentRead() if err.Error() != "Not ready to serve consistent reads" { t.Fatal("Server should NOT be ready for consistent reads") } @@ -510,7 +510,7 @@ func TestRPC_ReadyForConsistentReads(t *testing.T) { }() retry.Run(t, func(r *retry.R) { - if err := s.consistentRead(); err != nil { + if err := s.ConsistentRead(); err != nil { r.Fatalf("Expected server to be ready for consistent reads, got error %v", err) } }) @@ -1165,7 +1165,7 @@ func TestRPC_LocalTokenStrippedOnForward_GRPC(t *testing.T) { var conn *grpc.ClientConn { - client, resolverBuilder, balancerBuilder := newClientWithGRPCPlumbing(t, func(c *Config) { + client, resolverBuilder := newClientWithGRPCPlumbing(t, func(c *Config) { c.Datacenter = "dc2" c.PrimaryDatacenter = "dc1" c.RPCConfig.EnableStreaming = true @@ -1177,7 +1177,6 @@ func TestRPC_LocalTokenStrippedOnForward_GRPC(t *testing.T) { Servers: resolverBuilder, DialingFromServer: false, DialingFromDatacenter: "dc2", - BalancerBuilder: balancerBuilder, }) conn, err = pool.ClientConn("dc2") diff --git a/agent/consul/segment_oss.go b/agent/consul/segment_ce.go similarity index 88% rename from agent/consul/segment_oss.go rename to agent/consul/segment_ce.go index 034e79c54ef..44b8a14b2e7 100644 --- a/agent/consul/segment_oss.go +++ b/agent/consul/segment_ce.go @@ -11,7 +11,7 @@ import ( "github.com/hashicorp/consul/agent/structs" ) -var SegmentOSSSummaries = []prometheus.SummaryDefinition{ +var SegmentCESummaries = []prometheus.SummaryDefinition{ { Name: []string{"leader", "reconcile"}, Help: "Measures the time spent updating the raft store from the serf member information.", @@ -23,7 +23,7 @@ func (s *Server) LANSegmentAddr(name string) string { return "" } -// setupSegmentRPC returns an error if any segments are defined since the OSS +// setupSegmentRPC returns an error if any segments are defined since the CE // version of Consul doesn't support them. func (s *Server) setupSegmentRPC() (map[string]net.Listener, error) { if len(s.config.Segments) > 0 { @@ -33,7 +33,7 @@ func (s *Server) setupSegmentRPC() (map[string]net.Listener, error) { return nil, nil } -// setupSegments returns an error if any segments are defined since the OSS +// setupSegments returns an error if any segments are defined since the CE // version of Consul doesn't support them. func (s *Server) setupSegments(config *Config, rpcListeners map[string]net.Listener) error { if len(config.Segments) > 0 { @@ -43,6 +43,6 @@ func (s *Server) setupSegments(config *Config, rpcListeners map[string]net.Liste return nil } -// floodSegments is a NOP in the OSS version of Consul. +// floodSegments is a NOP in the CE version of Consul. func (s *Server) floodSegments(config *Config) { } diff --git a/agent/consul/server.go b/agent/consul/server.go index 5a68e780b41..bf1166d2b73 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -34,11 +34,13 @@ import ( "github.com/hashicorp/consul-net-rpc/net/rpc" "github.com/hashicorp/consul/acl" + "github.com/hashicorp/consul/agent/blockingquery" "github.com/hashicorp/consul/agent/consul/authmethod" "github.com/hashicorp/consul/agent/consul/authmethod/ssoauth" "github.com/hashicorp/consul/agent/consul/fsm" "github.com/hashicorp/consul/agent/consul/multilimiter" rpcRate "github.com/hashicorp/consul/agent/consul/rate" + "github.com/hashicorp/consul/agent/consul/reporting" "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/consul/stream" "github.com/hashicorp/consul/agent/consul/usagemetrics" @@ -52,6 +54,7 @@ import ( agentgrpc "github.com/hashicorp/consul/agent/grpc-internal" "github.com/hashicorp/consul/agent/grpc-internal/services/subscribe" "github.com/hashicorp/consul/agent/hcp" + hcpclient "github.com/hashicorp/consul/agent/hcp/client" logdrop "github.com/hashicorp/consul/agent/log-drop" "github.com/hashicorp/consul/agent/metadata" "github.com/hashicorp/consul/agent/pool" @@ -158,6 +161,8 @@ type raftStore interface { const requestLimitsBurstMultiplier = 10 +var _ blockingquery.FSMServer = (*Server)(nil) + // Server is Consul server which manages the service discovery, // health checking, DC forwarding, Raft, and multiple Serf pools. type Server struct { @@ -409,6 +414,21 @@ type Server struct { // embedded struct to hold all the enterprise specific data EnterpriseServer operatorServer *operator.Server + + // handles metrics reporting to HashiCorp + reportingManager *reporting.ReportingManager +} + +func (s *Server) DecrementBlockingQueries() uint64 { + return atomic.AddUint64(&s.queriesBlocking, ^uint64(0)) +} + +func (s *Server) GetShutdownChannel() chan struct{} { + return s.shutdownCh +} + +func (s *Server) IncrementBlockingQueries() uint64 { + return atomic.AddUint64(&s.queriesBlocking, 1) } type connHandler interface { @@ -728,6 +748,9 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server, incom s.overviewManager = NewOverviewManager(s.logger, s.fsm, s.config.MetricsReportingInterval) go s.overviewManager.Run(&lib.StopChannelContext{StopCh: s.shutdownCh}) + s.reportingManager = reporting.NewReportingManager(s.logger, getEnterpriseReportingDeps(flat), s, s.fsm.State()) + go s.reportingManager.Run(&lib.StopChannelContext{StopCh: s.shutdownCh}) + // Initialize external gRPC server - register services on external gRPC server. s.externalACLServer = aclgrpc.NewServer(aclgrpc.Config{ ACLsEnabled: s.config.ACLsEnabled, @@ -860,6 +883,7 @@ func newGRPCHandlerFromConfig(deps Deps, config *Config, s *Server) connHandler Datacenter: config.Datacenter, ConnectEnabled: config.ConnectEnabled, PeeringEnabled: config.PeeringEnabled, + FSMServer: s, }) s.peeringServer = p o := operator.NewServer(operator.Config{ @@ -1043,7 +1067,7 @@ func (s *Server) setupRaft() error { log = cacheStore // Create the snapshot store. - snapshots, err := raft.NewFileSnapshotStoreWithLogger(path, snapshotsRetained, s.logger.Named("snapshot")) + snapshots, err := raft.NewFileSnapshotStoreWithLogger(path, snapshotsRetained, s.logger.Named("raft.snapshot")) if err != nil { return err } @@ -1669,6 +1693,13 @@ func (s *Server) FSM() *fsm.FSM { return s.fsm } +func (s *Server) GetState() *state.Store { + if s == nil || s.FSM() == nil { + return nil + } + return s.FSM().State() +} + // Stats is used to return statistics for debugging and insight // for various sub-systems func (s *Server) Stats() map[string]map[string]string { @@ -1746,6 +1777,8 @@ func (s *Server) ReloadConfig(config ReloadableConfig) error { return err } + s.updateReportingConfig(config) + s.rpcLimiter.Store(rate.NewLimiter(config.RPCRateLimit, config.RPCMaxBurst)) if config.RequestLimits != nil { @@ -1850,13 +1883,14 @@ func (s *Server) trackLeaderChanges() { // hcpServerStatus is the callback used by the HCP manager to emit status updates to the HashiCorp Cloud Platform when // enabled. func (s *Server) hcpServerStatus(deps Deps) hcp.StatusCallback { - return func(ctx context.Context) (status hcp.ServerStatus, err error) { + return func(ctx context.Context) (status hcpclient.ServerStatus, err error) { status.Name = s.config.NodeName status.ID = string(s.config.NodeID) status.Version = cslversion.GetHumanVersion() status.LanAddress = s.config.RPCAdvertise.IP.String() status.GossipPort = s.config.SerfLANConfig.MemberlistConfig.AdvertisePort status.RPCPort = s.config.RPCAddr.Port + status.Datacenter = s.config.Datacenter tlsCert := s.tlsConfigurator.Cert() if tlsCert != nil { @@ -1898,6 +1932,8 @@ func (s *Server) hcpServerStatus(deps Deps) hcp.StatusCallback { status.ScadaStatus = deps.HCP.Provider.SessionStatus() } + status.ACL.Enabled = s.config.ACLsEnabled + return status, nil } } diff --git a/agent/consul/server_oss.go b/agent/consul/server_ce.go similarity index 94% rename from agent/consul/server_oss.go rename to agent/consul/server_ce.go index 4ae524b65c0..63de1bfe852 100644 --- a/agent/consul/server_oss.go +++ b/agent/consul/server_ce.go @@ -15,6 +15,7 @@ import ( "google.golang.org/grpc" "github.com/hashicorp/consul/acl" + "github.com/hashicorp/consul/agent/consul/reporting" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/lib" ) @@ -78,7 +79,7 @@ func (s *Server) removeFailedNode( } // lanPoolAllMembers only returns our own segment or partition's members, because -// OSS servers can't be in multiple segments or partitions. +// CE servers can't be in multiple segments or partitions. func (s *Server) lanPoolAllMembers() ([]serf.Member, error) { return s.LANMembersInAgentPartition(), nil } @@ -174,3 +175,12 @@ func addSerfMetricsLabels(conf *serf.Config, wan bool, segment string, partition conf.MetricLabels = append(conf.MetricLabels, networkMetric) } + +func (s *Server) updateReportingConfig(config ReloadableConfig) { + // no-op +} + +func getEnterpriseReportingDeps(deps Deps) reporting.EntDeps { + // no-op + return reporting.EntDeps{} +} diff --git a/agent/consul/server_ce_test.go b/agent/consul/server_ce_test.go new file mode 100644 index 00000000000..d5a2aff0820 --- /dev/null +++ b/agent/consul/server_ce_test.go @@ -0,0 +1,43 @@ +//go:build !consulent +// +build !consulent + +package consul + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/hashicorp/consul/testrpc" +) + +func TestAgent_ReloadConfig_Reporting(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + t.Parallel() + + dir1, s := testServerWithConfig(t, func(c *Config) { + c.Reporting.License.Enabled = false + }) + defer os.RemoveAll(dir1) + defer s.Shutdown() + + testrpc.WaitForTestAgent(t, s.RPC, "dc1") + + require.Equal(t, false, s.config.Reporting.License.Enabled) + + rc := ReloadableConfig{ + Reporting: Reporting{ + License: License{ + Enabled: true, + }, + }, + } + + require.NoError(t, s.ReloadConfig(rc)) + + // Check config reload is no-op + require.Equal(t, false, s.config.Reporting.License.Enabled) +} diff --git a/agent/consul/server_log_verification.go b/agent/consul/server_log_verification.go index 0c7e63e3a12..cb95b9aeeee 100644 --- a/agent/consul/server_log_verification.go +++ b/agent/consul/server_log_verification.go @@ -62,12 +62,12 @@ func makeLogVerifyReportFn(logger hclog.Logger) verifier.ReportFn { if r.WrittenSum > 0 && r.WrittenSum != r.ExpectedSum { // The failure occurred before the follower wrote to the log so it // must be corrupted in flight from the leader! - l2.Info("verification checksum FAILED: in-flight corruption", + l2.Error("verification checksum FAILED: in-flight corruption", "followerWriteChecksum", fmt.Sprintf("%08x", r.WrittenSum), "readChecksum", fmt.Sprintf("%08x", r.ReadSum), ) } else { - l2.Info("verification checksum FAILED: storage corruption", + l2.Error("verification checksum FAILED: storage corruption", "followerWriteChecksum", fmt.Sprintf("%08x", r.WrittenSum), "readChecksum", fmt.Sprintf("%08x", r.ReadSum), ) diff --git a/agent/consul/server_metadata.go b/agent/consul/server_metadata.go new file mode 100644 index 00000000000..742391e0b6a --- /dev/null +++ b/agent/consul/server_metadata.go @@ -0,0 +1,71 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package consul + +import ( + "encoding/json" + "io" + "os" + "time" +) + +// ServerMetadataFile is the name of the file on disk that server metadata +// should be written to. +const ServerMetadataFile = "server_metadata.json" + +// ServerMetadata represents specific metadata about a running server. +type ServerMetadata struct { + // LastSeenUnix is the timestamp a server was last seen, in Unix format. + LastSeenUnix int64 `json:"last_seen_unix"` +} + +// IsLastSeenStale checks whether the last seen timestamp is older than a given duration. +func (md *ServerMetadata) IsLastSeenStale(d time.Duration) bool { + lastSeen := time.Unix(md.LastSeenUnix, 0) + maxAge := time.Now().Add(-d) + + return lastSeen.Before(maxAge) +} + +// OpenServerMetadata is a helper function for opening the server metadata file +// with the correct permissions. +func OpenServerMetadata(filename string) (io.WriteCloser, error) { + return os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) +} + +type ServerMetadataReadFunc func(filename string) (*ServerMetadata, error) + +// ReadServerMetadata is a helper function for reading the contents of a server +// metadata file and unmarshaling the data from JSON. +func ReadServerMetadata(filename string) (*ServerMetadata, error) { + b, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + + var md ServerMetadata + if err := json.Unmarshal(b, &md); err != nil { + return nil, err + } + + return &md, nil +} + +// WriteServerMetadata writes server metadata to a file in JSON format. +func WriteServerMetadata(w io.Writer) error { + md := &ServerMetadata{ + LastSeenUnix: time.Now().Unix(), + } + + b, err := json.Marshal(md) + if err != nil { + return err + } + + if _, err := w.Write(b); err != nil { + return err + } + + return nil +} diff --git a/agent/consul/server_metadata_test.go b/agent/consul/server_metadata_test.go new file mode 100644 index 00000000000..d091bfdf363 --- /dev/null +++ b/agent/consul/server_metadata_test.go @@ -0,0 +1,68 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package consul + +import ( + "bytes" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type mockServerMetadataWriter struct { + writeErr error +} + +func (m *mockServerMetadataWriter) Write(p []byte) (n int, err error) { + if m.writeErr != nil { + return 0, m.writeErr + } + + return 1, nil +} + +func TestServerMetadata(t *testing.T) { + now := time.Now() + + t.Run("TestIsLastSeenStaleTrue", func(t *testing.T) { + // Create a server that is 48 hours old. + md := &ServerMetadata{ + LastSeenUnix: now.Add(-48 * time.Hour).Unix(), + } + + stale := md.IsLastSeenStale(24 * time.Hour) + assert.True(t, stale) + }) + + t.Run("TestIsLastSeenStaleFalse", func(t *testing.T) { + // Create a server that is 1 hour old. + md := &ServerMetadata{ + LastSeenUnix: now.Add(-1 * time.Hour).Unix(), + } + + stale := md.IsLastSeenStale(24 * time.Hour) + assert.False(t, stale) + }) +} + +func TestWriteServerMetadata(t *testing.T) { + t.Run("TestWriteError", func(t *testing.T) { + m := &mockServerMetadataWriter{ + writeErr: errors.New("write error"), + } + + err := WriteServerMetadata(m) + assert.Error(t, err) + }) + + t.Run("TestOK", func(t *testing.T) { + b := new(bytes.Buffer) + + err := WriteServerMetadata(b) + assert.NoError(t, err) + assert.True(t, b.Len() > 0) + }) +} diff --git a/agent/consul/server_serf.go b/agent/consul/server_serf.go index a515589303f..8c4c7600ab7 100644 --- a/agent/consul/server_serf.go +++ b/agent/consul/server_serf.go @@ -20,6 +20,7 @@ import ( "github.com/hashicorp/consul/lib" libserf "github.com/hashicorp/consul/lib/serf" "github.com/hashicorp/consul/logging" + "github.com/hashicorp/consul/types" ) const ( @@ -356,6 +357,7 @@ func (s *Server) lanNodeJoin(me serf.MemberEvent) { // Update server lookup s.serverLookup.AddServer(serverMeta) + s.router.AddServer(types.AreaLAN, serverMeta) // If we're still expecting to bootstrap, may need to handle this. if s.config.BootstrapExpect != 0 { @@ -377,6 +379,7 @@ func (s *Server) lanNodeUpdate(me serf.MemberEvent) { // Update server lookup s.serverLookup.AddServer(serverMeta) + s.router.AddServer(types.AreaLAN, serverMeta) } } @@ -515,5 +518,6 @@ func (s *Server) lanNodeFailed(me serf.MemberEvent) { // Update id to address map s.serverLookup.RemoveServer(serverMeta) + s.router.RemoveServer(types.AreaLAN, serverMeta) } } diff --git a/agent/consul/server_test.go b/agent/consul/server_test.go index 6084847fc89..1b87631d824 100644 --- a/agent/consul/server_test.go +++ b/agent/consul/server_test.go @@ -24,8 +24,6 @@ import ( "golang.org/x/time/rate" "google.golang.org/grpc" - "github.com/hashicorp/consul/agent/hcp" - "github.com/hashicorp/consul-net-rpc/net/rpc" "github.com/hashicorp/consul/agent/connect" @@ -33,6 +31,7 @@ import ( rpcRate "github.com/hashicorp/consul/agent/consul/rate" external "github.com/hashicorp/consul/agent/grpc-external" grpcmiddleware "github.com/hashicorp/consul/agent/grpc-middleware" + hcpclient "github.com/hashicorp/consul/agent/hcp/client" "github.com/hashicorp/consul/agent/metadata" "github.com/hashicorp/consul/agent/rpc/middleware" "github.com/hashicorp/consul/agent/structs" @@ -2075,10 +2074,10 @@ func TestServer_hcpManager(t *testing.T) { _, conf1 := testServerConfig(t) conf1.BootstrapExpect = 1 conf1.RPCAdvertise = &net.TCPAddr{IP: []byte{127, 0, 0, 2}, Port: conf1.RPCAddr.Port} - hcp1 := hcp.NewMockClient(t) - hcp1.EXPECT().PushServerStatus(mock.Anything, mock.MatchedBy(func(status *hcp.ServerStatus) bool { + hcp1 := hcpclient.NewMockClient(t) + hcp1.EXPECT().PushServerStatus(mock.Anything, mock.MatchedBy(func(status *hcpclient.ServerStatus) bool { return status.ID == string(conf1.NodeID) - })).Run(func(ctx context.Context, status *hcp.ServerStatus) { + })).Run(func(ctx context.Context, status *hcpclient.ServerStatus) { require.Equal(t, status.LanAddress, "127.0.0.2") }).Call.Return(nil) diff --git a/agent/consul/snapshot_endpoint.go b/agent/consul/snapshot_endpoint.go index 102bc0a38a5..197a3ec974a 100644 --- a/agent/consul/snapshot_endpoint.go +++ b/agent/consul/snapshot_endpoint.go @@ -68,14 +68,14 @@ func (s *Server) dispatchSnapshotRequest(args *structs.SnapshotRequest, in io.Re switch args.Op { case structs.SnapshotSave: if !args.AllowStale { - if err := s.consistentRead(); err != nil { + if err := s.ConsistentRead(); err != nil { return nil, err } } // Set the metadata here before we do anything; this should always be // pessimistic if we get more data while the snapshot is being taken. - s.setQueryMeta(&reply.QueryMeta, args.Token) + s.SetQueryMeta(&reply.QueryMeta, args.Token) // Take the snapshot and capture the index. snap, err := snapshot.New(s.logger, s.raft) diff --git a/agent/consul/state/acl.go b/agent/consul/state/acl.go index 1b7debe3690..072d1a5fe18 100644 --- a/agent/consul/state/acl.go +++ b/agent/consul/state/acl.go @@ -620,8 +620,35 @@ func aclTokenGetTxn(tx ReadTxn, ws memdb.WatchSet, value, index string, entMeta return nil, nil } +type ACLTokenListParameters struct { + Local bool + Global bool + Policy string + Role string + ServiceName string + MethodName string + MethodMeta *acl.EnterpriseMeta + EnterpriseMeta *acl.EnterpriseMeta +} + // ACLTokenList return a list of ACL Tokens that match the policy, role, and method. +// This function should be treated as deprecated, and ACLTokenListWithParameters should be preferred. +// +// Deprecated: use ACLTokenListWithParameters func (s *Store) ACLTokenList(ws memdb.WatchSet, local, global bool, policy, role, methodName string, methodMeta, entMeta *acl.EnterpriseMeta) (uint64, structs.ACLTokens, error) { + return s.ACLTokenListWithParameters(ws, ACLTokenListParameters{ + Local: local, + Global: global, + Policy: policy, + Role: role, + MethodName: methodName, + MethodMeta: methodMeta, + EnterpriseMeta: entMeta, + }) +} + +// ACLTokenListWithParameters returns a list of ACL Tokens that match the provided parameters. +func (s *Store) ACLTokenListWithParameters(ws memdb.WatchSet, params ACLTokenListParameters) (uint64, structs.ACLTokens, error) { tx := s.db.Txn(false) defer tx.Abort() @@ -634,43 +661,51 @@ func (s *Store) ACLTokenList(ws memdb.WatchSet, local, global bool, policy, role needLocalityFilter := false - if policy == "" && role == "" && methodName == "" { - if global == local { - iter, err = aclTokenListAll(tx, entMeta) + if params.Policy == "" && params.Role == "" && params.MethodName == "" && params.ServiceName == "" { + if params.Global == params.Local { + iter, err = aclTokenListAll(tx, params.EnterpriseMeta) } else { - iter, err = aclTokenList(tx, entMeta, local) + iter, err = aclTokenList(tx, params.EnterpriseMeta, params.Local) } - } else if policy != "" && role == "" && methodName == "" { - iter, err = aclTokenListByPolicy(tx, policy, entMeta) + } else if params.Policy != "" && params.Role == "" && params.MethodName == "" && params.ServiceName == "" { + // Find by policy + iter, err = aclTokenListByPolicy(tx, params.Policy, params.EnterpriseMeta) + needLocalityFilter = true + + } else if params.Policy == "" && params.Role != "" && params.MethodName == "" && params.ServiceName == "" { + // Find by role + iter, err = aclTokenListByRole(tx, params.Role, params.EnterpriseMeta) needLocalityFilter = true - } else if policy == "" && role != "" && methodName == "" { - iter, err = aclTokenListByRole(tx, role, entMeta) + } else if params.Policy == "" && params.Role == "" && params.MethodName != "" && params.ServiceName == "" { + // Find by methodName + iter, err = aclTokenListByAuthMethod(tx, params.MethodName, params.MethodMeta, params.EnterpriseMeta) needLocalityFilter = true - } else if policy == "" && role == "" && methodName != "" { - iter, err = aclTokenListByAuthMethod(tx, methodName, methodMeta, entMeta) + } else if params.Policy == "" && params.Role == "" && params.MethodName == "" && params.ServiceName != "" { + // Find by the service identity's serviceName + iter, err = aclTokenListByServiceName(tx, params.ServiceName, params.EnterpriseMeta) needLocalityFilter = true } else { - return 0, nil, fmt.Errorf("can only filter by one of policy, role, or methodName at a time") + return 0, nil, fmt.Errorf("can only filter by one of policy, role, serviceName, or methodName at a time") } if err != nil { return 0, nil, fmt.Errorf("failed acl token lookup: %v", err) } - if needLocalityFilter && global != local { + if needLocalityFilter && params.Global != params.Local { iter = memdb.NewFilterIterator(iter, func(raw interface{}) bool { token, ok := raw.(*structs.ACLToken) if !ok { return true } - if global && !token.Local { + if params.Global && !token.Local { return false - } else if local && token.Local { + } else if params.Local && token.Local { return false } @@ -695,7 +730,7 @@ func (s *Store) ACLTokenList(ws memdb.WatchSet, local, global bool, policy, role } // Get the table index. - idx := aclTokenMaxIndex(tx, nil, entMeta) + idx := aclTokenMaxIndex(tx, nil, params.EnterpriseMeta) return idx, result, nil } @@ -881,18 +916,18 @@ func aclPolicySetTxn(tx WriteTxn, idx uint64, policy *structs.ACLPolicy) error { } if existing != nil { - if policy.ID == structs.ACLPolicyGlobalManagementID { + if builtinPolicy, ok := structs.ACLBuiltinPolicies[policy.ID]; ok { // Only the name and description are modifiable - // Here we specifically check that the rules on the global management policy + // Here we specifically check that the rules on the builtin policy // are identical to the correct policy rules within the binary. This is opposed // to checking against the current rules to allow us to update the rules during // upgrades. - if policy.Rules != structs.ACLPolicyGlobalManagement { - return fmt.Errorf("Changing the Rules for the builtin global-management policy is not permitted") + if policy.Rules != builtinPolicy.Rules { + return fmt.Errorf("Changing the Rules for the builtin %s policy is not permitted", builtinPolicy.Name) } if policy.Datacenters != nil && len(policy.Datacenters) != 0 { - return fmt.Errorf("Changing the Datacenters of the builtin global-management policy is not permitted") + return fmt.Errorf("Changing the Datacenters of the builtin %s policy is not permitted", builtinPolicy.Name) } } } @@ -1059,8 +1094,8 @@ func aclPolicyDeleteTxn(tx WriteTxn, idx uint64, value string, fn aclPolicyGetFn policy := rawPolicy.(*structs.ACLPolicy) - if policy.ID == structs.ACLPolicyGlobalManagementID { - return fmt.Errorf("Deletion of the builtin global-management policy is not permitted") + if builtinPolicy, ok := structs.ACLBuiltinPolicies[policy.ID]; ok { + return fmt.Errorf("Deletion of the builtin %s policy is not permitted", builtinPolicy.Name) } return aclPolicyDeleteWithPolicy(tx, policy, idx) diff --git a/agent/consul/state/acl_oss.go b/agent/consul/state/acl_ce.go similarity index 97% rename from agent/consul/state/acl_oss.go rename to agent/consul/state/acl_ce.go index 68671307132..d1a176c0ade 100644 --- a/agent/consul/state/acl_oss.go +++ b/agent/consul/state/acl_ce.go @@ -73,6 +73,10 @@ func aclTokenListByAuthMethod(tx ReadTxn, authMethod string, _, _ *acl.Enterpris return tx.Get(tableACLTokens, indexAuthMethod, AuthMethodQuery{Value: authMethod}) } +func aclTokenListByServiceName(tx ReadTxn, serviceName string, entMeta *acl.EnterpriseMeta) (memdb.ResultIterator, error) { + return tx.Get(tableACLTokens, indexServiceName, Query{Value: serviceName}) +} + func aclTokenDeleteWithToken(tx WriteTxn, token *structs.ACLToken, idx uint64) error { // remove the token if err := tx.Delete(tableACLTokens, token); err != nil { diff --git a/agent/consul/state/acl_oss_test.go b/agent/consul/state/acl_ce_test.go similarity index 100% rename from agent/consul/state/acl_oss_test.go rename to agent/consul/state/acl_ce_test.go diff --git a/agent/consul/state/acl_schema.go b/agent/consul/state/acl_schema.go index c3fdff40a1b..521cb06edd9 100644 --- a/agent/consul/state/acl_schema.go +++ b/agent/consul/state/acl_schema.go @@ -19,6 +19,7 @@ const ( indexAccessor = "accessor" indexPolicies = "policies" indexRoles = "roles" + indexServiceName = "service-name" indexAuthMethod = "authmethod" indexLocality = "locality" indexName = "name" @@ -103,6 +104,15 @@ func tokensTableSchema() *memdb.TableSchema { writeIndex: indexExpiresLocalFromACLToken, }, }, + indexServiceName: { + Name: indexServiceName, + AllowMissing: true, + Unique: false, + Indexer: indexerMulti[Query, *structs.ACLToken]{ + readIndex: indexFromQuery, + writeIndexMulti: indexServiceNameFromACLToken, + }, + }, }, } } @@ -395,6 +405,21 @@ func indexExpiresFromACLToken(t *structs.ACLToken, local bool) ([]byte, error) { return b.Bytes(), nil } +func indexServiceNameFromACLToken(token *structs.ACLToken) ([][]byte, error) { + vals := make([][]byte, 0, len(token.ServiceIdentities)) + for _, id := range token.ServiceIdentities { + if id != nil && id.ServiceName != "" { + var b indexBuilder + b.String(strings.ToLower(id.ServiceName)) + vals = append(vals, b.Bytes()) + } + } + if len(vals) == 0 { + return nil, errMissingValueForIndex + } + return vals, nil +} + func authMethodsTableSchema() *memdb.TableSchema { return &memdb.TableSchema{ Name: tableACLAuthMethods, diff --git a/agent/consul/state/acl_test.go b/agent/consul/state/acl_test.go index 5e01514730f..2e4aeb5dceb 100644 --- a/agent/consul/state/acl_test.go +++ b/agent/consul/state/acl_test.go @@ -13,7 +13,6 @@ import ( "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/structs" - "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/proto/pbacl" ) @@ -28,16 +27,17 @@ const ( ) func setupGlobalManagement(t *testing.T, s *Store) { - policy := structs.ACLPolicy{ - ID: structs.ACLPolicyGlobalManagementID, - Name: "global-management", - Description: "Builtin Policy that grants unlimited access", - Rules: structs.ACLPolicyGlobalManagement, - } + policy := structs.ACLBuiltinPolicies[structs.ACLPolicyGlobalManagementID] policy.SetHash(true) require.NoError(t, s.ACLPolicySet(1, &policy)) } +func setupBuiltinGlobalReadOnly(t *testing.T, s *Store) { + policy := structs.ACLBuiltinPolicies[structs.ACLPolicyGlobalReadOnlyID] + policy.SetHash(true) + require.NoError(t, s.ACLPolicySet(2, &policy)) +} + func setupAnonymous(t *testing.T, s *Store) { token := structs.ACLToken{ AccessorID: acl.AnonymousTokenID, @@ -51,6 +51,7 @@ func setupAnonymous(t *testing.T, s *Store) { func testACLStateStore(t *testing.T) *Store { s := testStateStore(t) setupGlobalManagement(t, s) + setupBuiltinGlobalReadOnly(t, s) setupAnonymous(t, s) return s } @@ -182,6 +183,7 @@ func TestStateStore_ACLBootstrap(t *testing.T) { s := testStateStore(t) setupGlobalManagement(t, s) + setupBuiltinGlobalReadOnly(t, s) canBootstrap, index, err := s.CanBootstrapACLToken() require.NoError(t, err) @@ -209,6 +211,7 @@ func TestStateStore_ACLBootstrap(t *testing.T) { require.Equal(t, uint64(3), index) // Make sure the ACLs are in an expected state. + // nolint:staticcheck _, tokens, err := s.ACLTokenList(nil, true, true, "", "", "", nil, nil) require.NoError(t, err) require.Len(t, tokens, 1) @@ -223,6 +226,7 @@ func TestStateStore_ACLBootstrap(t *testing.T) { err = s.ACLBootstrap(32, index, token2.Clone()) require.NoError(t, err) + // nolint:staticcheck _, tokens, err = s.ACLTokenList(nil, true, true, "", "", "", nil, nil) require.NoError(t, err) require.Len(t, tokens, 2) @@ -844,18 +848,36 @@ func TestStateStore_ACLToken_List(t *testing.T) { AuthMethod: "test", Local: true, }, + // the serviceName specific token + &structs.ACLToken{ + AccessorID: "80c900e1-2fc5-4685-ae29-1b2d17fc30e4", + SecretID: "9d229cfd-ec4b-4d31-a6fd-ecbcb2a41d41", + ServiceIdentities: []*structs.ACLServiceIdentity{ + {ServiceName: "sn1"}, + }, + }, + // the serviceName specific token and local + &structs.ACLToken{ + AccessorID: "a14fa45e-0afe-4b44-961d-a430030ccfe2", + SecretID: "17f696b9-448a-4bd3-936b-08c92c66530f", + ServiceIdentities: []*structs.ACLServiceIdentity{ + {ServiceName: "sn1"}, + }, + Local: true, + }, } require.NoError(t, s.ACLTokenBatchSet(2, tokens, ACLTokenSetOptions{})) type testCase struct { - name string - local bool - global bool - policy string - role string - methodName string - accessors []string + name string + local bool + global bool + policy string + role string + methodName string + serviceName string + accessors []string } cases := []testCase{ @@ -871,6 +893,7 @@ func TestStateStore_ACLToken_List(t *testing.T) { "47eea4da-bda1-48a6-901c-3e36d2d9262f", // policy + global "54866514-3cf2-4fec-8a8a-710583831834", // mgmt + global "74277ae1-6a9b-4035-b444-2370fe6a2cb5", // authMethod + global + "80c900e1-2fc5-4685-ae29-1b2d17fc30e4", // serviceName + global "a7715fde-8954-4c92-afbc-d84c6ecdc582", // role + global }, }, @@ -884,6 +907,7 @@ func TestStateStore_ACLToken_List(t *testing.T) { accessors: []string{ "211f0360-ef53-41d3-9d4d-db84396eb6c0", // authMethod + local "4915fc9d-3726-4171-b588-6c271f45eecd", // policy + local + "a14fa45e-0afe-4b44-961d-a430030ccfe2", // serviceName + local "cadb4f13-f62a-49ab-ab3f-5a7e01b925d9", // role + local "f1093997-b6c7-496d-bfb8-6b1b1895641b", // mgmt + local }, @@ -978,6 +1002,30 @@ func TestStateStore_ACLToken_List(t *testing.T) { "74277ae1-6a9b-4035-b444-2370fe6a2cb5", // authMethod + global }, }, + { + name: "ServiceName - Local", + local: true, + global: false, + policy: "", + role: "", + methodName: "", + serviceName: "sn1", + accessors: []string{ + "a14fa45e-0afe-4b44-961d-a430030ccfe2", // serviceName + local + }, + }, + { + name: "ServiceName - Global", + local: false, + global: true, + policy: "", + role: "", + methodName: "", + serviceName: "sn1", + accessors: []string{ + "80c900e1-2fc5-4685-ae29-1b2d17fc30e4", // serviceName + global + }, + }, { name: "All", local: true, @@ -992,6 +1040,8 @@ func TestStateStore_ACLToken_List(t *testing.T) { "4915fc9d-3726-4171-b588-6c271f45eecd", // policy + local "54866514-3cf2-4fec-8a8a-710583831834", // mgmt + global "74277ae1-6a9b-4035-b444-2370fe6a2cb5", // authMethod + global + "80c900e1-2fc5-4685-ae29-1b2d17fc30e4", // serviceName + global + "a14fa45e-0afe-4b44-961d-a430030ccfe2", // serviceName + local "a7715fde-8954-4c92-afbc-d84c6ecdc582", // role + global "cadb4f13-f62a-49ab-ab3f-5a7e01b925d9", // role + local "f1093997-b6c7-496d-bfb8-6b1b1895641b", // mgmt + local @@ -999,14 +1049,27 @@ func TestStateStore_ACLToken_List(t *testing.T) { }, } - for _, tc := range []struct{ policy, role, methodName string }{ - {testPolicyID_A, testRoleID_A, "test"}, - {"", testRoleID_A, "test"}, - {testPolicyID_A, "", "test"}, - {testPolicyID_A, testRoleID_A, ""}, + for _, tc := range []struct{ policy, role, methodName, serviceName string }{ + {testPolicyID_A, testRoleID_A, "test", ""}, + {"", testRoleID_A, "test", ""}, + {testPolicyID_A, "", "test", ""}, + {testPolicyID_A, testRoleID_A, "", ""}, + {testPolicyID_A, "", "", "test"}, } { - t.Run(fmt.Sprintf("can't filter on more than one: %s/%s/%s", tc.policy, tc.role, tc.methodName), func(t *testing.T) { - _, _, err := s.ACLTokenList(nil, false, false, tc.policy, tc.role, tc.methodName, nil, nil) + t.Run(fmt.Sprintf("can't filter on more than one: %s/%s/%s/%s", tc.policy, tc.role, tc.methodName, tc.serviceName), func(t *testing.T) { + var err error + if tc.serviceName == "" { + // The legacy call can only be tested when the serviceName is not specified + // nolint:staticcheck + _, _, err = s.ACLTokenList(nil, false, false, tc.policy, tc.role, tc.methodName, nil, nil) + require.Error(t, err) + } + _, _, err = s.ACLTokenListWithParameters(nil, ACLTokenListParameters{ + Policy: tc.policy, + Role: tc.role, + MethodName: tc.methodName, + ServiceName: tc.serviceName, + }) require.Error(t, err) }) } @@ -1015,12 +1078,33 @@ func TestStateStore_ACLToken_List(t *testing.T) { tc := tc // capture range variable t.Run(tc.name, func(t *testing.T) { t.Parallel() - _, tokens, err := s.ACLTokenList(nil, tc.local, tc.global, tc.policy, tc.role, tc.methodName, nil, nil) - require.NoError(t, err) - require.Len(t, tokens, len(tc.accessors)) - tokens.Sort() - for i, token := range tokens { - require.Equal(t, tc.accessors[i], token.AccessorID) + // Test old function + if tc.serviceName == "" { + // nolint:staticcheck + _, tokens, err := s.ACLTokenList(nil, tc.local, tc.global, tc.policy, tc.role, tc.methodName, nil, nil) + require.NoError(t, err) + require.Len(t, tokens, len(tc.accessors)) + tokens.Sort() + for i, token := range tokens { + require.Equal(t, tc.accessors[i], token.AccessorID) + } + } + // Test new function + { + _, tokens, err := s.ACLTokenListWithParameters(nil, ACLTokenListParameters{ + Local: tc.local, + Global: tc.global, + Policy: tc.policy, + Role: tc.role, + ServiceName: tc.serviceName, + MethodName: tc.methodName, + }) + require.NoError(t, err) + require.Len(t, tokens, len(tc.accessors)) + tokens.Sort() + for i, token := range tokens { + require.Equal(t, tc.accessors[i], token.AccessorID) + } } }) } @@ -1075,6 +1159,7 @@ func TestStateStore_ACLToken_FixupPolicyLinks(t *testing.T) { require.Equal(t, "node-read-renamed", retrieved.Policies[0].Name) // list tokens without stale links + // nolint:staticcheck _, tokens, err := s.ACLTokenList(nil, true, true, "", "", "", nil, nil) require.NoError(t, err) @@ -1119,6 +1204,7 @@ func TestStateStore_ACLToken_FixupPolicyLinks(t *testing.T) { require.Len(t, retrieved.Policies, 0) // list tokens without stale links + // nolint:staticcheck _, tokens, err = s.ACLTokenList(nil, true, true, "", "", "", nil, nil) require.NoError(t, err) @@ -1204,6 +1290,7 @@ func TestStateStore_ACLToken_FixupRoleLinks(t *testing.T) { require.Equal(t, "node-read-role-renamed", retrieved.Roles[0].Name) // list tokens without stale links + // nolint:staticcheck _, tokens, err := s.ACLTokenList(nil, true, true, "", "", "", nil, nil) require.NoError(t, err) @@ -1248,6 +1335,7 @@ func TestStateStore_ACLToken_FixupRoleLinks(t *testing.T) { require.Len(t, retrieved.Roles, 0) // list tokens without stale links + // nolint:staticcheck _, tokens, err = s.ACLTokenList(nil, true, true, "", "", "", nil, nil) require.NoError(t, err) @@ -1428,7 +1516,7 @@ func TestStateStore_ACLPolicy_SetGet(t *testing.T) { ID: structs.ACLPolicyGlobalManagementID, Name: "global-management", Description: "Global Management", - Rules: structs.ACLPolicyGlobalManagement, + Rules: structs.ACLPolicyGlobalManagementRules, Datacenters: []string{"dc1"}, } @@ -1442,7 +1530,7 @@ func TestStateStore_ACLPolicy_SetGet(t *testing.T) { ID: structs.ACLPolicyGlobalManagementID, Name: "management", Description: "Modified", - Rules: structs.ACLPolicyGlobalManagement, + Rules: structs.ACLPolicyGlobalManagementRules, } require.NoError(t, s.ACLPolicySet(3, &policy)) @@ -1491,8 +1579,8 @@ func TestStateStore_ACLPolicy_SetGet(t *testing.T) { require.NoError(t, err) require.NotNil(t, rpolicy) require.Equal(t, "global-management", rpolicy.Name) - require.Equal(t, "Builtin Policy that grants unlimited access", rpolicy.Description) - require.Equal(t, structs.ACLPolicyGlobalManagement, rpolicy.Rules) + require.Equal(t, structs.ACLPolicyGlobalManagementDesc, rpolicy.Description) + require.Equal(t, structs.ACLPolicyGlobalManagementRules, rpolicy.Rules) require.Len(t, rpolicy.Datacenters, 0) require.Equal(t, uint64(1), rpolicy.CreateIndex) require.Equal(t, uint64(1), rpolicy.ModifyIndex) @@ -1662,31 +1750,39 @@ func TestStateStore_ACLPolicy_List(t *testing.T) { _, policies, err := s.ACLPolicyList(nil, nil) require.NoError(t, err) - require.Len(t, policies, 3) + require.Len(t, policies, 4) policies.Sort() require.Equal(t, structs.ACLPolicyGlobalManagementID, policies[0].ID) - require.Equal(t, "global-management", policies[0].Name) - require.Equal(t, "Builtin Policy that grants unlimited access", policies[0].Description) + require.Equal(t, structs.ACLPolicyGlobalManagementName, policies[0].Name) + require.Equal(t, structs.ACLPolicyGlobalManagementDesc, policies[0].Description) require.Empty(t, policies[0].Datacenters) require.NotEqual(t, []byte{}, policies[0].Hash) require.Equal(t, uint64(1), policies[0].CreateIndex) require.Equal(t, uint64(1), policies[0].ModifyIndex) - require.Equal(t, "a2719052-40b3-4a4b-baeb-f3df1831a217", policies[1].ID) - require.Equal(t, "acl-write-dc3", policies[1].Name) - require.Equal(t, "Can manage ACLs in dc3", policies[1].Description) - require.ElementsMatch(t, []string{"dc3"}, policies[1].Datacenters) - require.Nil(t, policies[1].Hash) + require.Equal(t, structs.ACLPolicyGlobalReadOnlyID, policies[1].ID) + require.Equal(t, structs.ACLPolicyGlobalReadOnlyName, policies[1].Name) + require.Equal(t, structs.ACLPolicyGlobalReadOnlyDesc, policies[1].Description) + require.Empty(t, policies[1].Datacenters) + require.NotEqual(t, []byte{}, policies[1].Hash) require.Equal(t, uint64(2), policies[1].CreateIndex) require.Equal(t, uint64(2), policies[1].ModifyIndex) - require.Equal(t, "a4f68bd6-3af5-4f56-b764-3c6f20247879", policies[2].ID) - require.Equal(t, "service-read", policies[2].Name) - require.Equal(t, "", policies[2].Description) - require.Empty(t, policies[2].Datacenters) + require.Equal(t, "a2719052-40b3-4a4b-baeb-f3df1831a217", policies[2].ID) + require.Equal(t, "acl-write-dc3", policies[2].Name) + require.Equal(t, "Can manage ACLs in dc3", policies[2].Description) + require.ElementsMatch(t, []string{"dc3"}, policies[2].Datacenters) require.Nil(t, policies[2].Hash) require.Equal(t, uint64(2), policies[2].CreateIndex) require.Equal(t, uint64(2), policies[2].ModifyIndex) + + require.Equal(t, "a4f68bd6-3af5-4f56-b764-3c6f20247879", policies[3].ID) + require.Equal(t, "service-read", policies[3].Name) + require.Equal(t, "", policies[3].Description) + require.Empty(t, policies[3].Datacenters) + require.Nil(t, policies[3].Hash) + require.Equal(t, uint64(2), policies[3].CreateIndex) + require.Equal(t, uint64(2), policies[3].ModifyIndex) } func TestStateStore_ACLPolicy_Delete(t *testing.T) { @@ -2675,16 +2771,19 @@ func TestStateStore_ACLAuthMethod_GlobalNameShadowing_TokenTest(t *testing.T) { } require.True(t, t.Run("list local only", func(t *testing.T) { + // nolint:staticcheck _, got, err := s.ACLTokenList(nil, true, false, "", "", "test", defaultEntMeta, defaultEntMeta) require.NoError(t, err) require.ElementsMatch(t, []string{methodDC2_tok1, methodDC2_tok2}, toList(got)) })) require.True(t, t.Run("list global only", func(t *testing.T) { + // nolint:staticcheck _, got, err := s.ACLTokenList(nil, false, true, "", "", "test", defaultEntMeta, defaultEntMeta) require.NoError(t, err) require.ElementsMatch(t, []string{methodDC1_tok1, methodDC1_tok2}, toList(got)) })) require.True(t, t.Run("list both", func(t *testing.T) { + // nolint:staticcheck _, got, err := s.ACLTokenList(nil, true, true, "", "", "test", defaultEntMeta, defaultEntMeta) require.NoError(t, err) require.ElementsMatch(t, []string{methodDC1_tok1, methodDC1_tok2, methodDC2_tok1, methodDC2_tok2}, toList(got)) @@ -2696,16 +2795,19 @@ func TestStateStore_ACLAuthMethod_GlobalNameShadowing_TokenTest(t *testing.T) { })) require.True(t, t.Run("list local only (after dc2 delete)", func(t *testing.T) { + // nolint:staticcheck _, got, err := s.ACLTokenList(nil, true, false, "", "", "test", defaultEntMeta, defaultEntMeta) require.NoError(t, err) require.Empty(t, got) })) require.True(t, t.Run("list global only (after dc2 delete)", func(t *testing.T) { + // nolint:staticcheck _, got, err := s.ACLTokenList(nil, false, true, "", "", "test", defaultEntMeta, defaultEntMeta) require.NoError(t, err) require.ElementsMatch(t, []string{methodDC1_tok1, methodDC1_tok2}, toList(got)) })) require.True(t, t.Run("list both (after dc2 delete)", func(t *testing.T) { + // nolint:staticcheck _, got, err := s.ACLTokenList(nil, true, true, "", "", "test", defaultEntMeta, defaultEntMeta) require.NoError(t, err) require.ElementsMatch(t, []string{methodDC1_tok1, methodDC1_tok2}, toList(got)) @@ -3496,6 +3598,7 @@ func TestStateStore_ACLTokens_Snapshot_Restore(t *testing.T) { require.NoError(t, s.ACLRoleBatchSet(2, roles, false)) // Read the restored ACLs back out and verify that they match. + // nolint:staticcheck idx, res, err := s.ACLTokenList(nil, true, true, "", "", "", nil, nil) require.NoError(t, err) require.Equal(t, uint64(4), idx) @@ -3570,7 +3673,6 @@ func TestStateStore_ACLPolicies_Snapshot_Restore(t *testing.T) { } func TestTokenPoliciesIndex(t *testing.T) { - lib.SeedMathRand() idIndex := &memdb.IndexSchema{ Name: "id", diff --git a/agent/consul/state/catalog.go b/agent/consul/state/catalog.go index 4d645013680..003bf409f4e 100644 --- a/agent/consul/state/catalog.go +++ b/agent/consul/state/catalog.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/hashicorp/go-memdb" - "github.com/mitchellh/copystructure" "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/configentry" @@ -900,12 +899,17 @@ func ensureServiceTxn(tx WriteTxn, idx uint64, node string, preserveIndexes bool return fmt.Errorf("failed updating gateway mapping: %s", err) } + if svc.PeerName == "" && sn.Name != "" { + if err := upsertKindServiceName(tx, idx, structs.ServiceKindConnectEnabled, sn); err != nil { + return fmt.Errorf("failed to persist service name as connect-enabled: %v", err) + } + } + + // Update the virtual IP for the service supported, err := virtualIPsSupported(tx, nil) if err != nil { return err } - - // Update the virtual IP for the service if supported { psn := structs.PeeredServiceName{Peer: svc.PeerName, ServiceName: sn} vip, err := assignServiceVirtualIP(tx, idx, psn) @@ -1181,7 +1185,7 @@ func serviceListTxn(tx ReadTxn, ws memdb.WatchSet, entMeta *acl.EnterpriseMeta, unique := make(map[structs.ServiceName]struct{}) for service := services.Next(); service != nil; service = services.Next() { svc := service.(*structs.ServiceNode) - unique[svc.CompoundServiceName()] = struct{}{} + unique[svc.CompoundServiceName().ServiceName] = struct{}{} } results := make(structs.ServiceList, 0, len(unique)) @@ -1915,17 +1919,17 @@ func (s *Store) deleteServiceTxn(tx WriteTxn, idx uint64, nodeName, serviceID st return fmt.Errorf("failed updating service-kind indexes: %w", err) } // Update the node indexes as the service information is included in node catalog queries. - if err := catalogUpdateNodesIndexes(tx, idx, entMeta, peerName); err != nil { + if err := catalogUpdateNodesIndexes(tx, idx, entMeta, svc.PeerName); err != nil { return fmt.Errorf("failed updating nodes indexes: %w", err) } - if err := catalogUpdateNodeIndexes(tx, idx, nodeName, entMeta, peerName); err != nil { + if err := catalogUpdateNodeIndexes(tx, idx, nodeName, entMeta, svc.PeerName); err != nil { return fmt.Errorf("failed updating node indexes: %w", err) } - name := svc.CompoundServiceName() + psn := svc.CompoundServiceName() if err := cleanupMeshTopology(tx, idx, svc); err != nil { - return fmt.Errorf("failed to clean up mesh-topology associations for %q: %v", name.String(), err) + return fmt.Errorf("failed to clean up mesh-topology associations for %q: %v", psn.String(), err) } q := Query{ @@ -1952,22 +1956,42 @@ func (s *Store) deleteServiceTxn(tx WriteTxn, idx uint64, nodeName, serviceID st if err := catalogUpdateServiceExtinctionIndex(tx, idx, entMeta, svc.PeerName); err != nil { return err } - psn := structs.PeeredServiceName{Peer: svc.PeerName, ServiceName: name} if err := freeServiceVirtualIP(tx, idx, psn, nil); err != nil { - return fmt.Errorf("failed to clean up virtual IP for %q: %v", name.String(), err) + return fmt.Errorf("failed to clean up virtual IP for %q: %v", psn.String(), err) } - if err := cleanupKindServiceName(tx, idx, svc.CompoundServiceName(), svc.ServiceKind); err != nil { - return fmt.Errorf("failed to persist service name: %v", err) + + if svc.PeerName == "" { + if err := cleanupKindServiceName(tx, idx, psn.ServiceName, svc.ServiceKind); err != nil { + return fmt.Errorf("failed to persist service name: %v", err) + } } } } else { return fmt.Errorf("Could not find any service %s: %s", svc.ServiceName, err) } + // Cleanup ConnectEnabled for this service if none exist. + if svc.PeerName == "" && (svc.ServiceKind == structs.ServiceKindConnectProxy || svc.ServiceConnect.Native) { + service := svc.ServiceName + if svc.ServiceKind == structs.ServiceKindConnectProxy { + service = svc.ServiceProxy.DestinationServiceName + } + sn := structs.ServiceName{Name: service, EnterpriseMeta: svc.EnterpriseMeta} + connectEnabled, err := serviceHasConnectEnabledInstances(tx, sn.Name, &sn.EnterpriseMeta) + if err != nil { + return fmt.Errorf("failed to search for connect instances for service %q: %w", sn.Name, err) + } + if !connectEnabled { + if err := cleanupKindServiceName(tx, idx, sn, structs.ServiceKindConnectEnabled); err != nil { + return fmt.Errorf("failed to cleanup connect-enabled service name: %v", err) + } + } + } + if svc.PeerName == "" { sn := structs.ServiceName{Name: svc.ServiceName, EnterpriseMeta: svc.EnterpriseMeta} if err := cleanupGatewayWildcards(tx, idx, sn, false); err != nil { - return fmt.Errorf("failed to clean up gateway-service associations for %q: %v", name.String(), err) + return fmt.Errorf("failed to clean up gateway-service associations for %q: %v", psn.String(), err) } } @@ -2672,7 +2696,7 @@ func (s *Store) CheckIngressServiceNodes(ws memdb.WatchSet, serviceName string, // De-dup services to lookup names := make(map[structs.ServiceName]struct{}) for _, n := range nodes { - names[n.CompoundServiceName()] = struct{}{} + names[n.CompoundServiceName().ServiceName] = struct{}{} } var results structs.CheckServiceNodes @@ -3634,7 +3658,7 @@ func updateGatewayNamespace(tx WriteTxn, idx uint64, service *structs.GatewaySer continue } - existing, err := tx.First(tableGatewayServices, indexID, service.Gateway, sn.CompoundServiceName(), service.Port) + existing, err := tx.First(tableGatewayServices, indexID, service.Gateway, sn.CompoundServiceName().ServiceName, service.Port) if err != nil { return fmt.Errorf("gateway service lookup failed: %s", err) } @@ -3731,6 +3755,27 @@ func serviceHasConnectInstances(tx WriteTxn, serviceName string, entMeta *acl.En return hasConnectInstance, hasNonConnectInstance, nil } +// serviceHasConnectEnabledInstances returns whether the given service name +// has a corresponding connect-proxy or connect-native instance. +// This function is mostly a clone of `serviceHasConnectInstances`, but it has +// an early return to improve performance and returns true if at least one +// connect-native instance exists. +func serviceHasConnectEnabledInstances(tx WriteTxn, serviceName string, entMeta *acl.EnterpriseMeta) (bool, error) { + query := Query{ + Value: serviceName, + EnterpriseMeta: *entMeta, + } + + svc, err := tx.First(tableServices, indexConnect, query) + if err != nil { + return false, fmt.Errorf("failed service lookup: %w", err) + } + if svc != nil { + return true, nil + } + return false, nil +} + // updateGatewayService associates services with gateways after an eligible event // ie. Registering a service in a namespace targeted by a gateway func updateGatewayService(tx WriteTxn, idx uint64, mapping *structs.GatewayService) error { @@ -4518,14 +4563,7 @@ func updateMeshTopology(tx WriteTxn, idx uint64, node string, svc *structs.NodeS var mapping *upstreamDownstream if existing, ok := obj.(*upstreamDownstream); ok { - rawCopy, err := copystructure.Copy(existing) - if err != nil { - return fmt.Errorf("failed to copy existing topology mapping: %v", err) - } - mapping, ok = rawCopy.(*upstreamDownstream) - if !ok { - return fmt.Errorf("unexpected topology type %T", rawCopy) - } + mapping := existing.DeepCopy() mapping.Refs[uid] = struct{}{} mapping.ModifyIndex = idx @@ -4567,7 +4605,10 @@ func updateMeshTopology(tx WriteTxn, idx uint64, node string, svc *structs.NodeS // cleanupMeshTopology removes a service from the mesh topology table // This is only safe to call when there are no more known instances of this proxy func cleanupMeshTopology(tx WriteTxn, idx uint64, service *structs.ServiceNode) error { - // TODO(peering): make this peering aware? + if service.PeerName != "" { + return nil + } + if service.ServiceKind != structs.ServiceKindConnectProxy { return nil } @@ -4588,14 +4629,7 @@ func cleanupMeshTopology(tx WriteTxn, idx uint64, service *structs.ServiceNode) // Do the updates in a separate loop so we don't trash the iterator. for _, m := range mappings { - rawCopy, err := copystructure.Copy(m) - if err != nil { - return fmt.Errorf("failed to copy existing topology mapping: %v", err) - } - copy, ok := rawCopy.(*upstreamDownstream) - if !ok { - return fmt.Errorf("unexpected topology type %T", rawCopy) - } + copy := m.DeepCopy() // Bail early if there's no reference to the proxy ID we're deleting if _, ok := copy.Refs[uid]; !ok { diff --git a/agent/consul/state/catalog_oss.go b/agent/consul/state/catalog_ce.go similarity index 99% rename from agent/consul/state/catalog_oss.go rename to agent/consul/state/catalog_ce.go index fccbf6984ef..ee599402d60 100644 --- a/agent/consul/state/catalog_oss.go +++ b/agent/consul/state/catalog_ce.go @@ -112,7 +112,7 @@ func catalogUpdateNodeExtinctionIndex(tx WriteTxn, idx uint64, _ *acl.Enterprise } func catalogInsertNode(tx WriteTxn, node *structs.Node) error { - // ensure that the Partition is always clear within the state store in OSS + // ensure that the Partition is always clear within the state store in CE node.Partition = "" // Insert the node and update the index. diff --git a/agent/consul/state/catalog_oss_test.go b/agent/consul/state/catalog_ce_test.go similarity index 100% rename from agent/consul/state/catalog_oss_test.go rename to agent/consul/state/catalog_ce_test.go diff --git a/agent/consul/state/catalog_events_oss.go b/agent/consul/state/catalog_events_ce.go similarity index 100% rename from agent/consul/state/catalog_events_oss.go rename to agent/consul/state/catalog_events_ce.go diff --git a/agent/consul/state/catalog_events_oss_test.go b/agent/consul/state/catalog_events_ce_test.go similarity index 92% rename from agent/consul/state/catalog_events_oss_test.go rename to agent/consul/state/catalog_events_ce_test.go index ace7cfe7120..8c29c692983 100644 --- a/agent/consul/state/catalog_events_oss_test.go +++ b/agent/consul/state/catalog_events_ce_test.go @@ -11,7 +11,7 @@ import ( "github.com/hashicorp/consul/agent/structs" ) -func TestEventPayloadCheckServiceNode_Subject_OSS(t *testing.T) { +func TestEventPayloadCheckServiceNode_Subject_CE(t *testing.T) { for desc, tc := range map[string]struct { evt EventPayloadCheckServiceNode sub string diff --git a/agent/consul/state/catalog_schema.deepcopy.go b/agent/consul/state/catalog_schema.deepcopy.go new file mode 100644 index 00000000000..406a7fdce79 --- /dev/null +++ b/agent/consul/state/catalog_schema.deepcopy.go @@ -0,0 +1,15 @@ +// generated by deep-copy -pointer-receiver -o ./catalog_schema.deepcopy.go -type upstreamDownstream ./; DO NOT EDIT. + +package state + +// DeepCopy generates a deep copy of *upstreamDownstream +func (o *upstreamDownstream) DeepCopy() *upstreamDownstream { + var cp upstreamDownstream = *o + if o.Refs != nil { + cp.Refs = make(map[string]struct{}, len(o.Refs)) + for k2, v2 := range o.Refs { + cp.Refs[k2] = v2 + } + } + return &cp +} diff --git a/agent/consul/state/catalog_test.go b/agent/consul/state/catalog_test.go index d354b9b094e..b5976bbd2b0 100644 --- a/agent/consul/state/catalog_test.go +++ b/agent/consul/state/catalog_test.go @@ -2577,20 +2577,49 @@ func TestStateStore_DeleteService(t *testing.T) { testRegisterService(t, s, 2, "node1", "service1") testRegisterCheck(t, s, 3, "node1", "service1", "check1", api.HealthPassing) - // Delete the service. + // register a node with a service on a cluster peer. + testRegisterNodeOpts(t, s, 4, "node1", func(n *structs.Node) error { + n.PeerName = "cluster-01" + return nil + }) + testRegisterServiceOpts(t, s, 5, "node1", "service1", func(service *structs.NodeService) { + service.PeerName = "cluster-01" + }) + + wsPeer := memdb.NewWatchSet() + _, ns, err := s.NodeServices(wsPeer, "node1", nil, "cluster-01") + require.Len(t, ns.Services, 1) + require.NoError(t, err) + ws := memdb.NewWatchSet() - _, _, err := s.NodeServices(ws, "node1", nil, "") + _, ns, err = s.NodeServices(ws, "node1", nil, "") + require.Len(t, ns.Services, 1) require.NoError(t, err) - if err := s.DeleteService(4, "node1", "service1", nil, ""); err != nil { - t.Fatalf("err: %s", err) + + { + // Delete the peered service. + err = s.DeleteService(6, "node1", "service1", nil, "cluster-01") + require.NoError(t, err) + require.True(t, watchFired(wsPeer)) + _, kindServiceNames, err := s.ServiceNamesOfKind(nil, structs.ServiceKindTypical) + require.NoError(t, err) + require.Len(t, kindServiceNames, 1) + require.Equal(t, "service1", kindServiceNames[0].Service.Name) } - if !watchFired(ws) { - t.Fatalf("bad") + + { + // Delete the service. + err = s.DeleteService(6, "node1", "service1", nil, "") + require.NoError(t, err) + require.True(t, watchFired(ws)) + _, kindServiceNames, err := s.ServiceNamesOfKind(nil, structs.ServiceKindTypical) + require.NoError(t, err) + require.Len(t, kindServiceNames, 0) } // Service doesn't exist. ws = memdb.NewWatchSet() - _, ns, err := s.NodeServices(ws, "node1", nil, "") + _, ns, err = s.NodeServices(ws, "node1", nil, "") if err != nil || ns == nil || len(ns.Services) != 0 { t.Fatalf("bad: %#v (err: %#v)", ns, err) } @@ -2605,15 +2634,15 @@ func TestStateStore_DeleteService(t *testing.T) { } // Index tables were updated. - assert.Equal(t, uint64(4), catalogChecksMaxIndex(tx, nil, "")) - assert.Equal(t, uint64(4), catalogServicesMaxIndex(tx, nil, "")) + assert.Equal(t, uint64(6), catalogChecksMaxIndex(tx, nil, "")) + assert.Equal(t, uint64(6), catalogServicesMaxIndex(tx, nil, "")) // Deleting a nonexistent service should be idempotent and not return an // error, nor fire a watch. - if err := s.DeleteService(5, "node1", "service1", nil, ""); err != nil { + if err := s.DeleteService(6, "node1", "service1", nil, ""); err != nil { t.Fatalf("err: %s", err) } - assert.Equal(t, uint64(4), catalogServicesMaxIndex(tx, nil, "")) + assert.Equal(t, uint64(6), catalogServicesMaxIndex(tx, nil, "")) if watchFired(ws) { t.Fatalf("bad") } @@ -8664,7 +8693,7 @@ func TestStateStore_EnsureService_ServiceNames(t *testing.T) { }, } - var idx uint64 + var idx, connectEnabledIdx uint64 testRegisterNode(t, s, idx, "node1") for _, svc := range services { @@ -8678,8 +8707,29 @@ func TestStateStore_EnsureService_ServiceNames(t *testing.T) { require.Len(t, gotNames, 1) require.Equal(t, svc.CompoundServiceName(), gotNames[0].Service) require.Equal(t, svc.Kind, gotNames[0].Kind) + if svc.Kind == structs.ServiceKindConnectProxy { + connectEnabledIdx = idx + } } + // A ConnectEnabled service should exist if a corresponding ConnectProxy or ConnectNative service exists. + verifyConnectEnabled := func(expectIdx uint64) { + gotIdx, gotNames, err := s.ServiceNamesOfKind(nil, structs.ServiceKindConnectEnabled) + require.NoError(t, err) + require.Equal(t, expectIdx, gotIdx) + require.Equal(t, []*KindServiceName{ + { + Kind: structs.ServiceKindConnectEnabled, + Service: structs.NewServiceName("foo", entMeta), + RaftIndex: structs.RaftIndex{ + CreateIndex: connectEnabledIdx, + ModifyIndex: connectEnabledIdx, + }, + }, + }, gotNames) + } + verifyConnectEnabled(connectEnabledIdx) + // Register another ingress gateway and there should be two names under the kind index newIngress := structs.NodeService{ Kind: structs.ServiceKindIngressGateway, @@ -8749,6 +8799,38 @@ func TestStateStore_EnsureService_ServiceNames(t *testing.T) { require.NoError(t, err) require.Equal(t, idx, gotIdx) require.Empty(t, got) + + // A ConnectEnabled entry should not be removed until all corresponding services are removed. + { + verifyConnectEnabled(connectEnabledIdx) + // Add a connect-native service. + idx++ + require.NoError(t, s.EnsureService(idx, "node1", &structs.NodeService{ + Kind: structs.ServiceKindTypical, + ID: "foo", + Service: "foo", + Address: "5.5.5.5", + Port: 5555, + EnterpriseMeta: *entMeta, + Connect: structs.ServiceConnect{ + Native: true, + }, + })) + verifyConnectEnabled(connectEnabledIdx) + + // Delete the proxy. This should not clean up the entry, because we still have a + // connect-native service registered. + idx++ + require.NoError(t, s.DeleteService(idx, "node1", "connect-proxy", entMeta, "")) + verifyConnectEnabled(connectEnabledIdx) + + // Remove the connect-native service to clear out the connect-enabled entry. + require.NoError(t, s.DeleteService(idx, "node1", "foo", entMeta, "")) + gotIdx, gotNames, err := s.ServiceNamesOfKind(nil, structs.ServiceKindConnectEnabled) + require.NoError(t, err) + require.Equal(t, idx, gotIdx) + require.Empty(t, gotNames) + } } func assertMaxIndexes(t *testing.T, tx ReadTxn, expect map[string]uint64, skip ...string) { diff --git a/agent/consul/state/config_entry_oss.go b/agent/consul/state/config_entry_ce.go similarity index 95% rename from agent/consul/state/config_entry_oss.go rename to agent/consul/state/config_entry_ce.go index ac4d1d6cb05..07e7167b383 100644 --- a/agent/consul/state/config_entry_oss.go +++ b/agent/consul/state/config_entry_ce.go @@ -62,7 +62,7 @@ func configIntentionsConvertToList(iter memdb.ResultIterator, _ *acl.EnterpriseM } // getExportedServicesMatchServicesNames returns a list of service names that are considered matches when -// found in a list of exported-services config entries. For OSS, namespace is not considered, so a match is one of: +// found in a list of exported-services config entries. For CE, namespace is not considered, so a match is one of: // - the service name matches // - the service name is a wildcard // diff --git a/agent/consul/state/config_entry_oss_test.go b/agent/consul/state/config_entry_ce_test.go similarity index 100% rename from agent/consul/state/config_entry_oss_test.go rename to agent/consul/state/config_entry_ce_test.go diff --git a/agent/consul/state/config_entry_intention.go b/agent/consul/state/config_entry_intention.go index 17183e7ceec..76b5a487af0 100644 --- a/agent/consul/state/config_entry_intention.go +++ b/agent/consul/state/config_entry_intention.go @@ -80,7 +80,7 @@ type ServiceIntentionSourceIndex struct { } // Compile-time assert that these interfaces hold to ensure that the -// methods correctly exist across the oss/ent split. +// methods correctly exist across the ce/ent split. var _ memdb.Indexer = (*ServiceIntentionSourceIndex)(nil) var _ memdb.MultiIndexer = (*ServiceIntentionSourceIndex)(nil) diff --git a/agent/consul/state/config_entry_intention_oss.go b/agent/consul/state/config_entry_intention_ce.go similarity index 100% rename from agent/consul/state/config_entry_intention_oss.go rename to agent/consul/state/config_entry_intention_ce.go diff --git a/agent/consul/state/coordinate_oss.go b/agent/consul/state/coordinate_ce.go similarity index 100% rename from agent/consul/state/coordinate_oss.go rename to agent/consul/state/coordinate_ce.go diff --git a/agent/consul/state/coordinate_oss_test.go b/agent/consul/state/coordinate_ce_test.go similarity index 100% rename from agent/consul/state/coordinate_oss_test.go rename to agent/consul/state/coordinate_ce_test.go diff --git a/agent/consul/state/deep-copy.sh b/agent/consul/state/deep-copy.sh new file mode 100755 index 00000000000..d976d921f3c --- /dev/null +++ b/agent/consul/state/deep-copy.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +readonly PACKAGE_DIR="$(dirname "${BASH_SOURCE[0]}")" +cd $PACKAGE_DIR + +# Uses: https://github.com/globusdigital/deep-copy +deep-copy \ + -pointer-receiver \ + -o ./catalog_schema.deepcopy.go \ + -type upstreamDownstream \ + ./ diff --git a/agent/consul/state/delay_oss.go b/agent/consul/state/delay_ce.go similarity index 100% rename from agent/consul/state/delay_oss.go rename to agent/consul/state/delay_ce.go diff --git a/agent/consul/state/graveyard_oss.go b/agent/consul/state/graveyard_ce.go similarity index 100% rename from agent/consul/state/graveyard_oss.go rename to agent/consul/state/graveyard_ce.go diff --git a/agent/consul/state/intention_oss.go b/agent/consul/state/intention_ce.go similarity index 100% rename from agent/consul/state/intention_oss.go rename to agent/consul/state/intention_ce.go diff --git a/agent/consul/state/kvs_oss.go b/agent/consul/state/kvs_ce.go similarity index 100% rename from agent/consul/state/kvs_oss.go rename to agent/consul/state/kvs_ce.go diff --git a/agent/consul/state/kvs_oss_test.go b/agent/consul/state/kvs_ce_test.go similarity index 100% rename from agent/consul/state/kvs_oss_test.go rename to agent/consul/state/kvs_ce_test.go diff --git a/agent/consul/state/operations_oss.go b/agent/consul/state/operations_ce.go similarity index 100% rename from agent/consul/state/operations_oss.go rename to agent/consul/state/operations_ce.go diff --git a/agent/consul/state/peering.go b/agent/consul/state/peering.go index 32e60450044..5fbd1b79f26 100644 --- a/agent/consul/state/peering.go +++ b/agent/consul/state/peering.go @@ -770,88 +770,181 @@ func exportedServicesForPeerTxn( maxIdx := peering.ModifyIndex entMeta := structs.NodeEnterpriseMetaInPartition(peering.Partition) - idx, conf, err := getExportedServicesConfigEntryTxn(tx, ws, nil, entMeta) + idx, exportConf, err := getExportedServicesConfigEntryTxn(tx, ws, nil, entMeta) if err != nil { return 0, nil, fmt.Errorf("failed to fetch exported-services config entry: %w", err) } if idx > maxIdx { maxIdx = idx } - if conf == nil { + if exportConf == nil { return maxIdx, &structs.ExportedServiceList{}, nil } var ( - normalSet = make(map[structs.ServiceName]struct{}) - discoSet = make(map[structs.ServiceName]struct{}) + // exportedServices will contain the listing of all service names that are being exported + // and will need to be queried for connect / discovery chain information. + exportedServices = make(map[structs.ServiceName]struct{}) + + // exportedConnectServices will contain the listing of all connect service names that are being exported. + exportedConnectServices = make(map[structs.ServiceName]struct{}) + + // namespaceConnectServices provides a listing of all connect service names for a particular partition+namespace pair. + namespaceConnectServices = make(map[acl.EnterpriseMeta]map[string]struct{}) + + // namespaceDiscoChains provides a listing of all disco chain names for a particular partition+namespace pair. + namespaceDiscoChains = make(map[acl.EnterpriseMeta]map[string]struct{}) ) - // At least one of the following should be true for a name for it to - // replicate: - // - // - are a discovery chain by definition (service-router, service-splitter, service-resolver) - // - have an explicit sidecar kind=connect-proxy - // - use connect native mode + // Helper function for inserting data and auto-creating maps. + insertEntry := func(m map[acl.EnterpriseMeta]map[string]struct{}, entMeta acl.EnterpriseMeta, name string) { + names, ok := m[entMeta] + if !ok { + names = make(map[string]struct{}) + m[entMeta] = names + } + names[name] = struct{}{} + } - for _, svc := range conf.Services { + // Build the set of all services that will be exported. + // Any possible namespace wildcards or "consul" services should be removed by this step. + for _, svc := range exportConf.Services { // Prevent exporting the "consul" service. if svc.Name == structs.ConsulServiceName { continue } - svcMeta := acl.NewEnterpriseMetaWithPartition(entMeta.PartitionOrDefault(), svc.Namespace) + svcEntMeta := acl.NewEnterpriseMetaWithPartition(entMeta.PartitionOrDefault(), svc.Namespace) + svcName := structs.NewServiceName(svc.Name, &svcEntMeta) - sawPeer := false + peerFound := false for _, consumer := range svc.Consumers { - name := structs.NewServiceName(svc.Name, &svcMeta) - - if _, ok := normalSet[name]; ok { - // Service was covered by a wildcard that was already accounted for - continue + if consumer.Peer == peering.Name { + peerFound = true + break } - if consumer.Peer != peering.Name { - continue + } + // Only look for more information if the matching peer was found. + if !peerFound { + continue + } + + // If this isn't a wildcard, we can simply add it to the list of services to watch and move to the next entry. + if svc.Name != structs.WildcardSpecifier { + exportedServices[svcName] = struct{}{} + continue + } + + // If all services in the namespace are exported by the wildcard, query those service names. + idx, typicalServices, err := serviceNamesOfKindTxn(tx, ws, structs.ServiceKindTypical, svcEntMeta) + if err != nil { + return 0, nil, fmt.Errorf("failed to get typical service names: %w", err) + } + if idx > maxIdx { + maxIdx = idx + } + for _, sn := range typicalServices { + // Prevent exporting the "consul" service. + if sn.Service.Name != structs.ConsulServiceName { + exportedServices[sn.Service] = struct{}{} } - sawPeer = true + } - if svc.Name != structs.WildcardSpecifier { - normalSet[name] = struct{}{} + // List all config entries of kind service-resolver, service-router, service-splitter, because they + // will be exported as connect services. + idx, discoChains, err := listDiscoveryChainNamesTxn(tx, ws, nil, svcEntMeta) + if err != nil { + return 0, nil, fmt.Errorf("failed to get discovery chain names: %w", err) + } + if idx > maxIdx { + maxIdx = idx + } + for _, sn := range discoChains { + // Prevent exporting the "consul" service. + if sn.Name != structs.ConsulServiceName { + exportedConnectServices[sn] = struct{}{} + insertEntry(namespaceDiscoChains, svcEntMeta, sn.Name) } } + } - // If the target peer is a consumer, and all services in the namespace are exported, query those service names. - if sawPeer && svc.Name == structs.WildcardSpecifier { - idx, typicalServices, err := serviceNamesOfKindTxn(tx, ws, structs.ServiceKindTypical, svcMeta) + // At least one of the following should be true for a name to replicate it as a *connect* service: + // - are a discovery chain by definition (service-router, service-splitter, service-resolver) + // - have an explicit sidecar kind=connect-proxy + // - use connect native mode + // - are registered with a terminating gateway + populateConnectService := func(sn structs.ServiceName) error { + // Load all disco-chains in this namespace if we haven't already. + if _, ok := namespaceDiscoChains[sn.EnterpriseMeta]; !ok { + // Check to see if we have a discovery chain with the same name. + idx, chains, err := listDiscoveryChainNamesTxn(tx, ws, nil, sn.EnterpriseMeta) if err != nil { - return 0, nil, fmt.Errorf("failed to get typical service names: %w", err) + return fmt.Errorf("failed to get connect services: %w", err) } if idx > maxIdx { maxIdx = idx } - for _, s := range typicalServices { - // Prevent exporting the "consul" service. - if s.Service.Name == structs.ConsulServiceName { - continue - } - normalSet[s.Service] = struct{}{} + for _, sn := range chains { + insertEntry(namespaceDiscoChains, sn.EnterpriseMeta, sn.Name) } + } + // Check to see if we have the connect service. + if _, ok := namespaceDiscoChains[sn.EnterpriseMeta][sn.Name]; ok { + exportedConnectServices[sn] = struct{}{} + // Do not early return because we have multiple watches that should be established. + } - // list all config entries of kind service-resolver, service-router, service-splitter? - idx, discoChains, err := listDiscoveryChainNamesTxn(tx, ws, nil, svcMeta) + // Load all services in this namespace if we haven't already. + if _, ok := namespaceConnectServices[sn.EnterpriseMeta]; !ok { + // This is more efficient than querying the service instance table. + idx, connectServices, err := serviceNamesOfKindTxn(tx, ws, structs.ServiceKindConnectEnabled, sn.EnterpriseMeta) if err != nil { - return 0, nil, fmt.Errorf("failed to get discovery chain names: %w", err) + return fmt.Errorf("failed to get connect services: %w", err) } if idx > maxIdx { maxIdx = idx } - for _, sn := range discoChains { - discoSet[sn] = struct{}{} + for _, ksn := range connectServices { + insertEntry(namespaceConnectServices, sn.EnterpriseMeta, ksn.Service.Name) } } + // Check to see if we have the connect service. + if _, ok := namespaceConnectServices[sn.EnterpriseMeta][sn.Name]; ok { + exportedConnectServices[sn] = struct{}{} + // Do not early return because we have multiple watches that should be established. + } + + // Check if the service is exposed via terminating gateways. + svcGateways, err := tx.Get(tableGatewayServices, indexService, sn) + if err != nil { + return fmt.Errorf("failed gateway lookup for %q: %w", sn.Name, err) + } + ws.Add(svcGateways.WatchCh()) + for svc := svcGateways.Next(); svc != nil; svc = svcGateways.Next() { + gs, ok := svc.(*structs.GatewayService) + if !ok { + return fmt.Errorf("failed converting to GatewayService for %q", sn.Name) + } + if gs.GatewayKind == structs.ServiceKindTerminatingGateway { + exportedConnectServices[sn] = struct{}{} + break + } + } + + return nil } - normal := maps.SliceOfKeys(normalSet) - disco := maps.SliceOfKeys(discoSet) + // Perform queries and check if each service is connect-enabled. + for sn := range exportedServices { + // Do not query for data if we already know it's a connect service. + if _, ok := exportedConnectServices[sn]; ok { + continue + } + if err := populateConnectService(sn); err != nil { + return 0, nil, err + } + } + // Fetch the protocol / targets for connect services. chainInfo := make(map[structs.ServiceName]structs.ExportedDiscoveryChainInfo) populateChainInfo := func(svc structs.ServiceName) error { if _, ok := chainInfo[svc]; ok { @@ -899,21 +992,17 @@ func exportedServicesForPeerTxn( return nil } - for _, svc := range normal { - if err := populateChainInfo(svc); err != nil { - return 0, nil, err - } - } - for _, svc := range disco { + for svc := range exportedConnectServices { if err := populateChainInfo(svc); err != nil { return 0, nil, err } } - structs.ServiceList(normal).Sort() + sortedServices := maps.SliceOfKeys(exportedServices) + structs.ServiceList(sortedServices).Sort() list := &structs.ExportedServiceList{ - Services: normal, + Services: sortedServices, DiscoChains: chainInfo, } @@ -1339,7 +1428,7 @@ func peersForServiceTxn( ) // Ensure the metadata is defaulted since we make assertions against potentially empty values below. - // In OSS this is a no-op. + // In CE this is a no-op. if entMeta == nil { entMeta = acl.DefaultEnterpriseMeta() } diff --git a/agent/consul/state/peering_oss.go b/agent/consul/state/peering_ce.go similarity index 100% rename from agent/consul/state/peering_oss.go rename to agent/consul/state/peering_ce.go diff --git a/agent/consul/state/peering_oss_test.go b/agent/consul/state/peering_ce_test.go similarity index 100% rename from agent/consul/state/peering_oss_test.go rename to agent/consul/state/peering_ce_test.go diff --git a/agent/consul/state/peering_test.go b/agent/consul/state/peering_test.go index 2c2caaab9a9..81933500d2f 100644 --- a/agent/consul/state/peering_test.go +++ b/agent/consul/state/peering_test.go @@ -1908,18 +1908,28 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { }, }, { + // Should be exported as both a normal and disco chain (resolver). Name: "mysql", Consumers: []structs.ServiceConsumer{ {Peer: "my-peering"}, }, }, { + // Should be exported as both a normal and disco chain (connect-proxy). Name: "redis", Consumers: []structs.ServiceConsumer{ {Peer: "my-peering"}, }, }, { + // Should only be exported as a normal service. + Name: "prometheus", + Consumers: []structs.ServiceConsumer{ + {Peer: "my-peering"}, + }, + }, + { + // Should not be exported (different peer consumer) Name: "mongo", Consumers: []structs.ServiceConsumer{ {Peer: "my-other-peering"}, @@ -1932,12 +1942,37 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { require.True(t, watchFired(ws)) ws = memdb.NewWatchSet() + // Register extra things so that disco chain entries appear. + lastIdx++ + require.NoError(t, s.EnsureNode(lastIdx, &structs.Node{ + Node: "node1", Address: "10.0.0.1", + })) + lastIdx++ + require.NoError(t, s.EnsureService(lastIdx, "node1", &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + ID: "redis-sidecar-proxy", + Service: "redis-sidecar-proxy", + Port: 5005, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "redis", + }, + })) + ensureConfigEntry(t, &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "mysql", + EnterpriseMeta: *defaultEntMeta, + }) + expect := &structs.ExportedServiceList{ Services: []structs.ServiceName{ { Name: "mysql", EnterpriseMeta: *defaultEntMeta, }, + { + Name: "prometheus", + EnterpriseMeta: *defaultEntMeta, + }, { Name: "redis", EnterpriseMeta: *defaultEntMeta, @@ -1998,17 +2033,21 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { ws = memdb.NewWatchSet() expect := &structs.ExportedServiceList{ + // Only "billing" shows up, because there are no other service instances running, + // and "consul" is never exported. Services: []structs.ServiceName{ { Name: "billing", EnterpriseMeta: *defaultEntMeta, }, }, + // Only "mysql" appears because there it has a service resolver. + // "redis" does not appear, because it's a sidecar proxy without a corresponding service, so the wildcard doesn't find it. DiscoChains: map[structs.ServiceName]structs.ExportedDiscoveryChainInfo{ - newSN("billing"): { + newSN("mysql"): { Protocol: "tcp", TCPTargets: []*structs.DiscoveryTarget{ - newTarget("billing", "", "dc1"), + newTarget("mysql", "", "dc1"), }, }, }, @@ -2025,13 +2064,17 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { ID: "payments", Service: "payments", Port: 5000, })) - // The proxy will be ignored. + // The proxy will cause "payments" to be output in the disco chains. It will NOT be output + // in the normal services list. lastIdx++ require.NoError(t, s.EnsureService(lastIdx, "foo", &structs.NodeService{ Kind: structs.ServiceKindConnectProxy, ID: "payments-proxy", Service: "payments-proxy", Port: 5000, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "payments", + }, })) lastIdx++ // The consul service should never be exported. @@ -2099,10 +2142,11 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { }, DiscoChains: map[structs.ServiceName]structs.ExportedDiscoveryChainInfo{ // NOTE: no consul-redirect here - newSN("billing"): { + // NOTE: no billing here, because it does not have a proxy. + newSN("payments"): { Protocol: "http", }, - newSN("payments"): { + newSN("mysql"): { Protocol: "http", }, newSN("resolver"): { @@ -2129,6 +2173,9 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { lastIdx++ require.NoError(t, s.DeleteConfigEntry(lastIdx, structs.ServiceSplitter, "splitter", nil)) + lastIdx++ + require.NoError(t, s.DeleteConfigEntry(lastIdx, structs.ServiceResolver, "mysql", nil)) + require.True(t, watchFired(ws)) ws = memdb.NewWatchSet() @@ -2160,6 +2207,51 @@ func TestStateStore_ExportedServicesForPeer(t *testing.T) { require.Equal(t, expect, got) }) + testutil.RunStep(t, "terminating gateway services are exported", func(t *testing.T) { + lastIdx++ + require.NoError(t, s.EnsureService(lastIdx, "foo", &structs.NodeService{ + ID: "term-svc", Service: "term-svc", Port: 6000, + })) + lastIdx++ + require.NoError(t, s.EnsureService(lastIdx, "foo", &structs.NodeService{ + Kind: structs.ServiceKindTerminatingGateway, + Service: "some-terminating-gateway", + ID: "some-terminating-gateway", + Port: 9000, + })) + lastIdx++ + require.NoError(t, s.EnsureConfigEntry(lastIdx, &structs.TerminatingGatewayConfigEntry{ + Kind: structs.TerminatingGateway, + Name: "some-terminating-gateway", + Services: []structs.LinkedService{{Name: "term-svc"}}, + })) + + expect := &structs.ExportedServiceList{ + Services: []structs.ServiceName{ + newSN("payments"), + newSN("term-svc"), + }, + DiscoChains: map[structs.ServiceName]structs.ExportedDiscoveryChainInfo{ + newSN("payments"): { + Protocol: "http", + }, + newSN("resolver"): { + Protocol: "http", + }, + newSN("router"): { + Protocol: "http", + }, + newSN("term-svc"): { + Protocol: "http", + }, + }, + } + idx, got, err := s.ExportedServicesForPeer(ws, id, "dc1") + require.NoError(t, err) + require.Equal(t, lastIdx, idx) + require.Equal(t, expect, got) + }) + testutil.RunStep(t, "deleting the config entry clears exported services", func(t *testing.T) { expect := &structs.ExportedServiceList{} diff --git a/agent/consul/state/query_oss.go b/agent/consul/state/query_ce.go similarity index 100% rename from agent/consul/state/query_oss.go rename to agent/consul/state/query_ce.go diff --git a/agent/consul/state/schema_oss.go b/agent/consul/state/schema_ce.go similarity index 100% rename from agent/consul/state/schema_oss.go rename to agent/consul/state/schema_ce.go diff --git a/agent/consul/state/schema_oss_test.go b/agent/consul/state/schema_ce_test.go similarity index 100% rename from agent/consul/state/schema_oss_test.go rename to agent/consul/state/schema_ce_test.go diff --git a/agent/consul/state/session_oss.go b/agent/consul/state/session_ce.go similarity index 100% rename from agent/consul/state/session_oss.go rename to agent/consul/state/session_ce.go diff --git a/agent/consul/state/state_store_oss_test.go b/agent/consul/state/state_store_ce_test.go similarity index 100% rename from agent/consul/state/state_store_oss_test.go rename to agent/consul/state/state_store_ce_test.go diff --git a/agent/consul/state/state_store_test.go b/agent/consul/state/state_store_test.go index 88e5418c8d7..e1b1bbc4ded 100644 --- a/agent/consul/state/state_store_test.go +++ b/agent/consul/state/state_store_test.go @@ -200,11 +200,27 @@ func testRegisterConnectService(t *testing.T, s *Store, idx uint64, nodeID, serv }) } +func testRegisterAPIService(t *testing.T, s *Store, idx uint64, nodeID, serviceID string) { + testRegisterGatewayService(t, s, structs.ServiceKindAPIGateway, idx, nodeID, serviceID) +} + +func testRegisterTerminatingService(t *testing.T, s *Store, idx uint64, nodeID, serviceID string) { + testRegisterGatewayService(t, s, structs.ServiceKindTerminatingGateway, idx, nodeID, serviceID) +} + func testRegisterIngressService(t *testing.T, s *Store, idx uint64, nodeID, serviceID string) { + testRegisterGatewayService(t, s, structs.ServiceKindIngressGateway, idx, nodeID, serviceID) +} + +func testRegisterMeshService(t *testing.T, s *Store, idx uint64, nodeID, serviceID string) { + testRegisterGatewayService(t, s, structs.ServiceKindMeshGateway, idx, nodeID, serviceID) +} + +func testRegisterGatewayService(t *testing.T, s *Store, kind structs.ServiceKind, idx uint64, nodeID, serviceID string) { svc := &structs.NodeService{ ID: serviceID, Service: serviceID, - Kind: structs.ServiceKindIngressGateway, + Kind: kind, Address: "1.1.1.1", Port: 1111, } @@ -224,6 +240,7 @@ func testRegisterIngressService(t *testing.T, s *Store, idx uint64, nodeID, serv t.Fatalf("bad service: %#v", result) } } + func testRegisterCheck(t *testing.T, s *Store, idx uint64, nodeID string, serviceID string, checkID types.CheckID, state string) { testRegisterCheckWithPartition(t, s, idx, diff --git a/agent/consul/state/usage.go b/agent/consul/state/usage.go index 5e3d7ce9cb3..dd693225e0a 100644 --- a/agent/consul/state/usage.go +++ b/agent/consul/state/usage.go @@ -22,6 +22,7 @@ var allConnectKind = []string{ string(structs.ServiceKindIngressGateway), string(structs.ServiceKindMeshGateway), string(structs.ServiceKindTerminatingGateway), + string(structs.ServiceKindAPIGateway), connectNativeInstancesTable, } @@ -126,11 +127,11 @@ func updateUsage(tx WriteTxn, changes Changes) error { // changed, in order to compare it with the finished memdb state. // Make sure to account for the fact that services can change their names. if serviceNameChanged(change) { - serviceNameChanges[svc.CompoundServiceName()] += 1 + serviceNameChanges[svc.CompoundServiceName().ServiceName] += 1 before := change.Before.(*structs.ServiceNode) - serviceNameChanges[before.CompoundServiceName()] -= 1 + serviceNameChanges[before.CompoundServiceName().ServiceName] -= 1 } else { - serviceNameChanges[svc.CompoundServiceName()] += delta + serviceNameChanges[svc.CompoundServiceName().ServiceName] += delta } case "kvs": diff --git a/agent/consul/state/usage_oss.go b/agent/consul/state/usage_ce.go similarity index 100% rename from agent/consul/state/usage_oss.go rename to agent/consul/state/usage_ce.go diff --git a/agent/consul/state/usage_test.go b/agent/consul/state/usage_test.go index b54e3dcfa0a..2699ef6f223 100644 --- a/agent/consul/state/usage_test.go +++ b/agent/consul/state/usage_test.go @@ -176,16 +176,25 @@ func TestStateStore_Usage_ServiceUsage(t *testing.T) { testRegisterConnectNativeService(t, s, 13, "node1", "service-native") testRegisterConnectNativeService(t, s, 14, "node2", "service-native") testRegisterConnectNativeService(t, s, 15, "node2", "service-native-1") + testRegisterIngressService(t, s, 16, "node1", "ingress") + testRegisterMeshService(t, s, 17, "node1", "mesh") + testRegisterTerminatingService(t, s, 18, "node1", "terminating") + testRegisterAPIService(t, s, 19, "node1", "api") + testRegisterAPIService(t, s, 20, "node2", "api") ws := memdb.NewWatchSet() idx, usage, err := s.ServiceUsage(ws) require.NoError(t, err) - require.Equal(t, idx, uint64(15)) - require.Equal(t, 5, usage.Services) - require.Equal(t, 8, usage.ServiceInstances) + require.Equal(t, idx, uint64(20)) + require.Equal(t, 9, usage.Services) + require.Equal(t, 13, usage.ServiceInstances) require.Equal(t, 2, usage.ConnectServiceInstances[string(structs.ServiceKindConnectProxy)]) require.Equal(t, 3, usage.ConnectServiceInstances[connectNativeInstancesTable]) require.Equal(t, 6, usage.BillableServiceInstances) + require.Equal(t, 2, usage.ConnectServiceInstances[string(structs.ServiceKindAPIGateway)]) + require.Equal(t, 1, usage.ConnectServiceInstances[string(structs.ServiceKindIngressGateway)]) + require.Equal(t, 1, usage.ConnectServiceInstances[string(structs.ServiceKindTerminatingGateway)]) + require.Equal(t, 1, usage.ConnectServiceInstances[string(structs.ServiceKindMeshGateway)]) testRegisterSidecarProxy(t, s, 16, "node2", "service2") diff --git a/agent/consul/subscribe_backend_test.go b/agent/consul/subscribe_backend_test.go index 770f4a61d92..5b88c5f1e81 100644 --- a/agent/consul/subscribe_backend_test.go +++ b/agent/consul/subscribe_backend_test.go @@ -39,7 +39,7 @@ func TestSubscribeBackend_IntegrationWithServer_TLSEnabled(t *testing.T) { require.NoError(t, err) defer server.Shutdown() - client, resolverBuilder, balancerBuilder := newClientWithGRPCPlumbing(t, configureTLS, clientConfigVerifyOutgoing) + client, resolverBuilder := newClientWithGRPCPlumbing(t, configureTLS, clientConfigVerifyOutgoing) // Try to join testrpc.WaitForLeader(t, server.RPC, "dc1") @@ -71,7 +71,6 @@ func TestSubscribeBackend_IntegrationWithServer_TLSEnabled(t *testing.T) { UseTLSForDC: client.tlsConfigurator.UseTLS, DialingFromServer: true, DialingFromDatacenter: "dc1", - BalancerBuilder: balancerBuilder, }) conn, err := pool.ClientConn("dc1") require.NoError(t, err) @@ -116,7 +115,6 @@ func TestSubscribeBackend_IntegrationWithServer_TLSEnabled(t *testing.T) { UseTLSForDC: client.tlsConfigurator.UseTLS, DialingFromServer: true, DialingFromDatacenter: "dc1", - BalancerBuilder: balancerBuilder, }) conn, err := pool.ClientConn("dc1") require.NoError(t, err) @@ -191,7 +189,7 @@ func TestSubscribeBackend_IntegrationWithServer_TLSReload(t *testing.T) { defer server.Shutdown() // Set up a client with valid certs and verify_outgoing = true - client, resolverBuilder, balancerBuilder := newClientWithGRPCPlumbing(t, configureTLS, clientConfigVerifyOutgoing) + client, resolverBuilder := newClientWithGRPCPlumbing(t, configureTLS, clientConfigVerifyOutgoing) testrpc.WaitForLeader(t, server.RPC, "dc1") @@ -204,7 +202,6 @@ func TestSubscribeBackend_IntegrationWithServer_TLSReload(t *testing.T) { UseTLSForDC: client.tlsConfigurator.UseTLS, DialingFromServer: true, DialingFromDatacenter: "dc1", - BalancerBuilder: balancerBuilder, }) conn, err := pool.ClientConn("dc1") require.NoError(t, err) @@ -284,7 +281,7 @@ func TestSubscribeBackend_IntegrationWithServer_DeliversAllMessages(t *testing.T codec := rpcClient(t, server) defer codec.Close() - client, resolverBuilder, balancerBuilder := newClientWithGRPCPlumbing(t) + client, resolverBuilder := newClientWithGRPCPlumbing(t) // Try to join testrpc.WaitForLeader(t, server.RPC, "dc1") @@ -346,7 +343,6 @@ func TestSubscribeBackend_IntegrationWithServer_DeliversAllMessages(t *testing.T UseTLSForDC: client.tlsConfigurator.UseTLS, DialingFromServer: true, DialingFromDatacenter: "dc1", - BalancerBuilder: balancerBuilder, }) conn, err := pool.ClientConn("dc1") require.NoError(t, err) @@ -376,14 +372,17 @@ func TestSubscribeBackend_IntegrationWithServer_DeliversAllMessages(t *testing.T "at least some of the subscribers should have received non-snapshot updates") } -func newClientWithGRPCPlumbing(t *testing.T, ops ...func(*Config)) (*Client, *resolver.ServerResolverBuilder, *balancer.Builder) { +func newClientWithGRPCPlumbing(t *testing.T, ops ...func(*Config)) (*Client, *resolver.ServerResolverBuilder) { _, config := testClientConfig(t) for _, op := range ops { op(config) } resolverBuilder := resolver.NewServerResolverBuilder(newTestResolverConfig(t, - "client."+config.Datacenter+"."+string(config.NodeID))) + "client."+config.Datacenter+"."+string(config.NodeID), + config.Datacenter, + "client", + )) resolver.Register(resolverBuilder) t.Cleanup(func() { @@ -392,6 +391,7 @@ func newClientWithGRPCPlumbing(t *testing.T, ops ...func(*Config)) (*Client, *re balancerBuilder := balancer.NewBuilder(resolverBuilder.Authority(), testutil.Logger(t)) balancerBuilder.Register() + t.Cleanup(balancerBuilder.Deregister) deps := newDefaultDeps(t, config) deps.Router = router.NewRouter( @@ -406,7 +406,7 @@ func newClientWithGRPCPlumbing(t *testing.T, ops ...func(*Config)) (*Client, *re t.Cleanup(func() { client.Shutdown() }) - return client, resolverBuilder, balancerBuilder + return client, resolverBuilder } type testLogger interface { diff --git a/agent/consul/system_metadata.go b/agent/consul/system_metadata.go index f1603225738..5ceb5fc5e71 100644 --- a/agent/consul/system_metadata.go +++ b/agent/consul/system_metadata.go @@ -4,7 +4,7 @@ import ( "github.com/hashicorp/consul/agent/structs" ) -func (s *Server) getSystemMetadata(key string) (string, error) { +func (s *Server) GetSystemMetadata(key string) (string, error) { _, entry, err := s.fsm.State().SystemMetadataGet(nil, key) if err != nil { return "", err @@ -16,7 +16,7 @@ func (s *Server) getSystemMetadata(key string) (string, error) { return entry.Value, nil } -func (s *Server) setSystemMetadataKey(key, val string) error { +func (s *Server) SetSystemMetadataKey(key, val string) error { args := &structs.SystemMetadataRequest{ Op: structs.SystemMetadataUpsert, Entry: &structs.SystemMetadataEntry{Key: key, Value: val}, diff --git a/agent/consul/system_metadata_test.go b/agent/consul/system_metadata_test.go index 633aa6bc407..b17790d2246 100644 --- a/agent/consul/system_metadata_test.go +++ b/agent/consul/system_metadata_test.go @@ -39,9 +39,9 @@ func TestLeader_SystemMetadata_CRUD(t *testing.T) { require.Len(t, entries, 0) // Create 3 - require.NoError(t, srv.setSystemMetadataKey("key1", "val1")) - require.NoError(t, srv.setSystemMetadataKey("key2", "val2")) - require.NoError(t, srv.setSystemMetadataKey("key3", "")) + require.NoError(t, srv.SetSystemMetadataKey("key1", "val1")) + require.NoError(t, srv.SetSystemMetadataKey("key2", "val2")) + require.NoError(t, srv.SetSystemMetadataKey("key3", "")) mapify := func(entries []*structs.SystemMetadataEntry) map[string]string { m := make(map[string]string) @@ -62,7 +62,7 @@ func TestLeader_SystemMetadata_CRUD(t *testing.T) { }, mapify(entries)) // Update one and delete one. - require.NoError(t, srv.setSystemMetadataKey("key3", "val3")) + require.NoError(t, srv.SetSystemMetadataKey("key3", "val3")) require.NoError(t, srv.deleteSystemMetadataKey("key1")) _, entries, err = state.SystemMetadataList(nil) diff --git a/agent/consul/txn_endpoint.go b/agent/consul/txn_endpoint.go index a1977b91966..e528ea72533 100644 --- a/agent/consul/txn_endpoint.go +++ b/agent/consul/txn_endpoint.go @@ -185,7 +185,7 @@ func (t *Txn) Read(args *structs.TxnReadRequest, reply *structs.TxnReadResponse) // We have to do this ourselves since we are not doing a blocking RPC. if args.RequireConsistent { - if err := t.srv.consistentRead(); err != nil { + if err := t.srv.ConsistentRead(); err != nil { return err } } @@ -217,7 +217,7 @@ func (t *Txn) Read(args *structs.TxnReadRequest, reply *structs.TxnReadResponse) reply.QueryMeta.ResultsFilteredByACLs = total != len(reply.Results) // We have to do this ourselves since we are not doing a blocking RPC. - t.srv.setQueryMeta(&reply.QueryMeta, args.Token) + t.srv.SetQueryMeta(&reply.QueryMeta, args.Token) return nil } diff --git a/agent/consul/usagemetrics/usagemetrics.go b/agent/consul/usagemetrics/usagemetrics.go index 9f1a928fb85..e968b9918df 100644 --- a/agent/consul/usagemetrics/usagemetrics.go +++ b/agent/consul/usagemetrics/usagemetrics.go @@ -13,6 +13,7 @@ import ( "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/logging" + "github.com/hashicorp/consul/version" ) var Gauges = []prometheus.GaugeDefinition{ @@ -92,6 +93,10 @@ var Gauges = []prometheus.GaugeDefinition{ Name: []string{"state", "billable_service_instances"}, Help: "Total number of billable service instances in the local datacenter.", }, + { + Name: []string{"version"}, + Help: "Represents the Consul version.", + }, } type getMembersFunc func() []serf.Member @@ -241,6 +246,7 @@ func (u *UsageMetricsReporter) runOnce() { } u.emitConfigEntryUsage(configUsage) + u.emitVersion() } func (u *UsageMetricsReporter) memberUsage() []serf.Member { @@ -263,3 +269,26 @@ func (u *UsageMetricsReporter) memberUsage() []serf.Member { return out } + +func (u *UsageMetricsReporter) emitVersion() { + // consul version metric with labels + metrics.SetGaugeWithLabels( + []string{"version"}, + 1, + []metrics.Label{ + {Name: "version", Value: versionWithMetadata()}, + {Name: "pre_release", Value: version.VersionPrerelease}, + }, + ) +} + +func versionWithMetadata() string { + vsn := version.Version + metadata := version.VersionMetadata + + if metadata != "" { + vsn += "+" + metadata + } + + return vsn +} diff --git a/agent/consul/usagemetrics/usagemetrics_oss.go b/agent/consul/usagemetrics/usagemetrics_ce.go similarity index 100% rename from agent/consul/usagemetrics/usagemetrics_oss.go rename to agent/consul/usagemetrics/usagemetrics_ce.go diff --git a/agent/consul/usagemetrics/usagemetrics_oss_test.go b/agent/consul/usagemetrics/usagemetrics_ce_test.go similarity index 93% rename from agent/consul/usagemetrics/usagemetrics_oss_test.go rename to agent/consul/usagemetrics/usagemetrics_ce_test.go index 002fc2a81c4..85888bc1a7d 100644 --- a/agent/consul/usagemetrics/usagemetrics_oss_test.go +++ b/agent/consul/usagemetrics/usagemetrics_ce_test.go @@ -4,6 +4,7 @@ package usagemetrics import ( + "fmt" "testing" "time" @@ -18,6 +19,7 @@ import ( "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/proto/pbpeering" "github.com/hashicorp/consul/sdk/testutil" + "github.com/hashicorp/consul/version" ) func newStateStore() (*state.Store, error) { @@ -146,6 +148,22 @@ var baseCases = map[string]testCase{ {Name: "kind", Value: "ingress-gateway"}, }, }, + "consul.usage.test.consul.state.connect_instances;datacenter=dc1;kind=api-gateway": { // Legacy + Name: "consul.usage.test.consul.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "api-gateway"}, + }, + }, + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "api-gateway"}, + }, + }, "consul.usage.test.consul.state.connect_instances;datacenter=dc1;kind=mesh-gateway": { // Legacy Name: "consul.usage.test.consul.state.connect_instances", Value: 0, @@ -437,6 +455,15 @@ var baseCases = map[string]testCase{ {Name: "kind", Value: "tcp-route"}, }, }, + // --- version --- + fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): { + Name: "consul.usage.test.version", + Value: 1, + Labels: []metrics.Label{ + {Name: "version", Value: versionWithMetadata()}, + {Name: "pre_release", Value: version.VersionPrerelease}, + }, + }, }, getMembersFunc: func() []serf.Member { return []serf.Member{} }, }, @@ -573,6 +600,22 @@ var baseCases = map[string]testCase{ {Name: "kind", Value: "ingress-gateway"}, }, }, + "consul.usage.test.consul.state.connect_instances;datacenter=dc1;kind=api-gateway": { // Legacy + Name: "consul.usage.test.consul.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "api-gateway"}, + }, + }, + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "api-gateway"}, + }, + }, "consul.usage.test.consul.state.connect_instances;datacenter=dc1;kind=mesh-gateway": { // Legacy Name: "consul.usage.test.consul.state.connect_instances", Value: 0, @@ -864,11 +907,20 @@ var baseCases = map[string]testCase{ {Name: "kind", Value: "tcp-route"}, }, }, + // --- version --- + fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): { + Name: "consul.usage.test.version", + Value: 1, + Labels: []metrics.Label{ + {Name: "version", Value: versionWithMetadata()}, + {Name: "pre_release", Value: version.VersionPrerelease}, + }, + }, }, }, } -func TestUsageReporter_emitNodeUsage_OSS(t *testing.T) { +func TestUsageReporter_emitNodeUsage_CE(t *testing.T) { cases := baseCases for name, tcase := range cases { @@ -907,7 +959,7 @@ func TestUsageReporter_emitNodeUsage_OSS(t *testing.T) { } } -func TestUsageReporter_emitPeeringUsage_OSS(t *testing.T) { +func TestUsageReporter_emitPeeringUsage_CE(t *testing.T) { cases := make(map[string]testCase) for k, v := range baseCases { eg := make(map[string]metrics.GaugeValue) @@ -1011,7 +1063,7 @@ func TestUsageReporter_emitPeeringUsage_OSS(t *testing.T) { } } -func TestUsageReporter_emitServiceUsage_OSS(t *testing.T) { +func TestUsageReporter_emitServiceUsage_CE(t *testing.T) { cases := make(map[string]testCase) for k, v := range baseCases { eg := make(map[string]metrics.GaugeValue) @@ -1028,6 +1080,9 @@ func TestUsageReporter_emitServiceUsage_OSS(t *testing.T) { require.NoError(t, s.EnsureNode(3, &structs.Node{Node: "baz", Address: "127.0.0.2"})) require.NoError(t, s.EnsureNode(4, &structs.Node{Node: "qux", Address: "127.0.0.3"})) + apigw := structs.TestNodeServiceAPIGateway(t) + apigw.ID = "api-gateway" + mgw := structs.TestNodeServiceMeshGateway(t) mgw.ID = "mesh-gateway" @@ -1042,16 +1097,17 @@ func TestUsageReporter_emitServiceUsage_OSS(t *testing.T) { require.NoError(t, s.EnsureRegistration(10, structs.TestRegisterIngressGateway(t))) require.NoError(t, s.EnsureService(11, "foo", mgw)) require.NoError(t, s.EnsureService(12, "foo", tgw)) - require.NoError(t, s.EnsureService(13, "bar", &structs.NodeService{ID: "db-native", Service: "db", Tags: nil, Address: "", Port: 5000, Connect: structs.ServiceConnect{Native: true}})) - require.NoError(t, s.EnsureConfigEntry(14, &structs.IngressGatewayConfigEntry{ + require.NoError(t, s.EnsureService(13, "foo", apigw)) + require.NoError(t, s.EnsureService(14, "bar", &structs.NodeService{ID: "db-native", Service: "db", Tags: nil, Address: "", Port: 5000, Connect: structs.ServiceConnect{Native: true}})) + require.NoError(t, s.EnsureConfigEntry(15, &structs.IngressGatewayConfigEntry{ Kind: structs.IngressGateway, Name: "foo", })) - require.NoError(t, s.EnsureConfigEntry(15, &structs.IngressGatewayConfigEntry{ + require.NoError(t, s.EnsureConfigEntry(16, &structs.IngressGatewayConfigEntry{ Kind: structs.IngressGateway, Name: "bar", })) - require.NoError(t, s.EnsureConfigEntry(16, &structs.IngressGatewayConfigEntry{ + require.NoError(t, s.EnsureConfigEntry(17, &structs.IngressGatewayConfigEntry{ Kind: structs.IngressGateway, Name: "baz", })) @@ -1092,22 +1148,22 @@ func TestUsageReporter_emitServiceUsage_OSS(t *testing.T) { } nodesAndSvcsCase.expectedGauges["consul.usage.test.consul.state.services;datacenter=dc1"] = metrics.GaugeValue{ // Legacy Name: "consul.usage.test.consul.state.services", - Value: 7, + Value: 8, Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, } nodesAndSvcsCase.expectedGauges["consul.usage.test.state.services;datacenter=dc1"] = metrics.GaugeValue{ Name: "consul.usage.test.state.services", - Value: 7, + Value: 8, Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, } nodesAndSvcsCase.expectedGauges["consul.usage.test.consul.state.service_instances;datacenter=dc1"] = metrics.GaugeValue{ // Legacy Name: "consul.usage.test.consul.state.service_instances", - Value: 9, + Value: 10, Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, } nodesAndSvcsCase.expectedGauges["consul.usage.test.state.service_instances;datacenter=dc1"] = metrics.GaugeValue{ Name: "consul.usage.test.state.service_instances", - Value: 9, + Value: 10, Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, } nodesAndSvcsCase.expectedGauges["consul.usage.test.consul.state.connect_instances;datacenter=dc1;kind=connect-proxy"] = metrics.GaugeValue{ // Legacy @@ -1158,6 +1214,22 @@ func TestUsageReporter_emitServiceUsage_OSS(t *testing.T) { {Name: "kind", Value: "ingress-gateway"}, }, } + nodesAndSvcsCase.expectedGauges["consul.usage.test.consul.state.connect_instances;datacenter=dc1;kind=api-gateway"] = metrics.GaugeValue{ // Legacy + Name: "consul.usage.test.consul.state.connect_instances", + Value: 1, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "api-gateway"}, + }, + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.connect_instances", + Value: 1, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "api-gateway"}, + }, + } nodesAndSvcsCase.expectedGauges["consul.usage.test.consul.state.connect_instances;datacenter=dc1;kind=mesh-gateway"] = metrics.GaugeValue{ // Legacy Name: "consul.usage.test.consul.state.connect_instances", Value: 1, @@ -1251,7 +1323,7 @@ func TestUsageReporter_emitServiceUsage_OSS(t *testing.T) { } } -func TestUsageReporter_emitKVUsage_OSS(t *testing.T) { +func TestUsageReporter_emitKVUsage_CE(t *testing.T) { cases := make(map[string]testCase) for k, v := range baseCases { eg := make(map[string]metrics.GaugeValue) diff --git a/agent/consul/usagemetrics/usagemetrics_test.go b/agent/consul/usagemetrics/usagemetrics_test.go index cf0cd01e01a..04aea120511 100644 --- a/agent/consul/usagemetrics/usagemetrics_test.go +++ b/agent/consul/usagemetrics/usagemetrics_test.go @@ -24,7 +24,7 @@ func assertEqualGaugeMaps(t *testing.T, expectedMap, foundMap map[string]metrics for key := range foundMap { if _, ok := expectedMap[key]; !ok { - t.Errorf("found unexpected gauge key: %s", key) + t.Errorf("found unexpected gauge key: %s with value: %v", key, foundMap[key]) } } diff --git a/agent/coordinate_endpoint_test.go b/agent/coordinate_endpoint_test.go index 4782ae72d0f..71492d909c4 100644 --- a/agent/coordinate_endpoint_test.go +++ b/agent/coordinate_endpoint_test.go @@ -40,9 +40,9 @@ func TestCoordinate_Disabled_Response(t *testing.T) { req, _ := http.NewRequest("PUT", "/should/not/care", nil) resp := httptest.NewRecorder() obj, err := tt(resp, req) - if err, ok := err.(HTTPError); ok { - if err.StatusCode != 401 { - t.Fatalf("expected status 401 but got %d", err.StatusCode) + if httpErr, ok := err.(HTTPError); ok { + if httpErr.StatusCode != 401 { + t.Fatalf("expected status 401 but got %d", httpErr.StatusCode) } } else { t.Fatalf("expected HTTP error but got %v", err) diff --git a/agent/dns.go b/agent/dns.go index 3dce6410d75..9f1c5fdd5ac 100644 --- a/agent/dns.go +++ b/agent/dns.go @@ -1043,7 +1043,7 @@ func (d *DNSServer) trimDomain(query string) string { longer, shorter = shorter, longer } - if strings.HasSuffix(query, longer) { + if strings.HasSuffix(query, "."+strings.TrimLeft(longer, ".")) { return strings.TrimSuffix(query, longer) } return strings.TrimSuffix(query, shorter) diff --git a/agent/dns_oss.go b/agent/dns_ce.go similarity index 100% rename from agent/dns_oss.go rename to agent/dns_ce.go diff --git a/agent/dns_oss_test.go b/agent/dns_ce_test.go similarity index 98% rename from agent/dns_oss_test.go rename to agent/dns_ce_test.go index c3cf264c687..26ee6dc2476 100644 --- a/agent/dns_oss_test.go +++ b/agent/dns_ce_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestDNS_OSS_PeeredServices(t *testing.T) { +func TestDNS_CE_PeeredServices(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } diff --git a/agent/dns_test.go b/agent/dns_test.go index 501564c66c2..cb8197c9bf7 100644 --- a/agent/dns_test.go +++ b/agent/dns_test.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/serf/coordinate" "github.com/miekg/dns" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" "github.com/hashicorp/consul/agent/config" "github.com/hashicorp/consul/agent/consul" @@ -3185,7 +3186,7 @@ func TestDNS_ServiceLookup_WanTranslation(t *testing.T) { } var out struct{} - require.NoError(t, a2.RPC(context.Background(), "Catalog.Register", args, &out)) + require.NoError(r, a2.RPC(context.Background(), "Catalog.Register", args, &out)) }) // Look up the SRV record via service and prepared query. @@ -3513,11 +3514,11 @@ func TestDNS_CaseInsensitiveServiceLookup(t *testing.T) { retry.Run(t, func(r *retry.R) { in, _, err := c.Exchange(m, a.DNSAddr()) if err != nil { - t.Fatalf("err: %v", err) + r.Fatalf("err: %v", err) } if len(in.Answer) != 1 { - t.Fatalf("question %v, empty lookup: %#v", question, in) + r.Fatalf("question %v, empty lookup: %#v", question, in) } }) } @@ -4883,21 +4884,26 @@ func TestDNS_TCP_and_UDP_Truncate(t *testing.T) { services := []string{"normal", "truncated"} for index, service := range services { numServices := (index * 5000) + 2 + var eg errgroup.Group for i := 1; i < numServices; i++ { - args := &structs.RegisterRequest{ - Datacenter: "dc1", - Node: fmt.Sprintf("%s-%d.acme.com", service, i), - Address: fmt.Sprintf("127.%d.%d.%d", 0, (i / 255), i%255), - Service: &structs.NodeService{ - Service: service, - Port: 8000, - }, - } + j := i + eg.Go(func() error { + args := &structs.RegisterRequest{ + Datacenter: "dc1", + Node: fmt.Sprintf("%s-%d.acme.com", service, j), + Address: fmt.Sprintf("127.%d.%d.%d", 0, (j / 255), j%255), + Service: &structs.NodeService{ + Service: service, + Port: 8000, + }, + } - var out struct{} - if err := a.RPC(context.Background(), "Catalog.Register", args, &out); err != nil { - t.Fatalf("err: %v", err) - } + var out struct{} + return a.RPC(context.Background(), "Catalog.Register", args, &out) + }) + } + if err := eg.Wait(); err != nil { + t.Fatalf("error registering: %v", err) } // Register an equivalent prepared query. @@ -7062,6 +7068,45 @@ func TestDNS_AltDomains_Overlap(t *testing.T) { } } +func TestDNS_AltDomain_DCName_Overlap(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + // this tests the DC name overlap with the consul domain/alt-domain + // we should get response when DC suffix is a prefix of consul alt-domain + t.Parallel() + a := NewTestAgent(t, ` + datacenter = "dc-test" + node_name = "test-node" + alt_domain = "test.consul." + `) + defer a.Shutdown() + testrpc.WaitForLeader(t, a.RPC, "dc-test") + + questions := []string{ + "test-node.node.dc-test.consul.", + "test-node.node.dc-test.test.consul.", + } + + for _, question := range questions { + m := new(dns.Msg) + m.SetQuestion(question, dns.TypeA) + + c := new(dns.Client) + in, _, err := c.Exchange(m, a.DNSAddr()) + if err != nil { + t.Fatalf("err: %v", err) + } + + require.Len(t, in.Answer, 1) + + aRec, ok := in.Answer[0].(*dns.A) + require.True(t, ok) + require.Equal(t, aRec.A.To4().String(), "127.0.0.1") + } +} + func TestDNS_PreparedQuery_AllowStale(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") diff --git a/agent/enterprise_delegate_oss.go b/agent/enterprise_delegate_ce.go similarity index 67% rename from agent/enterprise_delegate_oss.go rename to agent/enterprise_delegate_ce.go index 876c8837afe..39984898257 100644 --- a/agent/enterprise_delegate_oss.go +++ b/agent/enterprise_delegate_ce.go @@ -3,5 +3,5 @@ package agent -// enterpriseDelegate has no functions in OSS +// enterpriseDelegate has no functions in CE type enterpriseDelegate interface{} diff --git a/agent/envoyextensions/builtin/aws-lambda/aws_lambda.go b/agent/envoyextensions/builtin/aws-lambda/aws_lambda.go index bcff99a638a..35cc39b2d7b 100644 --- a/agent/envoyextensions/builtin/aws-lambda/aws_lambda.go +++ b/agent/envoyextensions/builtin/aws-lambda/aws_lambda.go @@ -39,7 +39,7 @@ func Constructor(ext api.EnvoyExtension) (extensioncommon.EnvoyExtender, error) if err := a.fromArguments(ext.Arguments); err != nil { return nil, err } - return &extensioncommon.BasicEnvoyExtender{ + return &extensioncommon.UpstreamEnvoyExtender{ Extension: &a, }, nil } @@ -62,7 +62,7 @@ func (a *awsLambda) validate() error { // CanApply returns true if the kind of the provided ExtensionConfiguration matches // the kind of the lambda configuration func (a *awsLambda) CanApply(config *extensioncommon.RuntimeConfig) bool { - return config.Kind == config.OutgoingProxyKind() + return config.Kind == config.UpstreamOutgoingProxyKind() } // PatchRoute modifies the routing configuration for a service of kind TerminatingGateway. If the kind is @@ -72,6 +72,11 @@ func (a *awsLambda) PatchRoute(r *extensioncommon.RuntimeConfig, route *envoy_ro return route, false, nil } + // Only patch outbound routes. + if extensioncommon.IsRouteToLocalAppCluster(route) { + return route, false, nil + } + for _, virtualHost := range route.VirtualHosts { for _, route := range virtualHost.Routes { action, ok := route.Action.(*envoy_route_v3.Route_Route) @@ -92,6 +97,11 @@ func (a *awsLambda) PatchRoute(r *extensioncommon.RuntimeConfig, route *envoy_ro // PatchCluster patches the provided envoy cluster with data required to support an AWS lambda function func (a *awsLambda) PatchCluster(_ *extensioncommon.RuntimeConfig, c *envoy_cluster_v3.Cluster) (*envoy_cluster_v3.Cluster, bool, error) { + // Only patch outbound clusters. + if extensioncommon.IsLocalAppCluster(c) { + return c, false, nil + } + transportSocket, err := makeUpstreamTLSTransportSocket(&envoy_tls_v3.UpstreamTlsContext{ Sni: "*.amazonaws.com", }) @@ -153,7 +163,12 @@ func (a *awsLambda) PatchCluster(_ *extensioncommon.RuntimeConfig, c *envoy_clus // PatchFilter patches the provided envoy filter with an inserted lambda filter being careful not to // overwrite the http filters. -func (a *awsLambda) PatchFilter(_ *extensioncommon.RuntimeConfig, filter *envoy_listener_v3.Filter) (*envoy_listener_v3.Filter, bool, error) { +func (a *awsLambda) PatchFilter(_ *extensioncommon.RuntimeConfig, filter *envoy_listener_v3.Filter, isInboundListener bool) (*envoy_listener_v3.Filter, bool, error) { + // Only patch outbound filters. + if isInboundListener { + return filter, false, nil + } + if filter.Name != "envoy.filters.network.http_connection_manager" { return filter, false, nil } diff --git a/agent/envoyextensions/builtin/aws-lambda/aws_lambda_test.go b/agent/envoyextensions/builtin/aws-lambda/aws_lambda_test.go index f0ac9c1e7c7..c24f4e2af8a 100644 --- a/agent/envoyextensions/builtin/aws-lambda/aws_lambda_test.go +++ b/agent/envoyextensions/builtin/aws-lambda/aws_lambda_test.go @@ -83,7 +83,7 @@ func TestConstructor(t *testing.T) { if tc.ok { require.NoError(t, err) - require.Equal(t, &extensioncommon.BasicEnvoyExtender{Extension: &tc.expected}, e) + require.Equal(t, &extensioncommon.UpstreamEnvoyExtender{Extension: &tc.expected}, e) } else { require.Error(t, err) } @@ -320,10 +320,11 @@ func TestPatchFilter(t *testing.T) { return v } tests := map[string]struct { - filter *envoy_listener_v3.Filter - expectFilter *envoy_listener_v3.Filter - expectBool bool - expectErr string + filter *envoy_listener_v3.Filter + isInboundFilter bool + expectFilter *envoy_listener_v3.Filter + expectBool bool + expectErr string }{ "invalid filter name is ignored": { filter: &envoy_listener_v3.Filter{Name: "something"}, @@ -413,6 +414,36 @@ func TestPatchFilter(t *testing.T) { }, expectBool: true, }, + "inbound filter ignored": { + filter: &envoy_listener_v3.Filter{ + Name: "envoy.filters.network.http_connection_manager", + ConfigType: &envoy_listener_v3.Filter_TypedConfig{ + TypedConfig: makeAny(&envoy_http_v3.HttpConnectionManager{ + HttpFilters: []*envoy_http_v3.HttpFilter{ + {Name: "one"}, + {Name: "two"}, + {Name: "envoy.filters.http.router"}, + {Name: "three"}, + }, + }), + }, + }, + expectFilter: &envoy_listener_v3.Filter{ + Name: "envoy.filters.network.http_connection_manager", + ConfigType: &envoy_listener_v3.Filter_TypedConfig{ + TypedConfig: makeAny(&envoy_http_v3.HttpConnectionManager{ + HttpFilters: []*envoy_http_v3.HttpFilter{ + {Name: "one"}, + {Name: "two"}, + {Name: "envoy.filters.http.router"}, + {Name: "three"}, + }, + }), + }, + }, + isInboundFilter: true, + expectBool: false, + }, } for name, tc := range tests { @@ -422,7 +453,7 @@ func TestPatchFilter(t *testing.T) { PayloadPassthrough: true, InvocationMode: "asynchronous", } - f, ok, err := l.PatchFilter(nil, tc.filter) + f, ok, err := l.PatchFilter(nil, tc.filter, tc.isInboundFilter) require.Equal(t, tc.expectBool, ok) if tc.expectErr == "" { require.NoError(t, err) diff --git a/agent/envoyextensions/builtin/http/localratelimit/copied.go b/agent/envoyextensions/builtin/http/localratelimit/copied.go deleted file mode 100644 index ec1d4988eb7..00000000000 --- a/agent/envoyextensions/builtin/http/localratelimit/copied.go +++ /dev/null @@ -1,58 +0,0 @@ -package localratelimit - -import ( - envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" - envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" - envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" - - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" -) - -// This is copied from xds and not put into the shared package because I'm not -// convinced it should be shared. - -func makeUpstreamTLSTransportSocket(tlsContext *envoy_tls_v3.UpstreamTlsContext) (*envoy_core_v3.TransportSocket, error) { - if tlsContext == nil { - return nil, nil - } - return makeTransportSocket("tls", tlsContext) -} - -func makeTransportSocket(name string, config proto.Message) (*envoy_core_v3.TransportSocket, error) { - any, err := anypb.New(config) - if err != nil { - return nil, err - } - return &envoy_core_v3.TransportSocket{ - Name: name, - ConfigType: &envoy_core_v3.TransportSocket_TypedConfig{ - TypedConfig: any, - }, - }, nil -} - -func makeEnvoyHTTPFilter(name string, cfg proto.Message) (*envoy_http_v3.HttpFilter, error) { - any, err := anypb.New(cfg) - if err != nil { - return nil, err - } - - return &envoy_http_v3.HttpFilter{ - Name: name, - ConfigType: &envoy_http_v3.HttpFilter_TypedConfig{TypedConfig: any}, - }, nil -} - -func makeFilter(name string, cfg proto.Message) (*envoy_listener_v3.Filter, error) { - any, err := anypb.New(cfg) - if err != nil { - return nil, err - } - - return &envoy_listener_v3.Filter{ - Name: name, - ConfigType: &envoy_listener_v3.Filter_TypedConfig{TypedConfig: any}, - }, nil -} diff --git a/agent/envoyextensions/builtin/http/localratelimit/ratelimit.go b/agent/envoyextensions/builtin/http/localratelimit/ratelimit.go deleted file mode 100644 index 3ffea6f1ff8..00000000000 --- a/agent/envoyextensions/builtin/http/localratelimit/ratelimit.go +++ /dev/null @@ -1,198 +0,0 @@ -package localratelimit - -import ( - "errors" - "fmt" - "time" - - envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" - envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" - envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" - envoy_ratelimit "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/local_ratelimit/v3" - envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" - envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" - envoy_resource_v3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" - "github.com/golang/protobuf/ptypes/wrappers" - "github.com/hashicorp/go-multierror" - "github.com/mitchellh/mapstructure" - "google.golang.org/protobuf/types/known/durationpb" - - "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/envoyextensions/extensioncommon" -) - -type ratelimit struct { - ProxyType string - - // Token bucket of the rate limit - MaxTokens *int - TokensPerFill *int - FillInterval *int - - // Percent of requests to be rate limited - FilterEnabled *uint32 - FilterEnforced *uint32 -} - -var _ extensioncommon.BasicExtension = (*ratelimit)(nil) - -// Constructor follows a specific function signature required for the extension registration. -func Constructor(ext api.EnvoyExtension) (extensioncommon.EnvoyExtender, error) { - var r ratelimit - if name := ext.Name; name != api.BuiltinLocalRatelimitExtension { - return nil, fmt.Errorf("expected extension name 'ratelimit' but got %q", name) - } - - if err := r.fromArguments(ext.Arguments); err != nil { - return nil, err - } - - return &extensioncommon.BasicEnvoyExtender{ - Extension: &r, - }, nil -} - -func (r *ratelimit) fromArguments(args map[string]interface{}) error { - if err := mapstructure.Decode(args, r); err != nil { - return fmt.Errorf("error decoding extension arguments: %v", err) - } - return r.validate() -} - -func (r *ratelimit) validate() error { - var resultErr error - - // NOTE: Envoy requires FillInterval value must be greater than 0. - // If unset, it is considered as 0. - if r.FillInterval == nil { - resultErr = multierror.Append(resultErr, fmt.Errorf("FillInterval(in second) is missing")) - } else if *r.FillInterval <= 0 { - resultErr = multierror.Append(resultErr, fmt.Errorf("FillInterval(in second) must be greater than 0, got %d", *r.FillInterval)) - } - - // NOTE: Envoy requires MaxToken value must be greater than 0. - // If unset, it is considered as 0. - if r.MaxTokens == nil { - resultErr = multierror.Append(resultErr, fmt.Errorf("MaxTokens is missing")) - } else if *r.MaxTokens <= 0 { - resultErr = multierror.Append(resultErr, fmt.Errorf("MaxTokens must be greater than 0, got %d", r.MaxTokens)) - } - - // TokensPerFill is allowed to unset. In this case, envoy - // uses its default value, which is 1. - if r.TokensPerFill != nil && *r.TokensPerFill <= 0 { - resultErr = multierror.Append(resultErr, fmt.Errorf("TokensPerFill must be greater than 0, got %d", *r.TokensPerFill)) - } - - if err := validateProxyType(r.ProxyType); err != nil { - resultErr = multierror.Append(resultErr, err) - } - - return resultErr -} - -// CanApply determines if the extension can apply to the given extension configuration. -func (p *ratelimit) CanApply(config *extensioncommon.RuntimeConfig) bool { - // rate limit is only applied to the service itself since the limit is - // aggregated from all downstream connections. - return string(config.Kind) == p.ProxyType && !config.IsUpstream() -} - -// PatchRoute does nothing. -func (p ratelimit) PatchRoute(_ *extensioncommon.RuntimeConfig, route *envoy_route_v3.RouteConfiguration) (*envoy_route_v3.RouteConfiguration, bool, error) { - return route, false, nil -} - -// PatchCluster does nothing. -func (p ratelimit) PatchCluster(_ *extensioncommon.RuntimeConfig, c *envoy_cluster_v3.Cluster) (*envoy_cluster_v3.Cluster, bool, error) { - return c, false, nil -} - -// PatchFilter inserts a http local rate_limit filter at the head of -// envoy.filters.network.http_connection_manager filters -func (p ratelimit) PatchFilter(_ *extensioncommon.RuntimeConfig, filter *envoy_listener_v3.Filter) (*envoy_listener_v3.Filter, bool, error) { - if filter.Name != "envoy.filters.network.http_connection_manager" { - return filter, false, nil - } - if typedConfig := filter.GetTypedConfig(); typedConfig == nil { - return filter, false, errors.New("error getting typed config for http filter") - } - - config := envoy_resource_v3.GetHTTPConnectionManager(filter) - if config == nil { - return filter, false, errors.New("error unmarshalling filter") - } - - tokenBucket := envoy_type_v3.TokenBucket{} - - if p.TokensPerFill != nil { - tokenBucket.TokensPerFill = &wrappers.UInt32Value{ - Value: uint32(*p.TokensPerFill), - } - } - if p.MaxTokens != nil { - tokenBucket.MaxTokens = uint32(*p.MaxTokens) - } - - if p.FillInterval != nil { - tokenBucket.FillInterval = durationpb.New(time.Duration(*p.FillInterval) * time.Second) - } - - var FilterEnabledDefault *envoy_core_v3.RuntimeFractionalPercent - if p.FilterEnabled != nil { - FilterEnabledDefault = &envoy_core_v3.RuntimeFractionalPercent{ - DefaultValue: &envoy_type_v3.FractionalPercent{ - Numerator: *p.FilterEnabled, - Denominator: envoy_type_v3.FractionalPercent_HUNDRED, - }, - } - } - - var FilterEnforcedDefault *envoy_core_v3.RuntimeFractionalPercent - if p.FilterEnforced != nil { - FilterEnforcedDefault = &envoy_core_v3.RuntimeFractionalPercent{ - DefaultValue: &envoy_type_v3.FractionalPercent{ - Numerator: *p.FilterEnforced, - Denominator: envoy_type_v3.FractionalPercent_HUNDRED, - }, - } - } - - ratelimitHttpFilter, err := makeEnvoyHTTPFilter( - "envoy.filters.http.local_ratelimit", - &envoy_ratelimit.LocalRateLimit{ - TokenBucket: &tokenBucket, - StatPrefix: "local_ratelimit", - FilterEnabled: FilterEnabledDefault, - FilterEnforced: FilterEnforcedDefault, - }, - ) - - if err != nil { - return filter, false, err - } - - changedFilters := make([]*envoy_http_v3.HttpFilter, 0, len(config.HttpFilters)+1) - - // The ratelimitHttpFilter is inserted as the first element of the http - // filter chain. - changedFilters = append(changedFilters, ratelimitHttpFilter) - changedFilters = append(changedFilters, config.HttpFilters...) - config.HttpFilters = changedFilters - - newFilter, err := makeFilter("envoy.filters.network.http_connection_manager", config) - if err != nil { - return filter, false, errors.New("error making new filter") - } - - return newFilter, true, nil -} - -func validateProxyType(t string) error { - if t != "connect-proxy" { - return fmt.Errorf("unexpected ProxyType %q", t) - } - - return nil -} diff --git a/agent/envoyextensions/builtin/http/localratelimit/ratelimit_test.go b/agent/envoyextensions/builtin/http/localratelimit/ratelimit_test.go deleted file mode 100644 index 5c68b1f51ea..00000000000 --- a/agent/envoyextensions/builtin/http/localratelimit/ratelimit_test.go +++ /dev/null @@ -1,160 +0,0 @@ -package localratelimit - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/envoyextensions/extensioncommon" -) - -func TestConstructor(t *testing.T) { - makeArguments := func(overrides map[string]interface{}) map[string]interface{} { - m := map[string]interface{}{ - "ProxyType": "connect-proxy", - } - - for k, v := range overrides { - m[k] = v - } - - return m - } - - cases := map[string]struct { - extensionName string - arguments map[string]interface{} - expected ratelimit - ok bool - expectedErrMsg string - }{ - "with no arguments": { - arguments: nil, - ok: false, - }, - "with an invalid name": { - arguments: makeArguments(map[string]interface{}{}), - extensionName: "bad", - ok: false, - }, - "MaxToken is missing": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 30, - "TokensPerFill": 5, - }), - expectedErrMsg: "MaxTokens is missing", - ok: false, - }, - "MaxTokens <= 0": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 30, - "TokensPerFill": 5, - "MaxTokens": 0, - }), - expectedErrMsg: "MaxTokens must be greater than 0", - ok: false, - }, - "FillInterval is missing": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "TokensPerFill": 5, - "MaxTokens": 10, - }), - expectedErrMsg: "FillInterval(in second) is missing", - ok: false, - }, - "FillInterval <= 0": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 0, - "TokensPerFill": 5, - "MaxTokens": 10, - }), - expectedErrMsg: "FillInterval(in second) must be greater than 0", - ok: false, - }, - "TokensPerFill <= 0": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 30, - "TokensPerFill": 0, - "MaxTokens": 10, - }), - expectedErrMsg: "TokensPerFill must be greater than 0", - ok: false, - }, - "FilterEnabled < 0": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 30, - "TokensPerFill": 5, - "MaxTokens": 10, - "FilterEnabled": -1, - }), - expectedErrMsg: "cannot parse 'FilterEnabled', -1 overflows uint", - ok: false, - }, - "FilterEnforced < 0": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 30, - "TokensPerFill": 5, - "MaxTokens": 10, - "FilterEnforced": -1, - }), - expectedErrMsg: "cannot parse 'FilterEnforced', -1 overflows uint", - ok: false, - }, - "valid everything": { - arguments: makeArguments(map[string]interface{}{ - "ProxyType": "connect-proxy", - "FillInterval": 30, - "MaxTokens": 20, - "TokensPerFill": 5, - }), - expected: ratelimit{ - ProxyType: "connect-proxy", - MaxTokens: intPointer(20), - FillInterval: intPointer(30), - TokensPerFill: intPointer(5), - }, - ok: true, - }, - } - - for n, tc := range cases { - t.Run(n, func(t *testing.T) { - - extensionName := api.BuiltinLocalRatelimitExtension - if tc.extensionName != "" { - extensionName = tc.extensionName - } - - svc := api.CompoundServiceName{Name: "svc"} - ext := extensioncommon.RuntimeConfig{ - ServiceName: svc, - EnvoyExtension: api.EnvoyExtension{ - Name: extensionName, - Arguments: tc.arguments, - }, - } - - e, err := Constructor(ext.EnvoyExtension) - - if tc.ok { - require.NoError(t, err) - require.Equal(t, &extensioncommon.BasicEnvoyExtender{Extension: &tc.expected}, e) - } else { - require.Error(t, err) - require.Contains(t, err.Error(), tc.expectedErrMsg) - } - }) - } -} - -func intPointer(i int) *int { - return &i -} diff --git a/agent/envoyextensions/builtin/lua/lua.go b/agent/envoyextensions/builtin/lua/lua.go index 76c802f6407..117912cd63f 100644 --- a/agent/envoyextensions/builtin/lua/lua.go +++ b/agent/envoyextensions/builtin/lua/lua.go @@ -61,11 +61,11 @@ func (l *lua) validate() error { // CanApply determines if the extension can apply to the given extension configuration. func (l *lua) CanApply(config *extensioncommon.RuntimeConfig) bool { - return string(config.Kind) == l.ProxyType && l.matchesListenerDirection(config) + return string(config.Kind) == l.ProxyType } -func (l *lua) matchesListenerDirection(config *extensioncommon.RuntimeConfig) bool { - return (config.IsUpstream() && l.Listener == "outbound") || (!config.IsUpstream() && l.Listener == "inbound") +func (l *lua) matchesListenerDirection(isInboundListener bool) bool { + return (!isInboundListener && l.Listener == "outbound") || (isInboundListener && l.Listener == "inbound") } // PatchRoute does nothing. @@ -79,7 +79,12 @@ func (l *lua) PatchCluster(_ *extensioncommon.RuntimeConfig, c *envoy_cluster_v3 } // PatchFilter inserts a lua filter directly prior to envoy.filters.http.router. -func (l *lua) PatchFilter(_ *extensioncommon.RuntimeConfig, filter *envoy_listener_v3.Filter) (*envoy_listener_v3.Filter, bool, error) { +func (l *lua) PatchFilter(_ *extensioncommon.RuntimeConfig, filter *envoy_listener_v3.Filter, isInboundListener bool) (*envoy_listener_v3.Filter, bool, error) { + // Make sure filter matches extension config. + if !l.matchesListenerDirection(isInboundListener) { + return filter, false, nil + } + if filter.Name != "envoy.filters.network.http_connection_manager" { return filter, false, nil } diff --git a/agent/envoyextensions/registered_extensions.go b/agent/envoyextensions/registered_extensions.go index fed8d3c59e8..c765df7c838 100644 --- a/agent/envoyextensions/registered_extensions.go +++ b/agent/envoyextensions/registered_extensions.go @@ -4,7 +4,6 @@ import ( "fmt" awslambda "github.com/hashicorp/consul/agent/envoyextensions/builtin/aws-lambda" - "github.com/hashicorp/consul/agent/envoyextensions/builtin/http/localratelimit" "github.com/hashicorp/consul/agent/envoyextensions/builtin/lua" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/envoyextensions/extensioncommon" @@ -14,9 +13,8 @@ import ( type extensionConstructor func(api.EnvoyExtension) (extensioncommon.EnvoyExtender, error) var extensionConstructors = map[string]extensionConstructor{ - api.BuiltinLuaExtension: lua.Constructor, - api.BuiltinAWSLambdaExtension: awslambda.Constructor, - api.BuiltinLocalRatelimitExtension: localratelimit.Constructor, + api.BuiltinLuaExtension: lua.Constructor, + api.BuiltinAWSLambdaExtension: awslambda.Constructor, } // ConstructExtension attempts to lookup and build an extension from the registry with the diff --git a/agent/envoyextensions/registered_extensions_test.go b/agent/envoyextensions/registered_extensions_test.go index 07f6e966845..3428dbed86a 100644 --- a/agent/envoyextensions/registered_extensions_test.go +++ b/agent/envoyextensions/registered_extensions_test.go @@ -1,9 +1,11 @@ package envoyextensions import ( + "fmt" "testing" "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/envoyextensions/extensioncommon" "github.com/stretchr/testify/require" ) @@ -58,3 +60,52 @@ func TestValidateExtensions(t *testing.T) { }) } } + +// This test is included here so that we can test all registered extensions without creating a cyclic dependency between +// envoyextensions and extensioncommon. +func TestUpstreamExtenderLimitations(t *testing.T) { + type testCase struct { + config *extensioncommon.RuntimeConfig + ok bool + errMsg string + } + unauthorizedExtensionCase := func(name string) testCase { + return testCase{ + config: &extensioncommon.RuntimeConfig{ + Kind: api.ServiceKindConnectProxy, + ServiceName: api.CompoundServiceName{Name: "api"}, + Upstreams: map[api.CompoundServiceName]*extensioncommon.UpstreamData{}, + IsSourcedFromUpstream: true, + EnvoyExtension: api.EnvoyExtension{ + Name: name, + }, + }, + ok: false, + errMsg: fmt.Sprintf("extension %q is not permitted to be applied via upstream service config", name), + } + } + cases := map[string]testCase{ + // Make sure future extensions are theoretically covered, even if not registered in the same way. + "unknown extension": unauthorizedExtensionCase("someotherextension"), + } + for name := range extensionConstructors { + // AWS Lambda is the only extension permitted to modify downstream proxy resources. + if name == api.BuiltinAWSLambdaExtension { + continue + } + cases[name] = unauthorizedExtensionCase(name) + } + + for n, tc := range cases { + t.Run(n, func(t *testing.T) { + extender := extensioncommon.UpstreamEnvoyExtender{} + _, err := extender.Extend(nil, tc.config) + if tc.ok { + require.NoError(t, err) + } else { + require.Error(t, err) + require.ErrorContains(t, err, tc.errMsg) + } + }) + } +} diff --git a/agent/grpc-external/limiter/limiter_test.go b/agent/grpc-external/limiter/limiter_test.go index cef6a4d4171..7f5b9654a0a 100644 --- a/agent/grpc-external/limiter/limiter_test.go +++ b/agent/grpc-external/limiter/limiter_test.go @@ -8,12 +8,8 @@ import ( "time" "github.com/stretchr/testify/require" - - "github.com/hashicorp/consul/lib" ) -func init() { lib.SeedMathRand() } - func TestSessionLimiter(t *testing.T) { lim := NewSessionLimiter() diff --git a/agent/grpc-external/metadata_test.go b/agent/grpc-external/options_test.go similarity index 100% rename from agent/grpc-external/metadata_test.go rename to agent/grpc-external/options_test.go diff --git a/agent/grpc-external/querymeta.go b/agent/grpc-external/querymeta.go new file mode 100644 index 00000000000..55b960255ff --- /dev/null +++ b/agent/grpc-external/querymeta.go @@ -0,0 +1,77 @@ +package external + +import ( + "fmt" + "reflect" + + "github.com/mitchellh/mapstructure" + "google.golang.org/grpc/metadata" + + "github.com/hashicorp/consul/agent/structs" +) + +func StringToQueryBackendDecodeHookFunc(f reflect.Type, t reflect.Type, data any) (any, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(structs.QueryBackend(0)) { + return data, nil + } + + name, ok := data.(string) + if !ok { + return data, fmt.Errorf("could not parse query backend as string") + } + + return structs.QueryBackendFromString(name), nil +} + +// QueryMetaFromGRPCMeta returns a structs.QueryMeta struct parsed from the metadata.MD, +// such as from a gRPC header or trailer. +func QueryMetaFromGRPCMeta(md metadata.MD) (structs.QueryMeta, error) { + var queryMeta structs.QueryMeta + + m := map[string]string{} + for k, v := range md { + m[k] = v[0] + } + + decodeHooks := mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + StringToQueryBackendDecodeHookFunc, + ) + + config := &mapstructure.DecoderConfig{ + Metadata: nil, + Result: &queryMeta, + WeaklyTypedInput: true, + DecodeHook: decodeHooks, + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return queryMeta, err + } + + err = decoder.Decode(m) + if err != nil { + return queryMeta, err + } + + return queryMeta, nil +} + +// GRPCMetadataFromQueryMeta returns a metadata struct with fields from the structs.QueryMeta attached. +// The return value is suitable for attaching to a gRPC header/trailer. +func GRPCMetadataFromQueryMeta(queryMeta structs.QueryMeta) (metadata.MD, error) { + md := metadata.MD{} + m := map[string]any{} + err := mapstructure.Decode(queryMeta, &m) + if err != nil { + return nil, err + } + for k, v := range m { + md.Set(k, fmt.Sprintf("%v", v)) + } + return md, nil +} diff --git a/agent/grpc-external/querymeta_test.go b/agent/grpc-external/querymeta_test.go new file mode 100644 index 00000000000..66c7136a3dd --- /dev/null +++ b/agent/grpc-external/querymeta_test.go @@ -0,0 +1,35 @@ +package external + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/hashicorp/consul/agent/structs" +) + +func TestQueryMetaFromGRPCMetaRoundTrip(t *testing.T) { + lastContact, err := time.ParseDuration("1s") + require.NoError(t, err) + + expected := structs.QueryMeta{ + Index: 42, + LastContact: lastContact, + KnownLeader: true, + ConsistencyLevel: "stale", + NotModified: true, + Backend: structs.QueryBackend(0), + ResultsFilteredByACLs: true, + } + + md, err := GRPCMetadataFromQueryMeta(expected) + require.NoError(t, err) + + actual, err := QueryMetaFromGRPCMeta(md) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, expected, actual) +} diff --git a/agent/grpc-external/services/peerstream/replication.go b/agent/grpc-external/services/peerstream/replication.go index 20ceac14892..f00b0b590a7 100644 --- a/agent/grpc-external/services/peerstream/replication.go +++ b/agent/grpc-external/services/peerstream/replication.go @@ -10,6 +10,7 @@ import ( newproto "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/cache" "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/structs" @@ -281,9 +282,11 @@ func (s *Server) handleUpsertExportedServiceList( exportedServices[snSidecarProxy] = struct{}{} serviceNames = append(serviceNames, sn) } - entMeta := structs.NodeEnterpriseMetaInPartition(partition) - _, serviceList, err := s.GetStore().ServiceList(nil, entMeta, peerName) + // Ensure we query services from all namespaces in this partition when we perform + // this query or else we may not propagate updates / deletes correctly. + entMeta := acl.NewEnterpriseMetaWithPartition(partition, acl.WildcardName) + _, serviceList, err := s.GetStore().ServiceList(nil, &entMeta, peerName) if err != nil { return err } @@ -338,7 +341,7 @@ func (s *Server) handleUpdateService( for _, nodeSnap := range snap.Nodes { // First register the node - skip the unchanged ones changed := true - if storedNode, ok := storedNodesMap[nodeSnap.Node.ID]; ok { + if storedNode, ok := storedNodesMap[nodeSnap.Node.Node]; ok { if storedNode.IsSame(nodeSnap.Node) { changed = false } @@ -354,7 +357,7 @@ func (s *Server) handleUpdateService( // Then register all services on that node - skip the unchanged ones for _, svcSnap := range nodeSnap.Services { changed = true - if storedSvcInst, ok := storedSvcInstMap[makeNodeSvcInstID(nodeSnap.Node.ID, svcSnap.Service.ID)]; ok { + if storedSvcInst, ok := storedSvcInstMap[makeNodeSvcInstID(nodeSnap.Node.Node, svcSnap.Service.ID)]; ok { if storedSvcInst.IsSame(svcSnap.Service) { changed = false } @@ -374,7 +377,7 @@ func (s *Server) handleUpdateService( for _, svcSnap := range nodeSnap.Services { for _, c := range svcSnap.Checks { changed := true - if chk, ok := storedChecksMap[makeNodeCheckID(nodeSnap.Node.ID, svcSnap.Service.ID, c.CheckID)]; ok { + if chk, ok := storedChecksMap[makeNodeCheckID(nodeSnap.Node.Node, svcSnap.Service.ID, c.CheckID)]; ok { if chk.IsSame(c) { changed = false } @@ -512,8 +515,10 @@ func (s *Server) handleUpdateService( // Delete any nodes that do not have any other services registered on them. for node := range unusedNodes { - nodeMeta := structs.NodeEnterpriseMetaInPartition(sn.PartitionOrDefault()) - _, ns, err := s.GetStore().NodeServices(nil, node, nodeMeta, peerName) + // The wildcard is used here so that all services, regardless of namespace are returned + // by the following query. Without this, the node might accidentally be cleaned up early. + wildcardNSMeta := acl.NewEnterpriseMetaWithPartition(sn.PartitionOrDefault(), acl.WildcardName) + _, ns, err := s.GetStore().NodeServiceList(nil, node, &wildcardNSMeta, peerName) if err != nil { return fmt.Errorf("failed to query services on node: %w", err) } @@ -526,10 +531,10 @@ func (s *Server) handleUpdateService( err = s.Backend.CatalogDeregister(&structs.DeregisterRequest{ Node: node, PeerName: peerName, - EnterpriseMeta: *nodeMeta, + EnterpriseMeta: *structs.NodeEnterpriseMetaInPartition(sn.PartitionOrDefault()), }) if err != nil { - ident := fmt.Sprintf("partition:%s/peer:%s/node:%s", nodeMeta.PartitionOrDefault(), peerName, node) + ident := fmt.Sprintf("partition:%s/peer:%s/node:%s", sn.PartitionOrDefault(), peerName, node) return fmt.Errorf("failed to deregister node %q: %w", ident, err) } } @@ -632,31 +637,35 @@ type nodeCheckIdentity struct { checkID string } -func makeNodeSvcInstID(nodeID types.NodeID, serviceID string) nodeSvcInstIdentity { +func makeNodeSvcInstID(node string, serviceID string) nodeSvcInstIdentity { return nodeSvcInstIdentity{ - nodeID: string(nodeID), + nodeID: node, serviceID: serviceID, } } -func makeNodeCheckID(nodeID types.NodeID, serviceID string, checkID types.CheckID) nodeCheckIdentity { +func makeNodeCheckID(node string, serviceID string, checkID types.CheckID) nodeCheckIdentity { return nodeCheckIdentity{ serviceID: serviceID, checkID: string(checkID), - nodeID: string(nodeID), + nodeID: node, } } -func buildStoredMap(storedInstances structs.CheckServiceNodes) (map[types.NodeID]*structs.Node, map[nodeSvcInstIdentity]*structs.NodeService, map[nodeCheckIdentity]*structs.HealthCheck) { - nodesMap := map[types.NodeID]*structs.Node{} +func buildStoredMap(storedInstances structs.CheckServiceNodes) ( + map[string]*structs.Node, + map[nodeSvcInstIdentity]*structs.NodeService, + map[nodeCheckIdentity]*structs.HealthCheck, +) { + nodesMap := map[string]*structs.Node{} svcInstMap := map[nodeSvcInstIdentity]*structs.NodeService{} checksMap := map[nodeCheckIdentity]*structs.HealthCheck{} for _, csn := range storedInstances { - nodesMap[csn.Node.ID] = csn.Node - svcInstMap[makeNodeSvcInstID(csn.Node.ID, csn.Service.ID)] = csn.Service + nodesMap[csn.Node.Node] = csn.Node + svcInstMap[makeNodeSvcInstID(csn.Node.Node, csn.Service.ID)] = csn.Service for _, chk := range csn.Checks { - checksMap[makeNodeCheckID(csn.Node.ID, csn.Service.ID, chk.CheckID)] = chk + checksMap[makeNodeCheckID(csn.Node.Node, csn.Service.ID, chk.CheckID)] = chk } } return nodesMap, svcInstMap, checksMap diff --git a/agent/grpc-external/services/peerstream/server.go b/agent/grpc-external/services/peerstream/server.go index 8a5e33e93c5..a67da563a03 100644 --- a/agent/grpc-external/services/peerstream/server.go +++ b/agent/grpc-external/services/peerstream/server.go @@ -119,7 +119,7 @@ type StateStore interface { ExportedServicesForPeer(ws memdb.WatchSet, peerID, dc string) (uint64, *structs.ExportedServiceList, error) ServiceDump(ws memdb.WatchSet, kind structs.ServiceKind, useKind bool, entMeta *acl.EnterpriseMeta, peerName string) (uint64, structs.CheckServiceNodes, error) CheckServiceNodes(ws memdb.WatchSet, serviceName string, entMeta *acl.EnterpriseMeta, peerName string) (uint64, structs.CheckServiceNodes, error) - NodeServices(ws memdb.WatchSet, nodeNameOrID string, entMeta *acl.EnterpriseMeta, peerName string) (uint64, *structs.NodeServices, error) + NodeServiceList(ws memdb.WatchSet, nodeNameOrID string, entMeta *acl.EnterpriseMeta, peerName string) (uint64, *structs.NodeServiceList, error) CAConfig(ws memdb.WatchSet) (uint64, *structs.CAConfiguration, error) TrustBundleListByService(ws memdb.WatchSet, service, dc string, entMeta acl.EnterpriseMeta) (uint64, []*pbpeering.PeeringTrustBundle, error) ServiceList(ws memdb.WatchSet, entMeta *acl.EnterpriseMeta, peerName string) (uint64, structs.ServiceList, error) diff --git a/agent/grpc-external/services/peerstream/stream_test.go b/agent/grpc-external/services/peerstream/stream_test.go index b21f35bccef..5f5d399af9e 100644 --- a/agent/grpc-external/services/peerstream/stream_test.go +++ b/agent/grpc-external/services/peerstream/stream_test.go @@ -547,7 +547,7 @@ func TestStreamResources_Server_StreamTracker(t *testing.T) { it := incrementalTime{ base: time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC), } - waitUntil := it.FutureNow(6) + waitUntil := it.FutureNow(7) srv, store := newTestServer(t, nil) srv.Tracker.setClock(it.Now) @@ -844,6 +844,13 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { Node: &structs.Node{Node: "foo", Address: "10.0.0.1"}, Service: &structs.NodeService{ID: "mysql-1", Service: "mysql", Port: 5000}, } + mysqlSidecar := &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "mysql-sidecar-proxy", + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "mysql", + }, + } lastIdx++ require.NoError(t, store.EnsureNode(lastIdx, mysql.Node)) @@ -851,6 +858,9 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { lastIdx++ require.NoError(t, store.EnsureService(lastIdx, "foo", mysql.Service)) + lastIdx++ + require.NoError(t, store.EnsureService(lastIdx, "foo", mysqlSidecar)) + mongoSvcDefaults := &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, Name: "mongo", @@ -870,6 +880,24 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { mysqlProxySN = structs.NewServiceName("mysql-sidecar-proxy", nil).String() ) + testutil.RunStep(t, "initial stream data is received", func(t *testing.T) { + expectReplEvents(t, client, + func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { + require.Equal(t, pbpeerstream.TypeURLPeeringTrustBundle, msg.GetResponse().ResourceURL) + // Roots tested in TestStreamResources_Server_CARootUpdates + }, + func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { + require.Equal(t, pbpeerstream.TypeURLExportedServiceList, msg.GetResponse().ResourceURL) + require.Equal(t, subExportedServiceList, msg.GetResponse().ResourceID) + require.Equal(t, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) + + var exportedServices pbpeerstream.ExportedServiceList + require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&exportedServices)) + require.ElementsMatch(t, []string{}, exportedServices.Services) + }, + ) + }) + testutil.RunStep(t, "exporting mysql leads to an UPSERT event", func(t *testing.T) { entry := &structs.ExportedServicesConfigEntry{ Name: "default", @@ -895,10 +923,6 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { require.NoError(t, store.EnsureConfigEntry(lastIdx, entry)) expectReplEvents(t, client, - func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { - require.Equal(t, pbpeerstream.TypeURLPeeringTrustBundle, msg.GetResponse().ResourceURL) - // Roots tested in TestStreamResources_Server_CARootUpdates - }, func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { // no mongo instances exist require.Equal(t, pbpeerstream.TypeURLExportedService, msg.GetResponse().ResourceURL) @@ -909,16 +933,6 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&nodes)) require.Len(t, nodes.Nodes, 0) }, - func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { - // proxies can't export because no mesh gateway exists yet - require.Equal(t, pbpeerstream.TypeURLExportedService, msg.GetResponse().ResourceURL) - require.Equal(t, mongoProxySN, msg.GetResponse().ResourceID) - require.Equal(t, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) - - var nodes pbpeerstream.ExportedService - require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&nodes)) - require.Len(t, nodes.Nodes, 0) - }, func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { require.Equal(t, pbpeerstream.TypeURLExportedService, msg.GetResponse().ResourceURL) require.Equal(t, mysqlSN, msg.GetResponse().ResourceID) @@ -938,17 +952,6 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&nodes)) require.Len(t, nodes.Nodes, 0) }, - // This event happens because this is the first test case and there are - // no exported services when replication is initially set up. - func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { - require.Equal(t, pbpeerstream.TypeURLExportedServiceList, msg.GetResponse().ResourceURL) - require.Equal(t, subExportedServiceList, msg.GetResponse().ResourceID) - require.Equal(t, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) - - var exportedServices pbpeerstream.ExportedServiceList - require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&exportedServices)) - require.ElementsMatch(t, []string{}, exportedServices.Services) - }, func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { require.Equal(t, pbpeerstream.TypeURLExportedServiceList, msg.GetResponse().ResourceURL) require.Equal(t, subExportedServiceList, msg.GetResponse().ResourceID) @@ -978,7 +981,7 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { expectReplEvents(t, client, func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { require.Equal(t, pbpeerstream.TypeURLExportedService, msg.GetResponse().ResourceURL) - require.Equal(t, mongoProxySN, msg.GetResponse().ResourceID) + require.Equal(t, mysqlProxySN, msg.GetResponse().ResourceID) require.Equal(t, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) var nodes pbpeerstream.ExportedService @@ -986,16 +989,26 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { require.Len(t, nodes.Nodes, 1) pm := nodes.Nodes[0].Service.Connect.PeerMeta - require.Equal(t, "grpc", pm.Protocol) + require.Equal(t, "tcp", pm.Protocol) spiffeIDs := []string{ - "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/mongo", + "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/mysql", "spiffe://11111111-2222-3333-4444-555555555555.consul/gateway/mesh/dc/dc1", } require.Equal(t, spiffeIDs, pm.SpiffeID) }, + ) + }) + + testutil.RunStep(t, "register service resolver to send proxy updates", func(t *testing.T) { + lastIdx++ + require.NoError(t, store.EnsureConfigEntry(lastIdx, &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "mongo", + })) + expectReplEvents(t, client, func(t *testing.T, msg *pbpeerstream.ReplicationMessage) { require.Equal(t, pbpeerstream.TypeURLExportedService, msg.GetResponse().ResourceURL) - require.Equal(t, mysqlProxySN, msg.GetResponse().ResourceID) + require.Equal(t, mongoProxySN, msg.GetResponse().ResourceID) require.Equal(t, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) var nodes pbpeerstream.ExportedService @@ -1003,9 +1016,9 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { require.Len(t, nodes.Nodes, 1) pm := nodes.Nodes[0].Service.Connect.PeerMeta - require.Equal(t, "tcp", pm.Protocol) + require.Equal(t, "grpc", pm.Protocol) spiffeIDs := []string{ - "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/mysql", + "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/mongo", "spiffe://11111111-2222-3333-4444-555555555555.consul/gateway/mesh/dc/dc1", } require.Equal(t, spiffeIDs, pm.SpiffeID) @@ -1032,7 +1045,7 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { require.Equal(r, mongo.Service.CompoundServiceName().String(), msg.GetResponse().ResourceID) var nodes pbpeerstream.ExportedService - require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&nodes)) + require.NoError(r, msg.GetResponse().Resource.UnmarshalTo(&nodes)) require.Len(r, nodes.Nodes, 1) }) }) @@ -1061,12 +1074,12 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { msg, err := client.RecvWithTimeout(100 * time.Millisecond) require.NoError(r, err) require.Equal(r, pbpeerstream.TypeURLExportedServiceList, msg.GetResponse().ResourceURL) - require.Equal(t, subExportedServiceList, msg.GetResponse().ResourceID) - require.Equal(t, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) + require.Equal(r, subExportedServiceList, msg.GetResponse().ResourceID) + require.Equal(r, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) var exportedServices pbpeerstream.ExportedServiceList - require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&exportedServices)) - require.Equal(t, []string{structs.ServiceName{Name: "mongo"}.String()}, exportedServices.Services) + require.NoError(r, msg.GetResponse().Resource.UnmarshalTo(&exportedServices)) + require.Equal(r, []string{structs.ServiceName{Name: "mongo"}.String()}, exportedServices.Services) }) }) @@ -1078,12 +1091,12 @@ func TestStreamResources_Server_ServiceUpdates(t *testing.T) { msg, err := client.RecvWithTimeout(100 * time.Millisecond) require.NoError(r, err) require.Equal(r, pbpeerstream.TypeURLExportedServiceList, msg.GetResponse().ResourceURL) - require.Equal(t, subExportedServiceList, msg.GetResponse().ResourceID) - require.Equal(t, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) + require.Equal(r, subExportedServiceList, msg.GetResponse().ResourceID) + require.Equal(r, pbpeerstream.Operation_OPERATION_UPSERT, msg.GetResponse().Operation) var exportedServices pbpeerstream.ExportedServiceList - require.NoError(t, msg.GetResponse().Resource.UnmarshalTo(&exportedServices)) - require.Len(t, exportedServices.Services, 0) + require.NoError(r, msg.GetResponse().Resource.UnmarshalTo(&exportedServices)) + require.Len(r, exportedServices.Services, 0) }) }) } @@ -1239,8 +1252,8 @@ func TestStreamResources_Server_DisconnectsOnHeartbeatTimeout(t *testing.T) { }) testutil.RunStep(t, "stream is disconnected due to heartbeat timeout", func(t *testing.T) { - disconnectTime := ptr(it.FutureNow(1)) retry.Run(t, func(r *retry.R) { + disconnectTime := ptr(it.StaticNow()) status, ok := srv.StreamStatus(testPeerID) require.True(r, ok) require.False(r, status.Connected) @@ -1410,7 +1423,7 @@ func makeClient(t *testing.T, srv *testServer, peerID string) *MockClient { }, })) - // Receive a services and roots subscription request pair from server + // Receive ExportedService, ExportedServiceList, and PeeringTrustBundle subscription requests from server receivedSub1, err := client.Recv() require.NoError(t, err) receivedSub2, err := client.Recv() @@ -1426,7 +1439,9 @@ func makeClient(t *testing.T, srv *testServer, peerID string) *MockClient { // Note that server address may not come as an initial message for _, resourceURL := range []string{ pbpeerstream.TypeURLExportedService, + pbpeerstream.TypeURLExportedServiceList, pbpeerstream.TypeURLPeeringTrustBundle, + // only dialers request, which is why this is absent below pbpeerstream.TypeURLPeeringServerAddresses, } { init := &pbpeerstream.ReplicationMessage{ @@ -1455,7 +1470,7 @@ func makeClient(t *testing.T, srv *testServer, peerID string) *MockClient { { Payload: &pbpeerstream.ReplicationMessage_Request_{ Request: &pbpeerstream.ReplicationMessage_Request{ - ResourceURL: pbpeerstream.TypeURLPeeringTrustBundle, + ResourceURL: pbpeerstream.TypeURLExportedServiceList, // The PeerID field is only set for the messages coming FROM // the establishing side and are going to be empty from the // other side. @@ -1466,7 +1481,7 @@ func makeClient(t *testing.T, srv *testServer, peerID string) *MockClient { { Payload: &pbpeerstream.ReplicationMessage_Request_{ Request: &pbpeerstream.ReplicationMessage_Request{ - ResourceURL: pbpeerstream.TypeURLPeeringServerAddresses, + ResourceURL: pbpeerstream.TypeURLPeeringTrustBundle, // The PeerID field is only set for the messages coming FROM // the establishing side and are going to be empty from the // other side. @@ -1578,7 +1593,11 @@ func Test_ExportedServicesCount(t *testing.T) { mst, err := srv.Tracker.Connected(peerID) require.NoError(t, err) - services := []string{"web", "api", "mongo"} + services := []string{ + structs.NewServiceName("web", nil).String(), + structs.NewServiceName("api", nil).String(), + structs.NewServiceName("mongo", nil).String(), + } update := cache.UpdateEvent{ CorrelationID: subExportedServiceList, Result: &pbpeerstream.ExportedServiceList{ @@ -1922,36 +1941,30 @@ func expectReplEvents(t *testing.T, client *MockClient, checkFns ...func(t *test } } -func Test_processResponse_ExportedServiceUpdates(t *testing.T) { - srv, store := newTestServer(t, func(c *Config) { - backend := c.Backend.(*testStreamBackend) - backend.leader = func() bool { - return false - } - }) - - type testCase struct { - name string - seed []*structs.RegisterRequest - input *pbpeerstream.ExportedService - expect map[string]structs.CheckServiceNodes - exportedServices []string - } - - peerName := "billing" - peerID := "1fabcd52-1d46-49b0-b1d8-71559aee47f5" - remoteMeta := pbcommon.NewEnterpriseMetaFromStructs(*structs.DefaultEnterpriseMetaInPartition("billing-ap")) - - // "api" service is imported from the billing-ap partition, corresponding to the billing peer. - // Locally it is stored to the default partition. - defaultMeta := *acl.DefaultEnterpriseMeta() - apiSN := structs.NewServiceName("api", &defaultMeta) +type PeeringProcessResponse_testCase struct { + name string + seed []*structs.RegisterRequest + inputServiceName structs.ServiceName + input *pbpeerstream.ExportedService + expect map[structs.ServiceName]structs.CheckServiceNodes + exportedServices []string +} +func processResponse_ExportedServiceUpdates( + t *testing.T, + srv *testServer, + store *state.Store, + localEntMeta acl.EnterpriseMeta, + peerName string, + tests []PeeringProcessResponse_testCase, +) *MutableStatus { // create a peering in the state store + peerID := "1fabcd52-1d46-49b0-b1d8-71559aee47f5" require.NoError(t, store.PeeringWrite(31, &pbpeering.PeeringWriteRequest{ Peering: &pbpeering.Peering{ - ID: peerID, - Name: peerName, + ID: peerID, + Name: peerName, + Partition: localEntMeta.PartitionOrDefault(), }, })) @@ -1959,7 +1972,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { mst, err := srv.Tracker.Connected(peerID) require.NoError(t, err) - run := func(t *testing.T, tc testCase) { + run := func(t *testing.T, tc PeeringProcessResponse_testCase) { // Seed the local catalog with some data to reconcile against. // and increment the tracker's imported services count var serviceNames []structs.ServiceName @@ -1973,14 +1986,14 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { in := &pbpeerstream.ReplicationMessage_Response{ ResourceURL: pbpeerstream.TypeURLExportedService, - ResourceID: apiSN.String(), + ResourceID: tc.inputServiceName.String(), Nonce: "1", Operation: pbpeerstream.Operation_OPERATION_UPSERT, Resource: makeAnyPB(t, tc.input), } // Simulate an update arriving for billing/api. - _, err = srv.processResponse(peerName, acl.DefaultPartitionName, mst, in) + _, err = srv.processResponse(peerName, localEntMeta.PartitionOrDefault(), mst, in) require.NoError(t, err) if len(tc.exportedServices) > 0 { @@ -1993,63 +2006,82 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { } // Simulate an update arriving for billing/api. - _, err = srv.processResponse(peerName, acl.DefaultPartitionName, mst, resp) + _, err = srv.processResponse(peerName, localEntMeta.PartitionOrDefault(), mst, resp) require.NoError(t, err) // Test the count and contents separately to ensure the count code path is hit. require.Equal(t, mst.GetImportedServicesCount(), len(tc.exportedServices)) require.ElementsMatch(t, mst.ImportedServices, tc.exportedServices) } - _, allServices, err := srv.GetStore().ServiceList(nil, &defaultMeta, peerName) + wildcardNS := acl.NewEnterpriseMetaWithPartition(localEntMeta.PartitionOrDefault(), acl.WildcardName) + _, allServices, err := srv.GetStore().ServiceList(nil, &wildcardNS, peerName) require.NoError(t, err) // This ensures that only services specified under tc.expect are stored. It includes // all exported services plus their sidecar proxies. for _, svc := range allServices { - _, ok := tc.expect[svc.Name] + _, ok := tc.expect[svc] require.True(t, ok) } for svc, expect := range tc.expect { - t.Run(svc, func(t *testing.T) { - _, got, err := srv.GetStore().CheckServiceNodes(nil, svc, &defaultMeta, peerName) + t.Run(svc.String(), func(t *testing.T) { + _, got, err := srv.GetStore().CheckServiceNodes(nil, svc.Name, &svc.EnterpriseMeta, peerName) require.NoError(t, err) requireEqualInstances(t, expect, got) }) } } - tt := []testCase{ + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + run(t, tc) + }) + } + return mst +} + +func Test_processResponse_ExportedServiceUpdates(t *testing.T) { + peerName := "billing" + localEntMeta := *acl.DefaultEnterpriseMeta() + + remoteMeta := *structs.DefaultEnterpriseMetaInPartition("billing-ap") + pbRemoteMeta := pbcommon.NewEnterpriseMetaFromStructs(remoteMeta) + + apiLocalSN := structs.NewServiceName("api", &localEntMeta) + redisLocalSN := structs.NewServiceName("redis", &localEntMeta) + tests := []PeeringProcessResponse_testCase{ { name: "upsert two service instances to the same node", - exportedServices: []string{"api"}, + exportedServices: []string{apiLocalSN.String()}, + inputServiceName: structs.NewServiceName("api", &remoteMeta), input: &pbpeerstream.ExportedService{ Nodes: []*pbservice.CheckServiceNode{ { Node: &pbservice.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: remoteMeta.Partition, + Partition: pbRemoteMeta.Partition, PeerName: peerName, }, Service: &pbservice.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, Checks: []*pbservice.HealthCheck{ { CheckID: "node-foo-check", Node: "node-foo", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, { CheckID: "api-1-check", ServiceID: "api-1", Node: "node-foo", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, }, @@ -2058,42 +2090,42 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Node: &pbservice.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: remoteMeta.Partition, + Partition: pbRemoteMeta.Partition, PeerName: peerName, }, Service: &pbservice.NodeService{ ID: "api-2", Service: "api", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, Checks: []*pbservice.HealthCheck{ { CheckID: "node-foo-check", Node: "node-foo", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, { CheckID: "api-2-check", ServiceID: "api-2", Node: "node-foo", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, }, }, }, }, - expect: map[string]structs.CheckServiceNodes{ - "api": { + expect: map[structs.ServiceName]structs.CheckServiceNodes{ + structs.NewServiceName("api", &localEntMeta): { { Node: &structs.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", // The remote billing-ap partition is overwritten for all resources with the local default. - Partition: defaultMeta.PartitionOrEmpty(), + Partition: localEntMeta.PartitionOrEmpty(), // The name of the peer "billing" is attached as well. PeerName: peerName, @@ -2101,21 +2133,21 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: []*structs.HealthCheck{ { CheckID: "node-foo-check", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, { CheckID: "api-1-check", ServiceID: "api-1", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, }, @@ -2124,27 +2156,27 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Node: &structs.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: defaultMeta.PartitionOrEmpty(), + Partition: localEntMeta.PartitionOrEmpty(), PeerName: peerName, }, Service: &structs.NodeService{ ID: "api-2", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: []*structs.HealthCheck{ { CheckID: "node-foo-check", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, { CheckID: "api-2-check", ServiceID: "api-2", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, }, @@ -2154,7 +2186,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, { name: "deleting a service with an empty exported service event", - exportedServices: []string{"api"}, + exportedServices: []string{apiLocalSN.String()}, seed: []*structs.RegisterRequest{ { ID: types.NodeID("af913374-68ea-41e5-82e8-6ffd3dffc461"), @@ -2163,7 +2195,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "api-2", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: structs.HealthChecks{ @@ -2181,41 +2213,43 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, }, }, - input: &pbpeerstream.ExportedService{}, - expect: map[string]structs.CheckServiceNodes{ - "api": {}, + inputServiceName: structs.NewServiceName("api", &remoteMeta), + input: &pbpeerstream.ExportedService{}, + expect: map[structs.ServiceName]structs.CheckServiceNodes{ + structs.NewServiceName("api", &localEntMeta): {}, }, }, { name: "upsert two service instances to different nodes", - exportedServices: []string{"api"}, + exportedServices: []string{apiLocalSN.String()}, + inputServiceName: structs.NewServiceName("api", &remoteMeta), input: &pbpeerstream.ExportedService{ Nodes: []*pbservice.CheckServiceNode{ { Node: &pbservice.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: remoteMeta.Partition, + Partition: pbRemoteMeta.Partition, PeerName: peerName, }, Service: &pbservice.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, Checks: []*pbservice.HealthCheck{ { CheckID: "node-foo-check", Node: "node-foo", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, { CheckID: "api-1-check", ServiceID: "api-1", Node: "node-foo", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, }, @@ -2224,60 +2258,60 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Node: &pbservice.Node{ ID: "c0f97de9-4e1b-4e80-a1c6-cd8725835ab2", Node: "node-bar", - Partition: remoteMeta.Partition, + Partition: pbRemoteMeta.Partition, PeerName: peerName, }, Service: &pbservice.NodeService{ ID: "api-2", Service: "api", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, Checks: []*pbservice.HealthCheck{ { CheckID: "node-bar-check", Node: "node-bar", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, { CheckID: "api-2-check", ServiceID: "api-2", Node: "node-bar", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, }, }, }, }, - expect: map[string]structs.CheckServiceNodes{ - "api": { + expect: map[structs.ServiceName]structs.CheckServiceNodes{ + structs.NewServiceName("api", &localEntMeta): { { Node: &structs.Node{ ID: "c0f97de9-4e1b-4e80-a1c6-cd8725835ab2", Node: "node-bar", - Partition: defaultMeta.PartitionOrEmpty(), + Partition: localEntMeta.PartitionOrEmpty(), PeerName: peerName, }, Service: &structs.NodeService{ ID: "api-2", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: []*structs.HealthCheck{ { CheckID: "node-bar-check", Node: "node-bar", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, { CheckID: "api-2-check", ServiceID: "api-2", Node: "node-bar", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, }, @@ -2288,7 +2322,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Node: "node-foo", // The remote billing-ap partition is overwritten for all resources with the local default. - Partition: defaultMeta.PartitionOrEmpty(), + Partition: localEntMeta.PartitionOrEmpty(), // The name of the peer "billing" is attached as well. PeerName: peerName, @@ -2296,21 +2330,21 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: []*structs.HealthCheck{ { CheckID: "node-foo-check", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, { CheckID: "api-1-check", ServiceID: "api-1", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, }, @@ -2320,7 +2354,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, { name: "deleting one service name from a node does not delete other service names", - exportedServices: []string{"api", "redis"}, + exportedServices: []string{apiLocalSN.String(), redisLocalSN.String()}, seed: []*structs.RegisterRequest{ { ID: types.NodeID("af913374-68ea-41e5-82e8-6ffd3dffc461"), @@ -2329,7 +2363,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "redis-2", Service: "redis", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: structs.HealthChecks{ @@ -2353,7 +2387,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: structs.HealthChecks{ @@ -2371,37 +2405,38 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, }, }, + inputServiceName: structs.NewServiceName("api", &remoteMeta), // Nil input is for the "api" service. input: &pbpeerstream.ExportedService{}, - expect: map[string]structs.CheckServiceNodes{ - "api": {}, + expect: map[structs.ServiceName]structs.CheckServiceNodes{ + structs.NewServiceName("api", &localEntMeta): {}, // Existing redis service was not affected by deletion. - "redis": { + structs.NewServiceName("redis", &localEntMeta): { { Node: &structs.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: defaultMeta.PartitionOrEmpty(), + Partition: localEntMeta.PartitionOrEmpty(), PeerName: peerName, }, Service: &structs.NodeService{ ID: "redis-2", Service: "redis", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: []*structs.HealthCheck{ { CheckID: "node-foo-check", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, { CheckID: "redis-2-check", ServiceID: "redis-2", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, }, @@ -2419,7 +2454,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "redis-2", Service: "redis", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: structs.HealthChecks{ @@ -2443,7 +2478,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "redis-2-sidecar-proxy", Service: "redis-sidecar-proxy", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: structs.HealthChecks{ @@ -2467,7 +2502,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: structs.HealthChecks{ @@ -2491,7 +2526,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "api-1-sidecar-proxy", Service: "api-sidecar-proxy", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: structs.HealthChecks{ @@ -2510,68 +2545,69 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, }, }, + inputServiceName: structs.NewServiceName("api", &remoteMeta), // Nil input is for the "api" service. input: &pbpeerstream.ExportedService{}, - exportedServices: []string{"redis"}, - expect: map[string]structs.CheckServiceNodes{ + exportedServices: []string{redisLocalSN.String()}, + expect: map[structs.ServiceName]structs.CheckServiceNodes{ // Existing redis service was not affected by deletion. - "redis": { + structs.NewServiceName("redis", &localEntMeta): { { Node: &structs.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: defaultMeta.PartitionOrEmpty(), + Partition: localEntMeta.PartitionOrEmpty(), PeerName: peerName, }, Service: &structs.NodeService{ ID: "redis-2", Service: "redis", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: []*structs.HealthCheck{ { CheckID: "node-foo-check", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, { CheckID: "redis-2-check", ServiceID: "redis-2", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, }, }, }, - "redis-sidecar-proxy": { + structs.NewServiceName("redis-sidecar-proxy", &localEntMeta): { { Node: &structs.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: defaultMeta.PartitionOrEmpty(), + Partition: localEntMeta.PartitionOrEmpty(), PeerName: peerName, }, Service: &structs.NodeService{ ID: "redis-2-sidecar-proxy", Service: "redis-sidecar-proxy", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: []*structs.HealthCheck{ { CheckID: "node-foo-check", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, { CheckID: "redis-2-sidecar-proxy-check", ServiceID: "redis-2-sidecar-proxy", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, }, @@ -2581,7 +2617,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, { name: "service checks are cleaned up when not present in a response", - exportedServices: []string{"api"}, + exportedServices: []string{apiLocalSN.String()}, seed: []*structs.RegisterRequest{ { ID: types.NodeID("af913374-68ea-41e5-82e8-6ffd3dffc461"), @@ -2590,7 +2626,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: structs.HealthChecks{ @@ -2608,19 +2644,20 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, }, }, + inputServiceName: structs.NewServiceName("api", &remoteMeta), input: &pbpeerstream.ExportedService{ Nodes: []*pbservice.CheckServiceNode{ { Node: &pbservice.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: remoteMeta.Partition, + Partition: pbRemoteMeta.Partition, PeerName: peerName, }, Service: &pbservice.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, Checks: []*pbservice.HealthCheck{ @@ -2629,20 +2666,20 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, }, }, - expect: map[string]structs.CheckServiceNodes{ + expect: map[structs.ServiceName]structs.CheckServiceNodes{ // Service check should be gone - "api": { + structs.NewServiceName("api", &localEntMeta): { { Node: &structs.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: defaultMeta.PartitionOrEmpty(), + Partition: localEntMeta.PartitionOrEmpty(), PeerName: peerName, }, Service: &structs.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: []*structs.HealthCheck{}, @@ -2652,7 +2689,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, { name: "node checks are cleaned up when not present in a response", - exportedServices: []string{"api", "redis"}, + exportedServices: []string{apiLocalSN.String(), redisLocalSN.String()}, seed: []*structs.RegisterRequest{ { ID: types.NodeID("af913374-68ea-41e5-82e8-6ffd3dffc461"), @@ -2661,7 +2698,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "redis-2", Service: "redis", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: structs.HealthChecks{ @@ -2685,7 +2722,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: structs.HealthChecks{ @@ -2703,19 +2740,20 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, }, }, + inputServiceName: structs.NewServiceName("api", &remoteMeta), input: &pbpeerstream.ExportedService{ Nodes: []*pbservice.CheckServiceNode{ { Node: &pbservice.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: remoteMeta.Partition, + Partition: pbRemoteMeta.Partition, PeerName: peerName, }, Service: &pbservice.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, Checks: []*pbservice.HealthCheck{ @@ -2724,27 +2762,27 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { CheckID: "api-1-check", ServiceID: "api-1", Node: "node-foo", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, }, }, }, }, - expect: map[string]structs.CheckServiceNodes{ + expect: map[structs.ServiceName]structs.CheckServiceNodes{ // Node check should be gone - "api": { + structs.NewServiceName("api", &localEntMeta): { { Node: &structs.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: defaultMeta.PartitionOrEmpty(), + Partition: localEntMeta.PartitionOrEmpty(), PeerName: peerName, }, Service: &structs.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: []*structs.HealthCheck{ @@ -2752,24 +2790,24 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { CheckID: "api-1-check", ServiceID: "api-1", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, }, }, }, - "redis": { + structs.NewServiceName("redis", &localEntMeta): { { Node: &structs.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: defaultMeta.PartitionOrEmpty(), + Partition: localEntMeta.PartitionOrEmpty(), PeerName: peerName, }, Service: &structs.NodeService{ ID: "redis-2", Service: "redis", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: []*structs.HealthCheck{ @@ -2777,7 +2815,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { CheckID: "redis-2-check", ServiceID: "redis-2", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, }, @@ -2787,7 +2825,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, { name: "replacing a service instance on a node cleans up the old instance", - exportedServices: []string{"api", "redis"}, + exportedServices: []string{apiLocalSN.String(), redisLocalSN.String()}, seed: []*structs.RegisterRequest{ { ID: types.NodeID("af913374-68ea-41e5-82e8-6ffd3dffc461"), @@ -2796,7 +2834,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "redis-2", Service: "redis", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: structs.HealthChecks{ @@ -2820,7 +2858,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { Service: &structs.NodeService{ ID: "api-1", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: structs.HealthChecks{ @@ -2838,20 +2876,21 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, }, }, + inputServiceName: structs.NewServiceName("api", &remoteMeta), input: &pbpeerstream.ExportedService{ Nodes: []*pbservice.CheckServiceNode{ { Node: &pbservice.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: remoteMeta.Partition, + Partition: pbRemoteMeta.Partition, PeerName: peerName, }, // New service ID and checks for the api service. Service: &pbservice.NodeService{ ID: "new-api-v2", Service: "api", - EnterpriseMeta: remoteMeta, + EnterpriseMeta: pbRemoteMeta, PeerName: peerName, }, Checks: []*pbservice.HealthCheck{ @@ -2870,19 +2909,19 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, }, }, - expect: map[string]structs.CheckServiceNodes{ - "api": { + expect: map[structs.ServiceName]structs.CheckServiceNodes{ + structs.NewServiceName("api", &localEntMeta): { { Node: &structs.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: defaultMeta.PartitionOrEmpty(), + Partition: localEntMeta.PartitionOrEmpty(), PeerName: peerName, }, Service: &structs.NodeService{ ID: "new-api-v2", Service: "api", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: []*structs.HealthCheck{ @@ -2895,24 +2934,24 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { CheckID: "new-api-v2-check", ServiceID: "new-api-v2", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, }, }, }, - "redis": { + structs.NewServiceName("redis", &localEntMeta): { { Node: &structs.Node{ ID: "af913374-68ea-41e5-82e8-6ffd3dffc461", Node: "node-foo", - Partition: defaultMeta.PartitionOrEmpty(), + Partition: localEntMeta.PartitionOrEmpty(), PeerName: peerName, }, Service: &structs.NodeService{ ID: "redis-2", Service: "redis", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, Checks: []*structs.HealthCheck{ @@ -2925,7 +2964,7 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { CheckID: "redis-2-check", ServiceID: "redis-2", Node: "node-foo", - EnterpriseMeta: defaultMeta, + EnterpriseMeta: localEntMeta, PeerName: peerName, }, }, @@ -2934,12 +2973,13 @@ func Test_processResponse_ExportedServiceUpdates(t *testing.T) { }, }, } - - for _, tc := range tt { - t.Run(tc.name, func(t *testing.T) { - run(t, tc) - }) - } + srv, store := newTestServer(t, func(c *Config) { + backend := c.Backend.(*testStreamBackend) + backend.leader = func() bool { + return false + } + }) + processResponse_ExportedServiceUpdates(t, srv, store, localEntMeta, peerName, tests) } // TestLogTraceProto tests that all PB trace log helpers redact the diff --git a/agent/grpc-external/services/peerstream/stream_tracker.go b/agent/grpc-external/services/peerstream/stream_tracker.go index f36ca055d10..61f8831ec78 100644 --- a/agent/grpc-external/services/peerstream/stream_tracker.go +++ b/agent/grpc-external/services/peerstream/stream_tracker.go @@ -369,9 +369,8 @@ func (s *MutableStatus) SetImportedServices(serviceNames []structs.ServiceName) defer s.mu.Unlock() s.ImportedServices = make([]string, len(serviceNames)) - for i, sn := range serviceNames { - s.ImportedServices[i] = sn.Name + s.ImportedServices[i] = sn.String() } } @@ -389,7 +388,7 @@ func (s *MutableStatus) SetExportedServices(serviceNames []structs.ServiceName) s.ExportedServices = make([]string, len(serviceNames)) for i, sn := range serviceNames { - s.ExportedServices[i] = sn.Name + s.ExportedServices[i] = sn.String() } } diff --git a/agent/grpc-external/services/peerstream/subscription_manager.go b/agent/grpc-external/services/peerstream/subscription_manager.go index 0f9174dd859..ec710e01758 100644 --- a/agent/grpc-external/services/peerstream/subscription_manager.go +++ b/agent/grpc-external/services/peerstream/subscription_manager.go @@ -143,7 +143,7 @@ func (m *subscriptionManager) handleEvent(ctx context.Context, state *subscripti pending := &pendingPayload{} m.syncNormalServices(ctx, state, evt.Services) if m.config.ConnectEnabled { - m.syncDiscoveryChains(state, pending, evt.ListAllDiscoveryChains()) + m.syncDiscoveryChains(state, pending, evt.DiscoChains) } err := pending.Add( @@ -744,7 +744,7 @@ func (m *subscriptionManager) NotifyStandardService( // // This name was chosen to match existing "sidecar service" generation logic // and similar logic in the Service Identity synthetic ACL policies. -const syntheticProxyNameSuffix = "-sidecar-proxy" +const syntheticProxyNameSuffix = structs.SidecarProxySuffix func generateProxyNameForDiscoveryChain(sn structs.ServiceName) structs.ServiceName { return structs.NewServiceName(sn.Name+syntheticProxyNameSuffix, &sn.EnterpriseMeta) diff --git a/agent/grpc-external/services/peerstream/subscription_manager_test.go b/agent/grpc-external/services/peerstream/subscription_manager_test.go index 6d7a41979ce..c7b77edec96 100644 --- a/agent/grpc-external/services/peerstream/subscription_manager_test.go +++ b/agent/grpc-external/services/peerstream/subscription_manager_test.go @@ -472,15 +472,40 @@ func TestSubscriptionManager_InitialSnapshot(t *testing.T) { Node: &structs.Node{Node: "foo", Address: "10.0.0.1"}, Service: &structs.NodeService{ID: "mysql-1", Service: "mysql", Port: 5000}, } + mysqlSidecar := structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "mysql-sidecar-proxy", + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "mysql", + }, + } backend.ensureNode(t, mysql.Node) backend.ensureService(t, "foo", mysql.Service) + backend.ensureService(t, "foo", &mysqlSidecar) mongo := &structs.CheckServiceNode{ - Node: &structs.Node{Node: "zip", Address: "10.0.0.3"}, - Service: &structs.NodeService{ID: "mongo-1", Service: "mongo", Port: 5000}, + Node: &structs.Node{Node: "zip", Address: "10.0.0.3"}, + Service: &structs.NodeService{ + ID: "mongo-1", + Service: "mongo", + Port: 5000, + }, + } + mongoSidecar := structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "mongo-sidecar-proxy", + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "mongo", + }, } backend.ensureNode(t, mongo.Node) backend.ensureService(t, "zip", mongo.Service) + backend.ensureService(t, "zip", &mongoSidecar) + + backend.ensureConfigEntry(t, &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "chain", + }) var ( mysqlCorrID = subExportedService + structs.NewServiceName("mysql", nil).String() diff --git a/agent/grpc-external/services/peerstream/testing.go b/agent/grpc-external/services/peerstream/testing.go index 4f0297a6c52..5eb575c06aa 100644 --- a/agent/grpc-external/services/peerstream/testing.go +++ b/agent/grpc-external/services/peerstream/testing.go @@ -150,6 +150,16 @@ func (t *incrementalTime) Now() time.Time { return t.base.Add(dur) } +// StaticNow returns the current internal clock without advancing it. +func (t *incrementalTime) StaticNow() time.Time { + t.mu.Lock() + defer t.mu.Unlock() + + dur := time.Duration(t.next) * time.Second + + return t.base.Add(dur) +} + // FutureNow will return a given future value of the Now() function. // The numerical argument indicates which future Now value you wanted. The // value must be > 0. diff --git a/agent/grpc-internal/balancer/balancer.go b/agent/grpc-internal/balancer/balancer.go index efd349c82c9..64521d4567f 100644 --- a/agent/grpc-internal/balancer/balancer.go +++ b/agent/grpc-internal/balancer/balancer.go @@ -65,21 +65,25 @@ import ( "google.golang.org/grpc/status" ) -// NewBuilder constructs a new Builder with the given name. -func NewBuilder(name string, logger hclog.Logger) *Builder { +// NewBuilder constructs a new Builder. Calling Register will add the Builder +// to our global registry under the given "authority" such that it will be used +// when dialing targets in the form "consul-internal:///...", this +// allows us to add and remove balancers for different in-memory agents during +// tests. +func NewBuilder(authority string, logger hclog.Logger) *Builder { return &Builder{ - name: name, - logger: logger, - byTarget: make(map[string]*list.List), - shuffler: randomShuffler(), + authority: authority, + logger: logger, + byTarget: make(map[string]*list.List), + shuffler: randomShuffler(), } } // Builder implements gRPC's balancer.Builder interface to construct balancers. type Builder struct { - name string - logger hclog.Logger - shuffler shuffler + authority string + logger hclog.Logger + shuffler shuffler mu sync.Mutex byTarget map[string]*list.List @@ -129,19 +133,15 @@ func (b *Builder) removeBalancer(targetURL string, elem *list.Element) { } } -// Name implements the gRPC Balancer interface by returning its given name. -func (b *Builder) Name() string { return b.name } - -// gRPC's balancer.Register method is not thread-safe, so we guard our calls -// with a global lock (as it may be called from parallel tests). -var registerLock sync.Mutex - -// Register the Builder in gRPC's global registry using its given name. +// Register the Builder in our global registry. Users should call Deregister +// when finished using the Builder to clean-up global state. func (b *Builder) Register() { - registerLock.Lock() - defer registerLock.Unlock() + globalRegistry.register(b.authority, b) +} - gbalancer.Register(b) +// Deregister the Builder from our global registry to clean up state. +func (b *Builder) Deregister() { + globalRegistry.deregister(b.authority) } // Rebalance randomizes the priority order of servers for the given target to diff --git a/agent/grpc-internal/balancer/balancer_test.go b/agent/grpc-internal/balancer/balancer_test.go index 830092ab3cb..8406e7c7b70 100644 --- a/agent/grpc-internal/balancer/balancer_test.go +++ b/agent/grpc-internal/balancer/balancer_test.go @@ -21,6 +21,8 @@ import ( "google.golang.org/grpc/stats" "google.golang.org/grpc/status" + "github.com/hashicorp/go-uuid" + "github.com/hashicorp/consul/agent/grpc-middleware/testutil/testservice" "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/consul/sdk/testutil/retry" @@ -34,12 +36,13 @@ func TestBalancer(t *testing.T) { server1 := runServer(t, "server1") server2 := runServer(t, "server2") - target, _ := stubResolver(t, server1, server2) + target, authority, _ := stubResolver(t, server1, server2) - balancerBuilder := NewBuilder(t.Name(), testutil.Logger(t)) + balancerBuilder := NewBuilder(authority, testutil.Logger(t)) balancerBuilder.Register() + t.Cleanup(balancerBuilder.Deregister) - conn := dial(t, target, balancerBuilder) + conn := dial(t, target) client := testservice.NewSimpleClient(conn) var serverName string @@ -78,12 +81,13 @@ func TestBalancer(t *testing.T) { server1 := runServer(t, "server1") server2 := runServer(t, "server2") - target, _ := stubResolver(t, server1, server2) + target, authority, _ := stubResolver(t, server1, server2) - balancerBuilder := NewBuilder(t.Name(), testutil.Logger(t)) + balancerBuilder := NewBuilder(authority, testutil.Logger(t)) balancerBuilder.Register() + t.Cleanup(balancerBuilder.Deregister) - conn := dial(t, target, balancerBuilder) + conn := dial(t, target) client := testservice.NewSimpleClient(conn) // Figure out which server we're talking to now, and which we should switch to. @@ -123,10 +127,11 @@ func TestBalancer(t *testing.T) { server1 := runServer(t, "server1") server2 := runServer(t, "server2") - target, _ := stubResolver(t, server1, server2) + target, authority, _ := stubResolver(t, server1, server2) - balancerBuilder := NewBuilder(t.Name(), testutil.Logger(t)) + balancerBuilder := NewBuilder(authority, testutil.Logger(t)) balancerBuilder.Register() + t.Cleanup(balancerBuilder.Deregister) // Provide a custom prioritizer that causes Rebalance to choose whichever // server didn't get our first request. @@ -137,7 +142,7 @@ func TestBalancer(t *testing.T) { }) } - conn := dial(t, target, balancerBuilder) + conn := dial(t, target) client := testservice.NewSimpleClient(conn) // Figure out which server we're talking to now. @@ -177,12 +182,13 @@ func TestBalancer(t *testing.T) { server1 := runServer(t, "server1") server2 := runServer(t, "server2") - target, res := stubResolver(t, server1, server2) + target, authority, res := stubResolver(t, server1, server2) - balancerBuilder := NewBuilder(t.Name(), testutil.Logger(t)) + balancerBuilder := NewBuilder(authority, testutil.Logger(t)) balancerBuilder.Register() + t.Cleanup(balancerBuilder.Deregister) - conn := dial(t, target, balancerBuilder) + conn := dial(t, target) client := testservice.NewSimpleClient(conn) // Figure out which server we're talking to now. @@ -233,7 +239,7 @@ func TestBalancer(t *testing.T) { }) } -func stubResolver(t *testing.T, servers ...*server) (string, *manual.Resolver) { +func stubResolver(t *testing.T, servers ...*server) (string, string, *manual.Resolver) { t.Helper() addresses := make([]resolver.Address, len(servers)) @@ -249,7 +255,10 @@ func stubResolver(t *testing.T, servers ...*server) (string, *manual.Resolver) { resolver.Register(r) t.Cleanup(func() { resolver.UnregisterForTesting(scheme) }) - return fmt.Sprintf("%s://", scheme), r + authority, err := uuid.GenerateUUID() + require.NoError(t, err) + + return fmt.Sprintf("%s://%s", scheme, authority), authority, r } func runServer(t *testing.T, name string) *server { @@ -309,12 +318,12 @@ func (s *server) Something(context.Context, *testservice.Req) (*testservice.Resp return &testservice.Resp{ServerName: s.name}, nil } -func dial(t *testing.T, target string, builder *Builder) *grpc.ClientConn { +func dial(t *testing.T, target string) *grpc.ClientConn { conn, err := grpc.Dial( target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig( - fmt.Sprintf(`{"loadBalancingConfig":[{"%s":{}}]}`, builder.Name()), + fmt.Sprintf(`{"loadBalancingConfig":[{"%s":{}}]}`, BuilderName), ), ) t.Cleanup(func() { diff --git a/agent/grpc-internal/balancer/registry.go b/agent/grpc-internal/balancer/registry.go new file mode 100644 index 00000000000..778b2c31c5f --- /dev/null +++ b/agent/grpc-internal/balancer/registry.go @@ -0,0 +1,69 @@ +package balancer + +import ( + "fmt" + "sync" + + gbalancer "google.golang.org/grpc/balancer" +) + +// BuilderName should be given in gRPC service configuration to enable our +// custom balancer. It refers to this package's global registry, rather than +// an instance of Builder to enable us to add and remove builders at runtime, +// specifically during tests. +const BuilderName = "consul-internal" + +// gRPC's balancer.Register method is thread-unsafe because it mutates a global +// map without holding a lock. As such, it's expected that you register custom +// balancers once at the start of your program (e.g. a package init function). +// +// In production, this is fine. Agents register a single instance of our builder +// and use it for the duration. Tests are where this becomes problematic, as we +// spin up several agents in-memory and register/deregister a builder for each, +// with its own agent-specific state, logger, etc. +// +// To avoid data races, we call gRPC's Register method once, on-package init, +// with a global registry struct that implements the Builder interface but +// delegates the building to N instances of our Builder that are registered and +// deregistered at runtime. We the dial target's host (aka "authority") which +// is unique per-agent to pick the correct builder. +func init() { + gbalancer.Register(globalRegistry) +} + +var globalRegistry = ®istry{ + byAuthority: make(map[string]*Builder), +} + +type registry struct { + mu sync.RWMutex + byAuthority map[string]*Builder +} + +func (r *registry) Build(cc gbalancer.ClientConn, opts gbalancer.BuildOptions) gbalancer.Balancer { + r.mu.RLock() + defer r.mu.RUnlock() + + auth := opts.Target.URL.Host + builder, ok := r.byAuthority[auth] + if !ok { + panic(fmt.Sprintf("no gRPC balancer builder registered for authority: %q", auth)) + } + return builder.Build(cc, opts) +} + +func (r *registry) Name() string { return BuilderName } + +func (r *registry) register(auth string, builder *Builder) { + r.mu.Lock() + defer r.mu.Unlock() + + r.byAuthority[auth] = builder +} + +func (r *registry) deregister(auth string) { + r.mu.Lock() + defer r.mu.Unlock() + + delete(r.byAuthority, auth) +} diff --git a/agent/grpc-internal/client.go b/agent/grpc-internal/client.go index 36431f2488a..38010ac2452 100644 --- a/agent/grpc-internal/client.go +++ b/agent/grpc-internal/client.go @@ -8,12 +8,12 @@ import ( "time" "google.golang.org/grpc" - gbalancer "google.golang.org/grpc/balancer" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/keepalive" "github.com/armon/go-metrics" + "github.com/hashicorp/consul/agent/grpc-internal/balancer" agentmiddleware "github.com/hashicorp/consul/agent/grpc-middleware" "github.com/hashicorp/consul/agent/metadata" "github.com/hashicorp/consul/agent/pool" @@ -22,8 +22,8 @@ import ( // grpcServiceConfig is provided as the default service config. // -// It configures our custom balancer (via the %s directive to interpolate its -// name) which will automatically switch servers on error. +// It configures our custom balancer which will automatically switch servers +// on error. // // It also enables gRPC's built-in automatic retries for RESOURCE_EXHAUSTED // errors *only*, as this is the status code servers will return for an @@ -41,7 +41,7 @@ import ( // but we're working on generating them automatically from the protobuf files const grpcServiceConfig = ` { - "loadBalancingConfig": [{"%s":{}}], + "loadBalancingConfig": [{"` + balancer.BuilderName + `":{}}], "methodConfig": [ { "name": [{}], @@ -131,12 +131,11 @@ const grpcServiceConfig = ` // ClientConnPool creates and stores a connection for each datacenter. type ClientConnPool struct { - dialer dialer - servers ServerLocator - gwResolverDep gatewayResolverDep - conns map[string]*grpc.ClientConn - connsLock sync.Mutex - balancerBuilder gbalancer.Builder + dialer dialer + servers ServerLocator + gwResolverDep gatewayResolverDep + conns map[string]*grpc.ClientConn + connsLock sync.Mutex } type ServerLocator interface { @@ -198,21 +197,14 @@ type ClientConnPoolConfig struct { // DialingFromDatacenter is the datacenter of the consul agent using this // pool. DialingFromDatacenter string - - // BalancerBuilder is a builder for the gRPC balancer that will be used. - BalancerBuilder gbalancer.Builder } // NewClientConnPool create new GRPC client pool to connect to servers using // GRPC over RPC. func NewClientConnPool(cfg ClientConnPoolConfig) *ClientConnPool { - if cfg.BalancerBuilder == nil { - panic("missing required BalancerBuilder") - } c := &ClientConnPool{ - servers: cfg.Servers, - conns: make(map[string]*grpc.ClientConn), - balancerBuilder: cfg.BalancerBuilder, + servers: cfg.Servers, + conns: make(map[string]*grpc.ClientConn), } c.dialer = newDialer(cfg, &c.gwResolverDep) return c @@ -251,9 +243,7 @@ func (c *ClientConnPool) dial(datacenter string, serverType string) (*grpc.Clien grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(c.dialer), grpc.WithStatsHandler(agentmiddleware.NewStatsHandler(metrics.Default(), metricsLabels)), - grpc.WithDefaultServiceConfig( - fmt.Sprintf(grpcServiceConfig, c.balancerBuilder.Name()), - ), + grpc.WithDefaultServiceConfig(grpcServiceConfig), // Keep alive parameters are based on the same default ones we used for // Yamux. These are somewhat arbitrary but we did observe in scale testing // that the gRPC defaults (servers send keepalives only every 2 hours, diff --git a/agent/grpc-internal/client_test.go b/agent/grpc-internal/client_test.go index ebd0601ad4c..f27808af6f2 100644 --- a/agent/grpc-internal/client_test.go +++ b/agent/grpc-internal/client_test.go @@ -13,7 +13,6 @@ import ( "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - gbalancer "google.golang.org/grpc/balancer" "github.com/hashicorp/consul/agent/grpc-internal/balancer" "github.com/hashicorp/consul/agent/grpc-internal/resolver" @@ -36,8 +35,8 @@ func TestNewDialer_WithTLSWrapper(t *testing.T) { require.NoError(t, err) t.Cleanup(logError(t, lis.Close)) - builder := resolver.NewServerResolverBuilder(newConfig(t)) - builder.AddServer(types.AreaWAN, &metadata.Server{ + builder := resolver.NewServerResolverBuilder(newConfig(t, "dc1", "server")) + builder.AddServer(types.AreaLAN, &metadata.Server{ Name: "server-1", ID: "ID1", Datacenter: "dc1", @@ -87,7 +86,7 @@ func TestNewDialer_WithALPNWrapper(t *testing.T) { p.Wait() }() - builder := resolver.NewServerResolverBuilder(newConfig(t)) + builder := resolver.NewServerResolverBuilder(newConfig(t, "dc1", "server")) builder.AddServer(types.AreaWAN, &metadata.Server{ Name: "server-1", ID: "ID1", @@ -142,8 +141,9 @@ func TestNewDialer_WithALPNWrapper(t *testing.T) { func TestNewDialer_IntegrationWithTLSEnabledHandler(t *testing.T) { // if this test is failing because of expired certificates // use the procedure in test/CA-GENERATION.md - res := resolver.NewServerResolverBuilder(newConfig(t)) - registerWithGRPC(t, res) + res := resolver.NewServerResolverBuilder(newConfig(t, "dc1", "server")) + bb := balancer.NewBuilder(res.Authority(), testutil.Logger(t)) + registerWithGRPC(t, res, bb) tlsConf, err := tlsutil.NewConfigurator(tlsutil.Config{ InternalRPC: tlsutil.ProtocolConfig{ @@ -159,16 +159,23 @@ func TestNewDialer_IntegrationWithTLSEnabledHandler(t *testing.T) { srv := newSimpleTestServer(t, "server-1", "dc1", tlsConf) md := srv.Metadata() - res.AddServer(types.AreaWAN, md) + res.AddServer(types.AreaLAN, md) t.Cleanup(srv.shutdown) + { + // Put a duplicate instance of this on the WAN that will + // fail if we accidentally use it. + srv := newPanicTestServer(t, hclog.Default(), "server-1", "dc1", nil) + res.AddServer(types.AreaWAN, srv.Metadata()) + t.Cleanup(srv.shutdown) + } + pool := NewClientConnPool(ClientConnPoolConfig{ Servers: res, TLSWrapper: TLSWrapper(tlsConf.OutgoingRPCWrapper()), UseTLSForDC: tlsConf.UseTLS, DialingFromServer: true, DialingFromDatacenter: "dc1", - BalancerBuilder: balancerBuilder(t, res.Authority()), }) conn, err := pool.ClientConn("dc1") @@ -190,8 +197,9 @@ func TestNewDialer_IntegrationWithTLSEnabledHandler_viaMeshGateway(t *testing.T) // use the procedure in test/CA-GENERATION.md gwAddr := ipaddr.FormatAddressPort("127.0.0.1", freeport.GetOne(t)) - res := resolver.NewServerResolverBuilder(newConfig(t)) - registerWithGRPC(t, res) + res := resolver.NewServerResolverBuilder(newConfig(t, "dc2", "server")) + bb := balancer.NewBuilder(res.Authority(), testutil.Logger(t)) + registerWithGRPC(t, res, bb) tlsConf, err := tlsutil.NewConfigurator(tlsutil.Config{ InternalRPC: tlsutil.ProtocolConfig{ @@ -244,7 +252,6 @@ func TestNewDialer_IntegrationWithTLSEnabledHandler_viaMeshGateway(t *testing.T) UseTLSForDC: tlsConf.UseTLS, DialingFromServer: true, DialingFromDatacenter: "dc2", - BalancerBuilder: balancerBuilder(t, res.Authority()), }) pool.SetGatewayResolver(func(addr string) string { return gwAddr @@ -266,21 +273,30 @@ func TestNewDialer_IntegrationWithTLSEnabledHandler_viaMeshGateway(t *testing.T) func TestClientConnPool_IntegrationWithGRPCResolver_Failover(t *testing.T) { count := 4 - res := resolver.NewServerResolverBuilder(newConfig(t)) - registerWithGRPC(t, res) + res := resolver.NewServerResolverBuilder(newConfig(t, "dc1", "server")) + bb := balancer.NewBuilder(res.Authority(), testutil.Logger(t)) + registerWithGRPC(t, res, bb) pool := NewClientConnPool(ClientConnPoolConfig{ Servers: res, UseTLSForDC: useTLSForDcAlwaysTrue, DialingFromServer: true, DialingFromDatacenter: "dc1", - BalancerBuilder: balancerBuilder(t, res.Authority()), }) for i := 0; i < count; i++ { name := fmt.Sprintf("server-%d", i) - srv := newSimpleTestServer(t, name, "dc1", nil) - res.AddServer(types.AreaWAN, srv.Metadata()) - t.Cleanup(srv.shutdown) + { + srv := newSimpleTestServer(t, name, "dc1", nil) + res.AddServer(types.AreaLAN, srv.Metadata()) + t.Cleanup(srv.shutdown) + } + { + // Put a duplicate instance of this on the WAN that will + // fail if we accidentally use it. + srv := newPanicTestServer(t, hclog.Default(), name, "dc1", nil) + res.AddServer(types.AreaWAN, srv.Metadata()) + t.Cleanup(srv.shutdown) + } } conn, err := pool.ClientConn("dc1") @@ -293,7 +309,7 @@ func TestClientConnPool_IntegrationWithGRPCResolver_Failover(t *testing.T) { first, err := client.Something(ctx, &testservice.Req{}) require.NoError(t, err) - res.RemoveServer(types.AreaWAN, &metadata.Server{ID: first.ServerName, Datacenter: "dc1"}) + res.RemoveServer(types.AreaLAN, &metadata.Server{ID: first.ServerName, Datacenter: "dc1"}) resp, err := client.Something(ctx, &testservice.Req{}) require.NoError(t, err) @@ -302,23 +318,32 @@ func TestClientConnPool_IntegrationWithGRPCResolver_Failover(t *testing.T) { func TestClientConnPool_ForwardToLeader_Failover(t *testing.T) { count := 3 - res := resolver.NewServerResolverBuilder(newConfig(t)) - registerWithGRPC(t, res) + res := resolver.NewServerResolverBuilder(newConfig(t, "dc1", "server")) + bb := balancer.NewBuilder(res.Authority(), testutil.Logger(t)) + registerWithGRPC(t, res, bb) pool := NewClientConnPool(ClientConnPoolConfig{ Servers: res, UseTLSForDC: useTLSForDcAlwaysTrue, DialingFromServer: true, DialingFromDatacenter: "dc1", - BalancerBuilder: balancerBuilder(t, res.Authority()), }) var servers []testServer for i := 0; i < count; i++ { name := fmt.Sprintf("server-%d", i) - srv := newSimpleTestServer(t, name, "dc1", nil) - res.AddServer(types.AreaWAN, srv.Metadata()) - servers = append(servers, srv) - t.Cleanup(srv.shutdown) + { + srv := newSimpleTestServer(t, name, "dc1", nil) + res.AddServer(types.AreaLAN, srv.Metadata()) + servers = append(servers, srv) + t.Cleanup(srv.shutdown) + } + { + // Put a duplicate instance of this on the WAN that will + // fail if we accidentally use it. + srv := newPanicTestServer(t, hclog.Default(), name, "dc1", nil) + res.AddServer(types.AreaWAN, srv.Metadata()) + t.Cleanup(srv.shutdown) + } } // Set the leader address to the first server. @@ -345,30 +370,43 @@ func TestClientConnPool_ForwardToLeader_Failover(t *testing.T) { require.Equal(t, resp.ServerName, servers[1].name) } -func newConfig(t *testing.T) resolver.Config { +func newConfig(t *testing.T, dc, agentType string) resolver.Config { n := t.Name() s := strings.Replace(n, "/", "", -1) s = strings.Replace(s, "_", "", -1) - return resolver.Config{Authority: strings.ToLower(s)} + return resolver.Config{ + Datacenter: dc, + AgentType: agentType, + Authority: strings.ToLower(s), + } } func TestClientConnPool_IntegrationWithGRPCResolver_MultiDC(t *testing.T) { dcs := []string{"dc1", "dc2", "dc3"} - res := resolver.NewServerResolverBuilder(newConfig(t)) - registerWithGRPC(t, res) + res := resolver.NewServerResolverBuilder(newConfig(t, "dc1", "server")) + bb := balancer.NewBuilder(res.Authority(), testutil.Logger(t)) + registerWithGRPC(t, res, bb) pool := NewClientConnPool(ClientConnPoolConfig{ Servers: res, UseTLSForDC: useTLSForDcAlwaysTrue, DialingFromServer: true, DialingFromDatacenter: "dc1", - BalancerBuilder: balancerBuilder(t, res.Authority()), }) for _, dc := range dcs { name := "server-0-" + dc srv := newSimpleTestServer(t, name, dc, nil) - res.AddServer(types.AreaWAN, srv.Metadata()) + if dc == "dc1" { + res.AddServer(types.AreaLAN, srv.Metadata()) + // Put a duplicate instance of this on the WAN that will + // fail if we accidentally use it. + srvBad := newPanicTestServer(t, hclog.Default(), name, dc, nil) + res.AddServer(types.AreaWAN, srvBad.Metadata()) + t.Cleanup(srvBad.shutdown) + } else { + res.AddServer(types.AreaWAN, srv.Metadata()) + } t.Cleanup(srv.shutdown) } @@ -386,18 +424,11 @@ func TestClientConnPool_IntegrationWithGRPCResolver_MultiDC(t *testing.T) { } } -func registerWithGRPC(t *testing.T, b *resolver.ServerResolverBuilder) { - resolver.Register(b) +func registerWithGRPC(t *testing.T, rb *resolver.ServerResolverBuilder, bb *balancer.Builder) { + resolver.Register(rb) + bb.Register() t.Cleanup(func() { - resolver.Deregister(b.Authority()) + resolver.Deregister(rb.Authority()) + bb.Deregister() }) } - -func balancerBuilder(t *testing.T, name string) gbalancer.Builder { - t.Helper() - - bb := balancer.NewBuilder(name, testutil.Logger(t)) - bb.Register() - - return bb -} diff --git a/agent/grpc-internal/handler_test.go b/agent/grpc-internal/handler_test.go index 96f6f036e0e..c8b2d5fd3fe 100644 --- a/agent/grpc-internal/handler_test.go +++ b/agent/grpc-internal/handler_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/consul/types" "github.com/hashicorp/go-hclog" @@ -13,6 +14,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/hashicorp/consul/agent/grpc-internal/balancer" "github.com/hashicorp/consul/agent/grpc-internal/resolver" "github.com/hashicorp/consul/agent/grpc-middleware/testutil/testservice" ) @@ -26,11 +28,12 @@ func TestHandler_PanicRecoveryInterceptor(t *testing.T) { Output: &buf, }) - res := resolver.NewServerResolverBuilder(newConfig(t)) - registerWithGRPC(t, res) + res := resolver.NewServerResolverBuilder(newConfig(t, "dc1", "server")) + bb := balancer.NewBuilder(res.Authority(), testutil.Logger(t)) + registerWithGRPC(t, res, bb) srv := newPanicTestServer(t, logger, "server-1", "dc1", nil) - res.AddServer(types.AreaWAN, srv.Metadata()) + res.AddServer(types.AreaLAN, srv.Metadata()) t.Cleanup(srv.shutdown) pool := NewClientConnPool(ClientConnPoolConfig{ @@ -38,7 +41,6 @@ func TestHandler_PanicRecoveryInterceptor(t *testing.T) { UseTLSForDC: useTLSForDcAlwaysTrue, DialingFromServer: true, DialingFromDatacenter: "dc1", - BalancerBuilder: balancerBuilder(t, res.Authority()), }) conn, err := pool.ClientConn("dc1") diff --git a/agent/grpc-internal/resolver/registry.go b/agent/grpc-internal/resolver/registry.go index 582103b9eae..61e2b8ffe73 100644 --- a/agent/grpc-internal/resolver/registry.go +++ b/agent/grpc-internal/resolver/registry.go @@ -20,10 +20,10 @@ func (r *registry) Build(target resolver.Target, cc resolver.ClientConn, opts re r.lock.RLock() defer r.lock.RUnlock() //nolint:staticcheck - res, ok := r.byAuthority[target.Authority] + res, ok := r.byAuthority[target.URL.Host] if !ok { //nolint:staticcheck - return nil, fmt.Errorf("no resolver registered for %v", target.Authority) + return nil, fmt.Errorf("no resolver registered for %v", target.URL.Host) } return res.Build(target, cc, opts) } diff --git a/agent/grpc-internal/resolver/resolver.go b/agent/grpc-internal/resolver/resolver.go index b5b76de6efc..0d43d0a3903 100644 --- a/agent/grpc-internal/resolver/resolver.go +++ b/agent/grpc-internal/resolver/resolver.go @@ -15,25 +15,45 @@ import ( // ServerResolvers updated when changes occur. type ServerResolverBuilder struct { cfg Config + // leaderResolver is used to track the address of the leader in the local DC. leaderResolver leaderResolver + // servers is an index of Servers by area and Server.ID. The map contains server IDs // for all datacenters. servers map[types.AreaID]map[string]*metadata.Server + // resolvers is an index of connections to the serverResolver which manages // addresses of servers for that connection. + // + // this is only applicable for non-leader conn types resolvers map[resolver.ClientConn]*serverResolver + // lock for all stateful fields (excludes config which is immutable). lock sync.RWMutex } type Config struct { + // Datacenter is the datacenter of this agent. + Datacenter string + + // AgentType is either 'server' or 'client' and is required. + AgentType string + // Authority used to query the server. Defaults to "". Used to support // parallel testing because gRPC registers resolvers globally. Authority string } func NewServerResolverBuilder(cfg Config) *ServerResolverBuilder { + if cfg.Datacenter == "" { + panic("ServerResolverBuilder needs Config.Datacenter to be nonempty") + } + switch cfg.AgentType { + case "server", "client": + default: + panic("ServerResolverBuilder needs Config.AgentType to be either server or client") + } return &ServerResolverBuilder{ cfg: cfg, servers: make(map[types.AreaID]map[string]*metadata.Server), @@ -53,6 +73,7 @@ func (s *ServerResolverBuilder) ServerForGlobalAddr(globalAddr string) (*metadat } } } + return nil, fmt.Errorf("failed to find Consul server for global address %q", globalAddr) } @@ -64,12 +85,12 @@ func (s *ServerResolverBuilder) Build(target resolver.Target, cc resolver.Client // If there's already a resolver for this connection, return it. // TODO(streaming): how would this happen since we already cache connections in ClientConnPool? - if resolver, ok := s.resolvers[cc]; ok { - return resolver, nil - } if cc == s.leaderResolver.clientConn { return s.leaderResolver, nil } + if resolver, ok := s.resolvers[cc]; ok { + return resolver, nil + } //nolint:staticcheck serverType, datacenter, err := parseEndpoint(target.Endpoint) @@ -116,6 +137,10 @@ func (s *ServerResolverBuilder) Authority() string { // AddServer updates the resolvers' states to include the new server's address. func (s *ServerResolverBuilder) AddServer(areaID types.AreaID, server *metadata.Server) { + if s.shouldIgnoreServer(areaID, server) { + return + } + s.lock.Lock() defer s.lock.Unlock() @@ -152,6 +177,10 @@ func DCPrefix(datacenter, suffix string) string { // RemoveServer updates the resolvers' states with the given server removed. func (s *ServerResolverBuilder) RemoveServer(areaID types.AreaID, server *metadata.Server) { + if s.shouldIgnoreServer(areaID, server) { + return + } + s.lock.Lock() defer s.lock.Unlock() @@ -173,14 +202,48 @@ func (s *ServerResolverBuilder) RemoveServer(areaID types.AreaID, server *metada } } +// shouldIgnoreServer is used to contextually decide if a particular kind of +// server should be accepted into a given area. +// +// On client agents it's pretty easy: clients only participate in the standard +// LAN, so we only accept servers from the LAN. +// +// On server agents it's a little less obvious. This resolver is ultimately +// used to have servers dial other servers. If a server is going to cross +// between datacenters (using traditional federation) then we want to use the +// WAN addresses for them, but if a server is going to dial a sibling server in +// the same datacenter we want it to use the LAN addresses always. To achieve +// that here we simply never allow WAN servers for our current datacenter to be +// added into the resolver, letting only the LAN instances through. +func (s *ServerResolverBuilder) shouldIgnoreServer(areaID types.AreaID, server *metadata.Server) bool { + if s.cfg.AgentType == "client" && areaID != types.AreaLAN { + return true + } + + if s.cfg.AgentType == "server" && + server.Datacenter == s.cfg.Datacenter && + areaID != types.AreaLAN { + return true + } + + return false +} + // getDCAddrs returns a list of the server addresses for the given datacenter. // This method requires that lock is held for reads. func (s *ServerResolverBuilder) getDCAddrs(dc string) []resolver.Address { + lanRequest := (s.cfg.Datacenter == dc) + var ( addrs []resolver.Address keptServerIDs = make(map[string]struct{}) ) - for _, areaServers := range s.servers { + for areaID, areaServers := range s.servers { + if (areaID == types.AreaLAN) != lanRequest { + // LAN requests only look at LAN data. WAN requests only look at + // WAN data. + continue + } for _, server := range areaServers { if server.Datacenter != dc { continue diff --git a/agent/grpc-internal/resolver/resolver_test.go b/agent/grpc-internal/resolver/resolver_test.go new file mode 100644 index 00000000000..ab6e403d7de --- /dev/null +++ b/agent/grpc-internal/resolver/resolver_test.go @@ -0,0 +1,195 @@ +package resolver + +import ( + "fmt" + "net" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + + "github.com/hashicorp/consul/agent/metadata" + "github.com/hashicorp/consul/types" +) + +func TestServerResolverBuilder(t *testing.T) { + const agentDC = "dc1" + + type testcase struct { + name string + agentType string // server/client + serverType string // server/leader + requestDC string + expectLAN bool + } + + run := func(t *testing.T, tc testcase) { + rs := NewServerResolverBuilder(newConfig(t, agentDC, tc.agentType)) + + endpoint := "" + if tc.serverType == "leader" { + endpoint = "leader.local" + } else { + endpoint = tc.serverType + "." + tc.requestDC + } + + cc := &fakeClientConn{} + _, err := rs.Build(resolver.Target{ + Scheme: "consul", + Authority: rs.Authority(), + Endpoint: endpoint, + }, cc, resolver.BuildOptions{}) + require.NoError(t, err) + + for i := 0; i < 3; i++ { + dc := fmt.Sprintf("dc%d", i+1) + for j := 0; j < 3; j++ { + wanIP := fmt.Sprintf("127.1.%d.%d", i+1, j+10) + name := fmt.Sprintf("%s-server-%d", dc, j+1) + wanMeta := newServerMeta(name, dc, wanIP, true) + + if tc.agentType == "server" { + rs.AddServer(types.AreaWAN, wanMeta) + } + + if dc == agentDC { + // register LAN/WAN pairs for the same instances + lanIP := fmt.Sprintf("127.0.%d.%d", i+1, j+10) + lanMeta := newServerMeta(name, dc, lanIP, false) + rs.AddServer(types.AreaLAN, lanMeta) + + if j == 0 { + rs.UpdateLeaderAddr(dc, lanIP) + } + } + } + } + + if tc.serverType == "leader" { + assert.Len(t, cc.state.Addresses, 1) + } else { + assert.Len(t, cc.state.Addresses, 3) + } + + for _, addr := range cc.state.Addresses { + addrPrefix := tc.requestDC + "-" + if tc.expectLAN { + addrPrefix += "127.0." + } else { + addrPrefix += "127.1." + } + assert.True(t, strings.HasPrefix(addr.Addr, addrPrefix), + "%q does not start with %q (returned WAN for LAN request)", addr.Addr, addrPrefix) + + if tc.expectLAN { + assert.False(t, strings.Contains(addr.ServerName, ".dc"), + "%q ends with datacenter suffix (returned WAN for LAN request)", addr.ServerName) + } else { + assert.True(t, strings.HasSuffix(addr.ServerName, "."+tc.requestDC), + "%q does not end with %q", addr.ServerName, "."+tc.requestDC) + } + } + } + + cases := []testcase{ + { + name: "server requesting local servers", + agentType: "server", + serverType: "server", + requestDC: agentDC, + expectLAN: true, + }, + { + name: "server requesting remote servers in dc2", + agentType: "server", + serverType: "server", + requestDC: "dc2", + expectLAN: false, + }, + { + name: "server requesting remote servers in dc3", + agentType: "server", + serverType: "server", + requestDC: "dc3", + expectLAN: false, + }, + // --------------- + { + name: "server requesting local leader", + agentType: "server", + serverType: "leader", + requestDC: agentDC, + expectLAN: true, + }, + // --------------- + { + name: "client requesting local server", + agentType: "client", + serverType: "server", + requestDC: agentDC, + expectLAN: true, + }, + { + name: "client requesting local leader", + agentType: "client", + serverType: "leader", + requestDC: agentDC, + expectLAN: true, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + run(t, tc) + }) + } +} + +func newServerMeta(name, dc, ip string, wan bool) *metadata.Server { + fullname := name + if wan { + fullname = name + "." + dc + } + return &metadata.Server{ + ID: name, + Name: fullname, + ShortName: name, + Datacenter: dc, + Addr: &net.IPAddr{IP: net.ParseIP(ip)}, + UseTLS: false, + } +} + +func newConfig(t *testing.T, dc, agentType string) Config { + n := t.Name() + s := strings.Replace(n, "/", "", -1) + s = strings.Replace(s, "_", "", -1) + return Config{ + Datacenter: dc, + AgentType: agentType, + Authority: strings.ToLower(s), + } +} + +// fakeClientConn implements resolver.ClientConn for tests +type fakeClientConn struct { + state resolver.State +} + +var _ resolver.ClientConn = (*fakeClientConn)(nil) + +func (f *fakeClientConn) UpdateState(state resolver.State) error { + f.state = state + return nil +} + +func (*fakeClientConn) ReportError(error) {} +func (*fakeClientConn) NewAddress(addresses []resolver.Address) {} +func (*fakeClientConn) NewServiceConfig(serviceConfig string) {} +func (*fakeClientConn) ParseServiceConfig(serviceConfigJSON string) *serviceconfig.ParseResult { + return nil +} diff --git a/agent/hcp/bootstrap/bootstrap.go b/agent/hcp/bootstrap/bootstrap.go index a55bbf49aee..d4235c532fc 100644 --- a/agent/hcp/bootstrap/bootstrap.go +++ b/agent/hcp/bootstrap/bootstrap.go @@ -7,7 +7,10 @@ package bootstrap import ( "bufio" "context" + "crypto/tls" + "crypto/x509" "encoding/json" + "encoding/pem" "errors" "fmt" "os" @@ -16,17 +19,22 @@ import ( "time" "github.com/hashicorp/consul/agent/config" - "github.com/hashicorp/consul/agent/hcp" + "github.com/hashicorp/consul/agent/connect" + hcpclient "github.com/hashicorp/consul/agent/hcp/client" "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/lib/retry" + "github.com/hashicorp/go-uuid" ) const ( - caFileName = "server-tls-cas.pem" - certFileName = "server-tls-cert.pem" - keyFileName = "server-tls-key.pem" - configFileName = "server-config.json" - subDir = "hcp-config" + subDir = "hcp-config" + + caFileName = "server-tls-cas.pem" + certFileName = "server-tls-cert.pem" + configFileName = "server-config.json" + keyFileName = "server-tls-key.pem" + tokenFileName = "hcp-management-token" + successFileName = "successful-bootstrap" ) type ConfigLoader func(source config.Source) (config.LoadResult, error) @@ -43,44 +51,80 @@ type UI interface { Error(string) } -// MaybeBootstrap will use the passed ConfigLoader to read the existing -// configuration, and if required attempt to bootstrap from HCP. It will retry -// until successful or a terminal error condition is found (e.g. permission -// denied). It must be passed a (CLI) UI implementation so it can deliver progress -// updates to the user, for example if it is waiting to retry for a long period. -func MaybeBootstrap(ctx context.Context, loader ConfigLoader, ui UI) (bool, ConfigLoader, error) { - loader = wrapConfigLoader(loader) - res, err := loader(nil) - if err != nil { - return false, nil, err - } - - // Check to see if this is a server and HCP is configured - - if !res.RuntimeConfig.IsCloudEnabled() { - // Not a server, let agent continue unmodified - return false, loader, nil - } +// RawBootstrapConfig contains the Consul config as a raw JSON string and the management token +// which either was retrieved from persisted files or from the bootstrap endpoint +type RawBootstrapConfig struct { + ConfigJSON string + ManagementToken string +} - ui.Output("Bootstrapping configuration from HCP") +// LoadConfig will attempt to load previously-fetched config from disk and fall back to +// fetch from HCP servers if the local data is incomplete. +// It must be passed a (CLI) UI implementation so it can deliver progress +// updates to the user, for example if it is waiting to retry for a long period. +func LoadConfig(ctx context.Context, client hcpclient.Client, dataDir string, loader ConfigLoader, ui UI) (ConfigLoader, error) { + ui.Output("Loading configuration from HCP") // See if we have existing config on disk - cfgJSON, ok := loadPersistedBootstrapConfig(res.RuntimeConfig, ui) - + // + // OPTIMIZE: We could probably be more intelligent about config loading. + // The currently implemented approach is: + // 1. Attempt to load data from disk + // 2. If that fails or the data is incomplete, block indefinitely fetching remote config. + // + // What if instead we had the following flow: + // 1. Attempt to fetch config from HCP. + // 2. If that fails, fall back to data on disk from last fetch. + // 3. If that fails, go into blocking loop to fetch remote config. + // + // This should allow us to more gracefully transition cases like when + // an existing cluster is linked, but then wants to receive TLS materials + // at a later time. Currently, if we observe the existing-cluster marker we + // don't attempt to fetch any additional configuration from HCP. + + cfg, ok := loadPersistedBootstrapConfig(dataDir, ui) if !ok { - // Fetch from HCP - ui.Info("Fetching configuration from HCP") - cfgJSON, err = doHCPBootstrap(ctx, res.RuntimeConfig, ui) + ui.Info("Fetching configuration from HCP servers") + + var err error + cfg, err = fetchBootstrapConfig(ctx, client, dataDir, ui) if err != nil { - return false, nil, fmt.Errorf("failed to bootstrap from HCP: %w", err) + return nil, fmt.Errorf("failed to bootstrap from HCP: %w", err) } ui.Info("Configuration fetched from HCP and saved on local disk") + } else { - ui.Info("Loaded configuration from local disk") + ui.Info("Loaded HCP configuration from local disk") + } // Create a new loader func to return - newLoader := func(source config.Source) (config.LoadResult, error) { + newLoader := bootstrapConfigLoader(loader, cfg) + return newLoader, nil +} + +// bootstrapConfigLoader is a ConfigLoader for passing bootstrap JSON config received from HCP +// to the config.builder. ConfigLoaders are functions used to build an agent's RuntimeConfig +// from various sources like files and flags. This config is contained in the config.LoadResult. +// +// The flow to include bootstrap config from HCP as a loader's data source is as follows: +// +// 1. A base ConfigLoader function (baseLoader) is created on agent start, and it sets the input +// source argument as the DefaultConfig. +// +// 2. When a server agent can be configured by HCP that baseLoader is wrapped in this bootstrapConfigLoader. +// +// 3. The bootstrapConfigLoader calls that base loader with the bootstrap JSON config as the +// default source. This data will be merged with other valid sources in the config.builder. +// +// 4. The result of the call to baseLoader() below contains the resulting RuntimeConfig, and we do some +// additional modifications to attach data that doesn't get populated during the build in the config pkg. +// +// Note that since the ConfigJSON is stored as the baseLoader's DefaultConfig, its data is the first +// to be merged by the config.builder and could be overwritten by user-provided values in config files or +// CLI flags. However, values set to RuntimeConfig after the baseLoader call are final. +func bootstrapConfigLoader(baseLoader ConfigLoader, cfg *RawBootstrapConfig) ConfigLoader { + return func(source config.Source) (config.LoadResult, error) { // Don't allow any further attempts to provide a DefaultSource. This should // only ever be needed later in client agent AutoConfig code but that should // be mutually exclusive from this bootstrapping mechanism since this is @@ -91,47 +135,57 @@ func MaybeBootstrap(ctx context.Context, loader ConfigLoader, ui UI) (bool, Conf return config.LoadResult{}, fmt.Errorf("non-nil config source provided to a loader after HCP bootstrap already provided a DefaultSource") } + // Otherwise, just call to the loader we were passed with our own additional // JSON as the source. - s := config.FileSource{ + // + // OPTIMIZE: We could check/log whether any fields set by the remote config were overwritten by a user-provided flag. + res, err := baseLoader(config.FileSource{ Name: "HCP Bootstrap", Format: "json", - Data: cfgJSON, + Data: cfg.ConfigJSON, + }) + if err != nil { + return res, fmt.Errorf("failed to load HCP Bootstrap config: %w", err) } - return loader(s) - } - return true, newLoader, nil + finalizeRuntimeConfig(res.RuntimeConfig, cfg) + return res, nil + } } -func wrapConfigLoader(loader ConfigLoader) ConfigLoader { - return func(source config.Source) (config.LoadResult, error) { - res, err := loader(source) - if err != nil { - return res, err - } +const ( + accessControlHeaderName = "Access-Control-Expose-Headers" + accessControlHeaderValue = "x-consul-default-acl-policy" +) - if res.RuntimeConfig.Cloud.ResourceID == "" { - res.RuntimeConfig.Cloud.ResourceID = os.Getenv("HCP_RESOURCE_ID") - } - return res, nil +// finalizeRuntimeConfig will set additional HCP-specific values that are not +// handled by the config.builder. +func finalizeRuntimeConfig(rc *config.RuntimeConfig, cfg *RawBootstrapConfig) { + rc.Cloud.ManagementToken = cfg.ManagementToken + + // HTTP response headers are modified for the HCP UI to work. + if rc.HTTPResponseHeaders == nil { + rc.HTTPResponseHeaders = make(map[string]string) + } + prevValue, ok := rc.HTTPResponseHeaders[accessControlHeaderName] + if !ok { + rc.HTTPResponseHeaders[accessControlHeaderName] = accessControlHeaderValue + } else { + rc.HTTPResponseHeaders[accessControlHeaderName] = prevValue + "," + accessControlHeaderValue } } -func doHCPBootstrap(ctx context.Context, rc *config.RuntimeConfig, ui UI) (string, error) { +// fetchBootstrapConfig will fetch boostrap configuration from remote servers and persist it to disk. +// It will retry until successful or a terminal error condition is found (e.g. permission denied). +func fetchBootstrapConfig(ctx context.Context, client hcpclient.Client, dataDir string, ui UI) (*RawBootstrapConfig, error) { w := retry.Waiter{ MinWait: 1 * time.Second, MaxWait: 5 * time.Minute, Jitter: retry.NewJitter(50), } - var bsCfg *hcp.BootstrapConfig - - client, err := hcp.NewClient(rc.Cloud) - if err != nil { - return "", err - } - + var bsCfg *hcpclient.BootstrapConfig for { // Note we don't want to shadow `ctx` here since we need that for the Wait // below. @@ -140,10 +194,10 @@ func doHCPBootstrap(ctx context.Context, rc *config.RuntimeConfig, ui UI) (strin resp, err := client.FetchBootstrap(reqCtx) if err != nil { - ui.Error(fmt.Sprintf("failed to fetch bootstrap config from HCP, will retry in %s: %s", + ui.Error(fmt.Sprintf("Error: failed to fetch bootstrap config from HCP, will retry in %s: %s", w.NextWait().Round(time.Second), err)) if err := w.Wait(ctx); err != nil { - return "", err + return nil, err } // Finished waiting, restart loop continue @@ -152,9 +206,24 @@ func doHCPBootstrap(ctx context.Context, rc *config.RuntimeConfig, ui UI) (strin break } - dataDir := rc.DataDir - shouldPersist := true - if dataDir == "" { + devMode := dataDir == "" + + cfgJSON, err := persistAndProcessConfig(dataDir, devMode, bsCfg) + if err != nil { + return nil, fmt.Errorf("failed to persist config for existing cluster: %w", err) + } + + return &RawBootstrapConfig{ + ConfigJSON: cfgJSON, + ManagementToken: bsCfg.ManagementToken, + }, nil +} + +// persistAndProcessConfig is called when we receive data from CCM. +// We validate and persist everything that was received, then also update +// the JSON config as needed. +func persistAndProcessConfig(dataDir string, devMode bool, bsCfg *hcpclient.BootstrapConfig) (string, error) { + if devMode { // Agent in dev mode, we still need somewhere to persist the certs // temporarily though to be able to start up at all since we don't support // inline certs right now. Use temp dir @@ -163,41 +232,96 @@ func doHCPBootstrap(ctx context.Context, rc *config.RuntimeConfig, ui UI) (strin return "", fmt.Errorf("failed to create temp dir for certificates: %w", err) } dataDir = tmp - shouldPersist = false } - // Persist the TLS cert files from the response since we need to refer to them - // as disk files either way. - if err := persistTLSCerts(dataDir, bsCfg); err != nil { - return "", fmt.Errorf("failed to persist TLS certificates to dir %q: %w", dataDir, err) + // Create subdir if it's not already there. + dir := filepath.Join(dataDir, subDir) + if err := lib.EnsurePath(dir, true); err != nil { + return "", fmt.Errorf("failed to ensure directory %q: %w", dir, err) } - // Update the config JSON to include those TLS cert files - cfgJSON, err := injectTLSCerts(dataDir, bsCfg.ConsulConfig) - if err != nil { - return "", fmt.Errorf("failed to inject TLS Certs into bootstrap config: %w", err) + + // Parse just to a map for now as we only have to inject to a specific place + // and parsing whole Config struct is complicated... + var cfg map[string]any + + if err := json.Unmarshal([]byte(bsCfg.ConsulConfig), &cfg); err != nil { + return "", fmt.Errorf("failed to unmarshal bootstrap config: %w", err) + } + + // Avoid ever setting an initial_management token from HCP now that we can + // separately bootstrap an HCP management token with a distinct accessor ID. + // + // CCM will continue to return an initial_management token because previous versions of Consul + // cannot bootstrap an HCP management token distinct from the initial management token. + // This block can be deleted once CCM supports tailoring bootstrap config responses + // based on the version of Consul that requested it. + acls, aclsOK := cfg["acl"].(map[string]any) + if aclsOK { + tokens, tokensOK := acls["tokens"].(map[string]interface{}) + if tokensOK { + delete(tokens, "initial_management") + } } - // Persist the final config we need to add for restarts. Assuming this wasn't - // a tmp dir to start with. - if shouldPersist { - if err := persistBootstrapConfig(dataDir, cfgJSON); err != nil { - return "", fmt.Errorf("failed to persist bootstrap config to dir %q: %w", dataDir, err) + var cfgJSON string + if bsCfg.TLSCert != "" { + if err := validateTLSCerts(bsCfg.TLSCert, bsCfg.TLSCertKey, bsCfg.TLSCAs); err != nil { + return "", fmt.Errorf("invalid certificates: %w", err) + } + + // Persist the TLS cert files from the response since we need to refer to them + // as disk files either way. + if err := persistTLSCerts(dir, bsCfg.TLSCert, bsCfg.TLSCertKey, bsCfg.TLSCAs); err != nil { + return "", fmt.Errorf("failed to persist TLS certificates to dir %q: %w", dataDir, err) + } + + // Store paths to the persisted TLS cert files. + cfg["ca_file"] = filepath.Join(dir, caFileName) + cfg["cert_file"] = filepath.Join(dir, certFileName) + cfg["key_file"] = filepath.Join(dir, keyFileName) + + // Convert the bootstrap config map back into a string + cfgJSONBytes, err := json.Marshal(cfg) + if err != nil { + return "", err } + cfgJSON = string(cfgJSONBytes) } + if !devMode { + // Persist the final config we need to add so that it is available locally after a restart. + // Assuming the configured data dir wasn't a tmp dir to start with. + if err := persistBootstrapConfig(dir, cfgJSON); err != nil { + return "", fmt.Errorf("failed to persist bootstrap config: %w", err) + } + + // HCP only returns the management token if it requires Consul to + // initialize it + if bsCfg.ManagementToken != "" { + if err := validateManagementToken(bsCfg.ManagementToken); err != nil { + return "", fmt.Errorf("invalid management token: %w", err) + } + if err := persistManagementToken(dir, bsCfg.ManagementToken); err != nil { + return "", fmt.Errorf("failed to persist HCP management token: %w", err) + } + } + + if err := persistSuccessMarker(dir); err != nil { + return "", fmt.Errorf("failed to persist success marker: %w", err) + } + } return cfgJSON, nil } -func persistTLSCerts(dataDir string, bsCfg *hcp.BootstrapConfig) error { - dir := filepath.Join(dataDir, subDir) +func persistSuccessMarker(dir string) error { + name := filepath.Join(dir, successFileName) + return os.WriteFile(name, []byte(""), 0600) - if bsCfg.TLSCert == "" || bsCfg.TLSCertKey == "" { - return fmt.Errorf("unexpected bootstrap response from HCP: missing TLS information") - } +} - // Create a subdir if it's not already there - if err := lib.EnsurePath(dir, true); err != nil { - return err +func persistTLSCerts(dir string, serverCert, serverKey string, caCerts []string) error { + if serverCert == "" || serverKey == "" { + return fmt.Errorf("unexpected bootstrap response from HCP: missing TLS information") } // Write out CA cert(s). We write them all to one file because Go's x509 @@ -208,7 +332,7 @@ func persistTLSCerts(dataDir string, bsCfg *hcp.BootstrapConfig) error { return err } bf := bufio.NewWriter(f) - for _, caPEM := range bsCfg.TLSCAs { + for _, caPEM := range caCerts { bf.WriteString(caPEM + "\n") } if err := bf.Flush(); err != nil { @@ -218,87 +342,240 @@ func persistTLSCerts(dataDir string, bsCfg *hcp.BootstrapConfig) error { return err } - if err := os.WriteFile(filepath.Join(dir, certFileName), []byte(bsCfg.TLSCert), 0600); err != nil { + if err := os.WriteFile(filepath.Join(dir, certFileName), []byte(serverCert), 0600); err != nil { return err } - if err := os.WriteFile(filepath.Join(dir, keyFileName), []byte(bsCfg.TLSCertKey), 0600); err != nil { + if err := os.WriteFile(filepath.Join(dir, keyFileName), []byte(serverKey), 0600); err != nil { return err } return nil } -func injectTLSCerts(dataDir string, bootstrapJSON string) (string, error) { - // Parse just to a map for now as we only have to inject to a specific place - // and parsing whole Config struct is complicated... - var cfg map[string]interface{} +// Basic validation to ensure a UUID was loaded and assumes the token is non-empty +func validateManagementToken(token string) error { + // note: we assume that the token is not an empty string + if _, err := uuid.ParseUUID(token); err != nil { + return errors.New("management token is not a valid UUID") + } + return nil +} + +func persistManagementToken(dir, token string) error { + name := filepath.Join(dir, tokenFileName) + return os.WriteFile(name, []byte(token), 0600) +} - if err := json.Unmarshal([]byte(bootstrapJSON), &cfg); err != nil { - return "", err +func persistBootstrapConfig(dir, cfgJSON string) error { + // Persist the important bits we got from bootstrapping. The TLS certs are + // already persisted, just need to persist the config we are going to add. + name := filepath.Join(dir, configFileName) + return os.WriteFile(name, []byte(cfgJSON), 0600) +} + +func loadPersistedBootstrapConfig(dataDir string, ui UI) (*RawBootstrapConfig, bool) { + if dataDir == "" { + // There's no files to load when in dev mode. + return nil, false + } + + dir := filepath.Join(dataDir, subDir) + + _, err := os.Stat(filepath.Join(dir, successFileName)) + if os.IsNotExist(err) { + // Haven't bootstrapped from HCP. + return nil, false + } + if err != nil { + ui.Warn("failed to check for config on disk, re-fetching from HCP: " + err.Error()) + return nil, false } - // Inject TLS cert files - cfg["ca_file"] = filepath.Join(dataDir, subDir, caFileName) - cfg["cert_file"] = filepath.Join(dataDir, subDir, certFileName) - cfg["key_file"] = filepath.Join(dataDir, subDir, keyFileName) + if err := checkCerts(dir); err != nil { + ui.Warn("failed to validate certs on disk, re-fetching from HCP: " + err.Error()) + return nil, false + } + + configJSON, err := loadBootstrapConfigJSON(dataDir) + if err != nil { + ui.Warn("failed to load bootstrap config from disk, re-fetching from HCP: " + err.Error()) + return nil, false + } - jsonBs, err := json.Marshal(cfg) + mgmtToken, err := loadManagementToken(dir) if err != nil { - return "", err + ui.Warn("failed to load HCP management token from disk, re-fetching from HCP: " + err.Error()) + return nil, false } - return string(jsonBs), nil + return &RawBootstrapConfig{ + ConfigJSON: configJSON, + ManagementToken: mgmtToken, + }, true } -func persistBootstrapConfig(dataDir, cfgJSON string) error { - // Persist the important bits we got from bootstrapping. The TLS certs are - // already persisted, just need to persist the config we are going to add. - name := filepath.Join(dataDir, subDir, configFileName) - return os.WriteFile(name, []byte(cfgJSON), 0600) +func loadBootstrapConfigJSON(dataDir string) (string, error) { + filename := filepath.Join(dataDir, subDir, configFileName) + + _, err := os.Stat(filename) + if os.IsNotExist(err) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("failed to check for bootstrap config: %w", err) + } + + // Attempt to load persisted config to check for errors and basic validity. + // Errors here will raise issues like referencing unsupported config fields. + _, err = config.Load(config.LoadOpts{ + ConfigFiles: []string{filename}, + HCL: []string{ + "server = true", + `bind_addr = "127.0.0.1"`, + fmt.Sprintf("data_dir = %q", dataDir), + }, + ConfigFormat: "json", + }) + if err != nil { + return "", fmt.Errorf("failed to parse local bootstrap config: %w", err) + } + + jsonBs, err := os.ReadFile(filename) + if err != nil { + return "", fmt.Errorf(fmt.Sprintf("failed to read local bootstrap config file: %s", err)) + } + return strings.TrimSpace(string(jsonBs)), nil +} + +func loadManagementToken(dir string) (string, error) { + name := filepath.Join(dir, tokenFileName) + bytes, err := os.ReadFile(name) + if os.IsNotExist(err) { + return "", errors.New("configuration files on disk are incomplete, missing: " + name) + } + if err != nil { + return "", fmt.Errorf("failed to read: %w", err) + } + + token := string(bytes) + if err := validateManagementToken(token); err != nil { + return "", fmt.Errorf("invalid management token: %w", err) + } + + return token, nil } -func loadPersistedBootstrapConfig(rc *config.RuntimeConfig, ui UI) (string, bool) { - // Check if the files all exist +func checkCerts(dir string) error { files := []string{ - filepath.Join(rc.DataDir, subDir, configFileName), - filepath.Join(rc.DataDir, subDir, caFileName), - filepath.Join(rc.DataDir, subDir, certFileName), - filepath.Join(rc.DataDir, subDir, keyFileName), - } - hasSome := false - for _, name := range files { - if _, err := os.Stat(name); errors.Is(err, os.ErrNotExist) { - // At least one required file doesn't exist, failed loading. This is not - // an error though - if hasSome { - ui.Warn("ignoring incomplete local bootstrap config files") - } - return "", false + filepath.Join(dir, caFileName), + filepath.Join(dir, certFileName), + filepath.Join(dir, keyFileName), + } + + missing := make([]string, 0) + for _, file := range files { + _, err := os.Stat(file) + if os.IsNotExist(err) { + missing = append(missing, file) + continue + } + if err != nil { + return err } - hasSome = true } - name := filepath.Join(rc.DataDir, subDir, configFileName) - jsonBs, err := os.ReadFile(name) + // If all the TLS files are missing, assume this is intentional. + // Existing clusters do not receive any TLS certs. + if len(missing) == len(files) { + return nil + } + + // If only some of the files are missing, something went wrong. + if len(missing) > 0 { + return fmt.Errorf("configuration files on disk are incomplete, missing: %v", missing) + } + + cert, key, caCerts, err := loadCerts(dir) + if err != nil { + return fmt.Errorf("failed to load certs from disk: %w", err) + } + + if err = validateTLSCerts(cert, key, caCerts); err != nil { + return fmt.Errorf("invalid certs on disk: %w", err) + } + return nil +} + +func loadCerts(dir string) (cert, key string, caCerts []string, err error) { + certPEMBlock, err := os.ReadFile(filepath.Join(dir, certFileName)) + if err != nil { + return "", "", nil, err + } + keyPEMBlock, err := os.ReadFile(filepath.Join(dir, keyFileName)) + if err != nil { + return "", "", nil, err + } + + caPEMs, err := os.ReadFile(filepath.Join(dir, caFileName)) + if err != nil { + return "", "", nil, err + } + caCerts, err = splitCACerts(caPEMs) if err != nil { - ui.Warn(fmt.Sprintf("failed to read local bootstrap config file, ignoring local files: %s", err)) - return "", false + return "", "", nil, fmt.Errorf("failed to parse CA certs: %w", err) + } + + return string(certPEMBlock), string(keyPEMBlock), caCerts, nil +} + +// splitCACerts takes a list of concatenated PEM blocks and splits +// them back up into strings. This is used because CACerts are written +// into a single file, but validated individually. +func splitCACerts(caPEMs []byte) ([]string, error) { + var out []string + + for { + nextBlock, remaining := pem.Decode(caPEMs) + if nextBlock == nil { + break + } + if nextBlock.Type != "CERTIFICATE" { + return nil, fmt.Errorf("PEM-block should be CERTIFICATE type") + } + + // Collect up to the start of the remaining bytes. + // We don't grab nextBlock.Bytes because it's not PEM encoded. + out = append(out, string(caPEMs[:len(caPEMs)-len(remaining)])) + caPEMs = remaining } - // Check this looks non-empty at least - jsonStr := strings.TrimSpace(string(jsonBs)) - // 50 is arbitrary but config containing the right secrets would always be - // bigger than this in JSON format so it is a reasonable test that this wasn't - // empty or just an empty JSON object or something. - if len(jsonStr) < 50 { - ui.Warn("ignoring incomplete local bootstrap config files") - return "", false + if len(out) == 0 { + return nil, errors.New("invalid CA certificate") + } + return out, nil +} + +// validateTLSCerts checks that the CA cert, server cert, and key on disk are structurally valid. +// +// OPTIMIZE: This could be improved by returning an error if certs are expired or close to expiration. +// However, that requires issuing new certs on bootstrap requests, since returning an error +// would trigger a re-fetch from HCP. +func validateTLSCerts(cert, key string, caCerts []string) error { + leaf, err := tls.X509KeyPair([]byte(cert), []byte(key)) + if err != nil { + return errors.New("invalid server certificate or key") + } + _, err = x509.ParseCertificate(leaf.Certificate[0]) + if err != nil { + return errors.New("invalid server certificate") } - // TODO we could parse the certificates and check they are still valid here - // and force a reload if not. We could also attempt to parse config and check - // it's all valid just in case the local config was really old and has - // deprecated fields or something? - return jsonStr, true + for _, caCert := range caCerts { + _, err = connect.ParseCert(caCert) + if err != nil { + return errors.New("invalid CA certificate") + } + } + return nil } diff --git a/agent/hcp/bootstrap/bootstrap_test.go b/agent/hcp/bootstrap/bootstrap_test.go new file mode 100644 index 00000000000..74b57e5f50a --- /dev/null +++ b/agent/hcp/bootstrap/bootstrap_test.go @@ -0,0 +1,473 @@ +package bootstrap + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/hashicorp/consul/agent/config" + "github.com/hashicorp/consul/agent/hcp" + hcpclient "github.com/hashicorp/consul/agent/hcp/client" + "github.com/hashicorp/consul/lib" + "github.com/hashicorp/consul/tlsutil" + "github.com/hashicorp/go-uuid" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +func TestBootstrapConfigLoader(t *testing.T) { + baseLoader := func(source config.Source) (config.LoadResult, error) { + return config.Load(config.LoadOpts{ + DefaultConfig: source, + HCL: []string{ + `server = true`, + `bind_addr = "127.0.0.1"`, + `data_dir = "/tmp/consul-data"`, + }, + }) + } + + bootstrapLoader := func(source config.Source) (config.LoadResult, error) { + return bootstrapConfigLoader(baseLoader, &RawBootstrapConfig{ + ConfigJSON: `{"bootstrap_expect": 8}`, + ManagementToken: "test-token", + })(source) + } + + result, err := bootstrapLoader(nil) + require.NoError(t, err) + + // bootstrap_expect and management token are injected from bootstrap config received from HCP. + require.Equal(t, 8, result.RuntimeConfig.BootstrapExpect) + require.Equal(t, "test-token", result.RuntimeConfig.Cloud.ManagementToken) + + // Response header is always injected from a constant. + require.Equal(t, "x-consul-default-acl-policy", result.RuntimeConfig.HTTPResponseHeaders[accessControlHeaderName]) +} + +func Test_finalizeRuntimeConfig(t *testing.T) { + type testCase struct { + rc *config.RuntimeConfig + cfg *RawBootstrapConfig + verifyFn func(t *testing.T, rc *config.RuntimeConfig) + } + run := func(t *testing.T, tc testCase) { + finalizeRuntimeConfig(tc.rc, tc.cfg) + tc.verifyFn(t, tc.rc) + } + + tt := map[string]testCase{ + "set header if not present": { + rc: &config.RuntimeConfig{}, + cfg: &RawBootstrapConfig{ + ManagementToken: "test-token", + }, + verifyFn: func(t *testing.T, rc *config.RuntimeConfig) { + require.Equal(t, "test-token", rc.Cloud.ManagementToken) + require.Equal(t, "x-consul-default-acl-policy", rc.HTTPResponseHeaders[accessControlHeaderName]) + }, + }, + "append to header if present": { + rc: &config.RuntimeConfig{ + HTTPResponseHeaders: map[string]string{ + accessControlHeaderName: "Content-Encoding", + }, + }, + cfg: &RawBootstrapConfig{ + ManagementToken: "test-token", + }, + verifyFn: func(t *testing.T, rc *config.RuntimeConfig) { + require.Equal(t, "test-token", rc.Cloud.ManagementToken) + require.Equal(t, "Content-Encoding,x-consul-default-acl-policy", rc.HTTPResponseHeaders[accessControlHeaderName]) + }, + }, + } + + for name, tc := range tt { + t.Run(name, func(t *testing.T) { + run(t, tc) + }) + } +} + +func boolPtr(value bool) *bool { + return &value +} + +func TestLoadConfig_Persistence(t *testing.T) { + type testCase struct { + // resourceID is the HCP resource ID. If set, a server is considered to be cloud-enabled. + resourceID string + + // devMode indicates whether the loader should not have a data directory. + devMode bool + + // verifyFn issues case-specific assertions. + verifyFn func(t *testing.T, rc *config.RuntimeConfig) + } + + run := func(t *testing.T, tc testCase) { + dir, err := os.MkdirTemp(os.TempDir(), "bootstrap-test-") + require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(dir) }) + + s := hcp.NewMockHCPServer() + s.AddEndpoint(TestEndpoint()) + + // Use an HTTPS server since that's what the HCP SDK expects for auth. + srv := httptest.NewTLSServer(s) + defer srv.Close() + + caCert, err := x509.ParseCertificate(srv.TLS.Certificates[0].Certificate[0]) + require.NoError(t, err) + + pool := x509.NewCertPool() + pool.AddCert(caCert) + clientTLS := &tls.Config{RootCAs: pool} + + baseOpts := config.LoadOpts{ + HCL: []string{ + `server = true`, + `bind_addr = "127.0.0.1"`, + fmt.Sprintf(`http_config = { response_headers = { %s = "Content-Encoding" } }`, accessControlHeaderName), + fmt.Sprintf(`cloud { client_id="test" client_secret="test" hostname=%q auth_url=%q resource_id=%q }`, + srv.Listener.Addr().String(), srv.URL, tc.resourceID), + }, + } + if tc.devMode { + baseOpts.DevMode = boolPtr(true) + } else { + baseOpts.HCL = append(baseOpts.HCL, fmt.Sprintf(`data_dir = %q`, dir)) + } + + baseLoader := func(source config.Source) (config.LoadResult, error) { + baseOpts.DefaultConfig = source + return config.Load(baseOpts) + } + + ui := cli.NewMockUi() + + // Load initial config to check whether bootstrapping from HCP is enabled. + initial, err := baseLoader(nil) + require.NoError(t, err) + + // Override the client TLS config so that the test server can be trusted. + initial.RuntimeConfig.Cloud.WithTLSConfig(clientTLS) + client, err := hcpclient.NewClient(initial.RuntimeConfig.Cloud) + require.NoError(t, err) + + loader, err := LoadConfig(context.Background(), client, initial.RuntimeConfig.DataDir, baseLoader, ui) + require.NoError(t, err) + + // Load the agent config with the potentially wrapped loader. + fromRemote, err := loader(nil) + require.NoError(t, err) + + // HCP-enabled cases should fetch from HCP on the first run of LoadConfig. + require.Contains(t, ui.OutputWriter.String(), "Fetching configuration from HCP") + + // Run case-specific verification. + tc.verifyFn(t, fromRemote.RuntimeConfig) + + require.Empty(t, fromRemote.RuntimeConfig.ACLInitialManagementToken, + "initial_management token should have been sanitized") + + if tc.devMode { + // Re-running the bootstrap func below isn't relevant to dev mode + // since they don't have a data directory to load data from. + return + } + + // Run LoadConfig again to exercise the logic of loading config from disk. + loader, err = LoadConfig(context.Background(), client, initial.RuntimeConfig.DataDir, baseLoader, ui) + require.NoError(t, err) + + fromDisk, err := loader(nil) + require.NoError(t, err) + + // HCP-enabled cases should fetch from disk on the second run. + require.Contains(t, ui.OutputWriter.String(), "Loaded HCP configuration from local disk") + + // Config loaded from disk should be the same as the one that was initially fetched from the HCP servers. + require.Equal(t, fromRemote.RuntimeConfig, fromDisk.RuntimeConfig) + } + + tt := map[string]testCase{ + "dev mode": { + devMode: true, + + resourceID: "organization/0b9de9a3-8403-4ca6-aba8-fca752f42100/" + + "project/0b9de9a3-8403-4ca6-aba8-fca752f42100/" + + "consul.cluster/new-cluster-id", + + verifyFn: func(t *testing.T, rc *config.RuntimeConfig) { + require.Empty(t, rc.DataDir) + + // Dev mode should have persisted certs since they can't be inlined. + require.NotEmpty(t, rc.TLS.HTTPS.CertFile) + require.NotEmpty(t, rc.TLS.HTTPS.KeyFile) + require.NotEmpty(t, rc.TLS.HTTPS.CAFile) + + // Find the temporary directory they got stored in. + dir := filepath.Dir(rc.TLS.HTTPS.CertFile) + + // Ensure we only stored the TLS materials. + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, entries, 3) + + haveFiles := make([]string, 3) + for i, entry := range entries { + haveFiles[i] = entry.Name() + } + + wantFiles := []string{caFileName, certFileName, keyFileName} + require.ElementsMatch(t, wantFiles, haveFiles) + }, + }, + "new cluster": { + resourceID: "organization/0b9de9a3-8403-4ca6-aba8-fca752f42100/" + + "project/0b9de9a3-8403-4ca6-aba8-fca752f42100/" + + "consul.cluster/new-cluster-id", + + // New clusters should have received and persisted the whole suite of config. + verifyFn: func(t *testing.T, rc *config.RuntimeConfig) { + dir := filepath.Join(rc.DataDir, subDir) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, entries, 6) + + files := []string{ + filepath.Join(dir, configFileName), + filepath.Join(dir, caFileName), + filepath.Join(dir, certFileName), + filepath.Join(dir, keyFileName), + filepath.Join(dir, tokenFileName), + filepath.Join(dir, successFileName), + } + for _, name := range files { + _, err := os.Stat(name) + require.NoError(t, err) + } + + require.Equal(t, filepath.Join(dir, certFileName), rc.TLS.HTTPS.CertFile) + require.Equal(t, filepath.Join(dir, keyFileName), rc.TLS.HTTPS.KeyFile) + require.Equal(t, filepath.Join(dir, caFileName), rc.TLS.HTTPS.CAFile) + + cert, key, caCerts, err := loadCerts(dir) + require.NoError(t, err) + + require.NoError(t, validateTLSCerts(cert, key, caCerts)) + }, + }, + "existing cluster": { + resourceID: "organization/0b9de9a3-8403-4ca6-aba8-fca752f42100/" + + "project/0b9de9a3-8403-4ca6-aba8-fca752f42100/" + + "consul.cluster/" + TestExistingClusterID, + + // Existing clusters should have only received and persisted the management token. + verifyFn: func(t *testing.T, rc *config.RuntimeConfig) { + dir := filepath.Join(rc.DataDir, subDir) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, entries, 3) + + files := []string{ + filepath.Join(dir, tokenFileName), + filepath.Join(dir, successFileName), + filepath.Join(dir, configFileName), + } + for _, name := range files { + _, err := os.Stat(name) + require.NoError(t, err) + } + }, + }, + } + + for name, tc := range tt { + t.Run(name, func(t *testing.T) { + run(t, tc) + }) + } +} + +func Test_loadPersistedBootstrapConfig(t *testing.T) { + type expect struct { + loaded bool + warning string + } + type testCase struct { + existingCluster bool + disableManagementToken bool + mutateFn func(t *testing.T, dir string) + expect expect + } + + run := func(t *testing.T, tc testCase) { + dataDir, err := os.MkdirTemp(os.TempDir(), "load-bootstrap-test-") + require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(dataDir) }) + + dir := filepath.Join(dataDir, subDir) + + // Do some common setup as if we received config from HCP and persisted it to disk. + require.NoError(t, lib.EnsurePath(dir, true)) + require.NoError(t, persistSuccessMarker(dir)) + + if !tc.existingCluster { + caCert, caKey, err := tlsutil.GenerateCA(tlsutil.CAOpts{}) + require.NoError(t, err) + + serverCert, serverKey, err := testLeaf(caCert, caKey) + require.NoError(t, err) + require.NoError(t, persistTLSCerts(dir, serverCert, serverKey, []string{caCert})) + + cfgJSON := `{"bootstrap_expect": 8}` + require.NoError(t, persistBootstrapConfig(dir, cfgJSON)) + } + + var token string + if !tc.disableManagementToken { + token, err = uuid.GenerateUUID() + require.NoError(t, err) + require.NoError(t, persistManagementToken(dir, token)) + } + + // Optionally mutate the persisted data to trigger errors while loading. + if tc.mutateFn != nil { + tc.mutateFn(t, dir) + } + + ui := cli.NewMockUi() + cfg, loaded := loadPersistedBootstrapConfig(dataDir, ui) + require.Equal(t, tc.expect.loaded, loaded, ui.ErrorWriter.String()) + if loaded { + require.Equal(t, token, cfg.ManagementToken) + require.Empty(t, ui.ErrorWriter.String()) + } else { + require.Nil(t, cfg) + require.Contains(t, ui.ErrorWriter.String(), tc.expect.warning) + } + } + + tt := map[string]testCase{ + "existing cluster with valid files": { + existingCluster: true, + // Don't mutate, files from setup are valid. + mutateFn: nil, + expect: expect{ + loaded: true, + warning: "", + }, + }, + "existing cluster no token": { + existingCluster: true, + disableManagementToken: true, + expect: expect{ + loaded: false, + }, + }, + "existing cluster no files": { + existingCluster: true, + mutateFn: func(t *testing.T, dir string) { + // Remove all files + require.NoError(t, os.RemoveAll(dir)) + }, + expect: expect{ + loaded: false, + // No warnings since we assume we need to fetch config from HCP for the first time. + warning: "", + }, + }, + "new cluster with valid files": { + // Don't mutate, files from setup are valid. + mutateFn: nil, + expect: expect{ + loaded: true, + warning: "", + }, + }, + "new cluster with no token": { + disableManagementToken: true, + expect: expect{ + loaded: false, + }, + }, + "new cluster some files": { + mutateFn: func(t *testing.T, dir string) { + // Remove one of the required files + require.NoError(t, os.Remove(filepath.Join(dir, certFileName))) + }, + expect: expect{ + loaded: false, + warning: "configuration files on disk are incomplete", + }, + }, + "new cluster no files": { + mutateFn: func(t *testing.T, dir string) { + // Remove all files + require.NoError(t, os.RemoveAll(dir)) + }, + expect: expect{ + loaded: false, + // No warnings since we assume we need to fetch config from HCP for the first time. + warning: "", + }, + }, + "new cluster invalid cert": { + mutateFn: func(t *testing.T, dir string) { + name := filepath.Join(dir, certFileName) + require.NoError(t, os.WriteFile(name, []byte("not-a-cert"), 0600)) + }, + expect: expect{ + loaded: false, + warning: "invalid server certificate", + }, + }, + "new cluster invalid CA": { + mutateFn: func(t *testing.T, dir string) { + name := filepath.Join(dir, caFileName) + require.NoError(t, os.WriteFile(name, []byte("not-a-ca-cert"), 0600)) + }, + expect: expect{ + loaded: false, + warning: "invalid CA certificate", + }, + }, + "new cluster invalid config flag": { + mutateFn: func(t *testing.T, dir string) { + name := filepath.Join(dir, configFileName) + require.NoError(t, os.WriteFile(name, []byte(`{"not_a_consul_agent_config_field" = "zap"}`), 0600)) + }, + expect: expect{ + loaded: false, + warning: "failed to parse local bootstrap config", + }, + }, + "existing cluster invalid token": { + existingCluster: true, + mutateFn: func(t *testing.T, dir string) { + name := filepath.Join(dir, tokenFileName) + require.NoError(t, os.WriteFile(name, []byte("not-a-uuid"), 0600)) + }, + expect: expect{ + loaded: false, + warning: "is not a valid UUID", + }, + }, + } + + for name, tc := range tt { + t.Run(name, func(t *testing.T) { + run(t, tc) + }) + } +} diff --git a/agent/hcp/bootstrap/testing.go b/agent/hcp/bootstrap/testing.go index b1a05a7e50b..d06c2d747c4 100644 --- a/agent/hcp/bootstrap/testing.go +++ b/agent/hcp/bootstrap/testing.go @@ -10,12 +10,11 @@ import ( "net/http" "strings" - gnmmod "github.com/hashicorp/hcp-sdk-go/clients/cloud-global-network-manager-service/preview/2022-02-15/models" - "github.com/hashicorp/hcp-sdk-go/resource" - "github.com/hashicorp/consul/agent/hcp" "github.com/hashicorp/consul/tlsutil" "github.com/hashicorp/go-uuid" + gnmmod "github.com/hashicorp/hcp-sdk-go/clients/cloud-global-network-manager-service/preview/2022-02-15/models" + "github.com/hashicorp/hcp-sdk-go/resource" ) // TestEndpoint returns an hcp.TestEndpoint to be used in an hcp.MockHCPServer. @@ -46,6 +45,29 @@ func handleBootstrap(data map[string]gnmmod.HashicorpCloudGlobalNetworkManager20 return resp, nil } +const TestExistingClusterID = "133114e7-9745-41ce-b1c9-9644a20d2952" + +func testLeaf(caCert, caKey string) (serverCert, serverKey string, err error) { + signer, err := tlsutil.ParseSigner(caKey) + if err != nil { + return "", "", err + } + + serverCert, serverKey, err = tlsutil.GenerateCert(tlsutil.CertOpts{ + Signer: signer, + CA: caCert, + Name: "server.dc1.consul", + Days: 30, + DNSNames: []string{"server.dc1.consul", "localhost"}, + IPAddresses: append([]net.IP{}, net.ParseIP("127.0.0.1")), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + }) + if err != nil { + return "", "", err + } + return serverCert, serverKey, nil +} + func generateClusterData(cluster resource.Resource) (gnmmod.HashicorpCloudGlobalNetworkManager20220215AgentBootstrapResponse, error) { resp := gnmmod.HashicorpCloudGlobalNetworkManager20220215AgentBootstrapResponse{ Cluster: &gnmmod.HashicorpCloudGlobalNetworkManager20220215Cluster{}, @@ -53,32 +75,28 @@ func generateClusterData(cluster resource.Resource) (gnmmod.HashicorpCloudGlobal ServerTLS: &gnmmod.HashicorpCloudGlobalNetworkManager20220215ServerTLS{}, }, } - - CACert, CAKey, err := tlsutil.GenerateCA(tlsutil.CAOpts{}) - if err != nil { - return resp, err + if cluster.ID == TestExistingClusterID { + token, err := uuid.GenerateUUID() + if err != nil { + return resp, err + } + resp.Bootstrap.ConsulConfig = "{}" + resp.Bootstrap.ManagementToken = token + return resp, nil } - resp.Bootstrap.ServerTLS.CertificateAuthorities = append(resp.Bootstrap.ServerTLS.CertificateAuthorities, CACert) - signer, err := tlsutil.ParseSigner(CAKey) + caCert, caKey, err := tlsutil.GenerateCA(tlsutil.CAOpts{}) if err != nil { return resp, err } - - cert, priv, err := tlsutil.GenerateCert(tlsutil.CertOpts{ - Signer: signer, - CA: CACert, - Name: "server.dc1.consul", - Days: 30, - DNSNames: []string{"server.dc1.consul", "localhost"}, - IPAddresses: append([]net.IP{}, net.ParseIP("127.0.0.1")), - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - }) + serverCert, serverKey, err := testLeaf(caCert, caKey) if err != nil { return resp, err } - resp.Bootstrap.ServerTLS.Cert = cert - resp.Bootstrap.ServerTLS.PrivateKey = priv + + resp.Bootstrap.ServerTLS.CertificateAuthorities = append(resp.Bootstrap.ServerTLS.CertificateAuthorities, caCert) + resp.Bootstrap.ServerTLS.Cert = serverCert + resp.Bootstrap.ServerTLS.PrivateKey = serverKey // Generate Config. We don't use the read config.Config struct because it // doesn't have `omitempty` which makes the output gross. We only want a tiny @@ -113,8 +131,9 @@ func generateClusterData(cluster resource.Resource) (gnmmod.HashicorpCloudGlobal // Enable HTTPS port, disable HTTP "ports": map[string]interface{}{ - "https": 8501, - "http": -1, + "https": 8501, + "http": -1, + "grpc_tls": 8503, }, // RAFT Peers @@ -125,16 +144,18 @@ func generateClusterData(cluster resource.Resource) (gnmmod.HashicorpCloudGlobal } // ACLs - management, err := uuid.GenerateUUID() + token, err := uuid.GenerateUUID() if err != nil { return resp, err } + resp.Bootstrap.ManagementToken = token + cfg["acl"] = map[string]interface{}{ "tokens": map[string]interface{}{ - "initial_management": management, - // Also setup the server's own agent token to be the same so it has + // Also setup the server's own agent token to be the management token so it has // permission to register itself. - "agent": management, + "agent": token, + "initial_management": token, }, "default_policy": "deny", "enabled": true, diff --git a/agent/hcp/client.go b/agent/hcp/client/client.go similarity index 78% rename from agent/hcp/client.go rename to agent/hcp/client/client.go index 5b8f942bd95..f04767e983c 100644 --- a/agent/hcp/client.go +++ b/agent/hcp/client/client.go @@ -1,4 +1,7 @@ -package hcp +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package client import ( "context" @@ -8,6 +11,8 @@ import ( httptransport "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" + + hcptelemetry "github.com/hashicorp/hcp-sdk-go/clients/cloud-consul-telemetry-gateway/preview/2023-04-14/client/consul_telemetry_service" hcpgnm "github.com/hashicorp/hcp-sdk-go/clients/cloud-global-network-manager-service/preview/2022-02-15/client/global_network_manager_service" gnmmod "github.com/hashicorp/hcp-sdk-go/clients/cloud-global-network-manager-service/preview/2022-02-15/models" "github.com/hashicorp/hcp-sdk-go/httpclient" @@ -17,11 +22,15 @@ import ( "github.com/hashicorp/consul/version" ) +// metricsGatewayPath is the default path for metrics export request on the Telemetry Gateway. +const metricsGatewayPath = "/v1/metrics" + // Client interface exposes HCP operations that can be invoked by Consul // //go:generate mockery --name Client --with-expecter --inpackage type Client interface { FetchBootstrap(ctx context.Context) (*BootstrapConfig, error) + FetchTelemetryConfig(ctx context.Context) (*TelemetryConfig, error) PushServerStatus(ctx context.Context, status *ServerStatus) error DiscoverServers(ctx context.Context) ([]string, error) } @@ -34,12 +43,14 @@ type BootstrapConfig struct { TLSCertKey string TLSCAs []string ConsulConfig string + ManagementToken string } type hcpClient struct { hc *httptransport.Runtime cfg config.CloudConfig gnm hcpgnm.ClientService + tgw hcptelemetry.ClientService resource resource.Resource } @@ -60,6 +71,8 @@ func NewClient(cfg config.CloudConfig) (Client, error) { } client.gnm = hcpgnm.New(client.hc, nil) + client.tgw = hcptelemetry.New(client.hc, nil) + return client, nil } @@ -75,11 +88,32 @@ func httpClient(c config.CloudConfig) (*httptransport.Runtime, error) { }) } +// FetchTelemetryConfig obtains telemetry configuration from the Telemetry Gateway. +func (c *hcpClient) FetchTelemetryConfig(ctx context.Context) (*TelemetryConfig, error) { + params := hcptelemetry.NewAgentTelemetryConfigParamsWithContext(ctx). + WithLocationOrganizationID(c.resource.Organization). + WithLocationProjectID(c.resource.Project). + WithClusterID(c.resource.ID) + + resp, err := c.tgw.AgentTelemetryConfig(params, nil) + if err != nil { + return nil, fmt.Errorf("failed to fetch from HCP: %w", err) + } + + if err := validateAgentTelemetryConfigPayload(resp); err != nil { + return nil, fmt.Errorf("invalid response payload: %w", err) + } + + return convertAgentTelemetryResponse(ctx, resp, c.cfg) +} + func (c *hcpClient) FetchBootstrap(ctx context.Context) (*BootstrapConfig, error) { + version := version.GetHumanVersion() params := hcpgnm.NewAgentBootstrapConfigParamsWithContext(ctx). WithID(c.resource.ID). WithLocationOrganizationID(c.resource.Organization). - WithLocationProjectID(c.resource.Project) + WithLocationProjectID(c.resource.Project). + WithConsulVersion(&version) resp, err := c.gnm.AgentBootstrapConfig(params, nil) if err != nil { @@ -103,6 +137,7 @@ func bootstrapConfigFromHCP(res *gnmmod.HashicorpCloudGlobalNetworkManager202202 TLSCertKey: serverTLS.PrivateKey, TLSCAs: serverTLS.CertificateAuthorities, ConsulConfig: res.Bootstrap.ConsulConfig, + ManagementToken: res.Bootstrap.ManagementToken, } } @@ -112,7 +147,7 @@ func (c *hcpClient) PushServerStatus(ctx context.Context, s *ServerStatus) error WithLocationOrganizationID(c.resource.Organization). WithLocationProjectID(c.resource.Project) - params.SetBody(&gnmmod.HashicorpCloudGlobalNetworkManager20220215AgentPushServerStateRequest{ + params.SetBody(hcpgnm.AgentPushServerStateBody{ ServerState: serverStatusToHCP(s), }) @@ -127,10 +162,12 @@ type ServerStatus struct { LanAddress string GossipPort int RPCPort int + Datacenter string Autopilot ServerAutopilot Raft ServerRaft TLS ServerTLSInfo + ACL ServerACLInfo ScadaStatus string } @@ -150,6 +187,10 @@ type ServerRaft struct { TimeSinceLastContact time.Duration } +type ServerACLInfo struct { + Enabled bool +} + type ServerTLSInfo struct { Enabled bool CertExpiry time.Time @@ -194,6 +235,10 @@ func serverStatusToHCP(s *ServerStatus) *gnmmod.HashicorpCloudGlobalNetworkManag }, Version: s.Version, ScadaStatus: s.ScadaStatus, + ACL: &gnmmod.HashicorpCloudGlobalNetworkManager20220215ACLInfo{ + Enabled: s.ACL.Enabled, + }, + Datacenter: s.Datacenter, } } diff --git a/agent/hcp/client/client_test.go b/agent/hcp/client/client_test.go new file mode 100644 index 00000000000..d4bae2ae4cb --- /dev/null +++ b/agent/hcp/client/client_test.go @@ -0,0 +1,123 @@ +package client + +import ( + "context" + "fmt" + "net/url" + "regexp" + "testing" + "time" + + "github.com/go-openapi/runtime" + hcptelemetry "github.com/hashicorp/hcp-sdk-go/clients/cloud-consul-telemetry-gateway/preview/2023-04-14/client/consul_telemetry_service" + "github.com/hashicorp/hcp-sdk-go/clients/cloud-consul-telemetry-gateway/preview/2023-04-14/models" + "github.com/stretchr/testify/require" +) + +type mockTGW struct { + mockResponse *hcptelemetry.AgentTelemetryConfigOK + mockError error +} + +func (m *mockTGW) AgentTelemetryConfig(params *hcptelemetry.AgentTelemetryConfigParams, authInfo runtime.ClientAuthInfoWriter, opts ...hcptelemetry.ClientOption) (*hcptelemetry.AgentTelemetryConfigOK, error) { + return m.mockResponse, m.mockError +} +func (m *mockTGW) GetLabelValues(params *hcptelemetry.GetLabelValuesParams, authInfo runtime.ClientAuthInfoWriter, opts ...hcptelemetry.ClientOption) (*hcptelemetry.GetLabelValuesOK, error) { + return hcptelemetry.NewGetLabelValuesOK(), nil +} +func (m *mockTGW) QueryRangeBatch(params *hcptelemetry.QueryRangeBatchParams, authInfo runtime.ClientAuthInfoWriter, opts ...hcptelemetry.ClientOption) (*hcptelemetry.QueryRangeBatchOK, error) { + return hcptelemetry.NewQueryRangeBatchOK(), nil +} +func (m *mockTGW) SetTransport(transport runtime.ClientTransport) {} + +type expectedTelemetryCfg struct { + endpoint string + labels map[string]string + filters string + refreshInterval time.Duration +} + +func TestFetchTelemetryConfig(t *testing.T) { + t.Parallel() + for name, tc := range map[string]struct { + mockResponse *hcptelemetry.AgentTelemetryConfigOK + mockError error + wantErr string + expected *expectedTelemetryCfg + }{ + "errorsWithFetchFailure": { + mockError: fmt.Errorf("failed to fetch from HCP"), + mockResponse: nil, + wantErr: "failed to fetch from HCP", + }, + "errorsWithInvalidPayload": { + mockResponse: &hcptelemetry.AgentTelemetryConfigOK{ + Payload: &models.HashicorpCloudConsulTelemetry20230414AgentTelemetryConfigResponse{}, + }, + mockError: nil, + wantErr: "invalid response payload", + }, + "success:": { + mockResponse: &hcptelemetry.AgentTelemetryConfigOK{ + Payload: &models.HashicorpCloudConsulTelemetry20230414AgentTelemetryConfigResponse{ + RefreshConfig: &models.HashicorpCloudConsulTelemetry20230414RefreshConfig{ + RefreshInterval: "1s", + }, + TelemetryConfig: &models.HashicorpCloudConsulTelemetry20230414TelemetryConfig{ + Endpoint: "https://test.com", + Labels: map[string]string{"test": "123"}, + Metrics: &models.HashicorpCloudConsulTelemetry20230414TelemetryMetricsConfig{ + IncludeList: []string{"consul", "test"}, + }, + }, + }, + }, + expected: &expectedTelemetryCfg{ + endpoint: "https://test.com/v1/metrics", + labels: map[string]string{"test": "123"}, + filters: "consul|test", + refreshInterval: 1 * time.Second, + }, + }, + } { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + c := &hcpClient{ + tgw: &mockTGW{ + mockError: tc.mockError, + mockResponse: tc.mockResponse, + }, + } + + telemetryCfg, err := c.FetchTelemetryConfig(context.Background()) + + if tc.wantErr != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErr) + require.Nil(t, telemetryCfg) + return + } + + urlEndpoint, err := url.Parse(tc.expected.endpoint) + require.NoError(t, err) + + regexFilters, err := regexp.Compile(tc.expected.filters) + require.NoError(t, err) + + expectedCfg := &TelemetryConfig{ + MetricsConfig: &MetricsConfig{ + Endpoint: urlEndpoint, + Filters: regexFilters, + Labels: tc.expected.labels, + }, + RefreshConfig: &RefreshConfig{ + RefreshInterval: tc.expected.refreshInterval, + }, + } + + require.NoError(t, err) + require.Equal(t, expectedCfg, telemetryCfg) + }) + } +} diff --git a/agent/hcp/client/metrics_client.go b/agent/hcp/client/metrics_client.go new file mode 100644 index 00000000000..3c5b5c4fb9d --- /dev/null +++ b/agent/hcp/client/metrics_client.go @@ -0,0 +1,167 @@ +package client + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "time" + + "github.com/hashicorp/go-cleanhttp" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-retryablehttp" + hcpcfg "github.com/hashicorp/hcp-sdk-go/config" + "github.com/hashicorp/hcp-sdk-go/resource" + colmetricpb "go.opentelemetry.io/proto/otlp/collector/metrics/v1" + metricpb "go.opentelemetry.io/proto/otlp/metrics/v1" + "golang.org/x/oauth2" + "google.golang.org/protobuf/proto" + + "github.com/hashicorp/consul/agent/hcp/telemetry" + "github.com/hashicorp/consul/version" +) + +const ( + // HTTP Client config + defaultStreamTimeout = 15 * time.Second + + // Retry config + // TODO: Eventually, we'd like to configure these values dynamically. + defaultRetryWaitMin = 1 * time.Second + defaultRetryWaitMax = 15 * time.Second + // defaultRetryMax is set to 0 to turn off retry functionality, until dynamic configuration is possible. + // This is to circumvent any spikes in load that may cause or exacerbate server-side issues for now. + defaultRetryMax = 0 + + // defaultErrRespBodyLength refers to the max character length of the body on a failure to export metrics. + // anything beyond we will truncate. + defaultErrRespBodyLength = 100 +) + +// cloudConfig represents cloud config for TLS abstracted in an interface for easy testing. +type CloudConfig interface { + HCPConfig(opts ...hcpcfg.HCPConfigOption) (hcpcfg.HCPConfig, error) + Resource() (resource.Resource, error) +} + +// otlpClient is an implementation of MetricsClient with a retryable http client for retries and to honor throttle. +// It also holds default HTTP headers to add to export requests. +type otlpClient struct { + client *retryablehttp.Client + header *http.Header +} + +// NewMetricsClient returns a configured MetricsClient. +// The current implementation uses otlpClient to provide retry functionality. +func NewMetricsClient(ctx context.Context, cfg CloudConfig) (telemetry.MetricsClient, error) { + if cfg == nil { + return nil, fmt.Errorf("failed to init telemetry client: provide valid cloudCfg (Cloud Configuration for TLS)") + } + + if ctx == nil { + return nil, fmt.Errorf("failed to init telemetry client: provide a valid context") + } + + logger := hclog.FromContext(ctx) + + c, err := newHTTPClient(cfg, logger) + if err != nil { + return nil, fmt.Errorf("failed to init telemetry client: %v", err) + } + + r, err := cfg.Resource() + if err != nil { + return nil, fmt.Errorf("failed to init telemetry client: %v", err) + } + + header := make(http.Header) + header.Set("content-type", "application/x-protobuf") + header.Set("x-hcp-resource-id", r.String()) + header.Set("x-channel", fmt.Sprintf("consul/%s", version.GetHumanVersion())) + + return &otlpClient{ + client: c, + header: &header, + }, nil +} + +// newHTTPClient configures the retryable HTTP client. +func newHTTPClient(cloudCfg CloudConfig, logger hclog.Logger) (*retryablehttp.Client, error) { + hcpCfg, err := cloudCfg.HCPConfig() + if err != nil { + return nil, err + } + + tlsTransport := cleanhttp.DefaultPooledTransport() + tlsTransport.TLSClientConfig = hcpCfg.APITLSConfig() + + var transport http.RoundTripper = &oauth2.Transport{ + Base: tlsTransport, + Source: hcpCfg, + } + + client := &http.Client{ + Transport: transport, + Timeout: defaultStreamTimeout, + } + + retryClient := &retryablehttp.Client{ + HTTPClient: client, + Logger: logger.Named("hcp_telemetry_client"), + RetryWaitMin: defaultRetryWaitMin, + RetryWaitMax: defaultRetryWaitMax, + RetryMax: defaultRetryMax, + CheckRetry: retryablehttp.DefaultRetryPolicy, + Backoff: retryablehttp.DefaultBackoff, + } + + return retryClient, nil +} + +// ExportMetrics is the single method exposed by MetricsClient to export OTLP metrics to the desired HCP endpoint. +// The endpoint is configurable as the endpoint can change during periodic refresh of CCM telemetry config. +// By configuring the endpoint here, we can re-use the same client and override the endpoint when making a request. +func (o *otlpClient) ExportMetrics(ctx context.Context, protoMetrics *metricpb.ResourceMetrics, endpoint string) error { + pbRequest := &colmetricpb.ExportMetricsServiceRequest{ + ResourceMetrics: []*metricpb.ResourceMetrics{protoMetrics}, + } + + body, err := proto.Marshal(pbRequest) + if err != nil { + return fmt.Errorf("failed to marshal the request: %w", err) + } + + req, err := retryablehttp.NewRequest(http.MethodPost, endpoint, bytes.NewBuffer(body)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + req.Header = *o.header + + resp, err := o.client.Do(req.WithContext(ctx)) + if err != nil { + return fmt.Errorf("failed to post metrics: %w", err) + } + defer resp.Body.Close() + + var respData bytes.Buffer + if _, err := io.Copy(&respData, resp.Body); err != nil { + return fmt.Errorf("failed to read body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + truncatedBody := truncate(respData.String(), defaultErrRespBodyLength) + return fmt.Errorf("failed to export metrics: code %d: %s", resp.StatusCode, truncatedBody) + } + + return nil +} + +func truncate(text string, width uint) string { + if len(text) <= int(width) { + return text + } + r := []rune(text) + trunc := r[:width] + return string(trunc) + "..." +} diff --git a/agent/hcp/client/metrics_client_test.go b/agent/hcp/client/metrics_client_test.go new file mode 100644 index 00000000000..4119e326e9d --- /dev/null +++ b/agent/hcp/client/metrics_client_test.go @@ -0,0 +1,175 @@ +package client + +import ( + "context" + "fmt" + "math/rand" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" + colpb "go.opentelemetry.io/proto/otlp/collector/metrics/v1" + metricpb "go.opentelemetry.io/proto/otlp/metrics/v1" + "google.golang.org/protobuf/proto" + + "github.com/hashicorp/consul/version" +) + +func TestNewMetricsClient(t *testing.T) { + for name, test := range map[string]struct { + wantErr string + cfg CloudConfig + ctx context.Context + }{ + "success": { + cfg: &MockCloudCfg{}, + ctx: context.Background(), + }, + "failsWithoutCloudCfg": { + wantErr: "failed to init telemetry client: provide valid cloudCfg (Cloud Configuration for TLS)", + cfg: nil, + ctx: context.Background(), + }, + "failsWithoutContext": { + wantErr: "failed to init telemetry client: provide a valid context", + cfg: MockCloudCfg{}, + ctx: nil, + }, + "failsHCPConfig": { + wantErr: "failed to init telemetry client", + cfg: MockCloudCfg{ + ConfigErr: fmt.Errorf("test bad hcp config"), + }, + ctx: context.Background(), + }, + "failsBadResource": { + wantErr: "failed to init telemetry client", + cfg: MockCloudCfg{ + ResourceErr: fmt.Errorf("test bad resource"), + }, + ctx: context.Background(), + }, + } { + t.Run(name, func(t *testing.T) { + client, err := NewMetricsClient(test.ctx, test.cfg) + if test.wantErr != "" { + require.Error(t, err) + require.Contains(t, err.Error(), test.wantErr) + return + } + + require.Nil(t, err) + require.NotNil(t, client) + }) + } +} + +var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZäöüÄÖÜ世界") + +func randStringRunes(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} + +func TestExportMetrics(t *testing.T) { + for name, test := range map[string]struct { + wantErr string + status int + largeBodyError bool + }{ + "success": { + status: http.StatusOK, + }, + "failsWithNonRetryableError": { + status: http.StatusBadRequest, + wantErr: "failed to export metrics: code 400", + }, + "failsWithNonRetryableErrorWithLongError": { + status: http.StatusBadRequest, + wantErr: "failed to export metrics: code 400", + largeBodyError: true, + }, + } { + t.Run(name, func(t *testing.T) { + randomBody := randStringRunes(1000) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, r.Header.Get("content-type"), "application/x-protobuf") + require.Equal(t, r.Header.Get("x-hcp-resource-id"), testResourceID) + require.Equal(t, r.Header.Get("x-channel"), fmt.Sprintf("consul/%s", version.GetHumanVersion())) + require.Equal(t, r.Header.Get("Authorization"), "Bearer test-token") + + body := colpb.ExportMetricsServiceResponse{} + bytes, err := proto.Marshal(&body) + + require.NoError(t, err) + + w.Header().Set("Content-Type", "application/x-protobuf") + w.WriteHeader(test.status) + if test.largeBodyError { + w.Write([]byte(randomBody)) + } else { + w.Write(bytes) + } + + })) + defer srv.Close() + + client, err := NewMetricsClient(context.Background(), MockCloudCfg{}) + require.NoError(t, err) + + ctx := context.Background() + metrics := &metricpb.ResourceMetrics{} + err = client.ExportMetrics(ctx, metrics, srv.URL) + + if test.wantErr != "" { + require.Error(t, err) + require.Contains(t, err.Error(), test.wantErr) + if test.largeBodyError { + truncatedBody := truncate(randomBody, defaultErrRespBodyLength) + require.Contains(t, err.Error(), truncatedBody) + } + return + } + + require.NoError(t, err) + }) + } +} + +func TestTruncate(t *testing.T) { + for name, tc := range map[string]struct { + body string + expectedSize int + }{ + "ZeroSize": { + body: "", + expectedSize: 0, + }, + "LessThanSize": { + body: "foobar", + expectedSize: 6, + }, + "defaultSize": { + body: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis vel tincidunt nunc, sed tristique risu", + expectedSize: 100, + }, + "greaterThanSize": { + body: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis vel tincidunt nunc, sed tristique risus", + expectedSize: 103, + }, + "greaterThanSizeWithUnicode": { + body: randStringRunes(1000), + expectedSize: 103, + }, + } { + t.Run(name, func(t *testing.T) { + truncatedBody := truncate(tc.body, defaultErrRespBodyLength) + truncatedRunes := []rune(truncatedBody) + require.Equal(t, len(truncatedRunes), tc.expectedSize) + }) + } +} diff --git a/agent/hcp/mock_Client.go b/agent/hcp/client/mock_Client.go similarity index 65% rename from agent/hcp/mock_Client.go rename to agent/hcp/client/mock_Client.go index 29bd27cbf1b..06853ceb86f 100644 --- a/agent/hcp/mock_Client.go +++ b/agent/hcp/client/mock_Client.go @@ -1,6 +1,6 @@ -// Code generated by mockery v2.15.0. DO NOT EDIT. +// Code generated by mockery v2.22.1. DO NOT EDIT. -package hcp +package client import ( context "context" @@ -26,6 +26,10 @@ func (_m *MockClient) DiscoverServers(ctx context.Context) ([]string, error) { ret := _m.Called(ctx) var r0 []string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]string, error)); ok { + return rf(ctx) + } if rf, ok := ret.Get(0).(func(context.Context) []string); ok { r0 = rf(ctx) } else { @@ -34,7 +38,6 @@ func (_m *MockClient) DiscoverServers(ctx context.Context) ([]string, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(ctx) } else { @@ -67,11 +70,20 @@ func (_c *MockClient_DiscoverServers_Call) Return(_a0 []string, _a1 error) *Mock return _c } +func (_c *MockClient_DiscoverServers_Call) RunAndReturn(run func(context.Context) ([]string, error)) *MockClient_DiscoverServers_Call { + _c.Call.Return(run) + return _c +} + // FetchBootstrap provides a mock function with given fields: ctx func (_m *MockClient) FetchBootstrap(ctx context.Context) (*BootstrapConfig, error) { ret := _m.Called(ctx) var r0 *BootstrapConfig + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*BootstrapConfig, error)); ok { + return rf(ctx) + } if rf, ok := ret.Get(0).(func(context.Context) *BootstrapConfig); ok { r0 = rf(ctx) } else { @@ -80,7 +92,6 @@ func (_m *MockClient) FetchBootstrap(ctx context.Context) (*BootstrapConfig, err } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(ctx) } else { @@ -113,6 +124,65 @@ func (_c *MockClient_FetchBootstrap_Call) Return(_a0 *BootstrapConfig, _a1 error return _c } +func (_c *MockClient_FetchBootstrap_Call) RunAndReturn(run func(context.Context) (*BootstrapConfig, error)) *MockClient_FetchBootstrap_Call { + _c.Call.Return(run) + return _c +} + +// FetchTelemetryConfig provides a mock function with given fields: ctx +func (_m *MockClient) FetchTelemetryConfig(ctx context.Context) (*TelemetryConfig, error) { + ret := _m.Called(ctx) + + var r0 *TelemetryConfig + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*TelemetryConfig, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *TelemetryConfig); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*TelemetryConfig) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockClient_FetchTelemetryConfig_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FetchTelemetryConfig' +type MockClient_FetchTelemetryConfig_Call struct { + *mock.Call +} + +// FetchTelemetryConfig is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockClient_Expecter) FetchTelemetryConfig(ctx interface{}) *MockClient_FetchTelemetryConfig_Call { + return &MockClient_FetchTelemetryConfig_Call{Call: _e.mock.On("FetchTelemetryConfig", ctx)} +} + +func (_c *MockClient_FetchTelemetryConfig_Call) Run(run func(ctx context.Context)) *MockClient_FetchTelemetryConfig_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockClient_FetchTelemetryConfig_Call) Return(_a0 *TelemetryConfig, _a1 error) *MockClient_FetchTelemetryConfig_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockClient_FetchTelemetryConfig_Call) RunAndReturn(run func(context.Context) (*TelemetryConfig, error)) *MockClient_FetchTelemetryConfig_Call { + _c.Call.Return(run) + return _c +} + // PushServerStatus provides a mock function with given fields: ctx, status func (_m *MockClient) PushServerStatus(ctx context.Context, status *ServerStatus) error { ret := _m.Called(ctx, status) @@ -151,6 +221,11 @@ func (_c *MockClient_PushServerStatus_Call) Return(_a0 error) *MockClient_PushSe return _c } +func (_c *MockClient_PushServerStatus_Call) RunAndReturn(run func(context.Context, *ServerStatus) error) *MockClient_PushServerStatus_Call { + _c.Call.Return(run) + return _c +} + type mockConstructorTestingTNewMockClient interface { mock.TestingT Cleanup(func()) diff --git a/agent/hcp/client/mock_CloudConfig.go b/agent/hcp/client/mock_CloudConfig.go new file mode 100644 index 00000000000..5f2ef50046d --- /dev/null +++ b/agent/hcp/client/mock_CloudConfig.go @@ -0,0 +1,47 @@ +package client + +import ( + "crypto/tls" + "net/url" + + hcpcfg "github.com/hashicorp/hcp-sdk-go/config" + "github.com/hashicorp/hcp-sdk-go/profile" + "github.com/hashicorp/hcp-sdk-go/resource" + "golang.org/x/oauth2" +) + +const testResourceID = "organization/test-org/project/test-project/test-type/test-id" + +type mockHCPCfg struct{} + +func (m *mockHCPCfg) Token() (*oauth2.Token, error) { + return &oauth2.Token{ + AccessToken: "test-token", + }, nil +} + +func (m *mockHCPCfg) APITLSConfig() *tls.Config { return nil } +func (m *mockHCPCfg) SCADAAddress() string { return "" } +func (m *mockHCPCfg) SCADATLSConfig() *tls.Config { return &tls.Config{} } +func (m *mockHCPCfg) APIAddress() string { return "" } +func (m *mockHCPCfg) PortalURL() *url.URL { return &url.URL{} } +func (m *mockHCPCfg) Profile() *profile.UserProfile { return nil } + +type MockCloudCfg struct { + ConfigErr error + ResourceErr error +} + +func (m MockCloudCfg) Resource() (resource.Resource, error) { + r := resource.Resource{ + ID: "test-id", + Type: "test-type", + Organization: "test-org", + Project: "test-project", + } + return r, m.ResourceErr +} + +func (m MockCloudCfg) HCPConfig(opts ...hcpcfg.HCPConfigOption) (hcpcfg.HCPConfig, error) { + return &mockHCPCfg{}, m.ConfigErr +} diff --git a/agent/hcp/client/telemetry_config.go b/agent/hcp/client/telemetry_config.go new file mode 100644 index 00000000000..162e1b8061e --- /dev/null +++ b/agent/hcp/client/telemetry_config.go @@ -0,0 +1,173 @@ +package client + +import ( + "context" + "errors" + "fmt" + "net/url" + "regexp" + "strings" + "time" + + "github.com/hashicorp/go-hclog" + hcptelemetry "github.com/hashicorp/hcp-sdk-go/clients/cloud-consul-telemetry-gateway/preview/2023-04-14/client/consul_telemetry_service" + + "github.com/hashicorp/consul/agent/hcp/config" +) + +var ( + // defaultMetricFilters is a regex that matches all metric names. + DefaultMetricFilters = regexp.MustCompile(".+") + + // Validation errors for AgentTelemetryConfigOK response. + errMissingPayload = errors.New("missing payload") + errMissingTelemetryConfig = errors.New("missing telemetry config") + errMissingRefreshConfig = errors.New("missing refresh config") + errMissingMetricsConfig = errors.New("missing metrics config") + errInvalidRefreshInterval = errors.New("invalid refresh interval") + errInvalidEndpoint = errors.New("invalid metrics endpoint") + errEmptyEndpoint = errors.New("empty metrics endpoint") +) + +// TelemetryConfig contains configuration for telemetry data forwarded by Consul servers +// to the HCP Telemetry gateway. +type TelemetryConfig struct { + MetricsConfig *MetricsConfig + RefreshConfig *RefreshConfig +} + +// MetricsConfig holds metrics specific configuration within TelemetryConfig. +type MetricsConfig struct { + Labels map[string]string + Filters *regexp.Regexp + Endpoint *url.URL + Disabled bool +} + +// RefreshConfig contains configuration for the periodic fetch of configuration from HCP. +type RefreshConfig struct { + RefreshInterval time.Duration +} + +// validateAgentTelemetryConfigPayload ensures the returned payload from HCP is valid. +func validateAgentTelemetryConfigPayload(resp *hcptelemetry.AgentTelemetryConfigOK) error { + if resp.Payload == nil { + return errMissingPayload + } + + if resp.Payload.TelemetryConfig == nil { + return errMissingTelemetryConfig + } + + if resp.Payload.RefreshConfig == nil { + return errMissingRefreshConfig + } + + if resp.Payload.TelemetryConfig.Metrics == nil { + return errMissingMetricsConfig + } + + return nil +} + +// convertAgentTelemetryResponse converts an AgentTelemetryConfig payload into a TelemetryConfig object. +func convertAgentTelemetryResponse(ctx context.Context, resp *hcptelemetry.AgentTelemetryConfigOK, cfg config.CloudConfig) (*TelemetryConfig, error) { + refreshInterval, err := time.ParseDuration(resp.Payload.RefreshConfig.RefreshInterval) + if err != nil { + return nil, fmt.Errorf("%w: %w", errInvalidRefreshInterval, err) + } + + telemetryConfig := resp.Payload.TelemetryConfig + metricsEndpoint, err := convertMetricEndpoint(telemetryConfig.Endpoint, telemetryConfig.Metrics.Endpoint) + if err != nil { + return nil, err + } + + metricsFilters := convertMetricFilters(ctx, telemetryConfig.Metrics.IncludeList) + metricLabels := convertMetricLabels(telemetryConfig.Labels, cfg) + + return &TelemetryConfig{ + MetricsConfig: &MetricsConfig{ + Endpoint: metricsEndpoint, + Labels: metricLabels, + Filters: metricsFilters, + Disabled: telemetryConfig.Metrics.Disabled, + }, + RefreshConfig: &RefreshConfig{ + RefreshInterval: refreshInterval, + }, + }, nil +} + +// convertMetricEndpoint returns a url for the export of metrics, if a valid endpoint was obtained. +// It returns no error, and no url, if an empty endpoint is retrieved (server not registered with CCM). +// It returns an error, and no url, if a bad endpoint is retrieved. +func convertMetricEndpoint(telemetryEndpoint string, metricsEndpoint string) (*url.URL, error) { + // Telemetry endpoint overriden by metrics specific endpoint, if given. + endpoint := telemetryEndpoint + if metricsEndpoint != "" { + endpoint = metricsEndpoint + } + + if endpoint == "" { + return nil, errEmptyEndpoint + } + + // Endpoint from CTW has no metrics path, so it must be added. + rawUrl := endpoint + metricsGatewayPath + u, err := url.ParseRequestURI(rawUrl) + if err != nil { + return nil, fmt.Errorf("%w: %w", errInvalidEndpoint, err) + } + + return u, nil +} + +// convertMetricFilters returns a valid regex used to filter metrics. +// if invalid filters are given, a defaults regex that allow all metrics is returned. +func convertMetricFilters(ctx context.Context, payloadFilters []string) *regexp.Regexp { + logger := hclog.FromContext(ctx) + validFilters := make([]string, 0, len(payloadFilters)) + for _, filter := range payloadFilters { + _, err := regexp.Compile(filter) + if err != nil { + logger.Error("invalid filter", "error", err) + continue + } + validFilters = append(validFilters, filter) + } + + if len(validFilters) == 0 { + logger.Error("no valid filters") + return DefaultMetricFilters + } + + // Combine the valid regex strings with OR. + finalRegex := strings.Join(validFilters, "|") + composedRegex, err := regexp.Compile(finalRegex) + if err != nil { + logger.Error("failed to compile final regex", "error", err) + return DefaultMetricFilters + } + + return composedRegex +} + +// convertMetricLabels returns a set of string pairs that must be added as attributes to all exported telemetry data. +func convertMetricLabels(payloadLabels map[string]string, cfg config.CloudConfig) map[string]string { + labels := make(map[string]string) + nodeID := string(cfg.NodeID) + if nodeID != "" { + labels["node_id"] = nodeID + } + + if cfg.NodeName != "" { + labels["node_name"] = cfg.NodeName + } + + for k, v := range payloadLabels { + labels[k] = v + } + + return labels +} diff --git a/agent/hcp/client/telemetry_config_test.go b/agent/hcp/client/telemetry_config_test.go new file mode 100644 index 00000000000..659a2ab0033 --- /dev/null +++ b/agent/hcp/client/telemetry_config_test.go @@ -0,0 +1,340 @@ +package client + +import ( + "context" + "net/url" + "regexp" + "testing" + "time" + + "github.com/hashicorp/hcp-sdk-go/clients/cloud-consul-telemetry-gateway/preview/2023-04-14/client/consul_telemetry_service" + "github.com/hashicorp/hcp-sdk-go/clients/cloud-consul-telemetry-gateway/preview/2023-04-14/models" + "github.com/stretchr/testify/require" + + "github.com/hashicorp/consul/agent/hcp/config" + "github.com/hashicorp/consul/types" +) + +func TestValidateAgentTelemetryConfigPayload(t *testing.T) { + t.Parallel() + for name, tc := range map[string]struct { + resp *consul_telemetry_service.AgentTelemetryConfigOK + wantErr error + }{ + "errorsWithNilPayload": { + resp: &consul_telemetry_service.AgentTelemetryConfigOK{}, + wantErr: errMissingPayload, + }, + "errorsWithNilTelemetryConfig": { + resp: &consul_telemetry_service.AgentTelemetryConfigOK{ + Payload: &models.HashicorpCloudConsulTelemetry20230414AgentTelemetryConfigResponse{ + RefreshConfig: &models.HashicorpCloudConsulTelemetry20230414RefreshConfig{}, + }, + }, + wantErr: errMissingTelemetryConfig, + }, + "errorsWithNilRefreshConfig": { + resp: &consul_telemetry_service.AgentTelemetryConfigOK{ + Payload: &models.HashicorpCloudConsulTelemetry20230414AgentTelemetryConfigResponse{ + TelemetryConfig: &models.HashicorpCloudConsulTelemetry20230414TelemetryConfig{}, + }, + }, + wantErr: errMissingRefreshConfig, + }, + "errorsWithNilMetricsConfig": { + resp: &consul_telemetry_service.AgentTelemetryConfigOK{ + Payload: &models.HashicorpCloudConsulTelemetry20230414AgentTelemetryConfigResponse{ + TelemetryConfig: &models.HashicorpCloudConsulTelemetry20230414TelemetryConfig{}, + RefreshConfig: &models.HashicorpCloudConsulTelemetry20230414RefreshConfig{}, + }, + }, + wantErr: errMissingMetricsConfig, + }, + "success": { + resp: &consul_telemetry_service.AgentTelemetryConfigOK{ + Payload: &models.HashicorpCloudConsulTelemetry20230414AgentTelemetryConfigResponse{ + TelemetryConfig: &models.HashicorpCloudConsulTelemetry20230414TelemetryConfig{ + Metrics: &models.HashicorpCloudConsulTelemetry20230414TelemetryMetricsConfig{}, + }, + RefreshConfig: &models.HashicorpCloudConsulTelemetry20230414RefreshConfig{}, + }, + }, + }, + } { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + err := validateAgentTelemetryConfigPayload(tc.resp) + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + return + } + require.NoError(t, err) + }) + } +} + +func TestConvertAgentTelemetryResponse(t *testing.T) { + validTestURL, err := url.Parse("https://test.com/v1/metrics") + require.NoError(t, err) + + validTestFilters, err := regexp.Compile("test|consul") + require.NoError(t, err) + + for name, tc := range map[string]struct { + resp *consul_telemetry_service.AgentTelemetryConfigOK + expectedTelemetryCfg *TelemetryConfig + wantErr error + }{ + "success": { + resp: &consul_telemetry_service.AgentTelemetryConfigOK{ + Payload: &models.HashicorpCloudConsulTelemetry20230414AgentTelemetryConfigResponse{ + TelemetryConfig: &models.HashicorpCloudConsulTelemetry20230414TelemetryConfig{ + Endpoint: "https://test.com", + Labels: map[string]string{"test": "test"}, + Metrics: &models.HashicorpCloudConsulTelemetry20230414TelemetryMetricsConfig{ + IncludeList: []string{"test", "consul"}, + }, + }, + RefreshConfig: &models.HashicorpCloudConsulTelemetry20230414RefreshConfig{ + RefreshInterval: "2s", + }, + }, + }, + expectedTelemetryCfg: &TelemetryConfig{ + MetricsConfig: &MetricsConfig{ + Endpoint: validTestURL, + Labels: map[string]string{"test": "test"}, + Filters: validTestFilters, + }, + RefreshConfig: &RefreshConfig{ + RefreshInterval: 2 * time.Second, + }, + }, + }, + "successBadFilters": { + resp: &consul_telemetry_service.AgentTelemetryConfigOK{ + Payload: &models.HashicorpCloudConsulTelemetry20230414AgentTelemetryConfigResponse{ + TelemetryConfig: &models.HashicorpCloudConsulTelemetry20230414TelemetryConfig{ + Endpoint: "https://test.com", + Labels: map[string]string{"test": "test"}, + Metrics: &models.HashicorpCloudConsulTelemetry20230414TelemetryMetricsConfig{ + IncludeList: []string{"[", "(*LF)"}, + }, + }, + RefreshConfig: &models.HashicorpCloudConsulTelemetry20230414RefreshConfig{ + RefreshInterval: "2s", + }, + }, + }, + expectedTelemetryCfg: &TelemetryConfig{ + MetricsConfig: &MetricsConfig{ + Endpoint: validTestURL, + Labels: map[string]string{"test": "test"}, + Filters: DefaultMetricFilters, + }, + RefreshConfig: &RefreshConfig{ + RefreshInterval: 2 * time.Second, + }, + }, + }, + "errorsWithInvalidRefreshInterval": { + resp: &consul_telemetry_service.AgentTelemetryConfigOK{ + Payload: &models.HashicorpCloudConsulTelemetry20230414AgentTelemetryConfigResponse{ + TelemetryConfig: &models.HashicorpCloudConsulTelemetry20230414TelemetryConfig{ + Metrics: &models.HashicorpCloudConsulTelemetry20230414TelemetryMetricsConfig{}, + }, + RefreshConfig: &models.HashicorpCloudConsulTelemetry20230414RefreshConfig{ + RefreshInterval: "300ws", + }, + }, + }, + wantErr: errInvalidRefreshInterval, + }, + "errorsWithInvalidEndpoint": { + resp: &consul_telemetry_service.AgentTelemetryConfigOK{ + Payload: &models.HashicorpCloudConsulTelemetry20230414AgentTelemetryConfigResponse{ + TelemetryConfig: &models.HashicorpCloudConsulTelemetry20230414TelemetryConfig{ + Metrics: &models.HashicorpCloudConsulTelemetry20230414TelemetryMetricsConfig{ + Endpoint: " ", + }, + }, + RefreshConfig: &models.HashicorpCloudConsulTelemetry20230414RefreshConfig{ + RefreshInterval: "1s", + }, + }, + }, + wantErr: errInvalidEndpoint, + }, + } { + t.Run(name, func(t *testing.T) { + telemetryCfg, err := convertAgentTelemetryResponse(context.Background(), tc.resp, config.CloudConfig{}) + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + require.Nil(t, telemetryCfg) + return + } + require.NoError(t, err) + require.Equal(t, tc.expectedTelemetryCfg, telemetryCfg) + }) + } +} + +func TestConvertMetricEndpoint(t *testing.T) { + t.Parallel() + for name, tc := range map[string]struct { + endpoint string + override string + expected string + wantErr error + }{ + "success": { + endpoint: "https://test.com", + expected: "https://test.com/v1/metrics", + }, + "successMetricsOverride": { + endpoint: "https://test.com", + override: "https://override.com", + expected: "https://override.com/v1/metrics", + }, + "errorWithEmptyEndpoints": { + endpoint: "", + override: "", + wantErr: errEmptyEndpoint, + }, + "errorWithInvalidURL": { + endpoint: " ", + override: "", + wantErr: errInvalidEndpoint, + }, + } { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + u, err := convertMetricEndpoint(tc.endpoint, tc.override) + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + require.Empty(t, u) + return + } + + require.NotNil(t, u) + require.NoError(t, err) + require.Equal(t, tc.expected, u.String()) + }) + } + +} + +func TestConvertMetricFilters(t *testing.T) { + t.Parallel() + for name, tc := range map[string]struct { + filters []string + expectedRegexString string + matches []string + wantErr string + wantMatch bool + }{ + "badFilterRegex": { + filters: []string{"(*LF)"}, + expectedRegexString: DefaultMetricFilters.String(), + matches: []string{"consul.raft.peers", "consul.mem.heap_size"}, + wantMatch: true, + }, + "emptyRegex": { + filters: []string{}, + expectedRegexString: DefaultMetricFilters.String(), + matches: []string{"consul.raft.peers", "consul.mem.heap_size"}, + wantMatch: true, + }, + "matchFound": { + filters: []string{"raft.*", "mem.*"}, + expectedRegexString: "raft.*|mem.*", + matches: []string{"consul.raft.peers", "consul.mem.heap_size"}, + wantMatch: true, + }, + "matchNotFound": { + filters: []string{"mem.*"}, + matches: []string{"consul.raft.peers", "consul.txn.apply"}, + expectedRegexString: "mem.*", + wantMatch: false, + }, + } { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + f := convertMetricFilters(context.Background(), tc.filters) + + require.Equal(t, tc.expectedRegexString, f.String()) + for _, metric := range tc.matches { + m := f.MatchString(metric) + require.Equal(t, tc.wantMatch, m) + } + }) + } +} + +func TestConvertMetricLabels(t *testing.T) { + t.Parallel() + for name, tc := range map[string]struct { + payloadLabels map[string]string + cfg config.CloudConfig + expectedLabels map[string]string + }{ + "Success": { + payloadLabels: map[string]string{ + "ctw_label": "test", + }, + cfg: config.CloudConfig{ + NodeID: types.NodeID("nodeyid"), + NodeName: "nodey", + }, + expectedLabels: map[string]string{ + "ctw_label": "test", + "node_id": "nodeyid", + "node_name": "nodey", + }, + }, + + "NoNodeID": { + payloadLabels: map[string]string{ + "ctw_label": "test", + }, + cfg: config.CloudConfig{ + NodeID: types.NodeID(""), + NodeName: "nodey", + }, + expectedLabels: map[string]string{ + "ctw_label": "test", + "node_name": "nodey", + }, + }, + "NoNodeName": { + payloadLabels: map[string]string{ + "ctw_label": "test", + }, + cfg: config.CloudConfig{ + NodeID: types.NodeID("nodeyid"), + NodeName: "", + }, + expectedLabels: map[string]string{ + "ctw_label": "test", + "node_id": "nodeyid", + }, + }, + "Empty": { + cfg: config.CloudConfig{ + NodeID: "", + NodeName: "", + }, + expectedLabels: map[string]string{}, + }, + } { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + labels := convertMetricLabels(tc.payloadLabels, tc.cfg) + require.Equal(t, labels, tc.expectedLabels) + }) + } +} diff --git a/agent/hcp/config/config.go b/agent/hcp/config/config.go index 98135710463..6e76f952609 100644 --- a/agent/hcp/config/config.go +++ b/agent/hcp/config/config.go @@ -3,7 +3,9 @@ package config import ( "crypto/tls" + "github.com/hashicorp/consul/types" hcpcfg "github.com/hashicorp/hcp-sdk-go/config" + "github.com/hashicorp/hcp-sdk-go/resource" ) // CloudConfig defines configuration for connecting to HCP services @@ -14,21 +16,42 @@ type CloudConfig struct { Hostname string AuthURL string ScadaAddress string + + // Management token used by HCP management plane. + // Cannot be set via config files. + ManagementToken string + + // TlsConfig for testing. + TLSConfig *tls.Config + + NodeID types.NodeID + NodeName string +} + +func (c *CloudConfig) WithTLSConfig(cfg *tls.Config) { + c.TLSConfig = cfg +} + +func (c *CloudConfig) Resource() (resource.Resource, error) { + return resource.FromString(c.ResourceID) } func (c *CloudConfig) HCPConfig(opts ...hcpcfg.HCPConfigOption) (hcpcfg.HCPConfig, error) { + if c.TLSConfig == nil { + c.TLSConfig = &tls.Config{} + } if c.ClientID != "" && c.ClientSecret != "" { opts = append(opts, hcpcfg.WithClientCredentials(c.ClientID, c.ClientSecret)) } if c.AuthURL != "" { - opts = append(opts, hcpcfg.WithAuth(c.AuthURL, &tls.Config{})) + opts = append(opts, hcpcfg.WithAuth(c.AuthURL, c.TLSConfig)) } if c.Hostname != "" { - opts = append(opts, hcpcfg.WithAPI(c.Hostname, &tls.Config{})) + opts = append(opts, hcpcfg.WithAPI(c.Hostname, c.TLSConfig)) } if c.ScadaAddress != "" { - opts = append(opts, hcpcfg.WithSCADA(c.ScadaAddress, &tls.Config{})) + opts = append(opts, hcpcfg.WithSCADA(c.ScadaAddress, c.TLSConfig)) } - opts = append(opts, hcpcfg.FromEnv()) + opts = append(opts, hcpcfg.FromEnv(), hcpcfg.WithoutBrowserLogin()) return hcpcfg.NewHCPConfig(opts...) } diff --git a/agent/hcp/deps.go b/agent/hcp/deps.go index 418d02620ee..1709df50545 100644 --- a/agent/hcp/deps.go +++ b/agent/hcp/deps.go @@ -1,23 +1,79 @@ package hcp import ( + "context" + "fmt" + + "github.com/armon/go-metrics" + "github.com/hashicorp/go-hclog" + + "github.com/hashicorp/consul/agent/hcp/client" "github.com/hashicorp/consul/agent/hcp/config" "github.com/hashicorp/consul/agent/hcp/scada" - "github.com/hashicorp/go-hclog" + "github.com/hashicorp/consul/agent/hcp/telemetry" ) // Deps contains the interfaces that the rest of Consul core depends on for HCP integration. type Deps struct { - Client Client + Client client.Client Provider scada.Provider + Sink metrics.MetricSink } -func NewDeps(cfg config.CloudConfig, logger hclog.Logger) (d Deps, err error) { - d.Client, err = NewClient(cfg) +func NewDeps(cfg config.CloudConfig, logger hclog.Logger) (Deps, error) { + ctx := context.Background() + ctx = hclog.WithContext(ctx, logger) + + hcpClient, err := client.NewClient(cfg) + if err != nil { + return Deps{}, fmt.Errorf("failed to init client: %w", err) + } + + provider, err := scada.New(cfg, logger.Named("scada")) + if err != nil { + return Deps{}, fmt.Errorf("failed to init scada: %w", err) + } + + metricsClient, err := client.NewMetricsClient(ctx, &cfg) if err != nil { - return + logger.Error("failed to init metrics client", "error", err) + return Deps{}, fmt.Errorf("failed to init metrics client: %w", err) } - d.Provider, err = scada.New(cfg, logger.Named("hcp.scada")) - return + sink, err := sink(ctx, metricsClient, NewHCPProvider(ctx, hcpClient)) + if err != nil { + // Do not prevent server start if sink init fails, only log error. + logger.Error("failed to init sink", "error", err) + } + + return Deps{ + Client: hcpClient, + Provider: provider, + Sink: sink, + }, nil +} + +// sink initializes an OTELSink which forwards Consul metrics to HCP. +// This step should not block server initialization, errors are returned, only to be logged. +func sink( + ctx context.Context, + metricsClient telemetry.MetricsClient, + cfgProvider *hcpProviderImpl, +) (metrics.MetricSink, error) { + logger := hclog.FromContext(ctx) + + reader := telemetry.NewOTELReader(metricsClient, cfgProvider) + sinkOpts := &telemetry.OTELSinkOpts{ + Reader: reader, + ConfigProvider: cfgProvider, + } + + sink, err := telemetry.NewOTELSink(ctx, sinkOpts) + if err != nil { + return nil, fmt.Errorf("failed to create OTELSink: %w", err) + } + + logger.Debug("initialized HCP metrics sink") + + return sink, nil } diff --git a/agent/hcp/deps_test.go b/agent/hcp/deps_test.go new file mode 100644 index 00000000000..ecf64489a1a --- /dev/null +++ b/agent/hcp/deps_test.go @@ -0,0 +1,25 @@ +package hcp + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/hashicorp/consul/agent/hcp/telemetry" +) + +type mockMetricsClient struct { + telemetry.MetricsClient +} + +func TestSink(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s, err := sink(ctx, mockMetricsClient{}, &hcpProviderImpl{}) + + require.NotNil(t, s) + require.NoError(t, err) +} diff --git a/agent/hcp/discover/discover.go b/agent/hcp/discover/discover.go index 8707a03a555..f2d109d084a 100644 --- a/agent/hcp/discover/discover.go +++ b/agent/hcp/discover/discover.go @@ -6,7 +6,7 @@ import ( "log" "time" - "github.com/hashicorp/consul/agent/hcp" + hcpclient "github.com/hashicorp/consul/agent/hcp/client" "github.com/hashicorp/consul/agent/hcp/config" ) @@ -29,7 +29,7 @@ func (p *Provider) Addrs(args map[string]string, l *log.Logger) ([]string, error return nil, err } - client, err := hcp.NewClient(cfg.CloudConfig) + client, err := hcpclient.NewClient(cfg.CloudConfig) if err != nil { return nil, err } diff --git a/agent/hcp/manager.go b/agent/hcp/manager.go index 9e3624f6af2..44f722c6773 100644 --- a/agent/hcp/manager.go +++ b/agent/hcp/manager.go @@ -5,6 +5,7 @@ import ( "sync" "time" + hcpclient "github.com/hashicorp/consul/agent/hcp/client" "github.com/hashicorp/consul/lib" "github.com/hashicorp/go-hclog" ) @@ -15,7 +16,7 @@ var ( ) type ManagerConfig struct { - Client Client + Client hcpclient.Client StatusFn StatusCallback MinInterval time.Duration @@ -44,7 +45,7 @@ func (cfg *ManagerConfig) nextHeartbeat() time.Duration { return min + lib.RandomStagger(max-min) } -type StatusCallback func(context.Context) (ServerStatus, error) +type StatusCallback func(context.Context) (hcpclient.ServerStatus, error) type Manager struct { logger hclog.Logger diff --git a/agent/hcp/manager_test.go b/agent/hcp/manager_test.go index cb4d729b7fa..2363739d620 100644 --- a/agent/hcp/manager_test.go +++ b/agent/hcp/manager_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + hcpclient "github.com/hashicorp/consul/agent/hcp/client" "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -12,12 +13,12 @@ import ( ) func TestManager_Run(t *testing.T) { - client := NewMockClient(t) - statusF := func(ctx context.Context) (ServerStatus, error) { - return ServerStatus{ID: t.Name()}, nil + client := hcpclient.NewMockClient(t) + statusF := func(ctx context.Context) (hcpclient.ServerStatus, error) { + return hcpclient.ServerStatus{ID: t.Name()}, nil } updateCh := make(chan struct{}, 1) - client.EXPECT().PushServerStatus(mock.Anything, &ServerStatus{ID: t.Name()}).Return(nil).Once() + client.EXPECT().PushServerStatus(mock.Anything, &hcpclient.ServerStatus{ID: t.Name()}).Return(nil).Once() mgr := NewManager(ManagerConfig{ Client: client, Logger: hclog.New(&hclog.LoggerOptions{Output: io.Discard}), @@ -36,26 +37,29 @@ func TestManager_Run(t *testing.T) { // Make sure after manager has stopped no more statuses are pushed. cancel() - mgr.SendUpdate() client.AssertExpectations(t) } func TestManager_SendUpdate(t *testing.T) { - client := NewMockClient(t) - statusF := func(ctx context.Context) (ServerStatus, error) { - return ServerStatus{ID: t.Name()}, nil + client := hcpclient.NewMockClient(t) + statusF := func(ctx context.Context) (hcpclient.ServerStatus, error) { + return hcpclient.ServerStatus{ID: t.Name()}, nil } updateCh := make(chan struct{}, 1) // Expect two calls, once during run startup and again when SendUpdate is called - client.EXPECT().PushServerStatus(mock.Anything, &ServerStatus{ID: t.Name()}).Return(nil).Twice() + client.EXPECT().PushServerStatus(mock.Anything, &hcpclient.ServerStatus{ID: t.Name()}).Return(nil).Twice() mgr := NewManager(ManagerConfig{ Client: client, Logger: hclog.New(&hclog.LoggerOptions{Output: io.Discard}), StatusFn: statusF, }) mgr.testUpdateSent = updateCh - go mgr.Run(context.Background()) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go mgr.Run(ctx) select { case <-updateCh: case <-time.After(time.Second): @@ -71,14 +75,14 @@ func TestManager_SendUpdate(t *testing.T) { } func TestManager_SendUpdate_Periodic(t *testing.T) { - client := NewMockClient(t) - statusF := func(ctx context.Context) (ServerStatus, error) { - return ServerStatus{ID: t.Name()}, nil + client := hcpclient.NewMockClient(t) + statusF := func(ctx context.Context) (hcpclient.ServerStatus, error) { + return hcpclient.ServerStatus{ID: t.Name()}, nil } updateCh := make(chan struct{}, 1) // Expect two calls, once during run startup and again when SendUpdate is called - client.EXPECT().PushServerStatus(mock.Anything, &ServerStatus{ID: t.Name()}).Return(nil).Twice() + client.EXPECT().PushServerStatus(mock.Anything, &hcpclient.ServerStatus{ID: t.Name()}).Return(nil).Twice() mgr := NewManager(ManagerConfig{ Client: client, Logger: hclog.New(&hclog.LoggerOptions{Output: io.Discard}), @@ -87,7 +91,11 @@ func TestManager_SendUpdate_Periodic(t *testing.T) { MinInterval: 100 * time.Millisecond, }) mgr.testUpdateSent = updateCh - go mgr.Run(context.Background()) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go mgr.Run(ctx) select { case <-updateCh: case <-time.After(time.Second): diff --git a/agent/hcp/scada/mock_Provider.go b/agent/hcp/scada/mock_Provider.go index 251178095bc..b9a0fd2d496 100644 --- a/agent/hcp/scada/mock_Provider.go +++ b/agent/hcp/scada/mock_Provider.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.15.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package scada @@ -7,6 +7,8 @@ import ( mock "github.com/stretchr/testify/mock" + provider "github.com/hashicorp/hcp-scada-provider" + time "time" ) @@ -23,6 +25,98 @@ func (_m *MockProvider) EXPECT() *MockProvider_Expecter { return &MockProvider_Expecter{mock: &_m.Mock} } +// AddMeta provides a mock function with given fields: _a0 +func (_m *MockProvider) AddMeta(_a0 ...provider.Meta) { + _va := make([]interface{}, len(_a0)) + for _i := range _a0 { + _va[_i] = _a0[_i] + } + var _ca []interface{} + _ca = append(_ca, _va...) + _m.Called(_ca...) +} + +// MockProvider_AddMeta_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddMeta' +type MockProvider_AddMeta_Call struct { + *mock.Call +} + +// AddMeta is a helper method to define mock.On call +// - _a0 ...provider.Meta +func (_e *MockProvider_Expecter) AddMeta(_a0 ...interface{}) *MockProvider_AddMeta_Call { + return &MockProvider_AddMeta_Call{Call: _e.mock.On("AddMeta", + append([]interface{}{}, _a0...)...)} +} + +func (_c *MockProvider_AddMeta_Call) Run(run func(_a0 ...provider.Meta)) *MockProvider_AddMeta_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]provider.Meta, len(args)-0) + for i, a := range args[0:] { + if a != nil { + variadicArgs[i] = a.(provider.Meta) + } + } + run(variadicArgs...) + }) + return _c +} + +func (_c *MockProvider_AddMeta_Call) Return() *MockProvider_AddMeta_Call { + _c.Call.Return() + return _c +} + +func (_c *MockProvider_AddMeta_Call) RunAndReturn(run func(...provider.Meta)) *MockProvider_AddMeta_Call { + _c.Call.Return(run) + return _c +} + +// DeleteMeta provides a mock function with given fields: _a0 +func (_m *MockProvider) DeleteMeta(_a0 ...string) { + _va := make([]interface{}, len(_a0)) + for _i := range _a0 { + _va[_i] = _a0[_i] + } + var _ca []interface{} + _ca = append(_ca, _va...) + _m.Called(_ca...) +} + +// MockProvider_DeleteMeta_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteMeta' +type MockProvider_DeleteMeta_Call struct { + *mock.Call +} + +// DeleteMeta is a helper method to define mock.On call +// - _a0 ...string +func (_e *MockProvider_Expecter) DeleteMeta(_a0 ...interface{}) *MockProvider_DeleteMeta_Call { + return &MockProvider_DeleteMeta_Call{Call: _e.mock.On("DeleteMeta", + append([]interface{}{}, _a0...)...)} +} + +func (_c *MockProvider_DeleteMeta_Call) Run(run func(_a0 ...string)) *MockProvider_DeleteMeta_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]string, len(args)-0) + for i, a := range args[0:] { + if a != nil { + variadicArgs[i] = a.(string) + } + } + run(variadicArgs...) + }) + return _c +} + +func (_c *MockProvider_DeleteMeta_Call) Return() *MockProvider_DeleteMeta_Call { + _c.Call.Return() + return _c +} + +func (_c *MockProvider_DeleteMeta_Call) RunAndReturn(run func(...string)) *MockProvider_DeleteMeta_Call { + _c.Call.Return(run) + return _c +} + // GetMeta provides a mock function with given fields: func (_m *MockProvider) GetMeta() map[string]string { ret := _m.Called() @@ -61,18 +155,26 @@ func (_c *MockProvider_GetMeta_Call) Return(_a0 map[string]string) *MockProvider return _c } +func (_c *MockProvider_GetMeta_Call) RunAndReturn(run func() map[string]string) *MockProvider_GetMeta_Call { + _c.Call.Return(run) + return _c +} + // LastError provides a mock function with given fields: func (_m *MockProvider) LastError() (time.Time, error) { ret := _m.Called() var r0 time.Time + var r1 error + if rf, ok := ret.Get(0).(func() (time.Time, error)); ok { + return rf() + } if rf, ok := ret.Get(0).(func() time.Time); ok { r0 = rf() } else { r0 = ret.Get(0).(time.Time) } - var r1 error if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { @@ -104,11 +206,20 @@ func (_c *MockProvider_LastError_Call) Return(_a0 time.Time, _a1 error) *MockPro return _c } +func (_c *MockProvider_LastError_Call) RunAndReturn(run func() (time.Time, error)) *MockProvider_LastError_Call { + _c.Call.Return(run) + return _c +} + // Listen provides a mock function with given fields: capability func (_m *MockProvider) Listen(capability string) (net.Listener, error) { ret := _m.Called(capability) var r0 net.Listener + var r1 error + if rf, ok := ret.Get(0).(func(string) (net.Listener, error)); ok { + return rf(capability) + } if rf, ok := ret.Get(0).(func(string) net.Listener); ok { r0 = rf(capability) } else { @@ -117,7 +228,6 @@ func (_m *MockProvider) Listen(capability string) (net.Listener, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(capability) } else { @@ -150,6 +260,11 @@ func (_c *MockProvider_Listen_Call) Return(_a0 net.Listener, _a1 error) *MockPro return _c } +func (_c *MockProvider_Listen_Call) RunAndReturn(run func(string) (net.Listener, error)) *MockProvider_Listen_Call { + _c.Call.Return(run) + return _c +} + // SessionStatus provides a mock function with given fields: func (_m *MockProvider) SessionStatus() string { ret := _m.Called() @@ -186,6 +301,11 @@ func (_c *MockProvider_SessionStatus_Call) Return(_a0 string) *MockProvider_Sess return _c } +func (_c *MockProvider_SessionStatus_Call) RunAndReturn(run func() string) *MockProvider_SessionStatus_Call { + _c.Call.Return(run) + return _c +} + // Start provides a mock function with given fields: func (_m *MockProvider) Start() error { ret := _m.Called() @@ -222,6 +342,11 @@ func (_c *MockProvider_Start_Call) Return(_a0 error) *MockProvider_Start_Call { return _c } +func (_c *MockProvider_Start_Call) RunAndReturn(run func() error) *MockProvider_Start_Call { + _c.Call.Return(run) + return _c +} + // Stop provides a mock function with given fields: func (_m *MockProvider) Stop() error { ret := _m.Called() @@ -258,6 +383,11 @@ func (_c *MockProvider_Stop_Call) Return(_a0 error) *MockProvider_Stop_Call { return _c } +func (_c *MockProvider_Stop_Call) RunAndReturn(run func() error) *MockProvider_Stop_Call { + _c.Call.Return(run) + return _c +} + // UpdateMeta provides a mock function with given fields: _a0 func (_m *MockProvider) UpdateMeta(_a0 map[string]string) { _m.Called(_a0) @@ -286,6 +416,11 @@ func (_c *MockProvider_UpdateMeta_Call) Return() *MockProvider_UpdateMeta_Call { return _c } +func (_c *MockProvider_UpdateMeta_Call) RunAndReturn(run func(map[string]string)) *MockProvider_UpdateMeta_Call { + _c.Call.Return(run) + return _c +} + type mockConstructorTestingTNewMockProvider interface { mock.TestingT Cleanup(func()) diff --git a/agent/hcp/telemetry/custom_metrics.go b/agent/hcp/telemetry/custom_metrics.go new file mode 100644 index 00000000000..d691dccde20 --- /dev/null +++ b/agent/hcp/telemetry/custom_metrics.go @@ -0,0 +1,14 @@ +package telemetry + +// Keys for custom Go Metrics metrics emitted only for the OTEL +// export (exporter.go) and transform (transform.go) failures and successes. +// These enable us to monitor OTEL operations. +var ( + internalMetricTransformFailure []string = []string{"hcp", "otel", "transform", "failure"} + + internalMetricExportSuccess []string = []string{"hcp", "otel", "exporter", "export", "success"} + internalMetricExportFailure []string = []string{"hcp", "otel", "exporter", "export", "failure"} + + internalMetricExporterShutdown []string = []string{"hcp", "otel", "exporter", "shutdown"} + internalMetricExporterForceFlush []string = []string{"hcp", "otel", "exporter", "force_flush"} +) diff --git a/agent/hcp/telemetry/doc.go b/agent/hcp/telemetry/doc.go new file mode 100644 index 00000000000..4ef18f39bd3 --- /dev/null +++ b/agent/hcp/telemetry/doc.go @@ -0,0 +1,12 @@ +// Package telemetry implements functionality to collect, aggregate, convert and export +// telemetry data in OpenTelemetry Protocol (OTLP) format. +// +// The entrypoint is the OpenTelemetry (OTEL) go-metrics sink which: +// - Receives metric data. +// - Aggregates metric data using the OTEL Go Metrics SDK. +// - Exports metric data using a configurable OTEL exporter. +// +// The package also provides an OTEL exporter implementation to be used within the sink, which: +// - Transforms metric data from the Metrics SDK OTEL representation to OTLP format. +// - Exports OTLP metric data to an external endpoint using a configurable client. +package telemetry diff --git a/agent/hcp/telemetry/gauge_store.go b/agent/hcp/telemetry/gauge_store.go new file mode 100644 index 00000000000..76dfb780666 --- /dev/null +++ b/agent/hcp/telemetry/gauge_store.go @@ -0,0 +1,77 @@ +package telemetry + +import ( + "context" + "sync" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// gaugeStore holds last seen Gauge values for a particular metric () in the store. +// OTEL does not currently have a synchronous Gauge instrument. Instead, it allows the registration of callbacks. +// The callbacks are called during export, where the Gauge value must be returned. +// This store is a workaround, which holds last seen Gauge values until the callback is called. +type gaugeStore struct { + store map[string]*gaugeValue + mutex sync.Mutex +} + +// gaugeValues are the last seen measurement for a Gauge metric, which contains a float64 value and labels. +type gaugeValue struct { + Value float64 + Attributes []attribute.KeyValue +} + +// NewGaugeStore returns an initialized empty gaugeStore. +func NewGaugeStore() *gaugeStore { + return &gaugeStore{ + store: make(map[string]*gaugeValue, 0), + } +} + +// LoadAndDelete will read a Gauge value and delete it. +// Once registered for a metric name, a Gauge callback will continue to execute every collection cycel. +// We must delete the value once we have read it, to avoid repeat values being sent. +func (g *gaugeStore) LoadAndDelete(key string) (*gaugeValue, bool) { + g.mutex.Lock() + defer g.mutex.Unlock() + + gauge, ok := g.store[key] + if !ok { + return nil, ok + } + + delete(g.store, key) + + return gauge, ok +} + +// Set adds a gaugeValue to the global gauge store. +func (g *gaugeStore) Set(key string, value float64, labels []attribute.KeyValue) { + g.mutex.Lock() + defer g.mutex.Unlock() + + gv := &gaugeValue{ + Value: value, + Attributes: labels, + } + + g.store[key] = gv +} + +// gaugeCallback returns a callback which gets called when metrics are collected for export. +func (g *gaugeStore) gaugeCallback(key string) metric.Float64Callback { + // Closures keep a reference to the key string, that get garbage collected when code completes. + return func(ctx context.Context, obs metric.Float64Observer) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + if gauge, ok := g.LoadAndDelete(key); ok { + obs.Observe(gauge.Value, metric.WithAttributes(gauge.Attributes...)) + } + return nil + } + } +} diff --git a/agent/hcp/telemetry/gauge_store_test.go b/agent/hcp/telemetry/gauge_store_test.go new file mode 100644 index 00000000000..1171ee379c3 --- /dev/null +++ b/agent/hcp/telemetry/gauge_store_test.go @@ -0,0 +1,89 @@ +package telemetry + +import ( + "context" + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" +) + +func TestGaugeStore(t *testing.T) { + t.Parallel() + + gaugeStore := NewGaugeStore() + + attributes := []attribute.KeyValue{ + { + Key: attribute.Key("test_key"), + Value: attribute.StringValue("test_value"), + }, + } + + gaugeStore.Set("test", 1.23, attributes) + + // Should store a new gauge. + val, ok := gaugeStore.LoadAndDelete("test") + require.True(t, ok) + require.Equal(t, val.Value, 1.23) + require.Equal(t, val.Attributes, attributes) + + // Gauge with key "test" have been deleted. + val, ok = gaugeStore.LoadAndDelete("test") + require.False(t, ok) + require.Nil(t, val) + + gaugeStore.Set("duplicate", 1.5, nil) + gaugeStore.Set("duplicate", 6.7, nil) + + // Gauge with key "duplicate" should hold the latest (last seen) value. + val, ok = gaugeStore.LoadAndDelete("duplicate") + require.True(t, ok) + require.Equal(t, val.Value, 6.7) +} + +func TestGaugeCallback_Failure(t *testing.T) { + t.Parallel() + + k := "consul.raft.apply" + gaugeStore := NewGaugeStore() + gaugeStore.Set(k, 1.23, nil) + + cb := gaugeStore.gaugeCallback(k) + ctx, cancel := context.WithCancel(context.Background()) + + cancel() + err := cb(ctx, nil) + require.ErrorIs(t, err, context.Canceled) +} + +// TestGaugeStore_Race induces a race condition. When run with go test -race, +// this test should pass if implementation is concurrency safe. +func TestGaugeStore_Race(t *testing.T) { + t.Parallel() + + gaugeStore := NewGaugeStore() + + wg := &sync.WaitGroup{} + samples := 100 + errCh := make(chan error, samples) + for i := 0; i < samples; i++ { + wg.Add(1) + key := fmt.Sprintf("consul.test.%d", i) + value := 12.34 + go func() { + defer wg.Done() + gaugeStore.Set(key, value, nil) + gv, _ := gaugeStore.LoadAndDelete(key) + if gv.Value != value { + errCh <- fmt.Errorf("expected value: '%f', but got: '%f' for key: '%s'", value, gv.Value, key) + } + }() + } + + wg.Wait() + + require.Empty(t, errCh) +} diff --git a/agent/hcp/telemetry/otel_exporter.go b/agent/hcp/telemetry/otel_exporter.go new file mode 100644 index 00000000000..2ba32693ab9 --- /dev/null +++ b/agent/hcp/telemetry/otel_exporter.go @@ -0,0 +1,104 @@ +package telemetry + +import ( + "context" + "fmt" + "net/url" + + goMetrics "github.com/armon/go-metrics" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/aggregation" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + metricpb "go.opentelemetry.io/proto/otlp/metrics/v1" +) + +// MetricsClient exports Consul metrics in OTLP format to the desired endpoint. +type MetricsClient interface { + ExportMetrics(ctx context.Context, protoMetrics *metricpb.ResourceMetrics, endpoint string) error +} + +// EndpointProvider provides the endpoint where metrics are exported to by the OTELExporter. +// EndpointProvider exposes the GetEndpoint() interface method to fetch the endpoint. +// This abstraction layer offers flexibility, in particular for dynamic configuration or changes to the endpoint. +// The OTELExporter calls the Disabled interface to verify that it should actually export metrics. +type EndpointProvider interface { + Disabled + GetEndpoint() *url.URL +} + +// otelExporter is a custom implementation of a OTEL Metrics SDK metrics.Exporter. +// The exporter is used by a OTEL Metrics SDK PeriodicReader to export aggregated metrics. +// This allows us to use a custom client - HCP authenticated MetricsClient. +type otelExporter struct { + client MetricsClient + endpointProvider EndpointProvider +} + +// newOTELExporter returns a configured OTELExporter. +func newOTELExporter(client MetricsClient, endpointProvider EndpointProvider) *otelExporter { + return &otelExporter{ + client: client, + endpointProvider: endpointProvider, + } +} + +// Temporality returns the Cumulative temporality for metrics aggregation. +// Telemetry Gateway stores metrics in Prometheus format, so use Cummulative aggregation as default. +func (e *otelExporter) Temporality(_ metric.InstrumentKind) metricdata.Temporality { + return metricdata.CumulativeTemporality +} + +// Aggregation returns the Aggregation to use for an instrument kind. +// The default implementation provided by the OTEL Metrics SDK library DefaultAggregationSelector panics. +// This custom version replicates that logic, but removes the panic. +func (e *otelExporter) Aggregation(kind metric.InstrumentKind) aggregation.Aggregation { + switch kind { + case metric.InstrumentKindObservableGauge: + return aggregation.LastValue{} + case metric.InstrumentKindHistogram: + return aggregation.ExplicitBucketHistogram{ + Boundaries: []float64{0, 5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10000}, + NoMinMax: false, + } + } + // for metric.InstrumentKindCounter and others, default to sum. + return aggregation.Sum{} +} + +// Export serializes and transmits metric data to a receiver. +func (e *otelExporter) Export(ctx context.Context, metrics *metricdata.ResourceMetrics) error { + if e.endpointProvider.IsDisabled() { + return nil + } + + endpoint := e.endpointProvider.GetEndpoint() + if endpoint == nil { + return nil + } + + otlpMetrics := transformOTLP(metrics) + if isEmpty(otlpMetrics) { + return nil + } + + err := e.client.ExportMetrics(ctx, otlpMetrics, endpoint.String()) + if err != nil { + goMetrics.IncrCounter(internalMetricExportFailure, 1) + return fmt.Errorf("failed to export metrics: %w", err) + } + + goMetrics.IncrCounter(internalMetricExportSuccess, 1) + return nil +} + +// ForceFlush is a no-op, as the MetricsClient client holds no state. +func (e *otelExporter) ForceFlush(ctx context.Context) error { + goMetrics.IncrCounter(internalMetricExporterForceFlush, 1) + return ctx.Err() +} + +// Shutdown is a no-op, as the MetricsClient is a HTTP client that requires no graceful shutdown. +func (e *otelExporter) Shutdown(ctx context.Context) error { + goMetrics.IncrCounter(internalMetricExporterShutdown, 1) + return ctx.Err() +} diff --git a/agent/hcp/telemetry/otel_exporter_test.go b/agent/hcp/telemetry/otel_exporter_test.go new file mode 100644 index 00000000000..6082f6fb0e3 --- /dev/null +++ b/agent/hcp/telemetry/otel_exporter_test.go @@ -0,0 +1,248 @@ +package telemetry + +import ( + "context" + "fmt" + "net/url" + "strings" + "testing" + "time" + + "github.com/armon/go-metrics" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/aggregation" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/resource" + metricpb "go.opentelemetry.io/proto/otlp/metrics/v1" +) + +const ( + testExportEndpoint = "https://test.com/v1/metrics" +) + +type mockMetricsClient struct { + exportErr error +} + +func (m *mockMetricsClient) ExportMetrics(ctx context.Context, protoMetrics *metricpb.ResourceMetrics, endpoint string) error { + return m.exportErr +} + +type mockEndpointProvider struct { + endpoint *url.URL + disabled bool +} + +func (m *mockEndpointProvider) GetEndpoint() *url.URL { return m.endpoint } +func (m *mockEndpointProvider) IsDisabled() bool { return m.disabled } + +func TestTemporality(t *testing.T) { + t.Parallel() + exp := &otelExporter{} + require.Equal(t, metricdata.CumulativeTemporality, exp.Temporality(metric.InstrumentKindCounter)) +} + +func TestAggregation(t *testing.T) { + t.Parallel() + for name, test := range map[string]struct { + kind metric.InstrumentKind + expAgg aggregation.Aggregation + }{ + "gauge": { + kind: metric.InstrumentKindObservableGauge, + expAgg: aggregation.LastValue{}, + }, + "counter": { + kind: metric.InstrumentKindCounter, + expAgg: aggregation.Sum{}, + }, + "histogram": { + kind: metric.InstrumentKindHistogram, + expAgg: aggregation.ExplicitBucketHistogram{Boundaries: []float64{0, 5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10000}, NoMinMax: false}, + }, + } { + test := test + t.Run(name, func(t *testing.T) { + t.Parallel() + exp := &otelExporter{} + require.Equal(t, test.expAgg, exp.Aggregation(test.kind)) + }) + } +} + +func TestExport(t *testing.T) { + t.Parallel() + for name, test := range map[string]struct { + wantErr string + metrics *metricdata.ResourceMetrics + client MetricsClient + provider EndpointProvider + }{ + "earlyReturnDisabledProvider": { + client: &mockMetricsClient{}, + provider: &mockEndpointProvider{ + disabled: true, + }, + }, + "earlyReturnWithoutEndpoint": { + client: &mockMetricsClient{}, + provider: &mockEndpointProvider{}, + }, + "earlyReturnWithoutScopeMetrics": { + client: &mockMetricsClient{}, + metrics: mutateMetrics(nil), + provider: &mockEndpointProvider{}, + }, + "earlyReturnWithoutMetrics": { + client: &mockMetricsClient{}, + metrics: mutateMetrics([]metricdata.ScopeMetrics{ + {Metrics: []metricdata.Metrics{}}, + }, + ), + provider: &mockEndpointProvider{}, + }, + "errorWithExportFailure": { + client: &mockMetricsClient{ + exportErr: fmt.Errorf("failed to export metrics."), + }, + metrics: mutateMetrics([]metricdata.ScopeMetrics{ + { + Metrics: []metricdata.Metrics{ + { + Name: "consul.raft.commitTime", + Data: metricdata.Gauge[float64]{}, + }, + }, + }, + }, + ), + provider: &mockEndpointProvider{ + endpoint: &url.URL{}, + }, + wantErr: "failed to export metrics", + }, + } { + test := test + t.Run(name, func(t *testing.T) { + t.Parallel() + provider := test.provider + if provider == nil { + u, err := url.Parse(testExportEndpoint) + require.NoError(t, err) + provider = &mockEndpointProvider{ + endpoint: u, + } + } + + exp := newOTELExporter(test.client, provider) + + err := exp.Export(context.Background(), test.metrics) + if test.wantErr != "" { + require.Error(t, err) + require.Contains(t, err.Error(), test.wantErr) + return + } + + require.NoError(t, err) + }) + } +} + +// TestExport_CustomMetrics tests that a custom metric (hcp.otel.exporter.*) is emitted +// for exporter operations. This test cannot be run in parallel as the metrics.NewGlobal() +// sets a shared global sink. +func TestExport_CustomMetrics(t *testing.T) { + for name, tc := range map[string]struct { + client MetricsClient + metricKey []string + operation string + }{ + "exportSuccessEmitsCustomMetric": { + client: &mockMetricsClient{}, + metricKey: internalMetricExportSuccess, + operation: "export", + }, + "exportFailureEmitsCustomMetric": { + client: &mockMetricsClient{ + exportErr: fmt.Errorf("client err"), + }, + metricKey: internalMetricExportFailure, + operation: "export", + }, + "shutdownEmitsCustomMetric": { + metricKey: internalMetricExporterShutdown, + operation: "shutdown", + }, + "forceFlushEmitsCustomMetric": { + metricKey: internalMetricExporterForceFlush, + operation: "flush", + }, + } { + t.Run(name, func(t *testing.T) { + // Init global sink. + serviceName := "test.transform" + cfg := metrics.DefaultConfig(serviceName) + cfg.EnableHostname = false + + sink := metrics.NewInmemSink(10*time.Second, 10*time.Second) + metrics.NewGlobal(cfg, sink) + + // Perform operation that emits metric. + u, err := url.Parse(testExportEndpoint) + require.NoError(t, err) + + exp := newOTELExporter(tc.client, &mockEndpointProvider{ + endpoint: u, + }) + + ctx := context.Background() + switch tc.operation { + case "flush": + exp.ForceFlush(ctx) + case "shutdown": + exp.Shutdown(ctx) + default: + exp.Export(ctx, inputResourceMetrics) + } + + // Collect sink metrics. + intervals := sink.Data() + require.Len(t, intervals, 1) + key := serviceName + "." + strings.Join(tc.metricKey, ".") + sv := intervals[0].Counters[key] + + // Verify count for transform failure metric. + require.NotNil(t, sv) + require.NotNil(t, sv.AggregateSample) + require.Equal(t, 1, sv.AggregateSample.Count) + }) + } +} + +func TestForceFlush(t *testing.T) { + t.Parallel() + exp := &otelExporter{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := exp.ForceFlush(ctx) + require.ErrorIs(t, err, context.Canceled) +} + +func TestShutdown(t *testing.T) { + t.Parallel() + exp := &otelExporter{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := exp.Shutdown(ctx) + require.ErrorIs(t, err, context.Canceled) +} + +func mutateMetrics(m []metricdata.ScopeMetrics) *metricdata.ResourceMetrics { + return &metricdata.ResourceMetrics{ + Resource: resource.Empty(), + ScopeMetrics: m, + } +} diff --git a/agent/hcp/telemetry/otel_sink.go b/agent/hcp/telemetry/otel_sink.go new file mode 100644 index 00000000000..6d520f09c8d --- /dev/null +++ b/agent/hcp/telemetry/otel_sink.go @@ -0,0 +1,291 @@ +package telemetry + +import ( + "bytes" + "context" + "errors" + "regexp" + "strings" + "sync" + "time" + + gometrics "github.com/armon/go-metrics" + "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/metric" + otelsdk "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + + "github.com/hashicorp/go-hclog" +) + +const ( + // defaultExportInterval is a default time interval between export of aggregated metrics. + // At the time of writing this is the same as the otelsdk.Reader's default export interval. + defaultExportInterval = 60 * time.Second + + // defaultExportTimeout is the time the otelsdk.Reader waits on an export before cancelling it. + // At the time of writing this is the same as the otelsdk.Reader's default export timeout default. + // + // note: in practice we are more likely to hit the http.Client Timeout in telemetry.MetricsClient. + // That http.Client Timeout is 15 seconds (at the time of writing). The otelsdk.Reader will use + // defaultExportTimeout for the entire Export call, but since the http.Client's Timeout is 15s, + // we should hit that first before reaching the 30 second timeout set here. + defaultExportTimeout = 30 * time.Second +) + +// Disabled should be implemented to turn on/off metrics processing +type Disabled interface { + // IsDisabled() can return true disallow the sink from accepting metrics. + IsDisabled() bool +} + +// ConfigProvider is required to provide custom metrics processing. +type ConfigProvider interface { + Disabled + // GetLabels should return a set of OTEL attributes added by default all metrics. + GetLabels() map[string]string + + // GetFilters should return filtesr that are required to enable metric processing. + // Filters act as an allowlist to collect only the required metrics. + GetFilters() *regexp.Regexp +} + +// OTELSinkOpts is used to provide configuration when initializing an OTELSink using NewOTELSink. +type OTELSinkOpts struct { + Reader otelsdk.Reader + ConfigProvider ConfigProvider +} + +// OTELSink captures and aggregates telemetry data as per the OpenTelemetry (OTEL) specification. +// Metric data is exported in OpenTelemetry Protocol (OTLP) wire format. +// This should be used as a Go Metrics backend, as it implements the MetricsSink interface. +type OTELSink struct { + // spaceReplacer cleans the flattened key by removing any spaces. + spaceReplacer *strings.Replacer + logger hclog.Logger + cfgProvider ConfigProvider + + // meterProvider is an OTEL MeterProvider, the entrypoint to the OTEL Metrics SDK. + // It handles reading/export of aggregated metric data. + // It enables creation and usage of an OTEL Meter. + meterProvider *otelsdk.MeterProvider + + // meter is an OTEL Meter, which enables the creation of OTEL instruments. + meter *otelmetric.Meter + + // Instrument stores contain an OTEL Instrument per metric name () + // for each gauge, counter and histogram types. + // An instrument allows us to record a measurement for a particular metric, and continuously aggregates metrics. + // We lazy load the creation of these intruments until a metric is seen, and use them repeatedly to record measurements. + gaugeInstruments map[string]otelmetric.Float64ObservableGauge + counterInstruments map[string]otelmetric.Float64Counter + histogramInstruments map[string]otelmetric.Float64Histogram + + // gaugeStore is required to hold last-seen values of gauges + // This is a workaround, as OTEL currently does not have synchronous gauge instruments. + // It only allows the registration of "callbacks", which obtain values when the callback is called. + // We must hold gauge values until the callback is called, when the measurement is exported, and can be removed. + gaugeStore *gaugeStore + + mutex sync.Mutex +} + +// NewOTELReader returns a configured OTEL PeriodicReader to export metrics every X seconds. +// It configures the reader with a custom OTELExporter with a MetricsClient to transform and export +// metrics in OTLP format to an external url. +func NewOTELReader(client MetricsClient, endpointProvider EndpointProvider) otelsdk.Reader { + return otelsdk.NewPeriodicReader( + newOTELExporter(client, endpointProvider), + otelsdk.WithInterval(defaultExportInterval), + otelsdk.WithTimeout(defaultExportTimeout), + ) +} + +// NewOTELSink returns a sink which fits the Go Metrics MetricsSink interface. +// It sets up a MeterProvider and Meter, key pieces of the OTEL Metrics SDK which +// enable us to create OTEL Instruments to record measurements. +func NewOTELSink(ctx context.Context, opts *OTELSinkOpts) (*OTELSink, error) { + if opts.Reader == nil { + return nil, errors.New("ferror: provide valid reader") + } + + if opts.ConfigProvider == nil { + return nil, errors.New("ferror: provide valid config provider") + } + + logger := hclog.FromContext(ctx).Named("otel_sink") + + // Setup OTEL Metrics SDK to aggregate, convert and export metrics periodically. + res := resource.NewSchemaless() + meterProvider := otelsdk.NewMeterProvider(otelsdk.WithResource(res), otelsdk.WithReader(opts.Reader)) + meter := meterProvider.Meter("github.com/hashicorp/consul/agent/hcp/telemetry") + + return &OTELSink{ + cfgProvider: opts.ConfigProvider, + spaceReplacer: strings.NewReplacer(" ", "_"), + logger: logger, + meterProvider: meterProvider, + meter: &meter, + gaugeStore: NewGaugeStore(), + gaugeInstruments: make(map[string]otelmetric.Float64ObservableGauge, 0), + counterInstruments: make(map[string]otelmetric.Float64Counter, 0), + histogramInstruments: make(map[string]otelmetric.Float64Histogram, 0), + }, nil +} + +// SetGauge emits a Consul gauge metric. +func (o *OTELSink) SetGauge(key []string, val float32) { + o.SetGaugeWithLabels(key, val, nil) +} + +// AddSample emits a Consul histogram metric. +func (o *OTELSink) AddSample(key []string, val float32) { + o.AddSampleWithLabels(key, val, nil) +} + +// IncrCounter emits a Consul counter metric. +func (o *OTELSink) IncrCounter(key []string, val float32) { + o.IncrCounterWithLabels(key, val, nil) +} + +// AddSampleWithLabels emits a Consul gauge metric that gets +// registed by an OpenTelemetry Histogram instrument. +func (o *OTELSink) SetGaugeWithLabels(key []string, val float32, labels []gometrics.Label) { + if o.cfgProvider.IsDisabled() { + return + } + + k := o.flattenKey(key) + if !o.allowedMetric(k) { + return + } + + // Set value in global Gauge store. + o.gaugeStore.Set(k, float64(val), o.labelsToAttributes(labels)) + + o.mutex.Lock() + defer o.mutex.Unlock() + + // If instrument does not exist, create it and register callback to emit last value in global Gauge store. + if _, ok := o.gaugeInstruments[k]; !ok { + // The registration of a callback only needs to happen once, when the instrument is created. + // The callback will be triggered every export cycle for that metric. + // It must be explicitly de-registered to be removed (which we do not do), to ensure new gauge values are exported every cycle. + inst, err := (*o.meter).Float64ObservableGauge(k, otelmetric.WithFloat64Callback(o.gaugeStore.gaugeCallback(k))) + if err != nil { + o.logger.Error("Failed to create gauge instrument", "error", err) + return + } + o.gaugeInstruments[k] = inst + } +} + +// AddSampleWithLabels emits a Consul sample metric that gets registed by an OpenTelemetry Histogram instrument. +func (o *OTELSink) AddSampleWithLabels(key []string, val float32, labels []gometrics.Label) { + if o.cfgProvider.IsDisabled() { + return + } + + k := o.flattenKey(key) + if !o.allowedMetric(k) { + return + } + + o.mutex.Lock() + defer o.mutex.Unlock() + + inst, ok := o.histogramInstruments[k] + if !ok { + histogram, err := (*o.meter).Float64Histogram(k) + if err != nil { + o.logger.Error("Failed create histogram instrument", "error", err) + return + } + inst = histogram + o.histogramInstruments[k] = inst + } + + attrs := o.labelsToAttributes(labels) + inst.Record(context.TODO(), float64(val), otelmetric.WithAttributes(attrs...)) +} + +// IncrCounterWithLabels emits a Consul counter metric that gets registed by an OpenTelemetry Histogram instrument. +func (o *OTELSink) IncrCounterWithLabels(key []string, val float32, labels []gometrics.Label) { + if o.cfgProvider.IsDisabled() { + return + } + + k := o.flattenKey(key) + if !o.allowedMetric(k) { + return + } + + o.mutex.Lock() + defer o.mutex.Unlock() + + inst, ok := o.counterInstruments[k] + if !ok { + counter, err := (*o.meter).Float64Counter(k) + if err != nil { + o.logger.Error("Failed to create counter instrument:", "error", err) + return + } + + inst = counter + o.counterInstruments[k] = inst + } + + attrs := o.labelsToAttributes(labels) + inst.Add(context.TODO(), float64(val), otelmetric.WithAttributes(attrs...)) +} + +// EmitKey unsupported. +func (o *OTELSink) EmitKey(key []string, val float32) {} + +// flattenKey key along with its labels. +func (o *OTELSink) flattenKey(parts []string) string { + buf := &bytes.Buffer{} + joined := strings.Join(parts, ".") + + o.spaceReplacer.WriteString(buf, joined) + + return buf.String() +} + +// filter checks the filter allowlist, if it exists, to verify if this metric should be recorded. +func (o *OTELSink) allowedMetric(key string) bool { + if filters := o.cfgProvider.GetFilters(); filters != nil { + return filters.MatchString(key) + } + + return true +} + +// labelsToAttributes converts go metrics and provider labels into OTEL format []attributes.KeyValue +func (o *OTELSink) labelsToAttributes(goMetricsLabels []gometrics.Label) []attribute.KeyValue { + providerLabels := o.cfgProvider.GetLabels() + + length := len(goMetricsLabels) + len(providerLabels) + if length == 0 { + return []attribute.KeyValue{} + } + + attrs := make([]attribute.KeyValue, 0, length) + // Convert provider labels to OTEL attributes. + for _, label := range goMetricsLabels { + attrs = append(attrs, attribute.KeyValue{ + Key: attribute.Key(label.Name), + Value: attribute.StringValue(label.Value), + }) + } + + // Convert provider labels to OTEL attributes. + for k, v := range providerLabels { + attrs = append(attrs, attribute.KeyValue{ + Key: attribute.Key(k), + Value: attribute.StringValue(v), + }) + } + + return attrs +} diff --git a/agent/hcp/telemetry/otel_sink_test.go b/agent/hcp/telemetry/otel_sink_test.go new file mode 100644 index 00000000000..21987dccdcf --- /dev/null +++ b/agent/hcp/telemetry/otel_sink_test.go @@ -0,0 +1,588 @@ +package telemetry + +import ( + "context" + "fmt" + "regexp" + "sort" + "strings" + "sync" + "testing" + + gometrics "github.com/armon/go-metrics" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/resource" +) + +type mockConfigProvider struct { + filter *regexp.Regexp + labels map[string]string + disabled bool +} + +func (m *mockConfigProvider) GetLabels() map[string]string { + return m.labels +} + +func (m *mockConfigProvider) GetFilters() *regexp.Regexp { + return m.filter +} + +func (m *mockConfigProvider) IsDisabled() bool { + return m.disabled +} + +var ( + expectedResource = resource.NewSchemaless() + + attrs = attribute.NewSet(attribute.KeyValue{ + Key: attribute.Key("node_id"), + Value: attribute.StringValue("test"), + }) + attrsWithMetricLabel = attribute.NewSet(attribute.KeyValue{ + Key: attribute.Key("metric.label"), + Value: attribute.StringValue("test"), + }, attribute.KeyValue{ + Key: attribute.Key("node_id"), + Value: attribute.StringValue("test"), + }) + + expectedSinkMetrics = map[string]metricdata.Metrics{ + "consul.raft.leader": { + Name: "consul.raft.leader", + Description: "", + Unit: "", + Data: metricdata.Gauge[float64]{ + DataPoints: []metricdata.DataPoint[float64]{ + { + Attributes: attrs, + Value: float64(float32(0)), + }, + }, + }, + }, + "consul.autopilot.healthy": { + Name: "consul.autopilot.healthy", + Description: "", + Unit: "", + Data: metricdata.Gauge[float64]{ + DataPoints: []metricdata.DataPoint[float64]{ + { + Attributes: attrsWithMetricLabel, + Value: float64(float32(1.23)), + }, + }, + }, + }, + "consul.raft.state.leader": { + Name: "consul.raft.state.leader", + Description: "", + Unit: "", + Data: metricdata.Sum[float64]{ + DataPoints: []metricdata.DataPoint[float64]{ + { + Attributes: attrs, + Value: float64(float32(23.23)), + }, + }, + }, + }, + "consul.raft.apply": { + Name: "consul.raft.apply", + Description: "", + Unit: "", + Data: metricdata.Sum[float64]{ + DataPoints: []metricdata.DataPoint[float64]{ + { + Attributes: attrsWithMetricLabel, + Value: float64(float32(1.44)), + }, + }, + }, + }, + "consul.raft.leader.lastContact": { + Name: "consul.raft.leader.lastContact", + Description: "", + Unit: "", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attrs, + Count: 1, + Sum: float64(float32(45.32)), + Min: metricdata.NewExtrema(float64(float32(45.32))), + Max: metricdata.NewExtrema(float64(float32(45.32))), + }, + }, + }, + }, + "consul.raft.commitTime": { + Name: "consul.raft.commitTime", + Description: "", + Unit: "", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attrsWithMetricLabel, + Count: 1, + Sum: float64(float32(26.34)), + Min: metricdata.NewExtrema(float64(float32(26.34))), + Max: metricdata.NewExtrema(float64(float32(26.34))), + }, + }, + }, + }, + } +) + +func TestNewOTELSink(t *testing.T) { + t.Parallel() + for name, test := range map[string]struct { + wantErr string + opts *OTELSinkOpts + }{ + "failsWithEmptyReader": { + wantErr: "ferror: provide valid reader", + opts: &OTELSinkOpts{ + Reader: nil, + ConfigProvider: &mockConfigProvider{}, + }, + }, + "failsWithEmptyConfigProvider": { + wantErr: "ferror: provide valid config provider", + opts: &OTELSinkOpts{ + Reader: metric.NewManualReader(), + }, + }, + "success": { + opts: &OTELSinkOpts{ + Reader: metric.NewManualReader(), + ConfigProvider: &mockConfigProvider{}, + }, + }, + } { + test := test + t.Run(name, func(t *testing.T) { + t.Parallel() + sink, err := NewOTELSink(context.Background(), test.opts) + if test.wantErr != "" { + require.Error(t, err) + require.Contains(t, err.Error(), test.wantErr) + return + } + + require.NotNil(t, sink) + }) + } +} + +func TestOTELSink(t *testing.T) { + t.Parallel() + + // Manual reader outputs the aggregated metrics when reader.Collect is called. + reader := metric.NewManualReader() + + ctx := context.Background() + opts := &OTELSinkOpts{ + Reader: reader, + ConfigProvider: &mockConfigProvider{ + filter: regexp.MustCompile("raft|autopilot"), + labels: map[string]string{ + "node_id": "test", + }, + }, + } + + sink, err := NewOTELSink(ctx, opts) + require.NoError(t, err) + + labels := []gometrics.Label{ + { + Name: "metric.label", + Value: "test", + }, + } + + sink.SetGauge([]string{"test", "bad_filter", "gauge"}, float32(0)) + sink.SetGauge([]string{"consul", "raft", "leader"}, float32(0)) + sink.SetGaugeWithLabels([]string{"consul", "autopilot", "healthy"}, float32(1.23), labels) + + sink.IncrCounter([]string{"test", "bad_filter", "counter"}, float32(23.23)) + sink.IncrCounter([]string{"consul", "raft", "state", "leader"}, float32(23.23)) + sink.IncrCounterWithLabels([]string{"consul", "raft", "apply"}, float32(1.44), labels) + + sink.AddSample([]string{"test", "bad_filter", "sample"}, float32(45.32)) + sink.AddSample([]string{"consul", "raft", "leader", "lastContact"}, float32(45.32)) + sink.AddSampleWithLabels([]string{"consul", "raft", "commitTime"}, float32(26.34), labels) + + var collected metricdata.ResourceMetrics + err = reader.Collect(ctx, &collected) + require.NoError(t, err) + + isSame(t, expectedSinkMetrics, collected) +} + +func TestOTELSinkDisabled(t *testing.T) { + reader := metric.NewManualReader() + ctx := context.Background() + + sink, err := NewOTELSink(ctx, &OTELSinkOpts{ + ConfigProvider: &mockConfigProvider{ + filter: regexp.MustCompile("raft"), + disabled: true, + }, + Reader: reader, + }) + require.NoError(t, err) + + sink.SetGauge([]string{"consul", "raft", "gauge"}, 1) + sink.IncrCounter([]string{"consul", "raft", "counter"}, 1) + sink.AddSample([]string{"consul", "raft", "sample"}, 1) + + var collected metricdata.ResourceMetrics + err = reader.Collect(ctx, &collected) + require.NoError(t, err) + require.Empty(t, collected.ScopeMetrics) +} + +func TestLabelsToAttributes(t *testing.T) { + for name, test := range map[string]struct { + providerLabels map[string]string + goMetricsLabels []gometrics.Label + expectedOTELAttributes []attribute.KeyValue + }{ + "emptyLabels": { + expectedOTELAttributes: []attribute.KeyValue{}, + }, + "emptyGoMetricsLabels": { + providerLabels: map[string]string{ + "node_id": "test", + }, + expectedOTELAttributes: []attribute.KeyValue{ + { + Key: attribute.Key("node_id"), + Value: attribute.StringValue("test"), + }, + }, + }, + "emptyProviderLabels": { + goMetricsLabels: []gometrics.Label{ + { + Name: "server_type", + Value: "internal", + }, + }, + expectedOTELAttributes: []attribute.KeyValue{ + { + Key: attribute.Key("server_type"), + Value: attribute.StringValue("internal"), + }, + }, + }, + "combinedLabels": { + goMetricsLabels: []gometrics.Label{ + { + Name: "server_type", + Value: "internal", + }, + { + Name: "method", + Value: "get", + }, + }, + providerLabels: map[string]string{ + "node_id": "test", + "node_name": "labels_test", + }, + expectedOTELAttributes: []attribute.KeyValue{ + { + Key: attribute.Key("server_type"), + Value: attribute.StringValue("internal"), + }, + { + Key: attribute.Key("method"), + Value: attribute.StringValue("get"), + }, + { + Key: attribute.Key("node_id"), + Value: attribute.StringValue("test"), + }, + { + Key: attribute.Key("node_name"), + Value: attribute.StringValue("labels_test"), + }, + }, + }, + } { + test := test + t.Run(name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + opts := &OTELSinkOpts{ + Reader: metric.NewManualReader(), + ConfigProvider: &mockConfigProvider{ + filter: regexp.MustCompile("raft|autopilot"), + labels: test.providerLabels, + }, + } + sink, err := NewOTELSink(ctx, opts) + require.NoError(t, err) + + require.Equal(t, test.expectedOTELAttributes, sink.labelsToAttributes(test.goMetricsLabels)) + }) + } +} + +func TestOTELSinkFilters(t *testing.T) { + t.Parallel() + for name, tc := range map[string]struct { + cfgProvider ConfigProvider + expected bool + }{ + "emptyMatch": { + cfgProvider: &mockConfigProvider{}, + expected: true, + }, + "matchingFilter": { + cfgProvider: &mockConfigProvider{ + filter: regexp.MustCompile("raft"), + }, + expected: true, + }, + "mismatchFilter": {cfgProvider: &mockConfigProvider{ + filter: regexp.MustCompile("test"), + }}, + } { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + testMetricKey := "consul.raft" + s, err := NewOTELSink(context.Background(), &OTELSinkOpts{ + ConfigProvider: tc.cfgProvider, + Reader: metric.NewManualReader(), + }) + require.NoError(t, err) + require.Equal(t, tc.expected, s.allowedMetric(testMetricKey)) + }) + } +} + +func TestOTELSink_Race(t *testing.T) { + reader := metric.NewManualReader() + ctx := context.Background() + defaultLabels := map[string]string{ + "node_id": "test", + } + opts := &OTELSinkOpts{ + Reader: reader, + ConfigProvider: &mockConfigProvider{ + filter: regexp.MustCompile("test"), + labels: defaultLabels, + }, + } + + sink, err := NewOTELSink(context.Background(), opts) + require.NoError(t, err) + + samples := 100 + expectedMetrics := generateSamples(samples, defaultLabels) + wg := &sync.WaitGroup{} + errCh := make(chan error, samples) + for k, v := range expectedMetrics { + wg.Add(1) + go func(k string, v metricdata.Metrics) { + defer wg.Done() + performSinkOperation(sink, k, v, errCh) + }(k, v) + } + wg.Wait() + + require.Empty(t, errCh) + + var collected metricdata.ResourceMetrics + err = reader.Collect(ctx, &collected) + require.NoError(t, err) + + isSame(t, expectedMetrics, collected) +} + +// generateSamples generates n of each gauges, counter and histogram measurements to use for test purposes. +func generateSamples(n int, labels map[string]string) map[string]metricdata.Metrics { + generated := make(map[string]metricdata.Metrics, 3*n) + attrs := *attribute.EmptySet() + + kvs := make([]attribute.KeyValue, 0, len(labels)) + for k, v := range labels { + kvs = append(kvs, attribute.KeyValue{Key: attribute.Key(k), Value: attribute.StringValue(v)}) + } + if len(kvs) > 0 { + attrs = attribute.NewSet(kvs...) + } + + for i := 0; i < n; i++ { + v := 12.3 + k := fmt.Sprintf("consul.test.gauges.%d", i) + generated[k] = metricdata.Metrics{ + Name: k, + Data: metricdata.Gauge[float64]{ + DataPoints: []metricdata.DataPoint[float64]{ + { + Attributes: attrs, + Value: float64(float32(v)), + }, + }, + }, + } + } + + for i := 0; i < n; i++ { + v := 22.23 + k := fmt.Sprintf("consul.test.sum.%d", i) + generated[k] = metricdata.Metrics{ + Name: k, + Data: metricdata.Sum[float64]{ + DataPoints: []metricdata.DataPoint[float64]{ + { + Attributes: attrs, + Value: float64(float32(v)), + }, + }, + }, + } + + } + + for i := 0; i < n; i++ { + v := 13.24 + k := fmt.Sprintf("consul.test.hist.%d", i) + generated[k] = metricdata.Metrics{ + Name: k, + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attrs, + Sum: float64(float32(v)), + Max: metricdata.NewExtrema(float64(float32(v))), + Min: metricdata.NewExtrema(float64(float32(v))), + Count: 1, + }, + }, + }, + } + } + + return generated +} + +// performSinkOperation emits a measurement using the OTELSink and calls wg.Done() when completed. +func performSinkOperation(sink *OTELSink, k string, v metricdata.Metrics, errCh chan error) { + key := strings.Split(k, ".") + data := v.Data + switch data.(type) { + case metricdata.Gauge[float64]: + gauge, ok := data.(metricdata.Gauge[float64]) + if !ok { + errCh <- fmt.Errorf("unexpected type assertion error for key: %s", key) + } + sink.SetGauge(key, float32(gauge.DataPoints[0].Value)) + case metricdata.Sum[float64]: + sum, ok := data.(metricdata.Sum[float64]) + if !ok { + errCh <- fmt.Errorf("unexpected type assertion error for key: %s", key) + } + sink.IncrCounter(key, float32(sum.DataPoints[0].Value)) + case metricdata.Histogram[float64]: + hist, ok := data.(metricdata.Histogram[float64]) + if !ok { + errCh <- fmt.Errorf("unexpected type assertion error for key: %s", key) + } + sink.AddSample(key, float32(hist.DataPoints[0].Sum)) + } +} + +func isSame(t *testing.T, expectedMap map[string]metricdata.Metrics, actual metricdata.ResourceMetrics) { + // Validate resource + require.Equal(t, expectedResource, actual.Resource) + + // Validate Metrics + require.NotEmpty(t, actual.ScopeMetrics) + actualMetrics := actual.ScopeMetrics[0].Metrics + require.Equal(t, len(expectedMap), len(actualMetrics)) + + for _, actual := range actualMetrics { + name := actual.Name + expected, ok := expectedMap[actual.Name] + require.True(t, ok, "metric key %s should be in expectedMetrics map", name) + isSameMetrics(t, expected, actual) + } +} + +// compareMetrics verifies if two metricdata.Metric objects are equal by ignoring the time component. +// avoid duplicate datapoint values to ensure predictable order of sort. +func isSameMetrics(t *testing.T, expected metricdata.Metrics, actual metricdata.Metrics) { + require.Equal(t, expected.Name, actual.Name, "different .Name field") + require.Equal(t, expected.Description, actual.Description, "different .Description field") + require.Equal(t, expected.Unit, actual.Unit, "different .Unit field") + + switch expectedData := expected.Data.(type) { + case metricdata.Gauge[float64]: + actualData, ok := actual.Data.(metricdata.Gauge[float64]) + require.True(t, ok, "different metric types: expected metricdata.Gauge[float64]") + + isSameDataPoint(t, expectedData.DataPoints, actualData.DataPoints) + case metricdata.Sum[float64]: + actualData, ok := actual.Data.(metricdata.Sum[float64]) + require.True(t, ok, "different metric types: expected metricdata.Sum[float64]") + + isSameDataPoint(t, expectedData.DataPoints, actualData.DataPoints) + case metricdata.Histogram[float64]: + actualData, ok := actual.Data.(metricdata.Histogram[float64]) + require.True(t, ok, "different metric types: expected metricdata.Histogram") + + isSameHistogramData(t, expectedData.DataPoints, actualData.DataPoints) + } +} + +func isSameDataPoint(t *testing.T, expected []metricdata.DataPoint[float64], actual []metricdata.DataPoint[float64]) { + require.Equal(t, len(expected), len(actual), "different datapoints length") + + // Sort for predictable data in order of lowest value. + sort.Slice(expected, func(i, j int) bool { + return expected[i].Value < expected[j].Value + }) + sort.Slice(actual, func(i, j int) bool { + return expected[i].Value < expected[j].Value + }) + + // Only verify the value and attributes. + for i, dp := range expected { + currActual := actual[i] + require.Equal(t, dp.Value, currActual.Value, "different datapoint value") + require.Equal(t, dp.Attributes, currActual.Attributes, "different attributes") + } +} + +func isSameHistogramData(t *testing.T, expected []metricdata.HistogramDataPoint[float64], actual []metricdata.HistogramDataPoint[float64]) { + require.Equal(t, len(expected), len(actual), "different histogram datapoint length") + + // Sort for predictable data in order of lowest sum. + sort.Slice(expected, func(i, j int) bool { + return expected[i].Sum < expected[j].Sum + }) + sort.Slice(actual, func(i, j int) bool { + return expected[i].Sum < expected[j].Sum + }) + + // Only verify the value and the attributes. + for i, dp := range expected { + currActual := actual[i] + require.Equal(t, dp.Sum, currActual.Sum, "different histogram datapoint .Sum value") + require.Equal(t, dp.Max, currActual.Max, "different histogram datapoint .Max value") + require.Equal(t, dp.Min, currActual.Min, "different histogram datapoint .Min value") + require.Equal(t, dp.Count, currActual.Count, "different histogram datapoint .Count value") + require.Equal(t, dp.Attributes, currActual.Attributes, "different attributes") + } +} diff --git a/agent/hcp/telemetry/otlp_transform.go b/agent/hcp/telemetry/otlp_transform.go new file mode 100644 index 00000000000..bbe6396ad65 --- /dev/null +++ b/agent/hcp/telemetry/otlp_transform.go @@ -0,0 +1,186 @@ +package telemetry + +import ( + "errors" + "fmt" + + goMetrics "github.com/armon/go-metrics" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + cpb "go.opentelemetry.io/proto/otlp/common/v1" + mpb "go.opentelemetry.io/proto/otlp/metrics/v1" + rpb "go.opentelemetry.io/proto/otlp/resource/v1" +) + +var ( + errAggregaton = errors.New("unsupported aggregation") + errTemporality = errors.New("unsupported temporality") +) + +// isEmpty verifies if the given OTLP protobuf metrics contains metric data. +// isEmpty returns true if no ScopeMetrics exist or all metrics within ScopeMetrics are empty. +func isEmpty(rm *mpb.ResourceMetrics) bool { + // No ScopeMetrics + if len(rm.ScopeMetrics) == 0 { + return true + } + + // If any inner metrics contain data, return false. + for _, v := range rm.ScopeMetrics { + if len(v.Metrics) != 0 { + return false + } + } + + // All inner metrics are empty. + return true +} + +// TransformOTLP returns an OTLP ResourceMetrics generated from OTEL metrics. If rm +// contains invalid ScopeMetrics, an error will be returned along with an OTLP +// ResourceMetrics that contains partial OTLP ScopeMetrics. +func transformOTLP(rm *metricdata.ResourceMetrics) *mpb.ResourceMetrics { + sms := scopeMetricsToPB(rm.ScopeMetrics) + return &mpb.ResourceMetrics{ + Resource: &rpb.Resource{ + Attributes: attributesToPB(rm.Resource.Iter()), + }, + ScopeMetrics: sms, + } +} + +// scopeMetrics returns a slice of OTLP ScopeMetrics. +func scopeMetricsToPB(scopeMetrics []metricdata.ScopeMetrics) []*mpb.ScopeMetrics { + out := make([]*mpb.ScopeMetrics, 0, len(scopeMetrics)) + for _, sm := range scopeMetrics { + ms := metricsToPB(sm.Metrics) + out = append(out, &mpb.ScopeMetrics{ + Scope: &cpb.InstrumentationScope{ + Name: sm.Scope.Name, + Version: sm.Scope.Version, + }, + Metrics: ms, + }) + } + return out +} + +// metrics returns a slice of OTLP Metric generated from OTEL metrics sdk ones. +func metricsToPB(metrics []metricdata.Metrics) []*mpb.Metric { + out := make([]*mpb.Metric, 0, len(metrics)) + for _, m := range metrics { + o, err := metricTypeToPB(m) + if err != nil { + goMetrics.IncrCounter(internalMetricTransformFailure, 1) + continue + } + out = append(out, o) + } + return out +} + +// metricType identifies the instrument type and converts it to OTLP format. +// only float64 values are accepted since the go metrics sink only receives float64 values. +func metricTypeToPB(m metricdata.Metrics) (*mpb.Metric, error) { + out := &mpb.Metric{ + Name: m.Name, + Description: m.Description, + Unit: m.Unit, + } + switch a := m.Data.(type) { + case metricdata.Gauge[float64]: + out.Data = &mpb.Metric_Gauge{ + Gauge: &mpb.Gauge{ + DataPoints: dataPointsToPB(a.DataPoints), + }, + } + case metricdata.Sum[float64]: + if a.Temporality != metricdata.CumulativeTemporality { + return out, fmt.Errorf("failed to convert metric to otel format: %w: %T", errTemporality, a) + } + out.Data = &mpb.Metric_Sum{ + Sum: &mpb.Sum{ + AggregationTemporality: mpb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMULATIVE, + IsMonotonic: a.IsMonotonic, + DataPoints: dataPointsToPB(a.DataPoints), + }, + } + case metricdata.Histogram[float64]: + if a.Temporality != metricdata.CumulativeTemporality { + return out, fmt.Errorf("failed to convert metric to otel format: %w: %T", errTemporality, a) + } + out.Data = &mpb.Metric_Histogram{ + Histogram: &mpb.Histogram{ + AggregationTemporality: mpb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMULATIVE, + DataPoints: histogramDataPointsToPB(a.DataPoints), + }, + } + default: + return out, fmt.Errorf("failed to convert metric to otel format: %w: %T", errAggregaton, a) + } + return out, nil +} + +// DataPoints returns a slice of OTLP NumberDataPoint generated from OTEL metrics sdk ones. +func dataPointsToPB(dataPoints []metricdata.DataPoint[float64]) []*mpb.NumberDataPoint { + out := make([]*mpb.NumberDataPoint, 0, len(dataPoints)) + for _, dp := range dataPoints { + ndp := &mpb.NumberDataPoint{ + Attributes: attributesToPB(dp.Attributes.Iter()), + StartTimeUnixNano: uint64(dp.StartTime.UnixNano()), + TimeUnixNano: uint64(dp.Time.UnixNano()), + } + + ndp.Value = &mpb.NumberDataPoint_AsDouble{ + AsDouble: dp.Value, + } + out = append(out, ndp) + } + return out +} + +// HistogramDataPoints returns a slice of OTLP HistogramDataPoint from OTEL metrics sdk ones. +func histogramDataPointsToPB(dataPoints []metricdata.HistogramDataPoint[float64]) []*mpb.HistogramDataPoint { + out := make([]*mpb.HistogramDataPoint, 0, len(dataPoints)) + for _, dp := range dataPoints { + sum := dp.Sum + hdp := &mpb.HistogramDataPoint{ + Attributes: attributesToPB(dp.Attributes.Iter()), + StartTimeUnixNano: uint64(dp.StartTime.UnixNano()), + TimeUnixNano: uint64(dp.Time.UnixNano()), + Count: dp.Count, + Sum: &sum, + BucketCounts: dp.BucketCounts, + ExplicitBounds: dp.Bounds, + } + if v, ok := dp.Min.Value(); ok { + hdp.Min = &v + } + if v, ok := dp.Max.Value(); ok { + hdp.Max = &v + } + out = append(out, hdp) + } + return out +} + +// attributes transforms items of an attribute iterator into OTLP key-values. +// Currently, labels are only key-value pairs. +func attributesToPB(iter attribute.Iterator) []*cpb.KeyValue { + l := iter.Len() + if iter.Len() == 0 { + return nil + } + + out := make([]*cpb.KeyValue, 0, l) + for iter.Next() { + kv := iter.Attribute() + av := &cpb.AnyValue{ + Value: &cpb.AnyValue_StringValue{ + StringValue: kv.Value.AsString(), + }, + } + out = append(out, &cpb.KeyValue{Key: string(kv.Key), Value: av}) + } + return out +} diff --git a/agent/hcp/telemetry/otlp_transform_test.go b/agent/hcp/telemetry/otlp_transform_test.go new file mode 100644 index 00000000000..fda64c8c634 --- /dev/null +++ b/agent/hcp/telemetry/otlp_transform_test.go @@ -0,0 +1,342 @@ +package telemetry + +import ( + "strings" + "testing" + "time" + + "github.com/armon/go-metrics" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/instrumentation" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.17.0" + cpb "go.opentelemetry.io/proto/otlp/common/v1" + mpb "go.opentelemetry.io/proto/otlp/metrics/v1" + rpb "go.opentelemetry.io/proto/otlp/resource/v1" +) + +var ( + // Common attributes for test cases. + start = time.Date(2000, time.January, 01, 0, 0, 0, 0, time.FixedZone("GMT", 0)) + end = start.Add(30 * time.Second) + + alice = attribute.NewSet(attribute.String("user", "alice")) + bob = attribute.NewSet(attribute.String("user", "bob")) + + pbAlice = &cpb.KeyValue{Key: "user", Value: &cpb.AnyValue{ + Value: &cpb.AnyValue_StringValue{StringValue: "alice"}, + }} + pbBob = &cpb.KeyValue{Key: "user", Value: &cpb.AnyValue{ + Value: &cpb.AnyValue_StringValue{StringValue: "bob"}, + }} + + // DataPoint test case : Histogram Datapoints (Histogram) + minA, maxA, sumA = 2.0, 4.0, 90.0 + minB, maxB, sumB = 4.0, 150.0, 234.0 + inputHDP = []metricdata.HistogramDataPoint[float64]{{ + Attributes: alice, + StartTime: start, + Time: end, + Count: 30, + Bounds: []float64{1, 5}, + BucketCounts: []uint64{0, 30, 0}, + Min: metricdata.NewExtrema(minA), + Max: metricdata.NewExtrema(maxA), + Sum: sumA, + }, { + Attributes: bob, + StartTime: start, + Time: end, + Count: 3, + Bounds: []float64{1, 5}, + BucketCounts: []uint64{0, 1, 2}, + Min: metricdata.NewExtrema(minB), + Max: metricdata.NewExtrema(maxB), + Sum: sumB, + }} + + expectedHDP = []*mpb.HistogramDataPoint{{ + Attributes: []*cpb.KeyValue{pbAlice}, + StartTimeUnixNano: uint64(start.UnixNano()), + TimeUnixNano: uint64(end.UnixNano()), + Count: 30, + Sum: &sumA, + ExplicitBounds: []float64{1, 5}, + BucketCounts: []uint64{0, 30, 0}, + Min: &minA, + Max: &maxA, + }, { + Attributes: []*cpb.KeyValue{pbBob}, + StartTimeUnixNano: uint64(start.UnixNano()), + TimeUnixNano: uint64(end.UnixNano()), + Count: 3, + Sum: &sumB, + ExplicitBounds: []float64{1, 5}, + BucketCounts: []uint64{0, 1, 2}, + Min: &minB, + Max: &maxB, + }} + // DataPoint test case : Number Datapoints (Gauge / Counter) + inputDP = []metricdata.DataPoint[float64]{ + {Attributes: alice, StartTime: start, Time: end, Value: 1.0}, + {Attributes: bob, StartTime: start, Time: end, Value: 2.0}, + } + + expectedDP = []*mpb.NumberDataPoint{ + { + Attributes: []*cpb.KeyValue{pbAlice}, + StartTimeUnixNano: uint64(start.UnixNano()), + TimeUnixNano: uint64(end.UnixNano()), + Value: &mpb.NumberDataPoint_AsDouble{AsDouble: 1.0}, + }, + { + Attributes: []*cpb.KeyValue{pbBob}, + StartTimeUnixNano: uint64(start.UnixNano()), + TimeUnixNano: uint64(end.UnixNano()), + Value: &mpb.NumberDataPoint_AsDouble{AsDouble: 2.0}, + }, + } + + invalidSumTemporality = metricdata.Metrics{ + Name: "invalid-sum", + Description: "Sum with invalid temporality", + Unit: "1", + Data: metricdata.Sum[float64]{ + Temporality: metricdata.DeltaTemporality, + IsMonotonic: false, + DataPoints: inputDP, + }, + } + + invalidSumAgg = metricdata.Metrics{ + Name: "unknown", + Description: "Unknown aggregation", + Unit: "1", + Data: metricdata.Sum[int64]{}, + } + + invalidHistTemporality = metricdata.Metrics{ + Name: "invalid-histogram", + Description: "Invalid histogram", + Unit: "1", + Data: metricdata.Histogram[float64]{ + Temporality: metricdata.DeltaTemporality, + DataPoints: inputHDP, + }, + } + + validFloat64Gauge = metricdata.Metrics{ + Name: "float64-gauge", + Description: "Gauge with float64 values", + Unit: "1", + Data: metricdata.Gauge[float64]{DataPoints: inputDP}, + } + + validFloat64Sum = metricdata.Metrics{ + Name: "float64-sum", + Description: "Sum with float64 values", + Unit: "1", + Data: metricdata.Sum[float64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: false, + DataPoints: inputDP, + }, + } + + validFloat64Hist = metricdata.Metrics{ + Name: "float64-histogram", + Description: "Histogram", + Unit: "1", + Data: metricdata.Histogram[float64]{ + Temporality: metricdata.CumulativeTemporality, + DataPoints: inputHDP, + }, + } + + // Metrics Test Case + // - 3 invalid metrics and 3 Valid to test filtering + // - 1 invalid metric type + // - 2 invalid cummulative temporalities (only cummulative supported) + // - 3 types (Gauge, Counter, and Histogram) supported + inputMetrics = []metricdata.Metrics{ + validFloat64Gauge, + validFloat64Sum, + validFloat64Hist, + invalidSumTemporality, + invalidHistTemporality, + invalidSumAgg, + } + + expectedMetrics = []*mpb.Metric{ + { + Name: "float64-gauge", + Description: "Gauge with float64 values", + Unit: "1", + Data: &mpb.Metric_Gauge{Gauge: &mpb.Gauge{DataPoints: expectedDP}}, + }, + { + Name: "float64-sum", + Description: "Sum with float64 values", + Unit: "1", + Data: &mpb.Metric_Sum{Sum: &mpb.Sum{ + AggregationTemporality: mpb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMULATIVE, + IsMonotonic: false, + DataPoints: expectedDP, + }}, + }, + { + Name: "float64-histogram", + Description: "Histogram", + Unit: "1", + Data: &mpb.Metric_Histogram{Histogram: &mpb.Histogram{ + AggregationTemporality: mpb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMULATIVE, + DataPoints: expectedHDP, + }}, + }, + } + + // ScopeMetrics Test Cases + inputScopeMetrics = []metricdata.ScopeMetrics{{ + Scope: instrumentation.Scope{ + Name: "test/code/path", + Version: "v0.1.0", + }, + Metrics: inputMetrics, + }} + + expectedScopeMetrics = []*mpb.ScopeMetrics{{ + Scope: &cpb.InstrumentationScope{ + Name: "test/code/path", + Version: "v0.1.0", + }, + Metrics: expectedMetrics, + }} + + // ResourceMetrics Test Cases + inputResourceMetrics = &metricdata.ResourceMetrics{ + Resource: resource.NewSchemaless( + semconv.ServiceName("test server"), + semconv.ServiceVersion("v0.1.0"), + ), + ScopeMetrics: inputScopeMetrics, + } + + expectedResourceMetrics = &mpb.ResourceMetrics{ + Resource: &rpb.Resource{ + Attributes: []*cpb.KeyValue{ + { + Key: "service.name", + Value: &cpb.AnyValue{ + Value: &cpb.AnyValue_StringValue{StringValue: "test server"}, + }, + }, + { + Key: "service.version", + Value: &cpb.AnyValue{ + Value: &cpb.AnyValue_StringValue{StringValue: "v0.1.0"}, + }, + }, + }, + }, + ScopeMetrics: expectedScopeMetrics, + } +) + +// TestTransformOTLP runs tests from the "bottom-up" of the metricdata data types. +func TestTransformOTLP(t *testing.T) { + t.Parallel() + // Histogram DataPoint Test Case (Histograms) + assert.Equal(t, expectedHDP, histogramDataPointsToPB(inputHDP)) + + // Number DataPoint Test Case (Counters / Gauges) + require.Equal(t, expectedDP, dataPointsToPB(inputDP)) + + // MetricType Error Test Cases + _, err := metricTypeToPB(invalidHistTemporality) + require.Error(t, err) + require.ErrorIs(t, err, errTemporality) + + _, err = metricTypeToPB(invalidSumTemporality) + require.Error(t, err) + require.ErrorIs(t, err, errTemporality) + + _, err = metricTypeToPB(invalidSumAgg) + require.Error(t, err) + require.ErrorIs(t, err, errAggregaton) + + // Metrics Test Case + m := metricsToPB(inputMetrics) + require.Equal(t, expectedMetrics, m) + require.Equal(t, len(expectedMetrics), 3) + + // Scope Metrics Test Case + sm := scopeMetricsToPB(inputScopeMetrics) + require.Equal(t, expectedScopeMetrics, sm) + + // // Resource Metrics Test Case + rm := transformOTLP(inputResourceMetrics) + require.Equal(t, expectedResourceMetrics, rm) +} + +// TestTransformOTLP_CustomMetrics tests that a custom metric (hcp.otel.transform.failure) is emitted +// when transform fails. This test cannot be run in parallel as the metrics.NewGlobal() +// sets a shared global sink. +func TestTransformOTLP_CustomMetrics(t *testing.T) { + for name, tc := range map[string]struct { + inputRM *metricdata.ResourceMetrics + expectedMetricCount int + }{ + "successNoMetric": { + inputRM: &metricdata.ResourceMetrics{ + // 3 valid metrics. + ScopeMetrics: []metricdata.ScopeMetrics{ + { + Metrics: []metricdata.Metrics{ + validFloat64Gauge, + validFloat64Hist, + validFloat64Sum, + }, + }, + }, + }, + }, + "failureEmitsMetric": { + // inputScopeMetrics contains 3 bad metrics. + inputRM: inputResourceMetrics, + expectedMetricCount: 3, + }, + } { + tc := tc + t.Run(name, func(t *testing.T) { + // Init global sink. + serviceName := "test.transform" + cfg := metrics.DefaultConfig(serviceName) + cfg.EnableHostname = false + + sink := metrics.NewInmemSink(10*time.Second, 10*time.Second) + metrics.NewGlobal(cfg, sink) + + // Perform operation that emits metric. + transformOTLP(tc.inputRM) + + // Collect sink metrics. + intervals := sink.Data() + require.Len(t, intervals, 1) + key := serviceName + "." + strings.Join(internalMetricTransformFailure, ".") + sv := intervals[0].Counters[key] + + if tc.expectedMetricCount == 0 { + require.Empty(t, sv) + return + } + + // Verify count for transform failure metric. + require.NotNil(t, sv) + require.NotNil(t, sv.AggregateSample) + require.Equal(t, 3, sv.AggregateSample.Count) + }) + } +} diff --git a/agent/hcp/telemetry_provider.go b/agent/hcp/telemetry_provider.go new file mode 100644 index 00000000000..84b8ae88f33 --- /dev/null +++ b/agent/hcp/telemetry_provider.go @@ -0,0 +1,182 @@ +package hcp + +import ( + "context" + "net/url" + "regexp" + "sync" + "time" + + "github.com/armon/go-metrics" + "github.com/go-openapi/runtime" + "github.com/hashicorp/go-hclog" + + "github.com/hashicorp/consul/agent/hcp/client" + "github.com/hashicorp/consul/agent/hcp/telemetry" +) + +var ( + // internalMetricRefreshFailure is a metric to monitor refresh failures. + internalMetricRefreshFailure []string = []string{"hcp", "telemetry_config_provider", "refresh", "failure"} + // internalMetricRefreshSuccess is a metric to monitor refresh successes. + internalMetricRefreshSuccess []string = []string{"hcp", "telemetry_config_provider", "refresh", "success"} + // defaultTelemetryConfigRefreshInterval is a default fallback in case the first HCP fetch fails. + defaultTelemetryConfigRefreshInterval = 1 * time.Minute +) + +// Ensure hcpProviderImpl implements telemetry provider interfaces. +var _ telemetry.ConfigProvider = &hcpProviderImpl{} +var _ telemetry.EndpointProvider = &hcpProviderImpl{} + +// hcpProviderImpl holds telemetry configuration and settings for continuous fetch of new config from HCP. +// it updates configuration, if changes are detected. +type hcpProviderImpl struct { + // cfg holds configuration that can be dynamically updated. + cfg *dynamicConfig + + // A reader-writer mutex is used as the provider is read heavy. + // OTEL components access telemetryConfig during metrics collection and export (read). + // Meanwhile, config is only updated when there are changes (write). + rw sync.RWMutex + // hcpClient is an authenticated client used to make HTTP requests to HCP. + hcpClient client.Client +} + +// dynamicConfig is a set of configurable settings for metrics collection, processing and export. +// fields MUST be exported to compute hash for equals method. +type dynamicConfig struct { + disabled bool + endpoint *url.URL + labels map[string]string + filters *regexp.Regexp + // refreshInterval controls the interval at which configuration is fetched from HCP to refresh config. + refreshInterval time.Duration +} + +// defaultDisabledCfg disables metric collection and contains default config values. +func defaultDisabledCfg() *dynamicConfig { + return &dynamicConfig{ + labels: map[string]string{}, + filters: client.DefaultMetricFilters, + refreshInterval: defaultTelemetryConfigRefreshInterval, + endpoint: nil, + disabled: true, + } +} + +// NewHCPProvider initializes and starts a HCP Telemetry provider. +func NewHCPProvider(ctx context.Context, hcpClient client.Client) *hcpProviderImpl { + h := &hcpProviderImpl{ + // Initialize with default config values. + cfg: defaultDisabledCfg(), + hcpClient: hcpClient, + } + + go h.run(ctx) + + return h +} + +// run continously checks for updates to the telemetry configuration by making a request to HCP. +func (h *hcpProviderImpl) run(ctx context.Context) { + // Try to initialize config once before starting periodic fetch. + h.updateConfig(ctx) + + ticker := time.NewTicker(h.cfg.refreshInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if newRefreshInterval := h.updateConfig(ctx); newRefreshInterval > 0 { + ticker.Reset(newRefreshInterval) + } + case <-ctx.Done(): + return + } + } +} + +// updateConfig makes a HTTP request to HCP to update metrics configuration held in the provider. +func (h *hcpProviderImpl) updateConfig(ctx context.Context) time.Duration { + logger := hclog.FromContext(ctx).Named("telemetry_config_provider") + + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + telemetryCfg, err := h.hcpClient.FetchTelemetryConfig(ctx) + if err != nil { + // Only disable metrics on 404 or 401 to handle the case of an unlinked cluster. + // For other errors such as 5XX ones, we continue metrics collection, as these are potentially transient server-side errors. + apiErr, ok := err.(*runtime.APIError) + if ok && apiErr.IsClientError() { + disabledMetricsCfg := defaultDisabledCfg() + h.modifyDynamicCfg(disabledMetricsCfg) + return disabledMetricsCfg.refreshInterval + } + + logger.Error("failed to fetch telemetry config from HCP", "error", err) + metrics.IncrCounter(internalMetricRefreshFailure, 1) + return 0 + } + + // newRefreshInterval of 0 or less can cause ticker Reset() panic. + newRefreshInterval := telemetryCfg.RefreshConfig.RefreshInterval + if newRefreshInterval <= 0 { + logger.Error("invalid refresh interval duration", "refreshInterval", newRefreshInterval) + metrics.IncrCounter(internalMetricRefreshFailure, 1) + return 0 + } + + newCfg := &dynamicConfig{ + filters: telemetryCfg.MetricsConfig.Filters, + endpoint: telemetryCfg.MetricsConfig.Endpoint, + labels: telemetryCfg.MetricsConfig.Labels, + refreshInterval: telemetryCfg.RefreshConfig.RefreshInterval, + disabled: telemetryCfg.MetricsConfig.Disabled, + } + + h.modifyDynamicCfg(newCfg) + + return newCfg.refreshInterval +} + +// modifyDynamicCfg acquires a write lock to update new configuration and emits a success metric. +func (h *hcpProviderImpl) modifyDynamicCfg(newCfg *dynamicConfig) { + h.rw.Lock() + h.cfg = newCfg + h.rw.Unlock() + + metrics.IncrCounter(internalMetricRefreshSuccess, 1) +} + +// GetEndpoint acquires a read lock to return endpoint configuration for consumers. +func (h *hcpProviderImpl) GetEndpoint() *url.URL { + h.rw.RLock() + defer h.rw.RUnlock() + + return h.cfg.endpoint +} + +// GetFilters acquires a read lock to return filters configuration for consumers. +func (h *hcpProviderImpl) GetFilters() *regexp.Regexp { + h.rw.RLock() + defer h.rw.RUnlock() + + return h.cfg.filters +} + +// GetLabels acquires a read lock to return labels configuration for consumers. +func (h *hcpProviderImpl) GetLabels() map[string]string { + h.rw.RLock() + defer h.rw.RUnlock() + + return h.cfg.labels +} + +// IsDisabled acquires a read lock and return true if metrics are enabled. +func (h *hcpProviderImpl) IsDisabled() bool { + h.rw.RLock() + defer h.rw.RUnlock() + + return h.cfg.disabled +} diff --git a/agent/hcp/telemetry_provider_test.go b/agent/hcp/telemetry_provider_test.go new file mode 100644 index 00000000000..5eb3c61c45c --- /dev/null +++ b/agent/hcp/telemetry_provider_test.go @@ -0,0 +1,428 @@ +package hcp + +import ( + "context" + "errors" + "fmt" + "net/url" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/armon/go-metrics" + "github.com/go-openapi/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/hashicorp/consul/agent/hcp/client" +) + +const ( + testRefreshInterval = 100 * time.Millisecond + testSinkServiceName = "test.telemetry_config_provider" + testRaceWriteSampleCount = 100 + testRaceReadSampleCount = 5000 +) + +var ( + // Test constants to verify inmem sink metrics. + testMetricKeyFailure = testSinkServiceName + "." + strings.Join(internalMetricRefreshFailure, ".") + testMetricKeySuccess = testSinkServiceName + "." + strings.Join(internalMetricRefreshSuccess, ".") +) + +type testConfig struct { + filters string + endpoint string + labels map[string]string + refreshInterval time.Duration + disabled bool +} + +func TestNewTelemetryConfigProvider_DefaultConfig(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Initialize new provider, but fail all HCP fetches. + mc := client.NewMockClient(t) + mc.EXPECT().FetchTelemetryConfig(mock.Anything).Return(nil, errors.New("failed to fetch config")) + + provider := NewHCPProvider(ctx, mc) + provider.updateConfig(ctx) + + // Assert provider has default configuration and metrics processing is disabled. + defaultCfg := &dynamicConfig{ + labels: map[string]string{}, + filters: client.DefaultMetricFilters, + refreshInterval: defaultTelemetryConfigRefreshInterval, + endpoint: nil, + disabled: true, + } + require.Equal(t, defaultCfg, provider.cfg) +} + +func TestTelemetryConfigProvider_UpdateConfig(t *testing.T) { + for name, tc := range map[string]struct { + mockExpect func(*client.MockClient) + metricKey string + initCfg *dynamicConfig + expected *dynamicConfig + expectedInterval time.Duration + }{ + "noChanges": { + initCfg: testDynamicCfg(&testConfig{ + endpoint: "http://test.com/v1/metrics", + filters: "test", + labels: map[string]string{ + "test_label": "123", + }, + refreshInterval: testRefreshInterval, + }), + mockExpect: func(m *client.MockClient) { + mockCfg, _ := testTelemetryCfg(&testConfig{ + endpoint: "http://test.com/v1/metrics", + filters: "test", + labels: map[string]string{ + "test_label": "123", + }, + refreshInterval: testRefreshInterval, + }) + m.EXPECT().FetchTelemetryConfig(mock.Anything).Return(mockCfg, nil) + }, + expected: testDynamicCfg(&testConfig{ + endpoint: "http://test.com/v1/metrics", + labels: map[string]string{ + "test_label": "123", + }, + filters: "test", + refreshInterval: testRefreshInterval, + }), + metricKey: testMetricKeySuccess, + expectedInterval: testRefreshInterval, + }, + "newConfig": { + initCfg: testDynamicCfg(&testConfig{ + endpoint: "http://test.com/v1/metrics", + filters: "test", + labels: map[string]string{ + "test_label": "123", + }, + refreshInterval: 2 * time.Second, + }), + mockExpect: func(m *client.MockClient) { + mockCfg, _ := testTelemetryCfg(&testConfig{ + endpoint: "http://newendpoint/v1/metrics", + filters: "consul", + labels: map[string]string{ + "new_label": "1234", + }, + refreshInterval: 2 * time.Second, + }) + m.EXPECT().FetchTelemetryConfig(mock.Anything).Return(mockCfg, nil) + }, + expected: testDynamicCfg(&testConfig{ + endpoint: "http://newendpoint/v1/metrics", + filters: "consul", + labels: map[string]string{ + "new_label": "1234", + }, + refreshInterval: 2 * time.Second, + }), + expectedInterval: 2 * time.Second, + metricKey: testMetricKeySuccess, + }, + "newConfigMetricsDisabled": { + initCfg: testDynamicCfg(&testConfig{ + endpoint: "http://test.com/v1/metrics", + filters: "test", + labels: map[string]string{ + "test_label": "123", + }, + refreshInterval: 2 * time.Second, + }), + mockExpect: func(m *client.MockClient) { + mockCfg, _ := testTelemetryCfg(&testConfig{ + endpoint: "", + filters: "consul", + labels: map[string]string{ + "new_label": "1234", + }, + refreshInterval: 2 * time.Second, + disabled: true, + }) + m.EXPECT().FetchTelemetryConfig(mock.Anything).Return(mockCfg, nil) + }, + expected: testDynamicCfg(&testConfig{ + endpoint: "", + filters: "consul", + labels: map[string]string{ + "new_label": "1234", + }, + refreshInterval: 2 * time.Second, + disabled: true, + }), + metricKey: testMetricKeySuccess, + expectedInterval: 2 * time.Second, + }, + "sameConfigInvalidRefreshInterval": { + initCfg: testDynamicCfg(&testConfig{ + endpoint: "http://test.com/v1/metrics", + filters: "test", + labels: map[string]string{ + "test_label": "123", + }, + refreshInterval: testRefreshInterval, + }), + mockExpect: func(m *client.MockClient) { + mockCfg, _ := testTelemetryCfg(&testConfig{ + refreshInterval: 0 * time.Second, + }) + m.EXPECT().FetchTelemetryConfig(mock.Anything).Return(mockCfg, nil) + }, + expected: testDynamicCfg(&testConfig{ + endpoint: "http://test.com/v1/metrics", + labels: map[string]string{ + "test_label": "123", + }, + filters: "test", + refreshInterval: testRefreshInterval, + }), + metricKey: testMetricKeyFailure, + expectedInterval: 0, + }, + "sameConfigHCPClientFailure": { + initCfg: testDynamicCfg(&testConfig{ + endpoint: "http://test.com/v1/metrics", + filters: "test", + labels: map[string]string{ + "test_label": "123", + }, + refreshInterval: testRefreshInterval, + }), + mockExpect: func(m *client.MockClient) { + m.EXPECT().FetchTelemetryConfig(mock.Anything).Return(nil, fmt.Errorf("failure")) + }, + expected: testDynamicCfg(&testConfig{ + endpoint: "http://test.com/v1/metrics", + filters: "test", + labels: map[string]string{ + "test_label": "123", + }, + refreshInterval: testRefreshInterval, + }), + metricKey: testMetricKeyFailure, + expectedInterval: 0, + }, + "disableMetrics404": { + initCfg: testDynamicCfg(&testConfig{ + endpoint: "http://test.com/v1/metrics", + filters: "test", + labels: map[string]string{ + "test_label": "123", + }, + refreshInterval: testRefreshInterval, + }), + mockExpect: func(m *client.MockClient) { + err := runtime.NewAPIError("404 failure", nil, 404) + m.EXPECT().FetchTelemetryConfig(mock.Anything).Return(nil, err) + }, + expected: defaultDisabledCfg(), + metricKey: testMetricKeySuccess, + expectedInterval: defaultTelemetryConfigRefreshInterval, + }, + } { + tc := tc + t.Run(name, func(t *testing.T) { + sink := initGlobalSink() + mockClient := client.NewMockClient(t) + tc.mockExpect(mockClient) + + provider := &hcpProviderImpl{ + hcpClient: mockClient, + cfg: tc.initCfg, + } + + newInterval := provider.updateConfig(context.Background()) + require.Equal(t, tc.expectedInterval, newInterval) + + // Verify endpoint provider returns correct config values. + require.Equal(t, tc.expected.endpoint, provider.GetEndpoint()) + require.Equal(t, tc.expected.filters, provider.GetFilters()) + require.Equal(t, tc.expected.labels, provider.GetLabels()) + require.Equal(t, tc.expected.disabled, provider.IsDisabled()) + + // Verify count for transform success metric. + interval := sink.Data()[0] + require.NotNil(t, interval, 1) + sv := interval.Counters[tc.metricKey] + assert.NotNil(t, sv.AggregateSample) + require.Equal(t, sv.AggregateSample.Count, 1) + }) + } +} + +// mockRaceClient is a mock HCP client that fetches TelemetryConfig. +// The mock TelemetryConfig returned can be manually updated at any time. +// It manages concurrent read/write access to config with a sync.RWMutex. +type mockRaceClient struct { + client.Client + cfg *client.TelemetryConfig + rw sync.RWMutex +} + +// updateCfg acquires a write lock and updates client config to a new value givent a count. +func (m *mockRaceClient) updateCfg(count int) (*client.TelemetryConfig, error) { + m.rw.Lock() + defer m.rw.Unlock() + + labels := map[string]string{fmt.Sprintf("label_%d", count): fmt.Sprintf("value_%d", count)} + + filters, err := regexp.Compile(fmt.Sprintf("consul_filter_%d", count)) + if err != nil { + return nil, err + } + + endpoint, err := url.Parse(fmt.Sprintf("http://consul-endpoint-%d.com", count)) + if err != nil { + return nil, err + } + + cfg := &client.TelemetryConfig{ + MetricsConfig: &client.MetricsConfig{ + Filters: filters, + Endpoint: endpoint, + Labels: labels, + }, + RefreshConfig: &client.RefreshConfig{ + RefreshInterval: testRefreshInterval, + }, + } + m.cfg = cfg + + return cfg, nil +} + +// FetchTelemetryConfig returns the current config held by the mockRaceClient. +func (m *mockRaceClient) FetchTelemetryConfig(ctx context.Context) (*client.TelemetryConfig, error) { + m.rw.RLock() + defer m.rw.RUnlock() + + return m.cfg, nil +} + +func TestTelemetryConfigProvider_Race(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + initCfg, err := testTelemetryCfg(&testConfig{ + endpoint: "test.com", + filters: "test", + labels: map[string]string{"test_label": "test_value"}, + refreshInterval: testRefreshInterval, + }) + require.NoError(t, err) + + m := &mockRaceClient{ + cfg: initCfg, + } + + // Start the provider goroutine, which fetches client TelemetryConfig every RefreshInterval. + provider := NewHCPProvider(ctx, m) + + for count := 0; count < testRaceWriteSampleCount; count++ { + // Force a TelemetryConfig value change in the mockRaceClient. + newCfg, err := m.updateCfg(count) + require.NoError(t, err) + // Force provider to obtain new client TelemetryConfig immediately. + // This call is necessary to guarantee TelemetryConfig changes to assert on expected values below. + provider.updateConfig(context.Background()) + + // Start goroutines to access label configuration. + wg := &sync.WaitGroup{} + kickOff(wg, testRaceReadSampleCount, provider, func(provider *hcpProviderImpl) { + require.Equal(t, provider.GetLabels(), newCfg.MetricsConfig.Labels) + }) + + // Start goroutines to access endpoint configuration. + kickOff(wg, testRaceReadSampleCount, provider, func(provider *hcpProviderImpl) { + require.Equal(t, provider.GetFilters(), newCfg.MetricsConfig.Filters) + }) + + // Start goroutines to access filter configuration. + kickOff(wg, testRaceReadSampleCount, provider, func(provider *hcpProviderImpl) { + require.Equal(t, provider.GetEndpoint(), newCfg.MetricsConfig.Endpoint) + }) + + wg.Wait() + } +} + +func kickOff(wg *sync.WaitGroup, count int, provider *hcpProviderImpl, check func(cfgProvider *hcpProviderImpl)) { + for i := 0; i < count; i++ { + wg.Add(1) + go func() { + defer wg.Done() + check(provider) + }() + } +} + +// initGlobalSink is a helper function to initialize a Go metrics inmemsink. +func initGlobalSink() *metrics.InmemSink { + cfg := metrics.DefaultConfig(testSinkServiceName) + cfg.EnableHostname = false + + sink := metrics.NewInmemSink(10*time.Second, 10*time.Second) + metrics.NewGlobal(cfg, sink) + + return sink +} + +// testDynamicCfg converts testConfig inputs to a dynamicConfig to be used in tests. +func testDynamicCfg(testCfg *testConfig) *dynamicConfig { + filters, _ := regexp.Compile(testCfg.filters) + + var endpoint *url.URL + if testCfg.endpoint != "" { + endpoint, _ = url.Parse(testCfg.endpoint) + } + return &dynamicConfig{ + endpoint: endpoint, + filters: filters, + labels: testCfg.labels, + refreshInterval: testCfg.refreshInterval, + disabled: testCfg.disabled, + } +} + +// testTelemetryCfg converts testConfig inputs to a TelemetryConfig to be used in tests. +func testTelemetryCfg(testCfg *testConfig) (*client.TelemetryConfig, error) { + filters, err := regexp.Compile(testCfg.filters) + if err != nil { + return nil, err + } + + var endpoint *url.URL + if testCfg.endpoint != "" { + u, err := url.Parse(testCfg.endpoint) + if err != nil { + return nil, err + } + endpoint = u + } + + return &client.TelemetryConfig{ + MetricsConfig: &client.MetricsConfig{ + Endpoint: endpoint, + Filters: filters, + Labels: testCfg.labels, + Disabled: testCfg.disabled, + }, + RefreshConfig: &client.RefreshConfig{ + RefreshInterval: testCfg.refreshInterval, + }, + }, nil +} diff --git a/agent/hcp/testing.go b/agent/hcp/testing.go index e32602777da..10e6a1114a6 100644 --- a/agent/hcp/testing.go +++ b/agent/hcp/testing.go @@ -10,6 +10,7 @@ import ( "sync" "time" + hcpgnm "github.com/hashicorp/hcp-sdk-go/clients/cloud-global-network-manager-service/preview/2022-02-15/client/global_network_manager_service" gnmmod "github.com/hashicorp/hcp-sdk-go/clients/cloud-global-network-manager-service/preview/2022-02-15/models" "github.com/hashicorp/hcp-sdk-go/resource" ) @@ -60,7 +61,7 @@ func (s *MockHCPServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.mu.Lock() defer s.mu.Unlock() - if r.URL.Path == "/oauth/token" { + if r.URL.Path == "/oauth2/token" { mockTokenResponse(w) return } @@ -133,30 +134,31 @@ func enforceMethod(w http.ResponseWriter, r *http.Request, methods []string) boo } func mockTokenResponse(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - w.Write([]byte(`{access_token: "token", token_type: "Bearer"}`)) + + w.Write([]byte(`{"access_token": "token", "token_type": "Bearer"}`)) } func (s *MockHCPServer) handleStatus(r *http.Request, cluster resource.Resource) (interface{}, error) { - var req gnmmod.HashicorpCloudGlobalNetworkManager20220215AgentPushServerStateRequest + var req hcpgnm.AgentPushServerStateBody if err := json.NewDecoder(r.Body).Decode(&req); err != nil { return nil, err } - status := req.ServerState log.Printf("STATUS UPDATE: server=%s version=%s leader=%v hasLeader=%v healthy=%v tlsCertExpiryDays=%1.0f", - status.Name, - status.Version, - status.Raft.IsLeader, - status.Raft.KnownLeader, - status.Autopilot.Healthy, - time.Until(time.Time(status.TLS.CertExpiry)).Hours()/24, + req.ServerState.Name, + req.ServerState.Version, + req.ServerState.Raft.IsLeader, + req.ServerState.Raft.KnownLeader, + req.ServerState.Autopilot.Healthy, + time.Until(time.Time(req.ServerState.TLS.CertExpiry)).Hours()/24, ) - s.servers[status.Name] = &gnmmod.HashicorpCloudGlobalNetworkManager20220215Server{ - GossipPort: status.GossipPort, - ID: status.ID, - LanAddress: status.LanAddress, - Name: status.Name, - RPCPort: status.RPCPort, + s.servers[req.ServerState.Name] = &gnmmod.HashicorpCloudGlobalNetworkManager20220215Server{ + GossipPort: req.ServerState.GossipPort, + ID: req.ServerState.ID, + LanAddress: req.ServerState.LanAddress, + Name: req.ServerState.Name, + RPCPort: req.ServerState.RPCPort, } return "{}", nil } diff --git a/agent/http.go b/agent/http.go index cb2c11b59ed..b8135b548db 100644 --- a/agent/http.go +++ b/agent/http.go @@ -164,7 +164,7 @@ func (s *HTTPHandlers) ReloadConfig(newCfg *config.RuntimeConfig) error { // // The first call must not be concurrent with any other call. Subsequent calls // may be concurrent with HTTP requests since no state is modified. -func (s *HTTPHandlers) handler(enableDebug bool) http.Handler { +func (s *HTTPHandlers) handler() http.Handler { // Memoize multiple calls. if s.h != nil { return s.h @@ -207,7 +207,15 @@ func (s *HTTPHandlers) handler(enableDebug bool) http.Handler { // handlePProf takes the given pattern and pprof handler // and wraps it to add authorization and metrics handlePProf := func(pattern string, handler http.HandlerFunc) { + wrapper := func(resp http.ResponseWriter, req *http.Request) { + + // If enableDebug register wrapped pprof handlers + if !s.agent.enableDebug.Load() && s.checkACLDisabled() { + resp.WriteHeader(http.StatusNotFound) + return + } + var token string s.parseToken(req, &token) @@ -242,14 +250,11 @@ func (s *HTTPHandlers) handler(enableDebug bool) http.Handler { handleFuncMetrics(pattern, s.wrap(bound, methods)) } - // If enableDebug or ACL enabled, register wrapped pprof handlers - if enableDebug || !s.checkACLDisabled() { - handlePProf("/debug/pprof/", pprof.Index) - handlePProf("/debug/pprof/cmdline", pprof.Cmdline) - handlePProf("/debug/pprof/profile", pprof.Profile) - handlePProf("/debug/pprof/symbol", pprof.Symbol) - handlePProf("/debug/pprof/trace", pprof.Trace) - } + handlePProf("/debug/pprof/", pprof.Index) + handlePProf("/debug/pprof/cmdline", pprof.Cmdline) + handlePProf("/debug/pprof/profile", pprof.Profile) + handlePProf("/debug/pprof/symbol", pprof.Symbol) + handlePProf("/debug/pprof/trace", pprof.Trace) if s.IsUIEnabled() { // Note that we _don't_ support reloading ui_config.{enabled, content_dir, diff --git a/agent/http_oss.go b/agent/http_ce.go similarity index 99% rename from agent/http_oss.go rename to agent/http_ce.go index 94eb575c36e..04c14b7f320 100644 --- a/agent/http_oss.go +++ b/agent/http_ce.go @@ -36,7 +36,7 @@ func (s *HTTPHandlers) validateEnterpriseIntentionPartition(logName, partition s return nil } - // No special handling for wildcard namespaces as they are pointless in OSS. + // No special handling for wildcard namespaces as they are pointless in CE. return HTTPError{ StatusCode: http.StatusBadRequest, @@ -51,7 +51,7 @@ func (s *HTTPHandlers) validateEnterpriseIntentionNamespace(logName, ns string, return nil } - // No special handling for wildcard namespaces as they are pointless in OSS. + // No special handling for wildcard namespaces as they are pointless in CE. return HTTPError{ StatusCode: http.StatusBadRequest, diff --git a/agent/http_oss_test.go b/agent/http_ce_test.go similarity index 94% rename from agent/http_oss_test.go rename to agent/http_ce_test.go index 4bb392f6185..b6cb8fe2bbb 100644 --- a/agent/http_oss_test.go +++ b/agent/http_ce_test.go @@ -59,7 +59,7 @@ func newHttpClient(timeout time.Duration) *http.Client { } } -func TestHTTPAPI_MethodNotAllowed_OSS(t *testing.T) { +func TestHTTPAPI_MethodNotAllowed_CE(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } @@ -127,7 +127,7 @@ func TestHTTPAPI_MethodNotAllowed_OSS(t *testing.T) { } } -func TestHTTPAPI_OptionMethod_OSS(t *testing.T) { +func TestHTTPAPI_OptionMethod_CE(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } @@ -141,7 +141,9 @@ func TestHTTPAPI_OptionMethod_OSS(t *testing.T) { uri := fmt.Sprintf("http://%s%s", a.HTTPAddr(), path) req, _ := http.NewRequest("OPTIONS", uri, nil) resp := httptest.NewRecorder() - a.srv.handler(true).ServeHTTP(resp, req) + + a.enableDebug.Store(true) + a.srv.handler().ServeHTTP(resp, req) allMethods := append([]string{"OPTIONS"}, methods...) if resp.Code != http.StatusOK { @@ -167,7 +169,7 @@ func TestHTTPAPI_OptionMethod_OSS(t *testing.T) { } } -func TestHTTPAPI_AllowedNets_OSS(t *testing.T) { +func TestHTTPAPI_AllowedNets_CE(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } @@ -187,7 +189,8 @@ func TestHTTPAPI_AllowedNets_OSS(t *testing.T) { req, _ := http.NewRequest(method, uri, nil) req.RemoteAddr = "192.168.1.2:5555" resp := httptest.NewRecorder() - a.srv.handler(true).ServeHTTP(resp, req) + a.enableDebug.Store(true) + a.srv.handler().ServeHTTP(resp, req) require.Equal(t, http.StatusForbidden, resp.Code, "%s %s", method, path) }) diff --git a/agent/http_test.go b/agent/http_test.go index 39963be0417..5329f703749 100644 --- a/agent/http_test.go +++ b/agent/http_test.go @@ -11,6 +11,7 @@ import ( "net/http" "net/http/httptest" "net/netip" + "net/url" "os" "path/filepath" "runtime" @@ -140,6 +141,95 @@ func TestHTTPServer_UnixSocket_FileExists(t *testing.T) { } } +func TestHTTPSServer_UnixSocket(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + if runtime.GOOS == "windows" { + t.SkipNow() + } + + tempDir := testutil.TempDir(t, "consul") + socket := filepath.Join(tempDir, "test.sock") + + a := StartTestAgent(t, TestAgent{ + UseHTTPS: true, + HCL: ` + addresses { + https = "unix://` + socket + `" + } + unix_sockets { + mode = "0777" + } + tls { + defaults { + ca_file = "../test/client_certs/rootca.crt" + cert_file = "../test/client_certs/server.crt" + key_file = "../test/client_certs/server.key" + } + } + `, + }) + defer a.Shutdown() + + // Ensure the socket was created + if _, err := os.Stat(socket); err != nil { + t.Fatalf("err: %s", err) + } + + // Ensure the mode was set properly + fi, err := os.Stat(socket) + if err != nil { + t.Fatalf("err: %s", err) + } + if fi.Mode().String() != "Srwxrwxrwx" { + t.Fatalf("bad permissions: %s", fi.Mode()) + } + + // Make an HTTP/2-enabled client, using the API helpers to set + // up TLS to be as normal as possible for Consul. + tlscfg := &api.TLSConfig{ + Address: "consul.test", + KeyFile: "../test/client_certs/client.key", + CertFile: "../test/client_certs/client.crt", + CAFile: "../test/client_certs/rootca.crt", + } + tlsccfg, err := api.SetupTLSConfig(tlscfg) + if err != nil { + t.Fatalf("err: %v", err) + } + transport := api.DefaultConfig().Transport + transport.TLSHandshakeTimeout = 30 * time.Second + transport.TLSClientConfig = tlsccfg + if err := http2.ConfigureTransport(transport); err != nil { + t.Fatalf("err: %v", err) + } + transport.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", socket) + } + client := &http.Client{Transport: transport} + + u, err := url.Parse("https://unix" + socket) + if err != nil { + t.Fatalf("err: %s", err) + } + u.Path = "/v1/agent/self" + u.Scheme = "https" + resp, err := client.Get(u.String()) + if err != nil { + t.Fatalf("err: %s", err) + } + defer resp.Body.Close() + + if body, err := io.ReadAll(resp.Body); err != nil || len(body) == 0 { + t.Fatalf("bad: %s %v", body, err) + } else if !strings.Contains(string(body), "NodeName") { + t.Fatalf("NodeName not found in results: %s", string(body)) + } +} + func TestSetupHTTPServer_HTTP2(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") @@ -151,9 +241,13 @@ func TestSetupHTTPServer_HTTP2(t *testing.T) { a := StartTestAgent(t, TestAgent{ UseHTTPS: true, HCL: ` - key_file = "../test/client_certs/server.key" - cert_file = "../test/client_certs/server.crt" - ca_file = "../test/client_certs/rootca.crt" + tls { + defaults { + ca_file = "../test/client_certs/rootca.crt" + cert_file = "../test/client_certs/server.crt" + key_file = "../test/client_certs/server.key" + } + } `, }) defer a.Shutdown() @@ -191,7 +285,9 @@ func TestSetupHTTPServer_HTTP2(t *testing.T) { err = setupHTTPS(httpServer, noopConnState, time.Second) require.NoError(t, err) - srvHandler := a.srv.handler(true) + a.enableDebug.Store(true) + + srvHandler := a.srv.handler() mux, ok := srvHandler.(*wrappedMux) require.True(t, ok, "expected a *wrappedMux, got %T", handler) mux.mux.HandleFunc("/echo", handler) @@ -386,7 +482,8 @@ func TestHTTPAPI_Ban_Nonprintable_Characters(t *testing.T) { t.Fatal(err) } resp := httptest.NewRecorder() - a.srv.handler(true).ServeHTTP(resp, req) + a.enableDebug.Store(true) + a.srv.handler().ServeHTTP(resp, req) if got, want := resp.Code, http.StatusBadRequest; got != want { t.Fatalf("bad response code got %d want %d", got, want) } @@ -409,7 +506,8 @@ func TestHTTPAPI_Allow_Nonprintable_Characters_With_Flag(t *testing.T) { t.Fatal(err) } resp := httptest.NewRecorder() - a.srv.handler(true).ServeHTTP(resp, req) + a.enableDebug.Store(true) + a.srv.handler().ServeHTTP(resp, req) // Key doesn't actually exist so we should get 404 if got, want := resp.Code, http.StatusNotFound; got != want { t.Fatalf("bad response code got %d want %d", got, want) @@ -548,7 +646,8 @@ func requireHasHeadersSet(t *testing.T, a *TestAgent, path string) { resp := httptest.NewRecorder() req, _ := http.NewRequest("GET", path, nil) - a.srv.handler(true).ServeHTTP(resp, req) + a.enableDebug.Store(true) + a.srv.handler().ServeHTTP(resp, req) hdrs := resp.Header() require.Equal(t, "*", hdrs.Get("Access-Control-Allow-Origin"), @@ -609,14 +708,16 @@ func TestAcceptEncodingGzip(t *testing.T) { // negotiation, but since this call doesn't go through a real // transport, the header has to be set manually req.Header["Accept-Encoding"] = []string{"gzip"} - a.srv.handler(true).ServeHTTP(resp, req) + a.enableDebug.Store(true) + a.srv.handler().ServeHTTP(resp, req) require.Equal(t, 200, resp.Code) require.Equal(t, "", resp.Header().Get("Content-Encoding")) resp = httptest.NewRecorder() req, _ = http.NewRequest("GET", "/v1/kv/long", nil) req.Header["Accept-Encoding"] = []string{"gzip"} - a.srv.handler(true).ServeHTTP(resp, req) + a.enableDebug.Store(true) + a.srv.handler().ServeHTTP(resp, req) require.Equal(t, 200, resp.Code) require.Equal(t, "gzip", resp.Header().Get("Content-Encoding")) } @@ -962,8 +1063,9 @@ func TestHTTPServer_PProfHandlers_EnableDebug(t *testing.T) { resp := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/debug/pprof/profile?seconds=1", nil) + a.enableDebug.Store(true) httpServer := &HTTPHandlers{agent: a.Agent} - httpServer.handler(true).ServeHTTP(resp, req) + httpServer.handler().ServeHTTP(resp, req) require.Equal(t, http.StatusOK, resp.Code) } @@ -981,7 +1083,7 @@ func TestHTTPServer_PProfHandlers_DisableDebugNoACLs(t *testing.T) { req, _ := http.NewRequest("GET", "/debug/pprof/profile", nil) httpServer := &HTTPHandlers{agent: a.Agent} - httpServer.handler(false).ServeHTTP(resp, req) + httpServer.handler().ServeHTTP(resp, req) require.Equal(t, http.StatusNotFound, resp.Code) } @@ -1062,7 +1164,8 @@ func TestHTTPServer_PProfHandlers_ACLs(t *testing.T) { t.Run(fmt.Sprintf("case %d (%#v)", i, c), func(t *testing.T) { req, _ := http.NewRequest("GET", fmt.Sprintf("%s?token=%s", c.endpoint, c.token), nil) resp := httptest.NewRecorder() - a.srv.handler(true).ServeHTTP(resp, req) + a.enableDebug.Store(true) + a.srv.handler().ServeHTTP(resp, req) assert.Equal(t, c.code, resp.Code) }) } @@ -1372,7 +1475,8 @@ func TestEnableWebUI(t *testing.T) { req, _ := http.NewRequest("GET", "/ui/", nil) resp := httptest.NewRecorder() - a.srv.handler(true).ServeHTTP(resp, req) + a.enableDebug.Store(true) + a.srv.handler().ServeHTTP(resp, req) require.Equal(t, http.StatusOK, resp.Code) // Validate that it actually sent the index page we expect since an error @@ -1401,7 +1505,8 @@ func TestEnableWebUI(t *testing.T) { { req, _ := http.NewRequest("GET", "/ui/", nil) resp := httptest.NewRecorder() - a.srv.handler(true).ServeHTTP(resp, req) + a.enableDebug.Store(true) + a.srv.handler().ServeHTTP(resp, req) require.Equal(t, http.StatusOK, resp.Code) require.Contains(t, resp.Body.String(), ` **Note:** This endpoint will always evaluate intentions with `Permissions` -defined as _deny_ intentions during. This endpoint is only suited for -networking layer 4 (e.g. TCP) integration. +-> **Note:** This endpoint will always evaluate matching intentions with L7 `Permissions` defined as _deny_ intentions because there is no request to check against. For performance and reliability reasons it is desirable to implement intention enforcement by listing [intentions that match the @@ -633,16 +559,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | ----------------------------- | -| `NO` | `none` | `none` | `intentions:read`1 | - -

- 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

+| `NO` | `none` | `none` | `intentions:read`

Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

| The corresponding CLI command is [`consul intention check`](/consul/commands/intention/check). @@ -693,16 +610,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | -------------------- | ----------------------------- | -| `YES` | `all` | `background refresh` | `intentions:read`1 | - -

- 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

+| `YES` | `all` | `background refresh` | `intentions:read`

Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

| The corresponding CLI command is [`consul intention match`](/consul/commands/intention/match). @@ -762,7 +670,7 @@ $ curl \ ## Methods to Specify Namespace -Connect intention endpoints +Intention endpoints support several methods for specifying the namespace of intention resources with the following order of precedence: 1. `ns` query parameter diff --git a/website/content/api-docs/discovery-chain.mdx b/website/content/api-docs/discovery-chain.mdx index a96f0e89fe5..ba222116ec6 100644 --- a/website/content/api-docs/discovery-chain.mdx +++ b/website/content/api-docs/discovery-chain.mdx @@ -9,7 +9,7 @@ description: The /discovery-chain endpoints are for interacting with the discove -> **1.6.0+:** The discovery chain API is available in Consul versions 1.6.0 and newer. ~> This is a low-level API primarily targeted at developers building external -[Connect proxy integrations](/consul/docs/connect/proxies/integrate). Future +[service mesh proxy integrations](/consul/docs/connect/proxies/integrate). Future high-level proxy integration APIs may obviate the need for this API over time. The `/discovery-chain` endpoint returns the compiled [discovery @@ -17,7 +17,7 @@ chain](/consul/docs/connect/l7-traffic/discovery-chain) for a service. This will fetch all related [configuration entries](/consul/docs/agent/config-entries) and render them into a form suitable -for use by a [connect proxy](/consul/docs/connect/proxies) implementation. This +for use by a [service mesh proxy](/consul/docs/connect/proxies) implementation. This is a key component of [L7 Traffic Management](/consul/docs/connect/l7-traffic). diff --git a/website/content/api-docs/features/consistency.mdx b/website/content/api-docs/features/consistency.mdx index bcd7d621d65..746b062ab4c 100644 --- a/website/content/api-docs/features/consistency.mdx +++ b/website/content/api-docs/features/consistency.mdx @@ -131,7 +131,7 @@ The following diagrams show the cross-datacenter request paths when Consul serve ### Consul DNS Queries -When DNS queries are issued to [Consul's DNS interface](/consul/docs/discovery/dns), +When DNS queries are issued to [Consul's DNS interface](/consul/docs/services/discovery/dns-overview), Consul uses the `stale` consistency mode by default when interfacing with its underlying Consul service discovery HTTP APIs ([Catalog](/consul/api-docs/catalog), [Health](/consul/api-docs/health), and [Prepared Query](/consul/api-docs/query)). diff --git a/website/content/api-docs/health.mdx b/website/content/api-docs/health.mdx index 645381a209b..bd373c91b91 100644 --- a/website/content/api-docs/health.mdx +++ b/website/content/api-docs/health.mdx @@ -222,8 +222,7 @@ The table below shows this endpoint's support for | `YES` 1 | `all` | `background refresh` | `node:read,service:read` |

- 1some query parameters will use the - streaming backend + 1some query parameters will use the streaming backend for blocking queries.

### Path Parameters @@ -410,13 +409,13 @@ following selectors and filter operations being supported: | `Service.Weights.Passing` | Equal, Not Equal | | `Service.Weights.Warning` | Equal, Not Equal | -## List Service Instances for Connect-enabled Service +## List Service Instances for Mesh-enabled Service This endpoint returns the service instances providing a -[Connect-capable](/consul/docs/connect) service in a given datacenter. +[mesh-capable](/consul/docs/connect) service in a given datacenter. This will include both proxies and native integrations. A service may -register both Connect-capable and incapable services at the same time, -so this endpoint may be used to filter only the Connect-capable endpoints. +register both mesh-capable and incapable services at the same time, +so this endpoint may be used to filter only the mesh-capable endpoints. @include 'http_api_results_filtered_by_acls.mdx' diff --git a/website/content/api-docs/operator/index.mdx b/website/content/api-docs/operator/index.mdx index 3be515ac9ce..283a5023623 100644 --- a/website/content/api-docs/operator/index.mdx +++ b/website/content/api-docs/operator/index.mdx @@ -17,7 +17,7 @@ If ACLs are enabled then a token with operator privileges may be required in order to use this interface. Check the [ACL Rules documentation](/consul/docs/security/acl/acl-rules#operator-rules) for more information. -Check the [Outage Recovery](/consul/tutorials/datacenter-operations/recovery-outage) tutorial for some examples of +Check the [Disaster recovery for Consul clusters](/consul/tutorials/datacenter-operations/recovery-outage) tutorial for some examples of how these capabilities are used. Please choose a sub-section in the navigation for more information. diff --git a/website/content/api-docs/operator/usage.mdx b/website/content/api-docs/operator/usage.mdx new file mode 100644 index 00000000000..b1dc75c39e0 --- /dev/null +++ b/website/content/api-docs/operator/usage.mdx @@ -0,0 +1,167 @@ +--- +layout: api +page_title: Usage - Operator - HTTP API +description: |- + The /operator/usage endpoint returns usage information about the number of + services, service instances and mesh-enabled service instances by + datacenter. +--- + +# Usage Operator HTTP API + +The `/operator/usage` endpoint returns usage information about the number of +services, service instances and mesh-enabled service instances by datacenter. + +| Method | Path | Produces | +| ------ | ----------------- | ------------------ | +| `GET` | `/operator/usage` | `application/json` | + +The table below shows this endpoint's support for +[blocking queries](/consul/api-docs/features/blocking), +[consistency modes](/consul/api-docs/features/consistency), +[agent caching](/consul/api-docs/features/caching), and +[required ACLs](/consul/api-docs/api-structure#authentication). + +| Blocking Queries | Consistency Modes | Agent Caching | ACL Required | +| ---------------- | ----------------- | ------------- | --------------- | +| `YES` | `all` | `none` | `operator:read` | + +The corresponding CLI command is [`consul operator usage instances`](/consul/commands/operator/usage). + +### Query Parameters + +- `global` `(bool: false)` - If present, usage information for all + known datacenters will be returned. By default, only the local datacenter's + usage information is returned. + +- `stale` `(bool: false)` - If the cluster does not currently have a leader, an + error will be returned. You can use the `?stale` query parameter to read the + Raft configuration from any of the Consul servers. + +### Sample Request + +```shell-session +$ curl \ + http://127.0.0.1:8500/v1/operator/usage +``` + +### Sample Response + + + + + +```json +{ + "Usage": { + "dc1": { + "Services": 1, + "ServiceInstances": 1, + "ConnectServiceInstances": { + "connect-native": 0, + "connect-proxy": 0, + "ingress-gateway": 0, + "mesh-gateway": 0, + "terminating-gateway": 0 + }, + "BillableServiceInstances": 0 + } + }, + "Index": 13, + "LastContact": 0, + "KnownLeader": true, + "ConsistencyLevel": "leader", + "NotModified": false, + "Backend": 0, + "ResultsFilteredByACLs": false +} +``` + + + + + +```json +{ + "Usage": { + "dc1": { + "Services": 1, + "ServiceInstances": 1, + "ConnectServiceInstances": { + "connect-native": 0, + "connect-proxy": 0, + "ingress-gateway": 0, + "mesh-gateway": 0, + "terminating-gateway": 0 + }, + "BillableServiceInstances": 0, + "PartitionNamespaceServices": { + "default": { + "default": 1 + } + }, + "PartitionNamespaceServiceInstances": { + "default": { + "default": 1 + } + }, + "PartitionNamespaceBillableServiceInstances": { + "default": { + "default": 1 + } + }, + "PartitionNamespaceConnectServiceInstances": { + "default": { + "default": { + "connect-native": 0, + "connect-proxy": 0, + "ingress-gateway": 0, + "mesh-gateway": 0, + "terminating-gateway": 0 + } + } + } + } + }, + "Index": 13, + "LastContact": 0, + "KnownLeader": true, + "ConsistencyLevel": "leader", + "NotModified": false, + "Backend": 0, + "ResultsFilteredByACLs": false +} +``` + + + + +- `Services` is the total number of unique service names registered in the + datacenter. + +- `ServiceInstances` is the total number of service instances registered in + the datacenter. + +- `ConnectServiceInstances` is the total number of mesh service instances + registered in the datacenter. + +- `BillableServiceInstances` is the total number of billable service instances + registered in the datacenter. This is only relevant in Consul Enterprise + and is the total service instance count, not including any mesh service + instances or any Consul server instances. + +- `PartitionNamespaceServices` is the total number + of unique service names registered in the datacenter, by partition and + namespace. + +- `PartitionNamespaceServiceInstances` is the total + number of service instances registered in the datacenter, by partition and + namespace. + +- `PartitionNamespaceBillableServiceInstances` is + the total number of billable service instances registered in the datacenter, + by partition and namespace. + +- `PartitionNamespaceConnectServiceInstances` is + the total number of mesh service instances registered in the datacenter, + by partition and namespace. diff --git a/website/content/api-docs/peering.mdx b/website/content/api-docs/peering.mdx index a6afb29d892..9478af104cf 100644 --- a/website/content/api-docs/peering.mdx +++ b/website/content/api-docs/peering.mdx @@ -154,7 +154,7 @@ The table below shows this endpoint's support for | Blocking Queries | Consistency Modes | Agent Caching | ACL Required | | ---------------- | ----------------- | ------------- | -------------- | -| `NO` | `consistent` | `none` | `peering:read` | +| `YES` | `consistent` | `none` | `peering:read` | ### Path Parameters @@ -263,9 +263,9 @@ The table below shows this endpoint's support for [agent caching](/consul/api-docs/features/caching), and [required ACLs](/consul/api-docs/api-structure#authentication). -| Blocking Queries | Consistency Modes | Agent Caching | ACL Required | -| ---------------- | ----------------- | ------------- | -------------- | -| `NO` | `consistent` | `none` | `peering:read` | +| Blocking Queries | Consistency Modes | Agent Caching | ACL Required | +| ---------------- | ----------------- | -------------------- | -------------- | +| `YES` | `consistent` | `background refresh` | `peering:read` | ### Query Parameters diff --git a/website/content/api-docs/query.mdx b/website/content/api-docs/query.mdx index b3e2dcc2525..c71e4c441d2 100644 --- a/website/content/api-docs/query.mdx +++ b/website/content/api-docs/query.mdx @@ -9,10 +9,9 @@ description: The /query endpoints manage and execute prepared queries in Consul. The `/query` endpoints create, update, destroy, and execute prepared queries. Prepared queries allow you to register a complex service query and then execute -it later via its ID or name to get a set of healthy nodes that provide a given -service. This is particularly useful in combination with Consul's -[DNS Interface](/consul/docs/discovery/dns#prepared-query-lookups) as it allows for much richer queries than -would be possible given the limited entry points exposed by DNS. +it later by specifying the query ID or name. Consul returns a set of healthy nodes that provide a given +service. Refer to +[Enable Dynamic DNS Queries](/consul/docs/services/discovery/dns-dynamic-lookups) for additional information. Check the [Geo Failover tutorial](/consul/tutorials/developer-discovery/automate-geo-failover) for details and examples for using prepared queries to implement geo failover for services. @@ -252,12 +251,12 @@ The table below shows this endpoint's support for key/value pairs that will be used for filtering the query results to services with the given metadata values present. -* `Connect` `(bool: false)` - If true, only [Connect-capable](/consul/docs/connect) services +* `Connect` `(bool: false)` - If true, only [mesh-capable](/consul/docs/connect) services for the specified service name will be returned. This includes both natively integrated services and proxies. For proxies, the proxy name may not match `Service`, because the proxy destination will. Any - constrains beyond the service name such as `Near`, `Tags`, and `NodeMeta` - are applied to Connect-capable service. + constraints beyond the service name such as `Near`, `Tags`, and `NodeMeta` + are applied to mesh-capable service. * `DNS` `(DNS: nil)` - Specifies DNS configuration @@ -536,8 +535,8 @@ be used. This is applied after any sorting or shuffling. - `connect` `(bool: false)` - If true, limit results to nodes that are - Connect-capable only. This parameter can also be specified directly on the template - itself to force all executions of a query to be Connect-only. See the + mesh-capable only. This parameter can also be specified directly on the template + itself to force all executions of a query to be mesh-only. See the template documentation for more information. ### Sample Request diff --git a/website/content/commands/acl/policy/update.mdx b/website/content/commands/acl/policy/update.mdx index e62dfa72d99..f64a1f79068 100644 --- a/website/content/commands/acl/policy/update.mdx +++ b/website/content/commands/acl/policy/update.mdx @@ -49,6 +49,8 @@ Usage: `consul acl policy update [options] [args]` the value is a file path to load the rules from. `-` may also be given to indicate that the rules are available on stdin. +~> Specifying `-rules` will overwrite existing rules. + - `-valid-datacenter=` - Datacenter that the policy should be valid within. This flag may be specified multiple times. diff --git a/website/content/commands/acl/token/update.mdx b/website/content/commands/acl/token/update.mdx index 28158e6db79..1a1703cb143 100644 --- a/website/content/commands/acl/token/update.mdx +++ b/website/content/commands/acl/token/update.mdx @@ -33,34 +33,62 @@ Usage: `consul acl token update [options]` - `-id=` - The Accessor ID of the token to read. It may be specified as a unique ID prefix but will error if the prefix matches multiple token Accessor IDs -- `merge-node-identities` - Merge the new node identities with the existing node +- `-merge-node-identities` - Deprecated. Merge the new node identities with the existing node identities. +~> This is deprecated and will be removed in a future Consul version. Use `append-node-identity` instead. -- `-merge-policies` - Merge the new policies with the existing policies. +- `-merge-policies` - Deprecated. Merge the new policies with the existing policies. -- `-merge-roles` - Merge the new roles with the existing roles. +~> This is deprecated and will be removed in a future Consul version. Use `append-policy-id` or `append-policy-name` +instead. -- `-merge-service-identities` - Merge the new service identities with the existing service identities. +- `-merge-roles` - Deprecated. Merge the new roles with the existing roles. + +~> This is deprecated and will be removed in a future Consul version. Use `append-role-id` or `append-role-name` +instead. + +- `-merge-service-identities` - Deprecated. Merge the new service identities with the existing service identities. + +~> This is deprecated and will be removed in a future Consul version. Use `append-service-identity` instead. - `-meta` - Indicates that token metadata such as the content hash and Raft indices should be shown for each entry. -- `-node-identity=` - Name of a node identity to use for this role. May +- `-node-identity=` - Name of a node identity to use for this role. Overwrites existing node identity. May be specified multiple times. Format is `NODENAME:DATACENTER`. Added in Consul 1.8.1. -- `-policy-id=` - ID of a policy to use for this token. May be specified multiple times. +- `-append-node-identity=` - Name of a node identity to add to this role. May + be specified multiple times. The token retains existing node identities. Format is `NODENAME:DATACENTER`. + +- `-policy-id=` - ID of a policy to use for this token. Overwrites existing policies. May be specified multiple times. + +- `-policy-name=` - Name of a policy to use for this token. Overwrites existing policies. May be specified multiple times. + +~> `-policy-id` and `-policy-name` will overwrite existing token policies. Use `-append-policy-id` or `-append-policy-name` to retain existing policies. -- `-policy-name=` - Name of a policy to use for this token. May be specified multiple times. +- `-append-policy-id=` - ID of policy to be added for this token. The token retains existing policies. May be specified multiple times. -- `-role-id=` - ID of a role to use for this token. May be specified multiple times. +- `-append-policy-name=` - Name of a policy to be added for this token. The token retains existing policies. May be specified multiple times. -- `-role-name=` - Name of a role to use for this token. May be specified multiple times. +- `-role-id=` - ID of a role to use for this token. Overwrites existing roles. May be specified multiple times. + +- `-role-name=` - Name of a role to use for this token. Overwrites existing roles. May be specified multiple times. + +~> `-role-id` and `-role-name` will overwrite existing roles. Use `-append-role-id` or `-append-role-name` to retain the existing roles. + +- `-append-role-id=` - ID of a role to add to this token. The token retains existing roles. May be specified multiple times. + +- `-append-role-name=` - Name of a role to add to this token. The token retains existing roles. May be specified multiple times. - `-service-identity=` - Name of a service identity to use for this - token. May be specified multiple times. Format is the `SERVICENAME` or + token. Overwrites existing service identities. May be specified multiple times. Format is the `SERVICENAME` or `SERVICENAME:DATACENTER1,DATACENTER2,...` +- `-append-service-identity=` - Name of a service identity to add to this + token. May be specified multiple times. The token retains existing service identities. + Format is the `SERVICENAME` or `SERVICENAME:DATACENTER1,DATACENTER2,...` + - `-format={pretty|json}` - Command output format. The default value is `pretty`. #### Enterprise Options diff --git a/website/content/commands/config/delete.mdx b/website/content/commands/config/delete.mdx index 3fc9e6618b0..134d6885e66 100644 --- a/website/content/commands/config/delete.mdx +++ b/website/content/commands/config/delete.mdx @@ -27,6 +27,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | Config Entry Kind | Required ACL | | ------------------- | ------------------ | +| api-gateway | `mesh:write` | | ingress-gateway | `operator:write` | | proxy-defaults | `operator:write` | | service-defaults | `service:write` | @@ -45,16 +46,16 @@ Usage: `consul config delete [options]` - `-kind` - Specifies the kind of the config entry to read. - `-name` - Specifies the name of the config entry to delete. The name of the - `proxy-defaults` config entry must be `global`, and the name of the `mesh` - config entry must be `mesh`. +`proxy-defaults` config entry must be `global`, and the name of the `mesh` +config entry must be `mesh`. - `-filename` - Specifies the file describing the config entry to delete. - `-cas` - Perform a Check-And-Set operation. Specifying this value also - requires the -modify-index flag to be set. The default value is false. +requires the -modify-index flag to be set. The default value is false. - `-modify-index=` - Unsigned integer representing the ModifyIndex of the - config entry. This is used in combination with the -cas flag. +config entry. This is used in combination with the -cas flag. #### Enterprise Options diff --git a/website/content/commands/config/list.mdx b/website/content/commands/config/list.mdx index c72e3e903de..1a70af17872 100644 --- a/website/content/commands/config/list.mdx +++ b/website/content/commands/config/list.mdx @@ -27,6 +27,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | Config Entry Kind | Required ACL | | ------------------- | ----------------- | +| api-gateway | `mesh:read` | | ingress-gateway | `service:read` | | proxy-defaults | `` | | service-defaults | `service:read` | diff --git a/website/content/commands/config/read.mdx b/website/content/commands/config/read.mdx index a50574aaed7..7a49482c5b3 100644 --- a/website/content/commands/config/read.mdx +++ b/website/content/commands/config/read.mdx @@ -28,6 +28,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | Config Entry Kind | Required ACL | | ------------------- | ----------------- | +| api-gateway | `mesh:read` | | ingress-gateway | `service:read` | | proxy-defaults | `` | | service-defaults | `service:read` | diff --git a/website/content/commands/config/write.mdx b/website/content/commands/config/write.mdx index 7e586aff759..bb98d8e4988 100644 --- a/website/content/commands/config/write.mdx +++ b/website/content/commands/config/write.mdx @@ -30,6 +30,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | Config Entry Kind | Required ACL | | ------------------- | ------------------ | +| api-gateway | `mesh:write` | | ingress-gateway | `operator:write` | | proxy-defaults | `operator:write` | | service-defaults | `service:write` | @@ -78,8 +79,9 @@ supported config entries. #### Service defaults -Service defaults control default global values for a service, such as the -protocol and Connect fields. +Service defaults control default global values for a service in the service mesh. +For example, the following configuration defines that all instances of the `web` +service use the `http` protocol. ```json { @@ -89,33 +91,23 @@ protocol and Connect fields. } ``` -- `Name` - Sets the name of the config entry. For service defaults, this must be - the name of the service being configured. - -- `Protocol` - Sets the protocol of the service. This is used by Connect proxies - for things like observability features. - -- `Connect` - This block contains Connect-related fields for the service. - - - `SidecarProxy` - Sets whether or not instances of this service should get a - sidecar proxy by default. +For more information, refer to the [service defaults configuration reference](/consul/docs/connect/config-entries/service-defaults). #### Proxy defaults -Proxy defaults allow for configuring global config defaults across all services -for Connect proxy config. Currently, only one global entry is supported. +Proxy defaults lets you configure global config defaults across all proxies +in the service mesh. Currently, it supports only one global entry. +For example, the following configuration overrides a default timeout for all +Envoy proxies. ```json { "Kind": "proxy-defaults", "Name": "global", "Config": { - "foo": 1 + "local_idle_timeout_ms": 20000 } } ``` -- `Name` - Sets the name of the config entry. Currently, only a single `proxy-defaults` - entry with the name `global` is supported. - -* `Config` - An arbitrary map of configuration values used by Connect proxies. +For more information, refer to the [proxy defaults configuration reference](/consul/docs/connect/config-entries/proxy-defaults). diff --git a/website/content/commands/connect/ca.mdx b/website/content/commands/connect/ca.mdx index 17a76a62221..350d18e6d31 100644 --- a/website/content/commands/connect/ca.mdx +++ b/website/content/commands/connect/ca.mdx @@ -10,14 +10,15 @@ description: > Command: `consul connect ca` -The CA connect command is used to interact with Consul Connect's Certificate Authority -subsystem. The command can be used to view or modify the current CA configuration. See the -[Connect CA documentation](/consul/docs/connect/ca) for more information. +This command is used to interact with Consul service mesh's Certificate Authority +managed by the connect subsystem. +It can be used to view or modify the current CA configuration. Refer to the +[service mesh CA documentation](/consul/docs/connect/ca) for more information. ```text Usage: consul connect ca [options] [args] - This command has subcommands for interacting with Consul Connect's + This command has subcommands for interacting with Consul service mesh's Certificate Authority (CA). Here are some simple examples, and more detailed examples are available @@ -34,8 +35,8 @@ Usage: consul connect ca [options] [args] For more examples, ask for subcommand help or view the documentation. Subcommands: - get-config Display the current Connect Certificate Authority (CA) configuration - set-config Modify the current Connect CA configuration + get-config Display the current service mesh Certificate Authority (CA) configuration + set-config Modify the current service mesh CA configuration ``` ## get-config diff --git a/website/content/commands/connect/envoy.mdx b/website/content/commands/connect/envoy.mdx index 90bccf2faf8..bb2cc5d77e8 100644 --- a/website/content/commands/connect/envoy.mdx +++ b/website/content/commands/connect/envoy.mdx @@ -11,7 +11,7 @@ Command: `consul connect envoy` The connect Envoy command is used to generate a bootstrap configuration for [Envoy proxy](https://envoyproxy.io) for use with [Consul -Connect](/consul/docs/connect/). +service mesh](/consul/docs/connect/). Refer to the [examples](#examples) for guidance on common use cases, such as [launching a service instance's sidecar proxy @@ -56,7 +56,7 @@ Usage: `consul connect envoy [options] [-- pass-through options]` ACL token from `-token` or the environment and so should be handled as a secret. This token grants the identity of any service it has `service:write` permission for and so can be used to access any upstream service that that service is - allowed to access by [Connect intentions](/consul/docs/connect/intentions). + allowed to access by [service mesh intentions](/consul/docs/connect/intentions). - `-envoy-version` - The version of envoy that is being started. Default is `1.23.1`. This is required so that the correct configuration can be generated. @@ -165,11 +165,9 @@ compatibility with Envoy and prevent potential issues. Default is `false`. If Envoy is configured as a terminating or mesh gateway, traffic from services within the mesh will be received at the specified IP and port. - If Envoy is configured as an ingress gateway, a `/ready` HTTP endpoint will be - instantiated at the specified IP and port. Consul uses `/ready` HTTP endpoints - to check gateway health. The specified IP will also be used by the ingress - gateway when instantiating user-defined listeners configured in the - [ingress gateway](/consul/docs/connect/gateways/ingress-gateway) configuration entry. + If Envoy is configured as an ingress gateway, Consul instantiates a `/ready` HTTP endpoint at the specified IP and port. Consul uses `/ready` HTTP endpoints + to check gateway health. Ingress gateways also use the specified IP when instantiating user-defined listeners configured in the + [ingress gateway configuration entry](/consul/docs/connect/config-entries/ingress-gateway). ~> **Note**: Ensure that user-defined ingress gateway listeners use a different port than the port specified in `-address` so that they do not diff --git a/website/content/commands/connect/expose.mdx b/website/content/commands/connect/expose.mdx index 97ca3910fc0..79be1e0e60a 100644 --- a/website/content/commands/connect/expose.mdx +++ b/website/content/commands/connect/expose.mdx @@ -2,7 +2,7 @@ layout: commands page_title: 'Commands: Connect Expose' description: > - The connect expose subcommand is used to expose a Connect-enabled service + The connect expose subcommand is used to expose a mesh-enabled service through an Ingress gateway by modifying the gateway's configuration and adding an intention to allow traffic from the gateway to the service. --- @@ -11,7 +11,7 @@ description: > Command: `consul connect expose` -The connect expose subcommand is used to expose a Connect-enabled service +The connect expose subcommand is used to expose a mesh-enabled service through an Ingress gateway by modifying the gateway's configuration and adding an intention to allow traffic from the gateway to the service. See the [Ingress gateway documentation](/consul/docs/connect/gateways/ingress-gateway) for more information @@ -20,7 +20,7 @@ about Ingress gateways. ```text Usage: consul connect expose [options] - Exposes a Connect-enabled service through the given ingress gateway, using the + Exposes a mesh-enabled service through the given ingress gateway, using the given protocol and port. ``` diff --git a/website/content/commands/connect/index.mdx b/website/content/commands/connect/index.mdx index 166b8ff8352..8ebf11a7bea 100644 --- a/website/content/commands/connect/index.mdx +++ b/website/content/commands/connect/index.mdx @@ -9,10 +9,10 @@ description: >- Command: `consul connect` -The `connect` command is used to interact with Consul -[Connect](/consul/docs/connect/intentions) subsystems. It exposes commands for -running the built-in mTLS proxy and viewing/updating the Certificate Authority -(CA) configuration. This command is available in Consul 1.2 and later. +The `connect` command is used to interact with the connect subsystem +that provides Consul's [service mesh](/consul/docs/connect) capabilities. +It exposes commands for running service mesh proxies and +for viewing/updating the service mesh Certificate Authority (CA) configuration. ## Usage @@ -24,22 +24,22 @@ the complete list of subcommands. ```text Usage: consul connect [options] [args] - This command has subcommands for interacting with Consul Connect. + This command has subcommands for interacting with Consul service mesh. Here are some simple examples, and more detailed examples are available in the subcommands or the documentation. - Run the built-in Connect mTLS proxy + Run the production service mesh proxy - $ consul connect proxy + $ consul connect envoy For more examples, ask for subcommand help or view the documentation. Subcommands: - ca Interact with the Consul Connect Certificate Authority (CA) - envoy Runs or Configures Envoy as a Connect proxy - expose Expose a Connect-enabled service through an Ingress gateway - proxy Runs a Consul Connect proxy + ca Interact with the Consul service mesh Certificate Authority (CA) + envoy Runs or configures Envoy as a service mesh proxy + expose Expose a mesh-enabled service through an Ingress gateway + proxy Runs a non-production, built-in service mesh sidecar proxy redirect-traffic Applies iptables rules for traffic redirection ``` diff --git a/website/content/commands/connect/proxy.mdx b/website/content/commands/connect/proxy.mdx index 2bdfb2a9a2b..db3192dbee4 100644 --- a/website/content/commands/connect/proxy.mdx +++ b/website/content/commands/connect/proxy.mdx @@ -3,7 +3,7 @@ layout: commands page_title: 'Commands: Connect Proxy' description: > The connect proxy subcommand is used to run the built-in mTLS proxy for - Connect. + Consul service mesh. --- # Consul Connect Proxy @@ -11,9 +11,9 @@ description: > Command: `consul connect proxy` The connect proxy command is used to run Consul's built-in mTLS proxy for -use with Connect. This can be used in production to enable a Connect-unaware -application to accept and establish Connect-based connections. This proxy -can also be used in development to connect to Connect-enabled services. +use with Consul service mesh. This can be used in production to enable a mesh-unaware +application to accept and establish mesh-based connections. This proxy +can also be used in development to connect to mesh-enabled services. ## Usage @@ -49,7 +49,7 @@ Usage: `consul connect proxy [options]` - `-upstream` - Upstream service to support connecting to. The format should be 'name:addr', such as 'db:8181'. This will make 'db' available on port 8181. When a regular TCP connection is made to port 8181, the proxy will service - discover "db" and establish a Connect mTLS connection identifying as + discover "db" and establish a Consul service mesh mTLS connection identifying as the `-service` value. This flag can be repeated multiple times. - `-listen` - Address to listen for inbound connections to the proxied service. @@ -60,7 +60,7 @@ Usage: `consul connect proxy [options]` `-listen`. - `-register` - Self-register with the local Consul agent, making this - proxy available as Connect-capable service in the catalog. This is only + proxy available as mesh-capable service in the catalog. This is only useful with `-listen`. - `-register-id` - Optional ID suffix for the service when `-register` is set to diff --git a/website/content/commands/connect/redirect-traffic.mdx b/website/content/commands/connect/redirect-traffic.mdx index ded81c4a942..44decfef407 100644 --- a/website/content/commands/connect/redirect-traffic.mdx +++ b/website/content/commands/connect/redirect-traffic.mdx @@ -13,7 +13,7 @@ Command: `consul connect redirect-traffic` The connect redirect-traffic command is used to apply traffic redirection rules to enforce all traffic to go through the [Envoy proxy](https://envoyproxy.io) when using [Consul -Service Mesh](/consul/docs/connect/) in the Transparent Proxy mode. +service mesh](/consul/docs/connect/) in the Transparent Proxy mode. This command requires `iptables` command line utility to be installed, and as a result, this command can currently only run on linux. diff --git a/website/content/commands/debug.mdx b/website/content/commands/debug.mdx index 3fea57f43e3..bebbe955a29 100644 --- a/website/content/commands/debug.mdx +++ b/website/content/commands/debug.mdx @@ -45,12 +45,12 @@ or otherwise. `Usage: consul debug [options]` By default, the debug command will capture an archive at the current path for -all targets for 2 minutes. +all targets for 5 minutes. #### Command Options - `-duration` - Optional, the total time to capture data for from the target agent. Must - be greater than the interval and longer than 10 seconds. Defaults to 2 minutes. + be greater than the interval and longer than 10 seconds. Defaults to 5 minutes. - `-interval` - Optional, the interval at which to capture dynamic data, such as heap and metrics. Must be longer than 5 seconds. Defaults to 30 seconds. @@ -79,7 +79,7 @@ information when `debug` is running. By default, it captures all information. | `host` | Information about resources on the host running the target agent such as CPU, memory, and disk. | | `members` | A list of all the WAN and LAN members in the cluster. | | `metrics` | Metrics from the in-memory metrics endpoint in the target, captured at the interval. | -| `logs` | `DEBUG` level logs for the target agent, captured for the duration. | +| `logs` | `TRACE` level logs for the target agent, captured for the duration. | | `pprof` | Golang heap, CPU, goroutine, and trace profiling. CPU and traces are captured for `duration` in a single file while heap and goroutine are separate snapshots for each `interval`. This information is not retrieved unless [`enable_debug`](/consul/docs/agent/config/config-files#enable_debug) is set to `true` on the target agent or ACLs are enable and an ACL token with `operator:read` is provided. | ## Examples diff --git a/website/content/commands/index.mdx b/website/content/commands/index.mdx index 2946d794ba4..a3e50081103 100644 --- a/website/content/commands/index.mdx +++ b/website/content/commands/index.mdx @@ -31,13 +31,13 @@ Available commands are: acl Interact with Consul's ACLs agent Runs a Consul agent catalog Interact with the catalog - connect Interact with Consul Connect + connect Interact with service mesh functionality debug Records a debugging archive for operators event Fire a new event exec Executes a command on Consul nodes force-leave Forces a member of the cluster to enter the "left" state info Provides debugging information for operators. - intention Interact with Connect service intentions + intention Interact with service mesh intentions join Tell Consul agent to join cluster keygen Generates a new encryption key keyring Manages gossip layer encryption keys @@ -56,6 +56,7 @@ Available commands are: services Interact with services snapshot Saves, restores and inspects snapshots of Consul server state tls Builtin helpers for creating CAs and certificates + troubleshoot Provides tools to troubleshoot Consul's service mesh configuration validate Validate config files/directories version Prints the Consul version watch Watch for changes in Consul diff --git a/website/content/commands/intention/check.mdx b/website/content/commands/intention/check.mdx index a0c384c6461..fadc8d224bd 100644 --- a/website/content/commands/intention/check.mdx +++ b/website/content/commands/intention/check.mdx @@ -31,16 +31,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | ACL Required | | ----------------------------- | -| `intentions:read`1 | - -

- 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

+| `intentions:read`

Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

| ## Usage diff --git a/website/content/commands/intention/create.mdx b/website/content/commands/intention/create.mdx index b41b9c502fb..71c491b06ad 100644 --- a/website/content/commands/intention/create.mdx +++ b/website/content/commands/intention/create.mdx @@ -25,16 +25,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | ACL Required | | ------------------------------ | -| `intentions:write`1 | - -

- 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

+| `intentions:write`

Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

| ## Usage diff --git a/website/content/commands/intention/delete.mdx b/website/content/commands/intention/delete.mdx index 1f6971c4906..ad2fca1c97e 100644 --- a/website/content/commands/intention/delete.mdx +++ b/website/content/commands/intention/delete.mdx @@ -19,16 +19,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | ACL Required | | ------------------------------ | -| `intentions:write`1 | - -

- 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

+| `intentions:write`

Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

| -> **Deprecated** - The one argument form of this command is deprecated in Consul 1.9.0. Intentions no longer need IDs when represented as diff --git a/website/content/commands/intention/get.mdx b/website/content/commands/intention/get.mdx index 6ac253c898e..84b3a86065d 100644 --- a/website/content/commands/intention/get.mdx +++ b/website/content/commands/intention/get.mdx @@ -24,16 +24,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | ACL Required | | ----------------------------- | -| `intentions:read`1 | - -

- 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

+| `intentions:read`

Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

| ## Usage diff --git a/website/content/commands/intention/index.mdx b/website/content/commands/intention/index.mdx index 9246ef9e050..f7ea711a27f 100644 --- a/website/content/commands/intention/index.mdx +++ b/website/content/commands/intention/index.mdx @@ -9,15 +9,14 @@ description: >- Command: `consul intention` -The `intention` command is used to interact with Connect +The `intention` command is used to interact with service mesh [intentions](/consul/docs/connect/intentions). It exposes commands for creating, updating, reading, deleting, checking, and managing intentions. This command is available in Consul 1.2 and later. -Intentions are managed primarily via -[`service-intentions`](/consul/docs/connect/config-entries/service-intentions) config -entries after Consul 1.9. Intentions may also be managed via the [HTTP -API](/consul/api-docs/connect/intentions). +Use the +[`service-intentions`](/consul/docs/connect/config-entries/service-intentions) configuration entry or the [HTTP +API](/consul/api-docs/connect/intentions) to manage intentions. ~> **Deprecated** - This command is deprecated in Consul 1.9.0 in favor of using the [config entry CLI command](/consul/commands/config/write). To create an diff --git a/website/content/commands/intention/list.mdx b/website/content/commands/intention/list.mdx index cc5130ac993..85a7d6b9d6c 100644 --- a/website/content/commands/intention/list.mdx +++ b/website/content/commands/intention/list.mdx @@ -19,16 +19,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | ACL Required | | ----------------------------- | -| `intentions:read`1 | - -

- 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

+| `intentions:read`

Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

| ## Usage diff --git a/website/content/commands/intention/match.mdx b/website/content/commands/intention/match.mdx index 4936e12ba35..4c727b1fc21 100644 --- a/website/content/commands/intention/match.mdx +++ b/website/content/commands/intention/match.mdx @@ -24,16 +24,7 @@ are not supported from commands, but may be from the corresponding HTTP endpoint | ACL Required | | ----------------------------- | -| `intentions:read`1 | - -

- 1 Intention ACL rules are specified as part of a{' '} - service rule. See{' '} - - Intention Management Permissions - {' '} - for more details. -

+| `intentions:read`

Define intention rules in the `service` policy. Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for additional information.

| ## Usage diff --git a/website/content/commands/license.mdx b/website/content/commands/license.mdx index 50cee37544a..762e66df43d 100644 --- a/website/content/commands/license.mdx +++ b/website/content/commands/license.mdx @@ -167,7 +167,8 @@ Licensed Features: Corresponding HTTP API Endpoint: [\[GET\] /v1/operator/license](/consul/api-docs/operator/license#getting-the-consul-license) -This command gets the Consul Enterprise license. +This command gets the Consul Enterprise license. If the leader hasn't been updated with the newer license, the followers +will display the outdated license in their GET output. The table below shows this command's [required ACLs](/consul/api-docs/api-structure#authentication). Configuration of [blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching) diff --git a/website/content/commands/members.mdx b/website/content/commands/members.mdx index 4e6b73cae28..ff1df561a6f 100644 --- a/website/content/commands/members.mdx +++ b/website/content/commands/members.mdx @@ -48,6 +48,12 @@ Usage: `consul members [options]` in the WAN gossip pool. These are generally all the server nodes in each datacenter. +- `-filter=` - Expression to use for filtering the results, + e.g., `-filter='Tags["dc"] == dc2'`. + See the [`/catalog/nodes` API documentation](/consul/api-docs/catalog#filtering) for a + description of what is filterable. + + #### Enterprise Options @include 'http_api_partition_options.mdx' diff --git a/website/content/commands/operator/index.mdx b/website/content/commands/operator/index.mdx index 14ba3290528..7c9f3760ed6 100644 --- a/website/content/commands/operator/index.mdx +++ b/website/content/commands/operator/index.mdx @@ -20,7 +20,7 @@ order to use this command. Requests are forwarded internally to the leader if required, so this can be run from any Consul node in a cluster. See the [ACL Guide](/consul/tutorials/security/access-control-setup-production) for more information. -See the [Outage Recovery](/consul/tutorials/datacenter-operations/recovery-outage) guide for some examples of how +See the [Disaster recovery for Consul clusters](/consul/tutorials/datacenter-operations/recovery-outage) guide for some examples of how this command is used. For an API to perform these operations programmatically, please see the documentation for the [Operator](/consul/api-docs/operator) endpoint. @@ -37,6 +37,7 @@ Subcommands: area Provides tools for working with network areas (Enterprise-only) autopilot Provides tools for modifying Autopilot configuration raft Provides cluster-level tools for Consul operators + usage Provides cluster-level usage information ``` For more information, examples, and usage about a subcommand, click on the name @@ -45,3 +46,4 @@ of the subcommand in the sidebar or one of the links below: - [area](/consul/commands/operator/area) - [autopilot](/consul/commands/operator/autopilot) - [raft](/consul/commands/operator/raft) +- [usage](/consul/commands/operator/usage) diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index 857a9ee1ac1..1b2464cb8b2 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -46,10 +46,10 @@ Usage: `consul operator raft list-peers -stale=[true|false]` The output looks like this: ```text -Node ID Address State Voter RaftProtocol -alice 127.0.0.1:8300 127.0.0.1:8300 follower true 2 -bob 127.0.0.2:8300 127.0.0.2:8300 leader true 3 -carol 127.0.0.3:8300 127.0.0.3:8300 follower true 2 +Node ID Address State Voter RaftProtocol Commit Index Trails Leader By +alice 127.0.0.1:8300 127.0.0.1:8300 follower true 2 1167 0 commits +bob 127.0.0.2:8300 127.0.0.2:8300 leader true 3 1167 - +carol 127.0.0.3:8300 127.0.0.3:8300 follower true 2 1159 8 commits ``` `Node` is the node name of the server, as known to Consul, or "(unknown)" if @@ -66,11 +66,15 @@ Raft configuration. `Voter` is "true" or "false", indicating if the server has a vote in the Raft configuration. +`Commit Index` is the last log index the server has a record of in its Raft log. + +`Trails Leader By` is the number of commits a follower trails the leader by. + #### Command Options - `-stale` - Enables non-leader servers to provide cluster state information. If the cluster is in an outage state without a leader, - we recommend setting this option to `true. + we recommend setting this option to `true`. Default is `false`. ## remove-peer @@ -109,7 +113,7 @@ The return code will indicate success or failure. Corresponding HTTP API Endpoint: [\[POST\] /v1/operator/raft/transfer-leader](/consul/api-docs/operator/raft#transfer-raft-leadership) -This command transfers Raft leadership to another server agent. If an `id` is provided, Consul transfers leadership to the server with that id. +This command transfers Raft leadership to another server agent. If an `id` is provided, Consul transfers leadership to the server with that id. Use this command to change leadership without restarting the leader node, which maintains quorum and workload capacity. diff --git a/website/content/commands/operator/usage.mdx b/website/content/commands/operator/usage.mdx new file mode 100644 index 00000000000..56d36486269 --- /dev/null +++ b/website/content/commands/operator/usage.mdx @@ -0,0 +1,100 @@ +--- +layout: commands +page_title: 'Commands: Operator Usage' +description: > + The operator usage command provides cluster-level tools for Consul operators + to view usage information, such as service and service instance counts. +--- + +# Consul Operator Usage + +Command: `consul operator usage` + +The Usage `operator` command provides cluster-level tools for Consul operators +to view usage information, such as service and service instance counts. + +```text +Usage: consul operator usage [options] + + # ... + +Subcommands: + instances Display service instance usage information +``` + +## instances + +Corresponding HTTP API Endpoint: [\[GET\] /v1/operator/usage](/consul/api-docs/operator/usage#operator-usage) + +This command retrieves usage information about the number of services registered in a given +datacenter. By default, the datacenter of the local agent is queried. + +The table below shows this command's [required ACLs](/consul/api-docs/api-structure#authentication). Configuration of +[blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching) +are not supported from commands, but may be from the corresponding HTTP endpoint. + +| ACL Required | +| --------------- | +| `operator:read` | + +Usage: `consul operator usage instances` + +The output looks like this: + +```text +$ consul operator usage instances +Billable Service Instances Total: 3 +dc1 Billable Service Instances: 3 + +Billable Services +Services Service instances +2 3 + +Connect Services +Type Service instances +connect-native 0 +connect-proxy 0 +ingress-gateway 0 +mesh-gateway 1 +terminating-gateway 0 +``` + +With the `-all-datacenters` flag: + +```text +$ consul operator usage instances -all-datacenters +Billable Service Instances Total: 4 +dc1 Billable Service Instances: 3 +dc2 Billable Service Instances: 1 + +Billable Services +Datacenter Services Service instances +dc1 2 3 +dc2 1 1 + +Total 3 4 + +Connect Services +Datacenter Type Service instances +dc1 connect-native 0 +dc1 connect-proxy 0 +dc1 ingress-gateway 0 +dc1 mesh-gateway 1 +dc1 terminating-gateway 0 +dc2 connect-native 0 +dc2 connect-proxy 0 +dc2 ingress-gateway 0 +dc2 mesh-gateway 1 +dc2 terminating-gateway 1 + +Total 3 +``` + +#### Command Options + +- `-all-datacenters` - Display service counts from all known datacenters. + Default is `false`. + +- `-billable` - Display only billable service information. Default is `false`. + +- `-connect` - Display only Consul service mesh component information. Default is `false`. diff --git a/website/content/commands/services/deregister.mdx b/website/content/commands/services/deregister.mdx index 5cc774422c0..79ea7cba27b 100644 --- a/website/content/commands/services/deregister.mdx +++ b/website/content/commands/services/deregister.mdx @@ -13,23 +13,19 @@ Corresponding HTTP API Endpoint: [\[PUT\] /v1/agent/service/deregister/:service_ The `services deregister` command deregisters a service with the local agent. Note that this command can only deregister services that were registered -with the agent specified (defaults to the local agent) and is meant to -be paired with `services register`. +with the agent specified and is intended to be paired with `services register`. +By default, the command deregisters services on the local agent. -This is just one method for service deregistration. If the service was -registered with a configuration file, then deleting that file and -[reloading](/consul/commands/reload) Consul is the correct method to -deregister. See [Service Definition](/consul/docs/discovery/services) for more -information about registering services generally. +We recommend deregistering services registered with a configuration file by deleting the file and [reloading](/consul/commands/reload) Consul. Refer to [Services Overview](/consul/docs/services/services) for additional information about services. -The table below shows this command's [required ACLs](/consul/api-docs/api-structure#authentication). Configuration of -[blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching) -are not supported from commands, but may be from the corresponding HTTP endpoint. +The following table shows the [ACLs](/consul/api-docs/api-structure#authentication) required to run the `consul services deregister` command: | ACL Required | | --------------- | | `service:write` | +You cannot use the Consul command line to configure [blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching), you can configure them from the corresponding HTTP endpoint. + ## Usage Usage: `consul services deregister [options] [FILE...]` diff --git a/website/content/commands/services/register.mdx b/website/content/commands/services/register.mdx index 8e3a9d1333e..cefa2359230 100644 --- a/website/content/commands/services/register.mdx +++ b/website/content/commands/services/register.mdx @@ -14,24 +14,16 @@ Corresponding HTTP API Endpoint: [\[PUT\] /v1/agent/service/register](/consul/ap The `services register` command registers a service with the local agent. This command returns after registration and must be paired with explicit service deregistration. This command simplifies service registration from -scripts, in dev mode, etc. +scripts. Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for information about other service registeration methods. -This is just one method of service registration. Services can also be -registered by placing a [service definition](/consul/docs/discovery/services) -in the Consul agent configuration directory and issuing a -[reload](/consul/commands/reload). This approach is easiest for -configuration management systems that other systems that have access to -the configuration directory. Clients may also use the -[HTTP API](/consul/api-docs/agent/service) directly. - -The table below shows this command's [required ACLs](/consul/api-docs/api-structure#authentication). Configuration of -[blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching) -are not supported from commands, but may be from the corresponding HTTP endpoint. +The following table shows the [ACLs](/consul/api-docs/api-structure#authentication) required to use the `consul services register` command: | ACL Required | | --------------- | | `service:write` | +You cannot use the Consul command line to configure [blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching), you can configure them from the corresponding HTTP endpoint. + ## Usage Usage: `consul services register [options] [FILE...]` @@ -65,9 +57,7 @@ The flags below should only be set if _no arguments_ are given. If no arguments are given, the flags below can be used to register a single service. -Note that the behavior of each of the fields below is exactly the same -as when constructing a standard [service definition](/consul/docs/discovery/services). -Please refer to that documentation for full details. +The following fields specify identical parameters in a standard service definition file. Refer to [Services Configuration Reference](/consul/docs/services/configuration/services-configuration-reference) for details about each configuration option. - `-id` - The ID of the service. This will default to `-name` if not set. diff --git a/website/content/commands/snapshot/save.mdx b/website/content/commands/snapshot/save.mdx index 63b0fbe48f3..cf77cd48a69 100644 --- a/website/content/commands/snapshot/save.mdx +++ b/website/content/commands/snapshot/save.mdx @@ -20,7 +20,8 @@ If ACLs are enabled, a management token must be supplied in order to perform a snapshot save. -> Note that saving a snapshot involves the server process writing the snapshot to a -temporary file on-disk before sending that file to the CLI client. The default location +temporary file on-disk before sending that file to the CLI client. Upon successful completion, +Consul removes the temporary file. The default location of the temporary file can vary depending on operating system, but typically is `/tmp`. You can get more detailed information on default locations in the Go documentation for [os.TempDir](https://golang.org/pkg/os/#TempDir). If you need to change this location, you can do so by setting the `TMPDIR` environment @@ -28,6 +29,7 @@ variable for the Consul server processes. Keep in mind that setting the environm the CLI client attempting to perform a snapshot save will have no effect. It _must_ be set in the context of the server process. If you're using Systemd to manage your Consul server processes, then adding `Environment=TMPDIR=/path/to/dir` to your Consul unit file will work. +As a result of the Raft snapshot, Consul also saves one snapshot file at `data_dir/raft/snapshots`. The table below shows this command's [required ACLs](/consul/api-docs/api-structure#authentication). Configuration of [blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching) @@ -47,6 +49,10 @@ Usage: `consul snapshot save [options] FILE` @include 'http_api_options_server.mdx' +- `-append-filename=` - Value can be - version,dc,node,status +Adds consul version, datacenter name, node name, and status (leader/follower) +to the file name before the extension separated by `-` + ## Examples To create a snapshot from the leader server and save it to "backup.snap": @@ -75,5 +81,16 @@ This is useful for situations where a cluster is in a degraded state and no leader is available. To target a specific server for a snapshot, you can run the `consul snapshot save` command on that specific server. +To create snapshot file with consul version and datacenter run + +```shell-session +$ consul snapshot save -append-filename version,dc backup.snap +#... +``` + +File name created will be like backup-%CONSUL_VERSION%-%DC_NAME%.snap +example - backup-1.17.0-dc1-local-machine-leader.tgz +Note Version is always the leader's consul version + Please see the [HTTP API](/consul/api-docs/snapshot) documentation for more details about snapshot internals. diff --git a/website/content/commands/troubleshoot/index.mdx b/website/content/commands/troubleshoot/index.mdx new file mode 100644 index 00000000000..0c992aab15c --- /dev/null +++ b/website/content/commands/troubleshoot/index.mdx @@ -0,0 +1,31 @@ +--- +layout: commands +page_title: 'Commands: Troubleshoot' +description: >- + The `consul troubleshoot` command provides tools to troubleshoot Consul's service mesh configuration. +--- + +# Consul Troubleshooting + +Command: `consul troubleshoot` + +Use the `troubleshoot` command to diagnose Consul service mesh configuration or network issues. For additional information about using the `troubleshoot` command, including explanations, requirements, usage instructions, refer to the [service-to-service troubleshooting overview](/consul/docs/troubleshoot/troubleshoot-services). + +## Usage + +```text +Usage: consul troubleshoot [options] + + # ... + +Subcommands: + + proxy Troubleshoots service mesh issues from the current Envoy instance + upstreams Gets upstream Envoy identifiers and IPs configured for the proxy +``` + +For more information, examples, and usage about a subcommand, click on the name +of the subcommand in the sidebar or one of the links below: + +- [proxy](/consul/commands/troubleshoot/proxy) +- [upstreams](/consul/commands/troubleshoot/upstreams) diff --git a/website/content/commands/troubleshoot/proxy.mdx b/website/content/commands/troubleshoot/proxy.mdx new file mode 100644 index 00000000000..d9749c0c254 --- /dev/null +++ b/website/content/commands/troubleshoot/proxy.mdx @@ -0,0 +1,65 @@ +--- +layout: commands +page_title: 'Commands: Troubleshoot Proxy' +description: >- + The `consul troubleshoot proxy` diagnoses Consul service mesh configuration and network issues to an upstream. +--- + +# Consul Troubleshoot Proxy + +Command: `consul troubleshoot proxy` + +The `troubleshoot proxy` command diagnoses Consul service mesh configuration and network issues to an upstream. For additional information about using the `troubleshoot proxy` command, including explanations, requirements, usage instructions, refer to the [service-to-service troubleshooting overview](/consul/docs/troubleshoot/troubleshoot-services). + +## Usage + +Usage: `consul troubleshoot proxy (-upstream-ip |-upstream-envoy-id ) [options]` +This command requires `-envoy-admin-endpoint` or `-upstream-ip` to specify the upstream. + +#### Command Options + +- `-envoy-admin-endpoint=` - Envoy admin endpoint address for the local Envoy instance. +Defaults to `127.0.0.1:19000`. + +- `-upstream-ip=` - The IP address of the upstream service; Use when the upstream is reached via a transparent proxy DNS address (KubeDNS or Consul DNS) + +- `-upstream-envoy-id=` - The Envoy identifier of the upstream service; Use when the upstream is explicitly configured on the downstream service. + +## Examples + +The following example illustrates how to troubleshoot Consul service mesh configuration and network issues to an upstream from a source proxy. + + ```shell-session + $ consul troubleshoot proxy -upstream-ip 10.4.6.160 + ==> Validation + ✓ Certificates are valid + ✓ Envoy has 0 rejected configurations + ✓ Envoy has detected 0 connection failure(s) + ✓ Listener for upstream "backend" found + ✓ Route for upstream "backend" found + ✓ Cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ Healthy endpoints for cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ Cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ! No healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + -> Check that your upstream service is healthy and running + -> Check that your upstream service is registered with Consul + -> Check that the upstream proxy is healthy and running + -> If you are explicitly configuring upstreams, ensure the name of the upstream is correct + ``` + +The following example troubleshoots Consul service mesh configuration and network issues from the local Envoy instance to the given upstream. + + ```shell-session + $ consul-k8s troubleshoot proxy -upstream-envoy-id db + ==> Validation + ✓ Certificates are valid + ✓ Envoy has 0 rejected configurations + ✓ Envoy has detected 0 connection failure(s) + ✓ Listener for upstream "db" found + ✓ Route for upstream "db" found + ✓ Cluster "db.default.dc1.internal.e08fa6d6-e91e-dfe0.consul" for upstream "db" found + ✓ Healthy endpoints for cluster "db.default.dc1.internal.e08fa6d6-e91e-dfe0.consul" for upstream "db" found + If you are still experiencing issues, you can: + -> Check intentions to ensure the upstream allows traffic from this source + -> If using transparent proxy, ensure DNS resolution is to the same IP you have verified here + ``` diff --git a/website/content/commands/troubleshoot/upstreams.mdx b/website/content/commands/troubleshoot/upstreams.mdx new file mode 100644 index 00000000000..425ec39e464 --- /dev/null +++ b/website/content/commands/troubleshoot/upstreams.mdx @@ -0,0 +1,35 @@ +--- +layout: commands +page_title: 'Commands: Troubleshoot Upstreams' +description: >- + The `consul troubleshoot upstreams` lists the available upstreams in the Consul service mesh from the current service. +--- + +# Consul Troubleshoot Upstreams + +Command: `consul troubleshoot upstreams` + +The `troubleshoot upstreams` lists the available upstreams in the Consul service mesh from the current service. For additional information about using the `troubleshoot upstreams` command, including explanations, requirements, usage instructions, refer to the [service-to-service troubleshooting overview](/consul/docs/troubleshoot/troubleshoot-services). + +## Usage + +Usage: `consul troubleshoot upstreams [options]` + +#### Command Options + +- `-envoy-admin-endpoint=` - Envoy admin endpoint address for the local Envoy instance. +Defaults to `127.0.0.1:19000`. + +## Examples + +Display all transparent proxy upstreams in Consul service mesh from the current Envoy instance. + + ```shell-session + $ consul troubleshoot upstreams + ==> Upstreams (explicit upstreams only) (0) + ==> Upstreams IPs (transparent proxy only) (1) + [10.4.6.160 240.0.0.3] true map[backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul] + If you cannot find the upstream address or cluster for a transparent proxy upstream: + - Check intentions: Tproxy upstreams are configured based on intentions. Make sure you have configured intentions to allow traffic to your upstream. + - To check that the right cluster is being dialed, run a DNS lookup for the upstream you are dialing. For example, run `dig backend.svc.consul` to return the IP address for the `backend` service. If the address you get from that is missing from the upstream IPs, it means that your proxy may be misconfigured. + ``` diff --git a/website/content/docs/agent/config/cli-flags.mdx b/website/content/docs/agent/config/cli-flags.mdx index ebcdb4c0764..82d44d02528 100644 --- a/website/content/docs/agent/config/cli-flags.mdx +++ b/website/content/docs/agent/config/cli-flags.mdx @@ -82,7 +82,7 @@ information. - `-dev` ((#\_dev)) - Enable development server mode. This is useful for quickly starting a Consul agent with all persistence options turned off, enabling an in-memory server which can be used for rapid prototyping or developing against - the API. In this mode, [Connect is enabled](/consul/docs/connect/configuration) and + the API. In this mode, [service mesh is enabled](/consul/docs/connect/configuration) and will by default create a new root CA certificate on startup. This mode is **not** intended for production use as it does not write any data to disk. The gRPC port is also defaulted to `8502` in this mode. @@ -92,7 +92,7 @@ information. only the given `-encrypt` key will be available on startup. This defaults to false. - `-enable-script-checks` ((#\_enable_script_checks)) This controls whether - [health checks that execute scripts](/consul/docs/discovery/checks) are enabled on this + [health checks that execute scripts](/consul/docs/services/usage/checks) are enabled on this agent, and defaults to `false` so operators must opt-in to allowing these. This was added in Consul 0.9.0. @@ -157,7 +157,7 @@ information. - `-segment` ((#\_segment)) - This flag is used to set the name of the network segment the agent belongs to. An agent can only join and communicate with other agents within its network segment. Ensure the [join - operation uses the correct port for this segment](/consul/docs/enterprise/network-segments#join_a_client_to_a_segment). + operation uses the correct port for this segment](/consul/docs/enterprise/network-segments/create-network-segment#configure-clients-to-join-segments). Review the [Network Segments documentation](/consul/docs/enterprise/network-segments/create-network-segment) for more details. By default, this is an empty string, which is the `` network segment. @@ -323,7 +323,7 @@ information. If Consul is running on a non-default Serf LAN port, you must specify the port number in the address when using the `-retry-join` flag. Alternatively, you can specify the custom port number as the default in the agent's [`ports.serf_lan`](/consul/docs/agent/config/config-files#serf_lan_port) configuration or with the [`-serf-lan-port`](#_serf_lan_port) command line flag when starting the agent. - If your network contains network segments, refer to the [network segements documentation](/consul/docs/enterprise/network-segments/create-network-segment) for additional information. + If your network contains network segments, refer to the [network segments documentation](/consul/docs/enterprise/network-segments/create-network-segment) for additional information. Here are some examples of using `-retry-join`: @@ -490,7 +490,7 @@ information. the data directory. This is useful when running multiple Consul agents on the same host for testing. This defaults to false in Consul prior to version 0.8.5 and in 0.8.5 and later defaults to true, so you must opt-in for host-based IDs. Host-based - IDs are generated using [gopsutil](https://github.com/shirou/gopsutil/tree/master/v3/host), which + IDs are generated using [gopsutil](https://github.com/shirou/gopsutil/), which is shared with HashiCorp's [Nomad](https://www.nomadproject.io/), so if you opt-in to host-based IDs then Consul and Nomad will use information on the host to automatically assign the same ID in both systems. diff --git a/website/content/docs/agent/config/config-files.mdx b/website/content/docs/agent/config/config-files.mdx index 82053a93567..c6536fa945d 100644 --- a/website/content/docs/agent/config/config-files.mdx +++ b/website/content/docs/agent/config/config-files.mdx @@ -7,6 +7,10 @@ description: >- # Agents Configuration File Reference ((#configuration_files)) +This topic describes the parameters for configuring Consul agents. For information about how to start Consul agents, refer to [Starting the Consul Agent](/consul/docs/agent#starting-the-consul-agent). + +## Overview + You can create one or more files to configure the Consul agent on startup. We recommend grouping similar configurations into separate files, such as ACL parameters, to make it easier to manage configuration changes. Using external files may be easier than @@ -18,13 +22,6 @@ easily readable and editable by both humans and computers. JSON formatted configuration consists of a single JSON object with multiple configuration keys specified within it. -Configuration files are used for more than just setting up the agent. -They are also used to provide check and service definitions that -announce the availability of system servers to the rest of the cluster. -These definitions are documented separately under [check configuration](/consul/docs/discovery/checks) and -[service configuration](/consul/docs/discovery/services) respectively. Service and check -definitions support being updated during a reload. - ```hcl @@ -66,15 +63,27 @@ telemetry { -# Configuration Key Reference ((#config_key_reference)) +### Time-to-live values + +Consul uses the Go `time` package to parse all time-to-live (TTL) values used in Consul agent configuration files. Specify integer and float values as a string and include one or more of the following units of time: + +- `ns` +- `us` +- `µs` +- `ms` +- `s` +- `m` +- `h` --> **Note:** All the TTL values described below are parsed by Go's `time` package, and have the following -[formatting specification](https://golang.org/pkg/time/#ParseDuration): "A -duration string is a possibly signed sequence of decimal numbers, each with -optional fraction and a unit suffix, such as '300ms', '-1.5h' or '2h45m'. -Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." +Examples: -## General +- `'300ms'` +- `'1.5h'` +- `'2h45m'` + +Refer to the [formatting specification](https://golang.org/pkg/time/#ParseDuration) for additional information. + +## General parameters - `addresses` - This is a nested object that allows setting bind addresses. In Consul 1.0 and later these can be set to a space-separated list @@ -236,8 +245,8 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." The initial RPC uses a JWT specified with either `intro_token`, `intro_token_file` or the `CONSUL_INTRO_TOKEN` environment variable to authorize the request. How the JWT token is verified is controlled by the `auto_config.authorizer` - object available for use on Consul servers. Enabling this option also turns - on Connect because it is vital for `auto_config`, more specifically the CA + object available for use on Consul servers. Enabling this option also enables + service mesh because it is vital for `auto_config`, more specifically the service mesh CA and certificates infrastructure. ~> **Warning:** Enabling `auto_config` conflicts with the [`auto_encrypt.tls`](#tls) feature. @@ -534,17 +543,17 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." - `license_path` This specifies the path to a file that contains the Consul Enterprise license. Alternatively the license may also be specified in either the `CONSUL_LICENSE` or `CONSUL_LICENSE_PATH` environment variables. See the [licensing documentation](/consul/docs/enterprise/license/overview) for more information about Consul Enterprise license management. Added in versions 1.10.0, 1.9.7 and 1.8.13. Prior to version 1.10.0 the value may be set for all agents to facilitate forwards compatibility with 1.10 but will only actually be used by client agents. -- `limits` Available in Consul 0.9.3 and later, this is a nested - object that configures limits that are enforced by the agent. Prior to Consul 1.5.2, - this only applied to agents in client mode, not Consul servers. The following parameters - are available: +- `limits`: This block specifies various types of limits that the Consul server agent enforces. - `http_max_conns_per_client` - Configures a limit of how many concurrent TCP connections a single client IP address is allowed to open to the agent's HTTP(S) server. This affects the HTTP(S) servers in both client and server agents. Default value is `200`. - `https_handshake_timeout` - Configures the limit for how long the HTTPS server in both client and server agents will wait for a client to complete a TLS handshake. This should be kept conservative as it limits how many connections an unauthenticated attacker can open if `verify_incoming` is being using to authenticate clients (strongly recommended in production). Default value is `5s`. - - `request_limits` - This object povides configuration for rate limiting RPC and gRPC requests on the consul server. As a result of rate limiting gRPC and RPC request, HTTP requests to the Consul server are rate limited. - - `mode` - Configures whether rate limiting is enabled or not as well as how it behaves through the use of 3 possible modes. The default value of "disabled" will prevent any rate limiting from occuring. A value of "permissive" will cause the system to track requests against the `read_rate` and `write_rate` but will only log violations and will not block and will allow the request to continue processing. A value of "enforcing" also tracks requests against the `read_rate` and `write_rate` but in addition to logging violations, the system will block the request from processings by returning an error. - - `read_rate` - Configures how frequently RPC, gRPC, and HTTP queries are allowed to happen. The rate limiter limits the rate to tokens per second equal to this value. See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets. - - `write_rate` - Configures how frequently RPC, gRPC, and HTTP write are allowed to happen. The rate limiter limits the rate to tokens per second equal to this value. See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets. + - `request_limits` - This object specifies configurations that limit the rate of RPC and gRPC requests on the Consul server. Limiting the rate of gRPC and RPC requests also limits HTTP requests to the Consul server. + - `mode` - String value that specifies an action to take if the rate of requests exceeds the limit. You can specify the following values: + - `permissive`: The server continues to allow requests and records an error in the logs. + - `enforcing`: The server stops accepting requests and records an error in the logs. + - `disabled`: Limits are not enforced or tracked. This is the default value for `mode`. + - `read_rate` - Integer value that specifies the number of read requests per second. Default is `-1` which represents infinity. + - `write_rate` - Integer value that specifies the number of write requests per second. Default is `-1` which represents infinity. - `rpc_handshake_timeout` - Configures the limit for how long servers will wait after a client TCP connection is established before they complete the connection handshake. When TLS is used, the same timeout applies to the TLS handshake separately from the initial protocol negotiation. All Consul clients should perform this immediately on establishing a new connection. This should be kept conservative as it limits how many connections an unauthenticated attacker can open if `verify_incoming` is being using to authenticate clients (strongly recommended in production). When `verify_incoming` is true on servers, this limits how long the connection socket and associated goroutines will be held open before the client successfully authenticates. Default value is `5s`. - `rpc_client_timeout` - Configures the limit for how long a client is allowed to read from an RPC connection. This is used to set an upper bound for calls to eventually terminate so that RPC connections are not held indefinitely. Blocking queries can override this timeout. Default is `60s`. - `rpc_max_conns_per_client` - Configures a limit of how many concurrent TCP connections a single source IP address is allowed to open to a single server. It affects both clients connections and other server connections. In general Consul clients multiplex many RPC calls over a single TCP connection so this can typically be kept low. It needs to be more than one though since servers open at least one additional connection for raft RPC, possibly more for WAN federation when using network areas, and snapshot requests from clients run over a separate TCP conn. A reasonably low limit significantly reduces the ability of an unauthenticated attacker to consume unbounded resources by holding open many connections. You may need to increase this if WAN federated servers connect via proxies or NAT gateways or similar causing many legitimate connections from a single source IP. Default value is `100` which is designed to be extremely conservative to limit issues with certain deployment patterns. Most deployments can probably reduce this safely. 100 connections on modern server hardware should not cause a significant impact on resource usage from an unauthenticated attacker though. @@ -642,7 +651,7 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." - `primary_datacenter` - This designates the datacenter which is authoritative for ACL information, intentions and is the root Certificate - Authority for Connect. It must be provided to enable ACLs. All servers and datacenters + Authority for service mesh. It must be provided to enable ACLs. All servers and datacenters must agree on the primary datacenter. Setting it on the servers is all you need for cluster-level enforcement, but for the APIs to forward properly from the clients, it must be set on them too. In Consul 0.8 and later, this also enables agent-level @@ -727,6 +736,12 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." - `server` Equivalent to the [`-server` command-line flag](/consul/docs/agent/config/cli-flags#_server). +- `server_rejoin_age_max` - controls the allowed maximum age of a stale server attempting to rejoin a cluster. + If the server has not ran during this period, it will refuse to start up again until an operator intervenes by manually deleting the `server_metadata.json` + file located in the data dir. + This is to protect clusters from instability caused by decommissioned servers accidentally being started again. + Note: the default value is 168h (equal to 7d) and the minimum value is 6h. + - `non_voting_server` - **This field is deprecated in Consul 1.9.1. See the [`read_replica`](#read_replica) field instead.** - `read_replica` - Equivalent to the [`-read-replica` command-line flag](/consul/docs/agent/config/cli-flags#_read_replica). @@ -914,10 +929,9 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." - `agent_master` ((#acl_tokens_agent_master)) **Renamed in Consul 1.11 to [`acl.tokens.agent_recovery`](#acl_tokens_agent_recovery).** - - `config_file_service_registration` ((#acl_tokens_config_file_service_registration)) - The ACL - token this agent uses to register services and checks from [service - definitions](/consul/docs/discovery/services) and [check definitions](/consul/docs/discovery/checks) found - in configuration files or in configuration fragments passed to the agent using the `-hcl` + - `config_file_service_registration` ((#acl_tokens_config_file_service_registration)) - Specifies the ACL + token the agent uses to register services and checks from [service](/consul/docs/services/usage/define-services) and [check](/consul/docs/services/usage/checks) definitions + specified in configuration files or fragments passed to the agent using the `-hcl` flag. If the `token` field is defined in the service or check definition, then that token is used to @@ -938,7 +952,7 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." - `replication` ((#acl_tokens_replication)) - The ACL token used to authorize secondary datacenters with the primary datacenter for replication - operations. This token is required for servers outside the [`primary_datacenter`](#primary_datacenter) when ACLs are enabled. This token may be provided later using the [agent token API](/consul/api-docs/agent#update-acl-tokens) on each server. This token must have at least "read" permissions on ACL data but if ACL token replication is enabled then it must have "write" permissions. This also enables Connect replication, for which the token will require both operator "write" and intention "read" permissions for replicating CA and Intention data. + operations. This token is required for servers outside the [`primary_datacenter`](#primary_datacenter) when ACLs are enabled. This token may be provided later using the [agent token API](/consul/api-docs/agent#update-acl-tokens) on each server. This token must have at least "read" permissions on ACL data but if ACL token replication is enabled then it must have "write" permissions. This also enables service mesh data replication, for which the token will require both operator "write" and intention "read" permissions for replicating CA and Intention data. ~> **Warning:** When enabling ACL token replication on the secondary datacenter, policies and roles already present in the secondary datacenter will be lost. For @@ -1080,7 +1094,10 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." - `bootstrap_expect` Equivalent to the [`-bootstrap-expect` command-line flag](/consul/docs/agent/config/cli-flags#_bootstrap_expect). -## Connect Parameters +## Service Mesh Parameters ((#connect-parameters)) + +The noun _connect_ is used throughout this documentation to refer to the connect +subsystem that provides Consul's service mesh capabilities. - `connect` This object allows setting options for the Connect feature. @@ -1088,14 +1105,14 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." - `enabled` ((#connect_enabled)) (Defaults to `true`) Controls whether Connect features are enabled on this agent. Should be enabled on all servers in the cluster - in order for Connect to function properly. + in order for service mesh to function properly. Will be set to `true` automatically if `auto_config.enabled` or `auto_encrypt.allow_tls` is `true`. - `enable_mesh_gateway_wan_federation` ((#connect_enable_mesh_gateway_wan_federation)) (Defaults to `false`) Controls whether cross-datacenter federation traffic between servers is funneled through mesh gateways. This was added in Consul 1.8.0. - `ca_provider` ((#connect_ca_provider)) Controls which CA provider to - use for Connect's CA. Currently only the `aws-pca`, `consul`, and `vault` providers are supported. + use for the service mesh's CA. Currently only the `aws-pca`, `consul`, and `vault` providers are supported. This is only used when initially bootstrapping the cluster. For an existing cluster, use the [Update CA Configuration Endpoint](/consul/api-docs/connect/ca#update-ca-configuration). @@ -1132,13 +1149,13 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." - `root_pki_path` ((#vault_ca_root_pki)) The path to use for the root CA pki backend in Vault. This can be an existing backend with a CA already - configured, or a blank/unmounted backend in which case Connect will automatically + configured, or a blank/unmounted backend in which case Consul will automatically mount/generate the CA. The Vault token given above must have `sudo` access to this backend, as well as permission to mount the backend at this path if it is not already mounted. - `intermediate_pki_path` ((#vault_ca_intermediate_pki)) - The path to use for the temporary intermediate CA pki backend in Vault. **Connect + The path to use for the temporary intermediate CA pki backend in Vault. **Consul will overwrite any data at this path in order to generate a temporary intermediate CA**. The Vault token given above must have `write` access to this backend, as well as permission to mount the backend at this path if it is not already @@ -1367,7 +1384,7 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." of `1ns` instead of 0. - `prefer_namespace` ((#dns_prefer_namespace)) **Deprecated in Consul 1.11. - Use the [canonical DNS format for enterprise service lookups](/consul/docs/discovery/dns#service-lookups-for-consul-enterprise) instead.** - + Use the [canonical DNS format for enterprise service lookups](/consul/docs/services/discovery/dns-static-lookups#service-lookups-for-consul-enterprise) instead.** - When set to `true`, in a DNS query for a service, a single label between the domain and the `service` label is treated as a namespace name instead of a datacenter. When set to `false`, the default, the behavior is the same as non-Enterprise @@ -1383,14 +1400,14 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." - `allow_tls` (Defaults to `false`) This option enables `auto_encrypt` on the servers and allows them to automatically distribute certificates - from the Connect CA to the clients. If enabled, the server can accept incoming - connections from both the built-in CA and the Connect CA, as well as their certificates. + from the service mesh CA to the clients. If enabled, the server can accept incoming + connections from both the built-in CA and the service mesh CA, as well as their certificates. Note, the server will only present the built-in CA and certificate, which the client can verify using the CA it received from `auto_encrypt` endpoint. If disabled, a client configured with `auto_encrypt.tls` will be unable to start. - `tls` (Defaults to `false`) Allows the client to request the - Connect CA and certificates from the servers, for encrypting RPC communication. + service mesh CA and certificates from the servers, for encrypting RPC communication. The client will make the request to any servers listed in the `-retry-join` option. This requires that every server to have `auto_encrypt.allow_tls` enabled. When both `auto_encrypt` options are used, it allows clients to receive certificates @@ -1516,6 +1533,16 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." part of the cluster before declaring it dead, giving that suspect node more time to refute if it is indeed still alive. The default is 6. +## Self-managed HCP Parameters + +- `cloud` This object specifies settings for connecting self-managed clusters to HCP. This was added in Consul 1.14 + + - `client_id` The OAuth2 client ID for authentication with HCP. This can be overridden using the `HCP_CLIENT_ID` environment variable. + + - `client_secret` The OAuth2 client secret for authentication with HCP. This can be overridden using the `HCP_CLIENT_SECRET` environment variable. + + - `resource_id` The HCP resource identifier. This can be overridden using the `HCP_RESOURCE_ID` environment variable. + ## Join Parameters - `rejoin_after_leave` Equivalent to the [`-rejoin` command-line flag](/consul/docs/agent/config/cli-flags#_rejoin). @@ -1586,15 +1613,89 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." ## Raft Parameters -- `raft_boltdb` ((#raft_boltdb)) This is a nested object that allows configuring - options for Raft's BoltDB based log store. - - - `NoFreelistSync` ((#NoFreelistSync)) Setting this to `true` will disable - syncing the BoltDB freelist to disk within the raft.db file. Not syncing - the freelist to disk will reduce disk IO required for write operations - at the expense of potentially increasing start up time due to needing - to scan the db to discover where the free space resides within the file. - +- `raft_boltdb` ((#raft_boltdb)) **These fields are deprecated in Consul v1.15.0. + Use [`raft_logstore`](#raft_logstore) instead.** This is a nested + object that allows configuring options for Raft's BoltDB-based log store. + + - `NoFreelistSync` **This field is deprecated in Consul v1.15.0. Use the + [`raft_logstore.boltdb.no_freelist_sync`](#raft_logstore_boltdb_no_freelist_sync) field + instead.** Setting this to `true` disables syncing the BoltDB freelist + to disk within the raft.db file. Not syncing the freelist to disk + reduces disk IO required for write operations at the expense of potentially + increasing start up time due to needing to scan the db to discover where the + free space resides within the file. + +- `raft_logstore` ((#raft_logstore)) This is a nested object that allows + configuring options for Raft's LogStore component which is used to persist + logs and crucial Raft state on disk during writes. This was added in Consul + v1.15.0. + + - `backend` ((#raft_logstore_backend)) Specifies which storage + engine to use to persist logs. Valid options are `boltdb` or `wal`. Default + is `boltdb`. The `wal` option specifies an experimental backend that + should be used with caution. Refer to + [Experimental WAL LogStore backend](/consul/docs/agent/wal-logstore) + for more information. + + - `disable_log_cache` ((#raft_logstore_disable_log_cache)) Disables the in-memory cache for recent logs. We recommend using it for performance testing purposes, as no significant improvement has been measured when the cache is disabled. While the in-memory log cache theoretically prevents disk reads for recent logs, recent logs are also stored in the OS page cache, which does not slow either the `boltdb` or `wal` backend's ability to read them. + + - `verification` ((#raft_logstore_verification)) This is a nested object that + allows configuring the online verification of the LogStore. Verification + provides additional assurances that LogStore backends are correctly storing + data. It imposes low overhead on servers and is safe to run in + production. It is most useful when evaluating a new backend + implementation. + + Verification must be enabled on the leader to have any effect and can be + used with any backend. When enabled, the leader periodically writes a + special "checkpoint" log message that includes the checksums of all log entries + written to Raft since the last checkpoint. Followers that have verification + enabled run a background task for each checkpoint that reads all logs + directly from the LogStore and then recomputes the checksum. A report is output + as an INFO level log for each checkpoint. + + Checksum failure should never happen and indicate unrecoverable corruption + on that server. The only correct response is to stop the server, remove its + data directory, and restart so it can be caught back up with a correct + server again. Please report verification failures including details about + your hardware and workload via GitHub issues. Refer to + [Experimental WAL LogStore backend](/consul/docs/agent/wal-logstore) + for more information. + + - `enabled` ((#raft_logstore_verification_enabled)) - Set to `true` to + allow this Consul server to write and verify log verification checkpoints + when elected leader. + + - `interval` ((#raft_logstore_verification_interval)) - Specifies the time + interval between checkpoints. There is no default value. You must + configure the `interval` and set [`enabled`](#raft_logstore_verification_enabled) + to `true` to correctly enable intervals. We recommend using an interval + between `30s` and `5m`. The performance overhead is insignificant when the + interval is set to `5m` or less. + + - `boltdb` ((#raft_logstore_boltdb)) - Object that configures options for + Raft's `boltdb` backend. It has no effect if the `backend` is not `boltdb`. + + - `no_freelist_sync` ((#raft_logstore_boltdb_no_freelist_sync)) - Set to + `true` to disable storing BoltDB's freelist to disk within the + `raft.db` file. Disabling freelist syncs reduces the disk IO required + for write operations, but could potentially increase start up time + because Consul must scan the database to find free space + within the file. + + - - `wal` ((#raft_logstore_wal)) - Object that configures the `wal` backend. + Refer to [Experimental WAL LogStore backend](/consul/docs/agent/wal-logstore) + for more information. + + - `segment_size_mb` ((#raft_logstore_wal_segment_size_mb)) - Integer value + that represents the target size in MB for each segment file before + rolling to a new segment. The default value is `64` and is suitable for + most deployments. While a smaller value may use less disk space because you + can reclaim space by deleting old segments sooner, the smaller segment that results + may affect performance because safely rotating to a new file more + frequently can impact tail latencies. Larger values are unlikely + to improve performance significantly. We recommend using this + configuration for performance testing purposes. - `raft_protocol` ((#raft_protocol)) Equivalent to the [`-raft-protocol` command-line flag](/consul/docs/agent/config/cli-flags#_raft_protocol). @@ -1718,8 +1819,7 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." geo location or datacenter, dc:sfo). By default, this is left blank and not used. - `disable_hostname` ((#telemetry-disable_hostname)) - This controls whether or not to prepend runtime telemetry with the machine's - hostname, defaults to false. + Set to `true` to stop prepending the machine's hostname to gauge-type metrics. Default is `false`. - `dogstatsd_addr` ((#telemetry-dogstatsd_addr)) This provides the address of a DogStatsD instance in the format `host:port`. DogStatsD is a protocol-compatible @@ -1731,6 +1831,9 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." of global tags that will be added to all telemetry packets sent to DogStatsD. It is a list of strings, where each string looks like "my_tag_name:my_tag_value". + - `enable_host_metrics` ((#telemetry-enable_host_metrics)) + This enables reporting of host metrics about system resources, defaults to false. + - `filter_default` ((#telemetry-filter_default)) This controls whether to allow metrics that have not been specified by the filter. Defaults to `true`, which will allow all metrics when no filters are provided. @@ -2104,7 +2207,7 @@ specially crafted certificate signed by the CA can be used to gain full access t ~> **Security Note:** `verify_server_hostname` *must* be set to true to prevent a compromised client from gaining full read and write access to all cluster - data *including all ACL tokens and Connect CA root keys*. + data *including all ACL tokens and service mesh CA root keys*. - `server_name` When provided, this overrides the [`node_name`](#_node) for the TLS certificate. It can be used to ensure that the certificate name matches diff --git a/website/content/docs/agent/config/index.mdx b/website/content/docs/agent/config/index.mdx index b2e3ac42c83..0ea4a030fb7 100644 --- a/website/content/docs/agent/config/index.mdx +++ b/website/content/docs/agent/config/index.mdx @@ -72,7 +72,7 @@ The following agent configuration options are reloadable at runtime: - These can be important in certain outage situations so being able to control them without a restart provides a recovery path that doesn't involve downtime. They generally shouldn't be changed otherwise. -- [RPC rate limiting](/consul/docs/agent/config/config-files#limits) +- [RPC rate limits](/consul/docs/agent/config/config-files#limits) - [HTTP Maximum Connections per Client](/consul/docs/agent/config/config-files#http_max_conns_per_client) - Services - TLS Configuration diff --git a/website/content/docs/agent/index.mdx b/website/content/docs/agent/index.mdx index bab9138e50d..c7bf699f87c 100644 --- a/website/content/docs/agent/index.mdx +++ b/website/content/docs/agent/index.mdx @@ -33,7 +33,7 @@ The following process describes the agent lifecycle within the context of an exi As a result, all nodes will eventually become aware of each other. 1. **Existing servers will begin replicating to the new node** if the agent is a server. -### Failures and Crashes +### Failures and crashes In the event of a network failure, some nodes may be unable to reach other nodes. Unreachable nodes will be marked as _failed_. @@ -48,7 +48,7 @@ catalog. Once the network recovers or a crashed agent restarts, the cluster will repair itself and unmark a node as failed. The health check in the catalog will also be updated to reflect the current state. -### Exiting Nodes +### Exiting nodes When a node leaves a cluster, it communicates its intent and the cluster marks the node as having _left_. In contrast to changes related to failures, all of the services provided by a node are immediately deregistered. @@ -61,6 +61,10 @@ interval of 72 hours (changing the reap interval is _not_ recommended due to its consequences during outage situations). Reaping is similar to leaving, causing all associated services to be deregistered. +## Limit traffic rates + +You can define a set of rate limiting configurations that help operators protect Consul servers from excessive or peak usage. The configurations enable you to gracefully degrade Consul servers to avoid a global interruption of service. Consul supports global server rate limiting, which lets configure Consul servers to deny requests that exceed the read or write limits. Refer to [Traffic Rate Limits Overview](/consul/docs/agent/limits). + ## Requirements You should run one Consul agent per server or host. @@ -73,7 +77,7 @@ Refer to the following sections for information about host, port, memory, and ot The [Datacenter Deploy tutorial](/consul/tutorials/production-deploy/reference-architecture#deployment-system-requirements) contains additional information, including licensing configuration, environment variables, and other details. -### Maximum Latency Network requirements +### Maximum latency network requirements Consul uses the gossip protocol to share information across agents. To function properly, you cannot exceed the protocol's maximum latency threshold. The latency threshold is calculated according to the total round trip time (RTT) for communication between all agents. Other network usages outside of Gossip are not bound by these latency requirements (i.e. client to server RPCs, HTTP API requests, xDS proxy configuration, DNS). @@ -82,7 +86,7 @@ For data sent between all Consul agents the following latency requirements must - Average RTT for all traffic cannot exceed 50ms. - RTT for 99 percent of traffic cannot exceed 100ms. -## Starting the Consul Agent +## Starting the Consul agent Start a Consul agent with the `consul` command and `agent` subcommand using the following syntax: @@ -111,7 +115,7 @@ $ consul agent -data-dir=tmp/consul -dev Agents are highly configurable, which enables you to deploy Consul to any infrastructure. Many of the default options for the `agent` command are suitable for becoming familiar with a local instance of Consul. In practice, however, several additional configuration options must be specified for Consul to function as expected. Refer to [Agent Configuration](/consul/docs/agent/config) topic for a complete list of configuration options. -### Understanding the Agent Startup Output +### Understanding the agent startup output Consul prints several important messages on startup. The following example shows output from the [`consul agent`](/consul/commands/agent) command: @@ -162,7 +166,7 @@ When running under `systemd` on Linux, Consul notifies systemd by sending this either the `join` or `retry_join` option has to be set and the service definition file has to have `Type=notify` set. -## Configuring Consul Agents +## Configuring Consul agents You can specify many options to configure how Consul operates when issuing the `consul agent` command. You can also create one or more configuration files and provide them to Consul at startup using either the `-config-file` or `-config-dir` option. @@ -180,7 +184,7 @@ $ consul agent -config-file=server.json The configuration options necessary to successfully use Consul depend on several factors, including the type of agent you are configuring (client or server), the type of environment you are deploying to (e.g., on-premise, multi-cloud, etc.), and the security options you want to implement (ACLs, gRPC encryption). The following examples are intended to help you understand some of the combinations you can implement to configure Consul. -### Common Configuration Settings +### Common configuration settings The following settings are commonly used in the configuration file (also called a service definition file when registering services with Consul) to configure Consul agents: @@ -195,10 +199,10 @@ The following settings are commonly used in the configuration file (also called | `addresses` | Block of nested objects that define addresses bound to the agent for internal cluster communication. | `"http": "0.0.0.0"` See the Agent Configuration page for [default address values](/consul/docs/agent/config/config-files#addresses) | | `ports` | Block of nested objects that define ports bound to agent addresses.
See (link to addresses option) for details. | See the Agent Configuration page for [default port values](/consul/docs/agent/config/config-files#ports) | -### Server Node in a Service Mesh +### Server node in a service mesh The following example configuration is for a server agent named "`consul-server`". The server is [bootstrapped](/consul/docs/agent/config/cli-flags#_bootstrap) and the Consul GUI is enabled. -The reason this server agent is configured for a service mesh is that the `connect` configuration is enabled. Connect is Consul's service mesh component that provides service-to-service connection authorization and encryption using mutual Transport Layer Security (TLS). Applications can use sidecar proxies in a service mesh configuration to establish TLS connections for inbound and outbound connections without being aware of Connect at all. See [Connect](/consul/docs/connect) for details. +The reason this server agent is configured for a service mesh is that the `connect` configuration is enabled. The connect subsystem provides Consul's service mesh capabilities, including service-to-service connection authorization and encryption using mutual Transport Layer Security (TLS). Applications can use sidecar proxies in a service mesh configuration to establish TLS connections for inbound and outbound connections without being aware of Consul service mesh at all. Refer to [Consul Service Mesh](/consul/docs/connect) for details. @@ -243,7 +247,7 @@ connect { -### Server Node with Encryption Enabled +### Server node with encryption enabled The following example shows a server node configured with encryption enabled. Refer to the [Security](/consul/docs/security) chapter for additional information about how to configure security options for Consul. @@ -313,10 +317,10 @@ tls { -### Client Node Registering a Service +### Client node registering a service Using Consul as a central service registry is a common use case. -The following example configuration includes common settings to register a service with a Consul agent and enable health checks (see [Checks](/consul/docs/discovery/checks) to learn more about health checks): +The following example configuration includes common settings to register a service with a Consul agent and enable health checks. Refer to [Define Health Checks](/consul/docs/services/usage/checks) to learn more about health checks. @@ -371,7 +375,7 @@ service { -## Client Node with Multiple Interfaces or IP addresses +## Client node with multiple interfaces or IP addresses The following example shows how to configure Consul to listen on multiple interfaces or IP addresses using a [go-sockaddr template]. @@ -422,7 +426,7 @@ advertise_addr = "{{ GetInterfaceIP \"en0\" }}" -## Stopping an Agent +## Stopping an agent An agent can be stopped in two ways: gracefully or forcefully. Servers and Clients both behave differently depending on the leave that is performed. There diff --git a/website/content/docs/agent/limits/index.mdx b/website/content/docs/agent/limits/index.mdx new file mode 100644 index 00000000000..75fb4f1ac0e --- /dev/null +++ b/website/content/docs/agent/limits/index.mdx @@ -0,0 +1,31 @@ +--- +layout: docs +page_title: Limit Traffic Rates Overview +description: Rate limiting is a set of Consul server agent configurations that you can use to mitigate the risks to Consul servers when clients send excessive requests to Consul resources. + +--- + +# Limit traffic rates overview + +This topic provides overview information about the traffic rates limits you can configure for Consul servers. + +## Introduction +You can configure global RPC rate limits to mitigate the risks to Consul servers when clients send excessive read or write requests to Consul resources. A read request is defined as any request that does not modify Consul internal state. A write request is defined as any request that modifies Consul internal state. Read and write requests are limited separately. + +## Rate limit modes +You can set one of the following modes, which determine how Consul servers react when the request limits are exceeded. + +- **Enforcing mode**: In this mode, the rate limiter denies requests to a server beyond a configurable rate. Consul generates metrics and logs to help operators understand their Consul load and configure limits accordingly. +- **Permissive mode**: The rate limiter allows requests if the limits are reached and produces metrics and logs to help operators understand their Consul load and configure limits accordingly. This mode is intended to help you configure limits and debug specific issues. +- **Disabled mode**: Disables the rate limiter. All requests are allowed and no logs or metrics are produced. This is the default mode. + +Refer to [`rate_limits`](/consul/docs/agent/config/config-files#request_limits) for additional configuration information. + +## Request denials +When an HTTP request is denied for rate limiting reason, Consul returns one of the following errors: + +- **429 Resource Exhausted**: Indicates that a server is not able to perform the request but that another server could potentially fulfill it. This error is most common on stale reads because any server may fulfill state read requests. To resolve this type of error, we recommend immediately retrying the request to another server. If the request came from a Consul client agent, the agent automatically retries the request up to the limit set in the [`rpc_hold_timeout`](/consul/docs/agent/config/config-files#rpc_hold_timeout) configuration . + +- **503 Service Unavailable**: Indicates that server is unable to perform the request and that no other server can fulfill the request, either. This usually occurs on consistent reads or for writes. In this case we recommend retrying according to an exponential backoff schedule. If the request came from a Consul client agent, the agent automatically retries the request according to the [`rpc_hold_timeout`](/consul/docs/agent/config/config-files#rpc_hold_timeout) configuration. + +Refer to [Rate limit reached on the server](/consul/docs/troubleshoot/common-errors#rate-limit-reached-on-the-server) for additional information. \ No newline at end of file diff --git a/website/content/docs/agent/limits/init-rate-limits.mdx b/website/content/docs/agent/limits/init-rate-limits.mdx new file mode 100644 index 00000000000..32057184530 --- /dev/null +++ b/website/content/docs/agent/limits/init-rate-limits.mdx @@ -0,0 +1,32 @@ +--- +layout: docs +page_title: Initialize Rate Limit Settings +description: Learn how to determins regular and peak loads in your network so that you can set the initial global rate limit configurations. +--- + +# Initialize rate limit settings + +In order to set limits for traffic, you must first understand regular and peak loads in your network. We recommend completing the following steps to benchmark request rates in your environment so that you can implement limits appropriate for your applications. + +1. Specify a global rate limit with arbitrary values in the agent configuration file based on the following conditions: + + - Environment where Consul servers are running + - Number of servers and the projected load + - Existing metrics expressing requests per second + +1. Set the `mode` to `permissive`. In the following example, Consul agents are allowed up to 1000 reads and 500 writes per second: + + ```hcl + request_limits { + mode = "permissive" + read_rate = 1000.0 + write_rate =500.0 + } + ``` + +1. Observe the logs and metrics for your application's typical cycle, such as a 24 hour period. Refer to [`log_file`](/consul/docs/agent/config/config-files#log_file) for information about where to retrieve logs. Call the [`/agent/metrics`](/consul/api-docs/agent#view-metrics) HTTP API endpoint and check the data for the following metrics: + + - `rpc.rate_limit.exceeded` with value `global/read` for label `limit_type` + - `rpc.rate_limit.exceeded` with value `global/write` for label `limit_type` + +1. If the limits are not reached, set the `mode` configuration to `enforcing`. Otherwise adjust and iterate limits. \ No newline at end of file diff --git a/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx b/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx new file mode 100644 index 00000000000..369d1b7b583 --- /dev/null +++ b/website/content/docs/agent/limits/set-global-traffic-rate-limits.mdx @@ -0,0 +1,108 @@ +--- +layout: docs +page_title: Set a Global Limit on Traffic Rates +description: Use global rate limits to prevent excessive rates of requests to Consul servers. +--- + +# Set a global limit on traffic rates + +This topic describes how to configure rate limits for RPC and gRPC traffic to the Consul server. + +## Introduction +Rate limits apply to each Consul server separately and are intended to limit the number of read requests or write requests to the server on the RPC and internal gRPC endpoints. + +Because all requests coming to a Consul server eventually perform an RPC or an internal gRPC request, these limits also apply to other user interfaces, such as the HTTP API interface, the CLI, and the external gRPC endpoint for services in the Consul service mesh. + +Refer to [Initialize Rate Limit Settings](/consul/docs/agent/limits/init-rate-limits) for additional information about right-sizing your gRPC request configurations. + +## Set a global rate limit for a Consul server +Configure the following settings in your Consul server configuration to limit the RPC and gRPC traffic rates. + +- Set the rate limiter [`mode`](/consul/docs/agent/config/config-files#mode-1) +- Set the [`read_rate`](/consul/docs/agent/config/config-files#read_rate) +- Set the [`write_rate`](/consul/docs/agent/config/config-files#write_rate) + +In the following example, the Consul server is configured to prevent more than `500` read and `200` write RPC calls: + + + +```hcl +limits = { + rate_limit = { + mode = "enforcing" + read_rate = 500 + write_rate = 200 + } +} +``` + +```json +{ + "limits" : { + "rate_limit" : { + "mode" : "enforcing", + "read_rate" : 500, + "write_rate" : 200 + } + } +} + +``` + + + +## Access rate limit logs +Consul prints a log line for each rate limit request. The log provides the information necessary for identifying the source of the request and the configured limit. Consul prints the log `DEBUG` log level and can drop the log to avoid affecting the server health. Dropping a log line increments the `rpc.rate_limit.log_dropped` metric. + +The following example log shows that RPC request from `127.0.0.1:53562` to `KVS.Apply` exceeded the limit: + + + +```shell-session +2023-02-17T10:01:15.565-0500 [DEBUG] agent.server.rpc-rate-limit: RPC +exceeded allowed rate limit: rpc=KVS.Apply source_addr=127.0.0.1:53562 +limit_type=global/write limit_enforced=false +``` + + +## Review rate limit metrics +Consul captures the following metrics associated with rate limits: + +- Type of limit +- Operation +- Rate limit mode + +Call the `agent/metrics` API endpoint to view the metrics associated with rate limits. Refer to [View Metrics](/consul/api-docs/agent#view-metrics) for API usage information. In the following example, Consul dropped a call to the `consul` service because it exceeded the limit by one call: + +```shell-session +$ curl http://127.0.0.1:8500/v1/agent/metrics +{ + . . . + "Counters": [ + { + "Name": "consul.rpc.rate_limit.exceeded", + "Count": 1, + "Sum": 1, + "Min": 1, + "Max": 1, + "Mean": 1, + "Stddev": 0, + "Labels": { + "service": "consul" + } + }, + { + "Name": "consul.rpc.rate_limit.log_dropped", + "Count": 1, + "Sum": 1, + "Min": 1, + "Max": 1, + "Mean": 1, + "Stddev": 0, + "Labels": {} + } + ], + . . . +``` + +Refer to [Telemetry]() for additional information. diff --git a/website/content/docs/agent/telemetry.mdx b/website/content/docs/agent/telemetry.mdx index 53d7ed91766..3d7eb97aff2 100644 --- a/website/content/docs/agent/telemetry.mdx +++ b/website/content/docs/agent/telemetry.mdx @@ -27,6 +27,10 @@ This telemetry information can be used for debugging or otherwise getting a better view of what Consul is doing. Review the [Monitoring and Metrics tutorial](/consul/tutorials/day-2-operations/monitor-datacenter-health?utm_source=docs) to learn how collect and interpret Consul data. +By default, all metric names of gauge type are prefixed with the hostname of the consul agent, e.g., +`consul..server.isLeader`. To disable prefixing the hostname, set +`telemetry.disable_hostname=true` in the [agent configuration](/consul/docs/agent/config/config-files#telemetry). + Additionally, if the [`telemetry` configuration options](/consul/docs/agent/config/config-files#telemetry) are provided, the telemetry information will be streamed to a [statsite](http://github.com/armon/statsite) or [statsd](http://github.com/etsy/statsd) server where @@ -138,9 +142,9 @@ you will need to apply a function such as InfluxDB's [`non_negative_difference() | Metric Name | Description | Unit | Type | | :--------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------- | :------ | -| `consul.client.rpc` | Increments whenever a Consul agent in client mode makes an RPC request to a Consul server | requests | counter | -| `consul.client.rpc.exceeded` | Increments whenever a Consul agent in client mode makes an RPC request to a Consul server gets rate limited by that agent's [`limits`](/consul/docs/agent/config/config-files#limits) configuration. | requests | counter | -| `consul.client.rpc.failed` | Increments whenever a Consul agent in client mode makes an RPC request to a Consul server and fails. | requests | counter | +| `consul.client.rpc` | Increments whenever a Consul agent makes an RPC request to a Consul server | requests | counter | +| `consul.client.rpc.exceeded` | Increments whenever a Consul agent makes an RPC request to a Consul server gets rate limited by that agent's [`limits`](/consul/docs/agent/config/config-files#limits) configuration. | requests | counter | +| `consul.client.rpc.failed` | Increments whenever a Consul agent makes an RPC request to a Consul server and fails. | requests | counter | **Why they're important:** These measurements indicate the current load created from a Consul agent, including when the load becomes high enough to be rate limited. A high RPC count, especially from `consul.client.rpcexceeded` meaning that the requests are being rate-limited, could imply a misconfigured Consul agent. @@ -294,7 +298,7 @@ This metric should be monitored to ensure that the license doesn't expire to pre | Metric Name | Description | Unit | Type | | :-------------------------------- | :--------------------------------------------------------------- | :---- | :---- | -| `consul.raft.boltdb.freelistBytes` | Represents the number of bytes necessary to encode the freelist metadata. When [`raft_boltdb.NoFreelistSync`](/consul/docs/agent/config/config-files#NoFreelistSync) is set to `false` these metadata bytes must also be written to disk for each committed log. | bytes | gauge | +| `consul.raft.boltdb.freelistBytes` | Represents the number of bytes necessary to encode the freelist metadata. When [`raft_logstore.boltdb.no_freelist_sync`](/consul/docs/agent/config/config-files#raft_logstore_boltdb_no_freelist_sync) is set to `false` these metadata bytes must also be written to disk for each committed log. | bytes | gauge | | `consul.raft.boltdb.logsPerBatch` | Measures the number of logs being written per batch to the db. | logs | sample | | `consul.raft.boltdb.storeLogs` | Measures the amount of time spent writing logs to the db. | ms | timer | | `consul.raft.boltdb.writeCapacity` | Theoretical write capacity in terms of the number of logs that can be written per second. Each sample outputs what the capacity would be if future batched log write operations were similar to this one. This similarity encompasses 4 things: batch size, byte size, disk performance and boltdb performance. While none of these will be static and its highly likely individual samples of this metric will vary, aggregating this metric over a larger time window should provide a decent picture into how this BoltDB store can perform | logs/second | sample | @@ -337,11 +341,12 @@ indicator of an actual issue, this metric can be used to diagnose why the `consu is high. If Bolt DB log storage performance becomes an issue and is caused by free list management then setting -[`raft_boltdb.NoFreelistSync`](/consul/docs/agent/config/config-files#NoFreelistSync) to `true` in the server's configuration +[`raft_logstore.boltdb.no_freelist_sync`](/consul/docs/agent/config/config-files#raft_logstore_boltdb_no_freelist_sync) to `true` in the server's configuration may help to reduce disk IO and log storage operation times. Disabling free list syncing will however increase the startup time for a server as it must scan the raft.db file for free space instead of loading the already populated free list structure. +Consul includes an experiment backend configuration that you can use instead of BoldDB. Refer to [Experimental WAL LogStore backend](/consul/docs/agent/wal-logstore) for more information. ## Metrics Reference @@ -352,37 +357,37 @@ This is a full list of metrics emitted by Consul. | `consul.acl.blocked.{check,service}.deregistration` | Increments whenever a deregistration fails for an entity (check or service) is blocked by an ACL. | requests | counter | | `consul.acl.blocked.{check,node,service}.registration` | Increments whenever a registration fails for an entity (check, node or service) is blocked by an ACL. | requests | counter | | `consul.api.http` | This samples how long it takes to service the given HTTP request for the given verb and path. Includes labels for `path` and `method`. `path` does not include details like service or key names, for these an underscore will be present as a placeholder (eg. path=`v1.kv._`) | ms | timer | -| `consul.client.rpc` | Increments whenever a Consul agent in client mode makes an RPC request to a Consul server. This gives a measure of how much a given agent is loading the Consul servers. Currently, this is only generated by agents in client mode, not Consul servers. | requests | counter | -| `consul.client.rpc.exceeded` | Increments whenever a Consul agent in client mode makes an RPC request to a Consul server gets rate limited by that agent's [`limits`](/consul/docs/agent/config/config-files#limits) configuration. This gives an indication that there's an abusive application making too many requests on the agent, or that the rate limit needs to be increased. Currently, this only applies to agents in client mode, not Consul servers. | rejected requests | counter | -| `consul.client.rpc.failed` | Increments whenever a Consul agent in client mode makes an RPC request to a Consul server and fails. | requests | counter | -| `consul.client.api.catalog_register.` | Increments whenever a Consul agent receives a catalog register request. | requests | counter | -| `consul.client.api.success.catalog_register.` | Increments whenever a Consul agent successfully responds to a catalog register request. | requests | counter | -| `consul.client.rpc.error.catalog_register.` | Increments whenever a Consul agent receives an RPC error for a catalog register request. | errors | counter | -| `consul.client.api.catalog_deregister.` | Increments whenever a Consul agent receives a catalog deregister request. | requests | counter | -| `consul.client.api.success.catalog_deregister.` | Increments whenever a Consul agent successfully responds to a catalog deregister request. | requests | counter | -| `consul.client.rpc.error.catalog_deregister.` | Increments whenever a Consul agent receives an RPC error for a catalog deregister request. | errors | counter | -| `consul.client.api.catalog_datacenters.` | Increments whenever a Consul agent receives a request to list datacenters in the catalog. | requests | counter | -| `consul.client.api.success.catalog_datacenters.` | Increments whenever a Consul agent successfully responds to a request to list datacenters. | requests | counter | -| `consul.client.rpc.error.catalog_datacenters.` | Increments whenever a Consul agent receives an RPC error for a request to list datacenters. | errors | counter | -| `consul.client.api.catalog_nodes.` | Increments whenever a Consul agent receives a request to list nodes from the catalog. | requests | counter | -| `consul.client.api.success.catalog_nodes.` | Increments whenever a Consul agent successfully responds to a request to list nodes. | requests | counter | -| `consul.client.rpc.error.catalog_nodes.` | Increments whenever a Consul agent receives an RPC error for a request to list nodes. | errors | counter | -| `consul.client.api.catalog_services.` | Increments whenever a Consul agent receives a request to list services from the catalog. | requests | counter | -| `consul.client.api.success.catalog_services.` | Increments whenever a Consul agent successfully responds to a request to list services. | requests | counter | -| `consul.client.rpc.error.catalog_services.` | Increments whenever a Consul agent receives an RPC error for a request to list services. | errors | counter | -| `consul.client.api.catalog_service_nodes.` | Increments whenever a Consul agent receives a request to list nodes offering a service. | requests | counter | -| `consul.client.api.success.catalog_service_nodes.` | Increments whenever a Consul agent successfully responds to a request to list nodes offering a service. | requests | counter | -| `consul.client.api.error.catalog_service_nodes.` | Increments whenever a Consul agent receives an RPC error for request to list nodes offering a service. | requests | counter | -| `consul.client.rpc.error.catalog_service_nodes.` | Increments whenever a Consul agent receives an RPC error for a request to list nodes offering a service.   | errors | counter | -| `consul.client.api.catalog_node_services.` | Increments whenever a Consul agent receives a request to list services registered in a node.   | requests | counter | -| `consul.client.api.success.catalog_node_services.` | Increments whenever a Consul agent successfully responds to a request to list services in a node.   | requests | counter | -| `consul.client.rpc.error.catalog_node_services.` | Increments whenever a Consul agent receives an RPC error for a request to list services in a node.   | errors | counter | +| `consul.client.rpc` | Increments whenever a Consul agent makes an RPC request to a Consul server. This gives a measure of how much a given agent is loading the Consul servers. Currently, this is only generated by agents in client mode, not Consul servers. | requests | counter | +| `consul.client.rpc.exceeded` | Increments whenever a Consul agent makes an RPC request to a Consul server gets rate limited by that agent's [`limits`](/consul/docs/agent/config/config-files#limits) configuration. This gives an indication that there's an abusive application making too many requests on the agent, or that the rate limit needs to be increased. Currently, this only applies to agents in client mode, not Consul servers. | rejected requests | counter | +| `consul.client.rpc.failed` | Increments whenever a Consul agent makes an RPC request to a Consul server and fails. | requests | counter | +| `consul.client.api.catalog_register` | Increments whenever a Consul agent receives a catalog register request. | requests | counter | +| `consul.client.api.success.catalog_register` | Increments whenever a Consul agent successfully responds to a catalog register request. | requests | counter | +| `consul.client.rpc.error.catalog_register` | Increments whenever a Consul agent receives an RPC error for a catalog register request. | errors | counter | +| `consul.client.api.catalog_deregister` | Increments whenever a Consul agent receives a catalog deregister request. | requests | counter | +| `consul.client.api.success.catalog_deregister` | Increments whenever a Consul agent successfully responds to a catalog deregister request. | requests | counter | +| `consul.client.rpc.error.catalog_deregister` | Increments whenever a Consul agent receives an RPC error for a catalog deregister request. | errors | counter | +| `consul.client.api.catalog_datacenters` | Increments whenever a Consul agent receives a request to list datacenters in the catalog. | requests | counter | +| `consul.client.api.success.catalog_datacenters` | Increments whenever a Consul agent successfully responds to a request to list datacenters. | requests | counter | +| `consul.client.rpc.error.catalog_datacenters` | Increments whenever a Consul agent receives an RPC error for a request to list datacenters. | errors | counter | +| `consul.client.api.catalog_nodes` | Increments whenever a Consul agent receives a request to list nodes from the catalog. | requests | counter | +| `consul.client.api.success.catalog_nodes` | Increments whenever a Consul agent successfully responds to a request to list nodes. | requests | counter | +| `consul.client.rpc.error.catalog_nodes` | Increments whenever a Consul agent receives an RPC error for a request to list nodes. | errors | counter | +| `consul.client.api.catalog_services` | Increments whenever a Consul agent receives a request to list services from the catalog. | requests | counter | +| `consul.client.api.success.catalog_services` | Increments whenever a Consul agent successfully responds to a request to list services. | requests | counter | +| `consul.client.rpc.error.catalog_services` | Increments whenever a Consul agent receives an RPC error for a request to list services. | errors | counter | +| `consul.client.api.catalog_service_nodes` | Increments whenever a Consul agent receives a request to list nodes offering a service. | requests | counter | +| `consul.client.api.success.catalog_service_nodes` | Increments whenever a Consul agent successfully responds to a request to list nodes offering a service. | requests | counter | +| `consul.client.api.error.catalog_service_nodes` | Increments whenever a Consul agent receives an RPC error for request to list nodes offering a service. | requests | counter | +| `consul.client.rpc.error.catalog_service_nodes` | Increments whenever a Consul agent receives an RPC error for a request to list nodes offering a service.   | errors | counter | +| `consul.client.api.catalog_node_services` | Increments whenever a Consul agent receives a request to list services registered in a node.   | requests | counter | +| `consul.client.api.success.catalog_node_services` | Increments whenever a Consul agent successfully responds to a request to list services in a node.   | requests | counter | +| `consul.client.rpc.error.catalog_node_services` | Increments whenever a Consul agent receives an RPC error for a request to list services in a node.   | errors | counter | | `consul.client.api.catalog_node_service_list` | Increments whenever a Consul agent receives a request to list a node's registered services. | requests | counter | | `consul.client.rpc.error.catalog_node_service_list` | Increments whenever a Consul agent receives an RPC error for request to list a node's registered services. | errors | counter | | `consul.client.api.success.catalog_node_service_list` | Increments whenever a Consul agent successfully responds to a request to list a node's registered services. | requests | counter | -| `consul.client.api.catalog_gateway_services.` | Increments whenever a Consul agent receives a request to list services associated with a gateway. | requests | counter | -| `consul.client.api.success.catalog_gateway_services.` | Increments whenever a Consul agent successfully responds to a request to list services associated with a gateway. | requests | counter | -| `consul.client.rpc.error.catalog_gateway_services.` | Increments whenever a Consul agent receives an RPC error for a request to list services associated with a gateway. | errors | counter | +| `consul.client.api.catalog_gateway_services` | Increments whenever a Consul agent receives a request to list services associated with a gateway. | requests | counter | +| `consul.client.api.success.catalog_gateway_services` | Increments whenever a Consul agent successfully responds to a request to list services associated with a gateway. | requests | counter | +| `consul.client.rpc.error.catalog_gateway_services` | Increments whenever a Consul agent receives an RPC error for a request to list services associated with a gateway. | errors | counter | | `consul.runtime.num_goroutines` | Tracks the number of running goroutines and is a general load pressure indicator. This may burst from time to time but should return to a steady state value. | number of goroutines | gauge | | `consul.runtime.alloc_bytes` | Measures the number of bytes allocated by the Consul process. This may burst from time to time but should return to a steady state value. | bytes | gauge | | `consul.runtime.heap_objects` | Measures the number of objects allocated on the heap and is a general memory pressure indicator. This may burst from time to time but should return to a steady state value. | number of objects | gauge | @@ -391,13 +396,13 @@ This is a full list of metrics emitted by Consul. | `consul.state.services` | Measures the current number of unique services registered with Consul, based on service name. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | | `consul.state.service_instances` | Measures the current number of unique service instances registered with Consul. It is only emitted by Consul servers. Added in v1.9.0. | number of objects | gauge | | `consul.state.kv_entries` | Measures the current number of entries in the Consul KV store. It is only emitted by Consul servers. Added in v1.10.3. | number of objects | gauge | -| `consul.state.connect_instances` | Measures the current number of unique connect service instances registered with Consul labeled by Kind (e.g. connect-proxy, connect-native, etc). Added in v1.10.4 | number of objects | gauge | +| `consul.state.connect_instances` | Measures the current number of unique mesh service instances registered with Consul labeled by Kind (e.g. connect-proxy, connect-native, etc). Added in v1.10.4 | number of objects | gauge | | `consul.state.config_entries` | Measures the current number of configuration entries registered with Consul labeled by Kind (e.g. service-defaults, proxy-defaults, etc). See [Configuration Entries](/consul/docs/connect/config-entries) for more information. Added in v1.10.4 | number of objects | gauge | | `consul.members.clients` | Measures the current number of client agents registered with Consul. It is only emitted by Consul servers. Added in v1.9.6. | number of clients | gauge | | `consul.members.servers` | Measures the current number of server agents registered with Consul. It is only emitted by Consul servers. Added in v1.9.6. | number of servers | gauge | | `consul.dns.stale_queries` | Increments when an agent serves a query within the allowed stale threshold. | queries | counter | -| `consul.dns.ptr_query.` | Measures the time spent handling a reverse DNS query for the given node. | ms | timer | -| `consul.dns.domain_query.` | Measures the time spent handling a domain query for the given node. | ms | timer | +| `consul.dns.ptr_query` | Measures the time spent handling a reverse DNS query for the given node. | ms | timer | +| `consul.dns.domain_query` | Measures the time spent handling a domain query for the given node. | ms | timer | | `consul.system.licenseExpiration` | This measures the number of hours remaining on the agents license. | hours | gauge | | `consul.version` | Represents the Consul version. | agents | gauge | @@ -418,7 +423,7 @@ These metrics are used to monitor the health of the Consul servers. | `consul.raft.applied_index` | Represents the raft applied index. | index | gauge | | `consul.raft.apply` | Counts the number of Raft transactions occurring over the interval, which is a general indicator of the write load on the Consul servers. | raft transactions / interval | counter | | `consul.raft.barrier` | Counts the number of times the agent has started the barrier i.e the number of times it has issued a blocking call, to ensure that the agent has all the pending operations that were queued, to be applied to the agent's FSM. | blocks / interval | counter | -| `consul.raft.boltdb.freelistBytes` | Represents the number of bytes necessary to encode the freelist metadata. When [`raft_boltdb.NoFreelistSync`](/consul/docs/agent/config/config-files#NoFreelistSync) is set to `false` these metadata bytes must also be written to disk for each committed log. | bytes | gauge | +| `consul.raft.boltdb.freelistBytes` | Represents the number of bytes necessary to encode the freelist metadata. When [`raft_logstore.boltdb.no_freelist_sync`](/consul/docs/agent/config/config-files#raft_logstore_boltdb_no_freelist_sync) is set to `false` these metadata bytes must also be written to disk for each committed log. | bytes | gauge | | `consul.raft.boltdb.freePageBytes` | Represents the number of bytes of free space within the raft.db file. | bytes | gauge | | `consul.raft.boltdb.getLog` | Measures the amount of time spent reading logs from the db. | ms | timer | | `consul.raft.boltdb.logBatchSize` | Measures the total size in bytes of logs being written to the db in a single batch. | bytes | sample | @@ -452,12 +457,17 @@ These metrics are used to monitor the health of the Consul servers. | `consul.raft.last_index` | Represents the raft applied index. | index | gauge | | `consul.raft.leader.dispatchLog` | Measures the time it takes for the leader to write log entries to disk. | ms | timer | | `consul.raft.leader.dispatchNumLogs` | Measures the number of logs committed to disk in a batch. | logs | gauge | +| `consul.raft.logstore.verifier.checkpoints_written` | Counts the number of checkpoint entries written to the LogStore. | checkpoints | counter | +| `consul.raft.logstore.verifier.dropped_reports` | Counts how many times the verifier routine was still busy when the next checksum came in and so verification for a range was skipped. If you see this happen, consider increasing the interval between checkpoints with [`raft_logstore.verification.interval`](/consul/docs/agent/config/config-files#raft_logstore_verification) | reports dropped | counter | +| `consul.raft.logstore.verifier.ranges_verified` | Counts the number of log ranges for which a verification report has been completed. Refer to [Monitor Raft metrics and logs for WAL](/consul/docs/agent/wal-logstore/monitoring) for more information. | log ranges verifications | counter | +| `consul.raft.logstore.verifier.read_checksum_failures` | Counts the number of times a range of logs between two check points contained at least one disk corruption. Refer to [Monitor Raft metrics and logs for WAL](/consul/docs/agent/wal-logstore/monitoring) for more information. | disk corruptions | counter | +| `consul.raft.logstore.verifier.write_checksum_failures` | Counts the number of times a follower has a different checksum to the leader at the point where it writes to the log. This could be caused by either a disk-corruption on the leader (unlikely) or some other corruption of the log entries in-flight. | in-flight corruptions | counter | | `consul.raft.leader.lastContact` | Measures the time since the leader was last able to contact the follower nodes when checking its leader lease. It can be used as a measure for how stable the Raft timing is and how close the leader is to timing out its lease.The lease timeout is 500 ms times the [`raft_multiplier` configuration](/consul/docs/agent/config/config-files#raft_multiplier), so this telemetry value should not be getting close to that configured value, otherwise the Raft timing is marginal and might need to be tuned, or more powerful servers might be needed. See the [Server Performance](/consul/docs/install/performance) guide for more details. | ms | timer | | `consul.raft.leader.oldestLogAge` | The number of milliseconds since the _oldest_ log in the leader's log store was written. This can be important for replication health where write rate is high and the snapshot is large as followers may be unable to recover from a restart if restoring takes longer than the minimum value for the current leader. Compare this with `consul.raft.fsm.lastRestoreDuration` and `consul.raft.rpc.installSnapshot` to monitor. In normal usage this gauge value will grow linearly over time until a snapshot completes on the leader and the log is truncated. Note: this metric won't be emitted until the leader writes a snapshot. After an upgrade to Consul 1.10.0 it won't be emitted until the oldest log was written after the upgrade. | ms | gauge | | `consul.raft.replication.heartbeat` | Measures the time taken to invoke appendEntries on a peer, so that it doesn't timeout on a periodic basis. | ms | timer | | `consul.raft.replication.appendEntries` | Measures the time it takes to replicate log entries to followers. This is a general indicator of the load pressure on the Consul servers, as well as the performance of the communication between the servers. | ms | timer | -| `consul.raft.replication.appendEntries.rpc` | Measures the time taken by the append entries RFC, to replicate the log entries of a leader agent onto its follower agent(s) | ms | timer | -| `consul.raft.replication.appendEntries.logs` | Measures the number of logs replicated to an agent, to bring it up to speed with the leader's logs. | logs appended/ interval | counter | +| `consul.raft.replication.appendEntries.rpc` | Measures the time taken by the append entries RPC to replicate the log entries of a leader agent onto its follower agent(s). | ms | timer | +| `consul.raft.replication.appendEntries.logs` | Counts the number of logs replicated to an agent to bring it up to speed with the leader's logs. | logs appended/ interval | counter | | `consul.raft.restore` | Counts the number of times the restore operation has been performed by the agent. Here, restore refers to the action of raft consuming an external snapshot to restore its state. | operation invoked / interval | counter | | `consul.raft.restoreUserSnapshot` | Measures the time taken by the agent to restore the FSM state from a user's snapshot | ms | timer | | `consul.raft.rpc.appendEntries` | Measures the time taken to process an append entries RPC call from an agent. | ms | timer | @@ -476,7 +486,18 @@ These metrics are used to monitor the health of the Consul servers. | `consul.raft.state.follower` | Counts the number of times an agent has entered the follower mode. This happens when a new agent joins the cluster or after the end of a leader election. | follower state entered / interval | counter | | `consul.raft.transition.heartbeat_timeout` | The number of times an agent has transitioned to the Candidate state, after receive no heartbeat messages from the last known leader. | timeouts / interval | counter | | `consul.raft.verify_leader` | This metric doesn't have a direct correlation to the leader change. It just counts the number of times an agent checks if it is still the leader or not. For example, during every consistent read, the check is done. Depending on the load in the system, this metric count can be high as it is incremented each time a consistent read is completed. | checks / interval | Counter | -| `consul.rpc.accept_conn` | Increments when a server accepts an RPC connection. | connections | counter | +| `consul.raft.wal.head_truncations` | Counts how many log entries have been truncated from the head - i.e. the oldest entries. by graphing the rate of change over time you can see individual truncate calls as spikes. | logs entries truncated | counter | +| `consul.raft.wal.last_segment_age_seconds` | A gauge that is set each time we rotate a segment and describes the number of seconds between when that segment file was first created and when it was sealed. this gives a rough estimate how quickly writes are filling the disk. | seconds | gauge | +| `consul.raft.wal.log_appends` | Counts the number of calls to StoreLog(s) i.e. number of batches of entries appended. | calls | counter | +| `consul.raft.wal.log_entries_read` | Counts the number of log entries read. | log entries read | counter | +| `consul.raft.wal.log_entries_written` | Counts the number of log entries written. | log entries written | counter | +| `consul.raft.wal.log_entry_bytes_read` | Counts the bytes of log entry read from segments before decoding. actual bytes read from disk might be higher as it includes headers and index entries and possible secondary reads for large entries that don't fit in buffers. | bytes | counter | +| `consul.raft.wal.log_entry_bytes_written` | Counts the bytes of log entry after encoding with Codec. Actual bytes written to disk might be slightly higher as it includes headers and index entries. | bytes | counter | +| `consul.raft.wal.segment_rotations` | Counts how many times we move to a new segment file. | rotations | counter | +| `consul.raft.wal.stable_gets` | Counts how many calls to StableStore.Get or GetUint64. | calls | counter | +| `consul.raft.wal.stable_sets` | Counts how many calls to StableStore.Set or SetUint64. | calls | counter | +| `consul.raft.wal.tail_truncations` | Counts how many log entries have been truncated from the head - i.e. the newest entries. by graphing the rate of change over time you can see individual truncate calls as spikes. | logs entries truncated | counter | +| `consul.rpc.accept_conn` | Increments when a server accepts an RPC connection. | connections | counter | | `consul.rpc.rate_limit.exceeded` | Increments whenever an RPC is over a configured rate limit. In permissive mode, the RPC is still allowed to proceed. | RPCs | counter | | `consul.rpc.rate_limit.log_dropped` | Increments whenever a log that is emitted because an RPC exceeded a rate limit gets dropped because the output buffer is full. | log messages dropped | counter | | `consul.catalog.register` | Measures the time it takes to complete a catalog register operation. | ms | timer | @@ -518,12 +539,12 @@ These metrics are used to monitor the health of the Consul servers. | `consul.leader.replication.namespaces.status` | This will only be emitted by the leader in a secondary datacenter. The value will be a 1 if the last round of namespace replication was successful or 0 if there was an error. | healthy | gauge | | `consul.leader.replication.namespaces.index` | This will only be emitted by the leader in a secondary datacenter. Increments to the index of namespaces in the primary datacenter that have been successfully replicated. | index | gauge | | `consul.prepared-query.apply` | Measures the time it takes to apply a prepared query update. | ms | timer | -| `consul.prepared-query.explain` | Measures the time it takes to process a prepared query explain request. | ms | timer | -| `consul.prepared-query.execute` | Measures the time it takes to process a prepared query execute request. | ms | timer | | `consul.prepared-query.execute_remote` | Measures the time it takes to process a prepared query execute request that was forwarded to another datacenter. | ms | timer | +| `consul.prepared-query.execute` | Measures the time it takes to process a prepared query execute request. | ms | timer | +| `consul.prepared-query.explain` | Measures the time it takes to process a prepared query explain request. | ms | timer | | `consul.rpc.raft_handoff` | Increments when a server accepts a Raft-related RPC connection. | connections | counter | -| `consul.rpc.request_error` | Increments when a server returns an error from an RPC request. | errors | counter | | `consul.rpc.request` | Increments when a server receives a Consul-related RPC request. | requests | counter | +| `consul.rpc.request_error` | Increments when a server returns an error from an RPC request. | errors | counter | | `consul.rpc.query` | Increments when a server receives a read RPC request, indicating the rate of new read queries. See consul.rpc.queries_blocking for the current number of in-flight blocking RPC calls. This metric changed in 1.7.0 to only increment on the the start of a query. The rate of queries will appear lower, but is more accurate. | queries | counter | | `consul.rpc.queries_blocking` | The current number of in-flight blocking queries the server is handling. | queries | gauge | | `consul.rpc.cross-dc` | Increments when a server sends a (potentially blocking) cross datacenter RPC query. | queries | counter | @@ -647,17 +668,17 @@ are allowed for . | `consul.catalog.service.query-tag` | Increments for each catalog query for the given service with the given tag. | queries | counter | | `consul.catalog.service.query-tags` | Increments for each catalog query for the given service with the given tags. | queries | counter | | `consul.catalog.service.not-found` | Increments for each catalog query where the given service could not be found. | queries | counter | -| `consul.catalog.connect.query` | Increments for each connect-based catalog query for the given service. | queries | counter | -| `consul.catalog.connect.query-tag` | Increments for each connect-based catalog query for the given service with the given tag. | queries | counter | -| `consul.catalog.connect.query-tags` | Increments for each connect-based catalog query for the given service with the given tags. | queries | counter | -| `consul.catalog.connect.not-found` | Increments for each connect-based catalog query where the given service could not be found. | queries | counter | +| `consul.catalog.connect.query` | Increments for each mesh-based catalog query for the given service. | queries | counter | +| `consul.catalog.connect.query-tag` | Increments for each mesh-based catalog query for the given service with the given tag. | queries | counter | +| `consul.catalog.connect.query-tags` | Increments for each mesh-based catalog query for the given service with the given tags. | queries | counter | +| `consul.catalog.connect.not-found` | Increments for each mesh-based catalog query where the given service could not be found. | queries | counter | | `consul.mesh.active-root-ca.expiry` | The number of seconds until the root CA expires, updated every hour. | seconds | gauge | | `consul.mesh.active-signing-ca.expiry` | The number of seconds until the signing CA expires, updated every hour. | seconds | gauge | | `consul.agent.tls.cert.expiry` | The number of seconds until the Agent TLS certificate expires, updated every hour. | seconds | gauge | -## Connect Built-in Proxy Metrics +## Service Mesh Built-in Proxy Metrics -Consul Connect's built-in proxy is by default configured to log metrics to the +Consul service mesh's built-in proxy is by default configured to log metrics to the same sink as the agent that starts it. When running in this mode it emits some basic metrics. These will be expanded @@ -718,3 +739,32 @@ Consul attaches the following labels to metric values. | `peer_id` | The ID of a peer connected to the reporting cluster or leader. | Any UUID | | `partition` | Name of the partition that the peering is created in. | Any defined partition name in the cluster | +## Server Host Metrics + +Consul servers can report the following metrics about the host's system resources. +This feature must be enabled in the [agent telemetry configuration](/consul/docs/agent/config/config-files#telemetry-enable_host_metrics). +Note that if the Consul server is operating inside a container these metrics +still report host resource usage and do not report any resource limits placed +on the container. + +**Requirements:** +- Consul 1.15.3+ + +| Metric | Description | Unit | Type | +| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | ------- | +| `consul.host.memory.total` | The total physical memory in bytes | mixed | mixed | +| `consul.host.memory.available` | The available physical memory in bytes | mixed | mixed | +| `consul.host.memory.free` | The free physical memory in bytes | mixed | mixed | +| `consul.host.memory.used` | The used physical memory in bytes | mixed | mixed | +| `consul.host.memory.used_percent` | The used physical memory as a percentage of total physical memory | mixed | mixed | +| `consul.host.cpu.total` | The host's total cpu utilization +| `consul.host.cpu.user` | The cpu utilization in user space +| `consul.host.cpu.idle` | The cpu utilization in idle state +| `consul.host.cpu.iowait` | The cpu utilization in iowait state +| `consul.host.cpu.system` | The cpu utilization in system space +| `consul.host.disk.size` | The size in bytes of the data_dir disk +| `consul.host.disk.used` | The number of bytes used on the data_dir disk +| `consul.host.disk.available` | The number of bytes available on the data_dir disk +| `consul.host.disk.used_percent` | The percentage of disk space used on the data_dir disk +| `consul.host.disk.inodes_percent` | The percentage of inode usage on the data_dir disk +| `consul.host.uptime` | The uptime of the host in seconds diff --git a/website/content/docs/agent/wal-logstore/enable.mdx b/website/content/docs/agent/wal-logstore/enable.mdx new file mode 100644 index 00000000000..2a339fcf89e --- /dev/null +++ b/website/content/docs/agent/wal-logstore/enable.mdx @@ -0,0 +1,152 @@ +--- +layout: docs +page_title: Enable the experimental WAL LogStore backend +description: >- + Learn how to safely configure and test the experimental WAL backend in your Consul deployment. +--- + +# Enable the experimental WAL LogStore backend + +This topic describes how to safely configure and test the WAL backend in your Consul deployment. + +The overall process for enabling the WAL LogStore backend for one server consists of the following steps. In production environments, we recommend starting by enabling the backend on a single server . If you eventually choose to expand the test to further servers, you must repeat these steps for each one. + +1. Enable log verification. +1. Select target server to enable WAL. +1. Stop target server gracefully. +1. Remove data directory from target server. +1. Update target server's configuration. +1. Start the target server. +1. Monitor target server raft metrics and logs. + +!> **Experimental feature:** The WAL LogStore backend is experimental and may contain bugs that could cause data loss. Follow this guide to manage risk during testing. + +## Requirements + +- Consul v1.15 or later is required for all servers in the datacenter. Refer to the [standard upgrade procedure](/consul/docs/upgrading/instructions/general-process) and the [1.15 upgrade notes](/consul/docs/upgrading/upgrade-specific#consul-1-15-x) for additional information. +- A Consul cluster with at least three nodes are required to safely test the WAL backend without downtime. + +We recommend taking the following additional measures: + +- Take a snapshot prior to testing. +- Monitor Consul server metrics and logs, and set an alert on specific log events that occur when WAL is enabled. Refer to [Monitor Raft metrics and logs for WAL](/consul/docs/agent/wal-logstore/monitoring) for more information. +- Enable WAL in a pre-production environment and run it for a several days before enabling it in production. + +## Known issues + +The following issues were discovered after release of Consul 1.15.1 and will be +fixed in a future patch release. + + * A follower that is disconnected may be unable to catch up if it is using the WAL backend. + * Restoring user snapshots can break replication to WAL-enabled followers. + * Restoring user snapshots can cause a WAL-enabled leader to panic. + +## Risks + +While their likelihood remains low to very low, be aware of the following risks before implementing the WAL backend: + + - If WAL corrupts data on a Consul server agent, the server data cannot be recovered. Restart the server with an empty data directory and reload its state from the leader to resolve the issue. + - WAL may corrupt data or contain a defect that causes the server to panic and crash. WAL may not restart if the defect recurs when WAL reads from the logs on startup. Restart the server with an empty data directory and reload its state from the leader to resolve the issue. + - If WAL corrupts data, clients may read corrupted data from the Consul server, such as invalid IP addresses or unmatched tokens. This outcome is unlikely even if a recurring defect causes WAL to corrupt data because replication uses objects cached in memory instead of reads from disk. Restart the server with an empty data directory and reload its state from the leader to resolve the issue. + - If you enable a Consul OSS server to use WAL or enable WAL on a voting server with Consul Enterprise, WAL may corrupt the server's state, become the leader, and replicate the corrupted state to all other servers. In this case, restoring from backup is required to recover a completely uncorrupted state. Test WAL on a non-voting server in Enterprise to prevent this outcome. You can add a new non-voting server to the cluster to test with if there are no existing ones. + +## Enable log verification + +You must enable log verification on all voting servers in Enterprise and all servers in OSS because the leader writes verification checkpoints. + +1. On each voting server, add the following to the server's configuration file: + + ```hcl + raft_logstore { + verification { + enabled = true + interval = "60s" + } + } + ``` + +1. Restart the server to apply the changes. The `consul reload` command is not sufficient to apply `raft_logstore` configuration changes. +1. Run the `consul operator raft list-peers` command to wait for each server to become a healthy voter before moving on to the next. This may take a few minutes for large snapshots. + +When complete, the server's logs should contain verifier reports that appear like the following example: + +```log hideClipboard +2023-01-31T14:44:31.174Z [INFO] agent.server.raft.logstore.verifier: verification checksum OK: elapsed=488.463268ms leaderChecksum=f15db83976f2328c rangeEnd=357802 rangeStart=298132 readChecksum=f15db83976f2328c +``` + +## Select target server to enable WAL + +If you are using Consul OSS or Consul Enterprise without non-voting servers, select a follower server to enable WAL. As noted in [Risks](#risks), Consul Enterprise users with non-voting servers should first select a non-voting server, or consider adding another server as a non-voter to test on. + +Retrieve the current state of the servers by running the following command: + +```shell-session +$ consul operator raft list-peers +``` + +## Stop target server + +Stop the target server gracefully. For example, if you are using `systemd`, +run the following command: + +```shell-session +$ systemctl stop consul +``` + +If your environment uses configuration management automation that might interfere with this process, such as Chef or Puppet, you must disable them until you have completely enabled WAL as a storage backend. + +## Remove data directory from target server + +Temporarily moving the data directory to a different location is less destructive than deleting it. We recommend moving it in cases where you unsuccessfully enable WAL. Do not use the old data directory (`/data-dir/raft.bak`) for recovery after restarting the server. We recommend eventually deleting the old directory. + +The following example assumes the `data_dir` in the server's configuration is `/data-dir` and renames it to `/data-dir.bak`. + +```shell-session +$ mv /data-dir/raft /data-dir/raft.bak +``` + +When switching backends, you must always remove _the entire raft directory_, not just the `raft.db` file or `wal` directory. The log must always be consistent with the snapshots to avoid undefined behavior or data loss. + +## Update target server configuration + +Add the following to the target server's configuration file: + +```hcl +raft_logstore { + backend = "wal" + verification { + enabled = true + interval = "60s" + } +} +``` + +## Start target server + +Start the target server. For example, if you are using `systemd`, run the following command: + +```shell-session +$ systemctl start consul +``` + +Watch for the server to become a healthy voter again. + +```shell-session +$ consul operator raft list-peers +``` + +## Monitor target server Raft metrics and logs + +Refer to [Monitor Raft metrics and logs for WAL](/consul/docs/agent/wal-logstore/monitoring) for details. + +We recommend leaving the cluster in the test configuration for several days or weeks, as long as you observe no errors. An extended test provides more confidence that WAL operates correctly under varied workloads and during routine server restarts. If you observe any errors, end the test immediately and report them. + +If you disabled configuration management automation, consider reenabling it during the testing phase to pick up other updates for the host. You must ensure that it does not revert the Consul configuration file and remove the altered backend configuration. One way to do this is add the `raft_logstore` block to a separate file that is not managed by your automation. This file can either be added to the directory if [`-config-dir`](/consul/docs/agent/config/cli-flags#_config_dir) is used or as an additional [`-config-file`](/consul/docs/agent/config/cli-flags#_config_file) argument. + +## Next steps + +- If you observe any verification errors, performance anomalies, or other suspicious behavior from the target server during the test, you should immediately follow [the procedure to revert back to BoltDB](/consul/docs/agent/wal-logstore/revert-to-boltdb). Report failures through GitHub. + +- If you do not see errors and would like to expand the test further, you can repeat the above procedure on another target server. We suggest waiting after each test expansion and slowly rolling WAL out to other parts of your environment. Once the majority of your servers use WAL, any bugs not yet discovered may result in cluster unavailability. + +- If you wish to permanently enable WAL on all servers, repeat the steps described in this topic for each server. Even if `backend = "wal"` is set in logs, servers continue to use BoltDB if they find an existing raft.db file in the data directory. \ No newline at end of file diff --git a/website/content/docs/agent/wal-logstore/index.mdx b/website/content/docs/agent/wal-logstore/index.mdx new file mode 100644 index 00000000000..491255e8ed5 --- /dev/null +++ b/website/content/docs/agent/wal-logstore/index.mdx @@ -0,0 +1,53 @@ +--- +layout: docs +page_title: WAL LogStore Backend Overview +description: >- + The experimental WAL (write-ahead log) LogStore backend shipped in Consul 1.15 is intended to replace the BoltDB backend, improving performance and log storage issues. +--- + +# Experimental WAL LogStore backend overview + +This topic provides an overview of the WAL (write-ahead log) LogStore backend. +The WAL backend is an experimental feature. Refer to +[Requirements](/consul/docs/agent/wal-logstore/enable#requirements) for +supported environments and known issues. + +We do not recommend enabling the WAL backend in production without following +[our guide for safe +testing](/consul/docs/agent/wal-logstore/enable). + +## WAL versus BoltDB + +WAL implements a traditional log with rotating, append-only log files. WAL resolves many issues with the existing `LogStore` provided by the BoltDB backend. The BoltDB `LogStore` is a copy-on-write BTree, which is not optimized for append-only, write-heavy workloads. + +### BoltDB storage scalability issues + +The existing BoltDB log store inefficiently stores append-only logs to disk because it was designed as a full key-value database. It is a single file that only ever grows. Deleting the oldest logs, which Consul does regularly when it makes new snapshots of the state, leaves free space in the file. The free space must be tracked in a `freelist` so that BoltDB can reuse it on future writes. By contrast, a simple segmented log can delete the oldest log files from disk. + +A burst of writes at double or triple the normal volume can suddenly cause the log file to grow to several times its steady-state size. After Consul takes the next snapshot and truncates the oldest logs, the resulting file is mostly empty space. + +To track the free space, Consul must write extra metadata to disk with every write. The metadata is proportional to the amount of free pages, so after a large burst write latencies tend to increase. In some cases, the latencies cause serious performance degradation to the cluster. + +To mitigate risks associated with sudden bursts of log data, Consul tries to limit lots of logs from accumulating in the LogStore. Significantly larger BoltDB files are slower to append to because the tree is deeper and freelist larger. For this reason, Consul's default options associated with snapshots, truncating logs, and keeping the log history have been aggressively set toward keeping BoltDB small rather than using disk IO optimally. + +But the larger the file, the more likely it is to have a large freelist or suddenly form one after a burst of writes. For this reason, the many of Consul's default options asssociated with snapshots, truncating logs, and keeping the log history aggressively keep BoltDT small rather than uisng disk IO more efficiently. + +Other reliability issues, such as [raft replication capacity issues](/consul/docs/agent/telemetry#raft-replication-capacity-issues), are much simpler to solve without the performance concerns caused by storing more logs in BoltDB. + +### WAL approaches storage issues differently + +When directly measured, WAL is more performant than BoltDB because it solves a simpler storage problem. Despite this, some users may not notice a significant performance improvement from the upgrade with the same configuration and workload. In this case, the benefit of WAL is that retaining more logs does not affect write performance. As a result, strategies for reducing disk IO with slower snapshots or for keeping logs to permit slower followers to catch up with cluster state are all possible, increasing the reliability of the deployment. + +## WAL quality assurance + +The WAL backend has been tested thoroughly during development: + +- Every component in the WAL, such as [metadata management](https://github.com/hashicorp/raft-wal/blob/main/types/meta.go), [log file encoding](https://github.com/hashicorp/raft-wal/blob/main/types/segment.go) to actual [file-system interaction](https://github.com/hashicorp/raft-wal/blob/main/types/vfs.go) are abstracted so unit tests can simulate difficult-to-reproduce disk failures. + +- We used the [application-level intelligent crash explorer (ALICE)](https://github.com/hashicorp/raft-wal/blob/main/alice/README.md) to exhaustively simulate thousands of possible crash failure scenarios. WAL correctly recovered from all scenarios. + +- We ran hundreds of tests in a performance testing cluster with checksum verification enabled and did not detect data loss or corruption. We will continue testing before making WAL the default backend. + +We are aware of how complex and critical disk-persistence is for your data. + +We hope that many users at different scales will try WAL in their environments after upgrading to 1.15 or later and report success or failure so that we can confidently replace BoltDB as the default for new clusters in a future release. \ No newline at end of file diff --git a/website/content/docs/agent/wal-logstore/monitoring.mdx b/website/content/docs/agent/wal-logstore/monitoring.mdx new file mode 100644 index 00000000000..f4f81a986d2 --- /dev/null +++ b/website/content/docs/agent/wal-logstore/monitoring.mdx @@ -0,0 +1,85 @@ +--- +layout: docs +page_title: Monitor Raft metrics and logs for WAL +description: >- + Learn how to monitor Raft metrics emitted the experimental WAL (write-ahead log) LogStore backend shipped in Consul 1.15. +--- + +# Monitor Raft metrics and logs for WAL + +This topic describes how to monitor Raft metrics and logs if you are testing the WAL backend. We strongly recommend monitoring the Consul cluster, especially the target server, for evidence that the WAL backend is not functioning correctly. Refer to [Enable the experimental WAL LogStore backend](/consul/docs/agent/wal-logstore/enable) for additional information about the WAL backend. + +!> **Upgrade warning:** The WAL LogStore backend is experimental. + +## Monitor for checksum failures + +Log store verification failures on any server, regardless of whether you are running the BoltDB or WAL backed, are unrecoverable errors. Consul may report the following errors in logs. + +### Read failures: Disk Corruption + +```log hideClipboard +2022-11-15T22:41:23.546Z [ERROR] agent.raft.logstore: verification checksum FAILED: storage corruption rangeStart=1234 rangeEnd=3456 leaderChecksum=0xc1... readChecksum=0x45... +``` + +This indicates that the server read back data that is different from what it wrote to disk. This indicates corruption in the storage backend or filesystem. + +For convenience, Consul also increments a metric `consul.raft.logstore.verifier.read_checksum_failures` when this occurs. + +### Write failures: In-flight Corruption + +The following error indicates that the checksum on the follower did not match the leader when the follower received the logs. The error implies that the corruption happened in the network or software and not the log store: + +```log hideClipboard +2022-11-15T22:41:23.546Z [ERROR] agent.raft.logstore: verification checksum FAILED: in-flight corruption rangeStart=1234 rangeEnd=3456 leaderChecksum=0xc1... followerWriteChecksum=0x45... +``` + +It is unlikely that this error indicates an issue with the storage backend, but you should take the same steps to resolve and report it. + +The `consul.raft.logstore.verifier.write_checksum_failures` metric increments when this error occurs. + +## Resolve checksum failures + +If either type of corruption is detected, complete the instructions for [reverting to BoltDB](/consul/docs/agent/wal-logstore/revert-to-boltdb). If the server already uses BoltDB, the errors likely indicate a latent bug in BoltDB or a bug in the verification code. In both cases, you should follow the revert instructions. + +Report all verification failures as a [GitHub +issue](https://github.com/hashicorp/consul/issues/new?assignees=&labels=&template=bug_report.md&title=WAL:%20Checksum%20Failure). + +In your report, include the following: + - Details of your server cluster configuration and hardware + - Logs around the failure message + - Context for how long they have been running the configuration + - Any metrics or description of the workload you have. For example, how many raft + commits per second. Also include the performance metrics described on this page. + +We recommend setting up an alert on Consul server logs containing `verification checksum FAILED` or on the `consul.raft.logstore.verifier.{read|write}_checksum_failures` metrics. The sooner you respond to a corrupt server, the lower the chance of any of the [potential risks](/consul/docs/agent/wal-logstore/enable#risks) causing problems in your cluster. + +## Performance metrics + +The key performance metrics to watch are: + +- `consul.raft.commitTime` measures the time to commit new writes on a quorum of + servers. It should be the same or lower after deploying WAL. Even if WAL is + faster for your workload and hardware, it may not be reflected in `commitTime` + until enough followers are using WAL that the leader does not have to wait for + two slower followers in a cluster of five to catch up. + +- `consul.raft.rpc.appendEntries.storeLogs` measures the time spent persisting + logs to disk on each _follower_. It should be the same or lower for + WAL-enabled followers. + +- `consul.raft.replication.appendEntries.rpc` measures the time taken for each + `AppendEntries` RPC from the leader's perspective. If this is significantly + higher than `consul.raft.rpc.appendEntries` on the follower, it indicates a + known queuing issue in the Raft library and is unrelated to the backend. + Followers with WAL enabled should not be slower than the others. You can + determine which follower is associated with which metric by running the + `consul operator raft list-peers` command and matching the + `peer_id` label value to the server IDs listed. + +- `consul.raft.compactLogs` measures the time take to truncate the logs after a + snapshot. WAL-enabled servers should not be slower than BoltDB servers. + +- `consul.raft.leader.dispatchLog` measures the time spent persisting logs to + disk on the _leader_. It is only relevant if a WAL-enabled server becomes a + leader. It should be the same or lower than before when the leader was using + BoltDB. \ No newline at end of file diff --git a/website/content/docs/agent/wal-logstore/revert-to-boltdb.mdx b/website/content/docs/agent/wal-logstore/revert-to-boltdb.mdx new file mode 100644 index 00000000000..2bd0638b7cd --- /dev/null +++ b/website/content/docs/agent/wal-logstore/revert-to-boltdb.mdx @@ -0,0 +1,76 @@ +--- +layout: docs +page_title: Revert to BoltDB +description: >- + Learn how to revert Consul to the BoltDB backend after enabled the WAL (write-ahead log) LogStore backend shipped in Consul 1.15. +--- + +# Revert storage backend to BoltDB from WAL + +This topic describes how to revert your Consul storage backend from the experimental WAL LogStore backend to the default BoltDB. + +The overall process for reverting to BoltDB consists of the following steps. Repeat the steps for all Consul servers that you need to revert. + +1. Stop target server gracefully. +1. Remove data directory from target server. +1. Update target server's configuration. +1. Start target server. + +## Stop target server gracefully + +Stop the target server gracefully. For example, if you are using `systemd`, +run the following command: + +```shell-session +$ systemctl stop consul +``` + +If your environment uses configuration management automation that might interfere with this process, such as Chef or Puppet, you must disable them until you have completely revereted the storage backend. + +## Remove data directory from target server + +Temporarily moving the data directory to a different location is less destructive than deleting it. We recommend moving the data directory instead of deleted it in cases where you unsuccessfully enable WAL. Do not use the old data directory (`/data-dir/raft.bak`) for recovery after restarting the server. We recommend eventually deleting the old directory. + +The following example assumes the `data_dir` in the server's configuration is `/data-dir` and renames it to `/data-dir.wal.bak`. + +```shell-session +$ mv /data-dir/raft /data-dir/raft.wal.bak +``` + +When switching backend, you must always remove _the entire raft directory_ not just the `raft.db` file or `wal` directory. This is because the log must always be consistent with the snapshots to avoid undefined behavior or data loss. + +## Update target server's configuration + +Modify the `backend` in the target server's configuration file: + +```hcl +raft_logstore { + backend = "boltdb" + verification { + enabled = true + interval = "60s" + } +} +``` + +## Start target server + +Start the target server. For example, if you are using `systemd`, run the following command: + +```shell-session +$ systemctl start consul +``` + +Watch for the server to become a healthy voter again. + +```shell-session +$ consul operator raft list-peers +``` + +### Clean up old data directories + +If necessary, clean up any `raft.wal.bak` directories. Replace `/data-dir` with the value you specified in your configuration file. + +```shell-session +$ rm /data-dir/raft.bak +``` \ No newline at end of file diff --git a/website/content/docs/api-gateway/configuration/routes.mdx b/website/content/docs/api-gateway/configuration/routes.mdx index 4702a360831..27b52034024 100644 --- a/website/content/docs/api-gateway/configuration/routes.mdx +++ b/website/content/docs/api-gateway/configuration/routes.mdx @@ -168,7 +168,7 @@ The following example creates a route named `example-route` in namespace `gatewa ### rules.filters -The `filters` block defines steps for processing requests. You can configure filters to modify the properties of matching incoming requests and enable Consul API Gateway features, such as rewriting path prefixes (refer to [Reroute HTTP requests](/consul/docs/api-gateway/usage#reroute-http-requests) for additional information). +The `filters` block defines steps for processing requests. You can configure filters to modify the properties of matching incoming requests and enable Consul API Gateway features, such as rewriting path prefixes (refer to [Reroute HTTP requests](/consul/docs/api-gateway/usage/reroute-http-requests) for additional information). * Type: Array of objects * Required: Optional @@ -203,7 +203,7 @@ Specifies rules for rewriting the URL of incoming requests when `rules.filters.t ### rules.filters.urlRewrite.path -Specifies a list of objects that determine how Consul API Gateway rewrites URL paths (refer to [Reroute HTTP requests](/consul/docs/api-gateway/usage#reroute-http-requests) for additional information). +Specifies a list of objects that determine how Consul API Gateway rewrites URL paths (refer to [Reroute HTTP requests](/consul/docs/api-gateway/usage/reroute-http-requests) for additional information). The following table describes the parameters for `path`: diff --git a/website/content/docs/api-gateway/index.mdx b/website/content/docs/api-gateway/index.mdx index 1bdc0bfce9d..83372546ac4 100644 --- a/website/content/docs/api-gateway/index.mdx +++ b/website/content/docs/api-gateway/index.mdx @@ -1,13 +1,13 @@ --- layout: docs -page_title: Consul API Gateway Overview +page_title: API Gateway for Kubernetes Overview description: >- Consul API Gateway enables external network client access to a service mesh on Kubernetes and forwards requests based on path or header information. Learn about how the k8s Gateway API specification configures Consul API Gateway so you can control access and simplify traffic management. --- -# Consul API Gateway Overview +# API Gateway for Kubernetes overview -This topic provides an overview of the Consul API Gateway. +This topic provides an overview of the Consul API Gateway for deploying on Kubernetes. If you would like to deploy on virtual machines, refer to [API Gateways on Virtual Machines](/consul/docs/connect/gateways/api-gateway/usage). ## What is Consul API Gateway? diff --git a/website/content/docs/api-gateway/install.mdx b/website/content/docs/api-gateway/install.mdx index 1ee4c61c8fc..1d715c0c478 100644 --- a/website/content/docs/api-gateway/install.mdx +++ b/website/content/docs/api-gateway/install.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Install Consul API Gateway +page_title: Install API Gateway for Kubernetes description: >- Learn how to install custom resource definitions (CRDs) and configure the Helm chart so that you can run Consul API Gateway on your Kubernetes deployment. --- -# Install Consul API Gateway +# Install API Gateway for Kubernetes This topic describes how to install and configure Consul API Gateway. diff --git a/website/content/docs/api-gateway/tech-specs.mdx b/website/content/docs/api-gateway/tech-specs.mdx index 9695eb0126a..7fb142340de 100644 --- a/website/content/docs/api-gateway/tech-specs.mdx +++ b/website/content/docs/api-gateway/tech-specs.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Consul API Gateway Technical Specifications +page_title: API Gateway for Kubernetes Technical Specifications description: >- Consul API Gateway is a service mesh add-on for Kubernetes deployments. Learn about its requirements for system resources, ports, and component versions, its Enterprise limitations, and compatible k8s cloud environments. --- -# Consul API Gateway Technical Specifications +# API Gateway for Kubernetes Technical Specifications This topic describes the technical specifications associated with using Consul API Gateway. diff --git a/website/content/docs/api-gateway/upgrades.mdx b/website/content/docs/api-gateway/upgrades.mdx index 821a80dce8c..e67c3a0de9c 100644 --- a/website/content/docs/api-gateway/upgrades.mdx +++ b/website/content/docs/api-gateway/upgrades.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Upgrade Consul API Gateway +page_title: Upgrade API Gateway for Kubernetes description: >- Upgrade Consul API Gateway to use newly supported features. Learn about the requirements, procedures, and post-configuration changes involved in standard and specific version upgrades. --- -# Upgrade Consul API Gateway +# Upgrade API Gateway for Kubernetes This topic describes how to upgrade Consul API Gateway. diff --git a/website/content/docs/api-gateway/usage/route-to-peered-services.mdx b/website/content/docs/api-gateway/usage/route-to-peered-services.mdx index 33bc54cdb48..50924762397 100644 --- a/website/content/docs/api-gateway/usage/route-to-peered-services.mdx +++ b/website/content/docs/api-gateway/usage/route-to-peered-services.mdx @@ -12,13 +12,13 @@ This topic describes how to configure Consul API Gateway to route traffic to ser 1. Consul 1.14 or later 1. Verify that the [requirements](/consul/docs/api-gateway/tech-specs) have been met. 1. Verify that the Consul API Gateway CRDs and controller have been installed and applied. Refer to [Installation](/consul/docs/api-gateway/install) for details. -1. A peering connection must already be established between Consul clusters. Refer to [Cluster Peering on Kubernetes](/consul/docs/connect/cluster-peering/k8s) for instructions. -1. The Consul service that you want to route traffic to must be exported to the cluster containing your `Gateway`. Refer to [Cluster Peering on Kubernetes](/consul/docs/connect/cluster-peering/k8s) for instructions. +1. A peering connection must already be established between Consul clusters. Refer to [Cluster Peering on Kubernetes](/consul/docs/k8s/connect/cluster-peering/tech-specs) for instructions. +1. The Consul service that you want to route traffic to must be exported to the cluster containing your `Gateway`. Refer to [Cluster Peering on Kubernetes](/consul/docs/k8s/connect/cluster-peering/tech-specs) for instructions. 1. A `ServiceResolver` for the Consul service you want to route traffic to must be created in the cluster that contains your `Gateway`. Refer to [Service Resolver Configuration Entry](/consul/docs/connect/config-entries/service-resolver) for instructions. ## Configuration -Specify the following fields in your `MeshService` configuration to use this feature. Refer to the [MeshService configuration reference](/consul/docs/api-gateway/configuration/mesh) for details about the parameters. +Specify the following fields in your `MeshService` configuration to use this feature. Refer to the [MeshService configuration reference](/consul/docs/api-gateway/configuration/meshservice) for details about the parameters. - [`name`](/consul/docs/api-gateway/configuration/meshservice#name) - [`peer`](/consul/docs/api-gateway/configuration/meshservice#peer) diff --git a/website/content/docs/api-gateway/usage/usage.mdx b/website/content/docs/api-gateway/usage/usage.mdx index 6dc4dd57607..b9b864ce234 100644 --- a/website/content/docs/api-gateway/usage/usage.mdx +++ b/website/content/docs/api-gateway/usage/usage.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Use Consul API Gateway +page_title: Deploy API Gateway for Kubernetes description: >- Learn how to apply a configured Consul API Gateway to your Kubernetes cluster, review the required fields for rerouting HTTP requests, and troubleshoot an error message. --- -# Basic Consul API Gateway Usage +# Deploy API Gateway for Kubernetes This topic describes how to use Consul API Gateway. diff --git a/website/content/docs/architecture/anti-entropy.mdx b/website/content/docs/architecture/anti-entropy.mdx index 8c2b9bb9675..39d2a08156c 100644 --- a/website/content/docs/architecture/anti-entropy.mdx +++ b/website/content/docs/architecture/anti-entropy.mdx @@ -26,7 +26,7 @@ health checks and updating their local state. Services and checks within the context of an agent have a rich set of configuration options available. This is because the agent is responsible for generating information about its services and their health through the use of -[health checks](/consul/docs/discovery/checks). +[health checks](/consul/docs/services/usage/checks). #### Catalog @@ -117,8 +117,7 @@ the source of truth for tag information. For example, the Redis database and its monitoring service Redis Sentinel have this kind of relationship. Redis instances are responsible for much of their configuration, but Sentinels determine whether the Redis instance is a -primary or a secondary. Using the Consul service configuration item -[enable_tag_override](/consul/docs/discovery/services) you can instruct the -Consul agent on which the Redis database is running to NOT update the -tags during anti-entropy synchronization. For more information see -[Services](/consul/docs/discovery/services#enable-tag-override-and-anti-entropy) page. +primary or a secondary. Enable the +[`enable_tag_override`](/consul/docs/services/configuration/services-configuration-reference#enable_tag_override) parameter in your service definition file to tell the Consul agent where the Redis database is running to bypass +tags during anti-entropy synchronization. Refer to +[Modify anti-entropy synchronozation](/consul/docs/services/usage/define-services#modify-anti-entropy-synchronization) for additional information. diff --git a/website/content/docs/architecture/improving-consul-resilience.mdx b/website/content/docs/architecture/improving-consul-resilience.mdx index 31050f52ae5..bfcc9a8be47 100644 --- a/website/content/docs/architecture/improving-consul-resilience.mdx +++ b/website/content/docs/architecture/improving-consul-resilience.mdx @@ -31,7 +31,7 @@ Without a quorum, Consul experiences an outage: it cannot provide most of its capabilities because they rely on the availability of this state information. If Consul has an outage, normal operation can be restored by following the -[outage recovery guide](/consul/tutorials/datacenter-operations/recovery-outage). +[Disaster recovery for Consul clusters guide](/consul/tutorials/datacenter-operations/recovery-outage). If Consul is deployed with 3 servers, the quorum size is 2. The deployment can lose 1 server and still maintain quorum, so it has a fault tolerance of 1. diff --git a/website/content/docs/architecture/index.mdx b/website/content/docs/architecture/index.mdx index a36fa644b6b..a4656a7718b 100644 --- a/website/content/docs/architecture/index.mdx +++ b/website/content/docs/architecture/index.mdx @@ -55,7 +55,7 @@ You can also run Consul with an alternate service mesh configuration that deploy ## LAN gossip pool -Client and server agents participate in a LAN gossip pool so that they can distribute and perform node [health checks](/consul/docs/discovery/checks). Agents in the pool propagate the health check information across the cluster. Agent gossip communication occurs on port `8301` using UDP. Agent gossip falls back to TCP if UDP is not available. Refer to [Gossip Protocol](/consul/docs/architecture/gossip) for additional information. +Client and server agents participate in a LAN gossip pool so that they can distribute and perform node [health checks](/consul/docs/services/usage/checks). Agents in the pool propagate the health check information across the cluster. Agent gossip communication occurs on port `8301` using UDP. Agent gossip falls back to TCP if UDP is not available. Refer to [Gossip Protocol](/consul/docs/architecture/gossip) for additional information. The following simplified diagram shows the interactions between servers and clients. diff --git a/website/content/docs/architecture/scale.mdx b/website/content/docs/architecture/scale.mdx index 47176c60afd..1cf31162da8 100644 --- a/website/content/docs/architecture/scale.mdx +++ b/website/content/docs/architecture/scale.mdx @@ -51,7 +51,7 @@ To mitigate these risks, we recommend a maximum of 5,000 Consul client agents in 1. Run exactly one Consul agent per host in the infrastructure. 1. Break up the single Consul datacenter into multiple smaller datacenters. -1. Enterprise users can define [network segments](/consul/docs/enterprise/network-segments) to divide the single gossip pool in the Consul datacenter into multiple smaller pools. +1. Enterprise users can define [network segments](/consul/docs/enterprise/network-segments/network-segments-overview) to divide the single gossip pool in the Consul datacenter into multiple smaller pools. If appropriate for your use case, we recommend breaking up a single Consul datacenter into multiple smaller datacenters. Running multiple datacenters reduces your network’s blast radius more than applying network segments. @@ -236,7 +236,7 @@ Several factors influence Consul performance at scale when used primarily for it - Rate of catalog updates, which is affected by the following events: - A service instance’s health check status changes - A service instance’s node loses connectivity to Consul servers - - The contents of the [service definition file](/consul/docs/discovery/services#service-definition) changes + - The contents of the [service definition file](/consul/docs/services/configuration/services-configuration-reference) changes - Service instances are registered or deregistered - Orchestrators such as Kubernetes or Nomad move a service to a new node @@ -257,7 +257,7 @@ Consul ESM enables health checks and monitoring for external services. When usin Because Consul’s service mesh uses service discovery subsystems, service mesh performance is also optimized by deploying multiple small clusters with consistent numbers of service instances and watches. Service mesh performance is influenced by the following additional factors: - The [transparent proxy](/consul/docs/connect/transparent-proxy) feature causes client agents to listen for service instance updates across all services instead of a subset. To prevent performance issues, we recommend that you do not use the permissive intention, `default: allow`, with the transparent proxy feature. When combined, every service instance update propagates to every proxy, which causes additional server load. -- When you use the [built in CA provider](/consul/docs/connect/ca/consul#built-in-ca), Consul leaders are responsible for signing certificates used for mTLS across the service mesh. The impact on CPU utilization depends on the total number of service instances and configured certificate TTLs. You can use the [CA provider configuration options](/consul/docs/agent/config/config-files#common-ca-config-options) to control the number of requests a server processes. We recommend adjusting [`csr_max_concurrent`](/consul/docs/agent/config/config-files#ca_csr_max_concurrent) and [`csr_max_per_second`](/consul/docs/agent/config/config-files#ca_csr_max_concurrent) to suit your environment. +- When you use the [built-in service mesh CA provider](/consul/docs/connect/ca/consul#built-in-ca), Consul leaders are responsible for signing certificates used for mTLS across the service mesh. The impact on CPU utilization depends on the total number of service instances and configured certificate TTLs. You can use the [CA provider configuration options](/consul/docs/agent/config/config-files#common-ca-config-options) to control the number of requests a server processes. We recommend adjusting [`csr_max_concurrent`](/consul/docs/agent/config/config-files#ca_csr_max_concurrent) and [`csr_max_per_second`](/consul/docs/agent/config/config-files#ca_csr_max_concurrent) to suit your environment. ### K/V store diff --git a/website/content/docs/concepts/service-mesh.mdx b/website/content/docs/concepts/service-mesh.mdx index 334a6639f1c..2e793f2441c 100644 --- a/website/content/docs/concepts/service-mesh.mdx +++ b/website/content/docs/concepts/service-mesh.mdx @@ -21,8 +21,8 @@ Some of the benefits of a service mesh include; - automatic failover - traffic management - encryption -- observability and traceability, -- authentication and authorization, +- observability and traceability +- authentication and authorization - network automation A common use case for leveraging a service mesh is to achieve a [_zero trust_ model](https://www.consul.io/use-cases/zero-trust-networking). diff --git a/website/content/docs/connect/ca/aws.mdx b/website/content/docs/connect/ca/aws.mdx index e10a8f5cfac..cac5cb46e65 100644 --- a/website/content/docs/connect/ca/aws.mdx +++ b/website/content/docs/connect/ca/aws.mdx @@ -49,7 +49,7 @@ optional configuration value. Example configurations are shown below: - + @@ -116,7 +116,7 @@ ACM Private CA has several that restrict how fast certificates can be issued. This may impact how quickly large clusters can rotate all issued certificates. -Currently, the ACM Private CA provider for Connect has some additional +Currently, the ACM Private CA provider for service mesh has some additional limitations described below. ### Unable to Cross-sign Other CAs @@ -150,7 +150,7 @@ CA](https://aws.amazon.com/certificate-manager/pricing/) for actual cost information. Assume the following Consul datacenters exist and are configured to use ACM -Private CA as their Connect CA with the default leaf certificate lifetime of +Private CA as their service mesh CA with the default leaf certificate lifetime of 72 hours: | Datacenter | Primary | CA Resource Created | Number of service instances | diff --git a/website/content/docs/connect/ca/consul.mdx b/website/content/docs/connect/ca/consul.mdx index 23d7f2ab4d4..8d36c001211 100644 --- a/website/content/docs/connect/ca/consul.mdx +++ b/website/content/docs/connect/ca/consul.mdx @@ -7,12 +7,12 @@ description: >- # Built-In Certificate Authority for Service Mesh -Consul ships with a built-in CA system so that Connect can be +Consul ships with a built-in CA system so that service mesh can be easily enabled out of the box. The built-in CA generates and stores the root certificate and private key on Consul servers. It can also be configured with a custom certificate and private key if needed. -If Connect is enabled and no CA provider is specified, the built-in +If service mesh is enabled and no CA provider is specified, the built-in CA is the default provider used. The provider can be [updated and rotated](/consul/docs/connect/ca#root-certificate-rotation) at any point to migrate to a new provider. @@ -24,7 +24,7 @@ CA providers. ## Configuration -The built-in CA provider has no required configuration. Enabling Connect +The built-in CA provider has no required configuration. Enabling service mesh alone will configure the built-in CA provider, and will automatically generate a root certificate and private key: @@ -84,8 +84,8 @@ $ curl localhost:8500/v1/connect/ca/configuration } ``` -This is the default Connect CA configuration if nothing is explicitly set when -Connect is enabled - the PrivateKey and RootCert fields have not been set, so those have +This is the default service mesh CA configuration if nothing is explicitly set when +service mesh is enabled - the PrivateKey and RootCert fields have not been set, so those have been generated (as seen above in the roots list). There are two ways to have the Consul CA use a custom private key and root certificate: diff --git a/website/content/docs/connect/ca/index.mdx b/website/content/docs/connect/ca/index.mdx index 03502a7395a..13cc56c72d3 100644 --- a/website/content/docs/connect/ca/index.mdx +++ b/website/content/docs/connect/ca/index.mdx @@ -7,8 +7,8 @@ description: >- # Service Mesh Certificate Authority Overview -Certificate management in Connect is done centrally through the Consul -servers using the configured CA (Certificate Authority) provider. A CA provider +Service mesh certificate management is done centrally through the Consul +servers using the configured service mesh CA (Certificate Authority) provider. A CA provider manages root and intermediate certificates and performs certificate signing operations. The Consul leader orchestrates CA provider operations as necessary, such as when a service needs a new certificate or during CA rotation events. @@ -42,7 +42,7 @@ datacenter. CA initialization happens automatically when a new Consul leader is elected as long as -[Connect is enabled](/consul/docs/connect/configuration#agent-configuration), +[service mesh is enabled](/consul/docs/connect/configuration#agent-configuration), and the CA system has not already been initialized. This initialization process will generate the initial root certificates and setup the internal Consul server state. @@ -55,7 +55,7 @@ If a CA is already initialized, any changes to the CA configuration in the agent configuration file (including removing the configuration completely) will have no effect. -If no specific provider is configured when Connect is enabled, the built-in +If no specific provider is configured when service mesh is enabled, the built-in Consul CA provider will be used and a private key and root certificate will be generated automatically. @@ -133,7 +133,7 @@ Cross-Signing](#forced-rotation-without-cross-signing). This also automatically occurs when a completely different CA provider is configured (since this changes the root key). Therefore, this automatic rotation process can also be used to cleanly transition between CA providers. For example, -updating Connect to use Vault instead of the built-in CA. +updating the service mesh to use Vault instead of the built-in CA. During rotation, an intermediate CA certificate is requested from the new root, which is then cross-signed by the old root. This cross-signed certificate is diff --git a/website/content/docs/connect/ca/vault.mdx b/website/content/docs/connect/ca/vault.mdx index 54fe5f4b674..9c9abf8724f 100644 --- a/website/content/docs/connect/ca/vault.mdx +++ b/website/content/docs/connect/ca/vault.mdx @@ -19,7 +19,7 @@ This page describes how configure the Vault CA provider. - Vault 0.10.3 to 1.10.x. -~> **Compatibility note:** If you use Vault 1.11.0+ as Consul's Connect CA, versions of Consul released before Dec 13, 2022 will develop an issue with Consul control plane or service mesh communication ([GH-15525](https://github.com/hashicorp/consul/pull/15525)). Use or upgrade to a [Consul version that includes the fix](https://support.hashicorp.com/hc/en-us/articles/11308460105491#01GMC24E6PPGXMRX8DMT4HZYTW) to avoid this problem. +~> **Compatibility note:** If you use Vault 1.11.0+ as Consul's service mesh CA, versions of Consul released before Dec 13, 2022 will develop an issue with Consul control plane or service mesh communication ([GH-15525](https://github.com/hashicorp/consul/pull/15525)). Use or upgrade to a [Consul version that includes the fix](https://support.hashicorp.com/hc/en-us/articles/11308460105491#01GMC24E6PPGXMRX8DMT4HZYTW) to avoid this problem. ## Enable Vault as the CA @@ -32,7 +32,7 @@ Refer to the [Configuration Reference](#configuration-reference) for details abo The following example shows the required configurations for a default implementation: - + diff --git a/website/content/docs/connect/cluster-peering/create-manage-peering.mdx b/website/content/docs/connect/cluster-peering/create-manage-peering.mdx deleted file mode 100644 index dbbf0d4fc3a..00000000000 --- a/website/content/docs/connect/cluster-peering/create-manage-peering.mdx +++ /dev/null @@ -1,537 +0,0 @@ ---- -layout: docs -page_title: Cluster Peering - Create and Manage Connections -description: >- - Generate a peering token to establish communication, export services, and authorize requests for cluster peering connections. Learn how to create, list, read, check, and delete peering connections. ---- - - -# Create and Manage Cluster Peering Connections - -A peering token enables cluster peering between different datacenters. Once you generate a peering token, you can use it to establish a connection between clusters. Then you can export services and create intentions so that peered clusters can call those services. - -## Create a peering connection - -Cluster peering is enabled by default on Consul servers as of v1.14. For additional information, including options to disable cluster peering, refer to [Configuration Files](/consul/docs/agent/config/config-files). - -The process to create a peering connection is a sequence with multiple steps: - -1. Create a peering token -1. Establish a connection between clusters -1. Export services between clusters -1. Authorize services for peers - -You can generate peering tokens and initiate connections on any available agent using either the API, CLI, or the Consul UI. If you use the API or CLI, we recommend performing these operations through a client agent in the partition you want to connect. - -The UI does not currently support exporting services between clusters or authorizing services for peers. - -### Create a peering token - -To begin the cluster peering process, generate a peering token in one of your clusters. The other cluster uses this token to establish the peering connection. - -Every time you generate a peering token, a single-use establishment secret is embedded in the token. Because regenerating a peering token invalidates the previously generated secret, you must use the most recently created token to establish peering connections. - - - - -In `cluster-01`, use the [`/peering/token` endpoint](/consul/api-docs/peering#generate-a-peering-token) to issue a request for a peering token. - -```shell-session -$ curl --request POST --data '{"Peer":"cluster-02"}' --url http://localhost:8500/v1/peering/token -``` - -The CLI outputs the peering token, which is a base64-encoded string containing the token details. - -Create a JSON file that contains the first cluster's name and the peering token. - - - -```json -{ - "Peer": "cluster-01", - "PeeringToken": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImF1ZCI6IlNvbHIifQ.5T7L_L1MPfQ_5FjKGa1fTPqrzwK4bNSM812nW6oyjb8" -} -``` - - - - - - -In `cluster-01`, use the [`consul peering generate-token` command](/consul/commands/peering/generate-token) to issue a request for a peering token. - -```shell-session -$ consul peering generate-token -name cluster-02 -``` - -The CLI outputs the peering token, which is a base64-encoded string containing the token details. -Save this value to a file or clipboard to be used in the next step on `cluster-02`. - - - - - -1. In the Consul UI for the datacenter associated with `cluster-01`, click **Peers**. -1. Click **Add peer connection**. -1. In the **Generate token** tab, enter `cluster-02` in the **Name of peer** field. -1. Click the **Generate token** button. -1. Copy the token before you proceed. You cannot view it again after leaving this screen. If you lose your token, you must generate a new one. - - - - -### Establish a connection between clusters - -Next, use the peering token to establish a secure connection between the clusters. - - - - -In one of the client agents in "cluster-02," use `peering_token.json` and the [`/peering/establish` endpoint](/consul/api-docs/peering#establish-a-peering-connection) to establish the peering connection. This endpoint does not generate an output unless there is an error. - -```shell-session -$ curl --request POST --data @peering_token.json http://127.0.0.1:8500/v1/peering/establish -``` - -When you connect server agents through cluster peering, their default behavior is to peer to the `default` partition. To establish peering connections for other partitions through server agents, you must add the `Partition` field to `peering_token.json` and specify the partitions you want to peer. For additional configuration information, refer to [Cluster Peering - HTTP API](/consul/api-docs/peering). - -You can dial the `peering/establish` endpoint once per peering token. Peering tokens cannot be reused after being used to establish a connection. If you need to re-establish a connection, you must generate a new peering token. - - - - - -In one of the client agents in "cluster-02," issue the [`consul peering establish` command](/consul/commands/peering/establish) and specify the token generated in the previous step. The command establishes the peering connection. -The commands prints "Successfully established peering connection with cluster-01" after the connection is established. - -```shell-session -$ consul peering establish -name cluster-01 -peering-token token-from-generate -``` - -When you connect server agents through cluster peering, they peer their default partitions. -To establish peering connections for other partitions through server agents, you must add the `-partition` flag to the `establish` command and specify the partitions you want to peer. -For additional configuration information, refer to [`consul peering establish` command](/consul/commands/peering/establish) . - -You can run the `peering establish` command once per peering token. -Peering tokens cannot be reused after being used to establish a connection. -If you need to re-establish a connection, you must generate a new peering token. - - - - - -1. In the Consul UI for the datacenter associated with `cluster 02`, click **Peers** and then **Add peer connection**. -1. Click **Establish peering**. -1. In the **Name of peer** field, enter `cluster-01`. Then paste the peering token in the **Token** field. -1. Click **Add peer**. - - - - -### Export services between clusters - -After you establish a connection between the clusters, you need to create a configuration entry that defines the services that are available for other clusters. Consul uses this configuration entry to advertise service information and support service mesh connections across clusters. - -First, create a configuration entry and specify the `Kind` as `"exported-services"`. - - - -```hcl -Kind = "exported-services" -Name = "default" -Services = [ - { - ## The name and namespace of the service to export. - Name = "service-name" - Namespace = "default" - - ## The list of peer clusters to export the service to. - Consumers = [ - { - ## The peer name to reference in config is the one set - ## during the peering process. - Peer = "cluster-02" - } - ] - } -] -``` - - - -Then, add the configuration entry to your cluster. - -```shell-session -$ consul config write peering-config.hcl -``` - -Before you proceed, wait for the clusters to sync and make services available to their peers. You can issue an endpoint query to [check the peered cluster status](#check-peered-cluster-status). - -### Authorize services for peers - -Before you can call services from peered clusters, you must set service intentions that authorize those clusters to use specific services. Consul prevents services from being exported to unauthorized clusters. - -First, create a configuration entry and specify the `Kind` as `"service-intentions"`. Declare the service on "cluster-02" that can access the service in "cluster-01." The following example sets service intentions so that "frontend-service" can access "backend-service." - - - -```hcl -Kind = "service-intentions" -Name = "backend-service" - -Sources = [ - { - Name = "frontend-service" - Peer = "cluster-02" - Action = "allow" - } -] -``` - - - -If the peer's name is not specified in `Peer`, then Consul assumes that the service is in the local cluster. - -Then, add the configuration entry to your cluster. - -```shell-session -$ consul config write peering-intentions.hcl -``` - -### Authorize Service Reads with ACLs - -If ACLs are enabled on a Consul cluster, sidecar proxies that access exported services as an upstream must have an ACL token that grants read access. -Read access to all imported services is granted using either of the following rules associated with an ACL token: -- `service:write` permissions for any service in the sidecar's partition. -- `service:read` and `node:read` for all services and nodes, respectively, in sidecar's namespace and partition. -For Consul Enterprise, access is granted to all imported services in the service's partition. -These permissions are satisfied when using a [service identity](/consul/docs/security/acl/acl-roles#service-identities). - -Example rule files can be found in [Reading Servers](/consul/docs/connect/config-entries/exported-services#reading-services) in the `exported-services` config entry documentation. - -Refer to [ACLs System Overview](/consul/docs/security/acl) for more information on ACLs and their setup. - -## Manage peering connections - -After you establish a peering connection, you can get a list of all active peering connections, read a specific peering connection's information, check peering connection health, and delete peering connections. - -### List all peering connections - -You can list all active peering connections in a cluster. - - - - -After you establish a peering connection, [query the `/peerings/` endpoint](/consul/api-docs/peering#list-all-peerings) to get a list of all peering connections. For example, the following command requests a list of all peering connections on `localhost` and returns the information as a series of JSON objects: - -```shell-session -$ curl http://127.0.0.1:8500/v1/peerings - -[ - { - "ID": "462c45e8-018e-f19d-85eb-1fc1bcc2ef12", - "Name": "cluster-02", - "State": "ACTIVE", - "Partition": "default", - "PeerID": "e83a315c-027e-bcb1-7c0c-a46650904a05", - "PeerServerName": "server.dc1.consul", - "PeerServerAddresses": [ - "10.0.0.1:8300" - ], - "CreateIndex": 89, - "ModifyIndex": 89 - }, - { - "ID": "1460ada9-26d2-f30d-3359-2968aa7dc47d", - "Name": "cluster-03", - "State": "INITIAL", - "Partition": "default", - "Meta": { - "env": "production" - }, - "CreateIndex": 109, - "ModifyIndex": 119 - }, -] -``` - - - - -After you establish a peering connection, run the [`consul peering list`](/consul/commands/peering/list) command to get a list of all peering connections. -For example, the following command requests a list of all peering connections and returns the information in a table: - -```shell-session -$ consul peering list - -Name State Imported Svcs Exported Svcs Meta -cluster-02 ACTIVE 0 2 env=production -cluster-03 PENDING 0 0 - ``` - - - - -In the Consul UI, click **Peers**. The UI lists peering connections you created for clusters in a datacenter. - -The name that appears in the list is the name of the cluster in a different datacenter with an established peering connection. - - - -### Read a peering connection - -You can get information about individual peering connections between clusters. - - - - -After you establish a peering connection, [query the `/peering/` endpoint](/consul/api-docs/peering#read-a-peering-connection) to get peering information about for a specific cluster. For example, the following command requests peering connection information for "cluster-02" and returns the info as a JSON object: - -```shell-session -$ curl http://127.0.0.1:8500/v1/peering/cluster-02 - -{ - "ID": "462c45e8-018e-f19d-85eb-1fc1bcc2ef12", - "Name": "cluster-02", - "State": "INITIAL", - "PeerID": "e83a315c-027e-bcb1-7c0c-a46650904a05", - "PeerServerName": "server.dc1.consul", - "PeerServerAddresses": [ - "10.0.0.1:8300" - ], - "CreateIndex": 89, - "ModifyIndex": 89 -} -``` - - - - -After you establish a peering connection, run the [`consul peering read`](/consul/commands/peering/list) command to get peering information about for a specific cluster. -For example, the following command requests peering connection information for "cluster-02": - -```shell-session -$ consul peering read -name cluster-02 - -Name: cluster-02 -ID: 3b001063-8079-b1a6-764c-738af5a39a97 -State: ACTIVE -Meta: - env=production - -Peer ID: e83a315c-027e-bcb1-7c0c-a46650904a05 -Peer Server Name: server.dc1.consul -Peer CA Pems: 0 -Peer Server Addresses: - 10.0.0.1:8300 - -Imported Services: 0 -Exported Services: 2 - -Create Index: 89 -Modify Index: 89 - -``` - - - - -In the Consul UI, click **Peers**. The UI lists peering connections you created for clusters in that datacenter. Click the name of a peered cluster to view additional details about the peering connection. - - - -### Check peering connection health - -You can check the status of your peering connection to perform health checks. - -To confirm that the peering connection between your clusters remains healthy, query the [`health/service` endpoint](/consul/api-docs/health) of one cluster from the other cluster. For example, in "cluster-02," query the endpoint and add the `peer=cluster-01` query parameter to the end of the URL. - -```shell-session -$ curl \ - "http://127.0.0.1:8500/v1/health/service/?peer=cluster-01" -``` - -A successful query includes service information in the output. - -### Delete peering connections - -You can disconnect the peered clusters by deleting their connection. Deleting a peering connection stops data replication to the peer and deletes imported data, including services and CA certificates. - - - - -In "cluster-01," request the deletion through the [`/peering/ endpoint`](/consul/api-docs/peering#delete-a-peering-connection). - -```shell-session -$ curl --request DELETE http://127.0.0.1:8500/v1/peering/cluster-02 -``` - - - - -In "cluster-01," request the deletion through the [`consul peering delete`](/consul/commands/peering/list) command. - -```shell-session -$ consul peering delete -name cluster-02 - -Successfully submitted peering connection, cluster-02, for deletion -``` - - - - -In the Consul UI, click **Peers**. The UI lists peering connections you created for clusters in that datacenter. - -Next to the name of the peer, click **More** (three horizontal dots) and then **Delete**. Click **Delete** to confirm and remove the peering connection. - - - - -## L7 traffic management between peers - -The following sections describe how to enable L7 traffic management features between peered clusters. - -### Service resolvers for redirects and failover - -As of Consul v1.14, you can use [dynamic traffic management](/consul/docs/connect/l7-traffic) to configure your service mesh so that services automatically failover and redirect between peers. The following examples update the [`service-resolver` config entry](/consul/docs/connect/config-entries/) in `cluster-01` so that Consul redirects traffic intended for the `frontend` service to a backup instance in peer `cluster-02` when it detects multiple connection failures. - - - -```hcl -Kind = "service-resolver" -Name = "frontend" -ConnectTimeout = "15s" -Failover = { - "*" = { - Targets = [ - {Peer = "cluster-02"} - ] - } -} -``` - -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceResolver -metadata: - name: frontend -spec: - connectTimeout: 15s - failover: - '*': - targets: - - peer: 'cluster-02' - service: 'frontend' - namespace: 'default' -``` - -```json -{ - "ConnectTimeout": "15s", - "Kind": "service-resolver", - "Name": "frontend", - "Failover": { - "*": { - "Targets": [ - { - "Peer": "cluster-02" - } - ] - } - }, - "CreateIndex": 250, - "ModifyIndex": 250 -} -``` - - - -### Service splitters and custom routes - -The `service-splitter` and `service-router` configuration entry kinds do not support directly targeting a service instance hosted on a peer. To split or route traffic to a service on a peer, you must combine the definition with a `service-resolver` configuration entry that defines the service hosted on the peer as an upstream service. For example, to split traffic evenly between `frontend` services hosted on peers, first define the desired behavior locally: - - - -```hcl -Kind = "service-splitter" -Name = "frontend" -Splits = [ - { - Weight = 50 - ## defaults to service with same name as configuration entry ("frontend") - }, - { - Weight = 50 - Service = "frontend-peer" - }, -] -``` - -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceSplitter -metadata: - name: frontend -spec: - splits: - - weight: 50 - ## defaults to service with same name as configuration entry ("web") - - weight: 50 - service: frontend-peer -``` - -```json -{ - "Kind": "service-splitter", - "Name": "frontend", - "Splits": [ - { - "Weight": 50 - }, - { - "Weight": 50, - "Service": "frontend-peer" - } - ] -} -``` - - - -Then, create a local `service-resolver` configuration entry named `frontend-peer` and define a redirect targeting the peer and its service: - - - -```hcl -Kind = "service-resolver" -Name = "frontend-peer" -Redirect { - Service = frontend - Peer = "cluster-02" -} -``` - -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceResolver -metadata: - name: frontend-peer -spec: - redirect: - peer: 'cluster-02' - service: 'frontend' -``` - -```json -{ - "Kind": "service-resolver", - "Name": "frontend-peer", - "Redirect": { - "Service": "frontend", - "Peer": "cluster-02" - } -} -``` - - - diff --git a/website/content/docs/connect/cluster-peering/index.mdx b/website/content/docs/connect/cluster-peering/index.mdx index 77f4b8f4c73..f69f093330e 100644 --- a/website/content/docs/connect/cluster-peering/index.mdx +++ b/website/content/docs/connect/cluster-peering/index.mdx @@ -1,32 +1,37 @@ --- layout: docs -page_title: Service Mesh - What is Cluster Peering? +page_title: Cluster Peering Overview description: >- - Cluster peering establishes communication between independent clusters in Consul, allowing services to interact across datacenters. Learn about the cluster peering process, differences with WAN federation for multi-datacenter deployments, and technical constraints. + Cluster peering establishes communication between independent clusters in Consul, allowing services to interact across datacenters. Learn how cluster peering works, its differences with WAN federation for multi-datacenter deployments, and how to troubleshoot common issues. --- -# What is Cluster Peering? +# Cluster peering overview -You can create peering connections between two or more independent clusters so that services deployed to different partitions or datacenters can communicate. +This topic provides an overview of cluster peering, which lets you connect two or more independent Consul clusters so that services deployed to different partitions or datacenters can communicate. +Cluster peering is enabled in Consul by default. For specific information about cluster peering configuration and usage, refer to following pages. -## Overview +## What is cluster peering? -Cluster peering is a process that allows Consul clusters to communicate with each other. The cluster peering process consists of the following steps: +Consul supports cluster peering connections between two [admin partitions](/consul/docs/enterprise/admin-partitions) _in different datacenters_. Deployments without an Enterprise license can still use cluster peering because every datacenter automatically includes a default partition. Meanwhile, admin partitions _in the same datacenter_ do not require cluster peering connections because you can export services between them without generating or exchanging a peering token. -1. Create a peering token in one cluster. -1. Use the peering token to establish peering with a second cluster. -1. Export services between clusters. -1. Create intentions to authorize services for peers. +The following diagram describes Consul's cluster peering architecture. -This process establishes cluster peering between two [admin partitions](/consul/docs/enterprise/admin-partitions). Deployments without an Enterprise license can still use cluster peering because every datacenter automatically includes a `default` partition. +![Diagram of cluster peering with admin partitions](/img/cluster-peering-diagram.png) -For detailed instructions on establishing cluster peering connections, refer to [Create and Manage Peering Connections](/consul/docs/connect/cluster-peering/create-manage-peering). +In this diagram, the `default` partition in Consul DC 1 has a cluster peering connection with the `web` partition in Consul DC 2. Enforced by their respective mesh gateways, this cluster peering connection enables `Service B` to communicate with `Service C` as a service upstream. -> To learn how to peer clusters and connect services across peers in AWS Elastic Kubernetes Service (EKS) environments, complete the [Consul Cluster Peering on Kubernetes tutorial](/consul/tutorials/developer-mesh/cluster-peering-aws?utm_source=docs). +Cluster peering leverages several components of Consul's architecture to enforce secure communication between services: -### Differences between WAN federation and cluster peering +- A _peering token_ contains an embedded secret that securely establishes communication when shared symetrically between datacenters. Sharing this token enables each datacenter's server agents to recognize requests from authorized peers, similar to how the [gossip encryption key secures agent LAN gossip](/consul/docs/security/encryption#gossip-encryption). +- A _mesh gateway_ encrypts outgoing traffic, decrypts incoming traffic, and directs traffic to healthy services. Consul's service mesh features must be enabled in order to use mesh gateways. Mesh gateways support the specific admin partitions they are deployed on. Refer to [Mesh gateways](/consul/docs/connect/gateways/mesh-gateway) for more information. +- An _exported service_ communicates with downstreams deployed in other admin partitions. They are explicitly defined in an [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services). +- A _service intention_ secures [service-to-service communication in a service mesh](/consul/docs/connect/intentions). Intentions enable identity-based access between services by exchanging TLS certificates, which the service's sidecar proxy verifies upon each request. -WAN federation and cluster peering are different ways to connect Consul deployments. WAN federation connects multiple datacenters to make them function as if they were a single cluster, while cluster peering treats each datacenter as a separate cluster. As a result, WAN federation requires a primary datacenter to maintain and replicate global states such as ACLs and configuration entries, but cluster peering does not. +### Compared with WAN federation + +WAN federation and cluster peering are different ways to connect services through mesh gateways so that they can communicate across datacenters. WAN federation connects multiple datacenters to make them function as if they were a single cluster, while cluster peering treats each datacenter as a separate cluster. As a result, WAN federation requires a primary datacenter to maintain and replicate global states such as ACLs and configuration entries, but cluster peering does not. + +WAN federation and cluster peering also treat encrypted traffic differently. While mesh gateways between WAN federated datacenters use mTLS to keep data encrypted, mesh gateways between peers terminate mTLS sessions, decrypt data to HTTP services, and then re-encrypt traffic to send to services. Data must be decrypted in order to evaluate and apply dynamic routing rules at the destination cluster, which reduces coupling between peers. Regardless of whether you connect your clusters through WAN federation or cluster peering, human and machine users can use either method to discover services in other clusters or dial them through the service mesh. @@ -42,11 +47,47 @@ Regardless of whether you connect your clusters through WAN federation or cluste | Shares key/value stores | ✅ | ❌ | | Can replicate ACL tokens, policies, and roles | ✅ | ❌ | -## Important Cluster Peering Constraints +## Guidance + +The following resources are available to help you use Consul's cluster peering features. + +### Tutorials + +- To learn how to peer clusters and connect services across peers in AWS Elastic Kubernetes Service (EKS) and Google Kubernetes Engine (GKE) environments, complete the [Consul Cluster Peering on Kubernetes tutorial](/consul/tutorials/developer-mesh/cluster-peering). + +### Usage documentation + +- [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/establish-cluster-peering) +- [Manage cluster peering connections](/consul/docs/connect/cluster-peering/usage/manage-connections) +- [Manage L7 traffic with cluster peering](/consul/docs/connect/cluster-peering/usage/peering-traffic-management) + +### Kubernetes documentation + +- [Cluster peering on Kubernetes technical specifications](/consul/docs/k8s/connect/cluster-peering/tech-specs) +- [Establish cluster peering connections on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering) +- [Manage cluster peering connections on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/manage-peering) +- [Manage L7 traffic with cluster peering on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/l7-traffic) + +### HCP Consul documentation + +- [Cluster peering](/hcp/docs/consul/usage/cluster-peering) +- [Cluster peering topologies](/hcp/docs/consul/usage/cluster-peering/topologies) +- [Establish cluster peering connnections on HCP Consul](/hcp/docs/consul/usage/cluster-peering/create-connections) +- [Cluster peering with management plane](/hcp/docs/consul/usage/management-plane#cluster-peering) + +### Reference documentation + +- [Cluster peering technical specifications](/consul/docs/connect/cluster-peering/tech-specs) +- [HTTP API reference: `/peering/` endpoint](/consul/api-docs/peering) +- [CLI reference: `peering` command](/consul/commands/peering). + +## Basic troubleshooting -Consider the following technical constraints: +If you experience errors when using Consul's cluster peering features, refer to the following list of technical constraints. +- Peer names can only contain lowercase characters. - Services with node, instance, and check definitions totaling more than 8MB cannot be exported to a peer. -- Two admin partitions in the same datacenter cannot be peered. Use [`exported-services`](/consul/docs/connect/config-entries/exported-services#exporting-services-to-peered-clusters) directly. -- The `consul intention` CLI command is not supported. To manage intentions that specify services in peered clusters, use [configuration entries](/consul/docs/connect/config-entries/service-intentions). -- Accessing key/value stores across peers is not supported. +- Two admin partitions in the same datacenter cannot be peered. Use the [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services#exporting-services-to-peered-clusters) instead. +- To manage intentions that specify services in peered clusters, use [configuration entries](/consul/docs/connect/config-entries/service-intentions). The `consul intention` CLI command is not supported. +- The Consul UI does not support exporting services between clusters or creating service intentions. Use either the API or the CLI to complete these required steps when establishing new cluster peering connections. +- Accessing key/value stores across peers is not supported. \ No newline at end of file diff --git a/website/content/docs/connect/cluster-peering/k8s.mdx b/website/content/docs/connect/cluster-peering/k8s.mdx deleted file mode 100644 index 570442fd4a7..00000000000 --- a/website/content/docs/connect/cluster-peering/k8s.mdx +++ /dev/null @@ -1,593 +0,0 @@ ---- -layout: docs -page_title: Cluster Peering on Kubernetes -description: >- - If you use Consul on Kubernetes, learn how to enable cluster peering, create peering CRDs, and then manage peering connections in consul-k8s. ---- - -# Cluster Peering on Kubernetes - -To establish a cluster peering connection on Kubernetes, you need to enable several pre-requisite values in the Helm chart and create custom resource definitions (CRDs) for each side of the peering. - -The following Helm values are mandatory for cluster peering: -- [`global.tls.enabled = true`](/consul/docs/k8s/helm#v-global-tls-enabled) -- [`meshGateway.enabled = true`](/consul/docs/k8s/helm#v-meshgateway-enabled) - -The following CRDs are used to create and manage a peering connection: - -- `PeeringAcceptor`: Generates a peering token and accepts an incoming peering connection. -- `PeeringDialer`: Uses a peering token to make an outbound peering connection with the cluster that generated the token. - -Peering connections, including both data plane and control plane traffic, is routed through mesh gateways. -As of Consul v1.14, you can also [implement service failovers and redirects to control traffic](/consul/docs/connect/l7-traffic) between peers. - -> To learn how to peer clusters and connect services across peers in AWS Elastic Kubernetes Service (EKS) environments, complete the [Consul Cluster Peering on Kubernetes tutorial](/consul/tutorials/developer-mesh/cluster-peering-aws?utm_source=docs). - -## Prerequisites - -You must implement the following requirements to create and use cluster peering connections with Kubernetes: - -- Consul v1.14.0 or later -- At least two Kubernetes clusters -- The installation must be running on Consul on Kubernetes version 1.0.0 or later - -### Prepare for installation - -Complete the following procedure after you have provisioned a Kubernetes cluster and set up your kubeconfig file to manage access to multiple Kubernetes clusters. - -1. Use the `kubectl` command to export the Kubernetes context names and then set them to variables for future use. For more information on how to use kubeconfig and contexts, refer to the [Kubernetes docs on configuring access to multiple clusters](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/). - - You can use the following methods to get the context names for your clusters: - - - Use the `kubectl config current-context` command to get the context for the cluster you are currently in. - - Use the `kubectl config get-contexts` command to get all configured contexts in your kubeconfig file. - - ```shell-session - $ export CLUSTER1_CONTEXT= - $ export CLUSTER2_CONTEXT= - ``` - -1. To establish cluster peering through Kubernetes, create a `values.yaml` file with the following Helm values. **NOTE:** Mesh Gateway replicas are defaulted to 1 replica, and could be adjusted using the `meshGateway.replicas` value for higher availability. - - - - ```yaml - global: - name: consul - image: "hashicorp/consul:1.14.1" - peering: - enabled: true - tls: - enabled: true - meshGateway: - enabled: true - ``` - - - -### Install Consul on Kubernetes - -Install Consul on Kubernetes by using the CLI to apply `values.yaml` to each cluster. - - 1. In `cluster-01`, run the following commands: - - ```shell-session - $ export HELM_RELEASE_NAME=cluster-01 - ``` - - ```shell-session - $ helm install ${HELM_RELEASE_NAME} hashicorp/consul --create-namespace --namespace consul --version "1.0.1" --values values.yaml --set global.datacenter=dc1 --kube-context $CLUSTER1_CONTEXT - ``` - - 1. In `cluster-02`, run the following commands: - - ```shell-session - $ export HELM_RELEASE_NAME=cluster-02 - ``` - - ```shell-session - $ helm install ${HELM_RELEASE_NAME} hashicorp/consul --create-namespace --namespace consul --version "1.0.1" --values values.yaml --set global.datacenter=dc2 --kube-context $CLUSTER2_CONTEXT - ``` - -## Create a peering connection for Consul on Kubernetes - -To peer Kubernetes clusters running Consul, you need to create a peering token on one cluster (`cluster-01`) and share -it with the other cluster (`cluster-02`). The generated peering token from `cluster-01` will include the addresses of -the servers for that cluster. The servers for `cluster-02` will use that information to dial the servers in -`cluster-01`. Complete the following steps to create the peer connection. - -### Using mesh gateways for the peering connection -If the servers in `cluster-01` are not directly routable from the dialing cluster `cluster-02`, then you'll need to set up peering through mesh gateways. - -1. In `cluster-01` apply the `Mesh` custom resource so the generated token will have the mesh gateway addresses which will be routable from the other cluster. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: Mesh - metadata: - name: mesh - spec: - peering: - peerThroughMeshGateways: true - ``` - - - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT apply -f mesh.yaml - ``` - -1. In `cluster-02` apply the `Mesh` custom resource so that the servers for `cluster-02` will use their local mesh gateway to dial the servers for `cluster-01`. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: Mesh - metadata: - name: mesh - spec: - peering: - peerThroughMeshGateways: true - ``` - - - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply -f mesh.yaml - ``` - -### Create a peering token - -Peers identify each other using the `metadata.name` values you establish when creating the `PeeringAcceptor` and `PeeringDialer` CRDs. - -1. In `cluster-01`, create the `PeeringAcceptor` custom resource. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: PeeringAcceptor - metadata: - name: cluster-02 ## The name of the peer you want to connect to - spec: - peer: - secret: - name: "peering-token" - key: "data" - backend: "kubernetes" - ``` - - - -1. Apply the `PeeringAcceptor` resource to the first cluster. - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT apply --filename acceptor.yaml - ```` - -1. Save your peering token so that you can export it to the other cluster. - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT get secret peering-token --output yaml > peering-token.yaml - ``` - -### Establish a peering connection between clusters - -1. Apply the peering token to the second cluster. - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply --filename peering-token.yaml - ``` - -1. In `cluster-02`, create the `PeeringDialer` custom resource. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: PeeringDialer - metadata: - name: cluster-01 ## The name of the peer you want to connect to - spec: - peer: - secret: - name: "peering-token" - key: "data" - backend: "kubernetes" - ``` - - - -1. Apply the `PeeringDialer` resource to the second cluster. - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply --filename dialer.yaml - ``` - -### Configure the mesh gateway mode for traffic between services -Mesh gateways are required for service-to-service traffic between peered clusters. By default, this will mean that a -service dialing another service in a remote peer will dial the remote mesh gateway to reach that service. If you would -like to configure the mesh gateway mode such that this traffic always leaves through the local mesh gateway, you can use the `ProxyDefaults` CRD. - -1. In `cluster-01` apply the following `ProxyDefaults` CRD to configure the mesh gateway mode. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: ProxyDefaults - metadata: - name: global - spec: - meshGateway: - mode: local - ``` - - - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT apply -f proxy-defaults.yaml - ``` - -1. In `cluster-02` apply the following `ProxyDefaults` CRD to configure the mesh gateway mode. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: ProxyDefaults - metadata: - name: global - spec: - meshGateway: - mode: local - ``` - - - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply -f proxy-defaults.yaml - ``` - -### Export services between clusters - -The examples described in this section demonstrate how to export a service named `backend`. You should change instances of `backend` in the example code to the name of the service you want to export. - -1. For the service in `cluster-02` that you want to export, add the `"consul.hashicorp.com/connect-inject": "true"` annotation to your service's pods prior to deploying. The annotation allows the workload to join the mesh. It is highlighted in the following example: - - - - ```yaml - # Service to expose backend - apiVersion: v1 - kind: Service - metadata: - name: backend - spec: - selector: - app: backend - ports: - - name: http - protocol: TCP - port: 80 - targetPort: 9090 - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: backend - --- - # Deployment for backend - apiVersion: apps/v1 - kind: Deployment - metadata: - name: backend - labels: - app: backend - spec: - replicas: 1 - selector: - matchLabels: - app: backend - template: - metadata: - labels: - app: backend - annotations: - "consul.hashicorp.com/connect-inject": "true" - spec: - serviceAccountName: backend - containers: - - name: backend - image: nicholasjackson/fake-service:v0.22.4 - ports: - - containerPort: 9090 - env: - - name: "LISTEN_ADDR" - value: "0.0.0.0:9090" - - name: "NAME" - value: "backend" - - name: "MESSAGE" - value: "Response from backend" - ``` - - - -1. Deploy the `backend` service to the second cluster. - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply --filename backend.yaml - ``` - -1. In `cluster-02`, create an `ExportedServices` custom resource. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: ExportedServices - metadata: - name: default ## The name of the partition containing the service - spec: - services: - - name: backend ## The name of the service you want to export - consumers: - - peer: cluster-01 ## The name of the peer that receives the service - ``` - - - -1. Apply the `ExportedServices` resource to the second cluster. - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply --filename exported-service.yaml - ``` - -### Authorize services for peers - -1. Create service intentions for the second cluster. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: ServiceIntentions - metadata: - name: backend-deny - spec: - destination: - name: backend - sources: - - name: "*" - action: deny - - name: frontend - action: allow - peer: cluster-01 ## The peer of the source service - ``` - - - -1. Apply the intentions to the second cluster. - - - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT apply --filename intention.yaml - ``` - - - -1. Add the `"consul.hashicorp.com/connect-inject": "true"` annotation to your service's pods before deploying the workload so that the services in `cluster-01` can dial `backend` in `cluster-02`. To dial the upstream service from an application, configure the application so that that requests are sent to the correct DNS name as specified in [Service Virtual IP Lookups](/consul/docs/discovery/dns#service-virtual-ip-lookups). In the following example, the annotation that allows the workload to join the mesh and the configuration provided to the workload that enables the workload to dial the upstream service using the correct DNS name is highlighted. [Service Virtual IP Lookups for Consul Enterprise](/consul/docs/discovery/dns#service-virtual-ip-lookups-for-consul-enterprise) details how you would similarly format a DNS name including partitions and namespaces. - - - - ```yaml - # Service to expose frontend - apiVersion: v1 - kind: Service - metadata: - name: frontend - spec: - selector: - app: frontend - ports: - - name: http - protocol: TCP - port: 9090 - targetPort: 9090 - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: frontend - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: frontend - labels: - app: frontend - spec: - replicas: 1 - selector: - matchLabels: - app: frontend - template: - metadata: - labels: - app: frontend - annotations: - "consul.hashicorp.com/connect-inject": "true" - spec: - serviceAccountName: frontend - containers: - - name: frontend - image: nicholasjackson/fake-service:v0.22.4 - securityContext: - capabilities: - add: ["NET_ADMIN"] - ports: - - containerPort: 9090 - env: - - name: "LISTEN_ADDR" - value: "0.0.0.0:9090" - - name: "UPSTREAM_URIS" - value: "http://backend.virtual.cluster-02.consul" - - name: "NAME" - value: "frontend" - - name: "MESSAGE" - value: "Hello World" - - name: "HTTP_CLIENT_KEEP_ALIVES" - value: "false" - ``` - - - -1. Apply the service file to the first cluster. - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT apply --filename frontend.yaml - ``` - -1. Run the following command in `frontend` and then check the output to confirm that you peered your clusters successfully. - - - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT exec -it $(kubectl --context $CLUSTER1_CONTEXT get pod -l app=frontend -o name) -- curl localhost:9090 - - { - "name": "frontend", - "uri": "/", - "type": "HTTP", - "ip_addresses": [ - "10.16.2.11" - ], - "start_time": "2022-08-26T23:40:01.167199", - "end_time": "2022-08-26T23:40:01.226951", - "duration": "59.752279ms", - "body": "Hello World", - "upstream_calls": { - "http://backend.virtual.cluster-02.consul": { - "name": "backend", - "uri": "http://backend.virtual.cluster-02.consul", - "type": "HTTP", - "ip_addresses": [ - "10.32.2.10" - ], - "start_time": "2022-08-26T23:40:01.223503", - "end_time": "2022-08-26T23:40:01.224653", - "duration": "1.149666ms", - "headers": { - "Content-Length": "266", - "Content-Type": "text/plain; charset=utf-8", - "Date": "Fri, 26 Aug 2022 23:40:01 GMT" - }, - "body": "Response from backend", - "code": 200 - } - }, - "code": 200 - } - ``` - - - -## End a peering connection - -To end a peering connection, delete both the `PeeringAcceptor` and `PeeringDialer` resources. - -1. Delete the `PeeringDialer` resource from the second cluster. - - ```shell-session - $ kubectl --context $CLUSTER2_CONTEXT delete --filename dialer.yaml - ``` - -1. Delete the `PeeringAcceptor` resource from the first cluster. - - ```shell-session - $ kubectl --context $CLUSTER1_CONTEXT delete --filename acceptor.yaml - ```` - -1. Confirm that you deleted your peering connection in `cluster-01` by querying the the `/health` HTTP endpoint. The peered services should no longer appear. - - 1. Exec into the server pod for the first cluster. - - ```shell-session - $ kubectl exec -it consul-server-0 --context $CLUSTER1_CONTEXT -- /bin/sh - ``` - - 1. If you've enabled ACLs, export an ACL token to access the `/health` HTP endpoint for services. The bootstrap token may be used if an ACL token is not already provisioned. - - ```shell-session - $ export CONSUL_HTTP_TOKEN= - ``` - - 1. Query the the `/health` HTTP endpoint. The peered services should no longer appear. - - ```shell-session - $ curl "localhost:8500/v1/health/connect/backend?peer=cluster-02" - ``` - -## Recreate or reset a peering connection - -To recreate or reset the peering connection, you need to generate a new peering token from the cluster where you created the `PeeringAcceptor`. - -1. In the `PeeringAcceptor` CRD, add the annotation `consul.hashicorp.com/peering-version`. If the annotation already exists, update its value to a higher version. - - - - ```yaml - apiVersion: consul.hashicorp.com/v1alpha1 - kind: PeeringAcceptor - metadata: - name: cluster-02 - annotations: - consul.hashicorp.com/peering-version: "1" ## The peering version you want to set, must be in quotes - spec: - peer: - secret: - name: "peering-token" - key: "data" - backend: "kubernetes" - ``` - - - -1. After updating `PeeringAcceptor`, repeat the following steps to create a peering connection: - 1. [Create a peering token](#create-a-peering-token) - 1. [Establish a peering connection between clusters](#establish-a-peering-connection-between-clusters) - 1. [Export services between clusters](#export-services-between-clusters) - 1. [Authorize services for peers](#authorize-services-for-peers) - - Your peering connection is re-established with the updated token. - -~> **Note:** The only way to create or set a new peering token is to manually adjust the value of the annotation `consul.hashicorp.com/peering-version`. Creating a new token causes the previous token to expire. - -## Traffic management between peers - -As of Consul v1.14, you can use [dynamic traffic management](/consul/docs/connect/l7-traffic) to configure your service mesh so that services automatically failover and redirect between peers. - -To configure automatic service failovers and redirect, edit the `ServiceResolver` CRD so that traffic resolves to a backup service instance on a peer. The following example updates the `ServiceResolver` CRD in `cluster-01` so that Consul redirects traffic intended for the `frontend` service to a backup instance in `cluster-02` when it detects multiple connection failures to the primary instance. - - - -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceResolver -metadata: - name: frontend -spec: - connectTimeout: 15s - failover: - '*': - targets: - - peer: 'cluster-02' - service: 'backup' - namespace: 'default' -``` - - diff --git a/website/content/docs/connect/cluster-peering/tech-specs.mdx b/website/content/docs/connect/cluster-peering/tech-specs.mdx new file mode 100644 index 00000000000..929a25f924d --- /dev/null +++ b/website/content/docs/connect/cluster-peering/tech-specs.mdx @@ -0,0 +1,84 @@ +--- +layout: docs +page_title: Cluster Peering Technical Specifications +description: >- + Cluster peering connections in Consul interact with mesh gateways, sidecar proxies, exported services, and ACLs. Learn about the configuration requirements for these components. +--- + +# Cluster peering technical specifications + +This reference topic describes the technical specifications associated with using cluster peering in your deployments. These specifications include required Consul components and their configurations. To learn more about Consul's cluster peering feature, refer to [cluster peering overview](/consul/docs/connect/cluster-peering). + +For cluster peering requirements in Kubernetes deployments, refer to [cluster peering on Kubernetes technical specifications](/consul/docs/k8s/connect/cluster-peering/tech-specs). + +## Requirements + +Consul's default configuration supports cluster peering connections directly between clusters. In production environments, we recommend using mesh gateways to securely route service mesh traffic between partitions with cluster peering connections. + +In addition, make sure your Consul environment meets the following prerequisites: + +- Consul v1.14 or higher. +- Use [Envoy proxies](/consul/docs/connect/proxies/envoy). Envoy is the only proxy with mesh gateway capabilities in Consul. +- A local Consul agent is required to manage mesh gateway configurations. + +## Mesh gateway specifications + +To change Consul's default configuration and enable cluster peering through mesh gateways, use a mesh configuration entry to update your network's service mesh proxies globally: + +1. In a `mesh` configuration entry, set `PeerThroughMeshGateways` to `true`: + + + + ```hcl + Kind = "mesh" + Peering { + PeerThroughMeshGateways = true + } + ``` + + + +1. Write the configuration entry to Consul: + + ```shell + $ consul config write mesh-config.hcl + ``` + +When cluster peering through mesh gateways, consider the following deployment requirements: + +- A cluster requires a registered mesh gateway in order to export services to peers in other regions or cloud providers. +- The mesh gateway must also be registered in the same admin partition as the exported services and their `exported-services` configuration entry. An enterprise license is required to use multiple admin partitions with a single cluster of Consul servers. +- To use the `local` mesh gateway mode, you must register a mesh gateway in the importing cluster. +- Define the `Proxy.Config` settings using opaque parameters compatible with your proxy. Refer to the [Gateway options](/consul/docs/connect/proxies/envoy#gateway-options) and [Escape-hatch Overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides) documentation for additional Envoy proxy configuration information. + +### Mesh gateway modes + +By default, cluster peering connections use mesh gateways in [remote mode](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#remote). Be aware of these additional requirements when changing a mesh gateway's mode. + +- For mesh gateways that connect peered clusters, you can set the `mode` as either `remote` or `local`. +- The `none` mode is invalid for mesh gateways with cluster peering connections. + +Refer to [mesh gateway modes](/consul/docs/connect/gateways/mesh-gateway#modes) for more information. + +## Sidecar proxy specifications + +The Envoy proxies that function as sidecars in your service mesh require configuration in order to properly route traffic to peers. Sidecar proxies are defined in the [service definition](/consul/docs/services/usage/define-services). + +- Configure the `proxy.upstreams` parameters to route traffic to the correct service, namespace, and peer. Refer to the [`upstreams`](/consul/docs/connect/registration/service-registration#upstream-configuration-reference) documentation for details. +- The `proxy.upstreams.destination_name` parameter is always required. +- The `proxy.upstreams.destination_peer` parameter must be configured to enable cross-cluster traffic. +- The `proxy.upstream/destination_namespace` configuration is only necessary if the destination service is in a non-default namespace. + +## Exported service specifications + +The `exported-services` configuration entry is required in order for services to communicate across partitions with cluster peering connections. Basic guidance on using the `exported-services` configuration entry is included in [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/establish-peering#export-services-between-clusters). + +Refer to the [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services) reference for more information. + +## ACL specifications + +If ACLs are enabled, you must add tokens to grant the following permissions: + +- Grant `service:write` permissions to services that define mesh gateways in their server definition. +- Grant `service:read` permissions for all services on the partition. +- Grant `mesh:write` permissions to the mesh gateways that participate in cluster peering connections. This permission allows a leaf certificate to be issued for mesh gateways to terminate TLS sessions for HTTP requests. \ No newline at end of file diff --git a/website/content/docs/connect/cluster-peering/usage/establish-cluster-peering.mdx b/website/content/docs/connect/cluster-peering/usage/establish-cluster-peering.mdx new file mode 100644 index 00000000000..b8549c5c3fb --- /dev/null +++ b/website/content/docs/connect/cluster-peering/usage/establish-cluster-peering.mdx @@ -0,0 +1,269 @@ +--- +layout: docs +page_title: Establish Cluster Peering Connections +description: >- + Generate a peering token to establish communication, export services, and authorize requests for cluster peering connections. Learn how to establish peering connections with Consul's HTTP API, CLI or UI. +--- + +# Establish cluster peering connections + +This page details the process for establishing a cluster peering connection between services deployed to different datacenters. You can interact with Consul's cluster peering features using the CLI, the HTTP API, or the UI. The overall process for establishing a cluster peering connection consists of the following steps: + +1. Create a peering token in one cluster. +1. Use the peering token to establish peering with a second cluster. +1. Export services between clusters. +1. Create intentions to authorize services for peers. + +Cluster peering between services cannot be established until all four steps are complete. + +For Kubernetes guidance, refer to [Establish cluster peering connections on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering). For HCP Consul guidance, refer to [Establish cluster peering connections on HCP Consul](/hcp/docs/consul/usage/cluster-peering/create-connections). + +## Requirements + +You must meet the following requirements to use cluster peering: + +- Consul v1.14.1 or higher +- Services hosted in admin partitions on separate datacenters + +If you need to make services available to an admin partition in the same datacenter, do not use cluster peering. Instead, use the [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services) to make service upstreams available to other admin partitions in a single datacenter. + +### Mesh gateway requirements + +Consul's default configuration supports cluster peering connections directly between clusters. In production environments, we recommend using mesh gateways to securely route service mesh traffic between partitions with cluster peering connections. + +To enable cluster peering through mesh gateways and configure mesh gateways to support cluster peering, refer to [mesh gateway specifications](/consul/docs/connect/cluster-peering/tech-specs#mesh-gateway-specifications). + +## Create a peering token + +To begin the cluster peering process, generate a peering token in one of your clusters. The other cluster uses this token to establish the peering connection. + +Every time you generate a peering token, a single-use secret for establishing the secret is embedded in the token. Because regenerating a peering token invalidates the previously generated secret, you must use the most recently created token to establish peering connections. + + + + +1. In `cluster-01`, use the [`consul peering generate-token` command](/consul/commands/peering/generate-token) to issue a request for a peering token. + + ```shell-session + $ consul peering generate-token -name cluster-02 + ``` + + The CLI outputs the peering token, which is a base64-encoded string containing the token details. + +1. Save this value to a file or clipboard to use in the next step on `cluster-02`. + + + + +1. In `cluster-01`, use the [`/peering/token` endpoint](/consul/api-docs/peering#generate-a-peering-token) to issue a request for a peering token. + + ```shell-session + $ curl --request POST --data '{"Peer":"cluster-02"}' --url http://localhost:8500/v1/peering/token + ``` + + The CLI outputs the peering token, which is a base64-encoded string containing the token details. + +1. Create a JSON file that contains the first cluster's name and the peering token. + + + + ```json + { + "Peer": "cluster-01", + "PeeringToken": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImF1ZCI6IlNvbHIifQ.5T7L_L1MPfQ_5FjKGa1fTPqrzwK4bNSM812nW6oyjb8" + } + ``` + + + + + + +To begin the cluster peering process, generate a peering token in one of your clusters. The other cluster uses this token to establish the peering connection. + +Every time you generate a peering token, a single-use secret for establishing the secret is embedded in the token. Because regenerating a peering token invalidates the previously generated secret, you must use the most recently created token to establish peering connections. + +1. In the Consul UI for the datacenter associated with `cluster-01`, click **Peers**. +1. Click **Add peer connection**. +1. In the **Generate token** tab, enter `cluster-02` in the **Name of peer** field. +1. Click the **Generate token** button. +1. Copy the token before you proceed. You cannot view it again after leaving this screen. If you lose your token, you must generate a new one. + + + + +## Establish a connection between clusters + +Next, use the peering token to establish a secure connection between the clusters. + + + + +1. In one of the client agents deployed to "cluster-02," issue the [`consul peering establish` command](/consul/commands/peering/establish) and specify the token generated in the previous step. + + ```shell-session + $ consul peering establish -name cluster-01 -peering-token token-from-generate + "Successfully established peering connection with cluster-01" + ``` + +When you connect server agents through cluster peering, they peer their default partitions. To establish peering connections for other partitions through server agents, you must add the `-partition` flag to the `establish` command and specify the partitions you want to peer. For additional configuration information, refer to [`consul peering establish` command](/consul/commands/peering/establish). + +You can run the `peering establish` command once per peering token. Peering tokens cannot be reused after being used to establish a connection. If you need to re-establish a connection, you must generate a new peering token. + + + + +1. In one of the client agents in "cluster-02," use `peering_token.json` and the [`/peering/establish` endpoint](/consul/api-docs/peering#establish-a-peering-connection) to establish the peering connection. This endpoint does not generate an output unless there is an error. + + ```shell-session + $ curl --request POST --data @peering_token.json http://127.0.0.1:8500/v1/peering/establish + ``` + +When you connect server agents through cluster peering, their default behavior is to peer to the `default` partition. To establish peering connections for other partitions through server agents, you must add the `Partition` field to `peering_token.json` and specify the partitions you want to peer. For additional configuration information, refer to [Cluster Peering - HTTP API](/consul/api-docs/peering). + +You can dial the `peering/establish` endpoint once per peering token. Peering tokens cannot be reused after being used to establish a connection. If you need to re-establish a connection, you must generate a new peering token. + + + + + +1. In the Consul UI for the datacenter associated with `cluster 02`, click **Peers** and then **Add peer connection**. +1. Click **Establish peering**. +1. In the **Name of peer** field, enter `cluster-01`. Then paste the peering token in the **Token** field. +1. Click **Add peer**. + + + + +## Export services between clusters + +After you establish a connection between the clusters, you need to create an `exported-services` configuration entry that defines the services that are available for other clusters. Consul uses this configuration entry to advertise service information and support service mesh connections across clusters. + +An `exported-services` configuration entry makes services available to another admin partition. While it can target admin partitions either locally or remotely. Clusters peers always export services to remote partitions. Refer to [exported service consumers](/consul/docs/connect/config-entries/exported-services#consumers-1) for more information. + +You must use the Consul CLI to complete this step. The HTTP API and the Consul UI do not support `exported-services` configuration entries. + + + + +1. Create a configuration entry and specify the `Kind` as `"exported-services"`. + + + + ```hcl + Kind = "exported-services" + Name = "default" + Services = [ + { + ## The name and namespace of the service to export. + Name = "service-name" + Namespace = "default" + + ## The list of peer clusters to export the service to. + Consumers = [ + { + ## The peer name to reference in config is the one set + ## during the peering process. + Peer = "cluster-02" + } + ] + } + ] + ``` + + + +1. Add the configuration entry to your cluster. + + ```shell-session + $ consul config write peering-config.hcl + ``` + +Before you proceed, wait for the clusters to sync and make services available to their peers. To check the peered cluster status, [read the cluster peering connection](/consul/docs/connect/cluster-peering/usage/manage-connections#read-a-peering-connection). + + + + +## Authorize services for peers + +Before you can call services from peered clusters, you must set service intentions that authorize those clusters to use specific services. Consul prevents services from being exported to unauthorized clusters. + +You must use the HTTP API or the Consul CLI to complete this step. The Consul UI supports intentions for local clusters only. + + + + +1. Create a configuration entry and specify the `Kind` as `"service-intentions"`. Declare the service on "cluster-02" that can access the service in "cluster-01." In the following example, the service intentions configuration entry authorizes the `backend-service` to communicate with the `frontend-service` that is hosted on remote peer `cluster-02`: + + + + ```hcl + Kind = "service-intentions" + Name = "backend-service" + + Sources = [ + { + Name = "frontend-service" + Peer = "cluster-02" + Action = "allow" + } + ] + ``` + + + + If the peer's name is not specified in `Peer`, then Consul assumes that the service is in the local cluster. + +1. Add the configuration entry to your cluster. + + ```shell-session + $ consul config write peering-intentions.hcl + ``` + + + + +1. Create a configuration entry and specify the `Kind` as `"service-intentions"`. Declare the service on "cluster-02" that can access the service in "cluster-01." In the following example, the service intentions configuration entry authorizes the `backend-service` to communicate with the `frontend-service` that is hosted on remote peer `cluster-02`: + + + + ```hcl + Kind = "service-intentions" + Name = "backend-service" + + Sources = [ + { + Name = "frontend-service" + Peer = "cluster-02" + Action = "allow" + } + ] + ``` + + + + If the peer's name is not specified in `Peer`, then Consul assumes that the service is in the local cluster. + +1. Add the configuration entry to your cluster. + + ```shell-session + $ curl --request PUT --data @peering-intentions.hcl http://127.0.0.1:8500/v1/config + ``` + + + + +### Authorize service reads with ACLs + +If ACLs are enabled on a Consul cluster, sidecar proxies that access exported services as an upstream must have an ACL token that grants read access. + +Read access to all imported services is granted using either of the following rules associated with an ACL token: + +- `service:write` permissions for any service in the sidecar's partition. +- `service:read` and `node:read` for all services and nodes, respectively, in sidecar's namespace and partition. + +For Consul Enterprise, the permissions apply to all imported services in the service's partition. These permissions are satisfied when using a [service identity](/consul/docs/security/acl/acl-roles#service-identities). + +Refer to [Reading servers](/consul/docs/connect/config-entries/exported-services#reading-services) in the `exported-services` configuration entry documentation for example rules. + +For additional information about how to configure and use ACLs, refer to [ACLs system overview](/consul/docs/security/acl). \ No newline at end of file diff --git a/website/content/docs/connect/cluster-peering/usage/manage-connections.mdx b/website/content/docs/connect/cluster-peering/usage/manage-connections.mdx new file mode 100644 index 00000000000..a4e92373328 --- /dev/null +++ b/website/content/docs/connect/cluster-peering/usage/manage-connections.mdx @@ -0,0 +1,137 @@ +--- +layout: docs +page_title: Manage Cluster Peering Connections +description: >- + Learn how to list, read, and delete cluster peering connections using Consul. You can use the HTTP API, the CLI, or the Consul UI to manage cluster peering connections. +--- + +# Manage cluster peering connections + +This usage topic describes how to manage cluster peering connections using the CLI, the HTTP API, and the UI. + +After you establish a cluster peering connection, you can get a list of all active peering connections, read a specific peering connection's information, and delete peering connections. + +For Kubernetes-specific guidance for managing cluster peering connections, refer to [Manage cluster peering connections on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/manage-peering). + +## List all peering connections + +You can list all active peering connections in a cluster. + + + + + ```shell-session + $ consul peering list + Name State Imported Svcs Exported Svcs Meta + cluster-02 ACTIVE 0 2 env=production + cluster-03 PENDING 0 0 + ``` + +For more information, including optional flags and parameters, refer to the [`consul peering list` CLI command reference](/consul/commands/peering/list). + + + + +The following example shows how to format an API request to list peering connections: + + ```shell-session + $ curl --header "X-Consul-Token: 0137db51-5895-4c25-b6cd-d9ed992f4a52" http://127.0.0.1:8500/v1/peerings + ``` + +For more information, including optional parameters and sample responses, refer to the [`/peering` endpoint reference](/consul/api-docs/peering#list-all-peerings). + + + + +In the Consul UI, click **Peers**. + +The UI lists peering connections you created for clusters in a datacenter. The name that appears in the list is the name of the cluster in a different datacenter with an established peering connection. + + + + +## Read a peering connection + +You can get information about individual peering connections between clusters. + + + + + +The following example outputs information about a peering connection locally referred to as "cluster-02": + + ```shell-session + $ consul peering read -name cluster-02 + Name: cluster-02 + ID: 3b001063-8079-b1a6-764c-738af5a39a97 + State: ACTIVE + Meta: + env=production + + Peer ID: e83a315c-027e-bcb1-7c0c-a46650904a05 + Peer Server Name: server.dc1.consul + Peer CA Pems: 0 + Peer Server Addresses: + 10.0.0.1:8300 + + Imported Services: 0 + Exported Services: 2 + + Create Index: 89 + Modify Index: 89 + ``` + +For more information, including optional flags and parameters, refer to the [`consul peering read` CLI command reference](/consul/commands/peering/read). + + + + + ```shell-session + $ curl --header "X-Consul-Token: b23b3cad-5ea1-4413-919e-c76884b9ad60" http://127.0.0.1:8500/v1/peering/cluster-02 + ``` + +For more information, including optional parameters and sample responses, refer to the [`/peering` endpoint reference](/consul/api-docs/peering#read-a-peering-connection). + + + + +1. In the Consul UI, click **Peers**. + +1. Click the name of a peered cluster to view additional details about the peering connection. + + + + +## Delete peering connections + +You can disconnect the peered clusters by deleting their connection. Deleting a peering connection stops data replication to the peer and deletes imported data, including services and CA certificates. + + + + + The following examples deletes a peering connection to a cluster locally referred to as "cluster-02": + + ```shell-session + $ consul peering delete -name cluster-02 + Successfully submitted peering connection, cluster-02, for deletion + ``` + +For more information, including optional flags and parameters, refer to the [`consul peering delete` CLI command reference](/consul/commands/peering/delete). + + + + + ```shell-session + $ curl --request DELETE --header "X-Consul-Token: b23b3cad-5ea1-4413-919e-c76884b9ad60" http://127.0.0.1:8500/v1/peering/cluster-02 + ``` + +This endpoint does not return a response. For more information, including optional parameters, refer to the [`/peering` endpoint reference](/consul/api-docs/peering#delete-a-peering-connection). + + + +1. In the Consul UI, click **Peers**. The UI lists peering connections you created for clusters in that datacenter. +1. Next to the name of the peer, click **More** (three horizontal dots) and then **Delete**. +1. Click **Delete** to confirm and remove the peering connection. + + + \ No newline at end of file diff --git a/website/content/docs/connect/cluster-peering/usage/peering-traffic-management.mdx b/website/content/docs/connect/cluster-peering/usage/peering-traffic-management.mdx new file mode 100644 index 00000000000..240b56dc974 --- /dev/null +++ b/website/content/docs/connect/cluster-peering/usage/peering-traffic-management.mdx @@ -0,0 +1,168 @@ +--- +layout: docs +page_title: Cluster Peering L7 Traffic Management +description: >- + Combine service resolver configurations with splitter and router configurations to manage L7 traffic in Consul deployments with cluster peering connections. Learn how to define dynamic traffic rules to target peers for redirects and failover. +--- + +# Manage L7 traffic with cluster peering + +This usage topic describes how to configure and apply the [`service-resolver` configuration entry](/consul/docs/connect/config-entries/service-resolver) to set up redirects and failovers between services that have an existing cluster peering connection. + +For Kubernetes-specific guidance for managing L7 traffic with cluster peering, refer to [Manage L7 traffic with cluster peering on Kubernetes](/consul/docs/k8s/connect/cluster-peering/usage/l7-traffic). + +## Service resolvers for redirects and failover + +When you use cluster peering to connect datacenters through their admin partitions, you can use [dynamic traffic management](/consul/docs/connect/l7-traffic) to configure your service mesh so that services automatically forward traffic to services hosted on peer clusters. + +However, the `service-splitter` and `service-router` configuration entry kinds do not natively support directly targeting a service instance hosted on a peer. Before you can split or route traffic to a service on a peer, you must define the service hosted on the peer as an upstream service by configuring a failover in the `service-resolver` configuration entry. Then, you can set up a redirect in a second service resolver to interact with the peer service by name. + +For more information about formatting, updating, and managing configuration entries in Consul, refer to [How to use configuration entries](/consul/docs/agent/config-entries). + +## Configure dynamic traffic between peers + +To configure L7 traffic management behavior in deployments with cluster peering connections, complete the following steps in order: + +1. Define the peer cluster as a failover target in the service resolver configuration. + + The following examples update the [`service-resolver` configuration entry](/consul/docs/connect/config-entries/service-resolver) in `cluster-01` so that Consul redirects traffic intended for the `frontend` service to a backup instance in peer `cluster-02` when it detects multiple connection failures. + + + + ```hcl + Kind = "service-resolver" + Name = "frontend" + ConnectTimeout = "15s" + Failover = { + "*" = { + Targets = [ + {Peer = "cluster-02"} + ] + } + } + ``` + + ```json + { + "ConnectTimeout": "15s", + "Kind": "service-resolver", + "Name": "frontend", + "Failover": { + "*": { + "Targets": [ + { + "Peer": "cluster-02" + } + ] + } + }, + "CreateIndex": 250, + "ModifyIndex": 250 + } + ``` + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceResolver + metadata: + name: frontend + spec: + connectTimeout: 15s + failover: + '*': + targets: + - peer: 'cluster-02' + service: 'frontend' + namespace: 'default' + ``` + + + +1. Define the desired behavior in `service-splitter` or `service-router` configuration entries. + + The following example splits traffic evenly between `frontend` services hosted on peers by defining the desired behavior locally: + + + + ```hcl + Kind = "service-splitter" + Name = "frontend" + Splits = [ + { + Weight = 50 + ## defaults to service with same name as configuration entry ("frontend") + }, + { + Weight = 50 + Service = "frontend-peer" + }, + ] + ``` + + ```json + { + "Kind": "service-splitter", + "Name": "frontend", + "Splits": [ + { + "Weight": 50 + }, + { + "Weight": 50, + "Service": "frontend-peer" + } + ] + } + ``` + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceSplitter + metadata: + name: frontend + spec: + splits: + - weight: 50 + ## defaults to service with same name as configuration entry ("frontend") + - weight: 50 + service: frontend-peer + ``` + + + +1. Create a local `service-resolver` configuration entry named `frontend-peer` and define a redirect targeting the peer and its service: + + + + ```hcl + Kind = "service-resolver" + Name = "frontend-peer" + Redirect { + Service = frontend + Peer = "cluster-02" + } + ``` + + ```json + { + "Kind": "service-resolver", + "Name": "frontend-peer", + "Redirect": { + "Service": "frontend", + "Peer": "cluster-02" + } + } + ``` + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceResolver + metadata: + name: frontend-peer + spec: + redirect: + peer: 'cluster-02' + service: 'frontend' + ``` + + \ No newline at end of file diff --git a/website/content/docs/connect/config-entries/exported-services.mdx b/website/content/docs/connect/config-entries/exported-services.mdx index d3a793c891a..48ca7838419 100644 --- a/website/content/docs/connect/config-entries/exported-services.mdx +++ b/website/content/docs/connect/config-entries/exported-services.mdx @@ -9,8 +9,6 @@ description: >- This topic describes the `exported-services` configuration entry type. The `exported-services` configuration entry enables Consul to export service instances to other clusters from a single file and connect services across clusters. For additional information, refer to [Cluster Peering](/consul/docs/connect/cluster-peering) and [Admin Partitions](/consul/docs/enterprise/admin-partitions). --> **v1.11.0+:** This config entry is supported in Consul versions 1.11.0+. - ## Introduction To configure Consul to export services contained in a Consul Enterprise admin partition or Consul OSS datacenter to one or more additional clusters, create a new configuration entry and declare `exported-services` in the `kind` field. This configuration entry enables you to route traffic between services in different clusters. diff --git a/website/content/docs/connect/config-entries/index.mdx b/website/content/docs/connect/config-entries/index.mdx index 6d29a27cc38..de5acd1bce9 100644 --- a/website/content/docs/connect/config-entries/index.mdx +++ b/website/content/docs/connect/config-entries/index.mdx @@ -7,7 +7,7 @@ description: >- # Configuration Entry Overview -Configuration entries can be used to configure the behavior of Consul Connect. +Configuration entries can be used to configure the behavior of Consul service mesh. The following configuration entries are supported: diff --git a/website/content/docs/connect/config-entries/ingress-gateway.mdx b/website/content/docs/connect/config-entries/ingress-gateway.mdx index ee94412661c..cf553af14f1 100644 --- a/website/content/docs/connect/config-entries/ingress-gateway.mdx +++ b/website/content/docs/connect/config-entries/ingress-gateway.mdx @@ -1,566 +1,1525 @@ --- layout: docs -page_title: Ingress Gateway - Configuration Entry Reference +page_title: Ingress gateway configuration entry reference description: >- - The ingress gateway configuration entry kind defines behavior to secure incoming communication between the service mesh and external sources. Use the reference guide to learn about `""ingress-gateway""` config entry parameters and exposing TCP and HTTP listeners. + The ingress gateway configuration entry kind defines behavior for securing incoming communication between the service mesh and external sources. Learn about `""ingress-gateway""` config entry parameters for exposing TCP and HTTP listeners. --- -# Ingress Gateway Configuration Entry +# Ingress gateway configuration entry reference -This topic provides reference information for the `ingress-gateway` configuration entry. +This topic provides configuration reference information for the ingress gateway configuration entry. An ingress gateway is a type of proxy you register as a service in Consul to enable network connectivity from external services to services inside of the service mesh. Refer to [Ingress gateways overview](/consul/docs/connect/gateways/ingress-gateway) for additional information. -## Introduction +## Configuration model -You can define an `ingress-gateway` configuration entry to connect the Consul service mesh to a set of external services. The specification for ingress gateways include a `listeners` configuration, which exposes the service mesh to the external services. Use camel case (`IngressGateway`) to declare an ingress gateway configuration entry on Kubernetes. +The following list describes the configuration hierarchy, language-specific data types, default values if applicable, and requirements for the configuration entry. Click on a property name to view additional details. -Refer to the [Kubernetes Ingress Gateway](/consul/docs/k8s/connect/ingress-gateways) documentation for information about configuring ingress gateways on Kubernetes. + + + +- [`Kind`](#kind): string | must be `ingress-gateway` | required +- [`Name`](#name): string | required +- [`Namespace`](#namespace): string | `default` | +- [`Meta`](#meta): map of strings +- [`Partition`](#partition): string | `default` | +- [`TLS`](#tls): map + - [`Enabled`](#tls-enabled): boolean | `false` + - [`TLSMinVersion`](#tls-tlsminversion): string | `TLSv1_2` + - [`TLSMaxVersion`](#tls-tlsmaxversion): string + - [`CipherSuites`](#tls-ciphersuites): list of strings + - [`SDS`](#tls-sds): map of strings + - [`ClusterName`](#tls-sds): string + - [`CertResource`](#tls-sds): string +- [`Defaults`](#defaults): map + - [`MaxConnections`](#defaults-maxconnections): number + - [`MaxPendingRequests`](#defaults-maxpendingrequests): number + - [`MaxConcurrentRequests`](#defaults-maxconcurrentrequests): number + - [`PassiveHealthCheck`](#defaults-passivehealthcheck): map + - [`Interval`](#defaults-passivehealthcheck): number + - [`MaxFailures`](#defaults-passivehealthcheck): number + - [`EnforcingConsecutive5xx`](#defaults-passivehealthcheck): number + - [`MaxEjectionPercent`](#defaults-passivehealthcheck): number + - [`BaseEjectionTime`](#defaults-passivehealthcheck): string +- [`Listeners`](#listeners): list of maps + - [`Port`](#listeners-port): number | `0` + - [`Protocol`](#listeners-protocol): number | `tcp` + - [`Services`](#listeners-services): list of objects + - [`Name`](#listeners-services-name): string + - [`Namespace`](#listeners-services-namespace): string | + - [`Partition`](#listeners-services-partition): string | + - [`Hosts`](#listeners-services-hosts): List of strings | `.ingress.*` + - [`RequestHeaders`](#listeners-services-requestheaders): map + - [`Add`](#listeners-services-requestheaders): map of strings + - [`Set`](#listeners-services-requestheaders): map of strings + - [`Remove`](#listeners-services-requestheaders): list of strings + - [`ResponseHeaders`](#listeners-services-responseheaders): map + - [`Add`](#listeners-services-responseheaders): map of strings + - [`Set`](#listeners-services-responseheaders): map of strings + - [`Remove`](#listeners-services-responseheaders): list of strings + - [`TLS`](#listeners-services-tls): map + - [`SDS`](#listeners-services-tls-sds): map of strings + - [`ClusterName`](#listeners-services-tls-sds): string + - [`CertResource`](#listeners-services-tls-sds): string + - [`MaxConnections`](#listeners-services-maxconnections): number | `0` + - [`MaxPendingRequests`](#listeners-services-maxconnections): number | `0` + - [`MaxConcurrentRequests`](#listeners-services-maxconnections): number | `0` + - [`PassiveHealthCheck`](#listeners-services-passivehealthcheck): map + - [`Interval`](#listeners-services-passivehealthcheck): number + - [`MaxFailures`](#listeners-services-passivehealthcheck): number + - [`EnforcingConsecutive5xx`](#listeners-services-passivehealthcheck): number + - [`MaxEjectionPercent`](#listeners-services-passivehealthcheck): number + - [`BaseEjectionTime`](#listeners-services-passivehealthcheck): string + - [`TLS`](#listeners-tls): map + - [`Enabled`](#listeners-tls-enabled): boolean | `false` + - [`TLSMinVersion`](#listeners-tls-tlsminversion): string | `TLSv1_2` + - [`TLSMaxVersion`](#listeners-tls-tlsmaxversion): string + - [`CipherSuites`](#listeners-tls-ciphersuites): list of strings + - [`SDS`](#listeners-tls-sds): map of strings + - [`ClusterName`](#listeners-tls-sds): string + - [`CertResource`](#listeners-tls-sds): string + + -For other platforms, see [Ingress Gateway](/consul/docs/connect/gateways/ingress-gateway). + + +- [ `apiVersion`](#apiversion): string | must be set to `consul.hashicorp.com/v1alpha1` | required +- [`kind`](#kind): string | must be `IngressGateway` | required +- [`metadata`](#metadata): map of strings + - [`name`](#metadata-name): string | required + - [`namespace`](#metadata-namespace): string | `default` | +- [`spec`](#spec): map + - [`tls`](#spec-tls): map + - [`enabled`](#spec-tls-enabled): boolean | `false` + - [`tlsMinVersion`](#spec-tls-tlsminversion): string | `TLSv1_2` + - [`tlsMaxVersion`](#spec-tls-tlsmaxversion): string + - [`cipherSuites`](#spec-tls-ciphersuites): list of strings + - [`sds`](#spec-tls-sds): map of strings + - [`clusterName`](#spec-tls-sds): string + - [`certResource`](#spec-tls-sds): string + - [`defaults`](#spec-defaults): map + - [`maxConnections`](#spec-defaults-maxconnections): number + - [`maxPendingRequests`](#spec-defaults-maxpendingrequests): number + - [`maxConcurrentRequests`](#spec-defaults-maxconcurrentrequests): number + - [`passiveHealthCheck`](#spec-defaults-passivehealthcheck): map + - [`interval`](#spec-defaults-passivehealthcheck): string + - [`maxFailures`](#spec-defaults-passivehealthcheck): integer + - [`enforcingConsecutive5xx`](#spec-defaults-passivehealthcheck): number + - [`maxEjectionPercent`](#spec-defaults-passivehealthcheck): number + - [`baseEjectionTime`](#spec-defaults-passivehealthcheck): string + - [`listeners`](#spec-listeners): list of maps + - [`port`](#spec-listeners-port): number | `0` + - [`protocol`](#spec-listeners-protocol): number | `tcp` + - [`services`](#spec-listeners-services): list of maps + - [`name`](#spec-listeners-services-name): string + - [`namespace`](#spec-listeners-services-namespace): string | current namespace | + - [`partition`](#spec-listeners-services-partition): string | current partition | + - [`hosts`](#spec-listeners-services-hosts): list of strings | `.ingress.*` + - [`requestHeaders`](#spec-listeners-services-requestheaders): map + - [`add`](#spec-listeners-services-requestheaders): map of strings + - [`set`](#spec-listeners-services-requestheaders): map of strings + - [`remove`](#spec-listeners-services-requestheaders): list of strings + - [`responseHeaders`](#spec-listeners-services-responseheaders): map + - [`add`](#spec-listeners-services-responseheaders): map of strings + - [`set`](#spec-listeners-services-responseheaders): map of strings + - [`remove`](#spec-listeners-services-responseheaders): list of strings + - [`tls`](#spec-listeners-services-tls): map + - [`sds`](#spec-listeners-services-tls-sds): map of strings + - [`clusterName`](#spec-listeners-services-tls-sds): string + - [`certResource`](#spec-listeners-services-tls-sds): string + - [`maxConnections`](#spec-listeners-services-maxconnections): number | `0` + - [`maxPendingRequests`](#spec-listeners-services-maxconnections): number | `0` + - [`maxConcurrentRequests`](#spec-listeners-services-maxconnections): number | `0` + - [`passiveHealthCheck`](#spec-listeners-services-passivehealthcheck): map + - [`interval`](#spec-listeners-services-passivehealthcheck): string + - [`maxFailures`](#spec-listeners-services-passivehealthcheck): number + - [`enforcingConsecutive5xx`](#spec-listeners-services-passivehealthcheck): number + - [`maxEjectionPercent`](#spec-listeners-services-passivehealthcheck): integer + - [`baseEjectionTime`](#spec-listeners-services-passivehealthcheck): string + - [`tls`](#spec-listeners-tls): map + - [`enabled`](#spec-listeners-tls-enabled): boolean | `false` + - [`tlsMinVersion`](#spec-listeners-tls-tlsminversion): string | `TLSv1_2` + - [`tlsMaxVersion`](#spec-listeners-tls-tlsmaxversion): string + - [`cipherSuites`](#spec-listeners-tls-ciphersuites): list of strings + - [`sds`](#spec-listeners-tls-sds): map of strings + - [`clusterName`](#spec-listeners-tls-sds): string + - [`certResource`](#spec-listeners-tls-sds): string -## Usage + -1. Verify that your datacenter meets the conditions specified in the [Requirements](#requirements). -1. Create a file containing the configuration entry settings (see [Configuration](#configuration)). -1. Apply the configuration settings using one of the following methods: - - Kubernetes CRD: Refer to the [Custom Resource Definitions](/consul/docs/k8s/crds) documentation for details. - - Issue the `consul config write` command: Refer to the [Consul Config Write](/consul/commands/config/write) documentation for details. + -## Configuration +## Complete configuration -Use the following syntax to configure an ingress gateway. +When every field is defined, an ingress gateway configuration entry has the following form: - - -```hcl -Kind = "ingress-gateway" -Name = "" + +```hcl +Kind = "ingress-gateway" +Name = "" +Namespace = "" +Partition = "" +Meta = { + = "" +} +TLS = { + Enabled = false + TLSMinVersion = "TLSv1_2" + TLSMaxVersion = "" + CipherSuites = [ + "" + ] + SDS = { + ClusterName = "" + CertResource = "" + } +} +Defaults = { + MaxConnections = + MaxPendingRequests = + MaxConcurrentRequests = + PassiveHealthCheck = { + Interval = "" + MaxFailures = + EnforcingConsecutive5xx = + MaxEjectionPercent = + BaseEjectionTime = "" + } +} Listeners = [ { - Port = - Protocol = "" + Port = 0 + Protocol = "tcp" Services = [ { - Name = "" + Name = "" + Namespace = "" + Partition = "" + Hosts = [ + ".ingress.*" + ] + RequestHeaders = { + Add = { + RequestHeaderName = "" + } + Set = { + RequestHeaderName = "" + } + Remove = [ + "" + ] + } + ResponseHeaders = { + Add = { + ResponseHeaderName = "" + } + Set = { + ResponseHeaderName = "" + } + Remove = [ + "" + ] + } + TLS = { + SDS = { + ClusterName = "" + CertResource = "" + } + } + MaxConnections = + MaxPendingRequests = + MaxConcurrentRequests = + PassiveHealthCheck = { + Interval = "" + MaxFailures = + EnforcingConsecutive5xx = + MaxEjectionPercent = + BaseEjectionTime = "" + } + }] + TLS = { + Enabled = false + TLSMinVersion = "TLSv1_2" + TLSMaxVersion = "" + CipherSuites = [ + "" + ] + SDS = { + ClusterName = "" + CertResource = "" } - ] - + } } ] - ``` + + + + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 kind: IngressGateway metadata: - name: + name: + namespace: "" spec: + tls: + enabled: false + tlsSMinVersion: TLSv1_2 + tlsMaxVersion: "" + cipherSuites: + - + sds: + clusterName: + certResource: + defaults: + maxConnections: + maxPendingRequests: + maxConcurrentRequests: + passiveHealthCheck: + interval: "" + maxFailures: + enforcingConsecutive5xx: + maxEjectionPercent: + baseEjectionTime: "" listeners: - - port: - protocol: + - port: 0 + protocol: tcp services: - name: + namespace: + partition: + hosts: + - .ingress.* + requestHeaders: + add: + requestHeaderName: + set: + requestHeaderName: + remove: + - + responseHeaders: + add: + responseHeaderName: + set: + responseHeaderName: + remove: + - + tls: + sds: + clusterName: + certResource: + maxConnections: + maxPendingRequests: + maxConcurrentRequests: + passiveHealthCheck: + interval: "" + maxFailures: + enforcingConsecutive5xx: + maxEjectionPercent: + baseEjectionTime: "" + tls: + enabled: false + tlsMinVersion: TLSv1_2 + tlsMaxVersion: + cipherSuites: + - + sds: + clusterName: + certResource: ``` + + + + ```json { - "Kind": "ingress-gateway", - "Name": "", - "Listeners": [ + "Kind" : "ingress-gateway", + "Name" : "", + "Namespace" : "", + "Partition" : "", + "Meta": { + "" : "" + }, + "TLS" : { + "Enabled" : false, + "TLSMinVersion" : "TLSv1_2", + "TLSMaxVersion" : "", + "CipherSuites" : [ + "" + ], + "SDS": { + "ClusterName" : "", + "CertResource" : "" + } + }, + "Defaults" : { + "MaxConnections" : , + "MaxPendingRequests" : , + "MaxConcurrentRequests": , + "PassiveHealthCheck" : { + "interval": ", + "maxFailures": , + "enforcingConsecutive5xx":, + "maxEjectionPercent": , + "baseEjectionTime": "" + } + }, + "Listeners" : [ { - "Port": , - "Protocol": "", - "Services": [ + "Port" : 0, + "Protocol" : "tcp", + "Services" : [ { - "Name": "" + "Name" : "", + "Namespace" : "", + "Partition" : "", + "Hosts" : [ + ".ingress.*" + ], + "RequestHeaders" : { + "Add" : { + "RequestHeaderName" : "" + }, + "Set" : { + "RequestHeaderName" : "" + }, + "Remove" : [ + "" + ] + }, + "ResponseHeaders" : { + "Add" : { + "ResponseHeaderName" : "" + }, + "Set" : { + "ResponseHeaderName" : "" + }, + "Remove" : [ + "" + ] + }, + "TLS" : { + "SDS" : { + "ClusterName" : "", + "CertResource" : "" + } + }, + "MaxConnections" : , + "MaxPendingRequests" : , + "MaxConcurrentRequests" : , + "PassiveHealthCheck" : { + "interval": ", + "maxFailures": , + "enforcingConsecutive5xx":, + "maxEjectionPercent": , + "baseEjectionTime": "" } - ] + ], + "TLS" : { + "Enabled" : false, + "TLSMinVersion" : "TLSv1_2", + "TLSMaxVersion" : "", + "CipherSuites" : [ + "" + ], + "SDS" : { + "ClusterName" : "", + "CertResource" : "" + } + } } ] } - ``` - - - + -```hcl -Kind = "ingress-gateway" -Name = "" -Namespace = "" -Partition = "" +## Specification -Listeners = [ - { - Port = - Protocol = "" - Services = [ - { - Name = "" - } - ] +This section provides details about the fields you can configure in the ingress gateway configuration entry. - } -] + -``` + -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: IngressGateway -metadata: - name: - namespace: - partition: +### `Kind` -spec: - listeners: - - port: - protocol: - services: - - name: -``` +Specifies the type of configuration entry. Must be set to `ingress-gateway`. -```json -{ - "Kind": "ingress-gateway", - "Name": "", - "Namespace": "", - "Partition": "", +#### Values - "Listeners": [ - { - "Port": , - "Protocol": "", - "Services": [ - { - "Name": "" - } - ] - } - ] -} -``` +- Default: None +- This field is required. +- Data type: String value that must be set to `ingress-gateway`. - - - +### `Name` -Refer to the [Available Fields](#available-fields) section for complete information about all ingress gateway configuration entry options and to the [Example Configurations](#example-configurations) section for example use-cases. +Specifies a name for the gateway. The name is metadata that you can use to reference the configuration entry when performing Consul operations with the [`consul config` command](/consul/commands/config). -### Scope +#### Values -[Configuration entries](/consul/docs/agent/config-entries) are global in scope. A configuration entry for a gateway name applies across the default partition of all federated Consul datacenters. If ingress gateways in different Consul datacenters need to route to different sets of services within their datacenter then the ingress gateways **must** be registered with different names or partitions. See [Ingress Gateway](/consul/docs/connect/gateways/ingress-gateway) for more information. +- Default: None +- This field is required. +- Data type: String -### Wildcard Service Specification +### `Namespace` -Ingress gateways can optionally target all services within a Consul namespace by -specifying a wildcard `*` as the service name. A wildcard specifier allows -for a single listener to route traffic to all available services on the -Consul service mesh, differentiating between the services by their host/authority -header. +Specifies the namespace to apply the configuration entry in. Refer to [Namespaces](/consul/docs/enterprise/namespaces) for additional information about Consul namespaces. -A wildcard specifier provides the following properties for an ingress -gateway: +If unspecified, the ingress gateway is applied to the `default` namespace. You can override the namespace when using the [`/config` API endpoint](/consul/api-docs/config) to register the configuration entry by specifying the `ns` query parameter. -- All services with the same - [protocol](/consul/docs/connect/config-entries/ingress-gateway#protocol) as the - listener will be routable. -- The ingress gateway will route traffic based on the host/authority header, - expecting a value matching `.ingress.*`, or if using namespaces, - `.ingress..*`. This matches the [Consul DNS - ingress subdomain](/consul/docs/discovery/dns#ingress-service-lookups). +#### Values -A wildcard specifier cannot be set on a listener of protocol `tcp`. +- Default: `default`, +- Data type: String -### ACLs +### `Partition` -Configuration entries may be protected by [ACLs](/consul/docs/security/acl). +Specifies the admin partition that the ingress gateway applies to. The value must match the partition in which the gateway is registered. Refer to [Admin partitions](/consul/docs/enterprise/admin-partitions) for additional information. -Reading an `ingress-gateway` config entry requires `service:read` on the `Name` -field of the config entry. +If unspecified, the ingress gateway is applied to the `default` partition. You can override the partition when using the [`/config` API endpoint](/consul/api-docs/config) to register the configuration entry by specifying the `partition` query parameter. -Creating, updating, or deleting an `ingress-gateway` config entry requires -`operator:write`. +#### Values -### Example Configurations +- Default: `default +- Data type: String -The following examples describe possible use-cases for ingress gateway configuration entries. +### `Meta` -#### TCP listener +Defines an arbitrary set of key-value pairs to store in the Consul KV. -The following example sets up a TCP listener on an ingress gateway named `us-east-ingress` to proxy traffic to the `db` service. The Consul Enterprise version also posits the gateway listener inside the `default` [namespace](/consul/docs/enterprise/namespaces) and the `team-frontend` [admin partition](/consul/docs/enterprise/admin-partitions): +#### Values - - - +- Default: None +- Data type: Map of one or more key-value pairs. + - keys: String + - values: String, integer, or float -```hcl -Kind = "ingress-gateway" -Name = "us-east-ingress" +### `TLS` -Listeners = [ - { - Port = 3456 - Protocol = "tcp" - Services = [ - { - Name = "db" - } - ] - } -] -``` +Specifies the TLS configuration settings for the gateway. -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: IngressGateway -metadata: - name: us-east-ingress -spec: - listeners: - - port: 3456 - protocol: tcp - services: - - name: db -``` +#### Values -```json -{ - "Kind": "ingress-gateway", - "Name": "us-east-ingress", - "Listeners": [ - { - "Port": 3456, - "Protocol": "tcp", - "Services": [ - { - "Name": "db" - } - ] - } - ] -} -``` +- Default: No default +- Data type: Object that can contain the following fields: + - [`Enabled`](#tls-enabled) + - [`TLSMinVersion`](#tls-tlsminversion) + - [`TLSMaxVersion`](#tls-tlsmaxversion) + - [`CipherSuites`](#tls-ciphersuites) + - [`SDS`](#tls-sds) - +### `TLS.Enabled` - - - +Enables and disables TLS for the configuration entry. Set to `true` to enable built-in TLS for every listener on the gateway. TLS is disabled by default. -```hcl -Kind = "ingress-gateway" -Name = "us-east-ingress" -Namespace = "default" -Partition = "team-frontend" +When enabled, Consul adds each host defined in every service's `Hosts` field to the gateway's x509 certificate as a DNS subject alternative name (SAN). -Listeners = [ - { - Port = 3456 - Protocol = "tcp" - Services = [ - { - Namespace = "ops" - Name = "db" - } - ] - } -] -``` +#### Values -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: IngressGateway -metadata: - name: us-east-ingress - namespace: default -spec: - listeners: - - port: 3456 - protocol: tcp - services: - - name: db - namespace: ops -``` + - Default: `false` + - Data type: boolean -```json -{ - "Kind": "ingress-gateway", - "Name": "us-east-ingress", - "Namespace": "default", - "Partition": "team-frontend", - "Listeners": [ - { - "Port": 3456, - "Protocol": "tcp", - "Services": [ - { - "Namespace": "ops", - "Name": "db" - } - ] - } - ] -} -``` +### `TLS.TLSMinVersion` - +Specifies the minimum TLS version supported for gateway listeners. - - +#### Values -#### Wildcard HTTP Listener +- Default: Depends on the version of Envoy: + - Envoy v1.22.0 and later: `TLSv1_2` + - Older versions: `TLSv1_0` +- Data type: String with one of the following values: + - `TLS_AUTO` + - `TLSv1_0` + - `TLSv1_1` + - `TLSv1_2` + - `TLSv1_3` -In the following example, two listeners are configured on an ingress gateway named `us-east-ingress`: +### `TLS.TLSMaxVersion` -- The first listener is configured to listen on port `8080` and uses a wildcard (`*`) to proxy traffic to all services in the datacenter. -- The second listener exposes the `api` and `web` services on port `4567` at user-provided hosts. -- TLS is enabled on every listener. -- The `max_connections` of the ingress gateway proxy to each upstream cluster is set to 4096. +Specifies the maximum TLS version supported for gateway listeners. -The Consul Enterprise version implements the following additional configurations: +#### Values -- The ingress gateway is set up in the `default` [namespace](/consul/docs/enterprise/namespaces) and proxies traffic to all services in the `frontend` namespace. -- The `api` and `web` services are proxied to team-specific [admin partitions](/consul/docs/enterprise/admin-partitions): +- Default: Depends on the version of Envoy: + - Envoy v1.22.0 and later: `TLSv1_2` + - Older versions: `TLSv1_0` +- Data type: String with one of the following values: + - `TLS_AUTO` + - `TLSv1_0` + - `TLSv1_1` + - `TLSv1_2` + - `TLSv1_3` - - - +### `TLS.CipherSuites[]` -```hcl -Kind = "ingress-gateway" -Name = "us-east-ingress" +Specifies a list of cipher suites that gateway listeners support when negotiating connections using TLS 1.2 or older. If unspecified, the Consul applies the default for the version of Envoy in use. Refer to the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#envoy-v3-api-field-extensions-transport-sockets-tls-v3-tls parameters-cipher-suites) for details. -TLS { - Enabled = true -} +#### Values -Defaults { - MaxConnections = 4096 -} +- Default: None +- Data type: List of string values. Refer to the [Consul repository](https://github.com/hashicorp/consul/blob/v1.11.2/types/tls.go#L154-L169) for a list of supported ciphers. -Listeners = [ - { - Port = 8080 - Protocol = "http" - Services = [ - { - Name = "*" - } - ] - }, - { - Port = 4567 - Protocol = "http" - Services = [ - { - Name = "api" - Hosts = ["foo.example.com"] - }, - { - Name = "web" - Hosts = ["website.example.com"] - } - ] - } -] -``` +### `TSL.SDS` -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: IngressGateway -metadata: - name: us-east-ingress -spec: - tls: - enabled: true - listeners: - - port: 8080 - protocol: http - services: - - name: '*' - - port: 4567 - protocol: http - services: - - name: api - hosts: ['foo.example.com'] - - name: web - hosts: ['website.example.com'] -``` +Specifies parameters for loading the TLS certificates from an external SDS service. Refer to [Serve custom TLS certificates from an external service](/consul/docs/connect/gateways/ingress-gateway/tls-external-service) for additional information. + +Consul applies the SDS configuration specified in this field as defaults for all listeners defined in the gateway. You can override the SDS settings for per listener or per service defined in the listener. Refer to the following configurations for additional information: + +- [`Listeners.TLS.SDS`](#listeners-tls-sds): Configures SDS settings for all services in the listener. +- [`Listeners.Services.TLS.SDS`](#listeners-services-tls-sds): Configures SDS settings for a specific service defined in the listener. + +#### Values + +- Default: None +- Data type: Map containing the following fields: + - `ClusterName` + - `CertResource` + +The following table describes how to configure SDS parameters. + +| Parameter | Description | Data type | +| --- | --- | --- | +| `ClusterName` | Specifies the name of the SDS cluster where Consul should retrieve certificates. The cluster must be specified in the gateway's bootstrap configuration. | String | +| `CertResource` | Specifies an SDS resource name. Consul requests the SDS resource name when fetching the certificate from the SDS service. When set, Consul serves the certificate to all listeners over TLS unless a listener-specific TLS configuration overrides the SDS configuration. | String | + +### `Defaults` + +Specifies default configurations for connecting upstream services. + +#### Values + +- Default: None +- The data type is a map containing the following parameters: + + - [`MaxConnections`](#defaults-maxconnections) + - [`MaxPendingRequests`](#defaults-maxpendingrequests) + - [`MaxConcurrentRequests`](#defaults-maxconcurrentrequests) + +### `Defaults.MaxConnections` + +Specifies the maximum number of HTTP/1.1 connections a service instance is allowed to establish against the upstream. + +#### Values + +- Default value is `0`, which instructs Consul to use the proxy's configuration. For Envoy, the default is `1024`. +- Data type: Integer + +### `Defaults.MaxPendingRequests` + +Specifies the maximum number of requests that are allowed to queue while waiting to establish a connection. Listeners must use an L7 protocol for this configuration to take effect. Refer to [`Listeners.Protocol`](#listeners-protocol). + +#### Values + +- Default value is `0`, which instructs Consul to use the proxy's configuration. For Envoy, the default is `1024`. +- Data type: Integer + +### `Defaults.MaxConcurrentRequests` + +Specifies the maximum number of concurrent HTTP/2 traffic requests that are allowed at a single point in time. Listeners must use an L7 protocol for this configuration to take effect. Refer to [`Listeners.Protocol`](#listeners-protocol). + +#### Values + +- Default value is `0`, which instructs Consul to use the proxy's configuration. For Envoy, the default is `1024`. +- Data type: Integer + +### `Defaults.PassiveHealthCheck` + +Defines a passive health check configuration. Passive health checks remove hosts from the upstream cluster when they are unreachable or return errors. + +#### Values + +- Default: None +- Data type: Map + +The following table describes the configurations for passive health checks: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | + | `Interval` | Specifies the time between checks. | string | `0s` | + | `MaxFailures` | Specifies the number of consecutive failures allowed per check interval. If exceeded, Consul removes the host from the load balancer. | integer | `0` | + | `EnforcingConsecutive5xx` | Specifies a percentage that indicates how many times out of 100 that Consul ejects the host when it detects an outlier status. The outlier status is determined by consecutive errors in the 500-599 response range. | integer | `100` | + | `MaxEjectionPercent` | Specifies the maximum percentage of an upstream cluster that Consul ejects when the proxy reports an outlier. Consul ejects at least one host when an outlier is detected regardless of the value. | integer | `10` | + | `BaseEjectionTime` | Specifies the minimum amount of time that an ejected host must remain outside the cluster before rejoining. The real time is equal to the value of the `BaseEjectionTime` multiplied by the number of times the host has been ejected. | string | `30s` | + +### `Listeners[]` + +Specifies a list of listeners in the mesh for the gateway. Listeners are uniquely identified by their port number. + +#### Values + +- Default: None +- Data type: List of maps containing the following fields: + - [`Port`](#listeners-port) + - [`Protocol`](#listeners-protocol) + - [`Services[]`](#listeners-services) + - [`TLS`](#listeners-tls) + +### `Listeners[].Port` + +Specifies the port that the listener receives traffic on. The port is bound to the IP address specified in the [`-address`](/consul/commands/connect/envoy#address) flag when starting the gateway. The listener port must not conflict with the health check port. + +#### Values + +- Default: `0` +- Data type: Integer + +### `Listeners[].Protocol` + +Specifies the protocol associated with the listener. To enable L7 network management capabilities, specify one of the following values: + +- `http` +- `http2` +- `grpc` + +#### Values + +- Default: `tcp` +- Data type: String that contains one of the following values: + + - `tcp` + - `http` + - `http2` + - `grpc` + +### `Listeners[].Services[]` + +Specifies a list of services that the listener exposes to services outside the mesh. Each service must have a unique name. The `Namespace` field is required for Consul Enterprise datacenters. If the [`Listeners.Protocol`] field is set to `tcp`, then Consul can only expose one service. You can expose multiple services if the listener uses any other supported protocol. + +#### Values + +- Default: None +- Data type: List of maps that can contain the following fields: + - [`Name`](#listeners-services-name) + - [`Namespace`](#listeners-services-namespace) + - [`Partition`](#listeners-services-partition) + - [`Hosts`](#listeners-services-hosts) + - [`RequestHeaders`](#listeners-services-requestheaders) + - [`ResponseHeaders`](#listeners-services-responseheaders)` + - [`TLS`](#listeners-services-tls) + - [`MaxConnections`](#listeners-services-maxconnections) + - [`MaxPendingRequests`](#listeners-services-maxpendingrequests) + - [`MaxConcurrentRequests`](#listeners-services-maxconcurrentrequests) + - [`PassiveHealthCheck`](#listeners-services-passivehealthcheck) + +### `Listeners[].Services[].Name` + +Specifies the name of a service to expose to the listener. You can specify services in the following ways: + +- Provide the name of a service registered in the Consul catalog. +- Provide the name of a service defined in other configuration entries. Refer to [Service Mesh Traffic Management Overview](/consul/docs/connect/l7-traffic) for additional information. +- Provide a `*` wildcard to expose all services in the datacenter. Wild cards are not supported for listeners configured for TCP. Refer to [`Listeners[].Protocol`](#listeners-protocol) for additional information. + +#### Values + +- Default: None +- Data type: String + +### `Listeners[].Services[].Namespace` + +Specifies the namespace to use when resolving the location of the service. + +#### Values + +- Default: Current namespace +- Data type: String + +### `Listeners[].Services[].Partition` + +Specifies the admin partition to use when resolving the location of the service. + +#### Values + +- Default: Current partition +- Data type: String + +### `Listeners[].Services[].Hosts[]` + +Specifies one or more hosts that the listening services can receive requests on. The ingress gateway proxies external traffic to the specified services when external requests include `host` headers that match a host specified in this field. + +If unspecified, Consul matches requests to services using the `.ingress.*` domain. You cannot specify a host for listeners that communicate over TCP. You cannot specify a host when service names are specified with a `*` wildcard. Requests must include the correct host for Consul to proxy traffic to the service. + +When TLS is disabled, you can use the `*` wildcard to match all hosts. Disabling TLS may be suitable for testing and learning purposes, but we recommend enabling TLS in production environments. + +You can use the wildcard in the left-most DNS label to match a set of hosts. For example, `*.example.com` is valid, but `example.*` and `*-suffix.example.com` are invalid. + +#### Values + +- Default: None +- Data type: List of strings or `*` + +### `Listeners[].Services[].RequestHeaders` + +Specifies a set of HTTP-specific header modification rules applied to requests routed through the gateway. You cannot configure request headers if the listener protocol is set to `tcp`. Refer to [HTTP listener with Path-based Routing](#http-listener-with-path-based-routing) for an example configuration. + +#### Values + +- Default: None +- Data type: Object containing one or more fields that define header modification rules: + + - `Add`: Map of one or more key-value pairs + - `Set`: Map of one or more key-value pairs + - `Remove`: Map of one or more key-value pairs + +The following table describes how to configure values for request headers: + +| Rule | Description | Data type | +| --- | --- | --- | +| `Add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | Map of strings | +| `Set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | Map of strings | +| `Remove` | Defines a list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | List of strings | + +##### Use variable placeholders + +For `Add` and `Set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated at runtime. + +### `Listeners[].Services[].ResponseHeaders` + +Specifies a set of HTTP-specific header modification rules applied to responses routed through the gateway. You cannot configure response headers if the listener protocol is set to `tcp`. Refer to [HTTP listener with Path-based Routing](#http-listener-with-path-based-routing) for an example configuration. + +#### Values + +- Default: None +- Data type: Map containing one or more fields that define header modification rules: + + - `Add`: Map of one or more key-value pairs + - `Set`: Map of one or more key-value pairs + - `Remove`: Map of one or more key-value pairs + +The following table describes how to configure values for request headers: + +| Rule | Description | Data type | +| --- | --- | --- | +| `Add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | Map of strings | +| `Set` | Defines a set of key-value pairs to add to the response header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | Map of strings | +| `Remove` | Defines a list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | List of strings | + +##### Use variable placeholders + +For `Add` and `Set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated at runtime. + +### `Listeners[].Services[].TLS` + +Specifies a TLS configuration for a specific service. The settings in this configuration overrides the main [`TLS`](#tls) settings for the configuration entry. + +#### Values + +- Default: None +- Data type: Map + +### `Listeners[].Services[].TLS.SDS` + +Specifies parameters that configure the listener to load TLS certificates from an external SDS. Refer to [Serve custom TLS certificates from an external service](/consul/docs/connect/gateways/ingress-gateway/tls-external-service) for additional information. + +This configuration overrides the main [`TLS.SDS`](#tls-sds) settings for configuration entry. If unspecified, Consul applies the top-level [`TLS.SDS`](#tls-sds) settings. + +#### Values + +- Default: None +- Data type: Map containing the following fields: + + - `ClusterName` + - `CertResource` + +The following table describes how to configure SDS parameters. Refer to [Configure static SDS clusters](/consul/docs/connect/gateways/ingress-gateway/tls-external-service#configure-static-sds-clusters) for usage information: + +| Parameter | Description | Data type | +| `ClusterName` | Specifies the name of the SDS cluster where Consul should retrieve certificates. The cluster must be specified in the gateway's bootstrap configuration. | String | +| `CertResource` | Specifies an SDS resource name. Consul requests the SDS resource name when fetching the certificate from the SDS service. When set, Consul serves the certificate to all listeners over TLS unless a listener-specific TLS configuration overrides the SDS configuration. | String | + +### `Listeners[].Services[].MaxConnections` + +Specifies the maximum number of HTTP/1.1 connections a service instance is allowed to establish against the upstream. + +When defined, this field overrides the [`Defaults.MaxConnections`](#defaults-maxconnections) configuration. + +#### Values + +- Default: None +- Data type: Integer + +### `Listeners[].Services.MaxPendingRequests` + +Specifies the maximum number of requests that are allowed to queue while waiting to establish a connection. When defined, this field overrides the value specified in the [`Defaults.MaxPendingRequests`](#defaults-maxpendingrequests) field of the configuration entry. + +Listeners must use an L7 protocol for this configuration to take effect. Refer to [`Listeners.Protocol`](#listeners-protocol) for more information. + +#### Values + +- Default: None +- Data type: Integer + +### `Listeners[].Services[].MaxConcurrentRequests` + +Specifies the maximum number of concurrent HTTP/2 traffic requests that the service is allowed at a single point in time. This field overrides the value set in the [`Defaults.MaxConcurrentRequests`](#defaults-maxconcurrentrequests) field of the configuration entry. + +Listeners must use an L7 protocol for this configuration to take effect. Refer to [`Listeners.Protocol`](#listeners-protocol) for more information. + +#### Values + +- Default: None +- Data type: Integer + +### `Listeners[].Services[].PassiveHealthCheck` + +Defines a passive health check configuration for the service. Passive health checks remove hosts from the upstream cluster when the service is unreachable or returns errors. This field overrides the value set in the [`Defaults.PassiveHealthCheck`](#defaults-passivehealthcheck) field of the configuration entry. + +#### Values + +- Default: None +- Data type: Map + +The following table describes the configurations for passive health checks: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | + | `Interval` | Specifies the time between checks. | string | `0s` | + | `MaxFailures` | Specifies the number of consecutive failures allowed per check interval. If exceeded, Consul removes the host from the load balancer. | integer | `0` | + | `EnforcingConsecutive5xx` | Specifies a percentage that indicates how many times out of 100 that Consul ejects the host when it detects an outlier status. The outlier status is determined by consecutive errors in the 500-599 response range. | integer | `100` | + | `MaxEjectionPercent` | Specifies the maximum percentage of an upstream cluster that Consul ejects when the proxy reports an outlier. Consul ejects at least one host when an outlier is detected regardless of the value. | integer | `10` | + | `BaseEjectionTime` | Specifies the minimum amount of time that an ejected host must remain outside the cluster before rejoining. The real time is equal to the value of the `BaseEjectionTime` multiplied by the number of times the host has been ejected. | string | `30s` | + +### `Listeners[].TLS` + +Specifies the TLS configuration for the listener. If unspecified, Consul applies any [service-specific TLS configurations](#listeners-services-tls). If neither the listener- nor service-specific TLS configurations are specified, Consul applies the main [`TLS`](#tls) settings for the configuration entry. + +#### Values + +- Default: None +- Data type: Map that can contain the following fields: + - [`Enabled`](#listeners-tls-enabled) + - [`TLSMinVersion`](#listeners-tls-tlsminversion) + - [`TLSMaxVersion`](#listeners-tls-tlsmaxversion) + - [`CipherSuites`](#listeners-tls-ciphersuites) + - [`SDS`](#listeners-tls-sds) + +### `Listeners[].TLS.Enabled` + +Set to `true` to enable built-in TLS for the listener. If enabled, Consul adds each host defined in every service's `Hosts` field to the gateway's x509 certificate as a DNS subject alternative name (SAN). + +#### Values + + - Default: `false` + - Data type: boolean + +### `Listeners[].TLS.TLSMinVersion` + +Specifies the minimum TLS version supported for the listener. + +#### Values + +- Default: Depends on the version of Envoy: + - Envoy v1.22.0 and later: `TLSv1_2` + - Older versions: `TLSv1_0` +- Data type: String with one of the following values: + - `TLS_AUTO` + - `TLSv1_0` + - `TLSv1_1` + - `TLSv1_2` + - `TLSv1_3` + +### `Listeners[].TLS.TLSMaxVersion` + +Specifies the maximum TLS version supported for the listener. + +#### Values + +- Default: Depends on the version of Envoy: + - Envoy v1.22.0 and later: `TLSv1_2` + - Older versions: `TLSv1_0` +- Data type: String with one of the following values: + - `TLS_AUTO` + - `TLSv1_0` + - `TLSv1_1` + - `TLSv1_2` + - `TLSv1_3` + +### `Listeners[].TLS.CipherSuites` + +Specifies a list of cipher suites that the listener supports when negotiating connections using TLS 1.2 or older. If unspecified, the Consul applies the default for the version of Envoy in use. Refer to the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#envoy-v3-api-field-extensions-transport-sockets-tls-v3-tls parameters-cipher-suites) for details. + +#### Values + +- Default: None +- Data type: List of string values. Refer to the [Consul repository](https://github.com/hashicorp/consul/blob/v1.11.2/types/tls.go#L154-L169) for a list of supported ciphers. + +### `Listeners[].TLS.SDS` + +Specifies parameters for loading the TLS certificates from an external SDS service. Refer to [Serve custom TLS certificates from an external service](/consul/docs/connect/gateways/ingress-gateway/ingress-gateways-tls-external-service) for additional information. + +Consul applies the SDS configuration specified in this field to all services in the listener. You can override the `Listeners.TLS.SDS` configuration per service by configuring the [`Listeners.Services.TLS.SDS`](#listeners-services-tls-sds) settings for each service. + +#### Values + +- Default: None +- The data type is a map containing `ClusterName` and `CertResource` fields. + +The following table describes how to configure SDS parameters. Refer to [Configure static SDS clusters](/consul/docs/connect/gateways/ingress-gateway/ingress-gateways-tls-external-service#configure-static-sds-clusters) for usage information: + +| Parameter | Description | Data type | +| --- | --- | --- | +| `ClusterName` | Specifies the name of the SDS cluster where Consul should retrieve certificates. The cluster must be specified in the gateway's bootstrap configuration. | String | +| `CertResource` | Specifies an SDS resource name. Consul requests the SDS resource name when fetching the certificate from the SDS service. When set, Consul serves the certificate to all listeners over TLS unless a listener-specific TLS configuration overrides the SDS configuration. | String | + + + + + +### `apiVersion` + +Kubernetes-only parameter that specifies the version of the Consul API that the configuration entry maps to Kubernetes configurations. The value must be `consul.hashicorp.com/v1alpha1`. + +### `kind` + +Specifies the type of configuration entry to implement. Must be set to `IngressGateway`. + +#### Values + +- Default: None +- This field is required. +- Data type: String value that must be set to `IngressGateway`. + +### `metadata` + +Specifies metadata for the gateway. + +#### Values + +- Default: None +- This field is required +- Data type: Map the contains the following fields: + - [`name`](#metadata-name) + - [`namespace`](#metadata-namespace) + +### `metadata.name` + +Specifies a name for the gateway. The name is metadata that you can use to reference the configuration entry when performing Consul operations with the [`consul config` command](/consul/commands/config). + +#### Values + +- Default: None +- This field is required. +- Data type: String + +### `metadata.namespace` + +Specifies the namespace to apply the configuration entry in. Refer to [Namespaces](/consul/docs/enterprise/namespaces) for additional information about Consul namespaces. + +If unspecified, the ingress gateway is applied to the `default` namespace. You can override the namespace when using the [`/config` API endpoint](/consul/api-docs/config) to register the configuration entry by specifying the `ns` query parameter. + +#### Values + +- Default: `default`, +- Data type: String + +### `spec` + +Kubernetes-only field that contains all of the configurations for ingress gateway pods. + +#### Values + +- Default: None +- This field is required. +- Data type: Map containing the following fields: + - [`tls`](#tls) + - [`defaults`](#defaults) + - [`listeners`](#listeners) + +### `spec.tls` + +Specifies the TLS configuration settings for the gateway. + +#### Values + +- Default: No default +- Data type: Object that can contain the following fields: + - [`enabled`](#tls-enabled) + - [`tlsMinVersion`](#spec-tls-tlsminversion) + - [`tlsMaxVersion`](#spec-tls-tlsmaxversion) + - [`cipherSuites`](#spec-tls-tlsciphersuites) + - [`sds`](#spec-tls-sds) +### `spec.tls.enabled` + +Enables and disables TLS for the configuration entry. Set to `true` to enable built-in TLS for every listener on the gateway. TLS is disabled by default. + +When enabled, Consul adds each host defined in every service's `Hosts` field to the gateway's x509 certificate as a DNS subject alternative name (SAN). +#### Values + + - Default: `false` + - Data type: boolean +### `spec.tls.tlsMinVersion` + +Specifies the minimum TLS version supported for gateway listeners. + +#### Values + +- Default: Depends on the version of Envoy: + - Envoy v1.22.0 and later: `TLSv1_2` + - Older versions: `TLSv1_0` +- Data type: String with one of the following values: + - `TLS_AUTO` + - `TLSv1_0` + - `TLSv1_1` + - `TLSv1_2` + - `TLSv1_3` + +### `spec.tls.tlsMaxVersion` + +Specifies the maximum TLS version supported for gateway listeners. + +#### Values + +- Default: Depends on the version of Envoy: + - Envoy v1.22.0 and later: `TLSv1_2` + - Older versions: `TLSv1_0` +- Data type: String with one of the following values: + - `TLS_AUTO` + - `TLSv1_0` + - `TLSv1_1` + - `TLSv1_2` + - `TLSv1_3` + +### `spec.tls.cipherSuites[]` + +Specifies a list of cipher suites that gateway listeners support when negotiating connections using TLS 1.2 or older. If unspecified, the Consul applies the default for the version of Envoy in use. Refer to the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#envoy-v3-api-field-extensions-transport-sockets-tls-v3-tls parameters-cipher-suites) for details. + +#### Values + +- Default: None +- Data type: List of string values. Refer to the [Consul repository](https://github.com/hashicorp/consul/blob/v1.11.2/types/tls.go#L154-L169) for a list of supported ciphers. + +### `spec.tls.sds` + +Specifies parameters for loading the TLS certificates from an external SDS service. Refer to [Serve custom TLS certificates from an external service](/consul/docs/connect/gateways/ingress-gateway/ingress-gateways-tls-external-service) for additional information. + +Consul applies the SDS configuration specified in this field as defaults for all listeners defined in the gateway. You can override the SDS settings for per listener or per service defined in the listener. Refer to the following configurations for additional information: + +- [`spec.listeners.tls.sds`](#spec-listeners-tls-sds): Configures SDS settings for all services in the listener. +- [`spec.listeners.services.tls.sds`](#spec-listeners-services-tls-sds): Configures SDS settings for a specific service defined in the listener. + +#### Values + +- Default: None +- Data type: Map containing the following fields: + - [`clusterName`] + - [`certResource`] + +The following table describes how to configure SDS parameters. + +| Parameter | Description | Data type | +| --- | --- | --- | +| `clusterName` | Specifies the name of the SDS cluster where Consul should retrieve certificates. The cluster must be specified in the gateway's bootstrap configuration. | String | +| `certResource` | Specifies an SDS resource name. Consul requests the SDS resource name when fetching the certificate from the SDS service. When set, Consul serves the certificate to all listeners over TLS unless a listener-specific TLS configuration overrides the SDS configuration. | String | + +### `spec.defaults` + +Specifies default configurations for upstream connections. + +#### Values + +- Default: None +- The data type is a map containing the following parameters: + + - [`maxConnections`](#spec-defaults-maxconnections) + - [`maxPendingRequests`](#spec-defaults-maxpendingrequests) + - [`maxConcurrentRequests`](#spec-defaults-maxconcurrentrequests) + +### `spec.defaults.maxConnections` + +Specifies the maximum number of HTTP/1.1 connections a service instance is allowed to establish against the upstream. If unspecified, Consul uses Envoy's configuration. The default configuration for Envoy is `1024`. + +#### Values + +- Default: `0` +- Data type: Integer + +### `spec.defaults.maxPendingRequests` + +Specifies the maximum number of requests that are allowed to queue while waiting to establish a connection. Listeners must use an L7 protocol for this configuration to take effect. Refer to [`spec.listeners.Protocol`](#spec-listeners-protocol). + +If unspecified, Consul uses Envoy's configuration. The default for Envoy is `1024`. + +#### Values + +- Default: `0` +- Data type: Integer + +### `spec.defaults.maxConcurrentRequests` + +Specifies the maximum number of concurrent HTTP/2 traffic requests that are allowed at a single point in time. Listeners must use an L7 protocol for this configuration to take effect. Refer to [`spec.listeners.protocol`](#spec-listeners-protocol). + +If unspecified, Consul uses Envoy's configuration. The default for Envoy is `1024`. + +#### Values + +- Default: `0` +- Data type: Integer + +### `spec.defaults.passiveHealthCheck` + +Defines a passive health check configuration. Passive health checks remove hosts from the upstream cluster when they are unreachable or return errors. + +#### Values + +- Default: None +- Data type: Map + +The following table describes the configurations for passive health checks: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | + | `Interval` | Specifies the time between checks. | string | `0s` | + | `MaxFailures` | Specifies the number of consecutive failures allowed per check interval. If exceeded, Consul removes the host from the load balancer. | integer | `0` | + | `EnforcingConsecutive5xx` | Specifies a percentage that indicates how many times out of 100 that Consul ejects the host when it detects an outlier status. The outlier status is determined by consecutive errors in the 500-599 response range. | integer | `100` | + | `MaxEjectionPercent` | Specifies the maximum percentage of an upstream cluster that Consul ejects when the proxy reports an outlier. Consul ejects at least one host when an outlier is detected regardless of the value. | integer | `10` | + | `BaseEjectionTime` | Specifies the minimum amount of time that an ejected host must remain outside the cluster before rejoining. The real time is equal to the value of the `BaseEjectionTime` multiplied by the number of times the host has been ejected. | string | `30s` | + +### `spec.listeners[]` + +Specifies a list of listeners in the mesh for the gateway. Listeners are uniquely identified by their port number. + +#### Values + +- Default: None +- Data type: List of maps containing the following fields: + - [`port`](#spec-listeners-port) + - [`protocol`](#spec-listeners-protocol) + - [`services[]`](#spec-listeners-services) + - [`tls`](#spec-listeners-tls) + +### `spec.listeners[].port` + +Specifies the port that the listener receives traffic on. The port is bound to the IP address specified in the [`-address`](/consul/commands/connect/envoy#address) flag when starting the gateway. The listener port must not conflict with the health check port. + +#### Values + +- Default: `0` +- Data type: Integer + +### `spec.listeners[].protocol` + +Specifies the protocol associated with the listener. To enable L7 network management capabilities, specify one of the following values: + +- `http` +- `http2` +- `grpc` + +#### Values + +- Default: `tcp` +- Data type: String that contains one of the following values: + + - `tcp` + - `http` + - `http2` + - `grpc` + +### `spec.listeners[].services[]` + +Specifies a list of services that the listener exposes to services outside the mesh. Each service must have a unique name. The `namespace` field is required for Consul Enterprise datacenters. If the listener's [`protocol`](#spec-listeners-protocol) field is set to `tcp`, then Consul can only expose one service. You can expose multiple services if the listener uses any other supported protocol. + +#### Values + +- Default: None +- Data type: List of maps that can contain the following fields: + - [`name`](#spec-listeners-services-name) + - [`namespace`](#spec-listeners-services-namespace) + - [`partition`](#spec-listeners-services-partition) + - [`hosts`](#spec-listeners-services-hosts) + - [`requestHeaders`](#spec-listeners-services-requestheaders) + - [`responseHeaders`](#spec-listeners-services-responseheaders)` + - [`tlsLS`](#spec-listeners-services-tls) + - [`maxConnections`](#spec-listeners-services-maxconnections) + - [`maxPendingRequests`](#spec-listeners-services-maxpendingrequests) + - [`maxConcurrentRequests`](#spec-listeners-services-maxconcurrentrequests) + - [`passiveHealthCheck`](#spec-listeners-services-passivehealthcheck) + +### `spec.listeners[].services[].name` + +Specifies the name of a service to expose to the listener. You can specify services in the following ways: + +- Provide the name of a service registered in the Consul catalog. +- Provide the name of a service defined in other configuration entries. Refer to [Service Mesh Traffic Management Overview](/consul/docs/connect/l7-traffic) for additional information. Refer to [HTTP listener with path-based routes](#http-listener-with-path-based-routes) for an example. +- Provide a `*` wildcard to expose all services in the datacenter. Wild cards are not supported for listeners configured for TCP. Refer to [`spec.listeners.protocol`](#spec-listeners-protocol) for additional information. + +#### Values + +- Default: None +- Data type: String + +### `spec.listeners[].services[].namespace` + +Specifies the namespace to use when resolving the location of the service. + +#### Values + +- Default: Current namespace +- Data type: String + +### `spec.listeners[].services[].partition` + +Specifies the admin partition to use when resolving the location of the service. + +#### Values + +- Default: Current partition +- Data type: String + +### `spec.listeners[].services[].hosts[]` + +Specifies one or more hosts that the listening services can receive requests on. The ingress gateway proxies external traffic to the specified services when external requests include `host` headers that match a host specified in this field. + +If unspecified, Consul matches requests to services using the `.ingress.*` domain. You cannot specify a host for listeners that communicate over TCP. You cannot specify a host when service names are specified with a `*` wildcard. Requests must include the correct host for Consul to proxy traffic to the service. + +When TLS is disabled, you can use the `*` wildcard to match all hosts. Disabling TLS may be suitable for testing and learning purposes, but we recommend enabling TLS in production environments. + +You can use the wildcard in the left-most DNS label to match a set of hosts. For example, `*.example.com` is valid, but `example.*` and `*-suffix.example.com` are invalid. + +#### Values + +- Default: None +- Data type: List of strings or `*` + +### `spec.listeners[].services[].requestHeaders` + +Specifies a set of HTTP-specific header modification rules applied to requests routed through the gateway. You cannot configure request headers if the listener protocol is set to `tcp`. Refer to [HTTP listener with path-based routing](#http-listener-with-path-based-routing) for an example configuration. + +#### Values + +- Default: None +- Data type: Object containing one or more fields that define header modification rules: + + - `add`: Map of one or more key-value pairs + - `set`: Map of one or more key-value pairs + - `remove`: Map of one or more key-value pairs + +The following table describes how to configure values for request headers: + +| Rule | Description | Data type | +| --- | --- | --- | +| `add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | Map of strings | +| `set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | Map of strings | +| `remove` | Defines a list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | List of strings | + +##### Use variable placeholders + +For `add` and `set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated at runtime. + +### `spec.listeners[].services[].responseHeaders` + +Specifies a set of HTTP-specific header modification rules applied to responses routed through the gateway. You cannot configure response headers if the listener protocol is set to `tcp`. Refer to [HTTP listener with path-based routing](#http-listener-with-path-based-routing) for an example configuration. + +#### Values + +- Default: None +- Data type: Map containing one or more fields that define header modification rules: + + - `add`: Map of one or more key-value pairs + - `set`: Map of one or more key-value pairs + - `remove`: Map of one or more key-value pairs + +The following table describes how to configure values for request headers: + +| Rule | Description | Data type | +| --- | --- | --- | +| `add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | Map of strings | +| `set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | Map of strings | +| `remove` | Defines a list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | List of strings | + +##### Use variable placeholders + +For `add` and `set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated at runtime. + +### `spec.listeners[].services[].tls` + +Specifies a TLS configuration for a specific service. The settings in this configuration overrides the main [`tls`](#spec.tls) settings for the configuration entry. + +#### Values + +- Default: None +- Data type: Map + +### `spec.listeners[].services[].tls.sds` + +Specifies parameters that configure the listener to load TLS certificates from an external SDS. Refer to [Serve custom TLS certificates from an external service](/consul/docs/connect/gateways/ingress-gateway/tls-external-service) for additional information. + +If unspecified, Consul applies the [`sds`](#spec-tls-sds) settings configured for the ingress gateway. If both are specified, this configuration overrides the settings for configuration entry. + +#### Values + +- Default: None +- Data type: Map containing the following fields: -```json -{ - "Kind": "ingress-gateway", - "Name": "us-east-ingress", - "TLS": { - "Enabled": true - }, - "Listeners": [ - { - "Port": 8080, - "Protocol": "http", - "Services": [ - { - "Name": "*" - } - ] - }, - { - "Port": 4567, - "Protocol": "http", - "Services": [ - { - "Name": "api", - "Hosts": ["foo.example.com"] - }, - { - "Name": "web", - "Hosts": ["website.example.com"] - } - ] - } - ] -} -``` + - `clusterName` + - `certResource` - +The following table describes how to configure SDS parameters. Refer to [Serve custom TLS certificates from an external service](/consul/docs/connect/gateways/ingress-gateway/ingress-gateways-tls-external-service) for usage information: - - - +| Parameter | Description | Data type | +| --- | --- | --- | +| `clusterName` | Specifies the name of the SDS cluster where Consul should retrieve certificates. The cluster must be specified in the gateway's bootstrap configuration. | String | +| `certResource` | Specifies an SDS resource name. Consul requests the SDS resource name when fetching the certificate from the SDS service. When set, Consul serves the certificate to all listeners over TLS unless a listener-specific TLS configuration overrides the SDS configuration. | String | -```hcl -Kind = "ingress-gateway" -Name = "us-east-ingress" -Namespace = "default" +### `spec-listeners[].services[].maxConnections` -TLS { - Enabled = true -} +Specifies the maximum number of HTTP/1.1 connections a service instance is allowed to establish against the upstream. -Listeners = [ - { - Port = 8080 - Protocol = "http" - Services = [ - { - Namespace = "frontend" - Name = "*" - } - ] - }, - { - Port = 4567 - Protocol = "http" - Services = [ - { - Namespace = "frontend" - Name = "api" - Hosts = ["foo.example.com"] - Partition = "api-team" - }, - { - Namespace = "frontend" - Name = "web" - Hosts = ["website.example.com"] - Partition = "web-team" - } - ] - } -] -``` +A value specified in this field overrides the [`maxConnections`](#spec-defaults-maxconnections) field specified in the `defaults` configuration. -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: IngressGateway -metadata: - name: us-east-ingress - namespace: default -spec: - tls: - enabled: true - listeners: - - port: 8080 - protocol: http - services: - - name: '*' - namespace: frontend - - port: 4567 - protocol: http - services: - - name: api - namespace: frontend - hosts: ['foo.example.com'] - partition: api-team - - name: web - namespace: frontend - hosts: ['website.example.com'] - partition: web-team -``` +#### Values -```json -{ - "Kind": "ingress-gateway", - "Name": "us-east-ingress", - "Namespace": "default", - "TLS": { - "Enabled": true - }, - "Listeners": [ - { - "Port": 8080, - "Protocol": "http", - "Services": [ - { - "Namespace": "frontend", - "Name": "*" - } - ] - }, - { - "Port": 4567, - "Protocol": "http", - "Services": [ - { - "Namespace": "frontend", - "Name": "api", - "Hosts": ["foo.example.com"], - "Partition": "api-team" - }, - { - "Namespace": "frontend", - "Name": "web", - "Hosts": ["website.example.com"], - "Partition": "web-team" - } - ] - } - ] -} -``` +- Default: None +- Data type: Integer - +### `spec.listeners[].services.maxPendingRequests` + +Specifies the maximum number of requests that are allowed to queue while waiting to establish a connection. A value specified in this field overrides the [`maxPendingRequests`](#spec-defaults-maxpendingrequests) field specified in the `defaults` configuration. + +Listeners must use an L7 protocol for this configuration to take effect. Refer to [`spec.listeners.protocol`](#spec-listeners-protocol) for more information. + +#### Values + +- Default: None +- Data type: Integer + +### `spec.listeners[].services[].maxConcurrentRequests` + +Specifies the maximum number of concurrent HTTP/2 traffic requests that the service is allowed at a single point in time. A value specified in this field overrides the [`maxConcurrentRequests`](#spec-defaults-maxconcurrentrequests) field specified in the `defaults` configuration entry. + +Listeners must use an L7 protocol for this configuration to take effect. Refer to [`spec.listeners.protocol`](#spec-listeners-protocol) for more information. + +#### Values + +- Default: None +- Data type: Integer + +### `spec.listeners[].services[].passiveHealthCheck` + +Defines a passive health check configuration for the service. Passive health checks remove hosts from the upstream cluster when the service is unreachable or returns errors. Health checks specified for services override the health checks defined in the [`spec.defaults.passiveHealthCheck`](#spec-defaults-passivehealthcheck) configuration. + +#### Values + +- Default: None +- Data type: Map + +The following table describes the configurations for passive health checks: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | + | `Interval` | Specifies the time between checks. | string | `0s` | + | `MaxFailures` | Specifies the number of consecutive failures allowed per check interval. If exceeded, Consul removes the host from the load balancer. | integer | `0` | + | `EnforcingConsecutive5xx` | Specifies a percentage that indicates how many times out of 100 that Consul ejects the host when it detects an outlier status. The outlier status is determined by consecutive errors in the 500-599 response range. | integer | `100` | + | `MaxEjectionPercent` | Specifies the maximum percentage of an upstream cluster that Consul ejects when the proxy reports an outlier. Consul ejects at least one host when an outlier is detected regardless of the value. | integer | `10` | + | `BaseEjectionTime` | Specifies the minimum amount of time that an ejected host must remain outside the cluster before rejoining. The real time is equal to the value of the `BaseEjectionTime` multiplied by the number of times the host has been ejected. | string | `30s` | + +### `spec.listeners[].tls` + +Specifies the TLS configuration for the listener. If unspecified, Consul applies any [service-specific TLS configurations](#spec-listeners-services-tls). If neither the listener- nor service-specific TLS configurations are specified, Consul applies the main [`tls`](#tls) settings for the configuration entry. + +#### Values + +- Default: None +- Data type: Map that can contain the following fields: + - [`enabled`](#spec-listeners-tls-enabled) + - [`tlsMinVersion`](#spec-listeners-tls-tlsminversion) + - [`tlsMaxVersion`](#spec-listeners-tls-tlsmaxversion) + - [`cipherSuites`](#spec-listeners-tls-ciphersuites) + - [`sds`](#spec-listeners-tls-sds) + +### `spec.listeners[].tls.enabled` + +Set to `true` to enable built-in TLS for the listener. If enabled, Consul adds each host defined in every service's `Hosts` field to the gateway's x509 certificate as a DNS subject alternative name (SAN). + +#### Values + + - Default: `false` + - Data type: boolean + +### `spec.listeners[].tls.tlsMinVersion` + +Specifies the minimum TLS version supported for the listener. + +#### Values + +- Default: Depends on the version of Envoy: + - Envoy v1.22.0 and later: `TLSv1_2` + - Older versions: `TLSv1_0` +- Data type: String with one of the following values: + - `TLS_AUTO` + - `TLSv1_0` + - `TLSv1_1` + - `TLSv1_2` + - `TLSv1_3` + +### `spec.listeners[].tls.tlsMaxVersion` + +Specifies the maximum TLS version supported for the listener. + +#### Values + +- Default: Depends on the version of Envoy: + - Envoy v1.22.0 and later: `TLSv1_2` + - Older versions: `TLSv1_0` +- Data type: String with one of the following values: + - `TLS_AUTO` + - `TLSv1_0` + - `TLSv1_1` + - `TLSv1_2` + - `TLSv1_3` + +### `spec.listeners[].tls.cipherSuites` + +Specifies a list of cipher suites that the listener supports when negotiating connections using TLS 1.2 or older. If unspecified, the Consul applies the default for the version of Envoy in use. Refer to the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#envoy-v3-api-field-extensions-transport-sockets-tls-v3-tls parameters-cipher-suites) for details. + +#### Values + +- Default: None +- Data type: List of string values. Refer to the [Consul repository](https://github.com/hashicorp/consul/blob/v1.11.2/types/tls.go#L154-L169) for a list of supported ciphers. + +### `spec.listeners[].tls.sds` + +Specifies parameters for loading the TLS certificates from an external SDS service. Refer to [Serve custom TLS certificates from an external service](/consul/docs/connect/gateways/ingress-gateway/ingress-gateways-tls-external-service) for additional information. + +Consul applies the SDS configuration specified in this field to all services in the listener. You can override the `spec.listeners[].tls.sds` configuration per service by configuring the [`spec.listeners.services.tls.sds`](#spec-listeners-services-tls-sds) settings for each service. + +#### Values + +- Default: None +- Data type: Map containing the following fields + - `clusterName` + - `certResource` + +The following table describes how to configure SDS parameters. Refer to [Configure static SDS clusters](/consul/docs/connect/gateways/ingress-gateway/ingress-gateways-tls-external-service#configure-static-sds-clusters) for usage information: + +| Parameter | Description | Data type | +| --- | --- | --- | +| `clusterName` | Specifies the name of the SDS cluster where Consul should retrieve certificates. The cluster must be specified in the gateway's bootstrap configuration. | String | +| `certResource` | Specifies an SDS resource name. Consul requests the SDS resource name when fetching the certificate from the SDS service. When set, Consul serves the certificate to all listeners over TLS unless a listener-specific TLS configuration overrides the SDS configuration. | String | + -#### HTTP listener with Path-based Routing +## Examples -The following example sets up an HTTP listener on an ingress gateway named `us-east-ingress` to proxy -traffic to a virtual service named `api`. In the Consul Enterprise version, `us-east-ingress` is set up in the `default` namespace and `default` partition. +Refer to the following examples for common ingress gateway configuration patterns: +- [Define a TCP listener](#define-a-tcp-listener) +- [Use wildcards to define listeners](#use-wildcards-to-define-an-http-listener) +- [HTTP listener with path-based routes](#http-listener-with-path-based-routes) -In this use-case, internal-only debug headers should be stripped before responding to external clients. -Requests to internal services should also be labelled to indicate which gateway they came through. +### Define a TCP listener + +The following example sets up a TCP listener on an ingress gateway named `us-east-ingress` that proxies traffic to the `db` service. For Consul Enterprise, the `db` service can only listen for traffic in the `default` namespace inside the `team-frontend` admin partition: + +#### Consul OSS - - ```hcl @@ -569,19 +1528,11 @@ Name = "us-east-ingress" Listeners = [ { - Port = 80 - Protocol = "http" + Port = 3456 + Protocol = "tcp" Services = [ { - Name = "api" - RequestHeaders { - Add { - "x-gateway" = "us-east-ingress" - } - } - ResponseHeaders { - Remove = ["x-debug"] - } + Name = "db" } ] } @@ -595,16 +1546,10 @@ metadata: name: us-east-ingress spec: listeners: - - port: 80 - protocol: http + - port: 3456 + protocol: tcp services: - - name: api - requestHeaders: - add: - x-gateway: us-east-ingress - responseHeaders: - remove: - - x-debug + - name: db ``` ```json @@ -613,19 +1558,11 @@ spec: "Name": "us-east-ingress", "Listeners": [ { - "Port": 80, - "Protocol": "http", + "Port": 3456, + "Protocol": "tcp", "Services": [ { - "Name": "api", - "RequestHeaders": { - "Add": { - "x-gateway": "us-east-ingress" - } - }, - "ResponseHeaders": { - "Remove": ["x-debug"] - } + "Name": "db" } ] } @@ -635,32 +1572,24 @@ spec: - - +#### Consul Enterprise + ```hcl Kind = "ingress-gateway" Name = "us-east-ingress" Namespace = "default" -Partition = "default" +Partition = "team-frontend" Listeners = [ { - Port = 80 - Protocol = "http" + Port = 3456 + Protocol = "tcp" Services = [ { - Name = "api" - Namespace = "frontend" - RequestHeaders { - Add { - "x-gateway" = "us-east-ingress" - } - } - ResponseHeaders { - Remove = ["x-debug"] - } + Namespace = "ops" + Name = "db" } ] } @@ -675,17 +1604,11 @@ metadata: namespace: default spec: listeners: - - port: 80 - protocol: http + - port: 3456 + protocol: tcp services: - - name: api - namespace: frontend - requestHeaders: - add: - x-gateway: us-east-ingress - responseHeaders: - remove: - - x-debug + - name: db + namespace: ops ``` ```json @@ -693,23 +1616,15 @@ spec: "Kind": "ingress-gateway", "Name": "us-east-ingress", "Namespace": "default", - "Partition": "default", + "Partition": "team-frontend", "Listeners": [ { - "Port": 80, - "Protocol": "http", + "Port": 3456, + "Protocol": "tcp", "Services": [ { - "Name": "api", - "Namespace": "frontend", - "RequestHeaders": { - "Add": { - "x-gateway": "us-east-ingress" - } - }, - "ResponseHeaders": { - "Remove": ["x-debug"] - } + "Namespace": "ops", + "Name": "db" } ] } @@ -719,88 +1634,112 @@ spec: - - +### Use wildcards to define an HTTP listener -For this use-case, the `api` service is not an actual registered service. It exists as a virtual service for L7 configuration only. A `service-router` (`ServiceRouter` on Kubernetes) is defined for the virtual service that uses path-based routing to route requests to different backend services: +The following example gateway is named `us-east-ingress` and defines two listeners. The first listener is configured to listen on port `8080` and uses a wildcard (`*`) to proxy traffic to all services in the datacenter. The second listener exposes the `api` and `web` services on port `4567` at user-provided hosts. - - +TLS is enabled on every listener. The `max_connections` of the ingress gateway proxy to each upstream cluster is set to `4096`. + +The Consul Enterprise version implements the following additional configurations: + +- The ingress gateway is set up in the `default` [namespace](/consul/docs/enterprise/namespaces) and proxies traffic to all services in the `frontend` namespace. +- The `api` and `web` services are proxied to team-specific [admin partitions](/consul/docs/enterprise/admin-partitions): + +#### Consul OSS ```hcl -Kind = "service-router" -Name = "api" -Routes = [ +Kind = "ingress-gateway" +Name = "us-east-ingress" + +TLS { + Enabled = true +} + +Defaults { + MaxConnections = 4096 +} + +Listeners = [ { - Match { - HTTP { - PathPrefix = "/billing" + Port = 8080 + Protocol = "http" + Services = [ + { + Name = "*" } - } - - Destination { - Service = "billing-api" - } + ] }, { - Match { - HTTP { - PathPrefix = "/payments" + Port = 4567 + Protocol = "http" + Services = [ + { + Name = "api" + Hosts = ["foo.example.com"] + }, + { + Name = "web" + Hosts = ["website.example.com"] } - } - - Destination { - Service = "payments-api" - } + ] } ] ``` ```yaml apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceRouter +kind: IngressGateway metadata: - name: api + name: us-east-ingress spec: - routes: - - match: - http: - pathPrefix: '/billing' - destination: - service: billing-api - - match: - http: - pathPrefix: '/payments' - destination: - service: payments-api + tls: + enabled: true + listeners: + - port: 8080 + protocol: http + services: + - name: '*' + - port: 4567 + protocol: http + services: + - name: api + hosts: ['foo.example.com'] + - name: web + hosts: ['website.example.com'] ``` ```json { - "Kind": "service-router", - "Name": "api", - "Routes": [ + "Kind": "ingress-gateway", + "Name": "us-east-ingress", + "TLS": { + "Enabled": true + }, + "Listeners": [ { - "Match": { - "HTTP": { - "PathPrefix": "/billing" + "Port": 8080, + "Protocol": "http", + "Services": [ + { + "Name": "*" } - }, - "Destination": { - "Service": "billing-api" - } + ] }, { - "Match": { - "HTTP": { - "PathPrefix": "/payments" + "Port": 4567, + "Protocol": "http", + "Services": [ + { + "Name": "api", + "Hosts": ["foo.example.com"] + }, + { + "Name": "web", + "Hosts": ["website.example.com"] } - }, - "Destination": { - "Service": "payments-api" - } + ] } ] } @@ -808,512 +1747,118 @@ spec: - - +#### Consul Enterprise ```hcl -Kind = "service-router" -Name = "api" +Kind = "ingress-gateway" +Name = "us-east-ingress" Namespace = "default" -Partition = "default" -Routes = [ + +TLS { + Enabled = true +} + +Listeners = [ { - Match { - HTTP { - PathPrefix = "/billing" + Port = 8080 + Protocol = "http" + Services = [ + { + Namespace = "frontend" + Name = "*" } - } - - Destination { - Service = "billing-api" - Namespace = "frontend" - } + ] }, { - Match { - HTTP { - PathPrefix = "/payments" + Port = 4567 + Protocol = "http" + Services = [ + { + Namespace = "frontend" + Name = "api" + Hosts = ["foo.example.com"] + Partition = "api-team" + }, + { + Namespace = "frontend" + Name = "web" + Hosts = ["website.example.com"] + Partition = "web-team" } - } - - Destination { - Service = "payments-api" - Namespace = "frontend" - } + ] } ] ``` ```yaml apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceRouter +kind: IngressGateway metadata: - name: api + name: us-east-ingress namespace: default spec: - routes: - - match: - http: - pathPrefix: '/billing' - destination: - service: billing-api - namespace: frontend - - match: - http: - pathPrefix: '/payments' - destination: - service: payments-api - namespace: frontend + tls: + enabled: true + listeners: + - port: 8080 + protocol: http + services: + - name: '*' + namespace: frontend + - port: 4567 + protocol: http + services: + - name: api + namespace: frontend + hosts: ['foo.example.com'] + partition: api-team + - name: web + namespace: frontend + hosts: ['website.example.com'] + partition: web-team ``` ```json { - "Kind": "service-router", - "Name": "api", + "Kind": "ingress-gateway", + "Name": "us-east-ingress", "Namespace": "default", - "Partition": "default", - "Routes": [ + "TLS": { + "Enabled": true + }, + "Listeners": [ { - "Match": { - "HTTP": { - "PathPrefix": "/billing" + "Port": 8080, + "Protocol": "http", + "Services": [ + { + "Namespace": "frontend", + "Name": "*" } - }, - "Destination": { - "Service": "billing-api", - "Namespace": "frontend" - } + ] }, { - "Match": { - "HTTP": { - "PathPrefix": "/payments" + "Port": 4567, + "Protocol": "http", + "Services": [ + { + "Namespace": "frontend", + "Name": "api", + "Hosts": ["foo.example.com"], + "Partition": "api-team" + }, + { + "Namespace": "frontend", + "Name": "web", + "Hosts": ["website.example.com"], + "Partition": "web-team" } - }, - "Destination": { - "Service": "payments-api", - "Namespace": "frontend" - } + ] } ] } ``` - - - - -## Available Fields - -You can specify the following parameters to configure ingress gateway configuration entries. - -', - yaml: false, - }, - { - name: 'Namespace', - type: 'string: `default`', - enterprise: true, - description: - 'Specifies the namespace in which the configuration entry will apply. The value must match the namespace in which the gateway is registered.' + - ' If omitted, the namespace will be inherited from the `ns` request parameter (refer to the [`config` API endpoint documentation](/consul/api-docs/config#ns)).' + - ' or will default to the `default` namespace.', - yaml: false, - }, - { - name: 'Meta', - type: 'map: nil', - description: - 'Specifies arbitrary KV metadata pairs. Added in Consul 1.8.4.', - yaml: false, - }, - { - name: 'Partition', - type: `string: "default"`, - enterprise: true, - description: - 'Specifies the admin partition in which the configuration will apply. The value must match the partition in which the gateway is registered.' + - ' If omitted, the partition will be inherited from the request (refer to the [`config` API endpoint documentation](/consul/api-docs/config)).' + - ' See [Admin Partitions](/consul/docs/enterprise/admin-partitions) for additional information.', - yaml: false, - }, - { - name: 'metadata', - children: [ - { - name: 'name', - description: 'Set to the name of the service being configured.', - }, - { - name: 'namespace', - enterprise: true, - description: - 'Refer to the [Kubernetes Namespaces documentation for Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise). The `namespace` parameter is not supported in Consul OSS (see [Kubernetes Namespaces in Consul OSS](/consul/docs/k8s/crds#consul-oss)).', - }, - ], - hcl: false, - }, - { - name: 'TLS', - type: 'TLSConfig: ', - description: 'TLS configuration for this gateway.', - children: [ - { - name: 'Enabled', - type: 'bool: false', - description: { - hcl: - "Set this configuration to `true` to enable built-in TLS for every listener on the gateway.

If TLS is enabled, then each host defined in each service's `Hosts` fields will be added as a DNSSAN to the gateway's x509 certificate.", - yaml: - "Set this configuration to `true` to enable built-in TLS for every listener on the gateway.

If TLS is enabled, then each host defined in each service's `hosts` fields will be added as a DNSSAN to the gateway's x509 certificate.", - }, - }, - { - name: 'TLSMinVersion', - type: 'string: ""', - description: - "Set the default minimum TLS version supported for the gateway's listeners. One of `TLS_AUTO`, `TLSv1_0`, `TLSv1_1`, `TLSv1_2`, or `TLSv1_3`. If unspecified, Envoy v1.22.0 and newer [will default to TLS 1.2 as a min version](https://github.com/envoyproxy/envoy/pull/19330), while older releases of Envoy default to TLS 1.0.", - }, - { - name: 'TLSMaxVersion', - type: 'string: ""', - description: { - hcl: - "Set the default maximum TLS version supported for the gateway's listeners. Must be greater than or equal to `TLSMinVersion`. One of `TLS_AUTO`, `TLSv1_0`, `TLSv1_1`, `TLSv1_2`, or `TLSv1_3`.", - yaml: - "Set the default maximum TLS version supported for the gateway's listeners. Must be greater than or equal to `tls_min_version`. One of `TLS_AUTO`, `TLSv1_0`, `TLSv1_1`, `TLSv1_2`, or `TLSv1_3`.", - }, - }, - { - name: 'CipherSuites', - type: 'array: ', - description: `Set the default list of TLS cipher suites for the gateway's - listeners to support when negotiating connections using - TLS 1.2 or earlier. If unspecified, Envoy will use a - [default server cipher list](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#envoy-v3-api-field-extensions-transport-sockets-tls-v3-tlsparameters-cipher-suites). - The list of supported cipher suites can seen in - [\`consul/types/tls.go\`](https://github.com/hashicorp/consul/blob/v1.11.2/types/tls.go#L154-L169) - and is dependent on underlying support in Envoy. Future - releases of Envoy may remove currently-supported but - insecure cipher suites, and future releases of Consul - may add new supported cipher suites if any are added to - Envoy.`, - }, - { - name: 'SDS', - type: 'SDSConfig: ', - description: - 'Defines a set of parameters that configures the gateway to load TLS certificates from an external SDS service. See [SDS](/consul/docs/connect/gateways/ingress-gateway#sds) for more details on usage.

SDS properties defined in this field are used as defaults for all listeners on the gateway.', - children: [ - { - name: 'ClusterName', - type: 'string', - description: - "Specifies the name of the SDS cluster from which Consul should retrieve certificates. This cluster must be [specified in the Gateway's bootstrap configuration](/consul/docs/connect/gateways/ingress-gateway#sds).", - }, - { - name: 'CertResource', - type: 'string', - description: - 'Specifies an SDS resource name. Consul will request the SDS resource name when fetching the certificate from the SDS service. Setting this causes all listeners to be served exclusively over TLS with this certificate unless overridden by listener-specific TLS configuration.', - }, - ], - }, - ], - }, - { - name: 'Defaults', - type: 'IngressServiceConfig: ', - description: `Default configuration that applies to all upstreams.`, - children: [ - { - name: 'MaxConnections', - type: 'int: 0', - description: `The maximum number of connections a service instance - will be allowed to establish against the given upstream. Use this to limit - HTTP/1.1 traffic, since HTTP/1.1 has a request per connection. - If not specified, it uses the default value. For example, 1024 for Envoy proxy.`, - }, - { - name: 'MaxPendingRequests', - type: 'int: 0', - description: `The maximum number of requests that will be queued - while waiting for a connection to be established. For this configuration to - be respected, a L7 protocol must be defined in the \`protocol\` field. - If not specified, it uses the default value. For example, 1024 for Envoy proxy.`, - }, - { - name: 'MaxConcurrentRequests', - type: 'int: 0', - description: `The maximum number of concurrent requests that - will be allowed at a single point in time. Use this to limit HTTP/2 traffic, - since HTTP/2 has many requests per connection. For this configuration to be - respected, a L7 protocol must be defined in the \`protocol\` field. - If not specified, it uses the default value. For example, 1024 for Envoy proxy.`, - }, - { - name: 'PassiveHealthCheck', - type: 'PassiveHealthCheck: ', - description: - 'Passive health checks remove hosts from the upstream cluster that are unreachable or that return errors.', - children: [ - { - name: 'interval', - type: 'int: ', - description: - "The time in nanosecond between checks. Each check will cause hosts which have exceeded `max_failures` to be removed from the load balancer, and any hosts which have passed their ejection time to be returned to the load balancer. If not specified, it uses the default value. For example, 10s for Envoy proxy.", - }, - { - name: 'max_failures', - type: 'int: ', - description: - 'The number of consecutive failures that cause a host to be removed from the upstream cluster. If not specified, Consul uses the proxy\'s default value. For example, `5` for Envoy proxy.', - }, - { - name: 'enforcing_consecutive_5xx', - type: 'int: ', - description: - 'A percentage representing the chance that a host will be actually ejected when the proxy detects an outlier status through consecutive errors in the 500 code range. If not specified, Consul uses the proxy\'s default value. For example, `100` for Envoy proxy.', - }, - ], - }, - ], - }, - { - name: 'Listeners', - type: 'array: )', - description: - 'A list of listeners that the ingress gateway should setup, uniquely identified by their port number.', - children: [ - { - name: 'Port', - type: 'int: 0', - description: `The port on which the ingress listener should receive - traffic. The port will be bound to the IP address that - was specified in the [\`-address\`](/consul/commands/connect/envoy#address) - flag when starting the gateway. - Note: The ingress listener port must not conflict - with the health check port specified in the \`-address\` - flag.`, - }, - { - name: 'Protocol', - type: 'string: "tcp"', - description: - 'The protocol associated with the listener. One of `tcp`, `http`, `http2`, or `grpc`.', - }, - { - name: 'Services', - type: 'array: ', - description: `A list of services to be exposed via this listener. - For \`tcp\` listeners, only a single service is allowed. - Each service must have a unique name. A namespace is also required for - Consul Enterprise.`, - children: [ - { - name: 'Name', - type: 'string: ""', - description: `The name of the service that should be exposed - through this listener. This can be either a service registered in the - catalog, or a service defined only by [other config entries](/consul/docs/connect/l7-traffic). If the wildcard specifier, - \`*\`, is provided, then ALL services will be exposed through the listener. - This is not supported for listeners with protocol \`tcp\`.`, - }, - { - name: 'Namespace', - type: 'string: ""', - enterprise: true, - description: - 'The namespace from which to resolve the service if different than the existing namespace. The current namespace is used if unspecified.', - }, - { - name: 'Partition', - type: 'string: ""', - enterprise: true, - description: - 'The admin partition from which to resolve the service if different than the existing partition. The current partition is used if unspecified.', - }, - { - name: 'Hosts', - type: 'array: ', - description: `A list of hosts that specify what - requests will match this service. This cannot be used with a \`tcp\` - listener, and cannot be specified alongside a \`*\` service name. If not - specified, the default domain \`\.ingress.*\` will be used to - match services. Requests must send the correct host to be routed to - the defined service.

- The wildcard specifier, \`*\`, can be used by itself to match all traffic - coming to the ingress gateway, if TLS is not enabled. This allows a user - to route all traffic to a single service without specifying a host, - allowing simpler tests and demos. Otherwise, the wildcard specifier can - be used as part of the host to match multiple hosts, but only in the - leftmost DNS label. This ensures that all defined hosts are valid DNS - records. For example, \`*.example.com\` is valid, while \`example.*\` and - \`*-suffix.example.com\` are not.`, - }, - { - name: 'RequestHeaders', - type: 'HTTPHeaderModifiers: ', - description: `A set of [HTTP-specific header modification rules](/consul/docs/connect/config-entries/service-router#httpheadermodifiers) - that will be applied to requests routed to this service. - This cannot be used with a \`tcp\` listener.`, - }, - { - name: 'ResponseHeaders', - type: 'HTTPHeaderModifiers: ', - description: `A set of [HTTP-specific header modification rules](/consul/docs/connect/config-entries/service-router#httpheadermodifiers) - that will be applied to responses from this service. - This cannot be used with a \`tcp\` listener.`, - }, - { - name: 'TLS', - type: 'ServiceTLSConfig: ', - description: 'TLS configuration for this service.', - children: [ - { - name: 'SDS', - type: 'SDSConfig: ', - description: `Defines a set of parameters that configures the SDS source for the certificate for this specific service. - At least one custom host must be specified in \`Hosts\`. - The certificate retrieved from SDS will be served for all requests identifying one of the - \`Hosts\` values in the TLS Server Name Indication (SNI) header.`, - children: [ - { - name: 'ClusterName', - type: 'string', - description: - "The SDS cluster name to connect to to retrieve certificates. This cluster must be [specified in the Gateway's bootstrap configuration](/consul/docs/connect/gateways/ingress-gateway#sds).", - }, - { - name: 'CertResource', - type: 'string', - description: - 'The SDS resource name to request when fetching the certificate from the SDS service.', - }, - ], - }, - ], - }, - { - name: 'MaxConnections', - type: 'int: 0', - description: 'overrides for the [`Defaults` field](#available-fields)', - }, - { - name: 'MaxPendingRequests', - type: 'int: 0', - description: 'overrides for the [`Defaults` field](#available-fields)', - }, - { - name: 'MaxConcurrentRequests', - type: 'int: 0', - description: 'overrides for the [`Defaults` field](#available-fields)', - }, - { - name: 'PassiveHealthCheck', - type: 'PassiveHealthCheck: ', - description: 'overrides for the [`Defaults` field](#available-fields)', - children: [ - { - name: 'interval', - type: 'int: ', - }, - { - name: 'max_failures', - type: 'int: ', - }, - { - name: 'enforcing_consecutive_5xx', - type: 'int: ', - }, - ], - }, - ], - }, - { - name: 'TLS', - type: 'TLSConfig: ', - description: 'TLS configuration for this listener.', - children: [ - { - name: 'Enabled', - type: 'bool: false', - description: { - hcl: - "Set this configuration to `true` to enable built-in TLS for this listener.

If TLS is enabled, then each host defined in each service's `Hosts` field will be added as a DNSSAN to the gateway's x509 certificate. Note that even hosts from other listeners with TLS disabled will be added. TLS can not be disabled for individual listeners if it is enabled on the gateway.", - yaml: - "Set this configuration to `true` to enable built-in TLS for this listener.

If TLS is enabled, then each host defined in each service's `hosts` field will be added as a DNSSAN to the gateway's x509 certificate. Note that even hosts from other listeners with TLS disabled will be added. TLS can not be disabled for individual listeners if it is enabled on the gateway.", - }, - }, - { - name: 'TLSMinVersion', - type: 'string: ""', - description: - 'Set the minimum TLS version supported for this listener. One of `TLS_AUTO`, `TLSv1_0`, `TLSv1_1`, `TLSv1_2`, or `TLSv1_3`. If unspecified, Envoy v1.22.0 and newer [will default to TLS 1.2 as a min version](https://github.com/envoyproxy/envoy/pull/19330), while older releases of Envoy default to TLS 1.0.', - }, - { - name: 'TLSMaxVersion', - type: 'string: ""', - description: - 'Set the maximum TLS version supported for this listener. Must be greater than or equal to `TLSMinVersion`. One of `TLS_AUTO`, `TLSv1_0`, `TLSv1_1`, `TLSv1_2`, or `TLSv1_3`.', - }, - { - name: 'CipherSuites', - type: 'array: ', - description: `Set the list of TLS cipher suites to support when negotiating - connections using TLS 1.2 or earlier. If unspecified, - Envoy will use a - [default server cipher list](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#envoy-v3-api-field-extensions-transport-sockets-tls-v3-tlsparameters-cipher-suites). - The list of supported cipher suites can seen in - [\`consul/types/tls.go\`](https://github.com/hashicorp/consul/blob/v1.11.2/types/tls.go#L154-L169) - and is dependent on underlying support in Envoy. Future - releases of Envoy may remove currently-supported but - insecure cipher suites, and future releases of Consul - may add new supported cipher suites if any are added to Envoy.`, - }, - { - name: 'SDS', - type: 'SDSConfig: ', - description: - 'Defines a set of parameters that configures the listener to load TLS certificates from an external SDS service. See [SDS](/consul/docs/connect/gateways/ingress-gateway#sds) for more details on usage.

SDS properties set here will be used as defaults for all services on this listener.', - children: [ - { - name: 'ClusterName', - type: 'string', - description: - "The SDS cluster name to connect to to retrieve certificates. This cluster must be [specified in the Gateway's bootstrap configuration](/consul/docs/connect/gateways/ingress-gateway#sds).", - }, - { - name: 'CertResource', - type: 'string', - description: - 'The SDS resource name to request when fetching the certificate from the SDS service.', - }, - ], - }, - ], - }, - ], - }, - ]} -/> diff --git a/website/content/docs/connect/config-entries/mesh.mdx b/website/content/docs/connect/config-entries/mesh.mdx index 7900a8104a9..f83c1298052 100644 --- a/website/content/docs/connect/config-entries/mesh.mdx +++ b/website/content/docs/connect/config-entries/mesh.mdx @@ -7,8 +7,6 @@ description: >- # Mesh Configuration Entry --> **v1.10.0+:** This configuration entry is supported in Consul versions 1.10.0+. - The `mesh` configuration entry allows you to define a global default configuration that applies to all service mesh proxies. Settings in this config entry apply across all namespaces and federated datacenters. @@ -333,7 +331,7 @@ Note that the Kubernetes example does not include a `partition` field. Configura type: 'bool: false', description: `Determines whether sidecar proxies operating in transparent mode can proxy traffic to IP addresses not registered in Consul's mesh. If enabled, traffic will only be proxied - to upstream proxies or Connect-native services. If disabled, requests will be proxied as-is to the + to upstream proxies or mesh-native services. If disabled, requests will be proxied as-is to the original destination IP address. Consul will not encrypt the connection.`, }, ], diff --git a/website/content/docs/connect/config-entries/proxy-defaults.mdx b/website/content/docs/connect/config-entries/proxy-defaults.mdx index 5ac09c102f0..043d9926372 100644 --- a/website/content/docs/connect/config-entries/proxy-defaults.mdx +++ b/website/content/docs/connect/config-entries/proxy-defaults.mdx @@ -342,8 +342,8 @@ spec: { name: 'Config', type: 'map[string]arbitrary', - description: `An arbitrary map of configuration values used by Connect proxies. - The available configurations depend on the Connect proxy you use. + description: `An arbitrary map of configuration values used by service mesh proxies. + The available configurations depend on the mesh proxy you use. Any values that your proxy allows can be configured globally here. To explore these options please see the documentation for your chosen proxy.
  • [Envoy](/consul/docs/connect/proxies/envoy#proxy-config-options)
  • [Consul's built-in proxy](/consul/docs/connect/proxies/built-in#proxy-config-key-reference)
`, @@ -351,7 +351,8 @@ spec: { name: 'EnvoyExtensions', type: 'array: []', - description: `A list of extensions to modify Envoy proxy configuration.`, + description: `A list of extensions to modify Envoy proxy configuration.

+ Applying \`EnvoyExtensions\` to \`ProxyDefaults\` may produce unintended consequences. We recommend enabling \`EnvoyExtensions\` with [\`ServiceDefaults\`](/consul/docs/connect/config-entries/service-defaults#envoyextensions) in most cases.`, children: [ { name: 'Name', @@ -408,7 +409,7 @@ spec: name: 'MeshGateway', type: 'MeshGatewayConfig: ', description: `Controls the default - [mesh gateway configuration](/consul/docs/connect/gateways/mesh-gateway#connect-proxy-configuration) + [mesh gateway configuration](/consul/docs/connect/gateways/mesh-gateway#service-mesh-proxy-configuration) for all proxies. Added in v1.6.0.`, children: [ { @@ -425,7 +426,7 @@ spec: [expose path configuration](/consul/docs/connect/registration/service-registration#expose-paths-configuration-reference) for Envoy. Added in v1.6.2.

Exposing paths through Envoy enables a service to protect itself by only listening on localhost, while still allowing - non-Connect-enabled applications to contact an HTTP endpoint. + non-mesh-enabled applications to contact an HTTP endpoint. Some examples include: exposing a \`/metrics\` path for Prometheus or \`/healthz\` for kubelet liveness checks.`, children: [ { diff --git a/website/content/docs/connect/config-entries/service-defaults.mdx b/website/content/docs/connect/config-entries/service-defaults.mdx index 9245d8112c9..2e048e5b272 100644 --- a/website/content/docs/connect/config-entries/service-defaults.mdx +++ b/website/content/docs/connect/config-entries/service-defaults.mdx @@ -1,25 +1,1220 @@ --- layout: docs -page_title: Service Defaults - Configuration Entry Reference -description: >- - The service defaults configuration entry kind defines sets of default configurations that apply to all services in the mesh. Use the examples learn how to define a default protocol, default upstream configuration, and default terminating gateway. +page_title: Service Defaults Configuration Reference +description: -> + Use the service-defaults configuration entry to set default configurations for services, such as upstreams, protocols, and namespaces. Learn how to configure service-defaults. --- -# Service Defaults Configuration Entry -The `service-defaults` config entry kind (`ServiceDefaults` on Kubernetes) controls default global values for a -service, such as its protocol. +# Service Defaults Configuration Reference +This topic describes how to configure service defaults configuration entries. The service defaults configuration entry contains common configuration settings for service mesh services, such as upstreams and gateways. Refer to [Define service defaults](/consul/docs/services/usage/define-services#define-service-defaults) for usage information. -## Sample Config Entries +## Configuration model -### Default protocol +The following outline shows how to format the service defaults configuration entry. Click on a property name to view details about the configuration. --> **NOTE**: The default protocol can also be configured globally for all proxies -using the [proxy defaults](/consul/docs/connect/config-entries/proxy-defaults#default-protocol) -config entry. However, if the protocol value is specified in a service defaults -config entry for a given service, that value will take precedence over the -globally configured value from proxy defaults. + + + +- [`Kind`](#kind): string | required +- [`Name`](#name): string | required +- [`Namespace`](#namespace): string +- [`Partition`](#partition): string +- [`Meta`](#meta): map | no default +- [`Protocol`](#protocol): string | default: `tcp` +- [`BalanceInboundConnections`](#balanceinboundconnections): string | no default +- [`Mode`](#mode): string | no default +- [`UpstreamConfig`](#upstreamconfig): map | no default + - [`Overrides`](#upstreamconfig-overrides): map | no default + - [`Name`](#upstreamconfig-overrides-name): string | no default + - [`Namespace`](#upstreamconfig-overrides-namespace): string | no default + - [`Peer`](#upstreamconfig-overrides-peer): string | no default + - [`Protocol`](#upstreamconfig-overrides-protocol): string | no default + - [`ConnectTimeoutMs`](#upstreamconfig-overrides-connecttimeoutms): int | default: `5000` + - [`MeshGateway`](#upstreamconfig-overrides-meshgateway): map | no default + - [`mode`](#upstreamconfig-overrides-meshgateway): string | no default + - [`BalanceOutboundConnections`](#upstreamconfig-overrides-balanceoutboundconnections): string | no default + - [`Limits`](#upstreamconfig-overrides-limits): map | optional + - [`MaxConnections`](#upstreamconfig-overrides-limits): integer | `0` + - [`MaxPendingRequests`](#upstreamconfig-overrides-limits): integer | `0` + - [`MaxConcurrentRequests`](#upstreamconfig-overrides-limits): integer | `0` + - [`PassiveHealthCheck`](#upstreamconfig-overrides-passivehealthcheck): map | optional + - [`Interval`](#upstreamconfig-overrides-passivehealthcheck): string | `0s` + - [`MaxFailures`](#upstreamconfig-overrides-passivehealthcheck): integer | `0` + - [`EnforcingConsecutive5xx`](#upstreamconfig-overrides-passivehealthcheck): integer | `0` + - [`MaxEjectionPercent`](#upstreamconfig-overrides-passivehealthcheck): integer | `0` + - [`BaseEjectionTime`](#upstreamconfig-overrides-passivehealthcheck): string | `30s` + - [`Defaults`](#upstreamconfig-defaults): map | no default + - [`Protocol`](#upstreamconfig-defaults-protocol): string | no default + - [`ConnectTimeoutMs`](#upstreamconfig-defaults-connecttimeoutms): int | default: `5000` + - [`MeshGateway`](#upstreamconfig-defaults-meshgateway): map | no default + - [`mode`](#upstreamconfig-defaults-meshgateway): string | no default + - [`BalanceOutboundConnections`](#upstreamconfig-defaults-balanceoutboundconnections): string | no default + - [`Limits`](#upstreamconfig-defaults-limits): map | optional + - [`MaxConnections`](#upstreamconfig-defaults-limits): integer | `0` + - [`MaxPendingRequests`](#upstreamconfig-defaults-limits): integer | `0` + - [`MaxConcurrentRequests`](#upstreamconfig-defaults-limits): integer | `0` + - [`PassiveHealthCheck`](#upstreamconfig-defaults-passivehealthcheck): map | optional + - [`Interval`](#upstreamconfig-defaults-passivehealthcheck): string | `0s` + - [`MaxFailures`](#upstreamconfig-defaults-passivehealthcheck): integer | `0` + - [`EnforcingConsecutive5xx`](#upstreamconfig-defaults-passivehealthcheck): integer | `100` + - [`MaxEjectionPercent`](#upstreamconfig-defaults-passivehealthcheck): integer | `0` + - [`BaseEjectionTime`](#upstreamconfig-defaults-passivehealthcheck): string | `30s` +- [`TransparentProxy`](#transparentproxy): map | no default + - [`OutboundListenerPort`](#transparentproxy): integer | `15001` + - [`DialedDirectly`](#transparentproxy ): boolean | `false` +- [`EnvoyExtensions`](#envoyextensions): list | no default + - [`Name`](#envoyextensions): string | `""` + - [`Required`](#envoyextensions): string | `""` + - [`Arguments`](#envoyextensions): map | `nil` +- [`Destination`](#destination): map | no default + - [`Addresses`](#destination): list | no default + - [`Port`](#destination): integer | `0` +- [`MaxInboundConnections`](#maxinboundconnections): integer | `0` +- [`LocalConnectTimeoutMs`](#localconnecttimeoutms): integer | `0` +- [`LocalRequestTimeoutMs`](#localrequesttimeoutms): integer | `0` +- [`MeshGateway`](#meshgateway): map | no default + - [`Mode`](#meshgateway): string | no default +- [`ExternalSNI`](#externalsni): string | no default +- [`Expose`](#expose): map | no default + - [`Checks`](#expose-checks): boolean | `false` + - [`Paths`](#expose-paths): list | no default + - [`Path`](#expose-paths): string | no default + - [`LocalPathPort`](#expose-paths): integer | `0` + - [`ListenerPort`](#expose-paths): integer | `0` + - [`Protocol`](#expose-paths): string | `http` + + + + +- [`apiVersion`](#apiversion): string | must be set to `consul.hashicorp.com/v1alpha1` +- [`kind`](#kind): string | no default +- [`metadata`](#metadata): map | no default + - [`name`](#name): string | no default + - [`namespace`](#namespace): string | no default | +- [`spec`](#spec): map | no default + - [`protocol`](#protocol): string | default: `tcp` + - [`balanceInboundConnections`](#balanceinboundconnections): string | no default + - [`mode`](#mode): string | no default + - [`upstreamConfig`](#upstreamconfig): map | no default + - [`overrides`](#upstreamconfig-overrides): list | no default + - [`name`](#upstreamconfig-overrides-name): string | no default + - [`namespace`](#upstreamconfig-overrides-namespace): string | no default + - [`protocol`](#upstreamconfig-overrides-protocol): string | no default + - [`connectTimeoutMs`](#upstreamconfig-overrides-connecttimeoutms): int | default: `5000` + - [`meshGateway`](#upstreamconfig-overrides-meshgateway): map | no default + - [`mode`](#upstreamconfig-overrides-meshgateway): string | no default + - [`balanceOutboundConnections`](#overrides-balanceoutboundconnections): string | no default + - [`limits`](#upstreamconfig-overrides-limits): map | optional + - [`maxConnections`](#upstreamconfig-overrides-limits): integer | `0` + - [`maxPendingRequests`](#upstreamconfig-overrides-limits): integer | `0` + - [`maxConcurrentRequests`](#upstreamconfig-overrides-limits): integer | `0` + - [`passiveHealthCheck`](#upstreamconfig-overrides-passivehealthcheck): map | optional + - [`interval`](#upstreamconfig-overrides-passivehealthcheck): string | `0s` + - [`maxFailures`](#upstreamconfig-overrides-passivehealthcheck): integer | `0` + - [`enforcingConsecutive5xx`](#upstreamconfig-overrides-passivehealthcheck): integer | `100` + - [`maxEjectionPercent`](#upstreamconfig-overrides-passivehealthcheck): integer | `10` + - [`baseEjectionTime`](#upstreamconfig-overrides-passivehealthcheck): string | `30s` + - [`defaults`](#upstreamconfig-defaults): map | no default + - [`protocol`](#upstreamconfig-defaults-protocol): string | no default + - [`connectTimeoutMs`](#upstreamconfig-defaults-connecttimeoutms): int | default: `5000` + - [`meshGateway`](#upstreamconfig-defaults-meshgateway): map | no default + - [`mode`](#upstreamconfig-defaults-meshgateway): string | no default + - [`balanceOutboundConnections`](#upstreamconfig-defaults-balanceoutboundconnections): string | no default + - [`limits`](#upstreamconfig-defaults-limits): map | optional + - [`maxConnections`](#upstreamconfig-defaults-limits): integer | `0` + - [`maxPendingRequests`](#upstreamconfig-defaults-limits): integer | `0` + - [`maxConcurrentRequests`](#upstreamconfig-defaults-limits): integer | `0` + - [`passiveHealthCheck`](#upstreamconfig-defaults-passivehealthcheck): map | optional + - [`interval`](#upstreamconfig-defaults-passivehealthcheck): string | `0s` + - [`maxFailures`](#upstreamconfig-defaults-passivehealthcheck): integer | `0` + - [`enforcingConsecutive5xx`](#upstreamconfig-defaults-passivehealthcheck): integer | `100` + - [`maxEjectionPercent`](#upstreamconfig-defaults-passivehealthcheck): integer | `10` + - [`baseEjectionTime`](#upstreamconfig-defaults-passivehealthcheck): string | `30s` + - [`transparentProxy`](#transparentproxy): map | no default + - [`outboundListenerPort`](#transparentproxy): integer | `15001` + - [`dialedDirectly`](#transparentproxy): boolean | `false` + - [`envoyExtensions`](#envoyextensions): list | no default + - [`name`](#envoyextensions): string | `""` + - [`required`](#envoyextensions): string | `""` + - [`arguments`](#envoyextensions): map | `nil` + - [`destination`](#destination): map | no default + - [`addresses`](#destination): list | no default + - [`port`](#destination): integer | `0` + - [`maxInboundConnections`](#maxinboundconnections): integer | `0` + - [`localConnectTimeoutMs`](#localconnecttimeoutms): integer | `0` + - [`localRequestTimeoutMs`](#localrequesttimeoutms): integer | `0` + - [`meshGateway`](#meshgateway): map | no default + - [`mode`](#meshgateway): string | no default + - [`externalSNI`](#externalsni): string | no default + - [`expose`](#expose): map | no default + - [`checks`](#expose-checks): boolean | `false` + - [`paths`](#expose-paths): list | no default + - [`path`](#expose-paths): string | no default + - [`localPathPort`](#expose-paths): integer | `0` + - [`listenerPort`](#expose-paths): integer | `0` + - [`protocol`](#expose-paths): string | `http` + + + + +## Complete configuration + +When every field is defined, a service-defaults configuration entry has the following form: + + + + +```hcl +Kind = "service-defaults" +Name = "service_name" +Namespace = "namespace" +Partition = "partition" +Meta = { + Key = "value" +} +Protocol = "tcp" +BalanceInboundConnections = "exact_balance" +Mode = "transparent" +UpstreamConfig = { + Overrides = { + Name = "name-of-upstreams-to-override" + Namespace = "namespace-containing-upstreams-to-override" + Protocol = "http" + ConnectTimeoutMs = 100 + MeshGateway = { + mode = "remote" + } + BalanceOutboundConnections = "exact_balance" + Limits = { + MaxConnections = 10 + MaxPendingRequests = 50 + MaxConcurrentRequests = 100 + } + PassiveHealthCheck = { + Interval = "5s" + MaxFailures = 5 + EnforcingConsecutive5xx = 99 + MaxEjectionPercent = 10 + BaseEjectionTime = "30s" + } + } + Defaults = { + Protocol = "http2" + ConnectTimeoutMs = 2000 + MeshGateway = { + mode = "local" + } + BalanceOutboundConnections = "exact_balance" + Limits = { + MaxConnections = 100 + MaxPendingRequests = 500 + MaxConcurrentRequests = 1000 + } + PassiveHealthCheck = { + Interval = "1s" + MaxFailures = 1 + EnforcingConsecutive5xx = 89 + MaxEjectionPercent = 10 + BaseEjectionTime = "30s" + } + } +} +TransparentProxy = { + OutboundListenerPort = 15002 + DialedDirectly = true +} +Destination = { + Addresses = [ + "First IP address", + "Second IP address" + ] + Port = 88 +} +MaxInboundConnections = 100 +LocalConnectTimeoutMs = 10 +LocalRequestTimeoutMs = 10 +MeshGateway = { + Mode = "remote" +} +ExternalSNI = "sni-server-host" +Expose = { + Checks = true + Paths = [ + { + Path = "/local/dir" + LocalPathPort = 99 + LocalListenerPort = 98 + Protocol = "http2" + } + ] +} +``` + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceDefaults +metadata: + name: + namespace: +spec: + protocol: tcp + balanceInboundConnections: exact_balance + mode: transparent + upstreamConfig: + overrides: + - name: + namespace: + protocol: + connectTimeoutMs: 5000 + meshGateway: + mode: + balanceOutboundConnections: exact_balance + limits: + maxConnections: 0 + maxPendingRequests: 0 + maxConcurrentRequests: 0 + passiveHealthCheck: + interval: "10s" + maxFailures: 0 + enforcingConsecutive5xx: 100 + maxEjectionPercent: 10 + baseEjectionTime: "30s" + defaults: + protocol: + connectTimeoutMs: 5000 + meshGateway: + mode: + balanceOutboundConnections: exact_balance + limits: + maxConnections: 0 + maxPendingRequests: 0 + maxConcurrentRequests: 0 + passiveHealthCheck: + interval: "0s" + maxFailures: 0 + enforcingConsecutive5xx: 100 + maxEjectionPercent: 10 + baseEjectionTime: "30s" + transparentProxy: + outboundListenerPort: 15001 + dialedDirectly: false + destination: + addresses: + - + + port: 0 + maxInboundConnections: 0 + meshGateway: + mode: + externalSNI: + expose: + checks: false + paths: + - path: + localPathPort: 0 + listenerPort: 0 + protocol: http +``` + + + + + +```json +{ + "apiVersion": "consul.hashicorp.com/v1alpha1", + "kind": "ServiceDefaults", + "metadata": { + "name": "", + "namespace": "", + "partition": "" + }, + "spec": { + "protocol": "tcp", + "balanceInboundConnections": "exact_balance", + "mode": "transparent", + "upstreamConfig": { + "overrides": [ + { + "name": "", + "namespace": "", + "peer": "", + "protocol": "", + "connectTimeoutMs": 5000, + "meshGateway": { + "mode": "" + }, + "balanceOutboundConnections": "exact_balance", + "limits": { + "maxConnections": 0, + "maxPendingRequests": 0, + "maxConcurrentRequests": 0 + }, + "passiveHealthCheck": { + "interval": "0s", + "maxFailures": 0, + "enforcingConsecutive5xx": 100, + "maxEjectionPercent": 10, + "baseEjectionTime": "30s", + }, + } + ], + "defaults": { + "protocol": "", + "connectTimeoutMs": 5000, + "meshGateway": { + "mode": "" + }, + "balanceOutboundConnections": "exact_balance", + "limits": { + "maxConnections": 0, + "maxPendingRequests": 0, + "maxConcurrentRequests": 0 + }, + "passiveHealthCheck": { + "interval": "0s", + "maxFailures": 0, + "enforcingConsecutive5xx": 100, + "maxEjectionPercent": 10, + "baseEjectionTime": "30s", + } + } + }, + "transparentProxy": { + "outboundListenerPort": 15001, + "dialedDirectly": false + }, + "destination": { + "addresses": [ + "", + "" + ], + "port": 0 + }, + "maxInboundConnections": 0, + "meshGateway": { + "mode": "" + }, + "externalSNI": "", + "expose": { + "checks": false, + "paths": [ + { + "path": "", + "localPathPort": 0, + "listenerPort": 0, + "protocol": "http" + } + ] + } + } +} +``` + + + + + +## Specification + +This section provides details about the fields you can configure in the service defaults configuration entry. + + + + +### `Kind` + +Specifies the configuration entry type. + +#### Values + +- Default: none +- This field is required. +- Data type: String value that must be set to `service-defaults`. + +### `Name` + +Specifies the name of the service you are setting the defaults for. + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Namespace` + +Specifies the Consul namespace that the configuration entry applies to. + +#### Values + +- Default: `default` +- Data type: string + +### `Partition` + +Specifies the name of the name of the Consul admin partition that the configuration entry applies to. Refer to [Admin Partitions](/consul/docs/enterprise/admin-partitions) for additional information. + +#### Values + +- Default: `default` +- Data type: string + +### `Meta` + +Specifies a set of custom key-value pairs to add to the [Consul KV](/consul/docs/dynamic-app-config/kv) store. + +#### Values + +- Default: none +- Data type: Map of one or more key-value pairs. + - keys: string + - values: string, integer, or float + +### `Protocol` + +Specifies the default protocol for the service. In service mesh use cases, the `protocol` configuration is required to enable the following features and components: + +- [observability](/consul/docs/connect/observability) +- [service splitter configuration entry](/consul/docs/connect/config-entries/service-splitter) +- [service router configuration entry](/consul/docs/connect/config-entries/service-router) +- [L7 intentions](/consul/docs/connect/intentions#l7-traffic-intentions) + +You can set the global protocol for proxies in the [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults#default-protocol) configuration entry, but the protocol specified in the `service-defaults` configuration entry overrides the `proxy-defaults` configuration. + +#### Values + +- Default: `tcp` +- You can specify one of the following string values: + - `tcp` (default) + - `http` + - `http2` + - `grpc` + +Refer to [Set the default protocol](#set-the-default-protocol) for an example configuration. + +### `BalanceInboundConnections` + +Specifies the strategy for allocating inbound connections to the service across Envoy proxy threads. The only supported value is `exact_balance`. By default, no connections are balanced. Refer to the [Envoy documentation](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) for details. + +#### Values + +- Default: none +- Data type: string + +### `Mode` + +Specifies a mode for how the service directs inbound and outbound traffic. + +- Default: none +- You can specify the following string values: + - `direct`: The proxy's listeners must be dialed directly by the local application and other proxies. + - `transparent`: The service captures inbound and outbound traffic and redirects it through the proxy. The mode does not enable the traffic redirection. It instructs Consul to configure Envoy as if traffic is already being redirected. + + +### `UpstreamConfig` + +Controls default upstream connection settings and custom overrides for individual upstream services. If your network contains federated datacenters, individual upstream configurations apply to all pairs of source and upstream destination services in the network. Refer to the following fields for details: + +- [`UpstreamConfig.Overrides`](#upstreamconfig-overrides) +- [`UpstreamConfig.Defaults`](#upstreamconfig-defaults) + +#### Values + +- Default: none +- Data type: map + +### `UpstreamConfig.Overrides[]` + +Specifies options that override the [default upstream configurations](#upstreamconfig-defaults) for individual upstreams. + +#### Values + +- Default: none +- Data type: list + +### `UpstreamConfig.Overrides[].Name` + +Specifies the name of the upstream service that the configuration applies to. We recommend that you do not use the `*` wildcard to avoid applying the configuration to unintended upstreams. + +#### Values + +- Default: none +- Data type: string + +### `UpstreamConfig.Overrides[].Namespace` + +Specifies the namespace containing the upstream service that the configuration applies to. Do not use the `*` wildcard to prevent the configuration from appling to unintended upstreams. + +#### Values + +- Default: none +- Data type: string + +### `UpstreamConfig.Overrides[].Protocol` +Specifies the protocol to use for requests to the upstream listener. + +We recommend configuring the protocol in the main [`Protocol`](#protocol) field of the configuration entry so that you can leverage [L7 features](/consul/docs/connect/l7-traffic). Setting the protocol in an upstream configuration limits L7 management functionality. + +#### Values + +- Default: none +- Data type: string + + +### `UpstreamConfig.Overrides[].ConnectTimeoutMs` + +Specifies how long in milliseconds that the service should attempt to establish an upstream connection before timing out. + +We recommend configuring the upstream timeout in the [`connection_timeout`](/consul/docs/connect/config-entries/service-resolver#connecttimeout) field of the `service-resolver` configuration entry for the upstream destination service. Doing so enables you to leverage [L7 features](/consul/docs/connect/l7-traffic). Configuring the timeout in the `service-defaults` upstream configuration limits L7 management functionality. + +#### Values + +- Default: `5000` +- Data type: integer + +### `UpstreamConfig.Overrides[].MeshGateway` + +Map that contains the default mesh gateway `mode` field for the upstream. Refer to [Service Mesh Proxy Configuration](/consul/docs/connect/gateways/mesh-gateway#service-mesh-proxy-configuration) in the mesh gateway documentation for additional information. + +#### Values + +- Default: `none` +- You can specify the following string values for the `mode` field: + - `none`: The service does not make outbound connections through a mesh gateway. Instead, the service makes outbound connections directly to the destination services. + - `local`: The service mesh proxy makes an outbound connection to a gateway running in the same datacenter. + - `remote`: The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. + + +### `UpstreamConfig.Overrides[].BalanceOutboundConnections` + +Sets the strategy for allocating outbound connections from the upstream across Envoy proxy threads. + +#### Values + +The only supported value is `exact_balance`. By default, no connections are balanced. Refer to the [Envoy documentation](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) for details. + +- Default: none +- Data type: string + +### `UpstreamConfig.Overrides[].Limits` + +Map that specifies a set of limits to apply to when connecting to individual upstream services. + +#### Values + +The following table describes limits you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `MaxConnections` | Specifies the maximum number of connections a service instance can establish against the upstream. Define this limit for HTTP/1.1 traffic. | integer | `0` | +| `MaxPendingRequests` | Specifies the maximum number of requests that are queued while waiting for a connection to establish. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | +| `MaxConcurrentRequests` | Specifies the maximum number of concurrent requests. Define this limit for HTTP/2 traffic. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | + +Refer to the [upstream configuration example](#upstream-configuration) for additional guidance. + +### `UpstreamConfig.Overrides[].PassiveHealthCheck` + +Map that specifies a set of rules that enable Consul to remove hosts from the upstream cluster that are unreachable or that return errors. + +#### Values + +The following table describes passive health check parameters you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `Interval` | Specifies the time between checks. | string | `0s` | +| `MaxFailures` | Specifies the number of consecutive failures allowed per check interval. If exceeded, Consul removes the host from the load balancer. | integer | `0` | +| `EnforcingConsecutive5xx` | Specifies a percentage that indicates how many times out of 100 that Consul ejects the host when it detects an outlier status. The outlier status is determined by consecutive errors in the 500-599 response range. | integer | `100` | + | `MaxEjectionPercent` | Specifies the maximum percentage of an upstream cluster that Consul ejects when the proxy reports an outlier. Consul ejects at least one host when an outlier is detected regardless of the value. | integer | `10` | + | `BaseEjectionTime` | Specifies the minimum amount of time that an ejected host must remain outside the cluster before rejoining. The real time is equal to the value of the `BaseEjectionTime` multiplied by the number of times the host has been ejected. | string | `30s` | + +### `UpstreamConfig.Defaults` + +Specifies configurations that set default upstream settings. For information about overriding the default configurations for in for individual upstreams, refer to [`UpstreamConfig.Overrides`](#upstreamconfig-overrides). + +#### Values + +- Default: none +- Data type: map + +### `UpstreamConfig.Defaults.Protocol` + +Specifies default protocol for upstream listeners. + +We recommend configuring the protocol in the main [`Protocol`](#protocol) field of the configuration entry so that you can leverage [L7 features](/consul/docs/connect/l7-traffic). Setting the protocol in an upstream configuration limits L7 management functionality. + +- Default: none +- Data type: string + +### `UpstreamConfig.Defaults.ConnectTimeoutMs` + +Specifies how long in milliseconds that all services should continue attempting to establish an upstream connection before timing out. + +For non-Kubernetes environments, we recommend configuring the upstream timeout in the [`connection_timeout`](/consul/docs/connect/config-entries/service-resolver#connecttimeout) field of the `service-resolver` configuration entry for the upstream destination service. Doing so enables you to leverage [L7 features](/consul/docs/connect/l7-traffic). Configuring the timeout in the `service-defaults` upstream configuration limits L7 management functionality. + +- Default: `5000` +- Data type: integer + +### `UpstreamConfig.Defaults.MeshGateway` + +Specifies the default mesh gateway `mode` field for all upstreams. Refer to [Service Mesh Proxy Configuration](/consul/docs/connect/gateways/mesh-gateway#service-mesh-proxy-configuration) in the mesh gateway documentation for additional information. + +You can specify the following string values for the `mode` field: + +- `none`: The service does not make outbound connections through a mesh gateway. Instead, the service makes outbound connections directly to the destination services. +- `local`: The service mesh proxy makes an outbound connection to a gateway running in the same datacenter. +- `remote`: The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. + +### `UpstreamConfig.Defaults.BalanceOutboundConnections` + +Sets the strategy for allocating outbound connections from upstreams across Envoy proxy threads. The only supported value is `exact_balance`. By default, no connections are balanced. Refer to the [Envoy documentation](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) for details. + +- Default: none +- Data type: string + +### `UpstreamConfig.Defaults.Limits` + +Map that specifies a set of limits to apply to when connecting upstream services. The following table describes limits you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `MaxConnections` | Specifies the maximum number of connections a service instance can establish against the upstream. Define this limit for HTTP/1.1 traffic. | integer | `0` | +| `MaxPendingRequests` | Specifies the maximum number of requests that are queued while waiting for a connection to establish. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | +| `MaxConcurrentRequests` | Specifies the maximum number of concurrent requests. Define this limit for HTTP/2 traffic. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | + +### `UpstreamConfig.Defaults.PassiveHealthCheck` + +Map that specifies a set of rules that enable Consul to remove hosts from the upstream cluster that are unreachable or that return errors. The following table describes the health check parameters you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `Interval` | Specifies the time between checks. | string | `0s` | +| `MaxFailures` | Specifies the number of consecutive failures allowed per check interval. If exceeded, Consul removes the host from the load balancer. | integer | `0` | +| `EnforcingConsecutive5xx ` | Specifies a percentage that indicates how many times out of 100 that Consul ejects the host when it detects an outlier status. The outlier status is determined by consecutive errors in the 500-599 response range. | integer | `100` | + | `MaxEjectionPercent` | Specifies the maximum percentage of an upstream cluster that Consul ejects when the proxy reports an outlier. Consul ejects at least one host when an outlier is detected regardless of the value. | integer | `10` | + | `BaseEjectionTime` | Specifies the minimum amount of time that an ejected host must remain outside the cluster before rejoining. The real time is equal to the value of the `BaseEjectionTime` multiplied by the number of times the host has been ejected. | string | `30s` | + +### `TransparentProxy` + +Controls configurations specific to proxies in transparent mode. Refer to [Transparent Proxy](/consul/docs/connect/transparent-proxy) for additional information. + +You can configure the following parameters in the `TransparentProxy` block: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `OutboundListenerPort` | Specifies the port that the proxy listens on for outbound traffic. This must be the same port number where outbound application traffic is redirected. | integer | `15001` | +| `DialedDirectly` | Enables transparent proxies to dial the proxy instance's IP address directly when set to `true`. Transparent proxies commonly dial upstreams at the `"virtual"` tagged address, which load balances across instances. Dialing individual instances can be helpful for stateful services, such as a database cluster with a leader. | boolean | `false` | + +### `EnvoyExtensions` + +List of extensions to modify Envoy proxy configuration. Refer to [Envoy Extensions](/consul/docs/connect/proxies/envoy-extensions) for additional information. + +You can configure the following parameters in the `EnvoyExtensions` block: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `Name` | Name of the extension. | string | `""` | +| `Required` | When Required is true and the extension does not update any Envoy resources, an error is returned. Use this parameter to ensure that extensions required for secure communication are not unintentionally bypassed. | string | `""` | +| `Arguments` | Arguments to pass to the extension executable. | map | `nil` | + +### `Destination[]` -Set the default protocol for a service in the default namespace to HTTP: +Configures the destination for service traffic through terminating gateways. Refer to [Terminating Gateway](/consul/docs/connect/gateways/terminating-gateway) for additional information. + +You can configure the following parameters in the `Destination` block: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `Address` | Specifies a list of addresses for the destination. You can configure a list of hostnames and IP addresses. Wildcards are not supported. | list | none | +| `Port` | Specifies the port number of the destination. | integer | `0` | + +### `MaxInboundConnections` + +Specifies the maximum number of concurrent inbound connections to each service instance. + +- Default: `0` +- Data type: integer + +### `LocalConnectTimeoutMs` + +Specifies the number of milliseconds allowed for establishing connections to the local application instance before timing out. + +- Default: `5000` +- Data type: integer + +### `LocalRequestTimeoutMs` + +Specifies the timeout for HTTP requests to the local application instance. Applies to HTTP-based protocols only. If not specified, inherits the Envoy default for route timeouts. + +- Default: Inherits `15s` from Envoy as the default +- Data type: string + +### `MeshGateway` + +Specifies the default mesh gateway `mode` field for the service. Refer to [Service Mesh Proxy Configuration](/consul/docs/connect/gateways/mesh-gateway#service-mesh-proxy-configuration) in the mesh gateway documentation for additional information. + +You can specify the following string values for the `mode` field: + +- `none`: The service does not make outbound connections through a mesh gateway. Instead, the service makes outbound connections directly to the destination services. +- `local`: The service mesh proxy makes an outbound connection to a gateway running in the same datacenter. +- `remote`: The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. + +### `ExternalSNI` + +Specifies the TLS server name indication (SNI) when federating with an external system. + +- Default: none +- Data type: string + +### `Expose` + +Specifies default configurations for exposing HTTP paths through Envoy. Exposing paths through Envoy enables services to listen on localhost only. Applications that are not Consul service mesh-enabled can still contact an HTTP endpoint. Refer to [Expose Paths Configuration Reference](/consul/docs/connect/registration/service-registration#expose-paths-configuration-reference) for additional information and example configurations. + +- Default: none +- Data type: map + +### `Expose.Checks` + +Exposes all HTTP and gRPC checks registered with the agent if set to `true`. Envoy exposes listeners for the checks and only accepts connections originating from localhost or Consul's [`advertise_addr`](/consul/docs/agent/config/config-files#advertise_addr). The ports for the listeners are dynamically allocated from the agent's [`expose_min_port`](/consul/docs/agent/config/config-files#expose_min_port) and [`expose_max_port`](/consul/docs/agent/config/config-files#expose_max_port) configurations. + +We recommend enabling the `Checks` configuration when a Consul client cannot reach registered services over localhost, such as when Consul agents run in their own pods in Kubernetes. + +- Default: `false` +- Data type: boolean + +### `Expose.Paths[]` + +Specifies a list of configuration maps that define paths to expose through Envoy when `Expose.Checks` is set to `true`. You can configure the following parameters for each map in the list: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `Path` | Specifies the HTTP path to expose. You must prepend the path with a forward slash (`/`). | string | none | +| `LocalPathPort` | Specifies the port where the local service listens for connections to the path. | integer | `0` | +| `ListenPort` | Specifies the port where the proxy listens for connections. The port must be available. If the port is unavailable, Envoy does not expose a listener for the path and the proxy registration still succeeds. | integer | `0` | +| `Protocol` | Specifies the protocol of the listener. You can configure one of the following values:
  • `http`
  • `http2`: Use with gRPC traffic
    • | integer | `http` | + +     + + + +### `apiVersion` + +Specifies the version of the Consul API for integrating with Kubernetes. The value must be `consul.hashicorp.com/v1alpha1`. The `apiVersion` field is not supported for non-Kubernetes deployments. + +- Default: none +- This field is required. +- String value that must be set to `consul.hashicorp.com/v1alpha1`. + +### `kind` + +Specifies the configuration entry type. Must be ` ServiceDefaults`. + +- Required: required +- String value that must be set to `ServiceDefaults`. + +### `metadata` + +Map that contains the service name, namespace, and admin partition that the configuration entry applies to. + +#### Values + +- Default: none +- Map containing the following strings: + - [`name`](#name) + - [`namespace`](#namespace) + - [`partition`](#partition) + + +### `metadata.name` + +Specifies the name of the service you are setting the defaults for. + +#### Values + +- Default: none +- This field is required +- Data type: string + +### `metadata.namespace` + +Specifies the Consul namespace that the configuration entry applies to. Refer to [Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for information about how Consul namespaces map to Kubernetes Namespaces. Open source Consul distributions (Consul OSS) ignore the `metadata.namespace` configuration. + +- Default: `default` +- Data type: string + +### `spec` + +Map that contains the details about the `ServiceDefaults` configuration entry. The `apiVersion`, `kind`, and `metadata` fields are siblings of the `spec` field. All other configurations are children. + +### `spec.protocol` + +Specifies the default protocol for the service. In service service mesh use cases, the `protocol` configuration is required to enable the following features and components: + +- [observability](/consul/docs/connect/observability) +- [`service-splitter` configuration entry](/consul/docs/connect/config-entries/service-splitter) +- [`service-router` configuration entry](/consul/docs/connect/config-entries/service-router) +- [L7 intentions](/consul/docs/connect/intentions#l7-traffic-intentions) + +You can set the global protocol for proxies in the [`ProxyDefaults` configuration entry](/consul/docs/connect/config-entries/proxy-defaults#default-protocol), but the protocol specified in the `ServiceDefaults` configuration entry overrides the `ProxyDefaults` configuration. + +#### Values + +- Default: `tcp` +- You can specify one of the following string values: + - `tcp` + - `http` + - `http2` + - `grpc` + +Refer to [Set the default protocol](#set-the-default-protocol) for an example configuration. + +### `spec.balanceInboundConnections` + +Specifies the strategy for allocating inbound connections to the service across Envoy proxy threads. The only supported value is `exact_balance`. By default, no connections are balanced. Refer to the [Envoy documentation](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) for details. + +#### Values + +- Default: none +- Data type: string + +### `spec.mode` + +Specifies a mode for how the service directs inbound and outbound traffic. + +#### Values + +- Default: none +- Required: optional +- You can specified the following string values: + +- `direct`: The proxy's listeners must be dialed directly by the local application and other proxies. +- `transparent`: The service captures inbound and outbound traffic and redirects it through the proxy. The mode does not enable the traffic redirection. It instructs Consul to configure Envoy as if traffic is already being redirected. + +### `spec.upstreamConfig` + +Specifies a map that controls default upstream connection settings and custom overrides for individual upstream services. If your network contains federated datacenters, individual upstream configurations apply to all pairs of source and upstream destination services in the network. + +#### Values + +- Default: none +- Map that contains the following configurations: + - [`UpstreamConfig.Overrides`](#upstreamconfig-overrides) + - [`UpstreamConfig.Defaults`](#upstreamconfig-defaults) + +### `spec.upstreamConfig.overrides[]` + +Specifies options that override the [default upstream configurations](#spec-upstreamconfig-defaults) for individual upstreams. + +#### Values + +- Default: none +- Data type: list + +### `spec.upstreamConfig.overrides[].name` + +Specifies the name of the upstream service that the configuration applies to. Do not use the `*` wildcard to prevent the configuration from applying to unintended upstreams. + +#### Values + +- Default: none +- Data type: string + +### `spec.upstreamConfig.overrides[].namespace` + +Specifies the namespace containing the upstream service that the configuration applies to. Do not use the `*` wildcard to prevent the configuration from applying to unintended upstreams. + +#### Values + +- Default: none +- Data type: string + +### `spec.upstreamConfig.overrides[].protocol` + +Specifies the protocol to use for requests to the upstream listener. We recommend configuring the protocol in the main [`protocol`](#protocol) field of the configuration entry so that you can leverage [L7 features](/consul/docs/connect/l7-traffic). Setting the protocol in an upstream configuration limits L7 management functionality. + +#### Values + +- Default: inherits the main [`protocol`](#protocol) configuration +- Data type: string + + +### `spec.upstreamConfig.overrides[].connectTimeoutMs` + +Specifies how long in milliseconds that the service should attempt to establish an upstream connection before timing out. + +We recommend configuring the upstream timeout in the [`connectTimeout`](/consul/docs/connect/config-entries/service-resolver#connecttimeout) field of the `ServiceResolver` CRD for the upstream destination service. Doing so enables you to leverage [L7 features](/consul/docs/connect/l7-traffic). Configuring the timeout in the `ServiceDefaults` upstream configuration limits L7 management functionality. + +#### Values + +- Default: `5000` +- Data type: integer + +### `spec.upstreamConfig.overrides[].meshGateway.mode` + +Map that contains the default mesh gateway `mode` field for the upstream. Refer to [Connect Proxy Configuration](/consul/docs/connect/gateways/mesh-gateway#connect-proxy-configuration) in the mesh gateway documentation for additional information. + +#### Values + +You can specify the following string values for the `mode` field: + +- `none`: The service does not make outbound connections through a mesh gateway. Instead, the service makes outbound connections directly to the destination services. +- `local`: The service mesh proxy makes an outbound connection to a gateway running in the same datacenter. +- `remote`: The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. + +### `spec.upstreamConfig.overrides[].balanceInboundConnections` + +Sets the strategy for allocating outbound connections from the upstream across Envoy proxy threads. The only supported value is `exact_balance`. By default, no connections are balanced. Refer to the [Envoy documentation](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) for details. + +#### Values + +- Default: none +- Data type: string + +### `spec.upstreamConfig.overrides[].limits` + +Map that specifies a set of limits to apply to when connecting to individual upstream services. + +#### Values + +The following table describes limits you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `maxConnections` | Specifies the maximum number of connections a service instance can establish against the upstream. Define this limit for HTTP/1.1 traffic. | integer | `0` | +| `maxPendingRequests` | Specifies the maximum number of requests that are queued while waiting for a connection to establish. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | +| `maxConcurrentRequests` | Specifies the maximum number of concurrent requests. Define this limit for HTTP/2 traffic. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | + +### `spec.upstreamConfig.overrides[].passiveHealthCheck` + +Map that specifies a set of rules that enable Consul to remove hosts from the upstream cluster that are unreachable or that return errors. + +#### Values + +The following table describes passive health check parameters you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `interval` | Specifies the time between checks. | string | `0s` | +| `maxFailures` | Specifies the number of consecutive failures allowed per check interval. If exceeded, Consul removes the host from the load balancer. | integer | `0` | +| `enforcingConsecutive5xx ` | Specifies a percentage that indicates how many times out of 100 that Consul ejects the host when it detects an outlier status. The outlier status is determined by consecutive errors in the 500-599 response range. | integer | `100` | + | `maxEjectionPercent` | The maximum % of an upstream cluster that can be ejected due to outlier detection. Defaults to 10% but will eject at least one host regardless of the value. | integer | `10` | + | `baseEjectionTime` | The base time that a host is ejected for. The real time is equal to the base time multiplied by the number of times the host has been ejected and is capped by max_ejection_time (Default 300s). Defaults to 30000ms or 30s. | string | `30s` | + +### `spec.upstreamConfig.defaults` + +Map of configurations that set default upstream configurations for the service. For information about overriding the default configurations for in for individual upstreams, refer to [`spec.upstreamConfig.overrides`](#spec-upstreamconfig-overrides). + +#### Values + +- Default: none +- Data type: list + +### `spec.upstreamConfig.defaults.protocol` + +Specifies default protocol for upstream listeners. We recommend configuring the protocol in the main [`Protocol`](#protocol) field of the configuration entry so that you can leverage [L7 features](/consul/docs/connect/l7-traffic). Setting the protocol in an upstream configuration limits L7 management functionality. + +#### Values + +- Default: none +- Data type: string + +### `spec.upstreamConfig.default.connectTimeoutMs` + +Specifies how long in milliseconds that all services should continue attempting to establish an upstream connection before timing out. + +We recommend configuring the upstream timeout in the [`connectTimeout`](/consul/docs/connect/config-entries/service-resolver#connecttimeout) field of the `ServiceResolver` CRD for upstream destination services. Doing so enables you to leverage [L7 features](/consul/docs/connect/l7-traffic). Configuring the timeout in the `ServiceDefaults` upstream configuration limits L7 management functionality. + +#### Values + +- Default: `5000` +- Data type: integer + +### `spec.upstreamConfig.defaults.meshGateway.mode` + +Specifies the default mesh gateway `mode` field for all upstreams. Refer to [Service Mesh Proxy Configuration](/consul/docs/connect/gateways/mesh-gateway#service-mesh-proxy-configuration) in the mesh gateway documentation for additional information. + +#### Values + +You can specify the following string values for the `mode` field: + +- `none`: The service does not make outbound connections through a mesh gateway. Instead, the service makes outbound connections directly to the destination services. +- `local`: The service mesh proxy makes an outbound connection to a gateway running in the same datacenter. +- `remote`: The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. + +### `spec.upstreamConfig.defaults.balanceInboundConnections` + +Sets the strategy for allocating outbound connections from upstreams across Envoy proxy threads. The only supported value is `exact_balance`. By default, no connections are balanced. Refer to the [Envoy documentation](https://cloudnative.to/envoy/api-v3/config/listener/v3/listener.proto.html#config-listener-v3-listener-connectionbalanceconfig) for details. + +#### Values + +- Default: none +- Data type: string + +### `spec.upstreamConfig.defaults.limits` + +Map that specifies a set of limits to apply to when connecting upstream services. + +#### Values + +The following table describes limits you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `maxConnections` | Specifies the maximum number of connections a service instance can establish against the upstream. Define this limit for HTTP/1.1 traffic. | integer | `0` | +| `maxPendingRequests` | Specifies the maximum number of requests that are queued while waiting for a connection to establish. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | +| `maxConcurrentRequests` | Specifies the maximum number of concurrent requests. Define this limit for HTTP/2 traffic. An L7 protocol must be defined in the [`protocol`](#protocol) field for this limit to take effect. | integer | `0` | + +### `spec.upstreamConfig.defaults.passiveHealthCheck` +Map that specifies a set of rules that enable Consul to remove hosts from the upstream cluster that are unreachable or that return errors. + +#### Values + +The following table describes the health check parameters you can configure: + +| Limit | Description | Data type | Default | +| --- | --- | --- | --- | +| `interval` | Specifies the time between checks. | string | `0s` | +| `maxFailures` | Specifies the number of consecutive failures allowed per check interval. If exceeded, Consul removes the host from the load balancer. | integer | `0` | +| `enforcingConsecutive5xx ` | Specifies a percentage that indicates how many times out of 100 that Consul ejects the host when it detects an outlier status. The outlier status is determined by consecutive errors in the 500-599 response range. | integer | `100` | + | `MaxEjectionPercent` | Specifies the maximum percentage of an upstream cluster that Consul ejects when the proxy reports an outlier. Consul ejects at least one host when an outlier is detected regardless of the value. | integer | `10` | + | `BaseEjectionTime` | Specifies the minimum amount of time that an ejected host must remain outside the cluster before rejoining. The real time is equal to the value of the `BaseEjectionTime` multiplied by the number of times the host has been ejected. | string | `30s` | + +### `spec.transparentProxy` + +Map of configurations specific to proxies in transparent mode. Refer to [Transparent Proxy](/consul/docs/connect/transparent-proxy) for additional information. + +#### Values + +You can configure the following parameters in the `TransparentProxy` block: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `outboundListenerPort` | Specifies the port that the proxy listens on for outbound traffic. This must be the same port number where outbound application traffic is redirected. | integer | `15001` | +| `dialedDirectly` | Enables transparent proxies to dial the proxy instance's IP address directly when set to `true`. Transparent proxies commonly dial upstreams at the `"virtual"` tagged address, which load balances across instances. Dialing individual instances can be helpful for stateful services, such as a database cluster with a leader. | boolean | `false` | + +### `spec.envoyExtensions` + +List of extensions to modify Envoy proxy configuration. Refer to [Envoy Extensions](/consul/docs/connect/proxies/envoy-extensions) for additional information. + +#### Values + +You can configure the following parameters in the `EnvoyExtensions` block: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `name` | Name of the extension. | string | `""` | +| `required` | When Required is true and the extension does not update any Envoy resources, an error is returned. Use this parameter to ensure that extensions required for secure communication are not unintentionally bypassed. | string | `""` | +| `arguments` | Arguments to pass to the extension executable. | map | `nil` | + +### `spec.destination` + +Map of configurations that specify one or more destinations for service traffic routed through terminating gateways. Refer to [Terminating Gateway](/consul/docs/connect/gateways/terminating-gateway) for additional information. + +#### Values + +You can configure the following parameters in the `Destination` block: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `address` | Specifies a list of addresses for the destination. You can configure a list of hostnames and IP addresses. Wildcards are not supported. | list | none | +| `port` | Specifies the port number of the destination. | integer | `0` | + +### `spec.maxInboundConnections` + +Specifies the maximum number of concurrent inbound connections to each service instance. + +#### Values + +- Default: `0` +- Data type: integer + +### `spec.localConnectTimeoutMs` + +Specifies the number of milliseconds allowed for establishing connections to the local application instance before timing out. + +#### Values + +- Default: `5000` +- Data type: integer + +### `spec.localRequestTimeoutMs` + +Specifies the timeout for HTTP requests to the local application instance. Applies to HTTP-based protocols only. If not specified, inherits the Envoy default for route timeouts. + +#### Values + +- Default of `15s` is inherited from Envoy +- Data type: string + +### `spec.meshGateway.mode` +Specifies the default mesh gateway `mode` field for the service. Refer to [Service Mesh Proxy Configuration](/consul/docs/connect/gateways/mesh-gateway#service-mesh-proxy-configuration) in the mesh gateway documentation for additional information. + +#### Values + +You can specify the following string values for the `mode` field: + +- `none`: The service does not make outbound connections through a mesh gateway. Instead, the service makes outbound connections directly to the destination services. +- `local`: The service mesh proxy makes an outbound connection to a gateway running in the same datacenter. +- `remote`: The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. + +### `spec.externalSNI` + +Specifies the TLS server name indication (SNI) when federating with an external system. + +#### Values + +- Default: none +- Data type: string + +### `spec.expose` + +Specifies default configurations for exposing HTTP paths through Envoy. Exposing paths through Envoy enables services to listen on `localhost` only. Applications that are not Consul service mesh-enabled can still contact an HTTP endpoint. Refer to [Expose Paths Configuration Reference](/consul/docs/connect/registration/service-registration#expose-paths-configuration-reference) for additional information and example configurations. + +#### Values + +- Default: none +- Data type: string + +### `spec.expose.checks` + +Exposes all HTTP and gRPC checks registered with the agent if set to `true`. Envoy exposes listeners for the checks and only accepts connections originating from localhost or Consul's [`advertise_addr`](/consul/docs/agent/config/config-files#advertise_addr). The ports for the listeners are dynamically allocated from the agent's [`expose_min_port`](/consul/docs/agent/config/config-files#expose_min_port) and [`expose_max_port`](/consul/docs/agent/config/config-files#expose_max_port) configurations. + +We recommend enabling the `Checks` configuration when a Consul client cannot reach registered services over localhost, such as when Consul agents run in their own pods in Kubernetes. + +#### Values + +- Default: `false` +- Data type: boolean + +### `spec.expose.paths[]` + +Specifies an list of maps that define paths to expose through Envoy when `spec.expose.checks` is set to `true`. + +#### Values + +The following table describes the parameters for each map: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `path` | Specifies the HTTP path to expose. You must prepend the path with a forward slash (`/`). | string | none | +| `localPathPort` | Specifies the port where the local service listens for connections to the path. | integer | `0` | +| `listenPort` | Specifies the port where the proxy listens for connections. The port must be available. If the port is unavailable, Envoy does not expose a listener for the path and the proxy registration still succeeds. | integer | `0` | +| `protocol` | Specifies the protocol of the listener. You can configure one of the following values:
  • `http`
  • `http2`: Use with gRPC traffic
    • | integer | `http` | + +     +     + +## Example configurations + +The following examples describe common `service-defaults` configurations. + +### Set the default protocol + +In the following example, protocol for the `web` service in the `default` namespace is set to `http`: @@ -50,14 +1245,15 @@ spec: +You can also set the global default protocol for all proxies in the [`proxy-defaults` configuration entry](/consul/docs/connect/config-entries/proxy-defaults#default-protocol), but the protocol specified for individual service instances in the `service-defaults` configuration entry takes precedence over the globally-configured value set in the `proxy-defaults`. + ### Upstream configuration -Set default connection limits and mesh gateway mode across all upstreams -of "dashboard", and also override the mesh gateway mode used when dialing -its upstream "counting" service. +The following example sets default connection limits and mesh gateway mode across all upstreams of the `dashboard` service. +It also overrides the mesh gateway mode used when dialing its `counting` upstream service. @@ -140,10 +1336,7 @@ spec: -Set default connection limits and mesh gateway mode across all upstreams -of "dashboard" in the "product" namespace, -and also override the mesh gateway mode used when dialing -its upstream "counting" service in the "backend" namespace. +The following example configures the default connection limits and mesh gateway mode for all of the `counting` service's upstreams. It also overrides the mesh gateway mode used when dialing the `dashboard` service in the `frontend` namespace. @@ -234,8 +1427,8 @@ spec: ### Terminating gateway destination -Create a default destination that will be assigned to a terminating gateway. A destination -represents a location outside the Consul cluster. They can be dialed directly when transparent proxy mode is enabled. +The following examples creates a default destination assigned to a terminating gateway. A destination +represents a location outside the Consul cluster. Services can dial destinations dialed directly when transparent proxy mode is enabled. @@ -255,6 +1448,7 @@ represents a location outside the Consul cluster. They can be dialed directly wh metadata: name: test-destination spec: + protocol: tcp destination: addresses: - "test.com" @@ -266,7 +1460,7 @@ represents a location outside the Consul cluster. They can be dialed directly wh { "Kind": "service-defaults", "Name": "test-destination", - "Protocol": "http", + "Protocol": "tcp", "Destination": { "Addresses": ["test.com","test.org"], "Port": 443 @@ -276,6 +1470,7 @@ represents a location outside the Consul cluster. They can be dialed directly wh + diff --git a/website/content/docs/connect/config-entries/service-intentions.mdx b/website/content/docs/connect/config-entries/service-intentions.mdx index 619b4fb9fed..321b7f16058 100644 --- a/website/content/docs/connect/config-entries/service-intentions.mdx +++ b/website/content/docs/connect/config-entries/service-intentions.mdx @@ -1,40 +1,942 @@ --- layout: docs -page_title: Service Intentions - Configuration Entry Reference +page_title: Service intentions configuration entry reference description: >- - The service intentions configuration entry kind defines the communication permissions between service types. Use the reference guide to learn about `""service-intentions""` config entry parameters and how to authorize L4 and L7 communication int he service mesh with intentions. + Use the service intentions configuration entry to allow or deny traffic to services in the mesh from specific sources. Learn how to configure `service-intention` config entries --- -# Service Intentions Configuration Entry ((#service-intentions)) +# Service intentions configuration entry reference --> **1.9.0+:** This config entry is available in Consul versions 1.9.0 and newer. +This topic provides reference information for the service intentions configuration entry. Intentions are configurations for controlling access between services in the service mesh. A single service intentions configuration entry specifies one destination service and one or more L4 traffic sources, L7 traffic sources, or combination of traffic sources. Refer to [Service mesh intentions overview](/consul/docs/connect/intentions) for additional information. -The `service-intentions` config entry kind (`ServiceIntentions` on Kubernetes) controls Connect traffic -authorization for both networking layer 4 (e.g. TCP) and networking layer 7 -(e.g. HTTP). +## Configuration model -Service intentions config entries represent a collection of -[intentions](/consul/docs/connect/intentions) sharing a specific destination. All -intentions governing access to a specific destination are stored in a single -`service-intentions` config entry. +The following outline shows how to format the service intentions configuration entry. Click on a property name to view details about the configuration. -A single config entry may define a mix of both L4 and L7 style intentions, but -for a specific source L4 and L7 intentions are mutually exclusive. Only one -will apply at a time. Default behavior for L4 is configurable (regardless of -global setting) by defining a low precedence intention for that destination. + -## Interaction with other Config Entries + -L7 intentions within a config entry are restricted to only destination services -that define their protocol as HTTP-based via a corresponding -[`service-defaults`](/consul/docs/connect/config-entries/service-defaults) config entry -or globally via [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults) . +- [`Kind`](#kind): string | required | must be set to `service-intentions` +- [`Name`](#name): string | required +- [`Namespace`](#namespace): string | `default` | +- [`Partition`](#partition): string | `default` | +- [`Meta`](#meta): map | no default +- [`Sources`](#sources): list | no default + - [`Name`](#sources-name): string | no default + - [`Peer`](#sources-peer): string | no default + - [`Namespace`](#sources-namespace): string | no default | + - [`Partition`](#sources-partition): string | no default | + - [`Action`](#sources-action): string | no default | required for L4 intentions + - [`Permissions`](#sources-permissions): list | no default + - [`Action`](#sources-permissions-action): string | no default | required + - [`HTTP`](#sources-permissions-http): map | required + - [`PathExact`](#sources-permissions-http): string | no default + - [`PathPrefix`](#sources-permissions-http): string | no default + - [`PathRegex`](#sources-permissions-http): string | no default + - [`Methods`](#sources-permissions-http): list | no default + - [`Header`](#sources-permissions-http-header): list of maps |no default + - [`Name`](#sources-permissions-http-header): string | required + - [`Present`](#sources-permissions-http-header): boolean | `false` + - [`Exact`](#sources-permissions-http-header): string | no default + - [`Prefix`](#sources-permissions-http-header): string | no default + - [`Suffix`](#sources-permissions-http-header): string | no default + - [`Regex`](#sources-permissions-http-header): string | no default + - [`Invert`](#sources-permissions-http-header): boolean | `false` + - [`Precedence`](#sources-precedence): number | no default | _read-only_ + - [`Type`](#sources-type): string | `consul` + - [`Description`](#sources-description): string + - [`LegacyID`](#sources-legacyid): string | no default | _read-only_ + - [`LegacyMeta`](#sources-legacymeta): map | no default | _read-only_ + - [`LegacyCreateTime`](#sources-legacycreatetime): string | no default | _read-only_ + - [`LegacyUpdateTime`](#sources-legacyupdatetime): string | no default | _read-only_ -## Sample Config Entries + + -The following examples demonstrate potential use-cases for the `service-intentions` configuration entry. +- [`apiVersion`](#apiversion): string | must be set to `consul.hashicorp.com/v1alpha1` +- [`kind`](#kind): string | must be set to `ServiceIntentions` +- [`metadata`](#metadata): map | required + - [`name`](#metadata-name): string | required + - [`namespace`](#metadata-namespace): string | `default` | +- [`spec`](#spec): map | no default + - [`destination`](#spec-destination): map | no default + - [`name`](#spec-destination-name): string | required + - [`namespace`](#metadata-namespace): string | `default` | + - [`sources`](#spec-sources): list | no default + - [`name`](#spec-sources-name): string | no default + - [`peer`](#spec-sources-peer): string | no default + - [`namespace`](#spec-sources-namespace): string | no default | + - [`partition`](#spec-sources-partition): string | no default | + - [`action`](#spec-sources-action): string | no default | required for L4 intentions + - [`permissions`](#spec-sources-permissions): list | no default + - [`action`](#spec-sources-permissions-action): string | no default | required + - [`http`](#spec-sources-permissions-http): map | required + - [`pathExact`](#spec-sources-permissions-http): string | no default + - [`pathPrefix`](#spec-sources-permissions-http): string | no default + - [`pathRegex`](#spec-sources-permissions-http): string | no default + - [`methods`](#spec-sources-permissions-http): list | no default + - [`header`](#spec-sources-permissions-http-header): list of maps |no default + - [`name`](#spec-sources-permissions-http-header): string | required + - [`present`](#spec-sources-permissions-http-header): boolean | `false` + - [`exact`](#spec-sources-permissions-http-header): string | no default + - [`prefix`](#spec-sources-permissions-http-header): string | no default + - [`suffix`](#spec-sources-permissions-http-header): string | no default + - [`regex`](#spec-sources-permissions-http-header): string | no default + - [`invert`](#spec-sources-permissions-http-header): boolean | `false` + - [`type`](#spec-sources-type): string | `consul` + - [`description`](#spec-sources-description): string -### REST Access + + + +## Complete configuration + +When every field is defined, a service intentions configuration entry has the following form: + + + + + +```hcl +Kind = "service-intentions" +Name = "" +Namespace = "" # string +Partition = "" # string +Meta = { + "" = "" + "" = "" + } +Sources = [ + { + Name = "" # string + Peer = "" # string + Namespace = "" # string + Partition = "" # string + Action = "allow" or "deny" # string for L4 intentions + Permissions = [ + { + Action = "allow" or "deny" # string for L7 intenions + HTTP = { + PathExact = "" # string + PathPrefix = "" # string + PathRegex = "" # string + Methods = [ + "", # string + "" + ] + Header = [ + { + Name = "" # string + Present = # boolean + }, + { + Name = "" # string + Exact = "" # boolean + }, + { + Name = "" # string + Prefix = "" # string + }, + { + Name = "" # string + Suffix = "" # string + }, + { + Name = "" # string + Regex = "" # string + Invert = # boolean + } + ] + } + } + ] + Type = "consul" # string + Description = "" # string + Precedence = # number + LegacyID = # string + LegacyMeta = # string + LegacyCreateTime = # string + LegacyUpdateTime = # string + } +] +``` + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: service-intentions +metadata: + name: + namespace: +spec: + destination: + destination: + name: + namespace: + sources: + name: + peer: + namespace: + partition: + action: allow or deny + permissions: + - action: allow or deny + http: + pathExact: + pathPrefix: + pathRegex: + methods: + - + + header: + - name: + present: true + - name: + exact: false + - name: + prefix: + - name: + suffix: + - name: + regex: + invert: false + type: consul + description: +``` + + + + +```json +{ + "Kind":"service-intentions", + "Name":"", + "Namespace":"", + "Partition":"", + "Meta":{ + "key-1":"", + "key-2":"" + }, + "Sources":[ + { + "Name":"", + "Peer":"", + "Namespace":"", + "Partition":"", + "Action":"allow or deny", + "Permissions":[ + { + "Action":"allow or deny", + "HTTP":{ + "PathExact":"", + "PathPrefix":"", + "PathRegex":"", + "Methods":[ + "", + "" + ], + "Header":[ + { + "Name":"", + "Present":true + }, + { + "Name":"", + "Exact":false + }, + { + "Name":"", + "Prefix":"" + }, + { + "Name":"", + "Suffix":"" + }, + { + "Name":"", + "Regex":"", + "Invert":false + } + ] + } + } + ], + "Type":"consul", + "Description":"", + "Precedence":"", + "LegacyID":"", + "LegacyMeta":"", + "LegacyCreateTime":"", + "LegacyUpdateTime":"" + } + ] +} +``` + + + + + +## Specification + +This section provides details about the fields you can configure in the service intentions configuration entry. + + + + + +### `Kind` + +Specifies the type of configuration entry to implement. Must be set to `service-intentions`. + +#### Values + +- Default: None +- This field is required. +- Data type: String value that must be set to `service-intentions`. + +### `Name` + +Specifies a name of the destination service for all intentions defined in the configuration entry. + +#### Values + +- Default: Defaults to the name of the node after writing the entry to the Consul server. +- This field is required. +- Data type: String + +You can also specify a wildcard character (`*`) to match all services without intentions. Intentions that are applied with a wildcard, however, are not supported when defining L7 [`Permissions`](#sources-permissions). + +### `Namespace` + +Specifies the [namespace](/consul/docs/enterprise/namespaces) that the configuration entry applies to. Services in the namespace are the traffic destinations that the intentions allow or deny traffic to. + +#### Values + +- Default: `default` +- Data type: String + +You can also specify a wildcard character (`*`) to match all namespaces. Intentions that are applied with a wildcard, however, are not supported when defining L7 [`Permissions`](#sources-permissions). + +### `Partition` + +Specifies the [admin partition](/consul/docs/enterprise/admin-partitions) to apply the configuration entry. Services in the specified partition are the traffic destinations that the intentions allow or deny traffic to. + +#### Values + +- Default: `default` +- Data type: String + +### `Meta` + +Specifies key-value pairs to add to the KV store when the configuration entry is evaluated. + +#### Values + +- Default: None +- Data type: Map of one or more key-value pairs + - keys: String + - values: String, integer, or float + +### `Sources[]` + +List of configurations that define intention sources and the authorization granted to the sources. You can specify source configurations in any order, but Consul stores and evaluates them in order of reverse precedence at runtime. Refer to [`Precedence`](#sources-precedence) for additional information. + +#### Values + +- Default: None +- List of objects that contain the following fields: + - `Name` + - `Peer` + - `Namespace` + - `Partition` + - `Action` + - `Permissions` + - `Precedence` + - `Type` + - `Description` + - `LegacyID` + - `LegacyMeta` + - `LegacyCreateTime` + - `LegacyUpdateTime` + +### `Sources[].Name` + +Specifies the name of the source that the intention allows or denies traffic from. If [`Type`](#sources-type) is set to `consul`, then the value refers to the name of a Consul service. The source is not required to be registered into the Consul catalog. + +#### Values + +- Default: None +- This field is required. +- Data type: String + +### `Sources[].Peer` + +Specifies the name of a peered Consul cluster that the intention allows or denies traffic from. Refer to [Cluster peering overview](/consul/docs/connect/cluster-peering) for additional information about peers. + +The `Peer` and `Partition` fields are mutually exclusive. + +#### Values + +- Default: None +- Data type: String + +### `Sources[].Namespace` + +Specifies the traffic source namespace that the intention allows or denies traffic from. + +#### Values + +- Default: If [`Peer`](#sources-peer) is unspecified, defaults to the destination [`Namespace`](#namespace). +- Data type: String + +### `Sources[].Partition` + +Specifies the name of an admin partition that the intention allows or denies traffic from. Refer to [Admin Partitions](/consul/docs/enterprise/admin-partitions) for additional information about partitions. + +The `Peer` and `Partition` fields are mutually exclusive. + +#### Values + +- Default: If [`Peer`](#sources-peer) is unspecified, defaults to the destination [`Partition`](#partition). +- Data type: string + +### `Sources[].Action` + +Specifies the action to take when the source sends traffic to the destination service. The value is either `allow` or `deny`. Do not configure this field to apply L7 intentions to the same source. Configure the [`Permissions`](#sources-permissions) field instead. + +#### Values + +- Default: None +- This field is required for L4 intentions. +- Data type: String value set to either `allow` or `deny` + +Refer to the following examples for additional guidance: + +- [L4 Intentions for specific sources and destinations](#l4-intentions-for-specific-sources-and-destinations) +- [L4 intentions for all destinations](#l4-intentions-for-all-destinations) +- [L4 intentions for all sources](#l4-intentions-for-all-sources) +- [L4 and L7](#l4-and-l7) + +### `Sources[].Permissions[]` + +Specifies a list of permissions for L7 traffic sources. The list contains one or more actions and a set of match criteria for each action. + +Consul applies permissions in the order specified in the configuration. Beginning at the top of the list, Consul applies the first matching request and stops evaluating against the remaining configurations. + +For requests that do not match any of the defined permissions, Consul applies the intention behavior defined in the [`acl_default_policy`](/consul/docs/agent/config/config-files#acl_default_policy) configuration. + +Do not configure this field for L4 intentions. Use the [`Sources.Action`](#sources-action) parameter instead. + +The `Permissions` only applies to services with a compatible protocol. `Permissions` are not supported when the [`Name`](#name) or [`Namespace`](#namespace) field is configured with a wildcard because service instances or services in a namespace may use different protocols. + +#### Values + +- Default: None +- List of objects that contain the following fields: + - `Action` + - `HTTP` + +Refer to the following examples for additional guidance: + +- [Rest access](#rest-access) +- [gRPC](#grpc) +- [Cluster peering](#cluster-peering) +- [L4 and L7](#l4-and-l7) + +### `Sources[].Permissions[].Action` + +Specifies the action to take when the source sends traffic to the destination service. The value is either `allow` or `deny`. + +#### Values + +- Default: None +- This field is required. +- Data type: String value set to either `allow` or `deny`. + +### `Sources[].Permissions[].HTTP` + +Specifies a set of HTTP-specific match criteria. Consul applies the action defined in the [`Action`](#sources-permissions-action) field to source traffic that matches the criteria. + +#### Values + +- Default: None +- This field is required. +- Data type: Map + +The following table describes the parameters that the HTTP map may contain: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `PathExact` | Specifies an exact path to match on the HTTP request path. Do not specify `PathExact` if `PathPrefix` or `PathRegex` are configured in the same `HTTP` configuration. | string | none | +| `PathPrefix` | Specifies a path prefix to match on the HTTP request path. Do not specify `PathPrefix` if `PathExact` or `PathRegex` are configured in the same `HTTP` configuration. | string | none | +| `PathRegex` | Defines a regular expression to match on the HTTP request path. Do not specify `PathRegex` if `PathExact` or `PathPrefix` are configured in the same `HTTP` configuration. The regex syntax is proxy-specific. If using Envoy, refer to the [re2 documentation](https://github.com/google/re2/wiki/Syntax) for details. | string | none | +| `Methods` | Specifies a list of HTTP methods. Consul applies the permission if a request matches the `PathExact`, `PathPrefix`, `PathRegex`, or `Header`, and the source sent the request using one of the specified methods. Refer to the [Mozilla documentation](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) for a list of supported request headers. | list | All request methods | +| `Header` | Specifies a header name and matching criteria for HTTP request headers. Refer to [`Sources[].Permissions[].HTTP[].Header`](#sources-permissions-http-header) for details. | list of maps | none | + +### `Sources[].Permissions[].HTTP[].Header[]` + +Specifies a header name and matching criteria for HTTP request headers. The request header must match all specified criteria for the permission to apply. + +#### Values + +- Default: None +- Data type: list of objects + +Each member of the `Header` list is a map that contains a `Name` field and at least one match criterion. The following table describes the parameters that each member of the `Header` list may contain: + +| Parameter | Description | Data type | Required | +| --- | --- | --- | --- | +| `Name` | Specifies the name of the header to match. | string | required | +| `Present` | Enables a match if the header configured in the `Name` field appears in the request. Consul matches on any value as long as the header key appears in the request. Do not specify `Present` if `Exact`, `Prefix`, `Suffix`, or `Regex` are configured in the same `Header` configuration. | boolean | optional | +| `Exact` | Specifies a value for the header key set in the `Name` field. If the request header value matches the `Exact` value, Consul applies the permission. Do not specify `Exact` if `Present`, `Prefix`, `Suffix`, or `Regex` are configured in the same `Header` configuration. | string | optional | +| `Prefix` | Specifies a prefix value for the header key set in the `Name` field. If the request header value starts with the `Prefix` value, Consul applies the permission. Do not specify `Prefix` if `Present`, `Exact`, `Suffix`, or `Regex` are configured in the same `Header` configuration. | string | optional | +| `Suffix` | Specifies a suffix value for the header key set in the `Name` field. If the request header value ends with the `Suffix` value, Consul applies the permission. Do not specify `Suffix` if `Present`, `Exact`, `Prefix`, or `Regex` are configured in the same `Header` configuration. | string | optional | +| `Regex` | Specifies a regular expression pattern as the value for the header key set in the `Name` field. If the request header value matches the regex, Consul applies the permission. Do not specify `Regex` if `Present`, `Exact`, `Prefix`, or `Suffix` are configured in the same `Header` configuration. The regex syntax is proxy-specific. If using Envoy, refer to the [re2 documentation](https://github.com/google/re2/wiki/Syntax) for details. | string | optional | +| `Invert` | Inverts the matching logic configured in the `Header`. Default is `false`. | boolean | optional | + + +### `Sources[].Precedence` + +The `Precedence` field contains a read-only integer. Consul generates the value based on name configurations for the source and destination services. Refer to [Precedence and matching order](/consul/docs/connect/intentions/create-manage-intentions#precedence-and-matching-order) for additional information. + +### `Sources[].Type` + +Specifies the type of destination service that the configuration entry applies to. The only value supported is `consul`. + +#### Values + +- Default: `consul` +- Data type: String + +### `Sources[].Description` + +Specifies a description of the intention. Consul presents the description in API responses to assist other tools integrated into the network. + +#### Values + +- Default: None +- Data type: String + +### `Sources[].LegacyID` + +Read-only unique user ID (UUID) for the intention in the system. Consul generates the value and exposes it in the configuration entry so that legacy API endpoints continue to function. Refer to [Read Specific Intention by ID](/consul/api-docs/connect/intentions#read-specific-intention-by-id) for additional information. + +### `Sources[].LegacyMeta` + +Read-only set of arbitrary key-value pairs to attach to the intention. Consul generates the metadata and exposes it in the configuration entry so that legacy intention API endpoints continue to function. Refer to [Read Specific Intention by ID](/consul/api-docs/connect/intentions#read-specific-intention-by-id) for additional information. + +### `Sources[].CreateTime` + +Read-only timestamp for the intention creation. Consul exposes the timestamp in the configuration entry to allow legacy intention API endpoints to continue functioning. Refer to [Read Specific Intention by ID](/consul/api-docs/connect/intentions#read-specific-intention-by-id) for additional information. + +### `Sources[].LegacyUpdateTime` + +Read-only timestamp marking the most recent intention update. Consul exposes the timestamp in the configuration entry to allow legacy intention API endpoints to continue functioning. Refer to [Read Specific Intention by ID](/consul/api-docs/connect/intentions#read-specific-intention-by-id) for additional information. + + + + + +### `apiVersion` + +Specifies the version of the Consul API for integrating with Kubernetes. The value must be `consul.hashicorp.com/v1alpha1`. + +#### Values + +- Default: None +- This field is required. +- String value that must be set to `consul.hashicorp.com/v1alpha1`. + +### `kind` + +Specifies the type of configuration entry to implement. Must be set to `ServiceIntentions`. + +#### Values + +- Default: None +- This field is required. +- Data type: String value that must be set to `ServiceIntentions`. + +### `metadata` + +Map that contains an arbitrary name for the configuration entry and the namespace it applies to. + +#### Values + +- Default: None +- Data type: Map + +### `metadata.name` + +Specifies an arbitrary name for the configuration entry. Note that in other configuration entries, the `metadata.name` field specifies the name of the service that the settings apply to. For service intentions, the service that accepts the configurations is the _destination_ and is specified in the [`spec.destination.name`](#spec-destination-name) field. Refer to the following topics for additional information: + +- [ServiceIntentions Special Case (OSS)](/consul/docs/k8s/crds#serviceintentions-special-case) +- [ServiceIntentions Special Case (Enterprise)](/consul/docs/k8s/crds#serviceintentions-special-case-enterprise) + +#### Values + +- Default: None +- Data type: String + +### `metadata.namespace` + +Specifies the [namespace](/consul/docs/enterprise/namespaces) that the configuration entry applies to. Refer to [Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for information about how Consul namespaces map to Kubernetes Namespaces. Open source Consul distributions (Consul OSS) ignore the `metadata.namespace` configuration. + +#### Values + +- Default: `default` +- Data type: String + +### `spec` +Map that contains the details about the `ServiceIntentions` configuration entry. The `apiVersion`, `kind`, and `metadata` fields are siblings of the spec field. All other configurations are children. + +#### Values + +- Default: None +- This field is required. +- Data type: Map + +### `spec.destination` + +Map that identifies the destination name and destination namespace that source services are allowed or denied access to. + +#### Values + +- Default: None +- This field is required. +- Data type: Map + +### `spec.destination.name` + +Specifies the name of the destination service in the mesh that the intentions apply to. +You can also specify a wildcard character (`*`) to match all services that are missing intention settings. Intentions that are applied with a wildcard, however, are not supported when defining L7 [`permissions`](#spec-sources-permissions). + +#### Values + +- Default: None +- This field is required. +- Data type: String + +### `spec.metadata.namespace` + +Specifies the [namespace](/consul/docs/enterprise/namespaces) that the configuration entry applies to. You can also specify a wildcard character (`*`) to match all namespaces. Intentions that are applied with a wildcard, however, are not supported when defining L7 [`permissions`](#spec-sources-permissions). + +Refer to [Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for information about how Consul namespaces map to Kubernetes Namespaces. Open source Consul distributions (Consul OSS) ignore the `metadata.namespace` configuration. + +#### Values + +- Default: If not set, destination service namespace is inherited from the `connectInject.consulNamespaces` configuration. Refer to [ServiceIntentions Special Case (Enterprise)](/consul/docs/k8s/crds#serviceintentions-special-case-enterprise) for details. +- Data type: String + +### `spec.sources[]` + +List of configurations that define intention sources and the authorization granted to the sources. You can specify source configurations in any order, but Consul stores and evaluates them in order of reverse precedence at runtime. + +#### Values + +- Default: None +- List of objects that contain the following fields: + - `name` + - `peer` + - `namespace` + - `partition` + - `Action` + - `permissions` + - `type` + - `description` + +### `spec.sources[].name` + +Specifies the name of the source that the intention allows or denies traffic from. If [`type`](#sources-type) is set to `consul`, then the value refers to the name of a Consul service. The source is not required to be registered into the Consul catalog. + +#### Values + +- Default: None +- This field is required. +- Data type: String + +### `spec.sources[].peer` + +Specifies the name of a peered Consul cluster that the intention allows or denies traffic from. Refer to [Cluster peering overview](/consul/docs/connect/cluster-peering) for additional information about peers. The `peer` and `partition` fields are mutually exclusive. +#### Values + +- Default: None +- Data type: String + +### `spec.sources[].namespace` + +Specifies the traffic source namespace that the intention allows or denies traffic from. + +#### Values + +- Default: If [`peer`](#spec-sources-peer) is unspecified, defaults to the namespace specified in the [`spec.destination.namespace`](#spec-destination-namespace) field. +- Data type: String + +### `spec.sources[].partition` + +Specifies the name of an admin partition that the intention allows or denies traffic from. Refer to [Admin Partitions](/consul/docs/enterprise/admin-partitions) for additional information about partitions. The `peer` and `partition` fields are mutually exclusive. + +#### Values + +- Default: If [`peer`](#sources-peer) is unspecified, defaults to the partition specified in [`spec.destination.partition`](#spec-destination-partition). +- Data type: String + +### `spec.sources[].action` + +Specifies the action to take when the source sends traffic to the destination service. The value is either `allow` or `deny`. Do not configure this field for L7 intentions. Configure the [`spec.sources.permissions`](#spec-sources-permissions) field instead. + +#### Values + +- Default: None +- This field is required for L4 intentions. +- Data type: String value set to either `allow` or `deny` + +### `spec.sources[].permissions[]` + +Specifies a list of permissions for L7 traffic sources. The list contains one or more actions and a set of match criteria for each action. + +Consul applies permissions in the order specified in the configuration. Starting at the beginning of the list, Consul applies the first matching request and stops evaluating against the remaining configurations. + +For requests that do not match any of the defined permissions, Consul applies the intention behavior defined in the [`acl_default_policy`](/consul/docs/agent/config/config-files#acl_default_policy) configuration. + +Do not configure this field for L4 intentions. Use the [`spec.sources.action`](#sources-action) parameter instead. + +`permissions` configurations only apply to services with a compatible protocol. As a result, they are not supported when the [`spec.destination.name`](#spec-destination-name) or [`spec.destination.namespace`](#spec-destination-namespace) field is configured with a wildcard because service instances or services in a namespace may use different protocols. + +#### Values + +- Default: None +- List of objects that contain the following fields: + - `action` + - `http` + +### `spec.sources[].permissions[].action` + +Specifies the action to take when the source sends traffic to the destination service. The value is either `allow` or `deny`. + +#### Values + +- Default: None +- This field is required. +- Data type: String value set to either `allow` or `deny` + +### `spec.sources[].permissions[].http` + +Specifies a set of HTTP-specific match criteria. Consul applies the action defined in the [`spec.sources.permissions.action`](#spec-sources-permissions-action) field to source traffic that matches the criteria. + +#### Values + +- Default: None +- This field is required. +- Data type: Map + +The following table describes the parameters that the HTTP map may contain: + +| Parameter | Description | Data type | Default | +| --- | --- | --- | --- | +| `pathExact` | Specifies an exact path to match on the HTTP request path. Do not specify `pathExact` if `pathPrefix` or `pathRegex` are configured in the same `http` configuration. | string | none | +| `pathPrefix` | Specifies a path prefix to match on the HTTP request path. Do not specify `pathPrefix` if `pathExact` or `pathRegex` are configured in the same `http` configuration. | string | none | +| `pathRegex` | Defines a regular expression to match on the HTTP request path. Do not specify `pathRegex` if `pathExact` or `pathPrefix` are configured in the same `http` configuration. The regex syntax is proxy-specific. If using Envoy, refer to the [re2 documentation](https://github.com/google/re2/wiki/Syntax) for details. | string | none | +| `methods` | Specifies a list of HTTP methods. Consul applies the permission if a request matches the `pathExact`, `pathPrefix`, `pathRegex`, or `header`, and the source sent the request using one of the specified methods. Refer to the [Mozilla documentation](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) for a list of supported request headers. | list | All request methods | +| `header` | Specifies a header name and matching criteria for HTTP request headers. Refer to [`spec.sources[].permissions[].http[].header`](#spec-sources-permissions-http-header) for details. | list of maps | none | + +### `spec.sources[].permissions[].http[].header` + +Specifies a set of criteria for matching HTTP request headers. The request header must match all specified criteria for the permission to apply. + +#### Values + +- Default: None +- Data type: List of maps + +Each member of the `header` list is a map that contains a `name` field and at least one match criterion. The following table describes the parameters that each member of the `header` list may contain: + +| Parameter | Description | Data type | Required | +| --- | --- | --- | --- | +| `name` | Specifies the name of the header to match. | string | required | +| `present` | Enables a match if the header configured in the `name` field appears in the request. Consul matches on any value as long as the header key appears in the request. Do not specify `present` if `exact`, `prefix`, `suffix`, or `regex` are configured in the same `header` configuration. | boolean | optional | +| `Exact` | Specifies a value for the header key set in the `Name` field. If the request header value matches the `exact` value, Consul applies the permission. Do not specify `exact` if `present`, `prefix`, `suffix`, or `regex` are configured in the same `header` configuration. | string | optional | +| `prefix` | Specifies a prefix value for the header key set in the `name` field. If the request header value starts with the `prefix` value, Consul applies the permission. Do not specify `prefix` if `present`, `exact`, `suffix`, or `regex` are configured in the same `header` configuration. | string | optional | +| `suffix` | Specifies a suffix value for the header key set in the `name` field. If the request header value ends with the `suffix` value, Consul applies the permission. Do not specify `suffix` if `present`, `exact`, `prefix`, or `regex` are configured in the same `header` configuration. | string | optional | +| `regex` | Specifies a regular expression pattern as the value for the header key set in the `name` field. If the request header value matches the regex, Consul applies the permission. Do not specify `regex` if `present`, `exact`, `prefix`, or `suffix` are configured in the same `header` configuration. The regex syntax is proxy-specific. If using Envoy, refer to the [re2 documentation](https://github.com/google/re2/wiki/Syntax) for details. | string | optional | +| `invert` | Inverts the matching logic configured in the `header`. Default is `false`. | boolean | optional | + +### `spec.sources[].type` + +Specifies the type of destination service that the configuration entry applies to. The only value supported is `consul`. + +#### Values + +- Default: `consul` +- Data type: String + +### `spec.sources[].description` + +Specifies a description of the intention. Consul presents the description in API responses to assist other tools integrated into the network. + +#### Values + +- Default: None +- Data type: String + + + + + + +## Examples + +The following examples demonstrate potential use-cases for the service intentions configuration entry. + +### L4 Intentions for specific sources and destinations + +The following example configuration entry specifies an L4 intention that denies traffic from `web` to `db` service instances, but allows traffic from `api` to `db`. + + + +```hcl +Kind = "service-intentions" +Name = "db" +Sources = [ + { + Name = "web" + Action = "deny" + }, + { + Name = "api" + Action = "allow" + } +] +``` + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceIntentions +metadata: + name: db +spec: + destination: + name: db + sources: + - name: web + action: deny + - name: api + action: allow +``` + +```json +{ + "Kind": "service-intentions", + "Name": "db", + "Sources": [ + { + "Action": "deny", + "Name": "web" + }, + { + "Action": "allow", + "Name": "api" + } + ] +} +``` + + + +### L4 intentions for all destinations + +In the following L4 example, the destination is configured with a `*` wildcard. As a result, traffic from `web` service instances is denied for any service in the datacenter. + + + +```hcl +Kind = "service-intentions" +Name = "*" +Sources = [ + { + Name = "web" + Action = "deny" + } +] +``` + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceIntentions +metadata: + name: web-deny-all +spec: + destination: + name: * + sources: + - name: web + action: deny +``` + +```json +{ + "Kind": "service-intentions", + "Name": "*", + "Sources": [ + { + "Action": "deny", + "Name": "web" + } + ] +} +``` + + + +### L4 intentions for all sources + +In the following L4 example, the source is configured with a `*` wildcard. As a result, traffic from any service is denied to `db` service instances. + + + +```hcl +Kind = "service-intentions" +Name = "db" +Sources = [ + { + Name = "*" + Action = "deny" + } +] +``` + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceIntentions +metadata: + name: db +spec: + destination: + name: db + sources: + - name: * + action: deny +``` + +```json +{ + "Kind": "service-intentions", + "Name": "db", + "Sources": [ + { + "Action": "deny", + "Name": "*" + } + ] +} +``` + + +### REST access In the following example, the `admin-dashboard` and `report-generator` services have different levels of access when making REST calls: @@ -134,8 +1036,8 @@ spec: ### gRPC -In the following use-case, access to the `IssueRefund` gRPC service method is set to `deny`. Because gRPC method calls [are -HTTP/2](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md), an HTTP path-matching rule can be used to control traffic: +In the following example, Consul denies requests from `frontend-web` to the `IssueRefund` gRPC service. +Because gRPC method calls use the [HTTP/2 protocol](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md), you can apply an HTTP path-matching rule to control traffic: @@ -174,8 +1076,8 @@ Sources = [ } ] } - # NOTE: a default catch-all based on the default ACL policy will apply to - # unmatched connections and requests. Typically this will be DENY. + # A default catch-all based on the default ACL policy applies to + # unmatched connections and requests. This is typically DENY. ] ``` @@ -204,8 +1106,8 @@ spec: - action: allow http: pathPrefix: '/mycompany.BillingService/' - # NOTE: a default catch-all based on the default ACL policy will apply to - # unmatched connections and requests. Typically this will be DENY. + # A default catch-all based on the default ACL policy applies to + # unmatched connections and requests. This is typically DENY. ``` ```json @@ -249,7 +1151,7 @@ spec: ### L4 and L7 -You can mix and match L4 and L7 intentions per source: +In the following example, Consul enforces application layer intentions that deny requests to `api` from `hackathon-project` but allow requests from `web`. In the same configuration entry, Consul enforces network layer intentions that allow requests from `nightly-reconciler` that send `POST` requests to the `/v1/reconcile-data` HTTP endpoint: @@ -277,8 +1179,8 @@ Sources = [ } ] }, - # NOTE: a default catch-all based on the default ACL policy will apply to - # unmatched connections and requests. Typically this will be DENY. + # A default catch-all based on the default ACL policy applies to + # unmatched connections and requests. This is typically DENY. ] ``` @@ -301,8 +1203,8 @@ spec: http: pathExact: /v1/reconcile-data methods: ['POST'] - # NOTE: a default catch-all based on the default ACL policy will apply to - # unmatched connections and requests. Typically this will be DENY. + # A default catch-all based on the default ACL policy applies to + # unmatched connections and requests. This is typically DENY. ``` ```json @@ -335,404 +1237,53 @@ spec: ``` -## Available Fields +### Cluster peering -', - yaml: false, - }, - { - name: 'Namespace', - type: `string: "default"`, - enterprise: true, - description: - "Specifies the namespaces the config entry will apply to. This may be set to the wildcard character (`*`) to match all services in all namespaces that don't otherwise have intentions defined. Wildcard intentions cannot be used when defining L7 [`Permissions`](/consul/docs/connect/config-entries/service-intentions#permissions).", - yaml: false, - }, - { - name: 'Partition', - type: `string: "default"`, - enterprise: true, - description: - "Specifies the admin partition on which the configuration entry will apply. Wildcard characters (`*`) are not supported. Admin partitions must specified explicitly.", - yaml: false, - }, - { - name: 'Meta', - type: 'map: nil', - description: 'Specifies arbitrary KV metadata pairs.', - yaml: false, - }, - { - name: 'metadata', - children: [ - { - name: 'name', - description: - 'Unlike other config entries, the `metadata.name` field is not used to set the name of the service being configured. Instead, that is set in `spec.destination.name`. Thus this name can be set to anything. See [ServiceIntentions Special Case (OSS)](/consul/docs/k8s/crds#serviceintentions-special-case) or [ServiceIntentions Special Case (Enterprise)](/consul/docs/k8s/crds#serviceintentions-special-case-enterprise) for more details.', - }, - { - name: 'namespace', - description: - 'If running Consul Open Source, the namespace is ignored (see [Kubernetes Namespaces in Consul OSS](/consul/docs/k8s/crds#consul-oss)). If running Consul Enterprise see [Kubernetes Namespaces in Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for more details.', - }, - ], - hcl: false, - }, - { - name: 'destination', - hcl: false, - children: [ - { - name: 'name', - hcl: false, - type: 'string: ', - description: - "The name of the destination service for all intentions defined in this config entry. This may be set to the wildcard character (`*`) to match all services that don't otherwise have intentions defined. Wildcard intentions cannot be used when defining L7 [`Permissions`](/consul/docs/connect/config-entries/service-intentions#permissions).", - }, - { - name: 'namespace', - hcl: false, - enterprise: true, - type: 'string: ', - description: - "Specifies the namespaces the config entry will apply to. This may be set to the wildcard character (`*`) to match all services in all namespaces that don't otherwise have intentions defined. If not set, the namespace used will depend on the `connectInject.consulNamespaces` configuration. See [ServiceIntentions Special Case (Enterprise)](/consul/docs/k8s/crds#serviceintentions-special-case-enterprise) for more details. Wildcard intentions cannot be used when defining L7 [`Permissions`](/consul/docs/connect/config-entries/service-intentions#permissions).", - }, - ], - }, - { - name: 'Sources', - type: 'array', - description: `The list of all [intention sources and the authorization granted to those sources](#sourceintention). The order of - this list does not matter, but out of convenience Consul will always store - this reverse sorted by intention precedence, as that is the order that they - will be evaluated at enforcement time.`, - }, - ]} -/> +When using cluster peering connections, intentions secure your deployments with authorized service-to-service communication between remote datacenters. In the following example, the service intentions configuration entry authorizes the `backend-service` to communicate with the `frontend-service` that is hosted on remote peer `cluster-02`: -### `SourceIntention` - -', - description: { - hcl: - "The source of the intention. For a `Type` of `consul` this is the name of a Consul service. The service doesn't need to be registered.", - yaml: - "The source of the intention. For a `type` of `consul` this is the name of a Consul service. The service doesn't need to be registered.", - }, - }, - { - name: 'Peer', - type: 'string: ""', - description: { - hcl: - "Specifies the [peer](/consul/docs/connect/cluster-peering/index.mdx) of the source service. `Peer` is mutually exclusive with `Partition`.", - yaml: - "Specifies the [peer](/consul/docs/connect/cluster-peering/index.mdx) of the source service. `peer` is mutually exclusive with `partition`.", - }, - }, - { - name: 'Namespace', - enterprise: true, - type: 'string: ""', - description: { - hcl: - "The namespace of the source service. If `Peer` is empty, `Namespace` defaults to the namespace of the destination service (i.e. the config entry's namespace).", - yaml: - 'The namespace of the source service. If `peer` is empty, `namespace` defaults to the namespace of the destination service (i.e. `spec.destination.namespace`).', - }, - }, - { - name: 'Partition', - enterprise: true, - type: 'string: ""', - description: { - hcl: - "Specifies the admin partition of the source service. If `Peer` is empty, `Partition` defaults to the destination service's partition (i.e. the configuration entry's partition). `Partition` is mutually exclusive with `Peer`.", - yaml: - "Specifies the admin partition of the source service. If `peer` is empty, `partition` defaults to the destination service's partition (i.e. `spec.destination.partition`). `partition` is mutually exclusive with `peer`.", - }, - }, - { - name: 'Action', - type: 'string: ""', - description: { - hcl: - 'For an L4 intention this is required, and should be set to one of `"allow"` or `"deny"` for the action that should be taken if this intention matches a request.' + - '

    This should be omitted for an L7 intention as it is mutually exclusive with the `Permissions` field.', - yaml: - 'For an L4 intention this is required, and should be set to one of `"allow"` or `"deny"` for the action that should be taken if this intention matches a request.' + - '

    This should be omitted for an L7 intention as it is mutually exclusive with the `permissions` field.', - }, - }, - { - name: 'Permissions', - type: 'array', - description: { - hcl: `The list of all [additional L7 attributes](#intentionpermission) that extend the intention match criteria.

    - Permission precedence is applied top to bottom. For any given request the - first permission to match in the list is terminal and stops further - evaluation. As with L4 intentions, traffic that fails to match any of the - provided permissions in this intention will be subject to the default - intention behavior is defined by the default [ACL policy](/consul/docs/agent/config/config-files#acl_default_policy).

    - This should be omitted for an L4 intention as it is mutually exclusive with - the \`Action\` field.

    - Setting \`Permissions\` is not valid if a wildcard is used for the \`Name\` or \`Namespace\` because they can only be - applied to services with a compatible protocol.`, - yaml: `The list of all [additional L7 attributes](#intentionpermission) that extend the intention match criteria.

    - Permission precedence is applied top to bottom. For any given request the - first permission to match in the list is terminal and stops further - evaluation. As with L4 intentions, traffic that fails to match any of the - provided permissions in this intention will be subject to the default - intention behavior is defined by the default [ACL policy](/consul/docs/agent/config/config-files#acl_default_policy).

    - This should be omitted for an L4 intention as it is mutually exclusive with - the \`action\` field.

    - Setting \`permissions\` is not valid if a wildcard is used for the \`spec.destination.name\` or \`spec.destination.namespace\` - because they can only be applied to services with a compatible protocol.`, - }, - }, - { - name: 'Precedence', - type: 'int: ', - description: - 'An [integer precedence value](/consul/docs/connect/intentions#precedence-and-match-order) computed from the source and destination naming components.', - yaml: false, - }, - { - name: 'Type', - type: 'string: "consul"', - description: { - hcl: - 'The type for the `Name` value. This can be only "consul" today to represent a Consul service. If not provided, this will be defaulted to "consul".', - yaml: - 'The type for the `name` value. This can be only "consul" today to represent a Consul service. If not provided, this will be defaulted to "consul".', - }, - }, - { - name: 'Description', - type: 'string: ""', - description: - 'Description for the intention. This is not used by Consul, but is presented in API responses to assist tooling.', - }, - { - name: 'LegacyID', - type: 'string: ', - description: `This is the UUID to uniquely identify - this intention in the system. Cannot be set directly and is exposed here as - an artifact of the config entry migration and is primarily used to allow - legacy intention [API](/consul/api-docs/connect/intentions#update-intention-by-id) - [endpoints](/consul/api-docs/connect/intentions#read-specific-intention-by-id) to - continue to function for a period of time after [upgrading to 1.9.0](/consul/docs/upgrading/upgrade-specific#consul-1-9-0).`, - yaml: false, - }, - { - name: 'LegacyMeta', - type: 'map: ', - description: `Specified arbitrary KV - metadata pairs attached to the intention, rather than to the enclosing config - entry. Cannot be set directly and is exposed here as an artifact of the - config entry migration and is primarily used to allow legacy intention - [API](/consul/api-docs/connect/intentions#update-intention-by-id) - [endpoints](/consul/api-docs/connect/intentions#read-specific-intention-by-id) to - continue to function for a period of time after [upgrading to 1.9.0](/consul/docs/upgrading/upgrade-specific#consul-1-9-0).`, - yaml: false, - }, - { - name: 'LegacyCreateTime', - type: 'time: optional', - description: `The timestamp that this intention was - created. Cannot be set directly and is exposed here as an artifact of the - config entry migration and is primarily used to allow legacy intention - [API](/consul/api-docs/connect/intentions#update-intention-by-id) - [endpoints](/consul/api-docs/connect/intentions#read-specific-intention-by-id) to - continue to function for a period of time after [upgrading to 1.9.0](/consul/docs/upgrading/upgrade-specific#consul-1-9-0).`, - yaml: false, - }, - { - name: 'LegacyUpdateTime', - type: 'time: optional', - description: `The timestamp that this intention was - last updated. Cannot be set directly and is exposed here as an artifact of the - config entry migration and is primarily used to allow legacy intention - [API](/consul/api-docs/connect/intentions#update-intention-by-id) - [endpoints](/consul/api-docs/connect/intentions#read-specific-intention-by-id) to - continue to function for a period of time after [upgrading to 1.9.0](/consul/docs/upgrading/upgrade-specific#consul-1-9-0).`, - yaml: false, - }, - ]} -/> + -### `IntentionPermission` + ```hcl + Kind = "service-intentions" + Name = "backend-service" -', - description: - 'This is one of "allow" or "deny" for the action that should be taken if this permission matches a request.', - }, - { - name: 'HTTP', - type: 'IntentionHTTPPermission: ', - description: - 'A set of [HTTP-specific authorization criteria](#intentionhttppermission)', - }, - ]} -/> + Name = "frontend-service" + Peer = "cluster-02" + Action = "allow" + } + ] + ``` -### `IntentionHTTPPermission` + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceIntentions + metadata: + name: backend-deny + spec: + destination: + name: backend + sources: + - name: "*" + action: deny + - name: frontend + action: allow + peer: cluster-01 ## The peer of the source service + ``` -
    At most only one of `PathExact`, `PathPrefix`, or `PathRegex` may be configured.', - yaml: - 'Exact path to match on the HTTP request path.

    At most only one of `pathExact`, `pathPrefix`, or `pathRegex` may be configured.', - }, - }, - { - name: 'PathPrefix', - type: 'string: ""', - description: { - hcl: - 'Path prefix to match on the HTTP request path.

    At most only one of `PathExact`, `PathPrefix`, or `PathRegex` may be configured.', - yaml: - 'Path prefix to match on the HTTP request path.

    At most only one of `pathExact`, `pathPrefix`, or `pathRegex` may be configured.', - }, - }, - { - name: 'PathRegex', - type: 'string: ""', - description: { - hcl: - 'Regular expression to match on the HTTP request path.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `PathExact`, `PathPrefix`, or `PathRegex` may be configured.', - yaml: - 'Regular expression to match on the HTTP request path.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `pathExact`, `pathPrefix`, or `pathRegex` may be configured.', - }, - }, - { - name: 'Methods', - type: 'array', - description: - 'A list of HTTP methods for which this match applies. If unspecified all HTTP methods are matched. If provided the names must be a valid [method](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods).', - }, + ```json + { + "Kind": "service-intentions", + "Name": "backend-service", + "Sources": [ { - name: 'Header', - type: 'array', - description: - 'A set of criteria that can match on HTTP request headers. If more than one is configured all must match for the overall match to apply.', - children: [ - { - name: 'Name', - type: 'string: ', - description: 'Name of the header to match', - }, - { - name: 'Present', - type: 'bool: false', - description: { - hcl: - 'Match if the header with the given name is present with any value.

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name is present with any value.

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Exact', - type: 'string: ""', - description: { - hcl: - 'Match if the header with the given name is this value.

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name is this value.

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Prefix', - type: 'string: ""', - description: { - hcl: - 'Match if the header with the given name has this prefix.

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name has this prefix.

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Suffix', - type: 'string: ""', - description: { - hcl: - 'Match if the header with the given name has this suffix.

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name has this suffix.

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Regex', - type: 'string: ""', - description: { - hcl: - 'Match if the header with the given name matches this pattern.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name matches this pattern.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Invert', - type: 'bool: false', - description: 'Inverts the logic of the match', - }, - ], - }, - ]} -/> - -## ACLs - -Configuration entries may be protected by [ACLs](/consul/docs/security/acl). - -Reading a `service-intentions` config entry requires `intentions:read` on the resource. - -Creating, updating, or deleting a `service-intentions` config entry requires -`intentions:write` on the resource. - -Intention ACL rules are specified as part of a `service` rule. See [Intention -Management Permissions](/consul/docs/connect/intentions#intention-management-permissions) for -more details. - -## Regular Expression Syntax - -The actual syntax of the regular expression fields described here is entirely -proxy-specific. + "Name": "frontend-service", + "Peer": "cluster-02", + "Action": "allow" + } + ] + } + ``` -When using [Envoy](/consul/docs/connect/proxies/envoy) as a proxy, the syntax for -these fields is [RE2](https://github.com/google/re2/wiki/Syntax). +     diff --git a/website/content/docs/connect/config-entries/service-resolver.mdx b/website/content/docs/connect/config-entries/service-resolver.mdx index 9730beb132e..dcea5980549 100644 --- a/website/content/docs/connect/config-entries/service-resolver.mdx +++ b/website/content/docs/connect/config-entries/service-resolver.mdx @@ -1,40 +1,1247 @@ --- layout: docs -page_title: Service Resolver - Configuration Entry Reference -description: >- - The service resolver configuration entry kind defines subsets of service instances that satisfy upstream discovery requests. Use the reference guide to learn about `""service-resolver""` config entry parameters and how filtering by service subsets helps route traffic based on properties like version number. +page_title: Service Resolver Configuration Entry Reference +description: >- + Service resolver configuration entries are L7 traffic management tools for defining sets of service instances that resolve upstream requests and Consul’s behavior when resolving them. Learn how to write `service-resolver` config entries in HCL or YAML with a specification reference, configuration model, a complete example, and example code by use case. --- -# Service Resolver Configuration Entry +# Service resolver configuration entry reference + +This page provides reference information for service resolver configuration entries. Configure and apply service resolvers to create named subsets of service instances and define their behavior when satisfying upstream requests. + +Refer to [L7 traffic management overview](/consul/docs/connect/l7-traffic) for additional information. + +## Configuration model + +The following list outlines field hierarchy, language-specific data types, and requirements in the configuration entry. Click on a property name to view additional details, including default values. + + + + + +- [`Kind`](#kind): string | required | must be set to `service-resolver` +- [`Name`](#name): string | required +- [`Namespace`](#namespace): string | `default` +- [`Partition`](#partition): string | `default` +- [`Meta`](#meta): map +- [`ConnectTimeout`](#connecttimeout): string | `0s` +- [`RequestTimeout`](#requesttimeout): string | `15s` +- [`Subsets`](#subsets): map + - [`Filter`](#subsets): string + - [`OnlyPassing`](#subsets): boolean | `false` +- [`DefaultSubset`](#defaultsubset): string +- [`Redirect`](#redirect): map + - [`Service`](#redirect-service): string + - [`ServiceSubset`](#redirect-servicesubset): string + - [`Namespace`](#redirect-namespace): string + - [`Partition`](#redirect-partition): string | `default` + - [`Datacenter`](#redirect-datacenter): list + - [`Peer`](#redirect-peer): string +- [`Failover`](#failover): map + - [`Service`](#failover-service): string + - [`ServiceSubset`](#failover-servicesubset): string + - [`Namespace`](#failover-namespace): string + - [`Datacenters`](#failover-datacenters): list + - [`Targets`](#failover-targets): list + - [`Service`](#failover-targets-service): string + - [`ServiceSubset`](#failover-targets-servicesubset): string + - [`Namespace`](#failover-targets-namespace): string + - [`Partition`](#failover-targets-partition): string | `default` + - [`Datacenter`](#failover-targets-datacenter): string + - [`Peer`](#failover-targets-peer): string +- [`LoadBalancer`](#loadbalancer): map + - [`Policy`](#loadbalancer-policy): string + - [`LeastRequestConfig`](#loadbalancer-leastrequestconfig): map + - [`ChoiceCount`](#loadbalancer-leastrequestconfig-choicecount): integer | `2` + - [`RingHashConfig`](#loadbalancer-ringhashconfig): map + - [`MinimumRingSize`](#loadbalancer-ringhashconfig): integer | `1024` + - [`MaximumRingSize`](#loadbalancer-ringhashconfig): integer | `8192` + - [`HashPolicies`](#loadbalancer-hashpolicies): map + - [`Field`](#loadbalancer-hashpolicies-field): string + - [`FieldValue`](#loadbalancer-hashpolicies-fieldvalue): string + - [`CookieConfig`](#loadbalancer-hashpolicies-cookieconfig): map + - [`Session`](#loadbalancer-hashpolicies-cookieconfig-session): boolean | `false` + - [`TTL`](#loadbalancer-hashpolicies-cookieconfig-ttl): string + - [`Path`](#loadbalancer-hashpolicies-cookieconfig-path): string + - [`SourceIP`](#loadbalancer-hashpolicies-sourceip): boolean | `false` + - [`Terminal`](#loadbalancer-hashpolicies-terminal): boolean | `false` + + + + + +- [`apiVersion`](#apiversion): string | required | must be set to `consul.hashicorp.com/v1alpha1` +- [`kind`](#kind): string | required | must be set to `ServiceResolver` +- [`metadata`](#metadata): map | required + - [`name`](#metadata-name): string | required + - [`namespace`](#metadata-namespace): string | optional +- [`spec`](#spec): map | required + - [`connectTimeout`](#spec-connecttimeout): string | `0s` + - [`requestTimeout`](#spec-requesttimeout): string | `15s` + - [`subsets`](#spec-subsets): map + - [`filter`](#spec-subsets-filter): string + - [`onlyPassing`](#spec-subsets-onlypassing): boolean | `false` + - [`defaultSubset`](#spec-defaultsubset): string + - [`redirect`](#spec-redirect): map + - [`service`](#spec-redirect-service): string + - [`serviceSubset`](#spec-redirect-servicesubset): string + - [`namespace`](#spec-redirect-namespace): string + - [`partition`](#spec-redirect-partition): string + - [`datacenter`](#spec-redirect-datacenter): string + - [`peer`](#spec-redirect-peer): string + - [`failover`](#spec-failover): map + - [`service`](#spec-failover-service): string + - [`serviceSubset`](#spec-failover-servicesubset): string + - [`namespace`](#spec-failover-namespace): string + - [`datacenters`](#spec-failover-datacenters): string + - [`targets`](#spec-failover-targets): list + - [`service`](#spec-failover-targets-service): string + - [`serviceSubset`](#spec-failover-targets-servicesubset): string + - [`namespace`](#spec-failover-targets-namespace): string | `default` + - [`partition`](#spec-failover-targets-partition): string | `default` + - [`datacenter`](#spec-failover-targets-datacenter): string + - [`peer`](#spec-failover-targets-peer): string + - [`loadBalancer`](#spec-loadbalancer): map + - [`policy`](#spec-loadbalancer-policy): string + - [`leastRequestConfig`](#spec-loadbalancer-leastrequestconfig): map + - [`choiceCount`](#spec-loadbalancer-leastrequestconfig): integer | `2` + - [`ringHashConfig`](#spec-loadbalancer-ringhashconfig): map + - [`minimumRingSize`](#spec-loadbalancer-ringhashconfig): integer | `1024` + - [`maximumRingSize`](#spec-loadbalancer-ringhashconfig): integer | `8192` + - [`hashPolicies`](#spec-loadbalancer-hashpolicies): list + - [`field`](#spec-loadbalancer-hashpolicies-field): string + - [`fieldValue`](#spec-loadbalancer-hashpolicies-fieldvalue): string + - [`cookieConfig`](#spec-loadbalancer-hashpolicies-cookieconfig): map + - [`session`](#spec-loadbalancer-hashpolicies-cookieconfig-session): boolean | `false` + - [`ttl`](#spec-loadbalancer-hashpolicies-cookieconfig-ttl): string | `0s` + - [`path`](#spec-loadbalancer-hashpolicies-cookieconfig-path): string + - [`sourceIP`](#spec-loadbalancer-hashpolicies-sourceip): boolean | `false` + - [`terminal`](#spec-loadbalancer-hashpolicies-terminal): boolean | `false` + + + + +## Complete configuration + +When every field is defined, a service resolver configuration entry has the following form: + + + + --> **v1.8.4+:** On Kubernetes, the `ServiceResolver` custom resource is supported in Consul versions 1.8.4+.
    -**v1.6.0+:** On other platforms, this config entry is supported in Consul versions 1.6.0+. +```hcl +Kind = "service-resolver" ## required +Name = "" +Namespace = "" +Partition = "" +Meta = { + = "" +} + +ConnectTimeout = "10s" +RequestTimeout = "15s" +Subsets = { + = { + Filter = "" + OnlyPassing = true + } + = { + Filter = "" + OnlyPassing = true + } +} +DefaultSubset = "" + +Redirect = { + Service = "" + ServiceSubset = "" + Namespace = "" + Partition = "" + Datacenter = "" + Peer = "" +} + +Failover = { ## requires at least one of the following: Service, ServiceSubset, Namespace, Targets, Datacenters + = { + Targets = [ + { Service = "" }, + { ServiceSubset = "" }, + { Namespace = "" }, + { Partition = "" }, + { Datacenter = "" }, + { Peer = "" } + ] + } + "*" = { + Service = "" + ServiceSubset = "" + Namespace = "" + Datacenters = ["", ""] + } +} + +LoadBalancer = { + Policy = "random" + LeastRequestConfig = { ## requires Policy = "least_request" + ChoiceCount = 2 + RingHashConfig = { ## requires Policy = "ring_hash" + MinimumRingSize = 1024 + MaximumRingSize = 8192 + } + } + HashPolicies = [ + { + Field = "header" ## cannot specify with SourceIP + FieldValue = "" ## cannot specify with SourceIP + CookieConfig = { + Session = false + TTL = "0s" + Path = "" + } + SourceIP = false ## cannot specify with Field or FieldValue + Terminal = false + } + ] +} + +``` + +     + + + +```json +{ + "Kind":"service-resolver", // required + "Name":"", + "Namespace":"", + "Partition":"parition-configuration-applies-to>", + "Meta":{ + "":"" + }, + + "ConnectTimeout":"10s", + "RequestTimeout":"15s", + "Subsets":{ + "":{ + "Filter":"":{ + "Filter":"", + "OnlyPassing":true + } + }, + "DefaultSubset ":"", + + "Redirect":{ + "Service":"", + "ServiceSubset":"", + "Namespace":"", + "Partition":"", + "Datacenter":"", + "Peer":"" + }, + + "Failover":{ // requires at least one of the following": Service, ServiceSubset, Namespace, Targets, Datacenters + "":{ + "Targets":[ + {"Service":""}, + {"ServiceSubset":""}, + {"Namespace":""}, + {"Partition":""}, + {"Datacenter":""}, + {"Peer":""} + ] + }, + "*":{ + "Service ":"", + "ServiceSubset":"", + "Namespace":"", + "Datacenters":["", ""] + } + }, + + "LoadBalancer":{ + "Policy":"random", + "LeastRequestConfig":{ // requires Policy":"least_request" + "ChoiceCount":2 + }, + "RingHashConfig":{ // requires Policy":"ring_hash" + "MinimumRingSize":1024, + "MaximumRingSize":8192 + }, + "HashPolicies":[ + { + "Field":"header", // cannot specify with SourceIP + "FieldValue":"", // cannot specify with SourceIP + "CookieConfig":{ + "Session":false, + "TTL":"0s", + "Path":"" + }, + "SourceIP":false, // cannot specify with Field or FieldValue + "Terminal":false + } + ] + } +} +``` + + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 # required +kind: ServiceResolver # required +metadata: + name: + namespace: +spec: + connectTimeout: 10s + requestTimeout: 15s + subsets: + : + filter: + onlyPassing: false + : + filter: + onlyPassing: false + defaultSubset: + redirect: + service: + servicesubset: + namespace: + partition: + peer: + failover: # requires at least one of the following: service, serviceSubset, namespace, targets, datacenters + : + targets: + - service: + - serviceSubset: + - namespace: + - partition: + - datacenter: + - peer: + `*`: + service: + serviceSubset: + namespace: + datacenters: ["", ""] + loadBalancer: + policy: random + leastRequestConfig: # requires policy: leastRequestConfig + choiceCount: 2 + ringHashConfig: # requires policy: ringHashConfig + minimumRingSize: 1024 + maximumRingSize: 8192 + hashPolicies: + - field: header # cannot specify with SourceIP + - fieldValue: # cannot specify with SourceIP + - cookieConfig: + session: false + ttl: 0s + path: + - sourceIP: false # cannot specify with field or fieldValue + - terminal: false +``` + + + +     + +## Specification + +This section provides details about the fields you can configure in the configuration entry. + + + + + +### `Kind` + +Specifies the type of configuration entry to implement. Must be set to `service-resolver`. + +#### Values + +- Default: None +- This field is required. +- Data type: String value that must be set to `service-resolver`. + +### `Name` + +Specifies a name for the configuration entry. The name is metadata that you can use to reference the configuration entry when performing Consul operations, such as applying a configuration entry to a specific cluster. + +#### Values + +- Default: None +- This field is required. +- Data type: String + +### `Namespace` + +Specifies the namespace that the service resolver applies to. Refer to [namespaces](/consul/docs/enterprise/namespaces) for more information. + +#### Values + +- Default: None +- Data type: String + +### `Partition` + +Specifies the admin partition that the service resolver applies to. Refer to [admin partitions](/consul/docs/enterprise/admin-partitions) for more information. + +#### Values + +- Default: `default` +- Data type: String + +### `Meta` + +Specifies key-value pairs to add to the KV store. + +#### Values + +- Default: none +- Data type: Map of one or more key-value pairs + - ``: String + - ``: String or integer + +### `ConnectTimeout` + +Specifies the timeout duration for establishing new network connections to this service. By default, the duration is measured in nanoseconds (ns). + +#### Values + +- Default: None +- Data type: String + +### `RequestTimeout` + +Specifies the timeout duration for receiving an HTTP response from this service. When set to `0s`, the default value of `15s` is used instead. By default, the duration is measured in nanoseconds (ns). + +#### Values + +- Default: `15s` +- Data type: String + +### `Subsets` + +Specifies names for custom service subsets and the conditions under which service instances belong to each subset. Define a subset by specifying a key and a value where the key is the subset’s name and the value is a map that contains a [filtering expression](/consul/api-docs/features/filtering). If this parameter is not specified, only the unnamed default subset is usable. + +For additional guidance, refer to the [filter on service version configuration example](#filter-on-service-version). + +#### Values + +- Default: None +- Data type: Map containing a key-value pair. + - ``: String that names the subset. The string must be valid as a DNS subdomain element. + - ``: Map that can contain the following parameters: + + | Parameter | Description | Data type | Default | + | :------------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------- | + | `Filter` | Specifies an expression that filters the DNS elements of service instances that belong to the subset. If empty, all healthy instances of a service are returned. This expression can filter on the same DNS selectors as the [Health API endpoint](/consul/api-docs/health#filtering-2). For more information about creating and using expressions to filter, refer to [filtering](/consul/api-docs/features/filtering). | String | None | + | `OnlyPassing` | Determines if instances that return a warning from a health check are allowed to resolve a request. When set to `false`, instances with `passing` and `warning` states are considered healthy. When set to `true`, only instances with a `passing` health check state are considered healthy. | Boolean | `false` | + +### `DefaultSubset` + +Specifies a defined subset of service instances to use when no explicit subset is requested. If this parameter is not specified, Consul uses the unnamed default subset. + +#### Values + +- Default: None +- Data type: String + +### `Redirect` + +Specifies redirect instructions for local service traffic so that services deployed to a different network location resolve the upstream request instead. When this field is defined, Consul ignores all other fields in a service resolver configuration entry except for `Kind`, `Name`, `Namespace`. When there are multiple redirects defined for a single service, Consul uses only the first one it applies. + +#### Values + +- Default: None +- Data type: Map that can contain the following parameters: + - [`Service`](#redirect-service) + - [`ServiceSubset`](#redirect-servicesubset) + - [`Namespace`](#redirect-namespace) + - [`Partition`](#redirect-partition) + - [`Datacenter`](#redirect-datacenter) + - [`Peer`](#redirect-peer) + +### `Redirect{}.Service` + +Specifies the name of a service at the redirect’s destination that resolves local upstream requests. + +#### Values + +- Default: None +- Data type: String + +### `Redirect{}.ServiceSubset` + +Specifies the name of a subset of services at the redirect’s destination that resolves local upstream requests. If empty, the default subset is used. If specified, you must also specify at least one of the following in the same `Redirect` map: `Service`, `Namespace`, and`Datacenter`. + +#### Values + +- Default: None +- Data type: String + +### `Redirect{}.Namespace` + +Specifies the namespace at the redirect’s destination that resolves local upstream requests. + +#### Values + +- Default: None +- Data type: String + +### `Redirect{}.Partition` + +Specifies the admin partition at the redirect’s destination that resolves local upstream requests. + +#### Values + +- Default: None +- Data type: String + + +### `Redirect{}.Datacenter` + +Specifies the datacenter at the redirect’s destination that resolves local upstream requests. + +#### Values + +- Default: None +- Data type: String + +### `Redirect{}.Peer` + +Specifies the cluster with an active cluster peering connection at the redirect’s destination that resolves local upstream requests. Requires separately defined service intentions that [authorize services for peers](/consul/docs/connect/cluster-peering/usage/establish-cluster-peering#authorize-services-for-peers). When Consul runs a health check before resolving requests from the peer, it does not apply health checks that were defined on the peer and exported to the local cluster through the exported services configuration entry. + +#### Values + +- Default: None +- Data type: String + +### `Failover` + +Specifies controls for rerouting traffic to an alternate pool of service instances if the target service fails. + +This parameter is a map, and its key is the name of the local service subset that resolves to another location when it fails. You can specify a `"*"` wildcard to apply failovers to any subset. + +`Service`, `ServiceSubset`, `Namespace`, `Targets`, and `Datacenters` cannot all be empty at the same time. + +#### Values + +- Default: None +- Data type: Map that can contain the following parameters: + - [`Service`](#failover-service) + - [`ServiceSubset`](#failover-servicesubset) + - [`Namespace`](#failover-namespace) + - [`Datacenters`](#failover-datacenters) + - [`Targets`](#failover-targets) + +### `Failover{}.Service` + +Specifies the name of the service to resolve at the failover location during a failover scenario. + +#### Values + +- Default: None +- Data type: String + +### `Failover{}.ServiceSubset` + +Specifies the name of a subset of service instances to resolve at the failover location during a failover scenario. If empty, Consul uses the service’s [`DefaultSubset`](#defaultsubset). + +#### Values + +- Default: None +- Data type: String + +### `Failover{}.Namespace` + +Specifies the namespace at the failover location where the failover services are deployed. If empty, the current namespace is used. + +#### Values + +- Default: None +- Data type: String + +### `Failover{}.Datacenters` + +Specifies an ordered list of datacenters at the failover location to attempt connections to during a failover scenario. When Consul cannot establish a connection with the first datacenter in the list, it proceeds sequentially until establishing a connection with another datacenter. + +#### Values + +- Default: None +- Data type: String + +### `Failover{}.Targets` + +Specifies a fixed list of failover targets to try during failover. This list can express complicated failover scenarios. + +For examples, refer to the [failover example configurations](#service-failover). + +#### Values + +- Default: None +- Data type: List of maps that can contain the following parameters: + - [`Service`](#failover-targets-service) + - [`ServiceSubset`](#failover-targets-servicesubset) + - [`Namespace`](#failover-targets-namespace) + - [`Partition`](#failover-targets-partition) + - [`Datacenter`](#failover-targets-datacenter) + - [`Peer`](#failover-targets-peer) + +### `Failover{}.Targets[].Service` + +Specifies the service name to use for the failover target. If empty, the current service name is used. + +#### Values + +- Default: None +- Data type: String + +### `Failover{}.Targets[].ServiceSubset` + +Specifies the named subset to use for the failover target. If empty, the default subset for the requested service name is used. + +#### Values + +- Default: None +- Data type: String + +### `Failover{}.Targets[].Namespace` + +Specifies the namespace to use for the failover target. If empty, the default namespace is used. + +#### Values + +- Default: None +- Data type: String + +### `Failover{}.Targets[].Partition` + +Specifies the admin partition within the same datacenter to use for the failover target. If empty, the `default` partition is used. To use an admin partition in a different datacenter for the failover target, use the `Peer` field instead. + +#### Values + +- Default: None +- Data type: String + +### `Failover{}.Targets[].Datacenter` + +Specifies the WAN federated datacenter to use for the failover target. If empty, the current datacenter is used. To use a datacenter for the failover target that is connected with a cluster peering relationship rather than WAN federation, use the `Peer` field instead. + +#### Values + +- Default: None +- Data type: String + +### `Failover{}.Targets[].Peer` + +Specifies the destination cluster peer to resolve the target service name from. [Intentions](/consul/docs/connect/cluster-peering/create-manage-peering#authorize-services-for-peers) must be defined on the peered cluster so that the source service can access this failover target service as an upstream. When the peer name is specified, Consul uses Envoy's outlier detection to determine the health of the failover target based on whether services can communicate with the failover target. Consul ignores service health checks imported from a peer for failover targets because the checks do not account for service routers, splitters, and resolvers that may be defined in the peer for the target service. + +#### Values + +- Default: None +- Data type: String + +### `LoadBalancer` + +Specifies the load balancing policy and configuration for services issuing requests to this upstream. + +#### Values + +- Default: None +- Data type: Map that can contain the following parameters: + - [`Policy`](#loadbalancer-policy) + - [`RingHashConfig`](#loadbalancer-ringhashconfig) + - [`LeastRequestConfig`](#loadbalancer-leastrequestconfig) + - [`HashPolicies`](#loadbalancer-hashpolicies) + +### `LoadBalancer{}.Policy` + +Specifies the type of load balancing policy for selecting a host. Supported load balancing policies include: + +| Policy | Description | +| :----- | :-------------------------------------------------------------- | +| Random | The load balancer forwards a client request to an available server chosen at random from a fixed list. | +| Round robin | The load balancer proceeds through a fixed list of servers and forwards a client request to each available server in order. | +| Least request | The load balancer forwards a client request to the server with the fewest connections. When using this policy, you can specify additional parameters in [`LoadBalancer{}.LeastRequestConfig`](#loadbalancer-leastrequestconfig). | +| Ring hash | The load balancer forwards requests from the same client to the same group of servers. When using this policy, you can specify additional parameters in [`LoadBalancer{}.RingHashConfig`](#loadbalancer-ringhashconfig). | +| Maglev | The load balancer uses a hashing algorithm to spread requests evenly across servers. When using this policy, you can specify additional parameters in [`LoadBalancer{}.HashPolicies`](#loadbalancer-hashpolicies). | + + +#### Values + +- Default: None +- Data type: String containing one of the following values: + + - `random` + - `round_robin` + - `least_request` + - `ring_hash` + - `maglev` + +### `LoadBalancer{}.LeastRequestConfig` + +Specifies configuration for the `least_request` policy type. + +#### Values + +- Default: None +- Data type: Map containing the following parameter: + + | Parameter | Description | Data type | Default | + | :------------ | :------------------------------------------------------------------------------------------------- | --------- | ------- | + | `ChoiceCount` | Specifies the number of random healthy hosts from which to select the one with the least requests. | Integer | `2` | + +### `LoadBalancer{}.RingHashConfig` + +Specifies configuration for the `ring_hash` policy type. + +#### Values + +- Default: None +- Data type: List that can contain the following parameters: + + | Parameter | Description | Data type | Default | + | :------------ | :------------------------------------------------------------- | --------- | ------- | + | `MinimumRingSize` | Determines the minimum number of entries in the hash ring. | Integer | `1024` | + | `MaximumRingSize` | Determines the maximum number of entries in the hash ring. | Integer | `8192` | + +### `LoadBalancer{}.HashPolicies` + +Specifies a list of hash policies to use for hashing load balancing algorithms. Consul evaluates hash policies individually and combines them so that identical lists result in the same hash. If no hash policies are present or successfully evaluated, then Consul selects a random backend host. + +#### Values + +- Default: None +- Data type: List of maps for the following parameters: + - [`Field`](#loadbalancer-hashpolicies-field) + - [`FieldValue`](#loadbalancer-hashpolicies-fieldvalue) + - [`CookieConfig`](#loadbalancer-hashpolicies-cookieconfig) + - [`SourceIP`](#loadbalancer-hashpolicies-sourceip) + - [`Terminal`](#loadbalancer-hashpolicies-terminal) + +### `LoadBalancer{}.HashPolicies[].Field` + + Specifies the attribute type to hash on. You cannot specify the `Field` parameter if `SourceIP` is also configured. + + Supported attribute types include the following: + +| Attribute | Description | +| :--------- | :-------------------------------------------------------------- | +| Cookie | The load balancer uses a cookie to obtain a hash key. Refer to the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-msg-config-route-v3-routeaction-hashpolicy-cookie) for more information. | +| Header | The load balancer uses a request header to obtain a hash key. Refer to the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-msg-config-route-v3-routeaction-hashpolicy-header) for more information. | +| Query Parameter | The load balancer uses a URL query parameter to obtain the hash key. Refer to the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-msg-config-route-v3-routeaction-hashpolicy-queryparameter) for more information. | + +#### Values + +- Default: None +- Data type: String containing one of the following values: + - `cookie` + - `header` + - `query_parameter` + +### `LoadBalancer{}.HashPolicies[].FieldValue` + +Specifies the value to hash, such as a header name, cookie name, or a URL query parameter name. You cannot specify the `FieldValue` parameter if `SourceIP` is also configured. + +#### Values + +- Default: None +- Data type: String + +### `LoadBalancer{}.HashPolicies[].CookieConfig` + +Specifies additional configuration options for the `cookie` hash policy type. This field causes Envoy to generate a cookie for a client on its first request. + +#### Values + +- Default: None +- Data type: Map that can contain the following parameters: + + | Parameter | Description | Data type | Default | + | :------------- | :-------------------------------------------------------------------------------- | --------- | ------- | + | `Session` | Directs Consul to generate a session cookie with no expiration. | Boolean | `false` | + | `TTL` | Specifies the TTL for generated cookies. Cannot be specified for session cookies. | String | `0s` | + | `Path` | Specifies the path to set for the cookie. | String | None | + +### `LoadBalancer{}.HashPolicies[].SourceIP` + +Determines if the hash type should besource IP address. You cannot specify `SourceIP` if `Field` or `FieldValue` are configured. + +#### Values + +- Default: `false` +- Data type: Boolean + +### `LoadBalancer{}.HashPolicies[].Terminal` + + Determines if Consul should stop computing the hash when multiple hash policies are present. If a hash is computed when a terminal policy is evaluated, then that hash is used and subsequent hash policies are ignored. + +#### Values + +- Default: `false` +- Data type: Boolean + + + + + +### `apiVersion` + +Specifies the version of the Consul API for integrating with Kubernetes. The value must be `consul.hashicorp.com/v1alpha1`. + +#### Values + +- Default: None +- This field is required. +- String value that must be set to `consul.hashicorp.com/v1alpha1`. + +### `kind` + +Specifies the type of configuration entry to implement. Must be set to `ServiceResolver`. + +#### Values + +- Default: None +- This field is required. +- Data type: String value that must be set to `ServiceResolver`. + +## `metadata` + +Map that contains an arbitrary name for the configuration entry and the namespace it applies to. + +#### Values + +- Default: None +- Data type: Map + +### `metadata.name` + +Specifies a name for the configuration entry. The name is metadata that you can use to reference the configuration entry when performing Consul operations, such as applying a configuration entry to a specific cluster. + +#### Values -The `service-resolver` config entry kind (`ServiceResolver` on Kubernetes) controls which service instances -should satisfy Connect upstream discovery requests for a given service name. +- Default: None +- This field is required. +- Data type: String -If no resolver config is defined the chain assumes 100% of traffic goes to the -healthy instances of the default service in the current datacenter+namespace -and discovery terminates. +### `metadata.namespace` -## Interaction with other Config Entries +Specifies the namespace that the service resolver applies to. Refer to [namespaces](/consul/docs/enterprise/namespaces) for more information. -- Service resolver config entries are a component of [L7 Traffic - Management](/consul/docs/connect/l7-traffic). +#### Values -## UI +- Default: None +- Data type: String -Once a `service-resolver` is successfully entered, you can view it in the UI. Service routers, service splitters, and service resolvers can all be viewed by clicking on your service then switching to the *routing* tab. +### `spec` -![screenshot of service resolver in the UI](/img/l7-routing/Resolver.png) +Map that contains the details about the `ServiceResolver` configuration entry. The `apiVersion`, `kind`, and `metadata` fields are siblings of the spec field. All other configurations are children. -## Sample Config Entries +#### Values + +- Default: None +- This field is required. +- Data type: Map + +### `spec.connectTimeout` + +Specifies the timeout duration for establishing new network connections to this service. By default, the duration is measured in nanoseconds (ns). + +#### Values + +- Default: None +- Data type: String + +### `spec.requestTimeout` + +Specifies the timeout duration for receiving an HTTP response from this service. When set to `0s`, the default value of `15s` is used instead. By default, the duration is measured in nanoseconds (ns). + +#### Values + +- Default: `15s` +- Data type: String + +### `spec.subsets` + +Specifies names for custom service subsets and the conditions under which service instances belong to each subset. Define a subset by specifying a key and a value where the key is the subset’s name and the value is a map that contains a [filtering expression](/consul/api-docs/features/filtering). If this parameter is not specified, only the unnamed default subset is usable. + +For additional guidance, refer to the [filter on service version configuration example](#filter-on-service-version). + +#### Values + +- Default: None +- Data type: Map containing a key-value pair. + - ``: String that names the subset. The string must be valid as a DNS subdomain element. + - ``: Map that can contain the following parameters: + + | Parameter | Description | Data type | Default | + | :------------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------- | + | `filter` | Specifies an expression that filters the DNS elements of service instances that belong to the subset. If empty, all healthy instances of a service are returned. This expression can filter on the same DNS selectors as the [Health API endpoint](/consul/api-docs/health#filtering-2). For more information about creating and using expressions to filter, refer to [filtering](/consul/api-docs/features/filtering). | String | None | + | `onlyPassing` | Determines if instances that return a warning from a health check are allowed to resolve a request. When set to `false`, instances with `passing` and `warning` states are considered healthy. When set to `true`, only instances with a `passing` health check state are considered healthy. | Boolean | `false` | + +### `spec.defaultSubset` + +Specifies a defined subset of service instances to use when no explicit subset is requested. If this parameter is not specified, Consul uses the unnamed default subset. + +#### Values + +- Default: None +- Data type: String + +### `spec.redirect` + +Specifies redirect instructions for local service traffic so that services deployed to a different network location resolve the upstream request instead. When this field is defined, Consul ignores all other fields in a service resolver configuration entry except for `kind`, `name`, `namespace`. When there are multiple redirects defined for a single service, Consul uses only the first one it applies. + +#### Values + +- Default: None +- Data type: Map that can contain the following parameters: + - [`service`](#spec-redirect-service) + - [`serviceSubset`](#spec-redirect-servicesubset) + - [`namespace`](#spec-redirect-namespace) + - [`partition`](#spec-redirect-partition) + - [`datacenter`](#spec-redirect-datacenter) + - [`peer`](#spec-redirect-peer) + +### `spec.redirect.service` + +Specifies the name of a service at the redirect’s destination that resolves local upstream requests. + +#### Values + +- Default: None +- Data type: String + +### `spec.redirect.serviceSubset` + +Specifies the name of a subset of services at the redirect’s destination that resolves local upstream requests. If empty, the default subset is used. If specified, you must also specify at least one of the following in the same `redirect` map: `service`, `namespace`, and`datacenter`. + +#### Values + +- Default: None +- Data type: String + +### `spec.redirect.namespace` + +Specifies the namespace at the redirect’s destination that resolves local upstream requests. + +#### Values + +- Default: None +- Data type: String + +### `spec.redirect.partition` + +Specifies the admin partition at the redirect’s destination that resolves local upstream requests. + +#### Values + +- Default: None +- Data type: String + + +### `spec.redirect.datacenter` + +Specifies the datacenter at the redirect’s destination that resolves local upstream requests. + +#### Values + +- Default: None +- Data type: String + +### `spec.redirect.peer` + +Specifies the cluster with an active cluster peering connection at the redirect’s destination that resolves local upstream requests. Requires separately defined service intentions that [authorize services for peers](/consul/docs/connect/cluster-peering/usage/establish-cluster-peering#authorize-services-for-peers). When Consul runs a health check before resolving requests from the peer, it does not apply health checks that were defined on the peer and exported to the local cluster through the exported services configuration entry. + +#### Values + +- Default: None +- Data type: String + +### `spec.failover` + +Specifies controls for rerouting traffic to an alternate pool of service instances if the target service fails. + +This parameter is a map, and its key is the name of the local service subset that resolves to another location when it fails. You can specify a `"*"` wildcard to apply failovers to any subset. + +`service`, `serviceSubset`, `namespace`, `targets`, and `datacenters` cannot all be empty at the same time. + +#### Values + +- Default: None +- Data type: Map that can contain the following parameters: + - [`service`](#spec-failover-service) + - [`serviceSubset`](#spec-failover-servicesubset) + - [`namespace`](#spec-failover-namespace) + - [`datacenters`](#spec-failover-datacenters) + - [`targets`](#spec-failover-targets) + +### `spec.failover.service` + +Specifies the name of the service to resolve at the failover location during a failover scenario. + +#### Values + +- Default: None +- Data type: String + +### `spec.failover.serviceSubset` + +Specifies the name of a subset of service instances to resolve at the failover location during a failover scenario. If empty, Consul uses the service’s [`defaultSubset`](#defaultsubset). + +#### Values + +- Default: None +- Data type: String + +### `spec.failover.namespace` + +Specifies the namespace at the failover location where the failover services are deployed. If empty, the current namespace is used. + +#### Values + +- Default: None +- Data type: String + +### `spec.failover.datacenters` + +Specifies an ordered list of datacenters at the failover location to attempt connections to during a failover scenario. When Consul cannot establish a connection with the first datacenter in the list, it proceeds sequentially until establishing a connection with another datacenter. + +#### Values + +- Default: None +- Data type: String + +### `spec.failover.targets` + +Specifies a fixed list of failover targets to try during failover. This list can express complicated failover scenarios. + +For examples, refer to the [failover example configurations](#service-failover). + +#### Values + +- Default: None +- Data type: List of maps that can contain the following parameters: + - [`service`](#spec-failover-targets-service) + - [`serviceSubset`](#spec-failover-targets-servicesubset) + - [`namespace`](#spec-failover-targets-namespace) + - [`partition`](#spec-failover-targets-partition) + - [`datacenter`](#spec-failover-targets-datacenter) + - [`peer](#spec-failover-targets-peer) + +### `spec.failover.targets.service` + +Specifies the service name to use for the failover target. If empty, the current service name is used. + +#### Values + +- Default: None +- Data type: String + +### `spec.failover.targets.serviceSubset` + +Specifies the named subset to use for the failover target. If empty, the default subset for the requested service name is used. + +#### Values + +- Default: None +- Data type: String + +### `spec.failover.targets.namespace` + +Specifies the namespace to use for the failover target. If empty, the default namespace is used. + +#### Values + +- Default: None +- Data type: String + +### `spec.failover.targets.partition` + +Specifies the admin partition within the same datacenter to use for the failover target. If empty, the `default` partition is used. To use an admin partition in a different datacenter for the failover target, use the `peer` field instead. + +#### Values + +- Default: None +- Data type: String + +### `spec.failover.targets.datacenter` + +Specifies the WAN federated datacenter to use for the failover target. If empty, the current datacenter is used. To use a datacenter for the failover target that is connected with a cluster peering relationship rather than WAN federation, use the `peer` field instead. + +#### Values + +- Default: None +- Data type: String + +### `spec.failover.targets.peer` + +Specifies the destination cluster peer to resolve the target service name from. [Intentions](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering#authorize-services-for-peers) must be defined on the peered cluster so that the source service can access this failover target service as an upstream. When the peer name is specified, Consul uses Envoy's outlier detection to determine the health of the failover target based on whether services can communicate with the failover target. Consul ignores service health checks imported from a peer for failover targets because the checks do not account for service routers, splitters, and resolvers that may be defined in the peer for the target service. + +#### Values + +- Default: None +- Data type: String + +### `spec.loadBalancer` + +Specifies the load balancing policy and configuration for services issuing requests to this upstream. + +#### Values + +- Default: None +- Data type: Map that can contain the following parameters: + - [`policy`](#spec-loadbalancer-policy) + - [`leastRequestConfig`](#spec-loadbalancer-leastrequestconfig) + - [`ringHashConfig`](#spec-loadbalancer-ringhashconfig) + - [`hashPolicies`](#spec-loadbalancer-hashpolicies) + +### `spec.loadBalancer.policy` + +Specifies the type of load balancing policy for selecting a host. Supported load balancing policies include: + +| Policy | Description | +| :----- | :-------------------------------------------------------------- | +| Random | The load balancer forwards a client request to an available server chosen at random from a fixed list. | +| Round robin | The load balancer proceeds through a fixed list of servers and forwards a client request to each available server in order. | +| Least request | The load balancer forwards a client request to the server with the fewest connections. When using this policy, you can specify additional parameters in [`spec.loadBalancer.leastRequestConfig`](#spec-loadbalancer-leastrequestconfig). | +| Ring hash | The load balancer forwards requests from the same client to the same group of servers. When using this policy, you can specify additional parameters in [`spec.loadBalancer.ringHashConfig`](#spec-loadbalancer-ringhashconfig). | +| Maglev | The load balancer uses a hashing algorithm to spread requests evenly across servers. When using this policy, you can specify additional parameters in [`spec.loadBalancer.hashPolicies`](#spec-loadbalancer-hashpolicies). | + +#### Values + +- Default: None +- Data type: String containing one of the following values: + + - `random` + - `round_robin` + - `least_request` + - `ring_hash` + - `maglev` + +### `spec.loadBalancer.leastRequestConfig` + +Specifies configuration for the `least_request` policy type. + +#### Values + +- Default: None +- Data type: Map containing the following parameter: + + | Parameter | Description | Data type | Default | + | :------------ | :------------------------------------------------------------------------------------------------- | --------- | ------- | + | `choiceCount` | Specifies the number of random healthy hosts from which to select the one with the least requests. | Integer | `2` | + +### `spec.loadBalancer.ringHashConfig` + +Specifies configuration for the `ring_hash` policy type. + +#### Values + +- Default: None +- Data type: List that can contain the following parameters: + + | Parameter | Description | Data type | Default | + | :------------ | :------------------------------------------------------------- | --------- | ------- | + | `minimumRingSize` | Determines the minimum number of entries in the hash ring. | Integer | `1024` | + | `maximumRingSize` | Determines the maximum number of entries in the hash ring. | Integer | `8192` | + +### `spec.loadBalancer.hashPolicies` + +Specifies a list of hash policies to use for hashing load balancing algorithms. Consul evaluates hash policies individually and combines them so that identical lists result in the same hash. If no hash policies are present or successfully evaluated, then Consul selects a random backend host. + +#### Values + +- Default: None +- Data type: List that can contain the following parameters: + - [`field`](#spec-loadbalancer-hashpolicies-field) + - [`fieldValue`](#spec-loadbalancer-hashpolicies-fieldvalue) + - [`cookieConfig`](#spec-loadbalancer-hashpolicies-cookieconfig) + - [`sourceIP`](#spec-loadbalancer-hashpolicies-sourceip) + - [`terminal`](#spec-loadbalancer-hashpolicies-terminal) + +### `spec.loadBalancer.hashPolicies[].field` + + Specifies the attribute type to hash on. You cannot specify the `field` parameter if `sourceIP` is also configured. + +Supported attribute types include the following: + +| Attribute | Description | +| :--------- | :-------------------------------------------------------------- | +| Cookie | The load balancer uses a cookie to obtain a hash key. Refer to the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-msg-config-route-v3-routeaction-hashpolicy-cookie) for more information. | +| Header | The load balancer uses a request header to obtain a hash key. Refer to the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-msg-config-route-v3-routeaction-hashpolicy-header) for more information. | +| Query Parameter | The load balancer uses a URL query parameter to obtain the hash key. Refer to the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-msg-config-route-v3-routeaction-hashpolicy-queryparameter) for more information. | + +#### Values + +- Default: None +- Data type: String containing one of the following values: + - `cookie` + - `header` + - `query_parameter` + +### `spec.loadBalancer.hashPolicies[].fieldValue` + +Specifies the value to hash, such as a header name, cookie name, or a URL query parameter name. You cannot specify the `fieldValue` parameter if `sourceIP` is also configured. + +#### Values + +- Default: None +- Data type: String + +### `spec.loadBalancer.hashPolicies[].cookieConfig` + +Specifies additional configuration options for the `cookie` hash policy type. This field causes Envoy to generate a cookie for a client on its first request. + +#### Values + +- Default: None +- Data type: Map that can contain the following parameters: + + | Parameter | Description | Data type | Default | + | :------------- | :-------------------------------------------------------------------------------- | --------- | ------- | + | `session` | Directs Consul to generate a session cookie with no expiration. | Boolean | `false` | + | `ttl` | Specifies the TTL for generated cookies. Cannot be specified for session cookies. | String | `0s` | + | `path` | Specifies the path to set for the cookie. | String | None | + +### `spec.loadBalancer.hashPolicies[].sourceIP` + +Determines if the hash type should besource IP address. You cannot specify `sourceIP` if `field` or `fieldValue` are configured. + +#### Values + +- Default: `false` +- Data type: Boolean + +### `spec.loadBalancer.hashPolicies[].terminal` + +Determines if Consul should stop computing the hash when multiple hash policies are present. If a hash is computed when a terminal policy is evaluated, then that hash is used and subsequent hash policies are ignored. + +#### Values + +- Default: `false` +- Data type: Boolean + + + + +## Examples + +The following examples demonstrate common service resolver configuration patterns for specific use cases. ### Filter on service version -Create service subsets based on a version metadata and override the defaults: +The following example creates two subsets of the `web` service and assigns service instances to subsets based on each instance's version metadata. It also defines `v1` as the default subset. - + + ```hcl Kind = "service-resolver" @@ -50,6 +1257,10 @@ Subsets = { } ``` + + + + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 kind: ServiceResolver @@ -64,6 +1275,10 @@ spec: filter: 'Service.Meta.version == v2' ``` + + + + ```json { "Kind": "service-resolver", @@ -80,13 +1295,15 @@ spec: } ``` - +     +     -### Other datacenters +### Resolve virtual upstreams -Expose a set of services in another datacenter as a virtual service: +The folowing example uses the [`Redirect` parameter](#redirect) to expose a set of services to another datacenter as a virtual service. - + + ```hcl Kind = "service-resolver" @@ -97,6 +1314,10 @@ Redirect { } ``` + + + + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 kind: ServiceResolver @@ -108,6 +1329,10 @@ spec: datacenter: dc2 ``` + + + + ```json { "Kind": "service-resolver", @@ -119,13 +1344,15 @@ spec: } ``` - + + -### Failover +### Service failover -Enable failover for subset `v2` to `dc2`, and all other subsets to `dc3` or `dc4`: +The following example enables failover for subset `v2` to `dc2`. All other subsets attempt failover to `dc3`, and when it is not available, to `dc4`: - + + ```hcl Kind = "service-resolver" @@ -141,6 +1368,10 @@ Failover = { } ``` + + + + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 kind: ServiceResolver @@ -155,6 +1386,10 @@ spec: datacenters: ['dc3', 'dc4'] ``` + + + + ```json { "Kind": "service-resolver", @@ -171,12 +1406,15 @@ spec: } ``` - + + + +#### Failover targets for federation and cluster peering -Enable failover to a WAN federated datacenter and then a cluster peer for all -service subsets. +The following example enables failover to target a WAN federated datacenter for all service subsets. If the connection to `dc3` times out after 15 seconds, the service failover targets the peer with the establish cluster peering connection instead. - + + ```hcl Kind = "service-resolver" @@ -192,6 +1430,10 @@ Failover = { } ``` + + + + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 kind: ServiceResolver @@ -206,6 +1448,10 @@ spec: - peer: "peer-01" ``` + + + + ```json { "Kind": "service-resolver", @@ -222,13 +1468,15 @@ spec: } ``` - + + - - Failover to another datacenter and namespace for all service subsets. - +#### Failover for all service subsets - +The following example enables failover to the `secondary` namespace in another datacenter for all service subsets. + + + ```hcl Kind = "service-resolver" @@ -243,6 +1491,10 @@ Failover = { } ``` + + + + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 kind: ServiceResolver @@ -256,6 +1508,10 @@ spec: datacenters: ['dc2'] ``` + + + + ```json { "Kind": "service-resolver", @@ -271,13 +1527,15 @@ spec: } ``` - + + + +#### Failover to a namespace - - Failover within a datacenter and a different namespace. - +The following example enables failover to a different namespace in the same datacenter. - + + ```hcl Kind = "service-resolver" @@ -292,6 +1550,10 @@ Failover = { } ``` + + + + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 kind: ServiceResolver @@ -305,6 +1567,10 @@ spec: namespace: 'secondary' ``` + + + + ```json { "Kind": "service-resolver", @@ -320,13 +1586,15 @@ spec: } ``` - + + ### Consistent load balancing -Apply consistent load balancing for requests based on `x-user-id` header: +The following example applies a Maglev load balancing policy for requests with an `x-user-id` header: - + + ```hcl Kind = "service-resolver" @@ -343,6 +1611,10 @@ LoadBalancer = { } ``` + + + + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 kind: ServiceResolver @@ -356,6 +1628,10 @@ spec: fieldValue: x-user-id ``` + + + + ```json { "Kind": "service-resolver", @@ -372,431 +1648,5 @@ spec: } ``` - - -## Available Fields - -', - yaml: false, - }, - { - name: 'Namespace', - type: `string: "default"`, - enterprise: true, - description: - 'Specifies the namespace in which the configuration entry will apply.', - yaml: false, - }, - { - name: 'Partition', - type: `string: "default"`, - enterprise: true, - description: - 'Specifies the admin partition in which the configuration entry will apply.', - yaml: false, - }, - { - name: 'Meta', - type: 'map: nil', - description: - 'Specifies arbitrary KV metadata pairs. Added in Consul 1.8.4.', - yaml: false, - }, - { - name: 'metadata', - children: [ - { - name: 'name', - description: 'Set to the name of the service being configured.', - }, - { - name: 'namespace', - description: - 'If running Consul Open Source, the namespace is ignored (see [Kubernetes Namespaces in Consul OSS](/consul/docs/k8s/crds#consul-oss)). If running Consul Enterprise see [Kubernetes Namespaces in Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for more details.', - }, - ], - hcl: false, - }, - { - name: 'ConnectTimeout', - type: 'duration: 0s', - description: - 'The timeout for establishing new network connections to this service. The default unit of time is `ns`.', - }, - { - name: 'DefaultSubset', - type: 'string: ""', - description: - 'The subset to use when no explicit subset is requested. If empty, the unnamed subset is used.', - }, - { - name: 'Subsets', - type: 'map[string]ServiceResolverSubset', - description: - 'A map of subset name to subset definition for all usable named subsets of this service. The map key is the name of the subset and all names must be valid DNS subdomain elements.

    This may be empty, in which case only the unnamed default subset will be usable.', - children: [ - { - name: 'Filter', - type: 'string: ""', - description: `The filter expression for selecting instances of the - requested service. If empty, all healthy instances are returned. - This expression can filter on the same selectors as the - [Health API endpoint](/consul/api-docs/health#filtering-2).`, - }, - { - name: 'OnlyPassing', - type: 'bool: false', - description: `Specifies the behavior of the resolver's - health check interpretation. If this is set to false, instances with checks - in the passing as well as the warning states will be considered healthy. If - this is set to true, only instances with checks in the passing state will - be considered healthy.`, - }, - ], - }, - { - name: 'Redirect', - type: 'ServiceResolverRedirect: ', - description: { - hcl: `When configured, all attempts to resolve the service this resolver defines will be substituted for the supplied redirect EXCEPT when the redirect has already been applied. -

    - When \`Redirect\` is set, all other fields besides \`Kind\`, \`Name\`, \`Namespace\` and \`Redirect\` will be ignored.`, - yaml: `When configured, all attempts to resolve the service this resolver defines will be substituted for the supplied redirect EXCEPT when the redirect has already been applied. -

    - When \`redirect\` is set, all other fields besides \`redirect\` will be ignored.`, - }, - children: [ - { - name: 'Service', - type: 'string: ""', - description: 'A service to resolve instead of the current service.', - }, - { - name: 'ServiceSubset', - type: 'string: ""', - description: { - hcl: `A named subset of the given service to - resolve instead of one defined as that service's \`DefaultSubset\`. If empty, the - default subset is used. -

    - If this is specified at least one of \`Service\`, \`Datacenter\`, or \`Namespace\` - should be configured.`, - yaml: `A named subset of the given service to - resolve instead of one defined as that service's \`defaultSubset\`. If empty, the - default subset is used. -

    - If this is specified at least one of \`service\`, \`datacenter\`, or \`namespace\` - should be configured.`, - }, - }, - { - name: 'Namespace', - enterprise: true, - type: 'string: ""', - description: - 'Specifies the destination namespace to resolve the service from.', - }, - { - name: 'Partition', - enterprise: true, - type: 'string: ""', - description: - 'Specifies the destination admin partition to resolve the service from.', - }, - { - name: 'Datacenter', - type: 'string: ""', - description: - 'Specifies the destination datacenter to resolve the service from.', - }, - { - name: 'Peer', - type: 'string: ""', - description: - `Specifies the destination cluster peer to resolve the target service name from. - Ensure that [intentions are defined](/consul/docs/connect/cluster-peering/create-manage-peering#authorize-services-for-peers) - on the peered cluster to allow the source service to access this redirect target service as an upstream. When - the peer name is specified, Consul uses Envoy's outlier detection to determine - the health of the redirect target based on whether communication attempts to the - redirect target are generally successful. Service health checks imported from a peer - are not considered in the health of a redirect target because they do not account for service - routers, splitters, and resolvers that may be defined in the peer for the target service.`, - }, - ], - }, - { - name: 'Failover', - type: 'map[string]ServiceResolverFailover', - description: { - hcl: `Controls when and how to - reroute traffic to an alternate pool of service instances. -

    - The map is keyed by the service subset it applies to and the special - string \`"*"\` is a wildcard that applies to any subset not otherwise - specified here. -

    - \`Service\`, \`ServiceSubset\`, \`Namespace\`, \`Targets\`, and - \`Datacenters\` cannot all be empty at once.`, - yaml: `Controls when and how to - reroute traffic to an alternate pool of service instances. -

    - The map is keyed by the service subset it applies to and the special - string \`"*"\` is a wildcard that applies to any subset not otherwise - specified here. -

    - \`service\`, \`serviceSubset\`, \`namespace\`, \`targets\` and - \`datacenters\` cannot all be empty at once.`, - }, - children: [ - { - name: 'Service', - type: 'string: ""', - description: - 'The service to resolve instead of the default as the failover group of instances during failover.', - }, - { - name: 'ServiceSubset', - type: 'string: ""', - description: - 'The named subset of the requested service to resolve as the failover group of instances. If empty, the default subset for the requested service is used.', - }, - { - name: 'Namespace', - enterprise: true, - type: 'string: ""', - description: - 'The namespace to resolve the requested service from to form the failover group of instances. If empty, the current namespace is used.', - }, - { - name: 'Datacenters', - type: 'array', - description: 'A fixed list of datacenters to try during failover.', - }, - { - name: 'Targets', - type: 'array', - description: `A fixed list of failover targets to try during - failover. It allows operators to express complicated failover - scenarios such as between cluster peers, WAN federated datacenters, and local admin partitions.`, - children: [ - { - name: 'Service', - type: 'string: ""', - description: - 'The service name to use for the failover target. If empty, the current service name is used.', - }, - { - name: 'ServiceSubset', - type: 'string: ""', - description: - 'The named subset to use for the failover target. If empty, the default subset for the requested service name is used.', - }, - { - name: 'Namespace', - enterprise: true, - type: 'string: ""', - description: - `The namespace to use for the failover target. If empty, the \`default\` namespace is used.`, - }, - { - name: 'Partition', - enterprise: true, - type: 'string: ""', - description: - `The admin partition within the same datacenter to use for the failover target. - If empty, the \`default\` partition is used. - To use an admin partition in a different datacenter for the failover target, - use the \`Peer\` field instead.`, - }, - { - name: 'Datacenter', - type: 'string: ""', - description: - `The WAN federated datacenter to use for the failover target. - If empty, the current datacenter is used. - To use a datacenter for the failover target that is connected - with a cluster peering relationship rather than WAN federation, - use the \`Peer\` field instead.`, - }, - { - name: 'Peer', - type: 'string: ""', - description: - `Specifies the destination cluster peer to resolve the target service name from. - Ensure that [intentions are defined](/consul/docs/connect/cluster-peering/create-manage-peering#authorize-services-for-peers) - on the peered cluster to allow the source service to access this failover target service as an upstream. - When the peer name is specified, Consul uses Envoy's outlier detection - to determine the health of the failover target based on - whether communication attempts to the failover target are generally successful. - Service health checks imported from a peer are not considered in the health - of a failover target because they do not account for service routers, splitters, and resolvers - that may be defined in the peer for the target service.`, - }, - ], - }, - ], - }, - { - name: 'LoadBalancer', - type: 'LoadBalancer', - description: - 'Determines the load balancing policy and configuration for services issuing requests to this upstream. This option is available in Consul versions 1.9.0 and newer.', - children: [ - { - name: 'Policy', - type: 'string: ""', - description: - 'The load balancing policy used to select a host. One of: `random`, `round_robin`, `least_request`, `ring_hash`, `maglev`.', - }, - { - name: 'RingHashConfig', - type: 'RingHashConfig', - description: 'Configuration for the `ring_hash` policy type.', - children: [ - { - name: 'MinimumRingSize', - type: 'int: 1024', - description: - 'Determines the minimum number of entries in the hash ring.', - }, - { - name: 'MaximumRingSize', - type: 'int: 8192', - description: - 'Determines the maximum number of entries in the hash ring.', - }, - ], - }, - { - name: 'LeastRequestConfig', - type: 'LeastRequestConfig', - description: 'Configuration for the `least_request` policy type.', - children: [ - { - name: 'ChoiceCount', - type: 'int: 2', - description: - 'Determines the number of random healthy hosts from which to select the one with the least requests.', - }, - ], - }, - { - name: 'HashPolicies', - type: 'array', - description: `List of hash policies to use for - hashing load balancing algorithms. Hash policies are evaluated individually - and combined such that identical lists result in the same hash. - If no hash policies are present, or none are successfully evaluated, - then a random backend host will be selected.`, - children: [ - { - name: 'Field', - type: 'string: ""', - description: { - hcl: - 'The attribute type to hash on. Must be one of `header`, `cookie`, or `query_parameter`. Cannot be specified along with `SourceIP`.', - yaml: - 'The attribute type to hash on. Must be one of `header`, `cookie`, or `query_parameter`. Cannot be specified along with `sourceIP`.', - }, - }, - { - name: 'FieldValue', - type: 'string: ""', - description: { - hcl: - 'The value to hash. ie. header name, cookie name, URL query parameter name. Cannot be specified along with `SourceIP`.', - yaml: - 'The value to hash. ie. header name, cookie name, URL query parameter name. Cannot be specified along with `sourceIP`.', - }, - }, - { - name: 'CookieConfig', - type: 'CookieConfig', - description: - 'Additional configuration for the "cookie" hash policy type. This is specified to have Envoy generate a cookie for a client on its first request.', - children: [ - { - name: 'Session', - type: 'bool: false', - description: 'Generates a session cookie with no expiration.', - }, - { - name: 'TTL', - type: 'duration: 0s', - description: - 'TTL for generated cookies. Cannot be specified for session cookies.', - }, - { - name: 'Path', - type: 'string: ""', - description: 'The path to set for the cookie.', - }, - ], - }, - { - name: 'SourceIP', - type: 'bool: false', - description: { - hcl: - 'Determines whether the hash should be of the source IP address rather than of a field and field value. Cannot be specified along with `Field` or `FieldValue`.', - yaml: - 'Determines whether the hash should be of the source IP address rather than of a field and field value. Cannot be specified along with `field` or `fieldValue`.', - }, - }, - { - name: 'Terminal', - type: 'bool: false', - description: - 'Will short circuit the computation of the hash when multiple hash policies are present. If a hash is computed when a Terminal policy is evaluated, then that hash will be used and subsequent hash policies will be ignored.', - }, - ], - }, - ], - }, - ]} -/> - -## Service Subsets - -A service subset assigns a concrete name to a specific subset of discoverable -service instances within a datacenter, such as `"version2"` or `"canary"`. - -A service subset name is useful only when composed with an actual service name, -a specific datacenter, and namespace. - -All services have an unnamed default subset that will return all healthy -instances unfiltered. - -Subsets are defined in `service-resolver` configuration entries, but are -referenced by their names throughout the other configuration entry kinds. - -## ACLs - -Configuration entries may be protected by [ACLs](/consul/docs/security/acl). - -Reading a `service-resolver` config entry requires `service:read` on the resource. - -Creating, updating, or deleting a `service-resolver` config entry requires -`service:write` on the resource and `service:read` on any other service referenced by -name in these fields: - -- [`Redirect.Service`](#service) - -- [`Failover[].Service`](#service-1) + + diff --git a/website/content/docs/connect/config-entries/service-router.mdx b/website/content/docs/connect/config-entries/service-router.mdx index 1a12e3cdf9f..4362925f59f 100644 --- a/website/content/docs/connect/config-entries/service-router.mdx +++ b/website/content/docs/connect/config-entries/service-router.mdx @@ -1,55 +1,1314 @@ --- layout: docs -page_title: Service Router - Configuration Entry Reference -description: >- - The service router configuration entry kind defines where the service mesh routes requests based on L7 network information such as header or path. Use the reference guide to learn about `""service-router""` config entry parameters and how behaviors like request timeouts, retry behavior, header modification, and path rewriting can be applied to a request based on its header or path information. +page_title: Service Router Configuration Entry Reference +description: >- + Service router configuration entries are L7 traffic management tools for redirecting requests for a service to a particular instance or set of instances. Learn how to write `service-router` config entries in HCL or YAML with a specification reference, configuration model, a complete example, and example code by use case. --- -# Service Router Configuration Entry +# Service router configuration entry reference + +This page provides reference information for service router configuration entries. Service routers use L7 network information to redirect a traffic request for a service to one or more specific service instances. + +Refer to [L7 traffic management overview](/consul/docs/connect/l7-traffic) for additional information. + +## Configuration model + +The following list outlines field hierarchy, language-specific data types, and requirements in this configuration entry. Click on a property name to view additional details, including default values. + + + + + +- [`Kind`](#kind): string | required | must be set to `service-router` +- [`Name`](#name): string | required +- [`Namespace`](#namespace): string +- [`Partition`](#partition): string | `default` +- [`Meta`](#meta): map +- [`Routes`](#routes): list + - [`Match`](#routes-match): map + - [`HTTP`](#routes-match-http): map + - [`PathExact`](#routes-match-http-pathexact): string + - [`PathPrefix`](#routes-match-http-pathprefix): string + - [`PathRegex`](#routes-match-http-pathregex): string + - [`Methods`](#routes-match-http-methods): list + - [`Header`](#routes-match-http-header): list + - [`Name`](#routes-match-http-header-name): string + - [`Present`](#routes-match-http-header-present): boolean | `false` + - [`Exact`](#routes-match-http-header-exact): string + - [`Prefix`](#routes-match-http-header-prefix): string + - [`Suffix`](#routes-match-http-header-suffix): string + - [`Regex`](#routes-match-http-header-regex): string + - [`Invert`](#routes-match-http-header-invert): boolean | `false` + - [`QueryParam`](#routes-match-http-queryparam): list + - [`Name`](#routes-match-http-queryparam-name): string + - [`Present`](#routes-match-http-queryparam-present): boolean | `false` + - [`Exact`](#routes-match-http-queryparam-exact): string + - [`Regex`](#routes-match-http-queryparam-regex): string + - [`Destination`](#routes-destination): map + - [`Service`](#routes-destination-service): string + - [`ServiceSubset`](#routes-destination-servicesubset): string + - [`Namespace`](#routes-destination-namespace): string + - [`Partition`](#routes-destination-partition): string + - [`PrefixRewrite`](#routes-destination-prefixrewrite): string + - [`RequestTimeout`](#routes-destination-requesttimeout): integer | `0` + - [`IdleTimeout`](#routes-destination-idletimeout): integer | `0` + - [`NumRetries`](#routes-destination-numretries): integer | `1` + - [`RetryOnConnectFailure`](#routes-destination-retryonconnectfailure): boolean | `false` + - [`RetryOn`](#routes-destination-retryon): list + - [`RetryOnStatusCodes`](#routes-destination-retryonstatuscodes): list + - [`RequestHeaders`](#routes-destination-requestheaders): map + - [`Add`](#routes-destination-requestheaders): map + - [`Set`](#routes-destination-requestheaders): map + - [`Remove`](#routes-destination-requestheaders): map + - [`ResponseHeaders`](#routes-destination-responseheaders): map + - [`Add`](#routes-destination-responseheaders): map + - [`Set`](#routes-destination-responseheaders): map + - [`Remove`](#routes-destination-responseheaders): map + + + + + +- [`apiVersion`](#apiversion): string | required | must be set to `consul.hashicorp.com/v1alpha1` +- [`kind`](#kind): string | required | must be set to `ServiceRouter` +- [`metadata`](#metadata): map | required + - [`name`](#metadata-name): string | required + - [`namespace`](#metadata-namespace): string | optional +- [`spec`](#spec): map | required + - [`routes`](#spec-routes): list + - [`match`](#spec-routes-match): map + - [`http`](#spec-routes-match-http): map + - [`pathExact`](#spec-routes-match-http-pathexact): string + - [`pathPrefix`](#spec-routes-match-http-pathprefix): string + - [`pathRegex`](#spec-routes-match-http-pathregex): string + - [`methods`](#spec-routes-match-http-methods): list + - [`header`](#spec-routes-match-http-header): list + - [`name`](#spec-routes-match-http-header-name): string + - [`present`](#spec-routes-match-http-header-present): boolean | `false` + - [`exact`](#spec-routes-match-http-header-exact): string + - [`prefix`](#spec-routes-match-http-header-prefix): string + - [`suffix`](#spec-routes-match-http-header-suffix): string + - [`regex`](#spec-routes-match-http-header-regex): string + - [`invert`](#spec-routes-match-http-header-invert): boolean + - [`queryParam`](#spec-routes-match-http-queryparam): list + - [`name`](#spec-routes-match-http-queryparam-name): string + - [`present`](#spec-routes-match-http-queryparam-present): boolean | `false` + - [`regex`](#spec-routes-match-http-queryparam-regex): string + - [`destination`](#spec-routes-destination): map + - [`service`](#spec-routes-destination-service): string + - [`serviceSubset`](#spec-routes-destination-servicesubset): string + - [`namespace`](#spec-routes-destination-namespace): string + - [`partition`](#spec-routes-destination-partition): string + - [`prefixRewrite`](#spec-routes-destination-prefixrewrite): string + - [`requestTimeout`](#spec-routes-destination-requesttimeout): integer | `0` + - [`numRetries`](#spec-routes-destination-numretries): integer | `0` + - [`numRetries`](#spec-routes-destination-numretries): integer | `1` + - [`retryOnConnectFailure`](#spec-routes-destination-retryonconnectfailure): boolean | `false` + - [`retryOn`](#spec-routes-destination-retryon): list + - [`retryOnStatusCodes`](#spec-routes-destination-retryonstatuscodes): list + - [`requestHeaders`](#spec-routes-destination-requestheaders): map + - [`add`](#spec-routes-destination-requestheaders): map + - [`set`](#spec-routes-destination-requestheaders): map + - [`remove`](#spec-routes-destination-requestheaders): map + - [`responseHeaders`](#spec-routes-destination-responseheaders): map + - [`add`](#spec-routes-destination-responseheaders): map + - [`set`](#spec-routes-destination-responseheaders): map + - [`remove`](#spec-routes-destination-responseheaders): map + + + + +## Complete configuration + +When every field is defined, a service router configuration entry has the following form: + + + + -The `service-router` config entry kind (`ServiceRouter` on Kubernetes) controls Connect traffic routing and -manipulation at networking layer 7 (e.g. HTTP). +```hcl +Kind = "service-router" ## required +Name = "" ## required +Namespace = "" +Partition = "" +Meta = { + = "" +} + + +Routes = [ + { + Match { + HTTP { + PathExact = "" ## cannot specify with PathPrefix or PathRegex + }, + HTTP { + PathPrefix = "" ## cannot specify with PathExact or PathRegex + }, + HTTP { + PathRegex = "" ## cannot specify with PathPrefix or PathExact + }, + HTTP { + Methods = ["GET", "POST", "PUT"] + }, + HTTP { + Header = [ ## do not specify Present, Exact, Prefix, Suffix, and Regex in a single Header + { + Name = "" ## required when specifying Routes.Match.HTTP.Header + Present = false + Exact = "" + Prefix = "" + Suffix = "" + Regex = "" + Invert = false + } + ] + } + HTTP { + QueryParam = [ ## do not specify Present, Exact, and Regex in a single QueryParam + Name = "" ## required when specifying Routes.Match.HTTP.Header + Present = false + Exact = "" + Regex = "" + ] + } + }, + + + Destination { + Service = "" + ServiceSubset = "" + Namespace = "" + Partition = "" + PrefixRewrite = "" ## required specifying either Routes.Match.HTTP.PathPrefix or Routes.Match.HTTP.PathExact + RequestTimeout = 0 + IdleTimeout = 0 + NumRetries = 1 + RetryOnConnectFailure = false + RetryOn = ["reset", "unavailable"] + RetryOnStatusCodes = [500, 502, 503] + RequestHeaders = { + Set = { + "X-Web-Version" : "" + } + } + ResponseHeaders = { + Set = { + "X-Web-Version" : "" + } + } + } + } +] +``` + + + + + +```json +{ + "Kind": "service-router", // required + "Name": "", // required + "Namespace": "", + "Partition": "", + "Meta": { + "": "" + }, + + "Routes": [ + { + "Match": { + "HTTP": { + "PathExact": "" // cannot specify with PathPrefix or PathRegex + }, + "HTTP": { + "PathPrefix": "" // cannot specify with PathExact or PathRegex + }, + "HTTP": { + "PathRegex": "" // cannot specify with PathPrefix or PathExact + }, + "HTTP": { + "Methods": ["GET", "POST", "PUT"] + }, + "HTTP": { + "Header": [ // do not specify Present, Exact, Prefix, Suffix, and Regex in a single Header + { + "Name": "", // required when specifying Routes.Match.HTTP.Header + "Present": false, + "Exact": "", + "Prefix": "", + "Suffix": "", + "Regex": "", + "Invert": false, + } + ] + }, + "HTTP": { + "QueryParam": [ // do not specify Present, Exact, and Regex in a single QueryParam + "Name": "", // required when specifying Routes.Match.HTTP.Header + "Present": false, + "Exact": "", + "Regex": "" + ] + } + }, + + "Destination": { + "Service": "", + "ServiceSubset": "", + "Namespace": "", + "Partition": "", + "PrefixRewrite": "", // required specifying either Routes.Match.HTTP.PathPrefix or Routes.Match.HTTP.PathExact + "RequestTimeout": 0, + "IdleTimeout": 0, + "NumRetries": 1, + "RetryOnConnectFailure": false, + "RetryOn": ["reset", "unavailable"], + "RetryOnStatusCodes": [500, 502, 503], + "RequestHeaders": { + "Set": { + "X-Web-Version" : "" + } + }, + "ResponseHeaders": { + "Set": { + "X-Web-Version" : "" + } + } + } + } + ] +} +``` + + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 # required +kind: ServiceRouter # required +metadata: + name: + namespace: +spec: + routes: + - match: + http: + pathExact: ## cannot specify with pathPrefix or pathRegex + http: + pathPrefix: ## cannot specify with pathExact or pathRegex + http: + pathRegex: ## cannot specify with pathPrefix or pathExact + http: + methods: [GET, POST, PUT] + http: + header: ## do not specify present, exact, prefix, suffix, and regex in a single header + - name: ## required when specifying spec.routes.match.http.header + present: false + exact: + prefix: + suffix: + regex: + invert: false + http: + queryParam: ## do not specify present, exact, and regex in a single queryParam + - name: ## required when specifying spec.routes.match.http.header + present: false + exact: + regex: + + destination: + service: + serviceSubset: + namespace: + partition: + prefixRewrite: ## required specifying either spec.routes.match.http.pathPrefix or spec.routes.match.http.pathExact + requestTimeout: 0 + idleTimeout: 0 + numRetries: 1 + retryOnConnectFailure: false + retryOn: ['reset'] + retryOnStatusCodes: [500, 502, 503] + requestHeaders: + set: + x-Web-Version: + responseHeaders: + set: + x-Web-Version: +``` + + + + + +## Specification + +This section provides details about the fields you can configure in the service router configuration entry. + + + + + +### `Kind` + +Specifies the type of configuration entry to implement. + +#### Values + +- Default: none +- This field is required. +- Data type: String value that must be set to `service-router`. + +### `Name` + +Specifies a name for the configuration entry. The name is metadata that you can use to reference the configuration entry when performing Consul operations with the [`consul config` command](/consul/commands/config). + +#### Values + +- Default: None +- Data type: String + +### `Namespace` + +Specifies the namespace to apply the configuration entry to. Refer to [Namespaces](/consul/docs/enterprise/namespaces) for additional information about Consul namespaces. + +#### Values + +- Default: None +- Data type: String + +### `Partition` + +Specifies the admin partition to apply the configuration entry to. Refer to [Admin partitions](/consul/docs/enterprise/admin-partitions) for additional information. + +#### Values + +- Default: `default` +- Data type: String + +### `Meta` + +Specifies key-value pairs to add to the KV store. + +#### Values + +- Default: None +- Data type: Map of one or more key-value pairs + - ``: String + - ``: String or integer + +### `Routes` + +Defines the possible routes for L7 requests. Consul evaluates traffic against the list of routes in their order of appearance in the configuration entry. When multiple routes satisfy the request, Consul uses the first route that matches. Traffic that fails to match any of the provided routes is routed to the default service. + +#### Values + +- Default: None +- Data type: List that can contain the following parameters: + +- [`Match`](#routes-match) +- [`Destination`](#routes-destination) + +### `Routes[].Match` + +Describes a set of criteria that Consul compares incoming L7 traffic with. If empty or omitted, it acts as a catch-all. + +#### Values + +- Default: None +- Data type: Map that contains the [`Routes[].Match{}.HTTP`](#routes-match-http) parameter. + +### `Routes[].Match{}.HTTP` + +Specifies a set of HTTP criteria used to evaluate incoming L7 traffic for matches. + +When matching on the HTTP request path, you can only match on one path at a time. Do not configure `PathExact`, `PathPrefix`, and `PathRegex` in a single HTTP map. + +#### Values + +- Default: None +- Data type: Map that can contain the following parameters: + + - [`PathExact`](#routes-match-http-pathexact) + - [`PathPrefix`](#routes-match-http-pathprefix) + - [`PathRegex`](#routes-match-http-pathregex) + - [`Methods`](#routes-match-http-methods) + - [`Header`](#routes-match-http-header) + - [`QueryParam`](#routes-match-http-queryparam) + +### `Routes[].Match{}.HTTP{}.PathExact` + +Specifies the exact path to match on the HTTP request path. When using this field, do not configure `PathPrefix` or `PathRegex` in the same HTTP map. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Match{}.HTTP{}.PathPrefix` + +Specifies the path prefix to match on the HTTP request path. When using this field, do not configure `PathExact` or `PathRegex` in the same HTTP map. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Match{}.HTTP{}.PathRegex` + +Specifies a regular expression to match on the HTTP request path. When using this field, do not configure `PathExact` or `PathPrefix` in the same HTTP map. The syntax for the regular expression field is proxy-specific. When [using Envoy](/consul/docs/connect/proxies/envoy), refer to [the documentation for Envoy v1.11.2 or newer](https://github.com/google/re2/wiki/Syntax) or [the documentation for Envoy v1.11.1 or older](https://en.cppreference.com/w/cpp/regex/ecmascript), depending on the version of Envoy you use. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Match{}.HTTP{}.Methods` + +Specifies HTTP methods that the match applies to. If not specified, the request matches on all HTTP methods. If provided, the name must be a valid method formatted as a string. + +String values must be a valid [HTTP request method](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods). + +#### Values + +- Default: None +- Data type: List of strings. Each string must match one of the following values: + - `GET` + - `HEAD` + - `POST` + - `PUT` + - `DELETE` + - `CONNECT` + - `OPTIONS` + - `TRACE` + - `PATCH` + +### `Routes[].Match{}.HTTP{}.Header` + +Specifies information in the HTTP request header to match with. When more than one field is configured, all criteria must match for the service routing to apply. + +When using this field, do not configure `Present`, `Exact`, `Prefix`, `Suffix`, and `Regex` in the same HTTP map. You can use only one of these fields at a time when configuring match criteria for HTTP headers, as they are mutually exclusive with one another. + +#### Values + +- Default: None +- Data type: List containing one or more of the following parameters: + + - [`Name`](#routes-match-http-header-name) + - [`Present`](#routes-match-http-header-present) + - [`Exact`](#routes-match-http-header-exact) + - [`Prefix`](#routes-match-http-header-prefix) + - [`Suffix`](#routes-match-http-header-suffix) + - [`Regex`](#routes-match-http-header-regex) + - [`Invert`](#routes-match-http-header-invert) + +### `Routes[].Match{}.HTTP{}.Header[].Name` + +Specifies the name of the HTTP header to match. This field is required when using [`Routes[].Match{}.HTTP{}.Header`](#routes-match-http-header). + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Match{}.HTTP{}.Header[].Present` + +Specifies that a request matches when the value in the `Name` field is present anywhere in the HTTP header. When using this field, do not configure `Exact`, `Prefix`, `Suffix`, or `Regex` in the same HTTP map. + +#### Values + +- Default: `false` +- Data type: Boolean + +### `Routes[].Match{}.HTTP{}.Header[].Exact` + +Specifies that a request matches when the header with the given name is this exact value. When using this field, do not configure `Present`, `Prefix`, `Suffix`, or `Regex` in the same HTTP map. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Match{}.HTTP{}.Header[].Prefix` + +Specifies that a request matches when the header with the given name has this prefix. When using this field, do not configure `Present`, `Exact`, `Suffix`, or `Regex` in the same HTTP map. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Match{}.HTTP{}.Header[].Suffix` + +Specifies that a request matches when the header with the given name has this suffix. When using this field, do not configure `Present`, `Exact`, `Prefix`, or `Regex` in the same HTTP map. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Match{}.HTTP{}.Header[].Regex` + +Specifies that a request matches when the header with the given name matches this regular expression. When using this field, do not configure `Present`, `Exact`, `Prefix`, or `Suffix` in the same HTTP map . The syntax for the regular expression field is proxy-specific. When [using Envoy](/consul/docs/connect/proxies/envoy), refer to [the documentation for Envoy v1.11.2 or newer](https://github.com/google/re2/wiki/Syntax) or [the documentation for Envoy v1.11.1 or older](https://en.cppreference.com/w/cpp/regex/ecmascript), depending on the version of Envoy you use. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Match{}.HTTP{}.Header[].Invert` + +Specifies that the logic for the HTTP header match should be inverted. Requests with matching criteria are not routed. + +#### Values + +- Default: `false` +- Data type: Boolean + +### `Routes[].Match{}.HTTP{}.QueryParam` + +Specifies information to match to on HTTP query parameters. When more than one field is configured, all criteria must match for the service routing to apply. + +When using this field, do not configure `Present`, `Exact`, and `Regex` in a single map. You can use only one of these fields at a time when configuring match criteria for HTTP query parameters, as they are mutually exclusive with one another. + +#### Values + +- Default: None +- Data type: List of maps. Each map can contain one or more of the following parameters: + + - [`Name`](#routes-match-http-queryparam-name) + - [`Present`](#routes-match-http-queryparam-present) + - [`Exact`](#routes-match-http-queryparam-exact) + - [`Regex`](#routes-match-http-queryparam-regex) + +### `Routes[].Match{}.HTTP{}.QueryParam[].Name` + +Specifies the name of the HTTP query parameter to match. This value is required when using [`Routes[].Match{}.HTTP{}.QueryParam`](#routes-match-http-queryparam). + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Match{}.HTTP{}.QueryParam[].Present` + +Specifies that a request matches when the value in the `Name` field is present anywhere in the HTTP query parameter. When using this field, do not configure `Exact` or `Regex` in the same map. + +#### Values + +- Default: `false` +- Data type: Boolean + +### `Routes[].Match{}.HTTP{}.QueryParam[].Exact` + +Specifies that a request matches when the query parameter with the given name is this exact value. When using this field, do not configure `Present` or `Regex` in the same map. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Match{}.HTTP{}.QueryParam[].Regex` + +Specifies that a request matches when the query parameter with the given name matches this regular expression. When using this field, do not configure `Present` or `Exact` in the same map. The syntax for the regular expression field is proxy-specific. When [using Envoy](/consul/docs/connect/proxies/envoy), refer to [the documentation for Envoy v1.11.2 or newer](https://github.com/google/re2/wiki/Syntax) or [the documentation for Envoy v1.11.1 or older](https://en.cppreference.com/w/cpp/regex/ecmascript), depending on the version of Envoy you use. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Destination` + +Specifies the target service to route matching requests to, as well as behavior for the request to follow when routed. + +#### Values + +- Default: None +- Data type: Map containing one or more of the following parameters: + + - [`Service`](#routes-destination-service) + - [`ServiceSubset`](#routes-destination-servicesubset) + - [`Namespace`](#routes-destination-namespace) + - [`Partition`](#routes-destination-partition) + - [`PrefixRewrite`](#routes-destination-prefixrewrite) + - [`RequestTimeout`](#routes-destination-requesttimeout) + - [`IdleTimeout`](#routes-destination-idletimeout) + - [`NumRetries`](#routes-destination-numretries) + - [`RetryOnConnectFailure`](#routes-destination-retryonconnectfailure) + - [`RetryOn`](#routes-destination-retryon) + - [`RetryOnStatusCodes`](#routes-destination-retryonstatuscodes) + - [`RequestHeaders`](#routes-destination-requestheaders) + - [`ResponseHeaders`](#routes-destination-responseheaders) + +### `Routes[].Destination{}.Service` + +Specifies the name of the service to resolve. If this parameter is not specified, the default service name is inherited from the configuration entry’s [`Name` field](#name). + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Destination{}.ServiceSubset` + +Specifies a named subset of the given service to resolve instead of the one defined as that service's `DefaultSubset` in the [service resolver configuration entry](/consul/docs/connect/config-entries/service-resolver). If this parameter is not specified, the default subset is used. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Destination{}.Namespace` + +Specifies the Consul namespace to resolve the service from instead of the current namespace. If this parameter is not specified, the current namespace is used. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Destination{}.Partition` + +Specifies the Consul admin partition to resolve the service from instead of the current partition. If this parameter is not specified, the current partition is used. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Destination{}.PrefixRewrite` + +Specifies rewrites to the HTTP request path before proxying it to its final destination. This field requires that either [`Routes[].Match{}.HTTP{}.PathPrefix`](#routes-match-http-pathprefix) or [`Routes[].Match{}.HTTP{}.PathExact`](#routes-match-http-pathexact) be configured on this route. + +#### Values + +- Default: None +- Data type: String + +### `Routes[].Destination{}.RequestTimeout` + +Specifies the total amount of time permitted for the entire downstream request to be processed, including retry attempts. + +#### Values + +- Default: `0` +- Data type: Integer + +### `Routes[].Destination{}.IdleTimeout` + +Specifies the total amount of time permitted for the request stream to be idle. + +#### Values + +- Default: `0` +- Data type: Integer + +### `Routes[].Destination{}.NumRetries` + +Specifies the number of times to retry the request when a retry condition occurs. Configure this field and other retry fields in `Destination` to configure the logic for retry attempts. For examples, refer to the [retry logic example configurations](#retry-logic). You cannot set the value to `0`. To disable retries, unset all other retry settings: `RetryOnConnectFailure`, `RetryOn`, `RetryOnStatusCodes`. + +#### Values + +- Default: `1` +- Data type: Integer + +### `Routes[].Destination{}.RetryOnConnectFailure` + +Specifies that connection failure errors that trigger a retry request. Configure this field and other retry fields in `Destination` to configure the logic for retry attempts. For examples, refer to the [retry logic example configurations](#retry-logic). + +#### Values + +- Default: `false` +- Data type: Boolean + +### `Routes[].Destination{}.RetryOn[]` + +Specifies a list of conditions for Consul to retry requests based on the response from an upstream service. Configure this field and other retry fields in the `Destination` object to configure the logic for retry attempts. For examples, refer to the [retry logic example configurations](#retry-logic). + +The following retry conditions are supported: + +| Conditions | Description | +| :------------------- | :------------------------------------------------------------------------------------------------------- | +| `5xx` | Consul retries the request when an upstream responds with any 5xx error code or does not respond at all. | +| `gateway-error` | Consul retries the request when the upstream responds with a 502, 503, or 504 error. | +| `reset` | Consul retries the request when the upstream does not respond at all. | +| `connect-failure` | Consul retries the request when the connection to the upstream fails. | +| `envoy-ratelimited` | Consul retries the request when the header `x-envoy-ratelimited` is present. | +| `retriable-4xx` | Consul retries the request when the upstream responds with a retriable 4xx code. | +| `refused-stream` | Consul retries the request when the upstream resets the stream with a `REFUSED_STREAM` error code. | +| `cancelled` | Consul retries the request when the gRPC status code in the response headers is `cancelled`. | +| `deadline-exceeded` | Consul retries the request when the gRPC status code in the response headers is `deadline-exceeded`. | +| `internal` | Consul retries the request when the gRPC status code in the response headers is `internal`. | +| `resource-exhausted` | Consul retries the request when the gRPC status code in the response headers is `resource-exhausted`. | +| `unavailable` | Consul retries the request when the gRPC status code in the response headers is `unavailable`. | + +#### Values + +- Default: None +- Data type: List of strings. Strings must match one of the following values: + + - `5xx` + - `gateway-error` + - `reset` + - `connect-failure` + - `envoy-ratelimited` + - `retriable-4xx` + - `refused-stream` + - `cancelled` + - `deadline-exceeded` + - `internal` + - `resource-exhausted` + - `unavailable` + +### `Routes[].Destination{}.RetryOnStatusCodes` + +Specifies a list of integers for [HTTP response status codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) that trigger a retry request. Configure this field and other retry fields in `Destination` to configure the logic for retry attempts. For examples, refer to the [retry logic example configurations](#retry-logic) + +#### Values + +- Default: None +- Data type: List of integers + +### `Routes[].Destination{}.RequestHeaders` + +Specifies a set of HTTP-specific header modification rules applied to requests routed with the service router. You cannot configure request headers if the listener protocol is set to `tcp`. + +#### Values + +- Default: None +- Values: Object containing one or more fields that define header modification rules: + - `Add`: Map of one or more key-value pairs. + - `Set`: Map of one or more key-value pairs. + - `Remove`: Map of one or more key-value pairs. + +The following table describes how to configure values for request headers: + +| Rule | Description | Type | +| --- | --- | --- | +| `Add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `Set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `Remove` | Defines a list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | list of strings | + +#### Use variable placeholders + +For `Add` and `Set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated when the routing occurs. + +### `Routes[].Destination{}.ResponseHeaders` + +Specifies a set of HTTP-specific header modification rules applied to responses routed with the service router. You cannot configure request headers if the listener protocol is set to `tcp`. + +#### Values + +- Default: None +- Values: Object containing one or more fields that define header modification rules: + - `Add`: Map of one or more string key-value pairs. + - `Set`: Map of one or more string key-value pairs. + - `Remove`: Map of one or more string key-value pairs. + +The following table describes how to configure values for response headers: + +| Rule | Description | Type | +| --- | --- | --- | +| `Add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `Set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `Remove` | Defines a list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | list of strings | + +#### Use variable placeholders + +For `Add` and `Set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated when the routing occurs. + + + + + +### `apiVersion` + +Kubernetes-only parameter that specifies the version of the Consul API that the configuration entry maps to Kubernetes configurations. The value must be `consul.hashicorp.com/v1alpha1`. + +### `kind` + +Specifies the type of configuration entry to implement. + +#### Values + +- Default: None +- This field is required. +- Data type: String value that must be set to `ServiceRouter`. + +### `metadata.name` + +Specifies a name for the configuration entry. The name is metadata that you can use to reference the configuration entry when performing Consul operations, such as applying a configuration entry to a specific cluster. + +#### Values + +- Default: None +- Data type: String + +### `metadata.namespace` + +Specifies the Consul namespace to use for resolving the service. You can map Consul namespaces to Kubernetes Namespaces in different ways. Refer to [Custom Resource Definitions (CRDs) for Consul on Kubernetes](/consul/docs/k8s/crds#consul-enterprise) for additional information. + +#### Values + +- Default: None +- Data type: String + +### `spec` + +Map that contains the details about the `ServiceRouter` configuration entry. The `apiVersion`, `kind`, and `metadata` fields are siblings of the spec field. All other configurations are children. + +#### Values + +- Default: None +- This field is required. +- Data type: Object containing [`spec.routes`](#spec-routes) configuration + +### `spec.meta` + +Specifies key-value pairs to add to the KV store. + +#### Values + +- Default: none +- Data type: Map of one or more key-value pairs + - keys: String + - values: String, integer, or float + +### `spec.routes` + +Defines the possible routes for L7 requests. Consul evaluates traffic against the list of routes in their order of appearance in the configuration entry. When multiple routes satisfy the request, Consul uses the first route that matches. Traffic that fails to match any of the provided routes is routed to the default service. + +#### Values + +- Default: None +- Data type: List that can contain the following parameters: + +- [`match`](#spec-routes-match) +- [`destination`](#spec-routes-destination) + +### `spec.routes[].match` + +Describes a set of criteria that Consul compares incoming L7 traffic with. If empty or omitted, it acts as a catch-all. + +#### Values + +- Default: None +- Data type: Map that contains the [`spec.routes[].match.http`](#spec-routes-match-http) parameter. + +### `spec.routes[].match.http` + +Specifies a set of HTTP criteria used to evaluate incoming L7 traffic for matches. + +When matching on the HTTP request path, you can only match on one path at a time. Do not configure `pathExact`, `pathPrefix`, and `pathRegex` in a single HTTP map. + +#### Values + +- Default: None +- Data type: Map that can contain the following parameters: + + - [`pathExact`](#spec-routes-match-http-pathexact) + - [`pathPrefix`](#spec-routes-match-http-pathprefix) + - [`pathRegex`](#spec-routes-match-http-pathregex) + - [`methods`](#spec-routes-match-http-methods) + - [`header`](#spec-routes-match-http-header) + - [`queryParam`](#spec-routes-match-http-queryparam) + +### `spec.routes[].match.http.pathExact` + +Specifies the exact path to match on the HTTP request path. When using this field, do not configure `pathPrefix` or `pathRegex` in the same HTTP map. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].match.http.pathPrefix` + +Specifies the path prefix to match on the HTTP request path. When using this field, do not configure `pathExact` or `pathRegex` in the same HTTP map. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].match.http.pathRegex` + +Specifies a regular expression to match on the HTTP request path. When using this field, do not configure `pathExact` or `pathPrefix` in the same HTTP map. The syntax for the regular expression field is proxy-specific. When [using Envoy](/consul/docs/connect/proxies/envoy), refer to [the documentation for Envoy v1.11.2 or newer](https://github.com/google/re2/wiki/Syntax) or [the documentation for Envoy v1.11.1 or older](https://en.cppreference.com/w/cpp/regex/ecmascript), depending on the version of Envoy you use. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].match.http.methods` + +Specifies HTTP methods that the match applies to. If not specified, the request matches on all HTTP methods. If provided, the name must be a valid method formatted as a string. + +String values must be a valid [HTTP request method](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods). + +#### Values + +- Default: None +- Data type: List of strings. Each string must match one of the following values: + - `GET` + - `HEAD` + - `POST` + - `PUT` + - `DELETE` + - `CONNECT` + - `OPTIONS` + - `TRACE` + - `PATCH` + +### `spec.routes[].match.http.header` + +Specifies information to match to in the HTTP request header. When more than one field is configured, all criteria must match for the service routing to apply. + +When using this field, do not configure `present`, `exact`, `prefix`, `suffix`, and `regex` in the same HTTP map. You can use only one of these fields at a time when configuring match criteria for HTTP headers, as they are mutually exclusive with one another. + +#### Values + +- Default: None +- Data type: List containing one or more of the following parameters: -If a router is not explicitly configured or is configured with no routes then -the system behaves as if a router were configured sending all traffic to a -service of the same name. + - [`name`](#spec-routes-match-http-header-name) + - [`present`](#spec-routes-match-http-header-present) + - [`exact`](#spec-routes-match-http-header-exact) + - [`prefix`](#spec-routes-match-http-header-prefix) + - [`suffix`](#spec-routes-match-http-header-suffix) + - [`regex`](#spec-routes-match-http-header-regex) + - [`invert`](#spec-routes-match-http-header-invert) -## Requirements +### `spec.routes[].match.http.header.name` -- Consul [service mesh connect](/consul/docs/connect/configuration) enabled services -- Service to service communication over the protocol `http` +Specifies the name of the HTTP header to match. This field is required when using [`spec.routes[].match.http.header`](#spec-routes-match-http-header). -## Interaction with other Config Entries +#### Values -- Service router config entries are a component of [L7 Traffic - Management](/consul/docs/connect/l7-traffic). +- Default: None +- Data type: String -- Service router config entries are restricted to only services that define - their protocol as HTTP-based via a corresponding - [`service-defaults`](/consul/docs/connect/config-entries/service-defaults) config - entry or globally via - [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults) . +### `spec.routes[].match.http.header.present` -- Any route destination that omits the `ServiceSubset` field is eligible for - splitting via a - [`service-splitter`](/consul/docs/connect/config-entries/service-splitter) should - one be configured for that service, otherwise resolution proceeds according - to any configured - [`service-resolver`](/consul/docs/connect/config-entries/service-resolver). +Specifies that a request matches when the value in the `Name` field is present anywhere in the HTTP header. When using this field, do not configure `exact`, `prefix`, `suffix`, or `regex` in the same HTTP map. -## UI +#### Values -Once a `service-router` is successfully entered, you can view it in the UI. Service routers, service splitters, and service resolvers can all be viewed by clicking on your service then switching to the _routing_ tab. +- Default: `false` +- Data type: Boolean -![screenshot of service router in the UI](/img/l7-routing/Router.png) +### `spec.routes[].match.http.header.exact` -## Sample Config Entries +Specifies that a request matches when the header with the given name is this exact value. When using this field, do not configure `present`, `prefix`, `suffix`, or `regex` in the same HTTP map. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].match.http.header.prefix` + +Specifies that a request matches when the header with the given name has this prefix. When using this field, do not configure `present`, `exact`, `suffix`, or `regex` in the same HTTP map. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].match.http.header.suffix` + +Specifies that a request matches when the header with the given name has this suffix. When using this field, do not configure `present`, `exact`, `prefix`, or `regex` in the same HTTP map. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].match.http.header.regex` + +Specifies that a request matches when the header with the given name matches this regular expression. When using this field, do not configure `present`, `exact`, `prefix`, or `suffix` in the same HTTP map . The syntax for the regular expression field is proxy-specific. When [using Envoy](/consul/docs/connect/proxies/envoy), refer to [the documentation for Envoy v1.11.2 or newer](https://github.com/google/re2/wiki/Syntax) or [the documentation for Envoy v1.11.1 or older](https://en.cppreference.com/w/cpp/regex/ecmascript), depending on the version of Envoy you use. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].match.http.header.invert` + +Specifies that the logic for the HTTP header match should be inverted. Requests with matching criteria are not routed. + +#### Values + +- Default: `false` +- Data type: Boolean + +### `spec.routes[].match.http.queryParam` + +Specifies information to match to on HTTP query parameters. When more than one field is configured, all criteria must match for the service routing to apply. + +When using this field, do not configure `present`, `exact`, and `regex` in a single map. You can use only one of these fields at a time when configuring match criteria for HTTP query parameters, as they are mutually exclusive with one another. + +#### Values + +- Default: None +- Data type: List of maps. Each map can contain one or more of the following parameters: + + - [`name`](#spec-routes-match-http-queryparam-name) + - [`present`](#spec-routes-match-http-queryparam-present) + - [`regex`](#spec-routes-match-http-queryparam-regex) + +### `spec.routes[].match.http.queryParam[].name` + +Specifies the name of the HTTP query parameter to match. This value is required when using [`spec.routes[].match.http.queryParam`](#spec-routes-match-http-queryparam). + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].match.http.queryParam[].present` + +Specifies that a request matches when the value in the `name` field is present anywhere in the HTTP query parameter. When using this field, do not configure `exact` or `regex` in the same map. + +#### Values + +- Default: `false` +- Data type: Boolean + +### `spec.routes[].match.http.queryParam[].exact` + +Specifies that a request matches when the query parameter with the given name is this exact value. When using this field, do not configure `present` or `regex` in the same map. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].match.http.queryParam[].regex` + +Specifies that a request matches when the query parameter with the given name matches this regular expression. When using this field, do not configure `present` or `exact` in the same map. The syntax for the regular expression field is proxy-specific. When [using Envoy](/consul/docs/connect/proxies/envoy), refer to [the documentation for Envoy v1.11.2 or newer](https://github.com/google/re2/wiki/Syntax) or [the documentation for Envoy v1.11.1 or older](https://en.cppreference.com/w/cpp/regex/ecmascript), depending on the version of Envoy you use. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].destination` + +Specifies the target service to route matching requests to, as well as behavior for the request to follow when routed. + +#### Values + +- Default: None +- Data type: Map containing one or more of the following parameters: + + - [`service`](#spec-routes-destination-service) + - [`serviceSubset`](#spec-routes-destination-servicesubset) + - [`namespace`](#spec-routes-destination-namespace) + - [`partition`](#spec-routes-destination-partition) + - [`prefixRewrite`](#spec-routes-destination-prefixrewrite) + - [`requestTimeout`](#spec-routes-destination-requesttimeout) + - [`numRetries`](#spec-routes-destination-numretries) + - [`numRetries`](#spec-routes-destination-numretries) + - [`retryOnConnectFailure`](#spec-routes-destination-retryonconnectfailure) + - [`retryOn`](#spec-routes-destination-retryon) + - [`retryOnStatusCodes`](#spec-routes-destination-retryonstatuscodes) + - [`requestHeaders`](#spec-routes-destination-requestheaders) + - [`responseHeaders`](#spec-routes-destination-responseheaders) + +### `spec.routes[].destination.service` + +Specifies the name of the service to resolve. If this parameter is not specified, the default service name is inherited from the configuration entry’s [`metadata.name` field](#metadata-name). + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].destination.serviceSubset` + +Specifies a named subset of the given service to resolve instead of the one defined as that service's `defaultSubset` in the [service resolver configuration entry](/consul/docs/connect/config-entries/service-resolver). If this parameter is not specified, the default subset is used. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].destination.namespace` + +Specifies the Consul namespace to resolve the service from instead of the current namespace. If this parameter is not specified, the current namespace is used. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].destination.partition` + +Specifies the Consul admin partition to resolve the service from instead of the current partition. If this parameter is not specified, the current partition is used. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].destination.prefixRewrite` + +Specifies rewrites to the HTTP request path before proxying it to its final destination. This field requires that either [`spec.routes[].match.http.pathPrefix`](#spec-routes-match-http-pathprefix) or [`spec.routes[].match.http.pathExact`](#spec-routes-match-http-pathexact) be configured on this route. + +#### Values + +- Default: None +- Data type: String + +### `spec.routes[].destination.requestTimeout` + +Specifies the total amount of time permitted for the entire downstream request to be processed, including retry attempts. + +#### Values + +- Default: `0` +- Data type: Integer + +### `spec.routes[].destination.idleTimeout` + +Specifies the total amount of time permitted for the request stream to be idle. + +#### Values + +- Default: `0` +- Data type: Integer + +### `spec.routes[].destination.numRetries` + +Specifies the number of times to retry the request when a retry condition occurs. Configure this field and other retry fields in `spec.routes.destination` to configure the logic for retry attempts. For examples, refer to the [retry logic example configurations](#retry-logic). You cannot set the value to `0`. To disable retries, unset all other retry settings: `retryOnConnectFailure`, `retryOn`, `retryOnStatusCodes`. + +#### Values + +- Default: `1` +- Data type: Integer + +### `spec.routes[].destination.retryOnConnectFailure` + +Specifies that connection failure errors that trigger a retry request. Configure this field and other retry fields in `spec.routes[].destination` to configure the logic for retry attempts. For examples, refer to the [retry logic example configurations](#retry-logic). + +#### Values + +- Default: `false` +- Data type: Boolean + +### `spec.routes[].destination.retryOn` + +Specifies a list of conditions for Consul to retry requests based on the response from an upstream service. Configure this field and other retry fields in the `spec.routes[].destination` object to configure the logic for retry attempts. For examples, refer to the [retry logic example configurations](#retry-logic). + +The following retry conditions are supported: + +| Conditions | Description | +| :------------------- | :------------------------------------------------------------------------------------------------------- | +| `5xx` | Consul retries the request when an upstream responds with any 5xx error code or does not respond at all. | +| `gateway-error` | Consul retries the request when the upstream responds with a 502, 503, or 504 error. | +| `reset` | Consul retries the request when the upstream does not respond at all. | +| `connect-failure` | Consul retries the request when the connection to the upstream fails. | +| `envoy-ratelimited` | Consul retries the request when the header `x-envoy-ratelimited` is present. | +| `retriable-4xx` | Consul retries the request when the upstream responds with a retriable 4xx code. | +| `refused-stream` | Consul retries the request when the upstream resets the stream with a `REFUSED_STREAM` error code. | +| `cancelled` | Consul retries the request when the gRPC status code in the response headers is `cancelled`. | +| `deadline-exceeded` | Consul retries the request when the gRPC status code in the response headers is `deadline-exceeded`. | +| `internal` | Consul retries the request when the gRPC status code in the response headers is `internal`. | +| `resource-exhausted` | Consul retries the request when the gRPC status code in the response headers is `resource-exhausted`. | +| `unavailable` | Consul retries the request when the gRPC status code in the response headers is `unavailable`. | + +#### Values + +- Default: None +- Data type: List of strings. Strings must match one of the following values: + + - `5xx` + - `gateway-error` + - `reset` + - `connect-failure` + - `envoy-ratelimited` + - `retriable-4xx` + - `refused-stream` + - `cancelled` + - `deadline-exceeded` + - `internal` + - `resource-exhausted` + - `unavailable` + +### `spec.routes[].destination.retryOnStatusCodes` + +Specifies a list of integers for [HTTP response status codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) that trigger a retry request. Configure this field and other retry fields in `spec.routes[].destination` to configure the logic for retry attempts. For examples, refer to the [retry logic example configurations](#retry-logic) + +#### Values + +- Default: None +- Data type: List of integers + +### `spec.routes[].destination.requestHeaders` + +Specifies a set of HTTP-specific header modification rules applied to requests routed with the service router. You cannot configure request headers if the listener protocol is set to `tcp`. + +#### Values + +- Default: None +- Values: Object containing one or more fields that define header modification rules: + - `add`: Map of one or more key-value pairs. + - `set`: Map of one or more key-value pairs. + - `remove`: Map of one or more key-value pairs. + +The following table describes how to configure values for request headers: + +| Rule | Description | Type | +| --- | --- | --- | +| `add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `remove` | Defines a list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | list of strings | + +#### Use variable placeholders + +For `add` and `set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated when the routing occurs. + +### `spec.routes[].destination.responseHeaders` + +Specifies a set of HTTP-specific header modification rules applied to responses routed with the service router. You cannot configure request headers if the listener protocol is set to `tcp`. + +#### Values + +- Default: None +- Values: Object containing one or more fields that define header modification rules: + - `add`: Map of one or more string key-value pairs. + - `set`: Map of one or more string key-value pairs. + - `remove`: Map of one or more string key-value pairs. + +The following table describes how to configure values for response headers: + +| Rule | Description | Type | +| --- | --- | --- | +| `add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `remove` | Defines a list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | list of strings | + +#### Use variable placeholders + +For `add` and `set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated when the routing occurs. + + + + + +## Examples + +The following examples demonstrate common service router configuration patterns for specific use cases. ### Path prefix matching -Route HTTP requests with a path starting with `/admin` to a different service: +The following example routes HTTP requests for the `web` service to a service named `admin` when they have `/admin` at the start of their path. - + + ```hcl Kind = "service-router" @@ -66,10 +1325,13 @@ Routes = [ Service = "admin" } }, - # NOTE: a default catch-all will send unmatched traffic to "web" ] ``` + + + + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 kind: ServiceRouter @@ -82,9 +1344,12 @@ spec: pathPrefix: /admin destination: service: admin - # NOTE: a default catch-all will send unmatched traffic to "web" ``` + + + + ```json { "Kind": "service-router", @@ -104,13 +1369,15 @@ spec: } ``` - + + -### Header/query parameter matching +### Match a header and query parameter -Route HTTP requests with a special URL parameter or header to a canary subset: +The following example routes HTTP traffic to the `web` service to a subset of `canary` instances when the requests have `x-debug` in either the header or the URL parameter. - + + ```hcl Kind = "service-router" @@ -148,10 +1415,13 @@ Routes = [ ServiceSubset = "canary" } }, - # NOTE: a default catch-all will send unmatched traffic to "web" ] ``` + + + + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 kind: ServiceRouter @@ -175,9 +1445,12 @@ spec: destination: service: web serviceSubset: canary - # NOTE: a default catch-all will send unmatched traffic to "web" ``` + + + + ```json { "Kind": "service-router", @@ -219,14 +1492,15 @@ spec: } ``` - + + ### gRPC routing -Re-route a gRPC method to another service. Since gRPC method calls [are -HTTP/2](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md), we can use an HTTP path match rule to re-route traffic: +The following example routes gRPC requests to the `invoice-generator`service when they come from an HTTP path that is exact match for `mycompany.BillingService/GenerateInvoice`. Because gRPC method calls use HTTP/2, you can use an HTTP path match rule to re-route traffic. - + + ```hcl Kind = "service-router" @@ -243,10 +1517,13 @@ Routes = [ Service = "invoice-generator" } }, - # NOTE: a default catch-all will send unmatched traffic to "billing" ] ``` + + + + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 kind: ServiceRouter @@ -259,9 +1536,12 @@ spec: pathExact: /mycompany.BillingService/GenerateInvoice destination: service: invoice-generator - # NOTE: a default catch-all will send unmatched traffic to "billing" ``` + + + + ```json { "Kind": "service-router", @@ -281,21 +1561,25 @@ spec: } ``` - + + ### Retry logic -Enable retry logic by delegating this responsibility to Consul and the proxy. Review the [`ServiceRouteDestination`](#serviceroutedestination) block for more details. +The following example configures Consul so that when a request for the `orders` service passes through the service mesh, Consul routes the traffic to the `products` service or the `procurement` service based on the HTTP path that originated the request: - +- If it originates from the `/coffees` path, the request routes to the `products` service, times out after 15 seconds, and attempts 5 retries. +- If it originates from the `/orders` path, the request routes to the `procurement` services, times out after 10 seconds, and attempts 3 retries. + + ```hcl Kind = "service-router" Name = "orders" Routes = [ { - Match{ + Match { HTTP { PathPrefix = "/coffees" } @@ -303,14 +1587,14 @@ Routes = [ Destination { Service = "products" - RequestTimeout = "10s" - NumRetries = 3 + RequestTimeout = "15s" + NumRetries = 5 RetryOnConnectFailure = true RetryOn = ["reset"] } }, { - Match{ + Match { HTTP { PathPrefix = "/orders" } @@ -327,6 +1611,10 @@ Routes = [ ] ``` + + + + ```yaml apiVersion: consul.hashicorp.com/v1alpha1 kind: ServiceRouter @@ -339,8 +1627,8 @@ spec: pathExact: /coffees destination: service: products - requestTimeout: 10s - numRetries: 3 + requestTimeout: 15s + numRetries: 5 retryOnConnectFailure: true retryOn: ['reset'] - match: @@ -354,6 +1642,9 @@ spec: retryOn: ['reset'] ``` + + + ```json { @@ -367,8 +1658,8 @@ spec: } }, "Destination": { - "NumRetries": 3, - "RequestTimeout": "10s", + "NumRetries": 5, + "RequestTimeout": "15s", "RetryOnConnectFailure": true, "RetryOn": ["reset"], "Service": "procurement" @@ -392,413 +1683,6 @@ spec: } ``` - - -## Available Fields - -', - yaml: false, - }, - { - name: 'Namespace', - type: `string: "default"`, - enterprise: true, - description: - 'Specifies the namespace to which the configuration entry will apply.', - yaml: false, - }, - { - name: 'Partition', - type: `string: "default"`, - enterprise: true, - description: - 'Specifies the admin partition to which the configuration will apply.', - yaml: false, - }, - { - name: 'Meta', - type: 'map: nil', - description: - 'Specifies arbitrary KV metadata pairs. Added in Consul 1.8.4.', - yaml: false, - }, - { - name: 'metadata', - children: [ - { - name: 'name', - description: 'Set to the name of the service being configured.', - }, - { - name: 'namespace', - description: - 'If running Consul Open Source, the namespace is ignored (see [Kubernetes Namespaces in Consul OSS](/consul/docs/k8s/crds#consul-oss)). If running Consul Enterprise see [Kubernetes Namespaces in Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for more details.', - }, - ], - hcl: false, - }, - { - name: 'Routes', - type: 'array', - description: `The list of routes to consider when - processing L7 requests. The first route to match in the list is terminal and - stops further evaluation. Traffic that fails to match any of the provided - routes will be routed to the default service.`, - children: [ - { - name: 'Match', - type: 'ServiceRouteMatch: ', - description: `A set of criteria that can - match incoming L7 requests. If empty or omitted it acts as a catch-all.`, - children: [ - { - name: 'HTTP', - type: 'ServiceRouteHTTPMatch: ', - description: `A set of [HTTP-specific match criteria](#serviceroutehttpmatch).`, - }, - ], - }, - { - name: 'Destination', - type: 'ServiceRouteDestination: ', - description: `Controls [how to proxy](#serviceroutedestination) the matching request(s) to a service.`, - }, - ], - }, - ]} -/> - -### `ServiceRouteHTTPMatch` - -
    At most only one of `PathExact`, `PathPrefix`, or `PathRegex` may be configured.', - yaml: - 'Exact path to match on the HTTP request path.

    At most only one of `pathExact`, `pathPrefix`, or `pathRegex` may be configured.', - }, - }, - { - name: 'PathPrefix', - type: 'string: ""', - description: { - hcl: - 'Path prefix to match on the HTTP request path.

    At most only one of `PathExact`, `PathPrefix`, or `PathRegex` may be configured.', - yaml: - 'Path prefix to match on the HTTP request path.

    At most only one of `pathExact`, `pathPrefix`, or `pathRegex` may be configured.', - }, - }, - { - name: 'PathRegex', - type: 'string: ""', - description: { - hcl: - 'Regular expression to match on the HTTP request path.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `PathExact`, `PathPrefix`, or `PathRegex` may be configured.', - yaml: - 'Regular expression to match on the HTTP request path.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `pathExact`, `pathPrefix`, or `pathRegex` may be configured.', - }, - }, - { - name: 'Methods', - type: 'array', - description: - 'A list of HTTP methods for which this match applies. If unspecified all HTTP methods are matched. If provided the names must be a valid [method](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods).', - }, - { - name: 'Header', - type: 'array)', - description: - 'A set of criteria that can match on HTTP request headers. If more than one is configured all must match for the overall match to apply.', - children: [ - { - name: 'Name', - type: 'string: ', - description: 'Name of the header to match.', - }, - { - name: 'Present', - type: 'bool: false', - description: { - hcl: - 'Match if the header with the given name is present with any value.

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name is present with any value.

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Exact', - type: 'string: ""', - description: { - hcl: - 'Match if the header with the given name is this value.

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name is this value.

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Prefix', - type: 'string: ""', - description: { - hcl: - 'Match if the header with the given name has this prefix.

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name has this prefix.

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Suffix', - type: 'string: ""', - description: { - hcl: - 'Match if the header with the given name has this suffix.

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name has this suffix.

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Regex', - type: 'string: ""', - description: { - hcl: - 'Match if the header with the given name matches this pattern.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `Exact`, `Prefix`, `Suffix`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the header with the given name matches this pattern.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `exact`, `prefix`, `suffix`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Invert', - type: 'bool: false', - description: 'Inverts the logic of the match', - }, - ], - }, - { - name: 'QueryParam', - type: 'array)', - description: - 'A set of criteria that can match on HTTP query parameters. If more than one is configured all must match for the overall match to apply.', - children: [ - { - name: 'Name', - type: 'string: ', - description: 'The name of the query parameter to match on.', - }, - { - name: 'Present', - type: 'bool: false', - description: { - hcl: - 'Match if the query parameter with the given name is present with any value.

    At most only one of `Exact`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the query parameter with the given name is present with any value.

    At most only one of `exact`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Exact', - type: 'string: ""', - description: { - hcl: - 'Match if the query parameter with the given name is this value.

    At most only one of `Exact`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the query parameter with the given name is this value.

    At most only one of `exact`, `regex`, or `present` may be configured.', - }, - }, - { - name: 'Regex', - type: 'string: ""', - description: { - hcl: - 'Match if the query parameter with the given name matches this pattern.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `Exact`, `Regex`, or `Present` may be configured.', - yaml: - 'Match if the query parameter with the given name matches this pattern.

    The syntax is [described below](#regular-expression-syntax).

    At most only one of `exact`, `regex`, or `present` may be configured.', - }, - }, - ], - }, - ]} -/> - -### `ServiceRouteDestination` - -
    This requires that either `Match.HTTP.PathPrefix` or `Match.HTTP.PathExact` be configured on this route.', - yaml: - 'Defines how to rewrite the HTTP request path before proxying it to its final destination.

    This requires that either `match.http.pathPrefix` or `match.http.pathExact` be configured on this route.', - }, - }, - { - name: 'RequestTimeout', - type: 'duration: 0', - description: - 'The total amount of time permitted for the entire downstream request (and retries) to be processed.', - }, - { - name: 'IdleTimeout', - type: 'duration: 0', - description: - 'The total amount of time permitted for the request stream to be idle', - }, - { - name: 'NumRetries', - type: 'int: 1', - description: { - hcl: 'The number of times to retry the request when a retryable result occurs. You cannot set the value to `0`. To disable retries, unset all other retry settings (`RetryOnConnectFailure`, `RetryOn`, `RetryOnStatusCodes`)', - yaml: 'The number of times to retry the request when a retryable result occurs. You cannot set the value to `0`. To disable retries, unset all other retry settings (`retryOnConnectFailure`, `retryOn`, `retryOnStatusCodes`)', - } - }, - { - name: 'RetryOnConnectFailure', - type: 'bool: false', - description: 'Allows for connection failure errors to trigger a retry.', - }, - { - name: 'RetryOn', - type: 'array', - description: `Allows Consul to retry requests when they meet one of the following sets of conditions: - \`5xx\`, \`gateway-error\`, \`reset\`, \`connect-failure\`, \`envoy-ratelimited\`, \`retriable-4xx\`, - \`refused-stream\`, \`cancelled\`, \`deadline-exceeded\`, \`internal\`, \`resource-exhausted\`, or \`unavailable\``, - }, - { - name: 'RetryOnStatusCodes', - type: 'array', - description: - 'A list of HTTP response status codes that are eligible for retry.', - }, - { - name: 'RequestHeaders', - type: 'HTTPHeaderModifiers: ', - description: `A set of [HTTP-specific header modification rules](/consul/docs/connect/config-entries/service-router#httpheadermodifiers) - that will be applied to requests routed to this service. - This cannot be used with a \`tcp\` listener.`, - }, - { - name: 'ResponseHeaders', - type: 'HTTPHeaderModifiers: ', - description: `A set of [HTTP-specific header modification rules](/consul/docs/connect/config-entries/service-router#httpheadermodifiers) - that will be applied to responses from this service. - This cannot be used with a \`tcp\` listener.`, - }, - ]} -/> - -### `HTTPHeaderModifiers` - -: optional', - description: `The set of key/value pairs that specify header values to add. - Use header names as keys. Header names are _not_ case-sensitive. - If header values with the same name already exist, the value set here will - be appended and both will be present. - If Envoy is used as the proxy, the value may contain - [variable placeholders](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#custom-request-response-headers) for example - \`%DOWNSTREAM_REMOTE_ADDRESS%\` to interpolate dynamic request - metadata into the value added.`, - }, - { - name: 'Set', - type: 'map: optional', - description: `The set of key/value pairs that specify header values to add. - Use header names as keys. Header names are _not_ case-sensitive. - If header values with the same name already exist, the value set here will - _replace_ them. - If Envoy is used as the proxy, the value may contain - [variable placeholders](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#custom-request-response-headers) for example - \`%DOWNSTREAM_REMOTE_ADDRESS%\` to interpolate dynamic request - metadata into the value added.`, - }, - { - name: 'Remove', - type: 'array: optional', - description: `The set of header names to remove. Only headers - whose names are a case-insensitive exact match will be removed`, - }, - ]} -/> - -## ACLs - -Configuration entries may be protected by [ACLs](/consul/docs/security/acl). - -Reading a `service-router` config entry requires `service:read` on the resource. - -Creating, updating, or deleting a `service-router` config entry requires -`service:write` on the resource and `service:read` on any other service referenced by -name in these fields: - -- [`Routes[].Destination.Service`](#service) - -## Regular Expression Syntax - -The actual syntax of the regular expression fields described here is entirely -proxy-specific. - -When using [Envoy](/consul/docs/connect/proxies/envoy) as a proxy (the only supported proxy in Kubernetes), -the syntax for these fields is version specific: -| Envoy Version | Syntax | -| --------------- | ------------------------------------------------------------------- | -| 1.11.2 or newer | [documentation](https://github.com/google/re2/wiki/Syntax) | -| 1.11.1 or older | [documentation](https://en.cppreference.com/w/cpp/regex/ecmascript) | + + \ No newline at end of file diff --git a/website/content/docs/connect/config-entries/service-splitter.mdx b/website/content/docs/connect/config-entries/service-splitter.mdx index 4386fba281b..5d94a4e38a2 100644 --- a/website/content/docs/connect/config-entries/service-splitter.mdx +++ b/website/content/docs/connect/config-entries/service-splitter.mdx @@ -1,54 +1,575 @@ --- layout: docs -page_title: Service Splitter - Configuration Entry Reference -description: >- - The service splitter configuration entry kind defines how to divide service mesh traffic between service instances. Use the reference guide to learn about `""service-splitter""` config entry parameters and how it can be used for traffic management behaviors like canary rollouts, blue green deployment, and load balancing across environments. +page_title: Service Splitter Configuration Entry Reference +description: >- + Service splitter configuration entries are L7 traffic management tools for redirecting requests for a service to + multiple instances. Learn how to write `service-splitter` config entries in HCL or YAML with a specification + reference, configuration model, a complete example, and example code by use case. --- -# Service Splitter Configuration Entry +# Service Splitter Configuration Reference + +This reference page describes the structure and contents of service splitter configuration entries. Configure and apply service splitters to redirect a percentage of incoming traffic requests for a service to one or more specific service instances. + +## Configuration model + +The following list outlines field hierarchy, language-specific data types, and requirements in a service splitter configuration entry. Click on a property name to view additional details, including default values. + + + + + +- [`Kind`](#kind): string | required +- [`Name`](#name): string | required +- [`Namespace`](#namespace): string +- [`Partition`](#partition): string +- [`Meta`](#meta): map +- [`Splits`](#splits): map | required + - [`Weight`](#splits-weight): number | required + - [`Service`](#splits-service): string | required + - [`ServiceSubset`](#splits-servicesubset): string + - [`Namespace`](#splits-namespace): string + - [`Partition`](#splits-partition): string + - [`RequestHeaders`](#splits-requestheaders): map + - [`Add`](#splits-requestheaders): map + - [`Set`](#splits-requestheaders): map + - [`Remove`](#splits-requestheaders): map + - [`ResponseHeaders`](#splits-responseheaders): map + - [`Add`](#splits-responseheaders): map + - [`Set`](#splits-responseheaders): map + - [`Remove`](#splits-responseheaders): map + + + + + +- [`apiVersion`](#apiversion): string | required +- [`kind`](#kind): string | required +- [`metadata`](#metadata): object | required + - [`name`](#metadata-name): string | required + - [`namespace`](#metadata-namespace): string | optional +- [`spec`](#spec): object | required + - [`splits`](#spec-splits): list | required + - [`weight`](#spec-splits-weight): float32 | required + - [`service`](#spec-splits-service): string | required + - [`serviceSubset`](#spec-splits-servicesubset): string + - [`namespace`](#spec-splits-namespace): string + - [`partition`](#spec-splits-partition): string + - [`requestHeaders`](#spec-splits-requestheaders): HTTPHeaderModifiers + - [`add`](#spec-splits-requestheaders): map + - [`set`](#spec-splits-requestheaders): map + - [`remove`](#spec-splits-requestheaders): map + - [`responseHeaders`](#spec-splits-responseheaders): HTTPHeaderModifiers + - [`add`](#spec-splits-responseheaders): map + - [`set`](#spec-splits-responseheaders): map + - [`remove`](#spec-splits-responseheaders): map + + + + +## Complete configuration + +When every field is defined, a service splitter configuration entry has the following form: + + + + --> **v1.8.4+:** On Kubernetes, the `ServiceSplitter` custom resource is supported in Consul versions 1.8.4+.
    -**v1.6.0+:** On other platforms, this config entry is supported in Consul versions 1.6.0+. +```hcl +Kind = "service-splitter" ## string | required +Name = "config-entry-name" ## string | required +Namespace = "main" ## string +Partition = "partition" ## string +Meta = { ## map + key = "value" +} +Splits = [ ## list | required + { ## map + Weight = 90 ## number | required + Service = "service" ## string + ServiceSubset = "v1" ## string + Namespace = "target-namespace" ## string + Partition = "target-partition" ## string + RequestHeaders = { ## map + Set = { + "X-Web-Version" : "from-v1" + } + } + ResponseHeaders = { ## map + Set = { + "X-Web-Version" : "to-v1" + } + } + }, + { + Weight = 10 + Service = "service" + ServiceSubset = "v2" + Namespace = "target-namespace" + Partition = "target-partition" + RequestHeaders = { + Set = { + "X-Web-Version" : "from-v2" + } + } + ResponseHeaders = { + Set = { + "X-Web-Version" : "to-v2" + } + } + } +] +``` + +     + + + +```json +{ + "Kind" : "service-splitter", ## string | required + "Name" : "config-entry-name", ## string | required + "Namespace" : "main", ## string + "Partition" : "partition", ## string + "Meta" : { ## map + "_key_" : "_value_" + }, + "Splits" : [ ## list | required + { ## map + "Weight" : 90, ## number | required + "Service" : "service", ## string + "ServiceSubset" : "v1", ## string + "Namespace" : "target-namespace", ## string + "Partition" : "target-partition", ## string + "RequestHeaders" : { ## map + "Set" : { + "X-Web-Version": "from-v1" + } + }, + "ResponseHeaders" : { ## map + "Set" : { + "X-Web-Version": "to-v1" + } + } + }, + { + "Weight" : 10, + "Service" : "service", + "ServiceSubset" : "v2", + "Namespace" : "target-namespace", + "Partition" : "target-partition", + "RequestHeaders" : { + "Set" : { + "X-Web-Version": "from-v2" + } + }, + "ResponseHeaders" : { + "Set" : { + "X-Web-Version": "to-v2" + } + } + } + ] +} +``` + + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 # string | required +kind: ServiceSplitter # string | required +metadata: # object | required + name: config-entry-name # string | required + namespace: main # string +spec: + splits: # list + - weight: 90 # floating point | required + service: service # string + serviceSubset: v1 # string + namespace: target-namespace # string + partition: target-partition # string + requestHeaders: + set: + x-web-version: from-v1 # string + responseHeaders: + set: + x-web-version: to-v1 # string + - weight: 10 + service: service + serviceSubset: v2 + namespace: target-namespace + partition: target-partition + requestHeaders: + set: + x-web-version: from-v2 + responseHeaders: + set: + x-web-version: to-v2 +``` + + + +     + +## Specification + +This section provides details about the fields you can configure in the service splitter configuration entry. + + + + + +### `Kind` + +Specifies the type of configuration entry to implement. + +#### Values + +- Default: none +- This field is required. +- Data type: String value that must be set to `service-splitter`. + +### `Name` + +Specifies a name for the configuration entry. The name is metadata that you can use to reference the configuration entry when performing Consul operations, such as applying a configuration entry to a specific cluster. + +#### Values + +- Default: Defaults to the name of the node after writing the entry to the Consul server. +- This field is required. +- Data type: String + + +### `Namespace` + +Specifies the [namespace](/consul/docs/enterprise/namespaces) to apply the configuration entry. + +#### Values + +- Default: None +- Data type: String + +### `Partition` + +Specifies the [admin partition](/consul/docs/enterprise/admin-partitions) to apply the configuration entry. + +#### Values + +- Default: `Default` +- Data type: String + +### `Meta` + +Specifies key-value pairs to add to the KV store. + +#### Values + +- Default: none +- Data type: Map of one or more key-value pairs + - keys: String + - values: String, integer, or float + +### `Splits` + +Defines how much traffic to send to sets of service instances during a traffic split. + +#### Values + +- Default: None +- This field is required. +- Data type: list of objects that can contain the following fields: + - `Weight`: The sum of weights for a set of service instances must add up to 100. + - `Service`: This field is required. + - `ServiceSubset` + - `Namespace` + - `Partition` + - `RequestHeaders` + - `ResponseHeaders` + +### `Splits[].Weight` + +Specifies the percentage of traffic sent to the set of service instances specified in the [`Service`](#service) field. Each weight must be a floating integer between `0` and `100`. The smallest representable value is `.01`. The sum of weights across all splits must add up to `100`. + +#### Values + +- Default: `null` +- This field is required. +- Data type: Floating number from `.01` to `100`. + +### `Splits[].Service` + +Specifies the name of the service to resolve. + +#### Values + +- Default: Inherits the value of the [`Name`](#name) field. +- Data type: String + +### `Splits[].ServiceSubset` + +Specifies a subset of the service to resolve. A service subset assigns a name to a specific subset of discoverable service instances within a datacenter, such as `version2` or `canary`. All services have an unnamed default subset that returns all healthy instances. + +You can define service subsets in a [service resolver configuration entry](/consul/docs/connect/config-entries/service-resolver), which are referenced by their names throughout the other configuration entries. This field overrides the default subset value in the service resolver configuration entry. + +#### Values + +- Default: If empty, the `split` uses the default subset. +- Data type: String + +### `Splits[].Namespace` + +Specifies the [namespace](/consul/docs/enterprise/namespaces) to use in the FQDN when resolving the service. + +#### Values + +- Default: Inherits the value of [`Namespace`](#Namespace) from the top-level of the configuration entry. +- Data type: String + +### `Splits[].Partition` + +Specifies the [admin partition](/consul/docs/enterprise/admin-partitions) to use in the FQDN when resolving the service. + +#### Values + +- Default: By default, the `service-splitter` uses the [admin partition specified in the top-level configuration entry](#partition). +- Data type: String + +### `Splits[].RequestHeaders` + +Specifies a set of HTTP-specific header modification rules applied to requests routed with the service split. You cannot configure request headers if the listener protocol is set to `tcp`. Refer to [Set HTTP Headers](#set-http-headers) for an example configuration. + +#### Values + +- Default: None +- Values: Object containing one or more fields that define header modification rules + - `Add`: Map of one or more key-value pairs + - `Set`: Map of one or more key-value pairs + - `Remove`: Map of one or more key-value pairs + +The following table describes how to configure values for request headers: + +| Rule | Description | Type | +| --- | --- | --- | +| `Add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `Set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `Remove` | Defines an list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | list of strings | + +#### Use variable placeholders + +For `Add` and `Set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated when the split occurs. + + +### `Splits[].ResponseHeaders` + +Specifies a set of HTTP-specific header modification rules applied to responses routed with the service split. You cannot configure request headers if the listener protocol is set to `tcp`. Refer to [Set HTTP Headers](#set-http-headers) for an example configuration. + +#### Values + +- Default: None +- Values: Object containing one or more fields that define header modification rules + - `Add`: Map of one or more string key-value pairs + - `Set`: Map of one or more string key-value pairs + - `Remove`: Map of one or more string key-value pairs + +The following table describes how to configure values for response headers: + +| Rule | Description | Type | +| --- | --- | --- | +| `Add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `Set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `Remove` | Defines an list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | list of strings | + +#### Use variable placeholders + +For `Add` and `Set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated when the split occurs. + + + + + +### `apiVersion` + +Kubernetes-only parameter that specifies the version of the Consul API that the configuration entry maps to Kubernetes configurations. The value must be `consul.hashicorp.com/v1alpha1`. + +### `kind` + +Specifies the type of configuration entry to implement. + +#### Values + +- Default: none +- This field is required. +- Data type: String value that must be set to `serviceSplitter`. + +### `metadata.name` + +Specifies a name for the configuration entry. The name is metadata that you can use to reference the configuration entry when performing Consul operations, such as applying a configuration entry to a specific cluster. + +#### Values + +- Default: Inherits name from the host node +- This field is required. +- Data type: String + + +### `metadata.namespace` + +Specifies the Consul namespace to use for resolving the service. You can map Consul namespaces to Kubernetes Namespaces in different ways. Refer to [Custom Resource Definitions (CRDs) for Consul on Kubernetes](/consul/docs/k8s/crds#consul-enterprise) for additional information. + +#### Values + +- Default: None +- Data type: String + +### `spec` + +Kubernetes-only field that contains all of the configurations for service splitter pods. + +#### Values + +- Default: none +- This field is required. +- Data type: Object containing [`spec.splits`](#spec-splits) configuration + +### `spec.meta` + +Specifies key-value pairs to add to the KV store. + +#### Values + +- Default: none +- Data type: Map of one or more key-value pairs + - keys: String + - values: String, integer, or float + +### `spec.splits` + +Defines how much traffic to send to sets of service instances during a traffic split. + +#### Values + +- Default: None +- This field is required. +- Data type: list of objects that can contain the following fields: + - `weight`: The sum of weights for a set of service instances. The total defined value must add up to 100. + - `service`: This field is required. + - `serviceSubset` + - `namespace` + - `partition` + - `requestHeaders` + - `responseHeaders` -The `service-splitter` config entry kind (`ServiceSplitter` on Kubernetes) controls how to split incoming Connect -requests across different subsets of a single service (like during staged -canary rollouts), or perhaps across different services (like during a v2 -rewrite or other type of codebase migration). +### `spec.splits[].weight` -If no splitter config is defined for a service it is assumed 100% of traffic -flows to a service with the same name and discovery continues on to the -resolution stage. +Specifies the percentage of traffic sent to the set of service instances specified in the [`spec.splits.service`](#spec-splits-service) field. Each weight must be a floating integer between `0` and `100`. The smallest representable value is `.01`. The sum of weights across all splits must add up to `100`. -## Interaction with other Config Entries +#### Values -- Service splitter config entries are a component of [L7 Traffic - Management](/consul/docs/connect/l7-traffic). +- Default: `null` +- This field is required. +- Data type: Floating integer from `.01` to `100` -- Service splitter config entries are restricted to only services that define - their protocol as http-based via a corresponding - [`service-defaults`](/consul/docs/connect/config-entries/service-defaults) config - entry or globally via - [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults) . +### `spec.splits[].service` -- Any split destination that specifies a different `Service` field and omits - the `ServiceSubset` field is eligible for further splitting should a splitter - be configured for that other service, otherwise resolution proceeds according - to any configured - [`service-resolver`](/consul/docs/connect/config-entries/service-resolver). +Specifies the name of the service to resolve. -## UI +#### Values -Once a `service-splitter` is successfully entered, you can view it in the UI. Service routers, service splitters, and service resolvers can all be viewed by clicking on your service then switching to the _routing_ tab. +- Default: The service matching the configuration entry [`meta.name`](#metadata-name) field. +- Data type: String -![screenshot of service splitter in the UI](/img/l7-routing/Splitter.png) +### `spec.splits[].serviceSubset` -## Sample Config Entries +Specifies a subset of the service to resolve. This field overrides the `DefaultSubset`. + +#### Values + +- Default: Inherits the name of the default subset. +- Data type: String + +### `spec.splits[].namespace` + +Specifies the [namespace](/consul/docs/enterprise/namespaces) to use when resolving the service. + +#### Values + +- Default: The namespace specified in the top-level configuration entry. +- Data type: String + +### `spec.splits[].partition` + +Specifies which [admin partition](/consul/docs/enterprise/admin-partitions) to use in the FQDN when resolving the service. + +#### Values + +- Default: `default` +- Data type: String + +### `spec.splits[].requestHeaders` + +Specifies a set of HTTP-specific header modification rules applied to requests routed with the service split. You cannot configure request headers if the listener protocol is set to `tcp`. Refer to [Set HTTP Headers](#set-http-headers) for an example configuration. + +#### Values + +- Default: None +- Values: Object containing one or more fields that define header modification rules + - `add`: Map of one or more key-value pairs + - `set`: Map of one or more key-value pairs + - `remove`: Map of one or more key-value pairs + +The following table describes how to configure values for request headers: + +| Rule | Description | Type | +| --- | --- | --- | +| `add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `remove` | Defines an list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | list of strings | + +#### Use variable placeholders + +For `add` and `set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated when the split occurs. + +### `spec.splits[].responseHeaders` + +Specifies a set of HTTP-specific header modification rules applied to responses routed with the service split. You cannot configure request headers if the listener protocol is set to `tcp`. Refer to [Set HTTP Headers](#set-http-headers) for an example configuration. + +#### Values + +- Default: None +- Values: Object containing one or more fields that define header modification rules + - `add`: Map of one or more string key-value pairs + - `set`: Map of one or more string key-value pairs + - `remove`: Map of one or more string key-value pairs + +The following table describes how to configure values for response headers: + +| Rule | Description | Type | +| --- | --- | --- | +| `add` | Defines a set of key-value pairs to add to the header. Use header names as the keys. Header names are not case-sensitive. If header values with the same name already exist, the value is appended and Consul applies both headers. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `set` | Defines a set of key-value pairs to add to the request header or to replace existing header values with. Use header names as the keys. Header names are not case-sensitive. If header values with the same names already exist, Consul replaces the header values. You can [use variable placeholders](#use-variable-placeholders). | map of strings | +| `remove` | Defines an list of headers to remove. Consul removes only headers containing exact matches. Header names are not case-sensitive. | list of strings | + +#### Use variable placeholders + +For `add` and `set`, if the service is configured to use Envoy as the proxy, the value may contain variables to interpolate dynamic metadata into the value. For example, using the variable `%DOWNSTREAM_REMOTE_ADDRESS%` in your configuration entry allows you to pass a value that is generated when the split occurs. + + + + + +## Examples + +The following examples demonstrate common service splitter configuration patterns for specific use cases. ### Two subsets of same service Split traffic between two subsets of the same service: - + + + ```hcl Kind = "service-splitter" @@ -65,18 +586,9 @@ Splits = [ ] ``` -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceSplitter -metadata: - name: web -spec: - splits: - - weight: 90 - serviceSubset: v1 - - weight: 10 - serviceSubset: v2 -``` + + + ```json { @@ -95,13 +607,34 @@ spec: } ``` - + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceSplitter +metadata: + name: web +spec: + splits: + - weight: 90 + serviceSubset: v1 + - weight: 10 + serviceSubset: v2 +``` + + + + ### Two different services Split traffic between two services: - + + + ```hcl Kind = "service-splitter" @@ -118,18 +651,9 @@ Splits = [ ] ``` -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceSplitter -metadata: - name: web -spec: - splits: - - weight: 50 - # will default to service with same name as config entry ("web") - - weight: 50 - service: web-rewrite -``` + + + ```json { @@ -147,14 +671,35 @@ spec: } ``` - + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceSplitter +metadata: + name: web +spec: + splits: + - weight: 50 + # defaults to the service with same name as the configuration entry ("web") + - weight: 50 + service: web-rewrite +``` + + + + ### Set HTTP Headers Split traffic between two subsets with extra headers added so clients can tell which version: - + + + ```hcl Kind = "service-splitter" @@ -181,24 +726,9 @@ Splits = [ ] ``` -```yaml -apiVersion: consul.hashicorp.com/v1alpha1 -kind: ServiceSplitter -metadata: - name: web -spec: - splits: - - weight: 90 - serviceSubset: v1 - responseHeaders: - set: - x-web-version: v1 - - weight: 10 - serviceSubset: v2 - responseHeaders: - set: - x-web-version: v2 -``` + + + ```json { @@ -227,136 +757,31 @@ spec: } ``` - + -## Available Fields -', - yaml: false, - }, - { - name: 'Namespace', - type: `string: "default"`, - enterprise: true, - description: - 'Specifies the namespace to which the configuration entry will apply.', - yaml: false, - }, - { - name: 'Partition', - type: `string: "default"`, - enterprise: true, - description: - 'Specifies the admin partition to which the configuration entry will apply.', - yaml: false, - }, - { - name: 'Meta', - type: 'map: nil', - description: - 'Specifies arbitrary KV metadata pairs. Added in Consul 1.8.4.', - yaml: false, - }, - { - name: 'metadata', - children: [ - { - name: 'name', - description: 'Set to the name of the service being configured.', - }, - { - name: 'namespace', - description: - 'If running Consul Open Source, the namespace is ignored (see [Kubernetes Namespaces in Consul OSS](/consul/docs/k8s/crds#consul-oss)). If running Consul Enterprise see [Kubernetes Namespaces in Consul Enterprise](/consul/docs/k8s/crds#consul-enterprise) for more details.', - }, - ], - hcl: false, - }, - { - name: 'Splits', - type: 'array', - description: - 'Defines how much traffic to send to which set of service instances during a traffic split. The sum of weights across all splits must add up to 100.', - children: [ - { - name: 'weight', - type: 'float32: 0', - description: - 'A value between 0 and 100 reflecting what portion of traffic should be directed to this split. The smallest representable weight is 1/10000 or .01%', - }, - { - name: 'Service', - type: 'string: ""', - description: 'The service to resolve instead of the default.', - }, - { - name: 'ServiceSubset', - type: 'string: ""', - description: { - hcl: - "A named subset of the given service to resolve instead of one defined as that service's `DefaultSubset`. If empty the default subset is used.", - yaml: - "A named subset of the given service to resolve instead of one defined as that service's `defaultSubset`. If empty the default subset is used.", - }, - }, - { - name: 'Namespace', - enterprise: true, - type: 'string: ""', - description: - 'The namespace to resolve the service from instead of the current namespace. If empty, the current namespace is used.', - }, - { - name: 'Partition', - enterprise: true, - type: 'string: ""', - description: - 'The admin partition to resolve the service from instead of the current partition. If empty, the current partition is used.', - }, - { - name: 'RequestHeaders', - type: 'HTTPHeaderModifiers: ', - description: `A set of [HTTP-specific header modification rules](/consul/docs/connect/config-entries/service-router#httpheadermodifiers) - that will be applied to requests routed to this split. - This cannot be used with a \`tcp\` listener.`, - }, - { - name: 'ResponseHeaders', - type: 'HTTPHeaderModifiers: ', - description: `A set of [HTTP-specific header modification rules](/consul/docs/connect/config-entries/service-router#httpheadermodifiers) - that will be applied to responses from this split. - This cannot be used with a \`tcp\` listener.`, - }, - ], - }, - ]} -/> -## ACLs + -Configuration entries may be protected by [ACLs](/consul/docs/security/acl). - -Reading a `service-splitter` config entry requires `service:read` on the resource. +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceSplitter +metadata: + name: web +spec: + splits: + - weight: 90 + serviceSubset: v1 + responseHeaders: + set: + x-web-version: v1 + - weight: 10 + serviceSubset: v2 + responseHeaders: + set: + x-web-version: v2 +``` -Creating, updating, or deleting a `service-splitter` config entry requires -`service:write` on the resource and `service:read` on any other service referenced by -name in these fields: + -- [`Splits[].Service`](#service) + diff --git a/website/content/docs/connect/config-entries/terminating-gateway.mdx b/website/content/docs/connect/config-entries/terminating-gateway.mdx index b7e0f38b7c4..9c0804f9c39 100644 --- a/website/content/docs/connect/config-entries/terminating-gateway.mdx +++ b/website/content/docs/connect/config-entries/terminating-gateway.mdx @@ -7,12 +7,9 @@ description: >- # Terminating Gateway Configuration Entry --> **v1.8.4+:** On Kubernetes, the `TerminatingGateway` custom resource is supported in Consul versions 1.8.4+.
    -**v1.8.0+:** On other platforms, this config entry is supported in Consul versions 1.8.0+. - The `terminating-gateway` config entry kind (`TerminatingGateway` on Kubernetes) allows you to configure terminating gateways to proxy traffic from services in the Consul service mesh to services registered with Consul that do not have a -[Connect service sidecar proxy](/consul/docs/connect/proxies). The configuration is associated with the name of a gateway service +[service mesh sidecar proxy](/consul/docs/connect/proxies). The configuration is associated with the name of a gateway service and will apply to all instances of the gateway with that name. ~> [Configuration entries](/consul/docs/agent/config-entries) are global in scope. A configuration entry for a gateway name applies @@ -41,7 +38,7 @@ Terminating gateways can optionally target all services within a Consul namespac as the service name. Configuration options set on the wildcard act as defaults that can be overridden by options set on a specific service name. -Note that if the wildcard specifier is used, and some services in that namespace have a Connect sidecar proxy, +Note that if the wildcard specifier is used, and some services in that namespace have a service mesh sidecar proxy, traffic from the mesh to those services will be evenly load-balanced between the gateway and their sidecars. ## Sample Config Entries diff --git a/website/content/docs/connect/configuration.mdx b/website/content/docs/connect/configuration.mdx index 8fbd88fa325..5edf02886ba 100644 --- a/website/content/docs/connect/configuration.mdx +++ b/website/content/docs/connect/configuration.mdx @@ -2,26 +2,26 @@ layout: docs page_title: Service Mesh Configuration - Overview description: >- - Learn how to enable and configure Consul's service mesh capabilities in agent configurations, and how to integrate with schedulers like Kubernetes and Nomad. ""Connect"" is the subsystem that provides Consul’s service mesh capabilities. + Learn how to enable and configure Consul's service mesh capabilities in agent configurations, and how to integrate with schedulers like Kubernetes and Nomad. Consul's service mesh capabilities are provided by the ""connect"" subsystem. --- # Service Mesh Configuration Overview There are many configuration options exposed for Consul service mesh. The only option that must be set is the `connect.enabled` option on Consul servers to enable Consul service mesh. -All other configurations are optional and have reasonable defaults. +All other configurations are optional and have defaults suitable for many environments. -Consul Connect is the component shipped with Consul that enables service mesh functionality. The terms _Consul Connect_ and _Consul service mesh_ are used interchangeably throughout this documentation. +The noun _connect_ is used throughout this documentation to refer to the connect +subsystem that provides Consul's service mesh capabilities. +Where you encounter the _noun_ connect, it is usually functionality specific to +service mesh. --> **Tip:** Service mesh is enabled by default when running Consul in -dev mode with `consul agent -dev`. +## Agent configuration -## Agent Configuration - -Begin by enabling Connect for your Consul -cluster. By default, Connect is disabled. Enabling Connect requires changing +Begin by enabling service mesh for your Consul +cluster. By default, service is disabled. Enabling service mesh requires changing the configuration of only your Consul _servers_ (not client agents). To enable -Connect, add the following to a new or existing +service mesh, add the following to a new or existing [server configuration file](/consul/docs/agent/config/config-files). In an existing cluster, this configuration change requires a Consul server restart, which you can perform one server at a time to maintain availability. In HCL: @@ -40,17 +40,17 @@ connect { ```     -This will enable Connect and configure your Consul cluster to use the +This will enable service mesh and configure your Consul cluster to use the built-in certificate authority for creating and managing certificates. You may also configure Consul to use an external [certificate management system](/consul/docs/connect/ca), such as [Vault](https://www.vaultproject.io/). -Services and proxies may always register with Connect settings, but they will -fail to retrieve or verify any TLS certificates. This causes all Connect-based -connection attempts to fail until Connect is enabled on the server agents. +Services and proxies may always register with service mesh settings, but unless +service mesh is enabled on the server agents, their attempts to communicate will fail +because they have no means to obtain or verify service mesh TLS certificates. -Other optional Connect configurations that you can set in the server +Other optional service mesh configurations that you can set in the server configuration file include: - [certificate authority settings](/consul/docs/agent/config/config-files#connect) @@ -58,10 +58,10 @@ configuration file include: - [dev mode](/consul/docs/agent/config/cli-flags#_dev) - [server host name verification](/consul/docs/agent/config/config-files#tls_internal_rpc_verify_server_hostname) -If you would like to use Envoy as your Connect proxy you will need to [enable +If you would like to use Envoy as your service mesh proxy you will need to [enable gRPC](/consul/docs/agent/config/config-files#grpc_port). -Additionally if you plan on using the observability features of Connect, it can +Additionally if you plan on using the observability features of Consul service mesh, it can be convenient to configure your proxies and services using [configuration entries](/consul/docs/agent/config-entries) which you can interact with using the CLI or API, or by creating configuration entry files. You will want to enable @@ -70,27 +70,21 @@ configuration](/consul/docs/agent/config/config-files#enable_central_service_con clients, which allows each service's proxy configuration to be managed centrally via API. -!> **Security note:** Enabling Connect is enough to try the feature but doesn't -automatically ensure complete security. Please read the [Connect production +!> **Security note:** Enabling service mesh is enough to try the feature but doesn't +automatically ensure complete security. Please read the [service mesh production tutorial](/consul/tutorials/developer-mesh/service-mesh-production-checklist) to understand the additional steps needed for a secure deployment. -## Centralized Proxy and Service Configuration - -To account for common Connect use cases where you have many instances of the -same service, and many colocated sidecar proxies, Consul allows you to customize -the settings for all of your proxies or all the instances of a given service at -once using [Configuration Entries](/consul/docs/agent/config-entries). +## Centralized proxy and service configuration -You can override centralized configurations for individual proxy instances in -their +If your network contains many instances of the same service and many colocated sidecar proxies, you can specify global settings for proxies or services in [Configuration Entries](/consul/docs/agent/config-entries). You can override the centralized configurations for individual proxy instances in their [sidecar service definitions](/consul/docs/connect/registration/sidecar-service), and the default protocols for service instances in their [service -registrations](/consul/docs/discovery/services). +definitions](/consul/docs/services/usage/define-services). ## Schedulers -Consul Connect is especially useful if you are using an orchestrator like Nomad +Consul service mesh is especially useful if you are using an orchestrator like Nomad or Kubernetes, because these orchestrators can deploy thousands of service instances which frequently move hosts. Sidecars for each service can be configured through these schedulers, and in some cases they can automate Consul configuration, @@ -98,16 +92,16 @@ sidecar deployment, and service registration. ### Nomad -Connect can be used with Nomad to provide secure service-to-service +Consul service mesh can be used with Nomad to provide secure service-to-service communication between Nomad jobs and task groups. The ability to use the dynamic -port feature of Nomad makes Connect particularly easy to use. Learn about how to -configure Connect on Nomad by reading the +port feature of Nomad makes Consul service mesh particularly easy to use. Learn about how to +configure Consul service mesh on Nomad by reading the [integration documentation](/consul/docs/connect/nomad). ### Kubernetes -The Consul Helm chart can automate much of Consul Connect's configuration, and +The Consul Helm chart can automate much of Consul's service mesh configuration, and makes it easy to automatically inject Envoy sidecars into new pods when they are deployed. Learn about the [Helm chart](/consul/docs/k8s/helm) in general, or if you are already familiar with it, check out its -[connect specific configurations](/consul/docs/k8s/connect). +[service mesh specific configurations](/consul/docs/k8s/connect). diff --git a/website/content/docs/connect/connect-internals.mdx b/website/content/docs/connect/connect-internals.mdx index 826a2bde222..73a3cc1e898 100644 --- a/website/content/docs/connect/connect-internals.mdx +++ b/website/content/docs/connect/connect-internals.mdx @@ -11,7 +11,10 @@ This topic describes how many of the core features of Consul's service mesh func It is not a prerequisite, but this information will help you understand how Consul service mesh behaves in more complex scenarios. -Consul Connect is the component shipped with Consul that enables service mesh functionality. The terms _Consul Connect_ and _Consul service mesh_ are used interchangeably throughout this documentation. +The noun _connect_ is used throughout this documentation to refer to the connect +subsystem that provides Consul's service mesh capabilities. +Where you encounter the _noun_ connect, it is usually functionality specific to +service mesh. To try service mesh locally, complete the [Getting Started with Consul service mesh](/consul/tutorials/kubernetes-deploy/service-mesh?utm_source=docs) @@ -59,7 +62,7 @@ in-memory data. ## Agent Caching and Performance -To enable fast responses on endpoints such as the [agent Connect +To enable fast responses on endpoints such as the [agent connect API](/consul/api-docs/agent/connect), the Consul agent locally caches most Consul service mesh-related data and sets up background [blocking queries](/consul/api-docs/features/blocking) against the server to update the cache in the background. This allows most API calls diff --git a/website/content/docs/connect/connectivity-tasks.mdx b/website/content/docs/connect/connectivity-tasks.mdx index c81c0ebda54..bd5a4bc66f6 100644 --- a/website/content/docs/connect/connectivity-tasks.mdx +++ b/website/content/docs/connect/connectivity-tasks.mdx @@ -59,7 +59,7 @@ Services outside the mesh do not have sidecar proxies or are not [integrated nat These may be services running on legacy infrastructure or managed cloud services running on infrastructure you do not control. -Terminating gateways effectively act as egress proxies that can represent one or more services. They terminate Connect +Terminating gateways effectively act as egress proxies that can represent one or more services. They terminate service mesh mTLS connections, enforce Consul intentions, and forward requests to the appropriate destination. These gateways also simplify authorization from dynamic service addresses. Consul's intentions determine whether diff --git a/website/content/docs/connect/dataplane/consul-dataplane.mdx b/website/content/docs/connect/dataplane/consul-dataplane.mdx index b4661d86123..ab59a5ba60c 100644 --- a/website/content/docs/connect/dataplane/consul-dataplane.mdx +++ b/website/content/docs/connect/dataplane/consul-dataplane.mdx @@ -7,7 +7,7 @@ description: >- # Consul Dataplane CLI Reference -The `consul-dataplane` command interacts with the binary for [simplified service mesh with Consul Dataplane](/consul/docs/k8s/dataplane). Use this command to install Consul Dataplane, configure its Envoy proxies, and secure Dataplane deployments. +The `consul-dataplane` command interacts with the binary for [simplified service mesh with Consul Dataplane](/consul/docs/connect/dataplane). Use this command to install Consul Dataplane, configure its Envoy proxies, and secure Dataplane deployments. ## Usage diff --git a/website/content/docs/connect/dev.mdx b/website/content/docs/connect/dev.mdx index 6f494bc3751..e31a2f423a6 100644 --- a/website/content/docs/connect/dev.mdx +++ b/website/content/docs/connect/dev.mdx @@ -8,17 +8,17 @@ description: >- # Service Mesh Debugging It is often necessary to connect to a service for development or debugging. -If a service only exposes a Connect listener, then we need a way to establish +If a service only exposes a service mesh listener, then we need a way to establish a mutual TLS connection to the service. The [`consul connect proxy` command](/consul/commands/connect/proxy) can be used for this task on any machine with access to a Consul agent (local or remote). -Restricting access to services only via Connect ensures that the only way to +Restricting access to services only via service mesh ensures that the only way to connect to a service is through valid authorization of the [intentions](/consul/docs/connect/intentions). This can extend to developers and operators, too. -## Connecting to Connect-only Services +## Connecting to Mesh-only Services As an example, let's assume that we have a PostgreSQL database running that we want to connect to via `psql`, but the only non-loopback listener is diff --git a/website/content/docs/connect/distributed-tracing.mdx b/website/content/docs/connect/distributed-tracing.mdx index 1c601ec27fc..939cde52d5a 100644 --- a/website/content/docs/connect/distributed-tracing.mdx +++ b/website/content/docs/connect/distributed-tracing.mdx @@ -234,8 +234,11 @@ config to take effect. 1. Requests through [Ingress Gateways](/consul/docs/connect/gateways/ingress-gateway) will not be traced unless the header `x-client-trace-id: 1` is set (see [hashicorp/consul#6645](https://github.com/hashicorp/consul/issues/6645)). -1. Consul does not currently support interoperation with [OpenTelemetry](https://opentelemetry.io/) libraries due to - Envoy not yet having support. +1. Consul's proxies do not currently support [OpenTelemetry](https://opentelemetry.io/) spans, as Envoy has not + [fully implemented](https://github.com/envoyproxy/envoy/issues/9958) it. Instead, you can add + OpenTelemetry libraries to your application to emit spans for other + [tracing protocols](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/observability/tracing) + supported by Envoy, such as Zipkin or Jaeger. 1. Tracing is only supported with Envoy proxies, not the built-in proxy. diff --git a/website/content/docs/connect/gateways/api-gateway/configuration/api-gateway.mdx b/website/content/docs/connect/gateways/api-gateway/configuration/api-gateway.mdx new file mode 100644 index 00000000000..11ad78f2d27 --- /dev/null +++ b/website/content/docs/connect/gateways/api-gateway/configuration/api-gateway.mdx @@ -0,0 +1,331 @@ +--- +layout: docs +page_title: API Gateway Configuration Entry Reference +description: Learn how to configure a Consul API Gateway on VMs. +--- + +# API gateway configuration entry reference + +This topic provides reference information for the API gateway configuration entry that you can deploy to networks in virtual machine (VM) environments. For reference information about configuring Consul API gateways on Kubernetes, refer to [Gateway Resource Configuration](/consul/docs/api-gateway/configuration/gateway). + +## Introduction + +A gateway is a type of network infrastructure that determines how service traffic should be handled. Gateways contain one or more listeners that bind to a set of hosts and ports. An HTTP Route or TCP Route can then attach to a gateway listener to direct traffic from the gateway to a service. + +## Configuration model + +The following list outlines field hierarchy, language-specific data types, and requirements in an `api-gateway` configuration entry. Click on a property name to view additional details, including default values. + +- [`Kind`](#kind): string | must be `"api-gateway"` +- [`Name`](#name): string | no default +- [`Namespace`](#namespace): string | no default +- [`Partition`](#partition): string | no default +- [`Meta`](#meta): map | no default +- [`Listeners`](#listeners): list of objects | no default + - [`Name`](#listeners-name): string | no default + - [`Port`](#listeners-port): number | no default + - [`Hostname`](#listeners-hostname): string | `"*"` + - [`Protocol`](#listeners-protocol): string | `"tcp"` + - [`TLS`](#listeners-tls): map | none + - [`MinVersion`](#listeners-tls-minversion): string | no default + - [`MaxVersion`](#listeners-tls-maxversion): string | no default + - [`CipherSuites`](#listeners-tls-ciphersuites): list of strings | Envoy default cipher suites + - [`Certificates`](#listeners-tls-certificates): list of objects | no default + - [`Kind`](#listeners-tls-certificates-kind): string | must be `"inline-certificate"` + - [`Name`](#listeners-tls-certificates-name): string | no default + - [`Namespace`](#listeners-tls-certificates-namespace): string | no default + - [`Partition`](#listeners-tls-certificates-partition): string | no default + +## Complete configuration + +When every field is defined, an `api-gateway` configuration entry has the following form: + + + +```hcl +Kind = "api-gateway" +Name = "" +Namespace = "" +Partition = "" + +Meta = { + = "" +} + +Listeners = [ + { + Port = + Name = "" + Protocol = "" + TLS = { + MaxVersion = "" + MinVersion = "" + CipherSuites = [ + "" + ] + Certificates = [ + { + Kind = "inline-certificate" + Name = "" + Namespace = "" + Partition = "" + } + ] + } + } +] +``` + +```json +{ + "Kind": "api-gateway", + "Name": "", + "Namespace": "", + "Partition": "", + "Meta": { + "": "" + }, + "Listeners": [ + { + "Name": "", + "Port": , + "Protocol": "", + "TLS": { + "MaxVersion": "", + "MinVersion": "", + "CipherSuites": [ + "" + ], + "Certificates": [ + { + "Kind": "inline-certificate", + "Name": "", + "Namespace": "", + "Partition": "" + } + ] + } + } + ] +} +``` + + + +## Specification + +This section provides details about the fields you can configure in the +`api-gateway` configuration entry. + +### `Kind` + +Specifies the type of configuration entry to implement. This must be +`api-gateway`. + +#### Values + +- Default: none +- This field is required. +- Data type: string value that must be set to `"api-gateway"`. + +### `Name` + +Specifies a name for the configuration entry. The name is metadata that you can +use to reference the configuration entry when performing Consul operations, +such as applying a configuration entry to a specific cluster. + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Meta` + +Specifies an arbitrary set of key-value pairs to associate with the gateway. + +#### Values + +- Default: none +- Data type: map containing one or more keys and string values. + +### `Listeners[]` + +Specifies a list of listeners that gateway should set up. Listeners are +uniquely identified by their port number. + +#### Values + +- Default: none +- This field is required. +- Data type: List of maps. Each member of the list contains the following fields: + - [`Name`](#listeners-name) + - [`Port`](#listeners-port) + - [`Hostname`](#listeners-hostname) + - [`Protocol`](#listeners-protocol) + - [`TLS`](#listeners-tls) + +### `Listeners[].Name` + +Specifies the unique name for the listener. This field accepts letters, numbers, and hyphens. + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Listeners[].Port` + +Specifies the port number that the listener receives traffic on. + +#### Values + +- Default: `0` +- This field is required. +- Data type: integer + +### `Listeners[].Hostname` + +Specifies the hostname that the listener receives traffic on. + +#### Values + +- Default: `"*"` +- This field is optional. +- Data type: string + +### `Listeners[].Protocol` + +Specifies the protocol associated with the listener. + +#### Values + +- Default: none +- This field is required. +- The data type is one of the following string values: `"tcp"` or `"http"`. + +### `Listeners[].TLS` + +Specifies the TLS configurations for the listener. + +#### Values + +- Default: none +- Map that contains the following fields: + - [`MaxVersion`](#listeners-tls-maxversion) + - [`MinVersion`](#listeners-tls-minversion) + - [`CipherSuites`](#listeners-tls-ciphersuites) + - [`Certificates`](#listeners-tls-certificates) + +### `Listeners[].TLS.MaxVersion` + +Specifies the maximum TLS version supported for the listener. + +#### Values + +- Default depends on the version of Envoy: + - Envoy 1.22.0 and later default to `TLSv1_2` + - Older versions of Envoy default to `TLSv1_0` +- Data type is one of the following string values: + - `TLS_AUTO` + - `TLSv1_0` + - `TLSv1_1` + - `TLSv1_2` + - `TLSv1_3` + +### `Listeners[].TLS.MinVersion` + +Specifies the minimum TLS version supported for the listener. + +#### Values + +- Default: none +- Data type is one of the following string values: + - `TLS_AUTO` + - `TLSv1_0` + - `TLSv1_1` + - `TLSv1_2` + - `TLSv1_3` + +### `Listeners[].TLS.CipherSuites[]` + +Specifies a list of cipher suites that the listener supports when negotiating connections using TLS 1.2 or older. + +#### Values + +- Defaults to the ciphers supported by the version of Envoy in use. Refer to the + [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#envoy-v3-api-field-extensions-transport-sockets-tls-v3-tlsparameters-cipher-suites) + for details. +- Data type: List of string values. Refer to the + [Consul repository](https://github.com/hashicorp/consul/blob/v1.11.2/types/tls.go#L154-L169) + for a list of supported ciphers. + +### `Listeners[].TLS.Certificates[]` + +The list of references to inline certificates that the listener uses for TLS termination. + +#### Values + +- Default: None +- Data type: List of maps. Each member of the list has the following fields: + - [`Kind`](#listeners-tls-certificates-kind) + - [`Name`](#listeners-tls-certificates-name) + - [`Namespace`](#listeners-tls-certificates-namespace) + - [`Partition`](#listeners-tls-certificates-partition) + +### `Listeners[].TLS.Certificates[].Kind` + +The list of references to inline-certificates that the listener uses for TLS termination. + +#### Values + +- Default: None +- This field is required and must be set to `"inline-certificate"`. +- Data type: string + +### `Listeners[].TLS.Certificates[].Name` + +The list of references to inline certificates that the listener uses for TLS termination. + +#### Values + +- Default: None +- This field is required. +- Data type: string + +### `Listeners[].TLS.Certificates[].Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) where the certificate can be found. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Listeners[].TLS.Certificates[].Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) where the certificate can be found. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string diff --git a/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx b/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx new file mode 100644 index 00000000000..997e2bbf692 --- /dev/null +++ b/website/content/docs/connect/gateways/api-gateway/configuration/http-route.mdx @@ -0,0 +1,679 @@ +--- +layout: docs +page_title: HTTP Route Configuration +description: Learn how to configure an HTTP Route bound to an API Gateway on VMs. +--- + +# HTTP route configuration reference + +This topic provides reference information for the gateway routes configuration entry. Refer to [Route Resource Configuration](/consul/docs/api-gateway/configuration/routes) for information about configuring API gateway routes in Kubernetes environments. + +## Configuration model + +The following list outlines field hierarchy, language-specific data types, and +requirements in an `http-route` configuration entry. Click on a property name +to view additional details, including default values. + +- [`Kind`](#kind): string | must be `http-route` +- [`Name`](#name): string | no default +- [`Namespace`](#namespace): string | no default +- [`Partition`](#partition): string | no default +- [`Meta`](#meta): map | no default +- [`Hostnames`](#hostnames): list | no default +- [`Parents`](#parents): list | no default + - [`Kind`](#parents-kind): string | must be `api-gateway` + - [`Name`](#parents-name): string | no default + - [`Namespace`](#parents-namespace): string | no default + - [`Partition`](#parents-partition): string | no default + - [`SectionName`](#parents-sectionname): string | no default +- [`Rules`](#rules): list | no default + - [`Filters`](#rules-filters): map | no default + - [`Headers`](#rules-filters-headers): list | no default + - [`Add`](#rules-filters-headers-add): map | no default + - [`Remove`](#rules-filters-headers-remove): list | no default + - [`Set`](#rules-filters-headers-set): map | no default + - [`URLRewrite`](#rules-filters-urlrewrite): map | no default + - [`Path`](#rules-filters-urlrewrite-path): string | no default + - [`Matches`](#rules-matches): list | no default + - [`Headers`](#rules-matches-headers): list | no default + - [`Match`](#rules-matches-headers-match): string | no default + - [`Name`](#rules-matches-headers-name): string | no default + - [`Value`](#rules-matches-headers-value): string | no default + - [`Method`](#rules-matches-method): string | no default + - [`Path`](#rules-matches-path): map | no default + - [`Match`](#rules-matches-path-match): string | no default + - [`Value`](#rules-matches-path-value): string | no default + - [`Query`](#rules-matches-query): list | no default + - [`Match`](#rules-matches-query-match): string | no default + - [`Name`](#rules-matches-query-name): string | no default + - [`Value`](#rules-matches-query-value): string | no default + - [`Services`](#rules-services): list | no default + - [`Name`](#rules-services-name): string | no default + - [`Namespace`](#rules-services-namespace): string + - [`Partition`](#rules-services-partition): string + - [`Weight`](#rules-services-weight): number | `1` + - [`Filters`](#rules-services-filters): map | no default + - [`Headers`](#rules-services-filters-headers): list | no default + - [`Add`](#rules-services-filters-headers-add): map | no default + - [`Remove`](#rules-services-filters-headers-remove): list | no default + - [`Set`](#rules-services-filters-headers-set): map | no default + - [`URLRewrite`](#rules-services-filters-urlrewrite): map | no default + - [`Path`](#rules-services-filters-urlrewrite-path): string | no default + +## Complete configuration + +When every field is defined, an `http-route` configuration entry has the following form: + + + +```hcl +Kind = "http-route" +Name = "" +Namespace = "" +Partition = "" +Meta = { + "" = "" +} +Hostnames = [""] + +Parents = [ + { + Kind = "api-gateway" + Name = "" + Namespace = "" + Partition = "" + SectionName = "" + } +] + +Rules = [ + { + Filters = { + Headers = [ + { + Add = { + "" = "" + } + Remove = [ + "" + ] + Set = { + "" = "" + } + } + ] + URLRewrite = { + Path = "" + } + } + Matches = [ + { + Headers = [ + { + Match = "" + Name = "" + Value = "" + } + ] + Method = "" + Path = { + Match = "" + Value = "" + } + Query = [ + { + Match = "" + Name = "" + Value = "" + } + ] + } + ] + Services = [ + { + Name = "" + Namespace = "" + Partition = "" + Weight = "" + Filters = { + Headers = [ + { + Add = { + "" = "" + } + Remove = [ + "" + ] + Set = { + "" = "" + } + } + ] + URLRewrite = { + Path = "" + } + } + } + ] + } +] + +``` + +```json +{ + "Kind": "http-route", + "Name": "", + "Namespace": "", + "Partition": "", + "Meta": { + "": "" + }, + "Hostnames": [ + "" + ], + "Parents": [ + { + "Kind": "api-gateway", + "Name": "", + "Namespace": "", + "Partition": "", + "SectionName": "" + } + ], + "Rules": [ + { + "Filters": [ + { + "Headers": [ + { + "Add": [ + { + "": "" + } + ], + "Remove": ["
    "], + "Set": [ + { + "": "" + } + ] + } + ], + "URLRewrite": [ + { + "Path": "" + } + ] + } + ], + "Matches": [ + { + "Headers": [ + { + "Match": "", + "Name": "", + "Value": "" + } + ], + "Method": "", + "Path": [ + { + "Match": "", + "Value": "" + } + ], + "Query": [ + { + "Match": "", + "Name": "", + "Value": "" + } + ] + } + ], + "Services": [ + { + "Name": "", + "Namespace": "", + "Partition": "", + "Weight": "", + "Filters": [ + { + "Headers": [ + { + "Add": [ + { + "" + } + ], + "Remove": ["
    "], + "Set": [ + { + "" + } + ] + } + ], + "URLRewrite": [ + { + "Path": "" + } + ] + } + ] + } + ] + } + ] +} +``` + + + +## Specification + +This section provides details about the fields you can configure in the `http-route` configuration entry. + +### `Kind` + +Specifies the type of configuration entry to implement. For HTTP routes, this must be `http-route`. + +#### Values + +- Default: none +- This field is required. +- Data type: string value that must be set to `"http-route"`. + +### `Name` + +Specifies a name for the configuration entry. The name is metadata that you can +use to reference the configuration entry when performing Consul operations, +such as applying a configuration entry to a specific cluster. + +#### Values + +- Default: Defaults to the name of the node after writing the entry to the Consul server. +- This field is required. +- Data type: string + +### `Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Meta` + +Specifies an arbitrary set of key-value pairs to associate with the route. + +#### Values + +- Default: none +- Data type: map containing one or more keys and string values. + +### `Parents[]` + +Specifies the list of gateways that this route binds to. + +#### Values + +- Default: none +- Data type: List of map. Each member of the list contains the following fields: + - `Kind` + - `Name` + - `Namespace` + - `Partition` + - `SectionName` + +### `Parents[].Kind` + +Specifies the type of resource to bind to. This field is required and must be +set to `"api-gateway"` + +#### Values + +- Default: none +- This field is required. +- Data type: string value set to `"api-gateway"` + +### `Parents[].Name` + +Specifies the name of the api-gateway to bind to. + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Parents[].Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Parents[].Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Parents[].SectionName` + +Specifies the name of the listener to bind to on the `api-gateway`. If left +empty, this route binds to _all listeners_ on the parent gateway. + +#### Values + +- Default: "" +- Data type: string + +### `Rules[]` + +Specifies the list of HTTP-based routing rules that this route uses to construct a route table. + +#### Values + +- Default: +- Data type: List of maps. Each member of the list contains the following fields: + - `Filters` + - `Matches` + - `Services` + +### `Rules[].Filters` + +Specifies the list of HTTP-based filters used to modify a request prior to routing it to the upstream service. + +#### Values + +- Default: none +- Data type: Map that contains the following fields: + - `Headers` + - `UrlRewrite` + +### `Rules[].Filters.Headers[]` + +Defines operations to perform on matching request headers when an incoming request matches the `Rules.Matches` configuration. + +#### Values + +This field contains the following configuration objects: + +| Parameter | Description | Type | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | +| `set` | Configure this field to rewrite the HTTP request header. It specifies the name of an HTTP header to overwrite and the new value to set. Any existing values associated with the header name are overwritten. You can specify the following configurations:
    • `name`: Required string that specifies the name of the HTTP header to set.
    • `value`: Required string that specifies the value of the HTTP header to set.
    | List of maps | +| `add` | Configure this field to append the request header with a new value. It specifies the name of an HTTP header to append and the values to add. You can specify the following configurations:
    • `name`: Required string that specifies the name of the HTTP header to append.
    • `value`: Required string that specifies the value of the HTTP header to add.
    | List of maps | +| `remove` | Configure this field to specify an array of header names to remove from the request header. | List of strings | + +### `Rules[].Filters.URLRewrite` + +Specifies rule for rewriting the URL of incoming requests when an incoming request matches the `Rules.Matches` configuration. + +#### Values + +- Default: none +- This field is a map that contains a `Path` field. + +### Rules[].Filters.URLRewrite.Path + +Specifies a path that determines how Consul API Gateway rewrites a URL path. Refer to [Reroute HTTP requests](/consul/docs/api-gateway/usage/reroute-http-requests) for additional information. + +#### Values + +The following table describes the parameters for `path`: + +| Parameter | Description | Type | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------ | +| `replacePrefixMatch` | Specifies a value that replaces the path prefix for incoming HTTP requests. The operation only affects the path prefix. The rest of the path is unchanged. | String | +| `type` | Specifies the type of replacement to use for the URL path. You can specify the following values:
    • `ReplacePrefixMatch`: Replaces the matched prefix of the URL path (default).
    | String | + +### `Rules[].Matches[]` + +Specifies the matching criteria used in the routing table. When an incoming +request matches the given HTTPMatch configuration, traffic routes to +services specified in the [`Rules.Services`](#rules-services) field. + +#### Values + +- Default: none +- Data type: List containing maps. Each member of the list contains the following fields: + - `Headers` + - `Method` + - `Path` + - `Query` + +### `Rules[].Matches[].Headers[]` + +Specifies rules for matching incoming request headers. You can specify multiple rules in a list, as well as multiple lists of rules. If all rules in a single list are satisfied, then the route forwards the request to the appropriate service defined in the [`Rules.Services`](#rules-services) configuration. You can create multiple `Header[]` lists to create a range of matching criteria. When at least one list of matching rules are satisfied, the route forwards the request to the appropriate service defined in the [`Rules.Services`](#rules-services) configuration. + +#### Values + +- Default: none +- Data type: List containing maps with the following fields: + - `Match` + - `Name` + - `Value` + +### `Rules.Matches.Headers.Match` + +Specifies type of match for headers: `"exact"`, `"prefix"`, or `"regex"`. + +#### Values + +- Default: none +- Data type: string + +### `Rules.Matches.Headers.Name` + +Specifies the name of the header to match. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Matches.Headers.Value` + +Specifies the value of the header to match. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Matches[].Method` + +Specifies a list of strings that define matches based on HTTP request method. + +#### Values + +Specify one of the following string values: + +- `HEAD` +- `POST` +- `PUT` +- `PATCH` +- `GET` +- `DELETE` +- `OPTIONS` +- `TRACE` +- `CONNECT` + +### `Rules[].Matches[].Path` + +Specifies the HTTP method to match. + +#### Values + +- Default: none +- Data type: map containing the following fields: + - `Match` + - `Value` + +### `Rules[].Matches[].Path.Match` + +Specifies type of match for the path: `"exact"`, `"prefix"`, or `"regex"`. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Matches[].Path.Value` + +Specifies the value of the path to match. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Matches[].Query[]` + +Specifies how a match is completed on a request’s query parameters. + +#### Values + +- Default: none +- Data type: List of map that contains the following fields: + - `Match` + - `Name` + - `Value` + +### `Rules[].Matches[].Query[].Match` + +Specifies type of match for query parameters: `"exact"`, `"prefix"`, or `"regex"`. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Matches[].Query[].Name` + +Specifies the name of the query parameter to match. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Matches[].Query[].Value` + +Specifies the value of the query parameter to match. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Services[]` + +Specifies the service that the API gateway routes incoming requests to when the +requests match the `Rules.Matches` configuration. + +#### Values + +- Default: none +- This field contains a list of maps. Each member of the list contains the following fields: + - `Name` + - `Weight` + - `Filters` + - `Namespace` + - `Partition` + +### `Rules[].Services[].Name` + +Specifies the name of an HTTP-based service to route to. + +#### Values + +- Default: none +- Data type: string + +### `Rules[].Services[].Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Rules[].Services.Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Rules[].Services[].Weight` + +Specifies the proportion of requests forwarded to the specified service. If no weight is specified, or if the specified +weight is set to less than or equal to `0`, the weight is normalized to `1`. The +proportion is determined by dividing the value of the weight by the sum of all +weights in the service list. For non-zero values, there may be some deviation +from the exact proportion depending on the precision an implementation +supports. Weight is not a percentage and the sum of weights does not need to +equal 100. + +#### Values + +- Default: none +- Data type: integer + +### `Rules[].Services[].Filters` + +Specifies the list of HTTP-based filters used to modify a request prior to +routing it to this upstream service. + +#### Values + +- Default: none +- Data type: Map that contains the following fields: + - `Headers` + - `UrlRewrite` + +### `Rules[].Services[].Filters.Headers[]` + +Defines operations to perform on matching request headers. + +#### Values + +This field contains the following configuration objects: + +| Parameter | Description | Type | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | +| `set` | Configure this field to rewrite the HTTP request header. It specifies the name of an HTTP header to overwrite and the new value to set. Any existing values associated with the header name are overwritten. You can specify the following configurations:
    • `name`: Required string that specifies the name of the HTTP header to set.
    • `value`: Required string that specifies the value of the HTTP header to set.
    | List of maps | +| `add` | Configure this field to append the request header with a new value. It specifies the name of an HTTP header to append and the values to add. You can specify the following configurations:
    • `name`: Required string that specifies the name of the HTTP header to append.
    • `value`: Required string that specifies the value of the HTTP header to add.
    | List of maps | +| `remove` | Configure this field to specify an array of header names to remove from the request header. | List of strings | + +### `Rules[].Services[].Filters.URLRewrite` + +Specifies rule for rewriting the URL of incoming requests. + +#### Values + +- Default: none +- This field is a map that contains a `Path` field. diff --git a/website/content/docs/connect/gateways/api-gateway/configuration/inline-certificate.mdx b/website/content/docs/connect/gateways/api-gateway/configuration/inline-certificate.mdx new file mode 100644 index 00000000000..4fca7c54ee6 --- /dev/null +++ b/website/content/docs/connect/gateways/api-gateway/configuration/inline-certificate.mdx @@ -0,0 +1,127 @@ +--- +layout: docs +page_title: Inline Certificate Configuration Reference +description: Learn how to configure an inline certificate bound to an API Gateway on VMs. +--- + +# Inline certificate configuration reference + +This topic provides reference information for the gateway inline certificate +configuration entry. For information about certificate configuration for Kubernetes environments, refer to [Gateway Resource Configuration](/consul/docs/api-gateway/configuration/gateway). + +## Configuration model + +The following list outlines field hierarchy, language-specific data types, and +requirements in an `inline-certificate` configuration entry. Click on a property name +to view additional details, including default values. + +- [`Kind`](#kind): string | must be `"inline-certificate"` +- [`Name`](#name): string | no default +- [`Namespace`](#namespace): string | no default +- [`Partition`](#partition): string | no default +- [`Meta`](#meta): map | no default +- [`Certificate`](#certificate): string | no default +- [`PrivateKey`](#privatekey): string | no default + +## Complete configuration + +When every field is defined, an `inline-certificate` configuration entry has the following form: + + + +```HCL +Kind = "inline-certificate" +Name = "" + +Meta = { + "" = "" +} + +Certificate = "" +PrivateKey = "" +``` + +```JSON +{ + "Kind": "inline-certificate", + "Name": "", + "Meta": { + "any key": "any value" + } + "Certificate": "", + "PrivateKey": "" +} +``` + + + +## Specification + +### `Kind` + +Specifies the type of configuration entry to implement. + +#### Values + +- Default: none +- This field is required. +- Data type: string that must equal `"inline-certificate"` + +### `Name` + +Specifies a name for the configuration entry. The name is metadata that you can +use to reference the configuration entry when performing Consul operations, such +as applying a configuration entry to a specific cluster. + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Meta` + +Specifies an arbitrary set of key-value pairs to associate with the gateway. + +#### Values + +- Default: none +- Data type: map containing one or more keys and string values. + +### `Certificate` + +Specifies the inline public certificate to use for TLS. + +#### Values + +- Default: none +- This field is required. +- Data type: string value of the public certificate + +### `PrivateKey` + +Specifies the inline private key to use for TLS. + +#### Values + +- Default: none +- This field is required. +- Data type: string value of the private key diff --git a/website/content/docs/connect/gateways/api-gateway/configuration/tcp-route.mdx b/website/content/docs/connect/gateways/api-gateway/configuration/tcp-route.mdx new file mode 100644 index 00000000000..23b32662d92 --- /dev/null +++ b/website/content/docs/connect/gateways/api-gateway/configuration/tcp-route.mdx @@ -0,0 +1,256 @@ +--- +layout: docs +page_title: TCP Route Configuration Reference +description: Learn how to configure a TCP Route that is bound to an API gateway on VMs. +--- + +# TCP route configuration Reference + +This topic provides reference information for the gateway TCP routes configuration +entry. Refer to [Route Resource Configuration](/consul/docs/api-gateway/configuration/routes) for information +about configuring API gateways in Kubernetes environments. + +## Configuration model + +The following list outlines field hierarchy, language-specific data types, and +requirements in an `tcp-route` configuration entry. Click on a property name to +view additional details, including default values. + +- [`Kind`](#kind): string | must be `"tcp-route"` +- [`Name`](#name): string | no default +- [`Namespace`](#namespace): string | no default +- [`Partition`](#partition): string | no default +- [`Meta`](#meta): map | no default +- [`Services`](#services): list | no default + - [`Name`](#services-name): string | no default + - [`Namespace`](#services-namespace): string | no default + - [`Partition`](#services-partition): string | no default +- [`Parents`](#parents): list | no default + - [`Kind`](#parents-kind): string | must be `"api-gateway"` + - [`Name`](#parents-name): string | no default + - [`Namespace`](#parents-namespace): string | no default + - [`Partition`](#parents-partition): string | no default + - [`SectionName`](#parents-sectionname): string | no default + +## Complete configuration + +When every field is defined, a `tcp-route` configuration entry has the following form: + + + +```HCL +Kind = "tcp-route" +Name = "" +Namespace = "" +Partition = "" + +Meta = { + "" = "" +} + +Services = [ + { + Name = "" + Namespace = "" + Partition = "" + } +] + + +Parents = [ + { + Kind = "api-gateway" + Name = "" + Namespace = "" + Partition = "" + SectionName = "" + } +] +``` + +```JSON +{ + "Kind": "tcp-route", + "Name": "", + "Namespace": "", + "Partition": "", + "Meta": { + "": "" + }, + "Services": [ + { + "Name": "" + "Namespace": "", + "Partition": "", + } + ], + "Parents": [ + { + "Kind": "api-gateway", + "Name": "", + "Namespace": "", + "Partition": "", + "SectionName": "" + } + ] +} +``` + + + +## Specification + +This section provides details about the fields you can configure in the +`tcp-route` configuration entry. + +### `Kind` + +Specifies the type of configuration entry to implement. This must be set to +`"tcp-route"`. + +#### Values + +- Default: none +- This field is required. +- Data type: string value that must be set to`tcp-route`. + +### `Name` + +Specifies a name for the configuration entry. The name is metadata that you can +use to reference the configuration entry when performing Consul operations, +such as applying a configuration entry to a specific cluster. + +#### Values + +- Default: Defaults to the name of the node after writing the entry to the Consul server. +- This field is required. +- Data type: string + +### `Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Meta` + +Specifies an arbitrary set of key-value pairs to associate with the gateway. + +#### Values + +- Default: none +- Data type: map containing one or more keys and string values. + +### `Services` + +Specifies a TCP-based service the API gateway routes incoming requests +to. You can only specify one service. + +#### Values + +- Default: none +- The data type is a list of maps. Each member of the list contains the following fields: + - [`Name`](#services-name) + - [`Namespace`](#services-namespace) + - [`Partition`](#services-partition) + +### `Services.Name` + +Specifies the list of TCP-based services to route to. You can specify a maximum of one service. + +#### Values + +- Default: none +- Data type: string + +### `Services.Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) where the service is located. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Services.Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) where the service is located. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Parents` + +Specifies the list of gateways that the route is bound to. + +#### Values + +- Default: none +- Data type: List of map. Each member of the list contains the following fields: + - [`Kind`](#parents-kind) + - [`Name`](#parents-name) + - [`Namespace`](#parents-namespace) + - [`Partition`](#parents-partition) + - [`SectionName`](#parents-sectionname) + +### `Parents.Kind` + +Specifies the type of resource to bind to. This field is required and must be +set to `"api-gateway"` + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Parents.Name` + +Specifies the name of the API gateway to bind to. + +#### Values + +- Default: none +- This field is required. +- Data type: string + +### `Parents.Namespace` + +Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) where the parent is located. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Parents.Partition` + +Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) where the parent is located. + +#### Values + +- Default: `"default"` in Enterprise +- Data type: string + +### `Parents.SectionName` + +Specifies the name of the listener defined in the [`api-gateway` configuration](/consul/docs/connect/gateways/api-gateway/configuration/api-gateway) that the route binds to. If the field is configured to an empty string, the route binds to all listeners on the parent gateway. + +#### Values + +- Default: `""` +- Data type: string diff --git a/website/content/docs/connect/gateways/api-gateway/index.mdx b/website/content/docs/connect/gateways/api-gateway/index.mdx new file mode 100644 index 00000000000..832fd943b2e --- /dev/null +++ b/website/content/docs/connect/gateways/api-gateway/index.mdx @@ -0,0 +1,28 @@ +--- +layout: docs +page_title: API Gateways Overview +description: API gateways are objects in Consul that enable ingress requests to services in your service mesh. Learn about API gateways for VMs in this overview. +--- + +# API gateway overview + +API gateways enable external network clients to access applications and services +running in a Consul datacenter. This type of network traffic is commonly +called _north-south_ network traffic because it refers to the flow of +data into and out of a specific environment. API gateways can also forward +requests from clients to specific destinations based on path or request +protocol. + +API gateways solve the following primary use cases: + +- **Control access at the point of entry**: Set the protocols of external connection + requests and secure inbound connections with TLS certificates from trusted + providers, such as Verisign and Let's Encrypt. +- **Simplify traffic management**: Load balance requests across services and route + traffic to the appropriate service by matching one or more criteria, such as + hostname, path, header presence or value, and HTTP method. + +Consul supports API +gateways for virtual machines and Kubernetes networks. Refer to the following documentation for next steps: +- [API Gateways on VMs](/consul/docs/connect/gateways/api-gateway/usage) +- [API Gateways for Kubernetes](/consul/docs/api-gateway). diff --git a/website/content/docs/connect/gateways/api-gateway/usage.mdx b/website/content/docs/connect/gateways/api-gateway/usage.mdx new file mode 100644 index 00000000000..9f314fd2f4e --- /dev/null +++ b/website/content/docs/connect/gateways/api-gateway/usage.mdx @@ -0,0 +1,211 @@ +--- +layout: docs +page_title: API Gateways on Virtual Machines +description: Learn how to configure and Consul API gateways and gateway routes on virtual machines so that you can enable ingress requests to services in your service mesh in VM environments. +--- + +# API gateways on virtual machines + +This topic describes how to deploy Consul API gateways to networks that operate +in virtual machine (VM) environments. If you want to implement an API gateway +in a Kubernetes environment, refer to [API Gateway for Kubernetes](/consul/docs/api-gateway). + +## Introduction + +Consul API gateways provide a configurable ingress points for requests into a Consul network. Use the following configuration entries to set up API gateways: + +- [API gateway](/consul/docs/connect/gateways/api-gateway/configuration/api-gateway): Provides an endpoint for requests to enter the network. Define listeners that expose ports on the endpoint for ingress. +- [HTTP routes](/consul/docs/connect/gateways/api-gateway/configuration/http-route) and [TCP routes](/consul/docs/connect/gateways/api-gateway/configuration/tcp-route): The routes attach to listeners defined in the gateway and control how requests route to services in the network. +- [Inline certificates](/consul/docs/connect/gateways/api-gateway/configuration/inline-certificate): Makes TLS certificates available to gateways so that requests between the user and the gateway endpoint are encrypted. + +You can configure and reuse configuration entries separately. You can define and attach routes and inline certificates to multiple gateways. + +The following steps describe the general workflow for deploying a Consul API +gateway to a VM environment: + +1. Create an API gateway configuration entry. The configuration entry includes + listener configurations and references to TLS certificates. +1. Deploy the API gateway configuration entry to create the listeners. +1. Create and deploy routes to bind to the gateway. + +Refer to [API Gateway for Kubernetes](/consul/docs/api-gateway) for information +about using Consul API gateway on Kubernetes. + +## Requirements + +The following requirements must be satisfied to use API gateways on VMs: + +- Consul 1.15 or later +- A Consul cluster with service mesh enabled. Refer to [`connect`](/consul/docs/agent/config/config-files#connect) +- Network connectivity between the machine deploying the API Gateway and a + Consul cluster agent or server + +If ACLs are enabled, you must present a token with the following permissions to +configure Consul and deploy API gateways: + +- `mesh: read` +- `mesh: write` + +Refer [Mesh Rules](/consul/docs/security/acl/acl-rules#mesh-rules) for +additional information about configuring policies that enable you to interact +with Consul API gateway configurations. + +## Create the API gateway configuration + +Create an API gateway configuration that defines listeners and TLS certificates +in the mesh. In the following example, the API gateway specifies an HTTP +listener on port `8443` that routes can use to connect external traffic to +services in the mesh. + +```hcl +Kind = "api-gateway" +Name = "my-gateway" + +// Each listener configures a port which can be used to access the Consul cluster +Listeners = [ + { + Port = 8443 + Name = "my-http-listener" + Protocol = "http" + TLS = { + Certificates = [ + { + Kind = "inline-certificate" + Name = "my-certificate" + } + ] + } + } +] +``` + +Refer to [API Gateway Configuration Reference](/consul/docs/connect/gateways/api-gateway/configuration/api-gateway) for +information about all configuration fields. + +Gateways and routes are eventually-consistent objects that provide feedback +about their current state through a series of status conditions. As a result, +you must manually check the route status to determine if the route +bound to the gateway successfully. + +## Deploy the API gateway + +Use the `consul config write` command to implement the API gateway +configuration entries. The following command applies the configuration entry +for the main gateway object: + +```shell-session +$ consul config write gateways.hcl +``` + +Run the following command to deploy an API gateway instance: + +```shell-session +$ consul connect envoy -gateway api -register -service my-api-gateway +``` + +The command directs Consul to configure Envoy as an API gateway. + +## Route requests + +Define route configurations and bind them to listeners configured on the +gateway so that Consul can route incoming requests to services in the mesh. +Create HTTP or TCP routes by setting the `Kind` parameter to `http-route` or +`tcp-route` and configuring rules that define request traffic flows. + +The following example routes requests from the listener on the API gateway at +port `8443` to services in Consul based on the path of the request. When an +incoming request starts at path `/`, Consul forwards 90 percent of the requests +to the `ui` service and 10 percent to `experimental-ui`. Consul also forwards +requests starting with `/api` to `api`. + +```hcl +Kind = "http-route" +Name = "my-http-route" + +// Rules define how requests will be routed +Rules = [ + // Send all requests to UI services with 10% going to the "experimental" UI + { + Matches = [ + { + Path = { + Match = "prefix" + Value = "/" + } + } + ] + Services = [ + { + Name = "ui" + Weight = 90 + }, + { + Name = "experimental-ui" + Weight = 10 + } + ] + }, + // Send all requests that start with the path `/api` to the API service + { + Matches = [ + { + Path = { + Match = "prefix" + Value = "/api" + } + } + ] + Services = [ + { + Name = "api" + } + ] + } +] + +Parents = [ + { + Kind = "api-gateway" + Name = "my-gateway" + SectionName = "my-http-listener" + } +] +``` + +Create this configuration by saving it to a file called `my-http-route.hcl` and using the command + +```shell-session +$ consul config write my-http-route.hcl +``` + +Refer to [HTTP Route Configuration Entry Reference](/consul/docs/connect/gateways/api-gateway/configuration/http-route) +and [TCP Route Configuration Entry Reference](/consul/docs/connect/gateways/api-gateway/configuration/tcp-route) for details about route configurations. + +## Add a TLS certificate + +Define an [`inline-certificate` configuration entry](/consul/docs/connect/gateways/api-gateway/configuration/inline-certificate) with a name matching the name in the [API gateway listener configuration](/consul/docs/connect/gateways/api-gateway/configuration/api-gateway#listeners) to bind the certificate to that listener. The inline certificate configuration entry takes a public certificate and private key in plaintext. + +The following example defines a certificate named `my-certificate`. API gateway configurations that specify `inline-certificate` in the `Certificate.Kind` field and `my-certificate` in the `Certificate.Name` field are able to use the certificate. + +```hcl +Kind = "inline-certificate" +Name = "my-certificate" + +Certificate = < **Mesh gateway tutorial**: Follow the [mesh gateway tutorial](/consul/tutorials/developer-mesh/service-mesh-gateways) to learn concepts associated with mesh gateways. @@ -64,7 +64,7 @@ Services outside the mesh do not have sidecar proxies or are not [integrated nat These may be services running on legacy infrastructure or managed cloud services running on infrastructure you do not control. -Terminating gateways effectively act as egress proxies that can represent one or more services. They terminate Connect +Terminating gateways effectively act as egress proxies that can represent one or more services. They terminate service mesh mTLS connections, enforce Consul intentions, and forward requests to the appropriate destination. These gateways also simplify authorization from dynamic service addresses. Consul's intentions determine whether diff --git a/website/content/docs/connect/gateways/ingress-gateway.mdx b/website/content/docs/connect/gateways/ingress-gateway.mdx deleted file mode 100644 index d75dbd5ebc0..00000000000 --- a/website/content/docs/connect/gateways/ingress-gateway.mdx +++ /dev/null @@ -1,281 +0,0 @@ ---- -layout: docs -page_title: Ingress Gateway | Service Mesh -description: >- - Ingress gateways listen for requests from external network locations and route authorized traffic to destinations in the service mesh. Use custom TLS certificates with ingress gateways through Envoy's gRPC Secret Discovery Service (SDS). ---- - -# Ingress Gateways - --> **1.8.0+:** This feature is available in Consul versions 1.8.0 and newer. - -Ingress gateways enable connectivity within your organizational network from services outside the Consul -service mesh to services in the mesh. An ingress gateway is -a type of proxy and must be registered as a service in Consul, with the -[kind](/consul/api-docs/agent/service#kind) set to "ingress-gateway". They are an -entrypoint for outside traffic and allow you to define what services should be -exposed and on what port. You configure an ingress gateway by defining a set of -[listeners](/consul/docs/connect/config-entries/ingress-gateway#listeners) that each map -to a set of backing -[services](/consul/docs/connect/config-entries/ingress-gateway#services). - -To enable easier service discovery, a new Consul [DNS -subdomain](/consul/docs/discovery/dns#ingress-service-lookups) is provided, on -`.ingress.`. - -For listeners with a -[protocol](/consul/docs/connect/config-entries/ingress-gateway#protocol) other than -`tcp`, multiple services can be specified for a single listener. In this -case, the ingress gateway relies on host/authority headers to decide the -service that should receive the traffic. The host used to match traffic -defaults to the [Consul DNS ingress -subdomain](/consul/docs/discovery/dns#ingress-service-lookups), but can be changed using -the [hosts](/consul/docs/connect/config-entries/ingress-gateway#hosts) field. - -![Ingress Gateway Architecture](/img/ingress-gateways.png) - -## Prerequisites - -Ingress gateways also require that your Consul datacenters are configured correctly: - -- You'll need to use Consul version 1.8.0 or newer. -- Consul [Connect](/consul/docs/agent/config/config-files#connect) must be enabled on the datacenter's Consul servers. -- [gRPC](/consul/docs/agent/config/config-files#grpc_port) must be enabled on all client agents. - -Currently, [Envoy](https://www.envoyproxy.io/) is the only proxy with ingress gateway capabilities in Consul. - -## Running and Using an Ingress Gateway - -For a complete example of how to allow external traffic inside your Consul service mesh, -review the [ingress gateway tutorial](/consul/tutorials/developer-mesh/service-mesh-ingress-gateways). - -## Ingress Gateway Configuration - -Ingress gateways are configured in service definitions and registered with Consul like other services, with two exceptions. -The first is that the [kind](/consul/api-docs/agent/service#kind) must be "ingress-gateway". Second, -the ingress gateway service definition may contain a `Proxy.Config` entry just like a -Connect proxy service, to define opaque configuration parameters useful for the actual proxy software. -For Envoy there are some supported [gateway options](/consul/docs/connect/proxies/envoy#gateway-options) as well as -[escape-hatch overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides). - --> **Note:** If ACLs are enabled, ingress gateways must be registered with a token granting `service:write` for the ingress gateway's service name, -`service:read` for all services in the ingress gateway's configuration entry, and `node:read` for all nodes of the services -in the ingress gateway's configuration entry. These privileges authorize the token to route communications to other Connect services. -If the Consul client agent on the gateway's node is not configured to use the default gRPC port, 8502, then the gateway's token -must also provide `agent:read` for its node's name in order to discover the agent's gRPC port. gRPC is used to expose Envoy's xDS API to Envoy proxies. - -~> [Configuration entries](/consul/docs/agent/config-entries) are global in scope. A configuration entry for a gateway name applies -across all federated Consul datacenters. If ingress gateways in different Consul datacenters need to route to different -sets of services within their datacenter, then the ingress gateways **must** be registered with different names. - - - - -## Custom TLS Certificates via Secret Discovery Service (SDS) - -~> **Advanced Topic:** This topic describes a low-level feature designed for -developers building integrations with custom TLS management solutions. - -Consul 1.11 added support for ingress gateways to serve TLS certificates to -inbound traffic that are sourced from an external service. The external service -must implement Envoy's [gRPC Secret Discovery -Service](https://www.envoyproxy.io/docs/envoy/latest/configuration/security/secret) -(or SDS) API. - -The following procedure describes how to configure an ingress gateway with TLS certificates from an SDS source. The instructions assume that you are familiar with Envoy configuration and the SDS protocol. - -### Configure Static SDS Cluster(s) - -Each Envoy proxy that makes up this Ingress Gateway must define one or more additional [static -clusters](/consul/docs/connect/proxies/envoy#envoy_extra_static_clusters_json) when registering. These additional clusters define how Envoy should connect to the required SDS service(s). Defining extra clusters in Envoy's bootstrap configuration requires a manual registration of the Ingress Gateway with Consul proxy. -It's not possible to use the `-register` flag with `consul connect envoy -gateway=ingress` to automatically register the proxy in this case. - -The cluster(s) must provide connection information and any necessary -authentication information such as mTLS credentials. - -The following example will demonstrate how to use: - - A DNS name to discover the SDS service addresses - - Local certificate files for TLS client authentication with the SDS server. - The certificates are assumed to be created and managed by some other - process. - - 1. **Register the proxy service.** - - The following Proxy Service Definition defines the additional cluster - configuration that will be provided to Envoy when it starts. With this TLS - configuration, Envoy will detect changes to the certificate and key files on - disk so an external process may maintain and rotate them without needing an - Envoy restart. - - ```hcl - // public-ingress.hcl - Services { - Name = "public-ingress" - Kind = "ingress-gateway" - - Proxy { - Config { - envoy_extra_static_clusters_json = <- + Ingress gateways enable you to connect external services to services in your mesh. Ingress gateways are a type of proxy that listens for requests from external network locations and route authorized traffic to destinations in the service mesh. +--- + +# Ingress gateways overview + +An ingress gateway is a type of proxy that enables network connectivity from external services to services inside the mesh. The following diagram describes the ingress gateway workflow: + +![Ingress Gateway Architecture](/img/ingress-gateways.png) + +## Workflow + +The following stages describe how to add an ingress gateway to your service mesh: + +1. Configure ingress gateway listeners: Create an ingress gateway configuration entry and specify which services to expose to external requests. The configuration entry allows you to define what services should be exposed, on what port, and by what hostname. You can expose services registered with Consul or expose virtual services defined in other configuration entries. Refer to [Ingress gateway configuration entry reference](/consul/docs/connect/config-entries/ingress-gateway) for details on the configuration parameters you can specify. + +1. Define an ingress gateway proxy service: Ingress gateways are a special-purpose proxy service that you can define and register in a similar manner to other services. When you register the ingress gateway service, Consul applies the configurations defined in the ingress gateway configuration reference. Refer to [Implement an ingress gateway](/consul/docs/connect/gateways/ingress-gateway/ingress-gateways-usage) for additional information. + +1. Start the network proxy: The ingress gateway proxy service accepts configurations from the configuration entry and directs requests to the exposed services. When the external traffic passes through the ingress gateway, your sidecar proxy handles the inbound and outbound connections between the exposed services and the gateway. Refer to [Service mesh proxy overview](/consul/docs/connect/proxies) to learn more about the proxies Consul supports. + +## Integrations with custom TLS management solutions + +You can configure the ingress gateway to retrieve and serve custom TLS certificates from external systems. This functionality is designed to help you integrate with custom TLS management software. Refer to [Serve custom TLS certificates from an external service](/consul/docs/connect/gateways/ingress-gateway/ingress-gateways-tls-external-service) for additional information. \ No newline at end of file diff --git a/website/content/docs/connect/gateways/ingress-gateway/ingress-gateways-tls-external-service.mdx b/website/content/docs/connect/gateways/ingress-gateway/ingress-gateways-tls-external-service.mdx new file mode 100644 index 00000000000..d33b8a68e13 --- /dev/null +++ b/website/content/docs/connect/gateways/ingress-gateway/ingress-gateways-tls-external-service.mdx @@ -0,0 +1,245 @@ +--- +layout: docs +page_title: Serve custom TLS certificates from an external service +description: Learn how to configure ingress gateways to serve TLS certificates from an external service to using secret discovery service. The SDS feature is designed for developers building integrations with custom TLS management solutions. +--- + +# Serve custom TLS certificates from an external service + +This is an advanced topic that describes how to configure ingress gateways to serve TLS certificates sourced from an external service to inbound traffic using secret discovery service (SDS). SDS is a low-level feature designed for developers building integrations with custom TLS management solutions. For instructions on more common ingress gateway implementations, refer to [Implement an ingress gateway](/consul/docs/connect/gateways/ingress-gateway/ingress-gateways-usage). + +## Overview + +The following process describes the general procedure for configuring ingress gateways to serve TLS certificates sourced from external services: + +1. Configure static SDS clusters in the ingress gateway service definition. +1. Register the service definition. +1. Configure TLS client authentication +1. Start Envoy. +1. Configure SDS settings in an ingress gateway configuration entry. +1. Register the ingress gateway configuration entry with Consul. + +## Requirements + +- The external service must implement Envoy's [gRPC secret discovery service (SDS) API](https://www.envoyproxy.io/docs/envoy/latest/configuration/security/secret). +- You should have some familiarity with Envoy configuration and the SDS protocol. +- The [`connect.enabled` parameter](/consul/docs/agent/config/config-files#connect) must be set to `true` for all server agents in the Consul datacenter. +- The [`ports.grpc` parameter](/consul/docs/agent/config/config-files#connect) must be configured for all server agents in the Consul datacenter. + +### ACL requirements + +If ACLs are enabled, you must present a token when registering ingress gateways that grant the following permissions: + +- `service:write` for the ingress gateway's service name +- `service:read` for all services in the ingress gateway's configuration entry +- `node:read` for all nodes of the services in the ingress gateway's configuration entry. + +These privileges authorize the token to route communications to other services in the mesh. If the Consul client agent on the gateway's node is not configured to use the default gRPC port, `8502`, then the gateway's token must also provide `agent:read` for its node's name in order to discover the agent's gRPC port. gRPC is used to expose Envoy's xDS API to Envoy proxies. + +## Configure static SDS clusters + +You must define one or more additional static clusters in the ingress gateway service definition for each Envoy proxy associated with the gateway. The additional clusters define how Envoy should connect to the required SDS services. + +Configure the static clusters in the [`Proxy.Config.envoy_envoy_extra_static_clusters_json`](/consul/docs/connect/proxies/envoy#envoy_extra_static_clusters_json) parameter in the service definition. + +The clusters must provide connection information and any necessary authentication information, such as mTLS credentials. + +You must manually register the ingress gateway with Consul proxy to define extra clusters in Envoy's bootstrap configuration. You can not use the `-register` flag with `consul connect envoy -gateway=ingress` to automatically register the proxy to define static clusters. + +In the following example, the `public-ingress` gateway includes a static cluster named `sds-cluster` that specifies paths to the SDS certificate and SDS certification validation files: + + + + +```hcl +Services { + Name = "public-ingress" + Kind = "ingress-gateway" + + Proxy { + Config { + envoy_extra_static_clusters_json = < + +Refer to the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/v1.17.2/api-v3/config/bootstrap/v3/bootstrap.proto#envoy-v3-api-field-config-bootstrap-v3-bootstrap-staticresources-clusters) for details about configuration parameters for SDS clusters. + +## Register the ingress gateway service definition + +Issue the `consul services register` command on the Consul agent on the Envoy proxy's node to register the service. The following example command registers an ingress gateway proxy from a `public-ingress.hcl` file: + +```shell-session +$ consul services register public-ingress.hcl +``` + +Refer to [Register services and health checks](/consul/docs/services/usage/register-services-checks) for additional information about registering services in Consul. + +## Configure TLS client authentication + +Store TLS client authentication files, certificate files, and keys on disk where the Envoy proxy runs and ensure that they are available to Consul. Refer to the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/bootstrap/bootstrap) for details on configuring authentication files. + +The following example specifies certificate chain: + + + + +```json +{ + "resources": [ + { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "name": "tls_sds", + "tls_certificate": { + "certificate_chain": { + "filename": "/certs/sds-client-auth.crt" + }, + "private_key": { + "filename": "/certs/sds-client-auth.key" + } + } + } + ] +} +``` + + +The following example specifies the validation context: + + + +```json +{ + "resources": [ + { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "name": "validation_context_sds", + "validation_context": { + "trusted_ca": { + "filename": "/certs/sds-ca.crt" + } + } + } + ] +} +``` + + + +## Start Envoy + +Issue the `consul connect envoy` command to bootstrap Envoy. The following example starts Envoy and registers it as a service called `public-ingress`: + +```shell-session +$ ​​consul connect envoy -gateway=ingress -service public-ingress +``` + +Refer to [Consul Connect Envoy](/consul/commands/connect/envoy) for additional information about using the `consul connect envoy` command. + +## Define an ingress gateway configuration entry + +Create an ingress gateway configuration entry that enables the gateway to use certificates from SDS. The configuration entry also maps downstream ingress listeners to upstream services. Configure the following fields: + +- [`Kind`](/consul/docs/connect/config-entries/ingress-gateway#kind): Set the value to `ingress-gateway`. +- [`Name`](/consul/docs/connect/config-entries/ingress-gateway#name): Consul applies the configuration entry settings to ingress gateway proxies with names that match the `Name` field. +- [`TLS`](/consul/docs/connect/config-entries/ingress-gateway#tls): The main `TLS` parameter for the configuration entry holds the SDS configuration. You can also specify TLS configurations per listener and per service. + - [`TLS.SDS`](/consul/docs/connect/config-entries/ingress-gateway#tls-sds): The `SDS` map includes the following configuration settings: + - [`ClusterName`](/consul/docs/connect/config-entries/ingress-gateway#tls-sds-clustername): Specifies the name of the cluster you specified when [configuring the SDS cluster](#configure-static-SDS-clusters). + - [`CertResource`](/consul/docs/connect/config-entries/ingress-gateway#tls-sds-certresource): Specifies the name of the certificate resource to load. +- [`Listeners`](/consul/docs/connect/config-entries/ingress-gateway#listeners): Specify one or more listeners. + - [`Listeners.Port`](/consul/docs/connect/config-entries/ingress-gateway#listeners-port): Specify a port for the listener. Each listener is uniquely identified by its port number. + - [`Listeners.Protocol`](/consul/docs/connect/config-entries/ingress-gateway#listeners-protocol): The default protocol is `tcp`, but you must specify the protocol used by the services you want to allow traffic from. + - [`Listeners.Services`](/consul/docs/connect/config-entries/ingress-gateway#listeners-services): The `Services` field contains the services that you want to expose to upstream services. The field contains several options and sub-configurations that enable granular control over ingress traffic, such as health check and TLS configurations. + +For Consul Enterprise service meshes, you may also need to configure the [`Partition`](/consul/docs/connect/config-entries/ingress-gateway#partition) and [`Namespace`](/consul/docs/connect/config-entries/ingress-gateway#namespace) fields for the gateway and for each exposed service. + +Refer to [Ingress gateway configuration entry reference](/consul/docs/connect/config-entries/ingress-gateway) for details about the supported parameters. + +The following example directs Consul to retrieve `example.com-public-cert` certificates from an SDS cluster named `sds-cluster` and serve them to all listeners: + + + +```hcl +Kind = "ingress-gateway" +Name = "public-ingress" + +TLS { + SDS { + ClusterName = "sds-cluster" + CertResource = "example.com-public-cert" + } +} + +Listeners = [ + { + Port = 8443 + Protocol = "http" + Services = ["*"] + } +] +``` + + + +## Register the ingress gateway configuration entry + +You can register the configuration entry using the [`consul config` command](/consul/commands/config) or by calling the [`/config` API endpoint](/consul/api-docs/config). Refer to [How to Use Configuration Entries](/consul/docs/agent/config-entries) for details about applying configuration entries. + +The following example registers an ingress gateway configuration entry named `public-ingress-cfg.hcl` that is stored on the local system: + +```shell-session +$ consul config write public-ingress-cfg.hcl +``` + +The Envoy instance starts a listener on the port specified in the configuration entry and fetches the TLS certificate named from the SDS server. \ No newline at end of file diff --git a/website/content/docs/connect/gateways/ingress-gateway/ingress-gateways-usage.mdx b/website/content/docs/connect/gateways/ingress-gateway/ingress-gateways-usage.mdx new file mode 100644 index 00000000000..0e42511b120 --- /dev/null +++ b/website/content/docs/connect/gateways/ingress-gateway/ingress-gateways-usage.mdx @@ -0,0 +1,117 @@ +--- +layout: docs +page_title: Implement an ingress gateway +description: Learn how to implement ingress gateways, which are Consul service mesh constructs that listen for requests from external network locations and route authorized traffic to destinations in the service mesh. +--- + +# Implement an ingress gateway + +This topic describes how to add ingress gateways to your Consul service mesh. Ingress gateways enable connectivity within your organizational network by allowing services outside of the service mesh to send traffic to services in the mesh. Refer to [Ingress gateways overview](/consul/docs/connect/gateways/ingress-gateway/) for additional information about ingress gateways. + +This topic describes ingress gateway usage for virtual machine (VM) environments. Refer to [Configure ingress gateways for Consul on Kubernetes](/consul/docs/k8s/connect/ingress-gateways) for instructions on how to implement ingress gateways in Kubernetes environments. + +## Overview + +Ingress gateways are a type of proxy service included with Consul. Complete the following steps to set up an ingress gateway: + +1. Define listeners and the services they expose. Specify these details in an ingress gateway configuration entry. +1. Register an ingress gateway service. Define the services in a service definition file. +1. Start the ingress gateway. This step deploys the Envoy proxy that functions as the ingress gateway. + +After specifying listeners and services in the ingress gateway configuration entry, you can register the gateway service and start Envoy with a single CLI command instead of completing these steps separately. Refer [Register an ingress service on Envoy startup](#register-an-ingress-service-on-envoy-startup). + +## Requirements +- Service mesh must be enabled for all agents. Set the [`connect.enabled` parameter](/consul/docs/agent/config/config-files#connect) to `true` to enable service mesh. +- The gRPC port must be configured for all server agents in the datacenter. Specify the gRPC port in the [`ports.grpc` parameter](/consul/docs/agent/config/config-files#grpc_port). We recommend setting the port to `8502` to simplify configuration when ACLs are enabled. Refer to [ACL requirements](#acl-requirements) for additional information. +- You must use Envoy for sidecar proxies in your service mesh. Refer to [Envoy Proxy Configuration for Service Mesh](/consul/docs/connect/proxies/envoy) for supported versions. +### ACL requirements +If ACLs are enabled, you must present a token when registering ingress gateways that grant the following permissions: + +`service:write` for the ingress gateway's service name +`service:read` for all services in the ingress gateway's configuration entry +`node:read` for all nodes of the services in the ingress gateway's configuration entry. + +These privileges authorize the token to route communications to other services in the mesh. If the Consul client agent on the gateway's node is not configured to use the default `8502` gRPC port, then the gateway's token must also provide `agent:read` for its node's name in order to discover the agent's gRPC port. gRPC is used to expose Envoy's xDS API to Envoy proxies. + +## Expose services + +Define and apply an ingress gateway configuration entry to specify which services in the mesh to expose to external services. +### Define an ingress gateway configuration entry +Ingress gateway configuration entries map downstream ingress listeners to upstream services. When you register an ingress gateway proxy that matches the configuration entry name, Consul applies the settings specified in the configuration entry. Configure the following fields: + +- [`Kind`](/consul/docs/connect/config-entries/ingress-gateway#kind): Set the value to `ingress-gateway`. +- [`Name`](/consul/docs/connect/config-entries/ingress-gateway#name): Consul applies the configuration entry settings to ingress gateway proxies with names that match the `Name` field. +- [`Listeners`](/consul/docs/connect/config-entries/ingress-gateway#listeners): Specify one or more listeners. + - [`Listeners.Port`](/consul/docs/connect/config-entries/ingress-gateway#listeners-port): Specify a port for the listener. Each listener is uniquely identified by its port number. + - [`Listeners.Protocol`](/consul/docs/connect/config-entries/ingress-gateway#listeners-protocol): The default protocol is `tcp`, but you must specify the protocol used by the services you want to allow traffic from. + - [`Listeners.Services`](/consul/docs/connect/config-entries/ingress-gateway#listeners-services): The `Services` field contains the services that you want to expose to upstream services. The field contains several options and sub-configurations that enable granular control over ingress traffic, such as health check and TLS configurations. + +For Consul Enterprise service meshes, you may also need to configure the [`Partition`](/consul/docs/connect/config-entries/ingress-gateway#partition) and [`Namespace`](/consul/docs/connect/config-entries/ingress-gateway#namespace) fields for the gateway and for each exposed service. + +Refer to [Ingress gateway configuration entry reference](/consul/docs/connect/config-entries/ingress-gateway) for details about the supported parameters. +### Register an ingress gateway configuration entry + +You can register the configuration entry using the [`consul config` command](/consul/commands/config) or by calling the [`/config` API endpoint](/consul/api-docs/config). Refer to [How to Use Configuration Entries](/consul/docs/agent/config-entries) for details about applying configuration entries. + +The following example registers an ingress gateway configuration entry named `public-ingress.hcl` that is stored on the local system: + +```shell-session +$ consul config write public-ingress.hcl +``` +## Deploy an ingress gateway service + +To deploy an ingress gateway service, create a service definition and register it with Consul. + +You can also define an ingress gateway service and register it with Consul while starting an Envoy proxy from the command line. Refer to [Register an ingress service on Envoy startup](#register-an-ingress-service-on-envoy-startup) for details. +### Create a service definition for the ingress gateway + +Consul applies the settings defined in the ingress gateway configuration entry to ingress gateway services that match the configuration entry name. Refer to [Define services](/consul/docs/services/usage/define-services) for additional information about defining services in Consul. + +The following fields are required for the ingress gateway service definition: + +[`Kind`](/consul/docs/services/configuration/services-configuration-reference#kind): The field must be set to `ingress-gateway`. +[`Name`](/consul/docs/services/configuration/services-configuration-reference#name): The name should match the value specified for the `Name` field in the configuration entry. + +All other service definition fields are optional, but we recommend defining health checks to verify the health of the gateway. Refer to [Services configuration reference](/consul/docs/services/configuration/services-configuration-reference) for information about defining services. + +### Register the ingress gateway proxy service + +You can register the ingress gateway using API or CLI. Refer to [Register services and health checks](/consul/docs/services/usage/register-services-checks) for instructions on registering services in Consul. + +The following example registers an ingress gateway defined in `ingress-gateway.hcl` from the Consul CLI: + +```shell-session +$ consul services register ingress-service.hcl +``` +## Start an Envoy proxy + +Run the `consul connect envoy` command to start Envoy. Specify the name of the ingress gateway service and include the `-gateway=ingress` flag. Refer to [Consul Connect Envoy](/consul/commands/connect/envoy) for details about using the command. + +The following example starts Envoy for the `ingress-service` gateway service: + +```shell-session +$ consul connect envoy -gateway=ingress ingress-service' +``` + +### Register an ingress service on Envoy startup + +You can also automatically register the ingress gateway service when starting the Envoy proxy. Specify the following flags with the `consul connect envoy` command: + +- `-gateway=ingress` +- `-register`: This +- `-service=` + +The following example starts Envoy and registers an ingress gateway service named `ingress-service` bound to the agent address at port `8888`: + +```shell-session +$ consul connect envoy -gateway=ingress -register -service ingress-service + -address '{{ GetInterfaceIP "eth0" }}:8888' +``` +You cannot register the ingress gateway service and start the proxy at the same time if you configure the gateway to retrieve and serve TLS certificates from their external downstreams. Refer to [Serve custom TLS certificates from an external service](/consul/docs/connect/gateways/ingress-gateway/ingress-gateways-tls-external-service) for more information. + +## Additional Envoy configurations + +Ingress gateways support additional Envoy gateway options and escape-hatch overrides. Specify gateway options in the ingress gateway service definition to use them. To use escape-hatch overrides, you must add them to your global proxy defaults configuration entry. Refer to the following documentation for additional information: + +- [Gateway options](/consul/docs/connect/proxies/envoy#gateway-options) +- [Escape-hatch overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides) diff --git a/website/content/docs/connect/gateways/mesh-gateway/index.mdx b/website/content/docs/connect/gateways/mesh-gateway/index.mdx index e5c5229dfcf..bcac5555278 100644 --- a/website/content/docs/connect/gateways/mesh-gateway/index.mdx +++ b/website/content/docs/connect/gateways/mesh-gateway/index.mdx @@ -18,7 +18,7 @@ Mesh gateways can be used with any of the following Consul configrations for man * [Mesh gateways can be used to route service-to-service traffic between datacenters](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters) * [Mesh gateways can be used to route all WAN traffic, including from Consul servers](/consul/docs/connect/gateways/mesh-gateway/wan-federation-via-mesh-gateways) 2. Cluster Peering - * [Mesh gateways can be used to route service-to-service traffic between datacenters](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers) + * [Mesh gateways can be used to route service-to-service traffic between datacenters](/consul/docs/connect/cluster-peering/usage/establish-cluster-peering) * [Mesh gateways can be used to route control-plane traffic from Consul servers](/consul/docs/connect/gateways/mesh-gateway/peering-via-mesh-gateways) 3. Admin Partitions * [Mesh gateways can be used to route service-to-service traffic between admin partitions in the same Consul datacenter](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-partitions) @@ -57,17 +57,17 @@ Configure the following settings to register the mesh gateway as a service in Co Each upstream associated with a service mesh proxy can be configured so that it is routed through a mesh gateway. Depending on your network, the proxy's connection to the gateway can operate in one of the following modes: -* `none` - No gateway is used and a service mesh connect proxy makes its outbound connections directly +* `none` - No gateway is used and a service mesh sidecar proxy makes its outbound connections directly to the destination services. This is the default for WAN federation. This setting is invalid for peered clusters and will be treated as remote instead. -* `local` - The service mesh connect proxy makes an outbound connection to a gateway running in the +* `local` - The service mesh sidecar proxy makes an outbound connection to a gateway running in the same datacenter. That gateway is responsible for ensuring that the data is forwarded to gateways in the destination datacenter. -* `remote` - The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. +* `remote` - The service mesh sidecar proxy makes an outbound connection to a gateway running in the destination datacenter. The gateway forwards the data to the final destination service. This is the default for peered clusters. -### Connect Proxy Configuration +### Service Mesh Proxy Configuration Set the proxy to the preferred [mode](#modes) to configure the service mesh proxy. You can specify the mode globally or within child configurations to control proxy behaviors at a lower level. Consul recognizes the following order of precedence if the gateway mode is configured in multiple locations the order of precedence: @@ -82,7 +82,7 @@ Use the following example configurations to help you understand some of the comm ### Enabling Gateways Globally -The following `proxy-defaults` configuration will enable gateways for all Connect services in the `local` mode. +The following `proxy-defaults` configuration will enable gateways for all mesh services in the `local` mode. @@ -104,7 +104,7 @@ Name: global ### Enabling Gateways Per Service -The following `service-defaults` configuration will enable gateways for all Connect services with the name `web`. +The following `service-defaults` configuration will enable gateways for all mesh services with the name `web`. diff --git a/website/content/docs/connect/gateways/mesh-gateway/peering-via-mesh-gateways.mdx b/website/content/docs/connect/gateways/mesh-gateway/peering-via-mesh-gateways.mdx index e538a5bc86d..806d161fcdc 100644 --- a/website/content/docs/connect/gateways/mesh-gateway/peering-via-mesh-gateways.mdx +++ b/website/content/docs/connect/gateways/mesh-gateway/peering-via-mesh-gateways.mdx @@ -7,9 +7,7 @@ description: >- # Enabling Peering Control Plane Traffic -In addition to [service-to-service traffic routing](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers), -we recommend routing control plane traffic between cluster peers through mesh gateways -to simplfy networking requirements. +This topic describes how to configure a mesh gateway to route control plane traffic between Consul clusters that share a peer connection. For information about routing service traffic between cluster peers through a mesh gateway, refer to [Enabling Service-to-service Traffic Across Admin Partitions](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-partitions). Control plane traffic between cluster peers includes the initial secret handshake and the bi-directional stream replicating peering data. @@ -59,7 +57,8 @@ For Consul Enterprise clusters, mesh gateways must be registered in the "default -In addition to the [ACL Configuration](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers#acl-configuration) necessary for service-to-service traffic, mesh gateways that route peering control plane traffic must be granted `peering:read` access to all peerings. +In addition to the [ACL Configuration](/consul/docs/connect/cluster-peering/tech-specs#acl-specifications) necessary for service-to-service traffic, mesh gateways that route peering control plane traffic must be granted `peering:read` access to all peerings. + This access allows the mesh gateway to list all peerings in a Consul cluster and generate unique routing per peered datacenter. @@ -80,7 +79,8 @@ peering = "read" -In addition to the [ACL Configuration](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers#acl-configuration) necessary for service-to-service traffic, mesh gateways that route peering control plane traffic must be granted `peering:read` access to all peerings in all partitions. +In addition to the [ACL Configuration](/consul/docs/connect/cluster-peering/tech-specs#acl-specifications) necessary for service-to-service traffic, mesh gateways that route peering control plane traffic must be granted `peering:read` access to all peerings in all partitions. + This access allows the mesh gateway to list all peerings in a Consul cluster and generate unique routing per peered partition. diff --git a/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-partitions.mdx b/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-partitions.mdx index d993b70f698..4e269d00fd4 100644 --- a/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-partitions.mdx +++ b/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-partitions.mdx @@ -36,7 +36,7 @@ Consul can only translate mesh gateway registration information into Envoy confi Sidecar proxies that send traffic to an upstream service through a gateway need to know the location of that gateway. They discover the gateway based on their sidecar proxy registrations. Consul can only translate the gateway registration information into Envoy configuration. -Sidecar proxies that do not send upstream traffic through a gateway are not affected when you deploy gateways. If you are using Consul's built-in proxy as a Connect sidecar it will continue to work for intra-datacenter traffic and will receive incoming traffic even if that traffic has passed through a gateway. +Sidecar proxies that do not send upstream traffic through a gateway are not affected when you deploy gateways. If you are using Consul's built-in proxy as a service mesh sidecar it will continue to work for intra-datacenter traffic and will receive incoming traffic even if that traffic has passed through a gateway. ## Configuration @@ -61,7 +61,7 @@ Depending on your network, the proxy's connection to the gateway can operate in * `remote` - The service mesh connect proxy makes an outbound connection to a gateway running in the destination datacenter. The gateway forwards the data to the final destination service. -### Connect Proxy Configuration +### Service Mesh Proxy Configuration Set the proxy to the preferred [mode](#modes) to configure the service mesh proxy. You can specify the mode globally or within child configurations to control proxy behaviors at a lower level. Consul recognizes the following order of precedence if the gateway mode is configured in multiple locations the order of precedence: @@ -76,7 +76,7 @@ Use the following example configurations to help you understand some of the comm ### Enabling Gateways Globally -The following `proxy-defaults` configuration will enable gateways for all Connect services in the `local` mode. +The following `proxy-defaults` configuration will enable gateways for all mesh services in the `local` mode. @@ -99,7 +99,7 @@ Name: global ### Enabling Gateways Per Service -The following `service-defaults` configuration will enable gateways for all Connect services with the name `web`. +The following `service-defaults` configuration will enable gateways for all mesh services with the name `web`. diff --git a/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers.mdx b/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers.mdx deleted file mode 100644 index ee5a3e38d38..00000000000 --- a/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers.mdx +++ /dev/null @@ -1,60 +0,0 @@ ---- -layout: docs -page_title: Enabling Service-to-service Traffic Across Peered Clusters -description: >- - Mesh gateways are specialized proxies that route data between services that cannot communicate directly. Learn how to enable service-to-service traffic across clusters in different datacenters or admin partitions that have an established peering connection. ---- - -# Enabling Service-to-service Traffic Across Peered Clusters - -Mesh gateways are required for you to route service mesh traffic between peered Consul clusters. Clusters can reside in different clouds or runtime environments where general interconnectivity between all services in all clusters is not feasible. - -At a minimum, a peered cluster exporting a service must have a mesh gateway registered. -For Enterprise, this mesh gateway must also be registered in the same partition as the exported service(s). -To use the `local` mesh gateway mode, there must also be a mesh gateway regsitered in the importing cluster. - -Unlike mesh gateways for WAN-federated datacenters and partitions, mesh gateways between peers terminate mTLS sessions to decrypt data to HTTP services and then re-encrypt traffic to send to services. Data must be decrypted in order to evaluate and apply dynamic routing rules at the destination cluster, which reduces coupling between peers. - -## Prerequisites - -To configure mesh gateways for cluster peering, make sure your Consul environment meets the following requirements: - -- Consul version 1.14.0 or newer. -- A local Consul agent is required to manage mesh gateway configuration. -- Use [Envoy proxies](/consul/docs/connect/proxies/envoy). Envoy is the only proxy with mesh gateway capabilities in Consul. - -## Configuration - -Configure the following settings to register and use the mesh gateway as a service in Consul. - -### Gateway registration - -- Specify `mesh-gateway` in the `kind` field to register the gateway with Consul. -- Define the `Proxy.Config` settings using opaque parameters compatible with your proxy. For Envoy, refer to the [Gateway Options](/consul/docs/connect/proxies/envoy#gateway-options) and [Escape-hatch Overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides) documentation for additional configuration information. - -Alternatively, you can also use the CLI to spin up and register a gateway in Consul. For additional information, refer to the [`consul connect envoy` command](/consul/commands/connect/envoy#mesh-gateways). - -### Sidecar registration - -- Configure the `proxy.upstreams` parameters to route traffic to the correct service, namespace, and peer. Refer to the [`upstreams` documentation](/consul/docs/connect/registration/service-registration#upstream-configuration-reference) for details. -- The service `proxy.upstreams.destination_name` is always required. -- The `proxy.upstreams.destination_peer` must be configured to enable cross-cluster traffic. -- The `proxy.upstream/destination_namespace` configuration is only necessary if the destination service is in a non-default namespace. - -### Service exports - -- Include the `exported-services` configuration entry to enable Consul to export services contained in a cluster to one or more additional clusters. For additional information, refer to the [Exported Services documentation](/consul/docs/connect/config-entries/exported-services). - -### ACL configuration - -If ACLs are enabled, you must add a token granting `service:write` for the gateway's service name and `service:read` for all services in the Enterprise admin partition or OSS datacenter to the gateway's service definition. -These permissions authorize the token to route communications for other Consul service mesh services. - -You must also grant `mesh:write` to mesh gateways routing peering traffic in the data plane. -This permission allows a leaf certificate to be issued for mesh gateways to terminate TLS sessions for HTTP requests. - -### Modes - -Modes are configurable as either `remote` or `local` for mesh gateways that connect peered clusters. -The `none` setting is invalid for mesh gateways in peered clusters and will be ignored by the gateway. -By default, all proxies connecting to peered clusters use mesh gateways in [remote mode](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#remote). diff --git a/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters.mdx b/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters.mdx index 520f6c01ff2..5a764c43724 100644 --- a/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters.mdx +++ b/website/content/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters.mdx @@ -29,10 +29,10 @@ Ensure that your Consul environment meets the following requirements. * Consul version 1.6.0 or newer. * A local Consul agent is required to manage its configuration. -* Consul [Connect](/consul/docs/agent/config/config-files#connect) must be enabled in both datacenters. +* Consul [service mesh](/consul/docs/agent/config/config-files#connect) must be enabled in both datacenters. * Each [datacenter](/consul/docs/agent/config/config-files#datacenter) must have a unique name. * Each datacenters must be [WAN joined](/consul/tutorials/networking/federation-gossip-wan). -* The [primary datacenter](/consul/docs/agent/config/config-files#primary_datacenter) must be set to the same value in both datacenters. This specifies which datacenter is the authority for Connect certificates and is required for services in all datacenters to establish mutual TLS with each other. +* The [primary datacenter](/consul/docs/agent/config/config-files#primary_datacenter) must be set to the same value in both datacenters. This specifies which datacenter is the authority for service mesh certificates and is required for services in all datacenters to establish mutual TLS with each other. * [gRPC](/consul/docs/agent/config/config-files#grpc_port) must be enabled. * If you want to [enable gateways globally](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#enabling-gateways-globally) you must enable [centralized configuration](/consul/docs/agent/config/config-files#enable_central_service_config). @@ -50,7 +50,7 @@ Consul can only translate mesh gateway registration information into Envoy confi Sidecar proxies that send traffic to an upstream service through a gateway need to know the location of that gateway. They discover the gateway based on their sidecar proxy registrations. Consul can only translate the gateway registration information into Envoy configuration. -Sidecar proxies that do not send upstream traffic through a gateway are not affected when you deploy gateways. If you are using Consul's built-in proxy as a Connect sidecar it will continue to work for intra-datacenter traffic and will receive incoming traffic even if that traffic has passed through a gateway. +Sidecar proxies that do not send upstream traffic through a gateway are not affected when you deploy gateways. If you are using Consul's built-in proxy as a service mesh sidecar it will continue to work for intra-datacenter traffic and will receive incoming traffic even if that traffic has passed through a gateway. ## Configuration @@ -66,18 +66,18 @@ Configure the following settings to register the mesh gateway as a service in Co Each upstream associated with a service mesh proxy can be configured so that it is routed through a mesh gateway. Depending on your network, the proxy's connection to the gateway can operate in one of the following modes (refer to the [mesh-architecture-diagram](#mesh-architecture-diagram)): -* `none` - (Default) No gateway is used and a service mesh connect proxy makes its outbound connections directly +* `none` - (Default) No gateway is used and a service mesh sidecar proxy makes its outbound connections directly to the destination services. -* `local` - The service mesh connect proxy makes an outbound connection to a gateway running in the +* `local` - The service mesh sidecar proxy makes an outbound connection to a gateway running in the same datacenter. That gateway is responsible for ensuring that the data is forwarded to gateways in the destination datacenter. Refer to the flow labeled `local` in the [mesh-architecture-diagram](#mesh-architecture-diagram). -* `remote` - The service mesh proxy makes an outbound connection to a gateway running in the destination datacenter. +* `remote` - The service mesh sidecar proxy makes an outbound connection to a gateway running in the destination datacenter. The gateway forwards the data to the final destination service. Refer to the flow labeled `remote` in the [mesh-architecture-diagram](#mesh-architecture-diagram). -### Connect Proxy Configuration +### Service Mesh Proxy Configuration Set the proxy to the preferred [mode](#modes) to configure the service mesh proxy. You can specify the mode globally or within child configurations to control proxy behaviors at a lower level. Consul recognizes the following order of precedence if the gateway mode is configured in multiple locations the order of precedence: @@ -92,7 +92,7 @@ Use the following example configurations to help you understand some of the comm ### Enabling Gateways Globally -The following `proxy-defaults` configuration will enable gateways for all Connect services in the `local` mode. +The following `proxy-defaults` configuration will enable gateways for all mesh services in the `local` mode. @@ -114,7 +114,7 @@ Name: global ### Enabling Gateways Per Service -The following `service-defaults` configuration will enable gateways for all Connect services with the name `web`. +The following `service-defaults` configuration will enable gateways for all mesh services with the name `web`. diff --git a/website/content/docs/connect/gateways/terminating-gateway.mdx b/website/content/docs/connect/gateways/terminating-gateway.mdx index 05f48eee5f4..17a926c0cd9 100644 --- a/website/content/docs/connect/gateways/terminating-gateway.mdx +++ b/website/content/docs/connect/gateways/terminating-gateway.mdx @@ -10,8 +10,8 @@ description: >- -> **1.8.0+:** This feature is available in Consul versions 1.8.0 and newer. Terminating gateways enable connectivity within your organizational network from services in the Consul service mesh to -services and [destinations](/consul/docs/connect/config-entries/service-defaults#terminating-gateway-destination) outside the mesh. These gateways effectively act as Connect proxies that can -represent more than one service. They terminate Connect mTLS connections, enforce intentions, +services and [destinations](/consul/docs/connect/config-entries/service-defaults#terminating-gateway-destination) outside the mesh. These gateways effectively act as service mesh proxies that can +represent more than one service. They terminate service mesh mTLS connections, enforce intentions, and forward requests to the appropriate destination. ![Terminating Gateway Architecture](/img/terminating-gateways.png) @@ -26,7 +26,7 @@ for filtering by instance. ## Security Considerations ~> We recommend that terminating gateways are not exposed to the WAN or open internet. This is because terminating gateways -hold certificates to decrypt Consul Connect traffic directed at them and may be configured with credentials to connect +hold certificates to decrypt Consul service mesh traffic directed at them and may be configured with credentials to connect to linked services. Connections over the WAN or open internet should flow through [mesh gateways](/consul/docs/connect/gateways/mesh-gateway) whenever possible since they are not capable of decrypting traffic or connecting directly to services. @@ -58,7 +58,7 @@ Each terminating gateway needs: Terminating gateways also require that your Consul datacenters are configured correctly: - You'll need to use Consul version 1.8.0 or newer. -- Consul [Connect](/consul/docs/agent/config/config-files#connect) must be enabled on the datacenter's Consul servers. +- Consul [service mesh](/consul/docs/agent/config/config-files#connect) must be enabled on the datacenter's Consul servers. - [gRPC](/consul/docs/agent/config/config-files#grpc_port) must be enabled on all client agents. Currently, [Envoy](https://www.envoyproxy.io/) is the only proxy with terminating gateway capabilities in Consul. @@ -68,9 +68,9 @@ Currently, [Envoy](https://www.envoyproxy.io/) is the only proxy with terminatin can only translate terminating gateway registration information into Envoy configuration, therefore the proxies acting as terminating gateways must be Envoy. -Connect proxies that send upstream traffic through a gateway aren't +Service mesh proxies that send upstream traffic through a gateway aren't affected when you deploy terminating gateways. If you are using non-Envoy proxies as -Connect proxies they will continue to work for traffic directed at services linked to +Service mesh proxies they will continue to work for traffic directed at services linked to a terminating gateway as long as they discover upstreams with the [/health/connect](/consul/api-docs/health#list-nodes-for-connect-capable-service) endpoint. @@ -84,7 +84,7 @@ services outside the mesh, review the [terminating gateway tutorial](/consul/tut Terminating gateways are configured in service definitions and registered with Consul like other services, with two exceptions. The first is that the [kind](/consul/api-docs/agent/service#kind) must be "terminating-gateway". Second, the terminating gateway service definition may contain a `Proxy.Config` entry just like a -Connect proxy service, to define opaque configuration parameters useful for the actual proxy software. +service mesh proxy service, to define opaque configuration parameters useful for the actual proxy software. For Envoy there are some supported [gateway options](/consul/docs/connect/proxies/envoy#gateway-options) as well as [escape-hatch overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides). diff --git a/website/content/docs/connect/index.mdx b/website/content/docs/connect/index.mdx index cbd2903a331..6bdc9989f52 100644 --- a/website/content/docs/connect/index.mdx +++ b/website/content/docs/connect/index.mdx @@ -5,17 +5,22 @@ description: >- Consul’s service mesh makes application and microservice networking secure and observable with identity-based authentication, mutual TLS (mTLS) encryption, and explicit service-to-service authorization enforced by sidecar proxies. Learn how Consul’s service mesh works and get started on VMs or Kubernetes. --- -# Consul Service Mesh +# Consul service mesh + +Consul service mesh provides service-to-service connection authorization and +encryption using mutual Transport Layer Security (TLS). -Consul Service Mesh provides service-to-service connection authorization and -encryption using mutual Transport Layer Security (TLS). Consul Connect is used interchangeably -with the name Consul Service Mesh and is what this document will use to refer to for Service Mesh functionality within Consul. Applications can use [sidecar proxies](/consul/docs/connect/proxies) in a service mesh configuration to -establish TLS connections for inbound and outbound connections without being aware of Connect at all. -Applications may also [natively integrate with Connect](/consul/docs/connect/native) for optimal performance and security. -Connect can help you secure your services and provide data about service-to-service communications. +establish TLS connections for inbound and outbound connections without being aware of the service mesh at all. +Applications may also [natively integrate with Consul service mesh](/consul/docs/connect/native) for optimal performance and security. +Consul service mesh can help you secure your services and provide data about service-to-service communications. + +The noun _connect_ is used throughout this documentation to refer to the connect +subsystem that provides Consul's service mesh capabilities. +Where you encounter the _noun_ connect, it is usually functionality specific to +service mesh. -Review the video below to learn more about Consul Connect from HashiCorp's co-founder Armon. +Review the video below to learn more about Consul service mesh from HashiCorp's co-founder Armon. -## Application Security +## Application security -Connect enables secure deployment best-practices with automatic +Consul service mesh enables secure deployment best-practices with automatic service-to-service encryption, and identity-based authorization. -Connect uses the registered service identity (rather than IP addresses) to +Consul uses the registered service identity, rather than IP addresses, to enforce access control with [intentions](/consul/docs/connect/intentions). This -makes it easier to reason about access control and enables services to be -rescheduled by orchestrators including Kubernetes and Nomad. Intention -enforcement is network agnostic, so Connect works with physical networks, cloud +makes it easier to control access and enables services to be +rescheduled by orchestrators, including Kubernetes and Nomad. Intention +enforcement is network agnostic, so Consul service mesh works with physical networks, cloud networks, software-defined networks, cross-cloud, and more. ## Observability -One of the key benefits of Consul Connect is the uniform and consistent view it can +One of the key benefits of Consul service mesh is the uniform and consistent view it can provide of all the services on your network, irrespective of their different -programming languages and frameworks. When you configure Consul Connect to use -sidecar proxies, those proxies "see" all service-to-service traffic and can -collect data about it. Consul Connect can configure Envoy proxies to collect +programming languages and frameworks. When you configure Consul service mesh to use +sidecar proxies, those proxies see all service-to-service traffic and can +collect data about it. Consul service mesh can configure Envoy proxies to collect layer 7 metrics and export them to tools like Prometheus. Correctly instrumented applications can also send open tracing data through Envoy. -## Getting Started With Consul Service Mesh +## Getting started with Consul service mesh -There are several ways to try Connect in different environments. +Complete the following tutorials try Consul service mesh in different environments: - The [Getting Started with Consul Service Mesh collection](/consul/tutorials/kubernetes-deploy/service-mesh?utm_source=docs) walks you through installing Consul as service mesh for Kubernetes using the Helm @@ -59,11 +64,10 @@ There are several ways to try Connect in different environments. - The [Secure Service-to-Service Communication tutorial](/consul/tutorials/developer-mesh/service-mesh-with-envoy-proxy?utm_source=docs) is a simple walk through of connecting two services on your local machine - using Consul Connect's built-in proxy and configuring your first intention. The guide also includes an introduction to - using Envoy as the Connect sidecar proxy. + and configuring your first intention. - The [Kubernetes tutorial](/consul/tutorials/kubernetes/kubernetes-minikube?utm_source=docs) - walks you through configuring Consul Connect in Kubernetes using the Helm + walks you through configuring Consul service mesh in Kubernetes using the Helm chart, and using intentions. You can run the guide on Minikube or an existing Kubernetes cluster. diff --git a/website/content/docs/connect/intentions.mdx b/website/content/docs/connect/intentions.mdx deleted file mode 100644 index b3b0371a93a..00000000000 --- a/website/content/docs/connect/intentions.mdx +++ /dev/null @@ -1,341 +0,0 @@ ---- -layout: docs -page_title: Service Mesh Intentions -description: >- - Intentions define communication permissions in the service mesh between microservices. Learn about configuration basics, wildcard intentions, precedence and match order, and protecting intention management with ACLs. ---- - -# Service Mesh Intentions - --> **1.9.0 and later:** This guide only applies in Consul versions 1.9.0 and -later. The documentation for the legacy intentions system is -[here](/consul/docs/connect/intentions-legacy). - -Intentions define access control for services via Connect and are used to -control which services may establish connections or make requests. Intentions -can be managed via the API, CLI, or UI. - -Intentions are enforced on inbound connections or requests by the -[proxy](/consul/docs/connect/proxies) or within a [natively integrated -application](/consul/docs/connect/native). - -Depending upon the [protocol] in use by the destination service, you can define -intentions to control Connect traffic authorization either at networking layer -4 (e.g. TCP) and application layer 7 (e.g. HTTP): - -- **Identity-based** - All intentions may enforce access based on identities - encoded within [TLS - certificates](/consul/docs/connect/connect-internals#mutual-transport-layer-security-mtls). - This allows for coarse all-or-nothing access control between pairs of - services. These work with for services with any [protocol] as they only - require awareness of the TLS handshake that wraps the opaque TCP connection. - These can also be thought of as **L4 intentions**. - -- **Application-aware** - Some intentions may additionally enforce access based - on [L7 request - attributes](/consul/docs/connect/config-entries/service-intentions#permissions) in - addition to connection identity. These may only be defined for services with - a [protocol] that is HTTP-based. These can also be thought of as **L7 - intentions**. - -At any given point in time, between any pair of services **only one intention -controls authorization**. This may be either an L4 intention or an L7 -intention, but at any given point in time only one of those applies. - -The [intention match API](/consul/api-docs/connect/intentions#list-matching-intentions) -should be periodically called to retrieve all relevant intentions for the -target destination. After verifying the TLS client certificate, the cached -intentions should be consulted for each incoming connection/request to -determine if it should be accepted or rejected. - -The default intention behavior is defined by the [`default_policy`](/consul/docs/agent/config/config-files#acl_default_policy) configuration. -If the configuration is set `allow`, then all service mesh Connect connections will be allowed by default. -If is set to `deny`, then all connections or requests will be denied by default. - -## Intention Basics - -You can define a [`service-intentions`](/consul/docs/connect/config-entries/service-intentions) configuration entry to create and manage intentions, as well as manage intentions through the Consul UI. You can also perform some intention-related tasks using the API and CLI commands. Refer to the [API](/consul/api-docs/connect/intentions) and [CLI](/consul/commands/intention) documentation for details. - -The following example shows a `service-intentions` configuration entry that specifies two intentions. Refer to the [`service-intentions`](/consul/docs/connect/config-entries/service-intentions) documentation for the full data model and additional examples. - - - -```hcl -Kind = "service-intentions" -Name = "db" -Sources = [ - { - Name = "web" - Action = "deny" - }, - { - Name = "api" - Action = "allow" - } -] -``` - -```json -{ - "Kind": "service-intentions", - "Name": "db", - "Sources": [ - { - "Action": "deny", - "Name": "web" - }, - { - "Action": "allow", - "Name": "api" - } - ] -} -``` - - - -This configuration entry defines two intentions with a common destination of `db`. The -first intention above is a deny intention with a source of `web`. This says -that connections from web to db are not allowed and the connection will be -rejected. The second intention is an allow intention with a source of `api`. -This says that connections from api to db are allowed and connections will be -accepted. - -### Wildcard Intentions - -You can use the `*` wildcard to match service names when defining an intention source or destination. The wildcard matches _any_ value, which enables you to set a wide initial scope when configuring intentions. - -The wildcard is supported in Consul Enterprise `namespace` fields (see [Namespaces](/consul/docs/enterprise/namespaces) for additional information), but it _is not supported_ in `partition` fields (see [Admin Partitions](/consul/docs/enterprise/admin-partitions) for additional information). - -In the following example, the `web` service cannot connect to _any_ service: - - - -```hcl -Kind = "service-intentions" -Name = "*" -Sources = [ - { - Name = "web" - Action = "deny" - } -] -``` - -```json -{ - "Kind": "service-intentions", - "Name": "*", - "Sources": [ - { - "Action": "deny", - "Name": "web" - } - ] -} -``` - - - -The `db` service is configured to deny all connection in the following example: - - - -```hcl -Kind = "service-intentions" -Name = "db" -Sources = [ - { - Name = "*" - Action = "deny" - } -] -``` - -```json -{ - "Kind": "service-intentions", - "Name": "db", - "Sources": [ - { - "Action": "deny", - "Name": "*" - } - ] -} -``` - - - - This example grants Prometheus access to any service in -any namespace. - - - -```hcl -Kind = "service-intentions" -Name = "*" -Namespace = "*" -Sources = [ - { - Name = "prometheus" - Namespace = "monitoring" - Action = "allow" - } -] -``` - -```json -{ - "Kind": "service-intentions", - "Name": "*", - "Namespace": "*", - "Sources": [ - { - "Action": "allow", - "Name": "prometheus", - "Namespace": "monitoring" - } - ] -} -``` - - - -### Enforcement - -For services that define their [protocol] as TCP, intentions mediate the -ability to **establish new connections**. When an intention is modified, -existing connections will not be affected. This means that changing a -connection from "allow" to "deny" today _will not_ kill the connection. - -For services that define their protocol as HTTP-based, intentions mediate the -ability to **issue new requests**. - -When an intention is modified, requests received after the modification will -use the latest intention rules to enforce access. This means that though -changing a connection from "allow" to "deny" today will not kill the -connection, it will correctly block new requests from being processed. - -## Precedence and Match Order - -Intentions are matched in an implicit order based on specificity, preferring -deny over allow. Specificity is determined by whether a value is an exact -specified value or is the wildcard value `*`. -The full precedence table is shown below and is evaluated -top to bottom, with larger numbers being evaluated first. - -| Source Namespace | Source Name | Destination Namespace | Destination Name | Precedence | -| ---------------- | ----------- | --------------------- | ---------------- | ---------- | -| Exact | Exact | Exact | Exact | 9 | -| Exact | `*` | Exact | Exact | 8 | -| `*` | `*` | Exact | Exact | 7 | -| Exact | Exact | Exact | `*` | 6 | -| Exact | `*` | Exact | `*` | 5 | -| `*` | `*` | Exact | `*` | 4 | -| Exact | Exact | `*` | `*` | 3 | -| Exact | `*` | `*` | `*` | 2 | -| `*` | `*` | `*` | `*` | 1 | - -The precedence value can be read from a -[field](/consul/docs/connect/config-entries/service-intentions#precedence) on the -`service-intentions` configuration entry after it is modified. Precedence cannot be -manually overridden today. - -The numbers in the table above are not stable. Their ordering will remain -fixed but the actual number values may change in the future. - --> - Namespaces are an Enterprise feature. In Consul -OSS the only allowable value for either namespace field is `"default"`. Other -rows in this table are not applicable. - -## Intention Management Permissions - -Intention management can be protected by [ACLs](/consul/docs/security/acl). -Permissions for intentions are _destination-oriented_, meaning the ACLs -for managing intentions are looked up based on the destination value -of the intention, not the source. - -Intention permissions are by default implicitly granted at `read` level -when granting `service:read` or `service:write`. This is because a -service registered that wants to use Connect needs `intentions:read` -for its own service name in order to know whether or not to authorize -connections. The following ACL policy will implicitly grant `intentions:read` -(note _read_) for service `web`. - - - -```hcl -service "web" { - policy = "write" -} -``` - -```json -{ - "service": [ - { - "web": [ - { - "policy": "write" - } - ] - } - ] -} -``` - - - -It is possible to explicitly specify intention permissions. For example, -the following policy will allow a service to be discovered without granting -access to read intentions for it. - -```hcl -service "web" { - policy = "read" - intentions = "deny" -} -``` - -Note that `intentions:read` is required for a token that a Connect-enabled -service uses to register itself or its proxy. If the token used does not -have `intentions:read` then the agent will be unable to resolve intentions -for the service and so will not be able to authorize any incoming connections. - -~> **Security Note:** Explicitly allowing `intentions:write` on the token you -provide to a service instance at registration time opens up a significant -additional vulnerability. Although you may trust the service _team_ to define -which inbound connections they accept, using a combined token for registration -allows a compromised instance to to redefine the intentions which allows many -additional attack vectors and may be hard to detect. We strongly recommend only -delegating `intentions:write` using tokens that are used by operations teams or -orchestrators rather than spread via application config, or only manage -intentions with management tokens. - -## Performance and Intention Updates - -The intentions for services registered with a Consul agent are cached -locally on that agent. They are then updated via a background blocking query -against the Consul servers. - -Supported [proxies] (such as [Envoy]) also cache this data within their own -configuration so that inbound connections or requests require no Consul agent -involvement during authorization. All actions in the data path of connections -happen within the proxy. - -Updates to intentions are propagated nearly instantly to agents since agents -maintain a continuous blocking query in the background for intention updates -for registered services. Proxies similarly use blocking queries to update -their local configurations quickly. - -Because all the intention data is cached locally, the agents or proxies can -fail static. Even if the agents are severed completely from the Consul servers, -or the proxies are severed completely from their local Consul agent, inbound -connection authorization continues to work indefinitely. Changes to intentions -will not be picked up until the partition heals, but will then automatically -take effect when connectivity is restored. - -[protocol]: /consul/docs/connect/config-entries/service-defaults#protocol -[proxies]: /consul/docs/connect/proxies -[envoy]: /consul/docs/connect/proxies/envoy diff --git a/website/content/docs/connect/intentions/create-manage-intentions.mdx b/website/content/docs/connect/intentions/create-manage-intentions.mdx new file mode 100644 index 00000000000..80e68ce890a --- /dev/null +++ b/website/content/docs/connect/intentions/create-manage-intentions.mdx @@ -0,0 +1,178 @@ +--- +layout: docs +page_title: Create and manage service intentions +description: >- + Learn how to create and manage Consul service mesh intentions using service-intentions config entries, the `consul intentions` command, and `/connect/intentions` API endpoint. +--- + +# Create and manage intentions + +This topic describes how to create and manage service intentions, which are configurations for controlling access between services in the service mesh. + +## Overview + +You can create single intentions or create them in batches using the Consul API, CLI, or UI. You can also define a service intention configuration entry that sets default intentions for all services in the mesh. Refer to [Service intentions overview](/consul/docs/connnect/intentions/intentions) for additional background information about intentions. + +## Requirements + +- At least two services must be registered in the datacenter. +- TLS must be enabled to enforce L4 intentions. Refer to [Encryption](/consul/docs/security/encryption) for additional information. + +### ACL requirements + +Consul grants permissions for creating and managing intentions based on the destination, not the source. When ACLs are enabled, services and operators must present a token linked to a policy that grants read and write permissions to the destination service. + +Consul implicitly grants `intentions:read` permissions to destination services when they are configured with `service:read` or `service:write` permissions. This is so that the services can allow or deny inbound connections when they attempt to join the service mesh. Refer to [Service rules](/consul/docs/security/acl/acl-rules#service-rules) for additional information about configuring ALCs for intentions. + +The default ACL policy configuration determines the default behavior for intentions. If the policy is set to `deny`, then all connections or requests are denied and you must enable them explicitly. Refer to [`default_policy`](/consul/docs/agent/config/config-files#acl_default_policy) for details. + +## Create an intention + +You can create and manage intentions one at a time using the Consul API, CLI, or UI You can specify one destination or multiple destinations in a single intention. + +### API + +Send a `PUT` request to the `/connect/intentions/exact` HTTP API endpoint and specify the following query parameters: + +- `source`: Service sending the request +- `destination`: Service responding to the request +- `ns`: Namespace of the destination service + +For L4 intentions, you must also specify the intention action in the request payload. + +The following example creates an intention that allows `web` to send request to `db`: + +```shell-session +$ curl --request PUT \ +--data ' { "Action": "allow" } ' \ +http://localhost:8500/v1/connect/intentions/exact\?source\=web\&destination\=db +``` + +Refer to the `/connect/intentions/exact` [HTTP API endpoint documentation](/consul/api-docs/connect/intentions) for additional information request payload parameters. + +For L7 intentions, specify the `Permissions` in the request payload to configure attributes for dynamically enforcing intentions. In the following example payload, Consul allows HTTP GET requests if the request body is empty: + + + +```json +{ + "Permissions": [ + { + "Action": "allow", + "HTTP": { + "Methods": ["GET"], + "Header": [ + { + "Name": "Content-Length", + "Exact": "0" + } + ] + } + } + ] +} + +``` + + + +The `Permissions` object specifies a list of permissions for L7 traffic sources. The list contains one or more actions and a set of match criteria for each action. Refer to the [`Sources[].Permissions[]` parameter](/consul/connect/config-entries/service-intentions#source-permissions) in the service intentions configuration entry reference for configuration details. + +To apply the intention, call the endpoint and pass the configuration file containing the attributes to the endpoint: + +```shell-session +$ curl --request PUT \ +--data @payload.json \ +http://localhost:8500/v1/connect/intentions/exact\?source\=svc1\&destination\=sv2 +``` +### CLI + +Use the `consul intention create` command according to the following syntax to create a new intention: + +```shell-session +$ consul intention create - +``` + +The following example creates an intention that allows `web` service instances to connect to `db` service instances: + +```shell-session +$ consul intention create -allow web db +``` + +You can use the asterisk (`*`) wildcard to specify multiple destination services. Refer to [Precedence and match order](/consul/docs/connect/intentions/create-manage-intentions#precedence-and-match-order) for additional information. + +### Consul UI + +1. Log into the Consul UI and choose **Services** from the sidebar menu. +1. Click on a service and then click the **Intentions* tab. +1. Click **Create** and choose the source service from the drop-down menu. +1. You can add an optional description. +1. Choose one of the following options: + 1. **Allow**: Allows the source service to send requests to the destination. + 1. **Deny**: Prevents the source service from sending requests to the destination. + 1. **Application Aware**: Enables you to specify L7 criteria for dynamically enforcing intentions. Refer to [Configure application aware settings](#configure-application-aware-settings) for additional information. +1. Click **Save**. + +Repeat the procedure as necessary to create additional intentions. + +#### Configure application aware settings + +You can use the Consul UI to configure L7 permissions. + +1. Click **Add permission** to open the permission editor. +1. Enable the **Allow** or **Deny** option. +1. You can specify a path, request method, and request headers to match. All criteria must be satisfied for Consul to enforce the permission. Refer to the [`Sources[].Permissions[]` parameter](/consul/docs/connect/config-entries/service-intentions#sources-permissions) in the service intentions configuration entry reference for information about the available configuration fields. +1. Click **Save**. + +Repeat the procedure as necessary to create additional permissions. + +## Create multiple intentions + +You can create a service intentions configuration entry to specify default intentions for your service mesh. You can specify default settings for L4 or L7 application-aware traffic. + +### Define a service intention configuration entry + +Configure the following fields: + + + + + +- [`Kind`](/consul/docs/connect/config-entries/service-intentions#kind): Declares the type of configuration entry. Must be set to `service-intentions`. +- [`Name`](/consul/docs/connect/config-entries/service-intentions#kind): Specifies the name of the destination service for intentions defined in the configuration entry. You can use a wildcard character (*) to set L4 intentions for all services that are not protected by specific intentions. Wildcards are not supported for L7 intentions. +- [`Sources`](/consul/docs/connect/config-entries/service-intentions#sources): Specifies an unordered list of all intention sources and the authorizations granted to those sources. Consul stores and evaluates the list in reverse order sorted by intention precedence. +- [`Sources.Action`](/consul/docs/connect/config-entries/service-intentions#sources-action) or [`Sources.Permissions`](/consul/docs/connect/config-entries/service-intentions#sources-permissions): For L4 intentions, set the `Action` field to "allow" or "deny" so that Consul can enforce intentions that match the source service. For L7 intentions, configure the `Permissions` settings, which define a set of application-aware attributes for dynamically matching incoming requests. The `Actions` and `Permissions` settings are mutually exclusive. + + + + + +- [`apiVersion`](/consul/docs/connect/config-entries/service-intentions#apiversion): Specifies the Consul API version. Must be set to `consul.hashicorp.com/v1alpha1`. +- [`kind`](/consul/docs/connect/config-entries/service-intentions#kind): Declares the type of configuration entry. Must be set to `ServiceIntentions`. +- [`spec.destination.name`](/consul/docs/connect/config-entries/service-intentions#spec-destination-name): Specifies the name of the destination service for intentions defined in the configuration entry. You can use a wildcard character (*) to set L4 intentions for all services that are not protected by specific intentions. Wildcards are not supported for L7 intentions. +- [`spec.sources`](/consul/docs/connect/config-entries/service-intentions#spec-sources): Specifies an unordered list of all intention sources and the authorizations granted to those sources. Consul stores and evaluates the list in reverse order sorted by intention precedence. +- [`spec.sources.action`](/consul/docs/connect/config-entries/service-intentions#spec-sources-action) or [`spec.sources.permissions`](/consul/docs/connect/config-entries/service-intentions#spec-sources-permissions): For L4 intentions, set the `action` field to "allow" or "deny" so that Consul can enforce intentions that match the source service. For L7 intentions, configure the `permissions` settings, which define a set of application-aware attributes for dynamically matching incoming requests. The `actions` and `permissions` settings are mutually exclusive. + + + + + +Refer to the [service intentions configuration entry](/consul/docs/connect/config-entries/service-intentions) reference documentation for details about all configuration options. + +Refer to the [example service intentions configurations](/consul/docs/connect/config-entries/service-intentions#examples) for additional guidance. + +#### Interaction with other configuration entries + +L7 intentions defined in a configuration entry are restricted to destination services +configured with an HTTP-based protocol as defined in a corresponding +[service defaults configuration entry](/consul/docs/connect/config-entries/service-defaults) +or globally in a [proxy defaults configuration entry](/consul/docs/connect/config-entries/proxy-defaults). + +### Apply the service intentions configuration entry + +You can apply the configuration entry using the [`consul config` command](/consul/commands/config) or by calling the [`/config` API endpoint](/consul/api-docs/config). In Kubernetes environments, apply the `ServiceIntentions` custom resource definitions (CRD) to implement and manage Consul configuration entries. + +Refer to the following topics for details about applying configuration entries: + +- [How to Use Configuration Entries](/consul/docs/agent/config-entries) +- [Custom Resource Definitions for Consul on Kubernetes](/consul/docs/k8s/crds) \ No newline at end of file diff --git a/website/content/docs/connect/intentions/index.mdx b/website/content/docs/connect/intentions/index.mdx new file mode 100644 index 00000000000..9cb0f36ee1b --- /dev/null +++ b/website/content/docs/connect/intentions/index.mdx @@ -0,0 +1,91 @@ +--- +layout: docs +page_title: Service mesh intentions overview +description: >- + Intentions are access controls that allow or deny incoming requests to services in the mesh. +--- + +# Service intentions overview + +This topic provides overview information about Consul intentions, which are mechanisms that control traffic communication between services in the Consul service mesh. + +![Diagram showing how service intentions control access between services](/img/consul-connect/consul-service-mesh-intentions-overview.svg) + +## Intention types + +Intentions control traffic communication between services at the network layer, also called _L4_ traffic, or the application layer, also called _L7 traffic_. The protocol that the destination service uses to send and receive traffic determines the type of authorization the intention can enforce. + +### L4 traffic intentions + +If the destination service uses TCP or any non-HTTP-based protocol, then intentions can control traffic based on identities encoded in mTLS certificates. Refer to [Mutual transport layer security (mTLS)](/consul/docs/connect/connect-internals#mutual-transport-layer-security-mtls) for additional information. + +This implementation allows broad all-or-nothing access control between pairs of services. The only requirement is that the service is aware of the TLS handshake that wraps the opaque TCP connection. + +### L7 traffic intentions + +If the destination service uses an HTTP-based protocol, then intentions can enforce access based on application-aware request attributes, in addition to identity-based enforcement, to control traffic between services. Refer to [Service intentions configuration reference](/consul/docs/connect/config-entries/service-intentions#permissions) for additional information. + +## Workflow + +You can manually create intentions from the Consul UI, API, or CLI. You can also enable Consul to dynamically create them by defining traffic routes in service intention configuration entries. Refer to [Create and manage intentions](/consul/docs/connect/intentions/create-manage-intentions) for details. + +### Enforcement + +The [proxy](/consul/docs/connect/proxies) or [natively-integrated +application](/consul/docs/connect/native) enforces intentions on inbound connections or requests. Only one intention can control authorization between a pair of services at any single point in time. + +L4 intentions mediate the ability to establish new connections. Modifying an intention does not have an effect on existing connections. As a result, changing a connection from `allow` to `deny` does not sever the connection. + +L7 intentions mediate the ability to issue new requests. When an intention is modified, requests received after the modification use the latest intention rules to enforce access. Changing a connection from `allow` to `deny` does not sever the connection, but doing so blocks new requests from being processed. + +### Caching + +The intentions for services registered with a Consul agent are cached locally on the agent. Supported proxies also cache intention data in their own configurations so that they can authorize inbound connections or requests without relying on the Consul agent. All actions in the data path of connections take place within the proxy. + +### Updates + +Consul propagates updates to intentions almost instantly as a result of the continuous blocking query the agent uses. A _blocking query_ is a Consul API feature that uses long polling to wait for potential changes. Refer to [Blocking Queries](/consul/api-docs/features/blocking) for additional information. Proxies also use blocking queries to quickly update their local configurations. + +Because all intention data is cached locally, authorizations for inbound connection persist, even if the agents are completely severed from the Consul servers or if the proxies are completely severed from their local Consul agent. If the connection is severed, Consul automatically applies changes to intentions when connectivity is restored. + +### Intention maintenance + +Services should periodically call the [intention match API](/consul/api-docs/connect/intentions#list-matching-intentions) to retrieve all relevant intentions for the target destination. After verifying the TLS client certificate, the cached intentions for each incoming connection or request determine if it should be accepted or rejected. + +## Precedence and match order + +Consul processes criteria defined in the service intention configuration entry to match incoming requests. When Consul finds a match, it applies the corresponding action specified in the configuration entry. The match criteria may include specific HTTP headers, request methods, or other attributes. Additionally, you can use regular expressions to programmatically match attributes. Refer to [Service intention configuration entry reference](/consul/docs/connect/config-entries/service-intentions) for details. + +Consul orders the matches based the following factors: + +- Specificity: Incoming requests that match attributes directly have the highest precedence. For example, intentions that are configured to deny traffic from services that send `POST` requests take precedence over intentions that allow traffic from methods configured with the wildcard value `*`. +- Authorization: Consul enforces `deny` over `allow` if match criteria are weighted equally. + +The following table shows match precedence in descending order: + +| Precedence | Source Namespace | Source Name | Destination Namespace | Destination Name | +| -----------| ---------------- | ------------| --------------------- | ---------------- | +| 9 | Exact | Exact | Exact | Exact | +| 8 | Exact | `*` | Exact | Exact | +| 7 | `*` | `*` | Exact | Exact | +| 6 | Exact | Exact | Exact | `*` | +| 5 | Exact | `*` | Exact | `*` | +| 4 | `*` | `*` | Exact | `*` | +| 3 | Exact | Exact | `*` | `*` | +| 2 | Exact | `*` | `*` | `*` | +| 1 | `*` | `*` | `*` | `*` | + +Consul prints the precedence value to the service intentions configuration entry after it processes the matching criteria. The value is read-only. Refer to +[`Precedence`](/consul/docs/connect/config-entries/service-intentions#precedence) for additional information. + +Namespaces are an Enterprise feature. In Consul OSS, the only allowable value for either namespace field is `"default"`. Other rows in the table are not applicable. + +The [intention match API](/consul/api-docs/connect/intentions#list-matching-intentions) +should be periodically called to retrieve all relevant intentions for the +target destination. After verifying the TLS client certificate, the cached +intentions should be consulted for each incoming connection/request to +determine if it should be accepted or rejected. + +The default intention behavior is defined by the [`default_policy`](/consul/docs/agent/config/config-files#acl_default_policy) configuration. +If the configuration is set `allow`, then all service-to-service connections in the mesh will be allowed by default. +If is set to `deny`, then all connections or requests will be denied by default. \ No newline at end of file diff --git a/website/content/docs/connect/intentions-legacy.mdx b/website/content/docs/connect/intentions/legacy.mdx similarity index 92% rename from website/content/docs/connect/intentions-legacy.mdx rename to website/content/docs/connect/intentions/legacy.mdx index 0e9e991d37c..df9ca5c6636 100644 --- a/website/content/docs/connect/intentions-legacy.mdx +++ b/website/content/docs/connect/intentions/legacy.mdx @@ -8,11 +8,9 @@ description: >- # Intentions in Legacy Mode ~> **1.8.x and earlier:** This document only applies in Consul versions 1.8.x -and before. If you are using version 1.9.0 or later please use the updated -documentation [here](/consul/docs/connect/intentions). +and before. If you are using version 1.9.0 or later, refer to the [current intentions documentation](/consul/docs/connect/intentions). -Intentions define access control for services via Connect and are used -to control which services may establish connections. Intentions can be +Intentions define access control for service-to-service connections in the service mesh. Intentions can be managed via the API, CLI, or UI. Intentions are enforced by the [proxy](/consul/docs/connect/proxies) @@ -24,8 +22,8 @@ connection must be terminated. The default intention behavior is defined by the default [ACL policy](/consul/docs/agent/config/config-files#acl_default_policy). If the default ACL policy is -"allow all", then all Connect connections are allowed by default. If the -default ACL policy is "deny all", then all Connect connections are denied by +"allow all", then all service-to-service connections in the mesh are allowed by default. If the +default ACL policy is "deny all", then all service-to-service connections are denied by default. ## Intention Basics @@ -131,7 +129,7 @@ of the intention, not the source. Intention permissions are by default implicitly granted at `read` level when granting `service:read` or `service:write`. This is because a -service registered that wants to use Connect needs `intentions:read` +service registered that wants to use service mesh needs `intentions:read` for its own service name in order to know whether or not to authorize connections. The following ACL policy will implicitly grant `intentions:read` (note _read_) for service `web`. @@ -153,7 +151,7 @@ service "web" { } ``` -Note that `intentions:read` is required for a token that a Connect-enabled +Note that `intentions:read` is required for a token that a mesh-enabled service uses to register itself or its proxy. If the token used does not have `intentions:read` then the agent will be unable to resolve intentions for the service and so will not be able to authorize any incoming connections. @@ -174,7 +172,7 @@ The intentions for services registered with a Consul agent are cached locally on that agent. They are then updated via a background blocking query against the Consul servers. -Connect connection attempts require only local agent +Service mesh connection attempts require only local agent communication for authorization and generally only impose microseconds of latency to the connection. All actions in the data path of connections require only local data to ensure minimal performance overhead. diff --git a/website/content/docs/connect/l7-traffic/discovery-chain.mdx b/website/content/docs/connect/l7-traffic/discovery-chain.mdx index a41353125e8..ae7582509c9 100644 --- a/website/content/docs/connect/l7-traffic/discovery-chain.mdx +++ b/website/content/docs/connect/l7-traffic/discovery-chain.mdx @@ -10,7 +10,7 @@ description: >- -> **1.6.0+:** This feature is available in Consul versions 1.6.0 and newer. ~> This topic is part of a [low-level API](/consul/api-docs/discovery-chain) -primarily targeted at developers building external [Connect proxy +primarily targeted at developers building external [Consul service mesh proxy integrations](/consul/docs/connect/proxies/integrate). The service discovery process can be modeled as a "discovery chain" which @@ -21,7 +21,7 @@ the discovery chain a user can control how proxy upstreams are ultimately resolved to specific instances for load balancing. -> **Note:** The discovery chain is currently only used to discover -[Connect](/consul/docs/connect) proxy upstreams. +[service mesh](/consul/docs/connect) proxy upstreams. ## Configuration @@ -69,7 +69,7 @@ various configuration entries interact in more complex ways. For example: To correctly interpret a collection of configuration entries as a valid discovery chain, we first compile them into a form more directly usable by the -layers responsible for configuring Connect sidecar proxies. +layers responsible for configuring service mesh sidecar proxies. You can interact with the compiler directly using the [discovery-chain API](/consul/api-docs/discovery-chain). @@ -100,7 +100,7 @@ The response is a single wrapped `CompiledDiscoveryChain` field: The chain encodes a digraph of [nodes](#discoverygraphnode) and [targets](#discoverytarget). Nodes are the compiled representation of various discovery chain stages and targets are instructions on how to use the [health -API](/consul/api-docs/health#list-nodes-for-connect-capable-service) to retrieve +API](/consul/api-docs/health#list-nodes-for-mesh-capable-service) to retrieve relevant service instance lists. You should traverse the nodes starting with [`StartNode`](#startnode). The @@ -228,7 +228,7 @@ A single node in the compiled discovery chain. be considered healthy. - `MeshGateway` `(MeshGatewayConfig)` - The [mesh gateway - configuration](/consul/docs/connect/gateways/mesh-gateway#connect-proxy-configuration) + configuration](/consul/docs/connect/gateways/mesh-gateway#service-mesh-proxy-configuration) to use when connecting to this target's service instances. - `Mode` `(string: "")` - One of `none`, `local`, or `remote`. diff --git a/website/content/docs/connect/l7-traffic/index.mdx b/website/content/docs/connect/l7-traffic/index.mdx index c15c3656edc..9f5fe8da2b5 100644 --- a/website/content/docs/connect/l7-traffic/index.mdx +++ b/website/content/docs/connect/l7-traffic/index.mdx @@ -12,7 +12,7 @@ description: >- Layer 7 traffic management allows operators to divide L7 traffic between different [subsets](/consul/docs/connect/config-entries/service-resolver#service-subsets) of -service instances when using Connect. +service instances when using service mesh. There are many ways you may wish to carve up a single datacenter's pool of services beyond simply returning all healthy instances for load balancing. @@ -28,7 +28,7 @@ and some [Envoy proxy escape hatches](/consul/docs/connect/proxies/envoy#escape- ## Stages -Connect proxy upstreams are discovered using a series of stages: routing, +Service mesh proxy upstreams are discovered using a series of stages: routing, splitting, and resolution. These stages represent different ways of managing L7 traffic. diff --git a/website/content/docs/connect/native/go.mdx b/website/content/docs/connect/native/go.mdx index 4c47bc315dd..df9080f17f7 100644 --- a/website/content/docs/connect/native/go.mdx +++ b/website/content/docs/connect/native/go.mdx @@ -7,26 +7,29 @@ description: >- # Service Mesh Native Integration for Go Applications --> **Note:** When calling `ConnectAuthorize()` on incoming connections this library -will return _deny_ if `Permissions` are defined on the matching intention. -The method is currently only suited for networking layer 4 (e.g. TCP) integration. - -We provide a library that makes it drop-in simple to integrate Connect +We provide a library that makes it drop-in simple to integrate Consul service mesh with most [Go](https://golang.org/) applications. This page shows examples -of integrating this library for accepting or establishing Connect-based -connections. For most Go applications, Connect can be natively integrated +of integrating this library for accepting or establishing mesh-based +connections. For most Go applications, Consul service mesh can be natively integrated in just a single line of code excluding imports and struct initialization. In addition to this, please read and understand the -[overview of Connect-Native integrations](/consul/docs/connect/native). -In particular, after integrating applications with Connect, they must declare -that they accept Connect-based connections via their service definitions. +[overview of service mesh native integrations](/consul/docs/connect/native). +In particular, after natively integrating applications with Consul service mesh, +they must declare that they accept mesh-based connections via their service definitions. + +The noun _connect_ is used throughout this documentation and the Go API +to refer to the connect subsystem that provides Consul's service mesh capabilities. ## Accepting Connections +-> **Note:** When calling `ConnectAuthorize()` on incoming connections this library +will return _deny_ if `Permissions` are defined on the matching intention. +The method is currently only suited for networking layer 4 (e.g. TCP) integration. + Any server that supports TLS (HTTP, gRPC, net/rpc, etc.) can begin -accepting Connect-based connections in just a few lines of code. For most -existing applications, converting the server to accept Connect-based +accepting mesh-based connections in just a few lines of code. For most +existing applications, converting the server to accept mesh-based connections will require only a one-line change excluding imports and structure initialization. @@ -55,7 +58,7 @@ func main() { svc, _ := connect.NewService("my-service", client) defer svc.Close() - // Creating an HTTP server that serves via Connect + // Creating an HTTP server that serves via service mesh server := &http.Server{ Addr: ":8080", TLSConfig: svc.ServerTLSConfig(), @@ -106,7 +109,7 @@ func main() { svc, _ := connect.NewService("my-service", client) defer svc.Close() - // Creating an HTTP server that serves via Connect + // Creating an HTTP server that serves via service mesh listener, _ := tls.Listen("tcp", ":8080", svc.ServerTLSConfig()) defer listener.Close() @@ -117,9 +120,9 @@ func main() { ## HTTP Clients -For Go applications that need to Connect to HTTP-based upstream dependencies, +For Go applications that need to connect to HTTP-based upstream dependencies, the Go library can construct an `*http.Client` that automatically establishes -Connect-based connections as long as Consul-based service discovery is used. +mesh-based connections as long as Consul-based service discovery is used. Example, followed by more details: @@ -152,7 +155,7 @@ section above. If your application is both a client and server, both the API client and service structure can be shared and reused. Next, we call `svc.HTTPClient()` to return a specially configured -`*http.Client`. This client will automatically established Connect-based +`*http.Client`. This client will automatically established mesh-based connections using Consul service discovery. Finally, we perform an HTTP `GET` request to a hypothetical userinfo service. @@ -189,7 +192,7 @@ the following specific limitations: ## Raw TLS Connection For a raw `net.Conn` TLS connection, the `svc.Dial` function can be used. -This will establish a connection to the desired service via Connect and +This will establish a connection to the desired service via the service mesh and return the `net.Conn`. This connection can then be used as desired. Example: diff --git a/website/content/docs/connect/native/index.mdx b/website/content/docs/connect/native/index.mdx index ffeb720f8bf..e8cc421af8a 100644 --- a/website/content/docs/connect/native/index.mdx +++ b/website/content/docs/connect/native/index.mdx @@ -2,18 +2,18 @@ layout: docs page_title: Service Mesh Native App Integration - Overview description: >- - When using sidecar proxies is not possible, applications can natively integrate with Consul service mesh, but have reduced access to service mesh features. Learn how “Connect-Native” apps use mTLS to authenticate with Consul and how to add integrations to service registrations. + When using sidecar proxies is not possible, applications can natively integrate with Consul service mesh, but have reduced access to service mesh features. Learn how "mesh-native" or "connect-native" apps use mTLS to authenticate with Consul and how to add integrations to service registrations. --- # Service Mesh Native App Integration Overview -~> **Note:** The Native App Integration does not support many of the Connect service +~> **Note:** The Native App Integration does not support many of the Consul's service mesh features, and is not under active development. The [Envoy proxy](/consul/docs/connect/proxies/envoy) should be used for most production environments. -Applications can natively integrate with the Connect API to support accepting -and establishing connections to other Connect services without the overhead of a +Applications can natively integrate with Consul's service mesh API to support accepting +and establishing connections to other mesh services without the overhead of a [proxy sidecar](/consul/docs/connect/proxies). This option is especially useful for applications that may be experiencing performance issues with the proxy sidecar deployment. This page will cover the high-level overview of @@ -21,20 +21,23 @@ integration, registering the service, etc. For language-specific examples, see the sidebar navigation to the left. It is also required if your service uses relies on a dynamic set of upstream services. -Connect is just basic mutual TLS. This means that almost any application -can easily integrate with Connect. There is no custom protocol in use; -any language that supports TLS can accept and establish Connect-based +Service mesh traffic is just basic mutual TLS. This means that almost any application +can easily integrate with Consul service mesh. There is no custom protocol in use; +any language that supports TLS can accept and establish mesh-based connections. We currently provide an easy-to-use [Go integration](/consul/docs/connect/native/go) to assist with the getting the proper certificates, verifying connections, etc. We plan to add helper libraries for other languages in the future. However, without library support, it is still possible for any major language -to integrate with Connect. +to integrate with Consul service mesh. + +The noun _connect_ is used throughout this documentation to refer to the connect +subsystem that provides Consul's service mesh capabilities. ## Overview -The primary work involved in natively integrating with Connect is +The primary work involved in natively integrating with service mesh is [acquiring the proper TLS certificate](/consul/api-docs/agent/connect#service-leaf-certificate), [verifying TLS certificates](/consul/api-docs/agent/connect#certificate-authority-ca-roots), and [authorizing inbound connections or requests](/consul/api-docs/connect/intentions#list-matching-intentions). @@ -54,8 +57,8 @@ Details on the steps are below: - **Service discovery** - This is normal service discovery using Consul, a static IP, or any other mechanism. If you're using Consul DNS, the - [`.connect`](/consul/docs/discovery/dns#connect-capable-service-lookups) - syntax to find Connect-capable endpoints for a service. After service + [`.connect`](/consul/docs/services/discovery/dns-static-lookups#service-mesh-enabled-service-lookups) + syntax to find mesh-capable endpoints for a service. After service discovery, choose one address from the list of **service addresses**. - **Mutual TLS** - As a client, connect to the discovered service address @@ -64,7 +67,7 @@ Details on the steps are below: as the client certificate. Verify the remote certificate against the [public CA roots](/consul/api-docs/agent/connect#certificate-authority-ca-roots). As a client, if the connection is established then you've established - a Connect-based connection and there are no further steps! + a mesh-based connection and there are no further steps! - **Authorization** - As a server accepting connections, verify the client certificate against the [public CA @@ -94,7 +97,7 @@ Details on the steps are below: HTTP) aware it can safely enforce intentions per _request_ instead of the coarser per _connection_ model. -## Updating Certificates and Certificate Roots +## Update certificates and certificate roots The leaf certificate and CA roots can be updated at any time and the natively integrated application must react to this relatively quickly @@ -102,8 +105,8 @@ so that new connections are not disrupted. This can be done through Consul blocking queries (HTTP long polling) or through periodic polling. The API calls for -[acquiring a leaf TLS certificate](/consul/api-docs/agent/connect#service-leaf-certificate) -and [reading CA roots](/consul/api-docs/agent/connect#certificate-authority-ca-roots) +[acquiring a service mesh TLS certificate](/consul/api-docs/agent/connect#service-leaf-certificate) +and [reading service mesh CA roots](/consul/api-docs/agent/connect#certificate-authority-ca-roots) both support [blocking queries](/consul/api-docs/features/blocking). By using blocking queries, an application can efficiently wait for an updated value. For example, @@ -121,7 +124,7 @@ certificate endpoints frequently, such as multiple times per minute. The overhead for the blocking queries (long or periodic polling) is minimal. The API calls are to the local agent and the local agent uses locally cached data multiplexed over a single TCP connection to the Consul leader. -Even if a single machine has 1,000 Connect-enabled services all blocking +Even if a single machine has 1,000 mesh-enabled services all blocking on certificate updates, this translates to only one TCP connection to the Consul server. @@ -129,14 +132,14 @@ Some language libraries such as the [Go library](/consul/docs/connect/native/go) automatically handle updating and locally caching the certificates. -## Service Registration +## Service registration -Connect-native applications must tell Consul that they support Connect +Mesh-native applications must tell Consul that they support service mesh natively. This enables the service to be returned as part of service -discovery for Connect-capable services, used by other Connect-native applications +discovery for service mesh-capable services used by other mesh-native applications and client [proxies](/consul/docs/connect/proxies). -This can be specified directly in the [service definition](/consul/docs/discovery/services): +You can enable native service mesh support directly in the [service definition](/consul/docs/services/configuration/services-configuration-reference#connect) by configuring the `connect` block. In the following example, the `redis` service is configured to support service mesh natively: ```json { @@ -150,6 +153,6 @@ This can be specified directly in the [service definition](/consul/docs/discover } ``` -Services that support Connect natively are still returned through the standard -service discovery mechanisms in addition to the Connect-only service discovery +Services that support service mesh natively are still returned through the standard +service discovery mechanisms in addition to the mesh-only service discovery mechanisms. diff --git a/website/content/docs/connect/nomad.mdx b/website/content/docs/connect/nomad.mdx index cd4849fbaf1..c65f07bc916 100644 --- a/website/content/docs/connect/nomad.mdx +++ b/website/content/docs/connect/nomad.mdx @@ -1,26 +1,26 @@ --- layout: docs -page_title: Sevice Mesh - Nomad Integration +page_title: Service Mesh - Nomad Integration description: >- Consul's service mesh can be applied to provide secure communication between services managed by Nomad's scheduler and orchestrator functions, including Nomad jobs and task groups. Use the guide and reference documentation to learn more. --- # Consul and Nomad Integration -Consul Connect can be used with [Nomad](https://www.nomadproject.io/) to provide +Consul service mesh can be used with [Nomad](https://www.nomadproject.io/) to provide secure service-to-service communication between Nomad jobs and task groups. Nomad is a simple, flexible scheduler and workload orchestrator. The ability to use the [dynamic port](/nomad/docs/job-specification/network#dynamic-ports) -feature of Nomad makes Connect reduces operational complexity. +feature of Nomad with Consul service mesh reduces operational complexity. For more information -about using Consul Connect with Nomad, select one of the following resources. +about using Consul service mesh with Nomad, select one of the following resources. -For a step-by-step guide on using Consul Connect with Nomad: +For a step-by-step guide on using Consul service mesh with Nomad: -- [Nomad Consul Connect Guide](/nomad/docs/integrations/consul-connect) +- [Nomad with Consul service mesh Guide](/nomad/docs/integrations/consul-connect) -For reference information about configuring Nomad jobs to use Consul Connect: +For reference information about configuring Nomad jobs to use Consul service mesh: - [Nomad Job Specification - `connect`](/nomad/docs/job-specification/connect) - [Nomad Job Specification - `sidecar_service`](/nomad/docs/job-specification/sidecar_service) diff --git a/website/content/docs/connect/observability/index.mdx b/website/content/docs/connect/observability/index.mdx index 3da463b389b..35db2edd4b4 100644 --- a/website/content/docs/connect/observability/index.mdx +++ b/website/content/docs/connect/observability/index.mdx @@ -7,7 +7,7 @@ description: >- # Service Mesh Observability Overview -In order to take advantage of Connect's L7 observability features you will need +In order to take advantage of the service mesh's L7 observability features you will need to: - Deploy sidecar proxies that are capable of emitting metrics with each of your @@ -26,7 +26,7 @@ configuration](/consul/docs/agent/config/config-files#enable_central_service_con If you are using Kubernetes, the Helm chart can simplify much of the configuration needed to enable observability. See our [Kubernetes observability docs](/consul/docs/k8s/connect/observability/metrics) for more information. -### Metrics Destination +### Metrics destination For Envoy the metrics destination can be configured in the proxy configuration entry's `config` section. @@ -39,20 +39,20 @@ config { } ``` -Find other possible metrics syncs in the [Connect Envoy documentation](/consul/docs/connect/proxies/envoy#bootstrap-configuration). +Find other possible metrics syncs in the [Envoy documentation](/consul/docs/connect/proxies/envoy#bootstrap-configuration). -### Service Protocol +### Service protocol -You can specify the [service protocol](/consul/docs/connect/config-entries/service-defaults#protocol) -in the `service-defaults` configuration entry. You can override it in the -[service registration](/consul/docs/discovery/services). By default, proxies only give -you L4 metrics. This protocol allows proxies to handle requests at the right L7 -protocol and emit richer L7 metrics. It also allows proxies to make per-request +You can specify the [`protocol`](/consul/docs/connect/config-entries/service-defaults#protocol) +for all service instances in the `service-defaults` configuration entry. You can also override the default protocol when defining and registering proxies in a service definition file. Refer to [Expose Paths Configuration Reference](/consul/docs/connect/registration/service-registration#expose-paths-configuration-reference) for additional information. + +By default, proxies only provide L4 metrics. +Defining the protocol allows proxies to handle requests at the L7 +protocol and emit L7 metrics. It also allows proxies to make per-request load balancing and routing decisions. -### Service Upstreams +### Service upstreams You can set the upstream for each service using the proxy's [`upstreams`](/consul/docs/connect/registration/service-registration#upstreams) -sidecar parameter, which can be defined in a service's [sidecar -registration](/consul/docs/connect/registration/sidecar-service). +sidecar parameter, which can be defined in a service's [sidecar registration](/consul/docs/connect/registration/sidecar-service). diff --git a/website/content/docs/connect/observability/ui-visualization.mdx b/website/content/docs/connect/observability/ui-visualization.mdx index b1a6f2a1808..8e109e74b15 100644 --- a/website/content/docs/connect/observability/ui-visualization.mdx +++ b/website/content/docs/connect/observability/ui-visualization.mdx @@ -14,8 +14,7 @@ show a service's immediate connectivity at a glance. It is not intended as a replacement for dedicated monitoring solutions, but rather as a quick overview of the state of a service and its connections within the Service Mesh. -The topology visualization requires services to be using [Consul -Connect](/consul/docs/connect) via [side-car proxies](/consul/docs/connect/proxies). +The topology visualization requires services to be using [service mesh](/consul/docs/connect) via [sidecar proxies](/consul/docs/connect/proxies). The visualization may optionally be configured to include a link to an external per-service dashboard. This is designed to provide convenient deep links to your @@ -718,7 +717,7 @@ Currently there are some limitations to this feature. access likely only has metrics for the local datacenter and a full solution would need additional proxying or exposing remote Prometheus servers on the network in remote datacenters. Later we may support an easy way to set this up - via Consul Connect but initially we don't attempt to fetch metrics in the UI + via Consul service mesh but initially we don't attempt to fetch metrics in the UI if you are browsing a remote datacenter. - **Built-in provider requires metrics proxy** Initially the built-in diff --git a/website/content/docs/connect/proxies/built-in.mdx b/website/content/docs/connect/proxies/built-in.mdx index 87f6fb4292f..c393d4648a8 100644 --- a/website/content/docs/connect/proxies/built-in.mdx +++ b/website/content/docs/connect/proxies/built-in.mdx @@ -8,11 +8,11 @@ description: >- # Built-in Proxy Configuration for Service Mesh ~> **Note:** The built-in proxy is not supported for production deployments. It does not -support many of the Connect service mesh features, and is not under active development. +support many of Consul's service mesh features, and is not under active development. The [Envoy proxy](/consul/docs/connect/proxies/envoy) should be used for production deployments. Consul comes with a built-in L4 proxy for testing and development with Consul -Connect service mesh. +service mesh. ## Proxy Config Key Reference @@ -61,7 +61,7 @@ All fields are optional with a reasonable default. that the proxy should use to connect to the local application instance. By default it assumes `127.0.0.1` as the address and takes the port from the service definition's `port` field. Note that allowing the application to listen on any non-loopback - address may expose it externally and bypass Connect's access enforcement. It may + address may expose it externally and bypass the service mesh's access enforcement. It may be useful though to allow non-standard loopback addresses or where an alternative known-private IP is available for example when using internal networking between containers. diff --git a/website/content/docs/connect/proxies/envoy-extensions/index.mdx b/website/content/docs/connect/proxies/envoy-extensions/index.mdx new file mode 100644 index 00000000000..ecd9e63e074 --- /dev/null +++ b/website/content/docs/connect/proxies/envoy-extensions/index.mdx @@ -0,0 +1,31 @@ +--- +layout: docs +page_title: Envoy Extensions +description: >- + Learn how Envoy extensions enables you to add support for additional Envoy features without modifying the Consul codebase. +--- + +# Envoy extensions overview + +This topic provides an overview of Envoy extensions in Consul service mesh deployments. You can modify Consul-generated Envoy resources to add additional functionality without modifying the Consul codebase. + +## Introduction + +Consul supports two methods for modifying Envoy behavior. You can either modify the Envoy resources Consul generates through [escape hatches](/consul/docs/connect/proxies/envoy#escape-hatch-overrides) or configure your services to use Envoy extensions using the `EnvoyExtension` parameter. Implementing escape hatches requires rewriting the Envoy resources so that they are compatible with Consul, a task that also requires understanding how Consul names Envoy resources and enforces intentions. + +Instead of modifying Consul code, you can configure your services to use Envoy extensions through the `EnvoyExtensions` field. This field is definable in [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults#envoyextensions) and [`service-defaults`](/consul/docs/connect/config-entries/service-defaults#envoyextensions) configuration entries. + +## Supported extensions + +Envoy extensions enable additional service mesh functionality in Consul by changing how the sidecar proxies behave. Extensions invoke custom code when traffic passes through an Envoy proxy. Consul supports the following extensions: + +- Lua +- Lambda + +### Lua + +The `lua` Envoy extension enables HTTP Lua filters in your Consul Envoy proxies. It allows you to run Lua scripts during Envoy requests and responses from Consul-generated Envoy resources. Refer to the [`lua`](/consul/docs/connect/proxies/envoy-extensions/usage/lua) documentation for more information. + +### Lambda + +The `lambda` Envoy extension enables services to make requests to AWS Lambda functions through the mesh as if they are a normal part of the Consul catalog. Refer to the [`lambda`](/consul/docs/connect/proxies/envoy-extensions/usage/lambda) documentation for more information. diff --git a/website/content/docs/connect/proxies/envoy-extensions/usage/lambda.mdx b/website/content/docs/connect/proxies/envoy-extensions/usage/lambda.mdx new file mode 100644 index 00000000000..ce155464196 --- /dev/null +++ b/website/content/docs/connect/proxies/envoy-extensions/usage/lambda.mdx @@ -0,0 +1,161 @@ +--- +layout: docs +page_title: Lambda Envoy Extension +description: >- + Learn how the `lambda` Envoy extension enables Consul to join AWS Lambda functions to its service mesh. +--- + +# Invoke Lambda functions in Envoy proxy + +The Lambda Envoy extension configures outbound traffic on upstream dependencies allowing mesh services to properly invoke AWS Lambda functions. Lambda functions appear in the catalog as any other Consul service. + +You can only enable the Lambda extension through `service-defaults`. This is because the Consul uses the `service-defaults` configuration entry name as the catalog name for the Lambda functions. + +## Specification + +The Lambda Envoy extension has the following arguments: + +| Arguments | Description | +| -------------------- | ------------------------------------------------------------------------------------------------ | +| `ARN` | Specifies the [AWS ARN](https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) for the service's Lambda. | +| `InvocationMode` | Determines if Consul configures the Lambda to be invoked using the synchronous or asynchronous [invocation mode](https://docs.aws.amazon.com/lambda/latest/operatorguide/invocation-modes.html). | +| `PayloadPassthrough` | Determines if the body Envoy receives is converted to JSON or directly passed to Lambda. | + +Be aware that unlike [manual lambda registration](/consul/docs/lambda/registration/manual#supported-meta-fields), region is inferred from the ARN when specified through an Envoy extension. + +## Workflow + +There are two steps to configure the Lambda Envoy extension: + +1. Configure EnvoyExtensions through `service-defaults`. +1. Apply the configuration entry. + +### Configure `EnvoyExtensions` + +To use the Lambda Envoy extension, you must configure and apply a `service-defaults` configuration entry. Consul uses the name of the entry as the Consul service name for the Lambdas in the catalog. Downstream services also use the name to invoke the Lambda. + +The following example configures the Lambda Envoy extension to create a service named `lambda` in the mesh that can invoke the associated Lambda function. + + + + + +```hcl +Kind = "service-defaults" +Name = "lambdaInvokingApp" +Protocol = "http" +EnvoyExtensions { + Name = "builtin/aws/lambda" + Arguments = { + ARN = "arn:aws:lambda:us-west-2:111111111111:function:lambda-1234" + } +} +``` + + + + + + +```hcl +{ + "kind": "service-defaults", + "name": "lambdaInvokingApp", + "protocol": "http", + "envoy_extensions": [{ + "name": "builtin/aws/lambda", + "arguments": { + "arn": "arn:aws:lambda:us-west-2:111111111111:function:lambda-1234" + } + }] +} + +``` + + + + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceDefaults +metadata: + name: lambda +spec: + protocol: http + envoyExtensions: + name = "builtin/aws/lambda" + arguments: + arn: arn:aws:lambda:us-west-2:111111111111:function:lambda-1234 +``` + + + + + + +For a full list of parameters for `EnvoyExtensions`, refer to the [`service-defaults`](/consul/docs/connect/config-entries/service-defaults#envoyextensions) and [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults#envoyextensions) configuration entries reference documentation. + +~> **Note:** You can only enable the Lambda extension through `service-defaults`. + +Refer to [Configuration specification](#configuration-specification) section to find a full list of arguments for the Lambda Envoy extension. + +### Apply the configuration entry + +Apply the `service-defaults` configuration entry. + + + + +```shell-session +$ consul config write lambda-envoy-extension.hcl +``` + + + + +```shell-session +$ consul config write lambda-envoy-extension.json +``` + + + + +```shell-session +$ kubectl apply lambda-envoy-extension.yaml +``` + + + + +## Examples + +In the following example, the Lambda Envoy extension adds a single Lambda function running in two regions into the mesh. Then, you can use the `lambda` service name to invoke it, as if it was any other service in the mesh. + + + +```hcl +Kind = "service-defaults" +Name = "lambda" +Protocol = "http" +EnvoyExtensions { + Name = "builtin/aws/lambda" + + Arguments = { + payloadPassthrough: false + arn: arn:aws:lambda:us-west-2:111111111111:function:lambda-1234 + } +} +EnvoyExtensions { + Name = "builtin/aws/lambda" + + Arguments = { + payloadPassthrough: false + arn: arn:aws:lambda:us-east-1:111111111111:function:lambda-1234 + } +} +``` + + \ No newline at end of file diff --git a/website/content/docs/connect/proxies/envoy-extensions/usage/lua.mdx b/website/content/docs/connect/proxies/envoy-extensions/usage/lua.mdx new file mode 100644 index 00000000000..496b7d5fa58 --- /dev/null +++ b/website/content/docs/connect/proxies/envoy-extensions/usage/lua.mdx @@ -0,0 +1,228 @@ +--- +layout: docs +page_title: Lua Envoy Extension +description: >- + Learn how the `lua` Envoy extension enables Consul to run Lua scripts during Envoy requests and responses from Consul-generated Envoy resources. +--- + +# Run Lua scripts in Envoy proxy + +The Lua Envoy extension enables the [HTTP Lua filter](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/lua_filter) in your Consul Envoy proxies, letting you run Lua scripts when requests and responses pass through Consul-generated Envoy resources. + +Envoy filters support setting and getting dynamic metadata, allowing a filter to share state information with subsequent filters. To set dynamic metadata, configure the HTTP Lua filter. Users can call `streamInfo:dynamicMetadata()` from Lua scripts to get the request's dynamic metadata. + +## Configuration specifications + +To use the Lua Envoy extension, configure the following arguments in the `EnvoyExtensions` block: + +| Arguments | Description | +| -------------- | ------------------------------------------------------------------------------------------------ | +| `ProxyType` | Determines the proxy type the extension applies to. The only supported value is `connect-proxy`. | +| `ListenerType` | Specifies if the extension is applied to the `inbound` or `outbound` listener. | +| `Script` | The Lua script that is configured to run by the HTTP Lua filter. | + +## Workflow + +There are two steps to configure the Lua Envoy extension: + +1. Configure EnvoyExtensions through `service-defaults` or `proxy-defaults`. +1. Apply the configuration entry. + +### Configure `EnvoyExtensions` + +To use Envoy extensions, you must configure and apply a `proxy-defaults` or `service-defaults` configuration entry with the Envoy extension. + +- When you configure Envoy extensions on `proxy-defaults`, they apply to every service. +- When you configure Envoy extensions on `service-defaults`, they apply to a specific service. + +Consul applies Envoy extensions configured in `proxy-defaults` before it applies extensions in `service-defaults`. As a result, the Envoy extension configuration in `service-defaults` may override configurations in `proxy-defaults`. + +The following example configures the Lua Envoy extension on every service by using the `proxy-defaults`. + + + + + +```hcl +Kind = "proxy-defaults" +Name = "global" +Protocol = "http" +EnvoyExtensions { + Name = "builtin/lua" + + Arguments = { + ProxyType = "connect-proxy" + Listener = "inbound" + Script = <<-EOS +function envoy_on_request(request_handle) + meta = request_handle:streamInfo():dynamicMetadata() + m = meta:get("consul") + request_handle:headers():add("x-consul-service", m["service"]) + request_handle:headers():add("x-consul-namespace", m["namespace"]) + request_handle:headers():add("x-consul-datacenter", m["datacenter"]) + request_handle:headers():add("x-consul-trust-domain", m["trust-domain"]) +end +EOS + } +} +``` + + + + + + +```hcl +{ + "kind": "proxy-defaults", + "name": "global", + "protocol": "http", + "envoy_extensions": [{ + "name": "builtin/lua", + "arguments": { + "proxy_type": "connect-proxy", + "listener": "inbound", + "script": "function envoy_on_request(request_handle)\nmeta = request_handle:streamInfo():dynamicMetadata()\nm = \nmeta:get("consul")\nrequest_handle:headers():add("x-consul-service", m["service"])\nrequest_handle:headers():add("x-consul-namespace", m["namespace"])\nrequest_handle:headers():add("x-consul-datacenter", m["datacenter"])\nrequest_handle:headers():add("x-consul-trust-domain", m["trust-domain"])\nend" + } + }] +} +``` + + + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ProxyDefaults +metadata: + name: global +spec: + protocol: http + envoyExtensions: + name = "builtin/lua" + arguments: + proxyType: "connect-proxy" + listener: "inbound" + script: |- +function envoy_on_request(request_handle) + meta = request_handle:streamInfo():dynamicMetadata() + m = meta:get("consul") + request_handle:headers():add("x-consul-service", m["service"]) + request_handle:headers():add("x-consul-namespace", m["namespace"]) + request_handle:headers():add("x-consul-datacenter", m["datacenter"]) + request_handle:headers():add("x-consul-trust-domain", m["trust-domain"]) +end +``` + + + + + +For a full list of parameters for `EnvoyExtensions`, refer to the [`service-defaults`](/consul/docs/connect/config-entries/service-defaults#envoyextensions) and [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults#envoyextensions) configuration entries reference documentation. + +!> **Warning:** Applying `EnvoyExtensions` to `ProxyDefaults` may produce unintended consequences. We recommend enabling `EnvoyExtensions` with `ServiceDefaults` in most cases. + +Refer to [Configuration specification](#configuration-specification) section to find a full list of arguments for the Lua Envoy extension. + +### Apply the configuration entry + +Apply the `proxy-defaults` or `service-defaults` configuration entry. + + + + +```shell-session +$ consul config write lua-envoy-extension-proxy-defaults.hcl +``` + + + + +```shell-session +$ consul config write lua-envoy-extension-proxy-defaults.json + +``` + + + + +```shell-session +$ kubectl apply lua-envoy-extension-proxy-defaults.yaml +``` + + + + +## Examples + +In the following example, the `service-defaults` configure the Lua Envoy extension to insert the HTTP Lua filter for service `myservice` and add the Consul service name to the`x-consul-service` header for all inbound requests. The `ListenerType` makes it so that the extension applies only on the inbound listener of the service's connect proxy. + + + +```hcl +Kind = "service-defaults" +Name = "myservice" +EnvoyExtensions = [ + { + Name = "builtin/lua" + + Arguments = { + ProxyType = "connect-proxy" + Listener = "inbound" + Script = < + +Alternatively, you can apply the same extension configuration to [`proxy-defaults`](/consul/docs/connect/config-entries/proxy-defaults#envoyextensions) configuration entries. + +You can also specify multiple Lua filters through the Envoy extensions. They will not override each other. + + + +```hcl +Kind = "service-defaults" +Name = "myservice" +EnvoyExtensions = [ + { + Name = "builtin/lua", + Arguments = { + ProxyType = "connect-proxy" + Listener = "inbound" + Script = <<-EOF +function envoy_on_request(request_handle) + meta = request_handle:streamInfo():dynamicMetadata() + m = meta:get("consul") + request_handle:headers():add("x-consul-datacenter", m["datacenter1"]) +end + EOF + } + }, + { + Name = "builtin/lua", + Arguments = { + ProxyType = "connect-proxy" + Listener = "inbound" + Script = <<-EOF +function envoy_on_request(request_handle) + meta = request_handle:streamInfo():dynamicMetadata() + m = meta:get("consul") + request_handle:headers():add("x-consul-datacenter", m["datacenter2"]) +end + EOF + } + } +] +``` + + \ No newline at end of file diff --git a/website/content/docs/connect/proxies/envoy.mdx b/website/content/docs/connect/proxies/envoy.mdx index 19e98a13676..557b65e0bdc 100644 --- a/website/content/docs/connect/proxies/envoy.mdx +++ b/website/content/docs/connect/proxies/envoy.mdx @@ -7,7 +7,7 @@ description: >- # Envoy Proxy Configuration for Service Mesh -Consul Connect has first class support for using +Consul service mesh has first class support for using [Envoy](https://www.envoyproxy.io) as a proxy. Consul configures Envoy by optionally exposing a gRPC service on the local agent that serves [Envoy's xDS configuration @@ -39,25 +39,23 @@ Consul supports **four major Envoy releases** at the beginning of each major Con | Consul Version | Compatible Envoy Versions | | ------------------- | -----------------------------------------------------------------------------------| -| 1.14.x | 1.24.0, 1.23.1, 1.22.5, 1.21.5 | +| 1.15.x | 1.25.9, 1.24.10, 1.23.12, 1.22.11 | +| 1.14.x | 1.24.10, 1.23.12, 1.22.11, 1.21.6 | | 1.13.x | 1.23.1, 1.22.5, 1.21.5, 1.20.7 | -| 1.12.x | 1.22.5, 1.21.5, 1.20.7, 1.19.5 | - -1. Envoy 1.20.1 and earlier are vulnerable to [CVE-2022-21654](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21654) and [CVE-2022-21655](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-21655). Both CVEs were patched in Envoy versions 1.18.6, 1.19.3, and 1.20.2. -Envoy 1.16.x and older releases are no longer supported (see [HCSEC-2022-07](https://discuss.hashicorp.com/t/hcsec-2022-07-consul-s-connect-service-mesh-affected-by-recent-envoy-security-releases/36332)). Consul 1.9.x clusters should be upgraded to 1.10.x and Envoy upgraded to the latest supported Envoy version for that release, 1.18.6. ### Envoy and Consul Dataplane -Consul Dataplane is a feature introduced in Consul v1.14. Because each version of Consul Dataplane supports one specific version of Envoy, you must use the following versions of Consul, Consul Dataplane, and Envoy together. +The Consul dataplane component was introduced in Consul v1.14 as a way to manage Envoy proxies without the use of Consul clients. Each new minor version of Consul is released with a new minor version of Consul dataplane, which packages both Envoy and the `consul-dataplane` binary in a single container image. For backwards compatability reasons, each new minor version of Consul will also support the previous minor version of Consul dataplane to allow for seamless upgrades. In addition, each minor version of Consul will support the next minor version of Consul dataplane to allow for extended dataplane support via newer versions of Envoy. -| Consul Version | Consul Dataplane Version | Bundled Envoy Version | -| ------------------- | ------------------------ | ---------------------- | -| 1.14.x | 1.0.x | 1.24.x | +| Consul Version | Consul Dataplane Version (Bundled Envoy Version) | +| ------------------- | ------------------------------------------------- | +| 1.15.x | 1.1.x (Envoy 1.25.x), 1.0.x (Envoy 1.24.x) | +| 1.14.x | 1.1.x (Envoy 1.25.x), 1.0.x (Envoy 1.24.x) | ## Getting Started To get started with Envoy and see a working example you can follow the [Using -Envoy with Connect](/consul/tutorials/developer-mesh/service-mesh-with-envoy-proxy?utm_source=docs) tutorial. +Envoy with Consul service mesh](/consul/tutorials/developer-mesh/service-mesh-with-envoy-proxy?utm_source=docs) tutorial. ## Configuration @@ -70,7 +68,7 @@ identity (node id) and the location of its local Consul agent from which it discovers all of its dynamic configuration. See [Bootstrap Configuration](#bootstrap-configuration) for more details. -The dynamic configuration Consul Connect provides to each Envoy instance includes: +The dynamic configuration Consul service mesh provides to each Envoy instance includes: - TLS certificates and keys to enable mutual authentication and keep certificates rotating. @@ -81,11 +79,11 @@ The dynamic configuration Consul Connect provides to each Envoy instance include - Configuration to [expose specific HTTP paths](/consul/docs/connect/registration/service-registration#expose-paths-configuration-reference). For more information on the parts of the Envoy proxy runtime configuration -that are currently controllable via Consul Connect see [Dynamic +that are currently controllable via Consul service mesh, refer to [Dynamic Configuration](#dynamic-configuration). We plan to enable more and more of Envoy's features through -Connect's first-class configuration over time, however some advanced users will +Consul service mesh's first-class configuration over time, however some advanced users will need additional control to configure Envoy in specific ways. To enable this, we provide several ["escape hatch"](#advanced-configuration) options that allow users to provide low-level raw Envoy config syntax for some sub-components in each @@ -113,7 +111,7 @@ If TLS is enabled on Consul, you will also need to add the following environment - [`CONSUL_CACERT`](/consul/commands#consul_cacert) - [`CONSUL_CLIENT_CERT`](/consul/commands#consul_client_cert) -- [`CONSUL_CLIENT_KEY`](//consulcommands#consul_client_key) +- [`CONSUL_CLIENT_KEY`](/consul/commands#consul_client_key) - [`CONSUL_HTTP_SSL`](/consul/commands#consul_http_ssl) ## Bootstrap Configuration @@ -126,7 +124,7 @@ Connect to a local Consul client agent and run the [`consul connect envoy` comma If you experience issues when bootstrapping Envoy proxies from the CLI, use the `-enable-config-gen-logging` flag to enable debug message logging. These logs can -help you troubleshoot issues that occur during the bootstrapping process. +help you troubleshoot issues that occur during the bootstrapping process. For more information about available flags and parameters, refer to the [`consul connect envoy CLI` reference](/consul/commands/connect/envoy). @@ -192,11 +190,15 @@ the [`sidecar_service`](/consul/docs/connect/registration/sidecar-service) block - `envoy_stats_flush_interval` - Configures Envoy's [`stats_flush_interval`](https://www.envoyproxy.io/docs/envoy/v1.17.2/api-v3/config/bootstrap/v3/bootstrap.proto#envoy-v3-api-field-config-bootstrap-v3-bootstrap-stats-flush-interval). +- `envoy_telemetry_collector_bind_socket_dir` - Specifies the directory where Envoy creates a Unix socket. + Envoy sends metrics to the socket where a Consul telemetry collector can collect them. + The socket is not configured by default. + The [Advanced Configuration](#advanced-configuration) section describes additional configurations that allow incremental or complete control over the bootstrap configuration generated. ### Bootstrap Envoy on Windows VMs -> Complete the [Connect Services on Windows Workloads to Consul Service Mesh tutorial](https://consul.io/consu/tutorials/consul-windows-workloads?utm_source=docs) to learn how to deploy Consul and use its service mesh on Windows VMs. +> Complete the [Connect Services on Windows Workloads to Consul Service Mesh tutorial](/consul/tutorials/developer-mesh/consul-windows-workloads) to learn how to deploy Consul and use its service mesh on Windows VMs. If you are running Consul on a Windows VM, attempting to bootstrap Envoy with the `consul connect envoy` command returns the following output: @@ -289,11 +291,11 @@ the [global `proxy-defaults` configuration entry](/consul/docs/connect/config-entries/proxy-defaults) to act as defaults that are inherited by all services. -- `protocol` - The protocol the service speaks. Connect's Envoy integration +- `protocol` - The protocol the service speaks. Consul service mesh's Envoy integration currently supports the following `protocol` values: - `tcp` - Unless otherwise specified this is the default, which causes Envoy - to proxy at L4. This provides all the security benefits of Connect's mTLS + to proxy at L4. This provides all the security benefits of the service mesh's mTLS and works for any TCP-based protocol. Load-balancing and metrics are available at the connection level. - `http` - This specifies that the service speaks HTTP/1.x. Envoy will setup an @@ -338,7 +340,7 @@ defaults that are inherited by all services. - `local_idle_timeout_ms` - In milliseconds, the idle timeout for HTTP requests to the local application instance. Applies to HTTP based protocols only. If not - specified, inherits the Envoy default for route idle timeouts (15s). A value of 0 + specified, inherits the Envoy default for route idle timeouts (15s). A value of 0 disables request timeouts. - `max_inbound_connections` - The maximum number of concurrent inbound connections @@ -499,13 +501,13 @@ manually code the entire configuration in protobuf JSON. ~> **Advanced Topic!** This section covers options that allow users to take almost complete control of Envoy's configuration. We provide these options so users can -experiment or take advantage of features not yet fully supported in Consul Connect. We +experiment or take advantage of features not yet fully supported in Consul service mesh. We plan to retain this ability in the future, but it should still be considered experimental because it requires in-depth knowledge of Envoy's configuration format. Users should consider Envoy version compatibility when using these features because they can configure Envoy in ways that are outside of Consul's control. Incorrect configuration could prevent all proxies in your mesh from functioning correctly, or bypass the security -guarantees Connect is designed to enforce. +guarantees Consul service mesh is designed to enforce. ### Configuration Formatting @@ -643,7 +645,7 @@ definition](/consul/docs/connect/registration/service-registration) or - `envoy_extra_static_listeners_json` - Similar to `envoy_extra_static_clusters_json` but appends one or more [Envoy listeners](https://www.envoyproxy.io/docs/envoy/v1.17.2/api-v3/config/listener/v3/listener.proto) to the array of [static listener](https://www.envoyproxy.io/docs/envoy/v1.17.2/api-v3/config/bootstrap/v3/bootstrap.proto#envoy-v3-api-field-config-bootstrap-v3-bootstrap-staticresources-listeners) definitions. - Can be used to setup limited access that bypasses Connect mTLS or + Can be used to setup limited access that bypasses the service mesh's mTLS or authorization for health checks or metrics. @@ -802,11 +804,11 @@ definition](/consul/docs/connect/registration/service-registration) or - Every `FilterChain` added to the listener will have its `TlsContext` overridden by the Connect TLS certificates and validation context. This - means there is no way to override Connect's mutual TLS for the public + means there is no way to override the service mesh's mutual TLS for the public listener. - Every `FilterChain` will have the `envoy.filters.{network|http}.rbac` filter prepended to the filters array to ensure that all inbound connections are - authorized by Connect. Before Consul 1.9.0 `envoy.ext_authz` was inserted instead. + authorized by the service mesh. Before Consul 1.9.0 `envoy.ext_authz` was inserted instead. @@ -994,8 +996,8 @@ warning. following exceptions: - Every `FilterChain` added to the listener will have its `TlsContext` - overridden by the Connect TLS certificates and validation context. This - means there is no way to override Connect's mutual TLS for the public + overridden by the service mesh TLS certificates and validation context. This + means there is no way to override the service mesh's mutual TLS for the public listener. diff --git a/website/content/docs/connect/proxies/index.mdx b/website/content/docs/connect/proxies/index.mdx index 799daebd4ed..ff344e56d2d 100644 --- a/website/content/docs/connect/proxies/index.mdx +++ b/website/content/docs/connect/proxies/index.mdx @@ -7,29 +7,23 @@ description: >- # Service Mesh Proxy Overview -A Connect-aware proxy enables unmodified applications to use Connect. A +Proxies enable unmodified applications to connect to other services in the service mesh. A per-service proxy sidecar transparently handles inbound and outbound service connections, automatically wrapping and verifying TLS connections. Consul -includes its own built-in L4 proxy and has first class support for Envoy. You -can choose other proxies to plug in as well. This section describes how to -configure Envoy or the built-in proxy using Connect, and how to integrate the +ships with a built-in L4 proxy and has first class support for Envoy. You +can plug other proxies into your environment as well. This section describes how to +configure Envoy or the built-in proxy using Consul service mesh, and how to integrate the proxy of your choice. -To ensure that services only allow external connections established via -the Connect protocol, you should configure all services to only accept connections on a loopback address. +To ensure that services only allow external connections established through +the service mesh protocol, you should configure all services to only accept connections on a loopback address. -~> **Deprecation Note:** Managed Proxies are a deprecated method for deploying -sidecar proxies, and have been removed in Consul 1.6. See [managed proxy -deprecation](/consul/docs/connect/proxies/managed-deprecated) for more -information. If you are using managed proxies we strongly recommend that you -switch service definitions for registering proxies. +## Dynamic upstreams require native integration -## Dynamic Upstreams Require Native Integration +Service mesh proxies do not support dynamic upstreams. If an application requires dynamic dependencies that are only available -at runtime, it must [natively integrate](/consul/docs/connect/native) -with Connect. After natively integrating, the HTTP API or -[DNS interface](/consul/docs/discovery/dns#connect-capable-service-lookups) -can be used. - -!> Connect proxies do not currently support dynamic upstreams. +at runtime, you must [natively integrate](/consul/docs/connect/native) +the application with Consul service mesh. After natively integrating, the HTTP API or +[DNS interface](/consul/docs/services/discovery/dns-static-lookups#service-mesh-enabled-service-lookups) +can be used. \ No newline at end of file diff --git a/website/content/docs/connect/proxies/integrate.mdx b/website/content/docs/connect/proxies/integrate.mdx index 20a203059d0..1ad9f4b9115 100644 --- a/website/content/docs/connect/proxies/integrate.mdx +++ b/website/content/docs/connect/proxies/integrate.mdx @@ -11,17 +11,20 @@ This topic describes the process and API endpoints you can use to extend proxies ## Overview -You can extend any proxy to support Connect. Consul ships with a built-in +You can extend any proxy to support Consul service mesh. Consul ships with a built-in proxy suitable for an out-of-the-box development experience, but you may require a more robust proxy solution for production environments. The proxy you integrate must be able to accept inbound connections and/or establish outbound connections identified as a particular service. In some cases, either ability may be acceptable, but both are generally required to support for full sidecar functionality. Sidecar proxies may support L4 or L7 network functionality. L4 integration is simpler and adequate for securing all traffic. L4 treats all traffic as TCP, however, so advanced routing or metrics features are not supported. -Full L7 support is built on top of L4 support. An L7 proxy integration supports most or all of the L7 traffic routing features in Connect by dynamically configuring routing, retries, and other L7 features. The built-in proxy only supports L4, while [Envoy](/consul/docs/connect/proxies/envoy) supports the full L7 feature set. +Full L7 support is built on top of L4 support. An L7 proxy integration supports most or all of the L7 traffic routing features in Consul service mesh by dynamically configuring routing, retries, and other L7 features. The built-in proxy only supports L4, while [Envoy](/consul/docs/connect/proxies/envoy) supports the full L7 feature set. Areas where the integration approach differs between L4 and L7 are identified in this topic. +The noun _connect_ is used throughout this documentation to refer to the connect +subsystem that provides Consul's service mesh capabilities. + ## Accepting Inbound Connections The proxy must accept TLS connections on some port to accept inbound connections. @@ -38,7 +41,7 @@ $ curl http://:8500/v1/agent/connect/ca/leaf/ -The client certificate from the inbound connection must be validated against the Connect CA root certificates. Call the [`/v1/agent/connect/ca/roots`] endpoint to obtain the root certificates from the Connect CA, e.g.: +The client certificate from the inbound connection must be validated against the service mesh CA root certificates. Call the [`/v1/agent/connect/ca/roots`] endpoint to obtain the root certificates from the service mesh CA, e.g.: @@ -103,7 +106,7 @@ leaf certificate. ## Establishing Outbound Connections -For outbound connections, the proxy should communicate with a Connect-capable +For outbound connections, the proxy should communicate with a mesh-capable endpoint for a service and provide a client certificate from the [`/v1/agent/connect/ca/leaf/`] API endpoint. The proxy must use the root certificate obtained from the [`/v1/agent/connect/ca/roots`] endpoint to verify the certificate served from the destination endpoint. @@ -126,7 +129,7 @@ fetching the discovery chain so that L7 features, such as HTTP routing rules, ar not returned. For each [target](/consul/docs/connect/l7-traffic/discovery-chain#targets) in the resulting -discovery chain, a list of healthy, Connect-capable endpoints may be fetched +discovery chain, a list of healthy, mesh-capable endpoints may be fetched from the [`/v1/health/connect/:service_id`] API endpoint as described in the [Service Discovery](#service-discovery) section. @@ -138,7 +141,7 @@ documentation for details about supported configuration parameters. ### Service Discovery -Proxies can use Consul's [service discovery API](https://consul.io/%60/v1/health/connect/:service_id%60) to return all available, Connect-capable endpoints for a given service. This endpoint supports a `cached` query parameter, which uses [agent caching](/consul/api-docs/features/caching) to improve +Proxies can use Consul's [service discovery API](/consul/api-docs/health#list-service-instances-for-mesh-enabled-service) to return all available, mesh-capable endpoints for a given service. This endpoint supports a `cached` query parameter, which uses [agent caching](/consul/api-docs/features/caching) to improve performance. The API package provides a [`UseCache`] query option to leverage caching. In addition to performance improvements, using the cache makes the mesh more resilient to Consul server outages. This is because the mesh "fails static" with the last known set of service instances still used, rather than errors on new connections. diff --git a/website/content/docs/connect/proxies/managed-deprecated.mdx b/website/content/docs/connect/proxies/managed-deprecated.mdx deleted file mode 100644 index 4ac7adeb3a8..00000000000 --- a/website/content/docs/connect/proxies/managed-deprecated.mdx +++ /dev/null @@ -1,278 +0,0 @@ ---- -layout: docs -page_title: Managed Proxy for Connect (Legacy) -description: >- - Consul's service mesh originally included a proxy manager that was deprecated in version 1.6. Learn about the reasons for its deprecation and how it worked with this legacy documentation. ---- - -# Managed Proxy for Connect Legacy Documentation - -Consul Connect was first released as a beta feature in Consul 1.2.0. The initial -release included a feature called "Managed Proxies". Managed proxies were -Connect proxies where the proxy process was started, configured, and stopped by -Consul. They were enabled via basic configurations within the service -definition. - -!> **Consul 1.6.0 removes Managed Proxies completely.** -This documentation is provided for prior versions only. You may consider using -[sidecar service -registrations](/consul/docs/connect/registration/sidecar-service) instead. - -Managed proxies have been deprecated since Consul 1.3 and have been fully removed -in Consul 1.6. Anyone using Managed Proxies should aim to change their workflow -as soon as possible to avoid issues with a later upgrade. - -After transitioning away from all managed proxy usage, the `proxy` subdirectory inside [`data_dir`](/consul/docs/agent/config/cli-flags#_data_dir) (specified in Consul config) can be deleted to remove extraneous configuration files and free up disk space. - -**new and known issues will not be fixed**. - -## Deprecation Rationale - -Originally managed proxies traded higher implementation complexity for an easier -"getting started" user experience. After seeing how Connect was investigated and -adopted during beta it became obvious that they were not the best trade off. - -Managed proxies only really helped in local testing or VM-per-service based -models whereas a lot of users jumped straight to containers where they are not -helpful. They also add only targeted fairly basic supervisor features which -meant most people would want to use something else in production for consistency -with other workloads. So the high implementation cost of building robust process -supervision didn't actually benefit most real use-cases. - -Instead of this Connect 1.3.0 introduces the concept of [sidecar service -registrations](/consul/docs/connect/registration/sidecar-service) which -have almost all of the benefits of simpler configuration but without any of the -additional process management complexity. As a result they can be used to -simplify configuration in both container-based and realistic production -supervisor settings. - -## Managed Proxy Documentation - -As the managed proxy features continue to be supported for now, the rest of this -page will document how they work in the interim. - --> **Deprecation Note:** It's _strongly_ recommended you do not build anything -using Managed proxies and consider using [sidecar service -registrations](/consul/docs/connect/registration/sidecar-service) instead. - -Managed proxies are given -a unique proxy-specific ACL token that allows read-only access to Connect -information for the specific service the proxy is representing. This ACL -token is more restrictive than can be currently expressed manually in -an ACL policy. - -The default managed proxy is a basic proxy built-in to Consul and written -in Go. Having a basic built-in proxy allows Consul to have a reasonable default -with performance that is good enough for most workloads. In some basic -benchmarks, the service-to-service communication over the built-in proxy -could sustain 5 Gbps with sub-millisecond latency. Therefore, -the performance impact of even the basic built-in proxy is minimal. - -Consul will be integrating with advanced proxies in the near future to support -more complex configurations and higher performance. The configuration below is -all for the built-in proxy. - --> **Security Note:** 1.) Managed proxies can only be configured -via agent configuration files. They _cannot_ be registered via the HTTP API. -And 2.) Managed proxies are not started at all if Consul is running as root. -Both of these default configurations help prevent arbitrary process -execution or privilege escalation. This behavior can be configured -[per-agent](/consul/docs/agent/config). - -### Lifecycle - -The Consul agent starts managed proxies on demand and supervises them, -restarting them if they crash. The lifecycle of the proxy process is decoupled -from the agent so if the agent crashes or is restarted for an upgrade, the -managed proxy instances will _not_ be stopped. - -Note that this behavior while desirable in production might leave proxy -processes running indefinitely if you manually stop the agent and clear its -data dir during testing. - -To terminate a managed proxy cleanly you need to deregister the service that -requested it. If the agent is already stopped and will not be restarted again, -you may choose to locate the proxy processes and kill them manually. - -While in `-dev` mode, unless a `-data-dir` is explicitly set, managed proxies -switch to being killed when the agent exits since it can't store state in order -to re-adopt them on restart. - -### Minimal Configuration - -Managed proxies are configured within a -[service definition](/consul/docs/discovery/services). The simplest possible -managed proxy configuration is an empty configuration. This enables the -default managed proxy and starts a listener for that service: - -```json -{ - "service": { - "name": "redis", - "port": 6379, - "connect": { "proxy": {} } - } -} -``` - -The listener is started on random port within the configured Connect -port range. It can be discovered using the -[DNS interface](/consul/docs/discovery/dns#connect-capable-service-lookups) -or -[Catalog API](#). -In most cases, service-to-service communication is established by -a proxy configured with upstreams (described below), which handle the -discovery transparently. - -### Upstream Configuration - -To transparently discover and establish Connect-based connections to -dependencies, they must be configured with a static port on the managed -proxy configuration: - -```json -{ - "service": { - "name": "web", - "port": 8080, - "connect": { - "proxy": { - "upstreams": [ - { - "destination_name": "redis", - "local_bind_port": 1234 - } - ] - } - } - } -} -``` - -In the example above, -"redis" is configured as an upstream with static port 1234 for service "web". -When a TCP connection is established on port 1234, the proxy -will find Connect-compatible "redis" services via Consul service discovery -and establish a TLS connection identifying as "web". - -~> **Security:** Any application that can communicate to the configured -static port will be able to masquerade as the source service ("web" in the -example above). You must either trust any loopback access on that port or -use namespacing techniques provided by your operating system. - --> **Deprecation Note:** versions 1.2.0 to 1.3.0 required specifying `upstreams` -as part of the opaque `config` that is passed to the proxy. However, since -1.3.0, the `upstreams` configuration is now specified directly under the -`proxy` key. Old service definitions using the nested config will continue to -work and have the values copied into the new location. This allows the upstreams -to be registered centrally rather than being part of the local-only config -passed to the proxy instance. - -For full details of the additional configurable options available when using the -built-in proxy see the [built-in proxy configuration -reference](/consul/docs/connect/configuration). - -### Prepared Query Upstreams - -The upstream destination may also be a -[prepared query](/consul/api-docs/query). -This allows complex service discovery behavior such as connecting to -the nearest neighbor or filtering by tags. - -For example, given a prepared query named "nearest-redis" that is -configured to route to the nearest Redis instance, an upstream can be -configured to route to this query. In the example below, any TCP connection -to port 1234 will attempt a Connect-based connection to the nearest Redis -service. - -```json -{ - "service": { - "name": "web", - "port": 8080, - "connect": { - "proxy": { - "upstreams": [ - { - "destination_name": "redis", - "destination_type": "prepared_query", - "local_bind_port": 1234 - } - ] - } - } - } -} -``` - -For full details of the additional configurable options available when using the -built-in proxy see the [built-in proxy configuration -reference](/consul/docs/connect/configuration). - -### Custom Managed Proxy - -[Custom proxies](/consul/docs/connect/proxies/integrate) can also be -configured to run as a managed proxy. To configure custom proxies, specify -an alternate command to execute for the proxy: - -```json -{ - "service": { - "name": "web", - "port": 8080, - "connect": { - "proxy": { - "exec_mode": "daemon", - "command": ["/usr/bin/my-proxy", "-flag-example"], - "config": { - "foo": "bar" - } - } - } - } -} -``` - -The `exec_mode` value specifies how the proxy is executed. The only -supported value at this time is "daemon". The command is the binary and -any arguments to execute. -The "daemon" mode expects a proxy to run as a long-running, blocking -process. It should not double-fork into the background. The custom -proxy should retrieve its configuration (such as the port to run on) -via the [custom proxy integration APIs](/consul/docs/connect/proxies/integrate). - -The default proxy command can be changed at an agent-global level -in the agent configuration. An example in HCL format is shown below. - -``` -connect { - proxy_defaults { - command = ["/usr/bin/my-proxy"] - } -} -``` - -With this configuration, all services registered without an explicit -proxy command will use `my-proxy` instead of the default built-in proxy. - -The `config` key is an optional opaque JSON object which will be passed through -to the proxy via the proxy configuration endpoint to allow any configuration -options the proxy needs to be specified. See the [built-in proxy -configuration reference](/consul/docs/connect/configuration) -for details of config options that can be passed when using the built-in proxy. - -### Managed Proxy Logs - -Managed proxies have both stdout and stderr captured in log files in the agent's -`data_dir`. They can be found in -`/proxy/logs/-std{err,out}.log`. - -The built-in proxy will inherit its log level from the agent so if the agent is -configured with `log_level = DEBUG`, a proxy it starts will also output `DEBUG` -level logs showing service discovery, certificate and authorization information. - -~> **Note:** In `-dev` mode there is no `data_dir` unless one is explicitly -configured so logging is disabled. You can access logs by providing the -[`-data-dir`](/consul/docs/agent/config/cli-flags#_data_dir) CLI option. If a data dir is -configured, this will also cause proxy processes to stay running when the agent -terminates as described in [Lifecycle](#lifecycle). diff --git a/website/content/docs/connect/registration/index.mdx b/website/content/docs/connect/registration/index.mdx index 24c7a812519..96db3278877 100644 --- a/website/content/docs/connect/registration/index.mdx +++ b/website/content/docs/connect/registration/index.mdx @@ -7,14 +7,13 @@ description: >- # Service Mesh Proxy Overview -To make Connect aware of proxies you will need to register them in a [service -definition](/consul/docs/discovery/services), just like you would register any other service with Consul. This section outlines your options for registering Connect proxies, either using independent registrations, or in nested sidecar registrations. +To enable service mesh proxies, you must define and register them with Consul. Proxies are a type of service in Consul that facilitate highly secure communication between services in a service mesh. The topics in the section outline your options for registering service mesh proxies. You can register proxies independently or nested inside a sidecar service registration. -## Proxy Service Registration +## Proxy service registration To register proxies with independent proxy service registrations, you can define them in either in config files or via the API just like any other service. Learn more about all of the options you can define when registering your proxy service in the [proxy registration documentation](/consul/docs/connect/registration/service-registration). -## Sidecar Service Registration +## Sidecar service registration To reduce the amount of boilerplate needed for a sidecar proxy, application service definitions may define an inline sidecar service block. This is an opinionated diff --git a/website/content/docs/connect/registration/service-registration.mdx b/website/content/docs/connect/registration/service-registration.mdx index 2bfe409b5b8..57050476dcf 100644 --- a/website/content/docs/connect/registration/service-registration.mdx +++ b/website/content/docs/connect/registration/service-registration.mdx @@ -7,7 +7,8 @@ description: >- # Register a Service Mesh Proxy Outside of a Service Registration -This topic describes how to declare a proxy as a `connect-proxy` in service definitions. The `kind` must be declared and information about the service they represent must be provided to function as a Consul service mesh proxy. +This topic describes how to declare a service mesh proxy in a service definition. The `kind` must be declared and information about the service they represent must be provided to function as a Consul service mesh proxy. + ## Configuration @@ -52,7 +53,7 @@ You can specify several additional parameters to configure the proxy to meet you ### Example -In the following example, a proxy named `redis-proxy` is registered as a service mesh proxy. It proxies to the `redis` service and is available at port `8181`. As a result, any service mesh clients searching for a Connect-capable endpoint for `redis` will find this proxy. +In the following example, a proxy named `redis-proxy` is registered as a service mesh proxy. It proxies to the `redis` service and is available at port `8181`. As a result, any service mesh clients searching for a mesh-capable endpoint for `redis` will find this proxy. @@ -178,9 +179,7 @@ You can configure the service mesh proxy to create listeners for upstream servic ### Upstream Configuration Examples -Upstreams support multiple destination types. The following examples include information about each implementation. - --> **Snake case**: The examples in this topic use `snake_case` because the syntax is supported in configuration files and API registrations. See [Service Definition Parameter Case](/consul/docs/discovery/services#service-definition-parameter-case) for additional information. +Upstreams support multiple destination types. The following examples include information about each implementation. Note that the examples in this topic use snake case, which is a convention that separates words with underscores, because the format is supported in configuration files and API registrations. @@ -300,10 +299,7 @@ The proxy will default to `direct` mode if a mode cannot be determined from the The following examples show additional configuration for transparent proxies. -Added in v1.10.0. - --> Note that `snake_case` is used here as it works in both [config file and API -registrations](/consul/docs/discovery/services#service-definition-parameter-case). +Note that the examples in this topic use snake case, which is a convention that separates words with underscores, because the format is supported in configuration files and API registrations. #### Configure a proxy listener for outbound traffic on port 22500 @@ -328,8 +324,7 @@ registrations](/consul/docs/discovery/services#service-definition-parameter-case The following examples show all possible mesh gateway configurations. --> Note that `snake_case` is used here as it works in both [config file and API -registrations](/consul/docs/discovery/services#service-definition-parameter-case). + Note that the examples in this topic use snake case, which is a convention that separates words with underscores, because the format is supported in configuration files and API registrations. #### Using a Local/Egress Gateway in the Local Datacenter @@ -383,11 +378,10 @@ registrations](/consul/docs/discovery/services#service-definition-parameter-case The following examples show possible configurations to expose HTTP paths through Envoy. Exposing paths through Envoy enables a service to protect itself by only listening on localhost, while still allowing -non-Connect-enabled applications to contact an HTTP endpoint. +non-mesh-enabled applications to contact an HTTP endpoint. Some examples include: exposing a `/metrics` path for Prometheus or `/healthz` for kubelet liveness checks. - --> Note that `snake_case` is used here as it works in both [config file and API -registrations](/consul/docs/discovery/services#service-definition-parameter-case). + +Note that the examples in this topic use snake case, which is a convention that separates words with underscores, because the format is supported in configuration files and API registrations. #### Expose listeners in Envoy for HTTP and GRPC checks registered with the local Consul agent diff --git a/website/content/docs/connect/registration/sidecar-service.mdx b/website/content/docs/connect/registration/sidecar-service.mdx index adf5ef20f03..46e14869c31 100644 --- a/website/content/docs/connect/registration/sidecar-service.mdx +++ b/website/content/docs/connect/registration/sidecar-service.mdx @@ -7,20 +7,16 @@ description: >- # Register a Service Mesh Proxy in a Service Registration -Connect proxies are typically deployed as "sidecars" that run on the same node -as the single service instance that they handle traffic for. They might be on -the same VM or running as a separate container in the same network namespace. +This topic describes how to declare a proxy as a _sidecar_ proxy. +Sidecar proxies run on the same node as the single service instance that they handle traffic for. +They may be on the same VM or running as a separate container in the same network namespace. -To simplify the configuration experience when deploying a sidecar for a service -instance, Consul 1.3 introduced a new field in the Connect block of the [service -definition](/consul/docs/discovery/services). +## Configuration -The `connect.sidecar_service` field is a complete nested service definition on -which almost any regular service definition field can be set. The exceptions are -[noted below](#limitations). If used, the service definition is treated -identically to another top-level service definition. The value of the nested -definition is that _all fields are optional_ with some opinionated defaults -applied that make setting up a sidecar proxy much simpler. +Add the `connect.sidecar_service` block to your service definition file and specify the parameters to configure sidecar proxy behavior. The `sidecar_service` block is a service definition that can contain most regular service definition fields. Refer to [Limitations](#limitations) for information about unsupported service definition fields for sidecar proxies. + +Consul treats sidecar proxy service definitions as a root-level service definition. All fields are optional in nested +definitions, which default to opinionated settings that are intended to reduce burden of setting up a sidecar proxy. ## Minimal Example @@ -134,7 +130,7 @@ proxy. - `kind` - Defaults to `connect-proxy`. This can't be overridden currently. - `check`, `checks` - By default we add a TCP check on the local address and port for the proxy, and a [service alias - check](/consul/docs/discovery/checks#alias) for the parent service. If either + check](/consul/docs/services/usage/checks#alias-checks) for the parent service. If either `check` or `checks` fields are set, only the provided checks are registered. - `proxy.destination_service_name` - Defaults to the parent service name. - `proxy.destination_service_id` - Defaults to the parent service ID. @@ -143,8 +139,7 @@ proxy. ## Limitations -Almost all fields in a [service definition](/consul/docs/discovery/services) may be -set on the `connect.sidecar_service` except for the following: +The following fields are not supported in the `connect.sidecar_service` block: - `id` - Sidecar services get an ID assigned and it is an error to override this. This ensures the agent can correctly deregister the sidecar service @@ -153,11 +148,8 @@ set on the `connect.sidecar_service` except for the following: unset this to make the registration be for a regular non-connect-proxy service. - `connect.sidecar_service` - Service definitions can't be nested recursively. -- `connect.proxy` - (Deprecated) [Managed - proxies](/consul/docs/connect/proxies/managed-deprecated) can't be defined on a - sidecar. - `connect.native` - Currently the `kind` is fixed to `connect-proxy` and it's - an error to register a `connect-proxy` that is also Connect-native. + an error to register a `connect-proxy` that is also service mesh-native. ## Lifecycle diff --git a/website/content/docs/connect/security.mdx b/website/content/docs/connect/security.mdx index 31fa331da63..42ce8d0d1e4 100644 --- a/website/content/docs/connect/security.mdx +++ b/website/content/docs/connect/security.mdx @@ -7,15 +7,15 @@ description: >- # Best Practices for Service Mesh Security -Connect enables secure service-to-service communication over mutual TLS. This +Consul service mesh enables secure service-to-service communication over mutual TLS. This provides both in-transit data encryption as well as authorization. This page -will document how to secure Connect. To try Connect locally, complete the +will document how to secure the service mesh. To try service mesh locally, complete the [Getting Started guide](/consul/tutorials/kubernetes-deploy/service-mesh?utm_source=docs) or for a full security model reference, see the dedicated [Consul security model](/consul/docs/security) page. When -setting up Connect in production, review this [tutorial](/consul/tutorials/developer-mesh/service-mesh-production-checklist?utm_source=consul.io&utm_medium=docs). +setting up service mesh in production, review this [tutorial](/consul/tutorials/developer-mesh/service-mesh-production-checklist?utm_source=consul.io&utm_medium=docs). -Connect will function in any Consul configuration. However, unless the checklist -below is satisfied, Connect is not providing the security guarantees it was +Consul service mesh will function in any Consul configuration. However, unless the checklist +below is satisfied, the service mesh is not providing the security guarantees it was built for. The checklist below can be incrementally adopted towards full security if you prefer to operate in less secure models initially. @@ -49,7 +49,7 @@ TCP and UDP encryption must be enabled to prevent plaintext communication between Consul agents. At a minimum, `verify_outgoing` should be enabled to verify server authenticity with each server having a unique TLS certificate. `verify_incoming` provides additional agent verification, but doesn't directly -affect Connect since requests must also always contain a valid ACL token. +affect service mesh since requests must also always contain a valid ACL token. Clients calling Consul APIs should be forced over encrypted connections. See the [Consul agent encryption page](/consul/docs/security/encryption) to @@ -72,7 +72,7 @@ general, the blast radius of unauthorized access for client agent directories is much smaller than servers. However, both must be protected against unauthorized access. -### Prevent Non-Connect Traffic to Services +### Prevent Non-Mesh Traffic to Services For services that are using [proxies](/consul/docs/connect/proxies) @@ -85,16 +85,16 @@ network namespacing techniques provided by the underlying operating system. For scenarios where multiple services are running on the same machine without isolation, these services must all be trusted. We call this the **trusted multi-tenancy** deployment model. Any service could theoretically -connect to any other service via the loopback listener, bypassing Connect +connect to any other service via the loopback listener, bypassing the service mesh completely. In this scenario, all services must be trusted _or_ isolation mechanisms must be used. For developer or operator access to a service, we recommend -using a local Connect proxy. This is documented in the +using a local service mesh proxy. This is documented in the [development and debugging guide](/consul/docs/connect/dev). **If non-proxy traffic can communicate with the service**, this traffic -will not be encrypted or authorized via Connect. +will not be encrypted or authorized via service mesh. ### Restrict Access to Envoy's Administration Interface diff --git a/website/content/docs/connect/transparent-proxy.mdx b/website/content/docs/connect/transparent-proxy.mdx index cf0b56c96e4..59a00e65077 100644 --- a/website/content/docs/connect/transparent-proxy.mdx +++ b/website/content/docs/connect/transparent-proxy.mdx @@ -276,7 +276,7 @@ spec: Additional services can query the [KubeDNS](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/) at `sample-app.default.svc.cluster.local` to reach `sample-app`. If ACLs are enabled and configured with default `deny` policies, the configuration also requires a [`ServiceIntention`](/consul/docs/connect/config-entries/service-intentions) to allow it to talk to `sample-app`. ### Headless Services -For services that are not addressed using a virtual cluster IP, you must configure the upstream service using the [DialedDirectly](/consul/docs/connect/config-entries/service-defaults#dialeddirectly) option. Then, use DNS to discover individual instance addresses and dial them through the transparent proxy. When this mode is enabled on the upstream, services present connect certificates for mTLS and intentions are enforced at the destination. +For services that are not addressed using a virtual cluster IP, you must configure the upstream service using the [DialedDirectly](/consul/docs/connect/config-entries/service-defaults#dialeddirectly) option. Then, use DNS to discover individual instance addresses and dial them through the transparent proxy. When this mode is enabled on the upstream, services present service mesh certificates for mTLS and intentions are enforced at the destination. Note that when dialing individual instances, Consul ignores the HTTP routing rules configured with configuration entries. The transparent proxy acts as a TCP proxy to the original destination IP address. diff --git a/website/content/docs/consul-vs-other/dns-tools-compare.mdx b/website/content/docs/consul-vs-other/dns-tools-compare.mdx index 6aab1440668..f4d27de8ab4 100644 --- a/website/content/docs/consul-vs-other/dns-tools-compare.mdx +++ b/website/content/docs/consul-vs-other/dns-tools-compare.mdx @@ -12,5 +12,5 @@ description: >- Consul was originally designed as a centralized service registry for any cloud environment that dynamically tracks services as they are added, changed, or removed within a compute infrastructure. Consul maintains a catalog of these registered services and their attributes, such as IP addresses or service name. For more information, refer to [What is Service Discovery?](/consul/docs/concepts/service-discovery). -As a result, Consul can also provide basic DNS functionality, including [lookups, alternate domains, and access controls](/consul/docs/discovery/dns). Since Consul is platform agnostic, you can retrieve service information across both cloud and on-premises data centers. Consul does not natively support some advanced DNS capabilities, such as filters or advanced routing logic. However, you can integrate Consul with existing DNS solutions, such as [NS1](https://help.ns1.com/hc/en-us/articles/360039417093-NS1-Consul-Integration-Overview) and [DNSimple](https://blog.dnsimple.com/2022/05/consul-integration/), to support these advanced capabilities. +As a result, Consul can also provide basic DNS functionality, including [lookups, alternate domains, and access controls](/consul/docs/services/discovery/dns-overview). Since Consul is platform agnostic, you can retrieve service information across both cloud and on-premises data centers. Consul does not natively support some advanced DNS capabilities, such as filters or advanced routing logic. However, you can integrate Consul with existing DNS solutions, such as [NS1](https://help.ns1.com/hc/en-us/articles/360039417093-NS1-Consul-Integration-Overview) and [DNSimple](https://blog.dnsimple.com/2022/05/consul-integration/), to support these advanced capabilities. diff --git a/website/content/docs/discovery/checks.mdx b/website/content/docs/discovery/checks.mdx deleted file mode 100644 index 0c2945602a2..00000000000 --- a/website/content/docs/discovery/checks.mdx +++ /dev/null @@ -1,885 +0,0 @@ ---- -layout: docs -page_title: Configure Health Checks -description: >- - Agents can be configured to periodically perform custom checks on the health of a service instance or node. Learn about the types of health checks and how to define them in agent and service configuration files. ---- - -# Health Checks - -One of the primary roles of the agent is management of system-level and application-level health -checks. A health check is considered to be application-level if it is associated with a -service. If not associated with a service, the check monitors the health of the entire node. - -Review the [health checks tutorial](/consul/tutorials/developer-discovery/service-registration-health-checks) -to get a more complete example on how to leverage health check capabilities in Consul. - -A check is defined in a configuration file or added at runtime over the HTTP interface. Checks -created via the HTTP interface persist with that node. - -There are several types of checks: - -- [`Script + Interval`](#script-check) - These checks invoke an external application - that performs the health check. - -- [`HTTP + Interval`](#http-check) - These checks make an HTTP `GET` request to the specified URL - in the health check definition. - -- [`TCP + Interval`](#tcp-check) - These checks attempt a TCP connection to the specified - address and port in the health check definition. - -- [`UDP + Interval`](#udp-check) - These checks direct the client to periodically send UDP datagrams - to the specified address and port in the health check definition. - -- [`OSService + Interval`](#osservice-check) - These checks periodically direct the Consul agent to monitor - the health of a service running on the host operating system. - -- [`Time to Live (TTL)`](#time-to-live-ttl-check) - These checks attempt an HTTP connection after a given TTL elapses. - -- [`Docker + Interval`](#docker-check) - These checks invoke an external application that - is packaged within a Docker container. - -- [`gRPC + Interval`](#grpc-check) - These checks are intended for applications that support the standard - [gRPC health checking protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - -- [`H2ping + Interval`](#h2ping-check) - These checks test an endpoint that uses HTTP/2 - by connecting to the endpoint and sending a ping frame. - -- [`Alias`](#alias-check) - These checks alias the health state of another registered - node or service. - - -## Registering a health check - -There are three ways to register a service with health checks: - -1. Start or reload a Consul agent with a service definition file in the - [agent's configuration directory](/consul/docs/agent#configuring-consul-agents). -1. Call the - [`/agent/service/register`](/consul/api-docs/agent/service#register-service) - HTTP API endpoint to register the service. -1. Use the - [`consul services register`](/consul/commands/services/register) - CLI command to register the service. - -When a service is registered using the HTTP API endpoint or CLI command, -the checks persist in the Consul data folder across Consul agent restarts. - -## Types of checks - -This section describes the available types of health checks you can use to -automatically monitor the health of a service instance or node. - --> **To manually mark a service unhealthy:** Use the maintenance mode - [CLI command](/consul/commands/maint) or - [HTTP API endpoint](/consul/api-docs/agent#enable-maintenance-mode) - to temporarily remove one or all service instances on a node - from service discovery DNS and HTTP API query results. - -### Script check - -Script checks periodically invoke an external application that performs the health check, -exits with an appropriate exit code, and potentially generates some output. -The specified `interval` determines the time between check invocations. -The output of a script check is limited to 4KB. -Larger outputs are truncated. - -By default, script checks are configured with a timeout equal to 30 seconds. -To configure a custom script check timeout value, -specify the `timeout` field in the check definition. -After reaching the timeout on a Windows system, -Consul waits for any child processes spawned by the script to finish. -After reaching the timeout on other systems, -Consul attempts to force-kill the script and any child processes it spawned. - -Script checks are not enabled by default. -To enable a Consul agent to perform script checks, -use one of the following agent configuration options: - -- [`enable_local_script_checks`](/consul/docs/agent/config/cli-flags#_enable_local_script_checks): - Enable script checks defined in local config files. - Script checks registered using the HTTP API are not allowed. -- [`enable_script_checks`](/consul/docs/agent/config/cli-flags#_enable_script_checks): - Enable script checks no matter how they are registered. - - ~> **Security Warning:** - Enabling non-local script checks in some configurations may introduce - a remote execution vulnerability known to be targeted by malware. - We strongly recommend `enable_local_script_checks` instead. - For more information, refer to - [this blog post](https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations). - -The following service definition file snippet is an example -of a script check definition: - - - -```hcl -check = { - id = "mem-util" - name = "Memory utilization" - args = ["/usr/local/bin/check_mem.py", "-limit", "256MB"] - interval = "10s" - timeout = "1s" -} -``` - -```json -{ - "check": { - "id": "mem-util", - "name": "Memory utilization", - "args": ["/usr/local/bin/check_mem.py", "-limit", "256MB"], - "interval": "10s", - "timeout": "1s" - } -} -``` - - - -#### Check script conventions - -A check script's exit code is used to determine the health check status: - -- Exit code 0 - Check is passing -- Exit code 1 - Check is warning -- Any other code - Check is failing - -Any output of the script is captured and made available in the -`Output` field of checks included in HTTP API responses, -as in this example from the [local service health endpoint](/consul/api-docs/agent/service#by-name-json). - -### HTTP check - -HTTP checks periodically make an HTTP `GET` request to the specified URL, -waiting the specified `interval` amount of time between requests. -The status of the service depends on the HTTP response code: any `2xx` code is -considered passing, a `429 Too ManyRequests` is a warning, and anything else is -a failure. This type of check -should be preferred over a script that uses `curl` or another external process -to check a simple HTTP operation. By default, HTTP checks are `GET` requests -unless the `method` field specifies a different method. Additional request -headers can be set through the `header` field which is a map of lists of -strings, such as `{"x-foo": ["bar", "baz"]}`. - -By default, HTTP checks are configured with a request timeout equal to 10 seconds. -To configure a custom HTTP check timeout value, -specify the `timeout` field in the check definition. -The output of an HTTP check is limited to approximately 4KB. -Larger outputs are truncated. -HTTP checks also support TLS. By default, a valid TLS certificate is expected. -Certificate verification can be turned off by setting the `tls_skip_verify` -field to `true` in the check definition. When using TLS, the SNI is implicitly -determined from the URL if it uses a hostname instead of an IP address. -You can explicitly set the SNI value by setting `tls_server_name`. - -Consul follows HTTP redirects by default. -To disable redirects, set the `disable_redirects` field to `true`. - -The following service definition file snippet is an example -of an HTTP check definition: - - - -```hcl -check = { - id = "api" - name = "HTTP API on port 5000" - http = "https://localhost:5000/health" - tls_server_name = "" - tls_skip_verify = false - method = "POST" - header = { - Content-Type = ["application/json"] - } - body = "{\"method\":\"health\"}" - disable_redirects = true - interval = "10s" - timeout = "1s" -} -``` - -```json -{ - "check": { - "id": "api", - "name": "HTTP API on port 5000", - "http": "https://localhost:5000/health", - "tls_server_name": "", - "tls_skip_verify": false, - "method": "POST", - "header": { "Content-Type": ["application/json"] }, - "body": "{\"method\":\"health\"}", - "interval": "10s", - "timeout": "1s" - } -} -``` - - - -### TCP check - -TCP checks periodically make a TCP connection attempt to the specified IP/hostname and port, waiting `interval` amount of time between attempts. -If no hostname is specified, it defaults to "localhost". -The health check status is `success` if the target host accepts the connection attempt, -otherwise the status is `critical`. In the case of a hostname that -resolves to both IPv4 and IPv6 addresses, an attempt is made to both -addresses, and the first successful connection attempt results in a -successful check. This type of check should be preferred over a script that -uses `netcat` or another external process to check a simple socket operation. - -By default, TCP checks are configured with a request timeout equal to 10 seconds. -To configure a custom TCP check timeout value, -specify the `timeout` field in the check definition. - -The following service definition file snippet is an example -of a TCP check definition: - - - -```hcl -check = { - id = "ssh" - name = "SSH TCP on port 22" - tcp = "localhost:22" - interval = "10s" - timeout = "1s" -} -``` - -```json -{ - "check": { - "id": "ssh", - "name": "SSH TCP on port 22", - "tcp": "localhost:22", - "interval": "10s", - "timeout": "1s" - } -} -``` - - - -### UDP check - -UDP checks periodically direct the Consul agent to send UDP datagrams -to the specified IP/hostname and port, -waiting `interval` amount of time between attempts. -The check status is set to `success` if any response is received from the targeted UDP server. -Any other result sets the status to `critical`. - -By default, UDP checks are configured with a request timeout equal to 10 seconds. -To configure a custom UDP check timeout value, -specify the `timeout` field in the check definition. -If any timeout on read exists, the check is still considered healthy. - -The following service definition file snippet is an example -of a UDP check definition: - - - -```hcl -check = { - id = "dns" - name = "DNS UDP on port 53" - udp = "localhost:53" - interval = "10s" - timeout = "1s" -} -``` - -```json -{ - "check": { - "id": "dns", - "name": "DNS UDP on port 53", - "udp": "localhost:53", - "interval": "10s", - "timeout": "1s" - } -} -``` - - - -### OSService check - -OSService checks periodically direct the Consul agent to monitor the health of a service running on -the host operating system as either a Windows service (Windows) or a SystemD service (Unix). -The check is logged as `healthy` if the service is running. -If it is stopped or not running, the status is `critical`. All other results set -the status to `warning`, which indicates that the check is not reliable because -an issue is preventing the check from determining the health of the service. - -The following service definition file snippet is an example -of an OSService check definition: - - - -```hcl -check = { - id = "myco-svctype-svcname-001" - name = "svcname-001 Windows Service Health" - service_id = "flash_pnl_1" - os_service = "myco-svctype-svcname-001" - interval = "10s" -} -``` - -```json -{ - "check": { - "id": "myco-svctype-svcname-001", - "name": "svcname-001 Windows Service Health", - "service_id": "flash_pnl_1", - "os_service": "myco-svctype-svcname-001", - "interval": "10s" - } -} -``` - - - -### Time to live (TTL) check - -TTL checks retain their last known state for the specified `ttl` duration. -The state of the check updates periodically over the HTTP interface. -If the `ttl` duration elapses before a new check update -is provided over the HTTP interface, -the check is set to `critical` state. - -This mechanism relies on the application to directly report its health. -For example, a healthy app can periodically `PUT` a status update to the HTTP endpoint. -Then, if the app is disrupted and unable to perform this update -before the TTL expires, the health check enters the `critical` state. -The endpoints used to update health information for a given check are: [pass](/consul/api-docs/agent/check#ttl-check-pass), -[warn](/consul/api-docs/agent/check#ttl-check-warn), [fail](/consul/api-docs/agent/check#ttl-check-fail), -and [update](/consul/api-docs/agent/check#ttl-check-update). TTL checks also persist their -last known status to disk. This persistence allows the Consul agent to restore the last known -status of the check across agent restarts. Persisted check status is valid through the -end of the TTL from the time of the last check. - -To manually mark a service unhealthy, -it is far more convenient to use the maintenance mode -[CLI command](/consul/commands/maint) or -[HTTP API endpoint](/consul/api-docs/agent#enable-maintenance-mode) -rather than a TTL health check with arbitrarily high `ttl`. - -The following service definition file snippet is an example -of a TTL check definition: - - - -```hcl -check = { - id = "web-app" - name = "Web App Status" - notes = "Web app does a curl internally every 10 seconds" - ttl = "30s" -} -``` - -```json -{ - "check": { - "id": "web-app", - "name": "Web App Status", - "notes": "Web app does a curl internally every 10 seconds", - "ttl": "30s" - } -} -``` - - - -### Docker check - -These checks depend on periodically invoking an external application that -is packaged within a Docker Container. The application is triggered within the running -container through the Docker Exec API. We expect that the Consul agent user has access -to either the Docker HTTP API or the unix socket. Consul uses `$DOCKER_HOST` to -determine the Docker API endpoint. The application is expected to run, perform a health -check of the service running inside the container, and exit with an appropriate exit code. -The check should be paired with an invocation interval. The shell on which the check -has to be performed is configurable, making it possible to run containers which -have different shells on the same host. -The output of a Docker check is limited to 4KB. -Larger outputs are truncated. - -Docker checks are not enabled by default. -To enable a Consul agent to perform Docker checks, -use one of the following agent configuration options: - -- [`enable_local_script_checks`](/consul/docs/agent/config/cli-flags#_enable_local_script_checks): - Enable script checks defined in local config files. - Script checks registered using the HTTP API are not allowed. - -- [`enable_script_checks`](/consul/docs/agent/config/cli-flags#_enable_script_checks): - Enable script checks no matter how they are registered. - - !> **Security Warning:** - We recommend using `enable_local_script_checks` instead of `enable_script_checks` in production - environments, as remote script checks are more vulnerable to malware attacks. Learn more about how [script checks can be exploited](https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations#how-script-checks-can-be-exploited). - -The following service definition file snippet is an example -of a Docker check definition: - - - -```hcl -check = { - id = "mem-util" - name = "Memory utilization" - docker_container_id = "f972c95ebf0e" - shell = "/bin/bash" - args = ["/usr/local/bin/check_mem.py"] - interval = "10s" -} -``` - -```json -{ - "check": { - "id": "mem-util", - "name": "Memory utilization", - "docker_container_id": "f972c95ebf0e", - "shell": "/bin/bash", - "args": ["/usr/local/bin/check_mem.py"], - "interval": "10s" - } -} -``` - - - -### gRPC check - -gRPC checks are intended for applications that support the standard -[gRPC health checking protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). -The state of the check will be updated by periodically probing the configured endpoint, -waiting `interval` amount of time between attempts. - -By default, gRPC checks are configured with a timeout equal to 10 seconds. -To configure a custom Docker check timeout value, -specify the `timeout` field in the check definition. - -gRPC checks default to not using TLS. -To enable TLS, set `grpc_use_tls` in the check definition. -If TLS is enabled, then by default, a valid TLS certificate is expected. -Certificate verification can be turned off by setting the -`tls_skip_verify` field to `true` in the check definition. -To check on a specific service instead of the whole gRPC server, -add the service identifier after the `gRPC` check's endpoint. - -The following example shows a gRPC check for a whole application: - - - -```hcl -check = { - id = "mem-util" - name = "Service health status" - grpc = "127.0.0.1:12345" - grpc_use_tls = true - interval = "10s" -} -``` - -```json -{ - "check": { - "id": "mem-util", - "name": "Service health status", - "grpc": "127.0.0.1:12345", - "grpc_use_tls": true, - "interval": "10s" - } -} -``` - - - -The following example shows a gRPC check for the specific `my_service` service: - - - -```hcl -check = { - id = "mem-util" - name = "Service health status" - grpc = "127.0.0.1:12345/my_service" - grpc_use_tls = true - interval = "10s" -} -``` - -```json -{ - "check": { - "id": "mem-util", - "name": "Service health status", - "grpc": "127.0.0.1:12345/my_service", - "grpc_use_tls": true, - "interval": "10s" - } -} -``` - - - -### H2ping check - -H2ping checks test an endpoint that uses http2 by connecting to the endpoint -and sending a ping frame, waiting `interval` amount of time between attempts. -If the ping is successful within a specified timeout, -then the check status is set to `success`. - -By default, h2ping checks are configured with a request timeout equal to 10 seconds. -To configure a custom h2ping check timeout value, -specify the `timeout` field in the check definition. - -TLS is enabled by default. -To disable TLS and use h2c, set `h2ping_use_tls` to `false`. -If TLS is not disabled, a valid certificate is required unless `tls_skip_verify` is set to `true`. - -The following service definition file snippet is an example -of an h2ping check definition: - - - -```hcl -check = { - id = "h2ping-check" - name = "h2ping" - h2ping = "localhost:22222" - interval = "10s" - h2ping_use_tls = false -} -``` - -```json -{ - "check": { - "id": "h2ping-check", - "name": "h2ping", - "h2ping": "localhost:22222", - "interval": "10s", - "h2ping_use_tls": false - } -} -``` - - - -### Alias check - -These checks alias the health state of another registered -node or service. The state of the check updates asynchronously, but is -nearly instant. For aliased services on the same agent, the local state is monitored -and no additional network resources are consumed. For other services and nodes, -the check maintains a blocking query over the agent's connection with a current -server and allows stale requests. If there are any errors in watching the aliased -node or service, the check state is set to `critical`. -For the blocking query, the check uses the ACL token set on the service or check definition. -If no ACL token is set in the service or check definition, -the blocking query uses the agent's default ACL token -([`acl.tokens.default`](/consul/docs/agent/config/config-files#acl_tokens_default)). - -~> **Configuration info**: The alias check configuration expects the alias to be -registered on the same agent as the one you are aliasing. If the service is -not registered with the same agent, `"alias_node": ""` must also be -specified. When using `alias_node`, if no service is specified, the check will -alias the health of the node. If a service is specified, the check will alias -the specified service on this particular node. - -The following service definition file snippet is an example -of an alias check for a local service: - - - -```hcl -check = { - id = "web-alias" - alias_service = "web" -} -``` - -```json -{ - "check": { - "id": "web-alias", - "alias_service": "web" - } -} -``` - - - -## Check definition - -This section covers some of the most common options for check definitions. -For a complete list of all check options, refer to the -[Register Check HTTP API endpoint documentation](/consul/api-docs/agent/check#json-request-body-schema). - --> **Casing for check options:** - The correct casing for an option depends on whether the check is defined in - a service definition file or an HTTP API JSON request body. - For example, the option `deregister_critical_service_after` in a service - definition file is instead named `DeregisterCriticalServiceAfter` in an - HTTP API JSON request body. - -#### General options - -- `name` `(string: )` - Specifies the name of the check. - -- `id` `(string: "")` - Specifies a unique ID for this check on this node. - - If unspecified, Consul defines the check id by: - - If the check definition is embedded within a service definition file, - a unique check id is auto-generated. - - Otherwise, the `id` is set to the value of `name`. - If names might conflict, you must provide unique IDs to avoid - overwriting existing checks with the same id on this node. - -- `interval` `(string: )` - Specifies - the frequency at which to run this check. - Required for all check types except TTL and alias checks. - - The value is parsed by Go's `time` package, and has the following - [formatting specification](https://golang.org/pkg/time/#ParseDuration): - - > A duration string is a possibly signed sequence of decimal numbers, each with - > optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". - > Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - -- `service_id` `(string: )` - Specifies - the ID of a service instance to associate this check with. - That service instance must be on this node. - If not specified, this check is treated as a node-level check. - For more information, refer to the - [service-bound checks](#service-bound-checks) section. - -- `status` `(string: "")` - Specifies the initial status of the health check as - "critical" (default), "warning", or "passing". For more details, refer to - the [initial health check status](#initial-health-check-status) section. - - -> **Health defaults to critical:** If health status it not initially specified, - it defaults to "critical" to protect against including a service - in discovery results before it is ready. - -- `deregister_critical_service_after` `(string: "")` - If specified, - the associated service and all its checks are deregistered - after this check is in the critical state for more than the specified value. - The value has the same formatting specification as the [`interval`](#interval) field. - - The minimum timeout is 1 minute, - and the process that reaps critical services runs every 30 seconds, - so it may take slightly longer than the configured timeout to trigger the deregistration. - This field should generally be configured with a timeout that's significantly longer than - any expected recoverable outage for the given service. - -- `notes` `(string: "")` - Provides a human-readable description of the check. - This field is opaque to Consul and can be used however is useful to the user. - For example, it could be used to describe the current state of the check. - -- `token` `(string: "")` - Specifies an ACL token used for any interaction - with the catalog for the check, including - [anti-entropy syncs](/consul/docs/architecture/anti-entropy) and deregistration. - - For alias checks, this token is used if a remote blocking query is necessary to watch the state of the aliased node or service. - -#### Success/failures before passing/warning/critical - -To prevent flapping health checks and limit the load they cause on the cluster, -a health check may be configured to become passing/warning/critical only after a -specified number of consecutive checks return as passing/critical. -The status does not transition states until the configured threshold is reached. - -- `success_before_passing` - Number of consecutive successful results required - before check status transitions to passing. Defaults to `0`. Added in Consul 1.7.0. - -- `failures_before_warning` - Number of consecutive unsuccessful results required - before check status transitions to warning. Defaults to the same value as that of - `failures_before_critical` to maintain the expected behavior of not changing the - status of service checks to `warning` before `critical` unless configured to do so. - Values higher than `failures_before_critical` are invalid. Added in Consul 1.11.0. - -- `failures_before_critical` - Number of consecutive unsuccessful results required - before check status transitions to critical. Defaults to `0`. Added in Consul 1.7.0. - -This feature is available for all check types except TTL and alias checks. -By default, both passing and critical thresholds are set to 0 so the check -status always reflects the last check result. - - - -```hcl -checks = [ - { - name = "HTTP TCP on port 80" - tcp = "localhost:80" - interval = "10s" - timeout = "1s" - success_before_passing = 3 - failures_before_warning = 1 - failures_before_critical = 3 - } -] -``` - -```json -{ - "checks": [ - { - "name": "HTTP TCP on port 80", - "tcp": "localhost:80", - "interval": "10s", - "timeout": "1s", - "success_before_passing": 3, - "failures_before_warning": 1, - "failures_before_critical": 3 - } - ] -} -``` - - - -## Initial health check status - -By default, when checks are registered against a Consul agent, the state is set -immediately to "critical". This is useful to prevent services from being -registered as "passing" and entering the service pool before they are confirmed -to be healthy. In certain cases, it may be desirable to specify the initial -state of a health check. This can be done by specifying the `status` field in a -health check definition, like so: - - - -```hcl -check = { - id = "mem" - args = ["/bin/check_mem", "-limit", "256MB"] - interval = "10s" - status = "passing" -} -``` - -```json -{ - "check": { - "id": "mem", - "args": ["/bin/check_mem", "-limit", "256MB"], - "interval": "10s", - "status": "passing" - } -} -``` - - - -The above service definition would cause the new "mem" check to be -registered with its initial state set to "passing". - -## Service-bound checks - -Health checks may optionally be bound to a specific service. This ensures -that the status of the health check will only affect the health status of the -given service instead of the entire node. Service-bound health checks may be -provided by adding a `service_id` field to a check configuration: - - - -```hcl -check = { - id = "web-app" - name = "Web App Status" - service_id = "web-app" - ttl = "30s" -} -``` - -```json -{ - "check": { - "id": "web-app", - "name": "Web App Status", - "service_id": "web-app", - "ttl": "30s" - } -} -``` - - - -In the above configuration, if the web-app health check begins failing, it will -only affect the availability of the web-app service. All other services -provided by the node will remain unchanged. - -## Agent certificates for TLS checks - -The [enable_agent_tls_for_checks](/consul/docs/agent/config/config-files#enable_agent_tls_for_checks) -agent configuration option can be utilized to have HTTP or gRPC health checks -to use the agent's credentials when configured for TLS. - -## Multiple check definitions - -Multiple check definitions can be defined using the `checks` (plural) -key in your configuration file. - - - -```hcl -checks = [ - { - id = "chk1" - name = "mem" - args = ["/bin/check_mem", "-limit", "256MB"] - interval = "5s" - }, - { - id = "chk2" - name = "/health" - http = "http://localhost:5000/health" - interval = "15s" - }, - { - id = "chk3" - name = "cpu" - args = ["/bin/check_cpu"] - interval = "10s" - }, - ... -] -``` - -```json -{ - "checks": [ - { - "id": "chk1", - "name": "mem", - "args": ["/bin/check_mem", "-limit", "256MB"], - "interval": "5s" - }, - { - "id": "chk2", - "name": "/health", - "http": "http://localhost:5000/health", - "interval": "15s" - }, - { - "id": "chk3", - "name": "cpu", - "args": ["/bin/check_cpu"], - "interval": "10s" - }, - ... - ] -} -``` - - diff --git a/website/content/docs/discovery/dns.mdx b/website/content/docs/discovery/dns.mdx deleted file mode 100644 index 90038ae2955..00000000000 --- a/website/content/docs/discovery/dns.mdx +++ /dev/null @@ -1,594 +0,0 @@ ---- -layout: docs -page_title: Find services with DNS -description: >- - For service discovery use cases, Domain Name Service (DNS) is the main interface to look up, query, and address Consul nodes and services. Learn how a Consul DNS lookup can help you find services by tag, name, namespace, partition, datacenter, or domain. ---- - -# Query services with DNS - -One of the primary query interfaces for Consul is DNS. -The DNS interface allows applications to make use of service -discovery without any high-touch integration with Consul. - -For example, instead of making HTTP API requests to Consul, -a host can use the DNS server directly via name lookups -like `redis.service.us-east-1.consul`. This query automatically -translates to a lookup of nodes that provide the `redis` service, -are located in the `us-east-1` datacenter, and have no failing health checks. -It's that simple! - -There are a number of configuration options that are important for the DNS interface, -specifically [`client_addr`](/consul/docs/agent/config/config-files#client_addr),[`ports.dns`](/consul/docs/agent/config/config-files#dns_port), -[`recursors`](/consul/docs/agent/config/config-files#recursors),[`domain`](/consul/docs/agent/config/config-files#domain), -[`alt_domain`](/consul/docs/agent/config/config-files#alt_domain), and [`dns_config`](/consul/docs/agent/config/config-files#dns_config). -By default, Consul will listen on 127.0.0.1:8600 for DNS queries in the `consul.` -domain, without support for further DNS recursion. Please consult the -[documentation on configuration options](/consul/docs/agent/config), -specifically the configuration items linked above, for more details. - -There are a few ways to use the DNS interface. One option is to use a custom -DNS resolver library and point it at Consul. Another option is to set Consul -as the DNS server for a node and provide a -[`recursors`](/consul/docs/agent/config/config-files#recursors) configuration so that non-Consul queries -can also be resolved. The last method is to forward all queries for the "consul." -domain to a Consul agent from the existing DNS server. Review the -[DNS Forwarding tutorial](/consul/tutorials/networking/dns-forwarding?utm_source=docs) for examples. - -You can experiment with Consul's DNS server on the command line using tools such as `dig`: - -```shell-session -$ dig @127.0.0.1 -p 8600 redis.service.dc1.consul. ANY -``` - --> **Note:** In DNS, all queries are case-insensitive. A lookup of `PostgreSQL.node.dc1.consul` will find all nodes named `postgresql`. - -## Node Lookups - -To resolve names, Consul relies on a very specific format for queries. -There are fundamentally two types of queries: node lookups and service lookups. -A node lookup, a simple query for the address of a named node, looks like this: - -```text -.node[.]. -``` - -For example, if we have a `foo` node with default settings, we could -look for `foo.node.dc1.consul.` The datacenter is an optional part of -the FQDN: if not provided, it defaults to the datacenter of the agent. -If we know `foo` is running in the same datacenter as our local agent, -we can instead use `foo.node.consul.` This convention allows for terse -syntax where appropriate while supporting queries of nodes in remote -datacenters as necessary. - -For a node lookup, the only records returned are A and AAAA records -containing the IP address, and TXT records containing the -`node_meta` values of the node. - -```shell-session -$ dig @127.0.0.1 -p 8600 foo.node.consul ANY - -; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 foo.node.consul ANY -; (1 server found) -;; global options: +cmd -;; Got answer: -;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 24355 -;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 1, ADDITIONAL: 0 -;; WARNING: recursion requested but not available - -;; QUESTION SECTION: -;foo.node.consul. IN ANY - -;; ANSWER SECTION: -foo.node.consul. 0 IN A 10.1.10.12 -foo.node.consul. 0 IN TXT "meta_key=meta_value" -foo.node.consul. 0 IN TXT "value only" - - -;; AUTHORITY SECTION: -consul. 0 IN SOA ns.consul. postmaster.consul. 1392836399 3600 600 86400 0 -``` - -By default the TXT records value will match the node's metadata key-value -pairs according to [RFC1464](https://www.ietf.org/rfc/rfc1464.txt). -Alternatively, the TXT record will only include the node's metadata value when the -node's metadata key starts with `rfc1035-`. - - -### Node Lookups for Consul Enterprise - -Consul nodes exist at the admin partition level within a datacenter. -By default, the partition and datacenter used in a [node lookup](#node-lookups) are -the partition and datacenter of the Consul agent that received the DNS query. - -Use the following query format to specify a partition for a node lookup: -```text -.node..ap..dc. -``` - -Consul server agents are in the `default` partition. -If DNS queries are addressed to Consul server agents, -node lookups to non-`default` partitions must explicitly specify -the partition of the target node. - -## Service Lookups - -A service lookup is used to query for service providers. Service queries support -two lookup methods: standard and strict [RFC 2782](https://tools.ietf.org/html/rfc2782). - -By default, SRV weights are all set at 1, but changing weights is supported using the -`Weights` attribute of the [service definition](/consul/docs/discovery/services). - -Note that DNS is limited in size per request, even when performing DNS TCP -queries. - -For services having many instances (more than 500), it might not be possible to -retrieve the complete list of instances for the service. - -When DNS SRV response are sent, order is randomized, but weights are not -taken into account. In the case of truncation different clients using weighted SRV -responses will have partial and inconsistent views of instances weights so the -request distribution could be skewed from the intended weights. In that case, -it is recommended to use the HTTP API to retrieve the list of nodes. - -### Standard Lookup - -The format of a standard service lookup is: - -```text -[.].service[.]. -``` - -The `tag` is optional, and, as with node lookups, the `datacenter` is as -well. If no tag is provided, no filtering is done on tag. If no -datacenter is provided, the datacenter of this Consul agent is assumed. - -If we want to find any redis service providers in our local datacenter, -we could query `redis.service.consul.` If we want to find the PostgreSQL -primary in a particular datacenter, we could query -`primary.postgresql.service.dc2.consul.` - -The DNS query system makes use of health check information to prevent routing -to unhealthy nodes. When a service query is made, any services failing their health -check or failing a node system check will be omitted from the results. To allow -for simple load balancing, the set of nodes returned is also randomized each time. -These mechanisms make it easy to use DNS along with application-level retries -as the foundation for an auto-healing service oriented architecture. - -For standard services queries, both A and SRV records are supported. SRV records -provide the port that a service is registered on, enabling clients to avoid relying -on well-known ports. SRV records are only served if the client specifically requests -them, like so: - -```shell-session -$ dig @127.0.0.1 -p 8600 consul.service.consul SRV - -; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 consul.service.consul ANY -; (1 server found) -;; global options: +cmd -;; Got answer: -;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 50483 -;; flags: qr aa rd; QUERY: 1, ANSWER: 3, AUTHORITY: 1, ADDITIONAL: 1 -;; WARNING: recursion requested but not available - -;; QUESTION SECTION: -;consul.service.consul. IN SRV - -;; ANSWER SECTION: -consul.service.consul. 0 IN SRV 1 1 8300 foobar.node.dc1.consul. - -;; ADDITIONAL SECTION: -foobar.node.dc1.consul. 0 IN A 10.1.10.12 -``` - -### RFC 2782 Lookup - -Valid formats for RFC 2782 SRV lookups depend on -whether you want to filter results based on a service tag: - -- No filtering on service tag: - - ```text - _._tcp[.service][.]. - ``` - -- Filtering on service tag specified in the RFC 2782 protocol field: - - ```text - _._[.service][.]. - ``` - -Per [RFC 2782](https://tools.ietf.org/html/rfc2782), SRV queries must -prepend an underscore (`_`) to the `service` and `protocol` values in a query to -prevent DNS collisions. -To perform no tag-based filtering, specify `tcp` in the RFC 2782 protocol field. -To filter results on a service tag, specify the tag in the RFC 2782 protocol field. - -Other than the query format and default `tcp` protocol/tag value, the behavior -of the RFC style lookup is the same as the standard style of lookup. - -If you registered the service `rabbitmq` on port 5672 and tagged it with `amqp`, -you could make an RFC 2782 query for its SRV record as `_rabbitmq._amqp.service.consul`: - -```shell-session -$ dig @127.0.0.1 -p 8600 _rabbitmq._amqp.service.consul SRV - -; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 _rabbitmq._amqp.service.consul ANY -; (1 server found) -;; global options: +cmd -;; Got answer: -;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 52838 -;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 -;; WARNING: recursion requested but not available - -;; QUESTION SECTION: -;_rabbitmq._amqp.service.consul. IN SRV - -;; ANSWER SECTION: -_rabbitmq._amqp.service.consul. 0 IN SRV 1 1 5672 rabbitmq.node1.dc1.consul. - -;; ADDITIONAL SECTION: -rabbitmq.node1.dc1.consul. 0 IN A 10.1.11.20 -``` - -Again, note that the SRV record returns the port of the service as well as its IP. - -#### SRV response for hosts in the .addr subdomain - -If a service registered to Consul has an explicit IP [`address`](/consul/api-docs/agent/service#address) -or tagged address(es) defined on the service registration, the hostname returned -in the target field of the answer section for the DNS SRV query for the service -will be in the format of `.addr..consul`. - - - - - -In the example below, the `rabbitmq` service has been registered with an explicit -IPv4 address of `192.0.2.10`. - - - -```hcl -node_name = "node1" - -services { - name = "rabbitmq" - address = "192.0.2.10" - port = 5672 -} -``` - -```json -{ - "node_name": "node1", - "services": [ - { - "name": "rabbitmq", - "address": "192.0.2.10", - "port": 5672 - } - ] -} -``` - - - -When performing an SRV query for this service, the SRV response contains a single -record with a hostname in the format of `.addr..consul`. - -```shell-session -$ dig @127.0.0.1 -p 8600 -t srv _rabbitmq._tcp.service.consul +short -1 1 5672 c000020a.addr.dc1.consul. -``` - -In this example, the hex-encoded IP from the returned hostname is `c000020a`. -Converting each hex octet to decimal reveals the IP address that was specified -in the service registration. - -```shell-session -$ echo -n "c000020a" | perl -ne 'printf("%vd\n", pack("H*", $_))' -192.0.2.10 -``` - - - - - -In the example below, the `rabbitmq` service has been registered with an explicit -IPv6 address of `2001:db8:1:2:cafe::1337`. - - - -```hcl -node_name = "node1" - -services { - name = "rabbitmq" - address = "2001:db8:1:2:cafe::1337" - port = 5672 -} -``` - -```json -{ - "node_name": "node1", - "services": [ - { - "name": "rabbitmq", - "address": "2001:db8:1:2:cafe::1337", - "port": 5672 - } - ] -} -``` - - - -When performing an SRV query for this service, the SRV response contains a single -record with a hostname in the format of `.addr..consul`. - -```shell-session -$ dig @127.0.0.1 -p 8600 -t srv _rabbitmq._tcp.service.consul +short -1 1 5672 20010db800010002cafe000000001337.addr.dc1.consul. -``` - -In this example, the hex-encoded IP from the returned hostname is -`20010db800010002cafe000000001337`. This is the fully expanded IPv6 address with -colon separators removed. - -The following command re-adds the colon separators to display the fully expanded -IPv6 address that was specified in the service registration. - -```shell-session -$ echo -n "20010db800010002cafe000000001337" | perl -ne 'printf join(":", unpack("(A4)*", $_))."\n"' -2001:0db8:0001:0002:cafe:0000:0000:1337 -``` - - - - - -### Service Lookups for Consul Enterprise - -By default, all service lookups use the `default` namespace -within the partition and datacenter of the Consul agent that received the DNS query. -To lookup services in another namespace, partition, and/or datacenter, -use the [canonical format](#canonical-format). - -Consul server agents are in the `default` partition. -If DNS queries are addressed to Consul server agents, -service lookups to non-`default` partitions must explicitly specify -the partition of the target service. - -To lookup services imported from a cluster peer, -refer to [service virtual IP lookups for Consul Enterprise](#service-virtual-ip-lookups-for-consul-enterprise) instead. - -#### Canonical format - -Use the following query format to specify namespace, partition, and/or datacenter -for `.service`, `.connect`, `.virtual`, and `.ingress` service lookup types. -All three fields (`namespace`, `partition`, `datacenter`) are optional. -```text -[.].service[..ns][..ap][..dc] -``` - -#### Alternative formats for specifying namespace - -Though the [canonical format](#canonical-format) is recommended for readability, -you can use the following query formats specify namespace but not partition: - -- Specify both namespace and datacenter: - - ```text - [.].service... - ``` - -- **Deprecated in Consul 1.11:** - Specify namespace without a datacenter, - which requires that DNS queries are addressed to a Consul agent with - [`dns_config.prefer_namespace`](/consul/docs/agent/config/config-files#dns_prefer_namespace) - set to `true`: - - ```text - [.].service.. - ``` - -### Prepared Query Lookups - -The following formats are valid for prepared query lookups: - -- Standard lookup - - ```text - .query[.]. - ``` - -- [RFC 2782](https://tools.ietf.org/html/rfc2782) SRV lookup - - ```text - _._tcp.query[.]. - ``` - -The `datacenter` is optional, and if not provided, the datacenter of this Consul -agent is assumed. - -The `query name or id` is the given name or ID of an existing -[Prepared Query](/consul/api-docs/query). These behave like standard service -queries but provide a much richer set of features, such as filtering by multiple -tags and automatically failing over to look for services in remote datacenters if -no healthy nodes are available in the local datacenter. Consul 0.6.4 and later also -added support for [prepared query templates](/consul/api-docs/query#prepared-query-templates) -which can match names using a prefix match, allowing one template to apply to -potentially many services. - -To allow for simple load balancing, the set of nodes returned is randomized each time. -Both A and SRV records are supported. SRV records provide the port that a service is -registered on, enabling clients to avoid relying on well-known ports. SRV records are -only served if the client specifically requests them. - -### Connect-Capable Service Lookups - -To find Connect-capable services: - -```text -.connect. -``` - -This will find all [Connect-capable](/consul/docs/connect) -endpoints for the given `service`. A Connect-capable endpoint may be -both a proxy for a service or a natively integrated Connect application. -The DNS interface does not differentiate the two. - -Most services will use a [proxy](/consul/docs/connect/proxies) that handles -service discovery automatically and therefore won't use this DNS format. -This DNS format is primarily useful for [Connect-native](/consul/docs/connect/native) -applications. - -This endpoint currently only finds services within the same datacenter -and doesn't support tags. This DNS interface will be expanded over time. -If you need more complex behavior, please use the -[catalog API](/consul/api-docs/catalog). - -### Service Virtual IP Lookups - -To find the unique virtual IP allocated for a service: - -```text -.virtual[.]. -``` - -This will return the unique virtual IP for any [Connect-capable](/consul/docs/connect) -service. Each Connect service has a virtual IP assigned to it by Consul - this is used -by sidecar proxies for the [Transparent Proxy](/consul/docs/connect/transparent-proxy) feature. -The peer name is an optional part of the FQDN, and it is used to query for the virtual IP -of a service imported from that peer. - -The virtual IP is also added to the service's [Tagged Addresses](/consul/docs/discovery/services#tagged-addresses) -under the `consul-virtual` tag. - -#### Service Virtual IP Lookups for Consul Enterprise - -By default, a service virtual IP lookup uses the `default` namespace -within the partition and datacenter of the Consul agent that received the DNS query. - -To lookup services imported from a cluster peered partition or open-source datacenter, -specify the namespace and peer name in the lookup: -```text -.virtual[.].. -``` - -To lookup services not imported from a cluster peer, -refer to [service lookups for Consul Enterprise](#service-lookups-for-consul-enterprise) instead. - -### Ingress Service Lookups - -To find ingress-enabled services: - -```text -.ingress. -``` - -This will find all [ingress gateway](/consul/docs/connect/gateways/ingress-gateway) -endpoints for the given `service`. - -This endpoint currently only finds services within the same datacenter -and doesn't support tags. This DNS interface will be expanded over time. -If you need more complex behavior, please use the -[catalog API](/consul/api-docs/catalog). - -### UDP Based DNS Queries - -When the DNS query is performed using UDP, Consul will truncate the results -without setting the truncate bit. This is to prevent a redundant lookup over -TCP that generates additional load. If the lookup is done over TCP, the results -are not truncated. - -## Alternative Domain - -By default, Consul responds to DNS queries in the `consul` domain, -but you can set a specific domain for responding to DNS queries by configuring the [`domain`](/consul/docs/agent/config/config-files#domain) parameter. - -In some instances, Consul may need to respond to queries in more than one domain, -such as during a DNS migration or to distinguish between internal and external queries. - -Consul versions 1.5.2+ can be configured to respond to DNS queries on an alternative domain -through the [`alt_domain`](/consul/docs/agent/config/config-files#alt_domain) agent configuration -option. As of Consul versions 1.11.0+, Consul's DNS response will use the same domain as was used in the query; -in prior versions, the response may use the primary [`domain`](/consul/docs/agent/config/config-files#domain) no matter which -domain was used in the query. - -In the following example, the `alt_domain` parameter is set to `test-domain`: - -```hcl - alt_domain = "test-domain" -``` - -```shell-session -$ dig @127.0.0.1 -p 8600 consul.service.test-domain SRV -``` - -The following responses are returned: - -``` -;; QUESTION SECTION: -;consul.service.test-domain. IN SRV - -;; ANSWER SECTION: -consul.service.test-domain. 0 IN SRV 1 1 8300 machine.node.dc1.test-domain. - -;; ADDITIONAL SECTION: -machine.node.dc1.test-domain. 0 IN A 127.0.0.1 -machine.node.dc1.test-domain. 0 IN TXT "consul-network-segment=" -``` - --> **PTR queries:** Responses to PTR queries (`.in-addr.arpa.`) will always use the -[primary domain](/consul/docs/agent/config/config-files#domain) (not the alternative domain), -as there is no way for the query to specify a domain. - -## Caching - -By default, all DNS results served by Consul set a 0 TTL value. This disables -caching of DNS results. However, there are many situations in which caching is -desirable for performance and scalability. This is discussed more in the tutorial -for [DNS caching](/consul/tutorials/networking/dns-caching). - -## WAN Address Translation - -By default, Consul DNS queries will return a node's local address, even when -being queried from a remote datacenter. If you need to use a different address -to reach a node from outside its datacenter, you can configure this behavior -using the [`advertise-wan`](/consul/docs/agent/config/cli-flags#_advertise-wan) and -[`translate_wan_addrs`](/consul/docs/agent/config/config-files#translate_wan_addrs) configuration -options. - -## DNS with ACLs - -In order to use the DNS interface when -[Access Control Lists (ACLs)](/consul/docs/security/acl) -are enabled, you must first create ACL tokens with the necessary policies. - -Consul agents resolve DNS requests using one of the preconfigured tokens below, -listed in order of precedence: - -1. The agent's [`default` token](/consul/docs/agent/config/config-files#acl_tokens_default). -2. The built-in [`anonymous` token](/consul/docs/security/acl/acl-tokens#built-in-tokens). - Because the anonymous token is used when any request is made to Consul without - explicitly specifying a token, production deployments should not apply policies - needed for DNS to this token. - -Consul will either accept or deny the request depending on whether the token -has the appropriate authorization. The following table describes the available -DNS lookups and required policies when ACLs are enabled: - -| Lookup | Type | Description | ACLs Required | -| ------------------------------------------------------------------------------ | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `*.node.consul` | [Node](#node-lookups) | Allow resolving DNS requests for the target node (i.e., `.node.consul`) | [`node:read`](/consul/docs/security/acl/acl-rules#node-rules) | -| `*.service.consul`, `*.connect.consul`, `*.ingress.consul`, `*.virtual.consul` | [Service: standard](#service-lookups) | Allow resolving DNS requests for target service (e.g., `.service.consul`) instances running on ACL-authorized nodes | [`service:read`](/consul/docs/security/acl/acl-rules#service-rules), [`node:read`](/consul/docs/security/acl/acl-rules#node-rules) | -| `*.query.consul` | [Service: prepared query](#prepared-query-lookups) | Allow resolving DNS requests for [service instances specified](/consul/api-docs/query#service-1) by the target prepared query (i.e., `.query.consul`) running on ACL-authorized nodes | [`query:read`](/consul/docs/security/acl/acl-rules#prepared-query-rules), [`service:read`](/consul/docs/security/acl/acl-rules#service-rules), [`node:read`](/consul/docs/security/acl/acl-rules#node-rules) | - -For guidance on how to configure an appropriate token for DNS, refer to the -securing Consul with ACLs guides for: - -- [Production Environments](/consul/tutorials/security/access-control-setup-production#token-for-dns) -- [Development Environments](/consul/tutorials/day-0/access-control-setup?utm_source=docs#enable-acls-on-consul-clients) diff --git a/website/content/docs/discovery/services.mdx b/website/content/docs/discovery/services.mdx deleted file mode 100644 index 9eba2e85bb5..00000000000 --- a/website/content/docs/discovery/services.mdx +++ /dev/null @@ -1,701 +0,0 @@ ---- -layout: docs -page_title: Register Services with Service Definitions -description: >- - Define and register services and their health checks with Consul to make a service available for service discovery or service mesh access. Learn how to format service definitions with this reference page and sample code. ---- - -# Register Services with Service Definitions - -One of the main goals of service discovery is to provide a catalog of available -services. To that end, the agent provides a simple service definition format -to declare the availability of a service and to potentially associate it with -a health check. A health check associated with a service is considered to be an -application-level check. Define services in a configuration file or add it at -runtime using the HTTP interface. - -Complete the [Getting Started tutorials](/consul/tutorials/get-started-vms/virtual-machine-gs-service-discovery) to get hands-on experience registering a simple service with a health check on your local machine. - -## Service Definition - -Configure a service by providing the service definition to the agent. You can -either specify the configuration file using the `-config-file` option, or specify -the directory containing the service definition file with the `-config-dir` option. -Consul can load service definitions saved as `.json` or `.hcl` files. - -Send a `SIGHUP` to the running agent or use [`consul reload`](/consul/commands/reload) to check for new service definitions or to -update existing services. Alternatively, the service can be [registered dynamically](/consul/api-docs/agent/service#register-service) -using the [HTTP API](/consul/api-docs). - -A service definition contains a set of parameters that specify various aspects of the service, including how it is discovered by other services in the network. -All possible parameters are included in the following example, but only the top-level `service` parameter and its `name` parameter child are required by default. - - - -```hcl -service { - name = "redis" - id = "redis" - port = 80 - tags = ["primary"] - - meta = { - custom_meta_key = "custom_meta_value" - } - - tagged_addresses = { - lan = { - address = "192.168.0.55" - port = 8000 - } - - wan = { - address = "198.18.0.23" - port = 80 - } - } - - port = 8000 - socket_path = "/tmp/redis.sock" - enable_tag_override = false - - checks = [ - { - args = ["/usr/local/bin/check_redis.py"] - interval = "10s" - } - ] - - kind = "connect-proxy" - proxy_destination = "redis" - - proxy = { - destination_service_name = "redis" - destination_service_id = "redis1" - local_service_address = "127.0.0.1" - local_service_port = 9090 - local_service_socket_path = "/tmp/redis.sock" - mode = "transparent" - - transparent_proxy { - outbound_listener_port = 22500 - } - - mesh_gateway = { - mode = "local" - } - - expose = { - checks = true - - paths = [ - { - path = "/healthz" - local_path_port = 8080 - listener_port = 21500 - protocol = "http2" - } - ] - } - } - - connect = { - native = false - } - - weights = { - passing = 5 - warning = 1 - } - - token = "233b604b-b92e-48c8-a253-5f11514e4b50" - namespace = "foo" -} -``` - -```json -{ - "service": { - "id": "redis", - "name": "redis", - "tags": ["primary"], - "address": "", - "meta": { - "meta": "for my service" - }, - "tagged_addresses": { - "lan": { - "address": "192.168.0.55", - "port": 8000, - }, - "wan": { - "address": "198.18.0.23", - "port": 80 - } - }, - "port": 8000, - "socket_path": "/tmp/redis.sock", - "enable_tag_override": false, - "checks": [ - { - "args": ["/usr/local/bin/check_redis.py"], - "interval": "10s" - } - ], - "kind": "connect-proxy", - "proxy_destination": "redis", // Deprecated - "proxy": { - "destination_service_name": "redis", - "destination_service_id": "redis1", - "local_service_address": "127.0.0.1", - "local_service_port": 9090, - "local_service_socket_path": "/tmp/redis.sock", - "mode": "transparent", - "transparent_proxy": { - "outbound_listener_port": 22500 - }, - "config": {}, - "upstreams": [], - "mesh_gateway": { - "mode": "local" - }, - "expose": { - "checks": true, - "paths": [ - { - "path": "/healthz", - "local_path_port": 8080, - "listener_port": 21500, - "protocol": "http2" - } - ] - } - }, - "connect": { - "native": false, - "sidecar_service": {} - "proxy": { // Deprecated - "command": [], - "config": {} - } - }, - "weights": { - "passing": 5, - "warning": 1 - }, - "token": "233b604b-b92e-48c8-a253-5f11514e4b50", - "namespace": "foo" - } -} -``` - - - -The following table describes the available parameters for service definitions. - -### `service` - -This is the root-level parameter that defines the service. You can specify the parameters to configure the service. - -| Parameter | Description | Default | Required | -| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | ---------------------------- | -| `id` | String value that specifies the service ID.

    If not specified, the value of the `name` field will be used.

    Services must have unique IDs per node, so you should specify unique values if the default `name` will conflict with other services.

    | Value of the `name` parameter | Optional | -| `name` | Specifies the name of the service.
    The value for this parameter is used as the ID if the `id` parameter is not specified.
    We recommend using valid DNS labels for service definition names for compatibility with external DNSs. | None | Required | -| `tags` | List of string values that can be used to add service-level labels.
    For example, you can define tags that distinguish between `primary` and `secondary` nodes or service versions.
    We recommend using valid DNS labels for service definition IDs for compatibility with external DNSs.
    Tag values are opaque to Consul.
    | None | Optional | -| `address` | String value that specifies a service-specific IP address or hostname.
    If no value is specified, the IP address of the agent node is used by default.
    There is no service-side validation of this parameter. | IP address of the agent node | Optional | -| `meta` | Object that defines a map of the max 64 key/value pairs.
    The meta object has the same limitations as the node meta object in the node definition.
    Meta data can be retrieved per individual instance of the service. All instances of a given service have their own copy of the meta data.
    See [Adding Meta Data](#adding-meta-data) for supported parameters.
    | None | Optional | -| `tagged_addresses` | Tagged addresses are additional addresses that may be defined for a node or service. See [Tagged Addresses](#tagged-addresses) for details. | None | Optional | -| `port` | Integer value that specifies a service-specific port number. The port number should be specified when the `address` parameter is defined to improve service discoverability. | Optional | -| `socket_path` | String value that specifies the path to the service socket.
    Specify this parameter to expose the service to the mesh if the service listens on a Unix Domain socket. | None | Optional | -| `enable_tag_override` | Boolean value that determines if the anti-entropy feature for the service is enabled.
    If set to `true`, then external agents can update this service in the catalog and modify the tags.
    Subsequent local sync operations by this agent will ignore the updated tags.
    This parameter only applies to the locally-registered service. If multiple nodes register the same service, the `enable_tag_override` configuration, and all other service configuration items, operate independently.
    Updating the tags for services registered on one node is independent from the same service (by name) registered on another node.
    See [anti-entropy syncs](/consul/docs/architecture/anti-entropy) for additional information.
    | False | Optional | -| `checks` | Array of objects that define health checks for the service. See [Health Checks](#health-checks) for details. | None | Optional | -| `kind` | String value that identifies the service as a Connect proxy. See [Connect](#connect) for details. | None | Optional | -| `proxy_destination` | String value that specifies the _name_ of the destination service that the service currently being configured proxies to.
    This parameter is deprecated. Use `proxy.destination_service` instead.
    See [Connect](#connect) for additional information. | None | Optional | -| `proxy` | Object that defines the destination services that the service currently being configured proxies to. See [Proxy](#proxy) for additional information. | None | Optional | -| `connect` | Object that configures a Consul Connect service mesh connection. See [Connect](#connect) for details. | None | Optional | -| `weights` | Object that configures the weight of the service in terms of its DNS service (SRV) response. See [DNS SRV Weights](#dns-srv-weights) for details. | None | Optional | -| `token` | String value specifying the ACL token to be used to register the service (if the ACL system is enabled). The token is required for the service to interact with the service catalog. See [Security Configurations](#security-configurations) for details. | None | Required if ACLs are enabled | -| `namespace` | String value specifying the Consul Namespace where the service should be registered. See [Security Configurations](#security-configurations) for details. | None | Optional | - -### Adding Meta Data - -You can add semantic meta data to the service using the `meta` parameter. This parameter defines a map of max 64 key/value pairs. You can specify the following parameters to define meta data for the service. - -| Parameter | Description | Default | Required | -| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -------- | -| `KEY` | String value that adds semantic metadata to the service.
    Keys can only have ASCII characters (`A` - `Z`, `a` - `z`, `0` - `9`, `_`, and `-`).
    Keys can not have special characters.
    Keys are limited to 128 characters.
    Values are limited to 512 characters. | None | Optional | - -### Security Configurations - -If the ACL system is enabled, specify a value for the `token` parameter to provide an ACL token. This token is -used for any interaction with the catalog for the service, including [anti-entropy syncs](/consul/docs/architecture/anti-entropy) and deregistration. - -Services registered in Consul clusters where both [Consul Namespaces](/consul/docs/enterprise/namespaces) -and the ACL system are enabled can be registered to specific namespaces that are associated with -ACL tokens scoped to the namespace. Services registered with a service definition -will not inherit the namespace associated with the ACL token specified in the `token` -field. The `namespace` _and_ the `token` parameters must be included in the service definition for the service to be registered to the -namespace that the ACL token is scoped to. - -### Health Checks - -You can add health checks to your service definition. Health checks perform several safety functions, such as allowing a web balancer to gracefully remove failing nodes and allowing a database -to replace a failed secondary. The health check functionality is strongly integrated into the DNS interface, as well. If a service is failing its health check or a node has any failing system-level check, the DNS interface will omit that -node from any service query. - -The health check name is automatically generated as `service:`. If there are multiple service checks -registered, the ID will be generated as `service::` where -`` is an incrementing number starting from `1`. - -Consul includes several check types with different options. Refer to the [health checks documentation](/consul/docs/discovery/checks) for details. - -### Proxy - -Service definitions allow for an optional proxy registration. Proxies used with Connect -are registered as services in Consul's catalog. -See the [Proxy Service Registration](/consul/docs/connect/registration/service-registration) reference -for the available configuration options. - -### Connect - -The `kind` parameter determines the service's role. Services can be configured to perform several roles, but you must omit the `kind` parameter for typical non-proxy instances. - -The following roles are supported for service entries: - -- `connect-proxy`: Defines the configuration for a connect proxy -- `ingress-gateway`: Defines the configuration for an [ingress gateway](/consul/docs/connect/config-entries/ingress-gateway) -- `mesh-gateway`: Defines the configuration for a [mesh gateway](/consul/docs/connect/gateways/mesh-gateway) -- `terminating-gateway`: Defines the configuration for a [terminating gateway](/consul/docs/connect/config-entries/terminating-gateway#terminating-gateway) - -In the service definition example described above, the service is registered as a proxy because the `kind` property is set to `connect-proxy`. -The `proxy` parameter is also required for Connect proxy registrations and is only valid if `kind` is `connect-proxy`. -Refer to the [Proxy Service Registration](/consul/docs/connect/registration/service-registration) documentation for details about this type. - -When the `kind` parameter is set to `connect-proxy`, the only required parameter for the `proxy` configuration is `destination_service_name`. -Refer to the [complete proxy configuration example](/consul/docs/connect/registration/service-registration#complete-configuration-example) for additional information. - -The `connect` field can be specified to configure [Connect](/consul/docs/connect) for a service. This field is available in Consul 1.2.0 and later. The following parameters are available. - -| Parameter | Description | Default | Required | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | -------- | -| `native` | Boolean value that advertises the service as [Connect-native](/consul/docs/connect/native).
    If set to `true`, do not configure a `sidecar_service`. | `false` | Optional | -| `sidecar_service` | Object that defines a nested service definition.
    Do not configure if `native` is set to `true`. | See [Sidecar Service Registration](/consul/docs/connect/registration/sidecar-service) for default configurations. | Optional | - --> **Non-service registration roles**: The `kind` values supported for configuration entries are different than what is supported for service registrations. Refer to the [Configuration Entries](/consul/docs/connect/config-entries) documentation for information about non-service registration types. - -#### Deprecated parameters - -Different Consul Connect parameters are supported for different Consul versions. The following table describes changes applicable to service discovery. - -| Parameter | Description | Consul version | Status | -| ------------------- | ---------------------------------------------------------------------------------------------------- | ---------------------------- | --------------------------------------------------------------------------- | -| `proxy_destination` | Specified the proxy destination **in the root level** of the definition file. | 1.2.0 to 1.3.0 | Deprecated since 1.5.0.
    Use `proxy.destination_service_name` instead. | -| `connect.proxy` | Specified "managed" proxies, [which have been deprecated](/consul/docs/connect/proxies/managed-deprecated). | 1.2.0 (beta) to 1.3.0 (beta) | Deprecated. | - -### DNS SRV Weights - -You can configure how the service responds to DNS SRV requests by specifying a set of states/weights in the `weights` field. - -#### `weights` - -When DNS SRV requests are made, the response will include the weights specified for the given state of the service. -This allows some instances to be given higher weight if they have more capacity. It also allows load reduction on -services with checks in `warning` status by giving passing instances a higher weight. - -| Parameter | Description | Default | Required | -| --------- | --------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | -------- | -| `STATE` | Integer value indicating its weight. A higher number indicates more weight. | If not specified, the following weights are used:
    `"passing" : 1`
    `"warning" : 1` | Optional | - -If a service is `critical`, it is excluded from DNS responses. -Services with warning checks are included in responses by default, but excluded if the optional param `only_passing = true` -is present in the agent DNS configuration or the `passing` query parameter is used via the API. - -### Enable Tag Override and Anti-Entropy - -Services may also contain a `token` field to provide an ACL token. This token is -used for any interaction with the catalog for the service, including -[anti-entropy syncs](/consul/docs/architecture/anti-entropy) and deregistration. - -You can optionally disable the anti-entropy feature for this service using the -`enable_tag_override` flag. External agents can modify tags on services in the -catalog, so subsequent sync operations can either maintain tag modifications or -revert them. If `enable_tag_override` is set to `TRUE`, the next sync cycle may -revert some service properties, **but** the tags would maintain the updated value. -If `enable_tag_override` is set to `FALSE`, the next sync cycle will revert any -updated service properties, **including** tags, to their original value. - -It's important to note that this applies only to the locally registered -service. If you have multiple nodes all registering the same service -their `enable_tag_override` configuration and all other service -configuration items are independent of one another. Updating the tags -for the service registered on one node is independent of the same -service (by name) registered on another node. If `enable_tag_override` is -not specified the default value is false. See [anti-entropy -syncs](/consul/docs/architecture/anti-entropy) for more info. - -For Consul 0.9.3 and earlier you need to use `enableTagOverride`. Consul 1.0 -supports both `enable_tag_override` and `enableTagOverride` but the latter is -deprecated and has been removed as of Consul 1.1. - -### Tagged Addresses - -Tagged addresses are additional addresses that may be defined for a node or -service. Tagged addresses can be used by remote agents and services as alternative -addresses for communicating with the given node or service. Multiple tagged -addresses may be configured on a node or service. - -The following example describes the syntax for defining a tagged address. - - - -```hcl -service { - name = "redis" - port = 80 - tagged_addresses { - = { - address = "
    " - port = port - } - } -} -``` - -```json -{ - "service": { - "name": "redis", - "port": 80, - "tagged_addresses": { - "": { - "address": "
    ", - "port": port - } - } - } -} -``` - - - -The following table provides an overview of the various tagged address types supported by Consul. - -| Type | Description | Tags | -| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------- | -| LAN | LAN addresses are intended to be directly accessible only from services within the same Consul data center. See [LAN tags](#lan-tags) for details. | `lan`
    `lan_ipv4`
    `lan_ipv6` | -| Virtual | Virtual tagged addresses are logical address types that can be configured on [Connect](/consul/docs/connect)-enabled services. The virtual address provides a fixed IP address that can be used by downstream services when connecting to an upstream service. See [Virtual tags](#virtual-tags) for details. | `virtual` | -| WAN | Define a WAN address for the service or node when it should be accessed at an alternate address by services in a remote datacenter. See [WAN tags](#wan-tags) for details. | `wan`
    `wan_ipv4`
    `wan_ipv6` | - -#### LAN tags - -- `lan` - The IPv4 LAN address at which the node or service is accessible. -- `lan_ipv4` - The IPv4 LAN address at which the node or service is accessible. -- `lan_ipv6` - The IPv6 LAN address at which the node or service is accessible. - - - - - -```hcl -service { - name = "redis" - address = "192.0.2.10" - port = 80 - tagged_addresses { - lan = { - address = "192.0.2.10" - port = 80 - } - lan_ipv4 = { - address = "192.0.2.10" - port = 80 - } - lan_ipv6 = { - address = "2001:db8:1:2:cafe::1337" - port = 80 - } - } -} -``` - - - - - -```json -{ - "service": { - "name": "redis", - "address": "192.0.2.10", - "port": 80, - "tagged_addresses": { - "lan": { - "address": "192.0.2.10", - "port": 80 - }, - "lan_ipv4": { - "address": "192.0.2.10", - "port": 80 - }, - "lan_ipv6": { - "address": "2001:db8:1:2:cafe::1337", - "port": 80 - } - } - } -} -``` - - - - -#### Virtual tags - -Connections to virtual addresses are load balanced across available instances of a service, provided the following conditions are satisfied: - -1. [Transparent proxy](/consul/docs/connect/transparent-proxy) is enabled for the - downstream and upstream services. -1. The upstream service is not configured for individual instances to be - [dialed directly](/consul/docs/connect/config-entries/service-defaults#dialeddirectly). - -Virtual addresses are not required to be routable IPs within the -network. They are strictly a control plane construct used to provide a fixed -address for the instances of a given logical service. Egress connections from -the proxy to an upstream service will be destined to the IP address of an -individual service instance, not the virtual address of the logical service. - -Use the following address tag to specify the logical address at which the -service can be reached by other services in the mesh. - -- `virtual` - The virtual IP address at which a logical service is reachable. - - - - - -```hcl -service { - name = "redis" - address = "192.0.2.10" - port = 80 - tagged_addresses { - virtual = { - address = "203.0.113.50" - port = 80 - } - } -} -``` - - - - - -```json -{ - "service": { - "name": "redis", - "address": "192.0.2.10", - "port": 80, - "tagged_addresses": { - "virtual": { - "address": "203.0.113.50", - "port": 80 - } - } - } -} -``` - - - - -#### WAN tags - -One or more of the following address tags can be configured for a node or service -to advertise how it should be accessed over the WAN. - -- `wan` - The IPv4 WAN address at which the node or service is accessible when - being dialed from a remote data center. -- `wan_ipv4` - The IPv4 WAN address at which the node or service is accessible - when being dialed from a remote data center. -- `wan_ipv6` - The IPv6 WAN address at which the node or service is accessible - when being dialed from a remote data center. - - - - - -```hcl -service { - name = "redis" - address = "192.0.2.10" - port = 80 - tagged_addresses { - wan = { - address = "198.51.100.200" - port = 80 - } - wan_ipv4 = { - address = "198.51.100.200" - port = 80 - } - wan_ipv6 = { - address = "2001:db8:5:6:1337::1eaf" - port = 80 - } - } -} -``` - - - - - -```json -{ - "service": { - "name": "redis", - "address": "192.0.2.10", - "port": 80, - "tagged_addresses": { - "wan": { - "address": "198.51.100.200", - "port": 80 - }, - "wan_ipv4": { - "address": "198.51.100.200", - "port": 80 - }, - "wan_ipv6": { - "address": "2001:db8:5:6:1337::1eaf", - "port": 80 - } - } - } -} -``` - - - - -## Multiple Service Definitions - -Multiple services definitions can be provided at once when registering services -via the agent configuration by using the plural `services` key (registering -multiple services in this manner is not supported using the HTTP API). - - - - - -```hcl -services { - id = "red0" - name = "redis" - tags = [ - "primary" - ] - address = "" - port = 6000 - checks = [ - { - args = ["/bin/check_redis", "-p", "6000"] - interval = "5s" - timeout = "20s" - } - ] -} -services { - id = "red1" - name = "redis" - tags = [ - "delayed", - "secondary" - ] - address = "" - port = 7000 - checks = [ - { - args = ["/bin/check_redis", "-p", "7000"] - interval = "30s" - timeout = "60s" - } - ] -} - -``` - - - - - -```json -{ - "services": [ - { - "id": "red0", - "name": "redis", - "tags": [ - "primary" - ], - "address": "", - "port": 6000, - "checks": [ - { - "args": ["/bin/check_redis", "-p", "6000"], - "interval": "5s", - "timeout": "20s" - } - ] - }, - { - "id": "red1", - "name": "redis", - "tags": [ - "delayed", - "secondary" - ], - "address": "", - "port": 7000, - "checks": [ - { - "args": ["/bin/check_redis", "-p", "7000"], - "interval": "30s", - "timeout": "60s" - } - ] - }, - ... - ] -} -``` - - - - -## Service and Tag Names with DNS - -Consul exposes service definitions and tags over the [DNS](/consul/docs/discovery/dns) -interface. DNS queries have a strict set of allowed characters and a -well-defined format that Consul cannot override. While it is possible to -register services or tags with names that don't match the conventions, those -services and tags will not be discoverable via the DNS interface. It is -recommended to always use DNS-compliant service and tag names. - -DNS-compliant service and tag names may contain any alpha-numeric characters, as -well as dashes. Dots are not supported because Consul internally uses them to -delimit service tags. - -## Service Definition Parameter Case - -For historical reasons Consul's API uses `CamelCased` parameter names in -responses, however its configuration file uses `snake_case` for both HCL and -JSON representations. For this reason the registration _HTTP APIs_ accept both -name styles for service definition parameters although APIs will return the -listings using `CamelCase`. - -Note though that **all config file formats require -`snake_case` fields**. We always document service definition examples using -`snake_case` and JSON since this format works in both config files and API -calls. diff --git a/website/content/docs/ecs/configuration-reference.mdx b/website/content/docs/ecs/configuration-reference.mdx index da0ee28f681..fed887bc835 100644 --- a/website/content/docs/ecs/configuration-reference.mdx +++ b/website/content/docs/ecs/configuration-reference.mdx @@ -185,7 +185,7 @@ Defines the Consul checks for the service. Each `check` object may contain the f | `method` | `string` | optional | Specifies the HTTP method to be used for an HTTP check. When no value is specified, `GET` is used. | | `name` | `string` | optional | The name of the check. | | `notes` | `string` | optional | Specifies arbitrary information for humans. This is not used by Consul internally. | -| `os_service` | `string` | optional | Specifies the name of a service on which to perform an [OS service check](/consul/docs/discovery/checks#osservice-check). The check runs according the frequency specified in the `interval` parameter. | +| `os_service` | `string` | optional | Specifies the name of a service on which to perform an [OS service check](/consul/docs/services/usage/checks#osservice-check). The check runs according the frequency specified in the `interval` parameter. | | `status` | `string` | optional | Specifies the initial status the health check. Must be one of `passing`, `warning`, `critical`, `maintenance`, or `null`. | | `successBeforePassing` | `integer` | optional | Specifies the number of consecutive successful results required before check status transitions to passing. | | `tcp` | `string` | optional | Specifies this is a TCP check. Must be an IP/hostname plus port to which a TCP connection is made every `interval`. | diff --git a/website/content/docs/ecs/index.mdx b/website/content/docs/ecs/index.mdx index 20c7b272967..963604188ae 100644 --- a/website/content/docs/ecs/index.mdx +++ b/website/content/docs/ecs/index.mdx @@ -22,7 +22,7 @@ traffic policy, and more. You can also connect service meshes so that services d Consul on ECS follows an [architecture](/consul/docs/architecture) similar to other platforms, but each ECS task is a Consul node. An ECS task runs the user application container(s), as well as a Consul client container for control plane communication and an [Envoy](https://envoyproxy.io/) sidecar proxy container to facilitate data plane communication for -[Consul Connect](/consul/docs/connect). +[Consul service mesh](/consul/docs/connect). For a detailed architecture overview, see the [Architecture](/consul/docs/ecs/architecture) page. @@ -30,8 +30,7 @@ For a detailed architecture overview, see the [Architecture](/consul/docs/ecs/ar There are several ways to get started with Consul with ECS. -- The [Serverless Consul Service Mesh with ECS and HCP](/consul/tutorials/cloud-production/consul-ecs-hcp?utm_source=docs) learn guide shows how to use Terraform to run Consul service mesh applications on ECS with managed Consul servers running in HashiCorp Cloud Platform (HCP). -- The [Service Mesh with ECS and Consul on EC2](/consul/tutorials/cloud-integrations/consul-ecs-ec2?utm_source=docs) learn guide shows how to use Terraform to run Consul service mesh applications on ECS with Consul servers running on EC2 instances. +- The [Integrate your AWS ECS services into Consul service mesh](/consul/tutorials/cloud-integrations/consul-ecs) tutorial shows how to use Terraform to run Consul service mesh applications on ECS with self-managed Consul or HCP-managed Consul. - The [Consul with Dev Server on Fargate](https://registry.terraform.io/modules/hashicorp/consul-ecs/aws/latest/examples/dev-server-fargate) example installation deploys a sample application in ECS using the Fargate launch type. - The [Consul with Dev Server on EC2](https://registry.terraform.io/modules/hashicorp/consul-ecs/aws/latest/examples/dev-server-ec2) example installation deploys a sample application in ECS using the EC2 launch type. diff --git a/website/content/docs/ecs/manual/install.mdx b/website/content/docs/ecs/manual/install.mdx index e648c6cf3ea..ffe7c805849 100644 --- a/website/content/docs/ecs/manual/install.mdx +++ b/website/content/docs/ecs/manual/install.mdx @@ -127,7 +127,7 @@ See the [ECS Task Definition](https://docs.aws.amazon.com/AmazonECS/latest/devel ## `sidecar-proxy` container -The `sidecar-proxy` container runs [Envoy proxy](/consul/docs/connect/proxies/envoy) for Consul Connect. In most cases, the container should contain the following parameters and values. +The `sidecar-proxy` container runs [Envoy proxy](/consul/docs/connect/proxies/envoy) for Consul service mesh. In most cases, the container should contain the following parameters and values. The `mountPoints` list must be set as shown in the following example. This will mount the shared `consul_data` volume into the `sidecar-proxy` container at the path `/consul`. This volume is where the `consul-ecs-mesh-init` container copies the `envoy-bootstrap.json` @@ -373,11 +373,11 @@ configuration to a shared volume. ### `CONSUL_ECS_CONFIG_JSON` -Configuration is passed to the `consul-ecs` binary in JSON format using the `CONSUL_ECS_CONFIG_JSON` environment variable. +Consul uses the `CONSUL_ECS_CONFIG_JSON` environment variable to passed configurations to the `consul-ecs` binary in JSON format. -The following is an example of the configuration that might be used for a service named `example-client-app` with one upstream -service name `example-server-app`. The `proxy` and `service` blocks include information used by `consul-ecs-mesh-init` to perform -[service registration](/consul/docs/discovery/services) with Consul during task startup. The same configuration format is used for +The following example configures a service named `example-client-app` with one upstream +service name `example-server-app`. The `proxy` and `service` blocks include information used by `consul-ecs-mesh-init` to register the service with Consul during task start up. +The same configuration format is used for the `consul-ecs-health-sync` container. ```json @@ -409,7 +409,7 @@ the `consul-ecs-health-sync` container. | `proxy.upstreams` | list | The upstream services that your application calls over the service mesh, if any. The `destinationName` and `localBindPort` fields are required. | | `service.name` | string | The name used to register this service into the Consul service catalog. | | `service.port` | integer | The port your application listens on. Set to `0` if your application does not listen on any port. | -| `service.checks` | list | Consul [checks](/consul/docs/discovery/checks) to include so that Consul can run health checks against your application. | +| `service.checks` | list | Consul [checks](/consul/docs/services/usage/checks) to include so that Consul can run health checks against your application. | See the [Configuration Reference](/consul/docs/ecs/configuration-reference) for a complete reference of fields. diff --git a/website/content/docs/ecs/terraform/secure-configuration.mdx b/website/content/docs/ecs/terraform/secure-configuration.mdx index c1997241c3b..07031f5490e 100644 --- a/website/content/docs/ecs/terraform/secure-configuration.mdx +++ b/website/content/docs/ecs/terraform/secure-configuration.mdx @@ -108,8 +108,9 @@ The following table describes the required input variables for the `acl-controll | `name_prefix` | string | AWS resources created by the `acl-controller` module will include this prefix in the resource name. | -If you are using Consul Enterprise, see Admin Partitions and Namespaces for -additional configuration required to support Consul Enterprise on ECS. + +If you are using Consul Enterprise, see the [Admin Partitions and Namespaces requirements documentation](/consul/docs/ecs/requirements) for additional configuration required to support Consul Enterprise on ECS. + ## Deploy your services diff --git a/website/content/docs/enterprise/admin-partitions.mdx b/website/content/docs/enterprise/admin-partitions.mdx index e5fdea32cfa..6b26360c935 100644 --- a/website/content/docs/enterprise/admin-partitions.mdx +++ b/website/content/docs/enterprise/admin-partitions.mdx @@ -23,7 +23,7 @@ Admin partitions exist a level above namespaces in the identity hierarchy. They As of Consul v1.11, every _datacenter_ contains a single administrative partition named `default` when created. With Consul Enterprise, operators have the option of creating multiple partitions within a single datacenter. --> **Preexisting resource nodes and namespaces**: Admin partitions were introduced in Consul 1.11. Resource nodes were not namespaced prior to 1.11. After upgrading to Consul 1.11 or later, all resource nodes will be namespaced. +-> **Preexisting nodes**: Admin partitions were introduced in Consul 1.11. Nodes existed in global scope prior to 1.11. After upgrading to Consul 1.11 or later, all nodes will be scoped to an admin partition, which will be the `default` partition when initially upgrading an existing deployment or for OSS versions. There are tutorials available to help you get started with admin partitions. @@ -53,7 +53,12 @@ Only resources in the `default` admin partition will be replicated to secondary ### DNS Queries -Client agents will be configured to operate within a specific admin partition. The DNS interface will only return results for the admin partition within the scope of the client. +When queried, the DNS interface returns results for a single admin partition. +The query may explicitly specify the admin partition to use in the lookup. +If you do not specify an admin partition in the query, +the lookup uses the admin partition of the Consul agent that received the query. +Server agents always exist within the `default` admin partition. +Client agents are configured to operate within a specific admin partition. ### Service Mesh Configurations @@ -61,14 +66,12 @@ The partition in which [`proxy-defaults`](/consul/docs/connect/config-entries/pr ### Cross-partition Networking -You can configure services to be discoverable by downstream services in any partition within the datacenter. Specify the upstream services that you want to be available for discovery by configuring the `exported-services` configuration entry in the partition where the services are registered. Refer to the [`exported-services` documentation](/consul/docs/connect/config-entries/exported-services) for details. Additionally, the requests made by downstream applications must have the correct DNS name for the Virtual IP Service lookup to occur. Service Virtual IP lookups allow for communications across Admin Partitions when using Transparent Proxy. Refer to the [Service Virtual IP Lookups for Consul Enterprise](/consul/docs/discovery/dns#service-virtual-ip-lookups-for-consul-enterprise) for additional information. +You can configure services to be discoverable by downstream services in any partition within the datacenter. Specify the upstream services that you want to be available for discovery by configuring the `exported-services` configuration entry in the partition where the services are registered. Refer to the [`exported-services` documentation](/consul/docs/connect/config-entries/exported-services) for details. Additionally, the requests made by downstream applications must have the correct DNS name for the Virtual IP Service lookup to occur. Service Virtual IP lookups allow for communications across Admin Partitions when using Transparent Proxy. Refer to the [Service Virtual IP Lookups for Consul Enterprise](/consul/docs/services/discovery/dns-static-lookups#service-virtual-ip-lookups-for-consul-enterprise) for additional information. ### Cluster Peering You can use [cluster peering](/consul/docs/connect/cluster-peering/) between two admin partitions to connect clusters owned by different operators. Without Consul Enterprise, cluster peering is limited to the `default` partitions in each datacenter. Enterprise users can [create and manage cluster peering connections](/consul/docs/connect/cluster-peering/create-manage-peering) between any two admin partitions as long as the partitions are in separate datacenters. It is not possible to establish cluster peering connections between two partitions in a single datacenter. -To use mesh gateways with admin partitions and cluster peering, refer to [Mesh Gateways between Peered Clusters](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers). - ## Requirements Your Consul configuration must meet the following requirements to use admin partitions. diff --git a/website/content/docs/enterprise/audit-logging.mdx b/website/content/docs/enterprise/audit-logging.mdx index 14d87265bff..75d3b33a410 100644 --- a/website/content/docs/enterprise/audit-logging.mdx +++ b/website/content/docs/enterprise/audit-logging.mdx @@ -82,6 +82,21 @@ audit { } } ``` + +```yaml +server: + auditLogs: + enabled: true + sinks: + - name: My Sink + type: file + format: json + path: /tmp/audit.json + delivery_guarantee: best-effort + rotate_duration: 24h + rotate_max_files: 15 + rotate_bytes: 25165824 +``` @@ -122,6 +137,18 @@ audit { } ``` +```yaml +server: + auditLogs: + enabled: true + sinks: + - name: My Sink + type: file + format: json + path: /dev/stdout + delivery_guarantee: best-effort +``` + diff --git a/website/content/docs/enterprise/index.mdx b/website/content/docs/enterprise/index.mdx index d03ab438d20..e63363e8106 100644 --- a/website/content/docs/enterprise/index.mdx +++ b/website/content/docs/enterprise/index.mdx @@ -116,17 +116,17 @@ Consul Enterprise feature availability can change depending on your server and c | Enterprise Feature | VM Client | K8s Client | ECS Client | | ----------------------------------------------------------------------- | :-------: | :--------: | :--------: | -| [Admin Partitions](/consul/docs/enterprise/admin-partitions) | ✅ | ✅ | ❌ | -| [Audit Logging](/consul/docs/enterprise/audit-logging) | ✅ | ✅ | ❌ | -| [Automated Server Backups](/consul/docs/enterprise/backups) | ✅ | ✅ | ❌ | +| [Admin Partitions](/consul/docs/enterprise/admin-partitions) | ✅ | ✅ | ✅ | +| [Audit Logging](/consul/docs/enterprise/audit-logging) | ✅ | ✅ | ✅ | +| [Automated Server Backups](/consul/docs/enterprise/backups) | ✅ | ✅ | ✅ | | [Automated Server Upgrades](/consul/docs/enterprise/upgrades) | ❌ | ❌ | ❌ | | [Enhanced Read Scalability](/consul/docs/enterprise/read-scale) | ❌ | ❌ | ❌ | -| [Namespaces](/consul/docs/enterprise/namespaces) | ✅ | ✅ | ❌ | -| [Network Areas](/consul/docs/enterprise/federation) | ✅ | ✅ | ❌ | +| [Namespaces](/consul/docs/enterprise/namespaces) | ✅ | ✅ | ✅ | +| [Network Areas](/consul/docs/enterprise/federation) | ✅ | ✅ | ✅ | | [Network Segments](/consul/docs/enterprise/network-segments/network-segments-overview) | ❌ | ❌ | ❌ | -| [OIDC Auth Method](/consul/docs/security/acl/auth-methods/oidc) | ✅ | ✅ | ❌ | +| [OIDC Auth Method](/consul/docs/security/acl/auth-methods/oidc) | ✅ | ✅ | ✅ | | [Redundancy Zones](/consul/docs/enterprise/redundancy) | ❌ | ❌ | ❌ | -| [Sentinel ](/consul/docs/enterprise/sentinel) | ✅ | ✅ | ❌ | +| [Sentinel ](/consul/docs/enterprise/sentinel) | ✅ | ✅ | ✅ | diff --git a/website/content/docs/enterprise/license/faq.mdx b/website/content/docs/enterprise/license/faq.mdx index 02841e9c6cf..d8a46ee2093 100644 --- a/website/content/docs/enterprise/license/faq.mdx +++ b/website/content/docs/enterprise/license/faq.mdx @@ -74,6 +74,8 @@ after license expiration and is defined in ~> **Starting with Consul 1.14, and patch releases 1.13.3 and 1.12.6, Consul will support non-terminating licenses**: Please contact your support representative for more details on non-terminating licenses. + An expired license will not allow Consul versions released after the expiration date to run. + It will not be possible to upgrade to a new version of Consul released after license expiration. ## Q: Does this affect client agents? @@ -136,7 +138,7 @@ When a customer deploys new clusters to a 1.10.0+ent release, they need to have New Consul cluster deployments using 1.10.0+ent will need to have a valid license on servers to successfully deploy. This valid license must be on-disk (auto-loaded) or as an environment variable. -Please see the [upgrade requirements](https://consul.io/faq#q-what-are-the-upgrade-requirements). +Please see the [upgrade requirements](/consul/docs/enterprise/license/faq#q-what-are-the-upgrade-requirements). ## Q: What is the migration path for customers who want to migrate from their existing license-as-applied-via-the-CLI flow to the license on disk flow? @@ -181,7 +183,7 @@ When downgrading to a version of Consul before 1.10.0+ent, customers will need t ## Q: Are there potential pitfalls when downgrading or upgrading Consul server instances? -~> Verify that you meet the [upgrade requirements](https://consul.io/faq#q-what-are-the-upgrade-requirements). +~> Verify that you meet the [upgrade requirements](/consul/docs/enterprise/license/faq#q-what-are-the-upgrade-requirements). Assume a scenario where there are three Consul server nodes: diff --git a/website/content/docs/enterprise/license/utilization-reporting.mdx b/website/content/docs/enterprise/license/utilization-reporting.mdx new file mode 100644 index 00000000000..6cbca18aa4b --- /dev/null +++ b/website/content/docs/enterprise/license/utilization-reporting.mdx @@ -0,0 +1,174 @@ +--- +page_title: Automated license utilization reporting +description: >- + Learn what data HashiCorp collects to meter Enterprise license utilization. Enable or disable reporting. Review sample payloads and logs. +--- + +# Automated license utilization reporting + +This topic describes how to enable automated license utilization reporting in Consul Enterprise. This feature automatically sends license utilization data to HashiCorp so that you do not have to manually collect and report it. It also enables you to review your license usage with the monitoring solution you already use, such as Splunk and Datadog, as you optimize and manage your deployments. + +## Introduction + +You can use automated license utilization report to understand how much additional networking infrastructure you can deploy under your current contract. This feature helps you protect against overutilization and budget for predicted consumption. + +Automated reporting shares the minimum data required to validate license utilization as defined in our contracts. This data mostly consists of computed metrics, and it will never contain Personal Identifiable Information (PII) or other sensitive information. Automated reporting shares the data with HashiCorp using a secure unidirectional HTTPS API and makes an auditable record in the product logs each time it submits a report. This process is GDPR-compliant. + +## Requirements + +Automated license utilization reporting does not support _air-gapped installations_, which are systems with no network interfaces. + +The following versions of Consul Enterprise support automated license utilization reporting: + +- Patch releases of Consul Enterprise v1.15.4 and newer. +- Patch releases of Consul Enterprise v1.14.8 and newer. +- Patch releases of Consul Enterprise v1.13.9 and newer. + +Download a supported release from the [Consul Versions](https://releases.hashicorp.com/consul/) page. + +## Enable automated reporting + +Before you enable automated reporting, make sure that outbound network traffic is configured correctly and upgrade your enterprise product to a version that supports it. If your installation is air-gapped or network settings are not in place, automated reporting will not work. + +To enable automated reporting, complete the following steps: + +1. [Allow outbound HTTPS traffic on port 443](#allow-outbound-https-traffic) +1. [Check product logs](#check-product-logs) + +### Allow outbound HTTPS traffic on port 443 + +Make sure that your network allows HTTPS egress on port 443 from `https://reporting.hashicorp.services` by adding the following IP adddresses to your allow-list: + +- `100.20.70.12` +- `35.166.5.222` +- `23.95.85.111` +- `44.215.244.1` + +### Check product logs + +Automatic license utilization reporting starts sending data within roughly 24 hours. Check the product logs for records that the data sent successfully. + + + +``` +[DEBUG] beginning snapshot export +[DEBUG] creating payload +[DEBUG] marshalling payload to json +[DEBUG] generating authentication headers +[DEBUG] creating request +[DEBUG] sending request +[DEBUG] performing request: method=POST url=https://census.license.hashicorp.services +[DEBUG] recording audit record +[INFO] reporting: Report sent: auditRecord={"payload":{"payload_version":"1","license_id":"d2cdd857-4202-5a45-70a6-e4b531050c34","product":"consul","product_version":"1.16.0-dev+ent","export_timestamp":"2023-05-26T20:09:13.753921087Z","snapshots":[{"snapshot_version":1,"snapshot_id":"0001J724F90F4XWQDSAA76ZQWA","process_id":"01H1CTJPC1S8H7Q45MKTJ689ZW","timestamp":"2023-05-26T20:09:13.753513962Z","schema_version":"1.0.0","service":"consul","metrics":{"consul.billable.nodes":{"key":"consul.billable.nodes","kind":"counter","mode":"write","value":2},"consul.billable.service_instances":{"key":"consul.billable.service_instances","kind":"counter","mode":"write","value":2}}}],"metadata":{}}} +[DEBUG] completed recording audit record +[DEBUG] export finished successfully" +``` + + + +If your installation is air-gapped or your network does not allow the correct egress, the logs show an error. + + + +``` +[DEBUG] reporting: beginning snapshot export +[DEBUG] reporting: creating payload +[DEBUG] reporting: marshalling payload to json +[DEBUG] reporting: generating authentication headers +[DEBUG] reporting: creating request +[DEBUG] reporting: sending request +[DEBUG] reporting: performing request: method=POST url=https://census.license.hashicorp.services +[DEBUG] reporting: error status code received: statusCode=403 +``` + + + +In this case, reconfigure your network to allow egress and check the logs again in roughly 24 hours to confirm that automated reporting works correctly. + +## Opt out + +If your installation is air-gapped or you want to manually collect and report on the same license utilization metrics, you can opt out of automated reporting. + +Manually reporting these metrics can be time consuming. Opting out of automated reporting does not mean that you also opt out from sending license utilization metrics. Customers who opt out of automated reporting are still required to manually collect and send license utilization metrics to HashiCorp. + +If you are considering opting out because you are worried about the data, we strongly recommend that you review the [example payloads](#example-payloads) before opting out. If you have concerns with any of the automatically reported data, raise these concerns with your account manager. + +There are two methods for opting out of automated reporting: + +- HCL configuration (recommended) +- Environment variable (requires restart) + +We recommend opting out in your product's configuration file because it does not require a system restart. Add the following block to your `configuration.hcl` or `configuration.json` file. + +```hcl +reporting { + license { + enabled = false + } +} +``` + +When opting out using an environment variable, the system provides a startup message confirming that you have disabled automated reporting. Set the following environment variable to disable automated reporting: + + + +```shell-session +$ export OPTOUT_LICENSE_REPORTING=true +``` + + + +After you set the environment variable, restart your system to complete the process for opting out. + +```shell-session +$ consul reload +``` + + +Check your product logs roughly 24 hours after opting out to make sure that the system is not trying to send reports. Keep in mind that if your configuration file and environment variable differ, the environment variable setting takes precedence. + +## Example payloads + +HashiCorp collects the following utilization data as JSON payloads: +`exporter_version` - The version of the licensing exporter + + + +```json +{ + "payload": { + "payload_version": "1", + "license_id": "d2cdd857-4202-5a45-70a6-e4b531050c34", + "product": "consul", + "product_version": "1.16.0-dev+ent", + "export_timestamp": "2023-05-26T20:09:13.753921087Z", + "snapshots": [ + { + "snapshot_version": 1, + "snapshot_id": "0001J724F90F4XWQDSAA76ZQWA", + "process_id": "01H1CTJPC1S8H7Q45MKTJ689ZW", + "timestamp": "2023-05-26T20:09:13.753513962Z", + "schema_version": "1.0.0", + "service": "consul", + "metrics": { + "consul.billable.nodes": { + "key": "consul.billable.nodes", + "kind": "counter", + "mode": "write", + "value": 2 + }, + "consul.billable.service_instances": { + "key": "consul.billable.service_instances", + "kind": "counter", + "mode": "write", + "value": 2 + } + } + } + ], + "metadata": {} + } +} +``` + + \ No newline at end of file diff --git a/website/content/docs/guides/index.mdx b/website/content/docs/guides/index.mdx index 02427c8b44b..24f96d794a0 100644 --- a/website/content/docs/guides/index.mdx +++ b/website/content/docs/guides/index.mdx @@ -33,6 +33,6 @@ Tracks include: - Day 1: Kubernetes Production Deployment - Maintenance and Monitoring Operations - Service Discovery and Consul DNS -- Service Segmentation and Consul Connect +- Service Segmentation and Consul Service Mesh - Service Configuration and Consul KV - Cloud and Load Balancer Integrations diff --git a/website/content/docs/install/cloud-auto-join.mdx b/website/content/docs/install/cloud-auto-join.mdx index 40e2f44edf0..82cf6b6d592 100644 --- a/website/content/docs/install/cloud-auto-join.mdx +++ b/website/content/docs/install/cloud-auto-join.mdx @@ -34,7 +34,7 @@ or via a configuration file: ## Auto-join with Network Segments -In order to use cloud auto-join with [Network Segments](/consul/docs/enterprise/network-segments), +In order to use cloud auto-join with [Network Segments](/consul/docs/enterprise/network-segments/network-segments-overview), you must reconfigure the Consul agent's Serf LAN port to match that of the segment you wish to join. diff --git a/website/content/docs/install/performance.mdx b/website/content/docs/install/performance.mdx index 7ab46b2d36e..778f4ad6698 100644 --- a/website/content/docs/install/performance.mdx +++ b/website/content/docs/install/performance.mdx @@ -172,9 +172,9 @@ To prevent any CPU spikes from a misconfigured client, RPC requests to the serve In addition, two [performance indicators](/consul/docs/agent/telemetry) — `consul.runtime.alloc_bytes` and `consul.runtime.heap_objects` — can help diagnose if the current sizing is not adequately meeting the load. -## Connect Certificate Signing CPU Limits +## Service Mesh Certificate Signing CPU Limits -If you enable [Connect](/consul/docs/connect), the leader server will need +If you enable [service mesh](/consul/docs/connect), the leader server will need to perform public key signing operations for every service instance in the cluster. Typically these operations are fast on modern hardware, however when the CA is changed or its key rotated, the leader will face an influx of @@ -194,11 +194,11 @@ operations could impact the server's normal work. To limit that, Consul since By default we set a limit of 50 per second which is reasonable on modest hardware but may be too low and impact rotation times if more than 1500 service -instances are using Connect in the cluster. `csr_max_per_second` is likely best +instances are using service mesh in the cluster. `csr_max_per_second` is likely best if you have fewer than four cores available since a whole core being used by signing is likely to impact the server stability if it's all or a large portion of the cores available. The downside is that you need to capacity plan: how many -service instances will need Connect certificates? What CSR rate can your server +service instances will need service mesh certificates? What CSR rate can your server tolerate without impacting stability? How fast do you want CA rotations to process? diff --git a/website/content/docs/integrate/download-tools.mdx b/website/content/docs/integrate/download-tools.mdx index a1263b7d817..71ad08504a1 100644 --- a/website/content/docs/integrate/download-tools.mdx +++ b/website/content/docs/integrate/download-tools.mdx @@ -15,7 +15,7 @@ These Consul tools are created and managed by the dedicated engineers at HashiCo - [Envconsul](https://github.com/hashicorp/envconsul) - Read and set environmental variables for processes from Consul. - [Consul API Gateway](https://github.com/hashicorp/consul-api-gateway/) - dedicated ingress solution for intelligently routing traffic to applications running on a Consul Service Mesh. -- [Consul ESM](https://github.com/hashicorp/consul-esm) - Provides external service monitoring for Consul. Complete the [tutorial](https://consul.io/(https://learn.hashicorp.com/tutorials/consul/service-registration-external-services?utm_source=docs)) to learn more. +- [Consul ESM](https://github.com/hashicorp/consul-esm) - Provides external service monitoring for Consul. Complete the [tutorial](/consul/tutorials/developer-discovery/service-registration-external-services?utm_source=docs) to learn more. - [Consul Migrate](https://github.com/hashicorp/consul-migrate) - Data migration tool to handle Consul upgrades to 0.5.1+ - [Consul Replicate](https://github.com/hashicorp/consul-replicate) - Consul cross-DC KV replication daemon. - [Consul Template](https://github.com/hashicorp/consul-template) - Generic template rendering and notifications with Consul. Complete the [tutorial](/consul/tutorials/developer-configuration/consul-template?utm_source=docs) to the learn more. @@ -47,14 +47,13 @@ These Consul tools are created and managed by the amazing members of the Consul - [fabio](https://github.com/eBay/fabio) - Fast, zero-conf, consul-aware load-balancing HTTP/HTTPS router - [files-to-consul-kv](https://github.com/bitsofinfo/files-to-consul-kv) - Ultra simple, configuration free CLI tool for syncing a directory structure of key-value files to Consul KV using the transactions API. Docker image available. Integrates easily into any CI/CD workflow. - [file2consul](https://github.com/joeatbayes/file2consul) - Update Consul values from git or files. Config loader with support for multiple environments. Provides variable expansion, interpolation, inheritance with overrides and ability to update multiple consul servers. Reduces cost of maintaining larger configuration sets between environments by reducing restatement and manual editing of similar or predictably changing config properties. MIT license, Written in GO. -- [Flightpath](https://docs.flightpath.xyz/) - An xDS server that can configure Envoy to act as an Edge proxy for Consul Connect enabled services +- [Flightpath](https://docs.flightpath.xyz/) - An xDS server that can configure Envoy to act as an Edge proxy for Consul service mesh enabled services - [git2consul](https://github.com/ryanbreen/git2consul) - Mirror the contents of a Git repository into Consul KVs - [gobetween](https://github.com/yyyar/gobetween) - Modern & minimalistic load balancer and reverse-proxy for the ☁️ Cloud era. - [Gonsul](https://github.com/miniclip/gonsul) - A Git to Consul standalone tool made in Go. Updates Consul KV from a repo with multiple strategies. - [gradle-consul-plugin](https://github.com/amirkibbar/red-apple) - A Consul Gradle plugin - [hashi-ui](https://github.com/jippi/hashi-ui) - A modern user interface for the Consul and Nomad - [HashiBox](https://github.com/nunchistudio/hashibox) - Vagrant environment to simulate highly-available cloud with Consul, Nomad, Vault, and optional support for Waypoint. OSS & Enterprise supported. -- [helios-consul](https://github.com/SVT/helios-consul) - Service registrar plugin for Helios - [Jenkins Consul Plugin](https://plugins.jenkins.io/consul) - Jenkins plugin for service discovery and K/V store - [marathon-consul](https://github.com/allegro/marathon-consul) - Service registry bridge for Marathon - [marathon-consul](https://github.com/CiscoCloud/marathon-consul) - Bridge from Marathon apps to the Consul K/V store diff --git a/website/content/docs/integrate/partnerships.mdx b/website/content/docs/integrate/partnerships.mdx index 0f9f7301b78..50188f07946 100644 --- a/website/content/docs/integrate/partnerships.mdx +++ b/website/content/docs/integrate/partnerships.mdx @@ -27,7 +27,7 @@ By leveraging Consul's RESTful HTTP API system, prospective partners are able to -**Data Plane**: These integrations extend Consul's certificate management, secure ACL configuration, observability metrics and logging, and service discovery that allows for dynamic service mapping APM and logging tools, extend sidecar proxies to support Consul connect, and extend API gateways to allow Consul to route incoming traffic to the proxies for Connect-enabled services. +**Data Plane**: These integrations extend Consul's certificate management, secure ACL configuration, observability metrics and logging, and service discovery that allows for dynamic service mapping APM and logging tools, extend sidecar proxies to support Consul service mesh, and extend API gateways to allow Consul to route incoming traffic to the proxies for mesh-enabled services. **Control Plane**: Consul has a client-server architecture and is the control plane for the service mesh. @@ -89,7 +89,7 @@ Here are links to resources, documentation, examples and best practices to guide - [Monitoring Consul with Datadog APM](https://www.datadoghq.com/blog/consul-datadog/) - [Monitor HCP Consul with New Relic Instant Observability](https://github.com/newrelic-experimental/hashicorp-quickstart-annex/blob/main/hcp-consul/README.md) - [HCP Consul and CloudFabrix AIOps Integration](https://bot-docs.cloudfabrix.io/Bots/consul/?h=consul) -- [Consul and SnappyFlow Full Stack Observability](https://docs.snappyflow.io/docs/integrations/hcp_consul) +- [Consul and SnappyFlow Full Stack Observability](https://docs.snappyflow.io/docs/Integrations/hcp_consul) **Network Performance Monitoring (NPM)** diff --git a/website/content/docs/internals/acl.mdx b/website/content/docs/internals/acl.mdx index 05c8d380191..87e35844179 100644 --- a/website/content/docs/internals/acl.mdx +++ b/website/content/docs/internals/acl.mdx @@ -10,7 +10,4 @@ description: >- # ACL System ((#version_8_acls)) -This content has been moved into the [ACL Guide](/consul/tutorials/security/access-control-setup-production). - -See [Complete ACL Coverage in Consul 0.8](/consul/docs/security/acl/acl-legacy) for details -about ACL changes in Consul 0.8 and later. +This content has been moved into the [ACL Guide](/consul/tutorials/security/access-control-setup-production). \ No newline at end of file diff --git a/website/content/docs/intro/index.mdx b/website/content/docs/intro/index.mdx index 50834137b4e..90a1759fe23 100644 --- a/website/content/docs/intro/index.mdx +++ b/website/content/docs/intro/index.mdx @@ -24,7 +24,7 @@ Consul interacts with the _data plane_ through proxies. The data plane is the pa The core Consul workflow consists of the following stages: -- **Register**: Teams add services to the Consul catalog, which is a central registry that lets services automatically discover each other without requiring a human operator to modify application code, deploy additional load balancers, or hardcode IP addresses. It is the runtime source of truth for all services and their addresses. Teams can manually [define and register services](/consul/docs/discovery/services) using the CLI or the API, or you can automate the process in Kubernetes with [service sync](/consul/docs/k8s/service-sync). Services can also include health checks so that Consul can monitor for unhealthy services. +- **Register**: Teams add services to the Consul catalog, which is a central registry that lets services automatically discover each other without requiring a human operator to modify application code, deploy additional load balancers, or hardcode IP addresses. It is the runtime source of truth for all services and their addresses. Teams can manually [define](/consul/docs/services/usage/define-services) and [register](/consul/docs/services/usage/register-services-checks) using the CLI or the API, or you can automate the process in Kubernetes with [service sync](/consul/docs/k8s/service-sync). Services can also include health checks so that Consul can monitor for unhealthy services. - **Query**: Consul’s identity-based DNS lets you find healthy services in the Consul catalog. Services registered with Consul provide health information, access points, and other data that help you control the flow of data through your network. Your services only access other services through their local proxy according to the identity-based policies you define. - **Secure**: After services locate upstreams, Consul ensures that service-to-service communication is authenticated, authorized, and encrypted. Consul service mesh secures microservice architectures with mTLS and can allow or restrict access based on service identities, regardless of differences in compute environments and runtimes. diff --git a/website/content/docs/k8s/annotations-and-labels.mdx b/website/content/docs/k8s/annotations-and-labels.mdx index d6ea11d4f1d..31c4f9d3919 100644 --- a/website/content/docs/k8s/annotations-and-labels.mdx +++ b/website/content/docs/k8s/annotations-and-labels.mdx @@ -2,20 +2,28 @@ layout: docs page_title: Annotations and Labels description: >- - Annotations and labels configure Consul sidecar properties and injection behavior when scheduling Kubernetes clusters. Learn about the annotations and labels that enable Consul’s service mesh and secure upstream communication on k8s in this reference guide. + Annotations and labels configure service mesh sidecar properties and injection behavior when scheduling Kubernetes clusters. Learn about the annotations and labels that enable Consul’s service mesh and secure upstream communication on k8s in this reference guide. --- # Annotations and Labels ## Overview -Consul on Kubernetes provides a few options for customizing how connect-inject behavior should be configured. +Consul on Kubernetes provides a few options for customizing how connect-inject or service sync behavior should be configured. This allows the user to configure natively configure Consul on select Kubernetes resources (i.e. pods, services). -- [Annotations](#annotations) -- [Labels](#labels) +- [Consul Service Mesh](#consul-service-mesh) + - [Annotations](#annotations) + - [Labels](#labels) +- [Service Sync](#service-sync) + - [Annotations](#annotations-1) -## Annotations +The noun _connect_ is used throughout this documentation to refer to the connect +subsystem that provides Consul's service mesh capabilities. + +## Consul Service Mesh + +### Annotations The following Kubernetes resource annotations could be used on a pod to control connect-inject behavior: @@ -69,85 +77,92 @@ The following Kubernetes resource annotations could be used on a pod to control public listener will listen on a dynamic port. - `consul.hashicorp.com/connect-service-upstreams` - The list of upstream - services that this pod needs to connect to via Connect along with a static + services that this pod needs to connect to via the service mesh along with a static local port to listen for those connections. When transparent proxy is enabled, - this annotation is optional. There are a few formats this annotation can take: - - - Unlabeled: - Use the unlabeled annotation format to specify a service name, Consul Enterprise namespaces and partitions, and - datacenters. To use [cluster peering](/consul/docs/connect/cluster-peering/k8s) with upstreams, use the following - labeled format. - - Service name: Place the service name at the beginning of the annotation to specify the upstream service. You can - also append the datacenter where the service is deployed (optional). - ```yaml - annotations: - "consul.hashicorp.com/connect-service-upstreams":"[service-name]:[port]:[optional datacenter]" - ``` + this annotation is optional. This annotation can be either _labeled_ or _unlabeled_. We recommend the labeled format because it has a more consistent syntax and can be used to reference cluster peers as upstreams. - - Namespace (requires Consul Enterprise 1.7+): Upstream services may be running in a different namespace. Place - the upstream namespace after the service name. For additional details about configuring the injector, refer to + - **Labeled**: - [Consul Enterprise Namespaces](#consul-enterprise-namespaces) . - ```yaml - annotations: - "consul.hashicorp.com/connect-service-upstreams":"[service-name].[service-namespace]:[port]:[optional datacenter]" - ``` - If the namespace is not specified, the annotation defaults to the namespace of the source service. - If you are not using Consul Enterprise 1.7+, Consul interprets the value placed in the namespace position as part of the service name. + The labeled annotation format allows you to reference any service as an upstream. You can specify a Consul Enterprise namespace. You can also specify an admin partition in the same datacenter, a cluster peer, or a WAN-federated datacenter. - - Admin partitions (requires Consul Enterprise 1.11+): Upstream services may be running in a different - partition. You must specify the namespace when specifying a partition. Place the partition name after the namespace. If you specify the name of the datacenter (optional), it must be the local datacenter. Communicating across partitions using this method is only supported within a - datacenter. For cross partition communication across datacenters, refer to [cluster - peering](/consul/docs/connect/cluster-peering/k8s). - ```yaml - annotations: - "consul.hashicorp.com/connect-service-upstreams":"[service-name].[service-namespace].[service-partition]:[port]:[optional datacenter]" - ``` - - [Prepared queries](/consul/api-docs/query): Prepend the annotation - with `prepared_query` and place the name of the query at the beginning of the string. - ```yaml - annotations: - 'consul.hashicorp.com/connect-service-upstreams': 'prepared_query:[query name]:[port]' - ``` - - - Labeled (requires Consul for Kubernetes v0.45.0+): - The labeled format is required when using the cluster peering feature and specifying an upstream in another - peer. You can specify a Consul Enterprise namespace, partition, or datacenter. The format supports only one peer, datacenter, or partition. - Service name: Place the service name at the beginning of the annotation followed by `.svc` to specify the upstream service. + ```yaml annotations: "consul.hashicorp.com/connect-service-upstreams":"[service-name].svc:[port]" ``` + - Peer or datacenter: Place the peer or datacenter after `svc.` followed by either `peer` or `dc` and the port number. + ```yaml annotations: "consul.hashicorp.com/connect-service-upstreams":"[service-name].svc.[service-peer].peer:[port]" ``` + ```yaml annotations: "consul.hashicorp.com/connect-service-upstreams":"[service-name].svc.[service-dc].dc:[port]" ``` - - Namespace (required Consul Enterprise): Place the namespace after `svc.` followed by `ns` and the port number. + + - Namespace (requires Consul Enterprise): Place the namespace after `svc.` followed by `ns` and the port number. + ```yaml annotations: "consul.hashicorp.com/connect-service-upstreams":"[service-name].svc.[service-namespace].ns:[port]" ``` - When specifying a peer, datacenter, or admin partition when namespaces are enabled, you must - provide the namespace . + + When namespaces are enabled, you must include the namespace in the annotation before specifying a cluster peer, WAN-federated datacenter, or admin partition in the same datacenter. + ```yaml annotations: "consul.hashicorp.com/connect-service-upstreams":"[service-name].svc.[service-namespace].ns.[service-peer].peer:[port]" ``` + ```yaml annotations: "consul.hashicorp.com/connect-service-upstreams":"[service-name].svc.[service-namespace].ns.[service-partition].ap:[port]" ``` + ```yaml annotations: "consul.hashicorp.com/connect-service-upstreams":"[service-name].svc.[service-namespace].ns.[service-dc].dc:[port]" ``` - - Multiple upstreams: Delimit multiple services or upstreams with commas. You can specify any of the unlabeled, labeled, or prepared query formats when using the supported versions for the formats. + - **Unlabeled**: + The unlabeled annotation format allows you to reference any service not in a cluster peer as an upstream. You can specify a Consul Enterprise namespace. You can also specify an admin partition in the same datacenter or a WAN-federated datacenter. Unlike the labeled annotation, you can also reference a prepared query as an upstream. + + - Service name: Place the service name at the beginning of the annotation to specify the upstream service. You also have the option to append the WAN federated datacenter where the service is deployed. + + - Namespace: Upstream services may be running in a different namespace. Place + the upstream namespace after the service name. For additional details about configuring the injector, refer to [Consul Enterprise namespaces](#consul-enterprise-namespaces) . + + ```yaml + annotations: + "consul.hashicorp.com/connect-service-upstreams":"[service-name].[service-namespace]:[port]:[optional datacenter]" + ``` + + If the namespace is not specified, the annotation defaults to the namespace of the source service. + Consul Enterprise v1.7 and older interprets the value placed in the namespace position as part of the service name. + + - Admin partitions: Upstream services may be running in a different + partition. When specifying a partition, you must also specify a namespace. Place the partition name after the namespace. If you specify the name of the datacenter, it must be the local datacenter. Communicating across partitions using this method is only supported within a + datacenter. For cross partition communication across datacenters, [establish a cluster + peering connection](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering) and set the upstream with a labeled annotation format. + + ```yaml + annotations: + "consul.hashicorp.com/connect-service-upstreams":"[service-name].[service-namespace].[service-partition]:[port]:[optional datacenter]" + ``` + + - Prepared queries: To reference a [prepared query](/consul/api-docs/query) in an upstream annotation, prepend the annotation + with `prepared_query` and then invoke the name of the query. + + ```yaml + annotations: + 'consul.hashicorp.com/connect-service-upstreams': 'prepared_query:[query name]:[port]' + ``` + + - **Multiple upstreams**: Delimit multiple services or upstreams with commas. You can specify any of the unlabeled, labeled, or prepared query formats when using the supported versions for the formats. ```yaml annotations: @@ -236,8 +251,8 @@ The following Kubernetes resource annotations could be used on a pod to control - `consul.hashicorp.com/prometheus-key-file` - Local filesystem path to a private key file for Envoy to use when serving TLS on the Prometheus metrics endpoint. Only applicable when `envoy_prometheus_bind_addr` is set in proxy config. -- `consul.hashicorp.com/service-metrics-port` - Set the port where the Connect service exposes metrics. -- `consul.hashicorp.com/service-metrics-path` - Set the path where the Connect service exposes metrics. +- `consul.hashicorp.com/service-metrics-port` - Set the port where the mesh service exposes metrics. +- `consul.hashicorp.com/service-metrics-path` - Set the path where the mesh service exposes metrics. - `consul.hashicorp.com/connect-inject-mount-volume` - Comma separated list of container names to mount the connect-inject volume into. The volume will be mounted at `/consul/connect-inject`. The connect-inject volume contains Consul internals data needed by the other sidecar containers, for example the `consul` binary, and the Pod's Consul ACL token. This data can be valuable for advanced use-cases, such as making requests to the Consul API from within application containers. - `consul.hashicorp.com/consul-sidecar-user-volume` - JSON objects as specified by the [Volume pod spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#volume-v1-core), that define volumes to add to the Envoy sidecar. ```yaml @@ -250,7 +265,7 @@ The following Kubernetes resource annotations could be used on a pod to control "consul.hashicorp.com/consul-sidecar-user-volume-mount": "[{\"name\": \"secrets-store-mount\", \"mountPath\": \"/mnt/secrets-store\"}]" ``` -## Labels +### Labels Resource labels could be used on a Kubernetes service to control connect-inject behavior. @@ -261,3 +276,45 @@ Resource labels could be used on a Kubernetes service to control connect-inject registration to ignore all services except for the one which should be used for routing requests using Consul. +## Service Sync + +### Annotations + +The following Kubernetes resource annotations could be used on a pod to [Service Sync](https://developer.hashicorp.com/consul/docs/k8s/service-sync) behavior: + +- `consul.hashicorp.com/service-sync`: If this is set to `true`, then the Kubernetes service is explicitly configured to be synced to Consul. + + ```yaml + annotations: + 'consul.hashicorp.com/service-sync': 'true' + ``` + +- `consul.hashicorp.com/service-port`: Configures the port to register to the Consul Catalog for the Kubernetes service. The annotation value may be a name of a port (recommended) or an exact port value. Refer to [service ports](https://developer.hashicorp.com/consul/docs/k8s/service-sync#service-ports) for more information. + + ```yaml + annotations: + 'consul.hashicorp.com/service-port': 'http' + ``` + +- `consul.hashicorp.com/service-tags`: A comma separated list of strings (without whitespace) to use for registering tags to the service registered to Consul. These custom tags automatically include the `k8s` tag which can't be disabled. + + ```yaml + annotations: + 'consul.hashicorp.com/service-tags': 'primary,foo' + ``` + +- `consul.hashicorp.com/service-meta-KEY`: A map for specifying service metadata for Consul services. The "KEY" below can be set to any key. This allows you to set multiple meta values. + + ```yaml + annotations: + 'consul.hashicorp.com/service-meta-KEY': 'value' + ``` + +- `consul.hashicorp.com/service-weight:` - Configures ability to support weighted loadbalancing by service annotation for Catalog Sync. The integer provided will be applied as a weight for the `passing` state for the health of the service. Refer to [weights](/consul/docs/services/configuration/services-configuration-reference#weights) in service configuration for more information on how this is leveraged for services in the Consul catalog. + + ```yaml + annotations: + consul.hashicorp.com/service-weight: 10 + ``` + + diff --git a/website/content/docs/k8s/compatibility.mdx b/website/content/docs/k8s/compatibility.mdx index 60f71bdcb85..5f2de78ba1c 100644 --- a/website/content/docs/k8s/compatibility.mdx +++ b/website/content/docs/k8s/compatibility.mdx @@ -11,13 +11,19 @@ For every release of Consul on Kubernetes, a Helm chart, `consul-k8s-control-pla ## Supported Consul and Kubernetes versions -Consul Kubernetes versions all of its components (`consul-k8s` CLI, `consul-k8s-control-plane`, and Helm chart) with a single semantic version. When installing or upgrading to a specific versions, ensure that you are using the correct Consul version with the compatible `consul-k8s` helm chart and/or CLI. +Consul Kubernetes versions all of its components (`consul-k8s` CLI, `consul-k8s-control-plane`, and Helm chart) with a single semantic version. When installing or upgrading to a specific versions, ensure that you are using the correct Consul version with the compatible Helm chart or `consul-k8s` CLI. -| Consul Version | Compatible consul-k8s Versions | Compatible Kubernetes Versions | +| Consul version | Compatible `consul-k8s` versions | Compatible Kubernetes versions | | -------------- | -------------------------------- | -------------------------------| +| 1.15.x | 1.1.x | 1.23.x - 1.26.x | | 1.14.x | 1.0.x | 1.22.x - 1.25.x | | 1.13.x | 0.49.x | 1.21.x - 1.24.x | -| 1.12.x | 0.43.0 - 0.49.x | 1.19.x - 1.22.x | + +### Version-specific upgrade requirements + +As of Consul v1.14.0, Kubernetes deployments use [Consul Dataplane](/consul/docs/connect/dataplane) instead of client agents. If you upgrade Consul from a version that uses client agents to a version that uses dataplanes, you must follow specific steps to update your Helm chart and remove client agents from the existing deployment. Refer to [Upgrading to Consul Dataplane](/consul/docs/k8s/upgrade#upgrading-to-consul-dataplane) for more information. + +The v1.0.0 release of the Consul on Kubernetes Helm chart also introduced a change to the [`externalServers[].hosts` parameter](/consul/docs/k8s/helm#v-externalservers-hosts). Previously, you were able to enter a provider lookup as a string in this field. Now, you must include `exec=` at the start of a string containing a provider lookup. Otherwise, the string is treated as a DNS name. Refer to the [`go-netaddrs` library and command line tool](https://github.com/hashicorp/go-netaddrs) for more information. ## Supported Envoy versions diff --git a/website/content/docs/k8s/connect/cluster-peering/tech-specs.mdx b/website/content/docs/k8s/connect/cluster-peering/tech-specs.mdx new file mode 100644 index 00000000000..cfe4ba7aebc --- /dev/null +++ b/website/content/docs/k8s/connect/cluster-peering/tech-specs.mdx @@ -0,0 +1,169 @@ +--- +layout: docs +page_title: Cluster Peering on Kubernetes Technical Specifications +description: >- + In Kubernetes deployments, cluster peering connections interact with mesh gateways, exported services, and ACLs. Learn about requirements specific to k8s, including required Helm values and custom resource definitions (CRDs). +--- + +# Cluster peering on Kubernetes technical specifications + +This reference topic describes the technical specifications associated with using cluster peering in your Kubernetes deployments. These specifications include [required Helm values](#helm-requirements) and [required custom resource definitions (CRDs)](#crd-requirements), as well as required Consul components and their configurations. To learn more about Consul's cluster peering feature, refer to [cluster peering overview](/consul/docs/connect/cluster-peering). + +For cluster peering requirements in non-Kubernetes deployments, refer to [cluster peering technical specifications](/consul/docs/connect/cluster-peering/tech-specs). + +## General requirements + +Make sure your Consul environment meets the following prerequisites: + +- Consul v1.14 or higher +- Consul on Kubernetes v1.0.0 or higher +- At least two Kubernetes clusters + +You must also configure the following service mesh components in order to establish cluster peering connections: + +- [Helm](#helm-requirements) +- [Custom resource definitions (CRD)](#crd-requirements) +- [Mesh gateways](#mesh-gateway-requirements) +- [Exported services](#exported-service-requirements) +- [ACLs](#acl-requirements) + +## Helm specifications + +Consul's default configuration supports cluster peering connections directly between clusters. In production environments, we recommend using mesh gateways to securely route service mesh traffic between partitions with cluster peering connections. The following values must be set in the Helm chart to enable mesh gateways: + +- [`global.tls.enabled = true`](/consul/docs/k8s/helm#v-global-tls-enabled) +- [`meshGateway.enabled = true`](/consul/docs/k8s/helm#v-meshgateway-enabled) + +Refer to the following example Helm configuration: + + + +```yaml +global: + name: consul + image: "hashicorp/consul:1.14.1" + peering: + enabled: true + tls: + enabled: true +meshGateway: + enabled: true +``` + + + +After mesh gateways are enabled in the Helm chart, you can separately [configure Mesh CRDs](#mesh-gateway-configuration-for-kubernetes). + +## CRD specifications + +You must create the following CRDs in order to establish a peering connection: + +- `PeeringAcceptor`: Generates a peering token and accepts an incoming peering connection. +- `PeeringDialer`: Uses a peering token to make an outbound peering connection with the cluster that generated the token. + +Refer to the following example CRDs: + + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: PeeringAcceptor +metadata: + name: cluster-02 ## The name of the peer you want to connect to +spec: + peer: + secret: + name: "peering-token" + key: "data" + backend: "kubernetes" +``` + + + + + +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: PeeringDialer +metadata: + name: cluster-01 ## The name of the peer you want to connect to +spec: + peer: + secret: + name: "peering-token" + key: "data" + backend: "kubernetes" +``` + + + + +## Mesh gateway specifications + +To change Consul's default configuration and enable cluster peering through mesh gateways, use a mesh configuration entry to update your network's service mesh proxies globally: + +1. In `cluster-01` create the `Mesh` custom resource with `peeringThroughMeshGateways` set to `true`. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: Mesh + metadata: + name: mesh + spec: + peering: + peerThroughMeshGateways: true + ``` + + + +1. Apply the mesh CRD to `cluster-01`. + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT apply -f mesh.yaml + ``` + +1. Apply the mesh CRD to `cluster-02`. + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply -f mesh.yaml + ``` + + + + For help setting up the cluster context variables used in this example, refer to [assign cluster IDs to environmental variables](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering#assign-cluster-ids-to-environmental-variables). + + + +When cluster peering through mesh gateways, consider the following deployment requirements: + +- A Consul cluster requires a registered mesh gateway in order to export services to peers in other regions or cloud providers. +- The mesh gateway must also be registered in the same admin partition as the exported services and their `exported-services` configuration entry. An enterprise license is required to use multiple admin partitions with a single cluster of Consul servers. +- To use the `local` mesh gateway mode, you must register a mesh gateway in the importing cluster. +- Define the `Proxy.Config` settings using opaque parameters compatible with your proxy. For additional Envoy proxy configuration information, refer to [Gateway options](/consul/docs/connect/proxies/envoy#gateway-options) and [Escape-hatch overrides](/consul/docs/connect/proxies/envoy#escape-hatch-overrides). + +### Mesh gateway modes + +By default, all cluster peering connections use mesh gateways in [remote mode](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters#remote). Be aware of these additional requirements when changing a mesh gateway's mode. + +- For mesh gateways that connect peered clusters, you can set the `mode` as either `remote` or `local`. +- The `none` mode is invalid for mesh gateways with cluster peering connections. + +To learn how to change the mesh gateway mode to `local` on your Kubernetes deployment, refer to [configure the mesh gateway mode for traffic between services](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering#configure-the-mesh-gateway-mode-for-traffic-between-services). + +## Exported service specifications + +The `exported-services` CRD is required in order for services to communicate across partitions with cluster peering connections. Basic guidance on using the `exported-services` configuration entry is included in [Establish cluster peering connections](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering#export-services-between-clusters). + +Refer to [`exported-services` configuration entry](/consul/docs/connect/config-entries/exported-services) for more information. + +## ACL specifications + +If ACLs are enabled, you must add tokens to grant the following permissions: + +- Grant `service:write` permissions to services that define mesh gateways in their server definition. +- Grant `service:read` permissions for all services on the partition. +- Grant `mesh:write` permissions to the mesh gateways that participate in cluster peering connections. This permission allows a leaf certificate to be issued for mesh gateways to terminate TLS sessions for HTTP requests. \ No newline at end of file diff --git a/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx b/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx new file mode 100644 index 00000000000..19e504b95d6 --- /dev/null +++ b/website/content/docs/k8s/connect/cluster-peering/usage/establish-peering.mdx @@ -0,0 +1,453 @@ +--- +layout: docs +page_title: Establish Cluster Peering Connections on Kubernetes +description: >- + To establish a cluster peering connection on Kubernetes, generate a peering token to establish communication. Then export services and authorize requests with service intentions. +--- + +# Establish cluster peering connections on Kubernetes + +This page details the process for establishing a cluster peering connection between services in a Consul on Kubernetes deployment. + +The overall process for establishing a cluster peering connection consists of the following steps: + +1. Create a peering token in one cluster. +1. Use the peering token to establish peering with a second cluster. +1. Export services between clusters. +1. Create intentions to authorize services for peers. + +Cluster peering between services cannot be established until all four steps are complete. + +For general guidance for establishing cluster peering connections, refer to [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/establish-cluster-peering). + +## Prerequisites + +You must meet the following requirements to use Consul's cluster peering features with Kubernetes: + +- Consul v1.14.1 or higher +- Consul on Kubernetes v1.0.0 or higher +- At least two Kubernetes clusters + +In Consul on Kubernetes, peers identify each other using the `metadata.name` values you establish when creating the `PeeringAcceptor` and `PeeringDialer` CRDs. For additional information about requirements for cluster peering on Kubernetes deployments, refer to [Cluster peering on Kubernetes technical specifications](/consul/docs/k8s/connect/cluster-peering/tech-specs). + +### Assign cluster IDs to environmental variables + +After you provision a Kubernetes cluster and set up your kubeconfig file to manage access to multiple Kubernetes clusters, you can assign your clusters to environmental variables for future use. + +1. Get the context names for your Kubernetes clusters using one of these methods: + + - Run the `kubectl config current-context` command to get the context for the cluster you are currently in. + - Run the `kubectl config get-contexts` command to get all configured contexts in your kubeconfig file. + +1. Use the `kubectl` command to export the Kubernetes context names and then set them to variables. For more information on how to use kubeconfig and contexts, refer to the [Kubernetes docs on configuring access to multiple clusters](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/). + + ```shell-session + $ export CLUSTER1_CONTEXT= + $ export CLUSTER2_CONTEXT= + ``` + +### Update the Helm chart + +To use cluster peering with Consul on Kubernetes deployments, update the Helm chart with [the required values](/consul/docs/k8s/connect/cluster-peering/tech-specs#helm-requirements). After updating the Helm chart, you can use the `consul-k8s` CLI to apply `values.yaml` to each cluster. + +1. In `cluster-01`, run the following commands: + + ```shell-session + $ export HELM_RELEASE_NAME1=cluster-01 + ``` + + ```shell-session + $ helm install ${HELM_RELEASE_NAME1} hashicorp/consul --create-namespace --namespace consul --version "1.0.1" --values values.yaml --set global.datacenter=dc1 --kube-context $CLUSTER1_CONTEXT + ``` + +1. In `cluster-02`, run the following commands: + + ```shell-session + $ export HELM_RELEASE_NAME2=cluster-02 + ``` + + ```shell-session + $ helm install ${HELM_RELEASE_NAME2} hashicorp/consul --create-namespace --namespace consul --version "1.0.1" --values values.yaml --set global.datacenter=dc2 --kube-context $CLUSTER2_CONTEXT + ``` + +### Configure the mesh gateway mode for traffic between services + +In Kubernetes deployments, you can configure mesh gateways to use `local` mode so that a service dialing a service in a remote peer dials the remote mesh gateway instead of the local mesh gateway. To configure the mesh gateway mode so that this traffic always leaves through the local mesh gateway, you can use the `ProxyDefaults` CRD. + +1. In `cluster-01` apply the following `ProxyDefaults` CRD to configure the mesh gateway mode. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ProxyDefaults + metadata: + name: global + spec: + meshGateway: + mode: local + ``` + + + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT apply -f proxy-defaults.yaml + ``` + +1. In `cluster-02` apply the following `ProxyDefaults` CRD to configure the mesh gateway mode. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ProxyDefaults + metadata: + name: global + spec: + meshGateway: + mode: local + ``` + + + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply -f proxy-defaults.yaml + ``` + +## Create a peering token + +To begin the cluster peering process, generate a peering token in one of your clusters. The other cluster uses this token to establish the peering connection. + +Every time you generate a peering token, a single-use secret for establishing the secret is embedded in the token. Because regenerating a peering token invalidates the previously generated secret, you must use the most recently created token to establish peering connections. + +1. In `cluster-01`, create the `PeeringAcceptor` custom resource. To ensure cluster peering connections are secure, the `metadata.name` field cannot be duplicated. Refer to the peer by a specific name. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: PeeringAcceptor + metadata: + name: cluster-02 ## The name of the peer you want to connect to + spec: + peer: + secret: + name: "peering-token" + key: "data" + backend: "kubernetes" + ``` + + + +1. Apply the `PeeringAcceptor` resource to the first cluster. + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT apply --filename acceptor.yaml + ``` + +1. Save your peering token so that you can export it to the other cluster. + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT get secret peering-token --output yaml > peering-token.yaml + ``` + +## Establish a connection between clusters + +Next, use the peering token to establish a secure connection between the clusters. + +1. Apply the peering token to the second cluster. + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply --filename peering-token.yaml + ``` + +1. In `cluster-02`, create the `PeeringDialer` custom resource. To ensure cluster peering connections are secure, the `metadata.name` field cannot be duplicated. Refer to the peer by a specific name. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: PeeringDialer + metadata: + name: cluster-01 ## The name of the peer you want to connect to + spec: + peer: + secret: + name: "peering-token" + key: "data" + backend: "kubernetes" + ``` + + + +1. Apply the `PeeringDialer` resource to the second cluster. + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply --filename dialer.yaml + ``` + +## Export services between clusters + +After you establish a connection between the clusters, you need to create an `exported-services` CRD that defines the services that are available to another admin partition. + +While the CRD can target admin partitions either locally or remotely, clusters peering always exports services to remote admin partitions. Refer to [exported service consumers](/consul/docs/connect/config-entries/exported-services#consumers-1) for more information. + + +1. For the service in `cluster-02` that you want to export, add the `"consul.hashicorp.com/connect-inject": "true"` annotation to your service's pods prior to deploying. The annotation allows the workload to join the mesh. It is highlighted in the following example: + + + + ```yaml + # Service to expose backend + apiVersion: v1 + kind: Service + metadata: + name: backend + spec: + selector: + app: backend + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 9090 + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: backend + --- + # Deployment for backend + apiVersion: apps/v1 + kind: Deployment + metadata: + name: backend + labels: + app: backend + spec: + replicas: 1 + selector: + matchLabels: + app: backend + template: + metadata: + labels: + app: backend + annotations: + "consul.hashicorp.com/connect-inject": "true" + spec: + serviceAccountName: backend + containers: + - name: backend + image: nicholasjackson/fake-service:v0.22.4 + ports: + - containerPort: 9090 + env: + - name: "LISTEN_ADDR" + value: "0.0.0.0:9090" + - name: "NAME" + value: "backend" + - name: "MESSAGE" + value: "Response from backend" + ``` + + + +1. Deploy the `backend` service to the second cluster. + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply --filename backend.yaml + ``` + +1. In `cluster-02`, create an `ExportedServices` custom resource. The name of the peer that consumes the service should be identical to the name set in the `PeeringDialer` CRD. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ExportedServices + metadata: + name: default ## The name of the partition containing the service + spec: + services: + - name: backend ## The name of the service you want to export + consumers: + - peer: cluster-01 ## The name of the peer that receives the service + ``` + + + +1. Apply the `ExportedServices` resource to the second cluster. + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply --filename exported-service.yaml + ``` + +## Authorize services for peers + +Before you can call services from peered clusters, you must set service intentions that authorize those clusters to use specific services. Consul prevents services from being exported to unauthorized clusters. + +1. Create service intentions for the second cluster. The name of the peer should match the name set in the `PeeringDialer` CRD. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceIntentions + metadata: + name: backend-deny + spec: + destination: + name: backend + sources: + - name: "*" + action: deny + - name: frontend + action: allow + peer: cluster-01 ## The peer of the source service + ``` + + + +1. Apply the intentions to the second cluster. + + + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT apply --filename intention.yaml + ``` + + + +1. Add the `"consul.hashicorp.com/connect-inject": "true"` annotation to your service's pods before deploying the workload so that the services in `cluster-01` can dial `backend` in `cluster-02`. To dial the upstream service from an application, configure the application so that that requests are sent to the correct DNS name as specified in [Service Virtual IP Lookups](/consul/docs/services/discovery/dns-static-lookups#service-virtual-ip-lookups). In the following example, the annotation that allows the workload to join the mesh and the configuration provided to the workload that enables the workload to dial the upstream service using the correct DNS name is highlighted. [Service Virtual IP Lookups for Consul Enterprise](/consul/docs/services/discovery/dns-static-lookups#service-virtual-ip-lookups-for-consul-enterprise) details how you would similarly format a DNS name including partitions and namespaces. + + + + ```yaml + # Service to expose frontend + apiVersion: v1 + kind: Service + metadata: + name: frontend + spec: + selector: + app: frontend + ports: + - name: http + protocol: TCP + port: 9090 + targetPort: 9090 + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: frontend + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: frontend + labels: + app: frontend + spec: + replicas: 1 + selector: + matchLabels: + app: frontend + template: + metadata: + labels: + app: frontend + annotations: + "consul.hashicorp.com/connect-inject": "true" + spec: + serviceAccountName: frontend + containers: + - name: frontend + image: nicholasjackson/fake-service:v0.22.4 + securityContext: + capabilities: + add: ["NET_ADMIN"] + ports: + - containerPort: 9090 + env: + - name: "LISTEN_ADDR" + value: "0.0.0.0:9090" + - name: "UPSTREAM_URIS" + value: "http://backend.virtual.cluster-02.consul" + - name: "NAME" + value: "frontend" + - name: "MESSAGE" + value: "Hello World" + - name: "HTTP_CLIENT_KEEP_ALIVES" + value: "false" + ``` + + + +1. Apply the service file to the first cluster. + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT apply --filename frontend.yaml + ``` + +1. Run the following command in `frontend` and then check the output to confirm that you peered your clusters successfully. + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT exec -it $(kubectl --context $CLUSTER1_CONTEXT get pod -l app=frontend -o name) -- curl localhost:9090 + ``` + + + + ```json + { + "name": "frontend", + "uri": "/", + "type": "HTTP", + "ip_addresses": [ + "10.16.2.11" + ], + "start_time": "2022-08-26T23:40:01.167199", + "end_time": "2022-08-26T23:40:01.226951", + "duration": "59.752279ms", + "body": "Hello World", + "upstream_calls": { + "http://backend.virtual.cluster-02.consul": { + "name": "backend", + "uri": "http://backend.virtual.cluster-02.consul", + "type": "HTTP", + "ip_addresses": [ + "10.32.2.10" + ], + "start_time": "2022-08-26T23:40:01.223503", + "end_time": "2022-08-26T23:40:01.224653", + "duration": "1.149666ms", + "headers": { + "Content-Length": "266", + "Content-Type": "text/plain; charset=utf-8", + "Date": "Fri, 26 Aug 2022 23:40:01 GMT" + }, + "body": "Response from backend", + "code": 200 + } + }, + "code": 200 + } + ``` + + + +### Authorize service reads with ACLs + +If ACLs are enabled on a Consul cluster, sidecar proxies that access exported services as an upstream must have an ACL token that grants read access. + +Read access to all imported services is granted using either of the following rules associated with an ACL token: + +- `service:write` permissions for any service in the sidecar's partition. +- `service:read` and `node:read` for all services and nodes, respectively, in sidecar's namespace and partition. + +For Consul Enterprise, the permissions apply to all imported services in the service's partition. These permissions are satisfied when using a [service identity](/consul/docs/security/acl/acl-roles#service-identities). + +Refer to [Reading servers](/consul/docs/connect/config-entries/exported-services#reading-services) in the `exported-services` configuration entry documentation for example rules. + +For additional information about how to configure and use ACLs, refer to [ACLs system overview](/consul/docs/security/acl). \ No newline at end of file diff --git a/website/content/docs/k8s/connect/cluster-peering/usage/l7-traffic.mdx b/website/content/docs/k8s/connect/cluster-peering/usage/l7-traffic.mdx new file mode 100644 index 00000000000..956298fe3cd --- /dev/null +++ b/website/content/docs/k8s/connect/cluster-peering/usage/l7-traffic.mdx @@ -0,0 +1,75 @@ +--- +layout: docs +page_title: Manage L7 Traffic With Cluster Peering on Kubernetes +description: >- + Combine service resolver configurations with splitter and router configurations to manage L7 traffic in Consul on Kubernetes deployments with cluster peering connections. Learn how to define dynamic traffic rules to target peers for redirects in k8s. +--- + +# Manage L7 traffic with cluster peering on Kubernetes + +This usage topic describes how to configure the `service-resolver` custom resource definition (CRD) to set up and manage L7 traffic between services that have an existing cluster peering connection in Consul on Kubernetes deployments. + +For general guidance for managing L7 traffic with cluster peering, refer to [Manage L7 traffic with cluster peering](/consul/docs/connect/cluster-peering/usage/peering-traffic-management). + +## Service resolvers for redirects and failover + +When you use cluster peering to connect datacenters through their admin partitions, you can use [dynamic traffic management](/consul/docs/connect/l7-traffic) to configure your service mesh so that services automatically forward traffic to services hosted on peer clusters. + +However, the `service-splitter` and `service-router` CRDs do not natively support directly targeting a service instance hosted on a peer. Before you can split or route traffic to a service on a peer, you must define the service hosted on the peer as an upstream service by configuring a failover in a `service-resolver` CRD. Then, you can set up a redirect in a second service resolver to interact with the peer service by name. + +For more information about formatting, updating, and managing configuration entries in Consul, refer to [How to use configuration entries](/consul/docs/agent/config-entries). + +## Configure dynamic traffic between peers + +To configure L7 traffic management behavior in deployments with cluster peering connections, complete the following steps in order: + +1. Define the peer cluster as a failover target in the service resolver configuration. + + The following example updates the [`service-resolver` CRD](/consul/docs/connect/config-entries/) in `cluster-01` so that Consul redirects traffic intended for the `frontend` service to a backup instance in peer `cluster-02` when it detects multiple connection failures. + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceResolver + metadata: + name: frontend + spec: + connectTimeout: 15s + failover: + '*': + targets: + - peer: 'cluster-02' + service: 'frontend' + namespace: 'default' + ``` + +1. Define the desired behavior in `service-splitter` or `service-router` CRD. + + The following example splits traffic evenly between `frontend` and `frontend-peer`: + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceSplitter + metadata: + name: frontend + spec: + splits: + - weight: 50 + ## defaults to service with same name as configuration entry ("frontend") + - weight: 50 + service: frontend-peer + ``` + +1. Create a second `service-resolver` configuration entry on the local cluster that resolves the name of the peer service you used when splitting or routing the traffic. + + The following example uses the name `frontend-peer` to define a redirect targeting the `frontend` service on the peer `cluster-02`: + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: ServiceResolver + metadata: + name: frontend-peer + spec: + redirect: + peer: 'cluster-02' + service: 'frontend' + ``` \ No newline at end of file diff --git a/website/content/docs/k8s/connect/cluster-peering/usage/manage-peering.mdx b/website/content/docs/k8s/connect/cluster-peering/usage/manage-peering.mdx new file mode 100644 index 00000000000..bc622fe15d3 --- /dev/null +++ b/website/content/docs/k8s/connect/cluster-peering/usage/manage-peering.mdx @@ -0,0 +1,121 @@ +--- +layout: docs +page_title: Manage Cluster Peering Connections on Kubernetes +description: >- + Learn how to list, read, and delete cluster peering connections using Consul on Kubernetes. You can also reset cluster peering connections on k8s deployments. +--- + +# Manage cluster peering connections on Kubernetes + +This usage topic describes how to manage cluster peering connections on Kubernetes deployments. + +After you establish a cluster peering connection, you can get a list of all active peering connections, read a specific peering connection's information, and delete peering connections. + +For general guidance for managing cluster peering connections, refer to [Manage L7 traffic with cluster peering](/consul/docs/connect/cluster-peering/usage/peering-traffic-management). + +## Reset a peering connection + +To reset the cluster peering connection, you need to generate a new peering token from the cluster where you created the `PeeringAcceptor` CRD. The only way to create or set a new peering token is to manually adjust the value of the annotation `consul.hashicorp.com/peering-version`. Creating a new token causes the previous token to expire. + +1. In the `PeeringAcceptor` CRD, add the annotation `consul.hashicorp.com/peering-version`. If the annotation already exists, update its value to a higher version. + + + + ```yaml + apiVersion: consul.hashicorp.com/v1alpha1 + kind: PeeringAcceptor + metadata: + name: cluster-02 + annotations: + consul.hashicorp.com/peering-version: "1" ## The peering version you want to set, must be in quotes + spec: + peer: + secret: + name: "peering-token" + key: "data" + backend: "kubernetes" + ``` + + + +1. After updating `PeeringAcceptor`, repeat all of the steps to [establish a new peering connection](/consul/docs/k8s/connect/cluster-peering/usage/establish-peering). + +## List all peering connections + +In Consul on Kubernetes deployments, you can list all active peering connections in a cluster using the Consul CLI. + +1. If necessary, [configure your CLI to interact with the Consul cluster](/consul/tutorials/get-started-kubernetes/kubernetes-gs-deploy#configure-your-cli-to-interact-with-consul-cluster). + +1. Run the [`consul peering list` CLI command](/consul/commands/peering/list). + + ```shell-session + $ consul peering list + Name State Imported Svcs Exported Svcs Meta + cluster-02 ACTIVE 0 2 env=production + cluster-03 PENDING 0 0 + ``` + +## Read a peering connection + +In Consul on Kubernetes deployments, you can get information about individual peering connections between clusters using the Consul CLI. + +1. If necessary, [configure your CLI to interact with the Consul cluster](/consul/tutorials/get-started-kubernetes/kubernetes-gs-deploy#configure-your-cli-to-interact-with-consul-cluster). + +1. Run the [`consul peering read` CLI command](/consul/commands/peering/read). + + ```shell-session + $ consul peering read -name cluster-02 + Name: cluster-02 + ID: 3b001063-8079-b1a6-764c-738af5a39a97 + State: ACTIVE + Meta: + env=production + + Peer ID: e83a315c-027e-bcb1-7c0c-a46650904a05 + Peer Server Name: server.dc1.consul + Peer CA Pems: 0 + Peer Server Addresses: + 10.0.0.1:8300 + + Imported Services: 0 + Exported Services: 2 + + Create Index: 89 + Modify Index: 89 + ``` + +## Delete peering connections + +To end a peering connection in Kubernetes deployments, delete both the `PeeringAcceptor` and `PeeringDialer` resources. + +1. Delete the `PeeringDialer` resource from the second cluster. + + ```shell-session + $ kubectl --context $CLUSTER2_CONTEXT delete --filename dialer.yaml + ``` + +1. Delete the `PeeringAcceptor` resource from the first cluster. + + ```shell-session + $ kubectl --context $CLUSTER1_CONTEXT delete --filename acceptor.yaml + ```` + +To confirm that you deleted your peering connection in `cluster-01`, query the the `/health` HTTP endpoint: + +1. Exec into the server pod for the first cluster. + + ```shell-session + $ kubectl exec -it consul-server-0 --context $CLUSTER1_CONTEXT -- /bin/sh + ``` + +1. If you've enabled ACLs, export an ACL token to access the `/health` HTP endpoint for services. The bootstrap token may be used if an ACL token is not already provisioned. + + ```shell-session + $ export CONSUL_HTTP_TOKEN= + ``` + +1. Query the the `/health` HTTP endpoint. Peered services with deleted connections should no longe appear. + + ```shell-session + $ curl "localhost:8500/v1/health/connect/backend?peer=cluster-02" + ``` \ No newline at end of file diff --git a/website/content/docs/k8s/connect/connect-ca-provider.mdx b/website/content/docs/k8s/connect/connect-ca-provider.mdx index 1699da1dab1..cdab172bcc2 100644 --- a/website/content/docs/k8s/connect/connect-ca-provider.mdx +++ b/website/content/docs/k8s/connect/connect-ca-provider.mdx @@ -1,13 +1,13 @@ --- layout: docs -page_title: Configure Certificate Authority (CA) for Consul on Kubernetes +page_title: Configure Service Mesh Certificate Authority (CA) for Consul on Kubernetes description: >- Consul includes a built-in CA, but when bootstrapping a cluster on k8s, you can configure your service mesh to use a custom certificate provider instead. Learn how to configure Vault as an external CA in primary and secondary datacenters and manually rotate Vault tokens. --- -# Configure Certificate Authority for Consul on Kubernetes +# Configure Service Mesh Certificate Authority for Consul on Kubernetes -If `connect` is enabled, the built-in Consul certificate authority (CA) is automatically enabled for the service mesh CA. You can use different CA providers with Consul service mesh. Refer to [Connect Certificate Management](/consul/docs/connect/ca) for supported providers. +If `connect` is enabled, the built-in Consul CA provider is automatically enabled for the service mesh certificate authority (CA). You can use different CA providers with Consul service mesh. Refer to [Service Mesh Certificate Management](/consul/docs/connect/ca) for supported providers. ## Overview @@ -21,20 +21,20 @@ To configure an external CA provider using the Consul Helm chart, complete the f To configure the Vault service mesh provider, refer to [Vault as the Service Mesh Certificate Provider on Kubernetes](/consul/docs/k8s/deployment-configurations/vault/data-integration/connect-ca). -## Configuring Vault as a Connect CA (Consul K8s 0.37.0 and earlier) +## Configuring Vault as a Service Mesh CA (Consul K8s 0.37.0 and earlier) -If you use Vault 1.11.0+ as Consul's Connect CA, versions of Consul released before Dec 13, 2022 will develop an issue with Consul control plane or service mesh communication ([GH-15525](https://github.com/hashicorp/consul/pull/15525)). Use or upgrade to a [Consul version that includes the fix](https://support.hashicorp.com/hc/en-us/articles/11308460105491#01GMC24E6PPGXMRX8DMT4HZYTW) to avoid this problem. +If you use Vault 1.11.0+ as Consul's service mesh CA, versions of Consul released before Dec 13, 2022 will develop an issue with Consul control plane or service mesh communication ([GH-15525](https://github.com/hashicorp/consul/pull/15525)). Use or upgrade to a [Consul version that includes the fix](https://support.hashicorp.com/hc/en-us/articles/11308460105491#01GMC24E6PPGXMRX8DMT4HZYTW) to avoid this problem. -The following instructions are only valid for Consul K8s CLI 0.37.0 and prior. It describes how to configure Vault as the Connect CA. You can configure other providers during initial bootstrap of the cluster by providing the appropriate [`ca_config`] and [`ca_provider`] values for your provider. +The following instructions are only valid for Consul K8s CLI 0.37.0 and prior. It describes how to configure Vault as the service mesh CA. You can configure other providers during initial bootstrap of the cluster by providing the appropriate [`ca_config`] and [`ca_provider`] values for your provider. --> **Auto-renewal:** If using Vault as your Connect CA, we strongly recommend Consul 1.8.5 or later, which includes support for token auto-renewal. If the Vault token is [renewable](/vault/api-docs/auth/token#renewable), then Consul automatically renews the token periodically. Otherwise, you must [manually rotate](#manually-rotating-vault-tokens) the Vault token before it expires. +-> **Auto-renewal:** If using Vault as your service mesh CA, we strongly recommend Consul 1.8.5 or later, which includes support for token auto-renewal. If the Vault token is [renewable](/vault/api-docs/auth/token#renewable), then Consul automatically renews the token periodically. Otherwise, you must [manually rotate](#manually-rotating-vault-tokens) the Vault token before it expires. ### Primary Datacenter -To configure Vault as a CA provider for Consul Connect, +To configure Vault as a CA provider for Consul service mesh, first, create a provider configuration JSON file. -Please refer to [Vault as a Connect CA](/consul/docs/connect/ca/vault) for the configuration options. +Please refer to [Vault as a service mesh CA](/consul/docs/connect/ca/vault) for the configuration options. You will need to provide a Vault token to the `token` property. Please refer to [these docs](/consul/docs/connect/ca/vault#token) for the permissions that the token needs to have. This token should be [renewable](/vault/api-docs/auth/token#renewable). @@ -142,7 +142,7 @@ $ kubectl exec consul-server-0 -- curl --silent http://localhost:8500/v1/connect ### Secondary Datacenters -To configure Vault as the Connect CA in secondary datacenters, you need to make sure that the Root CA is the same, +To configure Vault as the service mesh CA in secondary datacenters, you need to make sure that the Root CA is the same, but the intermediate is different for each datacenter. In the `connect` configuration for a secondary datacenter, you can specify a `intermediate_pki_path` that is, for example, prefixed with the datacenter for which this configuration is intended. diff --git a/website/content/docs/k8s/connect/index.mdx b/website/content/docs/k8s/connect/index.mdx index 7e30ab2be89..11acc590d4e 100644 --- a/website/content/docs/k8s/connect/index.mdx +++ b/website/content/docs/k8s/connect/index.mdx @@ -7,10 +7,15 @@ description: >- # How does Consul Service Mesh Work on Kubernetes? -[Consul Service Mesh](/consul/docs/connect) is a feature built into to Consul that enables +[Consul service mesh](/consul/docs/connect) is a feature built into to Consul that enables automatic service-to-service authorization and connection encryption across your Consul services. Consul Service Mesh can be used with Kubernetes to secure pod -communication with other pods and external Kubernetes services. _Consul Connect_ refers to the component in Consul that enables service mesh functionality. We sometimes use Connect to mean Consul service mesh throughout the documentation. +communication with other pods and external Kubernetes services. + +The noun _connect_ is used throughout this documentation to refer to the connect +subsystem that provides Consul's service mesh capabilities. +Where you encounter the _noun_ connect, it is usually functionality specific to +service mesh. Consul can automatically inject the sidecar running Envoy into pods in your cluster, making configuration for Kubernetes automatic. @@ -26,6 +31,30 @@ service is required to run services on the Consul service mesh. Installing Consul on Kubernetes with [`connect-inject` enabled](/consul/docs/k8s/connect#installation-and-configuration) adds a sidecar to all pods. By default, it enables service mesh functionality with Consul Dataplane by injecting an Envoy proxy. You can also configure Consul to inject a client agent sidecar to connect to your service mesh. Refer to [Simplified Service Mesh with Consul Dataplane](/consul/docs/connect/dataplane) for more information. +### Service names + +When the service is onboarded, the name registered in Consul is set to the name of the Kubernetes Service associated with the Pod. You can use the [`consul.hashicorp.com/connect-service` annotation](/consul/docs/k8s/annotations-and-labels#consul-hashicorp-com-connect-service) to specify a custom name for the service, but if ACLs are enabled then the name of the service registered in Consul must match the Pod's `ServiceAccount` name. + +### Transparent proxy mode + +By default, the Consul service mesh runs in transparent proxy mode. This mode forces inbound and outbound traffic through the sidecar proxy even though the service binds to all interfaces. Transparent proxy infers the location of upstream services using Consul service intentions, and also allows you to use Kubernetes DNS as you normally would for your workloads. + +When transparent proxy mode is enabled, all service-to-service traffic is required to use mTLS. When onboarding new services to service mesh, your network may have mixed mTLS and non-mTLS traffic, which can result in broken service-to-service communication. You can temporarily enable permissive mTLS mode during the onboarding process so that existing mesh services can accept traffic from services that are not yet fully onboarded. Permissive mTLS enables sidecar proxies to access both mTLS and non-mTLS traffic. Refer to [Onboard mesh services in transparent proxy mode](/consul/docs/k8s/connect/onboarding-tproxy-mode) for additional information. + +### Kubernetes service mesh workload scenarios + +-> **Note:** A Kubernetes Service is required in order to register services on the Consul service mesh. Consul monitors the lifecyle of the Kubernetes Service and its service instances using the service object. In addition, the Kubernetes service is used to register and de-register the service from Consul's catalog. + +The following configurations are examples for registering workloads on Kubernetes into Consul's service mesh in different scenarios. Each scenario provides an example Kubernetes manifest to demonstrate how to use Consul's service mesh with a specific Kubernetes workload type. + + - [Kubernetes Pods running as a deployment](#kubernetes-pods-running-as-a-deployment) + - [Connecting to mesh-enabled Services](#connecting-to-mesh-enabled-services) + - [Kubernetes Jobs](#kubernetes-jobs) + - [Kubernetes Pods with multiple ports](#kubernetes-pods-with-multiple-ports) + +#### Kubernetes Pods running as a deployment + + ```yaml apiVersion: v1 @@ -76,7 +105,9 @@ spec: serviceAccountName: static-server ``` -The only change for Connect is the addition of the + + +The only change for service mesh is the addition of the `consul.hashicorp.com/connect-inject` annotation. This enables injection for the Pod in this Deployment. The injector can also be [configured](/consul/docs/k8s/connect#installation-and-configuration) @@ -89,12 +120,12 @@ the _pod specification template_ as shown above. This will start a sidecar proxy that listens on port `20000` registered with Consul and proxies valid inbound connections to port 8080 in the pod. -To establish a connection to the pod using Connect, a client must use another Connect -proxy. The client Connect proxy will use Consul service discovery to find +To establish a connection to the pod using service mesh, a client must use another mesh +proxy. The client mesh proxy will use Consul service discovery to find all available upstream proxies and their public ports. In the example above, the server is listening on `:8080`. -By default, the Consul Service Mesh runs in [transparent proxy](/consul/docs/connect/transparent-proxy) mode. +By default, the Consul service mesh runs in [transparent proxy](/consul/docs/connect/transparent-proxy) mode. This means that even though the server binds to all interfaces, the inbound and outbound connections will automatically go through to the sidecar proxy. It also allows you to use Kubernetes DNS like you normally would without the @@ -106,17 +137,18 @@ proxy will enforce all inbound and outbound traffic to go through the Envoy prox The service name registered in Consul will be set to the name of the Kubernetes service associated with the Pod. This can be customized with the `consul.hashicorp.com/connect-service` -annotation. If using ACLs, this name must be the same as the Pod's `ServiceAccount` name. +annotation. If using ACLs, this name must be the same as the Pod's `ServiceAccount` name.a -### Connecting to Connect-Enabled Services +To establish a connection to the upstream Pod using service mesh, a client must dial the upstream workload using a mesh proxy. The client mesh proxy will use Consul service discovery to find all available upstream proxies and their public ports. + +### Connecting to mesh-enabled Services The example Deployment specification below configures a Deployment that is capable of establishing connections to our previous example "static-server" service. The connection to this static text service happens over an authorized and encrypted -connection via Connect. +connection via service mesh. --> **Note:** As of consul-k8s `v0.26.0` and Consul Helm `v0.32.0`, having a Kubernetes -Service is **required** to run services on the Consul Service Mesh. + ```yaml apiVersion: v1 @@ -162,6 +194,8 @@ spec: serviceAccountName: static-client ``` + + By default when ACLs are enabled or when ACLs default policy is `allow`, Consul will automatically configure proxies with all upstreams from the same datacenter. When ACLs are enabled with default `deny` policy, @@ -173,20 +207,20 @@ the injector will also set environment variables `_CONNECT_SERVICE_HOST` and `_CONNECT_SERVICE_PORT` in every container in the Pod for every defined upstream. This is analogous to the standard Kubernetes service environment variables, but point instead to the correct local proxy port to establish connections via -Connect. +service mesh. -We can verify access to the static text server using `kubectl exec`. +You can verify access to the static text server using `kubectl exec`. Because transparent proxy is enabled by default, -we use Kubernetes DNS to connect to our desired upstream. +use Kubernetes DNS to connect to your desired upstream. ```shell-session $ kubectl exec deploy/static-client -- curl --silent http://static-server/ "hello world" ``` -We can control access to the server using [intentions](/consul/docs/connect/intentions). +You can control access to the server using [intentions](/consul/docs/connect/intentions). If you use the Consul UI or [CLI](/consul/commands/intention/create) to -create a deny [intention](/consul/docs/connect/intentions) between +deny communication between "static-client" and "static-server", connections are immediately rejected without updating either of the running pods. You can then remove this intention to allow connections again. @@ -196,7 +230,95 @@ $ kubectl exec deploy/static-client -- curl --silent http://static-server/ command terminated with exit code 52 ``` -### Kubernetes Pods with Multiple ports +#### Kubernetes Jobs + +Kubernetes Jobs run pods that only make outbound requests to services on the mesh and successfully terminate when they are complete. In order to register a Kubernetes Job with the mesh, you must provide an integer value for the `consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds` annotation. Then, issue a request to the `http://127.0.0.1:20600/graceful_shutdown` API endpoint so that Kubernetes gracefully shuts down the `consul-dataplane` sidecar after the job is complete. + +Below is an example Kubernetes manifest that deploys a job correctly. + + + +```yaml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-job + namespace: default +--- +apiVersion: v1 +kind: Service +metadata: + name: test-job + namespace: default +spec: + selector: + app: test-job + ports: + - port: 80 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-job + namespace: default + labels: + app: test-job +spec: + template: + metadata: + annotations: + 'consul.hashicorp.com/connect-inject': 'true' + 'consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds': '5' + labels: + app: test-job + spec: + containers: + - name: test-job + image: alpine/curl:3.14 + ports: + - containerPort: 80 + command: + - /bin/sh + - -c + - | + echo "Started test job" + sleep 10 + echo "Killing proxy" + curl --max-time 2 -s -f -XPOST http://127.0.0.1:20600/graceful_shutdown + sleep 10 + echo "Ended test job" + serviceAccountName: test-job + restartPolicy: Never +``` + + + +Upon completing the job you should be able to verify that all containers are shut down within the pod. + +```shell-session +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +test-job-49st7 0/2 Completed 0 3m55s +``` + +```shell-session +$ kubectl get job +NAME COMPLETIONS DURATION AGE +test-job 1/1 30s 4m31s +``` + +In addition, based on the logs emitted by the pod you can verify that the proxy was shut down before the Job completed. + +```shell-session +$ kubectl logs test-job-49st7 -c test-job +Started test job +Killing proxy +Ended test job +``` + +#### Kubernetes Pods with multiple ports + To configure a pod with multiple ports to be a part of the service mesh and receive and send service mesh traffic, you will need to add configuration so that a Consul service can be registered per port. This is because services in Consul currently support a single port per service instance. @@ -208,6 +330,9 @@ First, decide on the names for the two Consul services that will correspond to t chooses the names `web` for `8080` and `web-admin` for `9090`. Create two service accounts for `web` and `web-admin`: + + + ```yaml apiVersion: v1 kind: ServiceAccount @@ -219,7 +344,14 @@ kind: ServiceAccount metadata: name: web-admin ``` + + + + Create two Service objects for `web` and `web-admin`: + + + ```yaml apiVersion: v1 kind: Service @@ -245,12 +377,17 @@ spec: port: 80 targetPort: 9090 ``` + + + `web` will target `containerPort` `8080` and select pods labeled `app: web`. `web-admin` will target `containerPort` `9090` and will also select the same pods. ~> Kubernetes 1.24+ only In Kubernetes 1.24+ you need to [create a Kubernetes secret](https://kubernetes.io/docs/concepts/configuration/secret/#service-account-token-secrets) for each multi-port service that references the ServiceAccount, and the Kubernetes secret must have the same name as the ServiceAccount: + + ```yaml apiVersion: v1 kind: Secret @@ -269,12 +406,15 @@ metadata: type: kubernetes.io/service-account-token ``` + + Create a Deployment with any chosen name, and use the following annotations: ```yaml -consul.hashicorp.com/connect-inject: true -consul.hashicorp.com/transparent-proxy: false -consul.hashicorp.com/connect-service: web,web-admin -consul.hashicorp.com/connect-service-port: 8080,9090 +annotations: + 'consul.hashicorp.com/connect-inject': 'true' + 'consul.hashicorp.com/transparent-proxy': 'false' + 'consul.hashicorp.com/connect-service': 'web,web-admin' + 'consul.hashicorp.com/connect-service-port': '8080,9090' ``` Note that the order the ports are listed in the same order as the service names, i.e. the first service name `web` corresponds to the first port, `8080`, and the second service name `web-admin` corresponds to the second port, `9090`. @@ -284,7 +424,10 @@ The service account on the pod spec for the deployment should be set to the firs serviceAccountName: web ``` -For reference, the full deployment example could look something like the following: +The following deployment example demonstrates the required annotations for the manifest. In addition, the previous YAML manifests can also be combined into a single manifest for easier deployment. + + + ```yaml apiVersion: apps/v1 kind: Deployment @@ -326,13 +469,61 @@ spec: serviceAccountName: web ``` + + After deploying the `web` application, you can test service mesh connections by deploying the `static-client` -application with the configuration in the [previous section](#connecting-to-connect-enabled-services) and add the -following annotation to the pod template on `static-client`: +application with the configuration in the [previous section](#connecting-to-mesh-enabled-services) and add the +`consul.hashicorp.com/connect-service-upstreams: 'web:1234,web-admin:2234'` annotation to the pod template on `static-client`: + + + ```yaml -consul.hashicorp.com/connect-service-upstreams: "web:1234,web-admin:2234" +apiVersion: v1 +kind: Service +metadata: + # This name will be the service name in Consul. + name: static-client +spec: + selector: + app: static-client + ports: + - port: 80 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: static-client +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-client +spec: + replicas: 1 + selector: + matchLabels: + app: static-client + template: + metadata: + name: static-client + labels: + app: static-client + annotations: + 'consul.hashicorp.com/connect-inject': 'true' + 'consul.hashicorp.com/connect-service-upstreams': 'web:1234,web-admin:2234' + spec: + containers: + - name: static-client + image: curlimages/curl:latest + # Just spin & wait forever, we'll use `kubectl exec` to demo + command: ['/bin/sh', '-c', '--'] + args: ['while true; do sleep 30; done;'] + # If ACLs are enabled, the serviceAccountName must match the Consul service name. + serviceAccountName: static-client ``` + + If you exec on to a static-client pod, using a command like: ```shell-session $ kubectl exec -it static-client-5bd667fbd6-kk6xs -- /bin/sh @@ -366,15 +557,15 @@ the Pod. So the upstream configuration can use the individual service names to r ## Installation and Configuration -The Connect sidecar proxy is injected via a +The service mesh sidecar proxy is injected via a [mutating admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#admission-webhooks) -provided by the +call the connect injector provided by the [consul-k8s project](https://github.com/hashicorp/consul-k8s). This enables the automatic pod mutation shown in the usage section above. Installation of the mutating admission webhook is automated using the [Helm chart](/consul/docs/k8s/installation/install). -To install the Connect injector, enable the Connect injection feature using +To install the connect injector, enable the connect injection feature using [Helm values](/consul/docs/k8s/helm#configuration-values) and upgrade the installation using `helm upgrade` for existing installs or `helm install` for a fresh install. @@ -395,7 +586,7 @@ To verify the installation, run the ["Accepting Inbound Connections"](/consul/docs/k8s/connect#accepting-inbound-connections) example from the "Usage" section above. After running this example, run `kubectl get pod static-server --output yaml`. In the raw YAML output, you should -see injected Connect containers and an annotation +see connect injected containers and an annotation `consul.hashicorp.com/connect-inject-status` set to `injected`. This confirms that injection is working properly. diff --git a/website/content/docs/k8s/connect/ingress-gateways.mdx b/website/content/docs/k8s/connect/ingress-gateways.mdx index 1d2072ae3a0..eb618035e35 100644 --- a/website/content/docs/k8s/connect/ingress-gateways.mdx +++ b/website/content/docs/k8s/connect/ingress-gateways.mdx @@ -11,7 +11,7 @@ description: >- ~> This topic requires familiarity with [Ingress Gateways](/consul/docs/connect/gateways/ingress-gateway). -This page describes how to enable external access to Connect Service Mesh services running inside Kubernetes using Consul ingress gateways. +This page describes how to enable external access through Consul ingress gateways to mesh services running inside Kubernetes. See [Ingress Gateways](/consul/docs/connect/gateways/ingress-gateway) for more information on use-cases and how it works. Adding an ingress gateway is a multi-step process that consists of the following steps: diff --git a/website/content/docs/k8s/connect/observability/metrics.mdx b/website/content/docs/k8s/connect/observability/metrics.mdx index c8db578268f..167f2c7613c 100644 --- a/website/content/docs/k8s/connect/observability/metrics.mdx +++ b/website/content/docs/k8s/connect/observability/metrics.mdx @@ -10,14 +10,14 @@ description: >- Consul on Kubernetes integrates with Prometheus and Grafana to provide metrics for Consul service mesh. The metrics available are: -- Connect service metrics -- Sidecar proxy metrics +- Mesh service metrics +- Mesh sidecar proxy metrics - Consul agent metrics - Ingress, terminating, and mesh gateway metrics Specific sidecar proxy metrics can also be seen in the Consul UI Topology Visualization view. This section documents how to enable each of these. -## Connect Service and Sidecar Metrics with Metrics Merging +## Mesh Service and Sidecar Metrics with Metrics Merging Prometheus annotations are used to instruct Prometheus to scrape metrics from Pods. Prometheus annotations only support scraping from one endpoint on a Pod, so Consul on Kubernetes supports metrics merging whereby service metrics and @@ -30,7 +30,7 @@ The diagram below illustrates how the metrics integration works when merging is [![Metrics Merging Architecture](/img/metrics-arch.png)](/img/metrics-arch.png) --> -Connect service metrics can be configured with the Helm values nested under [`connectInject.metrics`](/consul/docs/k8s/helm#v-connectinject-metrics). +Metrics for services in the mesh can be configured with the Helm values nested under [`connectInject.metrics`](/consul/docs/k8s/helm#v-connectinject-metrics). Metrics and metrics merging can be enabled by default for all connect-injected Pods with the following Helm values: diff --git a/website/content/docs/k8s/connect/terminating-gateways.mdx b/website/content/docs/k8s/connect/terminating-gateways.mdx index 1da48d0a542..319fedf71dd 100644 --- a/website/content/docs/k8s/connect/terminating-gateways.mdx +++ b/website/content/docs/k8s/connect/terminating-gateways.mdx @@ -40,7 +40,7 @@ terminatingGateways: The Helm chart may be deployed using the [Consul on Kubernetes CLI](/consul/docs/k8s/k8s-cli). ```shell-session -$ consul-k8s install -f values.yaml +$ consul-k8s install --config-file values.yaml ``` ## Accessing the Consul agent @@ -52,7 +52,7 @@ You can access the Consul server directly from your host by running `kubectl por ```shell-session -$ kubectl port-forward consul-server-0 8500 & +$ kubectl port-forward service/consul-server 8500 & ``` ```shell-session @@ -65,7 +65,7 @@ $ export CONSUL_HTTP_ADDR=http://localhost:8500 If TLS is enabled use port 8501: ```shell-session -$ kubectl port-forward consul-server-0 8501 & +$ kubectl port-forward service/consul-server 8501 & ``` ```shell-session @@ -102,6 +102,7 @@ you may register the service as a node in the Consul catalog. The [`destination`](/consul/docs/connect/config-entries/service-defaults#terminating-gateway-destination) field of the `ServiceDefaults` Custom Resource Definition (CRD) allows clients to dial an external service directly. For this method to work, [`TransparentProxy`](/consul/docs/connect/transparent-proxy) must be enabled. + The following table describes traffic behaviors when using the `destination` field to route traffic through a terminating gateway: | External Services Layer | Client dials | Client uses TLS | Allowed | Notes | @@ -205,7 +206,7 @@ true ### Update terminating gateway ACL role if ACLs are enabled -If ACLs are enabled, update the terminating gateway acl role to have `service: write` permissions on all of the services +If ACLs are enabled, update the terminating gateway ACL role to have `service:write` permissions on all of the services being represented by the gateway. Create a new policy that includes the write permission for the service you created. @@ -232,15 +233,14 @@ service "example-https" { } ``` -Fetch the ID of the terminating gateway token. +Obtain the ID of the terminating gateway role. ```shell-session -consul acl role list | grep -B 6 -- "- RELEASE_NAME-terminating-gateway-policy" | grep ID - -ID: +$ consul acl role list -format=json | jq --raw-output '[.[] | select(.Name | endswith("-terminating-gateway-acl-role"))] | if (. | length) == 1 then (. | first | .ID) else "Unable to determine the role ID because there are multiple roles matching this name.\n" | halt_error end' + ``` -Update the terminating gateway ACL token with the new policy. +Update the terminating gateway ACL role with the new policy. ```shell-session $ consul acl role update -id -policy-name example-https-write-policy @@ -379,7 +379,7 @@ deployment "static-client" successfully rolled out You can verify connectivity of the static-client and terminating gateway via a curl command. - + ```shell-session $ kubectl exec deploy/static-client -- curl -vvvs https://example.com/ diff --git a/website/content/docs/k8s/crds/index.mdx b/website/content/docs/k8s/crds/index.mdx index 342fe1aaa4c..6a68960a04d 100644 --- a/website/content/docs/k8s/crds/index.mdx +++ b/website/content/docs/k8s/crds/index.mdx @@ -16,8 +16,8 @@ You can specify the following values in the `kind` field. Click on a configurati - [`Mesh`](/consul/docs/connect/config-entries/mesh) - [`ExportedServices`](/consul/docs/connect/config-entries/exported-services) -- [`PeeringAcceptor`](/consul/docs/connect/cluster-peering/k8s#peeringacceptor) -- [`PeeringDialer`](/consul/docs/connect/cluster-peering/k8s#peeringdialer) +- [`PeeringAcceptor`](/consul/docs/k8s/connect/cluster-peering/tech-specs#peeringacceptor) +- [`PeeringDialer`](/consul/docs/k8s/connect/cluster-peering/tech-specs#peeringdialer) - [`ProxyDefaults`](/consul/docs/connect/config-entries/proxy-defaults) - [`ServiceDefaults`](/consul/docs/connect/config-entries/service-defaults) - [`ServiceSplitter`](/consul/docs/connect/config-entries/service-splitter) diff --git a/website/content/docs/k8s/crds/upgrade-to-crds.mdx b/website/content/docs/k8s/crds/upgrade-to-crds.mdx index 755b50f203a..0c09102a33e 100644 --- a/website/content/docs/k8s/crds/upgrade-to-crds.mdx +++ b/website/content/docs/k8s/crds/upgrade-to-crds.mdx @@ -14,7 +14,7 @@ you utilize the following: - [`connectInject.centralConfig.defaultProtocol`](#default-protocol) - [`connectInject.centralConfig.proxyDefaults`](#proxy-defaults) - [`meshGateway.globalMode`](#mesh-gateway-mode) -- [Connect annotation `consul.hashicorp.com/connect-service-protocol`](#connect-service-protocol-annotation) +- [connect annotation `consul.hashicorp.com/connect-service-protocol`](#connect-service-protocol-annotation) ## Central Config Enabled @@ -133,7 +133,7 @@ installation. ## connect-service-protocol Annotation -If any of your Connect services had the `consul.hashicorp.com/connect-service-protocol` +If any of your mesh services had the `consul.hashicorp.com/connect-service-protocol` annotation set, e.g. ```yaml diff --git a/website/content/docs/k8s/deployment-configurations/consul-enterprise.mdx b/website/content/docs/k8s/deployment-configurations/consul-enterprise.mdx index 48c4db1fa0c..29314c94354 100644 --- a/website/content/docs/k8s/deployment-configurations/consul-enterprise.mdx +++ b/website/content/docs/k8s/deployment-configurations/consul-enterprise.mdx @@ -11,7 +11,7 @@ You can use this Helm chart to deploy Consul Enterprise by following a few extra Find the license file that you received in your welcome email. It should have a `.hclic` extension. You will use the contents of this file to create a Kubernetes secret before installing the Helm chart. --> **Note:** This guide assumes you are storing your license as a Kubernetes Secret. If you would like to store the enterprise license in Vault, please reference [Storing the Enterprise License in Vault](/consul/docs/k8s/deployment-configuration/vault/data-integration/enterprise-license). +-> **Note:** This guide assumes you are storing your license as a Kubernetes Secret. If you would like to store the enterprise license in Vault, please reference [Storing the Enterprise License in Vault](/consul/docs/k8s/deployment-configurations/vault/data-integration/enterprise-license). You can use the following commands to create the secret with name `consul-ent-license` and key `key`: diff --git a/website/content/docs/k8s/deployment-configurations/multi-cluster/kubernetes.mdx b/website/content/docs/k8s/deployment-configurations/multi-cluster/kubernetes.mdx index 46aa7979e59..e407c7bedea 100644 --- a/website/content/docs/k8s/deployment-configurations/multi-cluster/kubernetes.mdx +++ b/website/content/docs/k8s/deployment-configurations/multi-cluster/kubernetes.mdx @@ -22,7 +22,7 @@ Consul treats each Kubernetes cluster as a separate Consul datacenter. In order to federate clusters, one cluster must be designated the primary datacenter. This datacenter will be responsible for creating the certificate authority that signs the TLS certificates -Connect uses to encrypt and authorize traffic. It also handles validating global ACL tokens. All other clusters +that Consul service mesh uses to encrypt and authorize traffic. It also handles validating global ACL tokens. All other clusters that are federated are considered secondaries. #### First Time Installation diff --git a/website/content/docs/k8s/deployment-configurations/servers-outside-kubernetes.mdx b/website/content/docs/k8s/deployment-configurations/servers-outside-kubernetes.mdx index 93c98d7f317..a794f3643d5 100644 --- a/website/content/docs/k8s/deployment-configurations/servers-outside-kubernetes.mdx +++ b/website/content/docs/k8s/deployment-configurations/servers-outside-kubernetes.mdx @@ -1,11 +1,11 @@ --- layout: docs -page_title: Join External Servers to Consul on Kubernetes +page_title: Join Kubernetes Clusters to external Consul Servers description: >- - Client agents that run on Kubernetes pods can join existing clusters whose server agents run outside of k8s. Learn how to expose gossip ports and bootstrap ACLs by configuring the Helm chart. + Kubernetes clusters can be joined to existing Consul clusters in a much simpler way with the introduction of Consul Dataplane. Learn how to add Kubernetes Clusters into an existing Consul cluster and bootstrap ACLs by configuring the Helm chart. --- -# Join External Servers to Consul on Kubernetes +# Join Kubernetes Clusters to external Consul Servers If you have a Consul cluster already running, you can configure your Consul on Kubernetes installation to join this existing cluster. @@ -14,9 +14,7 @@ The below `values.yaml` file shows how to configure the Helm chart to install Consul so that it joins an existing Consul server cluster. The `global.enabled` value first disables all chart components by default -so that each component is opt-in. This allows us to _only_ setup the client -agents. We then opt-in to the client agents by setting `client.enabled` to -`true`. +so that each component is opt-in. Next, configure `externalServers` to point it to Consul servers. The `externalServers.hosts` value must be provided and should be set to a DNS, an IP, @@ -37,8 +35,10 @@ externalServers: - **Note:** To join Consul on Kubernetes to an existing Consul server cluster running outside of Kubernetes, -refer to [Consul servers outside of Kubernetes](/consul/docs/k8s/deployment-configurations/servers-outside-kubernetes). +With the introduction of [Consul Dataplane](/consul/docs/connect/dataplane#what-is-consul-dataplane), Consul installation on Kubernetes is simplified by removing the Consul Client agents. +This requires the Helm installation and rest of the consul-k8s components installed on Kubernetes to talk to Consul Servers directly on various ports. +Before starting the installation, ensure that the Consul Servers are configured to have the gRPC port enabled `8502/tcp` using the [`ports.grpc = 8502`](/consul/docs/agent/config/config-files#grpc) configuration option. + ## Configuring TLS @@ -68,7 +68,7 @@ externalServers: If your HTTPS port is different from Consul's default `8501`, you must also set -`externalServers.httpsPort`. +`externalServers.httpsPort`. If the Consul servers are not running TLS enabled, use this config to set the HTTP port the servers are configured with (default `8500`). ## Configuring ACLs @@ -109,11 +109,15 @@ The bootstrap token requires the following minimal permissions: - `agent:read` if using WAN federation over mesh gateways Next, configure external servers. The Helm chart will use this configuration to talk to the Consul server's API -to create policies, tokens, and an auth method. If you are [enabling Consul Connect](/consul/docs/k8s/connect), +to create policies, tokens, and an auth method. If you are [enabling Consul service mesh](/consul/docs/k8s/connect), `k8sAuthMethodHost` should be set to the address of your Kubernetes API server so that the Consul servers can validate a Kubernetes service account token when using the [Kubernetes auth method](/consul/docs/security/acl/auth-methods/kubernetes) with `consul login`. +-> **Note:** If `externalServers.k8sAuthMethodHost` is set and you are also using WAN federation +(`global.federation.enabled` is set to `true`), ensure that `global.federation.k8sAuthMethodHost` is set to the same +value as `externalServers.k8sAuthMethodHost`. + ```yaml diff --git a/website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx b/website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx index b9694b39125..4f23ccddabe 100644 --- a/website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx +++ b/website/content/docs/k8s/deployment-configurations/single-dc-multi-k8s.mdx @@ -15,7 +15,8 @@ This example uses two Kubernetes clusters, but this approach could be extended t ## Requirements -* Consul-Helm version `v1.0.0` or higher +* `consul-k8s` v1.0.x or higher, and Consul 1.14.x or higher +* Kubernetes clusters must be able to communicate over LAN on a flat network. * Either the Helm release name for each Kubernetes cluster must be unique, or `global.name` for each Kubernetes cluster must be unique to prevent collisions of ACL resources with the same prefix. ## Prepare Helm release name ahead of installs @@ -50,6 +51,17 @@ global: gossipEncryption: secretName: consul-gossip-encryption-key secretKey: key +server: + exposeService: + enabled: true + type: NodePort + nodePort: + ## all are random nodePorts and you can set your own + http: 30010 + https: 30011 + serf: 30012 + rpc: 30013 + grpc: 30014 ui: service: type: NodePort @@ -65,6 +77,8 @@ The UI's service type is set to be `NodePort`. This is needed to connect to servers from another cluster without using the pod IPs of the servers, which are likely going to change. +Other services are exposed as `NodePort` services and configured with random port numbers. In this example, the `grpc` port is set to `30014`, which enables services to discover Consul servers using gRPC when connecting from another cluster. + To deploy, first generate the Gossip encryption key and save it as a Kubernetes secret. ```shell-session @@ -123,6 +137,8 @@ externalServers: hosts: ["10.0.0.4"] # The node port of the UI's NodePort service or the load balancer port. httpsPort: 31557 + # Matches the gRPC port of the Consul servers in the first cluster. + grpcPort: 30014 tlsServerName: server.dc1.consul # The address of the kube API server of this Kubernetes cluster k8sAuthMethodHost: https://kubernetes.example.com:443 @@ -147,6 +163,8 @@ NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE cluster1-consul-ui NodePort 10.0.240.80 443:31557/TCP 40h ``` +The `grpcPort: 30014` configuration refers to the gRPC port number specified in the `NodePort` configuration in the first cluster. + Set the `externalServer.tlsServerName` to `server.dc1.consul`. This the DNS SAN (Subject Alternative Name) that is present in the Consul server's certificate. This is required because the connection to the Consul servers uses the node IP, diff --git a/website/content/docs/k8s/deployment-configurations/vault/data-integration/connect-ca.mdx b/website/content/docs/k8s/deployment-configurations/vault/data-integration/connect-ca.mdx index 9cc6af4136d..7550eb44420 100644 --- a/website/content/docs/k8s/deployment-configurations/vault/data-integration/connect-ca.mdx +++ b/website/content/docs/k8s/deployment-configurations/vault/data-integration/connect-ca.mdx @@ -10,10 +10,10 @@ description: >- This topic describes how to configure the Consul Helm chart to use TLS certificates issued by Vault for Consul service mesh communication. -> **Note:** This feature requires Consul 1.11 or higher. As of v1.11, -Consul allows using Kubernetes auth methods to configure Connect CA. +Consul allows using Kubernetes auth methods to configure the service mesh CA. This allows for automatic token rotation once the renewal is no longer possible. -~> **Compatibility note:** If you use Vault 1.11.0+ as Consul's Connect CA, versions of Consul released before Dec 13, 2022 will develop an issue with Consul control plane or service mesh communication ([GH-15525](https://github.com/hashicorp/consul/pull/15525)). Use or upgrade to a [Consul version that includes the fix](https://support.hashicorp.com/hc/en-us/articles/11308460105491#01GMC24E6PPGXMRX8DMT4HZYTW) to avoid this problem. +~> **Compatibility note:** If you use Vault 1.11.0+ as Consul's service mesh CA, versions of Consul released before Dec 13, 2022 will develop an issue with Consul control plane or service mesh communication ([GH-15525](https://github.com/hashicorp/consul/pull/15525)). Use or upgrade to a [Consul version that includes the fix](https://support.hashicorp.com/hc/en-us/articles/11308460105491#01GMC24E6PPGXMRX8DMT4HZYTW) to avoid this problem. ## Overview @@ -46,7 +46,7 @@ Next, you will create Kubernetes auth roles for the Consul servers: $ vault write auth/kubernetes/role/consul-server \ bound_service_account_names= \ bound_service_account_namespaces= \ - policies= \ + policies= \ ttl=1h ``` @@ -58,7 +58,7 @@ $ helm template --release-name ${RELEASE_NAME} --show-only templates/server-serv ``` ## Update Consul on Kubernetes Helm chart -Now you can configure the Consul Helm chart to use Vault as the Connect CA provider: +Now you can configure the Consul Helm chart to use Vault as the service mesh (connect) CA provider: @@ -83,7 +83,7 @@ global: The `address` you provide to the `connectCA` configuration can be a Kubernetes DNS address if the Vault cluster is running the same Kubernetes cluster. The `rootPKIPath` and `intermediatePKIPath` should be the same as the ones -defined in your Connect CA policy. Behind the scenes, Consul will authenticate to Vault using a Kubernetes +defined in your service mesh CA policy. Behind the scenes, Consul will authenticate to Vault using a Kubernetes service account using the [Kubernetes auth method](/vault/docs/auth/kubernetes) and will use the Vault token for any API calls to Vault. If the Vault token can not be renewed, Consul will re-authenticate to generate a new Vault token. @@ -95,7 +95,7 @@ $ kubectl create secret generic vault-ca --from-file vault.ca=/path/to/your/vaul ### Secondary Datacenters -To configure Vault as the Connect CA in secondary datacenters, you need to make sure that the Root CA path is the same, +To configure Vault as the service mesh (connect) CA in secondary datacenters, you need to make sure that the Root CA path is the same, but the intermediate is different for each datacenter. In the `connectCA` Helm configuration for a secondary datacenter, you can specify a `intermediatePKIPath` that is, for example, prefixed with the datacenter for which this configuration is intended (e.g. `dc2/connect-intermediate`). diff --git a/website/content/docs/k8s/deployment-configurations/vault/data-integration/webhook-certs.mdx b/website/content/docs/k8s/deployment-configurations/vault/data-integration/webhook-certs.mdx index aec4fb1d69d..b615d31fabd 100644 --- a/website/content/docs/k8s/deployment-configurations/vault/data-integration/webhook-certs.mdx +++ b/website/content/docs/k8s/deployment-configurations/vault/data-integration/webhook-certs.mdx @@ -14,16 +14,16 @@ In a Consul Helm chart configuration that does not use Vault, `webhook-cert-mana When Vault is configured as the controller and connect inject Webhook Certificate Provider on Kubernetes: - `webhook-cert-manager` is no longer deployed to the cluster. - - controller and connect inject each get their webhook certificates from its own Vault PKI mount via the injected Vault Agent. - - controller and connect inject each need to be configured with its own Vault Role that has necessary permissions to receive certificates from its respective PKI mount. - - controller and connect inject each locally update its own `mutatingwebhookconfiguration` so that Kubernetes can relay events. + - Controller and connect inject each get their webhook certificates from its own Vault PKI mount via the injected Vault Agent. + - Controller and connect inject each need to be configured with its own Vault Role that has necessary permissions to receive certificates from its respective PKI mount. + - Controller and connect inject each locally update its own `mutatingwebhookconfiguration` so that Kubernetes can relay events. - Vault manages certificate rotation and rotates certificates to each webhook. To use Vault as the controller and connect inject Webhook Certificate Provider, we will need to modify the steps outlined in the [Data Integration](/consul/docs/k8s/deployment-configurations/vault/data-integration) section: These following steps will be repeated for each datacenter: 1. Create a Vault policy that authorizes the desired level of access to the secret. - 1. (Added) Create Vault PKI roles for controller and connect inject each that establish the domains that each is allowed to issue certificates for. + 1. (Added) Create Vault PKI roles for controller and connect inject that each establish the domains that each is allowed to issue certificates for. 1. Create Vault Kubernetes auth roles that link the policy to each Consul on Kubernetes service account that requires access. 1. Configure the Vault Kubernetes auth roles in the Consul on Kubernetes helm chart. @@ -74,44 +74,45 @@ Issue the following commands to enable and configure the PKI Secrets Engine to s 1. Create a policy that allows `["create", "update"]` access to the [certificate issuing URL](/vault/api-docs/secret/pki) so Consul controller and connect inject can fetch a new certificate/key pair and provide it to the Kubernetes `mutatingwebhookconfiguration`. - The path to the secret referenced in the `path` resource is the same value that you will configure in the `global.secretsBackend.vault.controller.tlsCert.secretName` and `global.secretsBackend.vault.connectInject.tlsCert.secretName` Helm configuration (refer to [Update Consul on Kubernetes Helm chart](#update-consul-on-kubernetes-helm-chart)). + The path to the secret referenced in the `path` resource is the same value that you will configure in the `global.secretsBackend.vault.controller.tlsCert.secretName` and `global.secretsBackend.vault.connectInject.tlsCert.secretName` Helm configuration (refer to [Update Consul on Kubernetes Helm chart](#update-consul-on-kubernetes-helm-chart)). - ```shell-session - $ vault policy write controller-tls-policy - < \ diff --git a/website/content/docs/k8s/deployment-configurations/vault/wan-federation.mdx b/website/content/docs/k8s/deployment-configurations/vault/wan-federation.mdx index 5c2badb0932..9eaa27cd3cf 100644 --- a/website/content/docs/k8s/deployment-configurations/vault/wan-federation.mdx +++ b/website/content/docs/k8s/deployment-configurations/vault/wan-federation.mdx @@ -609,7 +609,7 @@ Repeat the following steps for each datacenter in the cluster: 1. Configure and install Consul in the secondary datacenter (dc2). - -> **Note**: To configure Vault as the Connect CA in secondary datacenters, you need to make sure that the Root CA path is the same. The intermediate path is different for each datacenter. In the `connectCA` Helm configuration for a secondary datacenter, you can specify a `intermediatePKIPath` that is, for example, prefixed with the datacenter for which this configuration is intended (e.g. `dc2/connect-intermediate`). + -> **Note**: To configure Vault as the service mesh (connect) CA in secondary datacenters, you need to make sure that the Root CA path is the same. The intermediate path is different for each datacenter. In the `connectCA` Helm configuration for a secondary datacenter, you can specify a `intermediatePKIPath` that is, for example, prefixed with the datacenter for which this configuration is intended (e.g. `dc2/connect-intermediate`). diff --git a/website/content/docs/k8s/dns.mdx b/website/content/docs/k8s/dns.mdx index 5082919bd1d..0f34dd25078 100644 --- a/website/content/docs/k8s/dns.mdx +++ b/website/content/docs/k8s/dns.mdx @@ -8,10 +8,10 @@ description: >- # Resolve Consul DNS Requests in Kubernetes One of the primary query interfaces to Consul is the -[DNS interface](/consul/docs/discovery/dns). You can configure Consul DNS in +[DNS interface](/consul/docs/services/discovery/dns-overview). You can configure Consul DNS in Kubernetes using a [stub-domain configuration](https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#configure-stub-domain-and-upstream-dns-servers) -if using KubeDNS or a [proxy configuration](https://coredns.io/plugins/proxy/) if using CoreDNS. +if using KubeDNS or a [proxy configuration](https://coredns.io/plugins/forward/) if using CoreDNS. Once configured, DNS requests in the form `.service.consul` will resolve for services in Consul. This will work from all Kubernetes namespaces. diff --git a/website/content/docs/k8s/helm.mdx b/website/content/docs/k8s/helm.mdx index b5eb83c0df0..30f22a25cd5 100644 --- a/website/content/docs/k8s/helm.mdx +++ b/website/content/docs/k8s/helm.mdx @@ -35,6 +35,7 @@ Use these links to navigate to a particular top-level stanza. - [`webhookCertManager`](#h-webhookcertmanager) - [`prometheus`](#h-prometheus) - [`tests`](#h-tests) +- [`telemetryCollector`](#h-telemetrycollector) ## All Values @@ -58,7 +59,7 @@ Use these links to navigate to a particular top-level stanza. the prefix will be `-consul`. - `domain` ((#v-global-domain)) (`string: consul`) - The domain Consul will answer DNS queries for - (see `-domain` (https://www.consul.io/docs/agent/config/cli-flags#_domain)) and the domain services synced from + (Refer to [`-domain`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_domain)) and the domain services synced from Consul into Kubernetes will have, e.g. `service-name.service.consul`. - `peering` ((#v-global-peering)) - Configures the Cluster Peering feature. Requires Consul v1.14+ and Consul-K8s v1.0.0+. @@ -94,7 +95,7 @@ Use these links to navigate to a particular top-level stanza. - `imagePullSecrets` ((#v-global-imagepullsecrets)) (`array`) - Array of objects containing image pull secret names that will be applied to each service account. This can be used to reference image pull secrets if using a custom consul or consul-k8s-control-plane Docker image. - See https://kubernetes.io/docs/concepts/containers/images/#using-a-private-registry for reference. + Refer to https://kubernetes.io/docs/concepts/containers/images/#using-a-private-registry. Example: @@ -114,12 +115,13 @@ Use these links to navigate to a particular top-level stanza. https://github.com/hashicorp/consul/issues/1858. - `enablePodSecurityPolicies` ((#v-global-enablepodsecuritypolicies)) (`boolean: false`) - Controls whether pod security policies are created for the Consul components - created by this chart. See https://kubernetes.io/docs/concepts/policy/pod-security-policy/. + created by this chart. Refer to https://kubernetes.io/docs/concepts/policy/pod-security-policy/. - `secretsBackend` ((#v-global-secretsbackend)) - secretsBackend is used to configure Vault as the secrets backend for the Consul on Kubernetes installation. The Vault cluster needs to have the Kubernetes Auth Method, KV2 and PKI secrets engines enabled and have necessary secrets, policies and roles created prior to installing Consul. - See https://www.consul.io/docs/k8s/installation/vault for full instructions. + Refer to [Vault as the Secrets Backend](https://developer.hashicorp.com/consul/docs/k8s/deployment-configurations/vault) + documentation for full instructions. The Vault cluster _must_ not have the Consul cluster installed by this Helm chart as its storage backend as that would cause a circular dependency. @@ -177,11 +179,6 @@ Use these links to navigate to a particular top-level stanza. ``` and check the name of `metadata.name`. - - `controllerRole` ((#v-global-secretsbackend-vault-controllerrole)) (`string: ""`) - The Vault role to read Consul controller's webhook's - CA and issue a certificate and private key. - A Vault policy must be created which grants issue capabilities to - `global.secretsBackend.vault.controller.tlsCert.secretName`. - - `connectInjectRole` ((#v-global-secretsbackend-vault-connectinjectrole)) (`string: ""`) - The Vault role to read Consul connect-injector webhook's CA and issue a certificate and private key. A Vault policy must be created which grants issue capabilities to @@ -214,21 +211,21 @@ Use these links to navigate to a particular top-level stanza. The provider will be configured to use the Vault Kubernetes auth method and therefore requires the role provided by `global.secretsBackend.vault.consulServerRole` to have permissions to the root and intermediate PKI paths. - Please see https://www.consul.io/docs/connect/ca/vault#vault-acl-policies - for information on how to configure the Vault policies. + Please refer to [Vault ACL policies](https://developer.hashicorp.com/consul/docs/connect/ca/vault#vault-acl-policies) + documentation for information on how to configure the Vault policies. - `address` ((#v-global-secretsbackend-vault-connectca-address)) (`string: ""`) - The address of the Vault server. - `authMethodPath` ((#v-global-secretsbackend-vault-connectca-authmethodpath)) (`string: kubernetes`) - The mount path of the Kubernetes auth method in Vault. - `rootPKIPath` ((#v-global-secretsbackend-vault-connectca-rootpkipath)) (`string: ""`) - The path to a PKI secrets engine for the root certificate. - For more details, please refer to [Vault Connect CA configuration](https://www.consul.io/docs/connect/ca/vault#rootpkipath). + For more details, please refer to [Vault Connect CA configuration](https://developer.hashicorp.com/consul/docs/connect/ca/vault#rootpkipath). - `intermediatePKIPath` ((#v-global-secretsbackend-vault-connectca-intermediatepkipath)) (`string: ""`) - The path to a PKI secrets engine for the generated intermediate certificate. - For more details, please refer to [Vault Connect CA configuration](https://www.consul.io/docs/connect/ca/vault#intermediatepkipath). + For more details, please refer to [Vault Connect CA configuration](https://developer.hashicorp.com/consul/docs/connect/ca/vault#intermediatepkipath). - `additionalConfig` ((#v-global-secretsbackend-vault-connectca-additionalconfig)) (`string: {}`) - Additional Connect CA configuration in JSON format. - Please refer to [Vault Connect CA configuration](https://www.consul.io/docs/connect/ca/vault#configuration) + Please refer to [Vault Connect CA configuration](https://developer.hashicorp.com/consul/docs/connect/ca/vault#configuration) for all configuration options available for that provider. Example: @@ -245,22 +242,6 @@ Use these links to navigate to a particular top-level stanza. } ``` - - `controller` ((#v-global-secretsbackend-vault-controller)) - - - `tlsCert` ((#v-global-secretsbackend-vault-controller-tlscert)) - Configuration to the Vault Secret that Kubernetes will use on - Kubernetes CRD creation, deletion, and update, to get TLS certificates - used issued from vault to send webhooks to the controller. - - - `secretName` ((#v-global-secretsbackend-vault-controller-tlscert-secretname)) (`string: null`) - The Vault secret path that issues TLS certificates for controller - webhooks. - - - `caCert` ((#v-global-secretsbackend-vault-controller-cacert)) - Configuration to the Vault Secret that Kubernetes will use on - Kubernetes CRD creation, deletion, and update, to get CA certificates - used issued from vault to send webhooks to the controller. - - - `secretName` ((#v-global-secretsbackend-vault-controller-cacert-secretname)) (`string: null`) - The Vault secret path that contains the CA certificate for controller - webhooks. - - `connectInject` ((#v-global-secretsbackend-vault-connectinject)) - `caCert` ((#v-global-secretsbackend-vault-connectinject-cacert)) - Configuration to the Vault Secret that Kubernetes uses on @@ -278,7 +259,7 @@ Use these links to navigate to a particular top-level stanza. inject webhooks. - `gossipEncryption` ((#v-global-gossipencryption)) - Configures Consul's gossip encryption key. - (see `-encrypt` (https://www.consul.io/docs/agent/config/cli-flags#_encrypt)). + (Refer to [`-encrypt`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_encrypt)). By default, gossip encryption is not enabled. The gossip encryption key may be set automatically or manually. The recommended method is to automatically generate the key. To automatically generate and set a gossip encryption key, set autoGenerate to true. @@ -286,7 +267,7 @@ Use these links to navigate to a particular top-level stanza. To manually generate a gossip encryption key, set secretName and secretKey and use Consul to generate a key, saving this as a Kubernetes secret or Vault secret path and key. If `global.secretsBackend.vault.enabled=true`, be sure to add the "data" component of the secretName path as required by - the Vault KV-2 secrets engine [see example]. + the Vault KV-2 secrets engine [refer to example]. ```shell-session $ kubectl create secret generic consul-gossip-encryption-key --from-literal=key=$(consul keygen) @@ -307,12 +288,14 @@ Use these links to navigate to a particular top-level stanza. - `secretKey` ((#v-global-gossipencryption-secretkey)) (`string: ""`) - The key within the Kubernetes secret or Vault secret key that holds the gossip encryption key. + - `logLevel` ((#v-global-gossipencryption-loglevel)) (`string: ""`) - Override global log verbosity level for `gossip-encryption-autogenerate-job` pods. One of "trace", "debug", "info", "warn", or "error". + - `recursors` ((#v-global-recursors)) (`array: []`) - A list of addresses of upstream DNS servers that are used to recursively resolve DNS queries. These values are given as `-recursor` flags to Consul servers and clients. - See https://www.consul.io/docs/agent/config/cli-flags#_recursor for more details. + Refer to [`-recursor`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_recursor) for more details. If this is an empty array (the default), then Consul DNS will only resolve queries for the Consul top level domain (by default `.consul`). - - `tls` ((#v-global-tls)) - Enables TLS (https://learn.hashicorp.com/tutorials/consul/tls-encryption-secure) + - `tls` ((#v-global-tls)) - Enables [TLS](https://developer.hashicorp.com/consul/tutorials/security/tls-encryption-secure) across the cluster to verify authenticity of the Consul servers and clients. Requires Consul v1.4.1+. @@ -321,6 +304,8 @@ Use these links to navigate to a particular top-level stanza. authority (optional) and server and client certificates. This setting is required for [Cluster Peering](https://developer.hashicorp.com/consul/docs/connect/cluster-peering/k8s). + - `logLevel` ((#v-global-tls-loglevel)) (`string: ""`) - Override global log verbosity level. One of "trace", "debug", "info", "warn", or "error". + - `enableAutoEncrypt` ((#v-global-tls-enableautoencrypt)) (`boolean: false`) - If true, turns on the auto-encrypt feature on clients and servers. It also switches consul-k8s-control-plane components to retrieve the CA from the servers via the API. Requires Consul 1.7.1+. @@ -336,7 +321,7 @@ Use these links to navigate to a particular top-level stanza. - `verify` ((#v-global-tls-verify)) (`boolean: true`) - If true, `verify_outgoing`, `verify_server_hostname`, and `verify_incoming` for internal RPC communication will be set to `true` for Consul servers and clients. Set this to false to incrementally roll out TLS on an existing Consul cluster. - Please see https://consul.io/docs/k8s/operations/tls-on-existing-cluster + Please refer to [TLS on existing clusters](https://developer.hashicorp.com/consul/docs/k8s/operations/tls-on-existing-cluster) for more details. - `httpsOnly` ((#v-global-tls-httpsonly)) (`boolean: true`) - If true, the Helm chart will configure Consul to disable the HTTP port on @@ -372,13 +357,23 @@ Use these links to navigate to a particular top-level stanza. Note that we need the CA key so that we can generate server and client certificates. It is particularly important for the client certificates since they need to have host IPs - as Subject Alternative Names. In the future, we may support bringing your own server - certificates. + as Subject Alternative Names. If you are setting server certs yourself via `server.serverCert` + and you are not enabling clients (or clients are enabled with autoEncrypt) then you do not + need to provide the CA key. - `secretName` ((#v-global-tls-cakey-secretname)) (`string: null`) - The name of the Kubernetes or Vault secret that holds the CA key. - `secretKey` ((#v-global-tls-cakey-secretkey)) (`string: null`) - The key within the Kubernetes or Vault secret that holds the CA key. + - `annotations` ((#v-global-tls-annotations)) (`string: null`) - This value defines additional annotations for + tls init jobs. Format this value as a multi-line string. + + ```yaml + annotations: | + "sample/annotation1": "foo" + "sample/annotation2": "bar" + ``` + - `enableConsulNamespaces` ((#v-global-enableconsulnamespaces)) (`boolean: false`) - `enableConsulNamespaces` indicates that you are running Consul Enterprise v1.7+ with a valid Consul Enterprise license and would like to make use of configuration beyond registering everything into @@ -392,14 +387,22 @@ Use these links to navigate to a particular top-level stanza. for all Consul and consul-k8s-control-plane components. This requires Consul >= 1.4. - - `bootstrapToken` ((#v-global-acls-bootstraptoken)) - A Kubernetes or Vault secret containing the bootstrap token to use for - creating policies and tokens for all Consul and consul-k8s-control-plane components. - If set, we will skip ACL bootstrapping of the servers and will only - initialize ACLs for the Consul clients and consul-k8s-control-plane system components. + - `logLevel` ((#v-global-acls-loglevel)) (`string: ""`) - Override global log verbosity level. One of "trace", "debug", "info", "warn", or "error". + + - `bootstrapToken` ((#v-global-acls-bootstraptoken)) - A Kubernetes or Vault secret containing the bootstrap token to use for creating policies and + tokens for all Consul and consul-k8s-control-plane components. If `secretName` and `secretKey` + are unset, a default secret name and secret key are used. If the secret is populated, then + we will skip ACL bootstrapping of the servers and will only initialize ACLs for the Consul + clients and consul-k8s-control-plane system components. + If the secret is empty, then we will bootstrap ACLs on the Consul servers, and write the + bootstrap token to this secret. If ACLs are already bootstrapped on the servers, then the + secret must contain the bootstrap token. - `secretName` ((#v-global-acls-bootstraptoken-secretname)) (`string: null`) - The name of the Kubernetes or Vault secret that holds the bootstrap token. + If unset, this defaults to `{{ global.name }}-bootstrap-acl-token`. - `secretKey` ((#v-global-acls-bootstraptoken-secretkey)) (`string: null`) - The key within the Kubernetes or Vault secret that holds the bootstrap token. + If unset, this defaults to `token`. - `createReplicationToken` ((#v-global-acls-createreplicationtoken)) (`boolean: false`) - If true, an ACL token will be created that can be used in secondary datacenters for replication. This should only be set to true in the @@ -417,6 +420,23 @@ Use these links to navigate to a particular top-level stanza. - `secretKey` ((#v-global-acls-replicationtoken-secretkey)) (`string: null`) - The key within the Kubernetes or Vault secret that holds the replication token. + - `resources` ((#v-global-acls-resources)) (`map`) - The resource requests (CPU, memory, etc.) for the server-acl-init and server-acl-init-cleanup pods. + This should be a YAML map corresponding to a Kubernetes + [`ResourceRequirements``](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#resourcerequirements-v1-core) + object. + + Example: + + ```yaml + resources: + requests: + memory: '200Mi' + cpu: '100m' + limits: + memory: '200Mi' + cpu: '100m' + ``` + - `partitionToken` ((#v-global-acls-partitiontoken)) - partitionToken references a Vault secret containing the ACL token to be used in non-default partitions. This value should only be provided in the default partition and only when setting the `global.secretsBackend.vault.enabled` value to true. @@ -430,9 +450,9 @@ Use these links to navigate to a particular top-level stanza. - `tolerations` ((#v-global-acls-tolerations)) (`string: ""`) - tolerations configures the taints and tolerations for the server-acl-init and server-acl-init-cleanup jobs. This should be a multi-line string matching the - Tolerations (https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. + [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. - - `nodeSelector` ((#v-global-acls-nodeselector)) (`string: null`) - This value defines `nodeSelector` (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) + - `nodeSelector` ((#v-global-acls-nodeselector)) (`string: null`) - This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) labels for the server-acl-init and server-acl-init-cleanup jobs pod assignment, formatted as a multi-line string. Example: @@ -442,6 +462,23 @@ Use these links to navigate to a particular top-level stanza. beta.kubernetes.io/arch: amd64 ``` + - `annotations` ((#v-global-acls-annotations)) (`string: null`) - This value defines additional annotations for + acl init jobs. Format this value as a multi-line string. + + ```yaml + annotations: | + "sample/annotation1": "foo" + "sample/annotation2": "bar" + ``` + + - `argocd` ((#v-global-argocd)) - If argocd.enabled is set to true, following annotations are added to + job - server-acl-init-job + annotations - + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: HookSucceeded + + - `enabled` ((#v-global-argocd-enabled)) (`boolean: false`) + - `enterpriseLicense` ((#v-global-enterpriselicense)) - This value refers to a Kubernetes or Vault secret that you have created that contains your enterprise license. It is required if you are using an enterprise binary. Defining it here applies it to your cluster once a leader @@ -461,8 +498,9 @@ Use these links to navigate to a particular top-level stanza. - `enabled` ((#v-global-federation-enabled)) (`boolean: false`) - If enabled, this datacenter will be federation-capable. Only federation via mesh gateways is supported. Mesh gateways and servers will be configured to allow federation. - Requires `global.tls.enabled`, `meshGateway.enabled` and `connectInject.enabled` - to be true. Requires Consul 1.8+. + Requires `global.tls.enabled`, `connectInject.enabled`, and one of + `meshGateway.enabled` or `externalServers.enabled` to be true. + Requires Consul 1.8+. - `createFederationSecret` ((#v-global-federation-createfederationsecret)) (`boolean: false`) - If true, the chart will create a Kubernetes secret that can be imported into secondary datacenters so they can federate with this datacenter. The @@ -474,15 +512,18 @@ Use these links to navigate to a particular top-level stanza. - `primaryDatacenter` ((#v-global-federation-primarydatacenter)) (`string: null`) - The name of the primary datacenter. - - `primaryGateways` ((#v-global-federation-primarygateways)) (`array: []`) - A list of addresses of the primary mesh gateways in the form `:`. - (e.g. ["1.1.1.1:443", "2.3.4.5:443"] + - `primaryGateways` ((#v-global-federation-primarygateways)) (`array: []`) - A list of addresses of the primary mesh gateways in the form `:` + (e.g. `["1.1.1.1:443", "2.3.4.5:443"]`). - `k8sAuthMethodHost` ((#v-global-federation-k8sauthmethodhost)) (`string: null`) - If you are setting `global.federation.enabled` to true and are in a secondary datacenter, set `k8sAuthMethodHost` to the address of the Kubernetes API server of the secondary datacenter. This address must be reachable from the Consul servers in the primary datacenter. This auth method will be used to provision ACL tokens for Consul components and is different from the one used by the Consul Service Mesh. - Please see the [Kubernetes Auth Method documentation](https://consul.io/docs/acl/auth-methods/kubernetes). + Please refer to the [Kubernetes Auth Method documentation](https://developer.hashicorp.com/consul/docs/security/acl/auth-methods/kubernetes). + + If `externalServers.enabled` is set to true, `global.federation.k8sAuthMethodHost` and + `externalServers.k8sAuthMethodHost` should be set to the same value. You can retrieve this value from your `kubeconfig` by running: @@ -491,6 +532,8 @@ Use these links to navigate to a particular top-level stanza. -o jsonpath="{.clusters[?(@.name=='')].cluster.server}" ``` + - `logLevel` ((#v-global-federation-loglevel)) (`string: ""`) - Override global log verbosity level for the `create-federation-secret-job` pods. One of "trace", "debug", "info", "warn", or "error". + - `metrics` ((#v-global-metrics)) - Configures metrics for Consul service mesh - `enabled` ((#v-global-metrics-enabled)) (`boolean: false`) - Configures the Helm chart’s components @@ -509,6 +552,9 @@ Use these links to navigate to a particular top-level stanza. Envoy metrics on port `20200` at the `/metrics` path and all gateway pods will have Prometheus scrape annotations. Only applicable if `global.metrics.enabled` is true. + - `enableTelemetryCollector` ((#v-global-metrics-enabletelemetrycollector)) (`boolean: false`) - Configures the Helm chart’s components to forward envoy metrics for the Consul service mesh to the + consul-telemetry-collector. This includes gateway metrics and sidecar metrics. + - `imageConsulDataplane` ((#v-global-imageconsuldataplane)) (`string: hashicorp/consul-dataplane:`) - The name (and tag) of the consul-dataplane Docker image used for the connect-injected sidecar proxies and mesh, terminating, and ingress gateways. @@ -579,6 +625,19 @@ Use these links to navigate to a particular top-level stanza. anotherLabelKey: another-label-value ``` + - `trustedCAs` ((#v-global-trustedcas)) (`array: []`) - Optional PEM-encoded CA certificates that will be added to trusted system CAs. + + Example: + + ```yaml + trustedCAs: [ + | + -----BEGIN CERTIFICATE----- + MIIC7jCCApSgAwIBAgIRAIq2zQEVexqxvtxP6J0bXAwwCgYIKoZIzj0EAwIwgbkx + ... + ] + ``` + ### server ((#h-server)) - `server` ((#v-server)) - Server, when enabled, configures a server cluster to run. This should @@ -589,11 +648,13 @@ Use these links to navigate to a particular top-level stanza. Consul server cluster. If you're running Consul externally and want agents within Kubernetes to join that cluster, this should probably be false. + - `logLevel` ((#v-server-loglevel)) (`string: ""`) - Override global log verbosity level. One of "trace", "debug", "info", "warn", or "error". + - `image` ((#v-server-image)) (`string: null`) - The name of the Docker image (including any tag) for the containers running Consul server agents. - `replicas` ((#v-server-replicas)) (`integer: 1`) - The number of server agents to run. This determines the fault tolerance of - the cluster. Please see the deployment table (https://consul.io/docs/internals/consensus#deployment-table) + the cluster. Please refer to the [deployment table](https://developer.hashicorp.com/consul/docs/architecture/consensus#deployment-table) for more information. - `bootstrapExpect` ((#v-server-bootstrapexpect)) (`int: null`) - The number of servers that are expected to be running. @@ -632,8 +693,8 @@ Use these links to navigate to a particular top-level stanza. Vault Secrets backend: If you are using Vault as a secrets backend, a Vault Policy must be created which allows `["create", "update"]` capabilities on the PKI issuing endpoint, which is usually of the form `pki/issue/consul-server`. - Please see the following guide for steps to generate a compatible certificate: - https://learn.hashicorp.com/tutorials/consul/vault-pki-consul-secure-tls + Complete [this tutorial](https://developer.hashicorp.com/consul/tutorials/vault-secure/vault-pki-consul-secure-tls) + to learn how to generate a compatible certificate. Note: when using TLS, both the `server.serverCert` and `global.tls.caCert` which points to the CA endpoint of this PKI engine must be provided. @@ -672,19 +733,19 @@ Use these links to navigate to a particular top-level stanza. storage classes, the PersistentVolumeClaims would need to be manually created. A `null` value will use the Kubernetes cluster's default StorageClass. If a default StorageClass does not exist, you will need to create one. - Refer to the [Read/Write Tuning](https://www.consul.io/docs/install/performance#read-write-tuning) + Refer to the [Read/Write Tuning](https://developer.hashicorp.com/consul/docs/install/performance#read-write-tuning) section of the Server Performance Requirements documentation for considerations around choosing a performant storage class. - ~> **Note:** The [Reference Architecture](https://learn.hashicorp.com/tutorials/consul/reference-architecture#hardware-sizing-for-consul-servers) + ~> **Note:** The [Reference Architecture](https://developer.hashicorp.com/consul/tutorials/production-deploy/reference-architecture#hardware-sizing-for-consul-servers) contains best practices and recommendations for selecting suitable hardware sizes for your Consul servers. - - `connect` ((#v-server-connect)) (`boolean: true`) - This will enable/disable Connect (https://consul.io/docs/connect). Setting this to true + - `connect` ((#v-server-connect)) (`boolean: true`) - This will enable/disable [service mesh](https://developer.hashicorp.com/consul/docs/connect). Setting this to true _will not_ automatically secure pod communication, this setting will only enable usage of the feature. Consul will automatically initialize - a new CA and set of certificates. Additional Connect settings can be configured - by setting the `server.extraConfig` value. + a new CA and set of certificates. Additional service mesh settings can be configured + by setting the `server.extraConfig` value or by applying [configuration entries](https://developer.hashicorp.com/consul/docs/connect/config-entries). - `serviceAccount` ((#v-server-serviceaccount)) @@ -699,7 +760,7 @@ Use these links to navigate to a particular top-level stanza. - `resources` ((#v-server-resources)) (`map`) - The resource requests (CPU, memory, etc.) for each of the server agents. This should be a YAML map corresponding to a Kubernetes - ResourceRequirements (https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#resourcerequirements-v1-core) + [`ResourceRequirements``](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#resourcerequirements-v1-core) object. NOTE: The use of a YAML string is deprecated. Example: @@ -707,10 +768,10 @@ Use these links to navigate to a particular top-level stanza. ```yaml resources: requests: - memory: '100Mi' + memory: '200Mi' cpu: '100m' limits: - memory: '100Mi' + memory: '200Mi' cpu: '100m' ``` @@ -728,13 +789,18 @@ Use these links to navigate to a particular top-level stanza. - `server` ((#v-server-containersecuritycontext-server)) (`map`) - The consul server agent container + - `aclInit` ((#v-server-containersecuritycontext-aclinit)) (`map`) - The acl-init job + + - `tlsInit` ((#v-server-containersecuritycontext-tlsinit)) (`map`) - The tls-init job + - `updatePartition` ((#v-server-updatepartition)) (`integer: 0`) - This value is used to carefully control a rolling update of Consul server agents. This value specifies the - partition (https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions) - for performing a rolling update. Please read the linked Kubernetes documentation - and https://www.consul.io/docs/k8s/upgrade#upgrading-consul-servers for more information. + [partition](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions) + for performing a rolling update. Please read the linked Kubernetes + and [Upgrade Consul](https://developer.hashicorp.com/consul/docs/k8s/upgrade#upgrading-consul-servers) + documentation for more information. - - `disruptionBudget` ((#v-server-disruptionbudget)) - This configures the PodDisruptionBudget (https://kubernetes.io/docs/tasks/run-application/configure-pdb/) + - `disruptionBudget` ((#v-server-disruptionbudget)) - This configures the [`PodDisruptionBudget`](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) for the server cluster. - `enabled` ((#v-server-disruptionbudget-enabled)) (`boolean: true`) - Enables registering a PodDisruptionBudget for the server @@ -747,7 +813,7 @@ Use these links to navigate to a particular top-level stanza. --set 'server.disruptionBudget.maxUnavailable=0'` flag to the helm chart installation command because of a limitation in the Helm templating language. - - `extraConfig` ((#v-server-extraconfig)) (`string: {}`) - A raw string of extra JSON configuration (https://consul.io/docs/agent/options) for Consul + - `extraConfig` ((#v-server-extraconfig)) (`string: {}`) - A raw string of extra [JSON configuration](https://developer.hashicorp.com/consul/docs/agent/config/config-files) for Consul servers. This will be saved as-is into a ConfigMap that is read by the Consul server agents. This can be used to add additional configuration that isn't directly exposed by the chart. @@ -803,7 +869,7 @@ Use these links to navigate to a particular top-level stanza. - ... ``` - - `affinity` ((#v-server-affinity)) (`string`) - This value defines the affinity (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) + - `affinity` ((#v-server-affinity)) (`string`) - This value defines the [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) for server pods. It defaults to allowing only a single server pod on each node, which minimizes risk of the cluster becoming unusable if a node is lost. If you need to run more pods per node (for example, testing on Minikube), set this value @@ -824,12 +890,14 @@ Use these links to navigate to a particular top-level stanza. ``` - `tolerations` ((#v-server-tolerations)) (`string: ""`) - Toleration settings for server pods. This - should be a multi-line string matching the Tolerations - (https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. + should be a multi-line string matching the + [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) + array in a Pod spec. - `topologySpreadConstraints` ((#v-server-topologyspreadconstraints)) (`string: ""`) - Pod topology spread constraints for server pods. - This should be a multi-line YAML string matching the `topologySpreadConstraints` array - (https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) in a Pod Spec. + This should be a multi-line YAML string matching the + [`topologySpreadConstraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) + array in a Pod Spec. This requires K8S >= 1.18 (beta) or 1.19 (stable). @@ -847,7 +915,7 @@ Use these links to navigate to a particular top-level stanza. component: server ``` - - `nodeSelector` ((#v-server-nodeselector)) (`string: null`) - This value defines `nodeSelector` (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) + - `nodeSelector` ((#v-server-nodeselector)) (`string: null`) - This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) labels for server pod assignment, formatted as a multi-line string. Example: @@ -858,7 +926,7 @@ Use these links to navigate to a particular top-level stanza. ``` - `priorityClassName` ((#v-server-priorityclassname)) (`string: ""`) - This value references an existing - Kubernetes `priorityClassName` (https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) + Kubernetes [`priorityClassName`](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) that can be assigned to server pods. - `extraLabels` ((#v-server-extralabels)) (`map`) - Extra labels to attach to the server pods. This should be a YAML map. @@ -921,19 +989,19 @@ Use these links to navigate to a particular top-level stanza. feature, in case kubernetes cluster is behind egress http proxies. Additionally, it could be used to configure custom consul parameters. - - `snapshotAgent` ((#v-server-snapshotagent)) - Values for setting up and running snapshot agents - (https://consul.io/commands/snapshot/agent) + - `snapshotAgent` ((#v-server-snapshotagent)) - Values for setting up and running + [snapshot agents](https://developer.hashicorp.com/consul/commands/snapshot/agent) within the Consul clusters. They run as a sidecar with Consul servers. - `enabled` ((#v-server-snapshotagent-enabled)) (`boolean: false`) - If true, the chart will install resources necessary to run the snapshot agent. - `interval` ((#v-server-snapshotagent-interval)) (`string: 1h`) - Interval at which to perform snapshots. - See https://www.consul.io/commands/snapshot/agent#interval + Refer to [`interval`](https://developer.hashicorp.com/consul/commands/snapshot/agent#interval) - `configSecret` ((#v-server-snapshotagent-configsecret)) - A Kubernetes or Vault secret that should be manually created to contain the entire config to be used on the snapshot agent. This is the preferred method of configuration since there are usually storage - credentials present. Please see Snapshot agent config (https://consul.io/commands/snapshot/agent#config-file-options) + credentials present. Please refer to the [Snapshot agent config](https://developer.hashicorp.com/consul/commands/snapshot/agent#config-file-options) for details. - `secretName` ((#v-server-snapshotagent-configsecret-secretname)) (`string: null`) - The name of the Kubernetes secret or Vault secret path that holds the snapshot agent config. @@ -954,6 +1022,56 @@ Use these links to navigate to a particular top-level stanza. ... ``` + - `auditLogs` ((#v-server-auditlogs)) - Added in Consul 1.8, the audit object allow users to enable auditing + and configure a sink and filters for their audit logs. Please refer to + [audit logs](https://developer.hashicorp.com/consul/docs/enterprise/audit-logging) documentation + for further information. + + - `enabled` ((#v-server-auditlogs-enabled)) (`boolean: false`) - Controls whether Consul logs out each time a user performs an operation. + global.acls.manageSystemACLs must be enabled to use this feature. + + - `sinks` ((#v-server-auditlogs-sinks)) (`array`) - A single entry of the sink object provides configuration for the destination to which Consul + will log auditing events. + + Example: + + ```yaml + sinks: + - name: My Sink + type: file + format: json + path: /tmp/audit.json + delivery_guarantee: best-effort + rotate_duration: 24h + rotate_max_files: 15 + rotate_bytes: 25165824 + + ``` + + The sink object supports the following keys: + + - `name` - Name of the sink. + + - `type` - Type specifies what kind of sink this is. Currently only file sinks are available + + - `format` - Format specifies what format the events will be emitted with. Currently only `json` + events are emitted. + + - `path` - The directory and filename to write audit events to. + + - `delivery_guarantee` - Specifies the rules governing how audit events are written. Consul + only supports `best-effort` event delivery. + + - `mode` - The permissions to set on the audit log files. + + - `rotate_duration` - Specifies the interval by which the system rotates to a new log file. + At least one of `rotate_duration` or `rotate_bytes` must be configured to enable audit logging. + + - `rotate_bytes` - Specifies how large an individual log file can grow before Consul rotates to a new file. + At least one of rotate_bytes or rotate_duration must be configured to enable audit logging. + + - `rotate_max_files` - Defines the limit that Consul should follow before it deletes old log files. + ### externalServers ((#h-externalservers)) - `externalServers` ((#v-externalservers)) - Configuration for Consul servers when the servers are running outside of Kubernetes. @@ -991,7 +1109,10 @@ Use these links to navigate to a particular top-level stanza. - `k8sAuthMethodHost` ((#v-externalservers-k8sauthmethodhost)) (`string: null`) - If you are setting `global.acls.manageSystemACLs` and `connectInject.enabled` to true, set `k8sAuthMethodHost` to the address of the Kubernetes API server. This address must be reachable from the Consul servers. - Please see the Kubernetes Auth Method documentation (https://consul.io/docs/acl/auth-methods/kubernetes). + Please refer to the [Kubernetes Auth Method documentation](https://developer.hashicorp.com/consul/docs/security/acl/auth-methods/kubernetes). + + If `global.federation.enabled` is set to true, `global.federation.k8sAuthMethodHost` and + `externalServers.k8sAuthMethodHost` should be set to the same value. You could retrieve this value from your `kubeconfig` by running: @@ -1011,10 +1132,12 @@ Use these links to navigate to a particular top-level stanza. the resources necessary for a Consul client on every Kubernetes node. This _does not_ require `server.enabled`, since the agents can be configured to join an external cluster. + - `logLevel` ((#v-client-loglevel)) (`string: ""`) - Override global log verbosity level. One of "trace", "debug", "info", "warn", or "error". + - `image` ((#v-client-image)) (`string: null`) - The name of the Docker image (including any tag) for the containers running Consul client agents. - - `join` ((#v-client-join)) (`array: null`) - A list of valid `-retry-join` values (https://www.consul.io/docs/agent/config/cli-flags#_retry_join). + - `join` ((#v-client-join)) (`array: null`) - A list of valid [`-retry-join` values](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_retry_join). If this is `null` (default), then the clients will attempt to automatically join the server cluster running within Kubernetes. This means that with `server.enabled` set to true, clients will automatically @@ -1035,7 +1158,7 @@ Use these links to navigate to a particular top-level stanza. required for Connect. - `nodeMeta` ((#v-client-nodemeta)) - nodeMeta specifies an arbitrary metadata key/value pair to associate with the node - (see https://www.consul.io/docs/agent/config/cli-flags#_node_meta) + (refer to [`-node-meta`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_node_meta)) - `pod-name` ((#v-client-nodemeta-pod-name)) (`string: ${HOSTNAME}`) @@ -1079,7 +1202,7 @@ Use these links to navigate to a particular top-level stanza. - `tlsInit` ((#v-client-containersecuritycontext-tlsinit)) (`map`) - The tls-init initContainer - - `extraConfig` ((#v-client-extraconfig)) (`string: {}`) - A raw string of extra JSON configuration (https://consul.io/docs/agent/options) for Consul + - `extraConfig` ((#v-client-extraconfig)) (`string: {}`) - A raw string of extra [JSON configuration](https://developer.hashicorp.com/consul/docs/agent/config/config-files) for Consul clients. This will be saved as-is into a ConfigMap that is read by the Consul client agents. This can be used to add additional configuration that isn't directly exposed by the chart. @@ -1172,7 +1295,7 @@ Use these links to navigate to a particular top-level stanza. ``` - `priorityClassName` ((#v-client-priorityclassname)) (`string: ""`) - This value references an existing - Kubernetes `priorityClassName` (https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) + Kubernetes [`priorityClassName`](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) that can be assigned to client pods. - `annotations` ((#v-client-annotations)) (`string: null`) - This value defines additional annotations for @@ -1199,7 +1322,7 @@ Use these links to navigate to a particular top-level stanza. feature, in case kubernetes cluster is behind egress http proxies. Additionally, it could be used to configure custom consul parameters. - - `dnsPolicy` ((#v-client-dnspolicy)) (`string: null`) - This value defines the Pod DNS policy (https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy) + - `dnsPolicy` ((#v-client-dnspolicy)) (`string: null`) - This value defines the [Pod DNS policy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy) for client pods to use. - `hostNetwork` ((#v-client-hostnetwork)) (`boolean: false`) - hostNetwork defines whether or not we use host networking instead of hostPort in the event @@ -1209,7 +1332,8 @@ Use these links to navigate to a particular top-level stanza. combined with `dnsPolicy: ClusterFirstWithHostNet` - `updateStrategy` ((#v-client-updatestrategy)) (`string: null`) - updateStrategy for the DaemonSet. - See https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy. + Refer to the Kubernetes [Daemonset upgrade strategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) + documentation. This should be a multi-line string mapping directly to the updateStrategy Example: @@ -1232,7 +1356,7 @@ Use these links to navigate to a particular top-level stanza. - `enabled` ((#v-dns-enabled)) (`boolean: -`) - - `enableRedirection` ((#v-dns-enableredirection)) (`boolean: -`) - If true, services using Consul Connect will use Consul DNS + - `enableRedirection` ((#v-dns-enableredirection)) (`boolean: -`) - If true, services using Consul service mesh will use Consul DNS for default DNS resolution. The DNS lookups fall back to the nameserver IPs listed in /etc/resolv.conf if not found in Consul. @@ -1307,7 +1431,7 @@ Use these links to navigate to a particular top-level stanza. - `ingressClassName` ((#v-ui-ingress-ingressclassname)) (`string: ""`) - Optionally set the ingressClassName. - - `pathType` ((#v-ui-ingress-pathtype)) (`string: Prefix`) - pathType override - see: https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types + - `pathType` ((#v-ui-ingress-pathtype)) (`string: Prefix`) - pathType override - refer to: https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types - `hosts` ((#v-ui-ingress-hosts)) (`array`) - hosts is a list of host name to create Ingress rules. @@ -1343,16 +1467,17 @@ Use these links to navigate to a particular top-level stanza. - `enabled` ((#v-ui-metrics-enabled)) (`boolean: global.metrics.enabled`) - Enable displaying metrics in the UI. The default value of "-" will inherit from `global.metrics.enabled` value. - - `provider` ((#v-ui-metrics-provider)) (`string: prometheus`) - Provider for metrics. See - https://www.consul.io/docs/agent/options#ui_config_metrics_provider + - `provider` ((#v-ui-metrics-provider)) (`string: prometheus`) - Provider for metrics. Refer to + [`metrics_provider`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#ui_config_metrics_provider) This value is only used if `ui.enabled` is set to true. - `baseURL` ((#v-ui-metrics-baseurl)) (`string: http://prometheus-server`) - baseURL is the URL of the prometheus server, usually the service URL. This value is only used if `ui.enabled` is set to true. - - `dashboardURLTemplates` ((#v-ui-dashboardurltemplates)) - Corresponds to https://www.consul.io/docs/agent/options#ui_config_dashboard_url_templates configuration. + - `dashboardURLTemplates` ((#v-ui-dashboardurltemplates)) - Corresponds to [`dashboard_url_templates`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#ui_config_dashboard_url_templates) + configuration. - - `service` ((#v-ui-dashboardurltemplates-service)) (`string: ""`) - Sets https://www.consul.io/docs/agent/options#ui_config_dashboard_url_templates_service. + - `service` ((#v-ui-dashboardurltemplates-service)) (`string: ""`) - Sets [`dashboardURLTemplates.service`](https://developer.hashicorp.com/consul/docs/agent/config/config-files#ui_config_dashboard_url_templates_service). ### syncCatalog ((#h-synccatalog)) @@ -1372,8 +1497,8 @@ Use these links to navigate to a particular top-level stanza. to run the sync program. - `default` ((#v-synccatalog-default)) (`boolean: true`) - If true, all valid services in K8S are - synced by default. If false, the service must be annotated - (https://consul.io/docs/k8s/service-sync#sync-enable-disable) properly to sync. + synced by default. If false, the service must be [annotated](https://developer.hashicorp.com/consul/docs/k8s/service-sync#enable-and-disable-sync) + properly to sync. In either case an annotation can override the default. - `priorityClassName` ((#v-synccatalog-priorityclassname)) (`string: ""`) - Optional priorityClassName. @@ -1470,6 +1595,19 @@ Use these links to navigate to a particular top-level stanza. or may not be broadly accessible depending on your Kubernetes cluster. Set this to false to skip syncing ClusterIP services. + - `ingress` ((#v-synccatalog-ingress)) + + - `enabled` ((#v-synccatalog-ingress-enabled)) (`boolean: false`) - Syncs the hostname from a Kubernetes Ingress resource to service registrations + when a rule matched a service. Currently only supports host based routing and + not path based routing. The only supported path on an ingress rule is "/". + Set this to false to skip syncing Ingress services. + + Currently, port 80 is synced if there is not TLS entry for the hostname. Syncs the port + 443 if there is a TLS entry that matches the hostname. + + - `loadBalancerIPs` ((#v-synccatalog-ingress-loadbalancerips)) (`boolean: false`) - Requires syncIngress to be `true`. syncs the LoadBalancer IP from a Kubernetes Ingress + resource instead of the hostname to service registrations when a rule matched a service. + - `nodePortSyncType` ((#v-synccatalog-nodeportsynctype)) (`string: ExternalFirst`) - Configures the type of syncing that happens for NodePort services. The valid options are: ExternalOnly, InternalOnly, ExternalFirst. @@ -1486,7 +1624,7 @@ Use these links to navigate to a particular top-level stanza. - `secretKey` ((#v-synccatalog-aclsynctoken-secretkey)) (`string: null`) - The key within the Kubernetes secret that holds the acl sync token. - - `nodeSelector` ((#v-synccatalog-nodeselector)) (`string: null`) - This value defines `nodeSelector` (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) + - `nodeSelector` ((#v-synccatalog-nodeselector)) (`string: null`) - This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) labels for catalog sync pod assignment, formatted as a multi-line string. Example: @@ -1552,7 +1690,7 @@ Use these links to navigate to a particular top-level stanza. - `default` ((#v-connectinject-default)) (`boolean: false`) - If true, the injector will inject the Connect sidecar into all pods by default. Otherwise, pods must specify the - injection annotation (https://consul.io/docs/k8s/connect#consul-hashicorp-com-connect-inject) + [injection annotation](https://developer.hashicorp.com/consul/docs/k8s/connect#consul-hashicorp-com-connect-inject) to opt-in to Connect injection. If this is true, pods can use the same annotation to explicitly opt-out of injection. @@ -1570,7 +1708,7 @@ Use these links to navigate to a particular top-level stanza. This value is also overridable via the "consul.hashicorp.com/transparent-proxy-overwrite-probes" annotation. Note: This value has no effect if transparent proxy is disabled on the pod. - - `disruptionBudget` ((#v-connectinject-disruptionbudget)) - This configures the PodDisruptionBudget (https://kubernetes.io/docs/tasks/run-application/configure-pdb/) + - `disruptionBudget` ((#v-connectinject-disruptionbudget)) - This configures the [`PodDisruptionBudget`](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) for the service mesh sidecar injector. - `enabled` ((#v-connectinject-disruptionbudget-enabled)) (`boolean: true`) - This will enable/disable registering a PodDisruptionBudget for the @@ -1629,7 +1767,8 @@ Use these links to navigate to a particular top-level stanza. by the OpenShift platform. - `updateStrategy` ((#v-connectinject-cni-updatestrategy)) (`string: null`) - updateStrategy for the CNI installer DaemonSet. - See https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy. + Refer to the Kubernetes [Daemonset upgrade strategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) + documentation. This should be a multi-line string mapping directly to the updateStrategy Example: @@ -1653,7 +1792,7 @@ Use these links to navigate to a particular top-level stanza. persistent: true ``` - - `metrics` ((#v-connectinject-metrics)) - Configures metrics for Consul Connect services. All values are overridable + - `metrics` ((#v-connectinject-metrics)) - Configures metrics for Consul service mesh services. All values are overridable via annotations on a per-pod basis. - `defaultEnabled` ((#v-connectinject-metrics-defaultenabled)) (`string: -`) - If true, the connect-injector will automatically @@ -1718,7 +1857,7 @@ Use these links to navigate to a particular top-level stanza. - `imageConsul` ((#v-connectinject-imageconsul)) (`string: null`) - The Docker image for Consul to use when performing Connect injection. Defaults to global.image. - - `logLevel` ((#v-connectinject-loglevel)) (`string: ""`) - Override global log verbosity level. One of "debug", "info", "warn", or "error". + - `logLevel` ((#v-connectinject-loglevel)) (`string: ""`) - Sets the `logLevel` for the `consul-dataplane` sidecar and the `consul-connect-inject-init` container. When set, this value overrides the global log verbosity level. One of "debug", "info", "warn", or "error". - `serviceAccount` ((#v-connectinject-serviceaccount)) @@ -1731,7 +1870,19 @@ Use these links to navigate to a particular top-level stanza. "sample/annotation2": "bar" ``` - - `resources` ((#v-connectinject-resources)) (`map`) - The resource settings for connect inject pods. + - `resources` ((#v-connectinject-resources)) (`map`) - The resource settings for connect inject pods. The defaults, are optimized for getting started worklows on developer deployments. The settings should be tweaked for production deployments. + + - `requests` ((#v-connectinject-resources-requests)) + + - `memory` ((#v-connectinject-resources-requests-memory)) (`string: 200Mi`) - Recommended production default: 500Mi + + - `cpu` ((#v-connectinject-resources-requests-cpu)) (`string: 50m`) - Recommended production default: 250m + + - `limits` ((#v-connectinject-resources-limits)) + + - `memory` ((#v-connectinject-resources-limits-memory)) (`string: 200Mi`) - Recommended production default: 500Mi + + - `cpu` ((#v-connectinject-resources-limits-cpu)) (`string: 50m`) - Recommended production default: 250m - `failurePolicy` ((#v-connectinject-failurepolicy)) (`string: Fail`) - Sets the failurePolicy for the mutating webhook. By default this will cause pods not part of the consul installation to fail scheduling while the webhook is offline. This prevents a pod from skipping mutation if the webhook were to be momentarily offline. @@ -1742,12 +1893,12 @@ Use these links to navigate to a particular top-level stanza. - `namespaceSelector` ((#v-connectinject-namespaceselector)) (`string`) - Selector for restricting the webhook to only specific namespaces. Use with `connectInject.default: true` to automatically inject all pods in namespaces that match the selector. This should be set to a multiline string. - See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector + Refer to https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector for more details. - By default, we exclude the kube-system namespace since usually users won't - want those pods injected and also the local-path-storage namespace so that - Kind (Kubernetes In Docker) can provision Pods used to create PVCs. + By default, we exclude kube-system since usually users won't + want those pods injected and local-path-storage and openebs so that + Kind (Kubernetes In Docker) and [OpenEBS](https://openebs.io/) respectively can provision Pods used to create PVCs. Note that this exclusion is only supported in Kubernetes v1.21.1+. Example: @@ -1764,7 +1915,7 @@ Use these links to navigate to a particular top-level stanza. annotated. Use `["*"]` to automatically allow all k8s namespaces. For example, `["namespace1", "namespace2"]` will only allow pods in the k8s - namespaces `namespace1` and `namespace2` to have Connect sidecars injected + namespaces `namespace1` and `namespace2` to have Consul service mesh sidecars injected and registered with Consul. All other k8s namespaces will be ignored. To deny all namespaces, set this to `[]`. @@ -1829,8 +1980,8 @@ Use these links to navigate to a particular top-level stanza. If set to an empty string all service accounts can log in. This only has effect if ACLs are enabled. - See https://www.consul.io/docs/acl/acl-auth-methods.html#binding-rules - and https://www.consul.io/docs/acl/auth-methods/kubernetes.html#trusted-identity-attributes + Refer to Auth methods [Binding rules](https://developer.hashicorp.com/consul/docs/security/acl/auth-methods#binding-rules) + and [Trusted identiy attributes](https://developer.hashicorp.com/consul/docs/security/acl/auth-methods/kubernetes#trusted-identity-attributes) for more details. Requires Consul >= v1.5. @@ -1856,7 +2007,7 @@ Use these links to navigate to a particular top-level stanza. leads to unnecessary thread and memory usage and leaves unnecessary idle connections open. It is advised to keep this number low for sidecars and high for edge proxies. This will control the `--concurrency` flag to Envoy. - For additional information see also: https://blog.envoyproxy.io/envoy-threading-model-a8d44b922310 + For additional information, refer to https://blog.envoyproxy.io/envoy-threading-model-a8d44b922310 This setting can be overridden on a per-pod basis via this annotation: - `consul.hashicorp.com/consul-envoy-proxy-concurrency` @@ -1872,27 +2023,64 @@ Use these links to navigate to a particular top-level stanza. - `requests` ((#v-connectinject-sidecarproxy-resources-requests)) - - `memory` ((#v-connectinject-sidecarproxy-resources-requests-memory)) (`string: null`) - Recommended default: 100Mi + - `memory` ((#v-connectinject-sidecarproxy-resources-requests-memory)) (`string: null`) - Recommended production default: 100Mi - - `cpu` ((#v-connectinject-sidecarproxy-resources-requests-cpu)) (`string: null`) - Recommended default: 100m + - `cpu` ((#v-connectinject-sidecarproxy-resources-requests-cpu)) (`string: null`) - Recommended production default: 100m - `limits` ((#v-connectinject-sidecarproxy-resources-limits)) - - `memory` ((#v-connectinject-sidecarproxy-resources-limits-memory)) (`string: null`) - Recommended default: 100Mi + - `memory` ((#v-connectinject-sidecarproxy-resources-limits-memory)) (`string: null`) - Recommended production default: 100Mi + + - `cpu` ((#v-connectinject-sidecarproxy-resources-limits-cpu)) (`string: null`) - Recommended production default: 100m + + - `lifecycle` ((#v-connectinject-sidecarproxy-lifecycle)) (`map`) - Set default lifecycle management configuration for sidecar proxy. + These settings can be overridden on a per-pod basis via these annotations: + + - `consul.hashicorp.com/enable-sidecar-proxy-lifecycle` + - `consul.hashicorp.com/enable-sidecar-proxy-shutdown-drain-listeners` + - `consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds` + - `consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-port` + - `consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-shutdown-path` + + - `defaultEnabled` ((#v-connectinject-sidecarproxy-lifecycle-defaultenabled)) (`boolean: true`) + + - `defaultEnableShutdownDrainListeners` ((#v-connectinject-sidecarproxy-lifecycle-defaultenableshutdowndrainlisteners)) (`boolean: true`) + + - `defaultShutdownGracePeriodSeconds` ((#v-connectinject-sidecarproxy-lifecycle-defaultshutdowngraceperiodseconds)) (`integer: 30`) + + - `defaultGracefulPort` ((#v-connectinject-sidecarproxy-lifecycle-defaultgracefulport)) (`integer: 20600`) + + - `defaultGracefulShutdownPath` ((#v-connectinject-sidecarproxy-lifecycle-defaultgracefulshutdownpath)) (`string: /graceful_shutdown`) + + - `initContainer` ((#v-connectinject-initcontainer)) (`map`) - The resource settings for the Connect injected init container. If null, the resources + won't be set for the initContainer. The defaults are optimized for developer instances of + Kubernetes, however they should be tweaked with the recommended defaults as shown below to speed up service registration times. + + - `resources` ((#v-connectinject-initcontainer-resources)) - - `cpu` ((#v-connectinject-sidecarproxy-resources-limits-cpu)) (`string: null`) - Recommended default: 100m + - `requests` ((#v-connectinject-initcontainer-resources-requests)) - - `initContainer` ((#v-connectinject-initcontainer)) (`map`) - The resource settings for the Connect injected init container. + - `memory` ((#v-connectinject-initcontainer-resources-requests-memory)) (`string: 25Mi`) - Recommended production default: 150Mi + + - `cpu` ((#v-connectinject-initcontainer-resources-requests-cpu)) (`string: 50m`) - Recommended production default: 250m + + - `limits` ((#v-connectinject-initcontainer-resources-limits)) + + - `memory` ((#v-connectinject-initcontainer-resources-limits-memory)) (`string: 150Mi`) - Recommended production default: 150Mi + + - `cpu` ((#v-connectinject-initcontainer-resources-limits-cpu)) (`string: null`) - Recommended production default: 500m ### meshGateway ((#h-meshgateway)) - `meshGateway` ((#v-meshgateway)) - [Mesh Gateways](https://developer.hashicorp.com/consul/docs/connect/gateways/mesh-gateway) enable Consul Connect to work across Consul datacenters. - `enabled` ((#v-meshgateway-enabled)) (`boolean: false`) - If [mesh gateways](https://developer.hashicorp.com/consul/docs/connect/gateways/mesh-gateway) are enabled, a Deployment will be created that runs - gateways and Consul Connect will be configured to use gateways. + gateways and Consul service mesh will be configured to use gateways. This setting is required for [Cluster Peering](https://developer.hashicorp.com/consul/docs/connect/cluster-peering/k8s). Requirements: consul 1.6.0+ if using `global.acls.manageSystemACLs``. + - `logLevel` ((#v-meshgateway-loglevel)) (`string: ""`) - Override global log verbosity level for `mesh-gateway-deployment` pods. One of "trace", "debug", "info", "warn", or "error". + - `replicas` ((#v-meshgateway-replicas)) (`integer: 1`) - Number of replicas for the Deployment. - `wanAddress` ((#v-meshgateway-wanaddress)) - What gets registered as WAN address for the gateway. @@ -1924,7 +2112,7 @@ Use these links to navigate to a particular top-level stanza. - `port` ((#v-meshgateway-wanaddress-port)) (`integer: 443`) - Port that gets registered for WAN traffic. If source is set to "Service" then this setting will have no effect. - See the documentation for source as to which port will be used in that + Refer to the documentation for source as to which port will be used in that case. - `static` ((#v-meshgateway-wanaddress-static)) (`string: ""`) - If source is set to "Static" then this value will be used as the WAN @@ -1989,7 +2177,7 @@ Use these links to navigate to a particular top-level stanza. - `initServiceInitContainer` ((#v-meshgateway-initserviceinitcontainer)) (`map`) - The resource settings for the `service-init` init container. - - `affinity` ((#v-meshgateway-affinity)) (`string: null`) - This value defines the affinity (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) + - `affinity` ((#v-meshgateway-affinity)) (`string: null`) - This value defines the [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) for mesh gateway pods. It defaults to `null` thereby allowing multiple gateway pods on each node. But if one would prefer a mode which minimizes risk of the cluster becoming unusable if a node is lost, set this value to the value in the example below. @@ -2011,8 +2199,9 @@ Use these links to navigate to a particular top-level stanza. - `tolerations` ((#v-meshgateway-tolerations)) (`string: null`) - Optional YAML string to specify tolerations. - `topologySpreadConstraints` ((#v-meshgateway-topologyspreadconstraints)) (`string: ""`) - Pod topology spread constraints for mesh gateway pods. - This should be a multi-line YAML string matching the `topologySpreadConstraints` array - (https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) in a Pod Spec. + This should be a multi-line YAML string matching the + [`topologySpreadConstraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) + array in a Pod Spec. This requires K8S >= 1.18 (beta) or 1.19 (stable). @@ -2056,6 +2245,8 @@ Use these links to navigate to a particular top-level stanza. - `enabled` ((#v-ingressgateways-enabled)) (`boolean: false`) - Enable ingress gateway deployment. Requires `connectInject.enabled=true` and `client.enabled=true`. + - `logLevel` ((#v-ingressgateways-loglevel)) (`string: ""`) - Override global log verbosity level for `ingress-gateways-deployment` pods. One of "trace", "debug", "info", "warn", or "error". + - `defaults` ((#v-ingressgateways-defaults)) - Defaults sets default values for all gateway fields. With the exception of annotations, defining any of these values in the `gateways` list will override the default values provided here. Annotations will @@ -2102,7 +2293,7 @@ Use these links to navigate to a particular top-level stanza. - `resources` ((#v-ingressgateways-defaults-resources)) (`map`) - Resource limits for all ingress gateway pods - - `affinity` ((#v-ingressgateways-defaults-affinity)) (`string: null`) - This value defines the affinity (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) + - `affinity` ((#v-ingressgateways-defaults-affinity)) (`string: null`) - This value defines the [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) for ingress gateway pods. It defaults to `null` thereby allowing multiple gateway pods on each node. But if one would prefer a mode which minimizes risk of the cluster becoming unusable if a node is lost, set this value to the value in the example below. @@ -2124,8 +2315,9 @@ Use these links to navigate to a particular top-level stanza. - `tolerations` ((#v-ingressgateways-defaults-tolerations)) (`string: null`) - Optional YAML string to specify tolerations. - `topologySpreadConstraints` ((#v-ingressgateways-defaults-topologyspreadconstraints)) (`string: ""`) - Pod topology spread constraints for ingress gateway pods. - This should be a multi-line YAML string matching the `topologySpreadConstraints` array - (https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) in a Pod Spec. + This should be a multi-line YAML string matching the + [`topologySpreadConstraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) + array in a Pod Spec. This requires K8S >= 1.18 (beta) or 1.19 (stable). @@ -2170,7 +2362,7 @@ Use these links to navigate to a particular top-level stanza. `defaults`. Values defined here override the defaults except in the case of annotations where both will be applied. - - `name` ((#v-ingressgateways-gateways-name)) (`string: ingress-gateway`) + - `name` ((#v-ingressgateways-gateways-name)) (`string: ingress-gateway`) ### terminatingGateways ((#h-terminatinggateways)) @@ -2185,6 +2377,8 @@ Use these links to navigate to a particular top-level stanza. - `enabled` ((#v-terminatinggateways-enabled)) (`boolean: false`) - Enable terminating gateway deployment. Requires `connectInject.enabled=true` and `client.enabled=true`. + - `logLevel` ((#v-terminatinggateways-loglevel)) (`string: ""`) - Override global log verbosity level. One of "trace", "debug", "info", "warn", or "error". + - `defaults` ((#v-terminatinggateways-defaults)) - Defaults sets default values for all gateway fields. With the exception of annotations, defining any of these values in the `gateways` list will override the default values provided here. Annotations will @@ -2208,7 +2402,7 @@ Use these links to navigate to a particular top-level stanza. - `resources` ((#v-terminatinggateways-defaults-resources)) (`map`) - Resource limits for all terminating gateway pods - - `affinity` ((#v-terminatinggateways-defaults-affinity)) (`string: null`) - This value defines the affinity (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) + - `affinity` ((#v-terminatinggateways-defaults-affinity)) (`string: null`) - This value defines the [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) for terminating gateway pods. It defaults to `null` thereby allowing multiple gateway pods on each node. But if one would prefer a mode which minimizes risk of the cluster becoming unusable if a node is lost, set this value to the value in the example below. @@ -2230,8 +2424,9 @@ Use these links to navigate to a particular top-level stanza. - `tolerations` ((#v-terminatinggateways-defaults-tolerations)) (`string: null`) - Optional YAML string to specify tolerations. - `topologySpreadConstraints` ((#v-terminatinggateways-defaults-topologyspreadconstraints)) (`string: ""`) - Pod topology spread constraints for terminating gateway pods. - This should be a multi-line YAML string matching the `topologySpreadConstraints` array - (https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) in a Pod Spec. + This should be a multi-line YAML string matching the + [`topologySpreadConstraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) + array in a Pod Spec. This requires K8S >= 1.18 (beta) or 1.19 (stable). @@ -2285,7 +2480,7 @@ Use these links to navigate to a particular top-level stanza. `defaults`. Values defined here override the defaults except in the case of annotations where both will be applied. - - `name` ((#v-terminatinggateways-gateways-name)) (`string: terminating-gateway`) + - `name` ((#v-terminatinggateways-gateways-name)) (`string: terminating-gateway`) ### apiGateway ((#h-apigateway)) @@ -2306,7 +2501,7 @@ Use these links to navigate to a particular top-level stanza. - `enabled` ((#v-apigateway-managedgatewayclass-enabled)) (`boolean: true`) - When true a GatewayClass is configured to automatically work with Consul as installed by helm. - - `nodeSelector` ((#v-apigateway-managedgatewayclass-nodeselector)) (`string: null`) - This value defines `nodeSelector` (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) + - `nodeSelector` ((#v-apigateway-managedgatewayclass-nodeselector)) (`string: null`) - This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) labels for gateway pod assignment, formatted as a multi-line string. Example: @@ -2316,7 +2511,7 @@ Use these links to navigate to a particular top-level stanza. beta.kubernetes.io/arch: amd64 ``` - - `tolerations` ((#v-apigateway-managedgatewayclass-tolerations)) (`string: null`) - Toleration settings for gateway pods created with the managed gateway class. + - `tolerations` ((#v-apigateway-managedgatewayclass-tolerations)) (`string: null`) - Toleration settings for gateway pods created with the managed gateway class. This should be a multi-line string matching the [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. @@ -2333,7 +2528,7 @@ Use these links to navigate to a particular top-level stanza. ```yaml service: annotations: | - - external-dns.alpha.kubernetes.io/hostname + - external-dns.alpha.kubernetes.io/hostname ``` - `deployment` ((#v-apigateway-managedgatewayclass-deployment)) (`map`) - This value defines the number of pods to deploy for each Gateway as well as a min and max number of pods for all Gateways @@ -2370,10 +2565,10 @@ Use these links to navigate to a particular top-level stanza. ``` - `priorityClassName` ((#v-apigateway-controller-priorityclassname)) (`string: ""`) - This value references an existing - Kubernetes `priorityClassName` (https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) + Kubernetes [`priorityClassName`](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#pod-priority) that can be assigned to api-gateway-controller pods. - - `nodeSelector` ((#v-apigateway-controller-nodeselector)) (`string: null`) - This value defines `nodeSelector` (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) + - `nodeSelector` ((#v-apigateway-controller-nodeselector)) (`string: null`) - This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) labels for api-gateway-controller pod assignment, formatted as a multi-line string. Example: @@ -2384,7 +2579,7 @@ Use these links to navigate to a particular top-level stanza. ``` - `tolerations` ((#v-apigateway-controller-tolerations)) (`string: null`) - This value defines the tolerations for api-gateway-controller pod, this should be a multi-line string matching the - Tolerations (https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. + [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. - `service` ((#v-apigateway-controller-service)) - Configuration for the Service created for the api-gateway-controller @@ -2408,7 +2603,7 @@ Use these links to navigate to a particular top-level stanza. This should be a multi-line string matching the Toleration array in a PodSpec. - - `nodeSelector` ((#v-webhookcertmanager-nodeselector)) (`string: null`) - This value defines `nodeSelector` (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) + - `nodeSelector` ((#v-webhookcertmanager-nodeselector)) (`string: null`) - This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) labels for the webhook-cert-manager pod assignment, formatted as a multi-line string. Example: @@ -2433,6 +2628,78 @@ Use these links to navigate to a particular top-level stanza. - `enabled` ((#v-tests-enabled)) (`boolean: true`) +### telemetryCollector ((#h-telemetrycollector)) + +- `telemetryCollector` ((#v-telemetrycollector)) + + - `enabled` ((#v-telemetrycollector-enabled)) (`boolean: false`) - Enables the consul-telemetry-collector deployment + + - `logLevel` ((#v-telemetrycollector-loglevel)) (`string: ""`) - Override global log verbosity level. One of "trace", "debug", "info", "warn", or "error". + + - `image` ((#v-telemetrycollector-image)) (`string: hashicorp/consul-telemetry-collector:0.0.1`) - The name of the Docker image (including any tag) for the containers running + the consul-telemetry-collector + + - `resources` ((#v-telemetrycollector-resources)) (`map`) - The resource settings for consul-telemetry-collector pods. + + - `replicas` ((#v-telemetrycollector-replicas)) (`integer: 1`) - This value sets the number of consul-telemetry-collector replicas to deploy. + + - `customExporterConfig` ((#v-telemetrycollector-customexporterconfig)) (`string: null`) - This value defines additional configuration for the telemetry collector. It should be formatted as a multi-line + json blob string + + ```yaml + customExporterConfig: | + {"http_collector_endpoint": "other-otel-collector"} + ``` + + - `service` ((#v-telemetrycollector-service)) + + - `annotations` ((#v-telemetrycollector-service-annotations)) (`string: null`) - This value defines additional annotations for the server service account. This should be formatted as a multi-line + string. + + ```yaml + annotations: | + "sample/annotation1": "foo" + "sample/annotation2": "bar" + ``` + + - `serviceAccount` ((#v-telemetrycollector-serviceaccount)) + + - `annotations` ((#v-telemetrycollector-serviceaccount-annotations)) (`string: null`) - This value defines additional annotations for the telemetry-collector's service account. This should be formatted + as a multi-line string. + + ```yaml + annotations: | + "sample/annotation1": "foo" + "sample/annotation2": "bar" + ``` + + - `cloud` ((#v-telemetrycollector-cloud)) + + - `clientId` ((#v-telemetrycollector-cloud-clientid)) + + - `secretName` ((#v-telemetrycollector-cloud-clientid-secretname)) (`string: null`) + + - `secretKey` ((#v-telemetrycollector-cloud-clientid-secretkey)) (`string: null`) + + - `clientSecret` ((#v-telemetrycollector-cloud-clientsecret)) + + - `secretName` ((#v-telemetrycollector-cloud-clientsecret-secretname)) (`string: null`) + + - `secretKey` ((#v-telemetrycollector-cloud-clientsecret-secretkey)) (`string: null`) + + - `initContainer` ((#v-telemetrycollector-initcontainer)) + + - `resources` ((#v-telemetrycollector-initcontainer-resources)) (`map`) - The resource settings for consul-telemetry-collector initContainer. + + - `nodeSelector` ((#v-telemetrycollector-nodeselector)) (`string: null`) - Optional YAML string to specify a nodeSelector config. + + - `priorityClassName` ((#v-telemetrycollector-priorityclassname)) (`string: ""`) - Optional priorityClassName. + + - `extraEnvironmentVars` ((#v-telemetrycollector-extraenvironmentvars)) (`map`) - A list of extra environment variables to set within the stateful set. + These could be used to include proxy settings required for cloud auto-join + feature, in case kubernetes cluster is behind egress http proxies. Additionally, + it could be used to configure custom consul parameters. + ## Helm Chart Examples @@ -2450,7 +2717,7 @@ ui: type: LoadBalancer ``` -The below `values.yaml` results in a three server Consul Enterprise cluster with 100GB of storage and automatic Connect injection. +The below `values.yaml` results in a three server Consul Enterprise cluster with 100GB of storage and automatic connect injection. Note, this would require a secret that contains the enterprise license key. diff --git a/website/content/docs/k8s/index.mdx b/website/content/docs/k8s/index.mdx index f4d0ffe4c32..e1a8291abe3 100644 --- a/website/content/docs/k8s/index.mdx +++ b/website/content/docs/k8s/index.mdx @@ -59,7 +59,7 @@ There are several ways to try Consul with Kubernetes in different environments. - The [Consul and Kubernetes Deployment](/consul/tutorials/kubernetes/kubernetes-deployment-guide?utm_source=docs) tutorial covers the necessary steps to install and configure a new Consul cluster on Kubernetes in production. -- The [Secure Consul and Registered Services on Kubernetes](https://consul.io/consul/tutorials/kubernetes/kubernetes-secure-agents?utm_source=docs) tutorial covers +- The [Secure Consul and Registered Services on Kubernetes](/consul/tutorials/kubernetes/kubernetes-secure-agents?utm_source=docs) tutorial covers the necessary steps to secure a Consul cluster running on Kubernetes in production. - The [Layer 7 Observability with Consul Service Mesh](/consul/tutorials/kubernetes/kubernetes-layer7-observability) tutorial covers monitoring a diff --git a/website/content/docs/k8s/installation/install-cli.mdx b/website/content/docs/k8s/installation/install-cli.mdx index a628ede9573..d9d613516c4 100644 --- a/website/content/docs/k8s/installation/install-cli.mdx +++ b/website/content/docs/k8s/installation/install-cli.mdx @@ -128,20 +128,20 @@ Complete the following instructions to install a specific version of the CLI so 1. Download the appropriate version of Consul K8s CLI using the following `curl` command. Set the `$VERSION` environment variable to the appropriate version for your deployment. ```shell-session - $ export VERSION=1.0 && \ + $ export VERSION=1.1.1 && \ curl --location "https://releases.hashicorp.com/consul-k8s/${VERSION}/consul-k8s_${VERSION}_darwin_amd64.zip" --output consul-k8s-cli.zip ``` 1. Unzip the zip file output to extract the `consul-k8s` CLI binary. This overwrites existing files and also creates a `.consul-k8s` subdirectory in your `$HOME` folder. ```shell-session - $ unzip -o consul-k8s-cli.zip -d ~/.consul-k8s + $ unzip -o consul-k8s-cli.zip -d ~/consul-k8s ``` 1. Add the path to your directory. In order to persist the `$PATH` across sessions, dd it to your shellrc (i.e. shell run commands) file for the shell used by your terminal. ```shell-session - $ export PATH=$PATH:$HOME/.consul-k8s/ + $ export PATH=$PATH:$HOME/consul-k8s ``` 1. (Optional) Issue the `consul-k8s version` command to verify the installation. diff --git a/website/content/docs/k8s/installation/install.mdx b/website/content/docs/k8s/installation/install.mdx index 2872718166c..78f90f48079 100644 --- a/website/content/docs/k8s/installation/install.mdx +++ b/website/content/docs/k8s/installation/install.mdx @@ -85,7 +85,7 @@ or read the [Helm Chart Reference](/consul/docs/k8s/helm). ### Minimal `values.yaml` for Consul service mesh -The following `values.yaml` config file contains the minimum required settings to enable [Consul Service Mesh](https://consul.io/(/docs/k8s/connect)): +The following `values.yaml` config file contains the minimum required settings to enable [Consul Service Mesh](/consul/docs/k8s/connect): @@ -308,7 +308,7 @@ metadata: spec: containers: - name: example - image: 'consul:latest' + image: 'hashicorp/consul:latest' env: - name: HOST_IP valueFrom: @@ -345,7 +345,7 @@ spec: spec: containers: - name: example - image: 'consul:latest' + image: 'hashicorp/consul:latest' env: - name: HOST_IP valueFrom: diff --git a/website/content/docs/k8s/k8s-cli.mdx b/website/content/docs/k8s/k8s-cli.mdx index 402d2327b7d..df986a5885e 100644 --- a/website/content/docs/k8s/k8s-cli.mdx +++ b/website/content/docs/k8s/k8s-cli.mdx @@ -28,15 +28,64 @@ $ consul-k8s You can use the following commands with `consul-k8s`. + - [`config`](#config): Interact with helm configuration. + - [`config read`](#config-read): Read helm configuration of a Consul installation. - [`install`](#install): Install Consul on Kubernetes. - [`proxy`](#proxy): Inspect Envoy proxies managed by Consul. - [`proxy list`](#proxy-list): List all Pods running proxies managed by Consul. - [`proxy read`](#proxy-read): Inspect the Envoy configuration for a given Pod. + - [`proxy log`](#proxy-log): Inspect and modify the Envoy logging configuration for a given Pod. - [`status`](#status): Check the status of a Consul installation on Kubernetes. + - [`troubleshoot`](#troubleshoot): Troubleshoot Consul service mesh and networking issues from a given pod. - [`uninstall`](#uninstall): Uninstall Consul deployment. - [`upgrade`](#upgrade): Upgrade Consul on Kubernetes from an existing installation. - [`version`](#version): Print the version of the Consul on Kubernetes CLI. +### `config` + +The `config` command exposes the `read` subcommand that allows to read the helm configuration of a Consul installation. + +- [`config read`](#config-read): Read helm configuration of a Consul installation. + +### `config read` + +```shell-session +$ consul-k8s config read +``` + +| Flag | Description | Default | +| ------------------------------------ | ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| `-all-namespaces`, `-A` | `Boolean` List pods in all Kubernetes namespaces. | `false` | +| `-namespace`, `-n` | `String` The Kubernetes namespace to list proxies in. | Current [kubeconfig](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) namespace. | + +Refer to the [Global Options](#global-options) for additional options that you can use +when installing Consul on Kubernetes. + +#### Example Commands + +The following example command reads the Helm configuration in the `myNS` namespace. + +```shell-session +$ consul-k8s config read -namespace=myNS +``` + +``` +global: + cloud: + clientId: + secretKey: client-id + secretName: consul-hcp-client-id + clientSecret: + secretKey: client-secret + secretName: consul-hcp-client-secret + enabled: true + resourceId: + secretKey: resource-id + secretName: consul-hcp-resource-id + image: hashicorp/consul:1.14.7 + name: consul +``` + ### `install` The `install` command installs Consul on your Kubernetes cluster. @@ -100,6 +149,7 @@ Consul in your Kubernetes Cluster. - [`proxy list`](#proxy-list): List all Pods running proxies managed by Consul. - [`proxy read`](#proxy-read): Inspect the Envoy configuration for a given Pod. +- [`proxy log`](#proxy-log): Inspect and modify the Envoy logging configuration for a given Pod. ### `proxy list` @@ -447,6 +497,290 @@ $ consul-k8s proxy read backend-658b679b45-d5xlb -o raw } ``` +### `proxy log` + +The `proxy log` command allows you to inspect and modify the logging configuration of Envoy proxies running on a given Pod. + +```shell-session +$ consul-k8s proxy log +``` + +The command takes a required value, ``. This should be the full name +of a Kubernetes Pod. If a Pod is running more than one Envoy proxy managed by +Consul, as in the [Multiport configuration](/consul/docs/k8s/connect#kubernetes-pods-with-multiple-ports), +the terminal displays configuration information for all proxies in the pod. + +The following options are available. + +| Flag | Description | Default | +| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | +| `-namespace`, `-n` | `String` Specifies the namespace containing the target Pod. | Current [kubeconfig](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) namespace. | +| `-update-level`, `-u` | `String` Specifies the logger (optional) and the level to update.

    Use the following format to configure the same level for loggers: `-update-level `.

    You can also specify a comma-delineated list to configure levels for specific loggers, for example: `-update-level grpc:warning,http:info`.

    | none | +| `-reset`, `-r` | `String` Reset the log levels for all loggers back to the default of `info` | `info` | + +#### Example commands +In the following example, Consul returns the log levels for all of an Envoy proxy's loggers in a pod with the ID `server-697458b9f8-4vr29`: + +```shell-session +$ consul-k8s proxy log server-697458b9f8-4vr29 +Envoy log configuration for server-697458b9f8-4vr29 in namespace default: + +==> Log Levels for server-697458b9f8-4vr29 +Name Level +rds info +backtrace info +hc info +http info +io info +jwt info +rocketmq info +matcher info +runtime info +redis info +stats info +tap info +alternate_protocols_cache info +grpc info +init info +quic info +thrift info +wasm info +aws info +conn_handler info +ext_proc info +hystrix info +tracing info +dns info +oauth2 info +connection info +health_checker info +kafka info +mongo info +config info +admin info +forward_proxy info +misc info +websocket info +dubbo info +happy_eyeballs info +main info +client info +lua info +udp info +cache_filter info +filter info +multi_connection info +quic_stream info +router info +http2 info +key_value_store info +secret info +testing info +upstream info +assert info +ext_authz info +rbac info +decompression info +envoy_bug info +file info +pool info +``` + +The following command updates the log levels for all loggers of an Envoy proxy to `warning`. +```shell-session +$ consul-k8s proxy log server-697458b9f8-4vr29 -update-level warning +Envoy log configuration for server-697458b9f8-4vr29 in namespace default: + +==> Log Levels for server-697458b9f8-4vr29 +Name Level +pool warning +rbac warning +tracing warning +aws warning +cache_filter warning +decompression warning +init warning +assert warning +client warning +misc warning +udp warning +config warning +hystrix warning +key_value_store warning +runtime warning +admin warning +dns warning +jwt warning +redis warning +quic warning +alternate_protocols_cache warning +conn_handler warning +ext_proc warning +http warning +oauth2 warning +ext_authz warning +http2 warning +kafka warning +mongo warning +router warning +thrift warning +grpc warning +matcher warning +hc warning +multi_connection warning +wasm warning +dubbo warning +filter warning +upstream warning +backtrace warning +connection warning +io warning +main warning +happy_eyeballs warning +rds warning +tap warning +envoy_bug warning +rocketmq warning +file warning +forward_proxy warning +stats warning +health_checker warning +lua warning +secret warning +quic_stream warning +testing warning +websocket warning +``` +The following command updates the `grpc` log level to `error`, the `http` log level to `critical`, and the `runtime` log level to `debug` for pod ID `server-697458b9f8-4vr29` +```shell-session +$ consul-k8s proxy log server-697458b9f8-4vr29 -update-level grpc:error,http:critical,runtime:debug +Envoy log configuration for server-697458b9f8-4vr29 in namespace default: + +==> Log Levels for server-697458b9f8-4vr29 +Name Level +assert info +dns info +http critical +pool info +thrift info +udp info +grpc error +hc info +stats info +wasm info +alternate_protocols_cache info +ext_authz info +filter info +http2 info +key_value_store info +tracing info +cache_filter info +quic_stream info +aws info +io info +matcher info +rbac info +tap info +connection info +conn_handler info +rocketmq info +hystrix info +oauth2 info +redis info +backtrace info +file info +forward_proxy info +kafka info +config info +router info +runtime debug +testing info +happy_eyeballs info +ext_proc info +init info +lua info +health_checker info +misc info +envoy_bug info +jwt info +main info +quic info +upstream info +websocket info +client info +decompression info +mongo info +multi_connection info +rds info +secret info +admin info +dubbo info +``` +The following command resets the log levels for all loggers of an Envoy proxy in pod `server-697458b9f8-4vr29` to the default level of `info`. +```shell-session +$ consul-k8s proxy log server-697458b9f8-4vr29 -r +Envoy log configuration for server-697458b9f8-4vr29 in namespace default: + +==> Log Levels for server-697458b9f8-4vr29 +Name Level +ext_proc info +secret info +thrift info +tracing info +dns info +rocketmq info +happy_eyeballs info +hc info +io info +misc info +conn_handler info +key_value_store info +rbac info +hystrix info +wasm info +admin info +cache_filter info +client info +health_checker info +oauth2 info +runtime info +testing info +grpc info +upstream info +forward_proxy info +matcher info +pool info +aws info +decompression info +jwt info +tap info +assert info +redis info +http info +quic info +rds info +connection info +envoy_bug info +stats info +alternate_protocols_cache info +backtrace info +filter info +http2 info +init info +multi_connection info +quic_stream info +dubbo info +ext_authz info +main info +udp info +websocket info +config info +mongo info +router info +file info +kafka info +lua info +``` ### `status` The `status` command provides an overall status summary of the Consul on Kubernetes installation. It also provides the configuration that was used to deploy Consul K8s and information about the health of Consul servers and clients. This command does not take in any flags. @@ -490,6 +824,104 @@ $ consul-k8s status ✓ Consul clients healthy (3/3) ``` +### `troubleshoot` + +The `troubleshoot` command exposes two subcommands for troubleshooting Consul +service mesh and network issues from a given pod. + +- [`troubleshoot upstreams`](#troubleshoot-upstreams): List all Envoy upstreams in Consul service mesh from the given pod. +- [`troubleshoot proxy`](#troubleshoot-proxy): Troubleshoot Consul service mesh configuration and network issues between the given pod and the given upstream. + +### `troubleshoot upstreams` + +```shell-session +$ consul-k8s troubleshoot upstreams -pod +``` + +| Flag | Description | Default | +| ------------------------------------ | ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| `-namespace`, `-n` | `String` The Kubernetes namespace to list proxies in. | Current [kubeconfig](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) namespace. | + +#### Example Commands + +The following example displays all transparent proxy upstreams in Consul service mesh from the given pod. + + ```shell-session + $ consul-k8s troubleshoot upstreams -pod frontend-767ccfc8f9-6f6gx + + ==> Upstreams (explicit upstreams only) (0) + + ==> Upstreams IPs (transparent proxy only) (1) + [10.4.6.160 240.0.0.3] true map[backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul] + + If you cannot find the upstream address or cluster for a transparent proxy upstream: + - Check intentions: Tproxy upstreams are configured based on intentions. Make sure you have configured intentions to allow traffic to your upstream. + - To check that the right cluster is being dialed, run a DNS lookup for the upstream you are dialing. For example, run `dig backend.svc.consul` to return the IP address for the `backend` service. If the address you get from that is missing from the upstream IPs, it means that your proxy may be misconfigured. + ``` + +The following example displays all explicit upstreams from the given pod in the Consul service mesh. + + ```shell-session + $ consul-k8s troubleshoot upstreams -pod client-767ccfc8f9-6f6gx + + ==> Upstreams (explicit upstreams only) (1) + server + counting + + ==> Upstreams IPs (transparent proxy only) (0) + + If you cannot find the upstream address or cluster for a transparent proxy upstream: + - Check intentions: Tproxy upstreams are configured based on intentions. Make sure you have configured intentions to allow traffic to your upstream. + - To check that the right cluster is being dialed, run a DNS lookup for the upstream you are dialing. For example, run `dig backend.svc.consul` to return the IP address for the `backend` service. If the address you get from that is missing from the upstream IPs, it means that your proxy may be misconfigured. + ``` + +### `troubleshoot proxy` + +```shell-session +$ consul-k8s troubleshoot proxy -pod -upstream-ip +$ consul-k8s troubleshoot proxy -pod -upstream-envoy-id +``` + +| Flag | Description | Default | +| ------------------------------------ | ----------------------------------------------------------| ---------------------------------------------------------------------------------------------------------------------- | +| `-namespace`, `-n` | `String` The Kubernetes namespace to list proxies in. | Current [kubeconfig](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) namespace. | +| `-upstream-ip` | `String` The IP address of the upstream transparent proxy | | +| `-upstream-envoy-id` | `String` The Envoy identifier of the upstream | | + +#### Example Commands + +The following example troubleshoots the Consul service mesh configuration and network issues between the given pod and the given upstream IP. + + ```shell-session + $ consul-k8s troubleshoot proxy -pod frontend-767ccfc8f9-6f6gx -upstream-ip 10.4.6.160 + + ==> Validation + ✓ certificates are valid + ✓ Envoy has 0 rejected configurations + ✓ Envoy has detected 0 connection failure(s) + ✓ listener for upstream "backend" found + ✓ route for upstream "backend" found + ✓ cluster "backend.default.dc1.internal..consul" for upstream "backend" found + ✓ healthy endpoints for cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ cluster "backend2.default.dc1.internal..consul" for upstream "backend" found + ! no healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ``` + +The following example troubleshoots the Consul service mesh configuration and network issues between the given pod and the given upstream. + + ```shell-session + $ consul-k8s troubleshoot proxy -pod frontend-767ccfc8f9-6f6gx -upstream-envoy-id db + + ==> Validation + ✓ certificates are valid + ✓ Envoy has 0 rejected configurations + ✓ Envoy has detected 0 connection failure(s) + ! no listener for upstream "db" found + ! no route for upstream "backend" found + ! no cluster "db.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "db" found + ! no healthy endpoints for cluster "db.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "db" found + ``` + ### `uninstall` The `uninstall` command removes Consul from Kubernetes. diff --git a/website/content/docs/k8s/operations/tls-on-existing-cluster.mdx b/website/content/docs/k8s/operations/tls-on-existing-cluster.mdx index ee656f24389..32b89d326cd 100644 --- a/website/content/docs/k8s/operations/tls-on-existing-cluster.mdx +++ b/website/content/docs/k8s/operations/tls-on-existing-cluster.mdx @@ -13,7 +13,7 @@ you may want to configure TLS in a way that minimizes downtime to your applicati Consul already supports rolling out TLS on an existing cluster without downtime. However, depending on your Kubernetes use case, your upgrade procedure may be different. -## Gradual TLS Rollout without Consul Connect +## Gradual TLS Rollout without Consul Service Mesh If you do not use a service mesh, follow this process. @@ -39,7 +39,7 @@ If you do not use a service mesh, follow this process. 1. Repeat steps 1 and 2, turning on TLS verification by setting `global.tls.verify` to `true`. -## Gradual TLS Rollout with Consul Connect +## Gradual TLS Rollout with Consul Service Mesh Because the sidecar Envoy proxies need to talk to the Consul client agent regularly for service discovery, we can't enable TLS on the clients without also re-injecting a @@ -75,12 +75,12 @@ applications to it. 1. Run `helm upgrade` with the above config file. -1. At this point, all components (e.g., Consul Connect webhook and sync catalog) should be running +1. At this point, all components (e.g., Consul service mesh webhook and sync catalog) should be running on the new node pool. -1. Redeploy all your Connect-enabled applications. +1. Redeploy all your mesh-enabled applications. One way to trigger a redeploy is to run `kubectl drain` on the nodes in the old pool. - Now that the Connect webhook is TLS-aware, it adds TLS configuration to + Now that the service mesh webhook is TLS-aware, it adds TLS configuration to the sidecar proxy. Also, Kubernetes should schedule these applications on the new node pool. 1. Perform a rolling upgrade of the servers described in diff --git a/website/content/docs/k8s/service-sync.mdx b/website/content/docs/k8s/service-sync.mdx index 0cd6cb7c382..54ebbdd54d2 100644 --- a/website/content/docs/k8s/service-sync.mdx +++ b/website/content/docs/k8s/service-sync.mdx @@ -12,7 +12,7 @@ services are available to Consul agents and services in Consul can be available as first-class Kubernetes services. This functionality is provided by the [consul-k8s project](https://github.com/hashicorp/consul-k8s) and can be automatically installed and configured using the -[Consul Helm chart](/consul/docs/k8s/installation/install). +[Consul K8s Helm chart](/consul/docs/k8s/installation/install). ![screenshot of a Kubernetes service in the UI](/img/k8s-service.png) @@ -20,7 +20,7 @@ automatically installed and configured using the Consul catalog enable Kubernetes services to be accessed by any node that is part of the Consul cluster, including other distinct Kubernetes clusters. For non-Kubernetes nodes, they can access services using the standard -[Consul DNS](/consul/docs/discovery/dns) or HTTP API. +[Consul DNS](/consul/docs/services/discovery/dns-overview) or HTTP API. **Why sync Consul services to Kubernetes?** Syncing Consul services to Kubernetes services enables non-Kubernetes services to be accessed using kube-dns and Kubernetes-specific @@ -31,11 +31,7 @@ service discovery, including hosted services like databases. ~> Enabling both Service Mesh and Service Sync on the same Kubernetes services is not supported, as Service Mesh also registers Kubernetes service instances to Consul. Ensure that Service Sync is only enabled for namespaces and services that are not injected with the Consul sidecar for Service Mesh as described in [Sync Enable/Disable](/consul/docs/k8s/service-sync#sync-enable-disable). -The service sync uses an external long-running process in the -[consul-k8s project](https://github.com/hashicorp/consul-k8s). This process -can run either inside or outside of a Kubernetes cluster. However, running this process within -the Kubernetes cluster is generally easier since it is automated using the -[Helm chart](/consul/docs/k8s/helm). +The service sync feature deploys a long-running process which can run either inside or outside of a Kubernetes cluster. However, running this process within the Kubernetes cluster is generally easier since it is automated using the [Helm chart](/consul/docs/k8s/helm). The Consul server cluster can run either in or out of a Kubernetes cluster. The Consul server cluster does not need to be running on the same machine diff --git a/website/content/docs/k8s/upgrade/index.mdx b/website/content/docs/k8s/upgrade/index.mdx index ae12c8e3d9f..80507a45e07 100644 --- a/website/content/docs/k8s/upgrade/index.mdx +++ b/website/content/docs/k8s/upgrade/index.mdx @@ -5,14 +5,23 @@ description: >- Consul on Kubernetes relies on packages and binaries that have individual upgrade requirements. Learn how to update Helm configurations, Helm versions, Consul versions, and Consul agents, as well as how to determine what will change and its impact on your service mesh. --- -# Upgrading Consul on Kubernetes Components +# Upgrading Consul on Kubernetes components + +This topic describes considerations and strategies for upgrading Consul deployments running on Kubernetes clusters. In addition to upgrading the version of Consul, you may need to update your Helm chart or the release version of the Helm chart. + +## Version-specific upgrade requirements + +As of Consul v1.14.0, Kubernetes deployments use [Consul Dataplane](/consul/docs/connect/dataplane) instead of client agents. If you upgrade Consul from a version that uses client agents to a version that uses dataplanes, you must follow specific steps to update your Helm chart and remove client agents from the existing deployment. Refer to [Upgrading to Consul Dataplane](/consul/docs/k8s/upgrade#upgrading-to-consul-dataplane) for more information. + +The v1.0.0 release of the Consul on Kubernetes Helm chart also introduced a change to the [`externalServers[].hosts` parameter](/consul/docs/k8s/helm#v-externalservers-hosts). Previously, you were able to enter a provider lookup as a string in this field. Now, you must include `exec=` at the start of a string containing a provider lookup. Otherwise, the string is treated as a DNS name. Refer to the [`go-netaddrs` library and command line tool](https://github.com/hashicorp/go-netaddrs) for more information. ## Upgrade types We recommend updating Consul on Kubernetes when: + - You change your Helm configuration - A new Helm chart is released - - You want to upgrade your Consul version. + - You want to upgrade your Consul version ### Helm configuration changes @@ -38,9 +47,9 @@ For example, if you installed Consul with `connectInject.enabled: false` and you ``` **Before performing the upgrade, be sure you read the other sections on this page, - continuing at [Determining What Will Change](#determining-what-will-change).** + continuing at [Determine scope of changes](#determine-scope-of-changes).** -~> Note: You should always set the `--version` flag when upgrading Helm. Otherwise Helm uses the most up-to-date version in its local cache, which may result in an unintended upgrade. +~> Note: You should always set the `--version` flag when upgrading Helm. Otherwise, Helm uses the most up-to-date version in its local cache, which may result in an unintended upgrade. ### Upgrade Helm chart version @@ -87,7 +96,7 @@ If you want to upgrade to the latest `0.40.0` version, use the following procedu ``` **Before performing the upgrade, be sure you've read the other sections on this page, - continuing at [Determining What Will Change](#determining-what-will-change).** + continuing at [Determine scope of changes](#determine-scope-of-changes).** ### Upgrade Consul version @@ -126,9 +135,9 @@ to update to the new version. Before you upgrade to a new version: ``` **Before performing the upgrade, be sure you have read the other sections on this page, - continuing at [Determining What Will Change](#determining-what-will-change).** + continuing at [Determine scope of changes](#determine-scope-of-changes).** -~> Note: You should always set the `--version` flag when upgrading Helm. Otherwise Helm uses the most up-to-date version in its local cache, which may result in an unintended upgrade. +~> Note: You should always set the `--version` flag when upgrading Helm. Otherwise, Helm uses the most up-to-date version in its local cache, which may result in an unintended upgrade. ## Determine scope of changes @@ -175,7 +184,7 @@ that can be used. Initiate the server upgrade: 1. Change the `global.image` value to the desired Consul version. -1. Set the `server.updatePartition` value to the number of server replicas. By default there are 3 servers, so you would set this value to `3`. +1. Set the `server.updatePartition` value to the number of server replicas. By default, there are 3 servers, so you would set this value to `3`. 1. Set the `updateStrategy` for clients to `OnDelete`. @@ -219,24 +228,39 @@ In earlier versions, Consul on Kubernetes used client agents in its deployments. If you upgrade Consul from a version that uses client agents to a version the uses dataplanes, complete the following steps to upgrade your deployment safely and without downtime. -1. Before you upgrade, edit your Helm chart to enable Consul client agents by setting `client.enabled` and `client.updateStrategy`: +1. If ACLs are enabled, you must first upgrade to consul-k8s 0.49.8 or above. These versions expose the setting `connectInject.prepareDataplanesUpgrade` + which is required for no-downtime upgrades when ACLs are enabled. + + Set `connectInject.prepareDataplanesUpgrade` to `true` and then perform the upgrade to 0.49.8 or above (whichever is the latest in the 0.49.x series) ```yaml filename="values.yaml" - client: - enabled: true - updateStrategy: - type: OnDelete + connectInject: + prepareDataplanesUpgrade: true ``` -1. Add `consul.hashicorp.com/consul-k8s-version: 1.0.0` to the annotations for each pod you upgrade. +1. Consul dataplanes disables Consul clients by default, but during an upgrade you need to ensure Consul clients continue to run. Edit your Helm chart configuration and set the [`client.enabled`](/consul/docs/k8s/helm#v-client-enabled) field to `true` and specify an action for Consul to take during the upgrade process in the [`client.updateStrategy`](/consul/docs/k8s/helm#v-client-updatestrategy) field: -1. Follow our [recommended procedures to upgrade servers](#upgrading-consul-servers) on Kubernetes deployments to upgrade Helm values for the new version of Consul. + ```yaml filename="values.yaml" + client: + enabled: true + updateStrategy: | + type: OnDelete + ``` + +1. Follow our [recommended procedures to upgrade servers](#upgrade-consul-servers) on Kubernetes deployments to upgrade Helm values for the new version of Consul. The latest version of consul-k8s components may be in a CrashLoopBackoff state during the performance of the server upgrade from versions <1.14.x until all Consul servers are on versions >=1.14.x. Components in CrashLoopBackoff will not negatively affect the cluster because older versioned components will still be operating. Once all servers have been fully upgraded, the latest consul-k8s components will automatically restore from CrashLoopBackoff and older component versions will be spun down. 1. Run `kubectl rollout restart` to restart your service mesh applications. Restarting service mesh application causes Kubernetes to re-inject them with the webhook for dataplanes. 1. Restart all gateways in your service mesh. -1. Disable client agents in your Helm chart by deleting the `client` stanza or setting `client.enabled` to `false`. +1. Now that all services and gateways are using Consul dataplanes, disable client agents in your Helm chart by deleting the `client` stanza or setting `client.enabled` to `false` and running a `consul-k8s` or Helm upgrade. + +1. If ACLs are enabled, outdated ACL tokens will persist a result of the upgrade. You can manually delete the tokens to declutter your Consul environment. + + Outdated connect-injector tokens have the following description: `token created via login: {"component":"connect-injector"}`. Do not delete + the tokens that have a description where `pod` is a key, for example `token created via login: {"component":"connect-injector","pod":"default/consul-connect-injector-576b65747c-9547x"}`). The dataplane-enabled connect inject pods use these tokens. + + You can also review the creation date for the tokens and only delete the injector tokens created before your upgrade, but do not delete all old tokens without considering if they are still in use. Some tokens, such as the server tokens, are still necessary. ## Configuring TLS on an existing cluster diff --git a/website/content/docs/lambda/invocation.mdx b/website/content/docs/lambda/invocation.mdx index ebb12571ec2..cdfe774ef64 100644 --- a/website/content/docs/lambda/invocation.mdx +++ b/website/content/docs/lambda/invocation.mdx @@ -11,7 +11,7 @@ This topic describes how to invoke AWS Lambda functions from the Consul service ## Overview -You can invoke Lambda functions from the Consul service mesh through terminating gateways (recommended) or directly from connect proxies. +You can invoke Lambda functions from the Consul service mesh through terminating gateways (recommended) or directly from service mesh proxies. ### Terminating Gateway @@ -30,13 +30,13 @@ The following diagram shows the invocation procedure: -1. Make an HTTP request to the local Connect proxy. -1. The Connect proxy forwards the request to the terminating gateway. +1. Make an HTTP request to the local service mesh proxy. +1. The service mesh proxy forwards the request to the terminating gateway. 1. The terminating gateway invokes the function. -### Connect Proxy +### Service Mesh Proxy -You can invoke Lambda functions directly from a service's Connect proxy. +You can invoke Lambda functions directly from a service's mesh sidecar proxy. This method has the following limitations: - Intentions are unsupported. Consul enforces intentions by validating the client certificates presented when a connection is received. Lambda does not support client certificate validation, which prevents Consul from supporting intentions using this method. - Transparent proxies are unsupported. This is because Lambda services are not @@ -50,12 +50,12 @@ The following diagram shows the invocation procedure: -![Connect Proxy to Lambda](/img/connect_proxy_to_lambda.svg) +![Service Mesh Proxy to Lambda](/img/connect_proxy_to_lambda.svg) -1. Make an HTTP request to the local Connect proxy. -2. The Connect proxy invokes the Lambda. +1. Make an HTTP request to the local service mesh proxy. +2. The service mesh proxy invokes the Lambda. ## Invoke a Lambda Function Before you can invoke a Lambda function, register the service used to invoke diff --git a/website/content/docs/lambda/invoke-from-lambda.mdx b/website/content/docs/lambda/invoke-from-lambda.mdx index a7ec2913865..9ce0d087792 100644 --- a/website/content/docs/lambda/invoke-from-lambda.mdx +++ b/website/content/docs/lambda/invoke-from-lambda.mdx @@ -84,11 +84,11 @@ spec: ### Deploy the mesh gateway -The mesh gateway must be running and registered to the Lambda function’s Consul datacenter. Refer to the following documentation and tutorials for instructions: +The mesh gateway must be running and registered to the Lambda function’s Consul datacenter. Refer to the following documentation and tutorials for instructions: - [Mesh Gateways between WAN-Federated Datacenters](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-wan-datacenters) - [Mesh Gateways between Admin Partitions](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-partitions) -- [Mesh Gateways between Peered Clusters](/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers) +- [Establish cluster peering connections](/consul/docs/connect/cluster-peering/usage/establish-cluster-peering) - [Connect Services Across Datacenters with Mesh Gateways](/consul/tutorials/developer-mesh/service-mesh-gateways) ## Deploy the Lambda extension layer @@ -264,7 +264,7 @@ Define the following environment variables in your Lambda functions to configure ## Invoke the Lambda function -If _intentions_ are enabled in the Consul service mesh, you must create an intention that allows the Lambda function's Consul service to invoke all upstream services prior to invoking the Lambda function. Refer to [Service Mesh Intentions](/consul/docs/connect/intentions) for additional information. +If _intentions_ are enabled in the Consul service mesh, you must create an intention that allows the Lambda function's Consul service to invoke all upstream services prior to invoking the Lambda function. Refer to [Service mesh intentions](/consul/docs/connect/intentions) for additional information. There are several ways to invoke Lambda functions. In the following example, the `aws lambda invoke` CLI command invokes the function: diff --git a/website/content/docs/nia/configuration.mdx b/website/content/docs/nia/configuration.mdx index 4d4e8113f8e..bd67ef5c07e 100644 --- a/website/content/docs/nia/configuration.mdx +++ b/website/content/docs/nia/configuration.mdx @@ -197,7 +197,7 @@ Service registration requires that the [Consul token](/consul/docs/nia/configura | `default_check.enabled` | Optional | boolean | Enables CTS to create the default health check. | `true` | | `default_check.address` | Optional | string | The address to use for the default HTTP health check. Needs to include the scheme (`http`/`https`) and the port, if applicable. | `http://localhost:` or `https://localhost:`. Determined from the [port configuration](/consul/docs/nia/configuration#port) and whether [TLS is enabled](/consul/docs/nia/configuration#enabled-2) on the CTS API. | -The default health check is an [HTTP check](/consul/docs/discovery/checks#http-interval) that calls the [Health API](/consul/docs/nia/api/health). The following table describes the values CTS sets for this default check, corresponding to the [Consul register check API](/consul/api-docs/agent/check#register-check). If an option is not listed in this table, then CTS is using the default value. +The default health check is an [HTTP check](/consul/docs/services/usage/checks#http-checks) that calls the [Health API](/consul/docs/nia/api/health). The following table describes the values CTS sets for this default check, corresponding to the [Consul register check API](/consul/api-docs/agent/check#register-check). If an option is not listed in this table, then CTS is using the default value. | Parameter | Value | | --------- | ----- | diff --git a/website/content/docs/nia/index.mdx b/website/content/docs/nia/index.mdx index d24aaf2a521..f5368f036fa 100644 --- a/website/content/docs/nia/index.mdx +++ b/website/content/docs/nia/index.mdx @@ -11,7 +11,7 @@ Network Infrastructure Automation (NIA) enables dynamic updates to network infra CTS executes one or more automation tasks with the most recent service variable values from the Consul service catalog. Each task consists of a runbook automation written as a CTS compatible Terraform module using resources and data sources for the underlying network infrastructure. The `consul-terraform-sync` daemon runs on the same node as a Consul agent. -CTS is available as an open source and enterprise distribution. Follow the [Network Infrastructure Automation introduction tutorial](/consul/tutorials/network-infrastructure-automation/consul-terraform-sync-intro?utm_source=docs) to get started with CTS OSS or read more about [CTS Enterprise](/consul/docs/nia/enterprise). +CTS is available as an open source and enterprise distribution. Follow the [Automate your network configuration with Consul-Terraform-Sync tutorial](/consul/tutorials/network-infrastructure-automation/consul-terraform-sync-intro?utm_source=docs) to get started with CTS OSS or read more about [CTS Enterprise](/consul/docs/nia/enterprise). ## Use Cases diff --git a/website/content/docs/release-notes/consul-api-gateway/v0_4_x.mdx b/website/content/docs/release-notes/consul-api-gateway/v0_4_x.mdx index a6e82167d30..71fa38c8965 100644 --- a/website/content/docs/release-notes/consul-api-gateway/v0_4_x.mdx +++ b/website/content/docs/release-notes/consul-api-gateway/v0_4_x.mdx @@ -31,7 +31,7 @@ description: >- to rewrite the URL path in a client's HTTP request before sending the request to a service. For example, you could configure the gateway to change the path from `//store/checkout` to `//cart/checkout`. Refer to the [usage - documentation](/consul/docs/api-gateway/usage) for additional information. + documentation](/consul/docs/connect/gateways/api-gateway/usage) for additional information. ## What has Changed diff --git a/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx b/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx index 3d266b13350..8f185bb6f37 100644 --- a/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx +++ b/website/content/docs/release-notes/consul-k8s/v0_47_x.mdx @@ -9,7 +9,7 @@ description: >- ## Release Highlights -- **Cluster Peering (Beta)**: This release introduces support for Cluster Peering, which allows service connectivity between two independent clusters. Enabling peering will deploy the peering controllers and PeeringAcceptor and PeeringDialer CRDs. The new CRDs are used to establish a peering connection between two clusters. Refer to [Cluster Peering on Kubernetes](/consul/docs/connect/cluster-peering/k8s) for full instructions on using Cluster Peering on Kubernetes. +- **Cluster Peering (Beta)**: This release introduces support for cluster peering, which allows service connectivity between two independent clusters. When you enable cluster peering, Consul deploys the peering controllers and `PeeringAcceptor` and `PeeringDialer` CRDs. The new CRDs are used to establish a peering connection between two clusters. Refer to [Cluster Peering Overview](/consul/docs/connect/cluster-peering) for full instructions on using Cluster Peering on Kubernetes. - **Envoy Proxy Debugging CLI Commands**: This release introduces new commands to quickly identify proxies and troubleshoot Envoy proxies for sidecars and gateways. * Add `consul-k8s proxy list` command for displaying pods running Envoy managed by Consul. diff --git a/website/content/docs/release-notes/consul-k8s/v0_49_x.mdx b/website/content/docs/release-notes/consul-k8s/v0_49_x.mdx index 57a66e5c3e5..cb03589e1cf 100644 --- a/website/content/docs/release-notes/consul-k8s/v0_49_x.mdx +++ b/website/content/docs/release-notes/consul-k8s/v0_49_x.mdx @@ -44,3 +44,7 @@ The changelogs for this major release version and any maintenance versions are l ~> **Note:** The following link takes you to the changelogs on the GitHub website. - [0.49.0](https://github.com/hashicorp/consul-k8s/releases/tag/v0.49.0) +- [0.49.1](https://github.com/hashicorp/consul-k8s/releases/tag/v0.49.1) +- [0.49.2](https://github.com/hashicorp/consul-k8s/releases/tag/v0.49.2) +- [0.49.3](https://github.com/hashicorp/consul-k8s/releases/tag/v0.49.3) +- [0.49.4](https://github.com/hashicorp/consul-k8s/releases/tag/v0.49.4) diff --git a/website/content/docs/release-notes/consul-k8s/v1_0_x.mdx b/website/content/docs/release-notes/consul-k8s/v1_0_x.mdx index f34e2397c27..5de0b43f582 100644 --- a/website/content/docs/release-notes/consul-k8s/v1_0_x.mdx +++ b/website/content/docs/release-notes/consul-k8s/v1_0_x.mdx @@ -19,7 +19,8 @@ description: >- - `client.enabled` now defaults to `false`. Setting it to true will deploy client agents, however, none of the consul-k8s components will use clients for their operation. For Vault on Kubernetes using Consul deployed on Kubernetes as a storage backend, `client.enabled` should be set to `true` prior to upgrading. - `externalServers.grpcPort` default is now 8502 instead of 8503. -- `sync-catalog` now communicates directly to Consul servers. When communicating to servers outside of Kubernetes, use the `externalServers.hosts` stanza as described in [Join External Servers to Consul on Kubernetes](/consul/docs/k8s/deployment-configurations/servers-outside-kubernetes). +- `externalServers.hosts` no longer supports [cloud auto-join](/consul/docs/install/cloud-auto-join) strings directly. Instead, include an [`exec=`](https://github.com/hashicorp/go-netaddrs#command-line-tool-usage) string in the `externalServers.hosts` list to invoke the `discover` CLI. For example, the following string invokes the `discover` CLI with a cloud auto-join string: `exec=discover -q addrs provider=aws region=us-west-2 tag_key=consul-server tag_value=true`. The `discover` CLI is included in the official `hashicorp/consul-dataplane` images by default. +- `sync-catalog` now communicates directly to Consul servers. When communicating to servers outside of Kubernetes, use the `externalServers.hosts` stanza as described in [Join External Servers to Consul on Kubernetes](/consul/docs/k8s/deployment-configurations/servers-outside-kubernetes). - Consul snapshot agent runs as a sidecar to Consul servers. - `client.snapshotAgent` values are moved to `server.snapshotAgent`, with the exception of the following values: `client.snaphostAgent.replicas`, `client.snaphostAgent.serviceAccount` - `global.secretsBackend.vault.consulSnapshotAgentRole` value is now removed. You should now use the `global.secretsBackend.vault.consulServerRole` for access to any Vault secrets. @@ -61,3 +62,7 @@ The changelogs for this major release version and any maintenance versions are l ~> **Note:** The following link takes you to the changelogs on the GitHub website. - [1.0.0](https://github.com/hashicorp/consul-k8s/releases/tag/v1.0.0) +- [1.0.1](https://github.com/hashicorp/consul-k8s/releases/tag/v1.0.1) +- [1.0.2](https://github.com/hashicorp/consul-k8s/releases/tag/v1.0.2) +- [1.0.3](https://github.com/hashicorp/consul-k8s/releases/tag/v1.0.3) +- [1.0.4](https://github.com/hashicorp/consul-k8s/releases/tag/v1.0.4) diff --git a/website/content/docs/release-notes/consul-k8s/v1_1_x.mdx b/website/content/docs/release-notes/consul-k8s/v1_1_x.mdx new file mode 100644 index 00000000000..5c3b8a5619c --- /dev/null +++ b/website/content/docs/release-notes/consul-k8s/v1_1_x.mdx @@ -0,0 +1,61 @@ +--- +layout: docs +page_title: 1.1.x +description: >- + Consul on Kubernetes release notes for version 1.1.x +--- + +# Consul on Kubernetes 1.1.0 + +## Release Highlights + +- **Enhanced Envoy Access Logging:** Envoy access logs are now centrally managed via the `accessLogs` field within the ProxyDefaults CRD to allow operators to easily turn on access logs for all proxies within the service mesh. Refer to [Access logs overview](/consul/docs/connect/observability/access-logs) for more information. + +- **Consul Envoy Extensions:** The new Envoy extension system enables you to modify Consul-generated Envoy resources outside of the Consul binary. This will allow extensions to add, delete, and modify Envoy listeners, routes, clusters, and endpoints, enabling support for additional Envoy features without changes to the Consul codebase. +The new `envoyExtensions` field in the ProxyDefaults and ServiceDefaults CRDs enable built-in Envoy extensions. Refer to [Envoy extensions overview](/consul/docs/connect/proxies/envoy-extensions) for more information on how to use these extensions. + +- **Envoy Proxy Debugging CLI Commands**: This release adds a new command to quickly modify the log level of Envoy proxies for sidecars and gateways for easier debugging. +Refer to [consul-k8s CLI proxy log command](/consul/docs/k8s/k8s-cli#proxy-log) docs for more information. + * Add `consul-k8s proxy log podname` command for displaying current log levels or updating log levels for Envoy in a given pod. + + +## What's Changed + +- Connect inject now excludes the `openebs` namespace from sidecar injection by default. If you previously had pods in that namespace +that you wanted to be injected, you must now set namespaceSelector as follows: + + ```yaml + connectInject: + namespaceSelector: | + matchExpressions: + - key: "kubernetes.io/metadata.name" + operator: "NotIn" + values: ["kube-system","local-path-storage"] + ``` + +## Supported Software + +~> **Note:** Consul 1.14.x and 1.13.x are not supported. Please refer to [Supported Consul and Kubernetes versions](/consul/docs/k8s/compatibility#supported-consul-and-kubernetes-versions) for more detail on choosing the correct `consul-k8s` version. +- Consul 1.15.x. +- Consul Dataplane v1.1.x. Refer to [Envoy and Consul Dataplane](/consul/docs/connect/proxies/envoy#envoy-and-consul-dataplane) for details about Consul Dataplane versions and the available packaged Envoy version. +- Kubernetes 1.23.x - 1.26.x +- `kubectl` 1.23.x - 1.26.x +- Helm 3.6+ + +## Upgrading + +For detailed information on upgrading, please refer to the [Upgrades page](/consul/docs/k8s/upgrade) + +## Known Issues + +The following issues are known to exist in the v1.1.0 release: + +- Pod Security Standards that are configured for the [Pod Security Admission controller](https://kubernetes.io/blog/2022/08/25/pod-security-admission-stable/) are currently not supported by Consul K8s. OpenShift 4.11.x enables Pod Security Standards on Kubernetes 1.25 [by default](https://connect.redhat.com/en/blog/important-openshift-changes-pod-security-standards) and is also not supported. Support will be added in a future Consul K8s 1.0.x patch release. + +## Changelogs + +The changelogs for this major release version and any maintenance versions are listed below. + +~> **Note:** The following link takes you to the changelogs on the GitHub website. + +- [1.1.0](https://github.com/hashicorp/consul-k8s/releases/tag/v1.1.0) diff --git a/website/content/docs/release-notes/consul/v1_11_x.mdx b/website/content/docs/release-notes/consul/v1_11_x.mdx index ae8a72cca6a..3334c6fbde7 100644 --- a/website/content/docs/release-notes/consul/v1_11_x.mdx +++ b/website/content/docs/release-notes/consul/v1_11_x.mdx @@ -17,7 +17,7 @@ description: >- - **TLS Certificates for Ingress Gateways via an SDS source**: Ingress Gateways can now be configured to retrieve TLS certificates from an external SDS Service and load the TLS certificates for Ingress listeners. This configuration is set using the `ingress-gateway` configuration entry via the [SDS](/consul/docs/connect/config-entries/ingress-gateway#sds) stanza within the Ingress Gateway TLS configuration. -- **Vault Auth Method support for Connect CA Vault Provider**: Consul now supports configuring the Connect CA Vault provider to use auth methods for authentication to Vault. Consul supports using any non-deprecated auth method that is available in Vault v1.8.5, including AppRole, AliCloud, AWS, Azure, Cloud Foundry, GitHub, Google Cloud, JWT/OIDC, Kerberos, Kubernetes, LDAP, Oracle Cloud Infrastructure, Okta, Radius, TLS Certificates, and Username & Password. The Vault Auth Method for Connect CA Provider is utilized by default for the [Vault Secrets Backend](/consul/docs/k8s/deployment-configurations/vault) feature on Consul on Kubernetes. Utilizing a Vault Auth method would no longer require a Vault token to be managed or provisioned ahead of time to be used for authentication to Vault. +- **Vault Auth Method support for service mesh CA Vault Provider**: Consul now supports configuring the service mesh CA Vault provider to use auth methods for authentication to Vault. Consul supports using any non-deprecated auth method that is available in Vault v1.8.5, including AppRole, AliCloud, AWS, Azure, Cloud Foundry, GitHub, Google Cloud, JWT/OIDC, Kerberos, Kubernetes, LDAP, Oracle Cloud Infrastructure, Okta, Radius, TLS Certificates, and Username & Password. The Vault Auth Method for service mesh CA Provider is utilized by default for the [Vault Secrets Backend](/consul/docs/k8s/deployment-configurations/vault) feature on Consul on Kubernetes. Utilizing a Vault Auth method would no longer require a Vault token to be managed or provisioned ahead of time to be used for authentication to Vault. ## What's Changed diff --git a/website/content/docs/release-notes/consul/v1_12_x.mdx b/website/content/docs/release-notes/consul/v1_12_x.mdx index c1d0d4724d3..ebdaaed98b0 100644 --- a/website/content/docs/release-notes/consul/v1_12_x.mdx +++ b/website/content/docs/release-notes/consul/v1_12_x.mdx @@ -13,7 +13,7 @@ description: >- - **Per listener TLS Config**: It is now possible to configure TLS differently for each of Consul's listeners, such as HTTPS, gRPC, and the internal multiplexed RPC listener, using the `tls` stanza. Refer to [TLS Configuration Reference](/consul/docs/agent/config/config-files#tls-configuration-reference) for more details. -- **AWS Lambda**: Adds the ability to invoke AWS Lambdas through terminating gateways, which allows for cross-datacenter communication, transparent proxy, and intentions with Consul Service Mesh. Refer to [AWS Lambda](/consul/docs]/lambda) and [Invoke Lambda Functions](/consul/docs/lambda/invocation) for more details. +- **AWS Lambda**: Adds the ability to invoke AWS Lambdas through terminating gateways, which allows for cross-datacenter communication, transparent proxy, and intentions with Consul Service Mesh. Refer to [AWS Lambda](/consul/docs/lambda) and [Invoke Lambda Functions](/consul/docs/lambda/invocation) for more details. - **Mesh-wide TLS min/max versions and cipher suites**: Using the [Mesh](/consul/docs/connect/config-entries/mesh#tls) Config Entry or CRD, it is now possible to set TLS min/max versions and cipher suites for both inbound and outbound mTLS connections. diff --git a/website/content/docs/release-notes/consul/v1_13_x.mdx b/website/content/docs/release-notes/consul/v1_13_x.mdx index 736e84f9db5..dd3f7cfe309 100644 --- a/website/content/docs/release-notes/consul/v1_13_x.mdx +++ b/website/content/docs/release-notes/consul/v1_13_x.mdx @@ -15,7 +15,7 @@ description: >- - **Enables TLS on the Envoy Prometheus endpoint**: The Envoy prometheus endpoint can be enabled when `envoy_prometheus_bind_addr` is set and then secured over TLS using new CLI flags for the `consul connect envoy` command. These commands are: `-prometheus-ca-file`, `-prometheus-ca-path`, `-prometheus-cert-file` and `-prometheus-key-file`. The CA, cert, and key can be provided to Envoy by a Kubernetes mounted volume so that Envoy can watch the files and dynamically reload the certs when the volume is updated. -- **UDP Health Checks**: Adds the ability to register service discovery health checks that periodically send UDP datagrams to the specified IP/hostname and port. Refer to [UDP checks](/consul/docs/discovery/checks#udp-interval). +- **UDP Health Checks**: Adds the ability to register service discovery health checks that periodically send UDP datagrams to the specified IP/hostname and port. Refer to [UDP checks](/consul/docs//services/usage/checks#udp-checks). ## What's Changed diff --git a/website/content/docs/release-notes/consul/v1_15_x.mdx b/website/content/docs/release-notes/consul/v1_15_x.mdx new file mode 100644 index 00000000000..d90f918b637 --- /dev/null +++ b/website/content/docs/release-notes/consul/v1_15_x.mdx @@ -0,0 +1,100 @@ +--- +layout: docs +page_title: 1.15.x +description: >- + Consul release notes for version 1.15.x +--- + +# Consul 1.15.0 + +## Release Highlights + +- **Enhanced Envoy Access Logging:** Envoy access logs are now centrally managed via config entries and CRDs to allow operators to easily turn on access logs for all proxies within the service mesh. Refer to [Access logs overview](/consul/docs/connect/observability/access-logs) for more information. Additionally, the [Proxy default configuration entry](/consul/docs/connect/config-entries/proxy-defaults) shows you how to enable access logs centrally via the ProxyDefaults config entry or CRD. + +- **Consul Envoy Extensions:** The new Envoy extension system enables you to modify Consul-generated Envoy resources outside of the Consul binary. This will allow extensions to add, delete, and modify Envoy listeners, routes, clusters, and endpoints, enabling support for additional Envoy features without changes to the Consul codebase. +Current supported extensions include the [Lua](/consul/docs/connect/proxies/envoy-extensions#lua) and [AWS Lambda](/consul/docs/connect/proxies/envoy-extensions#lambda) extensions. Refer to [Envoy extensions overview](/consul/docs/connect/proxies/envoy-extensions) for more information on how to use these extensions for Consul service mesh. + +- **API Gateway support on Linux VM runtimes:** You can now deploy Consul API Gateway on Linux VM runtimes. API Gateway is built into Consul and, when deploying on Linux VM runtimes, is not separately installed software. Refer to [API gateway overview](/consul/docs/connect/gateways/api-gateway) for more information on API Gateway specifically for VM. + + ~> **Note:** Support for API Gateway on Linux VM runtimes is considered a "Beta" feature in Consul v1.15.0. HashiCorp expects to change it to a GA feature as part of a v1.15 patch release in the near future. + +- **Limit traffic rates to Consul servers:** You can now configure global RPC rate limits to mitigate the risks to Consul servers when clients send excessive read or write requests to Consul resources. Refer to [Limit traffic rates overview](/consul/docs/agent/limits) for more details on how to use the new troubleshooting commands. + +- **Service-to-service troubleshooting:** Consul includes a built-in tool for troubleshooting communication between services in a service mesh. The `consul troubleshoot` command enables you to validate communication between upstream and downstream Envoy proxies on VM and Kubernetes deployments. Refer to [Service-to-service troubleshooting overview](/consul/docs/troubleshoot/troubleshoot-services) for more details on how to use the new troubleshooting commands. +Refer to [Service-to-service troubleshooting overview](/consul/docs/troubleshoot/troubleshoot-services) for more details on how to use the new troubleshooting commands. + +- **Raft write-ahead log (Experimental):** Consul provides an experimental storage backend called write-ahead log (WAL). WAL implements a traditional log with rotating, append-only log files which resolves a number of performance issues with the current BoltDB storage backend. Refer to [Experimental WAL LogStore backend overview](/consul/docs/agent/wal-logstore) for more details. + + ~> **Note:** The new Raft write-ahead log storage backend is not recommended for production use cases yet, but is ready for testing by the general community. + +## What's Changed + +- ACL errors have now been ehanced to return descriptive errors when the specified resource cannot be found. Other ACL request errors provide more information about when a resource is missing. In addition, errors are now gracefully thrown when interacting with the ACL system before the ACL system been bootstrapped. + - The Delete Token/Policy/AuthMethod/Role/BindingRule endpoints now return 404 when the resource cannot be found. The new error format is as follows: + + ```log hideClipboard + Requested * does not exist: ACL not found", "* not found in namespace $NAMESPACE: ACL not found` + ``` + + - The Read Token/Policy/Role endpoints now return 404 when the resource cannot be found. The new error format is as follows: + + ```log hideClipboard + Cannot find * to delete + ``` + + - The Logout endpoint now returns a 401 error when the supplied token cannot be found. The new error format is as follows: + + ```log hideClipboard + Supplied token does not exist + ``` + + - The Token Self endpoint now returns 404 when the token cannot be found. The new error format is as follows: + + ```log hideClipboard + Supplied token does not exist + ``` + +- Consul v1.15.0 formally removes all uses of legacy ACLs and ACL policies from Consul. The legacy ACL system was deprecated in Consul v1.4.0 and removed in Consul v1.11.0. The documentation for the new ACL system can be found [here](/consul/docs/v1.14.x/security/acl). For information on how to migrate to the new ACL System, please read the [Migrate Legacy ACL Tokens tutorial](/consul/tutorials/security-operations/access-control-token-migration). +- The following agent flags are now deprecated: `-join`, `-join-wan`, `start_join`, and `start_join_wan`. These options are now aliases of `-retry-join`, `-retry-join-wan`, `retry_join`, and `retry_join_wan`, respectively. +- A `peer` field has been added to ServiceDefaults upstream overrides to make it possible to apply upstream overrides only to peer services. Prior to this change, overrides would be applied based on matching the namespace and name fields only, which means users could not have different configuration for local versus peer services. With this change, peer upstreams are only affected if the peer field matches the destination peer name. +- If you run the `consul connect envoy` command with an incompatible Envoy version, Consul will now error and exit. To ignore this check, use flag `--ignore-envoy-compatibility`. +- Ingress Gateway upstream clusters will have empty `outlier_detection` if passive health check is unspecified. + +## Upgrading + +For more detailed information, please refer to the [upgrade details page](/consul/docs/upgrading/upgrade-specific#consul-1-15-0) and the changelogs. + +## Known Issues + +The following issues are known to exist in the v1.15.x releases: + +- v1.15.0 - v1.15.1 contain a race condition that can cause + some service instances to lose their ability to communicate in the mesh after + [72 hours (LeafCertTTL)](/consul/docs/connect/ca/consul#leafcertttl) + due to a problem with leaf certificate rotation. + This is resolved in Consul v1.15.2. + +- For v1.15.0, Consul is reporting newer releases of Envoy (for example, v1.25.1) as not supported, even though these versions are listed as valid in the [Envoy compatilibity matrix](/consul/docs/connect/proxies/envoy#envoy-and-consul-client-agent). The following error would result for newer versions of Envoy: + + ```log hideClipboard + Envoy version 1.25.1 is not supported. If there is a reason you need to use this version of envoy use the ignore-envoy-compatibility flag. Using an unsupported version of Envoy is not recommended and your experience may vary. + ``` + + To workaround this issue on Consul v1.15.0, launch sidecar proxies + with the `ignore-envoy-compatiblity` flag: + + ```shell-session + $ consul connect envoy --ignore-envoy-compatibility + ``` + +- For v1.15.0, there is a known issue where `consul acl token read -self` requires an `-accessor-id`. This is resolved in Consul v1.15.1. + +- For v1.15.0, there is a known issue where search filters produced errors and resulted in lists not showing full results until being interacted with. This is resolved in Consul v1.15.1. + +## Changelogs + +The changelogs for this major release version and any maintenance versions are listed below. + +~> **Note:** These links take you to the changelogs on the GitHub website. + +- [1.15.0](https://github.com/hashicorp/consul/releases/tag/v1.15.0) diff --git a/website/content/docs/security/acl/acl-policies.mdx b/website/content/docs/security/acl/acl-policies.mdx index f5d005ffbb5..e1583f250a2 100644 --- a/website/content/docs/security/acl/acl-policies.mdx +++ b/website/content/docs/security/acl/acl-policies.mdx @@ -391,7 +391,11 @@ New installations of Consul ship with the following built-in policies. ### Global Management -The `global-management` policy grants unrestricted privileges to any token linked to it. The policy is assigned the reserved ID of `00000000-0000-0000-0000-000000000001`. You can rename the global management policy, but Consul will prevent you from modifying any other attributes, including the rule set and datacenter scope. +The `global-management` policy grants unrestricted privileges to any token linked to it. The policy is assigned the reserved ID of `00000000-0000-0000-0000-000000000001`. You can rename the global management policy, but Consul prevents you from modifying any other attributes, including the rule set and datacenter scope. + +### Global Read-Only + +The `builtin/global-read-only` policy grants unrestricted _read-only_ privileges to any token linked to it. The policy is assigned the reserved ID of `00000000-0000-0000-0000-000000000002`. You can rename the global read-only policy, but Consul prevents you from modifying any other attributes, including the rule set and datacenter scope. ### Namespace Management diff --git a/website/content/docs/security/acl/acl-rules.mdx b/website/content/docs/security/acl/acl-rules.mdx index e9cd5c8b128..280c0775036 100644 --- a/website/content/docs/security/acl/acl-rules.mdx +++ b/website/content/docs/security/acl/acl-rules.mdx @@ -34,7 +34,7 @@ The following resources are not covered by ACL policies: - The [Status API](/consul/api-docs/status) is used by servers when bootstrapping and exposes basic IP and port information about the servers, and does not allow modification of any state. - The datacenter listing operation of the [Catalog API](/consul/api-docs/catalog#list-datacenters) similarly exposes the names of known Consul datacenters, and does not allow modification of any state. -- The [connect CA roots endpoint](/consul/api-docs/connect/ca#list-ca-root-certificates) exposes just the public TLS certificate which other systems can use to verify the TLS connection with Consul. +- The [service mesh CA roots endpoint](/consul/api-docs/connect/ca#list-ca-root-certificates) exposes just the public TLS certificate which other systems can use to verify the TLS connection with Consul. -> **Consul Enterprise Namespace** - In addition to directly-linked policies, roles, and service identities, Consul Enterprise enables ACL policies and roles to be defined in the [Namespaces definition](/consul/docs/enterprise/namespaces#namespace-definition) (Consul Enterprise 1.7.0+). @@ -586,8 +586,7 @@ These actions may required an ACL token to complete. Use the following methods t This allows a single token to be used during all check registration operations. * Provide an ACL token with `service` and `check` definitions at registration time. This allows for greater flexibility and enables the use of multiple tokens on the same agent. - Refer to the [services](/consul/docs/discovery/services) and [checks](/consul/docs/discovery/checks) documentation for examples. - Tokens may also be passed to the [HTTP API](/consul/api-docs) for operations that require them. + Refer to the [services](/consul/docs/services/usage/define-services) and [checks](/consul/docs/services/usage/checks) documentation for examples. You can also pass tokens to the [HTTP API](/consul/api-docs) for operations that require them. ### Reading Imported Nodes @@ -815,12 +814,12 @@ to use for registration events: 2. Providing an ACL token with service and check definitions at registration time. This allows for greater flexibility and enables the use of multiple tokens on the same agent. Examples of what this looks like are available for - both [services](/consul/docs/discovery/services) and - [checks](/consul/docs/discovery/checks). Tokens may also be passed to the [HTTP - API](/consul/api-docs) for operations that require them. **Note:** all tokens + both [services](/consul/docs/services/usage/define-services) and + [checks](/consul/docs/services/usage/checks). Tokens may also be passed to the [HTTP + API](/consul/api-docs) for operations that require them. Note that all tokens passed to an agent are persisted on local disk to allow recovery from - restarts. See [`-data-dir` flag - documentation](/consul/docs/agent/config/cli-flags#_data_dir) for notes on securing + restarts. Refer to [`-data-dir` flag + documentation](/consul/docs/agent/config/cli-flags#_data_dir) for information about securing access. In addition to ACLs, in Consul 0.9.0 and later, the agent must be configured with @@ -870,7 +869,7 @@ service "app" { -Refer to [Intention Management Permissions](/consul/docs/connect/intentions#intention-management-permissions) +Refer to [ACL requirements for intentions](/consul/docs/connect/intentions/create-manage-intentions#acl-requirements) for more information about managing intentions access with service rules. ## Session Rules diff --git a/website/content/docs/security/acl/auth-methods/index.mdx b/website/content/docs/security/acl/auth-methods/index.mdx index c865589a853..c26cb05d21f 100644 --- a/website/content/docs/security/acl/auth-methods/index.mdx +++ b/website/content/docs/security/acl/auth-methods/index.mdx @@ -27,8 +27,8 @@ integrations allow for these credentials to be used to create ACL tokens with properly-scoped policies without additional operator intervention. In Consul 1.5.0 the focus is around simplifying the creation of tokens with the -privileges necessary to participate in a [Connect](/consul/docs/connect) -service mesh with minimal operator intervention. +privileges necessary to participate in a [service mesh](/consul/docs/connect) +with minimal operator intervention. ## Supported Types diff --git a/website/content/docs/security/acl/auth-methods/kubernetes.mdx b/website/content/docs/security/acl/auth-methods/kubernetes.mdx index b0a432884a0..13f76481c98 100644 --- a/website/content/docs/security/acl/auth-methods/kubernetes.mdx +++ b/website/content/docs/security/acl/auth-methods/kubernetes.mdx @@ -76,14 +76,13 @@ The Kubernetes service account corresponding to the configured [`ServiceAccountJWT`](/consul/docs/security/acl/auth-methods/kubernetes#serviceaccountjwt) needs to have access to two Kubernetes APIs: -- [**TokenReview**](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#create-tokenreview-v1-authentication-k8s-io) +- [**TokenReview**](https://kubernetes.io/docs/reference/kubernetes-api/authentication-resources/token-review-v1/) -> Kubernetes should be running with `--service-account-lookup`. This is defaulted to true in Kubernetes 1.7, but any versions prior should ensure the Kubernetes API server is started with this setting. -- [**ServiceAccount**](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#read-serviceaccount-v1-core) - (`get`) +- [**ServiceAccount**](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#service-account-tokens) The following is an example [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) diff --git a/website/content/docs/security/acl/auth-methods/oidc.mdx b/website/content/docs/security/acl/auth-methods/oidc.mdx index ea62c510083..7d22f01df7d 100644 --- a/website/content/docs/security/acl/auth-methods/oidc.mdx +++ b/website/content/docs/security/acl/auth-methods/oidc.mdx @@ -73,7 +73,7 @@ parameters are required to properly configure an auth method of type - `JWTSupportedAlgs` `(array)` - JWTSupportedAlgs is a list of supported signing algorithms. Defaults to `RS256`. ([Available - algorithms](https://github.com/hashicorp/consul/blob/main/vendor/github.com/coreos/go-oidc/jose.go#L7)) + algorithms](https://github.com/hashicorp/consul/blob/main/internal/go-sso/oidcauth/jwt.go)) - `BoundAudiences` `(array)` - List of `aud` claims that are valid for login; any match is sufficient. diff --git a/website/content/docs/security/acl/tokens/create/create-a-consul-esm-token.mdx b/website/content/docs/security/acl/tokens/create/create-a-consul-esm-token.mdx new file mode 100644 index 00000000000..cf33e95248f --- /dev/null +++ b/website/content/docs/security/acl/tokens/create/create-a-consul-esm-token.mdx @@ -0,0 +1,382 @@ +--- +layout: docs +page_title: Create tokens for for Consul external service monitor +description: >- + Learn how to create ACL tokens for the Consul external service monitor +--- + +# Create a Consul ESM token + +This topic describes how to create a token for the Consul external service monitor. + +## Introduction + +Consul external service monitor (ESM) can monitor third-party or external services in contexts where you are unable to run a Consul agent. To learn more about Consul ESM, refer to the [Register External Services with Consul Service Discovery](/consul/tutorials/developer-discovery/service-registration-external-services) tutorial. + + +## Requirements + +Core ACL functionality is available in all versions of Consul. + +Consul ESM must present a token linked to policies that grant the following permissions: + +* `agent:read`: Enables checking version compatibility and calculating network coordinates +* `key:write`: Enables storing state in the Consul KV store +* `node:read`: Enables discovering Consul nodes to monitor +* `node:write`: Enables updating status for the nodes that Consul ESM monitors +* `service:write`: Enables Consul ESM to register as a service in the catalog +* `session:write`: Enables Consul ESM is registered to acquire a leader lock +* `acl:read`: (Enterprise-only) Enables Consul ESM to scan namespaces for nodes and health checks to monitor + +Consul ESM only supports `default` admin partitions. + +@include 'create-token-requirements.mdx' + +## Consul ESM token in Consul OSS + +To create a token for Consul ESM, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants the appropriate permissions for Consul ESM running on an agent with the node name `agent1` to monitor two nodes named `node1` and `node2`. It allows Consul ESM to register into the catalog as the `consul-esm` service and write keys with the prefix `consul-esm/` in the Consul KV store. + + + +```hcl +agent "agent1" { + policy = "read" +} +key_prefix "consul-esm/" { + policy = "write" +} +node_prefix "" { + policy = "read" +} +service "consul-esm" { + policy = "write" +} +session "agent1" { + policy = "write" +} +node "node1" { + policy = "write" +} +node "node2" { + policy = "write" +} +``` + +```json +{ + "agent": { + "agent1": [{ + "policy": "read" + }] + }, + "key_prefix": { + "consul-esm/": [{ + "policy": "write" + }] + }, + "node": { + "node1": [{ + "policy": "write" + }], + "node2": [{ + "policy": "write" + }] + }, + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service": { + "consul-esm": [{ + "policy": "write" + }] + }, + "session": { + "agent1": [{ + "policy": "write" + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + +The following example registers a policy defined in `esm-policy.hcl`. + +```shell-session +$ consul acl policy create \ + -name "esm-policy" -rules @esm-policy.hcl \ + -description "Policy for Consul ESM" +``` + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + +The following example registers the policy defined in `esm-policy.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "esm-policy", + "Description": "Policy for Consul ESM", + "Rules": "agent \"agent1\" {\n policy = \"read\"\n}\nkey_prefix \"consul-esm/\" {\n policy = \"write\"\n}\nnode_prefix \"\" {\n policy = \"read\"\n}\nservice \"consul-esm\" {\n policy = \"write\"\n}\nsession \"agent1\" {\n policy = \"write\"\n}\nnode \"node1\" {\n policy = \"write\"\n}\nnode \"node2\" {\n policy = \"write\"\n}\n" +}' +``` + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following example creates an ACL token linked to the policy `esm-policy`. + +```shell-session +$ consul acl token create \ + -description "Token for Consul ESM" \ + -policy-name "esm-policy" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates an ACL token linked to the policy `esm-policy`. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "esm-policy" + } + ] +}' +``` + + + + + + +## Consul ESM token in Consul Enterprise + +To create a token for Consul ESM, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants the appropriate permissions for Consul ESM running on an agent named `agent1` to monitor two nodes named `node1` and `node2`. It allows Consul ESM to register into the catalog as the `consul-esm` service, to write keys with the prefix `consul-esm/` in the Consul KV store, and to scan the `default` and `ns1` namespaces for nodes and health checks to monitor. + + + +```hcl +partition "default" { + agent "agent1" { + policy = "read" + } + key_prefix "consul-esm/" { + policy = "write" + } + node_prefix "" { + policy = "read" + } + service "consul-esm" { + policy = "write" + } + session "agent1" { + policy = "write" + } + + node "node1" { + policy = "write" + } + node "node1" { + policy = "write" + } + + namespace "default" { + acl = "read" + } + namespace "ns1" { + acl = "read" + } +} +``` + +```json +{ + "partition": { + "default": [{ + "agent": { + "agent1": [{ + "policy": "read" + }] + }, + "key_prefix": { + "consul-esm/": [{ + "policy": "write" + }] + }, + "namespace": { + "default": [{ + "acl": "read" + }], + "ns1": [{ + "acl": "read" + }] + }, + "node": { + "node1": [{ + "policy": "write" + }, + { + "policy": "write" + }] + }, + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service": { + "consul-esm": [{ + "policy": "write" + }] + }, + "session": { + "agent1": [{ + "policy": "write" + }] + } + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + +You can specify an admin partition and namespace when creating policies in Consul Enterprise. The policy is only valid in the specified scopes. The example policy contains permissions for multiple namespaces in multiple partitions. You must create ACL policies that grant permissions for multiple namespaces in multiple partitions in the `default` namespace and the `default` partition. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + +The following command registers a policy defined in `esm-policy.hcl`. + +```shell-session +$ consul acl policy create -partition "default" -namespace "default" \ + -name "esm-policy" -rules @esm-policy.hcl \ + -description "Policy for Consul ESM" +``` + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + +The following example registers the policy defined in `esm-policy.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "esm-policy", + "Description": "Policy for Consul ESM", + "Partition": "default", + "Namespace": "default", + "Rules": "partition \"default\" {\n agent \"agent1\" {\n policy = \"read\"\n }\n key_prefix \"consul-esm/\" {\n policy = \"write\"\n }\n node_prefix \"\" {\n policy = \"read\"\n }\n service \"consul-esm\" {\n policy = \"write\"\n }\n session \"agent1\" {\n policy = \"write\"\n }\n\n node \"node1\" {\n policy = \"write\"\n }\n node \"node1\" {\n policy = \"write\"\n }\n\n namespace \"default\" {\n acl = \"read\"\n }\n namespace \"ns1\" {\n acl = \"read\"\n }\n}\n" +}' +``` + + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + +You can specify an admin partition and namespace when creating tokens in Consul Enterprise. The token must be created in the partition and namespace where the policy was created. The following example creates an ACL token in the `default` namespace in the `default` partition. + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + + +The following command creates the ACL token linked to the policy `esm-policy`. + +```shell-session +$ consul acl token create -partition "default" -namespace "default" \ + -description "Token for Consul ESM" \ + -policy-name "esm-policy" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates an ACL token linked to the policy `esm-policy`. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "esm-policy" + } + ], + "Partition": "default", + "Namespace": "default" +}' +``` + + + + diff --git a/website/content/docs/security/acl/tokens/create/create-a-dns-token.mdx b/website/content/docs/security/acl/tokens/create/create-a-dns-token.mdx new file mode 100644 index 00000000000..f4d87b7724b --- /dev/null +++ b/website/content/docs/security/acl/tokens/create/create-a-dns-token.mdx @@ -0,0 +1,331 @@ +--- +layout: docs +page_title: Create tokens for service registration +description: >- + Learn how to create ACL tokens to enable Consul DNS. +--- + +# Create a DNS token + +This topic describes how to create a token that enables the Consul DNS to query services in the network when ACLs are enabled. + +## Introduction + +The Consul binary ships with a DNS server that you can use for service discovery in your network. The agent that fulfills DNS lookups requires appropriate ACL permissions to discover services, nodes, and prepared queries registered in Consul. + +A Consul agent must be configured with a token linked to policies that grant the appropriate set of permissions. + +Specify the [`default`](/consul/docs/agent/config/config-files#acl_tokens_default) token to the Consul agent to authorize the agent to respond to DNS queries. Refer to [DNS usage overview](/consul/docs/services/discovery/dns-overview) for details on configuring and using Consul DNS. + +## Requirements + +Core ACL functionality is available in all versions of Consul. + +The DNS token must be linked to policies that grant the following permissions: + +* `service:read`: Enables the agent to perform service lookups for DNS +* `node:read`: Enables node lookups over DNS +* `query:read`: Enables the agent to perform prepared query lookups for DNS + +@include 'create-token-requirements.mdx' + +## DNS token in Consul OSS + +To create a token for DNS, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants the appropriate permissions to enable a Consul agent to respond to DNS queries. + + + +```hcl +node_prefix "" { + policy = "read" +} +service_prefix "" { + policy = "read" +} +query_prefix "" { + policy = "read" +} +``` + +```json +{ + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "query_prefix": { + "": [{ + "policy": "read" + }] + }, + "service_prefix": { + "": [{ + "policy": "read" + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + +The following example registers a policy defined in `dns-access.hcl`. + +```shell-session +$ consul acl policy create \ + -name "dns-access" -rules @dns-access.hcl \ + -description "DNS Policy" +``` + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + +The following example registers the policy defined in `dns-access.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "dns-access", + "Description": "DNS Policy", + "Rules": "node_prefix \"\" {\n policy = \"read\"\n}\nservice_prefix \"\" {\n policy = \"read\"\n}\nquery_prefix \"\" {\n policy = \"read\"\n}\n" +}' +``` + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following command creates the ACL token linked to the policy `dns-access`. + +```shell-session +$ consul acl token create \ + -description "DNS token" \ + -policy-name "dns-access" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates the ACL token linked to the policy `dns-access`. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "dns-access" + } + ] +}' +``` + + + + + +## DNS token in Consul Enterprise + +To create a token for DNS, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants the appropriate permissions to enable a Consul agent to respond to DNS queries for resources in any namespace in any partition. + + + +```hcl +partition_prefix "" { + namespace_prefix "" { + node_prefix "" { + policy = "read" + } + service_prefix "" { + policy = "read" + } + query_prefix "" { + policy = "read" + } + } +} +``` + +```json +{ + "partition_prefix": { + "": [{ + "namespace_prefix": { + "": [{ + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "query_prefix": { + "": [{ + "policy": "read" + }] + }, + "service_prefix": { + "": [{ + "policy": "read" + }] + } + }] + } + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + +You can specify an admin partition when creating policies in Consul Enterprise. The policy is only valid in the specified admin partition. The example policy contains permissions for multiple namespaces in multiple partitions. You must create ACL policies that grant permissions for multiple namespaces in multiple partitions in the `default` namespace and the `default` partition. + + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + +```shell-session +consul acl policy create -partition "default" -namespace "default" \ + -name dns-access -rules @dns-access.hcl \ + -description "DNS Policy" +``` + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + +The following example registers the policy defined in `dns-access.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "dns-access", + "Description": "DNS Policy", + "Partition": "default", + "Namespace": "default", + "Rules": "partition_prefix \"\" {\n namespace_prefix \"\" {\n node_prefix \"\" {\n policy = \"read\"\n }\n service_prefix \"\" {\n policy = \"read\"\n }\n query_prefix \"\" {\n policy = \"read\"\n }\n }\n}\n" +}' +``` + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following command creates the ACL token linked to the policy `dns-access`. + +```shell-session +$ consul acl token create -partition "default" -namespace "default" \ + -description "DNS token" \ + -policy-name "dns-access" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates the ACL token linked to the policy `dns-access`. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "dns-access" + } + ], + "Partition": "default", + "Namespace": "default" +}' +``` + + + + + +## Apply the token + +Configure the Consul agent with the token by either specifying the token in the agent configuration file or by using the `consul set-agent-token` command. + +### Apply the token in a file + +Specify the token in the [`default`](/consul/docs/agent/config/config-files#acl_tokens_default) field of the agent configuration file so that the agent can present it and register into the catalog on startup. + +```hcl +acl = { + enabled = true + tokens = { + default = "" + ... + } + ... +} +``` + +### Apply the token with a command + +Set the `default` token using the [`acl.token.default`](/consul/docs/agent/config/config-files#acl_tokens_default) command. The following command configures a running Consul agent token with the specified token. + +```shell-session +$ consul set-agent-token default +``` + diff --git a/website/content/docs/security/acl/tokens/create/create-a-mesh-gateway-token.mdx b/website/content/docs/security/acl/tokens/create/create-a-mesh-gateway-token.mdx new file mode 100644 index 00000000000..6e94395d31d --- /dev/null +++ b/website/content/docs/security/acl/tokens/create/create-a-mesh-gateway-token.mdx @@ -0,0 +1,508 @@ +--- +layout: docs +page_title: Create a token for mesh gateway registration +description: >- + Learn how to create ACL tokens that your mesh gateway can present to Consul servers so that they + can register with the Consul catalog. +--- + +# Create a mesh gateway token + +This topic describes how to create a token for a mesh gateway. + +## Introduction + +Mesh gateways enable service-to-service traffic between Consul datacenters or between Consul admin +partitions. They also enable datacenters to be federated across wide area networks. Refer to [Mesh +Gateways](/consul/docs/connect/gateways#mesh-gateways) for additional information. + +Gateways must present a token linked to policies that grant the appropriate set of permissions in +order to be discoverable and to route to other services in a mesh. + +## Requirements + +Core ACL functionality is available in all versions of Consul. + +The mesh gateway must present a token linked to a policy that grants the following permissions: + +* `mesh:write` to obtain leaf certificates for terminating TLS connections +* `peering:read` for Consul cluster peering through mesh gateways. If you are not using cluster + peering or if the mesh gateway is not in the `default` partition, then you can omit the + `peering:read` permission. +* `service:write` to allow the mesh gateway to register into the catalog +* `service:read` for all services and `node:read` for all nodes in order to discover and route to services +* `agent:read` to enable the `consul connect envoy` CLI command to automatically discover gRPC + settings from the Consul agent. If this command is not used to start the gateway or if the Consul + agent uses the default gRPC settings, then you can omit the `agent:read` permission. + +@include 'create-token-requirements.mdx' + +## Consul OSS + +To create a token for the mesh gateway, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants the appropriate permissions to register as a service named `mesh-gateway` and to operate as a mesh gateway. + + + +```hcl +mesh = "write" +peering = "read" +service "mesh-gateway" { + policy = "write" +} +service_prefix "" { + policy = "read" +} +node_prefix "" { + policy = "read" +} +agent_prefix "" { + policy = "read" +} +``` + +```json +{ + "mesh": "write", + "peering": "read", + "service": { + "mesh-gateway": [{ + "policy": "write" + }] + }, + "service_prefix": { + "": [{ + "policy": "read" + }] + }, + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "agent_prefix": { + "": [{ + "policy": "read" + }] + } +} +``` + + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + +The following commands create the ACL policy and token. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `mgw-register.hcl`: + +```shell-session +$ consul acl policy create \ + -name "mgw-register" -rules @mgw-register.hcl \ + -description "Mesh gateway policy" +``` + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `mgw-register.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + –-header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "mgw-register", + "Description": "Mesh gateway policy", + "Rules": "mesh = \"write\"\npeering = \"read\"\nservice \"mesh-gateway\" {\n policy = \"write\"\n}\nservice_prefix \"\" {\n policy = \"read\"\n}\nnode_prefix \"\" {\n policy = \"read\"\n}\nagent_prefix \"\" {\n policy = \"read\"\n}\n" +}' +``` + +Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following command creates the ACL token linked to the policy `mgw-register`. + +```shell-session +$ consul acl token create \ + -description "Mesh gateway token" \ + -policy-name "mgw-register" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + –-header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "mgw-register" + } + ] +}' +``` + + + + + +## Consul Enterprise in default partition + +To create a token for the mesh gateway, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +You can specify an admin partition and namespace when using Consul Enterprise. Mesh gateways must register into the `default` namespace. + +The following example policy is defined in a file. The policy grants the appropriate permissions to register as a service named `mesh-gateway` and to operate as a mesh gateway in the default partition. + + + +```hcl +mesh = "write" +partition_prefix "" { + peering = "read" +} +partition "default" { + namespace "default" { + service "mesh-gateway" { + policy = "write" + } + agent_prefix "" { + policy = "read" + } + } + namespace_prefix "" { + node_prefix "" { + policy = "read" + } + service_prefix "" { + policy = "read" + } + } +} +``` + +```json +{ + "mesh": "write", + "partition": { + "default": [{ + "namespace": { + "default": [{ + "service": { + "mesh-gateway": [{ + "policy": "write" + }] + }, + "agent_prefix": { + "": [{ + "policy": "read" + }] + } + }] + }, + "namespace_prefix": { + "": [{ + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service_prefix": { + "": [{ + "policy": "read" + }] + } + }] + } + }] + }, + "partition_prefix": { + "": [{ + "peering": "read" + }] + } +} +``` + + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + +The following commands create the ACL policy and token. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `mgw-register.hcl`: + +You can specify an admin partition when creating policies in Consul Enterprise. The policy is only valid in the specified admin partition. You must create the policy in the partition where the mesh gateway registers. The following example creates the policy in the `default` partition. + +```shell-session +$ consul acl policy create -partition "default" \ + -name mgw-register -rules @mgw-register.hcl \ + -description "Mesh gateway policy" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `mgw-register.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + –-header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "mgw-register", + "Description": "Mesh gateway policy", + "Partition": "default", + "Rules": "mesh = \"write\"\npeering = \"read\"\npartition_prefix \"\" {\n peering = \"read\"\n}\nnamespace \"default\" {\n service \"mesh-gateway\" {\n policy = \"write\"\n }\n agent_prefix \"\" {\n policy = \"read\"\n }\n}\nnamespace_prefix \"\" {\n node_prefix \"\" {\n \tpolicy = \"read\"\n }\n service_prefix \"\" {\n policy = \"read\"\n }\n}\n" +}' +``` + +Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +You can specify an admin partition when creating tokens in Consul Enterprise. The token is only valid in the specified admin partition. The token must be created in the partition where the mesh gateway registers. The following example creates the token in the partition `ptn1`. + +```shell-session +$ consul acl token create -partition "default" \ + -description "Mesh gateway token" \ + -policy-name "mgw-register" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +You can specify an admin partition when creating tokens in Consul Enterprise. The token is only valid in the specified admin partition. The token must be created in the partition where the mesh gateway registers. The following example creates the token in the partition `ptn1`. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + –-header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "mgw-register" + } + ], + "Partition": "default" +}' +``` + + + + + +## Consul Enterprise in non-default partition + +To create a token for the mesh gateway, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +You can specify an admin partition and namespace when using Consul Enterprise. Mesh gateways must register into the `default` namespace. To register a mesh gateway in a non-default partition, create the ACL policy and token in the partition where the mesh gateway registers. + +The following example policy is defined in a file. The policy grants the appropriate permissions to register as a service named `mesh-gateway` and to operate as a mesh gateway in a non-default partition. + + + +```hcl +mesh = "write" +namespace "default" { + service "mesh-gateway" { + policy = "write" + } + agent_prefix "" { + policy = "read" + } +} +namespace_prefix "" { + node_prefix "" { + policy = "read" + } + service_prefix "" { + policy = "read" + } +} +``` + +```json +{ + "mesh": "write", + "namespace": { + "default": [{ + "service": { + "mesh-gateway": [{ + "policy": "write" + }] + }, + "agent_prefix": { + "": [{ + "policy": "read" + }] + } + }] + }, + "namespace_prefix": { + "": [{ + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service_prefix": { + "": [{ + "policy": "read" + }] + } + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + +The following commands create the ACL policy and token. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `mgw-register.hcl`: + +You can specify an admin partition when creating policies in Consul Enterprise. The policy is only valid in the specified admin partition. You must create the policy in the partition where the mesh gateway registers. The following example creates the policy in the partition `ptn1`. + +```shell-session +$ consul acl policy create -partition "ptn1" \ + -name mgw-register -rules @mgw-register.hcl \ + -description "Mesh gateway policy" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `mgw-register.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + –-header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "mgw-register", + "Description": "Mesh gateway policy", + "Partition": "ptn1", + "Rules": "mesh = \"write\"\npeering = \"read\"\nnamespace \"default\" {\n service \"mesh-gateway\" {\n policy = \"write\"\n }\n agent_prefix \"\" {\n policy = \"read\"\n }\n}\nnamespace_prefix \"\" {\n node_prefix \"\" {\n \tpolicy = \"read\"\n }\n service_prefix \"\" {\n policy = \"read\"\n }\n}\n" +}' +``` + +Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +You can specify an admin partition when creating tokens in Consul Enterprise. The token is only valid in the specified admin partition. The token must be created in the partition where the mesh gateway registers. The following example creates the token in the partition `ptn1`. + +```shell-session +$ consul acl token create -partition "ptn1" \ + -description "Mesh gateway token" \ + -policy-name "mgw-register" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +You can specify an admin partition when creating tokens in Consul Enterprise. The token is only valid in the specified admin partition. The token must be created in the partition where the mesh gateway registers. The following example creates the token in the partition `ptn1`. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + –-header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "mgw-register" + } + ], + "Partition": "ptn1" +}' +``` + + + + diff --git a/website/content/docs/security/acl/tokens/create/create-a-replication-token.mdx b/website/content/docs/security/acl/tokens/create/create-a-replication-token.mdx new file mode 100644 index 00000000000..f38f4682fcc --- /dev/null +++ b/website/content/docs/security/acl/tokens/create/create-a-replication-token.mdx @@ -0,0 +1,312 @@ +--- +layout: docs +page_title: Create tokens for service registration +description: >- + Learn how to create ACL tokens that a server agent in a secondary datacenter can use for ACL token replication between WAN-federated datacenters. +--- + +# Create a replication token + +This topic describes how to configure an ACL token for ACL replication between WAN-federated datacenters. If your Consul clusters are connected through peer connections, ACL replication is not required. To learn more about cluster peering, refer to the [comparison between WAN federation and cluster peering](/consul/docs/connect/cluster-peering#compared-with-wan-federation). + +## Introduction + +Consul agents must present a token linked to policies that grant the appropriate set of permissions. +Specify the [`replication`](/consul/docs/agent/config/config-files#acl_tokens_replication) token on each server in a non-primary datacenter. For hands-on instructions on how to configure ACL replication across datacenters, refer to the [ACL Replication for Multiple Datacenters](/consul/tutorials/security-operations/access-control-replication-multiple-datacenters) tutorial. + + +## Requirements + +Core ACL functionality is available in all versions of Consul. + +For a Consul server agent with ACL replication enabled in a secondary datacenter, the token must be linked to a policy that grants the following permissions: + +* `acl:write`: Enables replication of ACL resources +* `operator:write`: Enables replication of the proxy-defaults configuration entry and enables CA certification signing in the secondary datacenter +* `service:read` and `intention:read`: Enables replication of the service-defaults and intentions configuration entries + +@include 'create-token-requirements.mdx' + +## Replication token in Consul OSS + +To create a token for ACL replication, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants the appropriate permissions for ACL replication. + + + +```hcl +acl = "write" +operator = "write" +service_prefix "" { + policy = "read" + intentions = "read" +} +``` + +```json +{ + "acl": "write", + "operator": "write", + "service_prefix": { + "": [{ + "intentions": "read", + "policy": "read" + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + +The following example registers a policy defined in `acl-replication.hcl`. + +```shell-session +$ consul acl policy create \ + -name "acl-replication" -rules @acl-replication.hcl \ + -description "ACL replication token" +``` + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + +The following example registers the policy defined in `acl-replication.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "acl-replication", + "Description": "ACL replication", + "Rules": "acl = \"write\"\noperator = \"write\"\nservice_prefix \"\" {\n policy = \"read\"\n intentions = \"read\"\n}\n" +}' +``` + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an auth method. + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following command creates the ACL token linked to the policy `acl-replication`. + +```shell-session +$ consul acl token create \ + -description "ACL replication" \ + -policy-name "acl-replication" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates the ACL token linked to the policy `acl-replication`. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "acl-replication" + } + ] +}' +``` + + + + + +## Replication token in Consul Enterprise + +To create a token for ACL replication, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The following example policy grants the appropriate permissions for ACL replication. + + + +```hcl +operator = "write" +service_prefix "" { + policy = "read" + intentions = "read" +} +namespace_prefix "" { + acl = "write" + service_prefix "" { + policy = "read" + intentions = "read" + } +} +``` + +```json +{ + "namespace_prefix": { + "": [{ + "acl": "write", + "service_prefix": { + "": [{ + "intentions": "read", + "policy": "read" + }] + } + }] + }, + "operator": "write", + "service_prefix": { + "": [{ + "intentions": "read", + "policy": "read" + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + +You can specify an admin partition, namespace, or both when registering policies in Consul Enterprise. Policies are only valid in the specified scopes. The policy for replication must be created in the `default` namespace and `default` partition. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + +The following example registers a policy defined in `acl-replication.hcl`. + +```shell-session +$ consul acl policy create -partition "default" -namespace "default" \ + -name "acl-replication" -rules @acl-replication.hcl \ + -description "ACL replication token" +``` + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + +The following example registers the policy defined in `acl-replication.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "acl-replication", + "Description": "ACL replication", + "Partition": "default", + "Namespace": "default", + "Rules": "operator = \"write\"\nservice_prefix \"\" {\n policy = \"read\"\n intentions = \"read\"\n}\nnamespace_prefix \"\" {\n acl = \"write\"\n service_prefix \"\" {\n policy = \"read\"\n intentions = \"read\"\n }\n}\n" +}' +``` + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +```shell-session +$ consul acl token create -partition "default" -namespace "default" \ + -description "ACL replication" \ + -policy-name "acl-replication" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates the ACL token linked to the policy `acl-replication`. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "acl-replication" + } + ], + "Partition": "default", + "Namespace": "default" +}' +``` + + + + + +## Apply the token + +Configure the Consul agent with the token by either specifying the token in the agent configuration file or by using the `consul set-agent-token` command. + +### Apply the token in a file + +Specify the token in the [`replication`](/consul/docs/agent/config/config-files#acl_tokens_replication) field of the agent configuration file so that the agent can present it and register into the catalog on startup. + +```hcl +acl = { + enabled = true + tokens = { + replication = "" + ... + } + ... +} +``` + +### Apply the token with a command + +Set the `replication` token using the [`consul set-agent-token`](/consul/commands/acl/set-agent-token) command. The following command configures a running Consul agent token with the specified token. + +```shell-session +$ consul set-agent-token replication +``` + diff --git a/website/content/docs/security/acl/tokens/create/create-a-service-token.mdx b/website/content/docs/security/acl/tokens/create/create-a-service-token.mdx new file mode 100644 index 00000000000..125390b3983 --- /dev/null +++ b/website/content/docs/security/acl/tokens/create/create-a-service-token.mdx @@ -0,0 +1,416 @@ +--- +layout: docs +page_title: Create tokens for service registration +description: >- + Learn how to create ACL tokens that your services can present to Consul servers so that they can register with the Consul catalog. +--- + +# Create a service token + +This topic describes how to create a token that you can use to register a service and discover services in the Consul catalog. If you are using Consul service mesh, a sidecar proxy can use the token to discover and route traffic to other services. + +## Introduction + +Services must present a token linked to policies that grant the appropriate set of permissions in order to be discoverable or to interact with other services in a mesh. + +### Service identities versus custom policies + +You can create tokens linked to custom policies or to service identities. [Service identities](/consul/docs/security/acl#service-identities) are constructs in Consul that enable you to quickly grant permissions for a group of services, rather than creating similar policies for each service. + +We recommend using a service identity to grant permissions for service discovery and service mesh use cases rather than creating a custom policy. This is because service identities automatically grant the service and its sidecar proxy `service:write`, `service:read`, and `node:read`. + +Your organization may have requirements or processes for deploying services in a way that is inconsistent with service and node identities. In these cases, you can create custom policies and link them to tokens. + +## Requirements + +Core ACL functionality is available in all versions of Consul. + +The service token must be linked to policies that grant the following permissions: + +* `service:write`: Enables the service to update the catalog. If service mesh is enabled, the service's sidecar proxy can also update the catalog. Note that this permission implicitly grants `intention:read` permission to sidecar proxies so that they can read and enforce intentions. Refer to [Intention Management Permissions](/consul/docs/connect/intentions#intention-management-permissions) for details. +* `service:read`: Enables the service to learn about other services in the network. If service mesh is enabled, the service's sidecar proxy can also learn about other services in the network. +* `node:read`: Enables the sidecar proxy to discover and route traffic to other services in the catalog if service mesh is enabled. + +@include 'create-token-requirements.mdx' + +## Service identity in Consul OSS + +Refer to [Service identities](/consul/docs/security/acl#service-identities) for information about creating service identities that you can link to tokens. + +You can manually create tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy or service identity to link to create a token. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following example creates an ACL token linked to a service identity for a service named `svc1`. + +```shell-session +$ consul acl token create \ + -description "Service token for svc1" \ + -service-identity "svc1" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify a service identity in the request body to create a token linked to the service identity. An ACL token linked to a policy with permissions to use the API endpoint is required. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates a token linked to a service identity named `svc1`: + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "ServiceIdentities": [ + { + "ServiceName": "svc1" + } + ] +}' +``` + + + + + +## Service identity in Consul Enterprise + +Refer to [Service identities](/consul/docs/security/acl#service-identities) for information about creating service identities that you can link to tokens. + +You can manually create tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy or service identity to link to create a token. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +You can specify an admin partition, namespace, or both when creating tokens in Consul Enterprise. The token can only include permissions in the specified scope, if any. The following example creates an ACL token that the service can use to register in the `ns1` namespace of partition `ptn1`: + +```shell-session +$ consul acl token create -partition "ptn1" -namespace "ns1" \ + -description "Service token for svc1" \ + -service-identity "svc1" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify a service identity in the request body to create a token linked to the service identity. An ACL token linked to a policy with permissions to use the API endpoint is required. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +You can specify an admin partition and namespace when creating tokens in Consul Enterprise. The token is only valid in the specified scopes. The following example creates an ACL token that the service can use to register in the `ns1` namespace of partition `ptn1`: + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "ServiceIdentities": [ + { + "ServiceName": "svc1" + } + ], + "Namespace": "ns1", + "Partition": "ptn1" +}' +``` + + + + + +## Custom policy in Consul OSS + +When you are unable to link tokens to a service identity, you can define policies, register them with Consul, and link the policies to tokens that enable services to register into the Consul catalog. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants the `svc1` service `write` permissions so that it can register into the catalog. For service mesh, the policy grants the `svc1-sidecar-proxy` service `write` permissions so that the sidecar proxy can register into the catalog. It grants service and node `read` permissions to discover and route to other services. + + + +```hcl +service "svc1" { + policy = "write" +} +service "svc1-sidecar-proxy" { + policy = "write" +} +service_prefix "" { + policy = "read" +} +node_prefix "" { + policy = "read" +} +``` + +```json +{ + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service": { + "svc1": [{ + "policy": "write" + }], + "svc1-sidecar-proxy": [{ + "policy": "write" + }] + }, + "service_prefix": { + "": [{ + "policy": "read" + }] + } +} +``` + + + + +### Register the policy with Consul + +After defining the policies, you can register them with Consul using the command line or API endpoint. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `svc1-register.hcl`: + + +```shell-session +$ consul acl policy create \ + -name "svc1-register" -rules @svc1-register.hcl \ + -description "Allow svc1 to register into the catalog" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl token create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `svc1-register.hcl`. You must embed policy rules in the `Rules` field of the request body + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "svc1-register", + "Description": "Allow svc1 to register into the catalog", + "Rules": "service \"svc1\" {\n policy = \"write\"\n}\nservice \"svc1-sidecar-proxy\" {\n policy = \"write\"\n}\nservice_prefix \"\" {\n policy = \"read\"\n}\nnode_prefix \"\" {\n policy = \"read\"\n}\n" +}' +``` + +Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + + + + + +### Link the policy to a token + +After registering the policies into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following commands create the ACL token linked to the policy `svc1-register`. + +```shell-session +$ consul acl token create \ + -description "Service token for svc1" \ + -policy-name "svc1-register" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates an ACL token that the `svc1` service can use to register in the `ns1` namespaces of partition `ptn1`: + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "svc1-register" + } + ] +}' +``` + + + + + +## Custom policy in Consul Enterprise + +When you are unable to link tokens to a service identity, you can define policies, register them with Consul, and link the policies to tokens that enable services to register into the Consul catalog. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +You can specify an admin partition and namespace when creating policies in Consul Enterprise. The policy is only valid in the specified scopes. + +The following example policy is defined in a file. The policy allows the `svc1` service to register in the `ns1` namespace of partition `ptn1`. For service mesh, the policy grants the `svc1-sidecar-proxy` service `write` permissions so that the sidecar proxy can register into the catalog. It grants service and node `read` permissions to discover and route to other services. + + + +```hcl +partition "ptn1" { + namespace "ns1" { + service "svc1" { + policy = "write" + } + service "svc1-sidecar-proxy" { + policy = "write" + } + service_prefix "" { + policy = "read" + } + node_prefix "" { + policy = "read" + } + } +} +``` + +```json +{ + "partition": { + "ptn1": [{ + "namespace": { + "ns1": [{ + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service": { + "svc1": [{ + "policy": "write" + }], + "svc1-sidecar-proxy": [{ + "policy": "write" + }] + }, + "service_prefix": { + "": [{ + "policy": "read" + }] + } + }] + } + }] + } +} +``` + + + + +### Register the policy with Consul + +After defining the policies, you can register them with Consul using the command line or API endpoint. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `svc1-register.hcl`: + + +```shell-session +$ consul acl policy create -partition "ptn1" -namespace "ns1" \ + -name "svc1-register" -rules @svc1-register.hcl \ + -description "Custom policy for service svc1" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl token create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `svc1-register.hcl`. You must embed policy rules in the `Rules` field of the request body + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "svc1-register", + "Description": "Allow svc1 to register into the catalog", + "Namespace": "ns1", + "Partition": "ptn1", + "Rules": "partition \"ptn1\" {\n namespace \"ns1\" {\n service \"svc1\" {\n policy = \"write\"\n }\n service \"svc1-sidecar-proxy\" {\n policy = \"write\"\n }\n service_prefix \"\" {\n policy = \"read\"\n }\n node_prefix \"\" {\n policy = \"read\"\n }\n }\n}\n" +}' +``` + +Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + + + + + +### Link the policy to a token + +After registering the policies into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +You can specify an admin partition and namespace when creating tokens in Consul Enterprise. The token is only valid in the specified scopes. The following example creates an ACL token that the service can use to register in the `ns1` namespace of partition `ptn1`: + +The following commands create the ACL token linked to the policy `svc1-register`. + +```shell-session +$ consul acl token create -partition "ptn1" -namespace "ns1" \ + -description "Service token for svc1" \ + -policy-name "svc1-register" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +You can specify an admin partition and namespace when creating tokens in Consul Enterprise. The token is only valid in the specified scopes. The following example creates an ACL token that the service can use to register in the `ns1` namespace of partition `ptn1`: + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "svc1-register" + } + ], + "Namespace": "ns1", + "Partition": "ptn1" +}' +``` + + + + diff --git a/website/content/docs/security/acl/tokens/create/create-a-snapshot-agent-token.mdx b/website/content/docs/security/acl/tokens/create/create-a-snapshot-agent-token.mdx new file mode 100644 index 00000000000..374498a93d7 --- /dev/null +++ b/website/content/docs/security/acl/tokens/create/create-a-snapshot-agent-token.mdx @@ -0,0 +1,173 @@ +--- +layout: docs +page_title: Create tokens for snapshot agents +description: >- + Learn how to create an ACL token for the Consul snapshot agent. +--- + +# Create a snapshot agent token + +This topic describes how to create a token for the Consul snapshot agent. + + + +## Introduction + +The `consul snapshot agent` command starts a process that takes snapshots of the state of the Consul +servers and either saves them locally or pushes them to a remote storage service. Refer to [Consul Snapshot Agent](/consul/commands/snapshot/agent) for additional information. + +## Requirements + +Core ACL functionality is available in all versions of Consul. + +### Requirements for the `agent` command + +The [`agent`](/consul/commands/snapshot/agent) subcommand requires [Consul Enterprise](https://www.hashicorp.com/products/consul/). All other [`snapshot` subcommands](/consul/commands/snapshot) are available in the open source version of Consul. + +### Snapshot agent ACL requirements + +The Consul snapshot agent must present a token linked to policies that grant the following set of permissions. + +* `acl:write`: Enables the agent read and snapshot ACL data +* `key:write`: Enables the agent to create a key in the Consul KV store that serves as a leader election lock when multiple snapshot agents are running in an environment +* `session:write`: Enables the agent to create sessions for the specified Consul node where it is running +* `service:write`: Enables the agent to register into the catalog + +@include 'create-token-requirements.mdx' + +## Create a token + +To create a token for the snapshot agent, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants the appropriate permissions for a snapshot agent running on a node named `server-1234` to register into the catalog as the `consul-snapshot` service. It uses the key `consul-snapshot/lock` for a leader election lock. + + + +```hcl +acl = "write" +key "consul-snapshot/lock" { + policy = "write" +} +session "server-1234" { + policy = "write" +} +service "consul-snapshot" { + policy = "write" +} +``` + +```json +{ + "acl": "write", + "key": { + "consul-snapshot/lock": [{ + "policy": "write" + }] + }, + "service": { + "consul-snapshot": [{ + "policy": "write" + }] + }, + "session": { + "server-1234": [{ + "policy": "write" + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + +You can specify an admin partition and namespace when creating policies in Consul Enterprise. Policies are only valid in the specified scopes. You must create the policy for the snapshot agent in the `default` namespace in the `default` partition. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + +The following example registers a policy defined in `snapshot-agent.hcl`: + +```shell-session +$ consul acl policy create -partition "default" -namespace "default" \ + -name snapshot-agent -rules @snapshot-agent.hcl \ + -description "Snapshot agent policy" +``` + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + +The following example registers the policy defined in `snapshot-agent.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "snapshot-agent", + "Description": "Snapshot agent policy", + "Partition": "default", + "Namespace": "default", + "Rules": "acl = \"write\"\nkey \"consul-snapshot/lock\" {\n policy = \"write\"\n}\nsession \"server-1234\" {\n policy = \"write\"\n}\nservice \"consul-snapshot\" {\n policy = \"write\"\n}\n" +}' +``` + + + + + +### Link the policy to a token + +After registering the policies into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + +You can specify an admin partition and namespace when creating tokens in Consul Enterprise. Tokens are only valid in the specified scopes. The snapshot agent token must be created in the `default` namespace in the `default` partition. + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following command creates the ACL token linked to the policy `snapshot-agent`. + +```shell-session +$ consul acl token create -partition "default" -namespace "default" \ + -description "Snapshot agent token" \ + -policy-name "snapshot-agent" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "snapshot-agent" + } + ], + "Partition": "default", + "Namespace": "default" +}' +``` + + + + + diff --git a/website/content/docs/security/acl/tokens/create/create-a-terminating-gateway-token.mdx b/website/content/docs/security/acl/tokens/create/create-a-terminating-gateway-token.mdx new file mode 100644 index 00000000000..5ba304a9f84 --- /dev/null +++ b/website/content/docs/security/acl/tokens/create/create-a-terminating-gateway-token.mdx @@ -0,0 +1,352 @@ +--- +layout: docs +page_title: Create a token for terminating gateway registration +description: >- + Learn how to create ACL tokens that your terminating gateway can present to Consul servers so that they can register with the Consul catalog. +--- + +# Create a terminating gateway token + +This topic describes how to create an ACL token that enables a terminating gateway to register with Consul. + +## Introduction + +Terminating gateways enable connectivity within your organizational network from services in the Consul service mesh to services and destinations outside the mesh. + +To learn how to configure terminating gateways, refer to the [Terminating Gateways](/consul/docs/connect/gateways/terminating-gateway#terminating-gateway-configuration) documentation and the [Understand Terminating Gateways](/consul/tutorials/developer-mesh/service-mesh-terminating-gateways) tutorial. + +## Requirements + +Core ACL functionality is available in all versions of Consul. + +The terminating gateway token must be linked to policies that grant the appropriate set of permissions in order to be discoverable and to forward traffic out of the mesh. The following permissions are required: + +* `service:write` to allow the terminating gateway to register into the catalog +* `service:write` for each service that it forwards traffic for +* `node:read` for the nodes of each service that it forwards traffic for +* `service:read` for all services and `node:read` for all nodes in order to discover and route to services +* `agent:read` to enable the `consul connect envoy` CLI command to automatically discover gRPC settings from the Consul agent. If this command is not used to start the gateway or if the Consul agent uses the default gRPC settings, then you can omit the `agent:read` permission. + +@include 'create-token-requirements.mdx' + +## Consul OSS + +To create a token for the terminating gateway, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants the appropriate permissions to register as a service named `terminating-gateway` and to operate as a terminating gateway. For this example, the terminating gateway forwards traffic for two services named `external-service-1` and `external-service-2`. The policy examples include `service:write` permissions for these services. If you have additional services, your policy must include `service:write` permissions for the additional services to be included in the policy rules. + + + +```hcl +service "terminating-gateway" { + policy = "write" +} +service "external-service-1" { + policy = "write" +} +service "external-service-2" { + policy = "write" +} +node_prefix "" { + policy = "read" +} +agent_prefix "" { + policy = "read" +} +``` + +```json +{ + "service": { + "terminating-gateway": [{ + "policy": "write" + }], + "external-service-1": [{ + "policy": "write" + }], + "external-service-2": [{ + "policy": "write" + }] + }, + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "agent_prefix": { + "": [{ + "policy": "read" + }] + } +} +``` + + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + +The following commands create the ACL policy and token. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `tgw-register.hcl`: + +```shell-session +$ consul acl policy create \ + -name "tgw-register" -rules @tgw-register.hcl \ + -description "Terminating gateway policy" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `tgw-register.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "tgw-register", + "Description": "Terminating gateway policy", + "Rules": "service \"terminating-gateway\" {\n policy = \"write\"\n}\nservice \"external-service-1\" {\n policy = \"write\"\n}\nservice \"external-service-2\" {\n policy = \"write\"\n}\nnode_prefix \"\" {\n policy = \"read\"\n}\nagent_prefix \"\" {\n policy = \"read\"\n}\n" +}' +``` + +Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following command creates the ACL token linked to the policy `tgw-register`. + +```shell-session +$ consul acl token create \ + -description "Terminating gateway token" \ + -policy-name "tgw-register" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "tgw-register" + } + ] +}' +``` + + + + + +## Consul Enterprise + +To create a token for the terminating gateway, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +You can specify an admin partition and namespace when creating policies in Consul Enterprise. The policy is only valid in the specified scopes. + +The following example policy is defined in a file. The policy grants the appropriate permissions for a terminating gateway to register as a service named `terminating-gateway` in namespace `ns1` in partition `ptn1`. + +For this example, the terminating gateway forwards traffic for the following two services: + +* `external-service-1` in the `default` namespace +* `external-service-2` in the `ns1` namespace + +The policy examples include `service:write` permissions for these services. If you have additional services, your policy must include `service:write` permissions for the additional services to be included in the policy rules. + +The policy contains permissions for resources in multiple namespaces. You must create ACL policies that grant permissions for multiple namespaces in the `default` namespace. + + + +```hcl +partition "ptn1" { + namespace "ns1" { + service "terminating-gateway" { + policy = "write" + } + node_prefix "" { + policy = "read" + } + service "external-service-2" { + policy = "write" + } + } + namespace "default" { + service "external-service-1" { + policy = "write" + } + node_prefix "" { + policy = "read" + } + agent_prefix "" { + policy = "read" + } + } +} +``` + +```json +{ + "partition": { + "ptn1": [{ + "namespace": { + "default": [{ + "agent_prefix": { + "": [{ + "policy": "read" + }] + }, + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service": { + "external-service-1": [{ + "policy": "write" + }] + } + }], + "ns1": [{ + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service": { + "external-service-2": [{ + "policy": "write" + }], + "terminating-gateway": [{ + "policy": "write" + }] + } + }] + } + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + +You can specify an admin partition and namespace when creating policies in Consul Enterprise. The policy is only valid in the specified admin partition and namespace. You must create the policy in the same partition where the terminating gateway is registered. If the terminating gateway requires permissions for multiple namespaces, then the policy must be created in the `default` namespace. The following example creates the policy in the partition `ptn1` and `default` namespace because the example policy contains permissions for multiple namespaces. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `tgw-register.hcl`: + +```shell-session +$ consul acl policy create -partition "ptn1" -namespace "default" \ + -name "tgw-register" -rules @tgw-register.hcl \ + -description "Terminating gateway policy" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `tgw-register.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "tgw-register", + "Description": "Terminating gateway policy", + "Partition": "ptn1", + "Namespace": "default", + "Rules": "partition \"ptn1\" {\n namespace \"ns1\" {\n service \"terminating-gateway\" {\n policy = \"write\"\n }\n node_prefix \"\" {\n policy = \"read\"\n }\n service \"external-service-2\" {\n policy = \"write\"\n }\n }\n namespace \"default\" {\n service \"external-service-1\" {\n policy = \"write\"\n }\n node_prefix \"\" {\n policy = \"read\"\n }\n agent_prefix \"\" {\n policy = \"read\"\n }\n }\n}\n" +}' +``` + +Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + +You can specify an admin partition when creating tokens in Consul Enterprise. The token is only valid in the specified admin partition. You must create the token in the partition where the terminating gateway is registered. If the terminating gateway requires permissions for multiple namespaces, then the token must be created in the `default` namespace. The following example creates the token in the `default` namespace in the `ptn1` partition because the example policy contains permissions for multiple namespaces. + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +```shell-session +$ consul acl token create -partition "ptn1" -namespace "default" \ + -description "Terminating gateway token" \ + -policy-name "tgw-register" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "tgw-register" + } + ], + "Partition": "ptn1", + "Namespace": "default" +}' +``` + + + + diff --git a/website/content/docs/security/acl/tokens/create/create-a-token-for-vault-consul-storage.mdx b/website/content/docs/security/acl/tokens/create/create-a-token-for-vault-consul-storage.mdx new file mode 100644 index 00000000000..ab2c0406149 --- /dev/null +++ b/website/content/docs/security/acl/tokens/create/create-a-token-for-vault-consul-storage.mdx @@ -0,0 +1,164 @@ +--- +layout: docs +page_title: Create tokens for service registration +description: Learn how to create an ACL token for Vault’s Consul storage backend. +--- + +# Create a token for Vault with Consul storage backend + +This topic describes how to create a token for Vault’s Consul storage backend. + +## Introduction + +If you are using Vault to manage secrets in your infrastructure, you can configure Vault to use Consul's key/value (KV) store as backend storage to persist Vault's data. Refer to the [Consul KV documentation](/consul/docs/dynamic-app-config/kv) and the [Vault storage documentation](/vault/docs/configuration/storage) for additional information. + +## Requirements + +Core ACL functionality is available in all versions of Consul. + +The Vault Consul storage backend must present a token linked to policies that grant the following permissions: + +* `agent:read`: Provides KV visibility to all agents +* `key:write`: Enables writing to the KV store +* `service:write`: Enables the Vault service to register into the catalog +* `session:write`: Enables the agent to initialize a new session + +@include 'create-token-requirements.mdx' + +## Create a token linked to a policy + +To create a token for Vault’s Consul storage backend, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants the appropriate permissions to enable Vault to register as a service named `vault` and provides access to the `vault/` path in Consul's KV store. + + + +```hcl +agent_prefix "" { + policy = "read" +} +key_prefix "vault/" { + policy = "write" +} +service "vault" { + policy = "write" +} +session_prefix "" { + policy = "write" +} +``` + +```json +{ + "agent_prefix": { + "": [{ + "policy": "read" + }] + }, + "key_prefix": { + "vault/": [{ + "policy": "write" + }] + }, + "service": { + "vault": [{ + "policy": "write" + }] + }, + "session_prefix": { + "": [{ + "policy": "write" + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + +The following example registers a policy defined in `vault-storage-backend.hcl`. + +```shell-session +$ consul acl policy create -partition "default" -namespace "default" \ + -name vault-storage-backend -rules @vault-storage-backend.hcl \ + -description "Policy for the Vault Consul storage backend" +``` + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + +The following example registers the policy defined in `vault-storage-backend.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "vault-storage-backend", + "Description": "Policy for the Vault Consul storage backend", + "Rules": "agent_prefix \"\" {\n policy = \"read\"\n}\nkey_prefix \"vault/\" {\n policy = \"write\"\n}\nservice \"vault\" {\n policy = \"write\"\n}\nsession_prefix \"\" {\n policy = \"write\"\n}\n" +}' +``` + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following command creates the ACL token linked to the policy `vault-storage-backend`. + +```shell-session +$ consul acl token create \ + -description "Token for the Vault Consul storage backend" \ + -policy-name "vault-storage-backend" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates the ACL token linked to the policy `vault-storage-backend`. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "vault-storage-backend" + } + ] +}' +``` + + + + + diff --git a/website/content/docs/security/acl/tokens/create/create-a-ui-token.mdx b/website/content/docs/security/acl/tokens/create/create-a-ui-token.mdx new file mode 100644 index 00000000000..9c1e9019b5e --- /dev/null +++ b/website/content/docs/security/acl/tokens/create/create-a-ui-token.mdx @@ -0,0 +1,557 @@ +--- +layout: docs +page_title: Create tokens for agent registration +description: >- + Learn how to create ACL tokens that your Consul agents can present to Consul servers so that they can join the Consul cluster. +--- + +# Create a UI token + +This topic describes how to create a token that you can use to view resources in the Consul UI. + +## Introduction + +To navigate the Consul UI when ACLs are enabled, log into the UI with a token linked to policies that grant an appropriate set of permissions. The UI is unable to display resources that the token does not have permission to access. + +## Requirements + +Core ACL functionality is available in all versions of Consul. + +@include 'create-token-requirements.mdx' + +## View catalog in Consul OSS + +This section describes how to create a token that grants read-only access to the catalog. This token allows users to view the catalog without the ability to make changes. To create the ACL token, define a policy, create the policy, and then link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy allows users that login with the token to view all services and nodes in the catalog. + + + +```hcl +service_prefix "" { + policy = "read" +} +node_prefix "" { + policy = "read" +} +``` + +```json +{ + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service_prefix": { + "": [{ + "policy": "read" + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policies, you can register them with Consul using the command line or API endpoint. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `ui-view-catalog.hcl`. + +```shell-session +$ consul acl policy create \ + -name "ui-view-catalog" -rules @ui-view-catalog.hcl \ + -description "Allow viewing the catalog" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `view-catalog.hcl`. You must embed the policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "ui-view-catalog", + "Description": "Allow viewing the catalog", + "Rules": "service_prefix \"\" {\n policy = \"read\"\n}\nnode_prefix \"\" {\n policy = \"read\"\n}\n" +}' +``` + + + + + +### Link the policy to a token + +After registering the policies into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following command creates the ACL token linked to the policy `ui-view-catalog`. + +```shell-session +$ consul acl token create \ + -description "UI token to view the catalog" \ + -policy-name "ui-view-catalog" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates an ACL token that you can use to login to the UI and view the catalog. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "ui-view-catalog" + } + ] +}' +``` + + + + + +## View catalog in Consul Enterprise + +This section describes how to create a token that grants read-only access to the catalog. This token allows users to view the catalog without the ability to make changes. To create the ACL token, define a policy, create the policy, and then link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The following policy allows users that log in with the token to view services and nodes in the catalog in any partition and in any namespace. The `operator:read` permission is needed to list partitions. Without this permission, you can still view resources within a partition but cannot easily navigate to other partitions in the Consul UI. + + + +```hcl +operator = "read" +partition_prefix "" { + namespace_prefix "" { + service_prefix "" { + policy = "read" + } + node_prefix "" { + policy = "read" + } + } +} +``` + +```json +{ + "partition_prefix": { + "": [{ + "namespace_prefix": { + "": [{ + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service_prefix": { + "": [{ + "policy": "read" + }] + } + }] + } + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policies, you can register them with Consul using the command line or API endpoint. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `ui-view-catalog.hcl`. + +You can specify an admin partition and namespace when registering policies in Consul Enterprise. Policies are only valid in the scopes specified during registration, but you can grant tokens registered in the `default` partition permission to access resources in a different partition than where the token was registered. Refer to the [admin partition documentation](/consul/docs/enterprise/admin-partitions#default-admin-partition) for additional information. + +The following example registers the policy in the `default` partition and the `default` namespace because the policy grants cross-partition and cross-namespace access. + +```shell-session +$ consul acl policy create -partition "default" -namespace "default" \ + -name "ui-view-catalog" -rules @ui-view-catalog.hcl \ + -description "Allow viewing the catalog" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `view-catalog.hcl`. You must embed the policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "ui-view-catalog", + "Description": "Allow viewing the catalog", + "Partition": "default", + "Namespace": "default", + "Rules": "partition_prefix \"\" {\n namespace_prefix \"\" {\n service_prefix \"\" {\n policy = \"read\"\n }\n node_prefix \"\" {\n policy = \"read\"\n }\n }\n}\n" +}' +``` + + + + + +### Link the policy to a token + +After registering the policies into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +```shell-session +$ consul acl token create -partition "default" -namespace "default" \ + -description "UI token to view the catalog" \ + -policy-name "ui-view-catalog" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +You can specify an admin partition and namespace when registering policies in Consul Enterprise. Policies are only valid in the scopes specified during registration, but you can grant tokens registered in the `default` partition permission to access resources in a different partition than where the token was registered. Refer to the [admin partition documentation](/consul/docs/enterprise/admin-partitions#default-admin-partition) for additional information. + +The following example registers the policy in the `default` partition and the `default` namespace because the policy grants cross-partition and cross-namespace access. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "ui-view-catalog" + } + ], + "Partition": "default", + "Namespace": "default" +}' +``` + + + + + +## View all resources in Consul OSS + +This section describes how to create a token with read-only access to all resources in the Consul UI. This token allows users to view any resources without the ability to make changes. To create the ACL token, define a policy, create the policy, and then link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy allows users that log in with the token to view all services and nodes in the catalog, all objects in the key/value store, all intentions, and all ACL resources. The `acl:read` permission does not allow viewing the token secret ids. + + + +```hcl +acl = "read" +key_prefix "" { + policy = "read" +} +node_prefix "" { + policy = "read" +} +operator = "read" +service_prefix "" { + policy = "read" + intentions = "read" +} +``` + +```json +{ + "acl": "read", + "key_prefix": { + "": [{ + "policy": "read" + }] + }, + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "operator": "read", + "service_prefix": { + "": [{ + "intentions": "read", + "policy": "read" + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policies, you can register them with Consul using the command line or API endpoint. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `ui-view-all.hcl`. + +```shell-session +$ consul acl policy create \ + -name "ui-view-all" -rules @ui-view-all.hcl \ + -description "Allow viewing all resources" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `ui-view-all.hcl`. You must embed the policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "ui-view-all", + "Description": "Allow viewing all resources", + "Rules": "acl = \"read\"\nkey_prefix \"\" {\n policy = \"read\"\n}\nnode_prefix \"\" {\n policy = \"read\"\n}\noperator = \"read\"\nservice_prefix \"\" {\n policy = \"read\"\n intentions = \"read\"\n}\n" +}' +``` + + + + + +### Link the policy to a token + +After registering the policies into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following command creates the ACL token linked to the policy `ui-view-all`. + +```shell-session +$ consul acl token create \ + -description "UI token to view all resources" \ + -policy-name "ui-view-all" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates an ACL token that you can use to login to the UI and view the catalog. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "ui-view-all" + } + ] +}' +``` + + + + + +## View all resources in Consul Enterprise + +This section describes how to create a token with read-only access to all resources in the Consul UI. This token allows users to view any resources without the ability to make changes. To create the ACL token, define a policy, create the policy, and then link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy allows users that log in with the token to view all services and nodes in the catalog, all objects in the key-value store, all intentions, and all ACL resources in any namespace and any partition. The `acl:read` permission does not allow viewing the token secret ids. + + + +```hcl +operator = "read" +partition_prefix "" { + namespace_prefix "" { + acl = "read" + key_prefix "" { + policy = "read" + } + node_prefix "" { + policy = "read" + } + service_prefix "" { + policy = "read" + intentions = "read" + } + } +} +``` + +```json +{ + "operator": "read", + "partition_prefix": { + "": [{ + "namespace_prefix": { + "": [{ + "acl": "read", + "key_prefix": { + "": [{ + "policy": "read" + }] + }, + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service_prefix": { + "": [{ + "intentions": "read", + "policy": "read" + }] + } + }] + } + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policies, you can register them with Consul using the command line or API endpoint. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `ui-view-all.hcl`. + +You can specify an admin partition and namespace when creating policies in Consul Enterprise. The policy is only valid in the specified scopes. Because the policy grants cross-partition and cross-namespace access, the policy must be created in the `default` partition and the `default` namespace. + +```shell-session +$ consul acl policy create -partition "default" -namespace "default" \ + -name "ui-view-all" -rules @ui-view-all.hcl \ + -description "Allow viewing all resources" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `ui-view-all.hcl`. You must embed the policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "ui-view-all", + "Description": "Allow viewing all resources", + "Partition": "default", + "Namespace": "default", + "Rules": "operator = \"read\"\npartition_prefix \"\" {\n namespace_prefix \"\" {\n acl = \"read\"\n key_prefix \"\" {\n policy = \"read\"\n }\n node_prefix \"\" {\n policy = \"read\"\n }\n service_prefix \"\" {\n policy = \"read\"\n intentions = \"read\"\n }\n }\n}\n" +}' +``` + + + + + +### Link the policy to a token + +After registering the policies into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +```shell-session +$ consul acl token create -partition "default" -namespace "default" \ + -description "UI token to view all resources" \ + -policy-name "ui-view-all" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +You can specify an admin partition and namespace when creating tokens in Consul Enterprise. The token is only valid in the specified scopes. Because the policy was created in the `default` partition and `default` namespace, the token must also be created in the `default` partition and `default` namespace. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "ui-view-all" + } + ], + "Partition": "default", + "Namespace": "default" +}' +``` + + + + diff --git a/website/content/docs/security/acl/tokens/create/create-an-agent-token.mdx b/website/content/docs/security/acl/tokens/create/create-an-agent-token.mdx new file mode 100644 index 00000000000..1fa06d7c4d3 --- /dev/null +++ b/website/content/docs/security/acl/tokens/create/create-an-agent-token.mdx @@ -0,0 +1,407 @@ +--- +layout: docs +page_title: Create tokens for agent registration +description: >- + Learn how to create ACL tokens that your Consul agents can present to Consul servers so that they can join the Consul cluster. +--- + +# Create an agent token + +This topic describes how to create a token that you can use to register an agent into the catalog. + +## Introduction + +Consul agents must present a token linked to policies that grant the appropriate set of permissions in order to register into the catalog and to discover services and nodes in the catalog. + +Specify the [`agent`](/consul/docs/agent/config/config-files#acl_tokens_agent) token to the Consul agent so that it can present the token when it registers into the catalog. + +### Node identities versus custom policies + +You can create tokens linked to custom policies or to node identities. [Node identities](/consul/docs/security/acl#node-identities) are constructs in Consul that enable you to quickly grant permissions for a group of agents, rather than create similar policies for each agent. + +We recommend using a node identity to grant permissions to the agent rather than creating a custom policy. This is because node identities automatically grant the node `node:write` and `service:read` permission. + +Your organization may have requirements or processes for deploying services in a way that is inconsistent with service and node identities. In these cases, you can create custom policies and link them to tokens. + +## Requirements + +Core ACL functionality is available in all versions of Consul. + +The agent token must be linked to policies that grant the following permissions: + +* `node:write`: Enables the agent to update the catalog. +* `service:read`: Enables the agent to discover other services in the catalog + +@include 'create-token-requirements.mdx' + +## Node identity in Consul OSS + +Refer to [Node identities](/consul/docs/security/acl#node-identities) for information about node identities that you can link to tokens. + +You can manually create tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy or node identity to link to create a token. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following command creates an ACL token linked to a node identity for a node named `node1` in the datacenter `dc1`. + +```shell-session +$ consul acl token create \ + -description "Agent token for node1" \ + -node-identity "node1:dc1" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify a node identity in the request body to create a token linked to the node identity. An ACL token linked to a policy with permissions to use the API endpoint is required. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates a token linked to a node identity named `node1`: + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "NodeIdentities": [ + { + "NodeName": "node1", + "Datacenter": "dc1" + } + ] +}' +``` + + + + + +## Node identity in Consul Enterprise + +Refer to [Node identities](/consul/docs/security/acl#node-identities) for information about node identities that you can link to tokens. + +You can manually create tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy or node identity to link to create a token. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +You can specify an admin partition when creating tokens in Consul Enterprise. The token is only valid in the specified admin partition. The following example creates an ACL token that the agent can use to register in partition `ptn1` in datacenter `dc1`: + +```shell-session +$ consul acl token create -partition "ptn1" \ + -description "Agent token for node1" \ + -node-identity "node1:dc1" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify a node identity in the request body to create a token linked to the node identity. An ACL token linked to a policy with permissions to use the API endpoint is required. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +You can specify an admin partition when creating a token in Consul Enterprise. The token is only valid in the specified admin partition. The following example creates an ACL token that the agent can use to register in the partition `ptn1` in datacenter `dc1`: + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "NodeIdentities": [ + { + "NodeName": "node1", + "Datacenter": "dc1" + } + ], + "Partition": "ptn1" +}' +``` + + + + + +## Custom policy in Consul OSS + +When you are unable to link tokens to a node identity, you can define policies, register them with Consul, and link the policies to tokens that enable nodes to register into the Consul catalog. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants `write` permission for node `node1` so that the Consul agent can register into the catalog. It grants `read` permissions to discover services in the catalog. + + + +```hcl +node "node1" { + policy = "write" +} +service_prefix "" { + policy = "read" +} +``` + +```json +{ + "node": { + "node1": [{ + "policy": "write" + }] + }, + "service_prefix": { + "": [{ + "policy": "read" + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `node1-register.hcl`: + +```shell-session +$ consul acl policy create \ + -name "node1-register" -rules @node1-register.hcl \ + -description "Custom policy for node1" \ +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `node1-register.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "node1-register", + "Description": "Allow node1 to register into the catalog", + "Rules": "node \"node1\" {\n policy = \"write\"\n}\nservice_prefix \"\" {\n policy = \"read\"\n}\n" +}' +``` + +Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + + + + + +### Link the policy to a token + +After registering the policies into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following command creates the ACL token linked to the policy `node1-register`. + +```shell-session +$ consul acl token create \ + -description "Agent token for node1" \ + -policy-name "node1-register" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +The following example creates an ACL token that the agent can use to register as node `node1` in the catalog: + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "node1-register" + } + ] +}' +``` + + + + + + + +## Custom policy in Consul Enterprise + +When you are unable to link tokens to a node identity, you can define policies, register them with Consul, and link the policies to tokens that enable nodes to register into the Consul catalog. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants the `write` permission for node `node1` in partition `ptn1` so that the Consul agent can register into the catalog. It grants `read` permissions to discover services in any namespace in the `ptn1` partition. + + + +```hcl +partition "ptn1" { + node "node1" { + policy = "write" + } + namespace_prefix "" { + service_prefix "" { + policy = "read" + } + } +} +``` + +```json +{ + "partition": { + "ptn1": [{ + "namespace_prefix": { + "": [{ + "service_prefix": { + "": [{ + "policy": "read" + }] + } + }] + }, + "node": { + "node1": [{ + "policy": "write" + }] + } + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `node1-register.hcl`: + +```shell-session +$ consul acl policy create -partition "ptn1" \ + -name "node1-register" -rules @node1-register.hcl \ + -description "Custom policy for node1" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `node1-register.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "node1-register", + "Description": "Allow node1 to register into the catalog", + "Partition": "ptn1", + "Rules": "partition \"ptn1\" {\n node \"node1\" {\n policy = \"write\"\n }\n namespace_prefix \"\" {\n service_prefix \"\" {\n policy = \"read\"\n }\n }\n}\n" +}' +``` + +Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +```shell-session +$ consul acl token create -partition "ptn1" \ + -description "Agent token for node1" \ + -policy-name "node1-register" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +You can specify an admin partition when creating tokens in Consul Enterprise. The token is only valid in the specified admin partition. The following example creates an ACL token that the agent can use to register as the node `node1` in the partition `ptn1`: + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "node1-register" + } + ], + "Partition": "ptn1" +}' +``` + + + + + +## Apply the token + +Configure the Consul agent to present the token by either specifying the token in the agent configuration file or by using the `consul set-agent-token` command. + +### Apply the token in a file + +Specify the token in the [`acl.token.agent`](/consul/docs/agent/config/config-files#acl_tokens_agent) field of the agent configuration file so that the agent can present it and register into the catalog on startup. + +```hcl +acl = { + enabled = true + tokens = { + agent = "" + ... + } + ... +} +``` + +### Apply the token with a command + +Set the `agent` token using the [`consul set-agent-token`](/consul/commands/acl/set-agent-token) command. The following command configures a running Consul agent token with the specified token. + +```shell-session +consul acl set-agent-token agent +``` diff --git a/website/content/docs/security/acl/tokens/create/create-an-ingress-gateway-token.mdx b/website/content/docs/security/acl/tokens/create/create-an-ingress-gateway-token.mdx new file mode 100644 index 00000000000..65e01369966 --- /dev/null +++ b/website/content/docs/security/acl/tokens/create/create-an-ingress-gateway-token.mdx @@ -0,0 +1,326 @@ +--- +layout: docs +page_title: Create a token for ingress gateway registration +description: >- + Learn how to create ACL tokens that your ingress gateway can present to Consul servers so that they can register with the Consul catalog. +--- + +# Create an ingress gateway token + +This topic describes how to create a token to enable an ingress gateway to register. + +## Introduction + +Gateways must present a token linked to policies that grant the appropriate set of permissions in order to register into the catalog and to route to other services in a mesh. + +## Requirements + +Core ACL functionality is available in all versions of Consul. + +The ingress gateway token must be linked to policies that grant the following permissions: + +* `service:write` to allow the ingress gateway to register into the catalog +* `service:read` for all services and `node:read` for all nodes in order to discover and route to services +* `agent:read` to enable the `consul connect envoy` CLI command to automatically discover gRPC settings from the Consul agent. If this command is not used to start the gateway or if the Consul agent uses the default gRPC settings, then you can omit the `agent:read` permission. + +@include 'create-token-requirements.mdx' + +## Consul OSS + +To create a token for the ingress gateway, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +The following example policy is defined in a file. The policy grants the ingress gateway the appropriate permissions to register as a service named `ingress-gateway` and to operate as an ingress gateway. + + + +```hcl +service "ingress-gateway" { + policy = "write" +} +node_prefix "" { + policy = "read" +} +service_prefix "" { + policy = "read" +} +agent_prefix "" { + policy = "read" +} +``` + +```json +{ + "agent_prefix": { + "": [{ + "policy": "read" + }] + }, + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service": { + "ingress-gateway": [{ + "policy": "write" + }] + }, + "service_prefix": { + "": [{ + "policy": "read" + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + +The following commands create the ACL policy and token. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `igw-register.hcl`: + +```shell-session +$ consul acl policy create \ + -name "igw-register" -rules @igw-register.hcl \ + -description "Ingress gateway policy" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `igw-register.hcl`. You must embed policy rules in the `Rules` field of the request body. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "igw-register", + "Description": "Ingress gateway policy", + "Rules": "service \"ingress-gateway\" {\n policy = \"write\"\n}\nnode_prefix \"\" {\n policy = \"read\"\n}\nservice_prefix \"\" {\n policy = \"read\"\n}\nagent_prefix \"\" {\n policy = \"read\"\n}\n" +}' +``` + +Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +The following command creates the ACL token linked to the policy `igw-register`. + +```shell-session +$ consul acl token create \ + -description "Ingress gateway token" \ + -policy-name "igw-register" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "igw-register" + } + ] +}' +``` + + + + + +## Consul Enterprise + +To create a token for the ingress gateway, you must define a policy, register the policy with Consul, and link the policy to a token. + +### Define a policy + +You can send policy definitions as command line or API arguments or define them in an external HCL or JSON file. Refer to [ACL Rules](/consul/docs/security/acl/acl-rules) for details about all of the rules you can use in your policies. + +You can specify an admin partition and namespace when creating policies in Consul Enterprise. The policy is only valid in the specified scopes. + +The following example policy is defined in a file. The policy allows an ingress gateway to register as a service named `ingress-gateway` in the `ptn1` partition and `ns1` namespace. The policy contains permissions for resources in multiple namespaces. You must create ACL policies that grant permissions for multiple namespaces in the `default` namespace. + + + +```hcl +partition "ptn1" { + namespace "ns1" { + service "ingress-gateway" { + policy = "write" + } + node_prefix "" { + policy = "read" + } + service_prefix "" { + policy = "read" + } + } + namespace "default" { + agent_prefix "" { + policy = "read" + } + } +} +``` + +```json +{ + "partition": { + "ptn1": [{ + "namespace": { + "default": [{ + "agent_prefix": { + "": [{ + "policy": "read" + }] + } + }], + "ns1": [{ + "node_prefix": { + "": [{ + "policy": "read" + }] + }, + "service": { + "ingress-gateway": [{ + "policy": "write" + }] + }, + "service_prefix": { + "": [{ + "policy": "read" + }] + } + }] + } + }] + } +} +``` + + + +### Register the policy with Consul + +After defining the policy, you can register the policy with Consul using the command line or API endpoint. + +The following commands create the ACL policy and token. + + + + + +Run the `consul acl policy create` command and specify the policy rules to create a policy. The following example registers a policy defined in `igw-register.hcl`: + +You can specify an admin partition and namespace when creating policies in Consul Enterprise. The policy is only valid in the specified admin partition and namespace. The following example creates the policy in the `default` namespace in the `ptn1` partition. The example policy contains permissions for resources in multiple namespaces. You must create ACL policies that grant permissions for multiple namespaces in the `default` namespace. + +```shell-session +$ consul acl policy create -partition "ptn1" -namespace "default" \ + -name "igw-register" -rules @igw-register.hcl \ + -description "Ingress gateway policy" +``` + +Refer to [Consul ACL Policy Create](/consul/commands/acl/policy/create) for details about the `consul acl policy create` command. + + + + + +Send a PUT request to the `/acl/policy` endpoint and specify the policy rules in the request body to create a policy. The following example registers the policy defined in `igw-register.hcl`. You must embed policy rules in the `Rules` field of the request body. + +You can specify an admin partition and namespace when creating tokens in Consul Enterprise. The token is only valid in the specified admin partition and namespace. The following example creates the token in the partition `ptn1` and namespace `ns1`. The example policy contains permissions for resources in multiple namespaces. You must create ACL policies that grant permissions for multiple namespaces in the `default` namespace. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/policy \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Name": "igw-register", + "Description": "Ingress gateway policy", + "Partition": "ptn1", + "Namespace": "default", + "Rules": "partition \"ptn1\" {\n namespace \"ns1\" {\n service \"ingress-gateway\" {\n policy = \"write\"\n }\n node_prefix \"\" {\n policy = \"read\"\n }\n service_prefix \"\" {\n policy = \"read\"\n }\n }\n namespace \"default\" {\n agent_prefix \"\" {\n policy = \"read\"\n }\n }\n}\n" +}' +``` + +Refer to [ACL Policy HTTP API](/consul/api-docs/acl/policies) for additional information about using the API endpoint. + + + + + +### Link the policy to a token + +After registering the policy into Consul, you can create and link tokens using the Consul command line or API endpoint. You can also enable Consul to dynamically create tokens from trusted external systems using an [auth method](/consul/docs/security/acl/auth-methods). + + + + + +Run the `consul acl token create` command and specify the policy name or ID to create a token linked to the policy. Refer to [Consul ACL Token Create](/consul/commands/acl/token/create) for details about the `consul acl token create` command. + +You can specify an admin partition and namespace when creating tokens in Consul Enterprise. The token is only valid in the specified admin partition and namespace. The following example creates the token in the partition `ptn1` and namespace `default`. The example policy contains permissions for resources in multiple namespaces. You must create ACL tokens linked to policies that grant permissions for multiple namespaces in the `default` namespace. + +```shell-session +$ consul acl token create -partition "ptn1" -namespace "default" \ + -description "Ingress gateway token" \ + -policy-name "igw-register" +``` + + + + + +Send a PUT request to the `/acl/token` endpoint and specify the policy name or ID in the request to create an ACL token linked to the policy. Refer to [ACL Token HTTP API](/consul/api-docs/acl/tokens) for additional information about using the API endpoint. + +You can specify an admin partition when creating tokens in Consul Enterprise. The token is only valid in the specified admin partition. You must create the token in the partition where the ingress gateway is registered. The following example creates the token in the partition `ptn1` and namespace `default`. + +```shell-session +$ curl --request PUT http://127.0.0.1:8500/v1/acl/token \ + --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" \ + --data '{ + "Policies": [ + { + "Name": "igw-register" + } + ], + "Partition": "ptn1", + "Namespace": "default" +}' +``` + + + + diff --git a/website/content/docs/security/acl/acl-tokens.mdx b/website/content/docs/security/acl/tokens/index.mdx similarity index 99% rename from website/content/docs/security/acl/acl-tokens.mdx rename to website/content/docs/security/acl/tokens/index.mdx index da14d52d88a..068d8efa50e 100644 --- a/website/content/docs/security/acl/acl-tokens.mdx +++ b/website/content/docs/security/acl/tokens/index.mdx @@ -66,7 +66,7 @@ service = { -Refer to the [service definitions documentation](/consul/docs/discovery/services#service-definition) for additional information. +Refer to [Services Configuration Reference](/consul/docs/services/configuration/services-configuration-reference) for additional information. ### Agent Requests diff --git a/website/content/docs/security/security-models/core.mdx b/website/content/docs/security/security-models/core.mdx index 270cdd6526b..92a5c1ac91c 100644 --- a/website/content/docs/security/security-models/core.mdx +++ b/website/content/docs/security/security-models/core.mdx @@ -102,21 +102,21 @@ environment and adapt these configurations accordingly. `server..` hostname for outgoing TLS connections. The default configuration does not verify the hostname of the certificate, only that it is signed by a trusted CA. This setting is critical to prevent a compromised client agent from being restarted as a server and having all cluster state including all ACL tokens and - Connect CA root keys replicated to it. This setting was introduced in 0.5.1. From version 0.5.1 to 1.4.0 we + service mesh CA root keys replicated to it. This setting was introduced in 0.5.1. From version 0.5.1 to 1.4.0 we documented that `verify_server_hostname` being true implied verify_outgoing however due to a bug this was not the case so setting only `verify_server_hostname` results in plaintext communication between client and server. See [CVE-2018-19653](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-19653) for more details. This is fixed in 1.4.1. - [`auto_encrypt`](/consul/docs/agent/config/config-files#auto_encrypt) - Enables automated TLS certificate distribution for client - agent RPC communication using the Connect CA. Using this configuration a [`ca_file`](/consul/docs/agent/config/config-files#tls_defaults_ca_file) + agent RPC communication using the service mesh CA. Using this configuration a [`ca_file`](/consul/docs/agent/config/config-files#tls_defaults_ca_file) and ACL token would still need to be distributed to client agents. - [`allow_tls`](/consul/docs/agent/config/config-files#allow_tls) - By default this is false, and should be set to true on server - agents to allow certificates to be automatically generated and distributed from the Connect CA to client agents. + agents to allow certificates to be automatically generated and distributed from the service mesh CA to client agents. - [`tls`](/consul/docs/agent/config/config-files#tls) - By default this is false, and should be set to true on client agents to - automatically request a client TLS certificate from the server's Connect CA. + automatically request a client TLS certificate from the server's service mesh CA. **Example Server Agent TLS Configuration** @@ -321,7 +321,7 @@ The following are parts of the core Consul threat model: This requires transport encryption to be enabled on the cluster and covers both TCP and UDP traffic. - **Consul agent-to-CA communication** - Communication between the Consul server and the configured certificate - authority provider for Connect is always encrypted. + authority provider for service mesh is always encrypted. - **Tampering of data in transit** - Any tampering should be detectable and cause Consul to avoid processing the request. @@ -338,13 +338,13 @@ The following are parts of the core Consul threat model: - **Denial of Service against a node** - DoS attacks against a node should not compromise the security stance of the software. -- **Connect-based Service-to-Service communication** - Communications between two Connect-enabled services (natively or - by proxy) should be secure from eavesdropping and provide authentication. This is achieved via mutual TLS. +- **Communication between mesh services** - Communications between two mesh-enabled services (natively or + by sidecar proxy) should be secure from eavesdropping and provide authentication. This is achieved via mutual TLS. The following are not part of the threat model for server agents: - **Access (read or write) to the Consul data directory** - All Consul servers, including non-leaders, persist the full - set of Consul state to this directory. The data includes all KV, service registrations, ACL tokens, Connect CA + set of Consul state to this directory. The data includes all KV, service registrations, ACL tokens, service mesh CA configuration, and more. Any read or write to this directory allows an attacker to access and tamper with that data. - **Access (read or write) to the Consul configuration directory** - Consul configuration can enable or disable the ACL @@ -353,14 +353,14 @@ The following are not part of the threat model for server agents: - **Memory access to a running Consul server agent** - If an attacker is able to inspect the memory state of a running Consul server agent the confidentiality of almost all Consul data may be compromised. If you're using an external - Connect CA, the root private key material is never available to the Consul process and can be considered safe. Service - Connect TLS certificates should be considered compromised; they are never persisted by server agents but do exist + service mesh CA, the root private key material is never available to the Consul process and can be considered safe. Service + service mesh TLS certificates should be considered compromised; they are never persisted by server agents but do exist in-memory during at least the duration of a Sign request. The following are not part of the threat model for client agents: - **Access (read or write) to the Consul data directory** - Consul clients will use the data directory to cache local - state. This includes local services, associated ACL tokens, Connect TLS certificates, and more. Read or write access + state. This includes local services, associated ACL tokens, service mesh TLS certificates, and more. Read or write access to this directory will allow an attacker to access this data. This data is typically a smaller subset of the full data of the cluster. - **Access (read or write) to the Consul configuration directory** - Consul client configuration files contain the @@ -371,18 +371,18 @@ The following are not part of the threat model for client agents: - **Memory access to a running Consul client agent** - The blast radius of this is much smaller than a server agent but the confidentiality of a subset of data can still be compromised. Particularly, any data requested against the agent's - API including services, KV, and Connect information may be compromised. If a particular set of data on the server was + API including services, KV, and service mesh information may be compromised. If a particular set of data on the server was never requested by the agent, it never enters the agent's memory since replication only exists between servers. An attacker could also potentially extract ACL tokens used for service registration on this agent, since the tokens must be stored in-memory alongside the registered service. -- **Network access to a local Connect proxy or service** - Communications between a service and a Connect-aware proxy +- **Network access to a local service mesh proxy or service** - Communications between a service and a mesh-aware proxy are generally unencrypted and must happen over a trusted network. This is typically a loopback device. This requires that other processes on the same machine are trusted, or more complex isolation mechanisms are used such as network - namespaces. This also requires that external processes cannot communicate to the Connect service or proxy (except on - the inbound port). Therefore, non-native Connect applications should only bind to non-public addresses. + namespaces. This also requires that external processes cannot communicate to the mesh service or sidecar proxy (except on + the inbound port). Therefore, non-native service mesh applications should only bind to non-public addresses. -- **Improperly Implemented Connect proxy or service** - A Connect proxy or natively integrated service must correctly +- **Improperly Implemented service mesh proxy or service** - A service mesh proxy or natively integrated service must correctly serve a valid leaf certificate, verify the inbound TLS client certificate, and call the Consul agent-local authorized endpoint. If any of this isn't performed correctly, the proxy or service may allow unauthenticated or unauthorized connections. diff --git a/website/content/docs/security/security-models/nia.mdx b/website/content/docs/security/security-models/nia.mdx index d6b312a4206..0e6de4982b1 100644 --- a/website/content/docs/security/security-models/nia.mdx +++ b/website/content/docs/security/security-models/nia.mdx @@ -76,7 +76,7 @@ security concerns accordingly. - **Run without Root** - The NIA daemon does not require root or other administrative privileges to operate. - **Protect NIA Daemon API Endpoint** - Any network endpoints provided by, or exposed to the NIA Daemon should be - protected using Consul Connect and appropriate firewall rules. + protected using Consul service mesh and appropriate firewall rules. - **Use a centralized logging solution** - Export log entries within [syslog](https://en.wikipedia.org/wiki/Syslog) generated from `consul-terraform-sync` to a centralized logging solution. diff --git a/website/content/docs/services/configuration/checks-configuration-reference.mdx b/website/content/docs/services/configuration/checks-configuration-reference.mdx new file mode 100644 index 00000000000..fee071de51b --- /dev/null +++ b/website/content/docs/services/configuration/checks-configuration-reference.mdx @@ -0,0 +1,55 @@ +--- +layout: docs +page_title: Health check configuration reference +description: -> + Use the health checks to direct safety functions, such as removing failing nodes and replacing secondary services. Learn how to configure health checks. +--- + +# Health check configuration reference + +This topic provides configuration reference information for health checks. For information about the different kinds of health checks and guidance on defining them, refer to [Define Health Checks]. + +## Introduction +Health checks perform several safety functions, such as allowing a web balancer to gracefully remove failing nodes and allowing a database to replace a failed secondary. You can configure health checks to monitor the health of the entire node. Refer to [Define Health Checks](/consul/docs/services/usage/checks) for information about how to define the differnet types of health checks available in Consul. + +## Check block +Specify health check options in the `check` block. To register two or more heath checks in the same configuration, use the [`checks` block](#checks-block). The following table describes the configuration options contained in the `check` block. + +| Parameter | Description | Check types | +| --- | --- | --- | +| `name` | Required string value that specifies the name of the check. Default is `service:`. If multiple service checks are registered, the autogenerated default is appended with colon and incrementing number starting with `1`. |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `id` | A unique string value that specifies an ID for the check. Default to the `name` value. If `name` values conflict, specify a unique ID to avoid overwriting existing checks with same ID on the same node. Consul auto-generates an ID if the check is defined in a service definition file. |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `notes` | String value that provides a human-readabiole description of the check. The contents are not visible to Consul. |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `interval` | Required string value that specifies how frequently to run the check. The `interval` parameter is required for supported check types. The value is parsed by the golang [time package formatting specification](https://golang.org/pkg/time/#ParseDuration). |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • Docker
  • gRPC
  • H2ping
    • | +| `timeout` | String value that specifies how long unsuccessful requests take to end with a timeout. The `timeout` is optional for the supported check types and has the following defaults:
  • Script: `30s`
  • HTTP: `10s`
  • TCP: `10s`
  • UDP: `10s`
  • gRPC: `10s`
  • H2ping: `10s`
    • |
  • Script
  • HTTP
  • TCP
  • UDP
  • gRPC
  • H2ping
    • | +| `status` | Optional string value that specifies the initial status of the health check. You can specify the following values:
  • `critical` (default)
  • `warning`
  • `passing`
    • |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `deregister_critical_service_after` | String value that specifies how long a service and its associated checks are allowed to be in a `critical` state. Consul deregisters services if they are `critical` for the specified amount of time. The value is parsed by the golang [time package formatting specification](https://golang.org/pkg/time/#ParseDuration) |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `success_before_passing` | Integer value that specifies how many consecutive times the check must pass before Consul marks the service or node as `passing`. Default is `0`. |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `failures_before_warning` | Integer value that specifies how many consecutive times the check must fail before Consul marks the service or node as `warning`. The value cannot be more than `failures_before_critical`. Defaults to the value specified for `failures_before_critical`. |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `failures_before_critical` | Integer value that specifies how many consecutive times the check must fail before Consul marks the service or node as `critical`. Default is `0`. |
  • Script
  • HTTP
  • TCP
  • UDP
  • OSService
  • TTL
  • Docker
  • gRPC
  • H2ping
  • Alias
    • | +| `args` | Specifies a list of arguments strings to pass to the command line. The list of values includes the path to a script file or external application to invoke and any additional parameters for running the script or application. |
  • Script
  • Docker
    • | +| `docker_container_id` | Specifies the Docker container ID in which to run an external health check application. Specify the external application with the `args` parameter. |
  • Docker
    • | +| `shell` | String value that specifies the type of command line shell to use for running the health check application. Specify the external application with the `args` parameter. |
  • Docker
    • | +| `grpc` | String value that specifies the gRPC endpoint, including port number, to send requests to. Append the endpoint with `:/` and a service identifier to check a specific service. The endpoint must support the [gRPC health checking protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). |
  • gRPC
    • | +| `grpc_use_tls` | Boolean value that enables TLS for gRPC checks when set to `true`. |
  • gRPC
    • | +| `h2ping` | String value that specifies the HTTP2 endpoint, including port number, to send HTTP2 requests to. |
  • H2ping
    • | +| `h2ping_use_tls` | Boolean value that enables TLS for H2ping checks when set to `true`. |
  • H2ping
    • | +| `http` | String value that specifies an HTTP endpoint to send requests to. |
  • HTTP
    • | +| `tls_server_name` | String value that specifies the name of the TLS server that issues certificates. Defaults to the SNI determined by the address specified in the `http` field. Set the `tls_skip_verify` to `false` to disable this field. |
  • HTTP
    • | +| `tls_skip_verify` | Boolean value that disbles TLS for HTTP checks when set to `true`. Default is `false`. |
  • HTTP
    • | +| `method` | String value that specifies the request method to send during HTTP checks. Default is `GET`. |
  • HTTP
    • | +| `header` | Object that specifies header fields to send in HTTP check requests. Each header specified in `header` object contains a list of string values. |
  • HTTP
    • | +| `body` | String value that contains JSON attributes to send in HTTP check requests. You must escap the quotation marks around the keys and values for each attribute. |
  • HTTP
    • | +| `disable_redirects` | Boolean value that prevents HTTP checks from following redirects if set to `true`. Default is `false`. |
  • HTTP
    • | +| `os_service` | String value that specifies the name of the name of a service to check during an OSService check. |
  • OSService
    • | +| `service_id` | String value that specifies the ID of a service instance to associate with an OSService check. That service instance must be on the same node as the check. If not specified, the check verifies the health of the node. |
  • OSService
    • | +| `tcp` | String value that specifies an IP address or host and port number for the check establish a TCP connection with. |
  • TCP
    • | +| `udp` | String value that specifies an IP address or host and port number for the check to send UDP datagrams to. |
  • UDP
    • | +| `ttl` | String value that specifies how long to wait for an update from an external process during a TTL check. |
  • TTL
    • | +| `alias_service` | String value that specifies a service or node that the service associated with the health check aliases. |
  • Alias
    • | + + + +## Checks block +You can define multiple health checks in a single `checks` block. The `checks` block is an array of objects that contain the configuration options described in the [`check` block configuration reference](#check-block). + diff --git a/website/content/docs/services/configuration/services-configuration-overview.mdx b/website/content/docs/services/configuration/services-configuration-overview.mdx new file mode 100644 index 00000000000..3c01f05ae7c --- /dev/null +++ b/website/content/docs/services/configuration/services-configuration-overview.mdx @@ -0,0 +1,29 @@ +--- +layout: docs +page_title: Services configuration overview +description: -> + This topic provides introduces the configuration items that enable you to register services with Consul so that they can connect to other services and nodes registered with Consul. +--- + +# Services configuration overview + +This topic provides introduces the configuration items that enable you to register services with Consul so that they can connect to other services and nodes registered with Consul. + +## Service definitions +A service definition contains a set of parameters that specify various aspects of the service, including how it is discovered by other services in the network. The service definition may also contain health check configurations. Refer to [Health Check Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for configuration details about health checks. + +Configure individual services and health checks by specifying parameters in the `service` block of a service definition file. Refer to [Define Services](/consul/docs/services/usage/define-services) for information about defining services. + +To register a service, provide the service definition to the Consul agent. Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for information about registering services. + +Consul supports service definitions written in JSON and HCL. + +## Service defaults +Use the `service-defaults` configuration entry to define the default parameters for service definitions. This enables you to configure common settings, such as namespace or partition for Consul Enterprise deployments, in a single definition. + +You can use `service-defaults` configuration entries on virtual machines and in Kubernetes environments. + +## ACLs +Services registered in Consul clusters where both Consul Namespaces and the ACL system are enabled can be registered to specific namespaces that are associated with ACL tokens scoped to the namespace. Services registered with a service definition will not inherit the namespace associated with the ACL token specified in the token field. The namespace and the token parameters must be included in the service definition for the service to be registered to the namespace that the ACL token is scoped to. + + diff --git a/website/content/docs/services/configuration/services-configuration-reference.mdx b/website/content/docs/services/configuration/services-configuration-reference.mdx new file mode 100644 index 00000000000..4614a4b2680 --- /dev/null +++ b/website/content/docs/services/configuration/services-configuration-reference.mdx @@ -0,0 +1,655 @@ +--- +layout: docs +page_title: Service configuration reference +description: Use the service definition to configure and register services to the Consul catalog, including services used as proxies in a Consul service mesh +--- + +# Services configuration reference + +This topic describes the options you can use to define services for registering them with Consul. Refer to the following topics for usage information: + +- [Define Services](/consul/docs/services/usage/define-services) +- [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) + +## Configuration model +The following outline shows how to format the configurations in the root `service` block. Click on a property name to view details about the configuration. + +- [`name`](#name): string | required +- [`id`](#id): string | optional +- [`address`](#address): string | optional +- [`port`](#port): integer | optional +- [`tags`](#tags): list of strings | optional +- [`meta`](#meta): object | optional + - [_`custom_meta_key`_](#meta): string | optional +- [`tagged_addresses`](#tagged_addresses): object | optional + - [`lan`](#tagged_addresses-lan): object | optional + - [`address`](#tagged_addresses-lan): string | optional + - [`port`](#tagged_addresses-lan): integer | optional + - [`wan`](#tagged_addresses-wan): object | optional + - [`address`](#tagged_addresses-wan): string | optional + - [`port`](#tagged_addresses-wan): integer | optional +- [`socket_path`](#socket_path): string | optional +- [`enable_tag_override`](#enable_tag_override): boolean | optional +- [`checks`](#checks) : list of objects | optional +- [`kind`](#kind): string | optional +- [`proxy`](#proxy): object | optional +- [`connect`](#connect): object | optional + - [`native`](#connect): boolean | optional + - [`sidecar_service`](#connect): object | optional +- [`weights`](#weights): object | optional + - [`passing`](#weights): integer | optional + - [`warning`](#weights): integer | optional +- [`token`](#token): string | required if ACLs are enabled +- [`namespace`](#namespace): string | optional | + +## Specification +This topic provides details about the configuration parameters. + +### name +Required value that specifies a name for the service. We recommend using valid DNS labels for service definition names for compatibility with external DNSs. The value for this parameter is used as the ID if the `id` parameter is not specified. + +- Type: string +- Default: none + +### id +Specifies an ID for the service. Services on the same node must have unique IDs. We recommend specifying unique values if the default name conflicts with other services. + +- Type: string +- Default: Value of the `name` field. + +### address +String value that specifies a service-specific IP address or hostname. +If no value is specified, the IP address of the agent node is used by default. +There is no service-side validation of this parameter. + +- Type: string +- Default: IP address of the agent node + +### port +Specifies a port number for the service. To improve service discoverability, we recommend specifying the port number, as well as an address in the [`tagged_addresses`](#tagged_addresses) parameter. + +- Type: integer +- Default: Port number of the agent + +### tags +Specifies a list of string values that add service-level labels. Tag values are opaque to Consul. We recommend using valid DNS labels for service definition IDs for compatibility with external DNSs. In the following example, the service is tagged as `v2` and `primary`: + +```hcl +tags = ["v2", "primary"] +``` + +Consul uses tags as an anti-entropy mechanism to maintain the state of the cluster. You can disable the anti-entropy feature for a service using the [`enable_tag_override`](#enable_tag_override) setting, which enables external agents to modify tags on services in the catalog. Refer to [Modify anti-entropy synchronization](/consul/docs/services/usage/define-services#modify-anti-entropy-synchronization) for additional usage information. + +### meta +The `meta` field contains custom key-value pairs that associate semantic metadata with the service. You can specify up to 64 pairs that meet the following requirements: + +- Keys and values must be strings. +- Keys can only contain ASCII characters (`A` -` Z`, `a`- `z`, `0` - `9`, `_`, and `-`). +- Keys can not have special characters. +- Keys are limited to 128 characters. +- Values are limited to 512 characters. + +In the following example, the `env` key is set to `prod`: + + + +```hcl +meta = { + env = "prod" +} +``` + +```json +"meta" : { + "env" : "prod" +} +``` + + + +### tagged_addresses +The `tagged_address` field is an object that configures additional addresses for a node or service. Remote agents and services can communicate with the service using a tagged address as an alternative to the address specified in the [`address`](#address) field. You can configure multiple addresses for a node or service. The following tags are supported: + +- [`lan`](#tagged_addresses-lan): IPv4 LAN address where the node or service is accessible. +- [`lan_ipv4`](#tagged_addresses-lan): IPv4 LAN address where the node or service is accessible. +- [`lan_ipv6`](#tagged_addresses-lan): IPv6 LAN address where the node or service is accessible. +- [`virtual`](#tagged_addresses-virtual): A fixed address for the instances of a given logical service. +- [`wan`](#tagged_addresses-wan): IPv4 WAN address where the node or service is accessible when dialed from a remote data center. +- [`wan_ipv4`](#tagged_addresses-wan): IPv4 WAN address where the node or service is accessible when dialed from a remote data center. +- [`wan_ipv6`](#tagged_addresses-lan): IPv6 WAN address at which the node or service is accessible when being dialed from a remote data center. + +### tagged_addresses.lan +Object that specifies either an IPv4 or IPv6 LAN address and port number where the service or node is accessible. You can specify one or more of the following fields: + +- `lan` +- `lan_ipv4` +- `lan_ipv6` + +The field contains the following parameters: + +- `address` +- `port` + +In the following example, the `redis` service has an IPv4 LAN address of `192.0.2.10:80` and IPv6 LAN address of `[2001:db8:1:2:cafe::1337]:80`: + + + +```hcl +service { + name = "redis" + address = "192.0.2.10" + port = 80 + tagged_addresses { + lan = { + address = "192.0.2.10" + port = 80 + } + lan_ipv4 = { + address = "192.0.2.10" + port = 80 + } + lan_ipv6 = { + address = "2001:db8:1:2:cafe::1337" + port = 80 + } + } +} +``` + +```json +{ + "service": { + "name": "redis", + "address": "192.0.2.10", + "port": 80, + "tagged_addresses": { + "lan": { + "address": "192.0.2.10", + "port": 80 + }, + "lan_ipv4": { + "address": "192.0.2.10", + "port": 80 + }, + "lan_ipv6": { + "address": "2001:db8:1:2:cafe::1337", + "port": 80 + } + } + } +} +``` + + + +### tagged_addresses.virtual +Object that specifies a fixed IP address and port number that downstream services in a service mesh can use to connect to the service. The `virtual` field contains the following parameters: + +- `address` +- `port` + +Virtual addresses are not required to be routable IPs within the network. They are strictly a control plane construct used to provide a fixed address for the instances of a logical service. Egress connections from the proxy to an upstream service go to the IP address of an individual service instance and not the virtual address of the logical service. + +If the following conditions are met, connections to virtual addresses are load balanced across available instances of a service, provided the following conditions are satisfied: + +1. [Transparent proxy](/consul/docs/connect/transparent-proxy) is enabled for the downstream and upstream services. +1. The upstream service is not configured for individual instances to be [dialed directly](/consul/docs/connect/config-entries/service-defaults#dialeddirectly). + +In the following example, the downstream services in the mesh can connect to the `redis` service at `203.0.113.50` on port `80`: + + + +```hcl +service { + name = "redis" + address = "192.0.2.10" + port = 80 + tagged_addresses { + virtual = { + address = "203.0.113.50" + port = 80 + } + } +} +``` + +```json +{ + "service": { + "name": "redis", + "address": "192.0.2.10", + "port": 80, + "tagged_addresses": { + "virtual": { + "address": "203.0.113.50", + "port": 80 + } + } + } +} +``` + + + +### tagged_addresses.wan +Object that specifies either an IPv4 or IPv6 WAN address and port number where the service or node is accessible from a remote datacenter. You can specify one or more of the following fields: + +- `wan` +- `wan_ipv4` +- `wan_ipv6` + +The field contains the following parameters: + +- `address` +- `port` + +In the following example, services or nodes in remote datacenters can reach the `redis` service at `198.51.100.200:80` and `[2001:db8:5:6:1337::1eaf]:80`: + + + +```hcl +service { + name = "redis" + address = "192.0.2.10" + port = 80 + tagged_addresses { + wan = { + address = "198.51.100.200" + port = 80 + } + wan_ipv4 = { + address = "198.51.100.200" + port = 80 + } + wan_ipv6 = { + address = "2001:db8:5:6:1337::1eaf" + port = 80 + } + } +} +``` + +```json +{ + "service": { + "name": "redis", + "address": "192.0.2.10", + "port": 80, + "tagged_addresses": { + "wan": { + "address": "198.51.100.200", + "port": 80 + }, + "wan_ipv4": { + "address": "198.51.100.200", + "port": 80 + }, + "wan_ipv6": { + "address": "2001:db8:5:6:1337::1eaf", + "port": 80 + } + } + } +} +``` + + + +### socket_path +String value that specifies the path to the service socket. Specify this parameter to expose the service to the mesh if the service listens on a Unix Domain socket. + +- Type: string +- Default: none + +### enable_tag_override +Boolean value that determines if the anti-entropy feature for the service is enabled. +Set to `true` to allow external Consul agents modify tags on the services in the Consul catalog. The local Consul agent ignores updated tags during subsequent sync operations. + +This parameter only applies to the locally-registered service. If multiple nodes register a service with the same `name`, the `enable_tag_override` configuration, and all other service configuration items, operate independently. + +Refer to [Modify anti-entropy synchronization](/consul/docs/services/usage/define-services#modify-anti-entropy-synchronization) for additional usage information. + +- Type: boolean +- Default: `false` + +### checks +The `checks` block contains an array of objects that define health checks for the service. Health checks perform several safety functions, such as allowing a web balancer to gracefully remove failing nodes and allowing a database to replace a failed secondary. Refer to [Health Check Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about configuring health checks. + +### kind +String value that identifies the service as a proxy and determines its role in the service mesh. Do not configure the `kind` parameter for non-proxy service instances. Refer to [Consul Service Mesh](/consul/docs/connect) for additional information. + +You can specify the following values: + +- `connect-proxy`: Defines the configuration for a service mesh proxy. Refer to [Register a Service Mesh Proxy Outside of a Service Registration](/consul/docs/connect/registration/service-registration) for details about registering a service as a service mesh proxy. +- `ingress-gateway`: Defines the configuration for an [ingress gateway](/consul/docs/connect/config-entries/ingress-gateway) +- `mesh-gateway`: Defines the configuration for a [mesh gateway](/consul/docs/connect/gateways/mesh-gateway) +- `terminating-gateway`: Defines the configuration for a [terminating gateway](/consul/docs/connect/gateways/terminating-gateway) + +For non-service registration roles, the `kind` field has a different context when used to define configuration entries, such as `service-defaults`. Refer to the documentation for the configuration entry you want to implement for additional information. + +### proxy +Object that specifies proxy configurations when the service is configured to operate as a proxy in a service mesh. Do not configure the `proxy` parameter for non-proxy service instances. Refer to [Register a Service Mesh Proxy Outside of a Service Registration](/consul/docs/connect/registration/service-registration) for details about registering your service as a service mesh proxy. Refer to [`kind`](#kind) for information about the types of proxies you can define. Services that you assign proxy roles to are registered as services in the catalog. + +### connect +Object that configures a Consul service mesh connection. You should only configure the `connect` block of parameters if you are using Consul service mesh. Refer to [Consul Service Mesh](/consul/docs/connect) for additional information. + +The following table describes the parameters that you can place in the `connect` block: + +| Parameter | Description | Default | +| --- | --- | --- | +| `native` | Boolean value that advertises the service as a native service mesh proxy. Use this parameter to integrate your application with the `connect` API. Refer to [Service Mesh Native App Integration Overview](/consul/docs/connect/native) for additional information. If set to `true`, do not configure a `sidecar_service`. | `false` | +| `sidecar_service` | Object that defines a sidecar proxy for the service. Do not configure if `native` is set to `true`. Refer to [Register a Service Mesh Proxy in a Service Registration](/consul/docs/connect/registration/sidecar-service) for usage and configuration details. | Refer to [Register a Service Mesh Proxy in a Service Registration](/consul/docs/connect/registration/sidecar-service) for default configurations. | + +### weights +Object that configures how the service responds to DNS SRV requests based on the service's health status. Configuring allows service instances with more capacity to respond to DNS SRV requests. It also reduces the load on services with checks in `warning` status by giving passing instances a higher weight. + +You can specify one or more of the following states and configure an integer value indicating its weight: + +- `passing` +- `warning` +- `critical` + +Larger integer values increase the weight state. Services have the following default weights: + +- `"passing" : 1` +- `"warning" : 1` + +Services in a `critical` state are excluded from DNS responses by default. Services with `warning` checks are included in responses by default. Refer to [Perform Static DNS Queries](/consul/docs/services/discovery/dns-static-lookups) for additional information. + +In the following example, service instances in a `passing` state respond to DNS SRV requests, while instances in a `critical` instance can still respond at a lower frequency: + + + +```hcl +service { + ## ... + weights = { + passing = 3 + warning = 2 + critical = 1 + } + ## ... +} +``` + +```json +"service": { + ## ... + "weights": { + "passing": 3, + "warning": 2, + "critical": 1 + }, + ## ... +} +``` + + + +### token +String value that specifies the ACL token to present when registering the service if ACLs are enabled. The token is required for the service to interact with the service catalog. + +If [ACLs](/consul/docs/security/acl) and [namespaces](/consul/docs/enterprise/namespaces) are enabled, you can register services scoped to the specific [`namespace`](#namespace) associated with the ACL token in a Consul cluster. + +Services registered with a service definition do not inherit the namespace associated with the ACL token specified in the token field. The `namespace` and `token` parameters must be included in the service definition for the service to be registered to the namespace that the ACL token is scoped to. + +- Type: string +- Default: none + +### namespace +String value that specifies the namespace in which to register the service. Refer [Namespaces](/consul/docs/enterprise/namespaces) for additional information. + +- Type: string +- Default: none + +## Multiple service definitions + +You can define multiple services in a single definition file in the `services` block. This enables you register multiple services in a single command. Note that the HTTP API does not support the `services` block. + + + +```hcl +services { + id = "red0" + name = "redis" + tags = [ + "primary" + ] + address = "" + port = 6000 + checks = [ + { + args = ["/bin/check_redis", "-p", "6000"] + interval = "5s" + timeout = "20s" + } + ] +} +services { + id = "red1" + name = "redis" + tags = [ + "delayed", + "secondary" + ] + address = "" + port = 7000 + checks = [ + { + args = ["/bin/check_redis", "-p", "7000"] + interval = "30s" + timeout = "60s" + } + ] +} +``` + +```json +{ + "services": [ + { + "id": "red0", + "name": "redis", + "tags": [ + "primary" + ], + "address": "", + "port": 6000, + "checks": [ + { + "args": ["/bin/check_redis", "-p", "6000"], + "interval": "5s", + "timeout": "20s" + } + ] + }, + { + "id": "red1", + "name": "redis", + "tags": [ + "delayed", + "secondary" + ], + "address": "", + "port": 7000, + "checks": [ + { + "args": ["/bin/check_redis", "-p", "7000"], + "interval": "30s", + "timeout": "60s" + } + ] + } + ] +} +``` + + + +## Example definition +The following example includes all possible parameters, but only the top-level `service` parameter and its `name` parameter are required by default. + + + +```hcl +service { + name = "redis" + id = "redis" + port = 80 + tags = ["primary"] + + meta = { + custom_meta_key = "custom_meta_value" + } + + tagged_addresses = { + lan = { + address = "192.168.0.55" + port = 8000 + } + + wan = { + address = "198.18.0.23" + port = 80 + } + } + + port = 8000 + socket_path = "/tmp/redis.sock" + enable_tag_override = false + + checks = [ + { + args = ["/usr/local/bin/check_redis.py"] + interval = "10s" + } + ] + + kind = "connect-proxy" + proxy_destination = "redis" + + proxy = { + destination_service_name = "redis" + destination_service_id = "redis1" + local_service_address = "127.0.0.1" + local_service_port = 9090 + local_service_socket_path = "/tmp/redis.sock" + mode = "transparent" + + transparent_proxy { + outbound_listener_port = 22500 + } + + mesh_gateway = { + mode = "local" + } + + expose = { + checks = true + + paths = [ + { + path = "/healthz" + local_path_port = 8080 + listener_port = 21500 + protocol = "http2" + } + ] + } + } + + connect = { + native = false + } + + weights = { + passing = 5 + warning = 1 + } + + token = "233b604b-b92e-48c8-a253-5f11514e4b50" + namespace = "foo" +} +``` + +```json +{ + "service": { + "id": "redis", + "name": "redis", + "tags": ["primary"], + "address": "", + "meta": { + "meta": "for my service" + }, + "tagged_addresses": { + "lan": { + "address": "192.168.0.55", + "port": 8000, + }, + "wan": { + "address": "198.18.0.23", + "port": 80 + } + }, + "port": 8000, + "socket_path": "/tmp/redis.sock", + "enable_tag_override": false, + "checks": [ + { + "args": ["/usr/local/bin/check_redis.py"], + "interval": "10s" + } + ], + "kind": "connect-proxy", + "proxy_destination": "redis", // Deprecated + "proxy": { + "destination_service_name": "redis", + "destination_service_id": "redis1", + "local_service_address": "127.0.0.1", + "local_service_port": 9090, + "local_service_socket_path": "/tmp/redis.sock", + "mode": "transparent", + "transparent_proxy": { + "outbound_listener_port": 22500 + }, + "config": {}, + "upstreams": [], + "mesh_gateway": { + "mode": "local" + }, + "expose": { + "checks": true, + "paths": [ + { + "path": "/healthz", + "local_path_port": 8080, + "listener_port": 21500, + "protocol": "http2" + } + ] + } + }, + "connect": { + "native": false, + "sidecar_service": {} + "proxy": { // Deprecated + "command": [], + "config": {} + } + }, + "weights": { + "passing": 5, + "warning": 1 + }, + "token": "233b604b-b92e-48c8-a253-5f11514e4b50", + "namespace": "foo" + } +} +``` + + + + + + diff --git a/website/content/docs/services/discovery/dns-configuration.mdx b/website/content/docs/services/discovery/dns-configuration.mdx new file mode 100644 index 00000000000..794be43a206 --- /dev/null +++ b/website/content/docs/services/discovery/dns-configuration.mdx @@ -0,0 +1,76 @@ +--- +layout: docs +page_title: Configure Consul DNS behavior +description: -> + Learn how to modify the default DNS behavior so that services and nodes can easily discover other services and nodes in your network. +--- + +# Configure Consul DNS behavior + +This topic describes the default behavior of the Consul DNS functionality and how to customize how Consul performs queries. + +## Introduction +The Consul DNS is the primary interface for querying records when Consul service mesh is disabled and your network runs in a non-Kubernetes environment. The DNS enables you to look up services and nodes registered with Consul using terminal commands instead of making HTTP API requests to Consul. Refer to the [Discover Consul Nodes and Services Overview](/consul/docs/services/discovery/dns-overview) for additional information. + +## Configure DNS behaviors +By default, the Consul DNS listens for queries at `127.0.0.1:8600` and uses the `consul` domain. Specify the following parameters in the agent configuration to determine DNS behavior when querying services: + +- [`client_addr`](/consul/docs/agent/config/config-files#client_addr) +- [`ports.dns`](/consul/docs/agent/config/config-files#dns_port) +- [`recursors`](/consul/docs/agent/config/config-files#recursors) +- [`domain`](/consul/docs/agent/config/config-files#domain) +- [`alt_domain`](/consul/docs/agent/config/config-files#alt_domain) +- [`dns_config`](/consul/docs/agent/config/config-files#dns_config) + +### Configure WAN address translation +By default, Consul DNS queries return a node's local address, even when being queried from a remote datacenter. You can configure the DNS to reach a node from outside its datacenter by specifying the address in the following configuration fields in the Consul agent: + +- [advertise-wan](/consul/docs/agent/config/cli-flags#_advertise-wan) +- [translate_wan_addrs](/consul//docs/agent/config/config-files#translate_wan_addrs) + +### Use a custom DNS resolver library +You can specify a list of addresses in the agent's [`recursors`](/consul/docs/agent/config/config-files#recursors) field to provide upstream DNS servers that recursively resolve queries that are outside the service domain for Consul. + +Nodes that query records outside the `consul.` domain resolve to an upstream DNS. You can specify IP addresses or use `go-sockaddr` templates. Consul resolves IP addresses in the specified order and ignores duplicates. + +### Enable non-Consul queries +You enable non-Consul queries to be resolved by setting Consul as the DNS server for a node and providing a [`recursors`](/consul/docs/agent/config/config-files#recursors) configuration. + +### Forward queries to an agent +You can forward all queries sent to the `consul.` domain from the existing DNS server to a Consul agent. Refer to [Forward DNS for Consul Service Discovery](/consul/tutorials/networking/dns-forwarding) for instructions. + +### Query an alternate domain +By default, Consul responds to DNS queries in the `consul` domain, but you can set a specific domain for responding to DNS queries by configuring the [`domain`](/consul/docs/agent/config/config-files#domain) parameter. + +You can also specify an additional domain in the [`alt_domain`](/consul/docs/agent/config/config-files#alt_domain) agent configuration option, which configures Consul to respond to queries in a secondary domain. Configuring an alternate domain may be useful during a DNS migration or to distinguish between internal and external queries, for example. + +Consul's DNS response uses the same domain as the query. + +In the following example, the `alt_domain` parameter in the agent configuration is set to `test-domain`, which enables operators to query the domain: + +```shell-session +$ dig @127.0.0.1 -p 8600 consul.service.test-domain SRV + +;; QUESTION SECTION: +;consul.service.test-domain. IN SRV + +;; ANSWER SECTION: +consul.service.test-domain. 0 IN SRV 1 1 8300 machine.node.dc1.test-domain. + +;; ADDITIONAL SECTION: +machine.node.dc1.test-domain. 0 IN A 127.0.0.1 +machine.node.dc1.test-domain. 0 IN TXT "consul-network-segment=" +``` +#### PTR queries +Responses to pointer record (PTR) queries, such as `.in-addr.arpa.`, always use the [primary domain](/consul/docs/agent/config/config-files#domain) and not the alternative domain. + +### Caching +By default, DNS results served by Consul are not cached. Refer to the [DNS Caching tutorial](/consul/tutorials/networking/dns-caching) for instructions on how to enable caching. + + + + + + + + diff --git a/website/content/docs/services/discovery/dns-dynamic-lookups.mdx b/website/content/docs/services/discovery/dns-dynamic-lookups.mdx new file mode 100644 index 00000000000..436998ad9c7 --- /dev/null +++ b/website/content/docs/services/discovery/dns-dynamic-lookups.mdx @@ -0,0 +1,110 @@ +--- +layout: docs +page_title: Enable dynamic DNS queries +description: -> + Learn how to dynamically query the Consul DNS using prepared queries, which enable robust service and node lookups. +--- + +# Enable dynamic DNS queries + +This topic describes how to dynamically query the Consul catalog using prepared queries. Prepared queries are configurations that enable you to register a complex service query and execute it on demand. For information about how to perform standard node and service lookups, refer to [Perform Static DNS Queries](/consul/docs/services/discovery/dns-static-lookups). + +## Introduction + +Prepared queries provide a rich set of lookup features, such as filtering by multiple tags and automatically failing over to look for services in remote datacenters if no healthy nodes are available in the local datacenter. You can also create prepared query templates that match names using a prefix match, allowing a single template to apply to potentially many services. Refer to [Query Consul Nodes and Services Overview](/consul/docs/services/discovery/dns-overview) for additional information about DNS query behaviors. + +## Requirements + +Consul 0.6.4 or later is required to create prepared query templates. + +### ACLs + +If ACLs are enabled, the querying service must present a token linked to permissions that enable read access for query, service, and node resources. Refer to the following documentation for information about creating policies to enable read access to the necessary resources: + +- [Prepared query rules](/consul/docs/security/acl/acl-rules#prepared-query-rules) +- [Service rules](/consul/docs/security/acl/acl-rules#service-rules) +- [Node rules](/consul/docs/security/acl/acl-rules#node-rules) + +## Create prepared queries + +Refer to the [prepared query reference](/consul/api-docs/query#create-prepared-query) for usage information. + +1. Specify the prepared query options in JSON format. The following prepared query targets all instances of the `redis` service in `dc1` and `dc2`: + + ```json + { + "Name": "my-query", + "Session": "adf4238a-882b-9ddc-4a9d-5b6758e4159e", + "Token": "", + "Service": { + "Service": "redis", + "Failover": { + "NearestN": 3, + "Datacenters": ["dc1", "dc2"] + }, + "Near": "node1", + "OnlyPassing": false, + "Tags": ["primary", "!experimental"], + "NodeMeta": { + "instance_type": "m3.large" + }, + "ServiceMeta": { + "environment": "production" + } + }, + "DNS": { + "TTL": "10s" + } + } + ``` + + Refer to the [prepared query configuration reference](/consul/api-docs/query#create-prepared-query) for information about all available options. + +1. Send the query in a POST request to the [`/query` API endpoint](/consul/api-docs/query). If the request is successful, Consul prints an ID for the prepared query. + + In the following example, the prepared query configuration is stored in the `payload.json` file: + + ```shell-session + $ curl --request POST --data @payload.json http://127.0.0.1:8500/v1/query + {"ID":"014af5ff-29e6-e972-dcf8-6ee602137127"}% + ``` + +1. To run the query, send a GET request to the endpoint and specify the ID returned from the POST call. + + ```shell-session + $ curl http://127.0.0.1:8500/v1/query/14af5ff-29e6-e972-dcf8-6ee602137127/execute\?near\=_agent + ``` + +## Execute prepared queries + +You can execute a prepared query using the standard lookup format or the strict RFC 2782 SRV lookup. + +### Standard lookup + +Use the following format to execute a prepared query using the standard lookup format: + +``` +.query[.]. +``` + +Refer [Standard lookups](/consul/docs/services/discovery/dns-static-lookups#standard-lookups) for additional information about the standard lookup format in Consul. + +### RFC 2782 SRV lookup + +Use the following format to execute a prepared query using the RFC 2782 lookup format: + +``` +_._tcp.query[.]. +``` + +For additional information about following the RFC 2782 SRV lookup format in Consul, refer to [RFC 2782 Lookup](/consul/docs/services/discovery/dns-static-lookups#rfc-2782-lookup). For general information about the RFC 2782 specification, refer to [A DNS RR for specifying the location of services \(DNS SRV\)](https://tools.ietf.org/html/rfc2782). + +### Lookup options + +The `datacenter` subdomain is optional. By default, the lookup queries the datacenter of this Consul agent. + +The `query name` or `id` subdomain is the name or ID of an existing prepared query. + +## Results + +To allow for simple load balancing, Consul returns the set of nodes in random order for each query. Prepared queries support A and SRV records. SRV records provide the port that a service is registered. Consul only serves SRV records if the client specifically requests them. diff --git a/website/content/docs/services/discovery/dns-overview.mdx b/website/content/docs/services/discovery/dns-overview.mdx new file mode 100644 index 00000000000..37eda715de2 --- /dev/null +++ b/website/content/docs/services/discovery/dns-overview.mdx @@ -0,0 +1,41 @@ +--- +layout: docs +page_title: DNS usage overview +description: >- + For service discovery use cases, Domain Name Service (DNS) is the main interface to look up, query, and address Consul nodes and services. Learn how a Consul DNS lookup can help you find services by tag, name, namespace, partition, datacenter, or domain. +--- + +# DNS usage overview + +This topic provides overview information about how to look up Consul nodes and services using the Consul DNS. + +## Consul DNS +The Consul DNS is the primary interface for discovering services registered in the Consul catalog. The DNS enables you to look up services and nodes registered with Consul using terminal commands instead of making HTTP API requests to Consul. + +We recommend using the DNS for service discovery in virtual machine (VM) environments because it removes the need to modify native applications so that they can consume the Consul service discovery APIs. + +The DNS has several default configurations, but you can customize how the server processes lookups. Refer to [Configure DNS Behaviors](/consul/docs/services/discovery/dns-configuration) for additional information. + +### DNS versus native app integration +You can use DNS to reach services registered with Consul or modify your application to natively consume the Consul service discovery HTTP APIs. + +We recommend using the DNS because it is less invasive. You do not have to modify your application with Consul to retrieve the service’s connection information. Instead, you can use a DNS fully qualified domain (FQDN) that conforms to Consul's lookup format to retreive the relevant information. + +Refer to [ Native App Integration](/consul/docs/connect/native) and its [Go package](/consul/docs/connect/native/go) for additional information. + +### DNS versus upstreams +If you are using Consul for service discovery and have not enabled service mesh features, then use the DNS to discover services and nodes in the Consul catalog. + +If you are using Consul for service mesh on VMs, you can use upstreams or DNS. We recommend using upstreams because you can query services and nodes without modifying the application code or environment variables. Refer to [Upstream Configuration Reference](/consul/docs/connect/registration/service-registration#upstream-configuration-reference) for additional information. + +If you are using Consul on Kubernetes, refer to [the upstreams annotation documentation](/consul/docs/k8s/annotations-and-labels#consul-hashicorp-com-connect-service-upstreams) for additional information. + +## Static queries +Node lookups and service lookups are the fundamental types of static queries. Depending on your use case, you may need to use different query methods and syntaxes to query the DNS for services and nodes. + +Consul relies on a very specific format for queries to resolve names. Note that all queries are case-sensitive. + +Refer to [Perform Static DNS Lookups](/consul/docs/services/discovery/dns-static-lookups) for details about how to perform node and service lookups. + +## Prepared queries +Prepared queries are configurations that enable you to register complex DNS queries. They provide lookup features that extend Consul's service discovery capabilities, such as filtering by multiple tags and automatically querying remote datacenters for services if no healthy nodes are available in the local datacenter. You can also create prepared query templates that match names using a prefix match, allowing a single template to apply to potentially many services. Refer to [Enable Dynamic DNS Queries](/consul/docs/services/discovery/dns-dynamic-lookups) for additional information. diff --git a/website/content/docs/services/discovery/dns-static-lookups.mdx b/website/content/docs/services/discovery/dns-static-lookups.mdx new file mode 100644 index 00000000000..353f4e2628e --- /dev/null +++ b/website/content/docs/services/discovery/dns-static-lookups.mdx @@ -0,0 +1,391 @@ +--- +layout: docs +page_title: Perform static DNS queries +description: -> + Learn how to use standard Consul DNS lookup formats to enable service discovery for services and nodes. +--- + +# Perform static DNS queries +This topic describes how to query the Consul DNS to look up nodes and services registered with Consul. Refer to [Enable Dynamic DNS Queries](/consul/docs/services/discovery/dns-dynamic-lookups) for information about using prepared queries. + +## Introduction +Node lookups and service lookups are the fundamental types of queries you can perform using the Consul DNS. Node lookups interrogate the catalog for named Consul agents. Service lookups interrogate the catalog for services registered with Consul. Refer to [DNS Usage Overivew](/consul/docs/services/discovery/dns-overview) for additional background information. + +## Requirements +All versions of Consul support DNS lookup features. + +### ACLs +If ACLs are enabled, you must present a token linked with the necessary policies. We recommend using a separate token in production deployments for querying the DNS. By default, Consul agents resolve DNS requests using the preconfigured tokens in order of precedence: + +The agent's [`default` token](/consul/docs/agent/config/config-files#acl_tokens_default) +The built-in [`anonymous` token](/consul/docs/security/acl/acl-tokens#built-in-tokens). + + +The following table describes the available DNS lookups and required policies when ACLs are enabled: + +| Lookup | Type | Description | ACLs Required | +| --- | --- | --- | --- | +| `*.node.consul` | Node | Allows Consul to resolve DNS requests for the target node. Example: `.node.consul` | `node:read` | +| `*.service.consul`
    `*.connect.consul`
    `*.ingress.consul`
    `*.virtual.consul` |Service: standard | Allows Consul to resolve DNS requests for target service instances running on ACL-authorized nodes. Example: `.service.consul` | `service:read`
    `node:read` | + +> **Tutorials**: For hands-on guidance on how to configure an appropriate token for DNS, refer to the tutorial for [Production Environments](/consul/tutorials/security/access-control-setup-production#token-for-dns) and [Development Environments](/consul/tutorials/day-0/access-control-setup#enable-acls-on-consul-clients). + +## Node lookups +Specify the name of the node, datacenter, and domain using the following FQDN syntax: + +```text +.node[..dc]. +``` + +The `datacenter` subdomain is optional. By default, the lookup queries the datacenter of the agent. + +By default, the domain is `consul`. Refer to [Configure DNS Behaviors](/consul/docs/services/discovery/dns-configuration) for information about using alternate domains. + +### Node lookup results + +Node lookups return A and AAAA records that contain the IP address and TXT records containing the `node_meta` values of the node. + +By default, TXT record values match the node's metadata key-value pairs according to [RFC1464](https://www.ietf.org/rfc/rfc1464.txt). If the metadata key starts with `rfc1035-`, the TXT record only includes the node's metadata value. + +The following example lookup queries the `foo` node in the `default` datacenter: + +```shell-session +$ dig @127.0.0.1 -p 8600 foo.node.consul ANY + +; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 foo.node.consul ANY +; (1 server found) +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 24355 +;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 1, ADDITIONAL: 0 +;; WARNING: recursion requested but not available + +;; QUESTION SECTION: +;foo.node.consul. IN ANY + +;; ANSWER SECTION: +foo.node.consul. 0 IN A 10.1.10.12 +foo.node.consul. 0 IN TXT "meta_key=meta_value" +foo.node.consul. 0 IN TXT "value only" + + +;; AUTHORITY SECTION: +consul. 0 IN SOA ns.consul. postmaster.consul. 1392836399 3600 600 86400 0 +``` + +### Node lookups for Consul Enterprise + +Consul Enterprise includes the admin partition concept, which is an abstraction that lets you define isolated administrative network areas. Refer to [Admin Partitions](/consul/docs/enterprise/admin-partitions) for additional information. + +Consul nodes reside in admin partitions within a datacenter. By default, node lookups query the same partition and datacenter of the Consul agent that received the DNS query. + +Use the following query format to specify a partition for a node lookup: + +``` +.node[..ap][..dc]. +``` + +Consul server agents are in the `default` partition. If you send a DNS query to Consul server agents, you must explicitly specify the partition of the target node if it is not `default`. + +## Service lookups +You can query the network for service providers using either the [standard lookup](#standard-lookup) method or [strict RFC 2782 lookup](#rfc-2782-lookup) method. + +By default, all SRV records are weighted equally in service lookup responses, but you can configure the weights using the [`Weights`](/consul/docs/services/configuration/services-configuration-reference#weights) attribute of the service definition. Refer to [Define Services](/consul/docs/services/usage/define-services) for additional information. + +The DNS protocol limits the size of requests, even when performing DNS TCP queries, which may affect your experience querying for services. For services with more than 500 instances, you may not be able to retrieve the complete list of instances for the service. Refer to [RFC 1035, Domain Names - Implementation and Specification](https://datatracker.ietf.org/doc/html/rfc1035#section-2.3.4) for additional information. + +Consul randomizes DNS SRV records and ignores weights specified in service configurations when printing responses. If records are truncated, each client using weighted SRV responses may have partial and inconsistent views of instance weights. As a result, the request distribution may be skewed from the intended weights. We recommend calling the [`/catalog/nodes` API endpoint](/consul/api-docs/catalog#list-nodes) to retrieve the complete list of nodes. You can apply query parameters to API calls to sort and filter the results. + +### Standard lookups +To perform standard service lookups, specify tags, the name of the service, datacenter, cluster peer, and domain using the following syntax to query for service providers: + +```text +[.].service[..dc][..peer]. +``` + +The `tag` subdomain is optional. It filters responses so that only service providers containing the tag appear. + +The `datacenter` subdomain is optional. By default, Consul interrogates the querying agent's datacenter. + +The `cluster-peer` name is optional, and specifies the [cluster peer](/consul/docs/connect/cluster-peering) whose [exported services](/consul/docs/connect/config-entries/exported-services) should be the target of the query. + +By default, the lookups query in the `consul` domain. Refer to [Configure DNS Behaviors](/consul/docs/services/discovery/dns-configuration) for information about using alternate domains. + +#### Standard lookup results +Standard services queries return A and SRV records. SRV records include the port that the service is registered on. SRV records are only served if the client specifically requests them. + +Services that fail their health check or that fail a node system check are omitted from the results. As a load balancing measure, Consul randomizes the set of nodes returned in the response. These mechanisms help you use DNS with application-level retries as the foundation for a self-healing service-oriented architecture. + +The following example retrieves the SRV records for any `redis` service registered in Consul. + +```shell-session +$ dig @127.0.0.1 -p 8600 consul.service.consul SRV + +; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 consul.service.consul ANY +; (1 server found) +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 50483 +;; flags: qr aa rd; QUERY: 1, ANSWER: 3, AUTHORITY: 1, ADDITIONAL: 1 +;; WARNING: recursion requested but not available + +;; QUESTION SECTION: +;consul.service.consul. IN SRV + +;; ANSWER SECTION: +consul.service.consul. 0 IN SRV 1 1 8300 foobar.node.dc1.consul. + +;; ADDITIONAL SECTION: +foobar.node.dc1.consul. 0 IN A 10.1.10.12 +``` + +The following example command and FQDN retrieves the SRV records for the primary Postgres service in the secondary datacenter: + +```shell-session hideClipboard +$ dig @127.0.0.1 -p 8600 primary.postgresql.service.dc2.consul SRV + +; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 primary.postgresql.service.dc2.consul ANY +; (1 server found) +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 50483 +;; flags: qr aa rd; QUERY: 1, ANSWER: 3, AUTHORITY: 1, ADDITIONAL: 1 +;; WARNING: recursion requested but not available + +;; QUESTION SECTION: +;consul.service.consul. IN SRV + +;; ANSWER SECTION: +consul.service.consul. 0 IN SRV 1 1 5432 primary.postgresql.service.dc2.consul. + +;; ADDITIONAL SECTION: +primary.postgresql.service.dc2.consul. 0 IN A 10.1.10.12 +``` + +### RFC 2782 lookup +Per [RFC 2782](https://tools.ietf.org/html/rfc2782), SRV queries must prepend `service` and `protocol` values with an underscore (`_`) to prevent DNS collisions. Use the following syntax to perform RFC 2782 lookups: + +```text +_._[.service][.]. +``` + +You can create lookups that filter results by placing service tags in the `protocol` field. Use the following syntax to create RFC 2782 lookups that filter results based on service tags: + +```text +_._[.service][.]. +``` + +The following example queries the `rabbitmq` service tagged with `amqp`, which returns an instance at `rabbitmq.node1.dc1.consul` on port `5672`: + +```shell-session +$ dig @127.0.0.1 -p 8600 _rabbitmq._amqp.service.consul SRV + +; <<>> DiG 9.8.3-P1 <<>> @127.0.0.1 -p 8600 _rabbitmq._amqp.service.consul ANY +; (1 server found) +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 52838 +;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 +;; WARNING: recursion requested but not available + +;; QUESTION SECTION: +;_rabbitmq._amqp.service.consul. IN SRV + +;; ANSWER SECTION: +_rabbitmq._amqp.service.consul. 0 IN SRV 1 1 5672 rabbitmq.node1.dc1.consul. + +;; ADDITIONAL SECTION: +rabbitmq.node1.dc1.consul. 0 IN A 10.1.11.20 +``` + +You can also perform RFC 2782 lookups that target a specific [cluster peer](/consul/docs/connect/cluster-peering) or datacenter by including `.dc` or `.peer` in the query labels: + +```text +_._[.service][..dc][..peer]. +``` + +The following example queries the `redis` service tagged with `tcp` for the cluster peer `phx1`, which returns two instances, one at `10.1.11.83:29081` and one at `10.1.11.86:29142`: + +```shell-session +dig @127.0.0.1 -p 8600 _redis._tcp.service.phx1.peer.consul SRV + +; <<>> DiG 9.18.15 <<>> @127.0.0.1 -p 8600 _redis._tcp.service.phx1.peer.consul SRV +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 40572 +;; flags: qr aa rd ra; QUERY: 1, ANSWER: 2, AUTHORITY: 0, ADDITIONAL: 2 + +;; OPT PSEUDOSECTION: +; EDNS: version: 0, flags:; udp: 1232 +;; QUESTION SECTION: +;_redis._tcp.service.phx1.peer.consul. IN SRV + +;; ANSWER SECTION: +_redis._tcp.service.phx1.peer.consul. 0 IN SRV 1 1 29081 0a000d53.addr.consul. +_redis._tcp.service.phx1.peer.consul. 0 IN SRV 1 1 29142 0a010d56.addr.consul. + +;; ADDITIONAL SECTION: +0a000d53.addr.consul. 0 IN A 10.1.11.83 +0a010d56.addr.consul. 0 IN A 10.1.11.86 +``` + +#### SRV responses for hosts in the .addr subdomain + +If a service registered with Consul is configured with an explicit IP address or addresses in the [`address`](/consul/docs/services/configuration/services-configuration-reference#address) or [`tagged_address`](/consul/docs/services/configuration/services-configuration-reference#tagged_address) parameter, then Consul returns the hostname in the target field of the answer section for the DNS SRV query according to the following format: + +```text +.addr..consul`. +``` + +In the following example, the `rabbitmq` service is registered with an explicit IPv4 address of `192.0.2.10`. + +```hcl +node_name = "node1" + +services { + name = "rabbitmq" + address = "192.0.2.10" + port = 5672 +} +{ + "node_name": "node1", + "services": [ + { + "name": "rabbitmq", + "address": "192.0.2.10", + "port": 5672 + } + ] +} +``` + +The following example SRV query response contains a single record with a hostname written as a hexadecimal value: + +```shell-session +$ dig @127.0.0.1 -p 8600 -t srv _rabbitmq._tcp.service.consul +short +1 1 5672 c000020a.addr.dc1.consul. +``` + +You can convert hex octets to decimals to reveal the IP address. The following example command converts the hostname expressed as `c000020a` into the IPv4 address specified in the service registration. + +``` +$ echo -n "c000020a" | perl -ne 'printf("%vd\n", pack("H*", $_))' +192.0.2.10 +``` + +In the following example, the `rabbitmq` service is registered with an explicit IPv6 address of `2001:db8:1:2:cafe::1337`. + +```hcl +node_name = "node1" + +services { + name = "rabbitmq" + address = "2001:db8:1:2:cafe::1337" + port = 5672 +} +{ + "node_name": "node1", + "services": [ + { + "name": "rabbitmq", + "address": "2001:db8:1:2:cafe::1337", + "port": 5672 + } + ] +} +``` + +The following example SRV query response contains a single record with a hostname written as a hexadecimal value: + +```shell-session +$ dig @127.0.0.1 -p 8600 -t SRV _rabbitmq._tcp.service.consul +short +1 1 5672 20010db800010002cafe000000001337.addr.dc1.consul. +``` + +The response contains the fully-expanded IPv6 address with colon separators removed. The following command re-adds the colon separators to display the fully expanded IPv6 address that was specified in the service registration. + +```shell-session +$ echo -n "20010db800010002cafe000000001337" | perl -ne 'printf join(":", unpack("(A4)*", $_))."\n"' +2001:0db8:0001:0002:cafe:0000:0000:1337 +``` + +### Service lookups for Consul Enterprise +You can perform the following types of service lookups to query for services in another namespace, partition, and datacenter: + +- `.service` +- `.connect` +- `.virtual` +- `.ingress` + +Use the following query format to specify namespace, partition, or datacenter: +``` +[.].service[..ns][..ap][..dc] +``` + +The `namespace`, `partition`, and `datacenter` are optional. By default, all service lookups use the `default` namespace within the partition and datacenter of the Consul agent that received the DNS query. + +Consul server agents reside in the `default` partition. If DNS queries are addressed to Consul server agents, you must explicitly specify the partition of the target service when querying for services in partitions other than `default`. + +To lookup services imported from a cluster peer, refer to [Service virtual IP lookups for Consul Enterprise](#service-virtual-ip-lookups-for-consul-enterprise). + +#### Alternative formats for specifying namespace + +Although we recommend using the format described in [Service lookups for Consul Enterprise](#service-lookups-for-consul-enterprise) for readability, you can use the alternate query format to specify namespaces but not partitions: + +``` +[.].service... +``` + +### Service mesh-enabled service lookups + +Add the `.connect` subdomain to query for service mesh-enabled services: + +```text +.connect. +``` + +This finds all service mesh-capable endpoints for the service. A service mesh-capable endpoint may be a proxy for a service or a natively integrated service mesh application. The DNS interface does not differentiate the two. + +Many services use a proxy that handles service discovery automatically. As a result, they may not use the DNS format, which is primarily for service mesh-native applications. +This endpoint only finds services within the same datacenter and does not support tags. Refer to the [`catalog` API endpoint](/consul/api-docs/catalog) for more complex behaviors. + +### Service virtual IP lookups + +Add the `.virtual` subdomain to queries to find the unique virtual IP allocated for a service: + +```text +.virtual[.]. +``` + +This returns the unique virtual IP for any service mesh-capable service. Each service mesh service has a virtual IP assigned to it by Consul. Sidecar proxies use the virtual IP to enable the [transparent proxy](/consul/docs/connect/transparent-proxy) feature. + +The peer name is an optional. The DNS uses it to query for the virtual IP of a service imported from the specified peer. + +Consul adds virtual IPs to the [`tagged_addresses`](/consul/docs/services/configuration/services-configuration-reference#tagged_addresses) field in the service definition under the `consul-virtual` tag. + +#### Service virtual IP lookups for Consul Enterprise + +By default, a service virtual IP lookup checks the `default` namespace within the partition and datacenter of the Consul agent that received the DNS query. +To lookup services imported from a partition in another cluster peered to the querying cluster or open-source datacenter, specify the namespace and peer name in the lookup: + +```text +.virtual[.].. +``` + +To lookup services in a cluster peer that have not been imported, refer to [Service lookups for Consul Enterprise](#service-lookups-for-consul-enterprise). + +### Ingress Service Lookups + +Add the `.ingress` subdomain to your DNS FQDN to find ingress-enabled services: + +```text +.ingress. +``` + +This finds all ingress gateway endpoints for the service. + +This endpoint finds services within the same datacenter and does not support tags. Refer to the [`catalog` API endpoint](/consul/api-docs/catalog) for more complex behaviors. + +### UDP-based DNS queries + +When the DNS query is performed using UDP, Consul truncateß the results without setting the truncate bit. This prevents a redundant lookup over TCP that generates additional load. If the lookup is done over TCP, the results are not truncated. \ No newline at end of file diff --git a/website/content/docs/services/services.mdx b/website/content/docs/services/services.mdx new file mode 100644 index 00000000000..7300b9d1aa8 --- /dev/null +++ b/website/content/docs/services/services.mdx @@ -0,0 +1,49 @@ +--- +layout: docs +page_title: Services overview +description: >- + Learn about services and service discovery workflows and concepts for virtual machine environments. +--- + +# Services overview + +This topic provides overview information about services and how to make them discoverable in Consul when your network operates on virtual machines. If service mesh is enabled in your network, refer to the following articles for additional information about connecting services in a mesh: + +- [How Service Mesh Works](/consul/docs/connect/connect-internals) +- [How Consul Service Mesh Works on Kubernetes](/consul/docs/k8s/connect) + +## Introduction + +A _service_ is an entity in your network that performs a specialized operation or set of related operations. In many contexts, a service is software that you want to make available to users or other programs with access to your network. Services can also refer to native Consul functionality, such as _service mesh proxies_ and _gateways_, that enable you to establish connections between different parts of your network. + +You can define and register services with Consul, which makes them discoverable to other services in the network. You can also define various types of health checks that perform several safety functions, such as allowing a web balancer to gracefully remove failing nodes and allowing a database to replace a failed secondary. + +## Workflow + +For service discovery, the core Consul workflow for services consists of three stages: + +1. **Define services and health checks**: A service definition lets you define various aspects of the service, including how it is discovered by other services in the network. You can define health checks in the service definitions to verify the health of the service. Refer to [Define Services](/consul/docs/services/usage/define-services) and [Define Health Checks](/consul/docs/services/usage/checks) for additional information. + +1. **Register services and health checks**: After defining your services and health checks, you must register them with a Consul agent. Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for additional information. + +1. **Query for services**: After registering your services and health checks, other services in your network can use the DNS to perform static or dynamic lookups to access your service. Refer to [DNS Usage Overview](/consul/docs/services/discovery/dns-overview) for additional information about the different ways to discover services in your datacenters. + + +## Service mesh use cases + +Consul redirects service traffic through sidecar proxies if you use Consul service mesh. As a result, you must specify upstream configurations in service definitions. The service mesh experience is different for virtual machine (VM) and Kubernetes environments. + +### Virtual machines + +You must define upstream services in the service definition. Consul uses the upstream configuration to bind the service with its upstreams. After registering the service, you must start a sidecar proxy on the VM to enable mesh connectivity. Refer to [Register a Service Mesh Proxy in a Service Registration](/consul/docs/connect/registration/sidecar-service) for details. + +### Kubernetes + +If you use Consul on Kubernetes, enable the service mesh injector in your Consul Helm chart and Consul automatically adds a sidecar to each of your pods using the Kubernetes `Service` definition as a reference. You can specify upstream annotations in the `Deployment` definition to bind upstream services to the pods. +Refer to [`connectInject`](/consul/docs/k8s/connect#installation-and-configuration) and [the upstreams annotation documentation](/consul/docs/k8s/annotations-and-labels#consul-hashicorp-com-connect-service-upstreams) for additional information. + +### Multiple services + +You can define common characteristics for services in your mesh, such as the admin partition, namespace, or upstreams, by creating and applying a `service-defaults` configuration entry. You can also define override configurations for specific upstreams or service instances. To use `service-defaults` configuraiton entries, you must enable Consul service mesh in your network. + +Refer to [Define Service Defaults](/consul/docs/services/usage/define-services#define-service-defaults) for additional information. \ No newline at end of file diff --git a/website/content/docs/services/usage/checks.mdx b/website/content/docs/services/usage/checks.mdx new file mode 100644 index 00000000000..e72b219dd0b --- /dev/null +++ b/website/content/docs/services/usage/checks.mdx @@ -0,0 +1,592 @@ +--- +layout: docs +page_title: Define health checks +description: -> + Learn how to configure different types of health checks for services you register with Consul. +--- + +# Define health checks +This topic describes how to create different types of health checks for your services. + + +## Overview +Health checks are configurations that verifies the health of a service or node. Health checks configurations are nested in the `service` block. Refer to [Define Services](/consul/docs/services/usage/define-services) for information about specifying other service parameters. + +You can define individual health checks for your service in separate `check` blocks or define multiple checks in a `checks` block. Refer to [Define multiple checks](#define-multiple-checks) for additional information. + +You can create several different kinds of checks: + +- _Script_ checks invoke an external application that performs the health check, exits with an appropriate exit code, and potentially generates output. Script checks are one of the most common types of checks. +- _HTTP_ checks make an HTTP GET request to the specified URL and wait for the specified amount of time. HTTP checks are one of the most common types of checks. +- _TCP_ checks attempt to connect to an IP or hostname and port over TCP and wait for the specified amount of time. +- _UDP_ checks send UDP datagrams to the specified IP or hostname and port and wait for the specified amount of time. +- _Time-to-live (TTL)_ checks are passive checks that await updates from the service. If the check does not receive a status update before the specified duration, the health check enters a `critical`state. +- _Docker_ checks are dependent on external applications packaged with a Docker container that are triggered by calls to the Docker `exec` API endpoint. +- _gRPC_ checks probe applications that support the standard gRPC health checking protocol. +- _H2ping_ checks test an endpoint that uses http2. The check connects to the endpoint and sends a ping frame. +- _Alias_ checks represent the health state of another registered node or service. + +If your network runs in a Kubernetes environment, you can sync service health information with Kubernetes health checks. Refer to [Configure Health Checks for Consul on Kubernetes](/consul/docs/k8s/connect/health) for details. + +### Registration + +After defining health checks, you must register the service containing the checks with Consul. Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for additional information. If the service is already registered, you can reload the service configuration file to implement your health check. Refer to [Reload](/consul/commands/reload) for additional information. + +## Define multiple checks + +You can define multiple checks for a service in a single `checks` block. The `checks` block contains an array of objects. The objects contain the configuration for each health check you want to implement. The following example includes two script checks named `mem` and `cpu` and an HTTP check that calls the `/health` API endpoint. + + + +```hcl +checks = [ + { + id = "chk1" + name = "mem" + args = ["/bin/check_mem", "-limit", "256MB"] + interval = "5s" + }, + { + id = "chk2" + name = "/health" + http = "http://localhost:5000/health" + interval = "15s" + }, + { + id = "chk3" + name = "cpu" + args = ["/bin/check_cpu"] + interval = "10s" + }, + ... +] +``` + +```json +{ + "checks": [ + { + "id": "chk1", + "name": "mem", + "args": ["/bin/check_mem", "-limit", "256MB"], + "interval": "5s" + }, + { + "id": "chk2", + "name": "/health", + "http": "http://localhost:5000/health", + "interval": "15s" + }, + { + "id": "chk3", + "name": "cpu", + "args": ["/bin/check_cpu"], + "interval": "10s" + }, + ... + ] +} +``` + + + +## Define initial health check status +When checks are registered against a Consul agent, they are assigned a `critical` status by default. This prevents services from registering as `passing` and entering the service pool before their health is verified. You can add the `status` parameter to the check definition to specify the initial state. In the following example, the check registers in a `passing` state: + + + +```hcl +check = { + id = "mem" + args = ["/bin/check_mem", "-limit", "256MB"] + interval = "10s" + status = "passing" +} +``` + +```json +{ + "check": [ + { + "args": [ + "/bin/check_mem", + "-limit", + "256MB" + ], + "id": "mem", + "interval": "10s", + "status": "passing" + } + ] +} +``` + + + +## Script checks +Script checks invoke an external application that performs the health check, exits with an appropriate exit code, and potentially generates output data. The output of a script check is limited to 4KB. Outputs that exceed the limit are truncated. + +Script checks timeout after 30 seconds by default, but you can configure a custom script check timeout value by specifying the `timeout` field in the check definition. When the timeout is reached on Windows, Consul waits for any child processes spawned by the script to finish. For any other system, Consul attempts to force-kill the script and any child processes it has spawned once the timeout has passed. + +### Script check configuration +To enable script checks, you must first enable the agent to send external requests, then configure the health check settings in the service definition: + +1. Add one of the following configurations to your agent configuration file to enable a script check: + - [`enable_local_script_checks`](/consul/docs/agent/config/cli-flags#enable_local_script_checks): Enable script checks defined in local configuration files. Script checks registered using the HTTP API are not allowed. + - [`enable_script_checks`](/consul/docs/agent/config/cli-flags#enable_script_checks): Enable script checks no matter how they are registered. + + !> **Security warning:** Enabling non-local script checks in some configurations may introduce a known remote execution vulnerability targeted by malware. We strongly recommend `enable_local_script_checks` instead. + +1. Specify the script to run in the `args` of the `check` block in your service configuration file. In the following example, a check named `Memory utilization` invokes the `check_mem.py` script every 10 seconds and times out if a response takes longer than one second: + + + + ```hcl + service { + ## ... + check = { + id = "mem-util" + name = "Memory utilization" + args = ["/usr/local/bin/check_mem.py", "-limit", "256MB"] + interval = "10s" + timeout = "1s" + } + } + ``` + + ```json + { + "service": [ + { + "check": { + "id": "mem-util", + "name": "Memory utilization", + "args": ["/usr/local/bin/check_mem.py", "-limit", "256MB"], + "interval": "10s", + "timeout": "1s" + } + } ] + } + ``` + + +Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +### Script check exit codes +The following exit codes returned by the script check determine the health check status: + +- Exit code 0 - Check is passing +- Exit code 1 - Check is warning +- Any other code - Check is failing + +Any output of the script is captured and made available in the `Output` field of checks included in HTTP API responses. Refer to the example described in the [local service health endpoint](/consul/api-docs/agent/service#by-name-json). + +## HTTP checks +_HTTP_ checks send an HTTP request to the specified URL and report the service health based on the [HTTP response code](#http-check-response-codes). We recommend using HTTP checks over [script checks](#script-checks) that use cURL or another external process to check an HTTP operation. + +### HTTP check configuration +Add an `http` field to the `check` block in your service definition file and specify the HTTP address, including port number, for the check to call. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +In the following example, an HTTP check named `HTTP API on port 5000` sends a `POST` request to the `health` endpoint every 10 seconds: + + + +```hcl +check = { + id = "api" + name = "HTTP API on port 5000" + http = "https://localhost:5000/health" + tls_server_name = "" + tls_skip_verify = false + method = "POST" + header = { + Content-Type = ["application/json"] + } + body = "{\"method\":\"health\"}" + disable_redirects = true + interval = "10s" + timeout = "1s" +} +``` + +```json +{ + "check": { + "id": "api", + "name": "HTTP API on port 5000", + "http": "https://localhost:5000/health", + "tls_server_name": "", + "tls_skip_verify": false, + "method": "POST", + "header": { "Content-Type": ["application/json"] }, + "body": "{\"method\":\"health\"}", + "interval": "10s", + "timeout": "1s" + } +} +``` + + +HTTP checks send GET requests by default, but you can specify another request method in the `method` field. You can send additional headers in the `header` block. The `header` block contains a key and an array of strings, such as `{"x-foo": ["bar", "baz"]}`. By default, HTTP checks timeout at 10 seconds, but you can specify a custom timeout value in the `timeout` field. + +HTTP checks expect a valid TLS certificate by default. You can disable certificate verification by setting the `tls_skip_verify` field to `true`. When using TLS and a host name is specified in the `http` field, the check automatically determines the SNI from the URL. If the `http` field is configured with an IP address or if you want to explicitly set the SNI, specify the name in the `tls_server_name` field. + +The check follows HTTP redirects configured in the network by default. Set the `disable_redirects` field to `true` to disable redirects. + +### HTTP check response codes +Responses larger than 4KB are truncated. The HTTP response determines the status of the service: + +- A `200`-`299` response code is healthy. +- A `429` response code indicating too many requests is a warning. +- All other response codes indicate a failure. + + +## TCP checks +TCP checks establish connections to the specified IPs or hosts. If the check successfully establishes a connection, the service status is reported as `success`. If the IP or host does not accept the connection, the service status is reported as `critical`. We recommend TCP checks over [script checks](#script-checks) that use netcat or another external process to check a socket operation. + +### TCP check configuration +Add a `tcp` field to the `check` block in your service definition file and specify the address, including port number, for the check to call. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +In the following example, a TCP check named `SSH TCP on port 22` attempts to connect to `localhost:22` every 10 seconds: + + + + +```hcl +check = { + id = "ssh" + name = "SSH TCP on port 22" + tcp = "localhost:22" + interval = "10s" + timeout = "1s" +} +``` + +```json +{ + "check": { + "id": "ssh", + "name": "SSH TCP on port 22", + "tcp": "localhost:22", + "interval": "10s", + "timeout": "1s" + } +} +``` + + + +If a hostname resolves to an IPv4 and an IPv6 address, Consul attempts to connect to both addresses. The first successful connection attempt results in a successful check. + +By default, TCP check requests timeout at 10 seconds, but you can specify a custom timeout in the `timeout` field. + +## UDP checks +UDP checks direct the Consul agent to send UDP datagrams to the specified IP or hostname and port. The check status is set to `success` if any response is received from the targeted UDP server. Any other result sets the status to `critical`. + +### UDP check configuration +Add a `udp` field to the `check` block in your service definition file and specify the address, including port number, for sending datagrams. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +In the following example, a UDP check named `DNS UDP on port 53` sends datagrams to `localhost:53` every 10 seconds: + + + +```hcl +check = { + id = "dns" + name = "DNS UDP on port 53" + udp = "localhost:53" + interval = "10s" + timeout = "1s" +} +``` + +```json +{ + "check": { + "id": "dns", + "name": "DNS UDP on port 53", + "udp": "localhost:53", + "interval": "10s", + "timeout": "1s" + } +} +``` + + + +By default, UDP checks timeout at 10 seconds, but you can specify a custom timeout in the `timeout` field. If any timeout on read exists, the check is still considered healthy. + +## OSService check +OSService checks if an OS service is running on the host. OSService checks support Windows services on Windows hosts or SystemD services on Unix hosts. The check logs the service as `healthy` if it is running. If the service is not running, the status is logged as `critical`. All other results are logged with `warning`. A `warning` status indicates that the check is not reliable because an issue is preventing it from determining the health of the service. + +### OSService check configurations +Add an `os_service` field to the `check` block in your service definition file and specify the name of the service to check. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +In the following example, an OSService check named `svcname-001 Windows Service Health` verifies that the `myco-svctype-svcname-001` service is running every 10 seconds: + + + +```hcl +check = { + id = "myco-svctype-svcname-001" + name = "svcname-001 Windows Service Health" + service_id = "flash_pnl_1" + os_service = "myco-svctype-svcname-001" + interval = "10s" +} +``` + +```json +{ + "check": { + "id": "myco-svctype-svcname-001", + "name": "svcname-001 Windows Service Health", + "service_id": "flash_pnl_1", + "os_service": "myco-svctype-svcname-001", + "interval": "10s" + } +} +``` + + + +## TTL checks +Time-to-live (TTL) checks wait for an external process to report the service's state to a Consul [`/agent/check` HTTP endpoint](/consul/api-docs/agent/check). If the check does not receive an update before the specified `ttl` duration, the check logs the service as `critical`. For example, if a healthy application is configured to periodically send a `PUT` request a status update to the HTTP endpoint, then the health check logs a `critical` state if the application is unable to send the update before the TTL expires. The check uses the following endpoints to update health information: + +- [pass](/consul/api-docs/agent/check#ttl-check-pass) +- [warn] (/consul/api-docs/agent/check#ttl-check-warn) +- [fail](/consul/api-docs/agent/check#ttl-check-fail) +- [update](/consul/api-docs/agent/check#ttl-check-update) + +TTL checks also persist their last known status to disk so that the Consul agent can restore the last known status of the check across restarts. Persisted check status is valid through the end of the TTL from the time of the last check. + +You can manually mark a service as unhealthy using the [`consul maint` CLI command](/consul/commands/maint) or [`agent/maintenance` HTTP API endpoint](/consul/api-docs/agent#enable-maintenance-mode), rather than waiting for a TTL health check if the `ttl` duration is high. + +### TTL check configuration +Add a `ttl` field to the `check` block in your service definition file and specify how long to wait for an update from the external process. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +In the following example, a TTL check named `Web App Status` logs the application as `critical` if a status update is not received every 30 seconds: + + + + +```hcl +check = { + id = "web-app" + name = "Web App Status" + notes = "Web app does a curl internally every 10 seconds" + ttl = "30s" +} +``` + +```json +{ + "check": { + "id": "web-app", + "name": "Web App Status", + "notes": "Web app does a curl internally every 10 seconds", + "ttl": "30s" + } +} +``` + + + +## Docker checks +Docker checks invoke an application packaged within a Docker container. The application should perform a health check and exit with an appropriate exit code. + +The application is triggered within the running container through the Docker `exec` API. You should have access to either the Docker HTTP API or the Unix socket. Consul uses the `$DOCKER_HOST` environment variable to determine the Docker API endpoint. + +The output of a Docker check is limited to 4KB. Larger outputs are truncated. + +### Docker check configuration +To enable Docker checks, you must first enable the agent to send external requests, then configure the health check settings in the service definition: + +1. Add one of the following configurations to your agent configuration file to enable a Docker check: + + - [`enable_local_script_checks`](/consul/docs/agent/config/cli-flags#enable_local_script_checks): Enable script checks defined in local config files. Script checks registered using the HTTP API are not allowed. + + - [`enable_script_checks`](/consul/docs/agent/config/cli-flags#enable_script_checks): Enable script checks no matter how they are registered. + + !> **Security warning**: Enabling non-local script checks in some configurations may introduce a known remote execution vulnerability targeted by malware. We strongly recommend `enable_local_script_checks` instead. +1. Configure the following fields in the `check` block in your service definition file: + - `docker_container_id`: The `docker ps` command is a common way to get the ID. + - `shell`: Specifies the shell to use for performing the check. Different containers can run different shells on the same host. + - `args`: Specifies the external application to invoke. + - `interval`: Specifies the interval for running the check. + +In the following example, a Docker check named `Memory utilization` invokes the `check_mem.py` application in container `f972c95ebf0e` every 10 seconds: + + + + +```hcl +check = { + id = "mem-util" + name = "Memory utilization" + docker_container_id = "f972c95ebf0e" + shell = "/bin/bash" + args = ["/usr/local/bin/check_mem.py"] + interval = "10s" +} +``` + +```json +{ + "check": { + "id": "mem-util", + "name": "Memory utilization", + "docker_container_id": "f972c95ebf0e", + "shell": "/bin/bash", + "args": ["/usr/local/bin/check_mem.py"], + "interval": "10s" + } +} +``` + + + +## gRPC checks +gRPC checks send a request to the specified endpoint. These checks are intended for applications that support the standard [gRPC health checking protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + +### gRPC check configuration +Add a `grpc` field to the `check` block in your service definition file and specify the endpoint, including port number, for sending requests. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +In the following example, a gRPC check named `Service health status` probes the entire application by sending requests to `127.0.0.1:12345` every 10 seconds: + + + +```hcl +check = { + id = "mem-util" + name = "Service health status" + grpc = "127.0.0.1:12345" + grpc_use_tls = true + interval = "10s" +} +``` + +```json +{ + "check": { + "id": "mem-util", + "name": "Service health status", + "grpc": "127.0.0.1:12345", + "grpc_use_tls": true, + "interval": "10s" + } +} +``` + + + +gRPC checks probe the entire gRPC server, but you can check on a specific service by adding the service identifier after the gRPC check's endpoint using the following format: `/:service_identifier`. + +In the following example, a gRPC check probes `my_service` in the application at `127.0.0.1:12345` every 10 seconds: + + + + +```hcl +check = { + id = "mem-util" + name = "Service health status" + grpc = "127.0.0.1:12345/my_service" + grpc_use_tls = true + interval = "10s" +} +``` + +```json +{ + "check": { + "id": "mem-util", + "name": "Service health status", + "grpc": "127.0.0.1:12345/my_service", + "grpc_use_tls": true, + "interval": "10s" + } +} +``` + + + +TLS is disabled for gRPC checks by default. You can enable TLS by setting `grpc_use_tls` to `true`. If TLS is enabled, you must either provide a valid TLS certificate or disable certificate verification by setting the `tls_skip_verify` field to `true`. + +By default, gRPC checks timeout after 10 seconds, but you can specify a custom duration in the `timeout` field. + +## H2ping checks +H2ping checks test an endpoint that uses HTTP2 by connecting to the endpoint and sending a ping frame. If the endpoint sends a response within the specified interval, the check status is set to `success`. + +### H2ping check configuration +Add an `h2ping` field to the `check` block in your service definition file and specify the HTTP2 endpoint, including port number, for the check to ping. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +In the following example, an H2ping check named `h2ping` pings the endpoint at `localhost:22222` every 10 seconds: + + + + +```hcl +check = { + id = "h2ping-check" + name = "h2ping" + h2ping = "localhost:22222" + interval = "10s" + h2ping_use_tls = false +} +``` + +```json +{ + "check": { + "id": "h2ping-check", + "name": "h2ping", + "h2ping": "localhost:22222", + "interval": "10s", + "h2ping_use_tls": false + } +} +``` + + + +TLS is enabled by default, but you can disable TLS by setting `h2ping_use_tls` to `false`. When TLS is disabled, the Consul sends pings over h2c. When TLS is enabled, a valid certificate is required unless `tls_skip_verify` is set to `true`. + +By default, H2ping checks timeout at 10 seconds, but you can specify a custom duration in the `timeout` field. + + +## Alias checks +Alias checks continuously report the health state of another registered node or service. If the alias experiences errors while watching the actual node or service, the check reports a`critical` state. Consul updates the alias and actual node or service state asynchronously but nearly instantaneously. + +For aliased services on the same agent, the check monitors the local state without consuming additional network resources. For services and nodes on different agents, the check maintains a blocking query over the agent's connection with a current server and allows stale requests. + +### ACLs +For the blocking query, the alias check presents the ACL token set on the actual service or the token configured in the check definition. If neither are available, the alias check falls back to the default ACL token set for the agent. Refer to [`acl.tokens.default`](/consul/docs/agent/config/config-files#acl_tokens_default) for additional information about the default ACL token. + +### Alias checks configuration +Add an `alias_service` field to the `check` block in your service definition file and specify the name of the service or node to alias. All other fields are optional. Refer to [Health Checks Configuration Reference](/consul/docs/services/configuration/checks-configuration-reference) for information about all health check configurations. + +In the following example, an alias check with the ID `web-alias` reports the health state of the `web` service: + + + + +```hcl +check = { + id = "web-alias" + alias_service = "web" +} +``` + +```json +{ + "check": { + "id": "web-alias", + "alias_service": "web" + } +} +``` + + + +By default, the alias must be registered with the same Consul agent as the alias check. If the service is not registered with the same agent, you must specify `"alias_node": ""` in the `check` configuration. If no service is specified and the `alias_node` field is enabled, the check aliases the health of the node. If a service is specified, the check will alias the specified service on this particular node. \ No newline at end of file diff --git a/website/content/docs/services/usage/define-services.mdx b/website/content/docs/services/usage/define-services.mdx new file mode 100644 index 00000000000..1e0490b9b3a --- /dev/null +++ b/website/content/docs/services/usage/define-services.mdx @@ -0,0 +1,450 @@ +--- +layout: docs +page_title: Define services +description: >- + Learn how to define services so that they are discoverable in your network. +--- + +# Define services + +This topic describes how to define services so that they can be discovered by other services. Refer to [Services Overview](/consul/docs/services/services) for additional information. + +## Overview + +You must tell Consul about the services deployed to your network if you want them to be discoverable. You can define services in a configuration file or send the service definition parameters as a payload to the `/agent/service/register` API endpoint. Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for details about how to register services with Consul. + +You can define multiple services individually using `service` blocks or group multiple services into the same `services` configuration block. Refer to [Define multiple services in a single file](#define-multiple-services-in-a-single-file) for additional information. + +If Consul service mesh is enabled in your network, you can use the [service defaults configuration entry](/consul/docs/connect/config-entries/service-defaults) to specify default global values for services. The configuration entry lets you define common service parameter, such as upstreams, namespaces, and partitions. Refer to [Define service defaults](#define-service-defaults) for additional information. + +## Requirements + +The core service discovery features are available in all versions of Consul. + +### Service defaults +To use the [service defaults configuration entry](#define-service-defaults), verify that your installation meets the following requirements: + +- Consul 1.5.0+ +- Consul 1.8.4+ is required to use the `ServiceDefaults` custom resource on Kubernetes + +### ACLs +If ACLs are enabled, resources in your network must present a token with `service:read` access to read a service defaults configuration entry. + +You must also present a token with `service:write` access to create, update, or delete a service defaults configuration entry. + +Service configurations must also contain and present an ACL token to perform anti-entropy syncs and deregistration operations. Refer to [Modify anti-entropy synchronozation](#modify-anti-entropy-synchronization) for additional information. + +On Consul Enterprise, you can register services with specific namespaces if the services' ACL tokens are scoped to the namespace. Services registered with a service definition do not inherit the namespace associated with the ACL token specified in the `token` field. The `namespace` and the `token` parameters must be included in the service definition for the service to be registered to the namespace that the ACL token is scoped to. + +## Define a service +Create a file for your service configurations and add a `service` block. The `service` block contains the parameters that configure various aspects of the service, including how it is discovered by other services in the network. The only required parameter is `name`. Refer to [Service Definition Reference](/consul/docs/services/configuration/services-configuration-reference) for details about the configuration options. + +For Kubernetes environments, you can enable the [`connectInject`](/consul/docs/k8s/connect#installation-and-configuration) configuration in your Consul Helm chart so that Consul automatically adds a sidecar to each of your pods. Consul uses the Kubernetes `Service` definition as a reference. + +The following example defines a service named `redis` that is available on port `80`. By default, the service has the IP address of the agent node. + + + + +```hcl +service { + name = "redis" + id = "redis" + port = 80 + tags = ["primary"] + + meta = { + custom_meta_key = "custom_meta_value" + } + + tagged_addresses = { + lan = { + address = "192.168.0.55" + port = 8000 + } + + wan = { + address = "198.18.0.23" + port = 80 + } + } +} +``` + + + + +```json +{ + "service": [ + { + "id": "redis", + "meta": [ + { + "custom_meta_key": "custom_meta_value" + } + ], + "name": "redis", + "port": 80, + "tagged_addresses": [ + { + "lan": [ + { + "address": "192.168.0.55", + "port": 8000 + } + ], + "wan": [ + { + "address": "198.18.0.23", + "port": 80 + } + ] + } + ], + "tags": [ + "primary" + ] + } + ] +} +``` + + + +```yaml +service: +- id: redis + meta: + - custom_meta_key: custom_meta_value + name: redis + port: 80 + tagged_addresses: + - lan: + - address: 192.168.0.55 + port: 8000 + wan: + - address: 198.18.0.23 + port: 80 + tags: + - primary +``` + + + +### Health checks + +You can add a `check` or `checks` block to your service configuration to define one or more health checks that monitor the health of your services. Refer to [Define Health Checks](/consul/docs/services/usage/checks) for additional information. + +### Register a service + +You can register your service using the [`consul services` command](/consul/commands/services) or by calling the [`/agent/services` API endpoint](/consul/api-docs/agent/service). Refer to [Register Services and Health Checks](/consul/docs/services/usage/register-services-checks) for details. + +## Define service defaults +If Consul service mesh is enabled in your network, you can define default values for services in your mesh by creating and applying a `service-defaults` configuration entry containing. Refer to [Service Mesh Configuration Overview](/consul/docs/connect/configuration) for additional information. + +Create a file for the configuration entry and specify the required fields. If you are authoring `service-defaults` in HCL or JSON, the `Kind` and `Name` fields are required. On Kubernetes, the `apiVersion`, `kind`, and `metadata.name` fields are required. Refer to [Service Defaults Reference](/consul/docs/connect/config-entries/service-defaults) for details about the configuration options. + +If you use Consul Enterprise, you can also specify the `Namespace` and `Partition` fields to apply the configuration to services in a specific namespace or partition. For Kubernetes environments, the configuration entry is always created in the same partition as the Kubernetes cluster. + +### Consul OSS example +The following example instructs services named `counting` to send up to `512` concurrent requests to a mesh gateway: + + + +```hcl +Kind = "service-defaults" +Name = "counting" + +UpstreamConfig = { + Defaults = { + MeshGateway = { + Mode = "local" + } + Limits = { + MaxConnections = 512 + MaxPendingRequests = 512 + MaxConcurrentRequests = 512 + } + } + + Overrides = [ + { + Name = "dashboard" + MeshGateway = { + Mode = "remote" + } + } + ] +} +``` +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceDefaults +metadata: + name: counting +spec: + upstreamConfig: + defaults: + meshGateway: + mode: local + limits: + maxConnections: 512 + maxPendingRequests: 512 + maxConcurrentRequests: 512 + overrides: + - name: dashboard + meshGateway: + mode: remote +``` +```json +{ + "Kind": "service-defaults", + "Name": "counting", + "UpstreamConfig": { + "Defaults": { + "MeshGateway": { + "Mode": "local" + }, + "Limits": { + "MaxConnections": 512, + "MaxPendingRequests": 512, + "MaxConcurrentRequests": 512 + } + }, + "Overrides": [ + { + "Name": "dashboard", + "MeshGateway": { + "Mode": "remote" + } + } + ] + } +} +``` + + +### Consul Enterprise example +The following example instructs services named `counting` in the `prod` namespace to send up to `512` concurrent requests to a mesh gateway: + + + +```hcl +Kind = "service-defaults" +Name = "counting" +Namespace = "prod" + +UpstreamConfig = { + Defaults = { + MeshGateway = { + Mode = "local" + } + Limits = { + MaxConnections = 512 + MaxPendingRequests = 512 + MaxConcurrentRequests = 512 + } + } + + Overrides = [ + { + Name = "dashboard" + MeshGateway = { + Mode = "remote" + } + } + ] +} +``` +```yaml +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceDefaults +metadata: + name: counting + namespace: prod +spec: + upstreamConfig: + defaults: + meshGateway: + mode: local + limits: + maxConnections: 512 + maxPendingRequests: 512 + maxConcurrentRequests: 512 + overrides: + - name: dashboard + meshGateway: + mode: remote +``` +```json +{ + "Kind": "service-defaults", + "Name": "counting", + "Namespace" : "prod", + "UpstreamConfig": { + "Defaults": { + "MeshGateway": { + "Mode": "local" + }, + "Limits": { + "MaxConnections": 512, + "MaxPendingRequests": 512, + "MaxConcurrentRequests": 512 + } + }, + "Overrides": [ + { + "Name": "dashboard", + "MeshGateway": { + "Mode": "remote" + } + } + ] + } +} +``` + + +### Apply service defaults + +You can apply your `service-defaults` configuration entry using the [`consul config` command](/consul/commands/config) or by calling the [`/config` API endpoint](/consul/api-docs/config). In Kubernetes environments, apply the `service-defaults` custom resource definitions (CRD) to implement and manage Consul configuration entries. + +Refer to the following topics for details about applying configuration entries: +- [How to Use Configuration Entries](/consul/docs/agent/config-entries) +- [Custom Resource Definitions for Consul on Kubernetes](/consul/docs/k8s/crds) + +## Define multiple services in a single file + +The `services` block contains an array of `service` objects. It is a wrapper that enables you to define multiple services in the service definition and instruct Consul to expect more than just a single service configuration. As a result, you can register multiple services in a single `consul services register` command. Note that the `/agent/service/register` API endpoint does not support the `services` parameter. + +In the following example, the service definition configures an instance of the `redis` service tagged as `primary` running on port `6000`. It also configures an instance of the service tagged as `secondary` running on port `7000`. + + + + + +```hcl +services { + id = "red0" + name = "redis" + tags = [ + "primary" + ] + address = "" + port = 6000 + checks = [ + { + args = ["/bin/check_redis", "-p", "6000"] + interval = "5s" + timeout = "20s" + } + ] +} +services { + id = "red1" + name = "redis" + tags = [ + "delayed", + "secondary" + ] + address = "" + port = 7000 + checks = [ + { + args = ["/bin/check_redis", "-p", "7000"] + interval = "30s" + timeout = "60s" + } + ] +} + +``` + + + + + +```json +{ + "services": [ + { + "id": "red0", + "name": "redis", + "tags": [ + "primary" + ], + "address": "", + "port": 6000, + "checks": [ + { + "args": ["/bin/check_redis", "-p", "6000"], + "interval": "5s", + "timeout": "20s" + } + ] + }, + { + "id": "red1", + "name": "redis", + "tags": [ + "delayed", + "secondary" + ], + "address": "", + "port": 7000, + "checks": [ + { + "args": ["/bin/check_redis", "-p", "7000"], + "interval": "30s", + "timeout": "60s" + } + ] + }, + ... + ] +} +``` + + + + +## Modify anti-entropy synchronization + +By default, the Consul agent uses anti-entropy mechanisms to maintain information about services and service health, and synchronize local states with the Consul catalog. You can enable the `enable_tag_override` option in the service configuration, which lets external agents change the tags for a service. This can be useful in situations where an external monitoring service needs to be the source of truth for tag information. Refer [Anti-entropy](/consul/docs/architecture/anti-entropy) for details. + +Add the `enable_tag_override` option to the `service` block and set the value to `true`: + + + + +```hcl +service { + ## ... + enable_tag_override = true + ## ... +} +``` + +```json +"service": { + ## ... + "enable_tag_override": true, + ## ... +} +``` + + + +This configuration only applies to the locally registered service. Nodes that register the same service apply the `enable_tag_override` and other service configurations independently. The tags for a service registered on one node update are not affected by operations performed on services with the same name registered on other nodes. + +Refer to [`enable_tag_override`](/consul/docs/services/configuration/services-configuration-reference#enable_tag_override) for additional configuration information. + +## Services in service mesh environments +Defining services for service mesh environments on virtual machines and in Kubernetes requires a different workflow. + +### Define service mesh proxies +You can register services to function as service mesh or sidecar proxies so that they can facilitate communication between other services across your network. Refer to [Service Mesh Proxy Overview](/consul/docs/connect/registration) for additional information. + +### Define services in Kubernetes +You can enable the services running in Kubernetes and Consul to sync automatically. Doing so ensures that Kubernetes services are available to Consul agents and services in Consul can be available as first-class Kubernetes services. Refer to [Service Sync for Consul on Kubernetes](/consul/docs/k8s/service-sync) for details. \ No newline at end of file diff --git a/website/content/docs/services/usage/register-services-checks.mdx b/website/content/docs/services/usage/register-services-checks.mdx new file mode 100644 index 00000000000..07a3f20ad9c --- /dev/null +++ b/website/content/docs/services/usage/register-services-checks.mdx @@ -0,0 +1,68 @@ +--- +layout: docs +page_title: Register services and health checks +description: -> + Learn how to register services and health checks with Consul agents. +--- + +# Register services and health checks + +This topic describes how to register services and health checks with Consul in networks running on virtual machines (VM). Refer to [Define Services](/consul/docs/services/usage/define-services) and [Define Health Checks](/consul/docs/services/usage/checks) for information about how to define services and health checks. + +## Overview +Register services and health checks in VM environments by providing the service definition to a Consul agent. You can use several different methods to register services and health checks. + +- Start a Consul agent and pass the service definition in the [agent's configuration directory](/consul/docs/agent#configuring-consul-agents). +- Reload the a running Consul agent and pass the service definition in the [agent's configuration directory](/consul/docs/agent#configuring-consul-agents). Use this method when implementing changes to an existing service or health check definition. +- Use the [`consul services register` command](/consul/commands/services/register) to register new service and health checks with a running Consul agent. +- Call the [`/agent/service/register`](/consul/api-docs/agent/service#register-service) HTTP API endpoint to to register new service and health checks with a running Consul agent. +- Call the [`/agent/check/register`](/consul/api-docs/agent/check#register-check) HTTP API endpoint to register a health check independent from the service. + +When a service is registered using the HTTP API endpoint or CLI command, the checks persist in the Consul data folder. If the agent restarts, Consul uses the service and check configurations in the configuration directory to start the services. + +Note that health checks associated with a service are application-level checks. + +## Start an agent +We recommend registering services on startup because the service persists if the agent fails. Specify the directory containing the service definition with the `-config-dir` option on startup. When the Consul agent starts, it processes all configurations in the directory and registers any services contained in the configurations. In the following example, the Consul agent starts and loads the configurations contained in the `configs` directory: + +```shell-session +$ consul agent -config-dir configs +``` + +Refer to [Starting the Consul Agent](/consul/docs/agent#starting-the-consul-agent) for additional information. + +## Reload an agent +Store your service definition file in the directory containing your Consul configuration files and either send a `SIGHUP` signal to the Consul agent service or run the `consul reload` command. Refer to [Consul Reload](/consul/commands/reload) for additional information. + +## Register using the CLI +Run the `consul services register` command and specify the service definition file to register the services and health checks defined in the file. In the following example, a service named `web` is registered: + +```shell-session +$ consul services register -name=web services.hcl +``` + +Refer to [Consul Agent Service Registration](/consul/commands/services/register) for additional information about using the command. + +## Register using the API + +Use the following methods to register services and health checks using the HTTP API. + +### Register services +Send a `PUT` request to the `/agent/service/register` API endpoint to dynamically register a service and its associated health checks. To register health checks independently, [call the checks API endpoint](#call-the-checks-http-api-endpoint). + +The following example request registers the service defined in the `service.json` file. + +```shell-session +$ curl --request PUT --data @service.json http://localhost:8500/v1/agent/service/register +``` + +Refer to [Service - Agent HTTP API](/consul/api-docs/agent/service) for additional information about the `services` endpoint. + +### Register health checks +Send a `PUT` request to the `/agent/check/register` API endpoint to dynamically register a health check to the local Consul agent. The following example request registers a health check defined in a `payload.json` file. + +```shell-session +$ curl --request PUT --data @payload.json http://localhost:8500/v1/agent/check/register +``` + +Refer to [Check - Agent HTTP API](/consul/api-docs/agent/check) for additional information about the `check` endpoint. diff --git a/website/content/docs/troubleshoot/common-errors.mdx b/website/content/docs/troubleshoot/common-errors.mdx index d0e3328aad0..505d360b05e 100644 --- a/website/content/docs/troubleshoot/common-errors.mdx +++ b/website/content/docs/troubleshoot/common-errors.mdx @@ -5,16 +5,16 @@ description: >- Troubleshoot issues based on the error message. Common errors result from failed actions, timeouts, multiple entries, bad and expired certificates, invalid characters, syntax parsing, malformed responses, and exceeded deadlines. --- -# Common Error Messages +# Common error messages -When installing and running Consul, there are some common messages you might see. Usually they indicate an issue in your network or in your server's configuration. Some of the more common errors and their solutions are listed below. +This topic describes common messages that may appear when installing and running Consul. Errors usually they indicate an issue in your network or in your server's configuration. Refer to the [Troubleshooting Guide][troubleshooting] for help resolving error messages that do not appear on this page. -If you are getting an error message you don't see listed on this page, please consider following our general [Troubleshooting Guide][troubleshooting]. - -For common errors messages related to Kubernetes, please go to [Common errors on Kubernetes](#common-errors-on-kubernetes). +For common errors messages related to Kubernetes, refer to [Common errors on Kubernetes](#common-errors-on-kubernetes). ## Configuration file errors +The following errors are related to misconfigured files. + ### Multiple network interfaces ```text @@ -100,7 +100,7 @@ If a host does not properly implement half-close you may see an error message `[ This has been a [known issue](https://github.com/docker/libnetwork/issues/1204) in Docker, but may manifest in other environments as well. -## ACL Not Found +## ACL not found ```text RPC error making call: rpc error making call: ACL not found @@ -108,7 +108,9 @@ RPC error making call: rpc error making call: ACL not found This indicates that you have ACL enabled in your cluster, but you aren't passing a valid token. Make sure that when creating your tokens that they have the correct permissions set. In addition, you would want to make sure that an agent token is provided on each call. -## TLS and Certificates +## TLS and certificates + +The follow errors are related to TLS and certificate issues. ### Incorrect certificate or certificate name @@ -150,8 +152,20 @@ You have installed an Enterprise version of Consul. If you are an Enterprise cus -> **Note:** Enterprise binaries can be identified on our [download site][releases] by the `+ent` suffix. +## Rate limit reached on the server + +You may receive a `RESOURCE_EXHAUSTED` error from the Consul server if the maximum number of read or write requests per second have been reached. Refer to [Set a global limit on traffic rates](/consul/docs/agent/limits/set-global-traffic-rate-limits) for additional information. You can retry another server unless the number of retries is exhausted. If the number of retries is exhausted, you should implement an exponential backoff. + +The RESOURCE_EXHAUSTED RPC response is translated into a `429 Too Many Requests` error code on the HTTP interface. + +The server may respond as `UNAVAILABLE` if it is the leader node and the global write request rate limit is reached. The solution is to apply an exponential backoff until the leader has capacity to serve those requests. + +The `UNAVAILABLE` RPC response is translated into a `503 Service Unavailable` error code on the RPC requests sent through HTTP interface. + ## Common errors on Kubernetes +The following error messages are specific to Kubernetes issues. + ### Unable to connect to the Consul client on the same host If the pods are unable to connect to a Consul client running on the same host, @@ -199,6 +213,56 @@ as doing so gives the Consul client unnecessary access to all network traffic on We recommend raising an issue with the CNI you're using to add support for `hostPort` and switching back to `hostPort` eventually. +### consul-server-connection-manager: ACL auth method login failed: error="rpc error: code = PermissionDenied desc = Permission denied" + +If you see this error in the init container logs of service mesh pods, check that the pod has a service account name that matches its Service. +For example, this deployment: + +``` +apiVersion: v1 +kind: Service +metadata: + # This name will be the service name in Consul. + name: static-server +spec: + selector: + app: static-server + ports: + - protocol: TCP + port: 80 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-server +spec: + replicas: 1 + selector: + matchLabels: + app: static-server + template: + metadata: + name: static-server + labels: + app: static-server + annotations: + 'consul.hashicorp.com/connect-inject': 'true' + spec: + containers: + - name: static-server + image: hashicorp/http-echo:latest + args: + - -text="hello world" + - -listen=:8080 + ports: + - containerPort: 8080 + name: http + serviceAccountName: does-not-match +``` + +Will fail because the `serviceAccountName` is `does-not-match` instead of `static-server`. + [troubleshooting]: /consul/tutorials/datacenter-operations/troubleshooting [node_name]: /consul/docs/agent/config/config-files#node_name [retry_join]: /consul/docs/agent/config/cli-flags#retry-join diff --git a/website/content/docs/troubleshoot/troubleshoot-services.mdx b/website/content/docs/troubleshoot/troubleshoot-services.mdx new file mode 100644 index 00000000000..92a66881475 --- /dev/null +++ b/website/content/docs/troubleshoot/troubleshoot-services.mdx @@ -0,0 +1,150 @@ +--- +layout: docs +page_title: Service-to-service troubleshooting overview +description: >- + Consul includes a built-in tool for troubleshooting communication between services in a service mesh. Learn how to use the `consul troubleshoot` command to validate communication between upstream and downstream Envoy proxies on VM and Kubernetes deployments. +--- + +# Service-to-service troubleshooting overview + +This topic provides an overview of Consul’s built-in service-to-service troubleshooting capabilities. When communication between an upstream service and a downstream service in a service mesh fails, you can run the `consul troubleshoot` command to initiate a series of automated validation tests. + +For more information, refer to the [`consul troubleshoot` CLI documentation](/consul/commands/troubleshoot) or the [`consul-k8s troubleshoot` CLI reference](/consul/docs/k8s/k8s-cli#troubleshoot). + +## Introduction + +When communication between upstream and downstream services in a service mesh fails, you can diagnose the cause manually with one or more of Consul’s built-in features, including [health check queries](/consul/docs/services/usage/checks), [the UI topology view](/consul/docs/connect/observability/ui-visualization), and [agent telemetry metrics](/consul/docs/agent/telemetry#metrics-reference). + +The `consul troubleshoot` command performs several checks in sequence that enable you to discover issues that impede service-to-service communication. The process systematically queries the [Envoy administration interface API](https://www.envoyproxy.io/docs/envoy/latest/operations/admin) and the Consul API to determine the cause of the communication failure. + +The troubleshooting command validates service-to-service communication by checking for the following common issues: + +- Upstream service does not exist +- One or both hosts are unhealthy +- A filter affects the upstream service +- The CA has expired mTLS certificates +- The services have expired mTLS certificates + +Consul outputs the results of these validation checks to the terminal along with suggested actions to resolve the service communication failure. When it detects rejected configurations or connection failures, Consul also outputs Envoy metrics for services. + +### Envoy proxies in a service mesh + +Consul validates communication in a service mesh by checking the Envoy proxies that are deployed as sidecars for the upstream and downstream services. As a result, troubleshooting requires that [Consul’s service mesh features are enabled](/consul/docs/connect/configuration). + +For more information about using Envoy proxies with Consul, refer to [Envoy proxy configuration for service mesh](/consul/docs/connect/proxies/envoy). + +## Requirements + +- Consul v1.15 or later. +- For Kubernetes, the `consul-k8s` CLI must be installed. + +### Technical constraints + +When troubleshooting service-to-service communication issues, be aware of the following constraints: + +- The troubleshooting tool does not check service intentions. For more information about intentions, including precedence and match order, refer to [service mesh intentions](/consul/docs/connect/intentions). +- The troubleshooting tool validates one direct connection between a downstream service and an upstream service. You must run the `consul troubleshoot` command with the Envoy ID for an individual upstream service. It does support validating multiple connections simultaneously. +- The troubleshooting tool only validates Envoy configurations for sidecar proxies. As a result, the troubleshooting tool does not validate Envoy configurations on upstream proxies such as mesh gateways and terminating gateways. + +## Usage + +Using the service-to-service troubleshooting tool is a two-step process: + +1. Find the identifier for the upstream service. +1. Use the upstream’s identifier to validate communication. + +In deployments without transparent proxies, the identifier is the _Envoy ID for the upstream service’s sidecar proxy_. If you use transparent proxies, the identifier is the _upstream service’s IP address_. For more information about using transparent proxies, refer to [Enable transparent proxy mode](/consul/docs/connect/transparent-proxy). + +### Troubleshoot on VMs + +To troubleshoot service-to-service communication issues in deployments that use VMs or bare-metal servers: + +1. Run the `consul troubleshoot upstreams` command to retrieve the upstream information for the service that is experiencing communication failures. Depending on your network’s configuration, the upstream information is either an Envoy ID or an IP address. + + ```shell-session + $ consul troubleshoot upstreams + ==> Upstreams (explicit upstreams only) (0) + ==> Upstreams IPs (transparent proxy only) (1) + [10.4.6.160 240.0.0.3] true map[backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul] + If you cannot find the upstream address or cluster for a transparent proxy upstream: + - Check intentions: Tproxy upstreams are configured based on intentions. Make sure you have configured intentions to allow traffic to your upstream. + - To check that the right cluster is being dialed, run a DNS lookup for the upstream you are dialing. For example, run `dig backend.svc.consul` to return the IP address for the `backend` service. If the address you get from that is missing from the upstream IPs, it means that your proxy may be misconfigured. + ``` + +1. Run the `consul troubleshoot proxy` command and specify the Envoy ID or IP address with the `-upstream-ip` flag to identify the proxy you want to perform the troubleshooting process on. The following example uses the upstream IP to validate communication with the upstream service `backend`: + + ```shell-session + $ consul troubleshoot proxy -upstream-ip 10.4.6.160 + ==> Validation + ✓ Certificates are valid + ✓ Envoy has 0 rejected configurations + ✓ Envoy has detected 0 connection failure(s) + ✓ Listener for upstream "backend" found + ✓ Route for upstream "backend" found + ✓ Cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ Healthy endpoints for cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ Cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ! No healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + -> Check that your upstream service is healthy and running + -> Check that your upstream service is registered with Consul + -> Check that the upstream proxy is healthy and running + -> If you are explicitly configuring upstreams, ensure the name of the upstream is correct + ``` + +In the example output, troubleshooting upstream communication reveals that the `backend` service has two service instances running in datacenter `dc1`. One of the services is healthy, but Consul cannot detect healthy endpoints for the second service instance. This information appears in the following lines of the example: + +```text hideClipboard + ✓ Cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ Healthy endpoints for cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ Cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ! No healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found +``` + +The output from the troubleshooting process identifies service instances according to their [Consul DNS address](/consul/docs/services/discovery/dns-static-lookups#standard-lookup). Use the DNS information for failing services to diagnose the specific issues affecting the service instance. + +For more information, refer to the [`consul troubleshoot` CLI documentation](/consul/commands/troubleshoot). + +### Troubleshoot on Kubernetes + +To troubleshoot service-to-service communication issues in deployments that use Kubernetes, retrieve the upstream information for the pod that is experiencing communication failures and use the upstream information to identify the proxy you want to perform the troubleshooting process on. + +1. Run the `consul-k8s troubleshoot upstreams` command and specify the pod ID with the `-pod` flag to retrieve upstream information. Depending on your network’s configuration, the upstream information is either an Envoy ID or an IP address. The following example displays all transparent proxy upstreams in Consul service mesh from the given pod. + + ```shell-session + $ consul-k8s troubleshoot upstreams -pod frontend-767ccfc8f9-6f6gx + ==> Upstreams (explicit upstreams only) (0) + ==> Upstreams IPs (transparent proxy only) (1) + [10.4.6.160 240.0.0.3] true map[backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul] + If you cannot find the upstream address or cluster for a transparent proxy upstream: + - Check intentions: Tproxy upstreams are configured based on intentions. Make sure you have configured intentions to allow traffic to your upstream. + - To check that the right cluster is being dialed, run a DNS lookup for the upstream you are dialing. For example, run `dig backend.svc.consul` to return the IP address for the `backend` service. If the address you get from that is missing from the upstream IPs, it means that your proxy may be misconfigured. + ``` + +1. Run the `consul-k8s troubleshoot proxy` command and specify the pod ID and upstream IP address to identify the proxy you want to troubleshoot. The following example uses the upstream IP to validate communication with the upstream service `backend`: + + ```shell-session + $ consul-k8s troubleshoot proxy -pod frontend-767ccfc8f9-6f6gx -upstream-ip 10.4.6.160 + ==> Validation + ✓ certificates are valid + ✓ Envoy has 0 rejected configurations + ✓ Envoy has detected 0 connection failure(s) + ✓ listener for upstream "backend" found + ✓ route for upstream "backend" found + ✓ cluster "backend.default.dc1.internal..consul" for upstream "backend" found + ✓ healthy endpoints for cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ cluster "backend2.default.dc1.internal..consul" for upstream "backend" found + ! no healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ``` + +In the example output, troubleshooting upstream communication reveals that the `backend` service has two clusters in datacenter `dc1`. One of the clusters returns healthy endpoints, but Consul cannot detect healthy endpoints for the second cluster. This information appears in the following lines of the example: + + ```text hideClipboard + ✓ cluster "backend.default.dc1.internal..consul" for upstream "backend" found + ✓ healthy endpoints for cluster "backend.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found + ✓ cluster "backend2.default.dc1.internal..consul" for upstream "backend" found + ! no healthy endpoints for cluster "backend2.default.dc1.internal.e08fa6d6-e91e-dfe0-f6e1-ba097a828e31.consul" for upstream "backend" found +``` + +The output from the troubleshooting process identifies service instances according to their [Consul DNS address](/consul/docs/k8s/dns). Use the DNS information for failing services to diagnose the specific issues affecting the service instance. + +For more information, refer to the [`consul-k8s troubleshoot` CLI reference](/consul/docs/k8s/k8s-cli#troubleshoot). \ No newline at end of file diff --git a/website/content/docs/upgrading/index.mdx b/website/content/docs/upgrading/index.mdx index 27500a4cb4e..4b3ecad8d84 100644 --- a/website/content/docs/upgrading/index.mdx +++ b/website/content/docs/upgrading/index.mdx @@ -36,7 +36,8 @@ Consul is A, and version B is released. 2. On each Consul server agent, install version B of Consul. -3. One Consul server agent at a time, shut down version A via `consul leave` and restart with version B. Wait until +3. One Consul server agent at a time, use a service management system + (e.g., systemd, upstart, etc.) to restart the Consul service with version B. Wait until the server agent is healthy and has rejoined the cluster before moving on to the next server agent. diff --git a/website/content/docs/upgrading/instructions/general-process.mdx b/website/content/docs/upgrading/instructions/general-process.mdx index 373256cc547..0c560f71357 100644 --- a/website/content/docs/upgrading/instructions/general-process.mdx +++ b/website/content/docs/upgrading/instructions/general-process.mdx @@ -107,13 +107,7 @@ Take note of which agent is the leader. binary with the new one. **3.** The following steps must be done in order on the server agents, leaving the leader -agent for last. First force the server agent to leave the cluster with the following command: - -``` -consul leave -``` - -Then, use a service management system (e.g., systemd, upstart, etc.) to restart the Consul service. If +agent for last. First, use a service management system (e.g., systemd, upstart, etc.) to restart the Consul service. If you are not using a service management system, you must restart the agent manually. To validate that the agent has rejoined the cluster and is in sync with the leader, issue the @@ -173,7 +167,7 @@ all of your servers attempting to kick off leadership elections endlessly withou reaching a quorum and electing a leader. Most of these problems can be solved by following the steps outlined in our -[Outage Recovery](/consul/tutorials/datacenter-operations/recovery-outage) document. +[Disaster recovery for Consul clusters](/consul/tutorials/datacenter-operations/recovery-outage) document. If you are still having trouble after trying the recovery steps outlined there, then the following options for further assistance are available: diff --git a/website/content/docs/upgrading/instructions/index.mdx b/website/content/docs/upgrading/instructions/index.mdx index 8c2b3c08050..1ea7effb735 100644 --- a/website/content/docs/upgrading/instructions/index.mdx +++ b/website/content/docs/upgrading/instructions/index.mdx @@ -10,45 +10,43 @@ description: >- This document is intended to help users who find themselves many versions behind to upgrade safely. -## Upgrade Path - -Our recommended upgrade path is to move through the following sequence of versions: - -- 0.8.5 (final 0.8.x) -- 1.2.4 (final 1.2.x) -- 1.6.10 (final 1.6.x) -- 1.8.19 (final 1.8.x) -- 1.10.12 (final 1.10.x) -- Latest 1.12.x -- Latest 1.13.x ([at least 1.13.1](/consul/docs/upgrading/upgrade-specific#service-mesh-compatibility)) -- Latest 1.14.x - -## Getting Started - -To get instructions for your upgrade, follow the instructions given below for -your _currently installed_ release series until you are on the latest current version. -The upgrade guides will mention notable changes and link to relevant changelogs – -we recommend reviewing the changelog for versions between the one you are on and the -one you are upgrading to at each step to familiarize yourself with changes. - -Select your _currently installed_ release series: -- 1.13.x: work upwards from [1.14 upgrade notes](/consul/docs/upgrading/upgrade-specific#consul-1-14-x) -- 1.12.x: work upwards from [1.13 upgrade notes](/consul/docs/upgrading/upgrade-specific#consul-1-13-x) -- 1.11.x: work upwards from [1.12 upgrade notes](/consul/docs/upgrading/upgrade-specific#consul-1-12-0) -- 1.10.x: work upwards from [1.11 upgrade notes](/consul/docs/upgrading/upgrade-specific#consul-1-11-0) -- [1.9.x](/consul/docs/upgrading/instructions/upgrade-to-1-10-x) -- [1.8.x](/consul/docs/upgrading/instructions/upgrade-to-1-10-x) -- [1.7.x](/consul/docs/upgrading/instructions/upgrade-to-1-8-x) -- [1.6.x](/consul/docs/upgrading/instructions/upgrade-to-1-8-x) -- [1.5.x](/consul/docs/upgrading/instructions/upgrade-to-1-6-x) -- [1.4.x](/consul/docs/upgrading/instructions/upgrade-to-1-6-x) -- [1.3.x](/consul/docs/upgrading/instructions/upgrade-to-1-6-x) -- [1.2.x](/consul/docs/upgrading/instructions/upgrade-to-1-6-x) -- [1.1.x](/consul/docs/upgrading/instructions/upgrade-to-1-2-x) -- [1.0.x](/consul/docs/upgrading/instructions/upgrade-to-1-2-x) -- [0.9.x](/consul/docs/upgrading/instructions/upgrade-to-1-2-x) -- [0.8.x](/consul/docs/upgrading/instructions/upgrade-to-1-2-x) - -If you are using <= 0.7.x, please contact support for assistance: -- OSS users without paid support plans can request help in our [Community Forum](https://discuss.hashicorp.com/c/consul/29) -- Enterprise and OSS users with paid support plans can contact [HashiCorp Support](https://support.hashicorp.com/) +## General Upgrade Path + +Each upgrade should jump at most 2 major versions, except where +[dedicated instructions](#dedicated-instructions-for-specific-upgrade-paths) +are provided for a larger jump between specific versions. +If your upgrade path has no applicable [dedicated instructions](#dedicated-instructions-for-specific-upgrade-paths), +review the [version-specific upgrade details](/consul/docs/upgrading/upgrade-specific) +to plan your upgrade, starting from the next version and working +upwards to your target version. + +For example, to upgrade from Consul 1.12 to Consul 1.15: + +1. Upgrade to Consul 1.14 as an intermediate step. + To plan, review the upgrade details for + [1.13](/consul/docs/upgrading/upgrade-specific#consul-1-13-x) and + [1.14](/consul/docs/upgrading/upgrade-specific#consul-1-14-x). +1. Upgrade to Consul 1.15. + To plan, review the upgrade details for + [1.15](/consul/docs/upgrading/upgrade-specific#consul-1-15-x). + +## Dedicated Instructions for Specific Upgrade Paths + +The following table provides links to dedicated instructions +for directly upgrading from a version in the starting range +to a destination version. + +| Starting Version Range | Destination Version | Upgrade Instructions | +| ---------------------- | ------------------- | -------------------- | +| 1.8.0 - 1.9.17 | 1.10.12 | Refer to [upgrading to latest 1.10.x](/consul/docs/upgrading/instructions/upgrade-to-1-10-x) | +| 1.6.9 - 1.8.18 | 1.8.19 | Refer to [upgrading to latest 1.8.x](/consul/docs/upgrading/instructions/upgrade-to-1-8-x) | +| 1.2.4 - 1.6.9 | 1.6.10 | Refer to [upgrading to latest 1.6.x](/consul/docs/upgrading/instructions/upgrade-to-1-6-x) | +| 0.8.5 - 1.2.3 | 1.2.4 | Refer to [upgrading to latest 1.2.x](/consul/docs/upgrading/instructions/upgrade-to-1-2-x) | + +For example, to upgrade from Consul 1.3.1 to latest 1.12: +1. Upgrade to Consul 1.6.10 using the dedicated instructions. +1. Upgrade to Consul 1.8.19 using the dedicated instructions. +1. Upgrade to Consul 1.10.12 using the dedicated instructions. +1. Upgrade to latest Consul 1.12.x after consulting the + [version-specific upgrade details](/consul/docs/upgrading/upgrade-specific) + for 1.11 and 1.12. \ No newline at end of file diff --git a/website/content/docs/upgrading/instructions/upgrade-to-1-6-x.mdx b/website/content/docs/upgrading/instructions/upgrade-to-1-6-x.mdx index c39aed602c0..d88560260eb 100644 --- a/website/content/docs/upgrading/instructions/upgrade-to-1-6-x.mdx +++ b/website/content/docs/upgrading/instructions/upgrade-to-1-6-x.mdx @@ -18,8 +18,8 @@ as part of this upgrade. The 1.6.x series is the last series that had support fo ACL tokens, so this migration _must_ happen before upgrading past the 1.6.x release series. Here is some documentation that may prove useful for reference during this upgrade process: -- [ACL System in Legacy Mode](/consul/docs/security/acl/acl-legacy) - You can find - information about legacy configuration options and differences between modes here. +- [Upgrading Legacy ACL tokens](/consul/tutorials/security-operations/access-control-token-migration) - You can find + information about upgrading legacy ACL tokens and differences between modes here. - [Configuration](/consul/docs/agent/config) - You can find more details around legacy ACL and new ACL configuration options here. Legacy ACL config options will be listed as deprecates as of 1.4.0. diff --git a/website/content/docs/upgrading/upgrade-specific.mdx b/website/content/docs/upgrading/upgrade-specific.mdx index 07d456ad816..79b0285d63c 100644 --- a/website/content/docs/upgrading/upgrade-specific.mdx +++ b/website/content/docs/upgrading/upgrade-specific.mdx @@ -16,6 +16,17 @@ upgrade flow. ## Consul 1.15.x +#### Service mesh compatibility ((#service-mesh-compatibility-1-15)) + +Upgrade to **Consul version 1.15.2 or later**. + +Consul versions 1.15.0 - 1.15.1 contain a race condition that can cause +some service instances to lose their ability to communicate in the mesh after +[72 hours (LeafCertTTL)](/consul/docs/connect/ca/consul#leafcertttl) +due to a problem with leaf certificate rotation. + +This bug is fixed in Consul versions 1.15.2 and newer. + #### Removing configuration options The `connect.enable_serverless_plugin` configuration option was removed. Lambda integration is now enabled by default. @@ -138,13 +149,13 @@ In Consul v1.15 and higher: ## Consul 1.14.x ### Service Mesh Compatibility -Prior to Consul 1.14, cluster peering or Consul connect were disabled by default. +Prior to Consul 1.14, cluster peering and Consul service mesh were disabled by default. A breaking change was made in Consul 1.14 that: - [Cluster Peering is enabled by default.](/consul/docs/connect/cluster-peering) Cluster peering and WAN federation can coexist, so there is no need to disable cluster peering to upgrade existing WAN federated datacenters. To disable cluster peering nonetheless, set [`peering.enabled`](/consul/docs/agent/config/config-files#peering_enabled) to `false`. -- [Consul Connect is enabled by default.](/consul/docs/connect) +- [Consul service mesh is enabled by default.](/consul/docs/connect) To disable, set [`connect.enabled`](/consul/docs/agent/config/config-files#connect_enabled) to `false`. The changes to Consul service mesh in version 1.14 are incompatible with Nomad 1.4.3 and @@ -209,7 +220,7 @@ Consul 1.13.0 contains a bug that prevents Consul server agents from restoring saved state on startup if the state 1. was generated before Consul 1.13 (such as during an upgrade), and -2. contained any Connect proxy registrations. +2. contained any service mesh proxy registrations. This bug is fixed in Consul versions 1.13.1 and newer. @@ -373,6 +384,12 @@ if the Vault policy is not modified as described. As a precaution, we recommend both modifying the Vault policy and upgrading to Consul 1.13.3 or later to avoid the breaking nature of that change. +### Nomad Incompatibility + +Nomad users should not upgrade to Consul 1.13.8 as an API change in Consul +prevents Nomad from correctly detecting the Consul agent version, resulting in +allocations not being placed in those clients. + ## Consul 1.12.x ((#consul-1-12-0)) ### Modify Vault Policy for Vault CA Provider @@ -499,6 +516,14 @@ to Consul 1.11.11 or later to avoid the breaking nature of that change. ### Licensing Changes +You can only upgrade to Consul Enterprise 1.10 from the following Enterprise versions: +- 1.8 release series: 1.8.13+ +- 1.9 release series: 1.9.7+ + +Other versions of Consul Enterprise are not forward compatible with v1.10 and will +cause issues during the upgrade that could result in agents failing to start due to +[changes in the way we manage licenses](/consul/docs/enterprise/license/faq). + Consul Enterprise 1.10 has removed temporary licensing capabilities from the binaries found on https://releases.hashicorp.com. Servers will no longer load a license previously set through the CLI or API. Instead the license must be present in the server's configuration @@ -743,7 +768,7 @@ registrations](/consul/docs/connect/registration/sidecar-service). There are two major features in Consul 1.4.0 that may impact upgrades: a [new ACL system](#acl-upgrade) and [multi-datacenter support for -Connect](#connect-multi-datacenter) in the Enterprise version. +service mesh](#multi-datacenter-service-mesh) in the Enterprise version. ### ACL Upgrade @@ -824,13 +849,13 @@ as soon as possible after upgrade, as well as updating any integrations to work with the the new ACL [Token](/consul/api-docs/acl/tokens) and [Policy](/consul/api-docs/acl/policies) APIs. -### Connect Multi-datacenter +### Multi-datacenter service mesh This only applies to users upgrading from an older version of Consul Enterprise to Consul Enterprise 1.4.0 (all license types). -In addition, this upgrade will only affect clusters where [Connect is enabled](/consul/docs/connect/configuration) on your servers before the migration. +In addition, this upgrade will only affect clusters where [service mesh is enabled](/consul/docs/connect/configuration) on your servers before the migration. -Connect multi-datacenter uses the same primary/secondary approach as ACLs and +Multi-datacenter service mesh uses the same primary/secondary approach as ACLs and will use the same [primary_datacenter](#primary-datacenter). When a secondary datacenter server restarts with 1.4.0 it will detect it is not the primary and begin an automatic bootstrap of multi-datacenter CA federation. @@ -839,11 +864,11 @@ Datacenters can be upgraded in either order; secondary datacenters will not switch into multi-datacenter mode until all servers in both the secondary and primary datacenter are detected to be running at least Consul 1.4.0. Secondary datacenters monitor this periodically (every few minutes) and will -automatically upgrade Connect to use a federated Certificate Authority when +automatically upgrade service mesh to use a federated Certificate Authority when they do. In general, migrating a Consul cluster from OSS to Enterprise will update the -CA to be federated automatically and without impact on Connect traffic. When +CA to be federated automatically and without impact on service mesh traffic. When upgrading Consul Enterprise 1.3.x to Consul Enterprise 1.4.0 upgrades the CA upgrade is seamless, however depending on the size of the cluster, _new_ connection attempts in the secondary datacenter might fail for a short window @@ -852,15 +877,12 @@ authorization endpoint validating originating cluster in a way that was not fully forwards compatible with migrating between cluster trust domains. That issue is fixed in 1.4.0 as part of General Availability. -Once migrated (typically a few seconds). Connect will use the primary +Once migrated (typically a few seconds). The service mesh will use the primary datacenter's Certificate Authority as the root of trust for all other datacenters. CA migration or root key changes in the primary will now rotate automatically and without loss of connectivity throughout all datacenters and workloads. -For more information see [Connect -Multi-datacenter](/consul/docs/enterprise). - ## Consul 1.3.0 This version added support for multiple tag filters in service discovery @@ -990,11 +1012,9 @@ config files loaded by Consul, even when using the [`-config-file`](/consul/docs/agent/config/cli-flags#_config_file) argument to specify a file directly. -#### Service Definition Parameter Case changed +#### Use Snake Case for Service Definition Parameters -All config file formats now require snake_case fields, so all CamelCased parameter -names should be changed before upgrading. -See [Service Definition Parameter Case](/consul/docs/discovery/services#service-definition-parameter-case) documentation for details. +Snake case, which is a convention that uses underscores between words in a configuration key, is required for all configuration file formats. Change any camel cased parameter to snake case equivalents before upgrading. #### Deprecated Options Have Been Removed @@ -1386,7 +1406,7 @@ the file. Consul 0.7 also uses a new, automatically-created raft/peers.info file to avoid ingesting the `peers.json` file on the first start after upgrading (the `peers.json` file is simply deleted on the first start after upgrading). -Please be sure to review the [Outage Recovery tutorial](/consul/tutorials/datacenter-operations/recovery-outage) +Please be sure to review the [Disaster recovery for Consul clusters tutorial](/consul/tutorials/datacenter-operations/recovery-outage) before upgrading for more details. ## Consul 0.6.4 diff --git a/website/content/partials/create-token-auth-methods.mdx b/website/content/partials/create-token-auth-methods.mdx new file mode 100644 index 00000000000..91287072850 --- /dev/null +++ b/website/content/partials/create-token-auth-methods.mdx @@ -0,0 +1,3 @@ +### Auth methods + +Auth methods are components that perform authentication against a trusted external party to authorize the creation of ACL tokens for use within the local datacenter. Refer to the [auth methods documentation](/consul/docs/security/acl/auth-methods) for details about how to leverage auth methods in your network. diff --git a/website/content/partials/create-token-requirements.mdx b/website/content/partials/create-token-requirements.mdx new file mode 100644 index 00000000000..bf4742719e8 --- /dev/null +++ b/website/content/partials/create-token-requirements.mdx @@ -0,0 +1,22 @@ +### Authentication + +You must provide an ACL token linked to a policy with `acl:write` permissions to create and modify ACL tokens and policies using the CLI or API. + +You can provide the token manually using the `-token` option on the command line, but we recommend setting the `CONSUL_HTTP_TOKEN` environment variable to simplify your workflow: + +```shell-session +$ export CONSUL_HTTP_TOKEN= +``` + +The Consul CLI automatically reads the `CONSUL_HTTP_TOKEN` environment variable so that you do not have to pass the token to every Consul CLI command. + +To authenticate calls to the Consul HTTP API, you must provide the token in the `X-Consul-Token` header for each call: + +```shell-session +$ curl --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" ... +``` + +To learn about alternative ways to authenticate, refer to the following documentation: + +* [CLI Authentication](/consul/commands#authentication) +* [API Authentication](/consul/api-docs/api-structure#authentication) diff --git a/website/data/api-docs-nav-data.json b/website/data/api-docs-nav-data.json index fb1dd87421b..66d8fa9a949 100644 --- a/website/data/api-docs-nav-data.json +++ b/website/data/api-docs-nav-data.json @@ -165,6 +165,10 @@ { "title": "Segment", "path": "operator/segment" + }, + { + "title": "Usage", + "path": "operator/usage" } ] }, diff --git a/website/data/commands-nav-data.json b/website/data/commands-nav-data.json index 62ed01a4004..ee491e9dfa7 100644 --- a/website/data/commands-nav-data.json +++ b/website/data/commands-nav-data.json @@ -425,6 +425,10 @@ { "title": "raft", "path": "operator/raft" + }, + { + "title": "usage", + "path": "operator/usage" } ] }, @@ -528,6 +532,23 @@ } ] }, + { + "title": "troubleshoot", + "routes": [ + { + "title": "Overview", + "path": "troubleshoot" + }, + { + "title": "upstreams", + "path": "troubleshoot/upstreams" + }, + { + "title": "proxy", + "path": "troubleshoot/proxy" + } + ] + }, { "title": "validate", "path": "validate" diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 16bf17d84ba..ea5fe2094d3 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -145,6 +145,10 @@ { "title": "Consul", "routes": [ + { + "title": "v1.15.x", + "path": "release-notes/consul/v1_15_x" + }, { "title": "v1.14.x", "path": "release-notes/consul/v1_14_x" @@ -174,6 +178,10 @@ { "title": "Consul K8s", "routes": [ + { + "title": "v1.1.x", + "path": "release-notes/consul-k8s/v1_1_x" + }, { "title": "v1.0.x", "path": "release-notes/consul-k8s/v1_0_x" @@ -193,7 +201,7 @@ ] }, { - "title": "Consul API Gateway", + "title": "API Gateway for Kubernetes", "routes": [ { "title": "v0.5.x", @@ -303,19 +311,70 @@ "divider": true }, { - "title": "Service Discovery", + "title": "Services", "routes": [ { - "title": "Register Services - Service Definitions", - "path": "discovery/services" + "title": "Overview", + "path": "services/services" }, { - "title": "Find Services - DNS Interface", - "path": "discovery/dns" + "title": "Usage", + "routes": [ + { + "title": "Define services", + "path": "services/usage/define-services" + }, + { + "title": "Define health checks", + "path": "services/usage/checks" + }, + { + "title": "Register services and health checks", + "path": "services/usage/register-services-checks" + } + ] }, { - "title": "Monitor Services - Check Definitions", - "path": "discovery/checks" + "title": "Discover services with DNS", + "routes": [ + { + "title": "Overview", + "path": "services/discovery/dns-overview" + }, + { + "title": "Configure DNS behavior", + "path": "services/discovery/dns-configuration" + }, + { + "title": "Perform static DNS lookups", + "path": "services/discovery/dns-static-lookups" + }, + { + "title": "Enable dynamic DNS lookups", + "path": "services/discovery/dns-dynamic-lookups" + } + ] + }, + { + "title": "Configuration", + "routes": [ + { + "title": "Overview", + "path": "services/configuration/services-configuration-overview" + }, + { + "title": "Services", + "path": "services/configuration/services-configuration-reference" + }, + { + "title": "Health checks", + "path": "services/configuration/checks-configuration-reference" + }, + { + "title": "Service defaults", + "href": "connect/config-entries/service-defaults" + } + ] } ] }, @@ -341,6 +400,42 @@ "title": "Overview", "path": "connect/config-entries" }, + { + "title": "API Gateway", + "href": "/consul/docs/connect/gateways/api-gateway/configuration/api-gateway", + "badge": { + "text": "BETA", + "type": "outlined", + "color": "neutral" + } + }, + { + "title": "HTTP Route", + "href": "/consul/docs/connect/gateways/api-gateway/configuration/http-route", + "badge": { + "text": "BETA", + "type": "outlined", + "color": "neutral" + } + }, + { + "title": "TCP Route", + "href": "/consul/docs/connect/gateways/api-gateway/configuration/tcp-route", + "badge": { + "text": "BETA", + "type": "outlined", + "color": "neutral" + } + }, + { + "title": "Inline Certificate", + "href": "/consul/docs/connect/gateways/api-gateway/configuration/inline-certificate", + "badge": { + "text": "BETA", + "type": "outlined", + "color": "neutral" + } + }, { "title": "Ingress Gateway", "path": "connect/config-entries/ingress-gateway" @@ -394,6 +489,28 @@ "title": "Envoy", "path": "connect/proxies/envoy" }, + { + "title": "Envoy Extensions", + "routes": [ + { + "title": "Overview", + "path": "connect/proxies/envoy-extensions" + }, + { + "title": "Usage", + "routes": [ + { + "title": "Run Lua scripts in Envoy proxies", + "path": "connect/proxies/envoy-extensions/usage/lua" + }, + { + "title": "Invoke Lambda functions in Envoy proxies", + "path": "connect/proxies/envoy-extensions/usage/lambda" + } + ] + } + ] + }, { "title": "Built-in Proxy", "path": "connect/proxies/built-in" @@ -401,11 +518,6 @@ { "title": "Proxy Integration", "path": "connect/proxies/integrate" - }, - { - "title": "Managed (Deprecated)", - "path": "connect/proxies/managed-deprecated", - "hidden": true } ] }, @@ -427,12 +539,25 @@ ] }, { - "title": "Service-to-service permissions - Intentions", - "path": "connect/intentions" - }, - { - "title": "Service-to-service permissions - Intentions (Legacy Mode)", - "path": "connect/intentions-legacy" + "title": "Service intentions", + "routes": [ + { + "title": "Overview", + "path": "connect/intentions" + }, + { + "title": "Create and manage service intentions", + "path": "connect/intentions/create-manage-intentions" + }, + { + "title": "Service intentions legacy mode", + "path": "connect/intentions/legacy" + }, + { + "title": "Configuration", + "href": "/consul/docs/connect/config-entries/service-intentions" + } + ] }, { "title": "Transparent Proxy", @@ -483,6 +608,45 @@ "title": "Overview", "path": "connect/gateways" }, + { + "title": "API Gateways", + "badge": { + "text": "BETA", + "type": "outlined", + "color": "neutral" + }, + "routes": [ + { + "title": "Overview", + "path": "connect/gateways/api-gateway" + }, + { + "title": "Usage", + "path": "connect/gateways/api-gateway/usage" + }, + { + "title": "Configuration", + "routes": [ + { + "title": "API Gateway", + "path": "connect/gateways/api-gateway/configuration/api-gateway" + }, + { + "title": "HTTP Route", + "path": "connect/gateways/api-gateway/configuration/http-route" + }, + { + "title": "TCP Route", + "path": "connect/gateways/api-gateway/configuration/tcp-route" + }, + { + "title": "Inline Certificate", + "path": "connect/gateways/api-gateway/configuration/inline-certificate" + } + ] + } + ] + }, { "title": "Mesh Gateways", "routes": [ @@ -505,16 +669,29 @@ { "title": "Enabling Peering Control Plane Traffic", "path": "connect/gateways/mesh-gateway/peering-via-mesh-gateways" - }, - { - "title": "Enabling Service-to-service Traffic Across Peered Clusters", - "path": "connect/gateways/mesh-gateway/service-to-service-traffic-peers" } ] }, { "title": "Ingress Gateways", - "path": "connect/gateways/ingress-gateway" + "routes": [ + { + "title": "Overview", + "path": "connect/gateways/ingress-gateway" + }, + { + "title": "Implement an ingress gateway", + "path": "connect/gateways/ingress-gateway/ingress-gateways-usage" + }, + { + "title": "Serve TLS certificates from an external service", + "path": "connect/gateways/ingress-gateway/ingress-gateways-tls-external-service" + }, + { + "title": "Configuration", + "href": "/consul/docs/connect/config-entries/ingress-gateway" + } + ] }, { "title": "Terminating Gateways", @@ -526,16 +703,29 @@ "title": "Cluster Peering", "routes": [ { - "title": "What is Cluster Peering?", + "title": "Overview", "path": "connect/cluster-peering" }, { - "title": "Create and Manage Peering Connections", - "path": "connect/cluster-peering/create-manage-peering" + "title": "Technical Specifications", + "path": "connect/cluster-peering/tech-specs" }, { - "title": "Cluster Peering on Kubernetes", - "path": "connect/cluster-peering/k8s" + "title": "Usage", + "routes": [ + { + "title": "Establish Cluster Peering Connections", + "path": "connect/cluster-peering/usage/establish-cluster-peering" + }, + { + "title": "Manage Cluster Peering Connections", + "path": "connect/cluster-peering/usage/manage-connections" + }, + { + "title": "Manage L7 Traffic With Cluster Peering", + "path": "connect/cluster-peering/usage/peering-traffic-management" + } + ] } ] }, @@ -642,7 +832,61 @@ }, { "title": "Tokens", - "path": "security/acl/acl-tokens" + "routes": [ + { + "title": "Overview", + "path": "security/acl/tokens" + }, + { + "title": "Create ACL Tokens", + "routes": [ + { + "title": "Create a service token", + "path": "security/acl/tokens/create/create-a-service-token" + }, + { + "title": "Create an agent token", + "path": "security/acl/tokens/create/create-an-agent-token" + }, + { + "title": "Create a UI token", + "path": "security/acl/tokens/create/create-a-ui-token" + }, + { + "title": "Create a mesh gateway token", + "path": "security/acl/tokens/create/create-a-mesh-gateway-token" + }, + { + "title": "Create an ingress gateway token", + "path": "security/acl/tokens/create/create-an-ingress-gateway-token" + }, + { + "title": "Create a terminating gateway token", + "path": "security/acl/tokens/create/create-a-terminating-gateway-token" + }, + { + "title": "Create a DNS token", + "path": "security/acl/tokens/create/create-a-dns-token" + }, + { + "title": "Create a replication token", + "path": "security/acl/tokens/create/create-a-replication-token" + }, + { + "title": "Create a snapshot agent token", + "path": "security/acl/tokens/create/create-a-snapshot-agent-token" + }, + { + "title": "Create a token for Vault's Consul storage backend", + "path": "security/acl/tokens/create/create-a-token-for-vault-consul-storage" + }, + { + "title": "Create a Consul ESM token", + "path": "security/acl/tokens/create/create-a-consul-esm-token" + } + ] + } + ] }, { "title": "Policies", @@ -734,6 +978,23 @@ } ] }, + { + "title": "Limit Traffic Rates", + "routes": [ + { + "title": "Overview", + "path": "agent/limits" + }, + { + "title": "Initialize Rate Limit Settings", + "path": "agent/limits/init-rate-limits" + }, + { + "title": "Set Global Traffic Rate Limits", + "path": "agent/limits/set-global-traffic-rate-limits" + } + ] + }, { "title": "Configuration Entries", "path": "agent/config-entries" @@ -750,6 +1011,27 @@ "title": "RPC", "path": "agent/rpc", "hidden": true + }, + { + "title": "Experimental WAL LogStore", + "routes": [ + { + "title": "Overview", + "path": "agent/wal-logstore" + }, + { + "title": "Enable WAL LogStore backend", + "path": "agent/wal-logstore/enable" + }, + { + "title": "Monitor Raft metrics and logs for WAL", + "path": "agent/wal-logstore/monitoring" + }, + { + "title": "Revert to BoltDB", + "path": "agent/wal-logstore/revert-to-boltdb" + } + ] } ] }, @@ -781,6 +1063,10 @@ { "title": "Troubleshoot", "routes": [ + { + "title": "Service-to-Service Troubleshooting", + "path": "troubleshoot/troubleshoot-services" + }, { "title": "Common Error Messages", "path": "troubleshoot/common-errors" @@ -805,7 +1091,6 @@ "title": "Architecture", "path": "k8s/architecture" }, - { "title": "Installation", "routes": [ @@ -963,6 +1248,32 @@ "title": "Admin Partitions", "href": "/docs/enterprise/admin-partitions" }, + { + "title": "Cluster Peering", + "routes": [ + { + "title": "Technical Specifications", + "path": "k8s/connect/cluster-peering/tech-specs" + }, + { + "title": "Usage", + "routes": [ + { + "title": "Establish Cluster Peering Connections", + "path": "k8s/connect/cluster-peering/usage/establish-peering" + }, + { + "title": "Manage Cluster Peering Connections", + "path": "k8s/connect/cluster-peering/usage/manage-peering" + }, + { + "title": "Manage L7 Traffic With Cluster Peering", + "path": "k8s/connect/cluster-peering/usage/l7-traffic" + } + ] + } + ] + }, { "title": "Transparent Proxy", "href": "/docs/connect/transparent-proxy" @@ -1281,6 +1592,10 @@ "title": "Overview", "path": "enterprise/license/overview" }, + { + "title": "Automated entitlement utilization reporting", + "path": "enterprise/license/utilization-reporting" + }, { "title": "FAQ", "path": "enterprise/license/faq" @@ -1293,7 +1608,7 @@ "divider": true }, { - "title": "Consul API Gateway", + "title": "API Gateway for Kubernetes", "routes": [ { "title": "Overview", @@ -1315,7 +1630,7 @@ "title": "Usage", "routes": [ { - "title": "Basic Usage", + "title": "Deploy", "path": "api-gateway/usage/usage" }, { diff --git a/website/public/img/cluster-peering-diagram.png b/website/public/img/cluster-peering-diagram.png new file mode 100644 index 00000000000..d00aa3eb0f3 Binary files /dev/null and b/website/public/img/cluster-peering-diagram.png differ diff --git a/website/public/img/connect_proxy_to_lambda.svg b/website/public/img/connect_proxy_to_lambda.svg index 1ba7a277937..85c0b7d3684 100644 --- a/website/public/img/connect_proxy_to_lambda.svg +++ b/website/public/img/connect_proxy_to_lambda.svg @@ -13,4 +13,4 @@ } - LambdaConnect ProxyService12 \ No newline at end of file + LambdaMesh Sidecar ProxyService12 \ No newline at end of file diff --git a/website/public/img/consul-connect/consul-service-mesh-intentions-overview.svg b/website/public/img/consul-connect/consul-service-mesh-intentions-overview.svg new file mode 100644 index 00000000000..cdd7ea0ad6e --- /dev/null +++ b/website/public/img/consul-connect/consul-service-mesh-intentions-overview.svg @@ -0,0 +1 @@ + diff --git a/website/public/img/terminating_gateway_to_lambda.svg b/website/public/img/terminating_gateway_to_lambda.svg index fbd414b258e..fd8c5b61957 100644 --- a/website/public/img/terminating_gateway_to_lambda.svg +++ b/website/public/img/terminating_gateway_to_lambda.svg @@ -13,4 +13,4 @@ } - TerminatingGatewayConnect ProxyService12Lambda3 \ No newline at end of file + TerminatingGatewayMesh Sidecar ProxyService12Lambda3 \ No newline at end of file diff --git a/website/redirects.js b/website/redirects.js index 8e31b56f340..5deb7617cc9 100644 --- a/website/redirects.js +++ b/website/redirects.js @@ -4,4 +4,67 @@ // modify or delete existing redirects without first verifying internally. // Next.js redirect documentation: https://nextjs.org/docs/api-reference/next.config.js/redirects -module.exports = [] +module.exports = [ + { + source: '/consul/docs/connect/cluster-peering/create-manage-peering', + destination: + '/consul/docs/connect/cluster-peering/usage/establish-cluster-peering', + permanent: true, + }, + { + source: '/consul/docs/connect/cluster-peering/usage/establish-peering', + destination: + '/consul/docs/connect/cluster-peering/usage/establish-cluster-peering', + permanent: true, + }, + { + source: '/consul/docs/connect/cluster-peering/k8s', + destination: '/consul/docs/k8s/connect/cluster-peering/tech-specs', + permanent: true, + }, + { + source: '/consul/docs/connect/intentions#intention-management-permissions', + destination: `/consul/docs/connect/intentions/create-manage-intentions#acl-requirements`, + permanent: true, + }, + { + source: '/consul/docs/connect/intentions#intention-basics', + destination: `/consul/docs/connect/intentions`, + permanent: true, + }, + { + source: '/consul/docs/v1.16.x/connect/transparent-proxy', + destination: '/consul/docs/v1.16.x/k8s/connect/transparent-proxy', + permanent: true, + }, + { + source: '/consul/docs/1.16.x/agent/limits/init-rate-limits', + destination: '/consul/docs/1.16.x/agent/limits/usage/init-rate-limits', + permanent: true, + }, + { + source: '/consul/docs/1.16.x/agent/limits/set-global-traffic-rate-limits', + destination: + '/consul/docs/1.16.x/agent/limits/usage/set-global-traffic-rate-limits', + permanent: true, + }, + { + source: + '/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-peers', + destination: + '/consul/docs/connect/cluster-peering/usage/establish-cluster-peering', + permanent: true, + }, + { + source: '/consul/docs/enterprise/sentinel', + destination: + '/consul/docs/dynamic-app-config/kv#using-sentinel-to-apply-policies-for-consul-kv', + permanent: true, + }, + { + source: + '/consul/docs/connect/gateways/mesh-gateway/service-to-service-traffic-datacenters', + destination: '/consul/docs/k8s/deployment-configurations/multi-cluster', + permanent: true, + }, +]