diff --git a/docs/config.json b/docs/config.json index 3f6cf8677ebab..000fdd952efd5 100644 --- a/docs/config.json +++ b/docs/config.json @@ -940,6 +940,10 @@ "title": "Cert Authority Rotation", "slug": "/management/operations/ca-rotation/" }, + { + "title": "Database Cert Authority Rotation", + "slug": "/management/operations/db-ca-rotation/" + }, { "title": "TLS Routing Migration", "slug": "/management/operations/tls-routing/", @@ -951,9 +955,11 @@ { "title": "Proxy Peering Migration", "slug": "/management/operations/proxy-peering/", - "forScopes": [ - "enterprise" - ] + "forScopes": ["enterprise"] + }, + { + "title": "Database CA Migrations", + "slug": "/management/operations/db-ca-migrations/" } ] }, diff --git a/docs/cspell.json b/docs/cspell.json index 108a966197577..b55487e76b181 100644 --- a/docs/cspell.json +++ b/docs/cspell.json @@ -630,6 +630,7 @@ "nodename", "nohup", "nologin", + "noout", "noprompt", "nosql", "nowait", diff --git a/docs/pages/database-access/reference/configuration.mdx b/docs/pages/database-access/reference/configuration.mdx index 748d61f06f014..de8fafd156fa6 100644 --- a/docs/pages/database-access/reference/configuration.mdx +++ b/docs/pages/database-access/reference/configuration.mdx @@ -110,11 +110,22 @@ spec: # Database connection endpoint. uri: "localhost:5432" - # Optional CA for validating the database certificate. - ca_cert: | - -----BEGIN CERTIFICATE----- - ... - -----END CERTIFICATE----- + # Optional TLS configuration. + tls: + # TLS verification mode. Valid options are: + # 'verify-full' - performs full certificate validation (default). + # 'verify-ca' - the same as `verify-full`, but skips the server name validation. + # 'insecure' - accepts any certificate provided by database (not recommended). + mode: verify-full + # Optional database DNS server name. It allows to override the DNS name on + # a client certificate when connecting to a database. + # Use only with 'verify-full' mode. + server_name: db.example.com + # Optional CA for validating the database certificate. + ca_cert: | + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- # Database admin user for automatic user provisioning. admin_user: diff --git a/docs/pages/management/operations.mdx b/docs/pages/management/operations.mdx index 4d6a3f80a184c..602159573383b 100644 --- a/docs/pages/management/operations.mdx +++ b/docs/pages/management/operations.mdx @@ -13,5 +13,7 @@ the [Cluster Administration Guides](./admin.mdx) section. - [Scaling](./operations/scaling.mdx): How to configure Teleport for large-scale deployments. - [Backup and Restore](./operations/backup-restore.mdx): Backing up and restoring the cluster. - [CA Rotation](./operations/ca-rotation.mdx): Rotating Teleport certificate authorities. +- [Database CA Rotation](./operations/db-ca-rotation.mdx): Rotating Teleport's `db` or `db_client` certificate authorities. - [TLS Routing Migration](./operations/tls-routing.mdx): Migrating your Teleport cluster to single-port TLS routing mode. - [Proxy Peering Migration](./operations/proxy-peering.mdx): Migrating your Teleport cluster to Proxy Peering mode. +- [Database CA Migrations](./operations/db-ca-migrations.mdx): Completing Teleport's Database CA migrations. diff --git a/docs/pages/management/operations/ca-rotation.mdx b/docs/pages/management/operations/ca-rotation.mdx index 2f1fa3e1e72c5..54001014d9d40 100644 --- a/docs/pages/management/operations/ca-rotation.mdx +++ b/docs/pages/management/operations/ca-rotation.mdx @@ -16,13 +16,20 @@ This section will show you how to rotate Teleport's certificate authority. If you are joining Teleport processes to a cluster via the Teleport Auth Service using a [join token](../../agents/join-services-to-your-cluster/join-token.mdx), each Teleport process will need a CA pin to trust the Auth Service. The CA pin will -change after each CA rotation. Make sure you use the *new* CA pin when adding -Teleport services after rotation. +change after each `host` CA rotation. Make sure you use the *new* CA pin when adding +Teleport services after `host` CA rotation. - -Teleport signs Windows Desktop certificates with the user certificate authority. -If the user CA is rotated, the new CA certificate must be exported and + +Teleport signs Windows Desktop client certificates with the `user` certificate +authority. +If the `user` CA is rotated, the new CA certificate must be exported and configured in group policy. + +Teleport signs self-hosted database host certificates with the `db` certificate +authority and signs database client certificates with the `db_client` CA. +If either of these CAs is rotated, then self-hosted databases must be +reconfigured. +Refer to [the Database CA Rotation Guide](./db-ca-rotation.mdx). ### Rotation phases @@ -54,7 +61,7 @@ There are two kinds of certificate rotations: - **Semi-automatic:** Teleport automatically transitions between phases of the rotation after some amount of time (known as a *grace period*) elapses. -For both types of rotations, the cluster goes through the phases in the +For both types of rotation, the cluster goes through the phases in the following order: - `standby` -> `init` -> `update_clients` -> `update_servers` -> `standby` @@ -63,7 +70,7 @@ Administrators can abort the rotation and revert all changes any time before the rotation is completed by entering the `rollback` phase. ```code -$ tctl auth rotate --phase=rollback --type= --manual +$ tctl auth rotate --manual --type= --phase=rollback ``` For example, if an admin has detected that some nodes failed to upgrade during @@ -90,7 +97,7 @@ of the cluster. Initiate the manual rotation of host certificate authorities: ```code -$ tctl auth rotate --phase=init --type= --manual +$ tctl auth rotate --manual --type= --phase=init Updated rotation phase to "init". To check status use 'tctl status' ``` @@ -134,7 +141,7 @@ transitions. Execute the transition from `init` to `update_clients`: ```code -$ tctl auth rotate --phase=update_clients --type= --manual +$ tctl auth rotate --manual --type= --phase=update_clients # Updated rotation phase to "update_clients". To check status use 'tctl status' $ tctl status # Cluster acme.cluster @@ -164,7 +171,7 @@ Now that all nodes have caught up, execute the transition from `update_clients` to `update_servers`: ```code -$ tctl auth rotate --phase=update_servers --type= --manual +$ tctl auth rotate --manual --type= --phase=update_servers # Updated rotation phase to "update_servers". To check status use 'tctl status' $ tctl status @@ -205,7 +212,7 @@ $ tsh ssh hello@terminal ```code -$ tctl auth rotate --phase=standby --type= --manual +$ tctl auth rotate --manual --type= --phase=standby ``` Verify that the rotation has completed with `tctl`: diff --git a/docs/pages/management/operations/db-ca-migrations.mdx b/docs/pages/management/operations/db-ca-migrations.mdx new file mode 100644 index 0000000000000..805a0c96a9238 --- /dev/null +++ b/docs/pages/management/operations/db-ca-migrations.mdx @@ -0,0 +1,176 @@ +--- +title: Database CA Migrations +description: How to complete Teleport Database CA migrations. +--- + +In Teleport, self-hosted databases must be configured with certificates to +enable mTLS authentication via the Teleport Database Service. + +Teleport 15 introduced a new `db_client` certificate authority (CA) to split the +responsibilities of the Teleport `db` CA, which was acting as both host and +client CA for Teleport self-hosted database access. + +Teleport's host/client database CA split is intended to limit the potential for +lateral movement to other resources in the event that a database instance's +private key is compromised. + +The `db` and `db_client` CAs were both introduced as an automatic migration +after upgrading to Teleport 10 and Teleport 15, respectively. + +This guide will explain why these CAs were created and how to determine if your +CA(s) should be rotated to complete the migration process. + +## Prerequisites + +(!docs/pages/includes/edition-prereqs-tabs.mdx!) + +- (!docs/pages/includes/tctl.mdx!) +- A Teleport cluster that was created prior to Teleport 15. + If your Teleport cluster was created with Teleport 15+, then this guide does + not apply to your cluster, because your `db` and `db_client` CAs were not + migrated. + +## Teleport `db` CA migration + +Your Teleport cluster's `db` CA can be used to issue certificates to self-hosted +databases. +This is convenient, because the Teleport Database Service trusts certificates +issued by the `db` CA by default, so there is no additional TLS configuration +in Teleport required. + +Alternatively, you can issue certificates to your self-hosted databases using +an external CA - you just need to configure the Teleport Database Service to +trust that CA when connecting to your database(s). + +
+For a static database defined in your Teleport Database Service `teleport.yaml` +configuration file, set `tls.ca_cert_file` to a file containing your CA's root +certificate. + +For a dynamic database, put your CA's root certificate in `spec.tls.ca_cert`. + +For examples and more information, consult the +[Database Access Configuration Reference](../../database-access/reference/configuration.mdx). +
+ +Prior to Teleport 10, the Teleport `host` CA was used to issue certificates to +self-hosted databases (via `tctl auth sign`). +The Teleport `db` CA was introduced to decouple self-hosted database CA rotation +from the rest of your Teleport cluster. +The idea is that you should be able to rotate the CA used for self-hosted +databases without affecting other resources connected to your cluster. +Likewise, when you rotate your cluster's `host` CA, you should not have to worry +about affecting self-hosted databases. + +To avoid breaking database access after upgrading to Teleport 10, Teleport +clusters are automatically migrated to create the `db` CA as a copy of +the `host` CA. + +If your cluster was upgraded to Teleport 10 and you use Teleport to issue +certificates to your self-hosted databases, then you should ensure that you have +completed the `db` CA migration. +Otherwise, if you later rotate just one CA for any reason, a copy of the old CA +will still exist. +While this does not necessarily lead to a vulnerability in your cluster, it is +bad security practice to keep an old CA around after rotating it. + +To complete the `db` CA migration: +- we recommend rotating your `host` CA +- we **strongly recommend** rotating your `db` CA + +## Teleport `db_client` CA migration + +The Teleport Database Service needs to authenticate itself to self-hosted +database(s) using a client certificate, which requires that you configure your +database(s) to trust Teleport's `db_client` CA. +Prior to the introduction of the `db_client` CA in Teleport 15, self-hosted +had to be configured to trust the Teleport `db` CA for client authentication. + +With the old approach - trusting the `db` CA for client connections - if a +database's private key is compromised, and a `db` certificate was issued for +that key, then it could be used to gain access to other databases. + +Not all self-hosted databases are vulnerable to lateral movement after a private +key compromise. +For example, MySQL and PostgreSQL both verify that a client's certificate +subject matches the client's database user. +Other databases only verify that a client's certificate is trusted, but do not +match the certificate subject to the database username. +For example, Cassandra, ScyllaDB, and Redis do not verify the client cert +subject. +All of these databases can be configured to require password authentication +after a successful mTLS handshake. +However, for defense in depth, these databases should only mTLS handshake with +a client that presents a `db_client` CA-issued certificate. + +If your Teleport cluster was upgraded to Teleport 15, then you should ensure +that you have completed the `db_client` migration. +To complete the `db_client` CA migration: +- we recommend rotating your `db` CA +- we **strongly recommend** rotating your `db_client` CA. +- we **strongly recommend** reconfiguring your databases' certificates after + you complete the `db_client` CA rotation. + + + If you use `tctl auth sign` to reconfigure a database's certificates during + a `db_client` CA rotation, then the trusted certificate output will include + both the old and the new CA certificates. + To complete the migration, you should reconfigure those databases again after + the rotation - that way they will only trust the new CA. + + If you don't want to reconfigure each database both during and after the + `db_client` CA rotation, and you do not mind temporarily losing connectivity + to your databases via Teleport, then you can just complete the `db_client` CA + rotation and reconfigure your databases afterward. + + +## 1/2. Check for Teleport CA migrations + +If you upgraded your cluster to Teleport 10 and you have never rotated your +`host` or `db` CAs, then you should complete the `db` CA migration. + +If you upgraded your cluster to Teleport 15 and you have never rotated your +`db` or `db_client` CAs, then you should complete the `db_client` CA migration. + +If you are unsure whether you need to complete the migration for either the `db` +or `db_client` CAs, you can check for duplicated CAs. +Use these commands to print the X.509 certificate serial number for your `host`, +`db`, and `db_client` CAs (in that order): + +```code +$ tctl auth export --type=tls-host | openssl x509 -noout -serial +$ tctl auth export --type=db | openssl x509 -noout -serial +$ tctl auth export --type=db-client | openssl x509 -noout -serial +``` + +If the `db` CA serial number matches the `host` CA serial number, then you +need to complete the `db` CA migration. + +If the `db_client` CA serial number matches the `db` CA serial number, then you +need to complete the `db_client` CA migration. + +## 2/2. Rotate CAs + +If you need to complete both the `db` and `db_client` migrations, then a single +rotation of each of the `host`, `db`, and `db_client` CAs is enough: you do not +need to rotate the `db` CA twice. + +If you need to rotate the `host` CA, we recommend completing that rotation +before starting either of the `db` or `db_client` CA rotations: do not rotate +other CAs in parallel with a `host` CA rotation. +For information about `host` CA rotation, refer to the +[CA Rotation Guide](./ca-rotation.mdx). + +Database CA rotations are a little different, because they involve configuring +external resources (self-hosted databases) with new certificates during the +rotation. +You can (and should) rotate the `db` and `db_client` CAs at the same time to +avoid repeating the database certificate reconfiguration steps. + +For details on rotating the `db` or `db_client` CA, refer to the +[Database CA Rotation Guide](./db-ca-rotation.mdx). + +## Further reading + +- How the [Teleport Certificate Authority](../../architecture/authentication.mdx) works. +- How [Teleport Database Access](../../database-access/architecture.mdx) works. diff --git a/docs/pages/management/operations/db-ca-rotation.mdx b/docs/pages/management/operations/db-ca-rotation.mdx new file mode 100644 index 0000000000000..0454968fcd205 --- /dev/null +++ b/docs/pages/management/operations/db-ca-rotation.mdx @@ -0,0 +1,228 @@ +--- +title: Database Cert Authority Rotation +description: How to rotate Teleport database certificate authorities +--- + +Teleport connects to self-hosted databases using mutual TLS, or mTLS for short. +To enable mTLS connections to your self-hosted database via Teleport, you have +to configure your database and Teleport so that they can complete an mTLS +handshake: essentially, they need to trust each other. +To that end, your Teleport cluster manages its own internal database certificate +authorities. +This guide will explain how to rotate your Teleport cluster's database CAs. + +## Prerequisites + +(!docs/pages/includes/edition-prereqs-tabs.mdx!) + +- (!docs/pages/includes/tctl.mdx!) + +## Database CA rotation + +Teleport uses two certificate authorities for self-hosted database access: + +- The `db` certificate authority is used to issue server certificates to + self-hosted databases. +- The `db_client` certificate authority is used to issue an ephemeral + client certificate to the Teleport Database Service when it connects to a + self-hosted database. + +Certificate authority rotation consists of several rotation phases. +The rotation phases have different effects depending on which CA is rotated. +This section will explain the differences and show you how to rotate your +database CAs. + +### `db` CA Rotation phases + +- `standby`: All operations have completed or haven't started yet. +- `init`: All Teleport cluster services are notified of the rotation. + A new `db` certificate authority is created in your cluster. + Remote trusted clusters must fetch the new CA. + `tctl auth sign` will issue database server certs with the new CA. +- `update_clients`: Ephemeral client certificates issued to the Teleport Proxy + Service will be issued by the new CA. +- `update_servers`: No effect during `db` CA rotation. +- `rollback`: The rotation was aborted and is rolling back to the old + certificate authority. + +### `db_client` CA Rotation phases + +- `standby`: All operations have completed or haven't started yet. +- `init`: All Teleport cluster services are notified of the rotation. + A new `db_client` certificate authority is created in your cluster. + `tctl auth sign` will output both the old and new certificate authorities + in its trusted CA output. +- `update_clients`: Ephemeral client certificates issued to the Teleport + Database Service will be issued by the new CA. +- `update_servers`: No effect during `db_client` CA rotation. +- `rollback`: The rotation was aborted and is rolling back to the old + certificate authority. + +### Rotation types + +There are two types of certificate authority rotation processes: + +- **Manual:** it is the cluster administrator's responsibility to transition + between each phase of the rotation while monitoring database connectivity. + Manual rotations are performed by providing the `--manual` flag and the + desired phase using the `--phase` flag to the `tctl auth rotate` command. +- **Semi-automatic:** Teleport automatically transitions between phases of the + rotation after some amount of time (known as a *grace period*) elapses. + +For both types of rotation, the cluster goes through the phases in the +following order: + +- `standby` -> `init` -> `update_clients` -> `update_servers` -> `standby` + +It does not make sense to use semi-automatic rotation when rotating the `db` +or `db_client` CA, because self-hosted databases are not automatically +reconfigured by the semi-automatic process. +Therefore this guide will only explain manual rotation. + +In manual mode we transition between phases while monitoring database +connectivity, and reconfigure databases with new CA certificates as needed. + +You will need to reconfigure any given self-hosted database only once during the +rotation: it does not need to be done at each step. +You can reconfigure your databases during any phase, although it's best to do +so during the `init` phase to avoid losing database access later in the rotation. + +If you lose connectivity to your databases during any phase prior to +`standby`, you can [roll back](#rollback) to the old certificate authority to +restore connectivity. + +## Step 1/4. Start the rotation + +Initiate manual rotation of the `db` or `db_client` certificate authority: +
+You should not rotate the `db` or `db_client` CA in parallel with other CA +rotations. +Rotating in parallel with other CA types, especially the `host` CA, is more +complicated, more likely to cause problems, and does not save you any effort. + +However, if you want to rotate both the `db` and `db_client` CA, then you can +(and should) save yourself some effort by rotating both of them at the same +time. + +You can rotate both `db` and `db_client` certificate authorities in parallel +by transitioning both CAs through each phase step. +Just use the `tctl auth rotate` command for each step with `--type=db` and +then `--type=db_client`. +
+ +```code +$ tctl auth rotate --manual --type= --phase=init +Updated rotation phase to "init". To check status use 'tctl status' +``` + +Use `tctl` to confirm that there is an active rotation in progress: +```code +$ tctl status +``` + + +If you are rotating the `db_client` CA and you do not want to lose access to +your self-hosted databases in the next phase, you should reconfigure your +databases now. +You do not need to reconfigure databases at this point if you are rotating only +the `db` CA, although there is no harm in doing so. + +Consult the appropriate +[Teleport Database Access Guide](../../database-access/guides) for your +databases before proceeding to the `update_clients` rotation phase. + + +## Step 2/4. Update clients + +Execute the transition from `init` to `update_clients`: + +```code +$ tctl auth rotate --manual --type= --phase=update_clients +# Updated rotation phase to "update_clients". To check status use 'tctl status' +``` + +If you are rotating the `db_client` CA, this is the phase where the Teleport +Database Service will start using client certificates issued by the new CA to +connect to databases. +You should verify that you can still access your databases after transitioning +to this phase. + +If you lose connectivity to your databases, you can reconfigure your +databases to restore access. +If that does not restore access or you are unable to reconfigure a database, +then [roll back](#rollback) to the old certificate authority. + +## Step 3/4. Update servers + +Execute the transition from `update_clients` to `update_servers`: + +```code +$ tctl auth rotate --manual --type= --phase=update_servers +# Updated rotation phase to "update_servers". To check status use 'tctl status' +``` + +This phase does not actually do anything for `db` or `db_client` CA rotation, +but it's required as part of the generalized Teleport CA rotation process before +we can finish the rotation. + +## Step 4/4. Finish the rotation + +If any of your self-hosted databases have not yet been reconfigured, you **will +lose access** to those databases after transitioning to the `standby` phase in +this final step. + +To avoid down time, consult the appropriate +[Teleport Database Access Guide](../../database-access/guides) and reconfigure +your databases before proceeding. +Otherwise, access may still be restored by reconfiguring your self-hosted +databases after this step. + +Execute the transition from `update_servers` to `standby`: + +```code +$ tctl auth rotate --manual --type= --phase=standby +# Updated rotation phase to "standby". To check status use 'tctl status' +``` + +## Rollback + +Rollback can only be performed before the rotation is completed, meaning before +the rotation transitions to `standby` from `update_servers`. + +The most common reason you would want to rollback if is if you cannot +reconfigure your databases. +If you have connectivity issues after reconfiguring a database, it's likely that +you simply misconfigured the database. +If you roll back the rotation, then you will have to manually revert any +database reconfigurations and start from step 1 to reattempt rotating your +certificate authorities, so consider this a method of last resort. + +First, enter the rollback phase with a manual phase transition: + +```code +$ tctl auth rotate --manual --type= --phase=rollback +# Updated rotation phase to "rollback". To check status use 'tctl status' +``` + +If you reconfigured any of your databases during the rotation, you will need +to reconfigure them again before transitioning to `standby` from the `rollback` +phase. + +Enter the standby phase manually: + +```code +$ tctl auth rotate --manual --type= --phase=standby +# Updated rotation phase to "standby". To check status use 'tctl status' +``` + +It is very unlikely that you will lose database connectivity during a `db` CA +rotation and need to rollback. +However, when rotating your `db_client` CA you will definitely lose access to +any database that was not reconfigured during the `init` rotation phase. +If you are unable to reconfigure those databases, then rolling back should +restore access to them immediately. + +## Further reading + +- How the [Teleport Certificate Authority](../../architecture/authentication.mdx) works. +- How [Teleport Database Access](../../database-access/architecture.mdx) works.