diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 3e16d0ebe..e76cca4aa 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -415,9 +415,11 @@ *** xref:manage:iceberg/about-iceberg-topics.adoc[] *** xref:manage:iceberg/specify-iceberg-schema.adoc[] *** xref:manage:iceberg/use-iceberg-catalogs.adoc[] +*** xref:manage:iceberg/rest-catalog/index.adoc[] +**** xref:manage:iceberg/iceberg-topics-aws-glue.adoc[AWS Glue] +**** xref:manage:iceberg/iceberg-topics-databricks-unity.adoc[Databricks Unity Catalog] +**** xref:manage:iceberg/redpanda-topics-iceberg-snowflake-catalog.adoc[Snowflake and Open Catalog] *** xref:manage:iceberg/query-iceberg-topics.adoc[] -*** xref:manage:iceberg/iceberg-topics-databricks-unity.adoc[Query Iceberg Topics with Databricks Unity Catalog] -*** xref:manage:iceberg/redpanda-topics-iceberg-snowflake-catalog.adoc[Query Iceberg Topics with Snowflake and Open Catalog] ** xref:manage:schema-reg/index.adoc[Schema Registry] *** xref:manage:schema-reg/schema-reg-overview.adoc[] *** xref:manage:schema-reg/schema-reg-ui.adoc[] diff --git a/modules/get-started/pages/whats-new-cloud.adoc b/modules/get-started/pages/whats-new-cloud.adoc index 3a6b95244..ca543de10 100644 --- a/modules/get-started/pages/whats-new-cloud.adoc +++ b/modules/get-started/pages/whats-new-cloud.adoc @@ -7,6 +7,14 @@ This page lists new features added to Redpanda Cloud. +== August 2025 + +=== Iceberg topics with AWS Glue + +A new xref:manage:iceberg/iceberg-topics-aws-glue.adoc[integration with AWS Glue Data Catalog] allows you to add Redpanda topics as Iceberg tables in your data lakehouse. The AWS Glue catalog integration is available in BYOC clusters with Redpanda version 25.2 and later. + +See xref:manage:iceberg/rest-catalog/index.adoc[] for supported Iceberg REST catalog integrations. + == July 2025 === Iceberg topics in Redpanda Cloud: GA diff --git a/modules/manage/pages/iceberg/iceberg-topics-aws-glue.adoc b/modules/manage/pages/iceberg/iceberg-topics-aws-glue.adoc new file mode 100644 index 000000000..8af4f52a1 --- /dev/null +++ b/modules/manage/pages/iceberg/iceberg-topics-aws-glue.adoc @@ -0,0 +1,6 @@ += Query Iceberg Topics using AWS Glue +:description: Add Redpanda topics as Iceberg tables that you can access through the AWS Glue Data Catalog. +:page-categories: Iceberg, Tiered Storage, Management, High Availability, Data Replication, Integration +:page-beta: true + +include::ROOT:manage:iceberg/iceberg-topics-aws-glue.adoc[tag=single-source] \ No newline at end of file diff --git a/modules/manage/pages/iceberg/rest-catalog/index.adoc b/modules/manage/pages/iceberg/rest-catalog/index.adoc new file mode 100644 index 000000000..d656620cc --- /dev/null +++ b/modules/manage/pages/iceberg/rest-catalog/index.adoc @@ -0,0 +1,3 @@ += Integrate with REST Catalogs +:description: Integrate Redpanda topics with managed Iceberg REST Catalogs. +:page-layout: index \ No newline at end of file diff --git a/modules/networking/pages/configure-private-service-connect-in-cloud-ui.adoc b/modules/networking/pages/configure-private-service-connect-in-cloud-ui.adoc index b0fe7aad6..512a0dfaa 100644 --- a/modules/networking/pages/configure-private-service-connect-in-cloud-ui.adoc +++ b/modules/networking/pages/configure-private-service-connect-in-cloud-ui.adoc @@ -3,4 +3,78 @@ :page-aliases: deploy:deployment-option/cloud/configure-private-service-connect-in-cloud-ui.adoc :env-byoc: true -include::networking:partial$psc-ui.adoc[] \ No newline at end of file +[NOTE] +==== + +* This guide is for configuring GCP Private Service Connect using the Redpanda Cloud UI. To configure and manage Private Service Connect on an existing cluster with *public* networking, you must use the xref:networking:gcp-private-service-connect.adoc[Cloud API for BYOC] or the xref:networking:dedicated/gcp/configure-psc-in-api.adoc[Cloud API for Dedicated]. +* The latest version of Redpanda GCP Private Service Connect (available March, 2025) supports zone affinity. This allows requests from Private Service Connect endpoints to stay within the same availability zone, avoiding additional networking costs. +* DEPRECATION: The original Redpanda GCP Private Service Connect is deprecated and will be removed in a future release. For more information, see xref:manage:maintenance.adoc#deprecated-features[Deprecated features]. +==== + + +The Redpanda GCP Private Service Connect service provides secure access to Redpanda Cloud from your own VPC network. Traffic over Private Service Connect does not go through the public internet because these connections are treated as their own private GCP service. While your VPC network has access to the Redpanda VPC network, Redpanda cannot access your VPC network. + +Consider using Private Service Connect if you have multiple VPC networks and could benefit from a more simplified approach to network management. + +[NOTE] +==== +* Each consumer VPC network can have one Private Service Connect endpoint connected to the Redpanda service attachment. +* Private Service Connect allows overlapping xref:networking:cidr-ranges.adoc[CIDR ranges] in VPC networks. +* The number of connections is limited only by your Redpanda xref:reference:tiers/index.adoc[usage tier]. Private Service Connect does not add extra connection limits. +* You control from which GCP projects connections are allowed. +==== + +== Requirements + +* Use the https://cloud.google.com/sdk/docs/install[gcloud^] command-line interface (CLI) to create the consumer-side resources, such as a consumer VPC network and forwarding rule, or to modify existing resources to use the Private Service Connect service attachment created for your cluster. +* The consumer VPC network must be in the same region as your Redpanda cluster. + +== Enable Private Service Connect for existing clusters + +. In the Redpanda Cloud UI, open your https://cloud.redpanda.com/clusters[cluster^], and click **Cluster settings**. +. Under Private Service Connect, click **Enable**. +ifdef::env-byoc[] +. For xref:get-started:cluster-types/byoc/gcp/vpc-byo-gcp.adoc[BYOVPC clusters], you need a PSC NAT subnet with `purpose` set to `PRIVATE_SERVICE_CONNECT`. You also need to create VPC network firewall rules to allow Private Service Connect traffic. You can use the `gcloud` CLI: ++ +NOTE: The firewall rules support up to 20 Redpanda brokers. If you have more than 20 brokers, or for help enabling Private Service Connect, contact https://support.redpanda.com/hc/en-us/requests/new[Redpanda support^]. ++ +[source,bash] +---- +gcloud compute networks subnets create \ + --project= \ + --network= \ + --region= \ + --range= \ + --purpose=PRIVATE_SERVICE_CONNECT +---- ++ +[source,bash] +---- +gcloud compute firewall-rules create redpanda-psc-ingress \ + --description="Allow access to Redpanda PSC endpoints" \ + --network="" \ + --project="" \ + --direction="INGRESS" \ + --target-tags="redpanda-node" \ + --source-ranges="10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/10" \ + --allow="tcp:30181,tcp:30282,tcp:30292,tcp:31004,tcp:31082-31101,tcp:31182-31201,tcp:31282-31301,tcp:32092-32111,tcp:32192-32211,tcp:32292-32311" +---- ++ +Provide your values for the following placeholders: ++ +- ``: The name of the PSC NAT subnet. +- ``: The host GCP project ID. +- ``: The name of the VPC network being used for your Redpanda Cloud cluster. +- ``: The region of the Redpanda Cloud cluster. +- ``: The CIDR range of the subnet. The mask should be at least `/29`. Each Private Service Connect connection takes up one IP address from the PSC NAT subnet, so the CIDR must be able to accommodate all projects from which connections to the service attachment will be issued. ++ +See the GCP documentation for https://cloud.google.com/vpc/docs/configure-private-service-connect-producer#add-subnet-psc[creating a subnet for Private Service Connect^]. +endif::[] +. For the accepted consumers list, you need the GCP project IDs from which incoming connections will be accepted. +. It may take several minutes for your cluster to update. When the update is complete, the Private Service Connect status in **Cluster settings** changes from **In progress** to **Enabled**. + +include::networking:partial$psc-ui.adoc[] + +== Disable Private Service Connect + +In **Cluster settings**, click **Disable**. Existing connections are closed after it is disabled. To connect using Private Service Connect again, you must re-enable it. \ No newline at end of file diff --git a/modules/networking/pages/dedicated/gcp/configure-psc-in-api.adoc b/modules/networking/pages/dedicated/gcp/configure-psc-in-api.adoc index 21700160d..f93d9ef15 100644 --- a/modules/networking/pages/dedicated/gcp/configure-psc-in-api.adoc +++ b/modules/networking/pages/dedicated/gcp/configure-psc-in-api.adoc @@ -15,26 +15,11 @@ Copy and store the resource group ID (UUID) from the URL in the browser. export RESOURCE_GROUP_ID= ---- -. Create VPC firewall rules to allow Private Service Connect traffic. Use the `gcloud` CLI to create the firewall rules: -+ -NOTE: The firewall rules support up to 20 Redpanda brokers. If you have more than 20 brokers, or for help enabling Private Service Connect, contact https://support.redpanda.com/hc/en-us/requests/new[Redpanda support^]. -+ -``` -gcloud compute firewall-rules create redpanda-psc \ - --description="Allow access to Redpanda PSC endpoints" \ - --network="" \ - --project="" \ - --direction="INGRESS" \ - --target-tags="redpanda-node" \ - --source-ranges="10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/10" \ - --allow="tcp:30181,tcp:30282,tcp:30292,tcp:31004,tcp:31082-31101,tcp:31182-31201,tcp:31282-31301,tcp:32092-32111,tcp:32192-32211,tcp:32292-32311" -``` - . Make a request to the xref:api:ROOT:cloud-controlplane-api.adoc#post-/v1/networks[`POST /v1/networks`] endpoint to create a network. + [,bash] ---- -network_post_body=`cat << EOF +NETWORK_POST_BODY=`cat << EOF { "cloud_provider": "CLOUD_PROVIDER_GCP", "cluster_type": "TYPE_DEDICATED", @@ -54,7 +39,7 @@ EOF` curl -vv -X POST \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $AUTH_TOKEN" \ --d "$network_post_body" $PUBLIC_API_ENDPOINT/v1/networks +-d "$NETWORK_POST_BODY" $PUBLIC_API_ENDPOINT/v1/networks ---- + Replace the following placeholder variables for the request body: @@ -94,7 +79,7 @@ export CLUSTER_POST_BODY=`cat << EOF "gcp_private_service_connect": { "enabled": true, "consumer_accept_list": - }, + } } EOF` @@ -110,7 +95,7 @@ curl -vv -X POST \ - ``: Provide the list of GCP zones where the brokers will be deployed. Format: `["", "", ""]` - ``: Choose a Redpanda Cloud cluster tier. For example, `tier-1-gcp-v2-x86`. - ``: Choose the Redpanda Cloud version. -- ``: The list of IDs of GCP projects from which Private Service Connect connection requests are accepted. Format: `[{"source": ""}, {"source": ""}, {"source": ""}]` +- ``: The list of IDs of GCP projects from which Private Service Connect connection requests are accepted. Format: `[{"source": ""}, {"source": ""}, {"source": ""}]` -- == Enable Private Service Connect on an existing cluster @@ -133,7 +118,7 @@ CLUSTER_PATCH_BODY=`cat << EOF { "gcp_private_service_connect": { "enabled": true, - "consumer_accept_list": + "consumer_accept_list": } } EOF` @@ -145,7 +130,7 @@ curl -v -X PATCH \ + Replace the following placeholder: + -``: a JSON list specifying the projects from which incoming connections will be accepted. All other sources. For example, `[{"source": "consumer-project-ID-1"},{"source": "consumer-project-ID-2"}]`. +``: A JSON list specifying the projects from which incoming connections will be accepted. All other sources are rejected. For example, `[{"source": "consumer-project-ID-1"},{"source": "consumer-project-ID-2"}]`. + Wait for the cluster to apply the new configuration (around 15 minutes). The Private Service Connect attachment is available when the cluster update is complete. To monitor the service attachment creation, run the following `gcloud` command with the project ID: + @@ -155,4 +140,23 @@ gcloud compute service-attachments list --project '' ---- -include::networking:partial$psc-api2.adoc[] \ No newline at end of file +include::networking:partial$psc-ui.adoc[] + +== Disable Private Service Connect + +Make a xref:api:ROOT:cloud-controlplane-api.adoc#patch-/v1/clusters/-cluster.id-[`PATCH /v1/clusters/{cluster.id}`] request to update the cluster to disable Private Service Connect. + +[,bash] +---- +CLUSTER_PATCH_BODY=`cat << EOF +{ + "gcp_private_service_connect": { + "enabled": false + } +} +EOF` +curl -v -X PATCH \ +-H "Content-Type: application/json" \ +-H "Authorization: Bearer $AUTH_TOKEN" \ +-d "$CLUSTER_PATCH_BODY" $PUBLIC_API_ENDPOINT/v1/clusters/$CLUSTER_ID +---- \ No newline at end of file diff --git a/modules/networking/pages/dedicated/gcp/configure-psc-in-ui.adoc b/modules/networking/pages/dedicated/gcp/configure-psc-in-ui.adoc index 7ae01b511..85af2f836 100644 --- a/modules/networking/pages/dedicated/gcp/configure-psc-in-ui.adoc +++ b/modules/networking/pages/dedicated/gcp/configure-psc-in-ui.adoc @@ -2,4 +2,78 @@ :description: Set up GCP Private Service Connect in the Redpanda Cloud UI. :env-dedicated: true -include::networking:partial$psc-ui.adoc[] \ No newline at end of file +[NOTE] +==== + +* This guide is for configuring GCP Private Service Connect using the Redpanda Cloud UI. To configure and manage Private Service Connect on an existing cluster with *public* networking, you must use the xref:networking:gcp-private-service-connect.adoc[Cloud API for BYOC] or the xref:networking:dedicated/gcp/configure-psc-in-api.adoc[Cloud API for Dedicated]. +* The latest version of Redpanda GCP Private Service Connect (available March, 2025) supports AZ affinity. This allows requests from Private Service Connect endpoints to stay within the same availability zone, avoiding additional networking costs. +* DEPRECATION: The original Redpanda GCP Private Service Connect is deprecated and will be removed in a future release. For more information, see xref:manage:maintenance.adoc#deprecated-features[Deprecated features]. +==== + + +The Redpanda GCP Private Service Connect service provides secure access to Redpanda Cloud from your VPC network. Traffic over Private Service Connect remains within GCP's private network, avoiding the public internet. Your VPC network can access the Redpanda VPC network, but Redpanda cannot access your VPC network. + +Consider using Private Service Connect if you have multiple VPC networks and could benefit from a more simplified approach to network management. + +[NOTE] +==== +* Each consumer VPC network can have one Private Service Connect endpoint connected to the Redpanda service attachment. +* Private Service Connect allows overlapping xref:networking:cidr-ranges.adoc[CIDR ranges] in VPC networks. +* The number of connections is limited only by your Redpanda usage tier. Private Service Connect does not add extra connection limits. +* You control from which GCP projects connections are allowed. +==== + +== Prerequisites + +* Use the https://cloud.google.com/sdk/docs/install[gcloud^] command-line interface (CLI) to create the consumer-side resources, such as a consumer VPC network and forwarding rule, or to modify existing resources to use the Private Service Connect service attachment created for your cluster. +* The consumer VPC network must be in the same region as your Redpanda cluster. + +== Enable Private Service Connect for existing clusters + +. In the Redpanda Cloud UI, open your https://cloud.redpanda.com/clusters[cluster^], and click **Cluster settings**. +. Under Private Service Connect, click **Enable**. +ifdef::env-byoc[] +. For xref:get-started:cluster-types/byoc/gcp/vpc-byo-gcp.adoc[BYOVPC clusters], you need a NAT subnet with `purpose` set to `PRIVATE_SERVICE_CONNECT`. You also need to create VPC network firewall rules to allow Private Service Connect traffic. You can use the `gcloud` CLI: ++ +NOTE: The firewall rules support up to 20 Redpanda brokers. If you have more than 20 brokers, or for help enabling Private Service Connect, contact https://support.redpanda.com/hc/en-us/requests/new[Redpanda support^]. ++ +[,bash] +---- +gcloud compute networks subnets create \ + --project= \ + --network= \ + --region= \ + --range= \ + --purpose=PRIVATE_SERVICE_CONNECT +---- ++ +[,bash] +---- +gcloud compute firewall-rules create redpanda-psc-ingress \ + --description="Allow access to Redpanda PSC endpoints" \ + --network="" \ + --project="" \ + --direction="INGRESS" \ + --target-tags="redpanda-node" \ + --source-ranges="10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/10" \ + --allow="tcp:30181,tcp:30282,tcp:30292,tcp:31004,tcp:31082-31101,tcp:31182-31201,tcp:31282-31301,tcp:32092-32111,tcp:32192-32211,tcp:32292-32311" +---- ++ +Provide your values for the following placeholders: ++ +- ``: The name of the NAT subnet. +- ``: The host GCP project ID. +- ``: The name of the VPC network being used for your Redpanda Cloud cluster. +- ``: The region of the Redpanda Cloud cluster. +- ``: The CIDR range of the subnet. The mask should be at least `/29`. Each Private Service Connect connection takes up one IP address from the NAT subnet, so the CIDR must be able to accommodate all projects from which connections to the service attachment will be issued. ++ +See the GCP documentation for https://cloud.google.com/vpc/docs/configure-private-service-connect-producer#add-subnet-psc[creating a subnet for Private Service Connect^]. +endif::[] +. For the accepted consumers list, you need the GCP project IDs from which incoming connections will be accepted. +. It may take several minutes for your cluster to update. When the update is complete, the Private Service Connect status in **Cluster settings** changes from **In progress** to **Enabled**. + +include::networking:partial$psc-ui.adoc[] + +== Disable Private Service Connect + +In **Cluster settings**, click **Disable**. Existing connections are closed after GCP Private Service Connect is disabled. To connect using Private Service Connect again, you must re-enable the service. \ No newline at end of file diff --git a/modules/networking/pages/gcp-private-service-connect.adoc b/modules/networking/pages/gcp-private-service-connect.adoc index ebfb3b353..240b14980 100644 --- a/modules/networking/pages/gcp-private-service-connect.adoc +++ b/modules/networking/pages/gcp-private-service-connect.adoc @@ -22,21 +22,21 @@ export RESOURCE_GROUP_ID= + [,bash] ---- -gcloud compute networks subnets create \ +gcloud compute networks subnets create \ --project= \ --network= \ --region= \ - --range= \ + --range= \ --purpose=PRIVATE_SERVICE_CONNECT ---- + Provide your values for the following placeholders: + -- ``: The name of the NAT subnet. +- ``: The name of the NAT subnet. - ``: The host GCP project ID. - ``: The name of the VPC being used for your Redpanda Cloud cluster. The name is used to identify this network in the Cloud UI. - ``: The GCP region of the Redpanda Cloud cluster. -- ``: The CIDR range of the subnet. The mask should be at least `/29`. Each Private Service Connect connection takes up one IP address from the NAT subnet, so the CIDR must be able to accommodate all projects from which connections to the service attachment will be issued. +- ``: The CIDR range of the subnet. The mask should be at least `/29`. Each Private Service Connect connection takes up one IP address from the NAT subnet, so the CIDR must be able to accommodate all projects from which connections to the service attachment will be issued. + See the GCP documentation for https://cloud.google.com/vpc/docs/configure-private-service-connect-producer#add-subnet-psc[creating a subnet for Private Service Connect^]. @@ -59,7 +59,7 @@ gcloud compute firewall-rules create redpanda-psc \ + [,bash] ---- -network_post_body=`cat << EOF +NETWORK_POST_BODY=`cat << EOF { "cloud_provider": "CLOUD_PROVIDER_GCP", "cluster_type": "TYPE_BYOC", @@ -79,7 +79,7 @@ EOF` curl -vv -X POST \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $AUTH_TOKEN" \ --d "$network_post_body" $PUBLIC_API_ENDPOINT/v1/networks +-d "$NETWORK_POST_BODY" $PUBLIC_API_ENDPOINT/v1/networks ---- + Replace the following placeholder variables for the request body: @@ -128,13 +128,13 @@ export CLUSTER_POST_BODY=`cat << EOF "secondary_ipv4_range_services": { "name": "" }, "k8s_master_ipv4_range": "" }, - "psc_nat_subnet_name": "" + "psc_nat_subnet_name": "", "agent_service_account": { "email": "" }, "connector_service_account": { "email": "" }, "console_service_account": { "email": "" }, "redpanda_cluster_service_account": { "email": "" }, "gke_service_account": { "email": "" }, - "tiered_storage_bucket": { "name" : "" }, + "tiered_storage_bucket": { "name" : "" } } } } @@ -154,12 +154,12 @@ Replace the following placeholders for the request body. Variables with a `byovp - ``: Provide the list of GCP zones where the brokers will be deployed. Format: `["", "", ""]` - ``: Choose a Redpanda Cloud cluster tier. For example, `tier-1-gcp-v2-x86`. - ``: Choose the Redpanda Cloud version. -- ``: The list of IDs of GCP projects from which Private Service Connect connection requests are accepted. Format: `[{"source": ""}, {"source": ""}, {"source": ""}]` +- ``: The list of IDs of GCP projects from which Private Service Connect connection requests are accepted. Format: `[{"source": ""}, {"source": ""}, {"source": ""}]` - ``: The name of the GCP subnet that was created for the cluster. - ``: The name of the IPv4 range designated for K8s pods. - ``: The name of the IPv4 range designated for services. - ``: The master IPv4 range. -- ``: The name of the GCP subnet that was created for Private Service Connect NAT. +- ``: The name of the GCP subnet that was created for Private Service Connect NAT. - ``: The email for the agent service account. - ``: The email for the connectors service account. - ``: The email for the console service account. @@ -182,7 +182,7 @@ export CLUSTER_ID= . For a *BYOC cluster*: + -- -- Run `rpk cloud byoc gcp apply` to ensure that the PSC subnets are created in your BYOC cluster. +- Run `rpk cloud byoc gcp apply` to ensure that the PSC NAT subnets are created in your BYOC cluster. ```bash rpk cloud byoc gcp apply --redpanda-id="${CLUSTER_ID}" --project-id='' ``` @@ -202,9 +202,7 @@ For a *BYOVPC cluster*: ```bash rpk cloud byoc gcp apply --redpanda-id="${CLUSTER_ID}" --project-id='' ``` --- - -. Make a request to the xref:api:ROOT:cloud-controlplane-api.adoc#patch-/v1/clusters/-cluster.id-[`PATCH /v1/clusters/{cluster.id}`] endpoint to update the cluster to include the newly-created Private Service Connect NAT subnet. +- Make a request to the xref:api:ROOT:cloud-controlplane-api.adoc#patch-/v1/clusters/-cluster.id-[`PATCH /v1/clusters/{cluster.id}`] endpoint to update the cluster to include the newly-created Private Service Connect NAT subnet. + [,bash] ---- @@ -226,7 +224,8 @@ curl -v -X PATCH \ + Replace the following placeholder: + -``: The name of the Private Service Connect NAT subnet. Use the fully-qualified name, for example `"projects//regions//subnetworks/"`. +``: The name of the Private Service Connect NAT subnet. Use the fully-qualified name, for example `"projects//regions//subnetworks/"`. +-- . Make a xref:api:ROOT:cloud-controlplane-api.adoc#patch-/v1/clusters/-cluster.id-[`PATCH /v1/clusters/{cluster.id}`] request to update the cluster to enable Private Service Connect. + @@ -236,7 +235,7 @@ CLUSTER_PATCH_BODY=`cat << EOF { "gcp_private_service_connect": { "enabled": true, - "consumer_accept_list": + "consumer_accept_list": } } EOF` @@ -248,7 +247,7 @@ curl -v -X PATCH \ + Replace the following placeholder: + -``: a JSON list specifying the projects from which incoming connections will be accepted. All other sources. For example, `[{"source": "consumer-project-ID-1"},{"source": "consumer-project-ID-2"}]`. +``: A JSON list specifying the projects from which incoming connections will be accepted. All other sources are rejected. For example, `[{"source": "consumer-project-ID-1"},{"source": "consumer-project-ID-2"}]`. + Wait for the cluster to apply the new configuration (around 15 minutes). The Private Service Connect attachment is available when the cluster update is complete. To monitor the service attachment creation, run the following `gcloud` command with the project ID: + @@ -257,5 +256,24 @@ Wait for the cluster to apply the new configuration (around 15 minutes). The Pri gcloud compute service-attachments list --project '' ---- -include::networking:partial$psc-api2.adoc[] +include::networking:partial$psc-ui.adoc[] + +== Disable Private Service Connect + +Make a xref:api:ROOT:cloud-controlplane-api.adoc#patch-/v1/clusters/-cluster.id-[`PATCH /v1/clusters/{cluster.id}`] request to update the cluster to disable Private Service Connect. + +[,bash] +---- +CLUSTER_PATCH_BODY=`cat << EOF +{ + "gcp_private_service_connect": { + "enabled": false + } +} +EOF` +curl -v -X PATCH \ +-H "Content-Type: application/json" \ +-H "Authorization: Bearer $AUTH_TOKEN" \ +-d "$CLUSTER_PATCH_BODY" $PUBLIC_API_ENDPOINT/v1/clusters/$CLUSTER_ID +---- diff --git a/modules/networking/partials/psc-api.adoc b/modules/networking/partials/psc-api.adoc index 5208add9a..a9ac4e0a6 100644 --- a/modules/networking/partials/psc-api.adoc +++ b/modules/networking/partials/psc-api.adoc @@ -1,27 +1,29 @@ [NOTE] ==== -* This guide is for configuring GCP Private Service Connect using the Redpanda Cloud API. To configure and manage Private Service Connect on an existing *public* cluster, you must use the Cloud API. See xref:networking:configure-private-service-connect-in-cloud-ui.adoc[Configure Private Service Connect in the Cloud UI] to set up the endpoint service using the Redpanda Cloud UI. +* This guide is for configuring GCP Private Service Connect using the Redpanda Cloud API. To configure and manage Private Service Connect on an existing cluster with *public* networking, you must use the Cloud API. See xref:networking:configure-private-service-connect-in-cloud-ui.adoc[Configure Private Service Connect in the Cloud UI] to set up the endpoint service using the Redpanda Cloud UI. * The latest version of Redpanda GCP Private Service Connect (available March, 2025) supports AZ affinity. This allows requests from Private Service Connect endpoints to stay within the same availability zone, avoiding additional networking costs. * DEPRECATION: The original Redpanda GCP Private Service Connect is deprecated and will be removed in a future release. For more information, see xref:manage:maintenance.adoc#deprecated-features[Deprecated features]. ==== -The Redpanda GCP Private Service Connect service provides secure access to Redpanda Cloud from your own VPC. Traffic over Private Service Connect does not go through the public internet because a Private Service Connect connection is treated as its own private GCP service. While your VPC has access to the Redpanda VPC, Redpanda cannot access your VPC. -Consider using Private Service Connect if you have multiple VPCs and could benefit from a more simplified approach to network management. +The Redpanda GCP Private Service Connect service provides secure access to Redpanda Cloud from your VPC network. Traffic over Private Service Connect remains within GCP's private network, avoiding the public internet. Your VPC network can access the Redpanda VPC network, but Redpanda cannot access your VPC network. + +Consider using Private Service Connect if you have multiple VPC networks and could benefit from a more simplified approach to network management. [NOTE] ==== -* Each client VPC can have one endpoint connected to Private Service Connect. +* Each consumer VPC network can have one Private Service Connect endpoint connected to the Redpanda service attachment. * Private Service Connect allows overlapping xref:networking:cidr-ranges.adoc[CIDR ranges] in VPC networks. -* The number of connections is limited only by your Redpanda usage tier. Private Service Connect does not add extra connection limits. +* The number of connections is limited only by your Redpanda xref:reference:tiers/index.adoc[usage tier]. Private Service Connect does not add extra connection limits. * You control from which GCP projects connections are allowed. ==== -== Requirements +== Prerequisites * In this guide, you use the xref:manage:api/cloud-api-overview.adoc[Redpanda Cloud API] to enable the Redpanda endpoint service for your clusters. Follow the steps on this page to <>. * Use the https://cloud.google.com/sdk/docs/install[gcloud^] command-line interface (CLI) to create the consumer-side resources, such as a VPC and forwarding rule, or to modify existing resources to use the Private Service Connect attachment created for your cluster. +* The consumer VPC network must be in the same region as your Redpanda cluster. == Get a Cloud API access token diff --git a/modules/networking/partials/psc-api2.adoc b/modules/networking/partials/psc-api2.adoc deleted file mode 100644 index 6539e9815..000000000 --- a/modules/networking/partials/psc-api2.adoc +++ /dev/null @@ -1,52 +0,0 @@ -== Deploy consumer-side resources - -For each VPC network, you must complete the following steps to successfully connect to the service and use Kafka API and other Redpanda services such as HTTP Proxy. - -. In **Cluster settings**, copy the **Service attachment URL** under **Private Service Connect**. Use this URL to create the Private Service Connect endpoint in GCP. - -. Create a private DNS zone. Use the cluster **DNS zone** value as the DNS name. -+ -[,bash] ----- -gcloud dns --project= managed-zones create --description="" --dns-name="" --visibility="private" --networks="" ----- - -. In the newly-created DNS zone, create a wildcard DNS record using the cluster **DNS record** value. -+ -[,bash] ----- -gcloud dns --project= record-sets create '*.' --zone="" --type="A" --ttl="300" --rrdatas="" ----- - -. Confirm that your GCP VPC firewall allows traffic to and from the Private Service Connect forwarding rule IP address, on the expected ports. - -== Access Redpanda services through VPC endpoint - -After you have enabled Private Service Connect for your cluster, your connection URLs are available in the *How to Connect* section of the cluster overview in the Redpanda Cloud UI. - -include::networking:partial$private-links-access-rp-services-through-vpc.adoc[] - -== Test the connection - -You can test the Private Service Connect connection from any VM or container in the consumer VPC. If configuring a client isn't possible right away, you can do these checks using `rpk` or curl: - -include::networking:partial$private-links-test-connection.adoc[] - -== Disable Private Service Connect - -Make a xref:api:ROOT:cloud-controlplane-api.adoc#patch-/v1/clusters/-cluster.id-[`PATCH /v1/clusters/{cluster.id}`] request to update the cluster to disable Private Service Connect. - -[,bash] ----- -CLUSTER_PATCH_BODY=`cat << EOF -{ - "gcp_private_service_connect": { - "enabled": false - } -} -EOF` -curl -v -X PATCH \ --H "Content-Type: application/json" \ --H "Authorization: Bearer $AUTH_TOKEN" \ --d "$CLUSTER_PATCH_BODY" $PUBLIC_API_ENDPOINT/v1/clusters/$CLUSTER_ID ----- \ No newline at end of file diff --git a/modules/networking/partials/psc-ui.adoc b/modules/networking/partials/psc-ui.adoc index d5d9d41fd..a58d6f168 100644 --- a/modules/networking/partials/psc-ui.adoc +++ b/modules/networking/partials/psc-ui.adoc @@ -1,97 +1,63 @@ -[NOTE] -==== - -* This guide is for configuring GCP Private Service Connect using the Redpanda Cloud UI. To configure and manage Private Service Connect on an existing *public* cluster, you must use the xref:networking:gcp-private-service-connect.adoc[Cloud API for BYOC] or the xref:networking:dedicated/gcp/configure-psc-in-api.adoc[Cloud API for Dedicated]. -* The latest version of Redpanda GCP Private Service Connect (available March, 2025) supports AZ affinity. This allows requests from Private Service Connect endpoints to stay within the same availability zone, avoiding additional networking costs. -* DEPRECATION: The original Redpanda GCP Private Service Connect is deprecated and will be removed in a future release. For more information, see xref:manage:maintenance.adoc#deprecated-features[Deprecated features]. -==== - - -The Redpanda GCP Private Service Connect service provides secure access to Redpanda Cloud from your own VPC. Traffic over Private Service Connect does not go through the public internet because these connections are treated as their own private GCP service. While your VPC has access to the Redpanda VPC, Redpanda cannot access your VPC. - -Consider using the endpoint services if you have multiple VPCs and could benefit from a more simplified approach to network management. - -[NOTE] -==== -* Each client VPC can have one endpoint connected to Private Service Connect. -* Private Service Connect allows overlapping xref:networking:cidr-ranges.adoc[CIDR ranges] in VPC networks. -* The number of connections is limited only by your Redpanda usage tier. Private Service Connect does not add extra connection limits. -* You control from which GCP projects connections are allowed. -==== +== Deploy consumer-side resources -== Requirements +For each consumer VPC network, you must complete the following steps to successfully connect to the service attachment and use the Kafka API and other Redpanda services, such as HTTP Proxy. -* Use the https://cloud.google.com/sdk/docs/install[gcloud^] command-line interface (CLI) to create the consumer-side resources, such as a client VPC and forwarding rule, or to modify existing resources to use the Private Service Connect attachment created for your cluster. -* The client VPC must be in the same region as your Redpanda cluster. +. In **Cluster settings**, copy the **DNS zone** and **Service attachment URL** under **Private Service Connect**. Use this URL to create the Private Service Connect endpoint in GCP. -== Enable endpoint service for existing clusters +. Get the name of the consumer VPC network and the subnet ``, where the Private Service Connect endpoint forwarding rule will be created. -. In the Redpanda Cloud UI, open your https://cloud.redpanda.com/clusters[cluster^], and click **Cluster settings**. -. Under Private Service Connect, click **Enable**. -ifdef::env-byoc[] -. For xref:get-started:cluster-types/byoc/gcp/vpc-byo-gcp.adoc[BYOVPC clusters], you need a NAT subnet with `purpose` set to `PRIVATE_SERVICE_CONNECT`. You also need to create VPC firewall rules to allow Private Service Connect traffic. You can use the `gcloud` CLI: -+ -NOTE: The firewall rules support up to 20 Redpanda brokers. If you have more than 20 brokers, or for help enabling Private Service Connect, contact https://support.redpanda.com/hc/en-us/requests/new[Redpanda support^]. +. Create a Private Service Connect IP address for the endpoint: + [,bash] +---- +gcloud compute addresses create --subnet= --addresses= --region= ---- -gcloud compute networks subnets create \ - --project= \ - --network= \ - --region= \ - --range= \ - --purpose=PRIVATE_SERVICE_CONNECT + +. Create the Private Service Connect endpoint forwarding rule: ++ +[,bash] +---- +gcloud compute forwarding-rules create --region= --network= --address= --target-service-attachment= ---- + +. Create firewall rules allowing egress traffic to the Private Service Connect endpoint: + [,bash] ---- -gcloud compute firewall-rules create redpanda-psc \ - --description="Allow access to Redpanda PSC endpoints" \ - --network="" \ - --project="" \ - --direction="INGRESS" \ - --target-tags="redpanda-node" \ - --source-ranges="10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/10" \ - --allow="tcp:30181,tcp:30282,tcp:30292,tcp:31004,tcp:31082-31101,tcp:31182-31201,tcp:31282-31301,tcp:32092-32111,tcp:32192-32211,tcp:32292-32311" +gcloud compute firewall-rules create redpanda-psc-egress \ + --description="Allow access to Redpanda PSC endpoint" \ + --network="" \ + --direction="EGRESS" \ + --destination-ranges= \ + --allow="tcp:443,tcp:30081,tcp:30282,tcp:30292,tcp:32092-32141,tcp:35082-35131,tcp:32192-32241,tcp:35182-35231,tcp:32292-32341,tcp:35282-35331" ---- -+ -Provide your values for the following placeholders: -+ -- ``: The name of the NAT subnet. -- ``: The host GCP project ID. -- ``: The name of the VPC being used for your Redpanda Cloud cluster. -- ``: The region of the Redpanda Cloud cluster. -- ``: The CIDR range of the subnet. The mask should be at least `/29`. Each Private Service Connect connection takes up one IP address from the NAT subnet, so the CIDR must be able to accommodate all projects from which connections to the service attachment will be issued. -+ -See the GCP documentation for https://cloud.google.com/vpc/docs/configure-private-service-connect-producer#add-subnet-psc[creating a subnet for Private Service Connect^]. -endif::[] -. For the accepted consumers list, you need the GCP project IDs from which incoming connections will be accepted. -. It may take several minutes for your cluster to update. When the update is complete, the Private Service Connect status in **Cluster settings** changes from **In progress** to **Enabled**. - -== Deploy consumer-side resources -For each VPC network, you must complete the following steps to successfully connect to the service and use Kafka API and other Redpanda services such as HTTP Proxy. - -. In **Cluster settings**, copy the **Service attachment URL** under **Private Service Connect**. Use this URL to create the Private Service Connect endpoint in GCP. - -. Create a private DNS zone. Use the cluster **DNS zone** value as the DNS name. +. Create a private DNS zone. Use the cluster **DNS zone** value as the DNS name: + [,bash] ---- -gcloud dns --project= managed-zones create --description="" --dns-name="" --visibility="private" --networks="" +gcloud dns managed-zones create \ + --project= \ + --description="Redpanda Private Service Connect DNS zone" \ + --dns-name="" \ + --visibility="private" \ + --networks="" ---- -. In the newly-created DNS zone, create a wildcard DNS record using the cluster **DNS record** value. +. In the newly-created DNS zone, create a wildcard DNS record using the cluster **DNS record** value: + [,bash] ---- -gcloud dns --project= record-sets create '*.' --zone="" --type="A" --ttl="300" --rrdatas="" +gcloud dns record-sets create '*.' \ + --project= \ + --zone="" \ + --type="A" \ + --ttl="300" \ + --rrdatas="" ---- -. Confirm that your GCP VPC firewall allows traffic to and from the Private Service Connect forwarding rule IP address, on the expected ports. - -== Access Redpanda services through VPC endpoint +== Access Redpanda services through Private Service Connect endpoint After you have enabled Private Service Connect for your cluster, your connection URLs are available in the *How to Connect* section of the cluster overview in the Redpanda Cloud UI. @@ -99,10 +65,6 @@ include::networking:partial$private-links-access-rp-services-through-vpc.adoc[] == Test the connection -You can test the connection to the endpoint service from any VM or container in the consumer VPC. If configuring a client isn't possible right away, you can do these checks using `rpk` or cURL: - -include::networking:partial$private-links-test-connection.adoc[] - -== Disable endpoint service +You can test the Private Service Connect connection from any VM or container in the consumer VPC. If configuring a client isn't possible right away, you can do these checks using `rpk` or curl: -In **Cluster settings**, click **Disable**. Existing connections are closed after GCP Private Service Connect is disabled. To connect using Private Service Connect again, you must re-enable the service. \ No newline at end of file +include::networking:partial$private-links-test-connection.adoc[] \ No newline at end of file diff --git a/modules/reference/pages/properties/cluster-properties.adoc b/modules/reference/pages/properties/cluster-properties.adoc index d2f92d3cb..4e9de17cf 100644 --- a/modules/reference/pages/properties/cluster-properties.adoc +++ b/modules/reference/pages/properties/cluster-properties.adoc @@ -8,4 +8,4 @@ NOTE: Some properties require a cluster restart for updates to take effect. This == Cluster configuration -include::ROOT:reference:properties/cluster-properties.adoc[tags=audit_enabled;audit_excluded_principals;audit_excluded_topics;data_transforms_enabled;data_transforms_logging_line_max_bytes;iceberg_catalog_type;iceberg_delete;iceberg_enabled;iceberg_rest_catalog_client_id;iceberg_rest_catalog_client_secret;iceberg_rest_catalog_token;iceberg_rest_catalog_authentication_mode;iceberg_rest_catalog_base_location;iceberg_rest_catalog_endpoint;iceberg_rest_catalog_oauth2_server_uri;iceberg_rest_catalog_prefix;iceberg_rest_catalog_request_timeout_ms;iceberg_default_partition_spec;iceberg_invalid_record_action;iceberg_target_lag_ms;iceberg_rest_catalog_trust;iceberg_rest_catalog_crl;data_transforms_per_function_memory_limit;data_transforms_binary_max_size;log_segment_ms;http_authentication;iceberg_catalog_base_location;default_topic_replications;minimum_topic_replications;oidc_discovery_url;oidc_principal_mapping;oidc_token_audience;sasl_mechanisms;tls_min_version;audit_log_num_partitions;data_transforms_per_core_memory_reservation;iceberg_disable_snapshot_tagging;enable_consumer_group_metrics] \ No newline at end of file +include::ROOT:reference:properties/cluster-properties.adoc[tags=audit_enabled;audit_excluded_principals;audit_excluded_topics;data_transforms_enabled;data_transforms_logging_line_max_bytes;iceberg_catalog_type;iceberg_delete;iceberg_enabled;iceberg_rest_catalog_client_id;iceberg_rest_catalog_client_secret;iceberg_rest_catalog_token;iceberg_rest_catalog_authentication_mode;iceberg_rest_catalog_base_location;iceberg_rest_catalog_endpoint;iceberg_rest_catalog_oauth2_server_uri;iceberg_rest_catalog_warehouse;iceberg_rest_catalog_request_timeout_ms;iceberg_default_partition_spec;iceberg_invalid_record_action;iceberg_target_lag_ms;iceberg_rest_catalog_trust;iceberg_rest_catalog_crl;data_transforms_per_function_memory_limit;data_transforms_binary_max_size;log_segment_ms;http_authentication;iceberg_catalog_base_location;default_topic_replications;minimum_topic_replications;oidc_discovery_url;oidc_principal_mapping;oidc_token_audience;sasl_mechanisms;tls_min_version;audit_log_num_partitions;data_transforms_per_core_memory_reservation;iceberg_disable_snapshot_tagging;enable_consumer_group_metrics] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 2b7517fa6..210faa794 100644 --- a/package-lock.json +++ b/package-lock.json @@ -884,9 +884,9 @@ } }, "node_modules/@redpanda-data/docs-extensions-and-macros": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/@redpanda-data/docs-extensions-and-macros/-/docs-extensions-and-macros-4.7.0.tgz", - "integrity": "sha512-E/dzevwpp8/3hKTyM2UUsd0DLz1o8zuJBwQP2+D8dg7jTlbgM6lfNTNupRWJRTP3M4FCeqL8SDralYdbt2zr8g==", + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/@redpanda-data/docs-extensions-and-macros/-/docs-extensions-and-macros-4.7.1.tgz", + "integrity": "sha512-S9OlnZYNaSk7Z3484fYpPXHOeo6Cp5cKKM1CJHtJp+tFfD7KT6gIU2BXmvX3buCXTwkjfgN0XwsdLs0sJ6G7Xw==", "license": "ISC", "dependencies": { "@asciidoctor/tabs": "^1.0.0-beta.6",