Skip to content

Commit

Permalink
docs: document crdb integration
Browse files Browse the repository at this point in the history
  • Loading branch information
sjwiesman committed Dec 19, 2024
1 parent 8203660 commit 18f1201
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 19 deletions.
44 changes: 25 additions & 19 deletions doc/user/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ disableKinds = ['taxonomy']
[params]
repo = "//github.com/MaterializeInc/materialize"
bannerMessage = "Need to run Materialize in your own private or public cloud? Get early access to **self-managed Materialize**!"
bannerLink="https://materialize.com/blog/self-managed/?utm_medium=docs-banner&utm_source=documentation&utm_campaign=FY25_Blogs&sf_campaign=701TR00000NoNFXYA3&utm_term=&utm_content="
bannerLink = "https://materialize.com/blog/self-managed/?utm_medium=docs-banner&utm_source=documentation&utm_campaign=FY25_Blogs&sf_campaign=701TR00000NoNFXYA3&utm_term=&utm_content="

[frontmatter]
publishDate = ["publishDate"]
Expand All @@ -20,7 +20,7 @@ publishDate = ["publishDate"]
[[menu.main]]
identifier = "get-started"
name = "Get started"
weight= 5
weight = 5

#
# Connect sources
Expand All @@ -29,14 +29,20 @@ weight= 5
[[menu.main]]
identifier = "ingest-data"
name = "Ingest data"
weight= 11
weight = 11

[[menu.main]]
name = "SQL Server"
parent = 'ingest-data'
identifier = 'sql-server'
weight = 15

[[menu.main]]
name = "CockroachDB"
parent = 'ingest-data'
identifier = 'crdb'
weight = 16

[[menu.main]]
name = "Redpanda"
parent = 'ingest-data'
Expand All @@ -53,7 +59,7 @@ weight = 30
identifier = "network-security"
name = "Network security"
parent = 'ingest-data'
weight= 35
weight = 35

#
# Transform data
Expand All @@ -62,7 +68,7 @@ weight= 35
[[menu.main]]
identifier = "transform-data"
name = "Transform data"
weight= 12
weight = 12

#
# Serve results
Expand All @@ -71,56 +77,56 @@ weight= 12
[[menu.main]]
identifier = "serve"
name = "Serve results"
weight= 13
weight = 13

[[menu.main]]
name = "Query using `SELECT`"
parent = "serve"
url = "/sql/select/"
weight= 5
weight = 5

[[menu.main]]
name = "Query using external tools"
identifier = "bi-tools"
parent = "serve"
weight= 10
weight = 10

[[menu.main]]
name = "Subscribe to results (`SUBSCRIBE`)"
parent = "serve"
url = "/sql/subscribe/"
weight= 15
weight = 15

[[menu.main]]
name = "Sink results to external tools"
identifier = "sink"
parent = "serve"
weight= 20
weight = 20

[[menu.main]]
name = "Kafka"
parent = "sink"
url = "/sql/create-sink"
weight= 20
weight = 20

[[menu.main]]
name = "Redpanda"
parent = "sink"
url = "/sql/create-sink"
weight= 30
weight = 30
#
# Manage Materialize
#

[[menu.main]]
identifier = "manage"
name = "Manage Materialize"
weight= 14
weight = 14

[[menu.main]]
name = "Monitoring"
identifier = "monitor"
weight= 5
weight = 5
parent = "manage"

#
Expand All @@ -130,15 +136,15 @@ parent = "manage"
[[menu.main]]
identifier = "reference"
name = "Reference"
weight= 15
weight = 15


[[menu.main]]
identifier = "cs_redpanda"
name = "Redpanda"
parent = "create-source"
url = "/sql/create-source/kafka"
weight= 10
weight = 10

[[menu.main]]
identifier = "csink_redpanda"
Expand Down Expand Up @@ -169,7 +175,7 @@ url = '/sql/create-sink'
[[menu.main]]
identifier = "integrations"
name = "Tools and integrations"
weight= 25
weight = 25

[[menu.main]]
identifier = "cli-reference"
Expand Down Expand Up @@ -204,13 +210,13 @@ weight = 70
name = "Security overview"
parent = "about"
url = "https://materialize.com/security-overview"
weight= 25
weight = 25

[[menu.main]]
name = "Responsible disclosure policy"
parent = "about"
url = "https://materialize.com/securitydisclosure"
weight= 30
weight = 30

[markup.goldmark.renderer]
# allow <a name="link-target">, the old syntax no longer works
Expand Down
79 changes: 79 additions & 0 deletions doc/user/content/ingest-data/cockroach.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
---
title: "CockroachDB CDC using Kafka and Changefeeds"
description: "How to propagate Change Data Capture (CDC) data from a CockroachDB database to Materialize"
menu:
main:
parent: "crdb"
name: "Using Kafka and Changefeeds"
identifier: "crdb-changefeeds"
weight: 16
---

Change Data Capture (CDC) allows you to track and propagate changes in a CockroachDB database to downstream consumers.
In this guide, we’ll cover how to use Materialize to create and efficiently maintain real-time materialized views on
top of CDC data.

{{< tip >}}
{{< guided-tour-blurb-for-ingest-data >}}
{{< /tip >}}

## Kafka + Changefeeds

Use [Changefeeds](https://www.cockroachlabs.com/docs/stable/change-data-capture-overview) and the [Kafka source](/sql/create-source/kafka/#using-debezium)
to propagate CDC data from CockroachDB to Materialize. Changefeeds capture
row-level changes resulting from `INSERT`, `UPDATE`, and `DELETE` operations in
the upstream database and publishes them as events to Kafka topics.

### Database setup

Before deploying a Debezium connector, ensure that the upstream database is
configured to support CDC.

1. Enable rangefeeds for the CockroachDB instance:

```sql
SET CLUSTER SETTING kv.rangefeed.enabled = true;
```

### Create Changefeeds

Create one changefeed for each table you want to publish to Materialize.
Each table will produce data to its own Kafka topic that can be consumed by Materialize.

```sql
CREATE CHANGEFEED FOR TABLE my_table
INTO 'kafka://broker:9092'
WITH format = avro,
confluent_schema_registry = 'http://registry:8081',
diff,
envelope=wrapped
```

Materialize recommends creating changefeeds with using format `avro` and enabling `diff` and using envelope `wrapped`.
This will emits change events using an envelope that matches Debezium contains detailed
information about upstream database operations, like the `before` and `after`
values for each record.
Please refer to the CockroachDB documentation for full details on changefeed configurations.

### Create a source

To create a source that interprets the changefeeds use the
[Debezium envelope](/sql/create-source/kafka/#using-debezium) in Materialize:

```mzsql
CREATE SOURCE kafka_repl
FROM KAFKA CONNECTION kafka_connection (TOPIC 'my_table')
FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_connection
ENVELOPE DEBEZIUM;
```

By default, the source will be created in the active cluster; to use a different
cluster, use the `IN CLUSTER` clause.

### Create a view

{{% ingest-data/ingest-data-kafka-debezium-view %}}

### Create an index on the view

{{% ingest-data/ingest-data-kafka-debezium-index %}}
10 changes: 10 additions & 0 deletions doc/user/content/integrations/_index.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,16 @@ database as a source requires enabling [**GTID-based binlog replication**](/sql/
| MySQL _(direct)_ | {{< supportLevel alpha >}} | See the [source documentation](/sql/create-source/mysql/) for more details, and the relevant integration guide for step-by-step instructions:<p></p><ul><li>[Amazon RDS for MySQL](/ingest-data/mysql/amazon-rds)</li><li>[Amazon Aurora for MySQL](/ingest-data/mysql/amazon-aurora)</li><li>[Azure DB for MySQL](/ingest-data/mysql/azure-db)</li><li>[Google Cloud SQL for MySQL](/ingest-data/mysql/google-cloud-sql)</li><li>[Self-hosted MySQL](/ingest-data/mysql/self-hosted)</li></ul> |
| MySQL _(via Debezium)_ | {{< supportLevel production >}} | See the [MySQL CDC guide](/integrations/cdc-mysql/) for a step-by-step breakdown of the integration. | |

### CockroachDB

CockroachDB is supported as a [**source**](/concepts/sources)
both through
[Changefeeds](/ingest-data/cockroachdb/) (via Kafka or Redpanda).

| Service | Support level | Notes | |
| ---------------------- | -------------------------------- | ---------------------------------------------------------------------------------------------------- | ----------- |
| CockroachDB _(via Changefeeds)_ | {{< supportLevel production >}} | See the [CockroachDB Changefeed Guide](/integrations/CockroachDB/) for a step-by-step breakdown of the integration. | |

### Other databases

{{< note >}}
Expand Down

0 comments on commit 18f1201

Please sign in to comment.