diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index e6dc47758..96ff79daf 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -167,8 +167,11 @@ *** xref:manage:topic-recovery.adoc[Topic Recovery] *** xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] ** xref:manage:schema-reg/index.adoc[Schema Registry] -*** xref:manage:schema-reg/schema-reg-overview.adoc[] -*** xref:manage:schema-reg/schema-reg-api.adoc[] +*** xref:manage:schema-reg/schema-reg-overview.adoc[Overview] +*** xref:manage:schema-reg/manage-schema-reg.adoc[] +**** xref:manage:schema-reg/schema-reg-api.adoc[API] +**** xref:console:ui/schema-reg.adoc[Redpanda Console] +**** xref:manage:kubernetes/k-schema-controller.adoc[Kubernetes] *** xref:manage:schema-reg/schema-id-validation.adoc[] *** xref:console:ui/schema-reg.adoc[Manage in Redpanda Console] ** xref:manage:console/index.adoc[Redpanda Console] diff --git a/modules/manage/examples/kubernetes/schema-crds.feature b/modules/manage/examples/kubernetes/schema-crds.feature new file mode 100644 index 000000000..0ffe46a1e --- /dev/null +++ b/modules/manage/examples/kubernetes/schema-crds.feature @@ -0,0 +1,106 @@ +@cluster:basic +Feature: Schema CRDs + Background: Cluster available + Given cluster "basic" is available + + @skip:gke @skip:aks @skip:eks + Scenario: Manage customer profile schema (Avro) + Given there is no schema "customer-profile" in cluster "basic" + When I apply Kubernetes manifest: + """ +# tag::customer-profile-avro-schema-manifest[] + # This manifest creates an Avro schema named "customer-profile" in the "basic" cluster. + # The schema defines a record with fields for customer ID, name, and age. + --- + apiVersion: cluster.redpanda.com/v1alpha2 + kind: Schema + metadata: + name: customer-profile + spec: + cluster: + clusterRef: + name: basic + schemaType: avro + compatibilityLevel: Backward + text: | + { + "type": "record", + "name": "CustomerProfile", + "fields": [ + { "type": "string", "name": "customer_id" }, + { "type": "string", "name": "name" }, + { "type": "int", "name": "age" } + ] + } +# end::customer-profile-avro-schema-manifest[] + """ + And schema "customer-profile" is successfully synced + Then I should be able to check compatibility against "customer-profile" in cluster "basic" + + @skip:gke @skip:aks @skip:eks + Scenario: Manage product catalog schema (Protobuf) + Given there is no schema "product-catalog" in cluster "basic" + When I apply Kubernetes manifest: + """ +# tag::product-catalog-protobuf-schema-manifest[] + # This manifest creates a Protobuf schema named "product-catalog" in the "basic" cluster. + # The schema defines a message "Product" with fields for product ID, name, price, and category. + --- + apiVersion: cluster.redpanda.com/v1alpha2 + kind: Schema + metadata: + name: product-catalog + spec: + cluster: + clusterRef: + name: basic + schemaType: protobuf + compatibilityLevel: Backward + text: | + syntax = "proto3"; + + message Product { + int32 product_id = 1; + string product_name = 2; + double price = 3; + string category = 4; + } +# end::product-catalog-protobuf-schema-manifest[] + """ + And schema "product-catalog" is successfully synced + Then I should be able to check compatibility against "product-catalog" in cluster "basic" + + @skip:gke @skip:aks @skip:eks + Scenario: Manage order event schema (JSON) + Given there is no schema "order-event" in cluster "basic" + When I apply Kubernetes manifest: + """ +# tag::order-event-json-schema-manifest[] + # This manifest creates a JSON schema named "order-event" in the "basic" cluster. + # The schema requires an "order_id" (string) and a "total" (number) field, with no additional properties allowed. + --- + apiVersion: cluster.redpanda.com/v1alpha2 + kind: Schema + metadata: + name: order-event + spec: + cluster: + clusterRef: + name: basic + schemaType: json + compatibilityLevel: None + text: | + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "order_id": { "type": "string" }, + "total": { "type": "number" } + }, + "required": ["order_id", "total"], + "additionalProperties": false + } +# end::order-event-json-schema-manifest[] + """ + And schema "order-event" is successfully synced + Then I should be able to check compatibility against "order-event" in cluster "basic" diff --git a/modules/manage/examples/kubernetes/topic-crds.feature b/modules/manage/examples/kubernetes/topic-crds.feature new file mode 100644 index 000000000..4b8218fb5 --- /dev/null +++ b/modules/manage/examples/kubernetes/topic-crds.feature @@ -0,0 +1,27 @@ +@cluster:basic +Feature: Topic CRDs + Background: Cluster available + Given cluster "basic" is available + + @skip:gke @skip:aks @skip:eks + Scenario: Manage topics + Given there is no topic "topic1" in cluster "basic" + When I apply Kubernetes manifest: + """ +# tag::basic-topic-example[] + # In this example manifest, a topic called "topic1" is created in a cluster called "basic". It has a replication factor of 1 and is distributed across a single partition. + --- + apiVersion: cluster.redpanda.com/v1alpha2 + kind: Topic + metadata: + name: topic1 + spec: + cluster: + clusterRef: + name: basic + partitions: 1 + replicationFactor: 1 +# end::basic-topic-example[] + """ + And topic "topic1" is successfully synced + Then I should be able to produce and consume from "topic1" in cluster "basic" \ No newline at end of file diff --git a/modules/manage/examples/kubernetes/user-crds.feature b/modules/manage/examples/kubernetes/user-crds.feature index 3712f6cb9..872b92845 100644 --- a/modules/manage/examples/kubernetes/user-crds.feature +++ b/modules/manage/examples/kubernetes/user-crds.feature @@ -25,7 +25,7 @@ Feature: User CRDs | jason | [{"type":"allow","resource":{"type":"cluster"},"operations":["Read"]}] | When I apply Kubernetes manifest: """ - # tag::manage-authn-only-manifest[] +# tag::manage-authn-only-manifest[] # In this example manifest, a user called "jason" is created in a cluster called "sasl". # The user's password is defined in a Secret called "jason-password". # This example assumes that you will create ACLs for this user separately. @@ -45,7 +45,7 @@ Feature: User CRDs secretKeyRef: name: jason-password key: password - # end::manage-authn-only-manifest[] +# end::manage-authn-only-manifest[] """ And user "jason" is successfully synced And I delete the CRD user "jason" @@ -58,7 +58,7 @@ Feature: User CRDs | travis | password | SCRAM-SHA-256 | When I apply Kubernetes manifest: """ - # tag::manage-authz-only-manifest[] +# tag::manage-authz-only-manifest[] # In this example manifest, an ACL called "travis" is created in a cluster called "sasl". # The ACL give an existing user called "travis" permissions to read from all topics whose names start with some-topic. # This example assumes that you already have a user called "travis" in your cluster. @@ -79,8 +79,8 @@ Feature: User CRDs name: some-topic patternType: prefixed operations: [Read] - # end::manage-authz-only-manifest[] +# end::manage-authz-only-manifest[] """ And user "travis" is successfully synced And I delete the CRD user "travis" - Then "travis" should be able to authenticate to the "sasl" cluster with password "password" and mechanism "SCRAM-SHA-256" + Then "travis" should be able to authenticate to the "sasl" cluster with password "password" and mechanism "SCRAM-SHA-256" \ No newline at end of file diff --git a/modules/manage/pages/kubernetes/k-schema-controller.adoc b/modules/manage/pages/kubernetes/k-schema-controller.adoc new file mode 100644 index 000000000..aaaa8606a --- /dev/null +++ b/modules/manage/pages/kubernetes/k-schema-controller.adoc @@ -0,0 +1,272 @@ += Manage Schemas with the Redpanda Operator +:description: Use the Schema resource to declaratively create and manage schemas as part of a Redpanda deployment in Kubernetes. +:page-categories: Management, Development +:env-kubernetes: true + +Use the Schema resource to declaratively create and manage schemas as part of a Redpanda deployment in Kubernetes. Each Schema resource maps to a schema in your Redpanda cluster, allowing you to define data structures, compatibility, and schema evolution in a declarative way. + +== Prerequisites + +Ensure you have the following: + +* *Kubectl*: Ensure the https://kubernetes.io/docs/tasks/tools/#kubectl[kubectl^] command-line tool is installed and configured to communicate with your cluster. +* *Redpanda cluster*: Ensure you have at least version v2.3.0-24.3.1 of the xref:deploy:deployment-option/self-hosted/kubernetes/k-production-deployment.adoc[Redpanda Operator] and a Redpanda resource deployed and accessible. + +== Create a schema + +. Define a schema using the Schema resource. Here's a basic example configuration that defines an Avro schema: ++ +.`schema.yaml` +[source,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Schema +metadata: + name: example-schema + namespace: +spec: + cluster: + clusterRef: + name: + schemaType: avro + compatibilityLevel: Backward + text: | + { + "type": "record", + "name": "ExampleRecord", + "fields": [ + { "type": "string", "name": "field1" }, + { "type": "int", "name": "field2" } + ] + } +---- ++ +Replace the following placeholders: ++ +- ``: The namespace in which to deploy the Schema resource. The Schema resource must be deployed in the same namespace as the Redpanda resource defined in `clusterRef.name`. +- ``: The name of the Redpanda resource that defines the Redpanda cluster to which you want to upload the schema. + +. Apply the manifest: ++ +[source,bash] +---- +kubectl apply -f schema.yaml --namespace +---- ++ +When the manifest is applied, the schema will be created in your Redpanda cluster. + +. Check the status of the Schema resource using the following command: ++ +[,bash] +---- +kubect get schema example-schema --namespace +---- + +. Verify that the schema was created in Redpanda: ++ +[source,bash] +---- +kubectl exec -it --namespace -- curl https://...svc.cluster.local:8081/subjects -sS --cacert /etc/tls/certs/default/ca.crt -w '\n' +---- ++ +Replace `` with the name of a Pod that's running Redpanda. + +== Schema examples + +These examples demonstrate how to define schemas in Avro, Protobuf, and JSON Schema formats. + +=== Create an Avro schema + +.`avro-schema.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/schema-crds.feature[tags=customer-profile-avro-schema-manifest,indent=0] +---- + +=== Create a Protobuf schema + +.`proto-schema.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/schema-crds.feature[tags=product-catalog-protobuf-schema-manifest,indent=0] +---- + +=== Create a JSON schema + +.`json-schema.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/schema-crds.feature[tags=order-event-json-schema-manifest,indent=0] +---- + +== Configuration + +The Schema resource in Redpanda offers various options to customize and control schema behavior. This section covers schema compatibility, schema references, and schema types, providing a detailed guide on using each of these features to maintain data integrity, manage dependencies, and facilitate schema evolution. + +You can find all configuration options for the Schema resource in the xref:reference:k-crd.adoc#k8s-api-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-schema[CRD reference]. + +.`schema.yaml` +[source,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Schema +metadata: + name: <.> + namespace: <.> +spec: + cluster: + clusterRef: + name: <.> + schemaType: avro <.> + compatibilityLevel: Backward <.> + references: [] <.> + text: | <.> + { + "type": "record", + "name": "test", + "fields": [ + { "type": "string", "name": "field1" }, + { "type": "int", "name": "field2" } + ] + } +---- + +<.> *Subject name*: The name of the subject for the schema. When data formats are updated, a new version of the schema can be registered under the same subject, enabling backward and forward compatibility. +<.> *Namespace*: The namespace in which to deploy the Schema resource. The Schema resource must be deployed in the same namespace as the Redpanda resource defined in `clusterRef.name`. +<.> *Cluster name*: The name of the Redpanda resource that defines the Redpanda cluster to which you want to upload the schema. +<.> *Compatibility level*: Defines the compatibility level for the schema. Options are `Backward` (default), `BackwardTransitive`, `Forward`, `ForwardTransitive` `Full`, `FullTransitive`, or `None`. See <>. +<.> *Schema type*: Specifies the type of the schema. Options are `avro` (default) or `protobuf`. For JSON Schema, include `"$schema":` in the `text` to indicate the JSON Schema draft version. See <>. +<.> *References*: Any references you want to add to other schemas. If no references are needed, this can be an empty list (default). See <>. +<.> *Schema body*: The body of the schema, which defines the data structure. + +=== Choose a schema type + +Redpanda's Schema Registry supports the following schema types: + +* *Avro*: A widely used serialization format in event-driven architectures. +* *Protobuf*: Popular for defining data structures in gRPC APIs and efficient data serialization. +* *JSON Schema*: Dynamic, schema-based validation for JSON documents. + +If no type is specified, Redpanda defaults to Avro. + +=== Choose a compatibility mode + +Compatibility modes determine how schema versions within a subject can evolve without breaking existing data consumers. Redpanda supports the following compatibility levels: + +* `None`: Disables compatibility checks, allowing any schema change. +* `Backward`: Consumers using the new schema (for example, version 10) can read data from producers using the previous schema (for example, version 9). +* `BackwardTransitive`: Enforces backward compatibility across all versions, not just the latest. +* `Forward`: Consumers using the previous schema (for example, version 9) can read data from producers using the new schema (for example, version 10). +* `ForwardTransitive`: Ensures forward compatibility across all schema versions. +* `Full`: Combines backward and forward compatibility, requiring that changes maintain compatibility in both directions. A new schema and the previous schema (for example, versions 10 and 9) are both backward and forward-compatible with each other. +* `FullTransitive`: Enforces full compatibility across all schema versions. + +For example, to set full compatibility, configure the Schema resource with: + +[source,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Schema +metadata: + name: fully-compatible-schema + namespace: redpanda +spec: + cluster: + clusterRef: + name: basic + schemaType: avro + compatibilityLevel: Full + text: | + { + "type": "record", + "name": "ExampleRecord", + "fields": [ + { "type": "string", "name": "field1" }, + { "type": "int", "name": "field2" } + ] + } +---- + +Compatibility settings are essential for maintaining data consistency, especially when updating schemas over time. + +=== Use schema references + +For complex data structures, you can define schema references that allow one schema to reference another, enabling modular and reusable schema components. Schema references are helpful for shared data structures across topics like product information or user profiles, reducing redundancy. + +NOTE: This feature is supported for Avro and Protobuf schemas. + +Define a schema reference using the `references` field. The reference includes the name, subject, and version of the referenced schema: + +[source,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Schema +metadata: + name: order-schema + namespace: redpanda +spec: + cluster: + clusterRef: + name: basic + references: + - name: product-schema + subject: product + version: 1 + text: | + { + "type": "record", + "name": "Order", + "fields": [ + { "name": "product", "type": "Product" } + ] + } +---- + +== Update a schema + +To update a schema, modify the Schema resource and apply the changes: + +[source,bash] +---- +kubectl apply -f .yaml --namespace +---- + +== Check schema version + +Ensure the schema has been versioned by running: + +[source,bash] +---- +kubectl get schema --namespace +---- + +You can also check specific versions of the schema: + +[source,bash] +---- +kubectl exec -it --namespace -- curl https://...svc.cluster.local:8081/schemas/ids/1 -sS --cacert /etc/tls/certs/default/ca.crt -w '\n' +kubectl exec -it --namespace -- curl https://...svc.cluster.local:8081/schemas/ids/2 -sS --cacert /etc/tls/certs/default/ca.crt -w '\n' +---- + +== Delete a schema + +To delete a schema, use the following command: + +[source,bash] +---- +kubectl delete schema --namespace redpanda +---- + +Verify that the schema was deleted by checking the Redpanda Schema Registry: + +[source,bash] +---- +kubectl exec -it --namespace -- curl https://...svc.cluster.local:8081/subjects -sS --cacert /etc/tls/certs/default/ca.crt -w '\n' +---- + +== Suggested reading + +For more details on using schemas in Redpanda, see: + +* xref:manage:schema-reg/index.adoc[] + diff --git a/modules/manage/pages/kubernetes/security/authentication/k-user-controller.adoc b/modules/manage/pages/kubernetes/security/authentication/k-user-controller.adoc index 092a6d700..5c46594f1 100644 --- a/modules/manage/pages/kubernetes/security/authentication/k-user-controller.adoc +++ b/modules/manage/pages/kubernetes/security/authentication/k-user-controller.adoc @@ -91,9 +91,9 @@ spec: operations: [Read,Write] ---- -== Configuration best practices +== Configuration -The following sections provide guidance on setting up user authentication, managing secrets, and defining ACLs within your Kubernetes environment. These recommendations ensure proper user management while minimizing manual interventions and preventing potential security issues. By following these best practices, you can ensure that user access and permissions are correctly configured and maintained across your Redpanda cluster. +The following sections provide guidance on setting up user authentication, managing secrets, and defining ACLs within your Kubernetes environment. These recommendations ensure proper user management while minimizing manual interventions and preventing potential security issues. You can find all configuration options for the User resource in the xref:reference:k-crd.adoc#k8s-api-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-user[CRD reference]. diff --git a/modules/manage/pages/schema-reg/manage-schema-reg.adoc b/modules/manage/pages/schema-reg/manage-schema-reg.adoc new file mode 100644 index 000000000..b0c456ae0 --- /dev/null +++ b/modules/manage/pages/schema-reg/manage-schema-reg.adoc @@ -0,0 +1,5 @@ += Use Schema Registry +:page-layout: index +:description: Learn how to add, delete, and update schemas in Redpanda Self-Managed. + +{description} \ No newline at end of file