feat: use Client Credentials for managing Keycloak Clients

docs/reference/configuration/Single Sign-On/overview.md

@@ -10,7 +10,11 @@ UDS Core automates Keycloak Client configuration, secret management, and advance
 
 ![Single Sign-On Flow Chart](https://github.com/defenseunicorns/uds-core/blob/main/docs/.images/diagrams/uds-core-operator-authservice-keycloak.svg?raw=true)
 
-When a new UDS Package CR with the `sso` configuration gets deployed, the UDS Operator creates a new Keycloak Client using the [Dynamic Client Registration](https://www.keycloak.org/securing-apps/client-registration). The Registration Token that is used for updating and removing the newly created Keycloak Client is stored in Pepr Store. Once the Keycloak Client is ready, and the `enableAuthserviceSelector` is defined in the spec, the UDS Operator deploys Istio [Request Authentication](https://istio.io/latest/docs/reference/config/security/request_authentication/) and [AuthorizationPolicy](https://istio.io/latest/docs/reference/config/security/authorization-policy/) for both JWT and Request Headers. Both actions combined enable seamless and transparent application authentication and authorization capabilities. 
+When a new UDS Package CR with the `sso` configuration gets deployed, the UDS Operator creates a new Keycloak Client. This process happens in one of two modes - using [Dynamic Client Registration](https://www.keycloak.org/securing-apps/client-registration) or [Keycloak Admin endpoint](https://www.keycloak.org/docs-api/latest/rest-api/index.html#_clients) for managing Clients. Depending on the Keycloak Realm configuration, the Operator automatically picks the right mode. In the case of the former mode, the Registration Token that is used for updating and removing the newly created Keycloak Client is stored in Pepr Store. The latter mode reads the Client Secrets from the `keycloak-client-secrets` Kubernetes Secret deployed in `keycloak` namespace. This Secret is managed automatically by the UDS Operator. Once the Keycloak Client is ready, and the `enableAuthserviceSelector` is defined in the spec, the UDS Operator deploys Istio [Request Authentication](https://istio.io/latest/docs/reference/config/security/request_authentication/) and [AuthorizationPolicy](https://istio.io/latest/docs/reference/config/security/authorization-policy/) for both JWT and Request Headers. Both actions combined, enables seamless and transparent application authentication and authorization capabilities. 
+
+## Rotating the UDS Operator Client Secret
+
+The UDS Operator uses a dedicated Client in Keycloak. In some cases, the Client Secret needs to be rotated. In order to do so, you need to manually modify the `keycloak-client-secrets` Kubernetes Secret in the `keycloak` namespace and delete the `uds-operator` key. The UDS Operator will automatically re-create it. 
 
 ## User Groups
 
@@ -20,7 +24,7 @@ UDS Core deploys Keycloak which has some preconfigured groups that applications
 
 #### Grafana
 
-Grafana [maps the groups](https://github.com/defenseunicorns/uds-core/blob/49cb11a058a9209cee7019fa552b8c0b2ef73368/src/grafana/values/values.yaml#L37) from Keycloak to it's internal `Admin` and `Viewer` groups.
+Grafana [maps the groups](https://github.com/defenseunicorns/uds-core/blob/49cb11a058a9209cee7019fa552b8c0b2ef73368/src/grafana/values/values.yaml#L37) from Keycloak to its internal `Admin` and `Viewer` groups.
 
 | Keycloak Group | Mapped Grafana Group |
 |----------------|----------------------|
@@ -31,7 +35,7 @@ If a user doesn't belong to either of these Keycloak groups the user will be una
 
 #### Neuvector
 
-Neuvector [maps the groups](https://github.com/defenseunicorns/uds-core/blob/main/src/neuvector/chart/templates/uds-package.yaml#L31-L35) from Keycloak to it's internal `admin` and `reader` groups.
+Neuvector [maps the groups](https://github.com/defenseunicorns/uds-core/blob/main/src/neuvector/chart/templates/uds-package.yaml#L31-L35) from Keycloak to its internal `admin` and `reader` groups.
 
 | Keycloak Group | Mapped Neuvector Group |
 |----------------|------------------------|

pepr.ts

@@ -14,6 +14,7 @@ import { registerCRDs } from "./src/pepr/operator/crd/register";
 import { patches } from "./src/pepr/patches";
 import { policies, startExemptionWatch } from "./src/pepr/policies";
 import { prometheus } from "./src/pepr/prometheus";
+import { setupKeycloakClientSecret } from "./src/pepr/operator/controllers/keycloak/config";
 
 const log = setupLogger(Component.STARTUP);
 
@@ -23,6 +24,7 @@ const log = setupLogger(Component.STARTUP);
   // KFC watch for exemptions and update in-memory map
   await startExemptionWatch();
   await setupAuthserviceSecret();
+  await setupKeycloakClientSecret();
   new PeprModule(cfg, [
     // UDS Core Operator
     operator,

src/keycloak/chart/templates/istio-admin.yaml

@@ -15,7 +15,7 @@ spec:
   rules:
     - to:
         - operation:
-            ports: 
+            ports:
               - "8080"
             paths:
               - "/admin*"
@@ -24,9 +24,10 @@ spec:
         - source:
             notNamespaces:
               - istio-admin-gateway
+              - "pepr-system"
     - to:
         - operation:
-            ports: 
+            ports:
               - "8080"
             paths:
             - /metrics*
@@ -37,22 +38,33 @@ spec:
             - monitoring
     - to:
         - operation:
-            ports: 
+            ports:
               - "8080"
             paths:
               # Never allow anonymous client registration except from the pepr-system namespace
               # This is another fallback protection, as the KC policy already blocks it
               - "/realms/{{ .Values.realm }}/clients-registrations/*"
       from:
         - source:
-            notNamespaces: 
+            notNamespaces:
               - "pepr-system"
+    - to:
+        - operation:
+            ports:
+              - "8080"
+            paths:
+              - "/admin/realms/{{ .Values.realm }}/clients"
+      from:
+        - source:
+            notNamespaces:
+              - pepr-system
+              - istio-admin-gateway
     - when:
         - key: request.headers[istio-mtls-client-certificate]
           values: ["*"]
       to:
         - operation:
-            ports: 
+            ports:
               - "8080"
       from:
         - source:

src/keycloak/chart/templates/statefulset.yaml

@@ -231,6 +231,8 @@ spec:
             - name: conf
               mountPath: /opt/keycloak/conf
               readOnly: true
+            - name: client-secrets
+              mountPath: /var/run/secrets/uds/client-secrets
       enableServiceLinks: {{ .Values.enableServiceLinks }}
       restartPolicy: {{ .Values.restartPolicy }}
       {{- with .Values.nodeSelector }}
@@ -254,6 +256,9 @@ spec:
       {{- end }}
       terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
       volumes:
+        - name: client-secrets
+          secret:
+            secretName: {{ include "keycloak.fullname" . }}-client-secrets
         - name: providers
           {{- if .Values.persistence.providers.enabled }}
           persistentVolumeClaim:

src/pepr/operator/README.md

@@ -155,6 +155,20 @@ spec:
           bearer_only: clientField(bearerOnly)
 ```
 
+### Controlling how UDS Operator interacts with Keycloak
+
+The UDS Operator can interact with Keycloak in two primary ways: using the Dynamic Client Registration or Client Credentials Grant. The method used is determined automatically or by specifying the environment variable `PEPR_KEYCLOAK_CLIENT_STRATEGY`.
+
+Dynamic Client Registration allows the UDS Operator to dynamically register new Clients in the Keycloak server. A successful registration flow results in the Registration Token to be stored in Pepr store, which can be later used for modifying and removing the client.
+
+Client Credentials Grant uses the OAuth 2.0 Client Credentials Grant to authenticate against the `uds-operator` client defined in Keycloak. This special client has a limited control over managing Keycloak Clients for the UDS Operator.
+
+The `PEPR_KEYCLOAK_CLIENT_STRATEGY` can be set to one of the following values:
+
+* `auto` (default): The UDS Operator will automatically determine the best strategy to use based on the Keycloak server configuration
+* `dynamic_client_registration`: The UDS Operator will use the Dynamic Client Registration strategy
+* `client_credentials`: The UDS Operator will use the Client Credentials Grant strategy
+
 ### Key Files and Folders
 
 ```bash

src/pepr/operator/controllers/keycloak/client-secret-sync.spec.ts

@@ -0,0 +1,61 @@
+/**
+ * Copyright 2025 Defense Unicorns
+ * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+ */
+
+import { describe, expect, it } from "@jest/globals";
+import {
+  updateKeycloakClientsSecret,
+  KEYCLOAK_CLIENT_SECRET_KEY,
+  KEYCLOAK_CLIENTS_SECRET_NAME,
+  KEYCLOAK_CLIENTS_SECRET_NAMESPACE,
+} from "./client-secret-sync";
+
+interface Config {
+  metadata: {
+    name: string;
+    namespace: string;
+  };
+  data: {
+    [key: string]: string;
+  };
+}
+
+const createConfig = (data: { [key: string]: string } = {}): Config => ({
+  metadata: {
+    name: KEYCLOAK_CLIENTS_SECRET_NAME,
+    namespace: KEYCLOAK_CLIENTS_SECRET_NAMESPACE,
+  },
+  data,
+});
+
+describe("updateKeycloakClientsSecret Tests", () => {
+  it("should generate a new secret if KEYCLOAK_CLIENT_SECRET_KEY does not exist", async () => {
+    const config = createConfig();
+
+    await updateKeycloakClientsSecret(config);
+
+    expect(config.data[KEYCLOAK_CLIENT_SECRET_KEY]).not.toBe("");
+  });
+
+  it("should generate a new secret if forceRotation is true", async () => {
+    const config = createConfig({
+      [KEYCLOAK_CLIENT_SECRET_KEY]: "existing-secret",
+    });
+
+    await updateKeycloakClientsSecret(config, true);
+
+    expect(config.data[KEYCLOAK_CLIENT_SECRET_KEY]).not.toBe("");
+    expect(config.data[KEYCLOAK_CLIENT_SECRET_KEY]).not.toBe("existing-secret");
+  });
+
+  it("should not generate a new secret if KEYCLOAK_CLIENT_SECRET_KEY exists and forceRotation is false", async () => {
+    const config = createConfig({
+      [KEYCLOAK_CLIENT_SECRET_KEY]: "existing-secret",
+    });
+
+    await updateKeycloakClientsSecret(config);
+
+    expect(config.data[KEYCLOAK_CLIENT_SECRET_KEY]).toBe("existing-secret");
+  });
+});

src/pepr/operator/controllers/keycloak/client-secret-sync.ts

@@ -0,0 +1,39 @@
+/**
+ * Copyright 2025 Defense Unicorns
+ * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+ */
+
+import { K8s, kind } from "pepr";
+import { v4 as uuidv4 } from "uuid";
+import { Component, setupLogger } from "../../../logger";
+
+export const KEYCLOAK_CLIENT_SECRET_KEY = "uds-operator";
+
+export const KEYCLOAK_CLIENTS_SECRET_NAMESPACE = "keycloak";
+export const KEYCLOAK_CLIENTS_SECRET_NAME = "keycloak-client-secrets";
+
+const log = setupLogger(Component.OPERATOR_CONFIG);
+
+/**
+ * Updates the Keycloak client secret in the provided config.
+ * If the secret does not exist or forceRotation is true, a new secret is generated.
+ * The secret is then applied to the Kubernetes cluster.
+ *
+ * @param {kind.Secret} config - The Kubernetes Secret object to update.
+ * @param {boolean} [forceRotation=false] - Whether to force rotation of the secret.
+ */
+export async function updateKeycloakClientsSecret(
+  config: kind.Secret,
+  forceRotation: boolean = false,
+) {
+  config.data = config.data || {};
+
+  // This might be a bug but it seems Zarf adds managedFields, which is prohibited in Secrets.
+  delete config.metadata?.managedFields;
+
+  if (!config.data[KEYCLOAK_CLIENT_SECRET_KEY] || forceRotation) {
+    log.info("Generating new Keycloak client secret");
+    config.data[KEYCLOAK_CLIENT_SECRET_KEY] = Buffer.from(uuidv4()).toString("base64");
+    await K8s(kind.Secret).Apply(config);
+  }
+}

src/pepr/operator/controllers/keycloak/client-sync.ts

@@ -6,21 +6,14 @@
 import { fetch, K8s, kind } from "pepr";
 
 import { Component, setupLogger } from "../../../logger";
-import { Store } from "../../common";
 import { Sso, UDSPackage } from "../../crd";
-import { getOwnerRef, purgeOrphans, retryWithDelay, sanitizeResourceName } from "../utils";
+import { getOwnerRef, purgeOrphans, sanitizeResourceName } from "../utils";
 import { Client, clientKeys } from "./types";
+import { createOrUpdateClient, deleteClient } from "./clients/keycloak-client";
 
-let apiURL =
-  "http://keycloak-http.keycloak.svc.cluster.local:8080/realms/uds/clients-registrations/default";
 const samlDescriptorUrl =
   "http://keycloak-http.keycloak.svc.cluster.local:8080/realms/uds/protocol/saml/descriptor";
 
-// Support dev mode with port-forwarded keycloak svc
-if (process.env.PEPR_MODE === "dev") {
-  apiURL = "http://localhost:8080/realms/uds/clients-registrations/default";
-}
-
 // Template regex to match clientField() references, see https://regex101.com/r/e41Dsk/3 for details
 const secretTemplateRegex = new RegExp(
   'clientField\\(([a-zA-Z]+)\\)(?:\\["?([\\w]+)"?\\]|(\\.json\\(\\)))?',
@@ -86,13 +79,13 @@ export async function purgeSSOClients(pkg: UDSPackage, newClients: string[] = []
   const currentClients = pkg.status?.ssoClients || [];
   const toRemove = currentClients.filter(client => !newClients.includes(client));
   for (const ref of toRemove) {
-    const storeKey = `sso-client-${ref}`;
-    const token = Store.getItem(storeKey);
-    if (token) {
-      await apiCall({ clientId: ref }, "DELETE", token);
-      await Store.removeItemAndWait(storeKey);
-    } else {
-      log.warn(pkg.metadata, `Failed to remove client ${ref}, token not found`);
+    try {
+      await deleteClient({ clientId: ref });
+    } catch (err) {
+      log.warn(
+        pkg.metadata,
+        `Failed to remove client ${ref}, package ${pkg.metadata?.namespace}/${pkg.metadata?.name}. Error: ${err.message}`,
+      );
       throw new Error(`Failed to remove client ${ref}, token not found`);
     }
   }
@@ -138,31 +131,22 @@ async function syncClient(
   const name = `sso-client-${clientReq.clientId}`;
   let client = convertSsoToClient(clientReq);
 
-  // Get keycloak client token from the store if this is an existing client
-  const token = Store.getItem(name);
-
   try {
-    // If an existing client is found, use the token to update the client
-    if (token && !isRetry) {
-      log.debug(pkg.metadata, `Found existing token for ${client.clientId}`);
-      client = await apiCall(client, "PUT", token);
-    } else {
-      log.debug(pkg.metadata, `Creating new client for ${client.clientId}`);
-      client = await apiCall(client);
-    }
+    client = await createOrUpdateClient(client, isRetry);
   } catch (err) {
     const msg =
       `Failed to process Keycloak request for client '${client.clientId}', package ` +
       `${pkg.metadata?.namespace}/${pkg.metadata?.name}. Error: ${err.message}`;
 
     // Throw the error if this is the retry or was an initial client creation attempt
-    if (isRetry || !token) {
+    // if (isRetry || !token) {
+    if (isRetry) {
       log.error(`${msg}, retry failed.`);
       // Throw the original error captured from the first attempt
       throw new Error(msg);
     } else {
       // Retry the request without the token in case we have a bad token stored
-      log.error(msg);
+      log.error(`${msg}, retrying...`);
 
       try {
         return await syncClient(clientReq, pkg, true);
@@ -178,18 +162,6 @@ async function syncClient(
     }
   }
 
-  // Write the new token to the store
-  try {
-    await retryWithDelay(async function setStoreToken() {
-      return Store.setItemAndWait(name, client.registrationAccessToken!);
-    }, log);
-  } catch {
-    throw Error(
-      `Failed to set token in store for client '${client.clientId}', package ` +
-        `${pkg.metadata?.namespace}/${pkg.metadata?.name}`,
-    );
-  }
-
   // Remove the registrationAccessToken from the client object to avoid problems (one-time use token)
   delete client.registrationAccessToken;
 
@@ -221,43 +193,6 @@ async function syncClient(
   return client;
 }
 
-async function apiCall(client: Partial<Client>, method = "POST", authToken = "") {
-  const req = {
-    body: JSON.stringify(client) as string | undefined,
-    method,
-    headers: {
-      "Content-Type": "application/json",
-    } as Record<string, string>,
-  };
-
-  let url = apiURL;
-
-  // When not creating a new client, add the client ID and registrationAccessToken
-  if (authToken) {
-    req.headers.Authorization = `Bearer ${authToken}`;
-    // Ensure that we URI encode the clientId in the request URL
-    url += `/${encodeURIComponent(client.clientId!)}`;
-  }
-
-  // Remove the body for DELETE requests
-  if (method === "DELETE" || method === "GET") {
-    delete req.body;
-  }
-
-  // Make the request
-  const resp = await fetch<Client>(url, req);
-
-  if (!resp.ok) {
-    if (resp.data) {
-      throw new Error(`${JSON.stringify(resp.statusText)}, ${JSON.stringify(resp.data)}`);
-    } else {
-      throw new Error(`${JSON.stringify(resp.statusText)}`);
-    }
-  }
-
-  return resp.data;
-}
-
 export function generateSecretData(client: Client, secretTemplate?: { [key: string]: string }) {
   if (secretTemplate) {
     log.debug(`Using secret template for client: ${client.clientId}`);