feat: use Client Credentials for managing Keycloak Clients

src/keycloak/chart/templates/istio-admin.yaml

@@ -15,7 +15,7 @@ spec:
   rules:
     - to:
         - operation:
-            ports: 
+            ports:
               - "8080"
             paths:
               - "/admin*"
@@ -24,9 +24,10 @@ spec:
         - source:
             notNamespaces:
               - istio-admin-gateway
+              - "pepr-system"
     - to:
         - operation:
-            ports: 
+            ports:
               - "8080"
             paths:
             - /metrics*
@@ -37,22 +38,33 @@ spec:
             - monitoring
     - to:
         - operation:
-            ports: 
+            ports:
               - "8080"
             paths:
               # Never allow anonymous client registration except from the pepr-system namespace
               # This is another fallback protection, as the KC policy already blocks it
               - "/realms/{{ .Values.realm }}/clients-registrations/*"
       from:
         - source:
-            notNamespaces: 
+            notNamespaces:
               - "pepr-system"
+    - to:
+        - operation:
+            ports:
+              - "8080"
+            paths:
+              - "/admin/realms/{{ .Values.realm }}/clients"
+      from:
+        - source:
+            notNamespaces:
+              - pepr-system
+              - istio-admin-gateway
     - when:
         - key: request.headers[istio-mtls-client-certificate]
           values: ["*"]
       to:
         - operation:
-            ports: 
+            ports:
               - "8080"
       from:
         - source:

src/keycloak/chart/templates/secret-client-secrets.yaml

@@ -0,0 +1,13 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "keycloak.fullname" . }}-client-secrets
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "keycloak.labels" . | nindent 4 }}
+type: Opaque
+data:
+  ignored-value: {{ "ignored-value" | b64enc }}

src/keycloak/chart/templates/statefulset.yaml

@@ -231,6 +231,8 @@ spec:
             - name: conf
               mountPath: /opt/keycloak/conf
               readOnly: true
+            - name: client-secrets
+              mountPath: /var/run/secrets/uds/client-secrets
       enableServiceLinks: {{ .Values.enableServiceLinks }}
       restartPolicy: {{ .Values.restartPolicy }}
       {{- with .Values.nodeSelector }}
@@ -254,6 +256,9 @@ spec:
       {{- end }}
       terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
       volumes:
+        - name: client-secrets
+          secret:
+            secretName: {{ include "keycloak.fullname" . }}-client-secrets
         - name: providers
           {{- if .Values.persistence.providers.enabled }}
           persistentVolumeClaim:

src/keycloak/chart/templates/uds-package.yaml

@@ -97,9 +97,9 @@ spec:
             - name: redirect-welcome
               uri:
                 exact: /
-            - name: redirect-admin
-              uri:
-                prefix: /admin
+{{/*            - name: redirect-admin*/}}
+{{/*              uri:*/}}
+{{/*                prefix: /admin*/}}
             - name: redirect-master-realm
               uri:
                 prefix: /realms/master

src/keycloak/chart/values.yaml

@@ -278,4 +278,6 @@ autoscaling:
           value: 1
           periodSeconds: 300
 
-env: []
+env:
+  - name: JAVA_OPTS_KC_HEAP
+    value: "-XX:MaxRAMPercentage=70 -XX:MinRAMPercentage=70 -XX:InitialRAMPercentage=50 -XX:MaxRAM=1G"

src/pepr/operator/README.md

@@ -155,6 +155,20 @@ spec:
           bearer_only: clientField(bearerOnly)
 ```
 
+### Controlling how UDS Operator interacts with Keycloak
+
+The UDS Operator can interact with Keycloak in two primary ways: using the Dynamic Client Registration or Client Credentials Grant. The method used is determined automatically or by specifying the environment variable `PEPR_KEYCLOAK_CLIENT_STRATEGY`.
+
+Dynamic Client Registration allows the UDS Operator to dynamically register new Clients in the Keycloak server. A successful registration flow results in the Registration Token to be stored in Pepr store, which can be later used for modifying and remocing the client.
+
+Client Credentials Grant uses the OAuth 2.0 Client Credentials Grant to authenticate against the `uds-operator` client defined in Keycloak. This special client has a limited control over managing Keycloak Clients for the UDS Operator.
+
+The `PEPR_KEYCLOAK_CLIENT_STRATEGY` can be set to one of the following values:
+
+* `auto` (default): The UDS Operator will automatically determine the best strategy to use based on the Keycloak server configuration
+* `dynamic_client_registration`: The UDS Operator will use the Dynamic Client Registration strategy
+* `client_credentials`: The UDS Operator will use the Client Credentials Grant strategy
+
 ### Key Files and Folders
 
 ```bash

src/pepr/operator/controllers/keycloak/client-secret-sync.ts

@@ -0,0 +1,25 @@
+/**
+ * Copyright 2025 Defense Unicorns
+ * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+ */
+
+import { K8s, kind } from "pepr";
+import { v4 as uuidv4 } from "uuid";
+import { Component, setupLogger } from "../../../logger";
+
+const KEYCLOAK_CLIENT_SECRET_KEY = "uds-operator";
+
+const log = setupLogger(Component.OPERATOR_CONFIG);
+
+export async function updateKeycloakClientsSecret(config: kind.Secret) {
+  config.data = config.data || {};
+
+  // This might be a bug but it seems Zarf adds managedFields, which is prohibited in Secrets.
+  delete config.metadata?.managedFields;
+
+  if (!config.data[KEYCLOAK_CLIENT_SECRET_KEY]) {
+    log.info("Generating new Keycloak client secret");
+    config.data[KEYCLOAK_CLIENT_SECRET_KEY] = Buffer.from(uuidv4()).toString("base64");
+    await K8s(kind.Secret).Apply(config);
+  }
+}

src/pepr/operator/controllers/keycloak/client-sync.ts

@@ -6,21 +6,22 @@
 import { fetch, K8s, kind } from "pepr";
 
 import { Component, setupLogger } from "../../../logger";
-import { Store } from "../../common";
 import { Sso, UDSPackage } from "../../crd";
-import { getOwnerRef, purgeOrphans, retryWithDelay, sanitizeResourceName } from "../utils";
+import { getOwnerRef, purgeOrphans, sanitizeResourceName } from "../utils";
 import { Client, clientKeys } from "./types";
+import { DynamicKeycloakClient } from "./keycloak-client";
 
-let apiURL =
-  "http://keycloak-http.keycloak.svc.cluster.local:8080/realms/uds/clients-registrations/default";
+let apiURL = "http://keycloak-http.keycloak.svc.cluster.local:8080";
 const samlDescriptorUrl =
   "http://keycloak-http.keycloak.svc.cluster.local:8080/realms/uds/protocol/saml/descriptor";
 
 // Support dev mode with port-forwarded keycloak svc
 if (process.env.PEPR_MODE === "dev") {
-  apiURL = "http://localhost:8080/realms/uds/clients-registrations/default";
+  apiURL = "http://localhost:8080";
 }
 
+const keycloakClient = new DynamicKeycloakClient(apiURL);
+
 // Template regex to match clientField() references, see https://regex101.com/r/e41Dsk/3 for details
 const secretTemplateRegex = new RegExp(
   'clientField\\(([a-zA-Z]+)\\)(?:\\["?([\\w]+)"?\\]|(\\.json\\(\\)))?',
@@ -86,13 +87,13 @@ export async function purgeSSOClients(pkg: UDSPackage, newClients: string[] = []
   const currentClients = pkg.status?.ssoClients || [];
   const toRemove = currentClients.filter(client => !newClients.includes(client));
   for (const ref of toRemove) {
-    const storeKey = `sso-client-${ref}`;
-    const token = Store.getItem(storeKey);
-    if (token) {
-      await apiCall({ clientId: ref }, "DELETE", token);
-      await Store.removeItemAndWait(storeKey);
-    } else {
-      log.warn(pkg.metadata, `Failed to remove client ${ref}, token not found`);
+    try {
+      await keycloakClient.delete({ clientId: ref });
+    } catch (err) {
+      log.warn(
+        pkg.metadata,
+        `Failed to remove client ${ref}, package ${pkg.metadata?.namespace}/${pkg.metadata?.name}. Error: ${err.message}`,
+      );
       throw new Error(`Failed to remove client ${ref}, token not found`);
     }
   }
@@ -138,31 +139,22 @@ async function syncClient(
   const name = `sso-client-${clientReq.clientId}`;
   let client = convertSsoToClient(clientReq);
 
-  // Get keycloak client token from the store if this is an existing client
-  const token = Store.getItem(name);
-
   try {
-    // If an existing client is found, use the token to update the client
-    if (token && !isRetry) {
-      log.debug(pkg.metadata, `Found existing token for ${client.clientId}`);
-      client = await apiCall(client, "PUT", token);
-    } else {
-      log.debug(pkg.metadata, `Creating new client for ${client.clientId}`);
-      client = await apiCall(client);
-    }
+    client = await keycloakClient.createOrUpdate(client);
   } catch (err) {
     const msg =
       `Failed to process Keycloak request for client '${client.clientId}', package ` +
       `${pkg.metadata?.namespace}/${pkg.metadata?.name}. Error: ${err.message}`;
 
     // Throw the error if this is the retry or was an initial client creation attempt
-    if (isRetry || !token) {
+    // if (isRetry || !token) {
+    if (isRetry) {
       log.error(`${msg}, retry failed.`);
       // Throw the original error captured from the first attempt
       throw new Error(msg);
     } else {
       // Retry the request without the token in case we have a bad token stored
-      log.error(msg);
+      log.error(`${msg}, retrying...`);
 
       try {
         return await syncClient(clientReq, pkg, true);
@@ -178,18 +170,6 @@ async function syncClient(
     }
   }
 
-  // Write the new token to the store
-  try {
-    await retryWithDelay(async function setStoreToken() {
-      return Store.setItemAndWait(name, client.registrationAccessToken!);
-    }, log);
-  } catch {
-    throw Error(
-      `Failed to set token in store for client '${client.clientId}', package ` +
-        `${pkg.metadata?.namespace}/${pkg.metadata?.name}`,
-    );
-  }
-
   // Remove the registrationAccessToken from the client object to avoid problems (one-time use token)
   delete client.registrationAccessToken;
 
@@ -221,43 +201,6 @@ async function syncClient(
   return client;
 }
 
-async function apiCall(client: Partial<Client>, method = "POST", authToken = "") {
-  const req = {
-    body: JSON.stringify(client) as string | undefined,
-    method,
-    headers: {
-      "Content-Type": "application/json",
-    } as Record<string, string>,
-  };
-
-  let url = apiURL;
-
-  // When not creating a new client, add the client ID and registrationAccessToken
-  if (authToken) {
-    req.headers.Authorization = `Bearer ${authToken}`;
-    // Ensure that we URI encode the clientId in the request URL
-    url += `/${encodeURIComponent(client.clientId!)}`;
-  }
-
-  // Remove the body for DELETE requests
-  if (method === "DELETE" || method === "GET") {
-    delete req.body;
-  }
-
-  // Make the request
-  const resp = await fetch<Client>(url, req);
-
-  if (!resp.ok) {
-    if (resp.data) {
-      throw new Error(`${JSON.stringify(resp.statusText)}, ${JSON.stringify(resp.data)}`);
-    } else {
-      throw new Error(`${JSON.stringify(resp.statusText)}`);
-    }
-  }
-
-  return resp.data;
-}
-
 export function generateSecretData(client: Client, secretTemplate?: { [key: string]: string }) {
   if (secretTemplate) {
     log.debug(`Using secret template for client: ${client.clientId}`);

src/pepr/operator/controllers/keycloak/keycloak-client.spec.ts

@@ -0,0 +1,60 @@
+/**
+ * Copyright 2025 Defense Unicorns
+ * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+ */
+
+import {
+  ClientCredentialsKeycloakClient,
+  DynamicClientRegistrationClient,
+  DynamicKeycloakClient,
+} from "./keycloak-client";
+
+describe("pickImplementation method", () => {
+  it("should return DynamicClientRegistrationClient when ENV_KEYCLOAK_CLIENT_IMPLEMENTATION is dynamic_client_registration", async () => {
+    process.env.PEPR_KEYCLOAK_CLIENT_STRATEGY =
+      DynamicKeycloakClient.ENV_KEYCLOAK_CLIENT_IMPLEMENTATION_DYNAMIC;
+    const client = new DynamicKeycloakClient("http://localhost");
+    const implementation = await client.pickImplementation();
+    expect(implementation).toBeInstanceOf(DynamicClientRegistrationClient);
+  });
+
+  it("should return ClientCredentialsKeycloakClient when ENV_KEYCLOAK_CLIENT_IMPLEMENTATION is client_credentials", async () => {
+    process.env.PEPR_KEYCLOAK_CLIENT_STRATEGY =
+      DynamicKeycloakClient.ENV_KEYCLOAK_CLIENT_IMPLEMENTATION_CLIENT_CREDENTIALS;
+    const client = new DynamicKeycloakClient("http://localhost");
+    const implementation = await client.pickImplementation();
+    expect(implementation).toBeInstanceOf(ClientCredentialsKeycloakClient);
+  });
+
+  it("should fallback to DynamicClientRegistrationClient if ClientCredentialsKeycloakClient is not properly configured", async () => {
+    delete process.env.PEPR_KEYCLOAK_CLIENT_STRATEGY;
+    const client = new DynamicKeycloakClient("http://localhost");
+    jest
+      .spyOn(client["clientCredentialsKeycloakClient"], "getAccessToken")
+      .mockImplementation(() => {
+        throw new Error("Client Credentials Keycloak Client is not properly configured");
+      });
+    const implementation = await client.pickImplementation();
+    expect(implementation).toBeInstanceOf(DynamicClientRegistrationClient);
+  });
+
+  it("should fallback use ClientCredentialsKeycloakClient if it is properly configured", async () => {
+    delete process.env.PEPR_KEYCLOAK_CLIENT_STRATEGY;
+    const client = new DynamicKeycloakClient("http://localhost");
+    jest
+      .spyOn(client["clientCredentialsKeycloakClient"], "getAccessToken")
+      .mockImplementation(async () => {
+        return Promise.resolve("mockAccessToken");
+      });
+    const implementation = await client.pickImplementation();
+    expect(implementation).toBeInstanceOf(ClientCredentialsKeycloakClient);
+  });
+
+  it("should throw an error for invalid Keycloak Client implementation", async () => {
+    process.env.PEPR_KEYCLOAK_CLIENT_STRATEGY = "invalid_strategy";
+    const client = new DynamicKeycloakClient("http://localhost");
+    await expect(client.pickImplementation()).rejects.toThrow(
+      "Invalid Keycloak Client implementation: invalid_strategy. Supported values: dynamic_client_registration, client_credentials",
+    );
+  });
+});