diff --git a/documentation/domains/Domain.json b/documentation/domains/Domain.json index faa96bd81d1..2249a07e5a5 100644 --- a/documentation/domains/Domain.json +++ b/documentation/domains/Domain.json @@ -715,6 +715,11 @@ "ProbeTuning": { "type": "object", "properties": { + "failureThreshold": { + "description": "Number of times the check is performed before giving up. Giving up in case of liveness probe means restarting the container. In case of readiness probe, the Pod will be marked Unready. Defaults to 1.", + "type": "number", + "minimum": 1 + }, "periodSeconds": { "description": "The number of seconds between checks.", "type": "number" @@ -723,6 +728,11 @@ "description": "The number of seconds with no response that indicates a failure.", "type": "number" }, + "successThreshold": { + "description": "Minimum number of times the check needs to pass for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness Probe.", + "type": "number", + "minimum": 1 + }, "initialDelaySeconds": { "description": "The number of seconds before the first check is performed.", "type": "number" diff --git a/documentation/domains/Domain.md b/documentation/domains/Domain.md index dc26c86cbd7..1b5fda99b26 100644 --- a/documentation/domains/Domain.md +++ b/documentation/domains/Domain.md @@ -255,8 +255,10 @@ The current status of the operation of the WebLogic domain. Updated automaticall | Name | Type | Description | | --- | --- | --- | +| `failureThreshold` | number | Number of times the check is performed before giving up. Giving up in case of liveness probe means restarting the container. In case of readiness probe, the Pod will be marked Unready. Defaults to 1. | | `initialDelaySeconds` | number | The number of seconds before the first check is performed. | | `periodSeconds` | number | The number of seconds between checks. | +| `successThreshold` | number | Minimum number of times the check needs to pass for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness Probe. | | `timeoutSeconds` | number | The number of seconds with no response that indicates a failure. | ### Shutdown diff --git a/documentation/domains/index.html b/documentation/domains/index.html index 3f61baa1531..7196af88db8 100644 --- a/documentation/domains/index.html +++ b/documentation/domains/index.html @@ -1636,6 +1636,11 @@ "ProbeTuning": { "type": "object", "properties": { + "failureThreshold": { + "description": "Number of times the check is performed before giving up. Giving up in case of liveness probe means restarting the container. In case of readiness probe, the Pod will be marked Unready. Defaults to 1.", + "type": "number", + "minimum": 1.0 + }, "periodSeconds": { "description": "The number of seconds between checks.", "type": "number" @@ -1644,6 +1649,11 @@ "description": "The number of seconds with no response that indicates a failure.", "type": "number" }, + "successThreshold": { + "description": "Minimum number of times the check needs to pass for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness Probe.", + "type": "number", + "minimum": 1.0 + }, "initialDelaySeconds": { "description": "The number of seconds before the first check is performed.", "type": "number" diff --git a/documentation/staging/content/userguide/managing-domains/domain-lifecycle/liveness-readiness-probe-customization.md b/documentation/staging/content/userguide/managing-domains/domain-lifecycle/liveness-readiness-probe-customization.md index 0f407e5c28b..0c5c075f074 100644 --- a/documentation/staging/content/userguide/managing-domains/domain-lifecycle/liveness-readiness-probe-customization.md +++ b/documentation/staging/content/userguide/managing-domains/domain-lifecycle/liveness-readiness-probe-customization.md @@ -15,18 +15,21 @@ This document describes how to customize the liveness and readiness probes for W #### Liveness probe customization -The liveness probe is configured to check that a server is alive by querying the Node Manager process. By default, the liveness probe is configured to check liveness every 45 seconds and to timeout after 5 seconds. If a pod fails the liveness probe, Kubernetes will restart that container. +The liveness probe is configured to check that a server is alive by querying the Node Manager process. By default, the liveness probe is configured to check liveness every 45 seconds, to timeout after 5 seconds, and to perform the first check after 30 seconds. The default success and failure threshold values are 1. If a pod fails the liveness probe, Kubernetes will restart that container. -You can customize the liveness probe interval and timeout using the `livenessProbe` attribute under the `serverPod` element of the domain resource. +You can customize the liveness probe initial delay, interval, timeout and failure threshold using the `livenessProbe` attribute under the `serverPod` element of the domain resource. -Following is an example configuration to change the liveness probe interval and timeout value. +Following is an example configuration to change the liveness probe interval, timeout and failure threshold value. ```yaml serverPod: livenessProbe: periodSeconds: 30 timeoutSeconds: 10 + failureThreshold: 3 ``` +**Note**: The liveness probe success threshold value must always be 1. See [Configure Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes) in the Kubernetes documentation for more details. + After the liveness probe script (livenessProbe.sh) performs its normal checks, you can customize the liveness probe by specifying a custom script, which will be invoked by livenessProbe.sh. You can specify the custom script either by using the `livenessProbeCustomScript` attribute in the domain resource or by setting the `LIVENESS_PROBE_CUSTOM_SCRIPT` environment variable using the `env` attribute under the `serverPod` element (see the configuration examples below). If the custom script fails with a non-zero exit status, the liveness probe will fail and Kubernetes will restart the container. @@ -82,12 +85,13 @@ The following operator-populated environment variables are available for use in By default, the readiness probe is configured to use the WebLogic Server ReadyApp framework. The ReadyApp framework allows fine customization of the readiness probe by the application's participation in the framework. For more details, see [Using the ReadyApp Framework](https://docs.oracle.com/en/middleware/fusion-middleware/weblogic-server/12.2.1.4/depgd/managing.html#GUID-C98443B1-D368-4CA4-A7A4-97B86FFD3C28). The readiness probe is used to determine if the server is ready to accept user requests. The readiness is used to determine when a server should be included in a load balancer's endpoints, in the case of a rolling restart, when a restarted server is fully started, and for various other purposes. -By default, the readiness probe is configured to check readiness every 5 seconds and to timeout after 5 seconds. You can customize the readiness probe interval and timeout using the `readinessProbe` attribute under the `serverPod` element of the domain resource. +By default, the readiness probe is configured to check readiness every 5 seconds, to timeout after 5 seconds, and to perform the first check after 30 seconds. The default success and failure thresholds values are 1. You can customize the readiness probe initial delay, interval, timeout, success and failure thresholds using the `readinessProbe` attribute under the `serverPod` element of the domain resource. -Following is an example configuration to change readiness probe interval and timeout value. +Following is an example configuration to change readiness probe interval, timeout and failure threshold value. ```yaml serverPod: readinessProbe: periodSeconds: 10 timeoutSeconds: 10 + failureThreshold: 3 ``` diff --git a/kubernetes/crd/domain-crd.yaml b/kubernetes/crd/domain-crd.yaml index 7cf0f652053..d29ba3b9994 100644 --- a/kubernetes/crd/domain-crd.yaml +++ b/kubernetes/crd/domain-crd.yaml @@ -5,7 +5,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - weblogic.sha256: 82e9e9e60da33f393aec1dc45c2854e06a569fea839d3339e3dd7f86ce8d837c + weblogic.sha256: 1948b4415bf4ebfc7637e2b3f75150cc8557624303df5f759124ce2a0d238d3c name: domains.weblogic.oracle spec: group: weblogic.oracle @@ -367,6 +367,13 @@ spec: description: Settings for the liveness probe associated with a WebLogic Server instance. properties: + failureThreshold: + description: Number of times the check is performed before + giving up. Giving up in case of liveness probe means + restarting the container. In case of readiness probe, + the Pod will be marked Unready. Defaults to 1. + minimum: 1.0 + type: number periodSeconds: description: The number of seconds between checks. type: number @@ -374,6 +381,13 @@ spec: description: The number of seconds with no response that indicates a failure. type: number + successThreshold: + description: Minimum number of times the check needs to + pass for the probe to be considered successful after + having failed. Defaults to 1. Must be 1 for liveness + Probe. + minimum: 1.0 + type: number initialDelaySeconds: description: The number of seconds before the first check is performed. @@ -1414,6 +1428,13 @@ spec: description: Settings for the readiness probe associated with a WebLogic Server instance. properties: + failureThreshold: + description: Number of times the check is performed before + giving up. Giving up in case of liveness probe means + restarting the container. In case of readiness probe, + the Pod will be marked Unready. Defaults to 1. + minimum: 1.0 + type: number periodSeconds: description: The number of seconds between checks. type: number @@ -1421,6 +1442,13 @@ spec: description: The number of seconds with no response that indicates a failure. type: number + successThreshold: + description: Minimum number of times the check needs to + pass for the probe to be considered successful after + having failed. Defaults to 1. Must be 1 for liveness + Probe. + minimum: 1.0 + type: number initialDelaySeconds: description: The number of seconds before the first check is performed. @@ -3111,6 +3139,14 @@ spec: with a WebLogic Server instance. type: object properties: + failureThreshold: + description: Number of times the check is performed + before giving up. Giving up in case of liveness probe + means restarting the container. In case of readiness + probe, the Pod will be marked Unready. Defaults to + 1. + type: number + minimum: 1.0 periodSeconds: description: The number of seconds between checks. type: number @@ -3118,6 +3154,13 @@ spec: description: The number of seconds with no response that indicates a failure. type: number + successThreshold: + description: Minimum number of times the check needs + to pass for the probe to be considered successful + after having failed. Defaults to 1. Must be 1 for + liveness Probe. + type: number + minimum: 1.0 initialDelaySeconds: description: The number of seconds before the first check is performed. @@ -4162,6 +4205,14 @@ spec: with a WebLogic Server instance. type: object properties: + failureThreshold: + description: Number of times the check is performed + before giving up. Giving up in case of liveness probe + means restarting the container. In case of readiness + probe, the Pod will be marked Unready. Defaults to + 1. + type: number + minimum: 1.0 periodSeconds: description: The number of seconds between checks. type: number @@ -4169,6 +4220,13 @@ spec: description: The number of seconds with no response that indicates a failure. type: number + successThreshold: + description: Minimum number of times the check needs + to pass for the probe to be considered successful + after having failed. Defaults to 1. Must be 1 for + liveness Probe. + type: number + minimum: 1.0 initialDelaySeconds: description: The number of seconds before the first check is performed. @@ -5880,6 +5938,13 @@ spec: description: Settings for the liveness probe associated with a WebLogic Server instance. properties: + failureThreshold: + description: Number of times the check is performed before + giving up. Giving up in case of liveness probe means restarting + the container. In case of readiness probe, the Pod will + be marked Unready. Defaults to 1. + minimum: 1.0 + type: number periodSeconds: description: The number of seconds between checks. type: number @@ -5887,6 +5952,12 @@ spec: description: The number of seconds with no response that indicates a failure. type: number + successThreshold: + description: Minimum number of times the check needs to pass + for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness Probe. + minimum: 1.0 + type: number initialDelaySeconds: description: The number of seconds before the first check is performed. @@ -6921,6 +6992,13 @@ spec: description: Settings for the readiness probe associated with a WebLogic Server instance. properties: + failureThreshold: + description: Number of times the check is performed before + giving up. Giving up in case of liveness probe means restarting + the container. In case of readiness probe, the Pod will + be marked Unready. Defaults to 1. + minimum: 1.0 + type: number periodSeconds: description: The number of seconds between checks. type: number @@ -6928,6 +7006,12 @@ spec: description: The number of seconds with no response that indicates a failure. type: number + successThreshold: + description: Minimum number of times the check needs to pass + for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness Probe. + minimum: 1.0 + type: number initialDelaySeconds: description: The number of seconds before the first check is performed. @@ -8532,6 +8616,14 @@ spec: with a WebLogic Server instance. type: object properties: + failureThreshold: + description: Number of times the check is performed + before giving up. Giving up in case of liveness probe + means restarting the container. In case of readiness + probe, the Pod will be marked Unready. Defaults to + 1. + type: number + minimum: 1.0 periodSeconds: description: The number of seconds between checks. type: number @@ -8539,6 +8631,13 @@ spec: description: The number of seconds with no response that indicates a failure. type: number + successThreshold: + description: Minimum number of times the check needs + to pass for the probe to be considered successful + after having failed. Defaults to 1. Must be 1 for + liveness Probe. + type: number + minimum: 1.0 initialDelaySeconds: description: The number of seconds before the first check is performed. @@ -9583,6 +9682,14 @@ spec: with a WebLogic Server instance. type: object properties: + failureThreshold: + description: Number of times the check is performed + before giving up. Giving up in case of liveness probe + means restarting the container. In case of readiness + probe, the Pod will be marked Unready. Defaults to + 1. + type: number + minimum: 1.0 periodSeconds: description: The number of seconds between checks. type: number @@ -9590,6 +9697,13 @@ spec: description: The number of seconds with no response that indicates a failure. type: number + successThreshold: + description: Minimum number of times the check needs + to pass for the probe to be considered successful + after having failed. Defaults to 1. Must be 1 for + liveness Probe. + type: number + minimum: 1.0 initialDelaySeconds: description: The number of seconds before the first check is performed. diff --git a/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java b/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java index 04dbc4f78f3..d361ed98ec3 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java @@ -33,7 +33,7 @@ public interface KubernetesConstants { int DEFAULT_MAX_CLUSTER_CONCURRENT_START_UP = 0; int DEFAULT_MAX_CLUSTER_CONCURRENT_SHUTDOWN = 1; - String CONTAINER_NAME = "weblogic-server"; + String WLS_CONTAINER_NAME = "weblogic-server"; String SCRIPT_CONFIG_MAP_NAME = "weblogic-scripts-cm"; String DOMAIN_DEBUG_CONFIG_MAP_SUFFIX = "-weblogic-domain-debug-cm"; diff --git a/operator/src/main/java/oracle/kubernetes/operator/ServerStatusReader.java b/operator/src/main/java/oracle/kubernetes/operator/ServerStatusReader.java index e41b9de2f01..164d2b1b9e6 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/ServerStatusReader.java +++ b/operator/src/main/java/oracle/kubernetes/operator/ServerStatusReader.java @@ -40,7 +40,7 @@ import oracle.kubernetes.utils.SystemClock; import oracle.kubernetes.weblogic.domain.model.ServerHealth; -import static oracle.kubernetes.operator.KubernetesConstants.CONTAINER_NAME; +import static oracle.kubernetes.operator.KubernetesConstants.WLS_CONTAINER_NAME; import static oracle.kubernetes.operator.ProcessingConstants.SERVER_HEALTH_MAP; import static oracle.kubernetes.operator.ProcessingConstants.SERVER_STATE_MAP; @@ -177,7 +177,7 @@ public NextAction apply(Packet packet) { try (LoggingContext stack = LoggingContext.setThreadContext().namespace(getNamespace(pod)).domainUid(getDomainUid(pod))) { - KubernetesExec kubernetesExec = EXEC_FACTORY.create(client, pod, CONTAINER_NAME); + KubernetesExec kubernetesExec = EXEC_FACTORY.create(client, pod, WLS_CONTAINER_NAME); kubernetesExec.setStdin(stdin); kubernetesExec.setTty(tty); proc = kubernetesExec.exec("/weblogic-operator/scripts/readState.sh"); diff --git a/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java b/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java index 445a97a44bd..022edb81959 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java +++ b/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java @@ -246,9 +246,13 @@ class PodTuning { public final int readinessProbeInitialDelaySeconds; public final int readinessProbeTimeoutSeconds; public final int readinessProbePeriodSeconds; + public final int readinessProbeSuccessThreshold; + public final int readinessProbeFailureThreshold; public final int livenessProbeInitialDelaySeconds; public final int livenessProbeTimeoutSeconds; public final int livenessProbePeriodSeconds; + public final int livenessProbeSuccessThreshold; + public final int livenessProbeFailureThreshold; public final long introspectorJobActiveDeadlineSeconds; /** @@ -256,25 +260,37 @@ class PodTuning { * @param readinessProbeInitialDelaySeconds readiness probe initial delay * @param readinessProbeTimeoutSeconds readiness probe timeout * @param readinessProbePeriodSeconds rediness probe period + * @param readinessProbeSuccessThreshold readiness probe success threshold + * @param readinessProbeFailureThreshold readiness probe failure threshold * @param livenessProbeInitialDelaySeconds liveness probe initial delay * @param livenessProbeTimeoutSeconds liveness probe timeout * @param livenessProbePeriodSeconds liveness probe period + * @param livenessProbeSuccessThreshold liveness probe success threshold + * @param livenessProbeFailureThreshold liveness probe failure threshold * @param introspectorJobActiveDeadlineSeconds introspector job active deadline */ public PodTuning( int readinessProbeInitialDelaySeconds, int readinessProbeTimeoutSeconds, int readinessProbePeriodSeconds, + int readinessProbeSuccessThreshold, + int readinessProbeFailureThreshold, int livenessProbeInitialDelaySeconds, int livenessProbeTimeoutSeconds, int livenessProbePeriodSeconds, + int livenessProbeSuccessThreshold, + int livenessProbeFailureThreshold, long introspectorJobActiveDeadlineSeconds) { this.readinessProbeInitialDelaySeconds = readinessProbeInitialDelaySeconds; this.readinessProbeTimeoutSeconds = readinessProbeTimeoutSeconds; this.readinessProbePeriodSeconds = readinessProbePeriodSeconds; + this.readinessProbeSuccessThreshold = readinessProbeSuccessThreshold; + this.readinessProbeFailureThreshold = readinessProbeFailureThreshold; this.livenessProbeInitialDelaySeconds = livenessProbeInitialDelaySeconds; this.livenessProbeTimeoutSeconds = livenessProbeTimeoutSeconds; this.livenessProbePeriodSeconds = livenessProbePeriodSeconds; + this.livenessProbeSuccessThreshold = livenessProbeSuccessThreshold; + this.livenessProbeFailureThreshold = livenessProbeFailureThreshold; this.introspectorJobActiveDeadlineSeconds = introspectorJobActiveDeadlineSeconds; } @@ -284,9 +300,13 @@ public String toString() { .append("readinessProbeInitialDelaySeconds", readinessProbeInitialDelaySeconds) .append("readinessProbeTimeoutSeconds", readinessProbeTimeoutSeconds) .append("readinessProbePeriodSeconds", readinessProbePeriodSeconds) + .append("readinessProbeSuccessThreshold", readinessProbeSuccessThreshold) + .append("readinessProbeFailureThreshold", readinessProbeFailureThreshold) .append("livenessProbeInitialDelaySeconds", livenessProbeInitialDelaySeconds) .append("livenessProbeTimeoutSeconds", livenessProbeTimeoutSeconds) .append("livenessProbePeriodSeconds", livenessProbePeriodSeconds) + .append("livenessProbeSuccessThreshold", livenessProbeSuccessThreshold) + .append("livenessProbeFailureThreshold", livenessProbeFailureThreshold) .toString(); } @@ -296,9 +316,13 @@ public int hashCode() { .append(readinessProbeInitialDelaySeconds) .append(readinessProbeTimeoutSeconds) .append(readinessProbePeriodSeconds) + .append(readinessProbeSuccessThreshold) + .append(readinessProbeFailureThreshold) .append(livenessProbeInitialDelaySeconds) .append(livenessProbeTimeoutSeconds) .append(livenessProbePeriodSeconds) + .append(livenessProbeSuccessThreshold) + .append(livenessProbeFailureThreshold) .toHashCode(); } @@ -315,9 +339,13 @@ public boolean equals(Object o) { .append(readinessProbeInitialDelaySeconds, pt.readinessProbeInitialDelaySeconds) .append(readinessProbeTimeoutSeconds, pt.readinessProbeTimeoutSeconds) .append(readinessProbePeriodSeconds, pt.readinessProbePeriodSeconds) + .append(readinessProbeSuccessThreshold, pt.readinessProbeSuccessThreshold) + .append(readinessProbeFailureThreshold, pt.readinessProbeFailureThreshold) .append(livenessProbeInitialDelaySeconds, pt.livenessProbeInitialDelaySeconds) .append(livenessProbeTimeoutSeconds, pt.livenessProbeTimeoutSeconds) .append(livenessProbePeriodSeconds, pt.livenessProbePeriodSeconds) + .append(livenessProbeSuccessThreshold, pt.livenessProbeSuccessThreshold) + .append(livenessProbeFailureThreshold, pt.livenessProbeFailureThreshold) .isEquals(); } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java b/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java index 749c32f69f7..2e58cfedf19 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java +++ b/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java @@ -83,9 +83,13 @@ private void update() { (int) readTuningParameter("readinessProbeInitialDelaySeconds", 30), (int) readTuningParameter("readinessProbeTimeoutSeconds", 5), (int) readTuningParameter("readinessProbePeriodSeconds", 5), + (int) readTuningParameter("readinessProbeSuccessThreshold", 1), + (int) readTuningParameter("readinessProbeFailureThreshold", 1), (int) readTuningParameter("livenessProbeInitialDelaySeconds", 30), (int) readTuningParameter("livenessProbeTimeoutSeconds", 5), (int) readTuningParameter("livenessProbePeriodSeconds", 45), + (int) readTuningParameter("livenessProbeSuccessThreshold", 1), + (int) readTuningParameter("livenessProbeFailureThreshold", 1), readTuningParameter("introspectorJobActiveDeadlineSeconds", 120)); FeatureGates featureGates = diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/BasePodStepContext.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/BasePodStepContext.java index bd8d3ae7469..6e35d03ab93 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/BasePodStepContext.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/BasePodStepContext.java @@ -310,7 +310,7 @@ protected boolean isK8sContainer(V1Container c) { } protected String getMainContainerName() { - return KubernetesConstants.CONTAINER_NAME; + return KubernetesConstants.WLS_CONTAINER_NAME; } protected String getAuxiliaryImagePaths(List auxiliaryImages, diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java index 753c9257ed4..3aefa02a4cc 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java @@ -876,7 +876,7 @@ private V1VolumeMount createRuntimeEncryptionSecretVolumeMount() { } protected String getContainerName() { - return KubernetesConstants.CONTAINER_NAME; + return KubernetesConstants.WLS_CONTAINER_NAME; } protected List getContainerCommand() { @@ -974,7 +974,13 @@ private V1Probe createReadinessProbe(TuningParameters.PodTuning tuning) { .initialDelaySeconds(getReadinessProbeInitialDelaySeconds(tuning)) .timeoutSeconds(getReadinessProbeTimeoutSeconds(tuning)) .periodSeconds(getReadinessProbePeriodSeconds(tuning)) - .failureThreshold(FAILURE_THRESHOLD); + .failureThreshold(getReadinessProbeFailureThreshold(tuning)); + + // Add the success threshold only if the value is non-default to avoid pod roll. + if (getReadinessProbeSuccessThreshold(tuning) != DEFAULT_SUCCESS_THRESHOLD) { + readinessProbe.successThreshold(getReadinessProbeSuccessThreshold(tuning)); + } + try { boolean istioEnabled = getDomain().isIstioEnabled(); if (istioEnabled) { @@ -1020,13 +1026,28 @@ private int getReadinessProbeInitialDelaySeconds(TuningParameters.PodTuning tuni .orElse(tuning.readinessProbeInitialDelaySeconds); } + private int getReadinessProbeSuccessThreshold(TuningParameters.PodTuning tuning) { + return Optional.ofNullable(getServerSpec().getReadinessProbe().getSuccessThreshold()) + .orElse(tuning.readinessProbeSuccessThreshold); + } + + private int getReadinessProbeFailureThreshold(TuningParameters.PodTuning tuning) { + return Optional.ofNullable(getServerSpec().getReadinessProbe().getFailureThreshold()) + .orElse(tuning.readinessProbeFailureThreshold); + } + private V1Probe createLivenessProbe(TuningParameters.PodTuning tuning) { - return new V1Probe() + V1Probe livenessProbe = new V1Probe() .initialDelaySeconds(getLivenessProbeInitialDelaySeconds(tuning)) .timeoutSeconds(getLivenessProbeTimeoutSeconds(tuning)) .periodSeconds(getLivenessProbePeriodSeconds(tuning)) - .failureThreshold(FAILURE_THRESHOLD) - .exec(execAction(LIVENESS_PROBE)); + .failureThreshold(getLivenessProbeFailureThreshold(tuning)); + + // Add the success threshold only if the value is non-default to avoid pod roll. + if (getLivenessProbeSuccessThreshold(tuning) != DEFAULT_SUCCESS_THRESHOLD) { + livenessProbe.successThreshold(getLivenessProbeSuccessThreshold(tuning)); + } + return livenessProbe.exec(execAction(LIVENESS_PROBE)); } private int getLivenessProbeInitialDelaySeconds(TuningParameters.PodTuning tuning) { @@ -1044,6 +1065,16 @@ private int getLivenessProbePeriodSeconds(TuningParameters.PodTuning tuning) { .orElse(tuning.livenessProbePeriodSeconds); } + private int getLivenessProbeSuccessThreshold(TuningParameters.PodTuning tuning) { + return Optional.ofNullable(getServerSpec().getLivenessProbe().getSuccessThreshold()) + .orElse(tuning.livenessProbeSuccessThreshold); + } + + private int getLivenessProbeFailureThreshold(TuningParameters.PodTuning tuning) { + return Optional.ofNullable(getServerSpec().getLivenessProbe().getFailureThreshold()) + .orElse(tuning.livenessProbeFailureThreshold); + } + private boolean mockWls() { return Boolean.getBoolean("mockWLS"); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/StepContextConstants.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/StepContextConstants.java index 2526190f079..80bfe3e523e 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/StepContextConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/StepContextConstants.java @@ -22,7 +22,7 @@ public interface StepContextConstants { String OPSS_WALLETFILE_MOUNT_PATH = "/weblogic-operator/opss-walletfile-secret"; String DEBUG_CM_MOUNTS_PATH = "/weblogic-operator/debug"; String NODEMGR_HOME = "/u01/nodemanager"; - int FAILURE_THRESHOLD = 1; + int DEFAULT_SUCCESS_THRESHOLD = 1; @SuppressWarnings("OctalInteger") int ALL_READ_AND_EXECUTE = 0555; diff --git a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java index 3f716f001e7..8b600f1b4ab 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java +++ b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java @@ -174,6 +174,8 @@ public class MessageKeys { public static final String AUXILIARY_IMAGE_VOLUME_NAME_NOT_DEFINED = "WLSDO-0026"; public static final String MONITORING_EXPORTER_CONFLICT_SERVER = "WLSDO-0027"; public static final String MONITORING_EXPORTER_CONFLICT_DYNAMIC_CLUSTER = "WLSDO-0028"; + public static final String INVALID_LIVENESS_PROBE_SUCCESS_THRESHOLD_VALUE = "WLSDO-0029"; + public static final String RESERVED_CONTAINER_NAME = "WLSDO-0030"; private MessageKeys() { } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/ClusterConfigurator.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/ClusterConfigurator.java index 4ad48bed067..335ae9e6e1b 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/ClusterConfigurator.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/ClusterConfigurator.java @@ -26,11 +26,13 @@ public interface ClusterConfigurator extends ServiceConfigurator { ClusterConfigurator withServerStartPolicy(String policy); - ClusterConfigurator withReadinessProbeSettings( - Integer initialDelay, Integer timeout, Integer period); + ClusterConfigurator withReadinessProbeSettings(Integer initialDelay, Integer timeout, Integer period); - ClusterConfigurator withLivenessProbeSettings( - Integer initialDelay, Integer timeout, Integer period); + ClusterConfigurator withReadinessProbeThresholds(Integer successThreshold, Integer failureThreshold); + + ClusterConfigurator withLivenessProbeSettings(Integer initialDelay, Integer timeout, Integer period); + + ClusterConfigurator withLivenessProbeThresholds(Integer successThreshold, Integer failureThreshold); /** * Add a node label to the Cluster's node selector. diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/DomainConfigurator.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/DomainConfigurator.java index e0ed09c94ef..613e5d80c15 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/DomainConfigurator.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/DomainConfigurator.java @@ -223,8 +223,7 @@ public abstract DomainConfigurator withConfigOverrideDistributionStrategy( * @param timeout the default timeout, in seconds. * @param period the default probe period, in seconds. */ - public abstract void withDefaultReadinessProbeSettings( - Integer initialDelay, Integer timeout, Integer period); + public abstract void withDefaultReadinessProbeSettings(Integer initialDelay, Integer timeout, Integer period); /** * Add auxiliary images for the domain resource. @@ -245,9 +244,18 @@ public abstract void withDefaultReadinessProbeSettings( * @param initialDelay the default initial delay, in seconds. * @param timeout the default timeout, in seconds. * @param period the default probe period, in seconds. + * + */ + public abstract void withDefaultLivenessProbeSettings(Integer initialDelay, Integer timeout, Integer period); + + /** + * Sets the default thresholds for the liveness probe. Any thresholds left null will default to the + * tuning parameters. + * + * @param successThreshold the default success threshold. + * @param failureThreshold the default failure threshold. */ - public abstract void withDefaultLivenessProbeSettings( - Integer initialDelay, Integer timeout, Integer period); + public abstract void withDefaultLivenessProbeThresholds(Integer successThreshold, Integer failureThreshold); /** * Sets the default server start policy ("ALWAYS", "NEVER" or "IF_NEEDED") for the domain. diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/ServerConfigurator.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/ServerConfigurator.java index 2ecfa4e2da4..2b33a7e6bd6 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/ServerConfigurator.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/ServerConfigurator.java @@ -25,11 +25,13 @@ public interface ServerConfigurator extends ServiceConfigurator { ServerConfigurator withServerStartPolicy(String startNever); - ServerConfigurator withLivenessProbeSettings( - Integer initialDelay, Integer timeout, Integer period); + ServerConfigurator withLivenessProbeSettings(Integer initialDelay, Integer timeout, Integer period); - ServerConfigurator withReadinessProbeSettings( - Integer initialDelay, Integer timeout, Integer period); + ServerConfigurator withLivenessProbeThresholds(Integer successThreshold, Integer failureThreshold); + + ServerConfigurator withReadinessProbeSettings(Integer initialDelay, Integer timeout, Integer period); + + ServerConfigurator withReadinessProbeThresholds(Integer successThreshold, Integer failureThreshold); /** * Add a node label to the Servers's node selector. diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/BaseConfiguration.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/BaseConfiguration.java index 6cc9dac4745..3f7e7b0a75b 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/BaseConfiguration.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/BaseConfiguration.java @@ -145,6 +145,10 @@ void setLivenessProbe(Integer initialDelay, Integer timeout, Integer period) { serverPod.setLivenessProbe(initialDelay, timeout, period); } + void setLivenessProbeThresholds(Integer successThreshold, Integer failureThreshold) { + serverPod.setLivenessProbeThresholds(successThreshold, failureThreshold); + } + ProbeTuning getLivenessProbe() { return serverPod.getLivenessProbeTuning(); } @@ -153,6 +157,10 @@ void setReadinessProbe(Integer initialDelay, Integer timeout, Integer period) { serverPod.setReadinessProbeTuning(initialDelay, timeout, period); } + void setReadinessProbeThresholds(Integer successThreshold, Integer failureThreshold) { + serverPod.setReadinessProbeThresholds(successThreshold, failureThreshold); + } + ProbeTuning getReadinessProbe() { return serverPod.getReadinessProbeTuning(); } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Domain.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Domain.java index 8864c03927f..5350830c4ab 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Domain.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Domain.java @@ -20,6 +20,7 @@ import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; import io.kubernetes.client.common.KubernetesObject; +import io.kubernetes.client.openapi.models.V1Container; import io.kubernetes.client.openapi.models.V1EnvVar; import io.kubernetes.client.openapi.models.V1LocalObjectReference; import io.kubernetes.client.openapi.models.V1ObjectMeta; @@ -45,7 +46,9 @@ import org.apache.commons.lang3.builder.ToStringBuilder; import static java.util.stream.Collectors.toSet; +import static oracle.kubernetes.operator.KubernetesConstants.WLS_CONTAINER_NAME; import static oracle.kubernetes.operator.helpers.LegalNames.toDns1123LegalName; +import static oracle.kubernetes.operator.helpers.StepContextConstants.DEFAULT_SUCCESS_THRESHOLD; import static oracle.kubernetes.utils.OperatorUtils.emptyToNull; /** @@ -796,6 +799,9 @@ public List getAfterIntrospectValidationFailures(Packet packet) { } class Validator { + public static final String ADMIN_SERVER_POD_SPEC_PREFIX = "spec.adminServer.serverPod"; + public static final String CLUSTER_SPEC_PREFIX = "spec.clusters"; + public static final String MS_SPEC_PREFIX = "spec.managedServers"; private final List failures = new ArrayList<>(); private final Set clusterNames = new HashSet<>(); private final Set serverNames = new HashSet<>(); @@ -815,6 +821,8 @@ List getValidationFailures(KubernetesResourceLookup kubernetesResources) verifyAuxiliaryImages(); verifyAuxiliaryImageVolumes(); addDuplicateAuxiliaryImageVolumeNames(); + verifyLivenessProbeSuccessThreshold(); + verifyContainerNameValidInPodSpec(); return failures; } @@ -1060,6 +1068,46 @@ private void checkDuplicateAuxiliaryImageVolume(AuxiliaryImageVolume auxiliaryIm } } + private void verifyLivenessProbeSuccessThreshold() { + Optional.ofNullable(getAdminServerSpec().getLivenessProbe()) + .ifPresent(probe -> verifySuccessThresholdValue(probe, ADMIN_SERVER_POD_SPEC_PREFIX + + ".livenessProbe.successThreshold")); + getSpec().getClusters().forEach(cluster -> + Optional.ofNullable(cluster.getLivenessProbe()) + .ifPresent(probe -> verifySuccessThresholdValue(probe, CLUSTER_SPEC_PREFIX + "[" + + cluster.getClusterName() + "].serverPod.livenessProbe.successThreshold"))); + getSpec().getManagedServers().forEach(managedServer -> + Optional.ofNullable(managedServer.getLivenessProbe()) + .ifPresent(probe -> verifySuccessThresholdValue(probe, MS_SPEC_PREFIX + "[" + + managedServer.getServerName() + "].serverPod.livenessProbe.successThreshold"))); + } + + private void verifySuccessThresholdValue(ProbeTuning probe, String prefix) { + if (probe.getSuccessThreshold() != null && probe.getSuccessThreshold() != DEFAULT_SUCCESS_THRESHOLD) { + failures.add(DomainValidationMessages.invalidLivenessProbeSuccessThresholdValue( + probe.getSuccessThreshold(), prefix)); + } + } + + private void verifyContainerNameValidInPodSpec() { + getAdminServerSpec().getContainers().forEach(container -> + isContainerNameReserved(container, ADMIN_SERVER_POD_SPEC_PREFIX + ".containers")); + getSpec().getClusters().forEach(cluster -> + cluster.getContainers().forEach(container -> + isContainerNameReserved(container, CLUSTER_SPEC_PREFIX + "[" + cluster.getClusterName() + + "].serverPod.containers"))); + getSpec().getManagedServers().forEach(managedServer -> + managedServer.getContainers().forEach(container -> + isContainerNameReserved(container, MS_SPEC_PREFIX + "[" + managedServer.getServerName() + + "].serverPod.containers"))); + } + + private void isContainerNameReserved(V1Container container, String prefix) { + if (container.getName().equals(WLS_CONTAINER_NAME)) { + failures.add(DomainValidationMessages.reservedContainerName(container.getName(), prefix)); + } + } + @Nonnull private Set getEnvNames() { return Optional.ofNullable(spec.getEnv()).stream() diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainCommonConfigurator.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainCommonConfigurator.java index 77f7d2bc206..61470499294 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainCommonConfigurator.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainCommonConfigurator.java @@ -52,8 +52,7 @@ public AdminServerConfigurator configureAdminServer() { } @Override - public void withDefaultReadinessProbeSettings( - Integer initialDelay, Integer timeout, Integer period) { + public void withDefaultReadinessProbeSettings(Integer initialDelay, Integer timeout, Integer period) { getDomainSpec().setReadinessProbe(initialDelay, timeout, period); } @@ -68,11 +67,16 @@ public DomainConfigurator withAuxiliaryImageVolumes(List a } @Override - public void withDefaultLivenessProbeSettings( - Integer initialDelay, Integer timeout, Integer period) { + public void withDefaultLivenessProbeSettings(Integer initialDelay, Integer timeout, Integer period) { getDomainSpec().setLivenessProbe(initialDelay, timeout, period); } + + @Override + public void withDefaultLivenessProbeThresholds(Integer successThreshold, Integer failureThreshold) { + getDomainSpec().setLivenessProbeThresholds(successThreshold, failureThreshold); + } + @Override public DomainConfigurator withDefaultServerStartPolicy(String startPolicy) { getDomainSpec().setServerStartPolicy(startPolicy); @@ -478,19 +482,29 @@ public ServerConfigurator withServerStartPolicy(String policy) { } @Override - public ServerConfigurator withLivenessProbeSettings( - Integer initialDelay, Integer timeout, Integer period) { + public ServerConfigurator withLivenessProbeSettings(Integer initialDelay, Integer timeout, Integer period) { server.setLivenessProbe(initialDelay, timeout, period); return this; } @Override - public ServerConfigurator withReadinessProbeSettings( - Integer initialDelay, Integer timeout, Integer period) { + public ServerConfigurator withLivenessProbeThresholds(Integer successThreshold, Integer failureThreshold) { + server.setLivenessProbeThresholds(successThreshold, failureThreshold); + return this; + } + + @Override + public ServerConfigurator withReadinessProbeSettings(Integer initialDelay, Integer timeout, Integer period) { server.setReadinessProbe(initialDelay, timeout, period); return this; } + @Override + public ServerConfigurator withReadinessProbeThresholds(Integer successThreshold, Integer failureThreshold) { + server.setReadinessProbeThresholds(successThreshold, failureThreshold); + return this; + } + @Override public ServerConfigurator withRequestRequirement(String resource, String quantity) { server.addRequestRequirement(resource, quantity); @@ -663,19 +677,29 @@ public ClusterConfigurator withServerStartPolicy(String policy) { } @Override - public ClusterConfigurator withReadinessProbeSettings( - Integer initialDelay, Integer timeout, Integer period) { + public ClusterConfigurator withReadinessProbeSettings(Integer initialDelay, Integer timeout, Integer period) { cluster.setReadinessProbe(initialDelay, timeout, period); return this; } @Override - public ClusterConfigurator withLivenessProbeSettings( - Integer initialDelay, Integer timeout, Integer period) { + public ClusterConfigurator withReadinessProbeThresholds(Integer successThreshold, Integer failureThreshold) { + cluster.setReadinessProbeThresholds(successThreshold, failureThreshold); + return this; + } + + @Override + public ClusterConfigurator withLivenessProbeSettings(Integer initialDelay, Integer timeout, Integer period) { cluster.setLivenessProbe(initialDelay, timeout, period); return this; } + @Override + public ClusterConfigurator withLivenessProbeThresholds(Integer successThreshold, Integer failureThreshold) { + cluster.setLivenessProbeThresholds(successThreshold, failureThreshold); + return this; + } + @Override public ClusterConfigurator withNodeSelector(String labelKey, String labelValue) { cluster.addNodeSelector(labelKey, labelValue); diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainValidationMessages.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainValidationMessages.java index 1557f09e208..b0b95f0d501 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainValidationMessages.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainValidationMessages.java @@ -166,4 +166,12 @@ public static String exceedMaxExternalServiceName( public static String noAvailablePortToUse(String domainUid, String serverName) { return getMessage(MessageKeys.NO_AVAILABLE_PORT_TO_USE_FOR_REST, domainUid, serverName); } + + public static String invalidLivenessProbeSuccessThresholdValue(int value, String prefix) { + return getMessage(MessageKeys.INVALID_LIVENESS_PROBE_SUCCESS_THRESHOLD_VALUE, value, prefix); + } + + public static String reservedContainerName(String name, String prefix) { + return getMessage(MessageKeys.RESERVED_CONTAINER_NAME, name, prefix); + } } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ProbeTuning.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ProbeTuning.java index 8304e53a32c..bdc71313b84 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ProbeTuning.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ProbeTuning.java @@ -5,6 +5,7 @@ import com.google.gson.annotations.SerializedName; import oracle.kubernetes.json.Description; +import oracle.kubernetes.json.Range; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; @@ -21,6 +22,19 @@ public class ProbeTuning { @SerializedName("timeoutSeconds") private Integer timeoutSeconds = null; + @Description("Number of times the check is performed before giving up. Giving up in " + + "case of liveness probe means restarting the container. In case of readiness probe, the Pod will be " + + "marked Unready. Defaults to 1.") + @SerializedName("failureThreshold") + @Range(minimum = 1) + Integer failureThreshold = null; + + @Description("Minimum number of times the check needs to pass for the probe to be considered successful" + + " after having failed. Defaults to 1. Must be 1 for liveness Probe.") + @SerializedName("successThreshold") + @Range(minimum = 1) + private Integer successThreshold = null; + public ProbeTuning() { } @@ -34,6 +48,12 @@ void copyValues(ProbeTuning fromProbe) { if (periodSeconds == null) { periodSeconds(fromProbe.periodSeconds); } + if (successThreshold == null) { + successThreshold(fromProbe.successThreshold); + } + if (failureThreshold == null) { + failureThreshold(fromProbe.failureThreshold); + } } public Integer getInitialDelaySeconds() { @@ -63,12 +83,32 @@ public ProbeTuning timeoutSeconds(Integer timeoutSeconds) { return this; } + public Integer getSuccessThreshold() { + return successThreshold; + } + + public ProbeTuning successThreshold(Integer successThreshold) { + this.successThreshold = successThreshold; + return this; + } + + public Integer getFailureThreshold() { + return failureThreshold; + } + + public ProbeTuning failureThreshold(Integer failureThreshold) { + this.failureThreshold = failureThreshold; + return this; + } + @Override public String toString() { return new ToStringBuilder(this) .append("initialDelaySeconds", initialDelaySeconds) .append("periodSeconds", periodSeconds) .append("timeoutSeconds", timeoutSeconds) + .append("successThreshold", successThreshold) + .append("failureThreshold", failureThreshold) .toString(); } @@ -88,6 +128,8 @@ public boolean equals(Object o) { .append(initialDelaySeconds, that.initialDelaySeconds) .append(periodSeconds, that.periodSeconds) .append(timeoutSeconds, that.timeoutSeconds) + .append(successThreshold, that.successThreshold) + .append(failureThreshold, that.failureThreshold) .isEquals(); } @@ -97,6 +139,8 @@ public int hashCode() { .append(initialDelaySeconds) .append(periodSeconds) .append(timeoutSeconds) + .append(successThreshold) + .append(failureThreshold) .toHashCode(); } } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerPod.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerPod.java index 10f5fe2c95c..db6eb3890a8 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerPod.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerPod.java @@ -409,6 +409,12 @@ void setReadinessProbeTuning(Integer initialDelay, Integer timeout, Integer peri .periodSeconds(period); } + void setReadinessProbeThresholds(Integer successThreshold, Integer failureThreshold) { + this.readinessProbe + .successThreshold(successThreshold) + .failureThreshold(failureThreshold); + } + List getAuxiliaryImages() { return this.auxiliaryImages; } @@ -428,6 +434,12 @@ void setLivenessProbe(Integer initialDelay, Integer timeout, Integer period) { .periodSeconds(period); } + void setLivenessProbeThresholds(Integer successThreshold, Integer failureThreshold) { + this.livenessProbe + .successThreshold(successThreshold) + .failureThreshold(failureThreshold); + } + void fillInFrom(ServerPod serverPod1) { for (V1EnvVar var : serverPod1.getV1EnvVars()) { addIfMissing(var); diff --git a/operator/src/main/resources/Operator.properties b/operator/src/main/resources/Operator.properties index 709d3d3bd8f..3dce5d3da63 100644 --- a/operator/src/main/resources/Operator.properties +++ b/operator/src/main/resources/Operator.properties @@ -182,6 +182,8 @@ WLSDO-0026=An item under ''spec.auxiliaryImageVolumes'' in the domain resource d the auxiliary volumes defined under ''spec.auxiliaryImageVolumes''. WLSDO-0027=The Monitoring Exporter port ''{0}'' specified by ''spec.monitoringExporter.port'' conflicts with WebLogic Server instance ''{1}'' port ''{2}'' WLSDO-0028=The Monitoring Exporter port ''{0}'' specified by ''spec.monitoringExporter.port'' conflicts with Cluster ''{1}'' dynamic server template port ''{2}'' +WLSDO-0029=Invalid value ''{0}'' for the liveness probe success threshold under ''{1}''. The liveness probe successThreshold value must be 1. +WLSDO-0030=The container name ''{0}'' specified under ''{1}'' is reserved for use by the operator. oneEnvVar=variable multipleEnvVars=variables diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/Matchers.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/Matchers.java index e7bcd7b8c51..5af4c479d1e 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/Matchers.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/Matchers.java @@ -256,20 +256,26 @@ private String getReadable() { public static class ProbeMatcher extends org.hamcrest.TypeSafeDiagnosingMatcher { private static final Integer EXPECTED_FAILURE_THRESHOLD = 1; + private static final Integer EXPECTED_SUCCESS_THRESHOLD = null; private final Integer expectedInitialDelay; private final Integer expectedTimeout; private final Integer expectedPeriod; + private final Integer expectedSuccessThreshold; + private final Integer expectedFailureThreshold; - private ProbeMatcher(int expectedInitialDelay, int expectedTimeout, int expectedPeriod) { + private ProbeMatcher(int expectedInitialDelay, int expectedTimeout, int expectedPeriod, + Integer expectedSuccessThreshold, int expectedFailureThreshold) { this.expectedInitialDelay = expectedInitialDelay; this.expectedTimeout = expectedTimeout; this.expectedPeriod = expectedPeriod; + this.expectedFailureThreshold = expectedFailureThreshold; + this.expectedSuccessThreshold = expectedSuccessThreshold; } - public static ProbeMatcher hasExpectedTuning( - int expectedInitialDelay, int expectedTimeout, int expectedPeriod) { + public static ProbeMatcher hasExpectedTuning(int expectedInitialDelay, int expectedTimeout, int expectedPeriod, + Integer successThreshold, Integer failureThreshold) { return new ProbeMatcher( - expectedInitialDelay, expectedTimeout, expectedPeriod); + expectedInitialDelay, expectedTimeout, expectedPeriod, successThreshold, failureThreshold); } @Override @@ -277,7 +283,8 @@ protected boolean matchesSafely(V1Probe item, Description mismatchDescription) { if (Objects.equals(expectedInitialDelay, item.getInitialDelaySeconds()) && Objects.equals(expectedTimeout, item.getTimeoutSeconds()) && Objects.equals(expectedPeriod, item.getPeriodSeconds()) - && Objects.equals(EXPECTED_FAILURE_THRESHOLD, item.getFailureThreshold())) { + && Objects.equals(expectedSuccessThreshold, item.getSuccessThreshold()) + && Objects.equals(expectedFailureThreshold, item.getFailureThreshold())) { return true; } else { mismatchDescription @@ -287,6 +294,8 @@ protected boolean matchesSafely(V1Probe item, Description mismatchDescription) { .appendValue(item.getTimeoutSeconds()) .appendText(", period ") .appendValue(item.getPeriodSeconds()) + .appendText(", successThreshold ") + .appendValue(item.getSuccessThreshold()) .appendText(" and failureThreshold ") .appendValue(item.getFailureThreshold()); @@ -303,6 +312,8 @@ public void describeTo(Description description) { .appendValue(expectedTimeout) .appendText(", period ") .appendValue(expectedPeriod) + .appendText(", successThreshold ") + .appendValue(EXPECTED_SUCCESS_THRESHOLD) .appendText(" and failureThreshold ") .appendValue(EXPECTED_FAILURE_THRESHOLD); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java index 4668f7bcbbe..677781c1927 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java @@ -103,13 +103,13 @@ import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.NUM_CONFIG_MAPS; import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.SECRETS_MD_5; import static oracle.kubernetes.operator.KubernetesConstants.ALWAYS_IMAGEPULLPOLICY; -import static oracle.kubernetes.operator.KubernetesConstants.CONTAINER_NAME; import static oracle.kubernetes.operator.KubernetesConstants.DEFAULT_EXPORTER_SIDECAR_PORT; import static oracle.kubernetes.operator.KubernetesConstants.DEFAULT_IMAGE; import static oracle.kubernetes.operator.KubernetesConstants.DOMAIN_DEBUG_CONFIG_MAP_SUFFIX; import static oracle.kubernetes.operator.KubernetesConstants.EXPORTER_CONTAINER_NAME; import static oracle.kubernetes.operator.KubernetesConstants.IFNOTPRESENT_IMAGEPULLPOLICY; import static oracle.kubernetes.operator.KubernetesConstants.SCRIPT_CONFIG_MAP_NAME; +import static oracle.kubernetes.operator.KubernetesConstants.WLS_CONTAINER_NAME; import static oracle.kubernetes.operator.LabelConstants.MII_UPDATED_RESTART_REQUIRED_LABEL; import static oracle.kubernetes.operator.LabelConstants.OPERATOR_VERSION; import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_ROLL_START_EVENT_GENERATED; @@ -191,6 +191,9 @@ public abstract class PodHelperTestBase extends DomainValidationBaseTest { private static final int CONFIGURED_DELAY = 21; private static final int CONFIGURED_TIMEOUT = 27; private static final int CONFIGURED_PERIOD = 35; + public static final int CONFIGURED_FAILURE_THRESHOLD = 1; + public static final int CONFIGURED_SUCCESS_THRESHOLD = 2; + private static final Integer DEFAULT_SUCCESS_THRESHOLD = null; private static final String LOG_HOME = "/shared/logs"; private static final String NODEMGR_HOME = "/u01/nodemanager"; private static final String CONFIGMAP_VOLUME_NAME = "weblogic-scripts-cm-volume"; @@ -620,7 +623,7 @@ void whenPodCreatedWithLatestImage_useAlwaysPullPolicy() { V1Container v1Container = getCreatedPodSpecContainer(); - assertThat(v1Container.getName(), equalTo(CONTAINER_NAME)); + assertThat(v1Container.getName(), equalTo(WLS_CONTAINER_NAME)); assertThat(v1Container.getImage(), equalTo(LATEST_IMAGE)); assertThat(v1Container.getImagePullPolicy(), equalTo(ALWAYS_IMAGEPULLPOLICY)); } @@ -848,7 +851,8 @@ void whenPodCreated_livenessProbeHasLivenessCommand() { void whenPodCreated_livenessProbeHasDefinedTuning() { assertThat( getCreatedPodSpecContainer().getLivenessProbe(), - hasExpectedTuning(LIVENESS_INITIAL_DELAY, LIVENESS_TIMEOUT, LIVENESS_PERIOD)); + hasExpectedTuning(LIVENESS_INITIAL_DELAY, LIVENESS_TIMEOUT, LIVENESS_PERIOD, DEFAULT_SUCCESS_THRESHOLD, + CONFIGURED_FAILURE_THRESHOLD)); } @Test @@ -862,7 +866,8 @@ void whenPodCreated_readinessProbeHasReadinessCommand() { void whenPodCreated_readinessProbeHasDefinedTuning() { assertThat( getCreatedPodSpecContainer().getReadinessProbe(), - hasExpectedTuning(READINESS_INITIAL_DELAY, READINESS_TIMEOUT, READINESS_PERIOD)); + hasExpectedTuning(READINESS_INITIAL_DELAY, READINESS_TIMEOUT, READINESS_PERIOD, DEFAULT_SUCCESS_THRESHOLD, + CONFIGURED_FAILURE_THRESHOLD)); } @Test @@ -908,19 +913,23 @@ void whenDistributionStrategyModified_dontReplacePod() { @Test void whenPodCreatedWithDomainV2Settings_livenessProbeHasConfiguredTuning() { configureServer() - .withLivenessProbeSettings(CONFIGURED_DELAY, CONFIGURED_TIMEOUT, CONFIGURED_PERIOD); + .withLivenessProbeSettings(CONFIGURED_DELAY, CONFIGURED_TIMEOUT, CONFIGURED_PERIOD) + .withLivenessProbeThresholds(CONFIGURED_SUCCESS_THRESHOLD, CONFIGURED_FAILURE_THRESHOLD); assertThat( getCreatedPodSpecContainer().getLivenessProbe(), - hasExpectedTuning(CONFIGURED_DELAY, CONFIGURED_TIMEOUT, CONFIGURED_PERIOD)); + hasExpectedTuning(CONFIGURED_DELAY, CONFIGURED_TIMEOUT, CONFIGURED_PERIOD, CONFIGURED_SUCCESS_THRESHOLD, + CONFIGURED_FAILURE_THRESHOLD)); } @Test void whenPodCreated_readinessProbeHasConfiguredTuning() { configureServer() - .withReadinessProbeSettings(CONFIGURED_DELAY, CONFIGURED_TIMEOUT, CONFIGURED_PERIOD); + .withReadinessProbeSettings(CONFIGURED_DELAY, CONFIGURED_TIMEOUT, CONFIGURED_PERIOD) + .withReadinessProbeThresholds(CONFIGURED_SUCCESS_THRESHOLD, CONFIGURED_FAILURE_THRESHOLD); assertThat( getCreatedPodSpecContainer().getReadinessProbe(), - hasExpectedTuning(CONFIGURED_DELAY, CONFIGURED_TIMEOUT, CONFIGURED_PERIOD)); + hasExpectedTuning(CONFIGURED_DELAY, CONFIGURED_TIMEOUT, CONFIGURED_PERIOD, + CONFIGURED_SUCCESS_THRESHOLD, CONFIGURED_FAILURE_THRESHOLD)); } @Test @@ -1642,7 +1651,7 @@ V1ObjectMeta createPodMetadata() { V1Container createPodSpecContainer() { return new V1Container() - .name(CONTAINER_NAME) + .name(WLS_CONTAINER_NAME) .image(LATEST_IMAGE) .imagePullPolicy(ALWAYS_IMAGEPULLPOLICY) .securityContext(new V1SecurityContext()) diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/TuningParametersStub.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/TuningParametersStub.java index e40bb720b4c..992468fa19e 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/TuningParametersStub.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/TuningParametersStub.java @@ -24,9 +24,13 @@ public abstract class TuningParametersStub implements TuningParameters { static final int READINESS_INITIAL_DELAY = 1; static final int READINESS_TIMEOUT = 2; static final int READINESS_PERIOD = 3; + static final int READINESS_SUCCESS_THRESHOLD = 1; + static final int READINESS_FAILURE_THRESHOLD = 1; static final int LIVENESS_INITIAL_DELAY = 4; static final int LIVENESS_PERIOD = 6; static final int LIVENESS_TIMEOUT = 5; + static final int LIVENESS_SUCCESS_THRESHOLD = 1; + static final int LIVENESS_FAILURE_THRESHOLD = 1; static final long INTROSPECTOR_JOB_ACTIVE_DEADLINE_SECONDS = 180L; // Call builder tuning @@ -52,9 +56,13 @@ public PodTuning getPodTuning() { READINESS_INITIAL_DELAY, READINESS_TIMEOUT, READINESS_PERIOD, + READINESS_SUCCESS_THRESHOLD, + READINESS_FAILURE_THRESHOLD, LIVENESS_INITIAL_DELAY, LIVENESS_TIMEOUT, LIVENESS_PERIOD, + LIVENESS_SUCCESS_THRESHOLD, + LIVENESS_FAILURE_THRESHOLD, INTROSPECTOR_JOB_ACTIVE_DEADLINE_SECONDS); } diff --git a/operator/src/test/java/oracle/kubernetes/weblogic/domain/model/DomainV2Test.java b/operator/src/test/java/oracle/kubernetes/weblogic/domain/model/DomainV2Test.java index 19876614c33..cec8125c213 100644 --- a/operator/src/test/java/oracle/kubernetes/weblogic/domain/model/DomainV2Test.java +++ b/operator/src/test/java/oracle/kubernetes/weblogic/domain/model/DomainV2Test.java @@ -29,6 +29,8 @@ import static oracle.kubernetes.operator.DomainSourceType.FromModel; import static oracle.kubernetes.operator.KubernetesConstants.DEFAULT_IMAGE; import static oracle.kubernetes.operator.KubernetesConstants.IFNOTPRESENT_IMAGEPULLPOLICY; +import static oracle.kubernetes.operator.helpers.PodHelperTestBase.CONFIGURED_FAILURE_THRESHOLD; +import static oracle.kubernetes.operator.helpers.PodHelperTestBase.CONFIGURED_SUCCESS_THRESHOLD; import static oracle.kubernetes.weblogic.domain.ChannelMatcher.channelWith; import static oracle.kubernetes.weblogic.domain.model.ConfigurationConstants.START_ALWAYS; import static oracle.kubernetes.weblogic.domain.model.ConfigurationConstants.START_NEVER; @@ -313,48 +315,66 @@ private V1EnvVar envVar(String name, String value) { @Test void livenessProbeSettings_returnsConfiguredValues() { - configureServer(SERVER1).withLivenessProbeSettings(INITIAL_DELAY, TIMEOUT, PERIOD); + configureServer(SERVER1) + .withLivenessProbeSettings(INITIAL_DELAY, TIMEOUT, PERIOD) + .withLivenessProbeThresholds(CONFIGURED_SUCCESS_THRESHOLD, CONFIGURED_FAILURE_THRESHOLD); ServerSpec spec = domain.getServer(SERVER1, CLUSTER_NAME); assertThat(spec.getLivenessProbe().getInitialDelaySeconds(), equalTo(INITIAL_DELAY)); assertThat(spec.getLivenessProbe().getTimeoutSeconds(), equalTo(TIMEOUT)); assertThat(spec.getLivenessProbe().getPeriodSeconds(), equalTo(PERIOD)); + assertThat(spec.getLivenessProbe().getSuccessThreshold(), equalTo(CONFIGURED_SUCCESS_THRESHOLD)); + assertThat(spec.getLivenessProbe().getFailureThreshold(), equalTo(CONFIGURED_FAILURE_THRESHOLD)); } @Test void whenLivenessProbeConfiguredOnMultipleLevels_useCombination() { configureDomain(domain).withDefaultLivenessProbeSettings(INITIAL_DELAY, -2, -3); - configureCluster(CLUSTER_NAME).withLivenessProbeSettings(null, TIMEOUT, -4); - configureServer(SERVER1).withLivenessProbeSettings(null, null, PERIOD); + configureCluster(CLUSTER_NAME) + .withLivenessProbeSettings(null, TIMEOUT, -4) + .withLivenessProbeThresholds(1, 1); + configureServer(SERVER1) + .withLivenessProbeSettings(null, null, PERIOD) + .withLivenessProbeThresholds(CONFIGURED_SUCCESS_THRESHOLD, CONFIGURED_FAILURE_THRESHOLD); ServerSpec spec = domain.getServer(SERVER1, CLUSTER_NAME); assertThat(spec.getLivenessProbe().getInitialDelaySeconds(), equalTo(INITIAL_DELAY)); assertThat(spec.getLivenessProbe().getTimeoutSeconds(), equalTo(TIMEOUT)); assertThat(spec.getLivenessProbe().getPeriodSeconds(), equalTo(PERIOD)); + assertThat(spec.getLivenessProbe().getSuccessThreshold(), equalTo(CONFIGURED_SUCCESS_THRESHOLD)); + assertThat(spec.getLivenessProbe().getFailureThreshold(), equalTo(CONFIGURED_FAILURE_THRESHOLD)); } @Test void readinessProbeSettings_returnsConfiguredValues() { - configureServer(SERVER1).withReadinessProbeSettings(INITIAL_DELAY, TIMEOUT, PERIOD); + configureServer(SERVER1) + .withReadinessProbeSettings(INITIAL_DELAY, TIMEOUT, PERIOD) + .withReadinessProbeThresholds(CONFIGURED_SUCCESS_THRESHOLD, CONFIGURED_FAILURE_THRESHOLD); ServerSpec spec = domain.getServer(SERVER1, CLUSTER_NAME); assertThat(spec.getReadinessProbe().getInitialDelaySeconds(), equalTo(INITIAL_DELAY)); assertThat(spec.getReadinessProbe().getTimeoutSeconds(), equalTo(TIMEOUT)); assertThat(spec.getReadinessProbe().getPeriodSeconds(), equalTo(PERIOD)); + assertThat(spec.getReadinessProbe().getSuccessThreshold(), equalTo(CONFIGURED_SUCCESS_THRESHOLD)); + assertThat(spec.getReadinessProbe().getFailureThreshold(), equalTo(CONFIGURED_FAILURE_THRESHOLD)); } @Test void whenReadinessProbeConfiguredOnMultipleLevels_useCombination() { configureDomain(domain).withDefaultReadinessProbeSettings(INITIAL_DELAY, -2, -3); configureCluster(CLUSTER_NAME).withReadinessProbeSettings(null, TIMEOUT, -4); - configureServer(SERVER1).withReadinessProbeSettings(null, null, PERIOD); + configureServer(SERVER1) + .withReadinessProbeSettings(null, null, PERIOD) + .withReadinessProbeThresholds(CONFIGURED_SUCCESS_THRESHOLD, CONFIGURED_FAILURE_THRESHOLD); ServerSpec spec = domain.getServer(SERVER1, CLUSTER_NAME); assertThat(spec.getReadinessProbe().getInitialDelaySeconds(), equalTo(INITIAL_DELAY)); assertThat(spec.getReadinessProbe().getTimeoutSeconds(), equalTo(TIMEOUT)); assertThat(spec.getReadinessProbe().getPeriodSeconds(), equalTo(PERIOD)); + assertThat(spec.getReadinessProbe().getSuccessThreshold(), equalTo(CONFIGURED_SUCCESS_THRESHOLD)); + assertThat(spec.getReadinessProbe().getFailureThreshold(), equalTo(CONFIGURED_FAILURE_THRESHOLD)); } @Test diff --git a/operator/src/test/java/oracle/kubernetes/weblogic/domain/model/DomainValidationTest.java b/operator/src/test/java/oracle/kubernetes/weblogic/domain/model/DomainValidationTest.java index ae2ebfe23d0..41dfece2eba 100644 --- a/operator/src/test/java/oracle/kubernetes/weblogic/domain/model/DomainValidationTest.java +++ b/operator/src/test/java/oracle/kubernetes/weblogic/domain/model/DomainValidationTest.java @@ -8,6 +8,7 @@ import java.util.List; import com.meterware.simplestub.Memento; +import io.kubernetes.client.openapi.models.V1Container; import io.kubernetes.client.openapi.models.V1LocalObjectReference; import oracle.kubernetes.operator.TuningParameters; import oracle.kubernetes.operator.helpers.KubernetesTestSupport; @@ -27,6 +28,7 @@ import static oracle.kubernetes.operator.DomainProcessorTestSetup.createTestDomain; import static oracle.kubernetes.operator.DomainSourceType.FromModel; import static oracle.kubernetes.operator.DomainSourceType.Image; +import static oracle.kubernetes.operator.KubernetesConstants.WLS_CONTAINER_NAME; import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_TOPOLOGY; import static oracle.kubernetes.operator.helpers.PodHelperTestBase.getAuxiliaryImage; import static org.hamcrest.Matchers.contains; @@ -458,6 +460,69 @@ void whenWebLogicCredentialsSecretNamespaceUndefined_useDomainNamespace() { assertThat(domain.getValidationFailures(resourceLookup), empty()); } + @Test + void whenLivenessProbeSuccessThresholdValueInvalidForDomain_reportError() { + configureDomain(domain).withDefaultLivenessProbeThresholds(2, 3); + + assertThat(domain.getValidationFailures(resourceLookup), + contains(stringContainsInOrder("Invalid value", "2", "liveness probe success threshold", + "adminServer"))); + } + + @Test + void whenLivenessProbeSuccessThresholdValueInvalidForCluster_reportError() { + configureDomain(domain) + .configureCluster("cluster-1") + .withLivenessProbeSettings(5, 4, 3).withLivenessProbeThresholds(2, 3); + + assertThat(domain.getValidationFailures(resourceLookup), + contains(stringContainsInOrder("Invalid value", "2", "liveness probe success threshold", + "cluster-1"))); + } + + @Test + void whenLivenessProbeSuccessThresholdValueInvalidForServer_reportError() { + configureDomain(domain) + .configureServer("managed-server1") + .withLivenessProbeSettings(5, 4, 3).withLivenessProbeThresholds(2, 3); + + assertThat(domain.getValidationFailures(resourceLookup), + contains(stringContainsInOrder("Invalid value", "2", "liveness probe success threshold", + "managed-server1"))); + } + + @Test + void whenReservedContainerNameUsedForDomain_reportError() { + configureDomain(domain) + .withContainer(new V1Container().name(WLS_CONTAINER_NAME)); + + assertThat(domain.getValidationFailures(resourceLookup), + contains(stringContainsInOrder("container name", WLS_CONTAINER_NAME, "adminServer", + "is reserved", "operator"))); + } + + @Test + void whenReservedContainerNameUsedForCluster_reportError() { + configureDomain(domain) + .configureCluster("cluster-1") + .withContainer(new V1Container().name(WLS_CONTAINER_NAME)); + + assertThat(domain.getValidationFailures(resourceLookup), + contains(stringContainsInOrder("container name", WLS_CONTAINER_NAME, "cluster-1", + "is reserved", "operator"))); + } + + @Test + void whenReservedContainerNameUsedForManagedServer_reportError() { + configureDomain(domain) + .configureServer("managed-server1") + .withContainer(new V1Container().name(WLS_CONTAINER_NAME)); + + assertThat(domain.getValidationFailures(resourceLookup), + contains(stringContainsInOrder("container name", WLS_CONTAINER_NAME, "managed-server1", + "is reserved", "operator"))); + } + @Test void whenWebLogicCredentialsSecretNameNotFound_reportError() { resourceLookup.undefineResource(SECRET_NAME, KubernetesResourceType.Secret, NS);