OWLS-72816 Generate event and status when operator can't scale cluster past maximum

docs-source/content/userguide/managing-domains/domain-events.md

@@ -29,6 +29,7 @@ The operator generates these event types, which indicate the following:
  *  `DomainProcessingRetrying`: The operator is going to retry the processing of a domain after it encountered an failure.
  *  `DomainProcessingCompleted`:  The operator successfully completed the processing of a domain resource.
  *  `DomainProcessingAborted`:  The operator stopped processing a domain when the operator encountered a fatal error or a failure that persisted after the specified maximum number of retries.
+ *  `InvalidReplicasValue`:  An invalid replicas value is found in a `Cluster` inside a domain resource. Please update the domain resource with a replicas value that is supported by the cluster whose name is specified in the event.
 
 #### Operator-generated event details
 
@@ -242,6 +243,39 @@ Source:
 Type:    Warning
 Events:  <none>
 
+```
+Example of a `InvalidReplicasValue` event:
+
+```none
+
+Name:             sample-domain1.InvalidReplicasValue.1607639075047
+Namespace:        sample-domain1-ns
+Labels:           weblogic.createdByOperator=true
+                  weblogic.domainUID=sample-domain1
+Annotations:      <none>
+API Version:      v1
+Event Time:       <nil>
+First Timestamp:  <nil>
+Involved Object:
+  API Version:   weblogic.oracle/v8
+  Kind:          Domain
+  Name:          sample-domain1
+  Namespace:     sample-domain1-ns
+Kind:            Event
+Last Timestamp:  2020-12-10T22:24:35Z
+Message:         Invalid replicas value in domain resource sample-domain1: Replica request of 5 exceeds the maximum dynamic server count of 2 configured for cluster cluster-1
+Metadata:
+  Creation Timestamp:  2020-12-10T22:24:35Z
+  Resource Version:    10071657
+  Self Link:           /api/v1/namespaces/sample-domain1-ns/events/sample-domain1.InvalidReplicasValue.1607639075047
+  UID:                 8a9a4396-509b-4314-b634-376ed9245896
+Reason:                InvalidReplicasValue
+Reporting Component:   weblogic.operator
+Reporting Instance:    weblogic-operator-67c75bc4bf-8m4sb
+Source:
+Type:    Warning
+Events:  <none>
+
 ```
 
 Example of domain processing completed after failure and retries:

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

@@ -655,6 +655,11 @@ public boolean wasInspectionRun() {
       return inspectionRun;
     }
 
+    @Override
+    public boolean isExplicitRecheck() {
+      return explicitRecheck;
+    }
+
     private boolean shouldContinue() {
       DomainPresenceInfo cachedInfo = getExistingDomainPresenceInfo(getNamespace(), getDomainUid());
 

operator/src/main/java/oracle/kubernetes/operator/EventConstants.java

@@ -14,6 +14,7 @@ public interface EventConstants {
   String DOMAIN_PROCESSING_FAILED_EVENT = "DomainProcessingFailed";
   String DOMAIN_PROCESSING_RETRYING_EVENT = "DomainProcessingRetrying";
   String DOMAIN_PROCESSING_ABORTED_EVENT = "DomainProcessingAborted";
+  String INVALID_REPLICAS_VALUE_EVENT = "InvalidReplicasValue";
   String EVENT_NORMAL = "Normal";
   String EVENT_WARNING = "Warning";
   String WEBLOGIC_OPERATOR_COMPONENT = "weblogic.operator";
@@ -31,4 +32,6 @@ public interface EventConstants {
       = "Retrying the processing of domain resource %s after one or more failed attempts";
   String DOMAIN_PROCESSING_ABORTED_PATTERN
       = "Aborting the processing of domain resource %s permanently due to: %s";
+  String INVALID_REPLICAS_VALUE_PATTERN
+      = "Invalid replicas value in domain resource %s: %s";
 }

operator/src/main/java/oracle/kubernetes/operator/MakeRightDomainOperation.java

@@ -52,6 +52,12 @@ static boolean isInspectionRequired(Packet packet) {
     return domainRequiresIntrospectionInCurrentMakeRight(packet) && !wasInspectionRun(packet);
   }
 
+  boolean isExplicitRecheck();
+
+  static boolean isExplicitRecheck(Packet packet) {
+    return fromPacket(packet).map(MakeRightDomainOperation::isExplicitRecheck).orElse(false);
+  }
+
   /**
    * Returns true if the packet contains info about a domain that requires introspection in a sequences of steps
    * before server pods are created or modified.

operator/src/main/java/oracle/kubernetes/operator/helpers/DomainValidationSteps.java

@@ -3,7 +3,6 @@
 
 package oracle.kubernetes.operator.helpers;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
@@ -24,10 +23,9 @@
 import oracle.kubernetes.operator.work.NextAction;
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
-import oracle.kubernetes.weblogic.domain.model.Cluster;
 import oracle.kubernetes.weblogic.domain.model.Domain;
+import oracle.kubernetes.weblogic.domain.model.DomainSpec;
 import oracle.kubernetes.weblogic.domain.model.KubernetesResourceLookup;
-import oracle.kubernetes.weblogic.domain.model.ManagedServer;
 
 import static java.lang.System.lineSeparator;
 import static oracle.kubernetes.operator.DomainStatusUpdater.BAD_DOMAIN;
@@ -146,33 +144,38 @@ static class ValidateDomainTopologyStep extends Step {
     }
 
 
-    private void logAndAddWarning(List<String> validationWarnings, String messageKey, Object... params) {
+    private void logAndAddWarning(DomainPresenceInfo info, String messageKey, Object... params) {
       LOGGER.warning(messageKey, params);
-      validationWarnings.add(LOGGER.formatMessage(messageKey, params));
+      info.addValidationWarning(LOGGER.formatMessage(messageKey, params));
     }
 
     private void validate(DomainPresenceInfo info, WlsDomainConfig wlsDomainConfig) {
-      List<String> validationWarnings = new ArrayList<>();
+      DomainSpec domainSpec = info.getDomain().getSpec();
 
-      Domain domain = info.getDomain();
+      info.clearValidationWarnings();
 
-      // log warnings for clusters that are specified in domain resource but not configured
+      // log warnings for each cluster that is specified in domain resource but not configured
       // in the WebLogic domain
-      for (Cluster cluster : domain.getSpec().getClusters()) {
-        if (!wlsDomainConfig.containsCluster(cluster.getClusterName())) {
-          logAndAddWarning(validationWarnings, MessageKeys.NO_CLUSTER_IN_DOMAIN, cluster.getClusterName());
-        }
-      }
-      // log warnings for managed servers that are specified in domain resource but not configured
+      domainSpec.getClusters().forEach(
+          c -> logAndAddWarningIfClusterDoesNotExist(wlsDomainConfig, c.getClusterName(), info));
+
+      // log warnings for each managed server that is specified in domain resource but not configured
       // in the WebLogic domain
-      for (ManagedServer server : domain.getSpec().getManagedServers()) {
-        if (!wlsDomainConfig.containsServer(server.getServerName())) {
-          logAndAddWarning(validationWarnings, MessageKeys.NO_MANAGED_SERVER_IN_DOMAIN, server.getServerName());
-        }
+      domainSpec.getManagedServers().forEach(
+          s -> logAndAddWarningIfServerDoesNotExist(wlsDomainConfig, s.getServerName(), info));
+    }
+
+    private void logAndAddWarningIfClusterDoesNotExist(WlsDomainConfig domainConfig, String clusterName,
+        DomainPresenceInfo info) {
+      if (!domainConfig.containsCluster(clusterName)) {
+        logAndAddWarning(info, MessageKeys.NO_CLUSTER_IN_DOMAIN, clusterName);
       }
-      info.clearValidationWarnings();
-      for (String warning: validationWarnings) {
-        info.addValidationWarning(warning);
+    }
+
+    private void logAndAddWarningIfServerDoesNotExist(WlsDomainConfig domainConfig, String serverName,
+        DomainPresenceInfo info) {
+      if (!domainConfig.containsServer(serverName)) {
+        logAndAddWarning(info, MessageKeys.NO_MANAGED_SERVER_IN_DOMAIN, serverName);
       }
     }
 

operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java

@@ -39,6 +39,8 @@
 import static oracle.kubernetes.operator.EventConstants.DOMAIN_PROCESSING_STARTING_PATTERN;
 import static oracle.kubernetes.operator.EventConstants.EVENT_NORMAL;
 import static oracle.kubernetes.operator.EventConstants.EVENT_WARNING;
+import static oracle.kubernetes.operator.EventConstants.INVALID_REPLICAS_VALUE_EVENT;
+import static oracle.kubernetes.operator.EventConstants.INVALID_REPLICAS_VALUE_PATTERN;
 import static oracle.kubernetes.operator.EventConstants.WEBLOGIC_OPERATOR_COMPONENT;
 import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_ABORTED;
 import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_COMPLETED;
@@ -254,6 +256,28 @@ public String getMessage(DomainPresenceInfo info, EventData eventData) {
       }
 
     },
+    INVALID_REPLICAS_VALUE {
+      @Override
+      public String getType() {
+        return EVENT_WARNING;
+      }
+
+      @Override
+      public String getReason() {
+        return INVALID_REPLICAS_VALUE_EVENT;
+      }
+
+      @Override
+      public String getPattern() {
+        return INVALID_REPLICAS_VALUE_PATTERN;
+      }
+
+      @Override
+      public String getMessage(DomainPresenceInfo info, EventData eventData) {
+        return String.format(INVALID_REPLICAS_VALUE_PATTERN,
+            info.getDomainUid(), Optional.ofNullable(eventData.message).orElse(""));
+      }
+    },
     EMPTY {
       @Override
       protected String getPattern() {

operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java

@@ -16,10 +16,13 @@
 import javax.annotation.Nonnull;
 
 import oracle.kubernetes.operator.DomainStatusUpdater;
+import oracle.kubernetes.operator.MakeRightDomainOperation;
 import oracle.kubernetes.operator.ProcessingConstants;
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerShutdownInfo;
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerStartupInfo;
+import oracle.kubernetes.operator.helpers.EventHelper.EventData;
+import oracle.kubernetes.operator.helpers.EventHelper.EventItem;
 import oracle.kubernetes.operator.helpers.PodHelper;
 import oracle.kubernetes.operator.logging.LoggingFacade;
 import oracle.kubernetes.operator.logging.LoggingFactory;
@@ -37,6 +40,7 @@
 import static java.util.Comparator.comparing;
 import static oracle.kubernetes.operator.DomainStatusUpdater.MANAGED_SERVERS_STARTING_PROGRESS_REASON;
 import static oracle.kubernetes.operator.DomainStatusUpdater.createProgressingStartedEventStep;
+import static oracle.kubernetes.operator.helpers.EventHelper.createEventStep;
 
 public class ManagedServersUpStep extends Step {
   static final String SERVERS_UP_MSG =
@@ -97,9 +101,11 @@ private static void insert(List<Step> steps, Step step) {
   public NextAction apply(Packet packet) {
     LOGGER.entering();
     DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
+    boolean isExplicitRecheck = MakeRightDomainOperation.isExplicitRecheck(packet);
     WlsDomainConfig config = (WlsDomainConfig) packet.get(ProcessingConstants.DOMAIN_TOPOLOGY);
 
-    ServersUpStepFactory factory = new ServersUpStepFactory(config, info.getDomain());
+    ServersUpStepFactory factory = new ServersUpStepFactory(config,
+        info.getDomain(), info, isExplicitRecheck);
 
     if (LOGGER.isFineEnabled()) {
       LOGGER.fine(SERVERS_UP_MSG, factory.domain.getDomainUid(), getRunningServers(info));
@@ -157,15 +163,21 @@ Step createServerStep(
   static class ServersUpStepFactory {
     final WlsDomainConfig domainTopology;
     final Domain domain;
+    final DomainPresenceInfo info;
+    final boolean skipEventCreation;
     List<ServerStartupInfo> startupInfos;
     List<ServerShutdownInfo> shutdownInfos = new ArrayList<>();
     final Collection<String> servers = new ArrayList<>();
     final Collection<String> preCreateServers = new ArrayList<>();
     final Map<String, Integer> replicas = new HashMap<>();
+    private Step eventStep;
 
-    ServersUpStepFactory(WlsDomainConfig domainTopology, Domain domain) {
+    ServersUpStepFactory(WlsDomainConfig domainTopology, Domain domain,
+        DomainPresenceInfo info, boolean skipEventCreation) {
       this.domainTopology = domainTopology;
       this.domain = domain;
+      this.info = info;
+      this.skipEventCreation = skipEventCreation;
     }
 
     /**
@@ -223,11 +235,8 @@ boolean exceedsMaxConfiguredClusterSize(WlsClusterConfig clusterConfig) {
     }
 
     private Step createNextStep(Step next) {
-      if (servers.isEmpty()) {
-        return next;
-      } else {
-        return new ManagedServerUpIteratorStep(getStartupInfos(), next);
-      }
+      Step nextStep = (servers.isEmpty()) ? next : new ManagedServerUpIteratorStep(getStartupInfos(), next);
+      return Optional.ofNullable(eventStep).map(s -> Step.chain(s, nextStep)).orElse(nextStep);
     }
 
     Collection<ServerStartupInfo> getStartupInfos() {
@@ -275,6 +284,11 @@ private void logIfReplicasExceedsClusterServersMax(WlsClusterConfig clusterConfi
             domain.getReplicaCount(clusterName),
             clusterConfig.getMaxDynamicClusterSize(),
             clusterName);
+        String message = LOGGER.formatMessage(MessageKeys.REPLICAS_EXCEEDS_TOTAL_CLUSTER_SERVER_COUNT,
+            domain.getReplicaCount(clusterName),
+            clusterConfig.getMaxDynamicClusterSize(),
+            clusterName);
+        addInvalidReplicasValueEventAndWarning(message);
       }
     }
 
@@ -286,13 +300,26 @@ private void logIfReplicasLessThanClusterServersMin(WlsClusterConfig clusterConf
                 domain.getReplicaCount(clusterName),
                 clusterConfig.getMinDynamicClusterSize(),
                 clusterName);
+        String message = LOGGER.formatMessage(MessageKeys.REPLICAS_LESS_THAN_TOTAL_CLUSTER_SERVER_COUNT,
+            domain.getReplicaCount(clusterName),
+            clusterConfig.getMinDynamicClusterSize(),
+            clusterName);
+        addInvalidReplicasValueEventAndWarning(message);
 
         // Reset current replica count so we don't scale down less than minimum
         // dynamic cluster size
         domain.setReplicaCount(clusterName, clusterConfig.getMinDynamicClusterSize());
       }
     }
 
+    private void addInvalidReplicasValueEventAndWarning(String message) {
+      LOGGER.warning(message);
+      if (!skipEventCreation) {
+        eventStep = createEventStep(new EventData(EventItem.INVALID_REPLICAS_VALUE, message));
+      }
+      info.addValidationWarning(message);
+    }
+
     private boolean lessThanMinConfiguredClusterSize(WlsClusterConfig clusterConfig) {
       if (clusterConfig != null) {
         String clusterName = clusterConfig.getClusterName();

operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java

@@ -8,6 +8,7 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import javax.annotation.Nonnull;
 
 import oracle.kubernetes.utils.OperatorUtils;
 import org.apache.commons.lang3.builder.EqualsBuilder;
@@ -377,6 +378,17 @@ public String getUpdateDynamicClusterSizePayload(final int clusterSize) {
     return "{ dynamicClusterSize: " + clusterSize + " }";
   }
 
+  /**
+   * Whether this cluster contains a server with the given server name,
+   * including servers that are both configured and dynamic servers.
+   *
+   * @param serverName server name to be checked
+   * @return True if the cluster contains a server with the given server name
+   */
+  boolean containsServer(@Nonnull String serverName) {
+    return getServerConfigs().stream().anyMatch(c -> serverName.equals(c.getName()));
+  }
+
   @Override
   public String toString() {
     return new ToStringBuilder(this)

operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java

@@ -11,6 +11,7 @@
 import javax.annotation.Nonnull;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Strings;
 import oracle.kubernetes.operator.logging.LoggingFacade;
 import oracle.kubernetes.operator.logging.LoggingFactory;
 import oracle.kubernetes.operator.logging.MessageKeys;
@@ -392,18 +393,15 @@ public synchronized boolean containsCluster(String clusterName) {
 
   /**
    * Whether the WebLogic domain contains a server with the given server name,
-   * not including servers that are part of a dynamic cluster.
+   * including standalone servers, and servers that belong to a configured or dynamic cluster.
    *
    * @param serverName server name to be checked
    * @return True if the WebLogic domain contains a server with the given server name
    */
   public synchronized boolean containsServer(String serverName) {
-    if (serverName != null && servers != null) {
-      for (WlsServerConfig serverConfig : servers) {
-        if (serverConfig.getName().equals(serverName)) {
-          return true;
-        }
-      }
+    if (!Strings.isNullOrEmpty(serverName)) {
+      return getServers().stream().anyMatch(s -> serverName.equals(s.getName()))
+          || getConfiguredClusters().stream().anyMatch(c -> c.containsServer(serverName));
     }
     return false;
   }

operator/src/main/resources/Operator.properties

@@ -145,7 +145,7 @@ WLSKO-0142=Failed to parse results from domain introspector for domain {0} due t
 WLSKO-0143=Failed to parse file {0} from domain introspector for domain {1} due to exception: {2}
 WLSKO-0144=Unable to start domain with domainUID {0} in namespace {1} after {2} attempts due to exception: {3}
 WLSKO-0145=Replacing pod {0} because: {1}
-WLSKO-0146=Replica request of {0} exceeds the maximum dynamic server count + server count of {1} configured for cluster {2}
+WLSKO-0146=Replica request of {0} exceeds the maximum dynamic server count of {1} configured for cluster {2}
 WLSKO-0147=Call {0} has failed with code {1}: message {2}. It has failed {3} times and will be retried up to {4} times.
 WLSKO-0148=Current Pod dump [{0}] vs expected pod [{1}].
 WLSKO-0150=Creating external channel service for WebLogic domain with UID: {0}.