diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java index 32a1a0d68b6..5974bed6f99 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java @@ -1,4 +1,4 @@ -// Copyright (c) 2017, 2021, Oracle and/or its affiliates. +// Copyright (c) 2017, 2022, Oracle and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. package oracle.kubernetes.operator.steps; @@ -15,6 +15,7 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; +import io.kubernetes.client.openapi.models.V1Pod; import oracle.kubernetes.operator.DomainStatusUpdater; import oracle.kubernetes.operator.MakeRightDomainOperation; import oracle.kubernetes.operator.ProcessingConstants; @@ -41,6 +42,7 @@ import static oracle.kubernetes.operator.DomainStatusUpdater.MANAGED_SERVERS_STARTING_PROGRESS_REASON; import static oracle.kubernetes.operator.DomainStatusUpdater.createProgressingStartedEventStep; import static oracle.kubernetes.operator.helpers.EventHelper.createEventStep; +import static oracle.kubernetes.operator.helpers.PodHelper.getPodServerName; public class ManagedServersUpStep extends Step { static final String SERVERS_UP_MSG = @@ -118,7 +120,7 @@ public NextAction apply(Packet packet) { LOGGER.fine(SERVERS_UP_MSG, factory.domain.getDomainUid(), getRunningServers(info)); } - Optional.ofNullable(config).ifPresent(wlsDomainConfig -> addServersToFactory(factory, wlsDomainConfig)); + Optional.ofNullable(config).ifPresent(wlsDomainConfig -> addServersToFactory(factory, wlsDomainConfig, info)); info.setServerStartupInfo(factory.getStartupInfos()); info.setServerShutdownInfo(factory.getShutdownInfos()); @@ -130,7 +132,8 @@ public NextAction apply(Packet packet) { packet); } - private void addServersToFactory(@Nonnull ServersUpStepFactory factory, @Nonnull WlsDomainConfig wlsDomainConfig) { + private void addServersToFactory(@Nonnull ServersUpStepFactory factory, @Nonnull WlsDomainConfig wlsDomainConfig, + DomainPresenceInfo info) { Set clusteredServers = new HashSet<>(); List pendingServers = new ArrayList<>(); @@ -145,6 +148,15 @@ private void addServersToFactory(@Nonnull ServersUpStepFactory factory, @Nonnull for (ServerConfig serverConfig : pendingServers) { factory.addServerIfNeeded(serverConfig.wlsServerConfig, serverConfig.wlsClusterConfig); } + + info.getServerPods().filter(pod -> !factory.getServers().contains(getPodServerName(pod))) + .filter(pod -> !getPodServerName(pod).equals(wlsDomainConfig.getAdminServerName())) + .forEach(pod -> shutdownServersNotPresentInDomainConfig(factory, pod)); + } + + private void shutdownServersNotPresentInDomainConfig(ServersUpStepFactory factory, V1Pod pod) { + WlsServerConfig serverConfig = new WlsServerConfig(getPodServerName(pod), pod.getMetadata().getName(), 0); + factory.addShutdownInfo(new ServerShutdownInfo(serverConfig, pod.getMetadata().getClusterName(), null, false)); } private void addClusteredServersToFactory( @@ -258,6 +270,10 @@ Collection getShutdownInfos() { return shutdownInfos; } + Collection getServers() { + return servers; + } + private void addStartupInfo(ServerStartupInfo startupInfo) { if (startupInfos == null) { startupInfos = new ArrayList<>(); diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ReadHealthStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ReadHealthStep.java index 9491936d96e..0c285afafad 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ReadHealthStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ReadHealthStep.java @@ -1,4 +1,4 @@ -// Copyright (c) 2018, 2021, Oracle and/or its affiliates. +// Copyright (c) 2018, 2022, Oracle and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. package oracle.kubernetes.operator.steps; @@ -191,6 +191,9 @@ static final class ReadHealthWithHttpStep extends Step { @Override public NextAction apply(Packet packet) { ReadHealthProcessing processing = new ReadHealthProcessing(packet, service, pod); + if (processing.getWlsServerConfig() == null) { + return doNext(packet); + } return doNext(createRequestStep(processing.createRequest(), new RecordHealthStep(getNext())), packet); } @@ -381,4 +384,4 @@ private static void logReadFailure(Packet packet) { MessageKeys.WLS_HEALTH_READ_FAILED, packet.get(ProcessingConstants.SERVER_NAME)); } -} +} \ No newline at end of file diff --git a/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java b/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java index 7b6a5a98a59..830fcbe0a11 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java @@ -1,4 +1,4 @@ -// Copyright (c) 2019, 2021, Oracle and/or its affiliates. +// Copyright (c) 2019, 2022, Oracle and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. package oracle.kubernetes.operator; @@ -6,6 +6,7 @@ import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -120,19 +121,29 @@ class DomainProcessorTest { private static final String ADMIN_NAME = "admin"; private static final String CLUSTER = "cluster"; - private static final int MAX_SERVERS = 60; + private static final String CLUSTER2 = "cluster-2"; + private static final int MAX_SERVERS = 5; private static final String MS_PREFIX = "managed-server"; private static final int MIN_REPLICAS = 2; private static final int NUM_ADMIN_SERVERS = 1; private static final int NUM_JOB_PODS = 1; - private static final String[] MANAGED_SERVER_NAMES = - IntStream.rangeClosed(1, MAX_SERVERS).mapToObj(DomainProcessorTest::getManagedServerName).toArray(String[]::new); + private static final String[] MANAGED_SERVER_NAMES = getManagedServerNames(CLUSTER); public static final String DOMAIN_NAME = "base_domain"; private TestUtils.ConsoleHandlerMemento consoleHandlerMemento; + private static String[] getManagedServerNames(String clusterName) { + return IntStream.rangeClosed(1, MAX_SERVERS) + .mapToObj(n -> getManagedServerName(n, clusterName)).toArray(String[]::new); + } + @Nonnull private static String getManagedServerName(int n) { - return MS_PREFIX + n; + return getManagedServerName(n, CLUSTER); + } + + @Nonnull + private static String getManagedServerName(int n, String clusterName) { + return clusterName + "-" + MS_PREFIX + n; } private final List mementos = new ArrayList<>(); @@ -163,13 +174,25 @@ V1JobStatus createNotCompletedStatus() { } private static WlsDomainConfig createDomainConfig() { - WlsClusterConfig clusterConfig = new WlsClusterConfig(CLUSTER); - for (String serverName : MANAGED_SERVER_NAMES) { - clusterConfig.addServerConfig(new WlsServerConfig(serverName, "domain1-" + serverName, 8001)); + return createDomainConfig(Collections.singletonList(CLUSTER), new ArrayList<>()); + } + + private static WlsDomainConfig createDomainConfig(List clusterNames, List independentServerNames) { + + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig(DOMAIN_NAME) + .withAdminServer(ADMIN_NAME, "domain1-admin-server", 7001); + for (String serverName : independentServerNames) { + wlsDomainConfig.addWlsServer(serverName, "domain-" + serverName, 8001); } - return new WlsDomainConfig(DOMAIN_NAME) - .withAdminServer(ADMIN_NAME, "domain1-admin-server", 7001) - .withCluster(clusterConfig); + for (String clusterName : clusterNames) { + WlsClusterConfig clusterConfig = new WlsClusterConfig(clusterName); + for (String serverName : getManagedServerNames(clusterName)) { + clusterConfig.addServerConfig(new WlsServerConfig(serverName, + "domain1-" + serverName, 8001)); + } + wlsDomainConfig.withCluster(clusterConfig); + } + return wlsDomainConfig; } @BeforeEach @@ -394,7 +417,7 @@ void whenMakeRightExecuted_ignoreNonOperatorPodDisruptionBudgets() { @Test void whenClusterReplicas2_server3WithAlwaysPolicy_establishMatchingPresence() { domainConfigurator.configureCluster(CLUSTER).withReplicas(2); - domainConfigurator.configureServer(MS_PREFIX + 3).withServerStartPolicy(START_ALWAYS); + domainConfigurator.configureServer(getManagedServerName(3)).withServerStartPolicy(START_ALWAYS); DomainPresenceInfo info = new DomainPresenceInfo(newDomain); processor.createMakeRightOperation(info).execute(); @@ -405,9 +428,9 @@ void whenClusterReplicas2_server3WithAlwaysPolicy_establishMatchingPresence() { assertServerPodAndServicePresent(info, ADMIN_NAME); for (Integer i : Arrays.asList(1,3)) { - assertServerPodAndServicePresent(info, MS_PREFIX + i); + assertServerPodAndServicePresent(info, getManagedServerName(i)); } - assertServerPodNotPresent(info, MS_PREFIX + 2); + assertServerPodNotPresent(info, getManagedServerName(2)); assertThat(info.getClusterService(CLUSTER), notNullValue()); assertThat(info.getPodDisruptionBudget(CLUSTER), notNullValue()); @@ -430,7 +453,7 @@ void whenClusterScaleUpToReplicas3_fromReplicas2_server3WithAlwaysPolicy_establi assertServerPodAndServicePresent(info, ADMIN_NAME); for (Integer i : Arrays.asList(1,2,3)) { - assertServerPodAndServicePresent(info, MS_PREFIX + i); + assertServerPodAndServicePresent(info, getManagedServerName(i)); } assertThat(info.getClusterService(CLUSTER), notNullValue()); @@ -442,7 +465,7 @@ void whenClusterScaleDownToReplicas1_fromReplicas2_server3WithAlwaysPolicy_estab establishPreviousIntrospection(null, Arrays.asList(1,3)); domainConfigurator.configureCluster(CLUSTER).withReplicas(1); - domainConfigurator.configureServer(MS_PREFIX + 3).withServerStartPolicy(START_ALWAYS); + domainConfigurator.configureServer(getManagedServerName(3)).withServerStartPolicy(START_ALWAYS); DomainPresenceInfo info = new DomainPresenceInfo(newDomain); processor.createMakeRightOperation(info).execute(); @@ -454,9 +477,9 @@ void whenClusterScaleDownToReplicas1_fromReplicas2_server3WithAlwaysPolicy_estab assertThat(runningPods.size(), equalTo(3)); assertServerPodAndServicePresent(info, ADMIN_NAME); - assertServerPodAndServicePresent(info, MS_PREFIX + 3); + assertServerPodAndServicePresent(info, getManagedServerName(3)); for (Integer i : Arrays.asList(1,2)) { - assertServerPodNotPresent(info, MS_PREFIX + i); + assertServerPodNotPresent(info, getManagedServerName(i)); } assertThat(info.getClusterService(CLUSTER), notNullValue()); @@ -468,7 +491,7 @@ void whenClusterReplicas3_server3And4WithAlwaysPolicy_establishMatchingPresence( domainConfigurator.configureCluster(CLUSTER).withReplicas(3); for (Integer i : Arrays.asList(3,4)) { - domainConfigurator.configureServer(MS_PREFIX + i).withServerStartPolicy(START_ALWAYS); + domainConfigurator.configureServer(getManagedServerName(i)).withServerStartPolicy(START_ALWAYS); } DomainPresenceInfo info = new DomainPresenceInfo(newDomain); @@ -480,9 +503,9 @@ void whenClusterReplicas3_server3And4WithAlwaysPolicy_establishMatchingPresence( assertServerPodAndServicePresent(info, ADMIN_NAME); for (Integer i : Arrays.asList(1,3,4)) { - assertServerPodAndServicePresent(info, MS_PREFIX + i); + assertServerPodAndServicePresent(info, getManagedServerName(i)); } - assertServerPodNotPresent(info, MS_PREFIX + 2); + assertServerPodNotPresent(info, getManagedServerName(2)); assertThat(info.getClusterService(CLUSTER), notNullValue()); } @@ -566,7 +589,7 @@ void whenClusterScaleUpToReplicas4_fromReplicas2_server3And4WithAlwaysPolicy_est establishPreviousIntrospection(null, Arrays.asList(1, 3, 4)); for (Integer i : Arrays.asList(3,4)) { - domainConfigurator.configureServer(MS_PREFIX + i).withServerStartPolicy(START_ALWAYS); + domainConfigurator.configureServer(getManagedServerName(i)).withServerStartPolicy(START_ALWAYS); } domainConfigurator.configureCluster(CLUSTER).withReplicas(4); @@ -581,7 +604,7 @@ void whenClusterScaleUpToReplicas4_fromReplicas2_server3And4WithAlwaysPolicy_est assertServerPodAndServicePresent(info, ADMIN_NAME); for (Integer i : Arrays.asList(1,2,3,4)) { - assertServerPodAndServicePresent(info, MS_PREFIX + i); + assertServerPodAndServicePresent(info, getManagedServerName(i)); } assertThat(info.getClusterService(CLUSTER), notNullValue()); @@ -592,7 +615,7 @@ void whenClusterReplicas2_server1And2And3WithAlwaysPolicy_establishMatchingPrese domainConfigurator.configureCluster(CLUSTER).withReplicas(2); for (Integer i : Arrays.asList(1,2,3)) { - domainConfigurator.configureServer(MS_PREFIX + i).withServerStartPolicy(START_ALWAYS); + domainConfigurator.configureServer(getManagedServerName(i)).withServerStartPolicy(START_ALWAYS); } DomainPresenceInfo info = new DomainPresenceInfo(newDomain); @@ -604,7 +627,7 @@ void whenClusterReplicas2_server1And2And3WithAlwaysPolicy_establishMatchingPrese assertServerPodAndServicePresent(info, ADMIN_NAME); for (Integer i : Arrays.asList(1,2,3)) { - assertServerPodAndServicePresent(info, MS_PREFIX + i); + assertServerPodAndServicePresent(info, getManagedServerName(i)); } assertThat(info.getClusterService(CLUSTER), notNullValue()); @@ -619,7 +642,7 @@ void whenClusterScaleDownToReplicas1_fromReplicas2_server1And2And3WithAlwaysPoli domainConfigurator.configureCluster(CLUSTER).withReplicas(1); for (Integer i : Arrays.asList(1,2,3)) { - domainConfigurator.configureServer(MS_PREFIX + i).withServerStartPolicy(START_ALWAYS); + domainConfigurator.configureServer(getManagedServerName(i)).withServerStartPolicy(START_ALWAYS); } DomainPresenceInfo info = new DomainPresenceInfo(newDomain); @@ -632,7 +655,7 @@ void whenClusterScaleDownToReplicas1_fromReplicas2_server1And2And3WithAlwaysPoli assertServerPodAndServicePresent(info, ADMIN_NAME); for (Integer i : Arrays.asList(1,2,3)) { - assertServerPodAndServicePresent(info, MS_PREFIX + i); + assertServerPodAndServicePresent(info, getManagedServerName(i)); } assertThat(info.getClusterService(CLUSTER), notNullValue()); } @@ -640,7 +663,7 @@ void whenClusterScaleDownToReplicas1_fromReplicas2_server1And2And3WithAlwaysPoli @Test void whenClusterReplicas2_server2NeverPolicy_establishMatchingPresence() { domainConfigurator.configureCluster(CLUSTER).withReplicas(2); - domainConfigurator.configureServer(MS_PREFIX + 2).withServerStartPolicy(START_NEVER); + domainConfigurator.configureServer(getManagedServerName(2)).withServerStartPolicy(START_NEVER); DomainPresenceInfo info = new DomainPresenceInfo(newDomain); processor.createMakeRightOperation(info).execute(); @@ -650,9 +673,9 @@ void whenClusterReplicas2_server2NeverPolicy_establishMatchingPresence() { assertServerPodAndServicePresent(info, ADMIN_NAME); for (Integer i : Arrays.asList(1,3)) { - assertServerPodAndServicePresent(info, MS_PREFIX + i); + assertServerPodAndServicePresent(info, getManagedServerName(i)); } - assertServerPodNotPresent(info, MS_PREFIX + 2); + assertServerPodNotPresent(info, getManagedServerName(2)); assertThat(info.getClusterService(CLUSTER), notNullValue()); } @@ -662,7 +685,7 @@ void whenClusterReplicas2_allServersExcept5NeverPolicy_establishMatchingPresence int[] servers = IntStream.rangeClosed(1, MAX_SERVERS).toArray(); for (int i : servers) { if (i != 5) { - domainConfigurator.configureServer(MS_PREFIX + i).withServerStartPolicy(START_NEVER); + domainConfigurator.configureServer(getManagedServerName(i)).withServerStartPolicy(START_NEVER); } } DomainPresenceInfo info = new DomainPresenceInfo(newDomain); @@ -675,9 +698,9 @@ void whenClusterReplicas2_allServersExcept5NeverPolicy_establishMatchingPresence assertServerPodAndServicePresent(info, ADMIN_NAME); for (int i : servers) { if (i != 5) { - assertServerPodAndServiceNotPresent(info, MS_PREFIX + i); + assertServerPodAndServiceNotPresent(info, getManagedServerName(i)); } else { - assertServerPodAndServicePresent(info, MS_PREFIX + i); + assertServerPodAndServicePresent(info, getManagedServerName(i)); } } @@ -811,7 +834,13 @@ private void establishPreviousIntrospection(Consumer domainSetup) throws } private void establishPreviousIntrospection(Consumer domainSetup, List msNumbers) - throws JsonProcessingException { + throws JsonProcessingException { + establishPreviousIntrospection(domainSetup, msNumbers, Collections.singletonList(CLUSTER), new ArrayList<>()); + } + + private void establishPreviousIntrospection(Consumer domainSetup, List msNumbers, + List clusterNames, List independentServers) + throws JsonProcessingException { if (domainSetup != null) { domainSetup.accept(domain); domainSetup.accept(newDomain); @@ -822,7 +851,7 @@ private void establishPreviousIntrospection(Consumer domainSetup, List recordJob((V1Job) j)); domainConfigurator.withIntrospectVersion(OLD_INTROSPECTION_STATE); } @@ -833,11 +862,13 @@ private String getCurrentImageSpecHash() { // define a config map with a topology to avoid the no-topology condition that always runs the introspector @SuppressWarnings("SameParameterValue") - private V1ConfigMap createIntrospectorConfigMap(String introspectionDoneValue) throws JsonProcessingException { + private V1ConfigMap createIntrospectorConfigMap(String introspectionDoneValue, List clusterNames, + List serverNames) throws JsonProcessingException { return new V1ConfigMap() .metadata(createIntrospectorConfigMapMeta(introspectionDoneValue)) - .data(new HashMap<>(Map.of(IntrospectorConfigMapConstants.TOPOLOGY_YAML, defineTopology(), - IntrospectorConfigMapConstants.DOMAIN_INPUTS_HASH, getCurrentImageSpecHash()))); + .data(new HashMap<>(Map.of(IntrospectorConfigMapConstants.TOPOLOGY_YAML, + defineTopology(clusterNames, serverNames), + IntrospectorConfigMapConstants.DOMAIN_INPUTS_HASH, getCurrentImageSpecHash()))); } private V1ObjectMeta createIntrospectorConfigMapMeta(@Nullable String introspectionDoneValue) { @@ -849,7 +880,11 @@ private V1ObjectMeta createIntrospectorConfigMapMeta(@Nullable String introspect } private String defineTopology() throws JsonProcessingException { - return IntrospectionTestUtils.createTopologyYaml(createDomainConfig()); + return defineTopology(Collections.singletonList(CLUSTER), new ArrayList<>()); + } + + private String defineTopology(List clusterNames, List serverNames) throws JsonProcessingException { + return IntrospectionTestUtils.createTopologyYaml(createDomainConfig(clusterNames, serverNames)); } @Test @@ -1196,6 +1231,38 @@ void whenDomainTypeIsFromModelDomainAndAdminServerModified_runIntrospectionJobFi assertThat(logRecords, containsFine(NOT_STARTING_DOMAINUID_THREAD)); } + @Test + void whenRunningClusterAndIndependentManagedServerRemovedFromDomainTopology_establishMatchingPresence() + throws JsonProcessingException { + establishPreviousIntrospection(null, Arrays.asList(1, 2, 3, 4), Arrays.asList(CLUSTER, CLUSTER2), + Arrays.asList("server-1")); + domainConfigurator.configureCluster(CLUSTER).withReplicas(4); + domainConfigurator.configureCluster(CLUSTER2).withReplicas(4); + DomainPresenceInfo info = new DomainPresenceInfo(newDomain); + processor.createMakeRightOperation(info).execute(); + + List runningPods = getRunningPods(); + //one introspector pod, one admin server pod, one independent server and four managed server pods for each cluster + assertThat(runningPods.size(), equalTo(11)); + + removeSecondClusterAndIndependentServerFromDomainTopology(); + processor.createMakeRightOperation(info).withExplicitRecheck().execute(); + + runningPods = getRunningPods(); + //one introspector pod, one admin server pod and four managed server pods for the one remaining cluster + assertThat(runningPods.size(), equalTo(6)); + } + + private void removeSecondClusterAndIndependentServerFromDomainTopology() throws JsonProcessingException { + testSupport.deleteResources(new V1ConfigMap() + .metadata(createIntrospectorConfigMapMeta(OLD_INTROSPECTION_STATE))); + testSupport.defineResources(new V1ConfigMap() + .metadata(createIntrospectorConfigMapMeta(OLD_INTROSPECTION_STATE)) + .data(new HashMap<>(Map.of(IntrospectorConfigMapConstants.TOPOLOGY_YAML, + IntrospectionTestUtils.createTopologyYaml(createDomainConfig()), + IntrospectorConfigMapConstants.DOMAIN_INPUTS_HASH, getCurrentImageSpecHash())))); + } + // todo after external service created, if adminService deleted, delete service @@ -1234,11 +1301,15 @@ private List getRunningPDBs() { } private void defineServerResources(String serverName) { - testSupport.defineResources(createServerPod(serverName), createServerService(serverName)); + defineServerResources(serverName, CLUSTER); + } + + private void defineServerResources(String serverName, String clusterName) { + testSupport.defineResources(createServerPod(serverName, clusterName), createServerService(serverName)); } /**/ - private V1Pod createServerPod(String serverName) { + private V1Pod createServerPod(String serverName, String clusterName) { Packet packet = new Packet(); packet .getComponents() @@ -1249,14 +1320,14 @@ private V1Pod createServerPod(String serverName) { packet.put(ProcessingConstants.SERVER_SCAN, domainConfig.getServerConfig(serverName)); return PodHelper.createAdminServerPodModel(packet); } else { - packet.put(ProcessingConstants.CLUSTER_NAME, CLUSTER); - packet.put(ProcessingConstants.SERVER_SCAN, getClusteredServerConfig(serverName)); + packet.put(ProcessingConstants.CLUSTER_NAME, clusterName); + packet.put(ProcessingConstants.SERVER_SCAN, getClusteredServerConfig(serverName, clusterName)); return PodHelper.createManagedServerPodModel(packet); } } - private WlsServerConfig getClusteredServerConfig(String serverName) { - return domainConfig.getClusterConfig(CLUSTER).getServerConfigs() + private WlsServerConfig getClusteredServerConfig(String serverName, String clusterName) { + return domainConfig.getClusterConfig(clusterName).getServerConfigs() .stream() .filter(c -> serverName.equals(c.getName())).findFirst() .orElseThrow();