From db0cc91fc76d013d1c1a54a7215bc5bc32160ae7 Mon Sep 17 00:00:00 2001 From: Anil Kedia Date: Tue, 12 Oct 2021 04:40:57 +0000 Subject: [PATCH 1/3] OWLS-93072 - Add "introspect failure retry count exceeded error" msg to domain status failure msg. --- .../operator/DomainStatusUpdater.java | 47 +++++++++ .../operator/ProcessingConstants.java | 12 ++- .../operator/DomainConditionMatcher.java | 4 +- .../helpers/DomainIntrospectorJobTest.java | 98 ++++++++++++++++--- 4 files changed, 143 insertions(+), 18 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java b/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java index 5c428840ba4..db7162551d9 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java +++ b/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java @@ -7,6 +7,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -54,13 +55,16 @@ import oracle.kubernetes.weblogic.domain.model.OnlineUpdate; import oracle.kubernetes.weblogic.domain.model.ServerHealth; import oracle.kubernetes.weblogic.domain.model.ServerStatus; +import org.jetbrains.annotations.NotNull; import static oracle.kubernetes.operator.LabelConstants.CLUSTERNAME_LABEL; import static oracle.kubernetes.operator.MIINonDynamicChangesMethod.CommitUpdateOnly; import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_TOPOLOGY; import static oracle.kubernetes.operator.ProcessingConstants.EXCEEDED_INTROSPECTOR_MAX_RETRY_COUNT_ERROR_MSG; +import static oracle.kubernetes.operator.ProcessingConstants.FATAL_ERROR_STATUS_MESSAGE; import static oracle.kubernetes.operator.ProcessingConstants.FATAL_INTROSPECTOR_ERROR; import static oracle.kubernetes.operator.ProcessingConstants.FATAL_INTROSPECTOR_ERROR_MSG; +import static oracle.kubernetes.operator.ProcessingConstants.INTROSPECTION_ERROR; import static oracle.kubernetes.operator.ProcessingConstants.MII_DYNAMIC_UPDATE; import static oracle.kubernetes.operator.ProcessingConstants.MII_DYNAMIC_UPDATE_RESTART_REQUIRED; import static oracle.kubernetes.operator.ProcessingConstants.SERVER_HEALTH_MAP; @@ -769,6 +773,49 @@ public FailureCountStep() { @Override void modifyStatus(DomainStatus domainStatus) { domainStatus.incrementIntrospectJobFailureCount(); + setStatusMessage(domainStatus); + } + + private void setStatusMessage(DomainStatus domainStatus) { + + if (hasExceededMaxRetryCount(domainStatus)) { + domainStatus.setMessage(String.join(System.lineSeparator(), EXCEEDED_INTROSPECTOR_MAX_RETRY_COUNT_ERROR_MSG, + INTROSPECTION_ERROR, getNonNullMessage(domainStatus))); + } else if (isFatalError(domainStatus)) { + domainStatus.setMessage(String.join(System.lineSeparator(), FATAL_ERROR_STATUS_MESSAGE, + INTROSPECTION_ERROR, getNonNullMessage(domainStatus))); + } else { + domainStatus.setMessage(String.join(System.lineSeparator(), getNonFatalErrorMessage(domainStatus), + INTROSPECTION_ERROR, getNonNullMessage(domainStatus))); + } + } + + private String getNonNullMessage(DomainStatus domainStatus) { + return Optional.ofNullable(domainStatus.getMessage()).orElse(getFailedConditionMessage(domainStatus)); + + } + + private String getFailedConditionMessage(DomainStatus domainStatus) { + List failedConditions = domainStatus.getConditions().stream() + .filter(c -> c.hasType(Failed) && c.getMessage() != null) + .collect(Collectors.toList()); + return failedConditions.size() >= 1 ? failedConditions.get(failedConditions.size() - 1).getMessage() + : ""; + } + + @NotNull + private String getNonFatalErrorMessage(DomainStatus domainStatus) { + return "Introspection failed on try " + domainStatus.getIntrospectJobFailureCount() + + " of " + DomainPresence.getDomainPresenceFailureRetryMaxCount() + "."; + } + + private boolean hasExceededMaxRetryCount(DomainStatus domainStatus) { + return domainStatus.getIntrospectJobFailureCount() >= DomainPresence.getDomainPresenceFailureRetryMaxCount(); + } + + private boolean isFatalError(DomainStatus domainStatus) { + return Optional.ofNullable(domainStatus.getMessage()) + .map(m -> m.contains(FATAL_INTROSPECTOR_ERROR)).orElse(false); } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java b/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java index 4b605d55486..49981e8cb48 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java @@ -53,9 +53,17 @@ public interface ProcessingConstants { String EXCEEDED_INTROSPECTOR_MAX_RETRY_COUNT_ERROR_MSG = "Stop introspection retry - " + "exceeded configured domainPresenceFailureRetryMaxCount: " + DomainPresence.getDomainPresenceFailureRetryMaxCount() - + " The domainPresenceFailureRetryMaxCount is an operator tuning parameter and can be controlled" - + " by adding it to the weblogic-operator-cm configmap."; + + ". The domainPresenceFailureRetryMaxCount is an operator tuning parameter and can be controlled" + + " by adding it to the weblogic-operator-cm configmap." + + " To force the introspector to start retrying again, update 'domain.spec.introspectVersion'."; String FATAL_INTROSPECTOR_ERROR_MSG = "Stop introspection retry - MII Fatal Error: "; String OPERATOR_EVENT_LABEL_FILTER = LabelConstants.getCreatedByOperatorSelector(); + + String INTROSPECTION_ERROR = "Introspection Error: "; + + String FATAL_ERROR_STATUS_MESSAGE = "Introspection encountered a fatal error and will not retry automatically." + + " Please resolve the error and then update 'domain.spec.introspectVersion' to force a retry."; + + } diff --git a/operator/src/test/java/oracle/kubernetes/operator/DomainConditionMatcher.java b/operator/src/test/java/oracle/kubernetes/operator/DomainConditionMatcher.java index d0323b1b7f5..daa43c560c7 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/DomainConditionMatcher.java +++ b/operator/src/test/java/oracle/kubernetes/operator/DomainConditionMatcher.java @@ -16,7 +16,7 @@ import org.hamcrest.TypeSafeDiagnosingMatcher; @SuppressWarnings("unused") -class DomainConditionMatcher extends TypeSafeDiagnosingMatcher { +public class DomainConditionMatcher extends TypeSafeDiagnosingMatcher { private final DomainConditionType expectedType; private String expectedStatus; private String expectedReason; @@ -26,7 +26,7 @@ private DomainConditionMatcher(DomainConditionType expectedType) { this.expectedType = expectedType; } - static DomainConditionMatcher hasCondition(DomainConditionType type) { + public static DomainConditionMatcher hasCondition(DomainConditionType type) { return new DomainConditionMatcher(type); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java index 9255f1150f7..40f63203847 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java @@ -36,6 +36,7 @@ import oracle.kubernetes.operator.work.FiberTestSupport; import oracle.kubernetes.operator.work.Step; import oracle.kubernetes.operator.work.TerminalStep; +import oracle.kubernetes.utils.SystemClock; import oracle.kubernetes.utils.TestUtils; import oracle.kubernetes.weblogic.domain.DomainConfigurator; import oracle.kubernetes.weblogic.domain.DomainConfiguratorFactory; @@ -62,6 +63,9 @@ import static oracle.kubernetes.operator.DomainUpPlanTest.StepChainMatcher.hasChainWithStepsInOrder; import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_INTROSPECTOR_JOB; import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_TOPOLOGY; +import static oracle.kubernetes.operator.ProcessingConstants.EXCEEDED_INTROSPECTOR_MAX_RETRY_COUNT_ERROR_MSG; +import static oracle.kubernetes.operator.ProcessingConstants.FATAL_ERROR_STATUS_MESSAGE; +import static oracle.kubernetes.operator.ProcessingConstants.INTROSPECTION_ERROR; import static oracle.kubernetes.operator.ProcessingConstants.JOBWATCHER_COMPONENT_NAME; import static oracle.kubernetes.operator.ProcessingConstants.JOB_POD_NAME; import static oracle.kubernetes.operator.helpers.DomainStatusMatcher.hasStatus; @@ -110,8 +114,10 @@ class DomainIntrospectorJobTest { IntStream.rangeClosed(1, MAX_SERVERS).mapToObj(n -> MS_PREFIX + n).toArray(String[]::new); private static final String SEVERE_PROBLEM_1 = "really bad"; private static final String SEVERE_MESSAGE_1 = "@[SEVERE] " + SEVERE_PROBLEM_1; + private static final String FATAL_PROBLEM_1 = "FatalIntrospectorError: really bad"; + private static final String FATAL_MESSAGE_1 = "@[SEVERE] " + FATAL_PROBLEM_1; public static final String TEST_VOLUME_NAME = "test"; - public static final String LAST_JOB_PROCESSED_ID = "some-unique-id"; + public static final String JOB_UID = "some-unique-id"; private final TerminalStep terminalStep = new TerminalStep(); private final Domain domain = createDomain(); @@ -122,6 +128,7 @@ class DomainIntrospectorJobTest { private final DomainConfigurator configurator = DomainConfiguratorFactory.forDomain(domain); private final EventRetryStrategyStub retryStrategy = createStrictStub(EventRetryStrategyStub.class); private final String jobPodName = LegalNames.toJobIntrospectorName(UID); + private TestUtils.ConsoleHandlerMemento consoleHandlerMemento; public DomainIntrospectorJobTest() { } @@ -133,7 +140,7 @@ private static String getJobName() { @BeforeEach public void setUp() throws Exception { mementos.add( - TestUtils.silenceOperatorLogger() + consoleHandlerMemento = TestUtils.silenceOperatorLogger() .collectLogMessages(logRecords, getMessageKeys()) .withLogLevel(Level.FINE) .ignoringLoggedExceptions(ApiException.class)); @@ -575,7 +582,7 @@ void whenJobLogContainsSevereError_incrementFailureCount() { @Test void whenReadJobLogCompletesWithSevereError_domainStatusContainsLastProcessedJobId() { testSupport.defineResources( - new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS).uid(LAST_JOB_PROCESSED_ID)) + new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS).uid(JOB_UID)) .status(new V1JobStatus())); IntrospectionTestUtils.defineResources(testSupport, SEVERE_MESSAGE_1); testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName())); @@ -584,24 +591,19 @@ void whenReadJobLogCompletesWithSevereError_domainStatusContainsLastProcessedJob final Domain updatedDomain = testSupport.getResources(DOMAIN).get(0); - assertThat(updatedDomain.getStatus().getLastIntrospectJobProcessedUid(), equalTo(LAST_JOB_PROCESSED_ID)); + assertThat(updatedDomain.getStatus().getLastIntrospectJobProcessedUid(), equalTo(JOB_UID)); logRecords.clear(); } @Test void whenReadJobLogCompletesWithoutSevereError_domainStatusContainsLastProcessedJobId() { - testSupport.defineResources( - new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS).uid(LAST_JOB_PROCESSED_ID)) - .status(new V1JobStatus())); - IntrospectionTestUtils.defineResources(testSupport, "passed"); - testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName())); + createIntrospectionLog("passed"); testSupport.runSteps(JobHelper.readDomainIntrospectorPodLog(terminalStep)); final Domain updatedDomain = testSupport.getResources(DOMAIN).get(0); - assertThat(updatedDomain.getStatus().getLastIntrospectJobProcessedUid(), equalTo(LAST_JOB_PROCESSED_ID)); - logRecords.clear(); + assertThat(updatedDomain.getStatus().getLastIntrospectJobProcessedUid(), equalTo(JOB_UID)); } @Test @@ -609,7 +611,7 @@ void whenDomainStatusContainsNullLastIntrospectProcessedJobUid_correctStepsExecu List nextSteps = new ArrayList<>(); domainPresenceInfo.getDomain() .setStatus(new DomainStatus().withLastIntrospectJobProcessedUid(null)); - V1Job job = new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS).uid(LAST_JOB_PROCESSED_ID)) + V1Job job = new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS).uid(JOB_UID)) .status(new V1JobStatus()); testSupport.defineResources(job); IntrospectionTestUtils.defineResources(testSupport, "passed"); @@ -626,8 +628,8 @@ void whenDomainStatusContainsNullLastIntrospectProcessedJobUid_correctStepsExecu void whenDomainStatusContainsProcessedJobIdSameAsCurrentJob_correctStepsExecuted() { List nextSteps = new ArrayList<>(); domainPresenceInfo.getDomain() - .setStatus(new DomainStatus().withLastIntrospectJobProcessedUid(LAST_JOB_PROCESSED_ID)); - V1Job job = new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS).uid(LAST_JOB_PROCESSED_ID)) + .setStatus(new DomainStatus().withLastIntrospectJobProcessedUid(JOB_UID)); + V1Job job = new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS).uid(JOB_UID)) .status(new V1JobStatus()); testSupport.defineResources(job); IntrospectionTestUtils.defineResources(testSupport, "passed"); @@ -667,6 +669,74 @@ void whenJobCreateFailsWith409Error_JobIsCreated() { logRecords.clear(); } + @Test + void whenJobLogContainsSevereError_domainStatusHasExpectedMessage() { + createIntrospectionLog(); + + testSupport.runSteps(JobHelper.readDomainIntrospectorPodLog(terminalStep)); + + assertThat(getUpdatedDomain().getStatus().getMessage(), equalTo(String.join(System.lineSeparator(), + "Introspection failed on try 1 of 2.", INTROSPECTION_ERROR, + SEVERE_PROBLEM_1))); + } + + private void createIntrospectionLog() { + createIntrospectionLog(SEVERE_MESSAGE_1); + } + + private void createIntrospectionLog(String logMessage) { + consoleHandlerMemento.ignoreMessage(getJobFailedMessageKey()); + consoleHandlerMemento.ignoreMessage(getJobFailedDetailMessageKey()); + testSupport.defineResources(createIntrospectorJob()); + IntrospectionTestUtils.defineResources(testSupport, logMessage); + testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName())); + } + + private V1Job createIntrospectorJob() { + return new V1Job().metadata(createJobMetadata()).status(new V1JobStatus()); + } + + private V1ObjectMeta createJobMetadata() { + return new V1ObjectMeta().name(getJobName()).namespace(NS).creationTimestamp(SystemClock.now()).uid(JOB_UID); + } + + @Test + void whenJobLogContainsSevereError_domainStatusHasExpectedMessage1() { + createIntrospectionLog(FATAL_MESSAGE_1); + + testSupport.runSteps(JobHelper.readDomainIntrospectorPodLog(terminalStep)); + + final Domain updatedDomain = testSupport.getResources(DOMAIN).get(0); + + assertThat(updatedDomain.getStatus().getMessage(), equalTo(String.join(System.lineSeparator(), + FATAL_ERROR_STATUS_MESSAGE, INTROSPECTION_ERROR, + FATAL_PROBLEM_1))); + logRecords.clear(); + } + + @Test + void whenJobLogContainsSevereError_domainStatusHasExpectedMessage2() { + createIntrospectionLog(); + + getUpdatedDomain().setStatus(createDomainStatusWithIntrospectJobFailureCount(2)); + + testSupport.runSteps(JobHelper.readDomainIntrospectorPodLog(terminalStep)); + + assertThat(getUpdatedDomain().getStatus().getMessage(), equalTo(String.join(System.lineSeparator(), + EXCEEDED_INTROSPECTOR_MAX_RETRY_COUNT_ERROR_MSG, INTROSPECTION_ERROR, + SEVERE_PROBLEM_1))); + } + + private DomainStatus createDomainStatusWithIntrospectJobFailureCount(int failureCount) { + final DomainStatus status = new DomainStatus(); + status.withIntrospectJobFailureCount(failureCount); + return status; + } + + private Domain getUpdatedDomain() { + return testSupport.getResources(DOMAIN).get(0); + } + private Cluster getCluster(String clusterName) { return domain.getSpec().getClusters().stream() .filter(c -> clusterName.equals(c.getClusterName())) From 5b9d2ee3f22aaf33b01f92695d24b865b2358d72 Mon Sep 17 00:00:00 2001 From: Anil Kedia Date: Tue, 12 Oct 2021 17:43:24 +0000 Subject: [PATCH 2/3] Cleanup and unit test refactoring. --- .../operator/DomainStatusUpdater.java | 19 ++-- .../operator/ProcessingConstants.java | 2 +- .../helpers/DomainIntrospectorJobTest.java | 91 ++++++++----------- 3 files changed, 49 insertions(+), 63 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java b/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java index db7162551d9..300407632b3 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java +++ b/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java @@ -61,7 +61,7 @@ import static oracle.kubernetes.operator.MIINonDynamicChangesMethod.CommitUpdateOnly; import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_TOPOLOGY; import static oracle.kubernetes.operator.ProcessingConstants.EXCEEDED_INTROSPECTOR_MAX_RETRY_COUNT_ERROR_MSG; -import static oracle.kubernetes.operator.ProcessingConstants.FATAL_ERROR_STATUS_MESSAGE; +import static oracle.kubernetes.operator.ProcessingConstants.FATAL_ERROR_DOMAIN_STATUS_MESSAGE; import static oracle.kubernetes.operator.ProcessingConstants.FATAL_INTROSPECTOR_ERROR; import static oracle.kubernetes.operator.ProcessingConstants.FATAL_INTROSPECTOR_ERROR_MSG; import static oracle.kubernetes.operator.ProcessingConstants.INTROSPECTION_ERROR; @@ -779,17 +779,20 @@ void modifyStatus(DomainStatus domainStatus) { private void setStatusMessage(DomainStatus domainStatus) { if (hasExceededMaxRetryCount(domainStatus)) { - domainStatus.setMessage(String.join(System.lineSeparator(), EXCEEDED_INTROSPECTOR_MAX_RETRY_COUNT_ERROR_MSG, - INTROSPECTION_ERROR, getNonNullMessage(domainStatus))); + domainStatus.setMessage(onSeparateLines(domainStatus, EXCEEDED_INTROSPECTOR_MAX_RETRY_COUNT_ERROR_MSG)); } else if (isFatalError(domainStatus)) { - domainStatus.setMessage(String.join(System.lineSeparator(), FATAL_ERROR_STATUS_MESSAGE, - INTROSPECTION_ERROR, getNonNullMessage(domainStatus))); + domainStatus.setMessage(onSeparateLines(domainStatus, FATAL_ERROR_DOMAIN_STATUS_MESSAGE)); } else { - domainStatus.setMessage(String.join(System.lineSeparator(), getNonFatalErrorMessage(domainStatus), - INTROSPECTION_ERROR, getNonNullMessage(domainStatus))); + domainStatus.setMessage(onSeparateLines(domainStatus, getNonFatalErrorStatusMessage(domainStatus))); } } + @NotNull + private String onSeparateLines(DomainStatus domainStatus, String exceededIntrospectorMaxRetryCountErrorMsg) { + return String.join(System.lineSeparator(), exceededIntrospectorMaxRetryCountErrorMsg, + INTROSPECTION_ERROR, getNonNullMessage(domainStatus)); + } + private String getNonNullMessage(DomainStatus domainStatus) { return Optional.ofNullable(domainStatus.getMessage()).orElse(getFailedConditionMessage(domainStatus)); @@ -804,7 +807,7 @@ private String getFailedConditionMessage(DomainStatus domainStatus) { } @NotNull - private String getNonFatalErrorMessage(DomainStatus domainStatus) { + private String getNonFatalErrorStatusMessage(DomainStatus domainStatus) { return "Introspection failed on try " + domainStatus.getIntrospectJobFailureCount() + " of " + DomainPresence.getDomainPresenceFailureRetryMaxCount() + "."; } diff --git a/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java b/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java index 49981e8cb48..551a0994fcb 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java @@ -62,7 +62,7 @@ public interface ProcessingConstants { String INTROSPECTION_ERROR = "Introspection Error: "; - String FATAL_ERROR_STATUS_MESSAGE = "Introspection encountered a fatal error and will not retry automatically." + String FATAL_ERROR_DOMAIN_STATUS_MESSAGE = "Introspection encountered a fatal error and will not retry automatically." + " Please resolve the error and then update 'domain.spec.introspectVersion' to force a retry."; diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java index 40f63203847..7196abe6b68 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java @@ -64,7 +64,7 @@ import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_INTROSPECTOR_JOB; import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_TOPOLOGY; import static oracle.kubernetes.operator.ProcessingConstants.EXCEEDED_INTROSPECTOR_MAX_RETRY_COUNT_ERROR_MSG; -import static oracle.kubernetes.operator.ProcessingConstants.FATAL_ERROR_STATUS_MESSAGE; +import static oracle.kubernetes.operator.ProcessingConstants.FATAL_ERROR_DOMAIN_STATUS_MESSAGE; import static oracle.kubernetes.operator.ProcessingConstants.INTROSPECTION_ERROR; import static oracle.kubernetes.operator.ProcessingConstants.JOBWATCHER_COMPONENT_NAME; import static oracle.kubernetes.operator.ProcessingConstants.JOB_POD_NAME; @@ -118,6 +118,7 @@ class DomainIntrospectorJobTest { private static final String FATAL_MESSAGE_1 = "@[SEVERE] " + FATAL_PROBLEM_1; public static final String TEST_VOLUME_NAME = "test"; public static final String JOB_UID = "some-unique-id"; + public static final String INFO_MESSAGE_1 = "informational message"; private final TerminalStep terminalStep = new TerminalStep(); private final Domain domain = createDomain(); @@ -539,10 +540,7 @@ void whenIntrospectorJobNotNeeded_doesNotValidatesDomainTopology() throws JsonPr @Test void whenJobLogContainsSevereError_logJobInfosOnDelete() { - testSupport.defineResources( - new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS)).status(new V1JobStatus())); - IntrospectionTestUtils.defineResources(testSupport, SEVERE_MESSAGE_1); - testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName())); + createIntrospectionLog(SEVERE_MESSAGE_1, false); testSupport.runSteps(JobHelper.deleteDomainIntrospectorJobStep(terminalStep)); @@ -553,10 +551,7 @@ void whenJobLogContainsSevereError_logJobInfosOnDelete() { @Test void whenJobLogContainsSevereError_logJobInfosOnReadPogLog() { - testSupport.defineResources( - new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS)).status(new V1JobStatus())); - IntrospectionTestUtils.defineResources(testSupport, SEVERE_MESSAGE_1); - testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName())); + createIntrospectionLog(SEVERE_MESSAGE_1, false); testSupport.runSteps(JobHelper.readDomainIntrospectorPodLog(terminalStep)); @@ -566,38 +561,25 @@ void whenJobLogContainsSevereError_logJobInfosOnReadPogLog() { @Test void whenJobLogContainsSevereError_incrementFailureCount() { - testSupport.defineResources( - new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS)).status(new V1JobStatus())); - IntrospectionTestUtils.defineResources(testSupport, SEVERE_MESSAGE_1); - testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName())); + createIntrospectionLog(SEVERE_MESSAGE_1); testSupport.runSteps(JobHelper.readDomainIntrospectorPodLog(terminalStep)); - final Domain updatedDomain = testSupport.getResources(DOMAIN).get(0); - - assertThat(updatedDomain.getStatus().getIntrospectJobFailureCount(), equalTo(1)); - logRecords.clear(); + assertThat(getUpdatedDomain().getStatus().getIntrospectJobFailureCount(), equalTo(1)); } @Test void whenReadJobLogCompletesWithSevereError_domainStatusContainsLastProcessedJobId() { - testSupport.defineResources( - new V1Job().metadata(new V1ObjectMeta().name(getJobName()).namespace(NS).uid(JOB_UID)) - .status(new V1JobStatus())); - IntrospectionTestUtils.defineResources(testSupport, SEVERE_MESSAGE_1); - testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName())); + createIntrospectionLog(SEVERE_MESSAGE_1); testSupport.runSteps(JobHelper.readDomainIntrospectorPodLog(terminalStep)); - final Domain updatedDomain = testSupport.getResources(DOMAIN).get(0); - - assertThat(updatedDomain.getStatus().getLastIntrospectJobProcessedUid(), equalTo(JOB_UID)); - logRecords.clear(); + assertThat(getUpdatedDomain().getStatus().getLastIntrospectJobProcessedUid(), equalTo(JOB_UID)); } @Test void whenReadJobLogCompletesWithoutSevereError_domainStatusContainsLastProcessedJobId() { - createIntrospectionLog("passed"); + createIntrospectionLog(INFO_MESSAGE_1); testSupport.runSteps(JobHelper.readDomainIntrospectorPodLog(terminalStep)); @@ -670,8 +652,8 @@ void whenJobCreateFailsWith409Error_JobIsCreated() { } @Test - void whenJobLogContainsSevereError_domainStatusHasExpectedMessage() { - createIntrospectionLog(); + void whenJobLogContainsSevereErrorAndRetriesLeft_domainStatusHasExpectedMessage() { + createIntrospectionLog(SEVERE_MESSAGE_1); testSupport.runSteps(JobHelper.readDomainIntrospectorPodLog(terminalStep)); @@ -680,28 +662,8 @@ void whenJobLogContainsSevereError_domainStatusHasExpectedMessage() { SEVERE_PROBLEM_1))); } - private void createIntrospectionLog() { - createIntrospectionLog(SEVERE_MESSAGE_1); - } - - private void createIntrospectionLog(String logMessage) { - consoleHandlerMemento.ignoreMessage(getJobFailedMessageKey()); - consoleHandlerMemento.ignoreMessage(getJobFailedDetailMessageKey()); - testSupport.defineResources(createIntrospectorJob()); - IntrospectionTestUtils.defineResources(testSupport, logMessage); - testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName())); - } - - private V1Job createIntrospectorJob() { - return new V1Job().metadata(createJobMetadata()).status(new V1JobStatus()); - } - - private V1ObjectMeta createJobMetadata() { - return new V1ObjectMeta().name(getJobName()).namespace(NS).creationTimestamp(SystemClock.now()).uid(JOB_UID); - } - @Test - void whenJobLogContainsSevereError_domainStatusHasExpectedMessage1() { + void whenJobLogContainsFatalError_domainStatusHasExpectedMessage() { createIntrospectionLog(FATAL_MESSAGE_1); testSupport.runSteps(JobHelper.readDomainIntrospectorPodLog(terminalStep)); @@ -709,14 +671,13 @@ void whenJobLogContainsSevereError_domainStatusHasExpectedMessage1() { final Domain updatedDomain = testSupport.getResources(DOMAIN).get(0); assertThat(updatedDomain.getStatus().getMessage(), equalTo(String.join(System.lineSeparator(), - FATAL_ERROR_STATUS_MESSAGE, INTROSPECTION_ERROR, + FATAL_ERROR_DOMAIN_STATUS_MESSAGE, INTROSPECTION_ERROR, FATAL_PROBLEM_1))); - logRecords.clear(); } @Test - void whenJobLogContainsSevereError_domainStatusHasExpectedMessage2() { - createIntrospectionLog(); + void whenJobLogContainsSevereErrorAndNumberOfRetriesExceedsMaxLimit_domainStatusHasExpectedMessage() { + createIntrospectionLog(SEVERE_MESSAGE_1); getUpdatedDomain().setStatus(createDomainStatusWithIntrospectJobFailureCount(2)); @@ -727,6 +688,28 @@ void whenJobLogContainsSevereError_domainStatusHasExpectedMessage2() { SEVERE_PROBLEM_1))); } + private void createIntrospectionLog(String logMessage) { + createIntrospectionLog(logMessage, true); + } + + private void createIntrospectionLog(String logMessage, boolean ignoreLogMessages) { + if (ignoreLogMessages) { + consoleHandlerMemento.ignoreMessage(getJobFailedMessageKey()); + consoleHandlerMemento.ignoreMessage(getJobFailedDetailMessageKey()); + } + testSupport.defineResources(createIntrospectorJob()); + IntrospectionTestUtils.defineResources(testSupport, logMessage); + testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName())); + } + + private V1Job createIntrospectorJob() { + return new V1Job().metadata(createJobMetadata()).status(new V1JobStatus()); + } + + private V1ObjectMeta createJobMetadata() { + return new V1ObjectMeta().name(getJobName()).namespace(NS).creationTimestamp(SystemClock.now()).uid(JOB_UID); + } + private DomainStatus createDomainStatusWithIntrospectJobFailureCount(int failureCount) { final DomainStatus status = new DomainStatus(); status.withIntrospectJobFailureCount(failureCount); From 9dd0aecc54b5b721fc5f4115079b38b733a2f96a Mon Sep 17 00:00:00 2001 From: Anil Kedia Date: Tue, 12 Oct 2021 17:49:30 +0000 Subject: [PATCH 3/3] Restore access modifier for class and method. --- .../oracle/kubernetes/operator/DomainConditionMatcher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operator/src/test/java/oracle/kubernetes/operator/DomainConditionMatcher.java b/operator/src/test/java/oracle/kubernetes/operator/DomainConditionMatcher.java index daa43c560c7..d0323b1b7f5 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/DomainConditionMatcher.java +++ b/operator/src/test/java/oracle/kubernetes/operator/DomainConditionMatcher.java @@ -16,7 +16,7 @@ import org.hamcrest.TypeSafeDiagnosingMatcher; @SuppressWarnings("unused") -public class DomainConditionMatcher extends TypeSafeDiagnosingMatcher { +class DomainConditionMatcher extends TypeSafeDiagnosingMatcher { private final DomainConditionType expectedType; private String expectedStatus; private String expectedReason; @@ -26,7 +26,7 @@ private DomainConditionMatcher(DomainConditionType expectedType) { this.expectedType = expectedType; } - public static DomainConditionMatcher hasCondition(DomainConditionType type) { + static DomainConditionMatcher hasCondition(DomainConditionType type) { return new DomainConditionMatcher(type); }