diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/bluegreen/BlueGreenConnectionPlugin.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/bluegreen/BlueGreenConnectionPlugin.java index bcb5be447..570366a39 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/bluegreen/BlueGreenConnectionPlugin.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/bluegreen/BlueGreenConnectionPlugin.java @@ -168,10 +168,13 @@ public Connection connect( isInitialConnection, connectFunc, this.pluginService); - if (conn == null) { this.bgStatus = this.pluginService.getStatus(BlueGreenStatus.class, this.bgdId); + if (this.bgStatus == null) { + this.endTimeNano.set(this.getNanoTime()); + return regularOpenConnection(connectFunc, isInitialConnection); + } routing = this.bgStatus.getConnectRouting().stream() .filter(r -> r.isMatch(hostSpec, hostRole)) @@ -274,6 +277,10 @@ public T execute( if (!result.isPresent()) { this.bgStatus = this.pluginService.getStatus(BlueGreenStatus.class, this.bgdId); + if (this.bgStatus == null) { + this.endTimeNano.set(this.getNanoTime()); + return jdbcMethodFunc.call(); + } routing = this.bgStatus.getExecuteRouting().stream() .filter(r -> r.isMatch(currentHostSpec, hostRole)) diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/bluegreen/BlueGreenStatusProvider.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/bluegreen/BlueGreenStatusProvider.java index 573001294..b6ec7dca8 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/bluegreen/BlueGreenStatusProvider.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/bluegreen/BlueGreenStatusProvider.java @@ -190,14 +190,14 @@ protected Properties getMonitoringProperties() { } protected void prepareStatus( - final BlueGreenRole role, - final BlueGreenInterimStatus interimStatus) { + final @NonNull BlueGreenRole role, + final @NonNull BlueGreenInterimStatus interimStatus) { this.processStatusLock.lock(); try { // Detect changes - int statusHash = interimStatus == null ? 0 : interimStatus.getCustomHashCode(); + int statusHash = interimStatus.getCustomHashCode(); int contextHash = this.getContextHash(); if (this.interimStatusHashes[role.getValue()] == statusHash && this.lastContextHash == contextHash) { @@ -240,10 +240,8 @@ protected void prepareStatus( } protected void updatePhase(BlueGreenRole role, BlueGreenInterimStatus interimStatus) { - - BlueGreenPhase latestInterimPhase = this.interimStatuses[role.getValue()] == null - ? BlueGreenPhase.NOT_CREATED - : this.interimStatuses[role.getValue()].blueGreenPhase; + BlueGreenInterimStatus roleStatus = this.interimStatuses[role.getValue()]; + BlueGreenPhase latestInterimPhase = roleStatus == null ? BlueGreenPhase.NOT_CREATED : roleStatus.blueGreenPhase; if (latestInterimPhase != null && interimStatus.blueGreenPhase != null @@ -292,10 +290,10 @@ protected void updateStatusCache() { protected void updateCorrespondingNodes() { this.correspondingNodes.clear(); - if (this.interimStatuses[BlueGreenRole.SOURCE.getValue()] != null - && !Utils.isNullOrEmpty(this.interimStatuses[BlueGreenRole.SOURCE.getValue()].startTopology) - && this.interimStatuses[BlueGreenRole.TARGET.getValue()] != null - && !Utils.isNullOrEmpty(this.interimStatuses[BlueGreenRole.TARGET.getValue()].startTopology)) { + BlueGreenInterimStatus sourceInterimStatus = this.interimStatuses[BlueGreenRole.SOURCE.getValue()]; + BlueGreenInterimStatus targetInterimStatus = this.interimStatuses[BlueGreenRole.TARGET.getValue()]; + if (sourceInterimStatus != null && !Utils.isNullOrEmpty(sourceInterimStatus.startTopology) + && targetInterimStatus != null && !Utils.isNullOrEmpty(targetInterimStatus.startTopology)) { HostSpec blueWriterHostSpec = getWriterHost(BlueGreenRole.SOURCE); HostSpec greenWriterHostSpec = getWriterHost(BlueGreenRole.TARGET); @@ -308,29 +306,30 @@ protected void updateCorrespondingNodes() { blueWriterHostSpec.getHost(), Pair.create(blueWriterHostSpec, greenWriterHostSpec)); } - if (!sortedGreenReaderHostSpecs.isEmpty()) { - int greenIndex = 0; - for (HostSpec blueHostSpec : sortedBlueReaderHostSpecs) { - this.correspondingNodes.put( - blueHostSpec.getHost(), Pair.create(blueHostSpec, sortedGreenReaderHostSpecs.get(greenIndex++))); - greenIndex %= sortedGreenReaderHostSpecs.size(); - } - } else { - // There's no green reader nodes. We have to map all blue reader nodes to a green writer. - for (HostSpec blueHostSpec : sortedBlueReaderHostSpecs) { - this.correspondingNodes.put( - blueHostSpec.getHost(), Pair.create(blueHostSpec, greenWriterHostSpec)); + if (!Utils.isNullOrEmpty(sortedBlueReaderHostSpecs)) { + // Map blue readers to green nodes. + if (!Utils.isNullOrEmpty(sortedGreenReaderHostSpecs)) { + int greenIndex = 0; + for (HostSpec blueHostSpec : sortedBlueReaderHostSpecs) { + this.correspondingNodes.put( + blueHostSpec.getHost(), Pair.create(blueHostSpec, sortedGreenReaderHostSpecs.get(greenIndex++))); + greenIndex %= sortedGreenReaderHostSpecs.size(); + } + } else { + // There's no green reader nodes. We have to map all blue reader nodes to the green writer. + for (HostSpec blueHostSpec : sortedBlueReaderHostSpecs) { + this.correspondingNodes.put( + blueHostSpec.getHost(), Pair.create(blueHostSpec, greenWriterHostSpec)); + } } } } - if (this.interimStatuses[BlueGreenRole.SOURCE.getValue()] != null - && !Utils.isNullOrEmpty(this.interimStatuses[BlueGreenRole.SOURCE.getValue()].hostNames) - && this.interimStatuses[BlueGreenRole.TARGET.getValue()] != null - && !Utils.isNullOrEmpty(this.interimStatuses[BlueGreenRole.TARGET.getValue()].hostNames)) { + if (sourceInterimStatus != null && !Utils.isNullOrEmpty(sourceInterimStatus.hostNames) + && targetInterimStatus != null && !Utils.isNullOrEmpty(targetInterimStatus.hostNames)) { - Set blueHosts = this.interimStatuses[BlueGreenRole.SOURCE.getValue()].hostNames; - Set greenHosts = this.interimStatuses[BlueGreenRole.TARGET.getValue()].hostNames; + Set blueHosts = sourceInterimStatus.hostNames; + Set greenHosts = targetInterimStatus.hostNames; // Find corresponding cluster hosts String blueClusterHost = blueHosts.stream().filter(this.rdsUtils::isWriterClusterDns) @@ -380,15 +379,25 @@ protected void updateCorrespondingNodes() { } } - protected HostSpec getWriterHost(final BlueGreenRole role) { - return this.interimStatuses[role.getValue()].startTopology.stream() + protected @Nullable HostSpec getWriterHost(final BlueGreenRole role) { + BlueGreenInterimStatus interimStatus = this.interimStatuses[role.getValue()]; + if (interimStatus == null) { + return null; + } + + return interimStatus.startTopology.stream() .filter(x -> x.getRole() == HostRole.WRITER) .findFirst() .orElse(null); } - protected List getReaderHosts(final BlueGreenRole role) { - return this.interimStatuses[role.getValue()].startTopology.stream() + protected @Nullable List getReaderHosts(final BlueGreenRole role) { + BlueGreenInterimStatus interimStatus = this.interimStatuses[role.getValue()]; + if (interimStatus == null) { + return null; + } + + return interimStatus.startTopology.stream() .filter(x -> x.getRole() != HostRole.WRITER) .sorted(Comparator.comparing(HostSpec::getHost)) .collect(Collectors.toList()); @@ -608,31 +617,43 @@ protected BlueGreenStatus getStatusOfPreparation() { protected List addSubstituteBlueWithIpAddressConnectRouting() { List connectRouting = new ArrayList<>(); - this.roleByHost.entrySet().stream() - .filter(x -> x.getValue() == BlueGreenRole.SOURCE && this.correspondingNodes.containsKey(x.getKey())) - .forEach(x -> { - HostSpec hostSpec = this.correspondingNodes.get(x.getKey()).getValue1(); - Optional blueIp = this.hostIpAddresses.get(hostSpec.getHost()); - HostSpec substituteHostSpecWithIp = blueIp == null || !blueIp.isPresent() - ? hostSpec - : this.hostSpecBuilder.copyFrom(hostSpec).host(blueIp.get()).build(); - - connectRouting.add(new SubstituteConnectRouting( - x.getKey(), - x.getValue(), - substituteHostSpecWithIp, - Collections.singletonList(hostSpec), - null)); - - connectRouting.add(new SubstituteConnectRouting( - this.getHostAndPort( - x.getKey(), - this.interimStatuses[x.getValue().getValue()].port), - x.getValue(), - substituteHostSpecWithIp, - Collections.singletonList(hostSpec), - null)); - }); + for (Map.Entry entry : this.roleByHost.entrySet()) { + String host = entry.getKey(); + BlueGreenRole role = entry.getValue(); + Pair nodePair = this.correspondingNodes.get(host); + if (role != BlueGreenRole.SOURCE || nodePair == null) { + continue; + } + + HostSpec blueHostSpec = nodePair.getValue1(); + Optional blueIp = this.hostIpAddresses.get(blueHostSpec.getHost()); + HostSpec blueIpHostSpec; + if (blueIp == null || !blueIp.isPresent()) { + blueIpHostSpec = blueHostSpec; + } else { + blueIpHostSpec = this.hostSpecBuilder.copyFrom(blueHostSpec).host(blueIp.get()).build(); + } + + connectRouting.add(new SubstituteConnectRouting( + host, + role, + blueIpHostSpec, + Collections.singletonList(blueHostSpec), + null)); + + BlueGreenInterimStatus interimStatus = this.interimStatuses[role.getValue()]; + if (interimStatus == null) { + continue; + } + + connectRouting.add(new SubstituteConnectRouting( + this.getHostAndPort(host, interimStatus.port), + role, + blueIpHostSpec, + Collections.singletonList(blueHostSpec), + null)); + } + return connectRouting; } @@ -815,8 +836,10 @@ protected void createPostRouting(List connectRouting) { .forEach(x -> { final String blueHost = x.getKey(); final boolean isBlueHostInstance = rdsUtils.isRdsInstance(blueHost); - HostSpec blueHostSpec = this.correspondingNodes.get(x.getKey()).getValue1(); - HostSpec greenHostSpec = this.correspondingNodes.get(x.getKey()).getValue2(); + + Pair nodePair = this.correspondingNodes.get(x.getKey()); + HostSpec blueHostSpec = nodePair == null ? null : nodePair.getValue1(); + HostSpec greenHostSpec = nodePair == null ? null : nodePair.getValue2(); if (greenHostSpec == null) { @@ -826,13 +849,13 @@ protected void createPostRouting(List connectRouting) { x.getValue(), this.bgdId)); - connectRouting.add(new SuspendUntilCorrespondingNodeFoundConnectRouting( - this.getHostAndPort( - blueHost, - this.interimStatuses[x.getValue().getValue()].port), - x.getValue(), - this.bgdId)); - + BlueGreenInterimStatus interimStatus = this.interimStatuses[x.getValue().getValue()]; + if (interimStatus != null) { + connectRouting.add(new SuspendUntilCorrespondingNodeFoundConnectRouting( + this.getHostAndPort(blueHost, interimStatus.port), + x.getValue(), + this.bgdId)); + } } else { final String greenHost = greenHostSpec.getHost(); Optional greenIp = this.hostIpAddresses.get(greenHostSpec.getHost()); @@ -841,11 +864,16 @@ protected void createPostRouting(List connectRouting) { : this.hostSpecBuilder.copyFrom(greenHostSpec).host(greenIp.get()).build(); // Check whether green host is already been connected with blue (no-prefixes) IAM host name. - List iamHosts = this.isAlreadySuccessfullyConnected(greenHost, blueHost) - // Green node has already changed its name, and it's not a new blue node (no prefixes). - ? Collections.singletonList(blueHostSpec) - // Green node isn't yet changed its name, so we need to try both possible IAM host options. - : Arrays.asList(greenHostSpec, blueHostSpec); + List iamHosts; + if (this.isAlreadySuccessfullyConnected(greenHost, blueHost)) { + // Green node has already changed its name, and it's not a new blue node (no prefixes). + iamHosts = blueHostSpec == null ? null : Collections.singletonList(blueHostSpec); + } else { + // Green node isn't yet changed its name, so we need to try both possible IAM host options. + iamHosts = blueHostSpec == null + ? Collections.singletonList(greenHostSpec) + : Arrays.asList(greenHostSpec, blueHostSpec); + } connectRouting.add(new SubstituteConnectRouting( blueHost, @@ -854,14 +882,15 @@ protected void createPostRouting(List connectRouting) { iamHosts, isBlueHostInstance ? (iamHost) -> this.registerIamHost(greenHost, iamHost) : null)); - connectRouting.add(new SubstituteConnectRouting( - this.getHostAndPort( - blueHost, - this.interimStatuses[x.getValue().getValue()].port), - x.getValue(), - greenHostSpecWithIp, - iamHosts, - isBlueHostInstance ? (iamHost) -> this.registerIamHost(greenHost, iamHost) : null)); + BlueGreenInterimStatus interimStatus = this.interimStatuses[x.getValue().getValue()]; + if (interimStatus != null) { + connectRouting.add(new SubstituteConnectRouting( + this.getHostAndPort(blueHost, interimStatus.port), + x.getValue(), + greenHostSpecWithIp, + iamHosts, + isBlueHostInstance ? (iamHost) -> this.registerIamHost(greenHost, iamHost) : null)); + } } }); } @@ -996,9 +1025,9 @@ protected void logSwitchoverFinalSummary() { return; } - PhaseTimeInfo timeZero = this.rollback - ? this.phaseTimeNano.get(BlueGreenPhase.PREPARATION.name()) - : this.phaseTimeNano.get(BlueGreenPhase.IN_PROGRESS.name()); + BlueGreenPhase timeZeroPhase = this.rollback ? BlueGreenPhase.PREPARATION : BlueGreenPhase.IN_PROGRESS; + String timeZeroKey = this.rollback ? timeZeroPhase.name() + " (rollback)" : timeZeroPhase.name(); + PhaseTimeInfo timeZero = this.phaseTimeNano.get(timeZeroKey); String divider = "----------------------------------------------------------------------------------\n"; String logMessage = diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/bluegreen/OnBlueGreenStatusChange.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/bluegreen/OnBlueGreenStatusChange.java index 5de76785b..f1168693f 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/bluegreen/OnBlueGreenStatusChange.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/bluegreen/OnBlueGreenStatusChange.java @@ -16,6 +16,7 @@ package software.amazon.jdbc.plugin.bluegreen; +@FunctionalInterface public interface OnBlueGreenStatusChange { void onBlueGreenStatusChanged(BlueGreenRole role, BlueGreenInterimStatus interimStatus); } diff --git a/wrapper/src/test/java/integration/container/tests/BlueGreenDeploymentTests.java b/wrapper/src/test/java/integration/container/tests/BlueGreenDeploymentTests.java index 21b858192..08b4cd71a 100644 --- a/wrapper/src/test/java/integration/container/tests/BlueGreenDeploymentTests.java +++ b/wrapper/src/test/java/integration/container/tests/BlueGreenDeploymentTests.java @@ -238,6 +238,7 @@ public void testSwitchover(TestDriver testDriver) throws SQLException, Interrupt hostId, host, testInstance.getPort(), dbName, startLatchAtomic, stop, finishLatchAtomic, results.get(hostId))); threadCount++; + threadFinishCount++; threads.add(getBlueDnsMonitoringThread( hostId, host, startLatchAtomic, stop, finishLatchAtomic, results.get(hostId))); diff --git a/wrapper/src/test/java/integration/host/TestEnvironment.java b/wrapper/src/test/java/integration/host/TestEnvironment.java index 153801780..791c3c0d6 100644 --- a/wrapper/src/test/java/integration/host/TestEnvironment.java +++ b/wrapper/src/test/java/integration/host/TestEnvironment.java @@ -824,9 +824,15 @@ private static void deAuthorizeIP(TestEnvironment env) { throw new RuntimeException(e); } } - env.auroraUtil.ec2DeauthorizesIP(env.runnerIP); - LOGGER.finest(String.format("Test runner IP %s de-authorized. Usage count: %d", - env.runnerIP, ipAddressUsageRefCount.get())); + + if (!env.reuseDb) { + env.auroraUtil.ec2DeauthorizesIP(env.runnerIP); + LOGGER.finest(String.format("Test runner IP %s de-authorized. Usage count: %d", + env.runnerIP, ipAddressUsageRefCount.get())); + } else { + LOGGER.finest("The IP address usage count hit 0, but the REUSE_RDS_DB was set to true, so IP " + + "de-authorization was skipped."); + } } else { LOGGER.finest("IP usage count: " + ipAddressUsageRefCount.get()); } @@ -1515,10 +1521,12 @@ private void deleteBlueGreenDeployment() throws InterruptedException { } private void deleteCustomClusterParameterGroup(String groupName) { - try { - this.auroraUtil.deleteCustomClusterParameterGroup(groupName); - } catch (Exception ex) { - LOGGER.finest(String.format("Error deleting cluster parameter group %s. %s", groupName, ex)); + if (!this.reuseDb) { + try { + this.auroraUtil.deleteCustomClusterParameterGroup(groupName); + } catch (Exception ex) { + LOGGER.finest(String.format("Error deleting cluster parameter group %s. %s", groupName, ex)); + } } } diff --git a/wrapper/src/test/java/software/amazon/jdbc/plugin/failover/ClusterAwareWriterFailoverHandlerTest.java b/wrapper/src/test/java/software/amazon/jdbc/plugin/failover/ClusterAwareWriterFailoverHandlerTest.java index 515325f7c..0944be13c 100644 --- a/wrapper/src/test/java/software/amazon/jdbc/plugin/failover/ClusterAwareWriterFailoverHandlerTest.java +++ b/wrapper/src/test/java/software/amazon/jdbc/plugin/failover/ClusterAwareWriterFailoverHandlerTest.java @@ -379,7 +379,7 @@ public void testFailedToConnect_taskAException_taskBWriterException() throws SQL when(mockPluginService.forceConnect(refEq(readerA), eq(properties))).thenReturn(mockReaderAConnection); when(mockPluginService.forceConnect(refEq(readerB), eq(properties))).thenReturn(mockReaderBConnection); when(mockPluginService.forceConnect(refEq(newWriterHost), eq(properties))).thenThrow(exception); - when(mockPluginService.isNetworkException(exception)).thenReturn(true); + when(mockPluginService.isNetworkException(eq(exception), any())).thenReturn(true); when(mockPluginService.getAllHosts()).thenReturn(newTopology);