From b476bb86564512dcaa888303b89904e93326c978 Mon Sep 17 00:00:00 2001 From: kai <64013698+jackie-coming@users.noreply.github.com> Date: Sat, 28 Dec 2024 16:36:54 +0800 Subject: [PATCH] feat: support incremental configuration synchronization client (#90) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: support incremental configuration synchronization client * code format * add Unknown sync mode * code format * code format * code format * code format * code format * 更新 RemoteConfigRepository.java Co-authored-by: Jason Song * 更新 ApolloConfig.java Co-authored-by: Jason Song * 更新 ApolloConfig.java Co-authored-by: Jason Song * 更新 ConfigurationChange.java Co-authored-by: Jason Song * 更新 ConfigSyncType.java Co-authored-by: Jason Song * 更新 ConfigSyncType.java Co-authored-by: Jason Song * 更新 ConfigSyncType.java Co-authored-by: Jason Song * 更新 ConfigurationChange.java Co-authored-by: Jason Song * 更新 ConfigurationChangeType.java Co-authored-by: Jason Song * 更新 ConfigurationChangeTypeUtils.java Co-authored-by: Jason Song * 更新 ConfigSyncType.java Co-authored-by: Jason Song * code format * code format * add UnknownSync action * Apply suggestions from code review --------- Co-authored-by: Jason Song --- CHANGES.md | 1 + .../internals/RemoteConfigRepository.java | 54 +++++- .../internals/RemoteConfigRepositoryTest.java | 164 ++++++++++++++++++ .../apollo/core/dto/ApolloConfig.java | 21 +++ .../apollo/core/dto/ConfigurationChange.java | 65 +++++++ .../apollo/core/enums/ConfigSyncType.java | 59 +++++++ .../core/enums/ConfigurationChangeType.java | 37 ++++ .../enums/ConfigurationChangeTypeUtils.java | 50 ++++++ 8 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ConfigurationChange.java create mode 100644 apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigSyncType.java create mode 100644 apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigurationChangeType.java create mode 100644 apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigurationChangeTypeUtils.java diff --git a/CHANGES.md b/CHANGES.md index 7705f8ea..c2f6ac72 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,7 @@ Apollo Java 2.4.0 * [Feature support pulling configuration information from multiple AppIds](https://github.com/apolloconfig/apollo-java/pull/70) * [Fix monitor arg cause npe](https://github.com/apolloconfig/apollo-java/pull/86) * [Fix the concurrent issue in SpringValueRegistry.scanAndClean](https://github.com/apolloconfig/apollo-java/pull/95) +* [Feature support incremental configuration synchronization client](https://github.com/apolloconfig/apollo-java/pull/90) ------------------ All issues and pull requests are [here](https://github.com/apolloconfig/apollo-java/milestone/4?closed=1) diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/RemoteConfigRepository.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/RemoteConfigRepository.java index e779c80b..3f6f794c 100644 --- a/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/RemoteConfigRepository.java +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/RemoteConfigRepository.java @@ -23,7 +23,10 @@ import com.ctrip.framework.apollo.core.ConfigConsts; import com.ctrip.framework.apollo.core.dto.ApolloConfig; import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; +import com.ctrip.framework.apollo.core.dto.ConfigurationChange; import com.ctrip.framework.apollo.core.dto.ServiceDTO; +import com.ctrip.framework.apollo.core.enums.ConfigSyncType; +import com.ctrip.framework.apollo.core.enums.ConfigurationChangeType; import com.ctrip.framework.apollo.core.schedule.ExponentialSchedulePolicy; import com.ctrip.framework.apollo.core.schedule.SchedulePolicy; import com.ctrip.framework.apollo.core.signature.Signature; @@ -49,6 +52,7 @@ import com.google.common.util.concurrent.RateLimiter; import com.google.gson.Gson; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -255,6 +259,24 @@ private ApolloConfig loadApolloConfig() { ApolloConfig result = response.getBody(); + if (result != null) { + ConfigSyncType configSyncType = ConfigSyncType.fromString(result.getConfigSyncType()); + + if (configSyncType == ConfigSyncType.INCREMENTAL_SYNC) { + ApolloConfig previousConfig = m_configCache.get(); + Map previousConfigurations = + (previousConfig != null) ? previousConfig.getConfigurations() : null; + result.setConfigurations( + mergeConfigurations(previousConfigurations, result.getConfigurationChanges())); + } else if (configSyncType == ConfigSyncType.UNKNOWN) { + String message = String.format( + "Invalid config sync type - %s", + result.getConfigSyncType()); + throw new ApolloConfigException(message, exception); + } + + } + logger.debug("Loaded config for {}: {}", m_namespace, result); return result; @@ -269,7 +291,7 @@ private ApolloConfig loadApolloConfig() { statusCodeException = new ApolloConfigStatusCodeException(ex.getStatusCode(), message); Tracer.logEvent(APOLLO_CLIENT_NAMESPACE_NOT_FOUND,m_namespace); - + } Tracer.logEvent(APOLLO_CONFIG_EXCEPTION, ExceptionUtil.getDetailMessage(statusCodeException)); transaction.setStatus(statusCodeException); @@ -363,4 +385,34 @@ private List getConfigServices() { return services; } + + Map mergeConfigurations(Map previousConfigurations, + List configurationChanges) { + Map newConfigurations = new HashMap<>(); + + if (previousConfigurations != null) { + newConfigurations = Maps.newHashMap(previousConfigurations); + } + + if (configurationChanges == null) { + return newConfigurations; + } + + for (ConfigurationChange change : configurationChanges) { + switch (ConfigurationChangeType.fromString(change.getConfigurationChangeType())) { + case ADDED: + case MODIFIED: + newConfigurations.put(change.getKey(), change.getNewValue()); + break; + case DELETED: + newConfigurations.remove(change.getKey()); + break; + default: + //do nothing + break; + } + } + + return newConfigurations; + } } diff --git a/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/RemoteConfigRepositoryTest.java b/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/RemoteConfigRepositoryTest.java index 0c64cf5f..dac8050a 100644 --- a/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/RemoteConfigRepositoryTest.java +++ b/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/RemoteConfigRepositoryTest.java @@ -34,7 +34,9 @@ import com.ctrip.framework.apollo.core.dto.ApolloConfig; import com.ctrip.framework.apollo.core.dto.ApolloConfigNotification; import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; +import com.ctrip.framework.apollo.core.dto.ConfigurationChange; import com.ctrip.framework.apollo.core.dto.ServiceDTO; +import com.ctrip.framework.apollo.core.enums.ConfigSyncType; import com.ctrip.framework.apollo.core.signature.Signature; import com.ctrip.framework.apollo.enums.ConfigSourceType; import com.ctrip.framework.apollo.exceptions.ApolloConfigException; @@ -53,6 +55,7 @@ import com.google.common.util.concurrent.SettableFuture; import com.google.gson.Gson; import java.lang.reflect.Type; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Properties; @@ -154,6 +157,141 @@ public void testLoadConfig() throws Exception { assertEquals(ConfigSourceType.REMOTE, remoteConfigRepository.getSourceType()); } + @Test + public void testLoadConfigWithIncrementalSync() throws Exception { + + String someKey = "someKey"; + String someValue = "someValue"; + String someKey1 = "someKey1"; + String someValue1 = "someKey1"; + Map configurations = Maps.newHashMap(); + configurations.put(someKey, someValue); + configurations.put(someKey1, someValue1); + ApolloConfig someApolloConfig = assembleApolloConfig(configurations); + + when(someResponse.getStatusCode()).thenReturn(200); + when(someResponse.getBody()).thenReturn(someApolloConfig); + + RemoteConfigRepository remoteConfigRepository = new RemoteConfigRepository(someAppId, + someNamespace); + + remoteConfigRepository.sync(); + + List configurationChanges = new ArrayList<>(); + String someNewValue = "someNewValue"; + configurationChanges.add(new ConfigurationChange(someKey, someNewValue, "MODIFIED")); + configurationChanges.add(new ConfigurationChange(someKey1, null, "DELETED")); + String someKey2 = "someKey2"; + String someValue2 = "someValue2"; + configurationChanges.add(new ConfigurationChange(someKey2, someValue2, "ADDED")); + ApolloConfig someApolloConfigWithIncrementalSync = assembleApolloConfigWithIncrementalSync( + configurationChanges); + + when(someResponse.getStatusCode()).thenReturn(200); + when(someResponse.getBody()).thenReturn(someApolloConfigWithIncrementalSync); + + remoteConfigRepository.sync(); + + Properties config = remoteConfigRepository.getConfig(); + + assertEquals(2, config.size()); + assertEquals("someNewValue", config.getProperty("someKey")); + assertEquals("someValue2", config.getProperty("someKey2")); + assertEquals(ConfigSourceType.REMOTE, remoteConfigRepository.getSourceType()); + remoteConfigLongPollService.stopLongPollingRefresh(); + } + + @Test + public void testMergeConfigurations() throws Exception { + String key1 = "key1"; + String value1 = "value1"; + String anotherValue1 = "anotherValue1"; + + String key3 = "key3"; + String value3 = "value3"; + Map previousConfigurations = ImmutableMap.of(key1, value1, key3, value3); + + List configurationChanges = new ArrayList<>(); + configurationChanges.add(new ConfigurationChange(key1, anotherValue1, "MODIFIED")); + String key2 = "key2"; + String value2 = "value2"; + configurationChanges.add(new ConfigurationChange(key2, value2, "ADDED")); + configurationChanges.add(new ConfigurationChange(key3, null, "DELETED")); + + RemoteConfigRepository remoteConfigRepository = new RemoteConfigRepository(someAppId, + someNamespace); + Map result = remoteConfigRepository.mergeConfigurations(previousConfigurations, + configurationChanges); + + assertEquals(2, result.size()); + assertEquals(anotherValue1, result.get(key1)); + assertEquals(value2, result.get(key2)); + } + + @Test + public void testMergeConfigurationWithPreviousConfigurationsIsNULL() throws Exception { + String key1 = "key1"; + String value1 = "value1"; + + String key3 = "key3"; + String value3 = "value3"; + + Map previousConfigurations = ImmutableMap.of(key1, value1, key3, value3); + + RemoteConfigRepository remoteConfigRepository = new RemoteConfigRepository(someAppId, + someNamespace); + Map result = remoteConfigRepository.mergeConfigurations(previousConfigurations, + null); + + assertEquals(2, result.size()); + assertEquals(value1, result.get(key1)); + assertEquals(value3, result.get(key3)); + } + + @Test + public void testMergeConfigurationWithChangesIsNULL() throws Exception { + String key1 = "key1"; + String value1 = "value1"; + String anotherValue1 = "anotherValue1"; + + String key3 = "key3"; + String value3 = "value3"; + + List configurationChanges = new ArrayList<>(); + configurationChanges.add(new ConfigurationChange(key1, anotherValue1, "MODIFIED")); + String key2 = "key2"; + String value2 = "value2"; + configurationChanges.add(new ConfigurationChange(key2, value2, "ADDED")); + configurationChanges.add(new ConfigurationChange(key3, null, "DELETED")); + + RemoteConfigRepository remoteConfigRepository = new RemoteConfigRepository(someAppId, + someNamespace); + Map result = remoteConfigRepository.mergeConfigurations(null, + configurationChanges); + + assertEquals(2, result.size()); + assertEquals(anotherValue1, result.get(key1)); + assertEquals(value2, result.get(key2)); + } + + @Test(expected = ApolloConfigException.class) + public void testGetRemoteConfigWithUnknownSync() throws Exception { + + ApolloConfig someApolloConfigWithUnknownSync = assembleApolloConfigWithUnknownSync( + new ArrayList<>()); + + when(someResponse.getStatusCode()).thenReturn(200); + when(someResponse.getBody()).thenReturn(someApolloConfigWithUnknownSync); + + RemoteConfigRepository remoteConfigRepository = new RemoteConfigRepository(someAppId, + someNamespace); + + //must stop the long polling before exception occurred + remoteConfigLongPollService.stopLongPollingRefresh(); + + remoteConfigRepository.getConfig(); + } + @Test public void testLoadConfigWithOrderedProperties() throws Exception { String someKey = "someKey"; @@ -371,6 +509,32 @@ private ApolloConfig assembleApolloConfig(Map configurations) { return apolloConfig; } + private ApolloConfig assembleApolloConfigWithIncrementalSync( + List configurationChanges) { + String someAppId = "appId"; + String someClusterName = "cluster"; + String someReleaseKey = "1"; + ApolloConfig apolloConfig = + new ApolloConfig(someAppId, someClusterName, someNamespace, someReleaseKey); + + apolloConfig.setConfigSyncType(ConfigSyncType.INCREMENTAL_SYNC.getValue()); + apolloConfig.setConfigurationChanges(configurationChanges); + return apolloConfig; + } + + private ApolloConfig assembleApolloConfigWithUnknownSync( + List configurationChanges) { + String someAppId = "appId"; + String someClusterName = "cluster"; + String someReleaseKey = "1"; + ApolloConfig apolloConfig = + new ApolloConfig(someAppId, someClusterName, someNamespace, someReleaseKey); + + apolloConfig.setConfigSyncType(ConfigSyncType.UNKNOWN.getValue()); + apolloConfig.setConfigurationChanges(configurationChanges); + return apolloConfig; + } + public static class MockConfigUtil extends ConfigUtil { @Override diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ApolloConfig.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ApolloConfig.java index 6ae5584f..39d589b9 100644 --- a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ApolloConfig.java +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ApolloConfig.java @@ -16,6 +16,7 @@ */ package com.ctrip.framework.apollo.core.dto; +import java.util.List; import java.util.Map; /** @@ -33,6 +34,10 @@ public class ApolloConfig { private String releaseKey; + private String configSyncType; + + private List configurationChanges; + public ApolloConfig() { } @@ -62,6 +67,14 @@ public String getReleaseKey() { return releaseKey; } + public String getConfigSyncType() { + return configSyncType; + } + + public List getConfigurationChanges() { + return configurationChanges; + } + public Map getConfigurations() { return configurations; } @@ -82,10 +95,18 @@ public void setReleaseKey(String releaseKey) { this.releaseKey = releaseKey; } + public void setConfigSyncType(String configSyncType) { + this.configSyncType = configSyncType; + } + public void setConfigurations(Map configurations) { this.configurations = configurations; } + public void setConfigurationChanges(List configurationChanges) { + this.configurationChanges = configurationChanges; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("ApolloConfig{"); diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ConfigurationChange.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ConfigurationChange.java new file mode 100644 index 00000000..3403df8f --- /dev/null +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ConfigurationChange.java @@ -0,0 +1,65 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.core.dto; + + +/** + * Holds the information for a Configuration change. + * + * @since 2.0.0 + */ +public class ConfigurationChange { + + private final String key; + private final String newValue; + private final String configurationChangeType; + + /** + * Constructor. + * + * @param key the key whose value is changed + * @param newValue the value after change + * @param configurationChangeType the change type + */ + public ConfigurationChange(String key, String newValue, String configurationChangeType) { + this.key = key; + this.newValue = newValue; + this.configurationChangeType = configurationChangeType; + } + + public String getKey() { + return key; + } + + public String getNewValue() { + return newValue; + } + + public String getConfigurationChangeType() { + return configurationChangeType; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ConfigChange{"); + sb.append(" key='").append(key).append('\''); + sb.append(", newValue='").append(newValue).append('\''); + sb.append(", configurationChangeType=").append(configurationChangeType); + sb.append('}'); + return sb.toString(); + } +} diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigSyncType.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigSyncType.java new file mode 100644 index 00000000..07e72aac --- /dev/null +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigSyncType.java @@ -0,0 +1,59 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.core.enums; + +import com.ctrip.framework.apollo.core.utils.StringUtils; + +/** + * This enum represents all the possible Configuration sync types + * + * @since 2.0.0 + */ +public enum ConfigSyncType { + FULL_SYNC("FullSync"), INCREMENTAL_SYNC("IncrementalSync"), UNKNOWN("Unknown"); + + private final String value; + + ConfigSyncType(String value) { + this.value = value; + } + + /** + * Transforms a given string to its matching {@link ConfigSyncType}. + * + * @param value the string that matches + * @return the matching {@link ConfigSyncType} + */ + public static ConfigSyncType fromString(String value) { + if (StringUtils.isEmpty(value)) { + return FULL_SYNC; + } + for (ConfigSyncType type : values()) { + if (type.value.equals(value)) { + return type; + } + } + return UNKNOWN; + } + + /** + * @return The string representation of the given {@link ConfigSyncType} + */ + public String getValue() { + return value; + } +} diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigurationChangeType.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigurationChangeType.java new file mode 100644 index 00000000..6a483102 --- /dev/null +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigurationChangeType.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.core.enums; + + +import com.google.common.base.Preconditions; + +/** + * This enum represents all the possible Configuration Change types + * + * @since 2.0.0 + */ +public enum ConfigurationChangeType { + ADDED, MODIFIED, DELETED, UNKNOWN; + + public static ConfigurationChangeType fromString(String changeType) { + ConfigurationChangeType configurationChangeType = ConfigurationChangeTypeUtils.transformChangeType( + changeType); + Preconditions.checkArgument(configurationChangeType != UNKNOWN, + String.format("ConfigurationChangeType %s is invalid", changeType)); + return configurationChangeType; + } +} diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigurationChangeTypeUtils.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigurationChangeTypeUtils.java new file mode 100644 index 00000000..c996ef0c --- /dev/null +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigurationChangeTypeUtils.java @@ -0,0 +1,50 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.core.enums; + +import com.ctrip.framework.apollo.core.utils.StringUtils; + +/** + * A utility class for the {@link ConfigurationChangeType} enum. + *

+ * The class provides simple functionalities that extend the capabilities of + * {@link ConfigurationChangeType} + * + * @since 2.0.0 + */ +public final class ConfigurationChangeTypeUtils { + + /** + * Transforms a given String to its matching {@link ConfigurationChangeType} + * + * @param changeType the String to convert + * @return the matching {@link ConfigurationChangeType} for the given String + */ + public static ConfigurationChangeType transformChangeType(String changeType) { + if (StringUtils.isBlank(changeType)) { + return ConfigurationChangeType.UNKNOWN; + } + + String cleanedChangeType = changeType.trim().toUpperCase(); + + try { + return ConfigurationChangeType.valueOf(cleanedChangeType); + } catch (IllegalArgumentException e) { + return ConfigurationChangeType.UNKNOWN; + } + } +}