From f654cb37c454c42033591ed16a61b89da9e62057 Mon Sep 17 00:00:00 2001 From: jason <2353220944@qq.com> Date: Tue, 26 Nov 2024 15:57:23 +0800 Subject: [PATCH] feat: support incremental configuration synchronization client --- .../internals/RemoteConfigRepository.java | 55 +++++++- .../internals/RemoteConfigRepositoryTest.java | 129 +++++++++++++++++- .../apollo/core/dto/ApolloConfig.java | 24 ++++ .../apollo/core/dto/ConfigurationChange.java | 64 +++++++++ .../apollo/core/enums/ConfigSyncType.java | 66 +++++++++ .../core/enums/ConfigurationChangeType.java | 25 ++++ pom.xml | 2 +- 7 files changed, 356 insertions(+), 9 deletions(-) create mode 100644 apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ConfigurationChange.java create mode 100644 apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigSyncType.java create mode 100644 apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigurationChangeType.java diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/RemoteConfigRepository.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/RemoteConfigRepository.java index 6008f63e..5b638ed0 100644 --- a/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/RemoteConfigRepository.java +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/RemoteConfigRepository.java @@ -21,7 +21,9 @@ import com.ctrip.framework.apollo.core.ConfigConsts; import com.ctrip.framework.apollo.core.dto.ApolloConfig; import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; +import com.ctrip.framework.apollo.core.dto.ConfigurationChange; import com.ctrip.framework.apollo.core.dto.ServiceDTO; +import com.ctrip.framework.apollo.core.enums.ConfigSyncType; import com.ctrip.framework.apollo.core.schedule.ExponentialSchedulePolicy; import com.ctrip.framework.apollo.core.schedule.SchedulePolicy; import com.ctrip.framework.apollo.core.signature.Signature; @@ -46,10 +48,8 @@ import com.google.common.net.UrlEscapers; import com.google.common.util.concurrent.RateLimiter; import com.google.gson.Gson; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Properties; + +import java.util.*; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -247,6 +247,24 @@ private ApolloConfig loadApolloConfig() { } ApolloConfig result = response.getBody(); + if(result!=null){ + ConfigSyncType configSyncType=ConfigSyncType.fromString(result.getConfigSyncType()); + + if(configSyncType!=null&&configSyncType.equals(ConfigSyncType.INCREMENTALSYNC)){ + + Map previousConfigurations=null; + + ApolloConfig previousConfig = m_configCache.get(); + + if(previousConfig!=null){ + previousConfigurations=previousConfig.getConfigurations(); + } + + result.setConfigurations(mergeConfigurations(previousConfigurations,result.getConfigurationChanges())); + + } + + } logger.debug("Loaded config for {}: {}", m_namespace, result); @@ -354,4 +372,33 @@ private List getConfigServices() { return services; } + + public Map mergeConfigurations(Map previousConfigurations,List configurationChanges) { + Map newConfigurations = new HashMap<>(); + + if(previousConfigurations!=null){ + newConfigurations=Maps.newHashMap(previousConfigurations); + } + + if (configurationChanges == null) { + return newConfigurations; + } + + for (ConfigurationChange change : configurationChanges) { + switch (change.getConfigurationChangeType()) { + case ADDED: + case MODIFIED: + newConfigurations.put(change.getKey(), change.getNewValue()); + break; + case DELETED: + newConfigurations.remove(change.getKey()); + break; + default: + //do nothing + break; + } + } + + return newConfigurations; + } } diff --git a/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/RemoteConfigRepositoryTest.java b/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/RemoteConfigRepositoryTest.java index 812ec962..f16f55e4 100644 --- a/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/RemoteConfigRepositoryTest.java +++ b/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/RemoteConfigRepositoryTest.java @@ -31,10 +31,9 @@ import static org.mockito.Mockito.when; import com.ctrip.framework.apollo.build.MockInjector; -import com.ctrip.framework.apollo.core.dto.ApolloConfig; -import com.ctrip.framework.apollo.core.dto.ApolloConfigNotification; -import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; -import com.ctrip.framework.apollo.core.dto.ServiceDTO; +import com.ctrip.framework.apollo.core.dto.*; +import com.ctrip.framework.apollo.core.enums.ConfigSyncType; +import com.ctrip.framework.apollo.core.enums.ConfigurationChangeType; import com.ctrip.framework.apollo.core.signature.Signature; import com.ctrip.framework.apollo.enums.ConfigSourceType; import com.ctrip.framework.apollo.exceptions.ApolloConfigException; @@ -53,6 +52,7 @@ import com.google.common.util.concurrent.SettableFuture; import com.google.gson.Gson; import java.lang.reflect.Type; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Properties; @@ -153,6 +153,116 @@ public void testLoadConfig() throws Exception { remoteConfigLongPollService.stopLongPollingRefresh(); } + @Test + public void testLoadConfigWithIncrementalSync() throws Exception { + + String someKey = "someKey"; + String someValue = "someValue"; + String someKey1 = "someKey1"; + String someValue1 = "someKey1"; + Map configurations = Maps.newHashMap(); + configurations.put(someKey, someValue); + configurations.put(someKey1, someValue1); + ApolloConfig someApolloConfig = assembleApolloConfig(configurations); + + when(someResponse.getStatusCode()).thenReturn(200); + when(someResponse.getBody()).thenReturn(someApolloConfig); + + RemoteConfigRepository remoteConfigRepository = new RemoteConfigRepository(someNamespace); + + remoteConfigRepository.sync(); + + + List configurationChanges=new ArrayList<>(); + String someNewValue = "someNewValue"; + configurationChanges.add(new ConfigurationChange(someKey, someNewValue, ConfigurationChangeType.MODIFIED)); + configurationChanges.add(new ConfigurationChange(someKey1, null, ConfigurationChangeType.DELETED)); + String someKey2 = "someKey2"; + String someValue2 = "someValue2"; + configurationChanges.add(new ConfigurationChange(someKey2, someValue2, ConfigurationChangeType.ADDED)); + ApolloConfig someApolloConfigWithIncrementalSync = assembleApolloConfigWithIncrementalSync(configurationChanges); + + when(someResponse.getStatusCode()).thenReturn(200); + when(someResponse.getBody()).thenReturn(someApolloConfigWithIncrementalSync); + + remoteConfigRepository.sync(); + + Properties config = remoteConfigRepository.getConfig(); + + assertEquals(2, config.size()); + assertEquals("someNewValue", config.getProperty("someKey")); + assertEquals("someValue2", config.getProperty("someKey2")); + assertEquals(ConfigSourceType.REMOTE, remoteConfigRepository.getSourceType()); + remoteConfigLongPollService.stopLongPollingRefresh(); + } + + @Test + public void testMergeConfigurations() throws Exception { + String key1 = "key1"; + String value1 = "value1"; + String anotherValue1 = "anotherValue1"; + + String key3 = "key3"; + String value3 = "value3"; + Map previousConfigurations = ImmutableMap.of(key1, value1,key3,value3); + + List configurationChanges=new ArrayList<>(); + configurationChanges.add(new ConfigurationChange(key1, anotherValue1, ConfigurationChangeType.MODIFIED)); + String key2 = "key2"; + String value2 = "value2"; + configurationChanges.add(new ConfigurationChange(key2, value2, ConfigurationChangeType.ADDED)); + configurationChanges.add(new ConfigurationChange(key3, null, ConfigurationChangeType.DELETED)); + + RemoteConfigRepository remoteConfigRepository = new RemoteConfigRepository(someNamespace); + Map result=remoteConfigRepository.mergeConfigurations(previousConfigurations, configurationChanges); + + assertEquals(2, result.size()); + assertEquals(anotherValue1, result.get(key1)); + assertEquals(value2, result.get(key2)); + } + @Test + public void testMergeConfigurationWithPreviousConfigurationsIsNULL() throws Exception { + String key1 = "key1"; + String value1 = "value1"; + + String key3 = "key3"; + String value3 = "value3"; + + Map previousConfigurations = ImmutableMap.of(key1, value1,key3,value3); + + + RemoteConfigRepository remoteConfigRepository = new RemoteConfigRepository(someNamespace); + Map result=remoteConfigRepository.mergeConfigurations(previousConfigurations, null); + + assertEquals(2, result.size()); + assertEquals(value1, result.get(key1)); + assertEquals(value3, result.get(key3)); + } + + @Test + public void testMergeConfigurationWithChangesIsNULL() throws Exception { + String key1 = "key1"; + String value1 = "value1"; + String anotherValue1 = "anotherValue1"; + + String key3 = "key3"; + String value3 = "value3"; + + List configurationChanges=new ArrayList<>(); + configurationChanges.add(new ConfigurationChange(key1, anotherValue1, ConfigurationChangeType.MODIFIED)); + String key2 = "key2"; + String value2 = "value2"; + configurationChanges.add(new ConfigurationChange(key2, value2, ConfigurationChangeType.ADDED)); + configurationChanges.add(new ConfigurationChange(key3, null, ConfigurationChangeType.DELETED)); + + RemoteConfigRepository remoteConfigRepository = new RemoteConfigRepository(someNamespace); + Map result=remoteConfigRepository.mergeConfigurations(null, configurationChanges); + + assertEquals(2, result.size()); + assertEquals(anotherValue1, result.get(key1)); + assertEquals(value2, result.get(key2)); + } + @Test public void testLoadConfigWithOrderedProperties() throws Exception { String someKey = "someKey"; @@ -374,6 +484,17 @@ private ApolloConfig assembleApolloConfig(Map configurations) { return apolloConfig; } + private ApolloConfig assembleApolloConfigWithIncrementalSync(List configurationChanges) { + String someAppId = "appId"; + String someClusterName = "cluster"; + String someReleaseKey = "1"; + ApolloConfig apolloConfig = + new ApolloConfig(someAppId, someClusterName, someNamespace, someReleaseKey); + + apolloConfig.setConfigSyncType(ConfigSyncType.INCREMENTALSYNC.getValue()); + apolloConfig.setConfigurationChanges(configurationChanges); + return apolloConfig; + } public static class MockConfigUtil extends ConfigUtil { diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ApolloConfig.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ApolloConfig.java index 6ae5584f..b1d296be 100644 --- a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ApolloConfig.java +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ApolloConfig.java @@ -16,6 +16,8 @@ */ package com.ctrip.framework.apollo.core.dto; + +import java.util.List; import java.util.Map; /** @@ -33,6 +35,10 @@ public class ApolloConfig { private String releaseKey; + private String configSyncType; + + private List configurationChanges; + public ApolloConfig() { } @@ -62,6 +68,14 @@ public String getReleaseKey() { return releaseKey; } + public String getConfigSyncType() { + return configSyncType; + } + + public List getConfigurationChanges() { + return configurationChanges; + } + public Map getConfigurations() { return configurations; } @@ -82,10 +96,20 @@ public void setReleaseKey(String releaseKey) { this.releaseKey = releaseKey; } + public void setConfigSyncType(String configSyncType) { + this.configSyncType = configSyncType; + } + + public void setConfigurations(Map configurations) { this.configurations = configurations; } + public void setConfigurationChanges(List configurationChanges) { + this.configurationChanges = configurationChanges; + } + + @Override public String toString() { final StringBuilder sb = new StringBuilder("ApolloConfig{"); diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ConfigurationChange.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ConfigurationChange.java new file mode 100644 index 00000000..506ea5a1 --- /dev/null +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/dto/ConfigurationChange.java @@ -0,0 +1,64 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.core.dto; + + +import com.ctrip.framework.apollo.core.enums.ConfigurationChangeType; + +/** + * Holds the information for a Configuration change. + * @author jason + */ +public class ConfigurationChange { + private final String key; + private final String newValue; + private final ConfigurationChangeType configurationChangeType; + + /** + * Constructor. + * @param key the key whose value is changed + * @param newValue the value after change + * @param configurationChangeType the change type + */ + public ConfigurationChange(String key, String newValue, ConfigurationChangeType configurationChangeType) { + this.key = key; + this.newValue = newValue; + this.configurationChangeType = configurationChangeType; + } + + public String getKey() { + return key; + } + public String getNewValue() { + return newValue; + } + + public ConfigurationChangeType getConfigurationChangeType() { + return configurationChangeType; + } + + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ConfigChange{"); + sb.append(" key='").append(key).append('\''); + sb.append(", newValue='").append(newValue).append('\''); + sb.append(", configurationChangeType=").append(configurationChangeType); + sb.append('}'); + return sb.toString(); + } +} diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigSyncType.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigSyncType.java new file mode 100644 index 00000000..48881558 --- /dev/null +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigSyncType.java @@ -0,0 +1,66 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.core.enums; + +import com.ctrip.framework.apollo.core.utils.StringUtils; + +import java.util.stream.Stream; + +/** + * This enum represents all the possible Configuration sync from apollo-config + * + * @author jason + */ +public enum ConfigSyncType { + FULLSYNC("FullSync"), INCREMENTALSYNC("IncrementalSync"); + + private final String value; + + ConfigSyncType(String value) { + this.value = value; + } + + + + /** + * Transforms a given string to its matching {@link ConfigSyncType}. + * + * @param value the string that matches + * @return the matching {@link ConfigSyncType} + * @throws IllegalArgumentException in case the value is empty or there is no + * matching {@link ConfigSyncType} + */ + public static ConfigSyncType fromString(String value) { + if (StringUtils.isEmpty(value)) { + return FULLSYNC; + } + for (ConfigSyncType type : values()) { + if (type.value.equals(value)) { + return type; + } + } + throw new IllegalArgumentException("Invalid ConfigSyncType: " + value); + + } + + /** + * @return The string representation of the given {@link ConfigSyncType} + */ + public String getValue() { + return value; + } +} diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigurationChangeType.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigurationChangeType.java new file mode 100644 index 00000000..7edb47af --- /dev/null +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/ConfigurationChangeType.java @@ -0,0 +1,25 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.core.enums; + + +/** + * @author jason + */ +public enum ConfigurationChangeType { + ADDED, MODIFIED, DELETED +} diff --git a/pom.xml b/pom.xml index 15949fc7..64e15f9e 100644 --- a/pom.xml +++ b/pom.xml @@ -60,7 +60,7 @@ - 2.2.0-SNAPSHOT + 2.5.0-SNAPSHOT 1.8 UTF-8 2.6.8