Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Reduce bootstrap time in the situation with large properties (apolloc…
Browse files Browse the repository at this point in the history
Shawyeok committed Jul 11, 2021
1 parent 72a0498 commit f262156
Showing 8 changed files with 219 additions and 20 deletions.
1 change: 1 addition & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
@@ -56,6 +56,7 @@ Apollo 1.9.0
* [set default session store-type](https://github.com/ctripcorp/apollo/pull/3812)
* [speed up the stale issue mark and close phase](https://github.com/ctripcorp/apollo/pull/3808)
* [feature: add the delegating password encoder for apollo-portal simple auth](https://github.com/ctripcorp/apollo/pull/3804)
* [Reduce bootstrap time in the situation with large properties](https://github.com/ctripcorp/apollo/pull/3816)
------------------
All issues and pull requests are [here](https://github.com/ctripcorp/apollo/milestone/6?closed=1)

Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@
import com.ctrip.framework.apollo.core.ApolloClientSystemConsts;
import com.ctrip.framework.apollo.core.ConfigConsts;
import com.ctrip.framework.apollo.core.utils.DeferredLogger;
import com.ctrip.framework.apollo.spring.config.CompositeConfigPropertySource;
import com.ctrip.framework.apollo.spring.config.ConfigPropertySourceFactory;
import com.ctrip.framework.apollo.spring.config.PropertySourcesConstants;
import com.ctrip.framework.apollo.spring.util.SpringInjector;
@@ -34,7 +35,6 @@
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.core.Ordered;
import org.springframework.core.env.CompositePropertySource;
import org.springframework.core.env.ConfigurableEnvironment;

/**
@@ -113,21 +113,26 @@ public void initialize(ConfigurableApplicationContext context) {
*/
protected void initialize(ConfigurableEnvironment environment) {

if (environment.getPropertySources().contains(PropertySourcesConstants.APOLLO_BOOTSTRAP_PROPERTY_SOURCE_NAME)) {
if (environment.getPropertySources()
.contains(PropertySourcesConstants.APOLLO_BOOTSTRAP_PROPERTY_SOURCE_NAME)) {
//already initialized, replay the logs that were printed before the logging system was initialized
DeferredLogger.replayTo();
return;
}

String namespaces = environment.getProperty(PropertySourcesConstants.APOLLO_BOOTSTRAP_NAMESPACES, ConfigConsts.NAMESPACE_APPLICATION);
String namespaces = environment
.getProperty(PropertySourcesConstants.APOLLO_BOOTSTRAP_NAMESPACES,
ConfigConsts.NAMESPACE_APPLICATION);
logger.debug("Apollo bootstrap namespaces: {}", namespaces);
List<String> namespaceList = NAMESPACE_SPLITTER.splitToList(namespaces);

CompositePropertySource composite = new CompositePropertySource(PropertySourcesConstants.APOLLO_BOOTSTRAP_PROPERTY_SOURCE_NAME);
CompositeConfigPropertySource composite = new CompositeConfigPropertySource(
PropertySourcesConstants.APOLLO_BOOTSTRAP_PROPERTY_SOURCE_NAME);
for (String namespace : namespaceList) {
Config config = ConfigService.getConfig(namespace);

composite.addPropertySource(configPropertySourceFactory.getConfigPropertySource(namespace, config));
composite.addPropertySource(
configPropertySourceFactory.getConfigPropertySource(namespace, config));
}

environment.getPropertySources().addFirst(composite);
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/*
* Copyright 2021 Apollo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.ctrip.framework.apollo.spring.config;

import com.ctrip.framework.apollo.ConfigChangeListener;
import com.ctrip.framework.apollo.model.ConfigChangeEvent;
import org.springframework.core.env.CompositePropertySource;
import org.springframework.core.env.PropertySource;

/**
* @author Shawyeok (shawyeok@outlook.com)
*/
public class CompositeConfigPropertySource extends CompositePropertySource implements
ConfigChangeListener {

private String[] names;

public CompositeConfigPropertySource(String name) {
super(name);
}

@Override
public String[] getPropertyNames() {
String[] propertyNames = this.names;
if (propertyNames == null) {
this.names = propertyNames = super.getPropertyNames();
}
return propertyNames;
}

@Override
public void addPropertySource(PropertySource<?> propertySource) {
super.addPropertySource(propertySource);
if (propertySource instanceof ConfigPropertySource) {
((ConfigPropertySource) propertySource).addChangeListener(this);
}
}

@Override
public void addFirstPropertySource(PropertySource<?> propertySource) {
super.addFirstPropertySource(propertySource);
if (propertySource instanceof ConfigPropertySource) {
((ConfigPropertySource) propertySource).addChangeListener(this);
}
}

@Override
public void onChange(ConfigChangeEvent changeEvent) {
// clear property names cache if any sources has changed
this.names = null;
}
}
Original file line number Diff line number Diff line change
@@ -35,6 +35,11 @@ public class ConfigPropertySource extends EnumerablePropertySource<Config> {
super(name, source);
}

@Override
public boolean containsProperty(String name) {
return this.source.getProperty(name, null) != null;
}

@Override
public String[] getPropertyNames() {
Set<String> propertyNames = this.source.getPropertyNames();
Original file line number Diff line number Diff line change
@@ -16,18 +16,18 @@
*/
package com.ctrip.framework.apollo.spring.config;

import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import com.ctrip.framework.apollo.build.ApolloInjector;
import com.ctrip.framework.apollo.spring.property.AutoUpdateConfigChangeListener;
import com.ctrip.framework.apollo.spring.util.SpringInjector;
import com.ctrip.framework.apollo.util.ConfigUtil;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Multimap;

import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;

import com.google.common.collect.Sets;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.springframework.beans.BeansException;
@@ -37,12 +37,8 @@
import org.springframework.context.EnvironmentAware;
import org.springframework.core.Ordered;
import org.springframework.core.PriorityOrdered;
import org.springframework.core.env.CompositePropertySource;
import org.springframework.core.env.ConfigurableEnvironment;
import org.springframework.core.env.Environment;

import java.util.Collection;
import java.util.Iterator;
import org.springframework.core.env.MutablePropertySources;
import org.springframework.core.env.PropertySource;

@@ -76,11 +72,13 @@ public void postProcessBeanFactory(ConfigurableListableBeanFactory beanFactory)
}

private void initializePropertySources() {
if (environment.getPropertySources().contains(PropertySourcesConstants.APOLLO_PROPERTY_SOURCE_NAME)) {
if (environment.getPropertySources()
.contains(PropertySourcesConstants.APOLLO_PROPERTY_SOURCE_NAME)) {
//already initialized
return;
}
CompositePropertySource composite = new CompositePropertySource(PropertySourcesConstants.APOLLO_PROPERTY_SOURCE_NAME);
CompositeConfigPropertySource composite = new CompositeConfigPropertySource(
PropertySourcesConstants.APOLLO_PROPERTY_SOURCE_NAME);

//sort by order asc
ImmutableSortedSet<Integer> orders = ImmutableSortedSet.copyOf(NAMESPACE_NAMES.keySet());
Original file line number Diff line number Diff line change
@@ -229,7 +229,7 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
TestApolloConfigChangeListenerBean1 bean = getBean(TestApolloConfigChangeListenerBean1.class, AppConfig3.class);

//PropertySourcesProcessor add listeners to listen config changed of all namespace
assertEquals(4, applicationListeners.size());
assertEquals(5, applicationListeners.size());
assertEquals(1, fxApolloListeners.size());

for (ConfigChangeListener listener : applicationListeners) {
@@ -302,7 +302,7 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
TestApolloChildConfigChangeListener bean = getBean(TestApolloChildConfigChangeListener.class, AppConfig7.class);

//PropertySourcesProcessor add listeners to listen config changed of all namespace
assertEquals(5, applicationListeners.size());
assertEquals(6, applicationListeners.size());
assertEquals(1, fxApolloListeners.size());

for (ConfigChangeListener listener : applicationListeners) {
@@ -461,8 +461,10 @@ public void testApolloConfigChangeListenerResolveExpressionSimple() {
// no using
verify(ignoreConfig, never()).addChangeListener(any(ConfigChangeListener.class));

// one invocation for spring value auto update and another for the @ApolloConfigChangeListener annotation
verify(applicationConfig, times(2)).addChangeListener(any(ConfigChangeListener.class));
// one invocation for spring value auto update
// one invocation for the @ApolloConfigChangeListener annotation
// one invocation for CompositeConfigPropertySource clear cache listener
verify(applicationConfig, times(3)).addChangeListener(any(ConfigChangeListener.class));
}

/**
Original file line number Diff line number Diff line change
@@ -111,7 +111,7 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
TestApolloConfigChangeListenerBean1.class);

//PropertySourcesProcessor add listeners to listen config changed of all namespace
assertEquals(4, applicationListeners.size());
assertEquals(5, applicationListeners.size());
assertEquals(1, fxApolloListeners.size());

for (ConfigChangeListener listener : applicationListeners) {
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
/*
* Copyright 2021 Apollo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.ctrip.framework.apollo.spring.config;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import com.ctrip.framework.apollo.ConfigChangeListener;
import com.ctrip.framework.apollo.model.ConfigChangeEvent;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import org.assertj.core.util.Arrays;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import org.springframework.core.env.PropertySource;

/**
* @author Shawyeok (shawyeok@outlook.com)
*/
@RunWith(MockitoJUnitRunner.class)
public class CompositeConfigPropertySourceTest {

private CompositeConfigPropertySource compositeSource;

@Mock
private ConfigPropertySource configPropertySource;

private List<ConfigChangeListener> listeners;

@Before
public void setUp() throws Exception {
compositeSource = new CompositeConfigPropertySource("testCompositeSource");
listeners = new LinkedList<>();
Mockito.doAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
ConfigChangeListener listener = invocation.getArgumentAt(0, ConfigChangeListener.class);
listeners.add(listener);
return Void.class;
}
}).when(configPropertySource).addChangeListener(any(ConfigChangeListener.class));
compositeSource.addPropertySource(configPropertySource);
}

@Test
public void testGetPropertyNames() {
String[] propertyNames = Arrays.array("propertyName");
String[] anotherPropertyNames = Arrays.array("propertyName", "anotherPropertyName");

when(configPropertySource.getPropertyNames()).thenReturn(propertyNames, anotherPropertyNames);

String[] returnedPropertyNames = compositeSource.getPropertyNames();
assertArrayEquals(propertyNames, returnedPropertyNames);
assertSame(returnedPropertyNames, compositeSource.getPropertyNames());

listeners.get(0).onChange(new ConfigChangeEvent(null, null));

returnedPropertyNames = compositeSource.getPropertyNames();
assertArrayEquals(anotherPropertyNames, returnedPropertyNames);
assertSame(returnedPropertyNames, compositeSource.getPropertyNames());
}

@Test
public void testAddPropertySource() {
verify(configPropertySource, times(1))
.addChangeListener(any(CompositeConfigPropertySource.class));
assertEquals(1, listeners.size());
assertTrue(compositeSource.getPropertySources().contains(configPropertySource));
}

@Test
public void testAddFirstPropertySource() {
ConfigPropertySource anotherSource = mock(ConfigPropertySource.class);
final List<ConfigChangeListener> anotherListenerList = new LinkedList<>();
Mockito.doAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
ConfigChangeListener listener = invocation.getArgumentAt(0, ConfigChangeListener.class);
anotherListenerList.add(listener);
return Void.class;
}
}).when(anotherSource).addChangeListener(any(ConfigChangeListener.class));
compositeSource.addFirstPropertySource(anotherSource);

Collection<PropertySource<?>> propertySources = compositeSource.getPropertySources();
Iterator<PropertySource<?>> it = propertySources.iterator();

assertEquals(2, propertySources.size());
assertEquals(1, anotherListenerList.size());
assertSame(anotherSource, it.next());
assertSame(configPropertySource, it.next());
}
}

0 comments on commit f262156

Please sign in to comment.