Skip to content

Commit

Permalink
Reduce bootstrap time in the situation with large properties (#3800)
Browse files Browse the repository at this point in the history
  • Loading branch information
Shawyeok committed Jul 11, 2021
1 parent 72a0498 commit f262156
Show file tree
Hide file tree
Showing 8 changed files with 219 additions and 20 deletions.
1 change: 1 addition & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ Apollo 1.9.0
* [set default session store-type](https://github.com/ctripcorp/apollo/pull/3812)
* [speed up the stale issue mark and close phase](https://github.com/ctripcorp/apollo/pull/3808)
* [feature: add the delegating password encoder for apollo-portal simple auth](https://github.com/ctripcorp/apollo/pull/3804)
* [Reduce bootstrap time in the situation with large properties](https://github.com/ctripcorp/apollo/pull/3816)
------------------
All issues and pull requests are [here](https://github.com/ctripcorp/apollo/milestone/6?closed=1)

Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.ctrip.framework.apollo.core.ApolloClientSystemConsts;
import com.ctrip.framework.apollo.core.ConfigConsts;
import com.ctrip.framework.apollo.core.utils.DeferredLogger;
import com.ctrip.framework.apollo.spring.config.CompositeConfigPropertySource;
import com.ctrip.framework.apollo.spring.config.ConfigPropertySourceFactory;
import com.ctrip.framework.apollo.spring.config.PropertySourcesConstants;
import com.ctrip.framework.apollo.spring.util.SpringInjector;
Expand All @@ -34,7 +35,6 @@
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.core.Ordered;
import org.springframework.core.env.CompositePropertySource;
import org.springframework.core.env.ConfigurableEnvironment;

/**
Expand Down Expand Up @@ -113,21 +113,26 @@ public void initialize(ConfigurableApplicationContext context) {
*/
protected void initialize(ConfigurableEnvironment environment) {

if (environment.getPropertySources().contains(PropertySourcesConstants.APOLLO_BOOTSTRAP_PROPERTY_SOURCE_NAME)) {
if (environment.getPropertySources()
.contains(PropertySourcesConstants.APOLLO_BOOTSTRAP_PROPERTY_SOURCE_NAME)) {
//already initialized, replay the logs that were printed before the logging system was initialized
DeferredLogger.replayTo();
return;
}

String namespaces = environment.getProperty(PropertySourcesConstants.APOLLO_BOOTSTRAP_NAMESPACES, ConfigConsts.NAMESPACE_APPLICATION);
String namespaces = environment
.getProperty(PropertySourcesConstants.APOLLO_BOOTSTRAP_NAMESPACES,
ConfigConsts.NAMESPACE_APPLICATION);
logger.debug("Apollo bootstrap namespaces: {}", namespaces);
List<String> namespaceList = NAMESPACE_SPLITTER.splitToList(namespaces);

CompositePropertySource composite = new CompositePropertySource(PropertySourcesConstants.APOLLO_BOOTSTRAP_PROPERTY_SOURCE_NAME);
CompositeConfigPropertySource composite = new CompositeConfigPropertySource(
PropertySourcesConstants.APOLLO_BOOTSTRAP_PROPERTY_SOURCE_NAME);
for (String namespace : namespaceList) {
Config config = ConfigService.getConfig(namespace);

composite.addPropertySource(configPropertySourceFactory.getConfigPropertySource(namespace, config));
composite.addPropertySource(
configPropertySourceFactory.getConfigPropertySource(namespace, config));
}

environment.getPropertySources().addFirst(composite);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/*
* Copyright 2021 Apollo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.ctrip.framework.apollo.spring.config;

import com.ctrip.framework.apollo.ConfigChangeListener;
import com.ctrip.framework.apollo.model.ConfigChangeEvent;
import org.springframework.core.env.CompositePropertySource;
import org.springframework.core.env.PropertySource;

/**
* @author Shawyeok ([email protected])
*/
public class CompositeConfigPropertySource extends CompositePropertySource implements
ConfigChangeListener {

private String[] names;

public CompositeConfigPropertySource(String name) {
super(name);
}

@Override
public String[] getPropertyNames() {
String[] propertyNames = this.names;
if (propertyNames == null) {
this.names = propertyNames = super.getPropertyNames();
}
return propertyNames;
}

@Override
public void addPropertySource(PropertySource<?> propertySource) {
super.addPropertySource(propertySource);
if (propertySource instanceof ConfigPropertySource) {
((ConfigPropertySource) propertySource).addChangeListener(this);
}
}

@Override
public void addFirstPropertySource(PropertySource<?> propertySource) {
super.addFirstPropertySource(propertySource);
if (propertySource instanceof ConfigPropertySource) {
((ConfigPropertySource) propertySource).addChangeListener(this);
}
}

@Override
public void onChange(ConfigChangeEvent changeEvent) {
// clear property names cache if any sources has changed
this.names = null;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ public class ConfigPropertySource extends EnumerablePropertySource<Config> {
super(name, source);
}

@Override
public boolean containsProperty(String name) {
return this.source.getProperty(name, null) != null;
}

@Override
public String[] getPropertyNames() {
Set<String> propertyNames = this.source.getPropertyNames();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,18 @@
*/
package com.ctrip.framework.apollo.spring.config;

import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import com.ctrip.framework.apollo.build.ApolloInjector;
import com.ctrip.framework.apollo.spring.property.AutoUpdateConfigChangeListener;
import com.ctrip.framework.apollo.spring.util.SpringInjector;
import com.ctrip.framework.apollo.util.ConfigUtil;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Multimap;

import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;

import com.google.common.collect.Sets;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.springframework.beans.BeansException;
Expand All @@ -37,12 +37,8 @@
import org.springframework.context.EnvironmentAware;
import org.springframework.core.Ordered;
import org.springframework.core.PriorityOrdered;
import org.springframework.core.env.CompositePropertySource;
import org.springframework.core.env.ConfigurableEnvironment;
import org.springframework.core.env.Environment;

import java.util.Collection;
import java.util.Iterator;
import org.springframework.core.env.MutablePropertySources;
import org.springframework.core.env.PropertySource;

Expand Down Expand Up @@ -76,11 +72,13 @@ public void postProcessBeanFactory(ConfigurableListableBeanFactory beanFactory)
}

private void initializePropertySources() {
if (environment.getPropertySources().contains(PropertySourcesConstants.APOLLO_PROPERTY_SOURCE_NAME)) {
if (environment.getPropertySources()
.contains(PropertySourcesConstants.APOLLO_PROPERTY_SOURCE_NAME)) {
//already initialized
return;
}
CompositePropertySource composite = new CompositePropertySource(PropertySourcesConstants.APOLLO_PROPERTY_SOURCE_NAME);
CompositeConfigPropertySource composite = new CompositeConfigPropertySource(
PropertySourcesConstants.APOLLO_PROPERTY_SOURCE_NAME);

//sort by order asc
ImmutableSortedSet<Integer> orders = ImmutableSortedSet.copyOf(NAMESPACE_NAMES.keySet());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
TestApolloConfigChangeListenerBean1 bean = getBean(TestApolloConfigChangeListenerBean1.class, AppConfig3.class);

//PropertySourcesProcessor add listeners to listen config changed of all namespace
assertEquals(4, applicationListeners.size());
assertEquals(5, applicationListeners.size());
assertEquals(1, fxApolloListeners.size());

for (ConfigChangeListener listener : applicationListeners) {
Expand Down Expand Up @@ -302,7 +302,7 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
TestApolloChildConfigChangeListener bean = getBean(TestApolloChildConfigChangeListener.class, AppConfig7.class);

//PropertySourcesProcessor add listeners to listen config changed of all namespace
assertEquals(5, applicationListeners.size());
assertEquals(6, applicationListeners.size());
assertEquals(1, fxApolloListeners.size());

for (ConfigChangeListener listener : applicationListeners) {
Expand Down Expand Up @@ -461,8 +461,10 @@ public void testApolloConfigChangeListenerResolveExpressionSimple() {
// no using
verify(ignoreConfig, never()).addChangeListener(any(ConfigChangeListener.class));

// one invocation for spring value auto update and another for the @ApolloConfigChangeListener annotation
verify(applicationConfig, times(2)).addChangeListener(any(ConfigChangeListener.class));
// one invocation for spring value auto update
// one invocation for the @ApolloConfigChangeListener annotation
// one invocation for CompositeConfigPropertySource clear cache listener
verify(applicationConfig, times(3)).addChangeListener(any(ConfigChangeListener.class));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
TestApolloConfigChangeListenerBean1.class);

//PropertySourcesProcessor add listeners to listen config changed of all namespace
assertEquals(4, applicationListeners.size());
assertEquals(5, applicationListeners.size());
assertEquals(1, fxApolloListeners.size());

for (ConfigChangeListener listener : applicationListeners) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
/*
* Copyright 2021 Apollo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.ctrip.framework.apollo.spring.config;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import com.ctrip.framework.apollo.ConfigChangeListener;
import com.ctrip.framework.apollo.model.ConfigChangeEvent;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import org.assertj.core.util.Arrays;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import org.springframework.core.env.PropertySource;

/**
* @author Shawyeok ([email protected])
*/
@RunWith(MockitoJUnitRunner.class)
public class CompositeConfigPropertySourceTest {

private CompositeConfigPropertySource compositeSource;

@Mock
private ConfigPropertySource configPropertySource;

private List<ConfigChangeListener> listeners;

@Before
public void setUp() throws Exception {
compositeSource = new CompositeConfigPropertySource("testCompositeSource");
listeners = new LinkedList<>();
Mockito.doAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
ConfigChangeListener listener = invocation.getArgumentAt(0, ConfigChangeListener.class);
listeners.add(listener);
return Void.class;
}
}).when(configPropertySource).addChangeListener(any(ConfigChangeListener.class));
compositeSource.addPropertySource(configPropertySource);
}

@Test
public void testGetPropertyNames() {
String[] propertyNames = Arrays.array("propertyName");
String[] anotherPropertyNames = Arrays.array("propertyName", "anotherPropertyName");

when(configPropertySource.getPropertyNames()).thenReturn(propertyNames, anotherPropertyNames);

String[] returnedPropertyNames = compositeSource.getPropertyNames();
assertArrayEquals(propertyNames, returnedPropertyNames);
assertSame(returnedPropertyNames, compositeSource.getPropertyNames());

listeners.get(0).onChange(new ConfigChangeEvent(null, null));

returnedPropertyNames = compositeSource.getPropertyNames();
assertArrayEquals(anotherPropertyNames, returnedPropertyNames);
assertSame(returnedPropertyNames, compositeSource.getPropertyNames());
}

@Test
public void testAddPropertySource() {
verify(configPropertySource, times(1))
.addChangeListener(any(CompositeConfigPropertySource.class));
assertEquals(1, listeners.size());
assertTrue(compositeSource.getPropertySources().contains(configPropertySource));
}

@Test
public void testAddFirstPropertySource() {
ConfigPropertySource anotherSource = mock(ConfigPropertySource.class);
final List<ConfigChangeListener> anotherListenerList = new LinkedList<>();
Mockito.doAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
ConfigChangeListener listener = invocation.getArgumentAt(0, ConfigChangeListener.class);
anotherListenerList.add(listener);
return Void.class;
}
}).when(anotherSource).addChangeListener(any(ConfigChangeListener.class));
compositeSource.addFirstPropertySource(anotherSource);

Collection<PropertySource<?>> propertySources = compositeSource.getPropertySources();
Iterator<PropertySource<?>> it = propertySources.iterator();

assertEquals(2, propertySources.size());
assertEquals(1, anotherListenerList.size());
assertSame(anotherSource, it.next());
assertSame(configPropertySource, it.next());
}
}

0 comments on commit f262156

Please sign in to comment.