Skip to content

Commit e8b41b6

Browse files
24kpure24kpure
authored
feat: reduce conflicts when update configmap in k8s #89 (#93)
## What's the purpose of this PR fix #89 ## Which issue(s) this PR fixes: fix #89 ## Brief changelog reduce conflicts when update configmap in k8s Follow this checklist to help us incorporate your contribution quickly and easily: - [x] Read the [Contributing Guide](https://github.com/apolloconfig/apollo/blob/master/CONTRIBUTING.md) before making this pull request. - [x] Write a pull request description that is detailed enough to understand what the pull request does, how, and why. - [x] Write necessary unit tests to verify the code. - [x] Run `mvn clean test` to make sure this pull request doesn't break anything. - [x] Update the [`CHANGES` log](https://github.com/apolloconfig/apollo-java/blob/master/CHANGES.md). <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Introduced a mechanism to reduce conflicts when updating ConfigMap in Kubernetes. - Enhanced access key secret retrieval for applications. - Added pod write permission controls for ConfigMap management. - **Improvements** - Refined configuration utility for better property handling. - Improved Kubernetes resource management logic. - **Testing** - Updated test suite to improve coverage of Kubernetes-related functionality. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent b476bb8 commit e8b41b6

File tree

4 files changed

+112
-12
lines changed

4 files changed

+112
-12
lines changed

‎CHANGES.md

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ Apollo Java 2.4.0
1414
* [Fix monitor arg cause npe](https://github.com/apolloconfig/apollo-java/pull/86)
1515
* [Fix the concurrent issue in SpringValueRegistry.scanAndClean](https://github.com/apolloconfig/apollo-java/pull/95)
1616
* [Feature support incremental configuration synchronization client](https://github.com/apolloconfig/apollo-java/pull/90)
17+
* [Feature reduce conflicts when update configmap in k8](https://github.com/apolloconfig/apollo-java/pull/93)
1718

1819
------------------
1920
All issues and pull requests are [here](https://github.com/apolloconfig/apollo-java/milestone/4?closed=1)

‎apollo-client/src/main/java/com/ctrip/framework/apollo/kubernetes/KubernetesManager.java

+64-1
Original file line numberDiff line numberDiff line change
@@ -18,27 +18,44 @@
1818

1919
import com.ctrip.framework.apollo.core.utils.StringUtils;
2020
import com.ctrip.framework.apollo.exceptions.ApolloConfigException;
21+
import com.google.common.annotations.VisibleForTesting;
22+
import com.google.common.base.Strings;
2123
import io.kubernetes.client.openapi.ApiClient;
2224
import io.kubernetes.client.openapi.ApiException;
2325
import io.kubernetes.client.openapi.apis.CoreV1Api;
2426
import io.kubernetes.client.openapi.models.V1ConfigMap;
2527
import io.kubernetes.client.openapi.models.V1ObjectMeta;
28+
import io.kubernetes.client.openapi.models.V1Pod;
29+
import io.kubernetes.client.openapi.models.V1PodList;
2630
import io.kubernetes.client.util.Config;
2731
import org.slf4j.Logger;
2832
import org.slf4j.LoggerFactory;
2933
import org.springframework.stereotype.Service;
3034

35+
import java.util.Comparator;
3136
import java.util.HashMap;
3237
import java.util.Map;
3338
import java.util.Objects;
3439
import java.util.concurrent.TimeUnit;
3540

41+
/**
42+
* Manages Kubernetes ConfigMap operations.
43+
* Required Kubernetes permissions:
44+
* - pods: [get, list] - For pod selection and write eligibility
45+
* - configmaps: [get, create, update] - For ConfigMap operations
46+
*/
3647
@Service
3748
public class KubernetesManager {
3849
private static final Logger logger = LoggerFactory.getLogger(KubernetesManager.class);
3950

51+
private static final String RUNNING_POD_FIELD_SELECTOR = "status.phase=Running";
52+
53+
private static final int MAX_SEARCH_NUM = 100;
54+
4055
private ApiClient client;
4156
private CoreV1Api coreV1Api;
57+
private int propertyKubernetesMaxWritePods = 3;
58+
private String localPodName = System.getenv("HOSTNAME");
4259

4360
public KubernetesManager() {
4461
try {
@@ -51,8 +68,11 @@ public KubernetesManager() {
5168
}
5269
}
5370

54-
public KubernetesManager(CoreV1Api coreV1Api) {
71+
@VisibleForTesting
72+
public KubernetesManager(CoreV1Api coreV1Api, String localPodName, int propertyKubernetesMaxWritePods) {
5573
this.coreV1Api = coreV1Api;
74+
this.localPodName = localPodName;
75+
this.propertyKubernetesMaxWritePods = propertyKubernetesMaxWritePods;
5676
}
5777

5878
private V1ConfigMap buildConfigMap(String name, String namespace, Map<String, String> data) {
@@ -132,6 +152,10 @@ public boolean updateConfigMap(String k8sNamespace, String name, Map<String, Str
132152
return false;
133153
}
134154

155+
if (!isWritePod(k8sNamespace)) {
156+
return true;
157+
}
158+
135159
int maxRetries = 5;
136160
int retryCount = 0;
137161
long waitTime = 100;
@@ -205,4 +229,43 @@ public boolean checkConfigMapExist(String k8sNamespace, String configMapName) {
205229
return false;
206230
}
207231
}
232+
233+
/**
234+
* check pod whether pod can write configmap
235+
*
236+
* @param k8sNamespace config map namespace
237+
* @return true if this pod can write configmap, false otherwise
238+
*/
239+
private boolean isWritePod(String k8sNamespace) {
240+
try {
241+
if (Strings.isNullOrEmpty(localPodName)) {
242+
return true;
243+
}
244+
V1Pod localPod = coreV1Api.readNamespacedPod(localPodName, k8sNamespace, null);
245+
V1ObjectMeta localMetadata = localPod.getMetadata();
246+
if (localMetadata == null || localMetadata.getLabels() == null) {
247+
return true;
248+
}
249+
String appName = localMetadata.getLabels().get("app");
250+
String labelSelector = "app=" + appName;
251+
252+
V1PodList v1PodList = coreV1Api.listNamespacedPod(k8sNamespace, null, null,
253+
null, RUNNING_POD_FIELD_SELECTOR, labelSelector,
254+
MAX_SEARCH_NUM, null, null
255+
, null, null);
256+
257+
return v1PodList.getItems().stream()
258+
.map(V1Pod::getMetadata)
259+
.filter(Objects::nonNull)
260+
//Make each node selects the same write nodes by sorting
261+
.filter(metadata -> metadata.getCreationTimestamp() != null)
262+
.sorted(Comparator.comparing(V1ObjectMeta::getCreationTimestamp))
263+
.map(V1ObjectMeta::getName)
264+
.limit(propertyKubernetesMaxWritePods)
265+
.anyMatch(localPodName::equals);
266+
} catch (Exception e) {
267+
logger.info("Error determining write pod eligibility:{}", e.getMessage(), e);
268+
return true;
269+
}
270+
}
208271
}

‎apollo-client/src/main/java/com/ctrip/framework/apollo/util/ConfigUtil.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -617,7 +617,7 @@ private void initClientMonitorExceptionQueueSize() {
617617
public int getMonitorExceptionQueueSize() {
618618
return monitorExceptionQueueSize;
619619
}
620-
620+
621621
private boolean getPropertyBoolean(String propertyName, String envName, boolean defaultVal) {
622622
String enablePropertyNamesCache = System.getProperty(propertyName);
623623
if (Strings.isNullOrEmpty(enablePropertyNamesCache)) {

‎apollo-client/src/test/java/com/ctrip/framework/apollo/kubernetes/KubernetesManagerTest.java

+46-10
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,30 @@
2121
import io.kubernetes.client.openapi.apis.CoreV1Api;
2222
import io.kubernetes.client.openapi.models.V1ConfigMap;
2323
import io.kubernetes.client.openapi.models.V1ObjectMeta;
24+
import io.kubernetes.client.openapi.models.V1Pod;
25+
import io.kubernetes.client.openapi.models.V1PodList;
2426
import org.junit.Before;
2527
import org.junit.Test;
28+
import org.mockito.Mockito;
2629

30+
import java.time.OffsetDateTime;
31+
import java.util.Collections;
2732
import java.util.HashMap;
2833
import java.util.Map;
2934

30-
import static org.mockito.Mockito.*;
31-
import static org.junit.Assert.*;
35+
import static org.junit.Assert.assertEquals;
36+
import static org.junit.Assert.assertFalse;
37+
import static org.junit.Assert.assertNull;
38+
import static org.junit.Assert.assertTrue;
39+
import static org.mockito.Mockito.any;
40+
import static org.mockito.Mockito.doReturn;
41+
import static org.mockito.Mockito.doThrow;
42+
import static org.mockito.Mockito.eq;
43+
import static org.mockito.Mockito.isNull;
44+
import static org.mockito.Mockito.mock;
45+
import static org.mockito.Mockito.times;
46+
import static org.mockito.Mockito.verify;
47+
import static org.mockito.Mockito.when;
3248

3349
public class KubernetesManagerTest {
3450

@@ -38,7 +54,7 @@ public class KubernetesManagerTest {
3854
@Before
3955
public void setUp() {
4056
coreV1Api = mock(CoreV1Api.class);
41-
kubernetesManager = new KubernetesManager(coreV1Api);
57+
kubernetesManager = new KubernetesManager(coreV1Api, "localPodName", 3);
4258

4359
MockInjector.setInstance(KubernetesManager.class, kubernetesManager);
4460
MockInjector.setInstance(CoreV1Api.class, coreV1Api);
@@ -58,13 +74,13 @@ public void testCreateConfigMapSuccess() throws Exception {
5874
.metadata(new V1ObjectMeta().name(name).namespace(namespace))
5975
.data(data);
6076

61-
when(coreV1Api.createNamespacedConfigMap(eq(namespace), eq(configMap), isNull(), isNull(), isNull(),isNull())).thenReturn(configMap);
77+
when(coreV1Api.createNamespacedConfigMap(eq(namespace), eq(configMap), isNull(), isNull(), isNull(), isNull())).thenReturn(configMap);
6278

6379
// act
6480
String result = kubernetesManager.createConfigMap(namespace, name, data);
6581

6682
// assert
67-
verify(coreV1Api, times(1)).createNamespacedConfigMap(eq(namespace), any(V1ConfigMap.class), isNull(), isNull(), isNull(),isNull());
83+
verify(coreV1Api, times(1)).createNamespacedConfigMap(eq(namespace), any(V1ConfigMap.class), isNull(), isNull(), isNull(), isNull());
6884
assert name.equals(result);
6985
}
7086

@@ -82,7 +98,7 @@ public void testCreateConfigMapNullData() throws Exception {
8298
String result = kubernetesManager.createConfigMap(namespace, name, data);
8399

84100
// assert
85-
verify(coreV1Api, times(1)).createNamespacedConfigMap(eq(namespace), any(V1ConfigMap.class), isNull(), isNull(), isNull(),isNull());
101+
verify(coreV1Api, times(1)).createNamespacedConfigMap(eq(namespace), any(V1ConfigMap.class), isNull(), isNull(), isNull(), isNull());
86102
assert name.equals(result);
87103
}
88104

@@ -135,20 +151,40 @@ public void testUpdateConfigMapSuccess() throws Exception {
135151
// arrange
136152
String namespace = "default";
137153
String name = "testConfigMap";
138-
Map<String, String> data = new HashMap<>();
139-
data.put("key", "value");
154+
155+
V1Pod pod = new V1Pod()
156+
.metadata(
157+
new V1ObjectMeta()
158+
.name("localPodName")
159+
.creationTimestamp(OffsetDateTime.now())
160+
.labels(Collections.singletonMap("app", "app")));
161+
V1PodList v1PodList = new V1PodList().addItemsItem(new V1Pod().metadata(pod.getMetadata()));
162+
163+
Map<String, String> existData = new HashMap<>();
164+
existData.put("key", "value");
140165
V1ConfigMap configMap = new V1ConfigMap();
141166
configMap.metadata(new V1ObjectMeta().name(name).namespace(namespace));
142-
configMap.data(data);
167+
configMap.data(existData);
143168

169+
when(coreV1Api.readNamespacedPod("localPodName", namespace, null)).thenReturn(pod);
170+
when(coreV1Api.listNamespacedPod(namespace, null, null,
171+
null, null, "app=app",
172+
null, null, null
173+
, null, null)).thenReturn(v1PodList);
144174
when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenReturn(configMap);
145175
when(coreV1Api.replaceNamespacedConfigMap(name, namespace, configMap, null, null, null, null)).thenReturn(configMap);
146176

147177
// act
148-
Boolean success = kubernetesManager.updateConfigMap(namespace, name, data);
178+
HashMap<String, String> updateData = new HashMap<>(existData);
179+
updateData.put("newKey","newValue");
180+
boolean success = kubernetesManager.updateConfigMap(namespace, name, updateData);
149181

150182
// assert
151183
assertTrue(success);
184+
Mockito.verify(coreV1Api, Mockito.times(1)).listNamespacedPod(namespace, null, null,
185+
null, "status.phase=Running", "app=app",
186+
100, null, null
187+
, null, null);
152188
}
153189

154190
/**

0 commit comments

Comments
 (0)