Merge branch 'master' into feature/config_cache_record_stats

youngzil · youngzil · commit fea247005b7a · 2024-10-18T08:35:46.000+08:00
# Conflicts:
#	CHANGES.md
diff --git a/CHANGES.md b/CHANGES.md
@@ -12,6 +12,7 @@ Apollo 2.4.0
 * [Fix link namespace published items show missing some items](https://github.com/apolloconfig/apollo/pull/5240)
 * [Feature: Add limit and whitelist for namespace count per appid+cluster](https://github.com/apolloconfig/apollo/pull/5228)
 * [Feature support the observe status access-key for pre-check and logging only](https://github.com/apolloconfig/apollo/pull/5236)
+* [Feature add limit for items count per namespace](https://github.com/apolloconfig/apollo/pull/5227)
 * [Feature: Add ConfigService cache record stats function](https://github.com/apolloconfig/apollo/pull/5247)
 ------------------
 All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1)
diff --git a/apollo-adminservice/src/main/java/com/ctrip/framework/apollo/adminservice/controller/ItemController.java b/apollo-adminservice/src/main/java/com/ctrip/framework/apollo/adminservice/controller/ItemController.java
@@ -17,6 +17,7 @@
 package com.ctrip.framework.apollo.adminservice.controller;
 
 import com.ctrip.framework.apollo.adminservice.aop.PreAcquireNamespaceLock;
+import com.ctrip.framework.apollo.biz.config.BizConfig;
 import com.ctrip.framework.apollo.biz.entity.Commit;
 import com.ctrip.framework.apollo.biz.entity.Item;
 import com.ctrip.framework.apollo.biz.entity.Namespace;
@@ -58,13 +59,14 @@ public class ItemController {
   private final NamespaceService namespaceService;
   private final CommitService commitService;
   private final ReleaseService releaseService;
+  private final BizConfig bizConfig;
 
-
-  public ItemController(final ItemService itemService, final NamespaceService namespaceService, final CommitService commitService, final ReleaseService releaseService) {
+  public ItemController(final ItemService itemService, final NamespaceService namespaceService, final CommitService commitService, final ReleaseService releaseService, final BizConfig bizConfig) {
     this.itemService = itemService;
     this.namespaceService = namespaceService;
     this.commitService = commitService;
     this.releaseService = releaseService;
+    this.bizConfig = bizConfig;
   }
 
   @PreAcquireNamespaceLock
@@ -78,6 +80,14 @@ public ItemDTO create(@PathVariable("appId") String appId,
     if (managedEntity != null) {
       throw BadRequestException.itemAlreadyExists(entity.getKey());
     }
+
+    if (bizConfig.isItemNumLimitEnabled()) {
+      int itemCount = itemService.findNonEmptyItemCount(entity.getNamespaceId());
+      if (itemCount >= bizConfig.itemNumLimit()) {
+        throw new BadRequestException("The maximum number of items (" + bizConfig.itemNumLimit() + ") for this namespace has been reached. Current item count is " + itemCount + ".");
+      }
+    }
+
     entity = itemService.save(entity);
     dto = BeanUtils.transform(ItemDTO.class, entity);
     commitService.createCommit(appId, clusterName, namespaceName, new ConfigChangeContentBuilder().createItem(entity).build(),
diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java
@@ -41,6 +41,8 @@ public class BizConfig extends RefreshableConfig {
 
   private static final int DEFAULT_MAX_NAMESPACE_NUM = 200;
 
+  private static final int DEFAULT_MAX_ITEM_NUM = 1000;
+
   private static final int DEFAULT_APPNAMESPACE_CACHE_REBUILD_INTERVAL = 60; //60s
   private static final int DEFAULT_GRAY_RELEASE_RULE_SCAN_INTERVAL = 60; //60s
   private static final int DEFAULT_APPNAMESPACE_CACHE_SCAN_INTERVAL = 1; //1s
@@ -117,6 +119,15 @@ public Set<String> namespaceNumLimitWhite() {
     return Sets.newHashSet(getArrayProperty("namespace.num.limit.white", new String[0]));
   }
 
+  public boolean isItemNumLimitEnabled() {
+    return getBooleanProperty("item.num.limit.enabled", false);
+  }
+
+  public int itemNumLimit() {
+    int limit = getIntProperty("item.num.limit", DEFAULT_MAX_ITEM_NUM);
+    return checkInt(limit, 5, Integer.MAX_VALUE, DEFAULT_MAX_ITEM_NUM);
+  }
+
   public Map<Long, Integer> namespaceValueLengthLimitOverride() {
     String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override");
     Map<Long, Integer> namespaceValueLengthOverride = Maps.newHashMap();
diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ItemRepository.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ItemRepository.java
@@ -64,4 +64,7 @@ public interface ItemRepository extends PagingAndSortingRepository<Item, Long> {
   @Query("update Item set IsDeleted = true, DeletedAt = ROUND(UNIX_TIMESTAMP(NOW(4))*1000), DataChange_LastModifiedBy = ?2 where NamespaceId = ?1 and IsDeleted = false")
   int deleteByNamespaceId(long namespaceId, String operator);
 
+  @Query("select count(*) from Item where namespaceId = :namespaceId and key <>''")
+  int countByNamespaceIdAndFilterKeyEmpty(@Param("namespaceId") long namespaceId);
+
 }
diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java
@@ -135,6 +135,10 @@ public List<Item> findItemsModifiedAfterDate(long namespaceId, Date date) {
     return itemRepository.findByNamespaceIdAndDataChangeLastModifiedTimeGreaterThan(namespaceId, date);
   }
 
+  public int findNonEmptyItemCount(long namespaceId) {
+    return itemRepository.countByNamespaceIdAndFilterKeyEmpty(namespaceId);
+  }
+
   public Page<Item> findItemsByKey(String key, Pageable pageable) {
     return itemRepository.findByKey(key, pageable);
   }
diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemSetService.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemSetService.java
@@ -16,6 +16,7 @@
  */
 package com.ctrip.framework.apollo.biz.service;
 
+import com.ctrip.framework.apollo.biz.config.BizConfig;
 import com.ctrip.framework.apollo.biz.entity.Audit;
 import com.ctrip.framework.apollo.biz.entity.Item;
 import com.ctrip.framework.apollo.biz.entity.Namespace;
@@ -25,6 +26,7 @@
 import com.ctrip.framework.apollo.common.exception.BadRequestException;
 import com.ctrip.framework.apollo.common.exception.NotFoundException;
 import com.ctrip.framework.apollo.common.utils.BeanUtils;
+import com.ctrip.framework.apollo.core.utils.StringUtils;
 import java.util.List;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
@@ -38,20 +40,23 @@ public class ItemSetService {
   private final CommitService commitService;
   private final ItemService itemService;
   private final NamespaceService namespaceService;
+  private final BizConfig bizConfig;
 
   public ItemSetService(
       final AuditService auditService,
       final CommitService commitService,
       final ItemService itemService,
-      final NamespaceService namespaceService) {
+      final NamespaceService namespaceService,
+      final BizConfig bizConfig) {
     this.auditService = auditService;
     this.commitService = commitService;
     this.itemService = itemService;
     this.namespaceService = namespaceService;
+    this.bizConfig = bizConfig;
   }
 
   @Transactional
-  public ItemChangeSets updateSet(Namespace namespace, ItemChangeSets changeSets){
+  public ItemChangeSets updateSet(Namespace namespace, ItemChangeSets changeSets) {
     return updateSet(namespace.getAppId(), namespace.getClusterName(), namespace.getNamespaceName(), changeSets);
   }
 
@@ -64,6 +69,16 @@ public ItemChangeSets updateSet(String appId, String clusterName,
       throw NotFoundException.namespaceNotFound(appId, clusterName, namespaceName);
     }
 
+    if (bizConfig.isItemNumLimitEnabled()) {
+      int itemCount = itemService.findNonEmptyItemCount(namespace.getId());
+      int createItemCount = (int) changeSet.getCreateItems().stream().filter(item -> !StringUtils.isEmpty(item.getKey())).count();
+      int deleteItemCount = (int) changeSet.getDeleteItems().stream().filter(item -> !StringUtils.isEmpty(item.getKey())).count();
+      itemCount = itemCount + createItemCount - deleteItemCount;
+      if (itemCount > bizConfig.itemNumLimit()) {
+        throw new BadRequestException("The maximum number of items (" + bizConfig.itemNumLimit() + ") for this namespace has been reached. Current item count is " + itemCount + ".");
+      }
+    }
+
     String operator = changeSet.getDataChangeLastModifiedBy();
     ConfigChangeContentBuilder configChangeContentBuilder = new ConfigChangeContentBuilder();
 
@@ -84,7 +99,7 @@ public ItemChangeSets updateSet(String appId, String clusterName,
 
     if (configChangeContentBuilder.hasContent()) {
       commitService.createCommit(appId, clusterName, namespaceName, configChangeContentBuilder.build(),
-                   changeSet.getDataChangeLastModifiedBy());
+                                 changeSet.getDataChangeLastModifiedBy());
     }
 
     return changeSet;
diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemSetServiceTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemSetServiceTest.java
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2024 Apollo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package com.ctrip.framework.apollo.biz.service;
+
+import static org.mockito.Mockito.when;
+
+import com.ctrip.framework.apollo.biz.AbstractIntegrationTest;
+import com.ctrip.framework.apollo.biz.config.BizConfig;
+import com.ctrip.framework.apollo.biz.entity.Item;
+import com.ctrip.framework.apollo.biz.entity.Namespace;
+import com.ctrip.framework.apollo.common.dto.ItemChangeSets;
+import com.ctrip.framework.apollo.common.dto.ItemDTO;
+import com.ctrip.framework.apollo.common.exception.BadRequestException;
+import org.junit.Assert;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.test.context.jdbc.Sql;
+
+public class ItemSetServiceTest extends AbstractIntegrationTest {
+
+  @MockBean
+  private BizConfig bizConfig;
+
+  @Autowired
+  private ItemService itemService;
+  @Autowired
+  private NamespaceService namespaceService;
+
+  @Autowired
+  private ItemSetService itemSetService;
+
+  @Test
+  @Sql(scripts = "/sql/itemset-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD)
+  @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD)
+  public void testUpdateSetWithoutItemNumLimit() {
+
+    when(bizConfig.itemKeyLengthLimit()).thenReturn(128);
+    when(bizConfig.itemValueLengthLimit()).thenReturn(20000);
+
+    when(bizConfig.isItemNumLimitEnabled()).thenReturn(false);
+    when(bizConfig.itemNumLimit()).thenReturn(5);
+
+    Namespace namespace = namespaceService.findOne(1L);
+
+    ItemChangeSets changeSets = new ItemChangeSets();
+    changeSets.addCreateItem(buildNormalItem(0L, namespace.getId(), "k6", "v6", "test item num limit", 6));
+    changeSets.addCreateItem(buildNormalItem(0L, namespace.getId(), "k7", "v7", "test item num limit", 7));
+
+    try {
+      itemSetService.updateSet(namespace, changeSets);
+    } catch (Exception e) {
+      Assert.fail();
+    }
+
+    int size = itemService.findNonEmptyItemCount(namespace.getId());
+    Assert.assertEquals(7, size);
+
+  }
+
+  @Test
+  @Sql(scripts = "/sql/itemset-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD)
+  @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD)
+  public void testUpdateSetWithItemNumLimit() {
+
+    when(bizConfig.itemKeyLengthLimit()).thenReturn(128);
+    when(bizConfig.itemValueLengthLimit()).thenReturn(20000);
+
+    when(bizConfig.isItemNumLimitEnabled()).thenReturn(true);
+    when(bizConfig.itemNumLimit()).thenReturn(5);
+
+    Namespace namespace = namespaceService.findOne(1L);
+    Item item9901 = itemService.findOne(9901);
+    Item item9902 = itemService.findOne(9902);
+
+    ItemChangeSets changeSets = new ItemChangeSets();
+    changeSets.addUpdateItem(buildNormalItem(item9901.getId(), item9901.getNamespaceId(), item9901.getKey(), item9901.getValue() + " update", item9901.getComment(), item9901.getLineNum()));
+    changeSets.addDeleteItem(buildNormalItem(item9902.getId(), item9902.getNamespaceId(), item9902.getKey(), item9902.getValue() + " update", item9902.getComment(), item9902.getLineNum()));
+    changeSets.addCreateItem(buildNormalItem(0L, item9901.getNamespaceId(), "k6", "v6", "test item num limit", 6));
+    changeSets.addCreateItem(buildNormalItem(0L, item9901.getNamespaceId(), "k7", "v7", "test item num limit", 7));
+
+    try {
+      itemSetService.updateSet(namespace, changeSets);
+      Assert.fail();
+    } catch (Exception e) {
+      Assert.assertTrue(e instanceof BadRequestException);
+    }
+
+    int size = itemService.findNonEmptyItemCount(namespace.getId());
+    Assert.assertEquals(5, size);
+
+  }
+
+  @Test
+  @Sql(scripts = "/sql/itemset-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD)
+  @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD)
+  public void testUpdateSetWithItemNumLimit2() {
+
+    when(bizConfig.itemKeyLengthLimit()).thenReturn(128);
+    when(bizConfig.itemValueLengthLimit()).thenReturn(20000);
+
+    when(bizConfig.isItemNumLimitEnabled()).thenReturn(true);
+    when(bizConfig.itemNumLimit()).thenReturn(5);
+
+    Namespace namespace = namespaceService.findOne(1L);
+    Item item9901 = itemService.findOne(9901);
+    Item item9902 = itemService.findOne(9902);
+
+    ItemChangeSets changeSets = new ItemChangeSets();
+    changeSets.addUpdateItem(buildNormalItem(item9901.getId(), item9901.getNamespaceId(), item9901.getKey(), item9901.getValue() + " update", item9901.getComment(), item9901.getLineNum()));
+    changeSets.addDeleteItem(buildNormalItem(item9902.getId(), item9902.getNamespaceId(), item9902.getKey(), item9902.getValue() + " update", item9902.getComment(), item9902.getLineNum()));
+    changeSets.addCreateItem(buildNormalItem(0L, item9901.getNamespaceId(), "k6", "v6", "test item num limit", 6));
+
+    try {
+      itemSetService.updateSet(namespace, changeSets);
+    } catch (Exception e) {
+      Assert.fail();
+    }
+
+    int size = itemService.findNonEmptyItemCount(namespace.getId());
+    Assert.assertEquals(5, size);
+
+  }
+
+
+  private ItemDTO buildNormalItem(Long id, Long namespaceId, String key, String value, String comment, int lineNum) {
+    ItemDTO item = new ItemDTO(key, value, comment, lineNum);
+    item.setId(id);
+    item.setNamespaceId(namespaceId);
+    return item;
+  }
+
+}
diff --git a/apollo-biz/src/test/resources/sql/itemset-test.sql b/apollo-biz/src/test/resources/sql/itemset-test.sql
@@ -0,0 +1,25 @@
+--
+-- Copyright 2024 Apollo Authors
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+INSERT INTO "Namespace" (`Id`, `AppId`, `ClusterName`, `NamespaceName`, `IsDeleted`, `DataChange_CreatedBy`, `DataChange_LastModifiedBy`)VALUES(1,'testApp', 'default', 'application', 0, 'apollo', 'apollo');
+
+INSERT INTO "Item" (`Id`, `NamespaceId`, "Key", "Type", "Value", `Comment`, `LineNum`)
+    VALUES
+        (9901, 1, 'k1', 0, 'v1', '', 1),
+        (9902, 1, 'k2', 2, 'v2', '', 2),
+        (9903, 1, 'k3', 0, 'v3', '', 3),
+        (9904, 1, 'k4', 0, 'v4', '', 4),
+        (9905, 1, 'k5', 0, 'v5', '', 5);
diff --git a/doc/images/item-num-limit-enabled.png b/doc/images/item-num-limit-enabled.png
diff --git a/doc/images/item-num-limit.png b/doc/images/item-num-limit.png
diff --git a/docs/en/portal/apollo-user-guide.md b/docs/en/portal/apollo-user-guide.md
@@ -531,6 +531,18 @@ Starting from version 2.4.0, apollo-portal provides the function of checking the
    ![item-num-limit](https://cdn.jsdelivr.net/gh/apolloconfig/apollo@master/doc/images/namespace-num-limit-white.png)
 
 
+## 6.5 Limitation on the number of configuration items in a single namespace
+Starting from version 2.4.0, apollo-portal provides the function of limiting the number of configuration items in a single namespace. This function is disabled by default and needs to be enabled by configuring the system `item.num.limit.enabled`. At the same time, the system parameter `item.num.limit` is provided to dynamically configure the upper limit of the number of items in a single Namespace.
+
+**Setting method：**
+1. Log in to the Apollo Configuration Center interface with a super administrator account
+2. Go to the `Administrator Tools - System Parameters - ConfigDB Configuration Management` page and add or modify the `item.num.limit.enabled` configuration item to true/false to enable/disable this function.
+
+   ![item-num-limit-enabled](https://cdn.jsdelivr.net/gh/apolloconfig/apollo@master/doc/images/item-num-limit-enabled.png)
+3. Go to the `Admin Tools - System Parameters - ConfigDB Configuration Management` page and add or modify the `item.num.limit` configuration item to configure the upper limit of the number of items under a single Namespace.
+
+   ![item-num-limit](https://cdn.jsdelivr.net/gh/apolloconfig/apollo@master/doc/images/item-num-limit.png)
+
 
 # VII. Best practices
 
diff --git a/docs/zh/portal/apollo-user-guide.md b/docs/zh/portal/apollo-user-guide.md
@@ -502,6 +502,22 @@ Apollo从1.6.0版本开始增加访问密钥机制，从而只有经过身份验
 
 
 
+## 6.5 单个命名空间下的配置项数量限制
+从2.4.0版本开始，apollo-portal提供了限制单个命名空间下的配置项数量的功能，此功能默认关闭，需要配置系统 `item.num.limit.enabled` 开启，同时提供了系统参数`item.num.limit`来动态配置单个Namespace下的item数量上限值
+
+**设置方法：**
+1. 用超级管理员账号登录到Apollo配置中心的界面
+2. 进入`管理员工具 - 系统参数 - ConfigDB 配置管理`页面新增或修改`item.num.limit.enabled`配置项为true/false 即可开启/关闭此功能
+
+   ![item-num-limit-enabled](https://cdn.jsdelivr.net/gh/apolloconfig/apollo@master/doc/images/item-num-limit-enabled.png)
+
+3. 进入`管理员工具 - 系统参数 - ConfigDB 配置管理`页面新增或修改`item.num.limit`配置项来配置单个Namespace下的item数量上限值
+
+   ![item-num-limit](https://cdn.jsdelivr.net/gh/apolloconfig/apollo@master/doc/images/item-num-limit.png)
+
+
+
+
 # 七、最佳实践
 
 ## 7.1 安全相关

Original file line number	Diff line number	Diff line change
`@@ -64,4 +64,7 @@ public interface ItemRepository extends PagingAndSortingRepository<Item, Long> {`
`64`	`64`	`@Query("update Item set IsDeleted = true, DeletedAt = ROUND(UNIX_TIMESTAMP(NOW(4))*1000), DataChange_LastModifiedBy = ?2 where NamespaceId = ?1 and IsDeleted = false")`
`65`	`65`	`int deleteByNamespaceId(long namespaceId, String operator);`
`66`	`66`
	`67`	`+ @Query("select count(*) from Item where namespaceId = :namespaceId and key <>''")`
	`68`	`+ int countByNamespaceIdAndFilterKeyEmpty(@Param("namespaceId") long namespaceId);`
	`69`	`+`
`67`	`70`	`}`
Original file line number	Diff line number	Diff line change
`@@ -135,6 +135,10 @@ public List<Item> findItemsModifiedAfterDate(long namespaceId, Date date) {`
`135`	`135`	`return itemRepository.findByNamespaceIdAndDataChangeLastModifiedTimeGreaterThan(namespaceId, date);`
`136`	`136`	`}`
`137`	`137`
	`138`	`+ public int findNonEmptyItemCount(long namespaceId) {`
	`139`	`+ return itemRepository.countByNamespaceIdAndFilterKeyEmpty(namespaceId);`
	`140`	`+ }`
	`141`	`+`
`138`	`142`	`public Page<Item> findItemsByKey(String key, Pageable pageable) {`
`139`	`143`	`return itemRepository.findByKey(key, pageable);`
`140`	`144`	`}`