Merge remote-tracking branch 'origin/master'

CHANGES.md

@@ -8,6 +8,14 @@ Apollo 2.4.0
 * [Update the server config link in system info page](https://github.com/apolloconfig/apollo/pull/5204)
 * [Feature support portal restTemplate Client connection pool config](https://github.com/apolloconfig/apollo/pull/5200)
 * [Feature added the ability for administrators to globally search for Value](https://github.com/apolloconfig/apollo/pull/5182)
+* [Fix: Resolve issues with duplicate comments and blank lines in configuration management](https://github.com/apolloconfig/apollo/pull/5232)
+* [Fix link namespace published items show missing some items](https://github.com/apolloconfig/apollo/pull/5240)
+* [Feature: Add limit and whitelist for namespace count per appid+cluster](https://github.com/apolloconfig/apollo/pull/5228)
+* [Feature support the observe status access-key for pre-check and logging only](https://github.com/apolloconfig/apollo/pull/5236)
+* [Feature add limit for items count per namespace](https://github.com/apolloconfig/apollo/pull/5227)
+* [Feature: Add ConfigService cache record stats function](https://github.com/apolloconfig/apollo/pull/5247)
+* [RefreshAdminServerAddressTask supports dynamic configuration of time interval](https://github.com/apolloconfig/apollo/pull/5248)
+* [Refactor: Configuration files uniformly use Kebab style](https://github.com/apolloconfig/apollo/pull/5262)
 
 ------------------
-All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1)
+All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1)

apollo-adminservice/src/main/java/com/ctrip/framework/apollo/adminservice/controller/AccessKeyController.java

@@ -16,6 +16,8 @@
  */
 package com.ctrip.framework.apollo.adminservice.controller;
 
+import static com.ctrip.framework.apollo.common.constants.AccessKeyMode.FILTER;
+
 import com.ctrip.framework.apollo.biz.entity.AccessKey;
 import com.ctrip.framework.apollo.biz.service.AccessKeyService;
 import com.ctrip.framework.apollo.common.dto.AccessKeyDTO;
@@ -27,6 +29,7 @@
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 /**
@@ -61,9 +64,11 @@ public void delete(@PathVariable String appId, @PathVariable long id, String ope
   }
 
   @PutMapping(value = "/apps/{appId}/accesskeys/{id}/enable")
-  public void enable(@PathVariable String appId, @PathVariable long id, String operator) {
+  public void enable(@PathVariable String appId, @PathVariable long id,
+      @RequestParam(required = false, defaultValue = "" + FILTER) int mode, String operator) {
     AccessKey entity = new AccessKey();
     entity.setId(id);
+    entity.setMode(mode);
     entity.setEnabled(true);
     entity.setDataChangeLastModifiedBy(operator);
 
@@ -74,6 +79,7 @@ public void enable(@PathVariable String appId, @PathVariable long id, String ope
   public void disable(@PathVariable String appId, @PathVariable long id, String operator) {
     AccessKey entity = new AccessKey();
     entity.setId(id);
+    entity.setMode(FILTER);
     entity.setEnabled(false);
     entity.setDataChangeLastModifiedBy(operator);
 

apollo-adminservice/src/main/java/com/ctrip/framework/apollo/adminservice/controller/ItemController.java

@@ -17,6 +17,7 @@
 package com.ctrip.framework.apollo.adminservice.controller;
 
 import com.ctrip.framework.apollo.adminservice.aop.PreAcquireNamespaceLock;
+import com.ctrip.framework.apollo.biz.config.BizConfig;
 import com.ctrip.framework.apollo.biz.entity.Commit;
 import com.ctrip.framework.apollo.biz.entity.Item;
 import com.ctrip.framework.apollo.biz.entity.Namespace;
@@ -58,13 +59,14 @@ public class ItemController {
   private final NamespaceService namespaceService;
   private final CommitService commitService;
   private final ReleaseService releaseService;
+  private final BizConfig bizConfig;
 
-
-  public ItemController(final ItemService itemService, final NamespaceService namespaceService, final CommitService commitService, final ReleaseService releaseService) {
+  public ItemController(final ItemService itemService, final NamespaceService namespaceService, final CommitService commitService, final ReleaseService releaseService, final BizConfig bizConfig) {
     this.itemService = itemService;
     this.namespaceService = namespaceService;
     this.commitService = commitService;
     this.releaseService = releaseService;
+    this.bizConfig = bizConfig;
   }
 
   @PreAcquireNamespaceLock
@@ -78,6 +80,14 @@ public ItemDTO create(@PathVariable("appId") String appId,
     if (managedEntity != null) {
       throw BadRequestException.itemAlreadyExists(entity.getKey());
     }
+
+    if (bizConfig.isItemNumLimitEnabled()) {
+      int itemCount = itemService.findNonEmptyItemCount(entity.getNamespaceId());
+      if (itemCount >= bizConfig.itemNumLimit()) {
+        throw new BadRequestException("The maximum number of items (" + bizConfig.itemNumLimit() + ") for this namespace has been reached. Current item count is " + itemCount + ".");
+      }
+    }
+
     entity = itemService.save(entity);
     dto = BeanUtils.transform(ItemDTO.class, entity);
     commitService.createCommit(appId, clusterName, namespaceName, new ConfigChangeContentBuilder().createItem(entity).build(),

apollo-adminservice/src/main/resources/application-database-discovery.properties

@@ -18,5 +18,5 @@ spring.cloud.discovery.enabled=false
 
 apollo.service.registry.enabled=true
 apollo.service.registry.cluster=default
-apollo.service.registry.heartbeatIntervalInSecond=10
+apollo.service.registry.heartbeat-interval-in-second=10
 

apollo-adminservice/src/main/resources/application.yml

@@ -38,17 +38,17 @@ logging:
 eureka:
   instance:
     hostname: ${hostname:localhost}
-    preferIpAddress: true
+    prefer-ip-address: true
     status-page-url-path: /info
     health-check-url-path: /health
   client:
-    serviceUrl:
+    service-url:
       # This setting will be overridden by eureka.service.url setting from ApolloConfigDB.ServerConfig or System Property
       # see com.ctrip.framework.apollo.biz.eureka.ApolloEurekaClientConfig
       defaultZone: http://${eureka.instance.hostname}:8080/eureka/
     healthcheck:
       enabled: true
-    eurekaServiceUrlPollIntervalSeconds: 60
+    eureka-service-url-poll-interval-seconds: 60
 
 management:
   health:

apollo-adminservice/src/test/resources/application.yml

@@ -23,11 +23,11 @@ server:
 eureka:
   instance:
     hostname: ${hostname:localhost}
-    preferIpAddress: true
+    prefer-ip-address: true
     status-page-url-path: /info
     health-check-url-path: /health
   client:
-    serviceUrl:
+    service-url:
       defaultZone: http://${eureka.instance.hostname}:8090/eureka/
     healthcheck:
       enabled: true

apollo-assembly/src/main/resources/application-database-discovery.properties

@@ -19,8 +19,8 @@ spring.cloud.discovery.enabled=false
 
 apollo.service.registry.enabled=true
 apollo.service.registry.cluster=default
-apollo.service.registry.heartbeatIntervalInSecond=10
+apollo.service.registry.heartbeat-interval-in-second=10
 
 apollo.service.discovery.enabled=true
 # health check by heartbeat, heartbeat time before 61s ago will be seemed as unhealthy
-apollo.service.discovery.healthCheckIntervalInSecond = 61
+apollo.service.discovery.health-check-interval-in-second = 61

apollo-assembly/src/main/resources/application.yml

@@ -30,7 +30,7 @@ spring:
 
 logging:
   file:
-    name: /opt/logs/100003171/apollo-assembly.log
+    name: /opt/logs/apollo-assembly.log
 
 management:
   health:
@@ -42,22 +42,22 @@ management:
 eureka:
   instance:
     hostname: ${hostname:localhost}
-    preferIpAddress: true
+    prefer-ip-address: true
     status-page-url-path: /info
     health-check-url-path: /health
   server:
-    peerEurekaNodesUpdateIntervalMs: 60000
-    enableSelfPreservation: false
+    peer-eureka-nodes-update-interval-ms: 60000
+    enable-self-preservation: false
   client:
-    serviceUrl:
+    service-url:
       # This setting will be overridden by eureka.service.url setting from ApolloConfigDB.ServerConfig or System Property
       # see com.ctrip.framework.apollo.biz.eureka.ApolloEurekaClientConfig
       defaultZone: http://${eureka.instance.hostname}:8080/eureka/
     healthcheck:
       enabled: true
-    eurekaServiceUrlPollIntervalSeconds: 60
+    eureka-service-url-poll-interval-seconds: 60
     fetch-registry: false
-    registerWithEureka: false
+    register-with-eureka: false
 
 server:
   compression:

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java

@@ -21,12 +21,14 @@
 import com.ctrip.framework.apollo.common.config.RefreshablePropertySource;
 import com.google.common.base.Strings;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
 import java.lang.reflect.Type;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import org.springframework.stereotype.Component;
@@ -36,6 +38,11 @@ public class BizConfig extends RefreshableConfig {
 
   private static final int DEFAULT_ITEM_KEY_LENGTH = 128;
   private static final int DEFAULT_ITEM_VALUE_LENGTH = 20000;
+
+  private static final int DEFAULT_MAX_NAMESPACE_NUM = 200;
+
+  private static final int DEFAULT_MAX_ITEM_NUM = 1000;
+
   private static final int DEFAULT_APPNAMESPACE_CACHE_REBUILD_INTERVAL = 60; //60s
   private static final int DEFAULT_GRAY_RELEASE_RULE_SCAN_INTERVAL = 60; //60s
   private static final int DEFAULT_APPNAMESPACE_CACHE_SCAN_INTERVAL = 1; //1s
@@ -99,6 +106,28 @@ public int itemValueLengthLimit() {
     return checkInt(limit, 5, Integer.MAX_VALUE, DEFAULT_ITEM_VALUE_LENGTH);
   }
 
+  public boolean isNamespaceNumLimitEnabled() {
+    return getBooleanProperty("namespace.num.limit.enabled", false);
+  }
+
+  public int namespaceNumLimit() {
+    int limit = getIntProperty("namespace.num.limit", DEFAULT_MAX_NAMESPACE_NUM);
+    return checkInt(limit, 0, Integer.MAX_VALUE, DEFAULT_MAX_NAMESPACE_NUM);
+  }
+
+  public Set<String> namespaceNumLimitWhite() {
+    return Sets.newHashSet(getArrayProperty("namespace.num.limit.white", new String[0]));
+  }
+
+  public boolean isItemNumLimitEnabled() {
+    return getBooleanProperty("item.num.limit.enabled", false);
+  }
+
+  public int itemNumLimit() {
+    int limit = getIntProperty("item.num.limit", DEFAULT_MAX_ITEM_NUM);
+    return checkInt(limit, 5, Integer.MAX_VALUE, DEFAULT_MAX_ITEM_NUM);
+  }
+
   public Map<Long, Integer> namespaceValueLengthLimitOverride() {
     String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override");
     Map<Long, Integer> namespaceValueLengthOverride = Maps.newHashMap();
@@ -205,6 +234,10 @@ public boolean isConfigServiceCacheEnabled() {
     return getBooleanProperty("config-service.cache.enabled", false);
   }
 
+  public boolean isConfigServiceCacheStatsEnabled() {
+    return getBooleanProperty("config-service.cache.stats.enabled", false);
+  }
+
   public boolean isConfigServiceCacheKeyIgnoreCase() {
     return getBooleanProperty("config-service.cache.key.ignore-case", false);
   }

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/entity/AccessKey.java

@@ -36,6 +36,9 @@ public class AccessKey extends BaseEntity {
   @Column(name = "`Secret`", nullable = false)
   private String secret;
 
+  @Column(name = "`Mode`")
+  private int mode;
+
   @Column(name = "`IsEnabled`", columnDefinition = "Bit default '0'")
   private boolean enabled;
 
@@ -55,6 +58,14 @@ public void setSecret(String secret) {
     this.secret = secret;
   }
 
+  public int getMode() {
+    return mode;
+  }
+
+  public void setMode(int mode) {
+    this.mode = mode;
+  }
+
   public boolean isEnabled() {
     return enabled;
   }
@@ -66,6 +77,6 @@ public void setEnabled(boolean enabled) {
   @Override
   public String toString() {
     return toStringHelper().add("appId", appId).add("secret", secret)
-        .add("enabled", enabled).toString();
+        .add("mode", mode).add("enabled", enabled).toString();
   }
 }

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ItemRepository.java

@@ -64,4 +64,7 @@ public interface ItemRepository extends PagingAndSortingRepository<Item, Long> {
   @Query("update Item set IsDeleted = true, DeletedAt = ROUND(UNIX_TIMESTAMP(NOW(4))*1000), DataChange_LastModifiedBy = ?2 where NamespaceId = ?1 and IsDeleted = false")
   int deleteByNamespaceId(long namespaceId, String operator);
 
+  @Query("select count(*) from Item where namespaceId = :namespaceId and key <>''")
+  int countByNamespaceIdAndFilterKeyEmpty(@Param("namespaceId") long namespaceId);
+
 }

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/NamespaceRepository.java

@@ -44,4 +44,6 @@ public interface NamespaceRepository extends PagingAndSortingRepository<Namespac
 
   int countByNamespaceNameAndAppIdNot(String namespaceName, String appId);
 
+  int countByAppIdAndClusterName(String appId, String clusterName);
+
 }

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/AccessKeyService.java

@@ -74,6 +74,7 @@ public AccessKey update(String appId, AccessKey entity) {
       throw BadRequestException.accessKeyNotExists();
     }
 
+    accessKey.setMode(entity.getMode());
     accessKey.setEnabled(entity.isEnabled());
     accessKey.setDataChangeLastModifiedBy(operator);
     accessKeyRepository.save(accessKey);

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java

@@ -135,6 +135,10 @@ public List<Item> findItemsModifiedAfterDate(long namespaceId, Date date) {
     return itemRepository.findByNamespaceIdAndDataChangeLastModifiedTimeGreaterThan(namespaceId, date);
   }
 
+  public int findNonEmptyItemCount(long namespaceId) {
+    return itemRepository.countByNamespaceIdAndFilterKeyEmpty(namespaceId);
+  }
+
   public Page<Item> findItemsByKey(String key, Pageable pageable) {
     return itemRepository.findByKey(key, pageable);
   }

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemSetService.java

@@ -16,6 +16,7 @@
  */
 package com.ctrip.framework.apollo.biz.service;
 
+import com.ctrip.framework.apollo.biz.config.BizConfig;
 import com.ctrip.framework.apollo.biz.entity.Audit;
 import com.ctrip.framework.apollo.biz.entity.Item;
 import com.ctrip.framework.apollo.biz.entity.Namespace;
@@ -25,6 +26,7 @@
 import com.ctrip.framework.apollo.common.exception.BadRequestException;
 import com.ctrip.framework.apollo.common.exception.NotFoundException;
 import com.ctrip.framework.apollo.common.utils.BeanUtils;
+import com.ctrip.framework.apollo.core.utils.StringUtils;
 import java.util.List;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
@@ -38,20 +40,23 @@ public class ItemSetService {
   private final CommitService commitService;
   private final ItemService itemService;
   private final NamespaceService namespaceService;
+  private final BizConfig bizConfig;
 
   public ItemSetService(
       final AuditService auditService,
       final CommitService commitService,
       final ItemService itemService,
-      final NamespaceService namespaceService) {
+      final NamespaceService namespaceService,
+      final BizConfig bizConfig) {
     this.auditService = auditService;
     this.commitService = commitService;
     this.itemService = itemService;
     this.namespaceService = namespaceService;
+    this.bizConfig = bizConfig;
   }
 
   @Transactional
-  public ItemChangeSets updateSet(Namespace namespace, ItemChangeSets changeSets){
+  public ItemChangeSets updateSet(Namespace namespace, ItemChangeSets changeSets) {
     return updateSet(namespace.getAppId(), namespace.getClusterName(), namespace.getNamespaceName(), changeSets);
   }
 
@@ -64,6 +69,16 @@ public ItemChangeSets updateSet(String appId, String clusterName,
       throw NotFoundException.namespaceNotFound(appId, clusterName, namespaceName);
     }
 
+    if (bizConfig.isItemNumLimitEnabled()) {
+      int itemCount = itemService.findNonEmptyItemCount(namespace.getId());
+      int createItemCount = (int) changeSet.getCreateItems().stream().filter(item -> !StringUtils.isEmpty(item.getKey())).count();
+      int deleteItemCount = (int) changeSet.getDeleteItems().stream().filter(item -> !StringUtils.isEmpty(item.getKey())).count();
+      itemCount = itemCount + createItemCount - deleteItemCount;
+      if (itemCount > bizConfig.itemNumLimit()) {
+        throw new BadRequestException("The maximum number of items (" + bizConfig.itemNumLimit() + ") for this namespace has been reached. Current item count is " + itemCount + ".");
+      }
+    }
+
     String operator = changeSet.getDataChangeLastModifiedBy();
     ConfigChangeContentBuilder configChangeContentBuilder = new ConfigChangeContentBuilder();
 
@@ -84,7 +99,7 @@ public ItemChangeSets updateSet(String appId, String clusterName,
 
     if (configChangeContentBuilder.hasContent()) {
       commitService.createCommit(appId, clusterName, namespaceName, configChangeContentBuilder.build(),
-                   changeSet.getDataChangeLastModifiedBy());
+                                 changeSet.getDataChangeLastModifiedBy());
     }
 
     return changeSet;