BanyanDB: Separate the storage configuration to an independent file: bydb.yaml. (#13097)

Author: wankai123

Diff for: apm-dist/src/main/assembly/binary.xml

@@ -61,6 +61,7 @@
                 <include>trace-sampling-policy-settings.yml</include>
                 <include>hierarchy-definition.yml</include>
                 <include>bydb.dependencies.properties</include>
+                <include>bydb.yml</include>
                 <include>oal/*.oal</include>
                 <include>fetcher-prom-rules/*.yaml</include>
                 <include>envoy-metrics-rules/**</include>

Diff for: docs/en/changes/changes.md

@@ -79,6 +79,7 @@
 * BanyanDB: fix alarm query result without sort. 
 * Add a component ID for Virtual thread executor.
 * Add more model installation log info for OAP storage initialization.
+* BanyanDB: Separate the storage configuration to an independent file: `bydb.yaml`.
 
 #### UI
 

Diff for: docs/en/setup/backend/storages/banyandb.md

@@ -22,76 +22,90 @@ org.apache.skywalking.oap.server.library.module.ModuleStartException: Incompatib
 ```
 
 ### Configuration
+In the `application.yml` file, select the BanyanDB storage provider:
 
 ```yaml
 storage:
   banyandb:
-    # Targets is the list of BanyanDB servers, separated by commas.
-    # Each target is a BanyanDB server in the format of `host:port`.
-    # If BanyanDB is deployed as a standalone server, the target should be the IP address or domain name and port of the BanyanDB server.
-    # If BanyanDB is deployed in a cluster, the targets should be the IP address or domain name and port of the `liaison` nodes, separated by commas.
-    targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912}
-    # The maximum number of records in a bulk write request.
-    # A larger value can improve write performance but also increases OAP and BanyanDB Server memory usage.
-    maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:10000}
-    # The minimum seconds between two bulk flushes.
-    # If the data in a bulk is less than maxBulkSize, the data will be flushed after this period.
-    # If the data in a bulk exceeds maxBulkSize, the data will be flushed immediately.
-    # A larger value can reduce write pressure on BanyanDB Server but increase data latency.
-    flushInterval: ${SW_STORAGE_BANYANDB_FLUSH_INTERVAL:15}
-    # The timeout in seconds for a bulk flush.
-    flushTimeout: ${SW_STORAGE_BANYANDB_FLUSH_TIMEOUT:10}
-    # The number of threads that write data to BanyanDB concurrently.
-    # A higher value can improve write performance but also increases CPU usage on both OAP and BanyanDB Server.
-    concurrentWriteThreads: ${SW_STORAGE_BANYANDB_CONCURRENT_WRITE_THREADS:15}
-    # The maximum size of the dataset when the OAP loads cache, such as network aliases.
-    resultWindowMaxSize: ${SW_STORAGE_BANYANDB_QUERY_MAX_WINDOW_SIZE:10000}
-    # The maximum size of metadata per query.
-    metadataQueryMaxSize: ${SW_STORAGE_BANYANDB_QUERY_MAX_SIZE:10000}
-    # The maximum number of trace segments per query.
-    segmentQueryMaxSize: ${SW_STORAGE_BANYANDB_QUERY_SEGMENT_SIZE:200}
-    # The maximum number of profile task queries in a request.
-    profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_TASK_SIZE:200}
-    # The batch size for querying profile data.
-    profileDataQueryBatchSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_DATA_BATCH_SIZE:100}
-    # If the BanyanDB server is configured with TLS, configure the TLS cert file path and enable TLS connection.
-    sslTrustCAPath: ${SW_STORAGE_BANYANDB_SSL_TRUST_CA_PATH:""}
-    # The group settings of record.
-    # `gr` is the short name of the group settings of record.
-    #
-    # The "normal" section defines settings for datasets not specified in "super".
-    # Each dataset will be grouped under a single group named "normal".
-    grNormalShardNum: ${SW_STORAGE_BANYANDB_GR_NORMAL_SHARD_NUM:1}
-    grNormalSIDays: ${SW_STORAGE_BANYANDB_GR_NORMAL_SI_DAYS:1}
-    grNormalTTLDays: ${SW_STORAGE_BANYANDB_GR_NORMAL_TTL_DAYS:3}
-    # "super" is a special dataset designed to store trace or log data that is too large for normal datasets.
-    # Each super dataset will be a separate group in BanyanDB, following the settings defined in the "super" section.
-    grSuperShardNum: ${SW_STORAGE_BANYANDB_GR_SUPER_SHARD_NUM:2}
-    grSuperSIDays: ${SW_STORAGE_BANYANDB_GR_SUPER_SI_DAYS:1}
-    grSuperTTLDays: ${SW_STORAGE_BANYANDB_GR_SUPER_TTL_DAYS:3}
-    # The group settings of metrics.
-    # `gm` is the short name of the group settings of metrics.
-    #
-    # OAP stores metrics based its granularity.
-    # Valid values are "day", "hour", and "minute". That means metrics will be stored in the three separate groups.
-    # Non-"minute" are governed by the "core.downsampling" setting.
-    # For example, if "core.downsampling" is set to "hour", the "hour" will be used, while "day" are ignored.
-    gmMinuteShardNum: ${SW_STORAGE_BANYANDB_GM_MINUTE_SHARD_NUM:2}
-    gmMinuteSIDays: ${SW_STORAGE_BANYANDB_GM_MINUTE_SI_DAYS:1}
-    gmMinuteTTLDays: ${SW_STORAGE_BANYANDB_GM_MINUTE_TTL_DAYS:7}
-    gmHourShardNum: ${SW_STORAGE_BANYANDB_GM_HOUR_SHARD_NUM:1}
-    gmHourSIDays: ${SW_STORAGE_BANYANDB_GM_HOUR_SI_DAYS:5}
-    gmHourTTLDays: ${SW_STORAGE_BANYANDB_GM_HOUR_TTL_DAYS:15}
-    gmDayShardNum: ${SW_STORAGE_BANYANDB_GM_DAY_SHARD_NUM:1}
-    gmDaySIDays: ${SW_STORAGE_BANYANDB_GM_DAY_SI_DAYS:15}
-    gmDayTTLDays: ${SW_STORAGE_BANYANDB_GM_DAY_TTL_DAYS:15}
-    # If the metrics is marked as "index_mode", the metrics will be stored in the "index" group.
-    # The "index" group is designed to store metrics that are used for indexing without value columns.
-    # Such as `service_traffic`, `network_address_alias`, etc.
-    # "index_mode" requires BanyanDB *0.8.0* or later.
-    gmIndexShardNum: ${SW_STORAGE_BANYANDB_GM_INDEX_SHARD_NUM:2}
-    gmIndexSIDays: ${SW_STORAGE_BANYANDB_GM_INDEX_SI_DAYS:15}
-    gmIndexTTLDays: ${SW_STORAGE_BANYANDB_GM_INDEX_TTL_DAYS:15}
+```
+
+Since 10.2.0, the banyandb configuration is separated to an independent configuration file: `bydb.yaml`:
+
+```yaml
+global:
+  # Targets is the list of BanyanDB servers, separated by commas.
+  # Each target is a BanyanDB server in the format of `host:port`.
+  # If BanyanDB is deployed as a standalone server, the target should be the IP address or domain name and port of the BanyanDB server.
+  # If BanyanDB is deployed in a cluster, the targets should be the IP address or domain name and port of the `liaison` nodes, separated by commas.
+  targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912}
+  # The maximum number of records in a bulk write request.
+  # A larger value can improve write performance but also increases OAP and BanyanDB Server memory usage.
+  maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:10000}
+  # The minimum seconds between two bulk flushes.
+  # If the data in a bulk is less than maxBulkSize, the data will be flushed after this period.
+  # If the data in a bulk exceeds maxBulkSize, the data will be flushed immediately.
+  # A larger value can reduce write pressure on BanyanDB Server but increase data latency.
+  flushInterval: ${SW_STORAGE_BANYANDB_FLUSH_INTERVAL:15}
+  # The timeout in seconds for a bulk flush.
+  flushTimeout: ${SW_STORAGE_BANYANDB_FLUSH_TIMEOUT:10}
+  # The number of threads that write data to BanyanDB concurrently.
+  # A higher value can improve write performance but also increases CPU usage on both OAP and BanyanDB Server.
+  concurrentWriteThreads: ${SW_STORAGE_BANYANDB_CONCURRENT_WRITE_THREADS:15}
+  # The maximum size of the dataset when the OAP loads cache, such as network aliases.
+  resultWindowMaxSize: ${SW_STORAGE_BANYANDB_QUERY_MAX_WINDOW_SIZE:10000}
+  # The maximum size of metadata per query.
+  metadataQueryMaxSize: ${SW_STORAGE_BANYANDB_QUERY_MAX_SIZE:10000}
+  # The maximum number of trace segments per query.
+  segmentQueryMaxSize: ${SW_STORAGE_BANYANDB_QUERY_SEGMENT_SIZE:200}
+  # The maximum number of profile task queries in a request.
+  profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_TASK_SIZE:200}
+  # The batch size for querying profile data.
+  profileDataQueryBatchSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_DATA_BATCH_SIZE:100}
+  asyncProfilerTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_ASYNC_PROFILER_TASK_QUERY_MAX_SIZE:200}
+  # If the BanyanDB server is configured with TLS, configure the TLS cert file path and enable TLS connection.
+  sslTrustCAPath: ${SW_STORAGE_BANYANDB_SSL_TRUST_CA_PATH:""}
+
+groups:
+  # The group settings of record.
+  #
+  # The "normal" section defines settings for datasets not specified in "super".
+  # Each dataset will be grouped under a single group named "normal".
+  recordsNormal:
+    shardNum:  ${SW_STORAGE_BANYANDB_GR_NORMAL_SHARD_NUM:1}
+    segmentInterval: ${SW_STORAGE_BANYANDB_GR_NORMAL_SI_DAYS:1}
+    ttl: ${SW_STORAGE_BANYANDB_GR_NORMAL_TTL_DAYS:3}
+  # "super" is a special dataset designed to store trace or log data that is too large for normal datasets.
+  # Each super dataset will be a separate group in BanyanDB, following the settings defined in the "super" section.
+  recordsSuper:
+    shardNum: ${SW_STORAGE_BANYANDB_GR_SUPER_SHARD_NUM:2}
+    segmentInterval: ${SW_STORAGE_BANYANDB_GR_SUPER_SI_DAYS:1}
+    ttl: ${SW_STORAGE_BANYANDB_GR_SUPER_TTL_DAYS:3}
+  # The group settings of metrics.
+  #
+  # OAP stores metrics based its granularity.
+  # Valid values are "day", "hour", and "minute". That means metrics will be stored in the three separate groups.
+  # Non-"minute" are governed by the "core.downsampling" setting.
+  # For example, if "core.downsampling" is set to "hour", the "hour" will be used, while "day" are ignored.
+  metricsMin:
+    shardNum: ${SW_STORAGE_BANYANDB_GM_MINUTE_SHARD_NUM:2}
+    segmentInterval: ${SW_STORAGE_BANYANDB_GM_MINUTE_SI_DAYS:1}
+    ttl: ${SW_STORAGE_BANYANDB_GM_MINUTE_TTL_DAYS:7}
+  metricsHour:
+    shardNum: ${SW_STORAGE_BANYANDB_GM_HOUR_SHARD_NUM:1}
+    segmentInterval: ${SW_STORAGE_BANYANDB_GM_HOUR_SI_DAYS:5}
+    ttl: ${SW_STORAGE_BANYANDB_GM_HOUR_TTL_DAYS:15}
+  metricsDay:
+    shardNum: ${SW_STORAGE_BANYANDB_GM_DAY_SHARD_NUM:1}
+    segmentInterval: ${SW_STORAGE_BANYANDB_GM_DAY_SI_DAYS:15}
+    ttl: ${SW_STORAGE_BANYANDB_GM_DAY_TTL_DAYS:15}
+  # If the metrics is marked as "index_mode", the metrics will be stored in the "index" group.
+  # The "index" group is designed to store metrics that are used for indexing without value columns.
+  # Such as `service_traffic`, `network_address_alias`, etc.
+  # "index_mode" requires BanyanDB *0.8.0* or later.
+  metadata:
+    shardNum: ${SW_STORAGE_BANYANDB_GM_INDEX_SHARD_NUM:2}
+    segmentInterval: ${SW_STORAGE_BANYANDB_GM_INDEX_SI_DAYS:15}
+    ttl: ${SW_STORAGE_BANYANDB_GM_INDEX_TTL_DAYS:15}
 
 ```
 

Diff for: oap-server/server-library/library-module/src/main/java/org/apache/skywalking/oap/server/library/module/ModuleDefine.java

@@ -18,14 +18,13 @@
 
 package org.apache.skywalking.oap.server.library.module;
 
-import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
-import java.util.Enumeration;
-import java.util.Properties;
 import java.util.ServiceLoader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.skywalking.oap.server.library.util.YamlConfigLoaderUtils.copyProperties;
+
 /**
  * A module definition.
  */
@@ -113,40 +112,6 @@ void prepare(ModuleManager moduleManager,
         loadedProvider.prepare();
     }
 
-    private void copyProperties(ModuleConfig dest, Properties src, String moduleName,
-                                String providerName) throws IllegalAccessException {
-        if (dest == null) {
-            return;
-        }
-        Enumeration<?> propertyNames = src.propertyNames();
-        while (propertyNames.hasMoreElements()) {
-            String propertyName = (String) propertyNames.nextElement();
-            Class<? extends ModuleConfig> destClass = dest.getClass();
-            try {
-                Field field = getDeclaredField(destClass, propertyName);
-                field.setAccessible(true);
-                field.set(dest, src.get(propertyName));
-            } catch (NoSuchFieldException e) {
-                LOGGER.warn(
-                    propertyName + " setting is not supported in " + providerName + " provider of " + moduleName + " module");
-            }
-        }
-    }
-
-    private Field getDeclaredField(Class<?> destClass, String fieldName) throws NoSuchFieldException {
-        if (destClass != null) {
-            Field[] fields = destClass.getDeclaredFields();
-            for (Field field : fields) {
-                if (field.getName().equals(fieldName)) {
-                    return field;
-                }
-            }
-            return getDeclaredField(destClass.getSuperclass(), fieldName);
-        }
-
-        throw new NoSuchFieldException();
-    }
-
     @Override
     public final ModuleProvider provider() throws DuplicateProviderException, ProviderNotFoundException {
         if (loadedProvider == null) {

Diff for: oap-server/server-library/library-module/src/main/java/org/apache/skywalking/oap/server/library/module/ModuleProvider.java

@@ -143,7 +143,7 @@ <T extends Service> T getService(Class<T> serviceType) throws ServiceNotProvided
         throw new ServiceNotProvidedException("Service " + serviceType.getName() + " should not be provided, based on moduleDefine define.");
     }
 
-    ModuleDefine getModule() {
+    public ModuleDefine getModule() {
         return moduleDefine;
     }
 

Diff for: oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/YamlConfigLoaderUtils.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.library.util;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.Properties;
+import lombok.extern.slf4j.Slf4j;
+import org.yaml.snakeyaml.Yaml;
+
+@Slf4j
+public class YamlConfigLoaderUtils {
+
+    public static void replacePropertyAndLog(final String propertyName,
+                                             final Object propertyValue,
+                                             final Properties target,
+                                             final Object providerName,
+                                             final Yaml yaml) {
+        final String valueString = PropertyPlaceholderHelper.INSTANCE.replacePlaceholders(
+            String.valueOf(propertyValue), target);
+        if (valueString.trim().length() == 0) {
+            target.replace(propertyName, valueString);
+            log.info("Provider={} config={} has been set as an empty string", providerName, propertyName);
+        } else {
+            // Use YAML to do data type conversion.
+            final Object replaceValue = convertValueString(valueString, yaml);
+            if (replaceValue != null) {
+                target.replace(propertyName, replaceValue);
+            }
+        }
+    }
+
+    public static Object convertValueString(final String valueString, final Yaml yaml) {
+        try {
+            Object replaceValue = yaml.load(valueString);
+            if (replaceValue instanceof String || replaceValue instanceof Integer || replaceValue instanceof Long || replaceValue instanceof Boolean || replaceValue instanceof ArrayList) {
+                return replaceValue;
+            } else {
+                return valueString;
+            }
+        } catch (Exception e) {
+            log.warn("yaml convert value type error, use origin values string. valueString={}", valueString, e);
+            return valueString;
+        }
+    }
+
+    public static void copyProperties(final Object dest,
+                                      final Properties src,
+                                      final String moduleName,
+                                      final String providerName) throws IllegalAccessException {
+        if (dest == null) {
+            return;
+        }
+        Enumeration<?> propertyNames = src.propertyNames();
+        while (propertyNames.hasMoreElements()) {
+            String propertyName = (String) propertyNames.nextElement();
+            Class<?> destClass = dest.getClass();
+            try {
+                Field field = getDeclaredField(destClass, propertyName);
+                field.setAccessible(true);
+                field.set(dest, src.get(propertyName));
+            } catch (NoSuchFieldException e) {
+                log.warn(
+                    propertyName + " setting is not supported in " + providerName + " provider of " + moduleName + " module");
+            }
+        }
+    }
+
+    public static Field getDeclaredField(final Class<?> destClass, final String fieldName) throws NoSuchFieldException {
+        if (destClass != null) {
+            Field[] fields = destClass.getDeclaredFields();
+            for (Field field : fields) {
+                if (field.getName().equals(fieldName)) {
+                    return field;
+                }
+            }
+            return getDeclaredField(destClass.getSuperclass(), fieldName);
+        }
+
+        throw new NoSuchFieldException();
+    }
+}

Diff for: oap-server/server-starter/pom.xml

@@ -316,6 +316,7 @@
                         <exclude>trace-sampling-policy-settings.yml</exclude>
                         <exclude>hierarchy-definition.yml</exclude>
                         <exclude>bydb.dependencies.properties</exclude>
+                        <exclude>bydb.yml</exclude>
                         <exclude>oal/</exclude>
                         <exclude>fetcher-prom-rules/</exclude>
                         <exclude>envoy-metrics-rules/</exclude>