Add more model installation log info for OAP storage initialization. (#13094)

wankai123 · web-flow · commit f7da99d6324b · 2025-03-10T11:01:32.000+08:00
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
@@ -77,7 +77,8 @@
 * Optimize metrics cache loading when trace latency greater than cache timeout. 
 * Allow calling `lang.groovy.GString` in DSL.
 * BanyanDB: fix alarm query result without sort. 
-* Add a component ID for Virtual thread executor
+* Add a component ID for Virtual thread executor.
+* Add more model installation log info for OAP storage initialization.
 
 #### UI
 
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelInstaller.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelInstaller.java
@@ -18,7 +18,9 @@
 
 package org.apache.skywalking.oap.server.core.storage.model;
 
+import lombok.Getter;
 import lombok.RequiredArgsConstructor;
+import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.oap.server.core.CoreModule;
 import org.apache.skywalking.oap.server.core.RunningMode;
@@ -38,20 +40,29 @@ public abstract class ModelInstaller implements ModelCreator.CreatingListener {
     @Override
     public void whenCreating(Model model) throws StorageException {
         if (RunningMode.isNoInitMode()) {
-            while (!isExists(model)) {
-                try {
-                    log.info(
-                        "table: {} does not exist. OAP is running in 'no-init' mode, waiting... retry 3s later.",
-                        model.getName()
-                    );
-                    Thread.sleep(3000L);
-                } catch (InterruptedException e) {
-                    log.error(e.getMessage());
+            while (true) {
+                InstallInfo info = isExists(model);
+                if (!info.isAllExist()) {
+                    try {
+                        log.info(
+                            "install info: {}.table for model: [{}] not all required resources exist. OAP is running in 'no-init' mode, waiting create or update... retry 3s later.",
+                            info.buildInstallInfoMsg(), model.getName()
+                        );
+                        Thread.sleep(3000L);
+                    } catch (InterruptedException e) {
+                        log.error(e.getMessage());
+                    }
+                } else {
+                    break;
                 }
             }
         } else {
-            if (!isExists(model)) {
-                log.info("table: {} does not exist", model.getName());
+            InstallInfo info = isExists(model);
+            if (!info.isAllExist()) {
+                log.info(
+                    "install info: {}. table for model: [{}] not all required resources exist, creating or updating...",
+                    info.buildInstallInfoMsg(), model.getName()
+                );
                 createTable(model);
             }
         }
@@ -74,10 +85,35 @@ protected final void overrideColumnName(String columnName, String newName) {
     /**
      * Check whether the storage entity exists. Need to implement based on the real storage.
      */
-    public abstract boolean isExists(Model model) throws StorageException;
+    public abstract InstallInfo isExists(Model model) throws StorageException;
 
     /**
      * Create the storage entity. All creations should be after the {@link #isExists(Model)} check.
      */
     public abstract void createTable(Model model) throws StorageException;
+
+    @Getter
+    @Setter
+    public abstract static class InstallInfo {
+        private final String modelName;
+        private final boolean timeSeries;
+        private final boolean superDataset;
+        private final String modelType;
+        private boolean allExist;
+
+        protected InstallInfo(Model model) {
+            this.modelName = model.getName();
+            this.timeSeries = model.isTimeSeries();
+            this.superDataset = model.isSuperDataset();
+            if (model.isMetric()) {
+                this.modelType = "metric";
+            } else if (model.isRecord()) {
+                this.modelType = "record";
+            } else {
+                this.modelType = "unknown";
+            }
+        }
+
+        public abstract String buildInstallInfoMsg();
+    }
 }
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexInstaller.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexInstaller.java
@@ -25,6 +25,8 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+import lombok.Getter;
+import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.banyandb.common.v1.BanyandbCommon;
 import org.apache.skywalking.banyandb.common.v1.BanyandbCommon.Group;
@@ -41,6 +43,7 @@
 import org.apache.skywalking.banyandb.v1.client.metadata.ResourceExist;
 import org.apache.skywalking.oap.server.core.CoreModule;
 import org.apache.skywalking.oap.server.core.RunningMode;
+import org.apache.skywalking.oap.server.core.analysis.DownSampling;
 import org.apache.skywalking.oap.server.core.config.DownSamplingConfigService;
 import org.apache.skywalking.oap.server.core.storage.StorageException;
 import org.apache.skywalking.oap.server.core.storage.model.Model;
@@ -62,21 +65,31 @@ public BanyanDBIndexInstaller(Client client, ModuleManager moduleManager, Banyan
     }
 
     @Override
-    public boolean isExists(Model model) throws StorageException {
+    public InstallInfo isExists(Model model) throws StorageException {
+        InstallInfoBanyanDB installInfo = new InstallInfoBanyanDB(model);
+        installInfo.setDownSampling(model.getDownsampling());
         if (!model.isTimeSeries()) {
-            return true;
+            installInfo.setTableName(model.getName());
+            installInfo.setAllExist(true);
+            return installInfo;
         }
         final DownSamplingConfigService downSamplingConfigService = moduleManager.find(CoreModule.NAME)
                                                          .provider()
                                                          .getService(DownSamplingConfigService.class);
         final MetadataRegistry.SchemaMetadata metadata = MetadataRegistry.INSTANCE.parseMetadata(
             model, config, downSamplingConfigService);
+        installInfo.setTableName(metadata.name());
+        installInfo.setKind(metadata.getKind());
+        installInfo.setGroup(metadata.getGroup());
         try {
             final BanyanDBClient c = ((BanyanDBStorageClient) this.client).client;
             // first check resource existence and create group if necessary
-            final boolean resourceExist = checkResourceExistence(metadata, c);
-            if (!resourceExist) {
-                return false;
+            final ResourceExist resourceExist = checkResourceExistence(metadata, c);
+            installInfo.setGroupExist(resourceExist.hasGroup());
+            installInfo.setTableExist(resourceExist.hasResource());
+            if (!resourceExist.hasResource()) {
+                installInfo.setAllExist(false);
+                return installInfo;
             } else {
                 // register models only locally(Schema cache) but not remotely
                 if (model.isRecord()) { // stream
@@ -108,7 +121,8 @@ public boolean isExists(Model model) throws StorageException {
                 if (remoteMeta == null) {
                     throw new IllegalStateException("inconsistent state: metadata:" + metadata + ", remoteMeta: null");
                 }
-                return true;
+                installInfo.setAllExist(true);
+                return installInfo;
             }
         } catch (BanyanDBException ex) {
             throw new StorageException("fail to check existence", ex);
@@ -206,7 +220,7 @@ private boolean checkGroup(MetadataRegistry.SchemaMetadata metadata, BanyanDBCli
             || g.getResourceOpts().getTtl().getNum() != metadata.getTtlDays();
     }
 
-    private boolean checkResourceExistence(MetadataRegistry.SchemaMetadata metadata,
+    private ResourceExist checkResourceExistence(MetadataRegistry.SchemaMetadata metadata,
                                            BanyanDBClient client) throws BanyanDBException {
         ResourceExist resourceExist;
         Group.Builder gBuilder
@@ -265,7 +279,7 @@ private boolean checkResourceExistence(MetadataRegistry.SchemaMetadata metadata,
                 groupAligned.add(metadata.getGroup());
             }
         }
-        return resourceExist.hasResource();
+        return resourceExist;
     }
 
     /**
@@ -559,4 +573,36 @@ private void checkTopNAggregation(Model model, BanyanDBClient client) throws Ban
             }
         }
     }
+
+    @Getter
+    @Setter
+    private static class InstallInfoBanyanDB extends InstallInfo {
+        private DownSampling downSampling;
+        private String tableName;
+        private MetadataRegistry.Kind kind;
+        private String group;
+        private boolean tableExist;
+        private boolean groupExist;
+
+        protected InstallInfoBanyanDB(Model model) {
+            super(model);
+        }
+
+        @Override
+        public String buildInstallInfoMsg() {
+            return "InstallInfoBanyanDB{" +
+                "modelName=" + getModelName() +
+                ", modelType=" + getModelType() +
+                ", timeSeries=" + isTimeSeries() +
+                ", superDataset=" + isSuperDataset() +
+                ", downSampling=" + downSampling.name() +
+                ", tableName=" + tableName +
+                ", kind=" + kind.name() +
+                ", group=" + group +
+                ", allResourcesExist=" + isAllExist() +
+                " [groupExist=" + groupExist +
+                ", tableExist=" + tableExist +
+                "]}";
+        }
+    }
 }
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
@@ -21,6 +21,7 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.google.common.base.Strings;
 import com.google.gson.Gson;
+import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.library.elasticsearch.response.Index;
@@ -81,12 +82,15 @@ protected IndexStructures getStructures() {
     }
 
     @Override
-    public boolean isExists(Model model) throws StorageException {
+    public InstallInfo isExists(Model model) throws StorageException {
+        InstallInfoES installInfo = new InstallInfoES(model, config);
         ElasticSearchClient esClient = (ElasticSearchClient) client;
         String tableName = IndexController.INSTANCE.getTableName(model);
         IndexController.LogicIndicesRegister.registerRelation(model, tableName);
+        installInfo.setTableName(esClient.formatIndexName(tableName));
         if (!model.isTimeSeries()) {
             boolean exist = esClient.isExistsIndex(tableName);
+            installInfo.setTableExist(exist);
             if (exist) {
                 Optional<Index> index = esClient.getIndex(tableName);
                 Mappings historyMapping = index.map(Index::getMappings).orElseGet(Mappings::new);
@@ -97,15 +101,21 @@ public boolean isExists(Model model) throws StorageException {
                     // or updating field types, it just cares about whether the data can be ingested without
                     // reporting errors.
                     exist = structures.containsFieldNames(tableName, createMapping(model));
+                    installInfo.setAllFieldsExist(exist);
                 } else {
                     boolean containsMapping = structures.containsMapping(tableName, createMapping(model));
-                    exist = containsMapping && structures.compareIndexSetting(tableName, createSetting(model));
+                    installInfo.setAllFieldsExist(containsMapping);
+                    boolean containsSetting = structures.compareIndexSetting(tableName, createSetting(model));
+                    installInfo.setAllIndexSettingsExist(containsSetting);
+                    exist = containsMapping && containsSetting;
                 }
             }
-            return exist;
+            installInfo.setAllExist(exist);
+            return installInfo;
         }
 
         boolean templateExists = esClient.isExistsTemplate(tableName);
+        installInfo.setTableExist(templateExists);
         final Optional<IndexTemplate> template = esClient.getTemplate(tableName);
 
         if ((templateExists && template.isEmpty()) || (!templateExists && template.isPresent())) {
@@ -123,12 +133,17 @@ public boolean isExists(Model model) throws StorageException {
             // because the no-init mode OAP server doesn't take responsibility for index settings.
             if (RunningMode.isNoInitMode()) {
                 exist = structures.containsFieldNames(tableName, createMapping(model));
+                installInfo.setAllFieldsExist(exist);
             } else {
                 boolean containsMapping = structures.containsMapping(tableName, createMapping(model));
-                exist = containsMapping && structures.compareIndexSetting(tableName, createSetting(model));
+                installInfo.setAllFieldsExist(containsMapping);
+                boolean containsSetting = structures.compareIndexSetting(tableName, createSetting(model));
+                installInfo.setAllIndexSettingsExist(containsSetting);
+                exist = containsMapping && containsSetting;
             }
         }
-        return exist;
+        installInfo.setAllExist(exist);
+        return installInfo;
     }
 
     @Override
@@ -373,4 +388,38 @@ protected Mappings createMapping(Model model) {
 
         return mappings;
     }
+
+    @Getter
+    @Setter
+    public static class InstallInfoES extends InstallInfo {
+        private String tableName;
+        private boolean tableExist;
+        private boolean allFieldsExist;
+        private boolean allIndexSettingsExist;
+        private StorageModuleElasticsearchConfig config;
+
+        protected InstallInfoES(Model model, StorageModuleElasticsearchConfig config) {
+            super(model);
+            this.config = config;
+        }
+
+        @Override
+        public String buildInstallInfoMsg() {
+            String tableNameMsg = isTimeSeries() ? "indexTemplateName=" + tableName : "indexName=" + tableName;
+            String tableExistMsg = isTimeSeries() ? "indexTemplateExists=" + tableExist : "indexExists=" + tableExist;
+            return "InstallInfoES:{" +
+                "modelName=" + getModelName() +
+                ", modelType=" + getModelType() +
+                ", timeSeries=" + isTimeSeries() +
+                ", superDataset=" + isSuperDataset() +
+                ", logicSharding=" + config.isLogicSharding() +
+                ", indexNamespace=" + config.getNamespace() +
+                ", " + tableNameMsg +
+                ", allResourcesExist=" + isAllExist() +
+                " [" + tableExistMsg +
+                ", allFieldsExist=" + allFieldsExist +
+                ", allIndexSettingsExist=" + allIndexSettingsExist +
+                "]}";
+        }
+    }
 }
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCTableInstaller.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCTableInstaller.java
@@ -19,6 +19,8 @@
 package org.apache.skywalking.oap.server.storage.plugin.jdbc.common;
 
 import com.google.gson.JsonObject;
+import lombok.Getter;
+import lombok.Setter;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.oap.server.core.analysis.DownSampling;
@@ -64,14 +66,16 @@ public JDBCTableInstaller(Client client, ModuleManager moduleManager) {
 
     @Override
     @SneakyThrows
-    public boolean isExists(Model model) {
+    public InstallInfo isExists(Model model) {
+        InstallInfoJDBC installInfo = new InstallInfoJDBC(model);
         TableMetaInfo.addModel(model);
 
         final var table = TableHelper.getLatestTableForWrite(model);
-
+        installInfo.setTableName(table);
         final var jdbcClient = (JDBCClient) client;
         if (!jdbcClient.tableExists(table)) {
-            return false;
+            installInfo.setAllExist(false);
+            return installInfo;
         }
 
         final var databaseColumns = getDatabaseColumns(table);
@@ -81,8 +85,9 @@ public boolean isExists(Model model) {
                 .map(ModelColumn::getColumnName)
                 .map(ColumnName::getStorageName)
                 .anyMatch(not(databaseColumns::contains));
-
-        return !isAnyColumnNotCreated;
+        installInfo.setAllColumnsExist(isAnyColumnNotCreated);
+        installInfo.setAllExist(!isAnyColumnNotCreated);
+        return installInfo;
     }
 
     @Override
@@ -277,4 +282,30 @@ private void createTable(String table, List<ModelColumn> columns, boolean isAddi
 
         executeSQL(sql);
     }
+
+    @Getter
+    @Setter
+    private static class InstallInfoJDBC extends InstallInfo {
+        private String tableName;
+        private boolean tableExist;
+        private boolean allColumnsExist;
+
+        protected InstallInfoJDBC(Model model) {
+            super(model);
+        }
+
+        @Override
+        public String buildInstallInfoMsg() {
+            return "InstallInfoJDBC{" +
+                "modelName=" + getModelName() +
+                ", modelType=" + getModelType() +
+                ", timeSeries=" + isTimeSeries() +
+                ", superDataset=" + isSuperDataset() +
+                ", tableName=" + tableName +
+                ", allResourcesExist=" + isAllExist() +
+                " [tableExist=" + tableExist +
+                ", allColumnsExist=" + allColumnsExist +
+                "]}";
+        }
+    }
 }
