Fix store check when offline & add cluster manager log file config (#385)

Author: solotzg

Diff for: cluster_manage/flash_cluster_manager.py

@@ -2,8 +2,8 @@
 import logging
 import os
 import socket
-import sys
 import time
+from logging.handlers import RotatingFileHandler
 
 import conf
 import etcd
@@ -253,12 +253,14 @@ def table_update(self):
 
 def main():
     flash_conf = conf.flash_conf
-
-    if not os.path.exists(flash_conf.tmp_path):
-        os.makedirs(flash_conf.tmp_path)
-
-    logging.basicConfig(filename='{}/flash_cluster_manager.log'.format(flash_conf.tmp_path), level=conf.log_level,
-                        format='%(asctime)s <%(levelname)s> %(name)s: %(message)s')
+    parent_path = os.path.dirname(flash_conf.log_path)
+    if not os.path.exists(parent_path):
+        os.makedirs(parent_path)
+
+    # keep at most 10G log files
+    logging.basicConfig(
+        handlers=[RotatingFileHandler(flash_conf.log_path, maxBytes=1024 * 1024 * 1024, backupCount=10)],
+        level=conf.log_level, format='%(asctime)s <%(levelname)s> %(name)s: %(message)s')
     logging.getLogger("requests").setLevel(logging.WARNING)
     logging.getLogger("urllib3").setLevel(logging.WARNING)
 

Diff for: cluster_manage/flash_tools.py

@@ -12,7 +12,7 @@ def __init__(self, file_path):
         self.pd_addrs = util.compute_addr_list(self.conf_toml['raft']['pd_addr'])
         self.http_port = self.conf_toml['http_port']
         self.data_path = self.conf_toml['path']
-        self.tmp_path = self.conf_toml['tmp_path']
+        tmp_path = self.conf_toml['tmp_path']
 
         p = self.conf_toml['flash']
         service_addr = p['service_addr']
@@ -25,6 +25,7 @@ def __init__(self, file_path):
         self.cluster_refresh_interval = min(
             int(flash_cluster['refresh_interval']), self.cluster_master_ttl)
         self.update_rule_interval = int(flash_cluster['update_rule_interval'])
+        self.log_path = flash_cluster.get('log', '{}/flash_cluster_manager.log'.format(tmp_path))
 
 
 def main():

Diff for: cluster_manage/pd_client.py

@@ -123,7 +123,7 @@ def get_store_by_labels(self, flash_label):
         for store in all_stores['stores']:
             store = store['store']
             for label in store.get('labels', []):
-                if label == flash_label and store.get('state_name') == 'Up':
+                if label == flash_label and store.get('state_name') in ('Up', 'Offline'):
                     res[store['id']] = store
         return res