diff --git a/.gitignore b/.gitignore index d6e426f..54f752e 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,7 @@ RemoteWPS1.sln /.vs/config/applicationhost.config /dist/ /src/wps_remote.egg-info/ + +/json_out.json +/json_out_1.json +/json_out_2.json diff --git a/setup.py b/setup.py index 5861c1c..d54ebac 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( name = "wps-remote", - version = "2.14.5", + version = "2.14.6", author = "GeoServer Developers", author_email = "geoserver-devel@lists.sourceforge.net", description = "A library that allows users to publish their executables as GeoServer WPS Processes through the XMPP protocol", diff --git a/src/wpsremote/configInstance.py b/src/wpsremote/configInstance.py index 141d0fa..3a3f291 100644 --- a/src/wpsremote/configInstance.py +++ b/src/wpsremote/configInstance.py @@ -25,15 +25,15 @@ def create(filePath, moreSerchPath=[], variables={}, raw=False, case_sensitive=F fp.close() config.items_without_defaults = types.MethodType(items_without_defaults,config) - config.get_list = types.MethodType(get_list_impl,config) - config.get_list_list = types.MethodType(get_list_list_impl,config) - config.get_list_int = types.MethodType(get_list_int_impl,config) - config.get_list_float = types.MethodType(get_list_float_impl,config) - config.get_password = types.MethodType(get_password,config) - config.get_path = types.MethodType(get_path,config) - config.get_list_path = types.MethodType( get_list_path_impl, config) + config.get_list = types.MethodType(get_list_impl, config) + config.get_list_list = types.MethodType(get_list_list_impl, config) + config.get_list_int = types.MethodType(get_list_int_impl, config) + config.get_list_float = types.MethodType(get_list_float_impl, config) + config.get_password = types.MethodType(get_password, config) + config.get_path = types.MethodType(get_path, config) + config.get_list_path = types.MethodType(get_list_path_impl, config) return config - + def items_without_defaults(target, section_name, raw=False): #return [s for s in target.items(section_name, raw=raw) if not s[0] in target.defaults().keys()] res=[] @@ -48,7 +48,7 @@ def items_without_defaults(target, section_name, raw=False): res.append(s) return res - + def get_list_impl(target, section, itemname): v=target.get(section, itemname) if (v==None or v==''): @@ -110,4 +110,4 @@ def get_path(target, section, itemname): p=target.get(section, itemname) if (p==None): return None - return path.path(p) \ No newline at end of file + return path.path(p) diff --git a/src/wpsremote/resource_monitor.py b/src/wpsremote/resource_monitor.py index 085dba9..b10c418 100644 --- a/src/wpsremote/resource_monitor.py +++ b/src/wpsremote/resource_monitor.py @@ -12,6 +12,10 @@ import thread import time import psutil +import logging + +logger = logging.getLogger("servicebot.resource_monitor") + class ResourceMonitor(threading.Thread): @@ -20,8 +24,8 @@ class ResourceMonitor(threading.Thread): cpu_perc = [] vmem_perc = [] lock = threading.Lock() - - def __init__(self, load_average_scan_minutes): + + def __init__(self, load_average_scan_minutes): threading.Thread.__init__(self) ResourceMonitor.load_average_scan_minutes = load_average_scan_minutes ResourceMonitor.lock.acquire() @@ -34,37 +38,44 @@ def __init__(self, load_average_scan_minutes): ResourceMonitor.lock.release() - def proc_is_running(self, proc_names): + def proc_is_running(self, proc_defs): for proc in psutil.process_iter(): try: - process = psutil.Process(proc.pid).as_dict() # Get the process info using PID - - pid = str(process["pid"]) - ppid = str(process["ppid"]) - status = process["status"] - - cpu_percent = process["cpu_percent"] - mem_percent = process["memory_percent"] - - rss = str(process["memory_info"].rss) - vms = str(process["memory_info"].vms) - username = process["username"] - name = process["name"] # Here is the process name - path = process["cwd"] - - for proc_name in proc_names: - if status.lower() == "running" and proc_name in name.lower(): - return True + process = psutil.Process(proc.pid) # Get the process info using PID + if process.is_running(): + pid = str(process.pid) + ppid = str(process.ppid) + status = process.status() + + cpu_percent = process.cpu_percent() + mem_percent = process.memory_percent() + + rss = str(process.memory_info().rss) + vms = str(process.memory_info().vms) + username = process.username() + name = process.name() # Here is the process name + path = process.cwd() + cmdline = ' '.join(process.cmdline()) + + print("Get the process info using (path, name, cmdline): [%s / %s / %s]" % (path, name, cmdline)) + for _p in proc_defs: + # logger.info("Look for process: [%s] / Status [%s]" % (_p, status.lower())) + # print("Look for process: [%s] / Status [%s]" % (_p, status.lower())) + if (status.lower() != "sleeping") and \ + ('name' in _p and _p['name'] in name) and \ + ('cwd' in _p and _p['cwd'] in path) and \ + ('cmdline' in _p and _p['cmdline'] in cmdline): + return True except: import traceback tb = traceback.format_exc() - # print(tb) - continue + logger.debug(tb) + print(tb) return False - def run(self): - while True: - ResourceMonitor.lock.acquire() + def run(self): + while True: + ResourceMonitor.lock.acquire() ResourceMonitor.vmem_perc[1] = (ResourceMonitor.vmem_perc[0] + ResourceMonitor.vmem_perc[1]) / 2.0 ResourceMonitor.vmem_perc[0] = (ResourceMonitor.vmem_perc[1] + psutil.virtual_memory().percent) / 2.0 @@ -73,4 +84,3 @@ def run(self): ResourceMonitor.cpu_perc[0] = psutil.cpu_percent(interval = (ResourceMonitor.load_average_scan_minutes*60), percpu= False) ResourceMonitor.lock.release() - diff --git a/src/wpsremote/servicebot.py b/src/wpsremote/servicebot.py index b15006e..de29a91 100644 --- a/src/wpsremote/servicebot.py +++ b/src/wpsremote/servicebot.py @@ -61,7 +61,8 @@ def __init__(self, remote_config_filepath, service_config_filepath): self._max_running_time = datetime.timedelta( seconds = serviceConfig.getint("DEFAULT", "max_running_time_seconds") ) try: - self._process_blacklist = serviceConfig.get_list("DEFAULT", "process_blacklist") + import json + self._process_blacklist = json.loads(serviceConfig.get("DEFAULT", "process_blacklist")) except: self._process_blacklist = [] @@ -180,9 +181,13 @@ def handle_getloadavg(self, getloadavg_message): loadavg = self._resource_monitor.cpu_perc[0] vmem = self._resource_monitor.vmem_perc[0] + logger.info("Scanning Running Process. Declared Black List: %s" % self._process_blacklist) if self._resource_monitor.proc_is_running(self._process_blacklist) == True: + logger.info("A process listed in blacklist is running! Setting loadavg and vmem to (100.0, 100.0)") loadavg = 100.0 vmem = 100.0 + else: + logger.info("No blacklisted process was found. Setting loadavg and vmem to (%s, %s)" % (loadavg, vmem)) outputs = dict() outputs['loadavg'] = [loadavg, 'Average Load on CPUs during the last 15 minutes.'] diff --git a/src/wpsremote/xmpp_data/configs/myservice/service.config b/src/wpsremote/xmpp_data/configs/myservice/service.config index 2d7f3a2..abf96cb 100644 --- a/src/wpsremote/xmpp_data/configs/myservice/service.config +++ b/src/wpsremote/xmpp_data/configs/myservice/service.config @@ -32,7 +32,7 @@ load_average_scan_minutes = 1 # . 'processbot') whenever one of the following process names are running. # . In other words, if one of the following processes are currently running on this machine, # . GeoServer won't send any WPS execute request until they are finished. -process_blacklist = [resource consuming process name1, resource consuming process name2] +process_blacklist = [{"cwd": "/mnt/d/work/code/python/geonode/geonode-master", "name": "celery", "cmdline": "-A geonode.celery_app:app worker -Q default,geonode,cleanup,update,email -B -E -l INFO"}] # ########################################### # # Inputs and Actions Declaration #