From 27e12d20d48dda9e9c4106f1d9e0ecd659f4a842 Mon Sep 17 00:00:00 2001 From: adraxyz Date: Tue, 29 Oct 2019 18:47:23 +0100 Subject: [PATCH] [Fixes #30] ProcessBot and ServiceBot tests --- src/wpsremote/processbot.py | 1 + .../xmpp_data/test/test_remote.config | 2 +- .../xmpp_data/test/test_service.config | 8 +- test/continoustest.bat | 1 + test/test_bot.py | 160 ++++++++++++++++++ test/test_computation_job_inputs.py | 9 +- test/test_computation_job_outputs.py | 3 + test/test_config_parser.py | 12 +- test_suite.sh | 3 + 9 files changed, 187 insertions(+), 12 deletions(-) create mode 100644 test/test_bot.py diff --git a/src/wpsremote/processbot.py b/src/wpsremote/processbot.py index 3262bc3..54314ca 100644 --- a/src/wpsremote/processbot.py +++ b/src/wpsremote/processbot.py @@ -183,6 +183,7 @@ def SpawnProcess(self): self._input_parameters_defs.set_values(self._input_values) # execute actions associated to input parameters + print(self._input_parameters_defs.__dict__) self._input_params_actions.execute(self._input_parameters_defs) # prepare cmd line diff --git a/src/wpsremote/xmpp_data/test/test_remote.config b/src/wpsremote/xmpp_data/test/test_remote.config index 7ebd4fa..6c11d13 100644 --- a/src/wpsremote/xmpp_data/test/test_remote.config +++ b/src/wpsremote/xmpp_data/test/test_remote.config @@ -23,7 +23,7 @@ mucServicePassword = geoserver resource_file_dir = ./xmpp_data/resource_dir -# . Configure this option (along with 'backup_on_wps_execution_shared_dir' +# . Configure this option (along with 'backup_on_wps_execution_shared_dir' # . on single outputs of 'service.config') in order to make a copy # . of the results into a shared folder before sending messages to XMPP # . WARNING: this option takes precedence on "UPLOADER" option diff --git a/src/wpsremote/xmpp_data/test/test_service.config b/src/wpsremote/xmpp_data/test/test_service.config index 160f2f5..7b615dc 100644 --- a/src/wpsremote/xmpp_data/test/test_service.config +++ b/src/wpsremote/xmpp_data/test/test_service.config @@ -7,7 +7,7 @@ namespace = default description = foo service executable_path = \code\etl executable_cmd = python %(executable_path)s\etl.py -output_dir = \tmp\oaaOnDemand +output_dir = xmpp_data/test/tmp unique_execution_id = 123 workdir = %(output_dir)s\%(unique_execution_id)s active = True @@ -22,7 +22,7 @@ description = Start date of optmizier list = [item_0, item_1, item_2] int_list = [0, 1, 2, 3] float_list = [0.12, 1.6, 2.55, 3.4] -path_list = [%(workdir)s, \tmp\oaaOnDemand] +path_list = [%(workdir)s, xmpp_data/test/tmp] [Action1] class = cmdline @@ -70,7 +70,7 @@ class = createJSONfile input_ref = asset target_filepath = %(workdir)s\asset_${json_path_expr}.json json_path_expr = ['Asset']['id'] -json_schema = .\configs\OAAonDemand\asset_schema.json +json_schema = ./src/wpsremote/xmpp_data/test/asset_schema.json [Const1] @@ -130,7 +130,7 @@ value = %(unique_execution_id)s [Action10] class = updateINIfile input_ref = uniqueId -source_filepath = .\configs\OAAonDemand\geoserverCommands_template.properties +source_filepath = ./src/wpsremote/xmpp_data/test/geoserverCommands_template.properties target_filepath= %(workdir)s\geoserverCommands.properties section = DEFAULT alias = unique_id diff --git a/test/continoustest.bat b/test/continoustest.bat index 1161d73..1be3e41 100644 --- a/test/continoustest.bat +++ b/test/continoustest.bat @@ -30,6 +30,7 @@ python test_process_input_parameters.py python test_computation_job_outputs.py python test_config_parser.py python test_bus_independent_messages.py +python test_bot.py python resource_cleaner.py IF "%1"=="" GOTO exit diff --git a/test/test_bot.py b/test/test_bot.py new file mode 100644 index 0000000..8990fee --- /dev/null +++ b/test/test_bot.py @@ -0,0 +1,160 @@ +# (c) 2019 Open Source Geospatial Foundation - all rights reserved +# (c) 2014 - 2015 Centre for Maritime Research and Experimentation (CMRE) +# (c) 2013 - 2014 German Aerospace Center (DLR) +# This code is licensed under the GPL 2.0 license, available at the root +# application directory. + +import unittest +import os +import pickle +import mock +import datetime +import psutil +from wpsremote.processbot import ProcessBot +from wpsremote.servicebot import ServiceBot +from wpsremote.resource_cleaner import Resource +from wpsremote.resource_monitor import ResourceMonitor +import wpsremote.configInstance as configInstance +from wpsremote.xmppBus import XMPPBus +from wpsremote.computation_job_inputs import ComputationJobInputs +from wpsremote.computational_job_input_actions import ComputationalJobInputActions +from wpsremote.output_parameters import OutputParameters +from wpsremote.path import path + +__author__ = "Alessio Fabiani" +__copyright__ = "Copyright 2019 Open Source Geospatial Foundation - all rights reserved" +__license__ = "GPL" + +# due to xmppMessages chdir +os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + + +class TestBot(unittest.TestCase): + + def setUp(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + self.remote_config = configInstance.create( + "./src/wpsremote/xmpp_data/test/test_remote.config" + ) + self.xmpp_bus = XMPPBus(self.remote_config, "service_name", "service_name_namespace") + self.xep_0045_plugin = self.xmpp_bus.xmpp.plugin["xep_0045"] + self.from_obj_mock = mock.Mock() + rooms = { + self.xmpp_bus._get_MUC_JId(): { + "resource": { + "role": "not_visitor" + } + } + } + self.xep_0045_plugin.rooms = rooms + self.from_obj_mock.resource = "resource" + test_msg = pickle.dumps({ + "test_message": "test message" + }) + msg = { + "type": "normal", + "body": "topic=request&id=123&baseURL=test_base_url&message={}".format(test_msg), + "from": self.from_obj_mock + } + # ExecuteMessage + self.execute_message = self.xmpp_bus.CreateIndipendentMessage(msg) + # cleaner + self.cleaner = Resource() + + def test_process_bot(self): + process_bot = ProcessBot( + "./src/wpsremote/xmpp_data/test/test_remote.config", + "./src/wpsremote/xmpp_data/test/test_service.config", + self.execute_message + ) + self.assertTrue(process_bot._active) + self.assertFalse(process_bot._finished) + self.assertIsInstance(process_bot._input_parameters_defs, ComputationJobInputs) + self.assertIsInstance(process_bot._input_params_actions, ComputationalJobInputActions) + self.assertEqual(process_bot._input_values, {"test_message": "test message"}) + self.assertEqual(process_bot._max_running_time, datetime.timedelta(0, 300)) + self.assertEqual(process_bot.max_execution_time(), datetime.timedelta(0, 300)) + self.assertIsInstance(process_bot._output_parameters_defs, OutputParameters) + self.assertEqual(process_bot._remote_wps_baseurl, "test_base_url") + self.assertEqual(process_bot._resource_file_dir, path("./xmpp_data/resource_dir")) + self.assertEqual(process_bot.get_resource_file_dir(), path("./xmpp_data/resource_dir")) + self.assertEqual(len(process_bot._stdout_action), 6) + for stdout_action in process_bot._stdout_action: + self.assertIn(stdout_action, ['ignore', 'progress', 'log', 'log', 'log', 'abort']) + self.assertEqual(len(process_bot._stdout_parser), 6) + for stdout_parser in process_bot._stdout_parser: + self.assertIn(stdout_parser, [ + '.*\\[DEBUG\\](.*)', + '.*\\[INFO\\] ProgressInfo\\:([-+]?[0-9]*\\.?[0-9]*)\\%', + '.*\\[(INFO)\\](.*)', + '.*\\[(WARN)\\](.*)', + '.*\\[(ERROR)\\](.*)', + '.*\\[(CRITICAL)\\](.*)' + ]) + self.assertEqual(process_bot._uniqueExeId, "123") + self.assertEqual(process_bot._uploader, None) + self.assertEqual(process_bot._wps_execution_shared_dir, None) + self.assertEqual(process_bot.get_wps_execution_shared_dir(), None) + self.assertEqual(process_bot.description, "foo service") + self.assertEqual(process_bot.namespace, "default") + self.assertEqual(process_bot.service, "Service") + self.assertEqual( + process_bot.workdir(), + process_bot._output_dir + "/" + process_bot._uniqueExeId + ) + # handle_finish + finish_msg = { + "type": "normal", + "body": "topic=finish", + "from": self.from_obj_mock + } + finished_message = self.xmpp_bus.CreateIndipendentMessage(finish_msg) + with self.assertRaises(SystemExit) as cm_finish: + process_bot.handle_finish(finished_message) + self.assertEqual(cm_finish.exception.code, 0) + # handle_abort + abort_msg = { + "type": "normal", + "body": "topic=abort", + "from": self.from_obj_mock + } + abort_message = self.xmpp_bus.CreateIndipendentMessage(abort_msg) + with self.assertRaises(SystemExit) as cm_abort: + process_bot.handle_abort(abort_message) + self.assertEqual(cm_abort.exception.code, -1) + self.cleaner.kill_processbot() + self.assertFalse(psutil.pid_exists(self.cleaner._processbot_pid)) + + def test_service_bot(self): + service_bot = ServiceBot( + "./src/wpsremote/xmpp_data/test/test_remote.config", + "./src/wpsremote/xmpp_data/test/test_service.config" + ) + self.assertTrue(service_bot._active) + self.assertIsInstance(service_bot._input_parameters_defs, ComputationJobInputs) + self.assertEqual(service_bot._max_running_time, datetime.timedelta(0, 300)) + self.assertEqual(service_bot._output_dir, path('xmpp_data/test/tmp')) + self.assertIsInstance(service_bot._output_parameters_defs, OutputParameters) + self.assertFalse(service_bot._process_blacklist) + self.assertTrue(service_bot._redirect_process_stdout_to_logger) + self.assertEqual( + service_bot._remote_config_filepath, + './src/wpsremote/xmpp_data/test/test_remote.config' + ) + self.assertIsNone(service_bot._remote_wps_endpoint) + self.assertEqual(service_bot._resource_file_dir, path('./xmpp_data/resource_dir')) + self.assertIsInstance(service_bot._resource_monitor, ResourceMonitor) + self.assertEqual( + service_bot._service_config_file, + './src/wpsremote/xmpp_data/test/test_service.config' + ) + self.assertIsNone(service_bot._wps_execution_shared_dir) + self.assertIsNotNone(service_bot.bus) + self.assertEqual(service_bot.description, 'foo service') + self.assertEqual(service_bot.namespace, 'default') + self.assertEqual(service_bot.running_process, {}) + self.assertEqual(service_bot.service, 'Service') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_computation_job_inputs.py b/test/test_computation_job_inputs.py index bfd6969..e6962de 100644 --- a/test/test_computation_job_inputs.py +++ b/test/test_computation_job_inputs.py @@ -7,7 +7,7 @@ import unittest import json import wpsremote.ConfigParser as ConfigParser - +import os from wpsremote import path from wpsremote import computation_job_param from wpsremote import computation_job_inputs @@ -124,6 +124,7 @@ def test_cmd_line_action_2_values(self): self.assertEquals(actions.get_cmd_line(), "--mypar1=1 --mypar2=abc") def test_create_json_file_action(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) p1 = computation_job_param.ComputationJobParam("mypar1", "application/json", "par 1", "par descr 1") inputs = computation_job_inputs.ComputationJobInputs() inputs.add_input(p1) @@ -142,6 +143,7 @@ def test_create_json_file_action(self): self.assertTrue(a1.exists()) def test_create_2_json_file_action(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) p1 = computation_job_param.ComputationJobParam("mypar1", "application/json", "par 1", @@ -164,6 +166,7 @@ def test_create_2_json_file_action(self): self.assertTrue(a1.exists()) def test_update_json_file_action_with_int(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) target_json_file = path.path("./json_out.json") if target_json_file.exists(): target_json_file.remove() @@ -187,6 +190,7 @@ def test_update_json_file_action_with_int(self): self.assertTrue(100, j['Config']['nEvaluations']) def test_update_json_file_action_with_string(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) target_json_file = path.path("./json_out.json") if target_json_file.exists(): target_json_file.remove() @@ -432,6 +436,7 @@ def test_cmdpar_order_from_dict(self): self.assertEquals(actions.get_cmd_line(), '-w . --k=2.4') def test_update_json_list_in_file(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) target_json_file = path.path("./json_out.json") if target_json_file.exists(): target_json_file.remove() @@ -452,6 +457,7 @@ def test_update_json_list_in_file(self): self.assertTrue(45.5, j['Config']['latLim'][1]) def test_update_json_list_in_file_two_times(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) target_json_file = path.path("./json_out.json") if target_json_file.exists(): target_json_file.remove() @@ -478,6 +484,7 @@ def test_update_json_list_in_file_two_times(self): self.assertTrue(76.5, j['Config']['latLim'][1]) def test_copyfile(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) target_json_file = path.path("./copy_of_source_json.json") if target_json_file.exists(): target_json_file.remove() diff --git a/test/test_computation_job_outputs.py b/test/test_computation_job_outputs.py index af6c5f2..481be92 100644 --- a/test/test_computation_job_outputs.py +++ b/test/test_computation_job_outputs.py @@ -5,6 +5,7 @@ # application directory. import unittest +import os import wpsremote.ConfigParser as ConfigParser from wpsremote import mockutils from wpsremote.output_file_parameter import ( @@ -128,6 +129,7 @@ class TestComputationJobOutputs(unittest.TestCase): ] def test_string_output_type_from_ini(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) ini_text = '''[Output1] name = result1 type = string @@ -355,6 +357,7 @@ def test_application_owc_output_type_from_ini(self): self.assertEqual('', owcfp.get_value()) def test_string_output_type(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) ofp = OutputFileParameter( "result1", { diff --git a/test/test_config_parser.py b/test/test_config_parser.py index bf15c19..95f808f 100644 --- a/test/test_config_parser.py +++ b/test/test_config_parser.py @@ -20,7 +20,7 @@ class TestConfigParser(unittest.TestCase): DEFAULT_ITEMS = [ ('service', 'Service'), ('namespace', 'default'), ('description', 'foo service'), ('executable_path', '\\code\\etl'), ('executable_cmd', 'python %(executable_path)s\\etl.py'), - ('output_dir', '\\tmp\\oaaOnDemand'), ('unique_execution_id', '123'), + ('output_dir', 'xmpp_data/test/tmp'), ('unique_execution_id', '123'), ('workdir', '%(output_dir)s\\%(unique_execution_id)s'), ('active', 'True'), ('max_running_time_seconds', '300'), ('servicepassword', 'admin') ] @@ -95,11 +95,11 @@ def test_items(self): items_list = [ ('service', 'Service'), ('namespace', 'default'), ('description', 'foo service'), ('executable_path', '\\code\\etl'), ('executable_cmd', 'python \\code\\etl\\etl.py'), - ('output_dir', '\\tmp\\oaaOnDemand'), ('unique_execution_id', '123'), - ('workdir', '\\tmp\\oaaOnDemand\\123'), ('active', 'True'), ('max_running_time_seconds', '300'), + ('output_dir', 'xmpp_data/test/tmp'), ('unique_execution_id', '123'), + ('workdir', 'xmpp_data/test/tmp\\123'), ('active', 'True'), ('max_running_time_seconds', '300'), ('servicepassword', 'admin'), ('class', 'updateJSONfile'), ('input_ref', 'timeHorizon'), ('source_filepath', '.\\configs\\OAAonDemand\\CMREOAA_MainConfigFile_template.json'), - ('target_filepath', '\\tmp\\oaaOnDemand\\123\\config.json'), + ('target_filepath', 'xmpp_data/test/tmp\\123\\config.json'), ('json_path_expr', "['Config']['timeHorizon']") ] cp = configInstance.create("./src/wpsremote/xmpp_data/test/test_service.config") @@ -171,7 +171,7 @@ def test_get_list_float_impl(self): self.assertIn(i, float_list) def test_get_list_path_impl(self): - path_list = ['\\tmp\\oaaOnDemand\\123', '\\tmp\\oaaOnDemand'] + path_list = ['xmpp_data/test/tmp\\123', 'xmpp_data/test/tmp'] cp = configInstance.create("./src/wpsremote/xmpp_data/test/test_service.config") items = configInstance.get_list_path_impl(cp, "Input1", "path_list") for i in items: @@ -187,7 +187,7 @@ def test_get_path(self): cp = configInstance.create("./src/wpsremote/xmpp_data/test/test_service.config") item = configInstance.get_path(cp, "DEFAULT", "output_dir") self.assertIsInstance(item, path.path) - self.assertEqual("\\tmp\\oaaOnDemand", item) + self.assertEqual("xmpp_data/test/tmp", item) if __name__ == '__main__': diff --git a/test_suite.sh b/test_suite.sh index 5a8213d..c154cf6 100644 --- a/test_suite.sh +++ b/test_suite.sh @@ -16,3 +16,6 @@ coverage run --source=src test/test_config_parser.py echo "Running... test_bus_independent_messages" coverage run --source=src test/test_bus_independent_messages.py +echo "Running... test_bot" +coverage run --source=src test/test_bot.py +