From ccc6bb2fec95e5fb9c80230e4a044643baebc964 Mon Sep 17 00:00:00 2001 From: leo-desbureaux-tellae <73165148+leo-desbureaux-tellae@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:21:16 +0200 Subject: [PATCH] feat!: scenario path management (#61) * new SimulationScenario class replacing SimulationParameters, manages scenario folders and data * changed signature of functions to accept a scenario folder path. Scenario path is no longer computed from the model code and scenario name * BREAKING CHANGE: now provide path to the scenario folder instead of path to the simulation parameters to run a simulation --- README.md | 4 +- docs/run/examples.rst | 6 +- docs/run/usage.rst | 8 +- .../basemodel/agent/operators/operator.py | 2 +- .../basemodel/algorithms/pal_zhang_GCH.py | 4 +- .../basemodel/environment/environment.py | 8 +- starling_sim/basemodel/input/dynamic_input.py | 19 +- .../basemodel/output/feature_factory.py | 2 +- .../basemodel/output/output_factory.py | 21 +- starling_sim/basemodel/parameters/__init__.py | 3 - .../parameters/simulation_parameters.py | 184 ------------------ starling_sim/basemodel/simulation_model.py | 18 +- starling_sim/model_simulator.py | 30 +-- starling_sim/models/FF_VS/model.py | 8 +- starling_sim/models/SB_VS/model.py | 8 +- starling_sim/models/SB_VS_R/model.py | 8 +- starling_sim/run.py | 4 +- starling_sim/simulation_scenario.py | 135 +++++++++++++ starling_sim/utils/paths.py | 77 +++----- starling_sim/utils/simulation_logging.py | 2 +- starling_sim/utils/test_models.py | 32 +-- 21 files changed, 252 insertions(+), 331 deletions(-) delete mode 100644 starling_sim/basemodel/parameters/__init__.py delete mode 100644 starling_sim/basemodel/parameters/simulation_parameters.py create mode 100644 starling_sim/simulation_scenario.py diff --git a/README.md b/README.md index 5c6d7d5..c140f1a 100644 --- a/README.md +++ b/README.md @@ -102,12 +102,12 @@ python3 main.py -e ### Usage Once the data is prepared, a scenario can be run from the project -root by running main.py with the path to the scenario parameters. +root by running main.py with the path to the scenario folder. Run one of the example scenarios, for instance: ```bash -python3 main.py data/models/SB_VS/example_nantes/inputs/Params.json +python3 main.py data/models/SB_VS/example_nantes/ ``` You will see the progression of the simulation with the logs that diff --git a/docs/run/examples.rst b/docs/run/examples.rst index 2eb2240..e027127 100644 --- a/docs/run/examples.rst +++ b/docs/run/examples.rst @@ -37,11 +37,11 @@ Run *** Example scenarios can be run like any other scenario by running main.py -with the path to the parameters file. For instance: +with the path to the scenario folder. For instance: .. code-block:: bash - python3 main.py data/models/SB_VS/example_nantes/inputs/Params.json + python3 main.py data/models/SB_VS/example_nantes/ You will then see the simulation logs display in the console, until the run finishes. If you find the logs too verbose, you can set the logging level to a higher level @@ -49,7 +49,7 @@ with the ``-l`` option (see simulation_logging.py for more information): .. code-block:: bash - python3 main.py -l 20 data/models/SB_VS/example_nantes/inputs/Params.json + python3 main.py -l 20 data/models/SB_VS/example_nantes/ ******* Outputs diff --git a/docs/run/usage.rst b/docs/run/usage.rst index 08965ac..b77390d 100644 --- a/docs/run/usage.rst +++ b/docs/run/usage.rst @@ -6,7 +6,7 @@ Usage Data preparation **************** -Simulation scenarios are launched from a file that contains +Simulation scenarios are launched from a parameter file that contains global parameters of the simulation. Simulation data must be placed in data/models///inputs @@ -22,11 +22,11 @@ Once you have setup your execution environment (either on your device or with Do and your simulation data (see :ref:`inout`), you can run your scenario. Simulations are run by executing the script main.py from the project root -with a path to the parameters file of the scenario. For instance: +with the path to your scenario folder. For instance: .. code-block:: bash - python3 main.py data/models/SB_VS/my_scenario/inputs/Params.json + python3 main.py data/models/SB_VS/my_scenario/ For more information about the options of main.py use the ``-h`` (or ``--help``) option: @@ -43,7 +43,7 @@ With Docker, this can be done in detached mode with .. code-block:: bash docker run -d -v "$(pwd)":/starling_dir/ --name container_name starling\ - bash -c "python3 main.py data/models/SB_VS/my_scenario/inputs/Params.json" + bash -c "python3 main.py data/models/SB_VS/my_scenario/" or in interactive mode with diff --git a/starling_sim/basemodel/agent/operators/operator.py b/starling_sim/basemodel/agent/operators/operator.py index 92df045..acc0389 100644 --- a/starling_sim/basemodel/agent/operators/operator.py +++ b/starling_sim/basemodel/agent/operators/operator.py @@ -326,7 +326,7 @@ def init_zone(self, zone_polygon): if isinstance(zone_polygon, str): filepath = scenario_agent_input_filepath( - self.sim.parameters["code"], self.sim.parameters["scenario"], zone_polygon + self.sim.scenario.scenario_folder, zone_polygon ) geojson = json_load(filepath) service_zone = geopandas_polygon_from_points( diff --git a/starling_sim/basemodel/algorithms/pal_zhang_GCH.py b/starling_sim/basemodel/algorithms/pal_zhang_GCH.py index c5b917f..6c45edb 100644 --- a/starling_sim/basemodel/algorithms/pal_zhang_GCH.py +++ b/starling_sim/basemodel/algorithms/pal_zhang_GCH.py @@ -276,7 +276,7 @@ def compute_station_variation(self, station_id): index = self.start_times.index(self.current_time) if index == len(self.start_times) - 1: - horizon = self.sim.parameters["limit"] + horizon = self.sim.scenario["limit"] else: horizon = self.start_times[index + 1] @@ -366,7 +366,7 @@ def append_depot_to_planning(self): def init_demand_dict(self): features = self.sim.dynamicInput.feature_list_from_file( - self.sim.parameters["dynamic_input_file"] + self.sim.scenario["dynamic_input_file"] ) self.sim.dynamicInput.pre_process_position_coordinates(features) demand_dict = {station: [] for station in self.operator.stations.keys()} diff --git a/starling_sim/basemodel/environment/environment.py b/starling_sim/basemodel/environment/environment.py index c5a3de7..2443fc6 100644 --- a/starling_sim/basemodel/environment/environment.py +++ b/starling_sim/basemodel/environment/environment.py @@ -10,7 +10,7 @@ class Environment: Describes an environment in which the simulation will take place """ - def __init__(self, parameters, network="osm"): + def __init__(self, scenario, network="osm"): """ Initialization of the environment, without import of data structures. @@ -21,11 +21,11 @@ def __init__(self, parameters, network="osm"): self.topologies = {} # get the topologies dict - topologies_dict = parameters["topologies"] + topologies_dict = scenario["topologies"] # get the 'store_paths' parameter - if "store_paths" in parameters: - store_paths = parameters["store_paths"] + if "store_paths" in scenario: + store_paths = scenario["store_paths"] else: store_paths = False diff --git a/starling_sim/basemodel/input/dynamic_input.py b/starling_sim/basemodel/input/dynamic_input.py index 63e96b2..2c1bc9d 100644 --- a/starling_sim/basemodel/input/dynamic_input.py +++ b/starling_sim/basemodel/input/dynamic_input.py @@ -55,7 +55,7 @@ def setup(self, simulation_model): # set the attribute of dynamic features self.dynamic_feature_list = self.feature_list_from_file( - self.sim.parameters["dynamic_input_file"] + self.sim.scenario["dynamic_input_file"] ) # sort list according to origin times @@ -64,7 +64,7 @@ def setup(self, simulation_model): ) # get the list of static features (present at the start of the simulation) - init_files = self.sim.parameters["init_input_file"] + init_files = self.sim.scenario["init_input_file"] # if there are several files, concatenate their feature lists if isinstance(init_files, list): @@ -135,10 +135,10 @@ def play_dynamic_input_(self): # see if an offset should be applied to the input origin time if ( - "early_dynamic_input" in self.sim.parameters - and self.sim.parameters["early_dynamic_input"] + "early_dynamic_input" in self.sim.scenario + and self.sim.scenario["early_dynamic_input"] ): - early_input_time_offset = self.sim.parameters["early_dynamic_input"] + early_input_time_offset = self.sim.scenario["early_dynamic_input"] else: early_input_time_offset = 0 @@ -283,10 +283,7 @@ def feature_list_from_file(self, filename): return [] # get the path to the input file - parameters = self.sim.parameters - filepath = scenario_agent_input_filepath( - parameters["code"], parameters["scenario"], filename - ) + filepath = scenario_agent_input_filepath(self.sim.scenario.scenario_folder, filename) # read the dict contained in input file try: @@ -315,7 +312,7 @@ def feature_list_from_file(self, filename): def make_demand_static(self): - if "make_static" in self.sim.parameters and self.sim.parameters["make_static"] in [ + if "make_static" in self.sim.scenario and self.sim.scenario["make_static"] in [ "all", "prebooked", "prebooked_only", @@ -325,7 +322,7 @@ def make_demand_static(self): # also add the agents of dynamic input that are prebooked dynamic_features = [] - make_static = self.sim.parameters["make_static"] + make_static = self.sim.scenario["make_static"] for feature in self.dynamic_feature_list: properties = feature["properties"] diff --git a/starling_sim/basemodel/output/feature_factory.py b/starling_sim/basemodel/output/feature_factory.py index 9a00b45..90b8cc4 100644 --- a/starling_sim/basemodel/output/feature_factory.py +++ b/starling_sim/basemodel/output/feature_factory.py @@ -109,7 +109,7 @@ def get_element_line_string(geojson_output, element): # get the list of route localisations and timestamps route_positions, route_timestamps = route_localisations( - event, geojson_output.sim.parameters["limit"], geojson_output.graphs[mode] + event, geojson_output.sim.scenario["limit"], geojson_output.graphs[mode] ) # add it to the agent's lists diff --git a/starling_sim/basemodel/output/output_factory.py b/starling_sim/basemodel/output/output_factory.py index b603f46..eb2d57a 100644 --- a/starling_sim/basemodel/output/output_factory.py +++ b/starling_sim/basemodel/output/output_factory.py @@ -46,18 +46,15 @@ def setup(self, simulation_model): self.sim = simulation_model # get output folder - output_folder = simulation_model.parameters["output_folder"] + output_folder = simulation_model.scenario.outputs_folder # get scenario - scenario = simulation_model.parameters["scenario"] + scenario = simulation_model.scenario.name # setup kpi outputs self.setup_kpi_output() - if not os.path.exists(output_folder): - os.mkdir(output_folder) - for kpi_output in self.kpi_outputs: # build the kpi output filename @@ -139,19 +136,19 @@ def extract_simulation(self, simulation_model): """ # traces output - if simulation_model.parameters["traces_output"]: + if simulation_model.scenario["traces_output"]: try: self.generate_trace_output(simulation_model) except: logging.warning(self.GENERATION_ERROR_FORMAT.format("traces")) # kpi output - if simulation_model.parameters["kpi_output"]: + if simulation_model.scenario["kpi_output"]: self.generate_kpi_output(simulation_model) # geojson output - if simulation_model.parameters["visualisation_output"]: + if simulation_model.scenario["visualisation_output"]: try: self.generate_geojson_output(simulation_model) except: @@ -192,9 +189,9 @@ def generate_trace_output(self, simulation_model): """ # get the scenario information and outfile - scenario = simulation_model.parameters["scenario"] - model_code = simulation_model.parameters["code"] - output_folder = simulation_model.parameters["output_folder"] + scenario = simulation_model.scenario.name + model_code = simulation_model.scenario.model + output_folder = simulation_model.scenario.outputs_folder filepath = output_folder + config["traces_format"].format(scenario=scenario) # open the trace file in write mode @@ -229,7 +226,7 @@ def generate_run_summary(self, simulation_model): :param simulation_model: """ - filepath = simulation_model.parameters["output_folder"] + RUN_SUMMARY_FILENAME + filepath = simulation_model.scenario.outputs_folder + RUN_SUMMARY_FILENAME # add run summary to output files self.sim.outputFactory.new_output_file(filepath, "application/json", content="run_summary") diff --git a/starling_sim/basemodel/parameters/__init__.py b/starling_sim/basemodel/parameters/__init__.py deleted file mode 100644 index e6f5232..0000000 --- a/starling_sim/basemodel/parameters/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -This package contains the modules related to the parameter's management -""" diff --git a/starling_sim/basemodel/parameters/simulation_parameters.py b/starling_sim/basemodel/parameters/simulation_parameters.py deleted file mode 100644 index 014a546..0000000 --- a/starling_sim/basemodel/parameters/simulation_parameters.py +++ /dev/null @@ -1,184 +0,0 @@ -""" -This module manages the parameters of the simulation -""" - -import logging -import os -from copy import deepcopy - -from starling_sim.utils.utils import json_load, validate_against_schema -from starling_sim.utils import paths - - -class SimulationParameters: - """ - This class stores the simulation parameters in a dict structure - - It must be extended to build the dict from different sources - - The parameters should be set and accessed using the builtin methods of a dict - """ - - BASE_PARAM_SCHEMA = "parameters.schema.json" - - def __init__(self, parameters, param_schema_path=None): - """ - This constructor must be extended to initialize the dict from specific arguments - """ - - # get JSON schema - if param_schema_path is None: - param_schema_path = paths.schemas_folder() + self.BASE_PARAM_SCHEMA - - self.schema = json_load(param_schema_path) - - # parameters validation - validate_against_schema(parameters, self.schema) - - # change date format from YYYY-MM-DD to YYYYMMDD - if "date" in parameters: - parameters["date"] = parameters["date"].replace("-", "") - - self._parametersDict = parameters - - def __getitem__(self, item): - """ - Method called when using 'simulationParameters[item]' - :param item: Name of the parameter accessed - :return: self._parametersDict[item] - """ - - if item not in self._parametersDict: - logging.error( - "Trying to access unknown parameter '" - + str(item) - + "'. Does it appears in the parameter entry ?" - ) - raise KeyError("The parameter '" + str(item) + "' does not exist") - - return self._parametersDict[item] - - def __setitem__(self, key, value): - """ - Method called when using 'simulationParameters[key] = value' - :param key: Name of the parameter - :param value: Value associated to the parameter - :return: - """ - - if key in self._parametersDict: - logging.warning("Replacing already existing parameter '" + str(key) + "'") - - self._parametersDict[key] = value - - def __contains__(self, item): - """ - Method called when using 'item in simulationParameters' - - :param item: - :return: True if item is in _parametersDict, False otherwise - """ - - return item in self._parametersDict - - def keys(self): - """ - Returns the keys of the _parametersDict attribute - - :return: keys of the _parametersDict attribute - """ - - return self._parametersDict.keys() - - def set_dict(self, params_dict): - """ - Sets the _parametersDict attribute with the given dict - - :param params_dict: - :return: - """ - - self._parametersDict = params_dict - - def copy_dict(self): - """ - Return a deepcopy of the simulation parameters. - - :return: deepcopy of simulation parameters - """ - - return deepcopy(self._parametersDict) - - -class ParametersFromDict(SimulationParameters): - """ - Get simulation parameters from an existing dict - """ - - def __init__(self, params_dict): - """ - Just set the _parametersDict - :param params_dict: - """ - - super().__init__(params_dict) - - -class ParametersFromFile(SimulationParameters): - """ - Get simulation parameters from a json file - """ - - def __init__(self, filepath): - """ - Initialisation of the param dict by reading params in a json file - :param filepath: - """ - - # check that the parameters file has the correct name - if os.path.basename(filepath) != paths.PARAMETERS_FILENAME: - raise ValueError("The parameters file must be named " + paths.PARAMETERS_FILENAME) - - # read json - json_param = json_load(filepath) - - # create empty dict - super().__init__(json_param) - - -# parameters management - - -def parameters_from_file(param_path): - """ - Generates a simulation parameters from the parameters file. - - :param param_path: path to parameters file - - :return: SimulationParameters object - """ - - # create simulation parameters - parameters = ParametersFromFile(param_path) - - add_paths_to_parameters(parameters) - - return parameters - - -def add_paths_to_parameters(parameters): - """ - Adds the input and output paths to the given parameters - using the 'code' and 'scenario' items - :param parameters: - :return: - """ - - model_code = parameters["code"] - scenario = parameters["scenario"] - - # input folder - parameters["input_folder"] = paths.scenario_input_folder(model_code, scenario) - - # output folder (will be generated by output) - parameters["output_folder"] = paths.scenario_output_folder(model_code, scenario) diff --git a/starling_sim/basemodel/simulation_model.py b/starling_sim/basemodel/simulation_model.py index 36ee8ab..d0b36cd 100644 --- a/starling_sim/basemodel/simulation_model.py +++ b/starling_sim/basemodel/simulation_model.py @@ -30,18 +30,18 @@ class SimulationModel: #: Agent types of the model and their modes modes = None - def __init__(self, parameters): + def __init__(self, scenario): """ Initialisation of the simulation model with instances of its different elements This constructor must be extended to instantiate the model elements using the correct classes - :param parameters: SimulationParameters object + :param scenario: SimulationScenario object """ # simulation parameters - self.parameters = parameters + self.scenario = scenario # run_summary self.runSummary = self.init_run_summary() @@ -50,7 +50,7 @@ def __init__(self, parameters): self.add_base_leaving_codes() # random seed for the simulation setup and run - self.randomSeed = parameters["seed"] + self.randomSeed = scenario["seed"] # information to be completed for the specific models @@ -87,7 +87,7 @@ def setup(self): logging.info("Simulation environment setup") self.environment.setup(self) - if "gtfs_timetables" in self.parameters: + if "gtfs_timetables" in self.scenario: logging.info("GTFS tables setup") self.setup_gtfs() @@ -105,14 +105,14 @@ def run(self): """ # if asked, add a process that logs the simulation time every hour - if "time_log" in self.parameters and self.parameters["time_log"]: + if "time_log" in self.scenario and self.scenario["time_log"]: self.scheduler.new_process(self.periodic_hour_log()) # create agents and add their loops self.scheduler.new_process(self.dynamicInput.play_dynamic_input_()) # run the simulation - self.scheduler.run(self.parameters["limit"]) + self.scheduler.run(self.scenario["limit"]) # trace the end of the simulation for all agents trace_simulation_end(self) @@ -156,7 +156,7 @@ def setup_gtfs(self): # import the gtfs timetable from the zip given in the parameters restrict_transfers = config["transfer_restriction"] - self.gtfs = import_gtfs_feed(self.parameters["gtfs_timetables"], restrict_transfers) + self.gtfs = import_gtfs_feed(self.scenario["gtfs_timetables"], restrict_transfers) def init_run_summary(self): """ @@ -175,7 +175,7 @@ def init_run_summary(self): summary["commit"] = get_git_revision_hash() # copy scenario parameters - summary["parameters"] = self.parameters.copy_dict() + summary["parameters"] = self.scenario.copy_parameters() # copy config summary["config"] = config.copy() diff --git a/starling_sim/model_simulator.py b/starling_sim/model_simulator.py index f3f5af0..16d103b 100644 --- a/starling_sim/model_simulator.py +++ b/starling_sim/model_simulator.py @@ -1,7 +1,7 @@ from starling_sim.models.SB_VS.model import Model as SB_VS_model from starling_sim.models.SB_VS_R.model import Model as SB_VS_R_model from starling_sim.models.FF_VS.model import Model as FF_VS_model -from starling_sim.basemodel.parameters.simulation_parameters import parameters_from_file +from starling_sim.simulation_scenario import SimulationScenario from starling_sim.utils.paths import model_import_path import logging @@ -46,23 +46,23 @@ def generate_output(self): self.simulationModel.generate_output() - def init_simulator_from_parameters(simulation_parameters, pkg): + def init_simulator_from_parameters(simulation_scenario, pkg): """ Returns a simulator initialised according to the model code contained in the given parameters - :param simulation_parameters: SimulationParameters containing "code" + :param simulation_scenario: SimulationParameters containing "code" :param pkg: name of the source package :return: ModelSimulator object """ # get the Model class - model_class = ModelSimulator.get_model_class(simulation_parameters["code"], pkg) + model_class = ModelSimulator.get_model_class(simulation_scenario["code"], pkg) # create a new instance of the simulation model try: - simulation_model = model_class(simulation_parameters) + simulation_model = model_class(simulation_scenario) except TypeError as e: logging.error( "Instantiation of {} failed with message :\n {}".format(model_class.__name__, e) @@ -117,30 +117,32 @@ def get_model_class(model_code, pkg): get_model_class = staticmethod(get_model_class) -def launch_simulation(parameters_path, pkg): +def launch_simulation(scenario_path, pkg): """ Realises the initialisation, setup, run and output of the simulation using the given parameters file. Displays logs of execution times - :param parameters_path: path to parameters files + :param scenario_path: path to scenario folder :param pkg: name of the source package """ - if parameters_path is None: - raise ValueError("No parameters file provided to simulation launcher.") + if scenario_path is None: + raise ValueError("No scenario folder provided to simulation launcher.") - # initialise simulation parameters from parameters file - simulation_parameters = parameters_from_file(parameters_path) + # initialise simulation scenario from folder path + simulation_scenario = SimulationScenario(scenario_path) + # read simulation parameters + simulation_scenario.get_scenario_parameters() # init the simulator logging.info( "Initializing simulator for the model code " - + simulation_parameters["code"] + + simulation_scenario["code"] + ", scenario " - + simulation_parameters["scenario"] + + simulation_scenario["scenario"] + "\n" ) - simulator = ModelSimulator.init_simulator_from_parameters(simulation_parameters, pkg) + simulator = ModelSimulator.init_simulator_from_parameters(simulation_scenario, pkg) # setup the simulator logging.info("Setting entries for: " + simulator.simulationModel.name) diff --git a/starling_sim/models/FF_VS/model.py b/starling_sim/models/FF_VS/model.py index 3c92607..5730ad1 100644 --- a/starling_sim/models/FF_VS/model.py +++ b/starling_sim/models/FF_VS/model.py @@ -21,18 +21,18 @@ class Model(SimulationModel): modes = {"user": ["walk"], "vehicle": [None, "walk"]} - def __init__(self, parameters): + def __init__(self, scenario): """ Initialisation of the classes used by the free-floating model - :param parameters: SimulationParameters object + :param scenario: SimulationScenario object """ - super().__init__(parameters) + super().__init__(scenario) # elements of the model self.agentPopulation = DictPopulation(self.agent_type_class.keys()) - self.environment = Environment(self.parameters) + self.environment = Environment(self.scenario) # inputs and outputs self.dynamicInput = Input(self.agent_type_class) diff --git a/starling_sim/models/SB_VS/model.py b/starling_sim/models/SB_VS/model.py index 7a82421..16dc1b0 100644 --- a/starling_sim/models/SB_VS/model.py +++ b/starling_sim/models/SB_VS/model.py @@ -26,18 +26,18 @@ class Model(SimulationModel): modes = {"user": ["walk"], "vehicle": ["station"], "station": [None, "walk"]} - def __init__(self, parameters): + def __init__(self, scenario): """ Initialisation of the classes used by the station-based model - :param parameters: SimulationParameters object + :param scenario: SimulationScenario object """ - super().__init__(parameters) + super().__init__(scenario) # elements of the model self.agentPopulation = DictPopulation(self.agent_type_class.keys()) - self.environment = Environment(self.parameters) + self.environment = Environment(self.scenario) # inputs and outputs self.dynamicInput = Input(self.agent_type_class) diff --git a/starling_sim/models/SB_VS_R/model.py b/starling_sim/models/SB_VS_R/model.py index 33d980b..f4ae735 100644 --- a/starling_sim/models/SB_VS_R/model.py +++ b/starling_sim/models/SB_VS_R/model.py @@ -35,18 +35,18 @@ class Model(SimulationModel): "operator": {"staff": "staff", "fleet": "station"}, } - def __init__(self, parameters): + def __init__(self, scenario): """ Initialisation of the classes used by the station-based with repositioning model - :param parameters: SimulationParameters object + :param scenario: SimulationScenario object """ - super().__init__(parameters) + super().__init__(scenario) # elements of the model self.agentPopulation = DictPopulation(self.agent_type_class.keys()) - self.environment = Environment(self.parameters) + self.environment = Environment(self.scenario) # inputs and outputs self.dynamicInput = Input(self.agent_type_class) diff --git a/starling_sim/run.py b/starling_sim/run.py index 1e27584..aa78518 100644 --- a/starling_sim/run.py +++ b/starling_sim/run.py @@ -17,7 +17,7 @@ def run_main(): parser = argparse.ArgumentParser(description="Starling agent-based simulation framework") - parser.add_argument("param_path", help="path to parameter file.", nargs="?") + parser.add_argument("scenario_path", help="path to scenario folder.", nargs="?") parser.add_argument( "-l", @@ -132,4 +132,4 @@ def run_main(): # launch simulation logging.info("Launching Starling {}\n".format(__version__)) - launch_simulation(input_args.param_path, input_args.package) + launch_simulation(input_args.scenario_path, input_args.package) diff --git a/starling_sim/simulation_scenario.py b/starling_sim/simulation_scenario.py new file mode 100644 index 0000000..9780276 --- /dev/null +++ b/starling_sim/simulation_scenario.py @@ -0,0 +1,135 @@ +""" +This module manages the parameters of the simulation +""" + +import logging +import os +import json +from copy import deepcopy + +from starling_sim.utils.utils import json_load, validate_against_schema +from starling_sim.utils import paths + + +class SimulationScenario: + """ + This class describes a simulation scenario folders and data. + """ + + BASE_PARAM_SCHEMA = "parameters.schema.json" + + def __init__(self, scenario_folder_path: str): + + # scenario folder path + self.scenario_folder = None + + # scenario inputs folder path + self.inputs_folder = None + + # scenario outputs folder path + self.outputs_folder = None + + # simulation parameters (can be accessed using SimulationScenario["key"]) + self.parameters = None + + # scenario model code + self.model = None + + # scenario name + self.name = None + + # set the scenario folders + self._set_scenario_folders(scenario_folder_path) + + def _set_scenario_folders(self, scenario_folder_path: str): + """ + Set the folder attributes from the scenario folder path. + + Paths are built according to the structure enforced by the functions of starling_sim.utils.paths. + + :param scenario_folder_path: path to the scenario folder + """ + + # check scenario folder + self.scenario_folder = os.path.join(scenario_folder_path, "") + if not os.path.exists(self.scenario_folder): + raise ValueError("The scenario folder does not exist : {}".format(self.scenario_folder)) + + # check inputs folder + self.inputs_folder = paths.scenario_inputs_folder(self.scenario_folder) + if not os.path.exists(self.inputs_folder): + raise ValueError("The inputs folder does not exist : {}".format(self.inputs_folder)) + + # create outputs folder if it does not exist + if "OUTPUT_FOLDER" in os.environ: + output_folder = os.path.join(os.environ["OUTPUT_FOLDER"], "") + else: + output_folder = paths.scenario_outputs_folder(self.scenario_folder) + self.outputs_folder = output_folder + if not os.path.exists(self.outputs_folder): + os.mkdir(self.outputs_folder) + + def get_scenario_parameters(self): + """ + Get and validate the scenario simulation parameters. + + Also get the scenario model and name from the parameters. + + The parameter file path is enforced by the scenario_parameters_filepath function. + """ + + try: + parameters_path = paths.scenario_parameters_filepath(self.scenario_folder) + self.parameters = json_load(parameters_path) + + except (FileNotFoundError, json.JSONDecodeError) as e: + raise ValueError("Error while loading the scenario parameters : {}".format(str(e))) + + # parameters validation + schema = json_load(paths.schemas_folder() + self.BASE_PARAM_SCHEMA) + validate_against_schema(self.parameters, schema) + + # change date format from YYYY-MM-DD to YYYYMMDD + if "date" in self.parameters: + self.parameters["date"] = self.parameters["date"].replace("-", "") + + # get model and scenario + self.model = self.parameters["code"] + self.name = self.parameters["scenario"] + + def __getitem__(self, item): + """ + Method called when using 'SimulationScenario[item]' + + :param item: Name of the parameter accessed + :return: self.parameters[item] + """ + + if item not in self.parameters: + logging.error( + "Trying to access unknown parameter '" + + str(item) + + "'. Does it appears in the parameter entry ?" + ) + raise KeyError("The parameter '" + str(item) + "' does not exist") + + return self.parameters[item] + + def __contains__(self, item): + """ + Method called when using 'item in simulationParameters' + + :param item: + :return: True if item is in self.parameters, False otherwise + """ + + return item in self.parameters + + def copy_parameters(self): + """ + Return a deepcopy of the simulation parameters. + + :return: deepcopy of simulation parameters + """ + + return deepcopy(self.parameters) diff --git a/starling_sim/utils/paths.py b/starling_sim/utils/paths.py index c7c3bdb..1a5b7d3 100644 --- a/starling_sim/utils/paths.py +++ b/starling_sim/utils/paths.py @@ -70,7 +70,7 @@ .. code-block:: bash - python3 main.py path/to/parameters.json --data-folder path/to/data_folder + python3 main.py path/to/scenario/ --data-folder path/to/data_folder/ However, the structure of the data repository must remain the same. This is ensured by the functions declared in starling_sim.utils.paths.py, that build the paths to the different folders by concatenating @@ -104,8 +104,6 @@ import starling_sim import os -_SEP = "/" - #: path to the data folder _DATA_FOLDER = "./data/" @@ -154,42 +152,42 @@ def common_inputs_folder(): """ Path to the common inputs folder """ - return data_folder() + COMMON_INPUTS_FOLDER + _SEP + return os.path.join(data_folder(), COMMON_INPUTS_FOLDER, "") def environment_folder(): """ Path to the environment folder. """ - return data_folder() + ENVIRONMENT_FOLDER_NAME + _SEP + return os.path.join(data_folder(), ENVIRONMENT_FOLDER_NAME, "") def osm_graphs_folder(): """ Path to the osm graphs folder. """ - return environment_folder() + OSM_GRAPHS_FOLDER_NAME + _SEP + return os.path.join(environment_folder(), OSM_GRAPHS_FOLDER_NAME, "") def graph_speeds_folder(): """ Path to the graph speeds folder. """ - return environment_folder() + GRAPH_SPEEDS_FOLDER_NAME + _SEP + return os.path.join(environment_folder(), GRAPH_SPEEDS_FOLDER_NAME, "") def gtfs_feeds_folder(): """ Path to the gtfs feeds folder. """ - return environment_folder() + GTFS_FEEDS_FOLDER_NAME + _SEP + return os.path.join(environment_folder(), GTFS_FEEDS_FOLDER_NAME, "") def models_folder(): """ Path to the models folder. """ - return data_folder() + MODELS_FOLDER_NAME + _SEP + return os.path.join(data_folder(), MODELS_FOLDER_NAME, "") def model_folder(model_code): @@ -198,40 +196,28 @@ def model_folder(model_code): :param model_code: code of the model """ - return models_folder() + model_code + _SEP - - -def scenario_folder(model_code, scenario): - """ - Path to the folder of the given scenario. - - :param model_code: code of the model - :param scenario: name of the scenario - """ - return model_folder(model_code) + scenario + _SEP + return os.path.join(models_folder(), model_code, "") -def scenario_input_folder(model_code, scenario): +def scenario_inputs_folder(scenario_folder): """ Path to the input folder of the given scenario. - :param model_code: code of the model - :param scenario: name of the scenario + :param scenario_folder: scenario folder path """ - return scenario_folder(model_code, scenario) + INPUT_FOLDER_NAME + _SEP + return os.path.join(scenario_folder, INPUT_FOLDER_NAME, "") -def scenario_parameters_filepath(model_code, scenario): +def scenario_parameters_filepath(scenario_folder): """ Path to the parameters file of the given scenario. - :param model_code: code of the model - :param scenario: name of the scenario + :param scenario_folder: scenario folder path """ - return scenario_input_folder(model_code, scenario) + PARAMETERS_FILENAME + return os.path.join(scenario_inputs_folder(scenario_folder), PARAMETERS_FILENAME) -def scenario_agent_input_filepath(model_code, scenario, filename): +def scenario_agent_input_filepath(scenario_folder, filename): """ Get the path to the scenario input file (dynamic or initialisation file). @@ -239,17 +225,16 @@ def scenario_agent_input_filepath(model_code, scenario, filename): look in the common inputs folder. If the file is not there, raise a FileNotFoundError. - :param model_code: code of the model - :param scenario: name of the scenario + :param scenario_folder: scenario folder path :param filename: name of the input file """ # complete the file path with the input folder path - filepath = scenario_input_folder(model_code, scenario) + filename + filepath = os.path.join(scenario_inputs_folder(scenario_folder), filename) # if the file does not exist, look in the common inputs folder if not os.path.exists(filepath): - filepath = common_inputs_folder() + filename + filepath = os.path.join(common_inputs_folder(), filename) if not os.path.exists(filepath): raise FileNotFoundError( "Input file {} not found in scenario inputs folder " @@ -259,42 +244,28 @@ def scenario_agent_input_filepath(model_code, scenario, filename): return filepath -def scenario_output_folder(model_code, scenario): +def scenario_outputs_folder(scenario_folder): """ - Path to the output folder of the given scenario. + Path to the output folder in the given scenario folder. - :param model_code: code of the model - :param scenario: name of the scenario + :param scenario_folder: scenario folder path """ - # check if the OUTPUT_FOLDER environment variable is provided - if "OUTPUT_FOLDER" in os.environ: - env_output_folder = os.environ["OUTPUT_FOLDER"] - # check that the folder exists - if not os.path.isdir(env_output_folder): - raise IOError("Environment variable OUTPUT_FOLDER does not point to a folder") - - # add missing folder separator - if not env_output_folder.endswith(_SEP): - env_output_folder = env_output_folder + _SEP - - return env_output_folder - else: - return scenario_folder(model_code, scenario) + OUTPUT_FOLDER_NAME + _SEP + return os.path.join(scenario_folder, OUTPUT_FOLDER_NAME, "") def starling_folder(): """ Path to the Starling folder. """ - return starling_sim.__path__[0] + _SEP + ".." + _SEP + return os.path.join(starling_sim.__path__[0], "..", "") def schemas_folder(): """ Path to the schemas folder. """ - return os.path.join(os.path.dirname(__file__), "..", SCHEMAS_FOLDER_NAME) + _SEP + return os.path.join(os.path.dirname(__file__), "..", SCHEMAS_FOLDER_NAME, "") def model_import_path(starling_pkg, model_code): diff --git a/starling_sim/utils/simulation_logging.py b/starling_sim/utils/simulation_logging.py index 75e4a1d..fc46f09 100644 --- a/starling_sim/utils/simulation_logging.py +++ b/starling_sim/utils/simulation_logging.py @@ -46,7 +46,7 @@ .. code-block:: bash - python3 main.py data/models/SB_VS/example_nantes/inputs/Params.json -l 20 + python3 main.py data/models/SB_VS/example_nantes/ -l 20 ****************** Simulation loggers diff --git a/starling_sim/utils/test_models.py b/starling_sim/utils/test_models.py index 0f00ff6..1a1da3d 100644 --- a/starling_sim/utils/test_models.py +++ b/starling_sim/utils/test_models.py @@ -3,6 +3,7 @@ from starling_sim.utils.simulation_logging import TEST_LOGGER from starling_sim.utils.utils import gz_decompression from starling_sim.utils import paths +from starling_sim.simulation_scenario import SimulationScenario import os import subprocess @@ -117,13 +118,17 @@ def test_models(model_code_list, pkg): def test_model(model_code, pkg): + model_folder_path = paths.model_folder(model_code) + # get the test scenarios of the model - test_scenarios = os.listdir(paths.model_folder(model_code)) + test_scenarios = os.listdir(model_folder_path) # test the scenarios for scenario in test_scenarios: try: - run_time = test_scenario(model_code, pkg, scenario) + scenario_folder_path = os.path.join(model_folder_path, scenario, "") + simulation_scenario = SimulationScenario(scenario_folder_path) + run_time = test_scenario(pkg, simulation_scenario) message = "Success ({} seconds)".format(run_time) except ValueError as e: message = str(e) @@ -131,39 +136,40 @@ def test_model(model_code, pkg): TEST_LOGGER.info("{}, {} : {}".format(model_code, scenario, message)) -def test_scenario(model_code, pkg, scenario): +def test_scenario(pkg, simulation_scenario): # get the scenario parameters file - parameters_path = paths.scenario_parameters_filepath(model_code, scenario) + # parameters_path = paths.scenario_parameters_filepath(model_code, scenario) + scenario_path = simulation_scenario.scenario_folder # test the existance of the scenario - if not os.path.exists(parameters_path): - raise ValueError("Scenario parameters not found") + if not os.path.exists(scenario_path): + raise ValueError("Scenario folder not found") # remove existing outputs - output_folder = paths.scenario_output_folder(model_code, scenario) + output_folder = simulation_scenario.outputs_folder if os.path.exists(output_folder): shutil.rmtree(output_folder) try: # run the scenario start = time.time() - launch_simulation(parameters_path, pkg) + launch_simulation(scenario_path, pkg) run_time = int(time.time() - start) except Exception as e: message = "Simulation crash ({})".format(str(e)) raise ValueError(message) # compare the scenario outputs with reference files - compare_scenario_outputs(model_code, scenario) + compare_scenario_outputs(simulation_scenario) return run_time -def compare_scenario_outputs(model_code, scenario): +def compare_scenario_outputs(simulation_scenario): # get the test files - test_scenario_output_folder = paths.scenario_output_folder(model_code, scenario) + test_scenario_output_folder = simulation_scenario.outputs_folder test_scenario_output_files = os.listdir(test_scenario_output_folder) # extract bz2 and gz archives @@ -174,8 +180,8 @@ def compare_scenario_outputs(model_code, scenario): gz_decompression(test_scenario_output_folder + output_file) # get the reference files - scenario_expected_outputs_folder = ( - paths.scenario_folder(model_code, scenario) + REFERENCE_OUTPUTS_FOLDER_NAME + "/" + scenario_expected_outputs_folder = os.path.join( + simulation_scenario.scenario_folder, REFERENCE_OUTPUTS_FOLDER_NAME, "" ) expected_output_files_list = os.listdir(scenario_expected_outputs_folder)