diff --git a/.pylintrc b/.pylintrc index 01af648..b62d52e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,5 +1,5 @@ [MESSAGES CONTROL] -disable=fixme,invalid-name,len-as-condition,no-else-return,no-member,empty-docstring +disable=fixme,invalid-name,len-as-condition,no-else-return,no-member,empty-docstring,too-many-positional-arguments [FORMAT] # Regexp for a line that is allowed to be longer than the limit. diff --git a/bbp_workflow/generation/workflow.py b/bbp_workflow/generation/workflow.py index 796a8e1..6e75f6a 100644 --- a/bbp_workflow/generation/workflow.py +++ b/bbp_workflow/generation/workflow.py @@ -11,7 +11,18 @@ @luigi.util.inherits(KgCfg, RemoteHostCfg) class SBOWorkflow(luigi.WrapperTask): - """Workflow class.""" + """Generator Workflow class for building detailed circuits. + + Args: + output_dir (Path): Output directory to write generated data. + config_url (str): ModelBuildingConfig NEXUS resource id. + account (str): Project account. + isolated (bool): + Whether the workflow will run in isolated mode. Isolated mode ignores activities from + previous executions. + target (str): The target generator execute. The workflow will run all tasks up until the + target task. + """ output_dir = luigi.PathParameter( significant=True, @@ -45,7 +56,7 @@ def host_config(self) -> dict: return self.clone(RemoteHostCfg).param_kwargs def requires(self): - + """Return required leaf task in the workflow DAG, determined by the `target` argument.""" generator_class = get_class_from_config_name(self.target) return generator_class( diff --git a/doc/source/api.rst b/doc/source/api.rst index c7a25a5..fe63abe 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -10,3 +10,4 @@ API Documentation bbp_workflow.simulation.util bbp_workflow.target bbp_workflow.task + bbp_workflow.generation.workflow diff --git a/doc/source/index.rst b/doc/source/index.rst index d775a46..a95a3da 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -34,5 +34,6 @@ Overview of the tasks available in the workflow engine: Home api + workflow dependencies changelog diff --git a/doc/source/workflow.rst b/doc/source/workflow.rst new file mode 100644 index 0000000..4576dde --- /dev/null +++ b/doc/source/workflow.rst @@ -0,0 +1,9 @@ +The Building Workflow +===================== + +.. toctree:: + + workflow_components/quickstart.rst + workflow_components/overview.rst + workflow_components/generator_hierarchy.rst + workflow_components/generator_definitions.rst diff --git a/doc/source/workflow_components/generator_definitions.rst b/doc/source/workflow_components/generator_definitions.rst new file mode 100644 index 0000000..df19833 --- /dev/null +++ b/doc/source/workflow_components/generator_definitions.rst @@ -0,0 +1,324 @@ +Generator Definitions +====================== + +.. _cell_composition_generator: + +CellCompositionGenerator +~~~~~~~~~~~~~~~~~~~~~~~~ + +Config: `CellCompositionConfig `_ + +The ``CellCompositionGenerator`` is responsible for manipulating an input CellComposition in terms of density. + + +The final resource generated by the generator is a new CellComposition incorporating the changes listed in overrides. + + +.. _cell_position_generator: + +CellPositionGenerator +~~~~~~~~~~~~~~~~~~~~~ + +Config: `CellPositionConfig `_ + +The ``CellPositionGenerator`` consumes the upstream CellComposition registered by :ref`CellCompositionGenerator ` and places cell somata, along with properties such as mtypes, and etype. + +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "node_sets_file": "path/to/nodesets/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + } + } + } + ], + "edges": [] + }, + "metadata": { + "status": "partial" + } + } + +where the node population has the following properties: + +.. code-block:: text + + etype + hemisphere + morph_class + mtype + region + subregion + synapse_class + x + y + z + +See `SONATA documentation `__ for column definitions. + +.. _mmodel_generator: + +MorphologyAssignmentGenerator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Config: `MorphologyAssignmentConfig `_ + +The ``MorphologyAssignmentGenerator`` takes the ``DetailedCircuit`` from :ref:`CellPositionGenerator ` and depending on the configuration assigns or/and synthesizes morphologies for each cell in the node population. + +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "metadata": { + "status": "partial" + }, + "node_sets_file": "path/to/nodests/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + "alternate_morphologies": { + "h5v1": "$BASE_DIR/morphologies", + "neurolucida-asc": "$BASE_DIR/morphologies" + } + } + } + } + ], + "edges": [] + } + } + +where the node population has the following properties: + +.. code-block:: text + + etype + hemisphere + morph_class + morphology + morphology_producer + mtype + orientation_w + orientation_x + orientation_y + orientation_z + region + subregion + synapse_class + x + y + z + +See `SONATA documentation `__ for column definitions. + + +.. _me_model_generator: + +MEModelGenerator +~~~~~~~~~~~~~~~~ + +Config: `MEModelConfig `_ + +The ``MEModelGenerator`` takes the ``DetailedCircuit`` from :ref:`MorphologyAssignmentGenerator ` and assigns emodels and emodel properties. + +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "node_sets_file": "path/to/nodesets/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + "partial": [ + "cell-properties", + "morphologies" + ], + "alternate_morphologies": { + "h5v1": "path/to/morphologies/dir", + "neurolucida-asc": "path/to/morphologies/dir" + }, + "biophysical_neuron_models_dir": "path/to/hoc/dir" + } + } + } + ], + "edges": [] + }, + "metadata": { + "status": "partial" + } + } + +where the node population has the following properties: + +.. _me_model_properties: + +.. code-block:: text + + dynamics_params/AIS_scaler + dynamics_params/holding_current + dynamics_params/input_resistance + dynamics_params/resting_potential + dynamics_params/soma_scaler + dynamics_params/threshold_current + etype + hemisphere + model_template + morph_class + morphology + morphology_producer + mtype + orientation_w + orientation_x + orientation_y + orientation_z + region + subregion + synapse_class + x + y + z + +See `SONATA documentation `__ for column definitions. + +.. _macro_generator: + +MacroConnectomeGenerator +~~~~~~~~~~~~~~~~~~~~~~~~ + +Config: `MacroConnectomeConfig `_ + +The ``MacroConnectomeGenerator`` is a :ref:`relay generator ` that propagates its config downstream to :ref:`MicroConnectomeGenerator `. + +The final resource of `MacroConnectomeGenerator` is a clone of its input config. + + +.. _micro_generator: + +MicroConnectomeGenerator +~~~~~~~~~~~~~~~~~~~~~~~~ + +Config: `MicroConnectomeConfig `_ + +The ``MicroConnectomeGenerator`` takes as an input the `MacroConnectomeConfig `_ from :ref:`MacroConnectomeGenerator ` and the ``DetailedCircuit`` from :ref:`MEModelGenerator ` and establishes the connectivity of the node population. + + +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "node_sets_file": "path/to/nodesets/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + "alternate_morphologies": { + "h5v1": "path/to/morphologies/dir", + "neurolucida-asc": "path/to/morphologies/dir" + }, + "biophysical_neuron_models_dir": "path/to/hoc/dir" + } + } + } + ], + "edges": [ + { + "edges_file": "path/to/edges/h5", + "populations": { + "root__neurons__root__neurons__chemical": { + "type": "chemical" + } + } + } + ] + }, + "metadata": { + "status": "partial" + } + } + +where the nodes file has the same properties as in the :ref:`MEModelGenerator ones ` and the edges file has the following properties: + + +.. code-block:: text + + afferent_center_x + afferent_center_y + afferent_center_z + afferent_section_id + afferent_section_pos + afferent_section_type + delay + syn_type_id + +See `SONATA documentation `__ for column definitions. + + +ConnectomeFilteringGenerator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Config: `SynapseConfig `_ + +The ``ConnectomeFilteringGenerator`` takes the ``DetailedCircuit`` from :ref:`MicroConnectomeGenerator ` and filters the according to the configuration. + +The final resource is a simulation-ready ``DetailedCircuit`` where the where the nodes file has the same properties as in the :ref:`MEModelGenerator ones ` and the edges file has the following properties: + +.. code-block:: text + + afferent_center_x + afferent_center_y + afferent_center_z + afferent_section_id + afferent_section_pos + afferent_section_type + conductance + conductance_scale_factor + decay_time + delay + depression_time + facilitation_time + n_rrp_vesicles + syn_property_rule + syn_type_id + u_hill_coefficient + u_syn + +See `SONATA documentation `__ for column definitions. diff --git a/doc/source/workflow_components/generator_hierarchy.rst b/doc/source/workflow_components/generator_hierarchy.rst new file mode 100644 index 0000000..50aeba0 --- /dev/null +++ b/doc/source/workflow_components/generator_hierarchy.rst @@ -0,0 +1,191 @@ +Generator Graph +=============== + +.. _generators: + +.. graphviz:: + + digraph SBOWorkflow{ + + ModelBuildingConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + CellCompositionConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + CellPositionConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + MorphologyAssignmentConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + MEModelConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + MacroConnectomeConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + MacroConnectomeConfig2 [ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "MacroConnectomeConfig" + ] + + MicroConnectomeConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + SynapseConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + CellComposition [ + shape = Mrecord style = filled fillcolor = lemonchiffon + target = "_top" + ] + + CellPositionDetailedCircuit [ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + MModelDetailedCircuit [ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + MEModelDetailedCircuit[ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + MicroDetailedCircuit[ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + FiltDetailedCircuit[ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + CellCompositionActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + CellPositionActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + MModelActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + MEModelActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + MacroActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + MicroActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + FiltActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + CellCompositionGenerator [ + shape = Mrecord color = black + ] + + CellPositionGenerator [ + shape = Mrecord color = black + ] + + MorphologyAssignmentGenerator [ + shape = Mrecord color = black + ] + + MEModelGenerator [ + shape = Mrecord color = black + ] + + MacroConnectomeGenerator [ + shape = Mrecord color = black + ] + + MicroConnectomeGenerator [ + shape = Mrecord color = black + ] + + ConnectomeFilteringGenerator [ + shape = Mrecord color = black + ] + + + ModelBuildingConfig -> CellCompositionConfig; + ModelBuildingConfig -> CellPositionConfig; + ModelBuildingConfig -> MorphologyAssignmentConfig; + ModelBuildingConfig -> MEModelConfig; + ModelBuildingConfig -> MacroConnectomeConfig; + ModelBuildingConfig -> MicroConnectomeConfig; + ModelBuildingConfig -> SynapseConfig; + + ModelBuildingConfig -> CellCompositionGenerator; + CellCompositionGenerator -> CellCompositionActivity [label = "target"]; + CellCompositionActivity -> CellComposition [label = "generated"]; + CellCompositionActivity -> CellCompositionConfig [label = "used_config"]; + CellComposition -> CellPositionGenerator; + + ModelBuildingConfig -> CellPositionGenerator; + CellPositionGenerator -> CellPositionActivity [label = "target"]; + CellPositionActivity -> CellPositionDetailedCircuit [label = "generated"]; + CellPositionActivity -> CellPositionConfig [label = "used_config"]; + CellPositionDetailedCircuit -> MorphologyAssignmentGenerator; + + + ModelBuildingConfig -> MorphologyAssignmentGenerator; + MorphologyAssignmentGenerator -> MModelActivity [label = "target"]; + MModelActivity -> MModelDetailedCircuit [label="generated"]; + MModelActivity -> MorphologyAssignmentConfig [label = "used_config"]; + MModelDetailedCircuit -> MEModelGenerator; + + ModelBuildingConfig -> MEModelGenerator; + MEModelGenerator -> MEModelActivity [label = "target"]; + MEModelActivity -> MEModelDetailedCircuit [label = "generated"]; + MEModelActivity -> MEModelConfig [label = "used_config"]; + MEModelDetailedCircuit -> MicroConnectomeGenerator; + + ModelBuildingConfig -> MacroConnectomeGenerator; + MacroConnectomeGenerator -> MacroActivity [label = "target"]; + MacroActivity -> MacroConnectomeConfig2 [label = "generated"]; + MacroActivity -> MacroConnectomeConfig [label = "used_config"]; + MacroConnectomeConfig2 -> MicroConnectomeGenerator; + + ModelBuildingConfig -> MicroConnectomeGenerator; + MicroConnectomeGenerator -> MicroActivity [label = "target"]; + MicroActivity -> MicroDetailedCircuit [label = "generated"]; + MicroActivity -> MicroConnectomeConfig [label = "used_config"]; + MicroDetailedCircuit -> ConnectomeFilteringGenerator; + + ModelBuildingConfig -> ConnectomeFilteringGenerator; + ConnectomeFilteringGenerator -> FiltActivity [label = "target"]; + FiltActivity -> FiltDetailedCircuit [label = "generated"]; + FiltActivity -> SynapseConfig [label = "used_config"]; + + } diff --git a/doc/source/workflow_components/overview.rst b/doc/source/workflow_components/overview.rst new file mode 100644 index 0000000..3188cae --- /dev/null +++ b/doc/source/workflow_components/overview.rst @@ -0,0 +1,104 @@ +Overview +======== + +.. _workflow: + +The Workflow Task +----------------- + +The :class:`Workflow ` task receives a `ModelBuildingConfig `_ resource id and a target task name. + +The target task will trigger the execution of all its required upstream tasks, ensuring that each necessary dependency is processed in sequence until the target task is reached and executed. + +For example, if cellPositionConfig is selected as the target task, Luigi will first process the ``CellPositionGenerator`` requirements, which directly depends on ``CellCompositionGenerator``. Therefore ``CellCompositionGenerator`` will be executed before ``CellPositionGenerator`` can run. + +.. note:: + + It is not necessary for the ModelBuildingConfig to include all the generator configs, however it must contain all configs up until the target task selected when executing the :ref:`workflow `. + + +.. _generator_layout: + +Generator: Task Layout +---------------------- + +A Generator is a Luigi Task in a predefined hierarchy of :ref:`generators ` which has NEXUS resources as inputs and outputs. + +.. graphviz:: + + + digraph generator_layout { + + rankdir = "LR" + + ModelBuildingConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 2 + ] + GeneratorConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 2 + ] + + UpstreamResource [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 2 + ] + Generator [ + shape = Mrecord color = black + label = "{Generator|main_config_url\lgenerator_config_name}" + width = 2 + ] + GeneratorTaskActivity [ + shape = record style = filled fillcolor = lightblue + width = 2 + ] + Resource [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 2 + ] + + ModelBuildingConfig -> Generator; + ModelBuildingConfig -> GeneratorConfig; + UpstreamResource -> Generator; + Generator -> GeneratorTaskActivity [label = "target"]; + GeneratorTaskActivity -> Resource [label = "generated"] + GeneratorTaskActivity -> GeneratorConfig [label = "used_config"] + + } + +The Generator produces an activity with the generated entity, registered to the knowledge graph. The Generated is completed if a target entity can be found in the database with the specific ``used_config`` input. + + +.. note:: + + Since the workflow registers and searches for resources in the knowledge graph, retriggering a task requires deprecating the corresponding activity associated with the input configuration. Without this, the target will always be found, and Luigi will consider the task as already completed, preventing re-execution. + + +.. _generator_anatomy: + +Generator: Anatomy & Variants +----------------------------- + +.. _generator_types: + +Types +~~~~~ + +There are two main types of generators: + +* Relay Generators +* Multi Variant Generators + +Relay generators are simple tasks that propagate the input config downstream by creating an activity with a clone of the config. An example of a Relay generator is the MacroConnectomeGenerator. + +Multi variant generators are the most common tasks, scattering variant tasks and then merging them to produce the final result. Each Generator that derives from MultiVariantGenerator implements a +scatter and optionally a merge method. + +Variants +~~~~~~~~ + +A Generator may launch one or more variant tasks. A variant is an executable tool identified by the triplet ``(generator_name, variant_name, version)``. For more info see the `variant documentation +`_. + +.. _blue_cwl_variant: https://blue-cwl.readthedocs.io/en/latest/concepts/variant.html#what-is-a-variant diff --git a/doc/source/workflow_components/quickstart.rst b/doc/source/workflow_components/quickstart.rst new file mode 100644 index 0000000..ccbe5c4 --- /dev/null +++ b/doc/source/workflow_components/quickstart.rst @@ -0,0 +1,41 @@ +Quickstart +========== + +Launching a building workflow can be achieved with the following command: + +.. code-block:: shell + + ./bbp-workflow-launch.sh bbp_workflow.generation.workflow.SBOWorkflow \ + --config-url $CONFIG_URL \ + --output-dir $OUTPUT_DIR \ + --host bbpv1.epfl.ch \ + --kg-base $NEXUS_BASE \ + --kg-org $NEXUS_ORG \ + --account $ACCOUNT \ + --workers 1 + +Or respectively from the bbp-workflow server: + +.. code-block:: shell + + bbp-workflow launch --config $LUIGI_CFG -f bbp_workflow.generation SBOWorkflow \ + config-url=$CONFIG_URL \ + target=cellPositionConfig \ + output-dir=$OUTPUT_DIR \ + host=$HOST \ + account=$ACCOUNT \ + +where $LUIGI_CFG is structured as follows: + +.. code-block:: text + + [DEFAULT] + workers: 1 + kg-base: https://staging.nise.bbp.epfl.ch/nexus/v1 + kg-org: bbp + kg-proj: mmb-point-neuron-framework-model + + [SBOWorkflow] + kg-base: https://staging.nise.bbp.epfl.ch/nexus/v1 + kg-org: bbp + kg-proj: mmb-point-neuron-framework-model