From d131d9f8dc9f29ea9c664f231f359dac4876e129 Mon Sep 17 00:00:00 2001 From: Eleftherios Zisis Date: Tue, 17 Sep 2024 16:54:51 +0200 Subject: [PATCH 1/6] Init docs --- bbp_workflow/generation/workflow.py | 14 +- doc/source/api.rst | 1 + doc/source/index.rst | 1 + doc/source/workflow.rst | 757 ++++++++++++++++++++++++++++ 4 files changed, 772 insertions(+), 1 deletion(-) create mode 100644 doc/source/workflow.rst diff --git a/bbp_workflow/generation/workflow.py b/bbp_workflow/generation/workflow.py index 796a8e1..3df1f2b 100644 --- a/bbp_workflow/generation/workflow.py +++ b/bbp_workflow/generation/workflow.py @@ -11,7 +11,18 @@ @luigi.util.inherits(KgCfg, RemoteHostCfg) class SBOWorkflow(luigi.WrapperTask): - """Workflow class.""" + """Generator Workflow class for building detailed circuits. + + Args: + output_dir (Path): Output directory to write generated data. + config_url (str): ModelBuildingConfig NEXUS resource id. + account (str): Project account. + isolated (bool): + Whether the workflow will run in isolated mode. Isolated mode ignores activities from + previous executions. + target (str): The target generator execute. The workflow will run all tasks up until the + target task. + """ output_dir = luigi.PathParameter( significant=True, @@ -45,6 +56,7 @@ def host_config(self) -> dict: return self.clone(RemoteHostCfg).param_kwargs def requires(self): + """Return required leaf task in the workflow DAG, determined by the `target` argument.""" generator_class = get_class_from_config_name(self.target) diff --git a/doc/source/api.rst b/doc/source/api.rst index c7a25a5..fe63abe 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -10,3 +10,4 @@ API Documentation bbp_workflow.simulation.util bbp_workflow.target bbp_workflow.task + bbp_workflow.generation.workflow diff --git a/doc/source/index.rst b/doc/source/index.rst index d775a46..a95a3da 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -34,5 +34,6 @@ Overview of the tasks available in the workflow engine: Home api + workflow dependencies changelog diff --git a/doc/source/workflow.rst b/doc/source/workflow.rst new file mode 100644 index 0000000..46e4ef3 --- /dev/null +++ b/doc/source/workflow.rst @@ -0,0 +1,757 @@ +The Building Workflow +===================== + +Quickstart +---------- + +Launching a building workflow can be achieved with the following command: + +.. code-block:: shell + + ./bbp-workflow-launch.sh bbp_workflow.generation.workflow.SBOWorkflow \ + --config-url $CONFIG_URL \ + --output-dir $OUTPUT_DIR \ + --host bbpv1.epfl.ch \ + --kg-base $NEXUS_BASE \ + --kg-org $NEXUS_ORG \ + --account $ACCOUNT \ + --workers 1 + +Or respectively from the bbp-workflow server: + +.. code-block:: shell + + bbp-workflow launch --config $LUIGI_CFG -f bbp_workflow.generation SBOWorkflow \ + config-url=$CONFIG_URL \ + target=cellPositionConfig \ + output-dir=$OUTPUT_DIR \ + host=$HOST \ + account=$ACCOUNT \ + +where $LUIGI_CFG is structured as follows: + +.. code-block:: text + + [DEFAULT] + workers: 1 + kg-base: https://staging.nise.bbp.epfl.ch/nexus/v1 + kg-org: bbp + kg-proj: mmb-point-neuron-framework-model + + [SBOWorkflow] + kg-base: https://staging.nise.bbp.epfl.ch/nexus/v1 + kg-org: bbp + kg-proj: mmb-point-neuron-framework-model + + +.. _workflow: + +Workflow +-------- + +The :class:`Workflow ` task receives a :ref:`ModelBuildingConfig ` resource id and a target task name. + +The target task will trigger the execution of all its required upstream tasks, ensuring that each necessary dependency is processed in sequence until the target task is reached and executed. + +For example, if cellPositionConfig is selected as the target task, Luigi will first process the ``CellPositionGenerator`` requirements, which directly depends on ``CellCompositionGenerator``. Therefore ``CellCompositionGenerator`` will be executed before ``CellPositionGenerator`` can run. + + +.. _model_config: + +ModelBuildingConfig +------------------- + +The ModelBuildingConfig incorporates all the generator configs as follows: + + +.. graphviz:: + + digraph foo{ + + rankdir = "LR" + splines = "ortho" + + ModelBuildingConfig[ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + CellCompositionConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + CellPositionConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + MorphologyAssignmentConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + MEModelConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + MacroConnectomeConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + MicroConnectomeConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + SynapseConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + ModelBuildingConfig -> CellCompositionConfig [label="configs[cellCompositionConfig]", labelheight=2]; + ModelBuildingConfig -> CellPositionConfig [label="configs[cellPositionConfig]"]; + ModelBuildingConfig -> MorphologyAssignmentConfig [label="configs[morphologyAssignmentConfig]"]; + ModelBuildingConfig -> MEModelConfig [label="configs[meModelConfig]"]; + ModelBuildingConfig -> MacroConnectomeConfig [label="configs[macroConnectomeConfig]"]; + ModelBuildingConfig -> MicroConnectomeConfig [label="configs[microConnectomeConfig]"]; + ModelBuildingConfig -> SynapseConfig [label="configs[synapseConfig]"]; + + + } + + +.. note:: + + It is not necessary for the config to include all the generator configs, however it must contain all configs up until the target task selected when executing the :ref:`workflow `. + + +.. _generator_layout: + +Generator: Task Layout +---------------------- + +A Generator is a Luigi Task in a predefined hierarchy of :ref:`generators ` which has NEXUS resources as inputs and outputs. + +.. graphviz:: + + + digraph generator_layout { + + rankdir = "LR" + + ModelBuildingConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 2 + ] + GeneratorConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 2 + ] + + UpstreamResource [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 2 + ] + Generator [ + shape = Mrecord color = black + label = "{Generator|main_config_url\lgenerator_config_name}" + width = 2 + ] + GeneratorTaskActivity [ + shape = record style = filled fillcolor = lightblue + width = 2 + ] + Resource [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 2 + ] + + ModelBuildingConfig -> Generator; + ModelBuildingConfig -> GeneratorConfig; + UpstreamResource -> Generator; + Generator -> GeneratorTaskActivity [label = "target"]; + GeneratorTaskActivity -> Resource [label = "generated"] + GeneratorTaskActivity -> GeneratorConfig [label = "used_config"] + + } + +The Generator produces an activity with the generated entity, registered to the knowledge graph. The Generated is completed if a target entity can be found in the database with the specific ``used_config`` input. + + +.. note:: + + Since the workflow registers and searches for resources in the knowledge graph, retriggering a task requires deprecating the corresponding activity associated with the input configuration. Without this, the target will always be found, and Luigi will consider the task as already completed, preventing re-execution. + + +.. _generators: + +Generator: Hierarchy +-------------------- + +.. graphviz:: + + digraph SBOWorkflow{ + + ModelBuildingConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + CellCompositionConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + CellPositionConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + MorphologyAssignmentConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + MEModelConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + MacroConnectomeConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + MacroConnectomeConfig2 [ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "MacroConnectomeConfig" + ] + + MicroConnectomeConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + SynapseConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + CellComposition [ + shape = Mrecord style = filled fillcolor = lemonchiffon + target = "_top" + ] + + CellPositionDetailedCircuit [ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + MModelDetailedCircuit [ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + MEModelDetailedCircuit[ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + MicroDetailedCircuit[ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + FiltDetailedCircuit[ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + CellCompositionActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + CellPositionActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + MModelActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + MEModelActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + MacroActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + MicroActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + FiltActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + CellCompositionGenerator [ + shape = Mrecord color = black + ] + + CellPositionGenerator [ + shape = Mrecord color = black + ] + + MorphologyAssignmentGenerator [ + shape = Mrecord color = black + ] + + MEModelGenerator [ + shape = Mrecord color = black + ] + + MacroConnectomeGenerator [ + shape = Mrecord color = black + ] + + MicroConnectomeGenerator [ + shape = Mrecord color = black + ] + + ConnectomeFilteringGenerator [ + shape = Mrecord color = black + ] + + + ModelBuildingConfig -> CellCompositionConfig; + ModelBuildingConfig -> CellPositionConfig; + ModelBuildingConfig -> MorphologyAssignmentConfig; + ModelBuildingConfig -> MEModelConfig; + ModelBuildingConfig -> MacroConnectomeConfig; + ModelBuildingConfig -> MicroConnectomeConfig; + ModelBuildingConfig -> SynapseConfig; + + ModelBuildingConfig -> CellCompositionGenerator; + CellCompositionGenerator -> CellCompositionActivity [label = "target"]; + CellCompositionActivity -> CellComposition [label = "generated"]; + CellCompositionActivity -> CellCompositionConfig [label = "used_config"]; + CellComposition -> CellPositionGenerator; + + ModelBuildingConfig -> CellPositionGenerator; + CellPositionGenerator -> CellPositionActivity [label = "target"]; + CellPositionActivity -> CellPositionDetailedCircuit [label = "generated"]; + CellPositionActivity -> CellPositionConfig [label = "used_config"]; + CellPositionDetailedCircuit -> MorphologyAssignmentGenerator; + + + ModelBuildingConfig -> MorphologyAssignmentGenerator; + MorphologyAssignmentGenerator -> MModelActivity [label = "target"]; + MModelActivity -> MModelDetailedCircuit [label="generated"]; + MModelActivity -> MorphologyAssignmentConfig [label = "used_config"]; + MModelDetailedCircuit -> MEModelGenerator; + + ModelBuildingConfig -> MEModelGenerator; + MEModelGenerator -> MEModelActivity [label = "target"]; + MEModelActivity -> MEModelDetailedCircuit [label = "generated"]; + MEModelActivity -> MEModelConfig [label = "used_config"]; + MEModelDetailedCircuit -> MicroConnectomeGenerator; + + ModelBuildingConfig -> MacroConnectomeGenerator; + MacroConnectomeGenerator -> MacroActivity [label = "target"]; + MacroActivity -> MacroConnectomeConfig2 [label = "generated"]; + MacroActivity -> MacroConnectomeConfig [label = "used_config"]; + MacroConnectomeConfig2 -> MicroConnectomeGenerator; + + ModelBuildingConfig -> MicroConnectomeGenerator; + MicroConnectomeGenerator -> MicroActivity [label = "target"]; + MicroActivity -> MicroDetailedCircuit [label = "generated"]; + MicroActivity -> MicroConnectomeConfig [label = "used_config"]; + MicroDetailedCircuit -> ConnectomeFilteringGenerator; + + ModelBuildingConfig -> ConnectomeFilteringGenerator; + ConnectomeFilteringGenerator -> FiltActivity [label = "target"]; + FiltActivity -> FiltDetailedCircuit [label = "generated"]; + FiltActivity -> SynapseConfig [label = "used_config"]; + + } + +Generator: Anatomy & Variants +----------------------------- + +Types +~~~~~ + +There are two main types of generators: + +* Relay Generators +* Multi Variant Generators + +Relay generators are simple tasks that propagate the input config downstream by creating an activity with a clone of the config. An example of a Relay generator is the MacroConnectomeGenerator. + +Multi variant generators are the most common tasks, scattering variant tasks and then merging them to produce the final result. Each Generator that derives from MultiVariantGenerator implements a +scatter and optionally a merge method. + +Variants +~~~~~~~~ + +A Generator may launch one or more variant tasks. A variant is an executable tool identified by the triplet ``(generator_name, variant_name, version)``. For more info see the `variant documentation +`_. + +.. _blue_cwl_variant: https://blue-cwl.readthedocs.io/en/latest/concepts/variant.html#what-is-a-variant + + +Generator: Configurations +------------------------- + +CellCompositionGenerator +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: json + + { + "http://api.brain-map.org/api/v2/data/Structure/997": { + "variantDefinition": { + "algorithm": "cell_composition_manipulation", + "version": "v3" + }, + "inputs": [ + { + "name": "base_cell_composition_id", + "type": "Dataset", + "id": "https://bbp.epfl.ch/neurosciencegraph/data/cellcompositions/54818e46-cf8c-4bd6-9b68-34dffbc8a68c?tag=v1.1.0" + } + ], + "configuration": { + "version": 1, + "unitCode": { + "density": "mm^-3" + }, + "overrides": {} + } + } + } + + +For the variant definition of the manipulation algorithm refer to `blue-cwl documentation `__. + + +CellPositionGenerator +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: json + + { + "http://api.brain-map.org/api/v2/data/Structure/997": { + "variantDefinition": { + "algorithm": "neurons_cell_position", + "version": "v3" + }, + "inputs": [], + "configuration": { + "place_cells": { + "soma_placement": "basic", + "density_factor": 1, + "sort_by": [ + "region", + "mtype" + ], + "seed": 0, + "mini_frequencies": false + } + } + } + } + + +MorphologyAssignmentGenerator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: json + + { + "variantDefinition": { + "topological_synthesis": { + "algorithm": "topological_synthesis", + "version": "v3" + }, + "placeholder_assignment": { + "algorithm": "placeholder_assignment", + "version": "v3" + } + }, + "defaults": { + "topological_synthesis": { + "@id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/fae6eb46-3007-41c6-af69-941a82aada68", + "@type": "CanonicalMorphologyModelConfig" + }, + "placeholder_assignment": { + "@id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/9503a07d-8337-48eb-8637-acc26b0f13bf", + "@type": "PlaceholderMorphologyConfig" + } + }, + "configuration": { + "topological_synthesis": { + "http://api.brain-map.org/api/v2/data/Structure/23": { + "https://bbp.epfl.ch/ontologies/core/bmo/GenericInhibitoryNeuronMType": {} + } + } + } + } + +For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ + +MEModelGenerator +~~~~~~~~~~~~~~~~ + +.. code-block:: json + + { + "variantDefinition": { + "neurons_me_model": { + "algorithm": "neurons_me_model", + "version": "v3" + } + }, + "defaults": { + "neurons_me_model": { + "@id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/2ec96e9f-7254-44b5-bbcb-fdea3e18f110", + "@type": [ + "PlaceholderEModelConfig", + "Entity" + ] + } + }, + "overrides": { + "neurons_me_model": {} + } + } + +.. code-block:: + +For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ + + +MacroConnectomeGenerator +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: + + { + "initial": { + "connection_strength": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/connectomestrength/8e285d4b-4d09-4357-98ae-9e9fc61face6", + "type": [ + "Entity", + "Dataset", + "WholeBrainConnectomeStrength" + ], + "rev": 10 + } + }, + "overrides": { + "connection_strength": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/wholebrainconnectomestrengths/9357f9b4-8e94-45cd-b701-8d18648a17a6", + "type": [ + "Entity", + "Dataset", + "WholeBrainConnectomeStrength" + ], + "rev": 1 + } + }, + "_ui_data": { + "editHistory": [] + } + } + + +MicroConnectomeGenerator +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: + + { + "variants": { + "placeholder__erdos_renyi": { + "algorithm": "placeholder", + "version": "v3", + "params": { + "weight": { + "type": "float32", + "unitCode": "#synapses/connection", + "default": 0 + }, + "nsynconn_mean": { + "type": "float32", + "unitCode": "#synapses/connection", + "default": 3 + }, + "nsynconn_std": { + "type": "float32", + "unitCode": "#synapses/connection", + "default": 1.5 + }, + "delay_velocity": { + "type": "float32", + "unitCode": "um/ms", + "default": 250 + }, + "delay_offset": { + "type": "float32", + "unitCode": "ms", + "default": 0.8 + } + } + }, + "placeholder__distance_dependent": { + "algorithm": "placeholder", + "version": "v3", + "params": { + "weight": { + "type": "float32", + "unitCode": "#synapses/connection", + "default": 0 + }, + "exponent": { + "type": "float32", + "unitCode": "1/um", + "default": 0.008 + }, + "nsynconn_mean": { + "type": "float32", + "unitCode": "#synapses/connection", + "default": 3 + }, + "nsynconn_std": { + "type": "float32", + "unitCode": "#synapses/connection", + "default": 1.5 + }, + "delay_velocity": { + "type": "float32", + "unitCode": "um/ms", + "default": 250 + }, + "delay_offset": { + "type": "float32", + "unitCode": "ms", + "default": 0.8 + } + } + } + }, + "initial": { + "variants": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/a46a442c-5baa-4a5c-9907-bfb359dd9e5d", + "rev": 9, + "type": [ + "Entity", + "Dataset", + "MicroConnectomeVariantSelection" + ] + }, + "placeholder__erdos_renyi": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/microconnectomedata/009413eb-e51b-40bc-9199-8b98bfc53f87", + "rev": 7, + "type": [ + "Entity", + "Dataset", + "MicroConnectomeData" + ] + }, + "placeholder__distance_dependent": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/microconnectomedata/c7e1d215-2dad-4216-8565-6b1e4c161f46", + "rev": 7, + "type": [ + "Entity", + "Dataset", + "MicroConnectomeData" + ] + } + }, + "overrides": { + "variants": { + "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/deee5e86-1d7b-45f6-8fad-259a71c35c6a", + "type": [ + "Entity", + "Dataset", + "MicroConnectomeVariantSelectionOverrides" + ], + "rev": 1 + }, + "placeholder__erdos_renyi": { + "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/36426136-201d-4dfd-93d9-b541e113a6bf", + "type": [ + "Entity", + "Dataset", + "MicroConnectomeDataOverrides" + ], + "rev": 1 + }, + "placeholder__distance_dependent": { + "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/4bb03c2b-b99d-4a5d-8a8b-12e1a30619aa", + "type": [ + "Entity", + "Dataset", + "MicroConnectomeDataOverrides" + ], + "rev": 1 + } + }, + "_ui_data": { + "editHistory": [] + } + } + +ConnectomeFilteringGenerator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: json + + { + "variantDefinition": { + "algorithm": "synapses", + "version": "v2" + }, + "defaults": { + "synapse_properties": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/synapticassignment/d57536aa-d576-4b3b-a89b-b7888f24eb21", + "type": [ + "Dataset", + "SynapticParameterAssignment" + ], + "rev": 9 + }, + "synapses_classification": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/synapticparameters/cf25c2bf-e6e4-4367-acd8-94004bfcfe49", + "type": [ + "Dataset", + "SynapticParameter" + ], + "rev": 6 + } + }, + "configuration": { + "synapse_properties": { + "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/839a8b83-1620-4fe7-8f58-658ded0ea1e8", + "type": [ + "Dataset", + "SynapticParameterAssignment" + ], + "rev": 1 + }, + "synapses_classification": { + "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/d133e408-bd00-41ca-9334-e5fab779ad99", + "type": [ + "Dataset", + "SynapticParameter" + ], + "rev": 3 + } + } + } + +For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ + From ecf6594672eb2fa047bfc121cd0e609db5ab4eb7 Mon Sep 17 00:00:00 2001 From: Eleftherios Zisis Date: Wed, 18 Sep 2024 14:32:15 +0200 Subject: [PATCH 2/6] Add more info --- doc/source/workflow.rst | 335 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 332 insertions(+), 3 deletions(-) diff --git a/doc/source/workflow.rst b/doc/source/workflow.rst index 46e4ef3..d80dcfe 100644 --- a/doc/source/workflow.rst +++ b/doc/source/workflow.rst @@ -377,9 +377,13 @@ Generator: Hierarchy } +.. _generator_anatomy: + Generator: Anatomy & Variants ----------------------------- +.. _generator_types: + Types ~~~~~ @@ -402,12 +406,18 @@ A Generator may launch one or more variant tasks. A variant is an executable too .. _blue_cwl_variant: https://blue-cwl.readthedocs.io/en/latest/concepts/variant.html#what-is-a-variant -Generator: Configurations -------------------------- +Generator: Definitions & Configurations +--------------------------------------- + +.. _cell_composition_generator: CellCompositionGenerator ~~~~~~~~~~~~~~~~~~~~~~~~ +The ``CellCompositionGenerator`` is responsible for manipulating an input CellComposition in terms of density. + +.. _cell_composition_config: + .. code-block:: json { @@ -433,13 +443,54 @@ CellCompositionGenerator } } +To add an override: + +.. code-block:: json + + { + "overrides": { + "http://api.brain-map.org/api/v2/data/Structure/23": { + "label": "Anterior amygdalar area", + "about": "BrainRegion", + "hasPart": { + "https://bbp.epfl.ch/ontologies/core/bmo/GenericExcitatoryNeuronMType?rev=6": { + "label": "GEN_mtype", + "about": "MType", + "hasPart": { + "https://bbp.epfl.ch/ontologies/core/bmo/GenericExcitatoryNeuronEType?rev=6": { + "label": "GEN_etype", + "about": "EType", + "composition": { + "neuron": { + "density": 1200 + } + } + } + } + } + } + } + } + } + For the variant definition of the manipulation algorithm refer to `blue-cwl documentation `__. +The final resource generated by the generator is a new CellComposition incorporating the changes listed in overrides. + +.. _cell_position_generator: CellPositionGenerator ~~~~~~~~~~~~~~~~~~~~~ +The ``CellPositionGenerator`` consumes the upstream CellComposition registered by :ref`CellCompositionGenerator ` and places cell somata, along with properties such as mtypes, and etype. + + +The ``CellPositionConfig`` has the following layout: + + +.. _cell_position_config: + .. code-block:: json { @@ -465,9 +516,58 @@ CellPositionGenerator } +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "node_sets_file": "path/to/nodesets/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + } + } + } + ], + "edges": [] + }, + "metadata": { + "status": "partial" + } + } + +where the node population has the following properties: + +.. code-block:: text + + etype + hemisphere + morph_class + mtype + region + subregion + synapse_class + x + y + z + +See `SONATA documentation `__ for column definitions. + +.. _mmodel_generator: + MorphologyAssignmentGenerator ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The ``MorphologyAssignmentGenerator`` takes the ``DetailedCircuit`` from :ref:`CellPositionGenerator ` and depending on the configuration assigns or/and synthesizes morphologies for each cell in the node population. + .. code-block:: json { @@ -502,9 +602,70 @@ MorphologyAssignmentGenerator For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "metadata": { + "status": "partial" + }, + "node_sets_file": "path/to/nodests/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + "alternate_morphologies": { + "h5v1": "$BASE_DIR/morphologies", + "neurolucida-asc": "$BASE_DIR/morphologies" + } + } + } + } + ], + "edges": [] + } + } + +where the node population has the following properties: + +.. code-block:: text + + etype + hemisphere + morph_class + morphology + morphology_producer + mtype + orientation_w + orientation_x + orientation_y + orientation_z + region + subregion + synapse_class + x + y + z + +See `SONATA documentation `__ for column definitions. + +.. _me_model_generator: + MEModelGenerator ~~~~~~~~~~~~~~~~ +The ``MEModelGenerator`` takes the ``DetailedCircuit`` from :ref:`MorphologyAssignmentGenerator ` and assigns emodels and emodel properties. + +.. _me_model_config: + .. code-block:: json { @@ -528,14 +689,87 @@ MEModelGenerator } } -.. code-block:: For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "node_sets_file": "path/to/nodesets/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + "partial": [ + "cell-properties", + "morphologies" + ], + "alternate_morphologies": { + "h5v1": "path/to/morphologies/dir", + "neurolucida-asc": "path/to/morphologies/dir" + }, + "biophysical_neuron_models_dir": "path/to/hoc/dir" + } + } + } + ], + "edges": [] + }, + "metadata": { + "status": "partial" + } + } + +where the node population has the following properties: + +.. _me_model_properties: + +.. code-block:: text + + dynamics_params/AIS_scaler + dynamics_params/holding_current + dynamics_params/input_resistance + dynamics_params/resting_potential + dynamics_params/soma_scaler + dynamics_params/threshold_current + etype + hemisphere + model_template + morph_class + morphology + morphology_producer + mtype + orientation_w + orientation_x + orientation_y + orientation_z + region + subregion + synapse_class + x + y + z + +See `SONATA documentation `__ for column definitions. + +.. _macro_generator: MacroConnectomeGenerator ~~~~~~~~~~~~~~~~~~~~~~~~ +The ``MacroConnectomeGenerator`` is a :ref:`relay generator ` that propagates its config downstream to :ref:`MicroConnectomeGenerator `. + +.. _macro_config: + .. code-block:: { @@ -566,10 +800,21 @@ MacroConnectomeGenerator } } +The final resource of `MacroConnectomeGenerator` is a clone of its input config. + + +.. _micro_generator: MicroConnectomeGenerator ~~~~~~~~~~~~~~~~~~~~~~~~ +The ``MicroConnectomeGenerator`` takes as an input the :ref:`MacroConnectomeConfig ` from :ref:`MacroConnectomeGenerator ` and the ``DetailedCircuit`` from :ref:`MEModelGenerator ` and establishes the connectivity of the node population. + + +The ``MicroConnectomeConfig`` has the following layout: + +.. _micro_config: + .. code-block:: { @@ -705,9 +950,70 @@ MicroConnectomeGenerator } } +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "node_sets_file": "path/to/nodesets/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + "alternate_morphologies": { + "h5v1": "path/to/morphologies/dir", + "neurolucida-asc": "path/to/morphologies/dir" + }, + "biophysical_neuron_models_dir": "path/to/hoc/dir" + } + } + } + ], + "edges": [ + { + "edges_file": "path/to/edges/h5", + "populations": { + "root__neurons__root__neurons__chemical": { + "type": "chemical" + } + } + } + ] + }, + "metadata": { + "status": "partial" + } + } + +where the nodes file has the same properties as in the :ref:`MEModelGenerator ones ` and the edges file has the following properties: + + +.. code-block:: text + + afferent_center_x + afferent_center_y + afferent_center_z + afferent_section_id + afferent_section_pos + afferent_section_type + delay + syn_type_id + +See `SONATA documentation `__ for column definitions. + + ConnectomeFilteringGenerator ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The ``ConnectomeFilteringGenerator`` takes the ``DetailedCircuit`` from :ref:`MicroConnectomeGenerator ` and filters the according to the configuration. + .. code-block:: json { @@ -755,3 +1061,26 @@ ConnectomeFilteringGenerator For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ +The final resource is a simulation-ready ``DetailedCircuit`` where the where the nodes file has the same properties as in the :ref:`MEModelGenerator ones ` and the edges file has the following properties: + +.. code-block:: text + + afferent_center_x + afferent_center_y + afferent_center_z + afferent_section_id + afferent_section_pos + afferent_section_type + conductance + conductance_scale_factor + decay_time + delay + depression_time + facilitation_time + n_rrp_vesicles + syn_property_rule + syn_type_id + u_hill_coefficient + u_syn + +See `SONATA documentation `__ for column definitions. From d74f389735eb821b2a18f113818b92714f38b6b7 Mon Sep 17 00:00:00 2001 From: Eleftherios Zisis Date: Wed, 18 Sep 2024 15:18:44 +0200 Subject: [PATCH 3/6] Rearrange --- doc/source/workflow.rst | 1087 +---------------- .../generator_definition_configuration.rst | 678 ++++++++++ .../generator_hierarchy.rst | 191 +++ doc/source/workflow_components/overview.rst | 171 +++ doc/source/workflow_components/quickstart.rst | 41 + 5 files changed, 1086 insertions(+), 1082 deletions(-) create mode 100644 doc/source/workflow_components/generator_definition_configuration.rst create mode 100644 doc/source/workflow_components/generator_hierarchy.rst create mode 100644 doc/source/workflow_components/overview.rst create mode 100644 doc/source/workflow_components/quickstart.rst diff --git a/doc/source/workflow.rst b/doc/source/workflow.rst index d80dcfe..c949357 100644 --- a/doc/source/workflow.rst +++ b/doc/source/workflow.rst @@ -1,1086 +1,9 @@ The Building Workflow ===================== -Quickstart ----------- +.. toctree:: -Launching a building workflow can be achieved with the following command: - -.. code-block:: shell - - ./bbp-workflow-launch.sh bbp_workflow.generation.workflow.SBOWorkflow \ - --config-url $CONFIG_URL \ - --output-dir $OUTPUT_DIR \ - --host bbpv1.epfl.ch \ - --kg-base $NEXUS_BASE \ - --kg-org $NEXUS_ORG \ - --account $ACCOUNT \ - --workers 1 - -Or respectively from the bbp-workflow server: - -.. code-block:: shell - - bbp-workflow launch --config $LUIGI_CFG -f bbp_workflow.generation SBOWorkflow \ - config-url=$CONFIG_URL \ - target=cellPositionConfig \ - output-dir=$OUTPUT_DIR \ - host=$HOST \ - account=$ACCOUNT \ - -where $LUIGI_CFG is structured as follows: - -.. code-block:: text - - [DEFAULT] - workers: 1 - kg-base: https://staging.nise.bbp.epfl.ch/nexus/v1 - kg-org: bbp - kg-proj: mmb-point-neuron-framework-model - - [SBOWorkflow] - kg-base: https://staging.nise.bbp.epfl.ch/nexus/v1 - kg-org: bbp - kg-proj: mmb-point-neuron-framework-model - - -.. _workflow: - -Workflow --------- - -The :class:`Workflow ` task receives a :ref:`ModelBuildingConfig ` resource id and a target task name. - -The target task will trigger the execution of all its required upstream tasks, ensuring that each necessary dependency is processed in sequence until the target task is reached and executed. - -For example, if cellPositionConfig is selected as the target task, Luigi will first process the ``CellPositionGenerator`` requirements, which directly depends on ``CellCompositionGenerator``. Therefore ``CellCompositionGenerator`` will be executed before ``CellPositionGenerator`` can run. - - -.. _model_config: - -ModelBuildingConfig -------------------- - -The ModelBuildingConfig incorporates all the generator configs as follows: - - -.. graphviz:: - - digraph foo{ - - rankdir = "LR" - splines = "ortho" - - ModelBuildingConfig[ - shape = Mrecord style = filled fillcolor = lemonchiffon - ] - - CellCompositionConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - CellPositionConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - MorphologyAssignmentConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - MEModelConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - MacroConnectomeConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - MicroConnectomeConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - SynapseConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - ModelBuildingConfig -> CellCompositionConfig [label="configs[cellCompositionConfig]", labelheight=2]; - ModelBuildingConfig -> CellPositionConfig [label="configs[cellPositionConfig]"]; - ModelBuildingConfig -> MorphologyAssignmentConfig [label="configs[morphologyAssignmentConfig]"]; - ModelBuildingConfig -> MEModelConfig [label="configs[meModelConfig]"]; - ModelBuildingConfig -> MacroConnectomeConfig [label="configs[macroConnectomeConfig]"]; - ModelBuildingConfig -> MicroConnectomeConfig [label="configs[microConnectomeConfig]"]; - ModelBuildingConfig -> SynapseConfig [label="configs[synapseConfig]"]; - - - } - - -.. note:: - - It is not necessary for the config to include all the generator configs, however it must contain all configs up until the target task selected when executing the :ref:`workflow `. - - -.. _generator_layout: - -Generator: Task Layout ----------------------- - -A Generator is a Luigi Task in a predefined hierarchy of :ref:`generators ` which has NEXUS resources as inputs and outputs. - -.. graphviz:: - - - digraph generator_layout { - - rankdir = "LR" - - ModelBuildingConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 2 - ] - GeneratorConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 2 - ] - - UpstreamResource [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 2 - ] - Generator [ - shape = Mrecord color = black - label = "{Generator|main_config_url\lgenerator_config_name}" - width = 2 - ] - GeneratorTaskActivity [ - shape = record style = filled fillcolor = lightblue - width = 2 - ] - Resource [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 2 - ] - - ModelBuildingConfig -> Generator; - ModelBuildingConfig -> GeneratorConfig; - UpstreamResource -> Generator; - Generator -> GeneratorTaskActivity [label = "target"]; - GeneratorTaskActivity -> Resource [label = "generated"] - GeneratorTaskActivity -> GeneratorConfig [label = "used_config"] - - } - -The Generator produces an activity with the generated entity, registered to the knowledge graph. The Generated is completed if a target entity can be found in the database with the specific ``used_config`` input. - - -.. note:: - - Since the workflow registers and searches for resources in the knowledge graph, retriggering a task requires deprecating the corresponding activity associated with the input configuration. Without this, the target will always be found, and Luigi will consider the task as already completed, preventing re-execution. - - -.. _generators: - -Generator: Hierarchy --------------------- - -.. graphviz:: - - digraph SBOWorkflow{ - - ModelBuildingConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - ] - - CellCompositionConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - ] - - CellPositionConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - ] - - MorphologyAssignmentConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - ] - - MEModelConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - ] - - MacroConnectomeConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - ] - - MacroConnectomeConfig2 [ - shape = Mrecord style = filled fillcolor = lemonchiffon - label = "MacroConnectomeConfig" - ] - - MicroConnectomeConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - ] - - SynapseConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - ] - - CellComposition [ - shape = Mrecord style = filled fillcolor = lemonchiffon - target = "_top" - ] - - CellPositionDetailedCircuit [ - shape = Mrecord style = filled fillcolor = lemonchiffon - label = "DetailedCircuit" - ] - - MModelDetailedCircuit [ - shape = Mrecord style = filled fillcolor = lemonchiffon - label = "DetailedCircuit" - ] - - MEModelDetailedCircuit[ - shape = Mrecord style = filled fillcolor = lemonchiffon - label = "DetailedCircuit" - ] - - MicroDetailedCircuit[ - shape = Mrecord style = filled fillcolor = lemonchiffon - label = "DetailedCircuit" - ] - - FiltDetailedCircuit[ - shape = Mrecord style = filled fillcolor = lemonchiffon - label = "DetailedCircuit" - ] - - CellCompositionActivity [ - shape = record style = filled fillcolor = lightblue - label = "GeneratorTaskActivity" - ] - - CellPositionActivity [ - shape = record style = filled fillcolor = lightblue - label = "GeneratorTaskActivity" - ] - - MModelActivity [ - shape = record style = filled fillcolor = lightblue - label = "GeneratorTaskActivity" - ] - - MEModelActivity [ - shape = record style = filled fillcolor = lightblue - label = "GeneratorTaskActivity" - ] - - MacroActivity [ - shape = record style = filled fillcolor = lightblue - label = "GeneratorTaskActivity" - ] - - MicroActivity [ - shape = record style = filled fillcolor = lightblue - label = "GeneratorTaskActivity" - ] - - FiltActivity [ - shape = record style = filled fillcolor = lightblue - label = "GeneratorTaskActivity" - ] - - CellCompositionGenerator [ - shape = Mrecord color = black - ] - - CellPositionGenerator [ - shape = Mrecord color = black - ] - - MorphologyAssignmentGenerator [ - shape = Mrecord color = black - ] - - MEModelGenerator [ - shape = Mrecord color = black - ] - - MacroConnectomeGenerator [ - shape = Mrecord color = black - ] - - MicroConnectomeGenerator [ - shape = Mrecord color = black - ] - - ConnectomeFilteringGenerator [ - shape = Mrecord color = black - ] - - - ModelBuildingConfig -> CellCompositionConfig; - ModelBuildingConfig -> CellPositionConfig; - ModelBuildingConfig -> MorphologyAssignmentConfig; - ModelBuildingConfig -> MEModelConfig; - ModelBuildingConfig -> MacroConnectomeConfig; - ModelBuildingConfig -> MicroConnectomeConfig; - ModelBuildingConfig -> SynapseConfig; - - ModelBuildingConfig -> CellCompositionGenerator; - CellCompositionGenerator -> CellCompositionActivity [label = "target"]; - CellCompositionActivity -> CellComposition [label = "generated"]; - CellCompositionActivity -> CellCompositionConfig [label = "used_config"]; - CellComposition -> CellPositionGenerator; - - ModelBuildingConfig -> CellPositionGenerator; - CellPositionGenerator -> CellPositionActivity [label = "target"]; - CellPositionActivity -> CellPositionDetailedCircuit [label = "generated"]; - CellPositionActivity -> CellPositionConfig [label = "used_config"]; - CellPositionDetailedCircuit -> MorphologyAssignmentGenerator; - - - ModelBuildingConfig -> MorphologyAssignmentGenerator; - MorphologyAssignmentGenerator -> MModelActivity [label = "target"]; - MModelActivity -> MModelDetailedCircuit [label="generated"]; - MModelActivity -> MorphologyAssignmentConfig [label = "used_config"]; - MModelDetailedCircuit -> MEModelGenerator; - - ModelBuildingConfig -> MEModelGenerator; - MEModelGenerator -> MEModelActivity [label = "target"]; - MEModelActivity -> MEModelDetailedCircuit [label = "generated"]; - MEModelActivity -> MEModelConfig [label = "used_config"]; - MEModelDetailedCircuit -> MicroConnectomeGenerator; - - ModelBuildingConfig -> MacroConnectomeGenerator; - MacroConnectomeGenerator -> MacroActivity [label = "target"]; - MacroActivity -> MacroConnectomeConfig2 [label = "generated"]; - MacroActivity -> MacroConnectomeConfig [label = "used_config"]; - MacroConnectomeConfig2 -> MicroConnectomeGenerator; - - ModelBuildingConfig -> MicroConnectomeGenerator; - MicroConnectomeGenerator -> MicroActivity [label = "target"]; - MicroActivity -> MicroDetailedCircuit [label = "generated"]; - MicroActivity -> MicroConnectomeConfig [label = "used_config"]; - MicroDetailedCircuit -> ConnectomeFilteringGenerator; - - ModelBuildingConfig -> ConnectomeFilteringGenerator; - ConnectomeFilteringGenerator -> FiltActivity [label = "target"]; - FiltActivity -> FiltDetailedCircuit [label = "generated"]; - FiltActivity -> SynapseConfig [label = "used_config"]; - - } - -.. _generator_anatomy: - -Generator: Anatomy & Variants ------------------------------ - -.. _generator_types: - -Types -~~~~~ - -There are two main types of generators: - -* Relay Generators -* Multi Variant Generators - -Relay generators are simple tasks that propagate the input config downstream by creating an activity with a clone of the config. An example of a Relay generator is the MacroConnectomeGenerator. - -Multi variant generators are the most common tasks, scattering variant tasks and then merging them to produce the final result. Each Generator that derives from MultiVariantGenerator implements a -scatter and optionally a merge method. - -Variants -~~~~~~~~ - -A Generator may launch one or more variant tasks. A variant is an executable tool identified by the triplet ``(generator_name, variant_name, version)``. For more info see the `variant documentation -`_. - -.. _blue_cwl_variant: https://blue-cwl.readthedocs.io/en/latest/concepts/variant.html#what-is-a-variant - - -Generator: Definitions & Configurations ---------------------------------------- - -.. _cell_composition_generator: - -CellCompositionGenerator -~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``CellCompositionGenerator`` is responsible for manipulating an input CellComposition in terms of density. - -.. _cell_composition_config: - -.. code-block:: json - - { - "http://api.brain-map.org/api/v2/data/Structure/997": { - "variantDefinition": { - "algorithm": "cell_composition_manipulation", - "version": "v3" - }, - "inputs": [ - { - "name": "base_cell_composition_id", - "type": "Dataset", - "id": "https://bbp.epfl.ch/neurosciencegraph/data/cellcompositions/54818e46-cf8c-4bd6-9b68-34dffbc8a68c?tag=v1.1.0" - } - ], - "configuration": { - "version": 1, - "unitCode": { - "density": "mm^-3" - }, - "overrides": {} - } - } - } - -To add an override: - -.. code-block:: json - - { - "overrides": { - "http://api.brain-map.org/api/v2/data/Structure/23": { - "label": "Anterior amygdalar area", - "about": "BrainRegion", - "hasPart": { - "https://bbp.epfl.ch/ontologies/core/bmo/GenericExcitatoryNeuronMType?rev=6": { - "label": "GEN_mtype", - "about": "MType", - "hasPart": { - "https://bbp.epfl.ch/ontologies/core/bmo/GenericExcitatoryNeuronEType?rev=6": { - "label": "GEN_etype", - "about": "EType", - "composition": { - "neuron": { - "density": 1200 - } - } - } - } - } - } - } - } - } - - -For the variant definition of the manipulation algorithm refer to `blue-cwl documentation `__. - -The final resource generated by the generator is a new CellComposition incorporating the changes listed in overrides. - -.. _cell_position_generator: - -CellPositionGenerator -~~~~~~~~~~~~~~~~~~~~~ - -The ``CellPositionGenerator`` consumes the upstream CellComposition registered by :ref`CellCompositionGenerator ` and places cell somata, along with properties such as mtypes, and etype. - - -The ``CellPositionConfig`` has the following layout: - - -.. _cell_position_config: - -.. code-block:: json - - { - "http://api.brain-map.org/api/v2/data/Structure/997": { - "variantDefinition": { - "algorithm": "neurons_cell_position", - "version": "v3" - }, - "inputs": [], - "configuration": { - "place_cells": { - "soma_placement": "basic", - "density_factor": 1, - "sort_by": [ - "region", - "mtype" - ], - "seed": 0, - "mini_frequencies": false - } - } - } - } - - -The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: - -.. code-block:: json - - { - "version": 2, - "manifest": { - "$BASE_DIR": "." - }, - "node_sets_file": "path/to/nodesets/json", - "networks": { - "nodes": [ - { - "nodes_file": "path/to/nodes/h5", - "populations": { - "root__neurons": { - "type": "biophysical", - } - } - } - ], - "edges": [] - }, - "metadata": { - "status": "partial" - } - } - -where the node population has the following properties: - -.. code-block:: text - - etype - hemisphere - morph_class - mtype - region - subregion - synapse_class - x - y - z - -See `SONATA documentation `__ for column definitions. - -.. _mmodel_generator: - -MorphologyAssignmentGenerator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``MorphologyAssignmentGenerator`` takes the ``DetailedCircuit`` from :ref:`CellPositionGenerator ` and depending on the configuration assigns or/and synthesizes morphologies for each cell in the node population. - -.. code-block:: json - - { - "variantDefinition": { - "topological_synthesis": { - "algorithm": "topological_synthesis", - "version": "v3" - }, - "placeholder_assignment": { - "algorithm": "placeholder_assignment", - "version": "v3" - } - }, - "defaults": { - "topological_synthesis": { - "@id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/fae6eb46-3007-41c6-af69-941a82aada68", - "@type": "CanonicalMorphologyModelConfig" - }, - "placeholder_assignment": { - "@id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/9503a07d-8337-48eb-8637-acc26b0f13bf", - "@type": "PlaceholderMorphologyConfig" - } - }, - "configuration": { - "topological_synthesis": { - "http://api.brain-map.org/api/v2/data/Structure/23": { - "https://bbp.epfl.ch/ontologies/core/bmo/GenericInhibitoryNeuronMType": {} - } - } - } - } - -For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ - -The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: - -.. code-block:: json - - { - "version": 2, - "manifest": { - "$BASE_DIR": "." - }, - "metadata": { - "status": "partial" - }, - "node_sets_file": "path/to/nodests/json", - "networks": { - "nodes": [ - { - "nodes_file": "path/to/nodes/h5", - "populations": { - "root__neurons": { - "type": "biophysical", - "alternate_morphologies": { - "h5v1": "$BASE_DIR/morphologies", - "neurolucida-asc": "$BASE_DIR/morphologies" - } - } - } - } - ], - "edges": [] - } - } - -where the node population has the following properties: - -.. code-block:: text - - etype - hemisphere - morph_class - morphology - morphology_producer - mtype - orientation_w - orientation_x - orientation_y - orientation_z - region - subregion - synapse_class - x - y - z - -See `SONATA documentation `__ for column definitions. - -.. _me_model_generator: - -MEModelGenerator -~~~~~~~~~~~~~~~~ - -The ``MEModelGenerator`` takes the ``DetailedCircuit`` from :ref:`MorphologyAssignmentGenerator ` and assigns emodels and emodel properties. - -.. _me_model_config: - -.. code-block:: json - - { - "variantDefinition": { - "neurons_me_model": { - "algorithm": "neurons_me_model", - "version": "v3" - } - }, - "defaults": { - "neurons_me_model": { - "@id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/2ec96e9f-7254-44b5-bbcb-fdea3e18f110", - "@type": [ - "PlaceholderEModelConfig", - "Entity" - ] - } - }, - "overrides": { - "neurons_me_model": {} - } - } - - -For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ - -The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: - -.. code-block:: json - - { - "version": 2, - "manifest": { - "$BASE_DIR": "." - }, - "node_sets_file": "path/to/nodesets/json", - "networks": { - "nodes": [ - { - "nodes_file": "path/to/nodes/h5", - "populations": { - "root__neurons": { - "type": "biophysical", - "partial": [ - "cell-properties", - "morphologies" - ], - "alternate_morphologies": { - "h5v1": "path/to/morphologies/dir", - "neurolucida-asc": "path/to/morphologies/dir" - }, - "biophysical_neuron_models_dir": "path/to/hoc/dir" - } - } - } - ], - "edges": [] - }, - "metadata": { - "status": "partial" - } - } - -where the node population has the following properties: - -.. _me_model_properties: - -.. code-block:: text - - dynamics_params/AIS_scaler - dynamics_params/holding_current - dynamics_params/input_resistance - dynamics_params/resting_potential - dynamics_params/soma_scaler - dynamics_params/threshold_current - etype - hemisphere - model_template - morph_class - morphology - morphology_producer - mtype - orientation_w - orientation_x - orientation_y - orientation_z - region - subregion - synapse_class - x - y - z - -See `SONATA documentation `__ for column definitions. - -.. _macro_generator: - -MacroConnectomeGenerator -~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``MacroConnectomeGenerator`` is a :ref:`relay generator ` that propagates its config downstream to :ref:`MicroConnectomeGenerator `. - -.. _macro_config: - -.. code-block:: - - { - "initial": { - "connection_strength": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/connectomestrength/8e285d4b-4d09-4357-98ae-9e9fc61face6", - "type": [ - "Entity", - "Dataset", - "WholeBrainConnectomeStrength" - ], - "rev": 10 - } - }, - "overrides": { - "connection_strength": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/wholebrainconnectomestrengths/9357f9b4-8e94-45cd-b701-8d18648a17a6", - "type": [ - "Entity", - "Dataset", - "WholeBrainConnectomeStrength" - ], - "rev": 1 - } - }, - "_ui_data": { - "editHistory": [] - } - } - -The final resource of `MacroConnectomeGenerator` is a clone of its input config. - - -.. _micro_generator: - -MicroConnectomeGenerator -~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``MicroConnectomeGenerator`` takes as an input the :ref:`MacroConnectomeConfig ` from :ref:`MacroConnectomeGenerator ` and the ``DetailedCircuit`` from :ref:`MEModelGenerator ` and establishes the connectivity of the node population. - - -The ``MicroConnectomeConfig`` has the following layout: - -.. _micro_config: - -.. code-block:: - - { - "variants": { - "placeholder__erdos_renyi": { - "algorithm": "placeholder", - "version": "v3", - "params": { - "weight": { - "type": "float32", - "unitCode": "#synapses/connection", - "default": 0 - }, - "nsynconn_mean": { - "type": "float32", - "unitCode": "#synapses/connection", - "default": 3 - }, - "nsynconn_std": { - "type": "float32", - "unitCode": "#synapses/connection", - "default": 1.5 - }, - "delay_velocity": { - "type": "float32", - "unitCode": "um/ms", - "default": 250 - }, - "delay_offset": { - "type": "float32", - "unitCode": "ms", - "default": 0.8 - } - } - }, - "placeholder__distance_dependent": { - "algorithm": "placeholder", - "version": "v3", - "params": { - "weight": { - "type": "float32", - "unitCode": "#synapses/connection", - "default": 0 - }, - "exponent": { - "type": "float32", - "unitCode": "1/um", - "default": 0.008 - }, - "nsynconn_mean": { - "type": "float32", - "unitCode": "#synapses/connection", - "default": 3 - }, - "nsynconn_std": { - "type": "float32", - "unitCode": "#synapses/connection", - "default": 1.5 - }, - "delay_velocity": { - "type": "float32", - "unitCode": "um/ms", - "default": 250 - }, - "delay_offset": { - "type": "float32", - "unitCode": "ms", - "default": 0.8 - } - } - } - }, - "initial": { - "variants": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/a46a442c-5baa-4a5c-9907-bfb359dd9e5d", - "rev": 9, - "type": [ - "Entity", - "Dataset", - "MicroConnectomeVariantSelection" - ] - }, - "placeholder__erdos_renyi": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/microconnectomedata/009413eb-e51b-40bc-9199-8b98bfc53f87", - "rev": 7, - "type": [ - "Entity", - "Dataset", - "MicroConnectomeData" - ] - }, - "placeholder__distance_dependent": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/microconnectomedata/c7e1d215-2dad-4216-8565-6b1e4c161f46", - "rev": 7, - "type": [ - "Entity", - "Dataset", - "MicroConnectomeData" - ] - } - }, - "overrides": { - "variants": { - "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/deee5e86-1d7b-45f6-8fad-259a71c35c6a", - "type": [ - "Entity", - "Dataset", - "MicroConnectomeVariantSelectionOverrides" - ], - "rev": 1 - }, - "placeholder__erdos_renyi": { - "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/36426136-201d-4dfd-93d9-b541e113a6bf", - "type": [ - "Entity", - "Dataset", - "MicroConnectomeDataOverrides" - ], - "rev": 1 - }, - "placeholder__distance_dependent": { - "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/4bb03c2b-b99d-4a5d-8a8b-12e1a30619aa", - "type": [ - "Entity", - "Dataset", - "MicroConnectomeDataOverrides" - ], - "rev": 1 - } - }, - "_ui_data": { - "editHistory": [] - } - } - -The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: - -.. code-block:: json - - { - "version": 2, - "manifest": { - "$BASE_DIR": "." - }, - "node_sets_file": "path/to/nodesets/json", - "networks": { - "nodes": [ - { - "nodes_file": "path/to/nodes/h5", - "populations": { - "root__neurons": { - "type": "biophysical", - "alternate_morphologies": { - "h5v1": "path/to/morphologies/dir", - "neurolucida-asc": "path/to/morphologies/dir" - }, - "biophysical_neuron_models_dir": "path/to/hoc/dir" - } - } - } - ], - "edges": [ - { - "edges_file": "path/to/edges/h5", - "populations": { - "root__neurons__root__neurons__chemical": { - "type": "chemical" - } - } - } - ] - }, - "metadata": { - "status": "partial" - } - } - -where the nodes file has the same properties as in the :ref:`MEModelGenerator ones ` and the edges file has the following properties: - - -.. code-block:: text - - afferent_center_x - afferent_center_y - afferent_center_z - afferent_section_id - afferent_section_pos - afferent_section_type - delay - syn_type_id - -See `SONATA documentation `__ for column definitions. - - -ConnectomeFilteringGenerator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``ConnectomeFilteringGenerator`` takes the ``DetailedCircuit`` from :ref:`MicroConnectomeGenerator ` and filters the according to the configuration. - -.. code-block:: json - - { - "variantDefinition": { - "algorithm": "synapses", - "version": "v2" - }, - "defaults": { - "synapse_properties": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/synapticassignment/d57536aa-d576-4b3b-a89b-b7888f24eb21", - "type": [ - "Dataset", - "SynapticParameterAssignment" - ], - "rev": 9 - }, - "synapses_classification": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/synapticparameters/cf25c2bf-e6e4-4367-acd8-94004bfcfe49", - "type": [ - "Dataset", - "SynapticParameter" - ], - "rev": 6 - } - }, - "configuration": { - "synapse_properties": { - "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/839a8b83-1620-4fe7-8f58-658ded0ea1e8", - "type": [ - "Dataset", - "SynapticParameterAssignment" - ], - "rev": 1 - }, - "synapses_classification": { - "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/d133e408-bd00-41ca-9334-e5fab779ad99", - "type": [ - "Dataset", - "SynapticParameter" - ], - "rev": 3 - } - } - } - -For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ - -The final resource is a simulation-ready ``DetailedCircuit`` where the where the nodes file has the same properties as in the :ref:`MEModelGenerator ones ` and the edges file has the following properties: - -.. code-block:: text - - afferent_center_x - afferent_center_y - afferent_center_z - afferent_section_id - afferent_section_pos - afferent_section_type - conductance - conductance_scale_factor - decay_time - delay - depression_time - facilitation_time - n_rrp_vesicles - syn_property_rule - syn_type_id - u_hill_coefficient - u_syn - -See `SONATA documentation `__ for column definitions. + workflow_components/quickstart.rst + workflow_components/overview.rst + workflow_components/generator_hierarchy.rst + workflow_components/generator_definition_configuration.rst diff --git a/doc/source/workflow_components/generator_definition_configuration.rst b/doc/source/workflow_components/generator_definition_configuration.rst new file mode 100644 index 0000000..ec01769 --- /dev/null +++ b/doc/source/workflow_components/generator_definition_configuration.rst @@ -0,0 +1,678 @@ +Generator: Definitions & Configurations +======================================= + +.. _cell_composition_generator: + +CellCompositionGenerator +~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``CellCompositionGenerator`` is responsible for manipulating an input CellComposition in terms of density. + +.. _cell_composition_config: + +.. code-block:: json + + { + "http://api.brain-map.org/api/v2/data/Structure/997": { + "variantDefinition": { + "algorithm": "cell_composition_manipulation", + "version": "v3" + }, + "inputs": [ + { + "name": "base_cell_composition_id", + "type": "Dataset", + "id": "https://bbp.epfl.ch/neurosciencegraph/data/cellcompositions/54818e46-cf8c-4bd6-9b68-34dffbc8a68c?tag=v1.1.0" + } + ], + "configuration": { + "version": 1, + "unitCode": { + "density": "mm^-3" + }, + "overrides": {} + } + } + } + +To add an override: + +.. code-block:: json + + { + "overrides": { + "http://api.brain-map.org/api/v2/data/Structure/23": { + "label": "Anterior amygdalar area", + "about": "BrainRegion", + "hasPart": { + "https://bbp.epfl.ch/ontologies/core/bmo/GenericExcitatoryNeuronMType?rev=6": { + "label": "GEN_mtype", + "about": "MType", + "hasPart": { + "https://bbp.epfl.ch/ontologies/core/bmo/GenericExcitatoryNeuronEType?rev=6": { + "label": "GEN_etype", + "about": "EType", + "composition": { + "neuron": { + "density": 1200 + } + } + } + } + } + } + } + } + } + + +For the variant definition of the manipulation algorithm refer to `blue-cwl documentation `__. + +The final resource generated by the generator is a new CellComposition incorporating the changes listed in overrides. + +.. _cell_position_generator: + +CellPositionGenerator +~~~~~~~~~~~~~~~~~~~~~ + +The ``CellPositionGenerator`` consumes the upstream CellComposition registered by :ref`CellCompositionGenerator ` and places cell somata, along with properties such as mtypes, and etype. + + +The ``CellPositionConfig`` has the following layout: + + +.. _cell_position_config: + +.. code-block:: json + + { + "http://api.brain-map.org/api/v2/data/Structure/997": { + "variantDefinition": { + "algorithm": "neurons_cell_position", + "version": "v3" + }, + "inputs": [], + "configuration": { + "place_cells": { + "soma_placement": "basic", + "density_factor": 1, + "sort_by": [ + "region", + "mtype" + ], + "seed": 0, + "mini_frequencies": false + } + } + } + } + + +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "node_sets_file": "path/to/nodesets/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + } + } + } + ], + "edges": [] + }, + "metadata": { + "status": "partial" + } + } + +where the node population has the following properties: + +.. code-block:: text + + etype + hemisphere + morph_class + mtype + region + subregion + synapse_class + x + y + z + +See `SONATA documentation `__ for column definitions. + +.. _mmodel_generator: + +MorphologyAssignmentGenerator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``MorphologyAssignmentGenerator`` takes the ``DetailedCircuit`` from :ref:`CellPositionGenerator ` and depending on the configuration assigns or/and synthesizes morphologies for each cell in the node population. + +.. code-block:: json + + { + "variantDefinition": { + "topological_synthesis": { + "algorithm": "topological_synthesis", + "version": "v3" + }, + "placeholder_assignment": { + "algorithm": "placeholder_assignment", + "version": "v3" + } + }, + "defaults": { + "topological_synthesis": { + "@id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/fae6eb46-3007-41c6-af69-941a82aada68", + "@type": "CanonicalMorphologyModelConfig" + }, + "placeholder_assignment": { + "@id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/9503a07d-8337-48eb-8637-acc26b0f13bf", + "@type": "PlaceholderMorphologyConfig" + } + }, + "configuration": { + "topological_synthesis": { + "http://api.brain-map.org/api/v2/data/Structure/23": { + "https://bbp.epfl.ch/ontologies/core/bmo/GenericInhibitoryNeuronMType": {} + } + } + } + } + +For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ + +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "metadata": { + "status": "partial" + }, + "node_sets_file": "path/to/nodests/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + "alternate_morphologies": { + "h5v1": "$BASE_DIR/morphologies", + "neurolucida-asc": "$BASE_DIR/morphologies" + } + } + } + } + ], + "edges": [] + } + } + +where the node population has the following properties: + +.. code-block:: text + + etype + hemisphere + morph_class + morphology + morphology_producer + mtype + orientation_w + orientation_x + orientation_y + orientation_z + region + subregion + synapse_class + x + y + z + +See `SONATA documentation `__ for column definitions. + +.. _me_model_generator: + +MEModelGenerator +~~~~~~~~~~~~~~~~ + +The ``MEModelGenerator`` takes the ``DetailedCircuit`` from :ref:`MorphologyAssignmentGenerator ` and assigns emodels and emodel properties. + +.. _me_model_config: + +.. code-block:: json + + { + "variantDefinition": { + "neurons_me_model": { + "algorithm": "neurons_me_model", + "version": "v3" + } + }, + "defaults": { + "neurons_me_model": { + "@id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/2ec96e9f-7254-44b5-bbcb-fdea3e18f110", + "@type": [ + "PlaceholderEModelConfig", + "Entity" + ] + } + }, + "overrides": { + "neurons_me_model": {} + } + } + + +For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ + +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "node_sets_file": "path/to/nodesets/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + "partial": [ + "cell-properties", + "morphologies" + ], + "alternate_morphologies": { + "h5v1": "path/to/morphologies/dir", + "neurolucida-asc": "path/to/morphologies/dir" + }, + "biophysical_neuron_models_dir": "path/to/hoc/dir" + } + } + } + ], + "edges": [] + }, + "metadata": { + "status": "partial" + } + } + +where the node population has the following properties: + +.. _me_model_properties: + +.. code-block:: text + + dynamics_params/AIS_scaler + dynamics_params/holding_current + dynamics_params/input_resistance + dynamics_params/resting_potential + dynamics_params/soma_scaler + dynamics_params/threshold_current + etype + hemisphere + model_template + morph_class + morphology + morphology_producer + mtype + orientation_w + orientation_x + orientation_y + orientation_z + region + subregion + synapse_class + x + y + z + +See `SONATA documentation `__ for column definitions. + +.. _macro_generator: + +MacroConnectomeGenerator +~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``MacroConnectomeGenerator`` is a :ref:`relay generator ` that propagates its config downstream to :ref:`MicroConnectomeGenerator `. + +.. _macro_config: + +.. code-block:: + + { + "initial": { + "connection_strength": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/connectomestrength/8e285d4b-4d09-4357-98ae-9e9fc61face6", + "type": [ + "Entity", + "Dataset", + "WholeBrainConnectomeStrength" + ], + "rev": 10 + } + }, + "overrides": { + "connection_strength": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/wholebrainconnectomestrengths/9357f9b4-8e94-45cd-b701-8d18648a17a6", + "type": [ + "Entity", + "Dataset", + "WholeBrainConnectomeStrength" + ], + "rev": 1 + } + }, + "_ui_data": { + "editHistory": [] + } + } + +The final resource of `MacroConnectomeGenerator` is a clone of its input config. + + +.. _micro_generator: + +MicroConnectomeGenerator +~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``MicroConnectomeGenerator`` takes as an input the :ref:`MacroConnectomeConfig ` from :ref:`MacroConnectomeGenerator ` and the ``DetailedCircuit`` from :ref:`MEModelGenerator ` and establishes the connectivity of the node population. + + +The ``MicroConnectomeConfig`` has the following layout: + +.. _micro_config: + +.. code-block:: + + { + "variants": { + "placeholder__erdos_renyi": { + "algorithm": "placeholder", + "version": "v3", + "params": { + "weight": { + "type": "float32", + "unitCode": "#synapses/connection", + "default": 0 + }, + "nsynconn_mean": { + "type": "float32", + "unitCode": "#synapses/connection", + "default": 3 + }, + "nsynconn_std": { + "type": "float32", + "unitCode": "#synapses/connection", + "default": 1.5 + }, + "delay_velocity": { + "type": "float32", + "unitCode": "um/ms", + "default": 250 + }, + "delay_offset": { + "type": "float32", + "unitCode": "ms", + "default": 0.8 + } + } + }, + "placeholder__distance_dependent": { + "algorithm": "placeholder", + "version": "v3", + "params": { + "weight": { + "type": "float32", + "unitCode": "#synapses/connection", + "default": 0 + }, + "exponent": { + "type": "float32", + "unitCode": "1/um", + "default": 0.008 + }, + "nsynconn_mean": { + "type": "float32", + "unitCode": "#synapses/connection", + "default": 3 + }, + "nsynconn_std": { + "type": "float32", + "unitCode": "#synapses/connection", + "default": 1.5 + }, + "delay_velocity": { + "type": "float32", + "unitCode": "um/ms", + "default": 250 + }, + "delay_offset": { + "type": "float32", + "unitCode": "ms", + "default": 0.8 + } + } + } + }, + "initial": { + "variants": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/a46a442c-5baa-4a5c-9907-bfb359dd9e5d", + "rev": 9, + "type": [ + "Entity", + "Dataset", + "MicroConnectomeVariantSelection" + ] + }, + "placeholder__erdos_renyi": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/microconnectomedata/009413eb-e51b-40bc-9199-8b98bfc53f87", + "rev": 7, + "type": [ + "Entity", + "Dataset", + "MicroConnectomeData" + ] + }, + "placeholder__distance_dependent": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/microconnectomedata/c7e1d215-2dad-4216-8565-6b1e4c161f46", + "rev": 7, + "type": [ + "Entity", + "Dataset", + "MicroConnectomeData" + ] + } + }, + "overrides": { + "variants": { + "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/deee5e86-1d7b-45f6-8fad-259a71c35c6a", + "type": [ + "Entity", + "Dataset", + "MicroConnectomeVariantSelectionOverrides" + ], + "rev": 1 + }, + "placeholder__erdos_renyi": { + "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/36426136-201d-4dfd-93d9-b541e113a6bf", + "type": [ + "Entity", + "Dataset", + "MicroConnectomeDataOverrides" + ], + "rev": 1 + }, + "placeholder__distance_dependent": { + "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/4bb03c2b-b99d-4a5d-8a8b-12e1a30619aa", + "type": [ + "Entity", + "Dataset", + "MicroConnectomeDataOverrides" + ], + "rev": 1 + } + }, + "_ui_data": { + "editHistory": [] + } + } + +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "node_sets_file": "path/to/nodesets/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + "alternate_morphologies": { + "h5v1": "path/to/morphologies/dir", + "neurolucida-asc": "path/to/morphologies/dir" + }, + "biophysical_neuron_models_dir": "path/to/hoc/dir" + } + } + } + ], + "edges": [ + { + "edges_file": "path/to/edges/h5", + "populations": { + "root__neurons__root__neurons__chemical": { + "type": "chemical" + } + } + } + ] + }, + "metadata": { + "status": "partial" + } + } + +where the nodes file has the same properties as in the :ref:`MEModelGenerator ones ` and the edges file has the following properties: + + +.. code-block:: text + + afferent_center_x + afferent_center_y + afferent_center_z + afferent_section_id + afferent_section_pos + afferent_section_type + delay + syn_type_id + +See `SONATA documentation `__ for column definitions. + + +ConnectomeFilteringGenerator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``ConnectomeFilteringGenerator`` takes the ``DetailedCircuit`` from :ref:`MicroConnectomeGenerator ` and filters the according to the configuration. + +.. code-block:: json + + { + "variantDefinition": { + "algorithm": "synapses", + "version": "v2" + }, + "defaults": { + "synapse_properties": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/synapticassignment/d57536aa-d576-4b3b-a89b-b7888f24eb21", + "type": [ + "Dataset", + "SynapticParameterAssignment" + ], + "rev": 9 + }, + "synapses_classification": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/synapticparameters/cf25c2bf-e6e4-4367-acd8-94004bfcfe49", + "type": [ + "Dataset", + "SynapticParameter" + ], + "rev": 6 + } + }, + "configuration": { + "synapse_properties": { + "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/839a8b83-1620-4fe7-8f58-658ded0ea1e8", + "type": [ + "Dataset", + "SynapticParameterAssignment" + ], + "rev": 1 + }, + "synapses_classification": { + "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/d133e408-bd00-41ca-9334-e5fab779ad99", + "type": [ + "Dataset", + "SynapticParameter" + ], + "rev": 3 + } + } + } + +For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ + +The final resource is a simulation-ready ``DetailedCircuit`` where the where the nodes file has the same properties as in the :ref:`MEModelGenerator ones ` and the edges file has the following properties: + +.. code-block:: text + + afferent_center_x + afferent_center_y + afferent_center_z + afferent_section_id + afferent_section_pos + afferent_section_type + conductance + conductance_scale_factor + decay_time + delay + depression_time + facilitation_time + n_rrp_vesicles + syn_property_rule + syn_type_id + u_hill_coefficient + u_syn + +See `SONATA documentation `__ for column definitions. diff --git a/doc/source/workflow_components/generator_hierarchy.rst b/doc/source/workflow_components/generator_hierarchy.rst new file mode 100644 index 0000000..48befc5 --- /dev/null +++ b/doc/source/workflow_components/generator_hierarchy.rst @@ -0,0 +1,191 @@ +Generator: Hierarchy +==================== + +.. _generators: + +.. graphviz:: + + digraph SBOWorkflow{ + + ModelBuildingConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + CellCompositionConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + CellPositionConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + MorphologyAssignmentConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + MEModelConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + MacroConnectomeConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + MacroConnectomeConfig2 [ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "MacroConnectomeConfig" + ] + + MicroConnectomeConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + SynapseConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + CellComposition [ + shape = Mrecord style = filled fillcolor = lemonchiffon + target = "_top" + ] + + CellPositionDetailedCircuit [ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + MModelDetailedCircuit [ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + MEModelDetailedCircuit[ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + MicroDetailedCircuit[ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + FiltDetailedCircuit[ + shape = Mrecord style = filled fillcolor = lemonchiffon + label = "DetailedCircuit" + ] + + CellCompositionActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + CellPositionActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + MModelActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + MEModelActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + MacroActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + MicroActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + FiltActivity [ + shape = record style = filled fillcolor = lightblue + label = "GeneratorTaskActivity" + ] + + CellCompositionGenerator [ + shape = Mrecord color = black + ] + + CellPositionGenerator [ + shape = Mrecord color = black + ] + + MorphologyAssignmentGenerator [ + shape = Mrecord color = black + ] + + MEModelGenerator [ + shape = Mrecord color = black + ] + + MacroConnectomeGenerator [ + shape = Mrecord color = black + ] + + MicroConnectomeGenerator [ + shape = Mrecord color = black + ] + + ConnectomeFilteringGenerator [ + shape = Mrecord color = black + ] + + + ModelBuildingConfig -> CellCompositionConfig; + ModelBuildingConfig -> CellPositionConfig; + ModelBuildingConfig -> MorphologyAssignmentConfig; + ModelBuildingConfig -> MEModelConfig; + ModelBuildingConfig -> MacroConnectomeConfig; + ModelBuildingConfig -> MicroConnectomeConfig; + ModelBuildingConfig -> SynapseConfig; + + ModelBuildingConfig -> CellCompositionGenerator; + CellCompositionGenerator -> CellCompositionActivity [label = "target"]; + CellCompositionActivity -> CellComposition [label = "generated"]; + CellCompositionActivity -> CellCompositionConfig [label = "used_config"]; + CellComposition -> CellPositionGenerator; + + ModelBuildingConfig -> CellPositionGenerator; + CellPositionGenerator -> CellPositionActivity [label = "target"]; + CellPositionActivity -> CellPositionDetailedCircuit [label = "generated"]; + CellPositionActivity -> CellPositionConfig [label = "used_config"]; + CellPositionDetailedCircuit -> MorphologyAssignmentGenerator; + + + ModelBuildingConfig -> MorphologyAssignmentGenerator; + MorphologyAssignmentGenerator -> MModelActivity [label = "target"]; + MModelActivity -> MModelDetailedCircuit [label="generated"]; + MModelActivity -> MorphologyAssignmentConfig [label = "used_config"]; + MModelDetailedCircuit -> MEModelGenerator; + + ModelBuildingConfig -> MEModelGenerator; + MEModelGenerator -> MEModelActivity [label = "target"]; + MEModelActivity -> MEModelDetailedCircuit [label = "generated"]; + MEModelActivity -> MEModelConfig [label = "used_config"]; + MEModelDetailedCircuit -> MicroConnectomeGenerator; + + ModelBuildingConfig -> MacroConnectomeGenerator; + MacroConnectomeGenerator -> MacroActivity [label = "target"]; + MacroActivity -> MacroConnectomeConfig2 [label = "generated"]; + MacroActivity -> MacroConnectomeConfig [label = "used_config"]; + MacroConnectomeConfig2 -> MicroConnectomeGenerator; + + ModelBuildingConfig -> MicroConnectomeGenerator; + MicroConnectomeGenerator -> MicroActivity [label = "target"]; + MicroActivity -> MicroDetailedCircuit [label = "generated"]; + MicroActivity -> MicroConnectomeConfig [label = "used_config"]; + MicroDetailedCircuit -> ConnectomeFilteringGenerator; + + ModelBuildingConfig -> ConnectomeFilteringGenerator; + ConnectomeFilteringGenerator -> FiltActivity [label = "target"]; + FiltActivity -> FiltDetailedCircuit [label = "generated"]; + FiltActivity -> SynapseConfig [label = "used_config"]; + + } diff --git a/doc/source/workflow_components/overview.rst b/doc/source/workflow_components/overview.rst new file mode 100644 index 0000000..035e197 --- /dev/null +++ b/doc/source/workflow_components/overview.rst @@ -0,0 +1,171 @@ +Overview +======== + +.. _workflow: + +The Workflow Task +----------------- + +The :class:`Workflow ` task receives a :ref:`ModelBuildingConfig ` resource id and a target task name. + +The target task will trigger the execution of all its required upstream tasks, ensuring that each necessary dependency is processed in sequence until the target task is reached and executed. + +For example, if cellPositionConfig is selected as the target task, Luigi will first process the ``CellPositionGenerator`` requirements, which directly depends on ``CellCompositionGenerator``. Therefore ``CellCompositionGenerator`` will be executed before ``CellPositionGenerator`` can run. + + +.. _model_config: + +ModelBuildingConfig +------------------- + +The ModelBuildingConfig incorporates all the generator configs as follows: + + +.. graphviz:: + + digraph foo{ + + rankdir = "LR" + splines = "ortho" + + ModelBuildingConfig[ + shape = Mrecord style = filled fillcolor = lemonchiffon + ] + + CellCompositionConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + CellPositionConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + MorphologyAssignmentConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + MEModelConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + MacroConnectomeConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + MicroConnectomeConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + SynapseConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 3 + ] + + ModelBuildingConfig -> CellCompositionConfig [label="configs[cellCompositionConfig]", labelheight=2]; + ModelBuildingConfig -> CellPositionConfig [label="configs[cellPositionConfig]"]; + ModelBuildingConfig -> MorphologyAssignmentConfig [label="configs[morphologyAssignmentConfig]"]; + ModelBuildingConfig -> MEModelConfig [label="configs[meModelConfig]"]; + ModelBuildingConfig -> MacroConnectomeConfig [label="configs[macroConnectomeConfig]"]; + ModelBuildingConfig -> MicroConnectomeConfig [label="configs[microConnectomeConfig]"]; + ModelBuildingConfig -> SynapseConfig [label="configs[synapseConfig]"]; + + + } + + +.. note:: + + It is not necessary for the config to include all the generator configs, however it must contain all configs up until the target task selected when executing the :ref:`workflow `. + + +.. _generator_layout: + +Generator: Task Layout +---------------------- + +A Generator is a Luigi Task in a predefined hierarchy of :ref:`generators ` which has NEXUS resources as inputs and outputs. + +.. graphviz:: + + + digraph generator_layout { + + rankdir = "LR" + + ModelBuildingConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 2 + ] + GeneratorConfig [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 2 + ] + + UpstreamResource [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 2 + ] + Generator [ + shape = Mrecord color = black + label = "{Generator|main_config_url\lgenerator_config_name}" + width = 2 + ] + GeneratorTaskActivity [ + shape = record style = filled fillcolor = lightblue + width = 2 + ] + Resource [ + shape = Mrecord style = filled fillcolor = lemonchiffon + width = 2 + ] + + ModelBuildingConfig -> Generator; + ModelBuildingConfig -> GeneratorConfig; + UpstreamResource -> Generator; + Generator -> GeneratorTaskActivity [label = "target"]; + GeneratorTaskActivity -> Resource [label = "generated"] + GeneratorTaskActivity -> GeneratorConfig [label = "used_config"] + + } + +The Generator produces an activity with the generated entity, registered to the knowledge graph. The Generated is completed if a target entity can be found in the database with the specific ``used_config`` input. + + +.. note:: + + Since the workflow registers and searches for resources in the knowledge graph, retriggering a task requires deprecating the corresponding activity associated with the input configuration. Without this, the target will always be found, and Luigi will consider the task as already completed, preventing re-execution. + + +.. _generator_anatomy: + +Generator: Anatomy & Variants +----------------------------- + +.. _generator_types: + +Types +~~~~~ + +There are two main types of generators: + +* Relay Generators +* Multi Variant Generators + +Relay generators are simple tasks that propagate the input config downstream by creating an activity with a clone of the config. An example of a Relay generator is the MacroConnectomeGenerator. + +Multi variant generators are the most common tasks, scattering variant tasks and then merging them to produce the final result. Each Generator that derives from MultiVariantGenerator implements a +scatter and optionally a merge method. + +Variants +~~~~~~~~ + +A Generator may launch one or more variant tasks. A variant is an executable tool identified by the triplet ``(generator_name, variant_name, version)``. For more info see the `variant documentation +`_. + +.. _blue_cwl_variant: https://blue-cwl.readthedocs.io/en/latest/concepts/variant.html#what-is-a-variant diff --git a/doc/source/workflow_components/quickstart.rst b/doc/source/workflow_components/quickstart.rst new file mode 100644 index 0000000..ccbe5c4 --- /dev/null +++ b/doc/source/workflow_components/quickstart.rst @@ -0,0 +1,41 @@ +Quickstart +========== + +Launching a building workflow can be achieved with the following command: + +.. code-block:: shell + + ./bbp-workflow-launch.sh bbp_workflow.generation.workflow.SBOWorkflow \ + --config-url $CONFIG_URL \ + --output-dir $OUTPUT_DIR \ + --host bbpv1.epfl.ch \ + --kg-base $NEXUS_BASE \ + --kg-org $NEXUS_ORG \ + --account $ACCOUNT \ + --workers 1 + +Or respectively from the bbp-workflow server: + +.. code-block:: shell + + bbp-workflow launch --config $LUIGI_CFG -f bbp_workflow.generation SBOWorkflow \ + config-url=$CONFIG_URL \ + target=cellPositionConfig \ + output-dir=$OUTPUT_DIR \ + host=$HOST \ + account=$ACCOUNT \ + +where $LUIGI_CFG is structured as follows: + +.. code-block:: text + + [DEFAULT] + workers: 1 + kg-base: https://staging.nise.bbp.epfl.ch/nexus/v1 + kg-org: bbp + kg-proj: mmb-point-neuron-framework-model + + [SBOWorkflow] + kg-base: https://staging.nise.bbp.epfl.ch/nexus/v1 + kg-org: bbp + kg-proj: mmb-point-neuron-framework-model From 85c780b47290cb96425d72ed7d98a3cfea584698 Mon Sep 17 00:00:00 2001 From: Eleftherios Zisis Date: Wed, 18 Sep 2024 15:48:59 +0200 Subject: [PATCH 4/6] Rearrange more --- doc/source/workflow.rst | 3 +- ...ation.rst => generator_configurations.rst} | 325 ++---------------- .../generator_definitions.rst | 322 +++++++++++++++++ .../generator_hierarchy.rst | 4 +- 4 files changed, 349 insertions(+), 305 deletions(-) rename doc/source/workflow_components/{generator_definition_configuration.rst => generator_configurations.rst} (55%) create mode 100644 doc/source/workflow_components/generator_definitions.rst diff --git a/doc/source/workflow.rst b/doc/source/workflow.rst index c949357..e20eaf6 100644 --- a/doc/source/workflow.rst +++ b/doc/source/workflow.rst @@ -6,4 +6,5 @@ The Building Workflow workflow_components/quickstart.rst workflow_components/overview.rst workflow_components/generator_hierarchy.rst - workflow_components/generator_definition_configuration.rst + workflow_components/generator_definitions.rst + workflow_components/generator_configurations.rst diff --git a/doc/source/workflow_components/generator_definition_configuration.rst b/doc/source/workflow_components/generator_configurations.rst similarity index 55% rename from doc/source/workflow_components/generator_definition_configuration.rst rename to doc/source/workflow_components/generator_configurations.rst index ec01769..85619e2 100644 --- a/doc/source/workflow_components/generator_definition_configuration.rst +++ b/doc/source/workflow_components/generator_configurations.rst @@ -1,15 +1,11 @@ -Generator: Definitions & Configurations -======================================= - -.. _cell_composition_generator: - -CellCompositionGenerator -~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``CellCompositionGenerator`` is responsible for manipulating an input CellComposition in terms of density. +Generator Configurations +======================== .. _cell_composition_config: +CellCompositionConfig +--------------------- + .. code-block:: json { @@ -65,24 +61,16 @@ To add an override: } } - For the variant definition of the manipulation algorithm refer to `blue-cwl documentation `__. -The final resource generated by the generator is a new CellComposition incorporating the changes listed in overrides. - -.. _cell_position_generator: - -CellPositionGenerator -~~~~~~~~~~~~~~~~~~~~~ -The ``CellPositionGenerator`` consumes the upstream CellComposition registered by :ref`CellCompositionGenerator ` and places cell somata, along with properties such as mtypes, and etype. +.. _cell_position_config: +CellPositionConfig +------------------ The ``CellPositionConfig`` has the following layout: - -.. _cell_position_config: - .. code-block:: json { @@ -108,57 +96,10 @@ The ``CellPositionConfig`` has the following layout: } -The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: - -.. code-block:: json - - { - "version": 2, - "manifest": { - "$BASE_DIR": "." - }, - "node_sets_file": "path/to/nodesets/json", - "networks": { - "nodes": [ - { - "nodes_file": "path/to/nodes/h5", - "populations": { - "root__neurons": { - "type": "biophysical", - } - } - } - ], - "edges": [] - }, - "metadata": { - "status": "partial" - } - } - -where the node population has the following properties: +.. _mmodel_config: -.. code-block:: text - - etype - hemisphere - morph_class - mtype - region - subregion - synapse_class - x - y - z - -See `SONATA documentation `__ for column definitions. - -.. _mmodel_generator: - -MorphologyAssignmentGenerator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``MorphologyAssignmentGenerator`` takes the ``DetailedCircuit`` from :ref:`CellPositionGenerator ` and depending on the configuration assigns or/and synthesizes morphologies for each cell in the node population. +MorphologyAssignmentConfig +-------------------------- .. code-block:: json @@ -194,70 +135,11 @@ The ``MorphologyAssignmentGenerator`` takes the ``DetailedCircuit`` from :ref:`C For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ -The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: - -.. code-block:: json - - { - "version": 2, - "manifest": { - "$BASE_DIR": "." - }, - "metadata": { - "status": "partial" - }, - "node_sets_file": "path/to/nodests/json", - "networks": { - "nodes": [ - { - "nodes_file": "path/to/nodes/h5", - "populations": { - "root__neurons": { - "type": "biophysical", - "alternate_morphologies": { - "h5v1": "$BASE_DIR/morphologies", - "neurolucida-asc": "$BASE_DIR/morphologies" - } - } - } - } - ], - "edges": [] - } - } - -where the node population has the following properties: - -.. code-block:: text - - etype - hemisphere - morph_class - morphology - morphology_producer - mtype - orientation_w - orientation_x - orientation_y - orientation_z - region - subregion - synapse_class - x - y - z - -See `SONATA documentation `__ for column definitions. - -.. _me_model_generator: - -MEModelGenerator -~~~~~~~~~~~~~~~~ - -The ``MEModelGenerator`` takes the ``DetailedCircuit`` from :ref:`MorphologyAssignmentGenerator ` and assigns emodels and emodel properties. - .. _me_model_config: +MEModelConfig +------------- + .. code-block:: json { @@ -284,84 +166,12 @@ The ``MEModelGenerator`` takes the ``DetailedCircuit`` from :ref:`MorphologyAssi For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ -The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: - -.. code-block:: json - - { - "version": 2, - "manifest": { - "$BASE_DIR": "." - }, - "node_sets_file": "path/to/nodesets/json", - "networks": { - "nodes": [ - { - "nodes_file": "path/to/nodes/h5", - "populations": { - "root__neurons": { - "type": "biophysical", - "partial": [ - "cell-properties", - "morphologies" - ], - "alternate_morphologies": { - "h5v1": "path/to/morphologies/dir", - "neurolucida-asc": "path/to/morphologies/dir" - }, - "biophysical_neuron_models_dir": "path/to/hoc/dir" - } - } - } - ], - "edges": [] - }, - "metadata": { - "status": "partial" - } - } - -where the node population has the following properties: - -.. _me_model_properties: - -.. code-block:: text - - dynamics_params/AIS_scaler - dynamics_params/holding_current - dynamics_params/input_resistance - dynamics_params/resting_potential - dynamics_params/soma_scaler - dynamics_params/threshold_current - etype - hemisphere - model_template - morph_class - morphology - morphology_producer - mtype - orientation_w - orientation_x - orientation_y - orientation_z - region - subregion - synapse_class - x - y - z - -See `SONATA documentation `__ for column definitions. - -.. _macro_generator: - -MacroConnectomeGenerator -~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``MacroConnectomeGenerator`` is a :ref:`relay generator ` that propagates its config downstream to :ref:`MicroConnectomeGenerator `. .. _macro_config: +MacroConnetomeConfig +-------------------- + .. code-block:: { @@ -392,21 +202,14 @@ The ``MacroConnectomeGenerator`` is a :ref:`relay generator ` t } } -The final resource of `MacroConnectomeGenerator` is a clone of its input config. +.. _micro_config: -.. _micro_generator: - -MicroConnectomeGenerator -~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``MicroConnectomeGenerator`` takes as an input the :ref:`MacroConnectomeConfig ` from :ref:`MacroConnectomeGenerator ` and the ``DetailedCircuit`` from :ref:`MEModelGenerator ` and establishes the connectivity of the node population. - +MicroConnectomeConfig +--------------------- The ``MicroConnectomeConfig`` has the following layout: -.. _micro_config: - .. code-block:: { @@ -542,69 +345,11 @@ The ``MicroConnectomeConfig`` has the following layout: } } -The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: - -.. code-block:: json - - { - "version": 2, - "manifest": { - "$BASE_DIR": "." - }, - "node_sets_file": "path/to/nodesets/json", - "networks": { - "nodes": [ - { - "nodes_file": "path/to/nodes/h5", - "populations": { - "root__neurons": { - "type": "biophysical", - "alternate_morphologies": { - "h5v1": "path/to/morphologies/dir", - "neurolucida-asc": "path/to/morphologies/dir" - }, - "biophysical_neuron_models_dir": "path/to/hoc/dir" - } - } - } - ], - "edges": [ - { - "edges_file": "path/to/edges/h5", - "populations": { - "root__neurons__root__neurons__chemical": { - "type": "chemical" - } - } - } - ] - }, - "metadata": { - "status": "partial" - } - } - -where the nodes file has the same properties as in the :ref:`MEModelGenerator ones ` and the edges file has the following properties: +.. _synapse_config: -.. code-block:: text - - afferent_center_x - afferent_center_y - afferent_center_z - afferent_section_id - afferent_section_pos - afferent_section_type - delay - syn_type_id - -See `SONATA documentation `__ for column definitions. - - -ConnectomeFilteringGenerator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``ConnectomeFilteringGenerator`` takes the ``DetailedCircuit`` from :ref:`MicroConnectomeGenerator ` and filters the according to the configuration. +SynapseConfig +------------- .. code-block:: json @@ -652,27 +397,3 @@ The ``ConnectomeFilteringGenerator`` takes the ``DetailedCircuit`` from :ref:`Mi } For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ - -The final resource is a simulation-ready ``DetailedCircuit`` where the where the nodes file has the same properties as in the :ref:`MEModelGenerator ones ` and the edges file has the following properties: - -.. code-block:: text - - afferent_center_x - afferent_center_y - afferent_center_z - afferent_section_id - afferent_section_pos - afferent_section_type - conductance - conductance_scale_factor - decay_time - delay - depression_time - facilitation_time - n_rrp_vesicles - syn_property_rule - syn_type_id - u_hill_coefficient - u_syn - -See `SONATA documentation `__ for column definitions. diff --git a/doc/source/workflow_components/generator_definitions.rst b/doc/source/workflow_components/generator_definitions.rst new file mode 100644 index 0000000..2dc1fd8 --- /dev/null +++ b/doc/source/workflow_components/generator_definitions.rst @@ -0,0 +1,322 @@ +Generator Definitions +====================== + +.. _cell_composition_generator: + +CellCompositionGenerator +~~~~~~~~~~~~~~~~~~~~~~~~ + +Config: :ref:`CellCompositionConfig ` + +The ``CellCompositionGenerator`` is responsible for manipulating an input CellComposition in terms of density. + + +The final resource generated by the generator is a new CellComposition incorporating the changes listed in overrides. + + +.. _cell_position_generator: + +CellPositionGenerator +~~~~~~~~~~~~~~~~~~~~~ + +Config: :ref:`CellPositionConfig ` + +The ``CellPositionGenerator`` consumes the upstream CellComposition registered by :ref`CellCompositionGenerator ` and places cell somata, along with properties such as mtypes, and etype. + +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "node_sets_file": "path/to/nodesets/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + } + } + } + ], + "edges": [] + }, + "metadata": { + "status": "partial" + } + } + +where the node population has the following properties: + +.. code-block:: text + + etype + hemisphere + morph_class + mtype + region + subregion + synapse_class + x + y + z + +See `SONATA documentation `__ for column definitions. + +.. _mmodel_generator: + +MorphologyAssignmentGenerator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Config: :ref:`MorphologyAssignmentConfig ` + +The ``MorphologyAssignmentGenerator`` takes the ``DetailedCircuit`` from :ref:`CellPositionGenerator ` and depending on the configuration assigns or/and synthesizes morphologies for each cell in the node population. + +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "metadata": { + "status": "partial" + }, + "node_sets_file": "path/to/nodests/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + "alternate_morphologies": { + "h5v1": "$BASE_DIR/morphologies", + "neurolucida-asc": "$BASE_DIR/morphologies" + } + } + } + } + ], + "edges": [] + } + } + +where the node population has the following properties: + +.. code-block:: text + + etype + hemisphere + morph_class + morphology + morphology_producer + mtype + orientation_w + orientation_x + orientation_y + orientation_z + region + subregion + synapse_class + x + y + z + +See `SONATA documentation `__ for column definitions. + + +.. _me_model_generator: + +MEModelGenerator +~~~~~~~~~~~~~~~~ + +Config: :ref:`MEModelConfig ` + +The ``MEModelGenerator`` takes the ``DetailedCircuit`` from :ref:`MorphologyAssignmentGenerator ` and assigns emodels and emodel properties. + +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "node_sets_file": "path/to/nodesets/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + "partial": [ + "cell-properties", + "morphologies" + ], + "alternate_morphologies": { + "h5v1": "path/to/morphologies/dir", + "neurolucida-asc": "path/to/morphologies/dir" + }, + "biophysical_neuron_models_dir": "path/to/hoc/dir" + } + } + } + ], + "edges": [] + }, + "metadata": { + "status": "partial" + } + } + +where the node population has the following properties: + +.. _me_model_properties: + +.. code-block:: text + + dynamics_params/AIS_scaler + dynamics_params/holding_current + dynamics_params/input_resistance + dynamics_params/resting_potential + dynamics_params/soma_scaler + dynamics_params/threshold_current + etype + hemisphere + model_template + morph_class + morphology + morphology_producer + mtype + orientation_w + orientation_x + orientation_y + orientation_z + region + subregion + synapse_class + x + y + z + +See `SONATA documentation `__ for column definitions. + +.. _macro_generator: + +MacroConnectomeGenerator +~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``MacroConnectomeGenerator`` is a :ref:`relay generator ` that propagates its config downstream to :ref:`MicroConnectomeGenerator `. + +The final resource of `MacroConnectomeGenerator` is a clone of its input config. + + +.. _micro_generator: + +MicroConnectomeGenerator +~~~~~~~~~~~~~~~~~~~~~~~~ + +Config: :ref:`MicroConnectomeConfig ` + +The ``MicroConnectomeGenerator`` takes as an input the :ref:`MacroConnectomeConfig ` from :ref:`MacroConnectomeGenerator ` and the ``DetailedCircuit`` from :ref:`MEModelGenerator ` and establishes the connectivity of the node population. + + +The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: + +.. code-block:: json + + { + "version": 2, + "manifest": { + "$BASE_DIR": "." + }, + "node_sets_file": "path/to/nodesets/json", + "networks": { + "nodes": [ + { + "nodes_file": "path/to/nodes/h5", + "populations": { + "root__neurons": { + "type": "biophysical", + "alternate_morphologies": { + "h5v1": "path/to/morphologies/dir", + "neurolucida-asc": "path/to/morphologies/dir" + }, + "biophysical_neuron_models_dir": "path/to/hoc/dir" + } + } + } + ], + "edges": [ + { + "edges_file": "path/to/edges/h5", + "populations": { + "root__neurons__root__neurons__chemical": { + "type": "chemical" + } + } + } + ] + }, + "metadata": { + "status": "partial" + } + } + +where the nodes file has the same properties as in the :ref:`MEModelGenerator ones ` and the edges file has the following properties: + + +.. code-block:: text + + afferent_center_x + afferent_center_y + afferent_center_z + afferent_section_id + afferent_section_pos + afferent_section_type + delay + syn_type_id + +See `SONATA documentation `__ for column definitions. + + +ConnectomeFilteringGenerator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Config: :ref:`SynapseConfig ` + +The ``ConnectomeFilteringGenerator`` takes the ``DetailedCircuit`` from :ref:`MicroConnectomeGenerator ` and filters the according to the configuration. + +The final resource is a simulation-ready ``DetailedCircuit`` where the where the nodes file has the same properties as in the :ref:`MEModelGenerator ones ` and the edges file has the following properties: + +.. code-block:: text + + afferent_center_x + afferent_center_y + afferent_center_z + afferent_section_id + afferent_section_pos + afferent_section_type + conductance + conductance_scale_factor + decay_time + delay + depression_time + facilitation_time + n_rrp_vesicles + syn_property_rule + syn_type_id + u_hill_coefficient + u_syn + +See `SONATA documentation `__ for column definitions. diff --git a/doc/source/workflow_components/generator_hierarchy.rst b/doc/source/workflow_components/generator_hierarchy.rst index 48befc5..50aeba0 100644 --- a/doc/source/workflow_components/generator_hierarchy.rst +++ b/doc/source/workflow_components/generator_hierarchy.rst @@ -1,5 +1,5 @@ -Generator: Hierarchy -==================== +Generator Graph +=============== .. _generators: From 9302e24bb01ed6cec781bb0521b64a05f7c84eea Mon Sep 17 00:00:00 2001 From: Eleftherios Zisis Date: Mon, 7 Oct 2024 15:16:40 +0200 Subject: [PATCH 5/6] Use entity-management links to config docs --- doc/source/workflow.rst | 1 - .../generator_configurations.rst | 399 ------------------ .../generator_definitions.rst | 16 +- doc/source/workflow_components/overview.rst | 71 +--- 4 files changed, 11 insertions(+), 476 deletions(-) delete mode 100644 doc/source/workflow_components/generator_configurations.rst diff --git a/doc/source/workflow.rst b/doc/source/workflow.rst index e20eaf6..4576dde 100644 --- a/doc/source/workflow.rst +++ b/doc/source/workflow.rst @@ -7,4 +7,3 @@ The Building Workflow workflow_components/overview.rst workflow_components/generator_hierarchy.rst workflow_components/generator_definitions.rst - workflow_components/generator_configurations.rst diff --git a/doc/source/workflow_components/generator_configurations.rst b/doc/source/workflow_components/generator_configurations.rst deleted file mode 100644 index 85619e2..0000000 --- a/doc/source/workflow_components/generator_configurations.rst +++ /dev/null @@ -1,399 +0,0 @@ -Generator Configurations -======================== - -.. _cell_composition_config: - -CellCompositionConfig ---------------------- - -.. code-block:: json - - { - "http://api.brain-map.org/api/v2/data/Structure/997": { - "variantDefinition": { - "algorithm": "cell_composition_manipulation", - "version": "v3" - }, - "inputs": [ - { - "name": "base_cell_composition_id", - "type": "Dataset", - "id": "https://bbp.epfl.ch/neurosciencegraph/data/cellcompositions/54818e46-cf8c-4bd6-9b68-34dffbc8a68c?tag=v1.1.0" - } - ], - "configuration": { - "version": 1, - "unitCode": { - "density": "mm^-3" - }, - "overrides": {} - } - } - } - -To add an override: - -.. code-block:: json - - { - "overrides": { - "http://api.brain-map.org/api/v2/data/Structure/23": { - "label": "Anterior amygdalar area", - "about": "BrainRegion", - "hasPart": { - "https://bbp.epfl.ch/ontologies/core/bmo/GenericExcitatoryNeuronMType?rev=6": { - "label": "GEN_mtype", - "about": "MType", - "hasPart": { - "https://bbp.epfl.ch/ontologies/core/bmo/GenericExcitatoryNeuronEType?rev=6": { - "label": "GEN_etype", - "about": "EType", - "composition": { - "neuron": { - "density": 1200 - } - } - } - } - } - } - } - } - } - -For the variant definition of the manipulation algorithm refer to `blue-cwl documentation `__. - - -.. _cell_position_config: - -CellPositionConfig ------------------- - -The ``CellPositionConfig`` has the following layout: - -.. code-block:: json - - { - "http://api.brain-map.org/api/v2/data/Structure/997": { - "variantDefinition": { - "algorithm": "neurons_cell_position", - "version": "v3" - }, - "inputs": [], - "configuration": { - "place_cells": { - "soma_placement": "basic", - "density_factor": 1, - "sort_by": [ - "region", - "mtype" - ], - "seed": 0, - "mini_frequencies": false - } - } - } - } - - -.. _mmodel_config: - -MorphologyAssignmentConfig --------------------------- - -.. code-block:: json - - { - "variantDefinition": { - "topological_synthesis": { - "algorithm": "topological_synthesis", - "version": "v3" - }, - "placeholder_assignment": { - "algorithm": "placeholder_assignment", - "version": "v3" - } - }, - "defaults": { - "topological_synthesis": { - "@id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/fae6eb46-3007-41c6-af69-941a82aada68", - "@type": "CanonicalMorphologyModelConfig" - }, - "placeholder_assignment": { - "@id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/9503a07d-8337-48eb-8637-acc26b0f13bf", - "@type": "PlaceholderMorphologyConfig" - } - }, - "configuration": { - "topological_synthesis": { - "http://api.brain-map.org/api/v2/data/Structure/23": { - "https://bbp.epfl.ch/ontologies/core/bmo/GenericInhibitoryNeuronMType": {} - } - } - } - } - -For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ - -.. _me_model_config: - -MEModelConfig -------------- - -.. code-block:: json - - { - "variantDefinition": { - "neurons_me_model": { - "algorithm": "neurons_me_model", - "version": "v3" - } - }, - "defaults": { - "neurons_me_model": { - "@id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/2ec96e9f-7254-44b5-bbcb-fdea3e18f110", - "@type": [ - "PlaceholderEModelConfig", - "Entity" - ] - } - }, - "overrides": { - "neurons_me_model": {} - } - } - - -For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ - - -.. _macro_config: - -MacroConnetomeConfig --------------------- - -.. code-block:: - - { - "initial": { - "connection_strength": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/connectomestrength/8e285d4b-4d09-4357-98ae-9e9fc61face6", - "type": [ - "Entity", - "Dataset", - "WholeBrainConnectomeStrength" - ], - "rev": 10 - } - }, - "overrides": { - "connection_strength": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/wholebrainconnectomestrengths/9357f9b4-8e94-45cd-b701-8d18648a17a6", - "type": [ - "Entity", - "Dataset", - "WholeBrainConnectomeStrength" - ], - "rev": 1 - } - }, - "_ui_data": { - "editHistory": [] - } - } - - -.. _micro_config: - -MicroConnectomeConfig ---------------------- - -The ``MicroConnectomeConfig`` has the following layout: - -.. code-block:: - - { - "variants": { - "placeholder__erdos_renyi": { - "algorithm": "placeholder", - "version": "v3", - "params": { - "weight": { - "type": "float32", - "unitCode": "#synapses/connection", - "default": 0 - }, - "nsynconn_mean": { - "type": "float32", - "unitCode": "#synapses/connection", - "default": 3 - }, - "nsynconn_std": { - "type": "float32", - "unitCode": "#synapses/connection", - "default": 1.5 - }, - "delay_velocity": { - "type": "float32", - "unitCode": "um/ms", - "default": 250 - }, - "delay_offset": { - "type": "float32", - "unitCode": "ms", - "default": 0.8 - } - } - }, - "placeholder__distance_dependent": { - "algorithm": "placeholder", - "version": "v3", - "params": { - "weight": { - "type": "float32", - "unitCode": "#synapses/connection", - "default": 0 - }, - "exponent": { - "type": "float32", - "unitCode": "1/um", - "default": 0.008 - }, - "nsynconn_mean": { - "type": "float32", - "unitCode": "#synapses/connection", - "default": 3 - }, - "nsynconn_std": { - "type": "float32", - "unitCode": "#synapses/connection", - "default": 1.5 - }, - "delay_velocity": { - "type": "float32", - "unitCode": "um/ms", - "default": 250 - }, - "delay_offset": { - "type": "float32", - "unitCode": "ms", - "default": 0.8 - } - } - } - }, - "initial": { - "variants": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/a46a442c-5baa-4a5c-9907-bfb359dd9e5d", - "rev": 9, - "type": [ - "Entity", - "Dataset", - "MicroConnectomeVariantSelection" - ] - }, - "placeholder__erdos_renyi": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/microconnectomedata/009413eb-e51b-40bc-9199-8b98bfc53f87", - "rev": 7, - "type": [ - "Entity", - "Dataset", - "MicroConnectomeData" - ] - }, - "placeholder__distance_dependent": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/microconnectomedata/c7e1d215-2dad-4216-8565-6b1e4c161f46", - "rev": 7, - "type": [ - "Entity", - "Dataset", - "MicroConnectomeData" - ] - } - }, - "overrides": { - "variants": { - "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/deee5e86-1d7b-45f6-8fad-259a71c35c6a", - "type": [ - "Entity", - "Dataset", - "MicroConnectomeVariantSelectionOverrides" - ], - "rev": 1 - }, - "placeholder__erdos_renyi": { - "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/36426136-201d-4dfd-93d9-b541e113a6bf", - "type": [ - "Entity", - "Dataset", - "MicroConnectomeDataOverrides" - ], - "rev": 1 - }, - "placeholder__distance_dependent": { - "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/4bb03c2b-b99d-4a5d-8a8b-12e1a30619aa", - "type": [ - "Entity", - "Dataset", - "MicroConnectomeDataOverrides" - ], - "rev": 1 - } - }, - "_ui_data": { - "editHistory": [] - } - } - - -.. _synapse_config: - -SynapseConfig -------------- - -.. code-block:: json - - { - "variantDefinition": { - "algorithm": "synapses", - "version": "v2" - }, - "defaults": { - "synapse_properties": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/synapticassignment/d57536aa-d576-4b3b-a89b-b7888f24eb21", - "type": [ - "Dataset", - "SynapticParameterAssignment" - ], - "rev": 9 - }, - "synapses_classification": { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/synapticparameters/cf25c2bf-e6e4-4367-acd8-94004bfcfe49", - "type": [ - "Dataset", - "SynapticParameter" - ], - "rev": 6 - } - }, - "configuration": { - "synapse_properties": { - "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/839a8b83-1620-4fe7-8f58-658ded0ea1e8", - "type": [ - "Dataset", - "SynapticParameterAssignment" - ], - "rev": 1 - }, - "synapses_classification": { - "id": "https://bbp.epfl.ch/data/bbp/mmb-point-neuron-framework-model/d133e408-bd00-41ca-9334-e5fab779ad99", - "type": [ - "Dataset", - "SynapticParameter" - ], - "rev": 3 - } - } - } - -For the variant definition of the placement algorithm refer to `blue-cwl documentation `__ diff --git a/doc/source/workflow_components/generator_definitions.rst b/doc/source/workflow_components/generator_definitions.rst index 2dc1fd8..df19833 100644 --- a/doc/source/workflow_components/generator_definitions.rst +++ b/doc/source/workflow_components/generator_definitions.rst @@ -6,7 +6,7 @@ Generator Definitions CellCompositionGenerator ~~~~~~~~~~~~~~~~~~~~~~~~ -Config: :ref:`CellCompositionConfig ` +Config: `CellCompositionConfig `_ The ``CellCompositionGenerator`` is responsible for manipulating an input CellComposition in terms of density. @@ -19,7 +19,7 @@ The final resource generated by the generator is a new CellComposition incorpora CellPositionGenerator ~~~~~~~~~~~~~~~~~~~~~ -Config: :ref:`CellPositionConfig ` +Config: `CellPositionConfig `_ The ``CellPositionGenerator`` consumes the upstream CellComposition registered by :ref`CellCompositionGenerator ` and places cell somata, along with properties such as mtypes, and etype. @@ -73,7 +73,7 @@ See `SONATA documentation ` +Config: `MorphologyAssignmentConfig `_ The ``MorphologyAssignmentGenerator`` takes the ``DetailedCircuit`` from :ref:`CellPositionGenerator ` and depending on the configuration assigns or/and synthesizes morphologies for each cell in the node population. @@ -138,7 +138,7 @@ See `SONATA documentation ` +Config: `MEModelConfig `_ The ``MEModelGenerator`` takes the ``DetailedCircuit`` from :ref:`MorphologyAssignmentGenerator ` and assigns emodels and emodel properties. @@ -216,6 +216,8 @@ See `SONATA documentation `_ + The ``MacroConnectomeGenerator`` is a :ref:`relay generator ` that propagates its config downstream to :ref:`MicroConnectomeGenerator `. The final resource of `MacroConnectomeGenerator` is a clone of its input config. @@ -226,9 +228,9 @@ The final resource of `MacroConnectomeGenerator` is a clone of its input config. MicroConnectomeGenerator ~~~~~~~~~~~~~~~~~~~~~~~~ -Config: :ref:`MicroConnectomeConfig ` +Config: `MicroConnectomeConfig `_ -The ``MicroConnectomeGenerator`` takes as an input the :ref:`MacroConnectomeConfig ` from :ref:`MacroConnectomeGenerator ` and the ``DetailedCircuit`` from :ref:`MEModelGenerator ` and establishes the connectivity of the node population. +The ``MicroConnectomeGenerator`` takes as an input the `MacroConnectomeConfig `_ from :ref:`MacroConnectomeGenerator ` and the ``DetailedCircuit`` from :ref:`MEModelGenerator ` and establishes the connectivity of the node population. The final resource generated by the generator is a partial ``DetailedCircuit`` with a circuit config as follows: @@ -293,7 +295,7 @@ See `SONATA documentation ` +Config: `SynapseConfig `_ The ``ConnectomeFilteringGenerator`` takes the ``DetailedCircuit`` from :ref:`MicroConnectomeGenerator ` and filters the according to the configuration. diff --git a/doc/source/workflow_components/overview.rst b/doc/source/workflow_components/overview.rst index 035e197..3188cae 100644 --- a/doc/source/workflow_components/overview.rst +++ b/doc/source/workflow_components/overview.rst @@ -6,82 +6,15 @@ Overview The Workflow Task ----------------- -The :class:`Workflow ` task receives a :ref:`ModelBuildingConfig ` resource id and a target task name. +The :class:`Workflow ` task receives a `ModelBuildingConfig `_ resource id and a target task name. The target task will trigger the execution of all its required upstream tasks, ensuring that each necessary dependency is processed in sequence until the target task is reached and executed. For example, if cellPositionConfig is selected as the target task, Luigi will first process the ``CellPositionGenerator`` requirements, which directly depends on ``CellCompositionGenerator``. Therefore ``CellCompositionGenerator`` will be executed before ``CellPositionGenerator`` can run. - -.. _model_config: - -ModelBuildingConfig -------------------- - -The ModelBuildingConfig incorporates all the generator configs as follows: - - -.. graphviz:: - - digraph foo{ - - rankdir = "LR" - splines = "ortho" - - ModelBuildingConfig[ - shape = Mrecord style = filled fillcolor = lemonchiffon - ] - - CellCompositionConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - CellPositionConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - MorphologyAssignmentConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - MEModelConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - MacroConnectomeConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - MicroConnectomeConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - SynapseConfig [ - shape = Mrecord style = filled fillcolor = lemonchiffon - width = 3 - ] - - ModelBuildingConfig -> CellCompositionConfig [label="configs[cellCompositionConfig]", labelheight=2]; - ModelBuildingConfig -> CellPositionConfig [label="configs[cellPositionConfig]"]; - ModelBuildingConfig -> MorphologyAssignmentConfig [label="configs[morphologyAssignmentConfig]"]; - ModelBuildingConfig -> MEModelConfig [label="configs[meModelConfig]"]; - ModelBuildingConfig -> MacroConnectomeConfig [label="configs[macroConnectomeConfig]"]; - ModelBuildingConfig -> MicroConnectomeConfig [label="configs[microConnectomeConfig]"]; - ModelBuildingConfig -> SynapseConfig [label="configs[synapseConfig]"]; - - - } - - .. note:: - It is not necessary for the config to include all the generator configs, however it must contain all configs up until the target task selected when executing the :ref:`workflow `. + It is not necessary for the ModelBuildingConfig to include all the generator configs, however it must contain all configs up until the target task selected when executing the :ref:`workflow `. .. _generator_layout: From f31607636320e50359e9f4671924c2675fb27dd1 Mon Sep 17 00:00:00 2001 From: Eleftherios Zisis Date: Mon, 7 Oct 2024 15:22:43 +0200 Subject: [PATCH 6/6] Fix format --- .pylintrc | 2 +- bbp_workflow/generation/workflow.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index 01af648..b62d52e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,5 +1,5 @@ [MESSAGES CONTROL] -disable=fixme,invalid-name,len-as-condition,no-else-return,no-member,empty-docstring +disable=fixme,invalid-name,len-as-condition,no-else-return,no-member,empty-docstring,too-many-positional-arguments [FORMAT] # Regexp for a line that is allowed to be longer than the limit. diff --git a/bbp_workflow/generation/workflow.py b/bbp_workflow/generation/workflow.py index 3df1f2b..6e75f6a 100644 --- a/bbp_workflow/generation/workflow.py +++ b/bbp_workflow/generation/workflow.py @@ -57,7 +57,6 @@ def host_config(self) -> dict: def requires(self): """Return required leaf task in the workflow DAG, determined by the `target` argument.""" - generator_class = get_class_from_config_name(self.target) return generator_class(