Switch to blue-cwl>=1.0.0

bbp_workflow/generation/entity.py

@@ -2,7 +2,6 @@
 
 """Entity definitions module."""
 
-from entity_management.core import Entity
 from entity_management.simulation import AttrOf, DataDownload, attributes
 from entity_management.workflow import BbpWorkflowActivity, BbpWorkflowConfig
 
@@ -36,17 +35,6 @@ class GeneratorTaskConfig(BbpWorkflowConfig):
     """Generator Configuration."""
 
 
-@attributes(
-    {
-        "generator_name": AttrOf(str),
-        "variant_name": AttrOf(str),
-        "version": AttrOf(str),
-    }
-)
-class VariantDefinition(Entity):
-    """Variant definition."""
-
-
 @attributes(
     {
         "distribution": AttrOf(DataDownload),

bbp_workflow/generation/generator.py

@@ -55,7 +55,7 @@ def scatter(self):
                         get_region_acronym(region_id),
                     )
                 ),
-                extra_inputs={"region": region_id},
+                extra_inputs={"region_id": region_id},
             )
             tasks[region_id] = task
 
@@ -89,7 +89,7 @@ def scatter(self):
                         get_region_acronym(region_id),
                     )
                 ),
-                extra_inputs={"region": region_id},
+                extra_inputs={"region_id": region_id},
             )
             tasks[region_id] = task
 

bbp_workflow/generation/generator_base.py

@@ -124,7 +124,7 @@ def _variant_upstream_inputs(self):
             return {}
 
         return {
-            input_name: generator_instance.output_resource.get_id()
+            f"{input_name}_id": generator_instance.output_resource.get_id()
             for input_name, generator_instance in requirements.items()
         }
 
@@ -143,7 +143,7 @@ def _variant_inputs(self, variant_id, data: dict, extra_inputs: Optional[dict] =
 
         Note: The variant definition id is added by default to all variants.
         """
-        inputs = {"variant": variant_id}
+        inputs = {"variant_id": variant_id}
 
         # keep only the name and id of the inputs in the data
         if "inputs" in data:
@@ -166,7 +166,7 @@ def _variant_inputs(self, variant_id, data: dict, extra_inputs: Optional[dict] =
                 filename="variant_task_parameterization.json",
                 content_type="application/json",
             )
-            inputs["configuration"] = configuration.get_id()
+            inputs["configuration_id"] = configuration.get_id()
 
         return inputs
 

bbp_workflow/generation/variant.py

@@ -294,6 +294,31 @@ class CommandLineToolTask(luigi.Task):
         visibility=ParameterVisibility.HIDDEN,
     )
 
+    def __repr__(self):
+        """
+        Build a task representation like `MyTask(param1=1.5, param2='5')`.
+
+        Note: CWL definitions are truncated to Tool(label: mylabel, ...)
+        """
+        params = self.get_params()
+        param_values = self.get_param_values(params, [], self.param_kwargs)
+
+        # Build up task id
+        repr_parts = []
+        param_objs = dict(params)
+        for param_name, param_value in param_values:
+            if param_objs[param_name].significant:
+                if param_name == "definition":
+                    repr_parts.append(f'definition={{"label": {param_value.label} ...}}')
+                else:
+                    repr_parts.append(
+                        f"{param_name}={param_objs[param_name].serialize(param_value)}"
+                    )
+
+        task_str = f"{self.get_task_family()}({', '.join(repr_parts)})"
+
+        return task_str
+
     @cached_property
     def output_dir(self):
         """Process output directory."""
@@ -365,6 +390,12 @@ class WorkflowStepTask(CommandLineToolTask):
         visibility=ParameterVisibility.HIDDEN,
     )
 
+    def _generate_log_path(self):
+        """Generate a timestamped log path."""
+        log_dir = create_dir(Path(self.output_dir, "logs"))
+        timestamp = datetime.now().strftime("%Y%m%dT%H%M%S")
+        return log_dir / f"{self.name}__{timestamp}.log"
+
     def requires(self):
         """Get the requirements of the current task from the cwl definition."""
         return {

pyproject.toml

@@ -40,7 +40,7 @@ dependencies = [
   "elephant",
   "neo",
   "pyarrow",
-  "blue-cwl",
+  "blue-cwl>=1.0.0",
   "brayns",
   "pandas",
   "bluepyemodel",

requirements.txt

@@ -16,16 +16,16 @@ asttokens==2.4.1
     #   stack-data
 async-property==0.2.2
     # via python-keycloak
-attrs==23.2.0
+attrs==24.2.0
     # via
     #   entity-management
     #   jsonschema
     #   referencing
-blue-cwl==0.1.0
+blue-cwl==1.0.0
     # via bbp-workflow (setup.py)
-bluepyefe==2.3.27
+bluepyefe==2.3.34
     # via bluepyemodel
-bluepyemodel==1.0.10
+bluepyemodel==1.0.15
     # via bbp-workflow (setup.py)
 bluepyopt==1.14.13
     # via bluepyemodel
@@ -35,7 +35,7 @@ bluepysnap==3.0.1
     #   brainbuilder
 brainbuilder==0.20.1
     # via blue-cwl
-brayns==3.8.2
+brayns==3.8.3
     # via bbp-workflow (setup.py)
 cached-property==1.5.2
     # via
@@ -46,7 +46,7 @@ certifi==2024.7.4
     #   httpcore
     #   httpx
     #   requests
-cffi==1.16.0
+cffi==1.17.0
     # via cryptography
 charset-normalizer==3.3.2
     # via requests
@@ -67,15 +67,15 @@ configparser==7.0.0
     # via bluepyemodel
 contourpy==1.2.1
     # via matplotlib
-cryptography==42.0.8
+cryptography==43.0.0
     # via jwcrypto
-currentscape==1.0.13
+currentscape==1.0.15
     # via bluepyemodel
 cycler==0.12.1
     # via matplotlib
 deap==1.4.1
     # via bluepyopt
-debugpy==1.8.2
+debugpy==1.8.5
     # via ipykernel
 decorator==5.1.1
     # via
@@ -89,7 +89,7 @@ devtools[pygments]==0.12.2
     # via entity-management
 docutils==0.21.2
     # via python-daemon
-efel==5.7.2
+efel==5.7.6
     # via
     #   bluepyefe
     #   bluepyemodel
@@ -132,7 +132,7 @@ h5py==3.11.0
     #   hdmf
     #   pynwb
     #   voxcell
-hdmf==3.14.2
+hdmf==3.14.3
     # via pynwb
 httpcore==1.0.5
     # via httpx
@@ -145,7 +145,7 @@ idna==3.7
     #   requests
 igor2==0.5.8
     # via bluepyefe
-importlib-resources==6.4.0
+importlib-resources==6.4.2
     # via bluepysnap
 ipykernel==6.29.5
     # via ipyparallel
@@ -192,7 +192,7 @@ jwcrypto==1.5.6
     # via python-keycloak
 kiwisolver==1.4.5
     # via matplotlib
-libsonata==0.1.27
+libsonata==0.1.28
     # via
     #   bbp-workflow (setup.py)
     #   blue-cwl
@@ -208,13 +208,13 @@ luigi==3.5.1
     #   luigi-tools
 luigi-tools==0.3.4
     # via bbp-workflow (setup.py)
-lxml==5.2.2
+lxml==5.3.0
     # via
     #   brainbuilder
     #   fz-td-recipe
 markupsafe==2.1.5
     # via jinja2
-matplotlib==3.9.1
+matplotlib==3.9.2
     # via
     #   bbp-workflow (setup.py)
     #   bluepyefe
@@ -224,7 +224,7 @@ matplotlib-inline==0.1.7
     # via
     #   ipykernel
     #   ipython
-more-itertools==10.3.0
+more-itertools==10.4.0
     # via
     #   bluepysnap
     #   morph-tool
@@ -240,7 +240,7 @@ morphio==3.3.9
     #   brainbuilder
     #   morph-tool
     #   neurom
-neo==0.13.1
+neo==0.13.2
     # via
     #   bbp-workflow (setup.py)
     #   bluepyefe
@@ -252,7 +252,7 @@ neurom==4.0.2
     # via
     #   bluepyemodel
     #   morph-tool
-neuron==8.2.4
+neuron==8.2.6
     # via
     #   bluepyemodel
     #   bluepyopt
@@ -341,9 +341,9 @@ psutil==6.0.0
     #   ipyparallel
 ptyprocess==0.7.0
     # via pexpect
-pure-eval==0.2.2
+pure-eval==0.2.3
     # via stack-data
-pyarrow==16.1.0
+pyarrow==17.0.0
     # via
     #   bbp-workflow (setup.py)
     #   blue-cwl
@@ -358,7 +358,7 @@ pygments==2.18.0
     # via
     #   devtools
     #   ipython
-pyjwt==2.8.0
+pyjwt==2.9.0
     # via entity-management
 pynrrd==1.0.0
     # via voxcell
@@ -379,17 +379,17 @@ python-dateutil==2.9.0.post0
     #   matplotlib
     #   pandas
     #   pynwb
-python-keycloak==4.2.1
+python-keycloak==4.3.0
     # via entity-management
 pytz==2024.1
     # via pandas
-pyyaml==6.0.1
+pyyaml==6.0.2
     # via
     #   bluepyemodel
     #   brainbuilder
     #   fz-td-recipe
     #   neurom
-pyzmq==26.0.3
+pyzmq==26.1.0
     # via
     #   ipykernel
     #   ipyparallel
@@ -414,7 +414,7 @@ requests==2.32.3
     #   voxcell
 requests-toolbelt==1.0.0
     # via python-keycloak
-rpds-py==0.19.0
+rpds-py==0.20.0
     # via
     #   jsonschema
     #   referencing
@@ -459,7 +459,7 @@ tornado==6.4.1
     #   ipyparallel
     #   jupyter-client
     #   luigi
-tqdm==4.66.4
+tqdm==4.66.5
     # via
     #   bluepyemodel
     #   brainbuilder
@@ -500,7 +500,7 @@ wcwidth==0.2.13
     # via prompt-toolkit
 websockets==12.0
     # via brayns
-xarray==2024.6.0
+xarray==2024.7.0
     # via
     #   bbp-workflow (setup.py)
     #   pint-xarray

tests/test_generation_variant.py

@@ -110,8 +110,8 @@ class MockTool:
             "_self": VARIANT_TOOL_URL,
             "@type": "Variant",
             "name": "foo|bar|v1",
-            "generator_name": "foo",
-            "variant_name": "bar",
+            "generatorName": "foo",
+            "variantName": "bar",
             "version": "v1",
         },
         distr_dict={
@@ -160,8 +160,8 @@ class MockWorkflow:
             "_self": VARIANT_WORKFLOW_URL,
             "@type": "Variant",
             "name": "foo|bar|v1",
-            "generator_name": "foo",
-            "variant_name": "bar",
+            "generatorName": "foo",
+            "variantName": "bar",
             "version": "v1",
         },
         distr_dict={

tox.ini

@@ -74,6 +74,12 @@ extras = docs
 commands = make html SPHINXOPTS=-W
 allowlist_externals = make
 
+[testenv:upgrade-requirements]
+description = Create or update requirements.txt, using the latest version of dependencies
+skip_install = True
+deps = pip-tools
+commands = python -m piptools compile --upgrade
+
 # E203: whitespace before ':'
 # E731: do not assign a lambda expression, use a def
 # W503: line break after binary operator
@@ -91,3 +97,9 @@ max-line-length = 100
 #   - D413: no blank line afer last section
 #   - D419: Docstring is empty
 add-ignore = D102,D107,D300,D301,D413,D419
+
+
+[gh-actions]
+python =
+  3.11: py311, lint, check-packaging
+  3.12: py312