From d1343eadd44defdcb10b556e028d8ae5356aa2f4 Mon Sep 17 00:00:00 2001 From: Gianluca Ficarelli <26835404+GianlucaFicarelli@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:11:43 +0200 Subject: [PATCH] Use CumulativeAnalysisReport.generation.activity when registering in Nexus (#6) Other changes: - Remove EnsuredList because not needed anymore - Remove analysis entities already moved to entity-management - Upgrade requirements.txt, needing entity-management>=1.5.0 --- CHANGELOG.rst | 21 +++++++ bbp_workflow/sbo/analysis/common/task.py | 20 +++--- bbp_workflow/sbo/analysis/emodel/task.py | 2 +- bbp_workflow/sbo/analysis/sim/task.py | 10 +-- bbp_workflow/sbo/emodel/task.py | 2 +- bbp_workflow/sbo/entity.py | 79 ------------------------ bbp_workflow/sbo/sim/task.py | 3 +- bbp_workflow/sbo/viz/task.py | 3 +- bbp_workflow/simulation/task.py | 4 +- requirements.txt | 31 +++++----- 10 files changed, 61 insertions(+), 114 deletions(-) delete mode 100644 bbp_workflow/sbo/entity.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c03750a..e90039a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,27 @@ Changelog ========= +Version 4.1.0 (2024-08-28) +-------------------------- + +Breaking changes +~~~~~~~~~~~~~~~~ + +- Use CumulativeAnalysisReport.generation.activity when registering custom analyses in Nexus. + +Improvements +~~~~~~~~~~~~ + +- Upgrade requirements.txt. + +Version 4.0.2 (2024-08-23) +-------------------------- + +Improvements +~~~~~~~~~~~~ + +- Add generatorName in VariantTaskConfig to obey schema. + Version 4.0.1 (2024-08-15) -------------------------- diff --git a/bbp_workflow/sbo/analysis/common/task.py b/bbp_workflow/sbo/analysis/common/task.py index edebc5e..0e5e7ff 100644 --- a/bbp_workflow/sbo/analysis/common/task.py +++ b/bbp_workflow/sbo/analysis/common/task.py @@ -10,8 +10,14 @@ from functools import cached_property from pathlib import Path +from entity_management.analysis import ( + AnalysisReportGeneration, + AnalysisReportGenerationActivity, + AnalysisSoftwareSourceCode, + CumulativeAnalysisReport, + MultiCumulativeAnalysisReport, +) from entity_management.base import Derivation -from entity_management.core import Contribution from luigi import ( BoolParameter, Config, @@ -28,11 +34,6 @@ from luigi.parameter import ParameterVisibility from bbp_workflow.luigi import CompleteTask, RemoteTarget, inherits -from bbp_workflow.sbo.entity import ( - AnalysisSoftwareSourceCode, - CumulativeAnalysisReport, - MultiCumulativeAnalysisReport, -) from bbp_workflow.sbo.task import BaseTask, MetaTask from bbp_workflow.settings import DEFAULT_HOST, L from bbp_workflow.slurm import _run_cmd, _run_sbatch, _sbatch @@ -534,13 +535,14 @@ def pre_publish_resource(self, activity): assert isinstance(task, AnalyseGeneric) if task.source_code_url: agent = AnalysisSoftwareSourceCode.from_url(task.source_code_url) - contributions = [Contribution(agent=agent)] + generation_activity = AnalysisReportGenerationActivity(wasAssociatedWith=agent) + generations = [AnalysisReportGeneration(activity=generation_activity)] else: L.warning( "The source code url for analysis %s is missing, the relation won't be created", index, ) - contributions = None + generations = None sub_resource = CumulativeAnalysisReport( name=task.report_name, description=task.report_description, @@ -548,7 +550,7 @@ def pre_publish_resource(self, activity): types=list(task.types), derivation=Derivation(entity=analysed_entity), index=index, - contribution=contributions, + generation=generations, ) sub_resource = self.publish(sub_resource, activity=activity) parts.append(sub_resource) diff --git a/bbp_workflow/sbo/analysis/emodel/task.py b/bbp_workflow/sbo/analysis/emodel/task.py index 1373eb5..38cba37 100644 --- a/bbp_workflow/sbo/analysis/emodel/task.py +++ b/bbp_workflow/sbo/analysis/emodel/task.py @@ -7,6 +7,7 @@ from pathlib import Path from blue_cwl.me_model.staging import stage_emodel +from entity_management.analysis import AnalysisReport, MultiEModelAnalysis from entity_management.base import Derivation from entity_management.emodel import EModel from luigi import DictParameter, ListParameter, Parameter, PathParameter @@ -18,7 +19,6 @@ MultiAnalyseGenericMeta, get_analysis_config_schema, ) -from bbp_workflow.sbo.entity import AnalysisReport, MultiEModelAnalysis from bbp_workflow.sbo.task import BaseTask from bbp_workflow.settings import DEFAULT_HOST from bbp_workflow.task import KgCfg diff --git a/bbp_workflow/sbo/analysis/sim/task.py b/bbp_workflow/sbo/analysis/sim/task.py index 04bee92..49d0600 100644 --- a/bbp_workflow/sbo/analysis/sim/task.py +++ b/bbp_workflow/sbo/analysis/sim/task.py @@ -6,6 +6,11 @@ from functools import cached_property from pathlib import Path +from entity_management.analysis import ( + AnalysisReport, + Derivation, + MultiCumulativeSimulationCampaignAnalysis, +) from entity_management.simulation import Simulation from luigi import DictParameter, ListParameter, Parameter @@ -15,11 +20,6 @@ MultiAnalyseGenericMeta, get_analysis_config_schema, ) -from bbp_workflow.sbo.entity import ( - AnalysisReport, - Derivation, - MultiCumulativeSimulationCampaignAnalysis, -) from bbp_workflow.sbo.sim.task import RunSimCampaign, RunSimCampaignMeta from bbp_workflow.settings import L from bbp_workflow.simulation.util import read_config diff --git a/bbp_workflow/sbo/emodel/task.py b/bbp_workflow/sbo/emodel/task.py index fbf2108..91a30cc 100644 --- a/bbp_workflow/sbo/emodel/task.py +++ b/bbp_workflow/sbo/emodel/task.py @@ -13,9 +13,9 @@ from bluepyemodel.emodel_pipeline.emodel_settings import EModelPipelineSettings from bluepyemodel.model.model_configurator import ModelConfigurator from entity_management.core import Entity +from entity_management.workflow import BbpWorkflowActivity from luigi import Parameter -from bbp_workflow.sbo.entity import BbpWorkflowActivity from bbp_workflow.sbo.task import BaseTask, MetaTask L = logging.getLogger(__name__) diff --git a/bbp_workflow/sbo/entity.py b/bbp_workflow/sbo/entity.py deleted file mode 100644 index aa1cd69..0000000 --- a/bbp_workflow/sbo/entity.py +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -"""Generic entities for SBO tasks.""" - -from entity_management.base import Derivation, attributes -from entity_management.core import Agent, Contribution, Entity -from entity_management.util import AttrOf -from entity_management.workflow import BbpWorkflowActivity - - -class EnsuredList(list): - """List to be used when the argument passed to the constructor might not be an iterable.""" - - def __init__(self, iterable=()): - """Init the object, always passing an iterable to the superclass. - - This may be needed when an attribute is defined as a list, but there is only one element, - and it's returned from Nexus as a single item instead of a list. - """ - if not isinstance(iterable, (list, tuple)): - iterable = (iterable,) - super().__init__(iterable) - - -@attributes({}) -class MultiCumulativeSimulationCampaignAnalysis(BbpWorkflowActivity): - """Activity generating multiple analyses of a Simulation Campaign.""" - - -@attributes({}) -class MultiEModelAnalysis(BbpWorkflowActivity): - """Activity generating multiple analyses of an EModel.""" - - -@attributes( - { - "name": AttrOf(str, default=None), - "description": AttrOf(str, default=None), - "codeRepository": AttrOf(str, default=None), - "subdirectory": AttrOf(str, default=None), - "command": AttrOf(str, default=None), - } -) -class AnalysisSoftwareSourceCode(Agent): - """AnalysisSoftwareSourceCode.""" - - -@attributes( - { - "derivation": AttrOf(Derivation), - "categories": AttrOf(EnsuredList[str], default=None), - "types": AttrOf(EnsuredList[str], default=None), - } -) -class AnalysisReport(Entity): - """AnalysisReport.""" - - -@attributes( - { - "derivation": AttrOf(Derivation), - "categories": AttrOf(EnsuredList[str], default=None), - "types": AttrOf(EnsuredList[str], default=None), - "hasPart": AttrOf(EnsuredList[AnalysisReport], default=None), - "index": AttrOf(int, default=None), - "contribution": AttrOf(EnsuredList[Contribution], default=None), - } -) -class CumulativeAnalysisReport(Entity): - """CumulativeAnalysisReport.""" - - -@attributes( - { - "hasPart": AttrOf(EnsuredList[CumulativeAnalysisReport], default=None), - } -) -class MultiCumulativeAnalysisReport(Entity): - """MultiCumulativeAnalysisReport.""" diff --git a/bbp_workflow/sbo/sim/task.py b/bbp_workflow/sbo/sim/task.py index 97834f4..1a82d20 100644 --- a/bbp_workflow/sbo/sim/task.py +++ b/bbp_workflow/sbo/sim/task.py @@ -8,6 +8,8 @@ from threading import Thread import numpy as np +from entity_management.analysis import AnalysisReport +from entity_management.base import Derivation from entity_management.simulation import Simulation as SimEntity from entity_management.simulation import ( SimulationCampaign, @@ -32,7 +34,6 @@ from bbp_workflow.luigi import RemoteTarget, inherits from bbp_workflow.sbo.circ.task import FindDetailedCircuit, FindDetailedCircuitMeta -from bbp_workflow.sbo.entity import AnalysisReport, Derivation from bbp_workflow.sbo.task import BaseTask, MetaTask from bbp_workflow.settings import ( BBP_WORKFLOW_SIF, diff --git a/bbp_workflow/sbo/viz/task.py b/bbp_workflow/sbo/viz/task.py index 93a97f2..232c253 100644 --- a/bbp_workflow/sbo/viz/task.py +++ b/bbp_workflow/sbo/viz/task.py @@ -6,6 +6,8 @@ from pathlib import Path import brayns +from entity_management.analysis import AnalysisReport +from entity_management.base import Derivation from entity_management.simulation import Simulation as SimEntity from entity_management.simulation import SimulationCampaignAnalysis from luigi import ( @@ -23,7 +25,6 @@ from bbp_workflow.luigi import RemoteTarget, inherits from bbp_workflow.sbo.analysis.sim.task import MultiAnalyseSimCampaignMeta -from bbp_workflow.sbo.entity import AnalysisReport, Derivation from bbp_workflow.sbo.sim.task import ReportsSimCampaignMeta, RunSimCampaign, RunSimCampaignMeta from bbp_workflow.sbo.task import BaseTask, MetaTask from bbp_workflow.settings import BBP_WORKFLOW_SIF, BRAYNS_SIF, DEFAULT_HOST, ENVIRONMENT diff --git a/bbp_workflow/simulation/task.py b/bbp_workflow/simulation/task.py index 1dcd992..7158e3b 100644 --- a/bbp_workflow/simulation/task.py +++ b/bbp_workflow/simulation/task.py @@ -415,7 +415,9 @@ def _handle_post_processors(self, config): if self.post_processors: sim_indices = _indices_to_ranges(_campaign_sim_indices(config)) path_prefix_name = Path(config.attrs["path_prefix"]) / config.name - cmd = f'python "$POST_PROCESSOR" {path_prefix_name/"config.json"} $SLURM_ARRAY_TASK_ID' + cmd = ( + f'python "$POST_PROCESSOR" {path_prefix_name / "config.json"} $SLURM_ARRAY_TASK_ID' + ) pre_cmd = dedent( f"""\ cd {path_prefix_name}/$SLURM_ARRAY_TASK_ID diff --git a/requirements.txt b/requirements.txt index 3067cb3..4ccc721 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile @@ -27,7 +27,7 @@ bluepyefe==2.3.34 # via bluepyemodel bluepyemodel==1.0.15 # via bbp-workflow (setup.py) -bluepyopt==1.14.13 +bluepyopt==1.14.15 # via bluepyemodel bluepysnap==3.0.1 # via @@ -35,7 +35,7 @@ bluepysnap==3.0.1 # brainbuilder brainbuilder==0.20.1 # via blue-cwl -brayns==3.8.3 +brayns==3.9.0 # via bbp-workflow (setup.py) cached-property==1.5.2 # via @@ -63,9 +63,9 @@ click==8.1.7 # neurom comm==0.2.2 # via ipykernel -configparser==7.0.0 +configparser==7.1.0 # via bluepyemodel -contourpy==1.2.1 +contourpy==1.3.0 # via matplotlib cryptography==43.0.0 # via jwcrypto @@ -89,14 +89,14 @@ devtools[pygments]==0.12.2 # via entity-management docutils==0.21.2 # via python-daemon -efel==5.7.6 +efel==5.7.8 # via # bluepyefe # bluepyemodel # bluepyopt elephant==1.1.0 # via bbp-workflow (setup.py) -entity-management==1.4.1 +entity-management==1.5.0 # via # bbp-workflow (setup.py) # blue-cwl @@ -136,16 +136,16 @@ hdmf==3.14.3 # via pynwb httpcore==1.0.5 # via httpx -httpx==0.27.0 +httpx==0.27.2 # via python-keycloak -idna==3.7 +idna==3.8 # via # anyio # httpx # requests igor2==0.5.8 # via bluepyefe -importlib-resources==6.4.2 +importlib-resources==6.4.4 # via bluepysnap ipykernel==6.29.5 # via ipyparallel @@ -364,7 +364,7 @@ pynrrd==1.0.0 # via voxcell pynwb==2.8.1 # via efel -pyparsing==3.1.2 +pyparsing==3.1.4 # via # matplotlib # rdflib @@ -389,12 +389,12 @@ pyyaml==6.0.2 # brainbuilder # fz-td-recipe # neurom -pyzmq==26.1.0 +pyzmq==26.2.0 # via # ipykernel # ipyparallel # jupyter-client -quantities==0.15.0 +quantities==0.16.0 # via # elephant # neo @@ -422,7 +422,7 @@ ruamel-yaml==0.18.6 # via hdmf ruamel-yaml-clib==0.2.8 # via ruamel-yaml -scipy==1.14.0 +scipy==1.14.1 # via # bluepyefe # bluepyemodel @@ -480,7 +480,6 @@ typing-extensions==4.12.2 # efel # flexcache # flexparser - # ipython # jwcrypto # luigi-tools # pint @@ -498,7 +497,7 @@ voxcell==3.1.9 # brainbuilder wcwidth==0.2.13 # via prompt-toolkit -websockets==12.0 +websockets==13.0 # via brayns xarray==2024.7.0 # via