feat: add WRROC models, validators & unit tests

elixir-cloud-aai · Aug 30, 2024 · ef290a5 · ef290a5
1 parent 7e85db7
commit ef290a5
Show file tree

Hide file tree

Showing 8 changed files with 491 additions and 395 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,10 +11,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v5
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 

diff --git a/crategen/converters/abstract_converter.py b/crategen/converters/abstract_converter.py
@@ -7,5 +7,5 @@ def convert_to_wrroc(self, data):
         """Convert data to WRROC format"""
 
     @abstractmethod
-    def convert_from_wrroc(self, wrroc_data):
+    def convert_from_wrroc(self, data):
         """Convert WRROC data to the original format"""
diff --git a/crategen/converters/tes_converter.py b/crategen/converters/tes_converter.py
@@ -1,7 +1,17 @@
-from pydantic import ValidationError
+from datetime import datetime
 
-from ..models.tes_models import TESData, TESInput, TESOutput, TESExecutor, TESResources, TESState, TESTaskLog
-from ..models.wrroc_models import WRROCDataTES, WRROCInputs, WRROCOutputs
+from pydantic import AnyUrl, ValidationError
+
+from ..models.tes_models import (
+    TESData,
+    TESExecutor,
+    TESInput,
+    TESOutput,
+    TESState,
+    TESTaskLog,
+)
+from ..models.wrroc_models import WRROCDataTES
+from ..validators import validate_wrroc_tes
 from .abstract_converter import AbstractConverter
 
 
@@ -11,10 +21,10 @@ def convert_to_wrroc(self, data: dict) -> dict:
         Convert TES data to WRROC format.
 
         Args:
-            data (dict): The input TES data.
+            data: The input TES data.
 
         Returns:
-            dict: The converted WRROC data.
+            The converted WRROC data.
 
         Raises:
             ValidationError: If TES data is invalid.
@@ -28,33 +38,33 @@ def convert_to_wrroc(self, data: dict) -> dict:
         executors = data_tes.executors
         end_time = data_tes.logs[0].end_time if data_tes.logs else None
 
-        # Convert to WRROC format
         wrroc_data = {
             "@id": data_tes.id,
             "name": data_tes.name,
             "description": data_tes.description,
             "instrument": executors[0].image if executors else None,
             "object": [
-                {"@id": input.url, "name": input.path, "type": input.type}
-                for input in data_tes.inputs
+                {"@id": input.url, "name": input.path} for input in data_tes.inputs
             ],
             "result": [
                 {"@id": output.url, "name": output.path} for output in data_tes.outputs
             ],
             "startTime": data_tes.creation_time,
             "endTime": end_time,
         }
+
+        validate_wrroc_tes(wrroc_data)
         return wrroc_data
 
     def convert_from_wrroc(self, data: dict) -> dict:
         """
         Convert WRROC data to TES format.
 
         Args:
-            data (dict): The input WRROC data.
+            data: The input WRROC data.
 
         Returns:
-            dict: The converted TES data.
+            The converted TES data.
 
         Raises:
             ValidationError: If WRROC data is invalid.
@@ -65,15 +75,39 @@ def convert_from_wrroc(self, data: dict) -> dict:
         except ValidationError as e:
             raise ValueError(f"Invalid WRROC data: {e.errors()}") from e
 
-        # Convert from WRROC to TES format
-        tes_data = {
-            "id": data_wrroc.id,
-            "name": data_wrroc.name,
-            "description": data_wrroc.description,
-            "executors": [{"image": data_wrroc.instrument}],
-            "inputs": [{"url": obj.id, "path": obj.name} for obj in data_wrroc.object],
-            "outputs": [{"url": res.id, "path": res.name} for res in data_wrroc.result],
-            "creation_time": data_wrroc.startTime,
-            "logs": [{"end_time": data_wrroc.endTime}],
-        }
-        return tes_data
+        # Convert URL strings to AnyUrl
+        tes_inputs = [TESInput(url=AnyUrl(url=obj.id), path=obj.name) for obj in data_wrroc.object]
+        tes_outputs = [TESOutput(url=AnyUrl(url=res.id), path=res.name) for res in data_wrroc.result]
+
+        # Ensure 'image' and 'command' fields are provided
+        tes_executors = [TESExecutor(image=data_wrroc.instrument or "", command=[])]  # Provide default empty list for command
+
+        # Ensure correct type for end_time (datetime)
+        end_time = datetime.fromisoformat(data_wrroc.endTime) if data_wrroc.endTime else None
+
+        tes_logs = [
+            TESTaskLog(
+                logs=[],
+                metadata=None,
+                start_time=None,
+                end_time=end_time, 
+                outputs=[],
+                system_logs=None
+            )
+        ]
+
+        tes_data = TESData(
+            id=data_wrroc.id,
+            name=data_wrroc.name,
+            description=data_wrroc.description,
+            executors=tes_executors,
+            inputs=tes_inputs,
+            outputs=tes_outputs,
+            creation_time=None, 
+            logs=tes_logs,
+            state=TESState.UNKNOWN
+        )
+
+        # Validate TES data before returning
+        tes_data = TESData(**tes_data.dict())
+        return tes_data.dict()
diff --git a/crategen/converters/wes_converter.py b/crategen/converters/wes_converter.py
@@ -1,8 +1,9 @@
 from pydantic import ValidationError
 
-from ..models.wes_models import WESData
+from ..models.wes_models import WESData, WESOutputs, WESRequest, WESRunLog
 from ..models.wrroc_models import WRROCDataWES
 from ..utils import convert_to_iso8601
+from ..validators import validate_wrroc_wes
 from .abstract_converter import AbstractConverter
 
 
@@ -12,10 +13,10 @@ def convert_to_wrroc(self, data: dict) -> dict:
         Convert WES data to WRROC format.
 
         Args:
-            data (dict): The input WES data.
+            data: The input WES data.
 
         Returns:
-            dict: The converted WRROC data.
+            The converted WRROC data.
 
         Raises:
             ValidationError: If WES data is invalid.
@@ -26,7 +27,6 @@ def convert_to_wrroc(self, data: dict) -> dict:
         except ValidationError as e:
             raise ValueError(f"Invalid WES data: {e.errors()}") from e
 
-        # Convert to WRROC format
         wrroc_data = {
             "@id": data_wes.run_id,
             "name": data_wes.run_log.name,
@@ -38,17 +38,20 @@ def convert_to_wrroc(self, data: dict) -> dict:
                 for output in data_wes.outputs
             ],
         }
+
+        # Validate WRROC data before returning
+        validate_wrroc_wes(wrroc_data)
         return wrroc_data
 
     def convert_from_wrroc(self, data: dict) -> dict:
         """
         Convert WRROC data to WES format.
 
         Args:
-            data (dict): The input WRROC data.
+            data: The input WRROC data.
 
         Returns:
-            dict: The converted WES data.
+            The converted WES data.
 
         Raises:
             ValidationError: If WRROC data is invalid.
@@ -61,18 +64,27 @@ def convert_from_wrroc(self, data: dict) -> dict:
                 f"Invalid WRROC data for WES conversion: {e.errors()}"
             ) from e
 
-        # Convert from WRROC to WES format
-        # Convert from WRROC to WES format
-        wes_data = {
-            "run_id": data_wrroc.id,
-            "run_log": {
-                "name": data_wrroc.name,
-                "start_time": data_wrroc.startTime,
-                "end_time": data_wrroc.endTime,
-            },
-            "state": data_wrroc.status,
-            "outputs": [
-                {"location": res.id, "name": res.name} for res in data_wrroc.result
-            ],
-        }
-        return wes_data
+        wes_outputs = [WESOutputs(location=res.id, name=res.name) for res in data_wrroc.result]
+        wes_run_log = WESRunLog(
+            name=data_wrroc.name,
+            start_time=data_wrroc.startTime,
+            end_time=data_wrroc.endTime
+        )
+        wes_request = WESRequest(
+            workflow_params={},  # Adjust as necessary
+            workflow_type="CWL",  # Example type, adjust as necessary
+            workflow_type_version="v1.0"  # Example version, adjust as necessary
+        )
+
+        wes_data = WESData(
+            run_id=data_wrroc.id,
+            request=wes_request,
+            state=data_wrroc.status,
+            run_log=wes_run_log,
+            task_logs=None,  # Provide appropriate value
+            outputs=wes_outputs
+        )
+
+        # Validate WES data before returning
+        wes_data = WESData(**wes_data.dict())
+        return wes_data.dict()
diff --git a/crategen/models/wrroc_models.py b/crategen/models/wrroc_models.py
@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import AnyUrl, BaseModel
+from pydantic import AnyUrl, BaseModel, Field
 
 
 class WRROCInputs(BaseModel):
@@ -12,7 +12,7 @@ class WRROCInputs(BaseModel):
         name (str): The name of the input.
     """
 
-    id: str
+    id: str = Field(alias='@id')
     name: str
 
 
@@ -25,7 +25,7 @@ class WRROCOutputs(BaseModel):
         name (str): The name of the output.
     """
 
-    id: str
+    id: str = Field(alias='@id')
     name: str
 
 
@@ -45,18 +45,19 @@ class WRROCDataBase(BaseModel):
         version (Optional[str]): The version of the WRROC entity.
     """
 
-    id: str
+    id: str = Field(alias='@id')
     name: str
     description: Optional[str] = ""
     instrument: Optional[str] = None
-    object: list[WRROCInputs]
-    result: list[WRROCOutputs]
+    object: list[WRROCInputs] = Field(default_factory=list)
+    result: list[WRROCOutputs] = Field(default_factory=list)
     startTime: Optional[str] = None
     endTime: Optional[str] = None
     version: Optional[str] = None
 
     class Config:
         extra = "allow"
+        allow_population_by_field_name = True  # Allows using field name for input data
 
 
 class WRROCData(WRROCDataBase):
@@ -101,7 +102,7 @@ class WRROCProcess(BaseModel):
         profiles (Optional[list[AnyUrl]]): URLs to the RO-Crate profiles used.
     """
 
-    id: str
+    id: str = Field(alias='@id')
     name: str
     description: Optional[str] = ""
     startTime: Optional[str] = None
@@ -111,6 +112,7 @@ class WRROCProcess(BaseModel):
 
     class Config:
         extra = "allow"
+        allow_population_by_field_name = True
 
 
 class WRROCWorkflow(WRROCProcess):
@@ -131,6 +133,7 @@ class WRROCWorkflow(WRROCProcess):
 
     class Config:
         extra = "allow"
+        allow_population_by_field_name = True
 
 
 class WRROCProvenance(WRROCWorkflow):
@@ -153,3 +156,4 @@ class WRROCProvenance(WRROCWorkflow):
 
     class Config:
         extra = "allow"
+        allow_population_by_field_name = True