From ab37fe6c0b40ef2dad3f803c50a96eb3c4390b83 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 27 Nov 2022 08:22:35 +0000 Subject: [PATCH 1/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/tests/test_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 82f5287970..7ef386f348 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1079,7 +1079,6 @@ def test_audit_shellcommandtask_file(tmpdir): shutil.copy("test.txt", tmpdir) shutil.copy("test2.txt", tmpdir) - cmd = "cat" file_in = tmpdir / "test.txt" file_in_2 = tmpdir / "test2.txt" @@ -1100,7 +1099,7 @@ def test_audit_shellcommandtask_file(tmpdir): }, ), ), - ( + ( "in_file_2", attr.ib( type=File, @@ -1139,6 +1138,7 @@ def test_audit_shellcommandtask_file(tmpdir): assert data["AtLocation"] == str(file_in_2) assert data["digest"] == test_file_hash_2 + def test_audit_shellcommandtask_version(tmpdir): import subprocess as sp From 419dadada332efc20394b86505c25c4accee4112 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Sun, 27 Nov 2022 00:24:27 -0800 Subject: [PATCH 2/7] Added prov_env message --- pydra/engine/audit.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index ece23239e6..7672f31cf9 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -6,7 +6,8 @@ from ..utils.messenger import send_message, make_message, gen_uuid, now, AuditFlag from .helpers import ensure_list, gather_runtime_info, hash_file from .specs import attr_fields, File, Directory - +import subprocess as sp +import platform class Audit: """Handle provenance tracking and resource utilization.""" @@ -171,7 +172,7 @@ def audit_check(self, flag): return self.audit_flags & flag def audit_task(self, task): - import subprocess as sp + label = task.name @@ -221,3 +222,29 @@ def audit_task(self, task): } self.audit_message(start_message, AuditFlag.PROV) + + env_id = f"uid:{gen_uuid()}" + os_plat = str(platform.platform()) + env_vars = str(os.environ) + # check if using conda environment + if "CONDA_PREFIX" in os.environ: + conda_env_path = str(os.environ["CONDA_PREFIX"]) + conda_env_name = conda_env_path.split("/")[-1] + + env_message = { + "@id": env_id, + "Label": "Conda environment: " + conda_env_name, + "EnvVars": env_vars, + "OperatingSystem": os_plat, + "Dependencies": "test" + } + else: + env_message = { + "@id": env_id, + "Label": None, + "EnvVars": env_vars, + "OperatingSystem": os_plat, + "Dependencies": "test" + } + # Fetch env info + self.audit_message(env_message, AuditFlag.PROV) \ No newline at end of file From f2231c0eea8d3af8eb483ba0ce454a24c0729de5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 27 Nov 2022 08:25:08 +0000 Subject: [PATCH 3/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 7672f31cf9..bacd995bef 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -9,6 +9,7 @@ import subprocess as sp import platform + class Audit: """Handle provenance tracking and resource utilization.""" @@ -173,7 +174,6 @@ def audit_check(self, flag): def audit_task(self, task): - label = task.name command = task.cmdline if hasattr(task.inputs, "executable") else None @@ -222,7 +222,7 @@ def audit_task(self, task): } self.audit_message(start_message, AuditFlag.PROV) - + env_id = f"uid:{gen_uuid()}" os_plat = str(platform.platform()) env_vars = str(os.environ) @@ -230,13 +230,13 @@ def audit_task(self, task): if "CONDA_PREFIX" in os.environ: conda_env_path = str(os.environ["CONDA_PREFIX"]) conda_env_name = conda_env_path.split("/")[-1] - + env_message = { "@id": env_id, "Label": "Conda environment: " + conda_env_name, "EnvVars": env_vars, "OperatingSystem": os_plat, - "Dependencies": "test" + "Dependencies": "test", } else: env_message = { @@ -244,7 +244,7 @@ def audit_task(self, task): "Label": None, "EnvVars": env_vars, "OperatingSystem": os_plat, - "Dependencies": "test" + "Dependencies": "test", } # Fetch env info - self.audit_message(env_message, AuditFlag.PROV) \ No newline at end of file + self.audit_message(env_message, AuditFlag.PROV) From 56b52951379b49f946db76b84a77643121f7b024 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Mon, 28 Nov 2022 14:09:03 -0800 Subject: [PATCH 4/7] Added tests for env --- pydra/engine/tests/test_task.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 7ef386f348..232a43456d 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1002,12 +1002,19 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] return a + b from glob import glob - + import platform funky = testfunc(a=2, audit_flags=AuditFlag.PROV, messengers=FileMessenger()) funky.cache_dir = tmpdir funky() message_path = tmpdir / funky.checksum / "messages" - + op_sys = platform.platform() + env_vars = str(os.environ) + if "CONDA_PREFIX" in os.environ: + conda_env_path = str(os.environ["CONDA_PREFIX"]) + conda_env_name = conda_env_path.split("/")[-1] + + else: + conda_env_name = None for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: data = json.load(f) @@ -1020,9 +1027,16 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] assert None == data["Label"] if "AssociatedWith" in data: assert None == data["AssociatedWith"] + if "OperatingSystem" in data: + assert op_sys == data["OperatingSystem"] + if "EnvVars" in data: + assert env_vars in data["EnvVars"] + if "Label" in data: + assert conda_env_name in data["Label"] + - # assert any(json_content) - + + def test_audit_shellcommandtask(tmpdir): args = "-l" @@ -1268,7 +1282,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] from glob import glob assert len(glob(str(tmpdir / funky.checksum / "proc*.log"))) == 1 - assert len(glob(str(message_path / "*.jsonld"))) == 7 + assert len(glob(str(message_path / "*.jsonld"))) == 8 # commented out to speed up testing collect_messages(tmpdir / funky.checksum, message_path, ld_op="compact") From 4662d8b6c5cd479424aedb5a8a2e908c12054707 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 22:12:14 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/tests/test_task.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 232a43456d..a9da72c2e8 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1003,6 +1003,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] from glob import glob import platform + funky = testfunc(a=2, audit_flags=AuditFlag.PROV, messengers=FileMessenger()) funky.cache_dir = tmpdir funky() @@ -1033,10 +1034,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] assert env_vars in data["EnvVars"] if "Label" in data: assert conda_env_name in data["Label"] - - - def test_audit_shellcommandtask(tmpdir): args = "-l" From 9a09a36e7cf0ff2c52b3450514dd70452d70daf5 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Mon, 28 Nov 2022 14:23:18 -0800 Subject: [PATCH 6/7] fixed nonetype issue --- pydra/engine/audit.py | 23 +++++++---------------- pydra/engine/tests/test_task.py | 2 +- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index bacd995bef..ac92505d52 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -9,7 +9,6 @@ import subprocess as sp import platform - class Audit: """Handle provenance tracking and resource utilization.""" @@ -174,6 +173,7 @@ def audit_check(self, flag): def audit_task(self, task): + label = task.name command = task.cmdline if hasattr(task.inputs, "executable") else None @@ -231,20 +231,11 @@ def audit_task(self, task): conda_env_path = str(os.environ["CONDA_PREFIX"]) conda_env_name = conda_env_path.split("/")[-1] - env_message = { - "@id": env_id, - "Label": "Conda environment: " + conda_env_name, - "EnvVars": env_vars, - "OperatingSystem": os_plat, - "Dependencies": "test", - } else: - env_message = { - "@id": env_id, - "Label": None, - "EnvVars": env_vars, - "OperatingSystem": os_plat, - "Dependencies": "test", - } + conda_env_name = str(None) + + env_message = {"@id": env_id, "Label": f"Conda environment: {conda_env_name}", "EnvVars": env_vars, "OperatingSystem": os_plat, "Dependencies": "test"} + + # Fetch env info - self.audit_message(env_message, AuditFlag.PROV) + self.audit_message(env_message, AuditFlag.PROV) \ No newline at end of file diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index a9da72c2e8..a2b6cba5b4 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1015,7 +1015,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] conda_env_name = conda_env_path.split("/")[-1] else: - conda_env_name = None + conda_env_name = str(None) for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: data = json.load(f) From 186bd09daac4a3c9f0ee950a3ed15ae534034011 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 22:23:41 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index ac92505d52..fcca05fbf3 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -9,6 +9,7 @@ import subprocess as sp import platform + class Audit: """Handle provenance tracking and resource utilization.""" @@ -173,7 +174,6 @@ def audit_check(self, flag): def audit_task(self, task): - label = task.name command = task.cmdline if hasattr(task.inputs, "executable") else None @@ -234,8 +234,13 @@ def audit_task(self, task): else: conda_env_name = str(None) - env_message = {"@id": env_id, "Label": f"Conda environment: {conda_env_name}", "EnvVars": env_vars, "OperatingSystem": os_plat, "Dependencies": "test"} - + env_message = { + "@id": env_id, + "Label": f"Conda environment: {conda_env_name}", + "EnvVars": env_vars, + "OperatingSystem": os_plat, + "Dependencies": "test", + } # Fetch env info - self.audit_message(env_message, AuditFlag.PROV) \ No newline at end of file + self.audit_message(env_message, AuditFlag.PROV)