Skip to content

Commit 6c843e5

Browse files
pagrubelpagrubel
authored
Merge pull request #1008 from lanl/prevent-multi-failed-tasks-archiving
Prevent simultaneous task failures multiple archive calls
2 parents 45c9827 + 924704c commit 6c843e5

19 files changed

+434
-23
lines changed

‎beeflow/common/gdb/gdb_driver.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ def set_task_output_glob(self, task, output_id, glob):
279279
def workflow_completed(self):
280280
"""Determine if a workflow has completed.
281281
282-
A workflow has completed if each of its final tasks has state 'COMPLETED'.
282+
A workflow has completed if each of its final tasks has finished or failed.
283283
284284
:rtype: bool
285285
"""

‎beeflow/common/gdb/neo4j_cypher.py

+5-2
Original file line numberDiff line numberDiff line change
@@ -716,7 +716,9 @@ def reset_workflow_id(tx, old_id, new_id):
716716

717717

718718
def final_tasks_completed(tx, wf_id):
719-
"""Return true if each of a workflow's final Task nodes has state 'COMPLETED'.
719+
"""Return true if each of a workflow's final Task nodes is in a finished state.
720+
721+
Finished states: 'COMPLETED', 'FAILED', 'SUBMIT_FAIL', 'BUILD_FAIL', or 'DEP_FAIL'.
720722
721723
:param wf_id: the workflow's id
722724
:type wf_id: str
@@ -725,7 +727,8 @@ def final_tasks_completed(tx, wf_id):
725727
restart = "|RESTARTED_FROM" if get_workflow_by_id(tx, wf_id)['restart'] else ""
726728
not_completed_query = ("MATCH (m:Metadata)-[:DESCRIBES]->(t:Task {workflow_id: $wf_id}) "
727729
f"WHERE NOT (t)<-[:DEPENDS_ON{restart}]-(:Task) "
728-
"AND m.state <> 'COMPLETED' "
730+
"AND NOT m.state IN "
731+
"['COMPLETED', 'FAILED', 'SUBMIT_FAIL', 'BUILD_FAIL', 'DEP_FAIL'] "
729732
"RETURN t IS NOT NULL LIMIT 1")
730733

731734
# False if at least one task with state not 'COMPLETED'

‎beeflow/common/gdb/neo4j_driver.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# pylint: disable=W0221
2+
13
"""Neo4j interface module.
24
35
Connection requires a valid URI, Username, and Password.
@@ -418,7 +420,7 @@ def set_task_output_glob(self, task, output_id, glob):
418420
def workflow_completed(self, workflow_id):
419421
"""Determine if a workflow in the Neo4j database has completed.
420422
421-
A workflow has completed if each of its final task nodes have state 'COMPLETED'.
423+
A workflow has completed if each of its final tasks has finished or failed.
422424
:param workflow_id: the workflow id
423425
:type workflow_id: str
424426
:rtype: bool

‎beeflow/data/cwl/bee_workflows/cgt-2branches/README.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
This example has two branches of cat-grep-tar. The second branch searches for a
2+
file that should not exist. That branch fails but the workflow can continue.
3+

‎beeflow/data/cwl/bee_workflows/cgt-2branches/cat.cwl

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env cwl-runner
2+
3+
cwlVersion: v1.0
4+
class: CommandLineTool
5+
baseCommand: cat
6+
stdout: cat.txt
7+
stderr: cat.err
8+
inputs:
9+
input_file:
10+
type: File
11+
inputBinding:
12+
position: 1
13+
outputs:
14+
contents:
15+
type: stdout
16+
cat_stderr:
17+
type: stderr

‎beeflow/data/cwl/bee_workflows/cgt-2branches/cat2.cwl

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env cwl-runner
2+
3+
cwlVersion: v1.0
4+
class: CommandLineTool
5+
baseCommand: cat
6+
stdout: cat2.txt
7+
stderr: cat2.err
8+
inputs:
9+
input_file2:
10+
type: File
11+
inputBinding:
12+
position: 1
13+
outputs:
14+
contents:
15+
type: stdout
16+
cat_stderr2:
17+
type: stderr

‎beeflow/data/cwl/bee_workflows/cgt-2branches/grep0.cwl

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env cwl-runner
2+
3+
cwlVersion: v1.0
4+
class: CommandLineTool
5+
baseCommand: grep
6+
stdout: occur0.txt
7+
inputs:
8+
word:
9+
type: string
10+
inputBinding:
11+
position: 1
12+
text_file:
13+
type: File
14+
inputBinding:
15+
position: 2
16+
outputs:
17+
occur:
18+
type: stdout

‎beeflow/data/cwl/bee_workflows/cgt-2branches/grep1.cwl

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env cwl-runner
2+
3+
cwlVersion: v1.0
4+
class: CommandLineTool
5+
baseCommand: grep
6+
stdout: occur1.txt
7+
inputs:
8+
word:
9+
type: string
10+
inputBinding:
11+
position: 1
12+
text_file:
13+
type: File
14+
inputBinding:
15+
position: 2
16+
outputs:
17+
occur:
18+
type: stdout

‎beeflow/data/cwl/bee_workflows/cgt-2branches/grep2.cwl

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env cwl-runner
2+
3+
cwlVersion: v1.0
4+
class: CommandLineTool
5+
baseCommand: grep
6+
stdout: occur2.txt
7+
inputs:
8+
word:
9+
type: string
10+
inputBinding:
11+
position: 1
12+
text_file:
13+
type: File
14+
inputBinding:
15+
position: 2
16+
outputs:
17+
occur:
18+
type: stdout

‎beeflow/data/cwl/bee_workflows/cgt-2branches/grep3.cwl

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env cwl-runner
2+
3+
cwlVersion: v1.0
4+
class: CommandLineTool
5+
baseCommand: grep
6+
stdout: occur3.txt
7+
inputs:
8+
word:
9+
type: string
10+
inputBinding:
11+
position: 1
12+
text_file:
13+
type: File
14+
inputBinding:
15+
position: 2
16+
outputs:
17+
occur:
18+
type: stdout

‎beeflow/data/cwl/bee_workflows/cgt-2branches/input.yml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
input_file: lorem.txt
2+
input_file2: none.txt
3+
word0: Vivamus
4+
word1: pulvinar
5+
word2: Morbi
6+
word3: vitae
7+
tarball_fname: out.tgz
8+
tarball_fname2: out2.tgz

‎beeflow/data/cwl/bee_workflows/cgt-2branches/lorem.txt

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut ut metus urna.
2+
Morbi tortor libero, suscipit hendrerit lacus et, condimentum porta orci.
3+
Maecenas vulputate lectus lorem, ac eleifend massa consectetur finibus. Integer
4+
vitae sem sit amet quam pharetra consectetur vel vel augue. Vivamus finibus
5+
metus mauris, sed ultricies purus placerat id. Vivamus eget auctor dui. Cras
6+
viverra rutrum neque, eu imperdiet ante sagittis ac. Donec iaculis, lacus sit
7+
amet mollis viverra, ipsum tortor luctus metus, sit amet dignissim orci massa
8+
non lectus. Nullam vitae dui placerat, condimentum nibh eget, dignissim erat.
9+
Phasellus gravida pretium facilisis. Phasellus pharetra mattis risus nec
10+
imperdiet.
11+
12+
Fusce facilisis finibus dolor. Nunc et posuere ante. Praesent blandit
13+
vestibulum egestas. Maecenas auctor nulla maximus tortor condimentum luctus.
14+
Orci varius natoque penatibus et magnis dis parturient montes, nascetur
15+
ridiculus mus. Quisque scelerisque turpis sed aliquet blandit. Praesent eget
16+
lobortis urna. Ut aliquam enim sit amet est ornare, sit amet viverra est
17+
tempus. Nunc blandit sollicitudin enim vel cursus. Fusce sit amet tincidunt
18+
eros. Nulla iaculis, est vitae consectetur imperdiet, ante urna aliquam libero,
19+
id vehicula tortor justo a dolor. Vestibulum id faucibus nisi, nec vehicula
20+
lacus.
21+
22+
Duis feugiat consequat quam eu lobortis. Aliquam erat volutpat. Integer egestas
23+
justo sit amet dui malesuada ullamcorper. Quisque tincidunt lacinia purus, id
24+
facilisis risus convallis id. Fusce ex ligula, consectetur ac dui sed, blandit
25+
suscipit diam. Vivamus sit amet porta dui, nec faucibus arcu. Phasellus et
26+
mauris eu elit molestie pulvinar eget ut orci. Quisque quam est, varius vitae
27+
tristique luctus, tincidunt non lectus. Donec luctus molestie ex. Morbi dui
28+
arcu, rhoncus volutpat felis ut, ultricies vestibulum ligula. Ut metus ex,
29+
mollis eget lacus sed, venenatis aliquam purus. Aenean id imperdiet tortor.
30+
31+
Phasellus mollis vulputate libero. In erat sapien, tempor nec libero ac, tempor
32+
lacinia velit. Curabitur vehicula, arcu eu mollis ultricies, ex ex rutrum
33+
risus, id accumsan eros lectus at velit. In sem erat, sagittis pulvinar porta
34+
et, pulvinar nec ligula. Ut rhoncus lorem vulputate aliquet cursus. Cras
35+
efficitur erat posuere, faucibus mi et, tincidunt quam. Suspendisse eleifend ac
36+
justo ac fermentum. Nulla in lorem nec neque lacinia pharetra. Donec eget elit
37+
id magna mollis interdum. Vivamus pellentesque diam volutpat sollicitudin
38+
mattis. Interdum et malesuada fames ac ante ipsum primis in faucibus. Nulla
39+
aliquam tellus porta sem venenatis imperdiet. Nunc pretium lorem sit amet ipsum
40+
aliquam, sed dapibus turpis consequat.
41+
42+
Aliquam elementum rhoncus placerat. Praesent augue urna, euismod sit amet
43+
dignissim nec, ultricies nec dui. Vestibulum ante ipsum primis in faucibus orci
44+
luctus et ultrices posuere cubilia curae; Nulla lobortis tincidunt leo.
45+
Maecenas finibus quam mauris, a molestie nisi sodales id. Morbi mollis, libero
46+
id tristique ultrices, purus leo faucibus lectus, in consectetur justo eros ut
47+
risus. Integer aliquam fermentum elit, id lacinia turpis tempus et. Donec vitae
48+
sem lobortis eros vulputate blandit nec vel leo. Nullam posuere aliquet dui non
49+
mattis.

‎beeflow/data/cwl/bee_workflows/cgt-2branches/tar.cwl

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env cwl-runner
2+
3+
cwlVersion: v1.0
4+
class: CommandLineTool
5+
baseCommand: tar
6+
inputs:
7+
tarball_fname:
8+
type: string
9+
inputBinding:
10+
position: 1
11+
prefix: -cf
12+
file0:
13+
type: File
14+
inputBinding:
15+
position: 2
16+
file1:
17+
type: File
18+
inputBinding:
19+
position: 3
20+
outputs:
21+
tarball:
22+
type: File
23+
outputBinding:
24+
glob: $(inputs.tarball_fname)

‎beeflow/data/cwl/bee_workflows/cgt-2branches/tar2.cwl

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env cwl-runner
2+
3+
cwlVersion: v1.0
4+
class: CommandLineTool
5+
baseCommand: tar
6+
inputs:
7+
tarball_fname2:
8+
type: string
9+
inputBinding:
10+
position: 1
11+
prefix: -cf
12+
file2:
13+
type: File
14+
inputBinding:
15+
position: 2
16+
file3:
17+
type: File
18+
inputBinding:
19+
position: 3
20+
outputs:
21+
tarball2:
22+
type: File
23+
outputBinding:
24+
glob: $(inputs.tarball_fname2)

‎beeflow/data/cwl/bee_workflows/cgt-2branches/workflow.cwl

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
#!/usr/bin/env cwl-runner
2+
3+
cwlVersion: v1.0
4+
class: Workflow
5+
inputs:
6+
input_file: File
7+
word0: string
8+
word1: string
9+
tarball_fname: string
10+
input_file2: File
11+
word2: string
12+
word3: string
13+
tarball_fname2: string
14+
15+
outputs:
16+
tarball:
17+
type: File
18+
outputSource: tar/tarball
19+
cat_stderr:
20+
type: File
21+
outputSource: cat/cat_stderr
22+
cat_stderr2:
23+
type: File
24+
outputSource: cat2/cat_stderr2
25+
tarball2:
26+
type: File
27+
outputSource: tar2/tarball2
28+
29+
steps:
30+
cat:
31+
run: cat.cwl
32+
in:
33+
input_file: input_file
34+
out: [contents, cat_stderr]
35+
grep0:
36+
run: grep0.cwl
37+
in:
38+
word: word0
39+
text_file: cat/contents
40+
out: [occur]
41+
grep1:
42+
run: grep1.cwl
43+
in:
44+
word: word1
45+
text_file: cat/contents
46+
out: [occur]
47+
tar:
48+
run: tar.cwl
49+
in:
50+
file0: grep0/occur
51+
file1: grep1/occur
52+
tarball_fname: tarball_fname
53+
out: [tarball]
54+
cat2:
55+
run: cat2.cwl
56+
in:
57+
input_file2: input_file2
58+
out: [contents,cat_stderr2]
59+
grep2:
60+
run: grep2.cwl
61+
in:
62+
word: word2
63+
text_file: cat2/contents
64+
out: [occur]
65+
grep3:
66+
run: grep3.cwl
67+
in:
68+
word: word3
69+
text_file: cat2/contents
70+
out: [occur]
71+
tar2:
72+
run: tar2.cwl
73+
in:
74+
file2: grep2/occur
75+
file3: grep3/occur
76+
tarball_fname2: tarball_fname2
77+
out: [tarball2]
78+

‎beeflow/tests/test_neo4j_cypher.py

+24
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
"""Tests for neo4j_cypher module."""
22

3+
from unittest.mock import call
34
import pytest
45
from beeflow.common.gdb import neo4j_cypher
56

@@ -54,3 +55,26 @@ def test_get_task_hints(mocker, hints, expected):
5455
)
5556
result = neo4j_cypher.get_task_hints(tx, "")
5657
assert result == expected
58+
59+
60+
@pytest.mark.parametrize(
61+
"restart, single, expected",
62+
[
63+
(True, None, True),
64+
(False, None, True),
65+
(True, True, False),
66+
(False, True, False),
67+
],
68+
)
69+
def test_final_tasks_completed(mocker, restart, single, expected):
70+
"""Regression test final_tasks_completed."""
71+
tx = mocker.MagicMock()
72+
tx.run().single.return_value = single
73+
mocker.patch(
74+
"beeflow.common.gdb.neo4j_cypher.get_workflow_by_id",
75+
return_value={"restart": restart},
76+
)
77+
result = neo4j_cypher.final_tasks_completed(tx, "WFID")
78+
assert ("|RESTARTED_FROM" in tx.run.mock_calls[1].args[0]) == restart
79+
assert tx.run.mock_calls[2] == call().single()
80+
assert result == expected

0 commit comments

Comments
 (0)