Replaced example folder with a simpler workflow

rwth-i6 · Mar 2, 2023 · 7216081 · 7216081
1 parent b3e2a98
commit 7216081
Show file tree

Hide file tree

Showing 18 changed files with 236 additions and 437 deletions.
diff --git a/example/README.md b/example/README.md
@@ -0,0 +1,9 @@
+This folder contains a simple example workflow.
+
+You can start the workflow by running:
+`sis manager config/workflow.main`
+
+- `config/workflow.py` contains the workflow definition, start here reading the code and comments
+- `recipe/` contains the modules which define the actual jobs
+- - `tools.py` continue reading here
+- - `splitter/__init__.py` read this file last
diff --git a/example/config/__init__.py b/example/config/__init__.py
diff --git a/example/config/dump.py b/example/config/dump.py
diff --git a/example/config/workflow.py b/example/config/workflow.py
@@ -0,0 +1,49 @@
+# This imports all things typically used sisyphus
+from sisyphus import *
+from recipe import tools
+from recipe import splitter
+
+
+# Workflow example, it is possible to have multiple workflows in the same file. Since this function is named `main`
+# you can call it by running sis manager config/workflow.main
+async def main():
+    # Initialize input data, tags are optional. The have no influence on the workflow,
+    # but make it easier to track which input influenced a job
+    input_data = tk.Path('input/lorem_upsum.txt', tags={'lorem_upsum'})
+
+    # Create new lines after . and remove empty spaces at the beginning and end of each line
+    sentences = tools.Pipeline(input_data, ["sed 's:\\.:\\n:g'", "sed 's:^ *::;s: *$::'"]).out
+
+    # Count lines, lines is a Variable meaning once it's computed its value can be accessed using .get()
+    # or by converting it into a string
+    lines = tools.WordCount(sentences).lines
+
+    # You can run computations on these variables even though they are not computed yet.
+    # The requested computation is stored in a wrapper object and only resolved when .get() is called
+    # or the object is converted into a string. An exception is raised if you call get() on an unfinished Variable
+    # which doesn't have a backup value.
+    middle = lines // 2
+    first_half = tools.Head(sentences, middle).out
+    tk.register_output('first_half', first_half)
+
+    # Tell Sisyphus that this output is needed and should be linked inside the output directory
+    tk.register_output('sentences', sentences)
+
+    # Split each paragraph into a new line and again register output
+    paragraphs_job = splitter.ParagraphSplitter(sentences)
+
+    # paragraphs_job.outputs() is a coroutine, the `await` keyword will make sure that the
+    # workflow will be stopped at this point and only continue once the paragraphs_job is finished
+    paragraphs = await paragraphs_job.outputs()
+    wc_per_paragraph = [tools.WordCount(paragraph) for paragraph in paragraphs]
+
+    for i, wc in enumerate(wc_per_paragraph):
+        # You can also register variables as output:
+        tk.register_output('paragraph.%02i.words' % i, wc.words)
+
+    # All jobs inside the wc_per_paragraph list will be computed after this line
+    await tk.async_run(wc_per_paragraph)
+
+    # We can now get the final value of all variables
+    max_word_per_paragraph = max(wc.words.get() for wc in wc_per_paragraph)
+    print(f"The largest paragraph has {max_word_per_paragraph} words")
diff --git a/example/data/5lines.txt b/example/data/5lines.txt
diff --git a/example/input/lorem_upsum.txt b/example/input/lorem_upsum.txt
@@ -0,0 +1,21 @@
+Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+
+Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.
+
+Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi.
+
+Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat.
+
+Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis.
+
+At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, At accusam aliquyam diam diam dolore dolores duo eirmod eos erat, et nonumy sed tempor et et invidunt justo labore Stet clita ea et gubergren, kasd magna no rebum. sanctus sea sed takimata ut vero voluptua. est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat.
+
+Consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus.
+
+Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+
+Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.
+
+Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi.
+
+Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo
diff --git a/example/recipe/__init__.py b/example/recipe/__init__.py
diff --git a/example/recipe/direct_load.py b/example/recipe/direct_load.py
diff --git a/example/recipe/math.py b/example/recipe/math.py
diff --git a/example/recipe/parallel/__init__.py b/example/recipe/parallel/__init__.py