feat: add terminator to Python DSL

argoproj-labs · Jul 21, 2021 · 8293394 · 8293394
1 parent e5face5
commit 8293394
Show file tree

Hide file tree

Showing 7 changed files with 49 additions and 30 deletions.
diff --git a/dsls/python/argo_dataflow/pipeline.py b/dsls/python/argo_dataflow/pipeline.py
@@ -171,13 +171,13 @@ def dump(self):
 
 
 class Step:
-    def __init__(self, name, sources=[], volumes=[]):
+    def __init__(self, name, sources=[], volumes=[], terminator=False):
         self._name = name
         self._sources = sources
         self._sinks = []
         self._scale = {}
         self._volumes = volumes
-        self._terminator = False
+        self._terminator = terminator
         self._annotations = []
 
     def log(self, name=None):
@@ -245,8 +245,8 @@ def dump(self):
 
 
 class ContainerStep(Step):
-    def __init__(self, name, image, args, fifo=False, volumes=[], volumeMounts=[], sources=[], env={}, resources={}):
-        super().__init__(name, sources=sources, volumes=volumes)
+    def __init__(self, name, image, args, fifo=False, volumes=[], volumeMounts=[], sources=[], env={}, resources={}, terminator=False):
+        super().__init__(name, sources=sources, volumes=volumes,terminator=terminator)
         self._image = image
         self._args = args
         self._fifo = fifo
@@ -295,8 +295,8 @@ def dump(self):
 
 
 class GitStep(Step):
-    def __init__(self, name, url, branch, path, image, sources=[], env=None):
-        super().__init__(name, sources=sources)
+    def __init__(self, name, url, branch, path, image, sources=[], env=None, terminator=False):
+        super().__init__(name, sources=sources, terminator=terminator)
         self._url = url
         self._branch = branch
         self._path = path
@@ -357,8 +357,8 @@ def dump(self):
 
 
 class CodeStep(Step):
-    def __init__(self, name, source=None, code=None, runtime=None, sources=[]):
-        super().__init__(name, sources=sources)
+    def __init__(self, name, source=None, code=None, runtime=None, sources=[], terminator=False):
+        super().__init__(name, sources=sources, terminator=terminator)
         if source:
             self._source = inspect.getsource(source)
         else:
@@ -404,9 +404,9 @@ def dump(self):
     def cat(self, name):
         return CatStep(name, sources=[self])
 
-    def container(self, name, image, args=[], fifo=False, volumes=[], volumeMounts=[], env={}, resources={}):
+    def container(self, name, image, args=[], fifo=False, volumes=[], volumeMounts=[], env={}, resources={},terminator=False):
         return ContainerStep(name, sources=[self], image=image, args=args, fifo=fifo, volumes=volumes,
-                             volumeMounts=volumeMounts, env=env, resources=resources)
+                             volumeMounts=volumeMounts, env=env, resources=resources,terminator=terminator)
 
     def expand(self, name):
         return ExpandStep(name, sources=[self])
@@ -434,8 +434,8 @@ def cat(name):
     return CatStep(name, [])
 
 
-def container(name, image, args, fifo=False, volumes=[], volumeMounts=[], env={}, resources={}):
-    return ContainerStep(name, sources=[], image=image, args=args, fifo=fifo, volumes=volumes,
+def container(name, image, args, fifo=False, volumes=[], volumeMounts=[], env={}, resources={}, terminator=False):
+    return ContainerStep(name, sources=[], terminator=terminator, image=image, args=args, fifo=fifo, volumes=volumes,
                          volumeMounts=volumeMounts, env=env, resources=resources)
 
 

diff --git a/examples/107-completion-pipeline.py b/examples/107-completion-pipeline.py
@@ -15,7 +15,7 @@
      .step(
         (container('main',
                     args=['sh', '-c', 'exit 0'],
-                    image='ubuntu:latest'
+                    image='golang:1.16'
                     )
          ))
      .save())
diff --git a/examples/107-completion-pipeline.yaml b/examples/107-completion-pipeline.yaml
@@ -21,5 +21,5 @@ spec:
       - sh
       - -c
       - exit 0
-      image: ubuntu:latest
+      image: golang:1.16
     name: main
diff --git a/examples/107-terminator-pipeline.py b/examples/107-terminator-pipeline.py
@@ -0,0 +1,23 @@
+from argo_dataflow import pipeline, container
+
+if __name__ == '__main__':
+    (pipeline("107-terminator")
+     .owner('argoproj-labs')
+     .describe("""This example demonstrates having a terminator step, and then terminating other steps
+      using different terminations strategies.
+      """)
+     .annotate('dataflow.argoproj.io/wait-for', 'Completed')
+     .step(
+        (container('main',
+                   args=['sh', '-c', 'cat'],
+                   image='golang:1.16'
+                   )
+         ))
+     .step(
+        (container('terminator',
+                   args=['sh', '-c', 'exit 0'],
+                   image='golang:1.16',
+                   terminator=True
+                   )
+         ))
+     .save())
diff --git a/examples/107-terminator-pipeline.yaml b/examples/107-terminator-pipeline.yaml
@@ -2,30 +2,26 @@ apiVersion: dataflow.argoproj.io/v1alpha1
 kind: Pipeline
 metadata:
   annotations:
-    dataflow.argoproj.io/description: |
-      This example demostrates having a terminator step, and then terminating other steps
-      using different terminations strategies.
+    dataflow.argoproj.io/description: "This example demonstrates having a terminator\
+      \ step, and then terminating other steps\n      using different terminations\
+      \ strategies.\n      "
+    dataflow.argoproj.io/owner: argoproj-labs
     dataflow.argoproj.io/wait-for: Completed
-  creationTimestamp: null
-  name: terminator
+  name: 107-terminator
 spec:
   steps:
   - container:
       args:
       - sh
       - -c
-      - |
-        cat
-      image: ubuntu:latest
-    name: ubuntu
-  - cat: {}
-    name: runner
+      - cat
+      image: golang:1.16
+    name: main
   - container:
       args:
       - sh
       - -c
-      - |
-        exit 0
-      image: ubuntu:latest
+      - exit 0
+      image: golang:1.16
     name: terminator
     terminator: true
diff --git a/examples/108-container-pipeline.py b/examples/108-container-pipeline.py
@@ -8,7 +8,7 @@
      .step(
         (container('main',
                    args=['sh', '-c', 'exit 0'],
-                   image='ubuntu:latest',
+                   image='golang:1.16',
                    env={'FOO': 'bar'},
                    resources={'requests': {'cpu': 1}}
                    )

diff --git a/examples/108-container-pipeline.yaml b/examples/108-container-pipeline.yaml
@@ -16,7 +16,7 @@ spec:
       env:
       - name: FOO
         value: bar
-      image: ubuntu:latest
+      image: golang:1.16
       resources:
         requests:
           cpu: 1