From d2f219f806277636ab5b4c41dd26ea87c887ec3a Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Tue, 8 Jun 2021 16:14:29 -0700 Subject: [PATCH] ok --- docs/EXAMPLES.md | 4 ++-- dsls/python/__init__.py | 2 +- examples/101-two-node-pipeline.yaml | 3 ++- examples/102-flatten-expand-pipeline.yaml | 6 ++++-- examples/108-container-pipeline.py | 1 + examples/108-container-pipeline.yaml | 1 + examples/201-vetinary-pipeline.yaml | 12 ++++++++---- examples/201-word-count-pipeline.yaml | 6 ++++-- examples/301-erroring-pipeline.py | 1 + examples/301-erroring-pipeline.yaml | 1 + examples/301-stan-pipeline.py | 1 + examples/301-stan-pipeline.yaml | 4 +++- examples/301-two-sinks-pipeline.yaml | 2 +- examples/301-two-sources-pipeline.yaml | 2 +- 14 files changed, 31 insertions(+), 15 deletions(-) diff --git a/docs/EXAMPLES.md b/docs/EXAMPLES.md index 9f267d36..7654f1a7 100644 --- a/docs/EXAMPLES.md +++ b/docs/EXAMPLES.md @@ -311,7 +311,7 @@ This example shows reading and writing to a STAN subject kubectl apply -f https://raw.githubusercontent.com/argoproj-labs/argo-dataflow/main/examples/301-stan-pipeline.yaml ``` -### [two-sinks](https://raw.githubusercontent.com/argoproj-labs/argo-dataflow/main/examples/301-two-sinks-pipeline.yaml) +### [301-two-sinks](https://raw.githubusercontent.com/argoproj-labs/argo-dataflow/main/examples/301-two-sinks-pipeline.yaml) This example has two sinks @@ -320,7 +320,7 @@ This example has two sinks kubectl apply -f https://raw.githubusercontent.com/argoproj-labs/argo-dataflow/main/examples/301-two-sinks-pipeline.yaml ``` -### [two-sources](https://raw.githubusercontent.com/argoproj-labs/argo-dataflow/main/examples/301-two-sources-pipeline.yaml) +### [301-two-sources](https://raw.githubusercontent.com/argoproj-labs/argo-dataflow/main/examples/301-two-sources-pipeline.yaml) This example has two sources diff --git a/dsls/python/__init__.py b/dsls/python/__init__.py index 7ea5cce8..2529c670 100644 --- a/dsls/python/__init__.py +++ b/dsls/python/__init__.py @@ -457,7 +457,7 @@ def dump(self): y = {'subject': self._subject} if self._parallel: y['parallel'] = self._parallel - x['stan'] = y + x['stan'] = y return x diff --git a/examples/101-two-node-pipeline.yaml b/examples/101-two-node-pipeline.yaml index 833971c3..85acb719 100644 --- a/examples/101-two-node-pipeline.yaml +++ b/examples/101-two-node-pipeline.yaml @@ -24,4 +24,5 @@ spec: - kafka: topic: output-topic sources: - - {} + - stan: + subject: a-b diff --git a/examples/102-flatten-expand-pipeline.yaml b/examples/102-flatten-expand-pipeline.yaml index a632ef85..0201bad8 100644 --- a/examples/102-flatten-expand-pipeline.yaml +++ b/examples/102-flatten-expand-pipeline.yaml @@ -23,10 +23,12 @@ spec: - stan: subject: flattened sources: - - {} + - stan: + subject: data - expand: {} name: expand sinks: - log: {} sources: - - {} + - stan: + subject: flattened diff --git a/examples/108-container-pipeline.py b/examples/108-container-pipeline.py index af2dbdee..674d9e22 100644 --- a/examples/108-container-pipeline.py +++ b/examples/108-container-pipeline.py @@ -4,6 +4,7 @@ (pipeline("108-container") .owner('argoproj-labs') .describe("""This example showcases container options.""") + .annotate('dataflow.argoproj.io/wait-for', 'Completed') .step( (container('main', args=['sh', '-c', 'exit 0'], diff --git a/examples/108-container-pipeline.yaml b/examples/108-container-pipeline.yaml index f216e4fe..96399c46 100644 --- a/examples/108-container-pipeline.yaml +++ b/examples/108-container-pipeline.yaml @@ -4,6 +4,7 @@ metadata: annotations: dataflow.argoproj.io/description: This example showcases container options. dataflow.argoproj.io/owner: argoproj-labs + dataflow.argoproj.io/wait-for: Completed name: 108-container spec: steps: diff --git a/examples/201-vetinary-pipeline.yaml b/examples/201-vetinary-pipeline.yaml index b03d3748..cdb27dee 100644 --- a/examples/201-vetinary-pipeline.yaml +++ b/examples/201-vetinary-pipeline.yaml @@ -33,7 +33,8 @@ spec: - stan: subject: cats sources: - - {} + - stan: + subject: pets - map: |- json("Meow! " + object(msg).name) name: process-cats @@ -41,7 +42,8 @@ spec: - kafka: topic: output-topic sources: - - {} + - stan: + subject: cats - filter: |- string(msg) contains "dog" name: filter-dogs @@ -49,7 +51,8 @@ spec: - stan: subject: dogs sources: - - {} + - stan: + subject: pets - map: |- json("Woof! " + object(msg).name) name: process-dogs @@ -57,4 +60,5 @@ spec: - kafka: topic: output-topic sources: - - {} + - stan: + subject: dogs diff --git a/examples/201-word-count-pipeline.yaml b/examples/201-word-count-pipeline.yaml index 29469940..834378bc 100644 --- a/examples/201-word-count-pipeline.yaml +++ b/examples/201-word-count-pipeline.yaml @@ -55,7 +55,8 @@ spec: - stan: subject: words sources: - - {} + - stan: + subject: lines - container: args: - bash @@ -75,5 +76,6 @@ spec: - kafka: topic: output-topic sources: - - {} + - stan: + subject: words terminator: true diff --git a/examples/301-erroring-pipeline.py b/examples/301-erroring-pipeline.py index 930f2cd0..4f2a9f52 100644 --- a/examples/301-erroring-pipeline.py +++ b/examples/301-erroring-pipeline.py @@ -12,6 +12,7 @@ def handler(msg): (pipeline("301-erroring") .owner('argoproj-labs') .describe("""This example showcases retry policies.""") + .annotate('dataflow.argoproj.io/wait-for', 'Errors') .step( (cron('*/3 * * * * *', retryPolicy='Always') .handler('always', handler=handler) diff --git a/examples/301-erroring-pipeline.yaml b/examples/301-erroring-pipeline.yaml index c1005407..d2fe7f2b 100644 --- a/examples/301-erroring-pipeline.yaml +++ b/examples/301-erroring-pipeline.yaml @@ -4,6 +4,7 @@ metadata: annotations: dataflow.argoproj.io/description: This example showcases retry policies. dataflow.argoproj.io/owner: argoproj-labs + dataflow.argoproj.io/wait-for: Errors name: 301-erroring spec: steps: diff --git a/examples/301-stan-pipeline.py b/examples/301-stan-pipeline.py index d014310f..81b84518 100644 --- a/examples/301-stan-pipeline.py +++ b/examples/301-stan-pipeline.py @@ -4,6 +4,7 @@ (pipeline("301-stan") .owner('argoproj-labs') .describe("""This example shows reading and writing to a STAN subject""") + .annotate('dataflow.argoproj.io/test', 'false') .step( (stan('input-subject') .cat('main') diff --git a/examples/301-stan-pipeline.yaml b/examples/301-stan-pipeline.yaml index d6809b65..b07d57b8 100644 --- a/examples/301-stan-pipeline.yaml +++ b/examples/301-stan-pipeline.yaml @@ -5,6 +5,7 @@ metadata: dataflow.argoproj.io/description: This example shows reading and writing to a STAN subject dataflow.argoproj.io/owner: argoproj-labs + dataflow.argoproj.io/test: 'false' name: 301-stan spec: steps: @@ -14,4 +15,5 @@ spec: - stan: subject: output-subject sources: - - {} + - stan: + subject: input-subject diff --git a/examples/301-two-sinks-pipeline.yaml b/examples/301-two-sinks-pipeline.yaml index 451f3e2c..29f257f9 100644 --- a/examples/301-two-sinks-pipeline.yaml +++ b/examples/301-two-sinks-pipeline.yaml @@ -5,7 +5,7 @@ metadata: dataflow.argoproj.io/description: | This example has two sinks creationTimestamp: null - name: two-sinks + name: 301-two-sinks spec: steps: - cat: {} diff --git a/examples/301-two-sources-pipeline.yaml b/examples/301-two-sources-pipeline.yaml index c65f49f3..821de781 100644 --- a/examples/301-two-sources-pipeline.yaml +++ b/examples/301-two-sources-pipeline.yaml @@ -5,7 +5,7 @@ metadata: dataflow.argoproj.io/description: | This example has two sources creationTimestamp: null - name: two-sources + name: 301-two-sources spec: steps: - cat: {}