Skip to content

Commit

Permalink
feat: support layout of cron source messages
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexec
alexec committed May 17, 2021
1 parent 290cde4 commit 9e6ab3d
Show file tree
Hide file tree
Showing 13 changed files with 280 additions and 166 deletions.
6 changes: 3 additions & 3 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ COPY .git/ .git/
COPY api/ api/
COPY shared/ shared/
COPY manager/ manager/
RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 go build -a -ldflags="-X 'github.com/argoproj-labs/argo-dataflow/shared/util.message=xx$(git log -n1 --oneline)'" -o bin/manager ./manager
RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 go build -a -ldflags="-X 'github.com/argoproj-labs/argo-dataflow/shared/util.message=$(git log -n1 --oneline)'" -o bin/manager ./manager

FROM gcr.io/distroless/static:nonroot AS controller
WORKDIR /
Expand All @@ -28,9 +28,9 @@ COPY .git/ .git/
COPY api/ api/
COPY shared/ shared/
COPY runner/ runner/
RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 go build -a -ldflags="-X 'github.com/argoproj-labs/argo-dataflow/shared/util.message=xx$(git log -n1 --oneline)'" -o bin/runner ./runner
RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 go build -a -ldflags="-X 'github.com/argoproj-labs/argo-dataflow/shared/util.message=$(git log -n1 --oneline)'" -o bin/runner ./runner
COPY kill/ kill/
RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 go build -a -ldflags="-X 'github.com/argoproj-labs/argo-dataflow/shared/util.message=xx$(git log -n1 --oneline)'" -o bin/kill ./kill
RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 go build -a -ldflags="-X 'github.com/argoproj-labs/argo-dataflow/shared/util.message=$(git log -n1 --oneline)'" -o bin/kill ./kill

FROM gcr.io/distroless/static:nonroot AS runner
WORKDIR /
Expand Down
2 changes: 2 additions & 0 deletions api/v1alpha1/cron.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,6 @@ package v1alpha1

type Cron struct {
Schedule string `json:"schedule" protobuf:"bytes,1,opt,name=schedule"`
// +kubebuilder:default="2006-01-02T15:04:05Z07:00"
Layout string `json:"layout,omitempty" protobuf:"bytes,2,opt,name=layout"`
}
365 changes: 203 additions & 162 deletions api/v1alpha1/generated.pb.go
View file

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions api/v1alpha1/generated.proto
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions config/ci.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -757,6 +757,9 @@ spec:
properties:
cron:
properties:
layout:
default: 2006-01-02T15:04:05Z07:00
type: string
schedule:
type: string
required:
Expand Down Expand Up @@ -2565,6 +2568,9 @@ spec:
properties:
cron:
properties:
layout:
default: 2006-01-02T15:04:05Z07:00
type: string
schedule:
type: string
required:
Expand Down
3 changes: 3 additions & 0 deletions config/crd/bases/dataflow.argoproj.io_pipelines.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1123,6 +1123,9 @@ spec:
properties:
cron:
properties:
layout:
default: 2006-01-02T15:04:05Z07:00
type: string
schedule:
type: string
required:
Expand Down
3 changes: 3 additions & 0 deletions config/crd/bases/dataflow.argoproj.io_steps.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1080,6 +1080,9 @@ spec:
properties:
cron:
properties:
layout:
default: 2006-01-02T15:04:05Z07:00
type: string
schedule:
type: string
required:
Expand Down
6 changes: 6 additions & 0 deletions config/default.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -757,6 +757,9 @@ spec:
properties:
cron:
properties:
layout:
default: 2006-01-02T15:04:05Z07:00
type: string
schedule:
type: string
required:
Expand Down Expand Up @@ -2565,6 +2568,9 @@ spec:
properties:
cron:
properties:
layout:
default: 2006-01-02T15:04:05Z07:00
type: string
schedule:
type: string
required:
Expand Down
6 changes: 6 additions & 0 deletions config/dev.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -757,6 +757,9 @@ spec:
properties:
cron:
properties:
layout:
default: 2006-01-02T15:04:05Z07:00
type: string
schedule:
type: string
required:
Expand Down Expand Up @@ -2565,6 +2568,9 @@ spec:
properties:
cron:
properties:
layout:
default: 2006-01-02T15:04:05Z07:00
type: string
schedule:
type: string
required:
Expand Down
6 changes: 6 additions & 0 deletions config/quick-start.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -757,6 +757,9 @@ spec:
properties:
cron:
properties:
layout:
default: 2006-01-02T15:04:05Z07:00
type: string
schedule:
type: string
required:
Expand Down Expand Up @@ -2565,6 +2568,9 @@ spec:
properties:
cron:
properties:
layout:
default: 2006-01-02T15:04:05Z07:00
type: string
schedule:
type: string
required:
Expand Down
15 changes: 15 additions & 0 deletions docs/EXAMPLES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,21 @@ It also shows an example of a pipelines terminates based on a single step's stat
kubectl apply -f https://raw.githubusercontent.com/argoproj-labs/argo-dataflow/main/examples/201-word-count-pipeline.yaml
```

### [Cron](https://raw.githubusercontent.com/argoproj-labs/argo-dataflow/main/examples/301-cron-pipeline.yaml)

This example uses a cron source.

You can format dates using a "layout":

https://golang.org/pkg/time/#Time.Format

By deafult, the layout is RFC3339.


```
kubectl apply -f https://raw.githubusercontent.com/argoproj-labs/argo-dataflow/main/examples/301-cron-pipeline.yaml
```

### [Erroring](https://raw.githubusercontent.com/argoproj-labs/argo-dataflow/main/examples/301-erroring-pipeline.yaml)

This example creates errors randomly
Expand Down
23 changes: 23 additions & 0 deletions examples/301-cron-pipeline.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
apiVersion: dataflow.argoproj.io/v1alpha1
kind: Pipeline
metadata:
annotations:
dataflow.argoproj.io/description: |
This example uses a cron source.
You can format dates using a "layout":
https://golang.org/pkg/time/#Time.Format
By deafult, the layout is RFC3339.
dataflow.argoproj.io/name: Cron
creationTimestamp: null
name: cron
spec:
steps:
- cat: {}
name: main
sources:
- cron:
layout: "15:04:05"
schedule: '*/3 * * * * *'
2 changes: 1 addition & 1 deletion runner/sidecar/sidecar.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ func connectSources(ctx context.Context, toMain func([]byte) error) error {
if c := source.Cron; c != nil {
logger.Info("connecting to source", "type", "cron", "schedule", c.Schedule)
_, err := x.AddFunc(c.Schedule, func() {
data := []byte(time.Now().Format(time.RFC3339))
data := []byte(time.Now().Format(c.Layout))
debug.Info("◷ cron →", "m", printable(data))
withLock(func() { sourceStatues.Set(source.Name, replica, printable(data)) })
if err := toMain(data); err != nil {
Expand Down

0 comments on commit 9e6ab3d

Please sign in to comment.