Introduce podGC strategy for deleting completed/successful pods (argo…

…proj#1234)

api/openapi-spec/swagger.json

@@ -541,6 +541,14 @@
         }
       }
     },
+    "io.argoproj.workflow.v1alpha1.PodGC": {
+      "description": "PodGC describes how to delete completed pods as they complete",
+      "properties": {
+        "strategy": {
+          "type": "string"
+        }
+      }
+    },
     "io.argoproj.workflow.v1alpha1.RawArtifact": {
       "description": "RawArtifact allows raw string content to be placed as an artifact in a container",
       "required": [
@@ -1265,6 +1273,10 @@
           "type": "integer",
           "format": "int64"
         },
+        "podGC": {
+          "description": "PodGC describes the strategy to use when to deleting completed pods",
+          "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.PodGC"
+        },
         "podPriority": {
           "description": "Priority to apply to workflow pods.",
           "type": "integer",

examples/pod-gc-strategy.yaml

@@ -0,0 +1,35 @@
+# pod gc provides the ability to delete pods automatically without deleting the workflow
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: pod-gc-strategy-
+spec:
+  entrypoint: pod-gc-strategy
+
+  podGC:
+    # pod gc strategy must be one of the following
+    # * OnPodCompletion - delete pods immediately when pod is completed (including errors/failures)
+    # * OnPodSuccess - delete pods immediately when pod is successful
+    # * OnWorkflowCompletion - delete pods when workflow is completed
+    # * OnWorkflowSuccess - delete pods when workflow is successful
+    strategy: OnPodSuccess
+
+  templates:
+  - name: pod-gc-strategy
+    steps:
+    - - name: fail
+        template: fail
+      - name: succeed
+        template: succeed
+
+  - name: fail
+    container:
+      image: alpine:3.7
+      command: [sh, -c]
+      args: ["exit 1"]
+
+  - name: succeed
+    container:
+      image: alpine:3.7
+      command: [sh, -c]
+      args: ["exit 0"]

pkg/apis/workflow/v1alpha1/workflow_types.go

@@ -52,6 +52,17 @@ const (
 	NodeTypeSuspend   NodeType = "Suspend"
 )
 
+// PodGCStrategy is the strategy when to delete completed pods for GC.
+type PodGCStrategy string
+
+// PodGCStrategy
+const (
+	PodGCOnPodCompletion      PodGCStrategy = "OnPodCompletion"
+	PodGCOnPodSuccess         PodGCStrategy = "OnPodSuccess"
+	PodGCOnWorkflowCompletion PodGCStrategy = "OnWorkflowCompletion"
+	PodGCOnWorkflowSuccess    PodGCStrategy = "OnWorkflowSuccess"
+)
+
 // TemplateGetter is an interface to get templates.
 type TemplateGetter interface {
 	GetNamespace() string
@@ -178,6 +189,9 @@ type WorkflowSpec struct {
 	// +optional
 	SchedulerName string `json:"schedulerName,omitempty"`
 
+	// PodGC describes the strategy to use when to deleting completed pods
+	PodGC *PodGC `json:"podGC,omitempty"`
+
 	// PriorityClassName to apply to workflow pods.
 	PodPriorityClassName string `json:"podPriorityClassName,omitempty"`
 
@@ -395,6 +409,11 @@ type Artifact struct {
 	Optional bool `json:"optional,omitempty"`
 }
 
+// PodGC describes how to delete completed pods as they complete
+type PodGC struct {
+	Strategy PodGCStrategy `json:"strategy,omitempty"`
+}
+
 // ArchiveStrategy describes how to archive files/directory when saving artifacts
 type ArchiveStrategy struct {
 	Tar  *TarStrategy  `json:"tar,omitempty"`
@@ -716,6 +735,11 @@ func (ws *WorkflowStatus) Completed() bool {
 	return isCompletedPhase(ws.Phase)
 }
 
+// Successful return whether or not the workflow has succeeded
+func (ws *WorkflowStatus) Successful() bool {
+	return ws.Phase == NodeSucceeded
+}
+
 // Remove returns whether or not the node has completed execution
 func (n NodeStatus) Completed() bool {
 	return isCompletedPhase(n.Phase) || n.IsDaemoned() && n.Phase != NodePending

workflow/common/util.go

@@ -19,6 +19,7 @@ import (
 	"github.com/valyala/fasttemplate"
 	apiv1 "k8s.io/api/core/v1"
 	apierr "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -444,6 +445,21 @@ func addPodMetadata(c kubernetes.Interface, field, podName, namespace, key, valu
 	return err
 }
 
+const deleteRetries = 3
+
+// DeletePod deletes a pod. Ignores NotFound error
+func DeletePod(c kubernetes.Interface, podName, namespace string) error {
+	var err error
+	for attempt := 0; attempt < deleteRetries; attempt++ {
+		err = c.CoreV1().Pods(namespace).Delete(podName, &metav1.DeleteOptions{})
+		if err == nil || apierr.IsNotFound(err) {
+			break
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return err
+}
+
 // IsPodTemplate returns whether the template corresponds to a pod
 func IsPodTemplate(tmpl *wfv1.Template) bool {
 	if tmpl.Container != nil || tmpl.Script != nil || tmpl.Resource != nil {