WIP: Implement deployment lifecycle hooks

pkg/cmd/server/origin/master.go

@@ -52,6 +52,7 @@ import (
 	deploycontroller "github.com/openshift/origin/pkg/deploy/controller/deployment"
 	deployconfigcontroller "github.com/openshift/origin/pkg/deploy/controller/deploymentconfig"
 	imagechangecontroller "github.com/openshift/origin/pkg/deploy/controller/imagechange"
+	execnewpodcontroller "github.com/openshift/origin/pkg/deploy/controller/lifecycle/execnewpod"
 	deployconfiggenerator "github.com/openshift/origin/pkg/deploy/generator"
 	deployregistry "github.com/openshift/origin/pkg/deploy/registry/deploy"
 	deployconfigregistry "github.com/openshift/origin/pkg/deploy/registry/deployconfig"
@@ -698,6 +699,7 @@ func (c *MasterConfig) RunDeployerPodController() {
 	_, kclient := c.DeploymentControllerClients()
 	factory := deployerpodcontroller.DeployerPodControllerFactory{
 		KubeClient: kclient,
+		Codec:      latest.Codec,
 	}
 
 	controller := factory.Create()
@@ -733,6 +735,16 @@ func (c *MasterConfig) RunDeploymentImageChangeTriggerController() {
 	controller.Run()
 }
 
+func (c *MasterConfig) RunExecNewPodController() {
+	_, kclient := c.DeploymentControllerClients()
+	factory := &execnewpodcontroller.ExecNewPodControllerFactory{
+		KubeClient: kclient,
+	}
+
+	controller := factory.Create()
+	controller.Run()
+}
+
 // RouteAllocator returns a route allocation controller.
 func (c *MasterConfig) RouteAllocator() *routeallocationcontroller.RouteAllocationController {
 	factory := routeallocationcontroller.RouteAllocationControllerFactory{

pkg/deploy/api/types.go

@@ -52,6 +52,8 @@ type DeploymentStrategy struct {
 	Type DeploymentStrategyType `json:"type,omitempty"`
 	// CustomParams are the input to the Custom deployment strategy.
 	CustomParams *CustomDeploymentStrategyParams `json:"customParams,omitempty"`
+	// Lifecycle provides optional hooks into the deployment process.
+	Lifecycle *Lifecycle `json:"lifecycle,omitempty"`
 }
 
 // DeploymentStrategyType refers to a specific DeploymentStrategy implementation.
@@ -74,6 +76,85 @@ type CustomDeploymentStrategyParams struct {
 	Command []string `json:"command,omitempty"`
 }
 
+// Lifecycle describes actions the system should take in response to
+// deployment lifecycle events. The deployment process blocks while executing
+// lifecycle handlers. A HandleFailurePolicy determines what action is taken
+// in response to a failed handler.
+type Lifecycle struct {
+	// Pre is called immediately before the deployment strategy executes.
+	Pre *Handler `json:"pre,omitempty"`
+	// Post is called immediately after the deployment strategy executes.
+	// NOTE: AbortHandlerFailurePolicy is not supported for Post.
+	Post *Handler `json:"post,omitempty"`
+}
+
+// Handler defines a specific deployment lifecycle action.
+type Handler struct {
+	// Status is the current status of the handler.
+	Status DeploymentLifecycleStatus `json:"status"`
+	// FailurePolicy specifies what action to take if the handler fails.
+	FailurePolicy HandlerFailurePolicy `json:"failurePolicy"`
+	// ExecNewPod specifies the action to take.
+	ExecNewPod *ExecNewPodAction `json:"execNewPod,omitempty"`
+}
+
+// DeploymentLifecycleStatus represents the status of a lifecycle action.
+type DeploymentLifecycleStatus string
+
+const (
+	// DeploymentLifecycleStatusPending means the action has not yet executed.
+	DeploymentLifecycleStatusPending DeploymentLifecycleStatus = "Pending"
+	// DeploymentLifecycleStatusRunning means the action is currently executing.
+	DeploymentLifecycleStatusRunning DeploymentLifecycleStatus = "Running"
+	// DeploymentLifecycleStatusComplete means the action completed
+	// successfully, taking into account failure policy.
+	DeploymentLifecycleStatusComplete DeploymentLifecycleStatus = "Complete"
+	// DeploymentLifecycleStatusFailed means the action failed to execute,
+	// taking into account failure policy.
+	DeploymentLifecycleStatusFailed DeploymentLifecycleStatus = "Failed"
+	// DeploymentLifecycleStatusCompleteWithErrors means the action failed to
+	// execute, but the failure was ignored due to failure policy.
+	DeploymentLifecycleStatusCompleteWithErrors DeploymentLifecycleStatus = "CompleteWithErrors"
+)
+
+// HandlerFailurePolicy describes the action to take if a handler fails.
+type HandlerFailurePolicy string
+
+const (
+	// HandlerFailurePolicyRetry means retry the handler until it succeeds.
+	HandlerFailurePolicyRetry HandlerFailurePolicy = "Retry"
+	// HandlerFailurePolicyAbort means abort the deployment (if possible).
+	HandlerFailurePolicyAbort HandlerFailurePolicy = "Abort"
+	// HandlerFailurePolicyIgnore means ignore failure and continue the deployment.
+	HandlerFailurePolicyIgnore HandlerFailurePolicy = "Ignore"
+)
+
+// ExecNewPodAction runs a command in a new pod based on the specified
+// container which is assumed to be part of the deployment template.
+type ExecNewPodAction struct {
+	// Command is the action command and its arguments.
+	Command []string `json:"command"`
+	// Env is a set of environment variables to supply to the action's container.
+	Env []kapi.EnvVar `json:"env,omitempty"`
+	// ContainerName is the name of a container in the deployment pod
+	// template whose Docker image will be used for the action's container.
+	ContainerName string `json:"containerName"`
+}
+
+// These annotations are used to track pods for ExecNewPodAction.
+const (
+	// PreExecNewPodActionPodAnnotation is the name of a pre-deployment pod.
+	PreExecNewPodActionPodAnnotation = "openshift.io/deployment.lifecycle.pre.execnewpod.pod"
+	// PreExecNewPodActionPodPhaseAnnotation is the phase of a pre-deployment
+	// pod and is used to track its status and outcome.
+	PreExecNewPodActionPodPhaseAnnotation = "openshift.io/deployment.lifecycle.pre.execnewpod.phase"
+	// PostExecNewPodActionPodAnnotation is the name of a post-deployment pod.
+	PostExecNewPodActionPodAnnotation = "openshift.io/deployment.lifecycle.post.execnewpod.pod"
+	// PostDeploymentHookPodPhaseAnnotation is the phase of a post-deployment
+	// pod and is used to track its status and outcome.
+	PostExecNewPodActionPodPhaseAnnotation = "openshift.io/deployment.lifecycle.post.execnewpod.phase"
+)
+
 // A DeploymentList is a collection of deployments.
 // DEPRECATED: Like Deployment, this is no longer used.
 type DeploymentList struct {

pkg/deploy/api/v1beta1/types.go

@@ -53,6 +53,8 @@ type DeploymentStrategy struct {
 	Type DeploymentStrategyType `json:"type,omitempty"`
 	// CustomParams are the input to the Custom deployment strategy.
 	CustomParams *CustomDeploymentStrategyParams `json:"customParams,omitempty"`
+	// Lifecycle provides optional hooks into the deployment process.
+	Lifecycle *Lifecycle `json:"lifecycle,omitempty"`
 }
 
 // DeploymentStrategyType refers to a specific DeploymentStrategy implementation.
@@ -75,6 +77,82 @@ type CustomDeploymentStrategyParams struct {
 	Command []string `json:"command,omitempty"`
 }
 
+// Lifecycle describes actions the system should take in response to
+// deployment lifecycle events. The deployment process blocks while executing
+// lifecycle handlers. A HandleFailurePolicy determines what action is taken
+// in response to a failed handler.
+type Lifecycle struct {
+	// Pre is called immediately before the deployment strategy executes.
+	Pre *Handler `json:"pre,omitempty"`
+	// Post is called immediately after the deployment strategy executes.
+	// NOTE: AbortHandlerFailurePolicy is not supported for Post.
+	Post *Handler `json:"post,omitempty"`
+}
+
+// Handler defines a specific deployment lifecycle action.
+type Handler struct {
+	// Status is the current status of the handler.
+	Status DeploymentLifecycleStatus `json:"status"`
+	// FailurePolicy specifies what action to take if the handler fails.
+	FailurePolicy HandlerFailurePolicy `json:"failurePolicy"`
+	// ExecNewPod specifies the action to take.
+	ExecNewPod *ExecNewPodAction `json:"execNewPod,omitempty"`
+}
+
+// DeploymentLifecycleStatus represents the status of a lifecycle action.
+type DeploymentLifecycleStatus string
+
+const (
+	// DeploymentLifecycleStatusPending means the action has not yet executed.
+	DeploymentLifecycleStatusPending DeploymentLifecycleStatus = "Pending"
+	// DeploymentLifecycleStatusRunning means the action is currently executing.
+	DeploymentLifecycleStatusRunning DeploymentLifecycleStatus = "Running"
+	// DeploymentLifecycleStatusComplete means the action completed
+	// successfully, taking into account failure policy.
+	DeploymentLifecycleStatusComplete DeploymentLifecycleStatus = "Complete"
+	// DeploymentLifecycleStatusFailed means the action failed to execute,
+	// taking into account failure policy.
+	DeploymentLifecycleStatusFailed DeploymentLifecycleStatus = "Failed"
+)
+
+// HandlerFailurePolicy describes the action to take if a handler fails.
+type HandlerFailurePolicy string
+
+const (
+	// HandlerFailurePolicyRetry means retry the handler until it succeeds.
+	HandlerFailurePolicyRetry HandlerFailurePolicy = "Retry"
+	// HandlerFailurePolicyAbort means abort the deployment (if possible).
+	HandlerFailurePolicyAbort HandlerFailurePolicy = "Abort"
+	// HandlerFailurePolicyIgnore means ignore failure and continue the deployment.
+	HandlerFailurePolicyIgnore HandlerFailurePolicy = "Ignore"
+)
+
+// ExecNewPodAction runs a command in a new pod based on the specified
+// container which is assumed to be part of the deployment template.
+type ExecNewPodAction struct {
+	// Command is the action command and its arguments.
+	Command []string `json:"command"`
+	// Env is a set of environment variables to supply to the action's container.
+	Env []kapi.EnvVar `json:"env,omitempty"`
+	// ContainerName is the name of a container in the deployment pod
+	// template whose Docker image will be used for the action's container.
+	ContainerName string `json:"containerName"`
+}
+
+// These annotations are used to track pods for ExecNewPodAction.
+const (
+	// PreExecNewPodActionPodAnnotation is the name of a pre-deployment pod.
+	PreExecNewPodActionPodAnnotation = "openshift.io/deployment.lifecycle.pre.execnewpod.pod"
+	// PreExecNewPodActionPodPhaseAnnotation is the phase of a pre-deployment
+	// pod and is used to track its status and outcome.
+	PreExecNewPodActionPodPhaseAnnotation = "openshift.io/deployment.lifecycle.pre.execnewpod.phase"
+	// PostExecNewPodActionPodAnnotation is the name of a post-deployment pod.
+	PostExecNewPodActionPodAnnotation = "openshift.io/deployment.lifecycle.post.execnewpod.pod"
+	// PostDeploymentHookPodPhaseAnnotation is the phase of a post-deployment
+	// pod and is used to track its status and outcome.
+	PostExecNewPodActionPodPhaseAnnotation = "openshift.io/deployment.lifecycle.post.execnewpod.phase"
+)
+
 // A DeploymentList is a collection of deployments.
 // DEPRECATED: Like Deployment, this is no longer used.
 type DeploymentList struct {

pkg/deploy/controller/deployerpod/controller.go

@@ -8,6 +8,7 @@ import (
 	kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 
 	deployapi "github.com/openshift/origin/pkg/deploy/api"
+	lifecycle "github.com/openshift/origin/pkg/deploy/controller/lifecycle"
 )
 
 // DeployerPodController keeps a deployment's status in sync with the deployer pod
@@ -17,6 +18,7 @@ import (
 type DeployerPodController struct {
 	// deploymentClient provides access to deployments.
 	deploymentClient deploymentClient
+	lifecycleManager lifecycle.Interface
 }
 
 // Handle syncs pod's status with any associated deployment.
@@ -36,10 +38,35 @@ func (c *DeployerPodController) Handle(pod *kapi.Pod) error {
 	currentStatus := statusFor(deployment)
 	nextStatus := currentStatus
 
+statusSwitch:
 	switch pod.Status.Phase {
 	case kapi.PodRunning:
 		nextStatus = deployapi.DeploymentStatusRunning
 	case kapi.PodSucceeded, kapi.PodFailed:
+		// Before finalizing the status of the deployment, let any post action
+		// execute. The post action can't cause the deployment to be failed at
+		// this point.
+		postStatus, err := c.lifecycleManager.Status(lifecycle.PostDeploymentContext, deployment)
+		if err != nil {
+			return fmt.Errorf("couldn't determine post hook status for %s: %s", labelForDeployment(deployment), err)
+		}
+		switch postStatus {
+		case deployapi.DeploymentLifecycleStatusPending:
+			err := c.lifecycleManager.Execute(lifecycle.PostDeploymentContext, deployment)
+			if err != nil {
+				return fmt.Errorf("couldn't execute post hook for %s: %v", labelForDeployment(deployment), err)
+			}
+			// block the deployment
+			break statusSwitch
+		case deployapi.DeploymentLifecycleStatusRunning:
+			// block the deployment
+			break statusSwitch
+		case deployapi.DeploymentLifecycleStatusFailed,
+			deployapi.DeploymentLifecycleStatusComplete,
+			deployapi.DeploymentLifecycleStatusCompleteWithErrors:
+			// continue the deployment
+		}
+
 		nextStatus = deployapi.DeploymentStatusComplete
 		// Detect failure based on the container state
 		for _, info := range pod.Status.ContainerStatuses {

pkg/deploy/controller/deployerpod/factory.go

@@ -15,6 +15,8 @@ import (
 
 	controller "github.com/openshift/origin/pkg/controller"
 	deployapi "github.com/openshift/origin/pkg/deploy/api"
+	lifecycle "github.com/openshift/origin/pkg/deploy/controller/lifecycle"
+	execnewpod "github.com/openshift/origin/pkg/deploy/controller/lifecycle/execnewpod"
 	deployutil "github.com/openshift/origin/pkg/deploy/util"
 )
 
@@ -24,6 +26,8 @@ import (
 type DeployerPodControllerFactory struct {
 	// KubeClient is a Kubernetes client.
 	KubeClient kclient.Interface
+	// Codec is used for encoding/decoding.
+	Codec runtime.Codec
 }
 
 // Create creates a DeployerPodController.
@@ -56,6 +60,21 @@ func (factory *DeployerPodControllerFactory) Create() controller.RunnableControl
 	}
 	cache.NewPoller(pollFunc, 10*time.Second, podQueue).Run()
 
+	lifecycleManager := &lifecycle.LifecycleManager{
+		Plugins: []lifecycle.Plugin{
+			&execnewpod.Plugin{
+				PodClient: &execnewpod.PodClientImpl{
+					CreatePodFunc: func(namespace string, pod *kapi.Pod) (*kapi.Pod, error) {
+						return factory.KubeClient.Pods(namespace).Create(pod)
+					},
+				},
+			},
+		},
+		DecodeConfig: func(deployment *kapi.ReplicationController) (*deployapi.DeploymentConfig, error) {
+			return deployutil.DecodeDeploymentConfig(deployment, factory.Codec)
+		},
+	}
+
 	podController := &DeployerPodController{
 		deploymentClient: &deploymentClientImpl{
 			getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
@@ -65,6 +84,7 @@ func (factory *DeployerPodControllerFactory) Create() controller.RunnableControl
 				return factory.KubeClient.ReplicationControllers(namespace).Update(deployment)
 			},
 		},
+		lifecycleManager: lifecycleManager,
 	}
 
 	return &controller.RetryController{

pkg/deploy/controller/deployment/controller.go

@@ -9,6 +9,7 @@ import (
 	kerrors "github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
 
 	deployapi "github.com/openshift/origin/pkg/deploy/api"
+	lifecycle "github.com/openshift/origin/pkg/deploy/controller/lifecycle"
 	deployutil "github.com/openshift/origin/pkg/deploy/util"
 )
 
@@ -32,6 +33,8 @@ type DeploymentController struct {
 	makeContainer func(strategy *deployapi.DeploymentStrategy) (*kapi.Container, error)
 	// decodeConfig knows how to decode the deploymentConfig from a deployment's annotations.
 	decodeConfig func(deployment *kapi.ReplicationController) (*deployapi.DeploymentConfig, error)
+	// todo
+	lifecycleManager lifecycle.Interface
 }
 
 // fatalError is an error which can't be retried.
@@ -45,8 +48,37 @@ func (c *DeploymentController) Handle(deployment *kapi.ReplicationController) er
 	currentStatus := statusFor(deployment)
 	nextStatus := currentStatus
 
+statusSwitch:
 	switch currentStatus {
 	case deployapi.DeploymentStatusNew:
+		// Before kicking off the strategy, handle any pre action and ensure the
+		// action has reached a terminal status. A failed pre action may cause the
+		// deployment to fail.
+		preStatus, err := c.lifecycleManager.Status(lifecycle.PreDeploymentContext, deployment)
+		if err != nil {
+			return fatalError(fmt.Sprintf("couldn't determine pre hook status for %s: %s", labelForDeployment(deployment), err))
+		}
+
+		switch preStatus {
+		case deployapi.DeploymentLifecycleStatusPending:
+			err := c.lifecycleManager.Execute(lifecycle.PreDeploymentContext, deployment)
+			if err != nil {
+				return fatalError(fmt.Sprintf("couldn't execute pre hook for %s: %v", labelForDeployment(deployment), err))
+			}
+			// block the deployment
+			break statusSwitch
+		case deployapi.DeploymentLifecycleStatusRunning:
+			// block the deployment
+			break statusSwitch
+		case deployapi.DeploymentLifecycleStatusFailed:
+			// fail the deployment
+			nextStatus = deployapi.DeploymentStatusFailed
+			break statusSwitch
+		case deployapi.DeploymentLifecycleStatusComplete,
+			deployapi.DeploymentLifecycleStatusCompleteWithErrors:
+			// continue the deployment
+		}
+
 		podTemplate, err := c.makeDeployerPod(deployment)
 		if err != nil {
 			return fatalError(fmt.Sprintf("couldn't make deployer pod for %s: %v", labelForDeployment(deployment), err))