Verify that Trials were successfully deleted (#1288)

* Verify that trials were deleted
Update suggestion status

* Update suggestion requests

* Fix tests

* Fix comment

* Add recorder to test controllers

* Travis test

* Change resume exp trial condition

* Modify e2e for from volume experiment

* Fix IsRestarting check

* Fix comment

pkg/apis/controller/suggestions/v1beta1/util.go

@@ -5,6 +5,11 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const (
+	// SuggestionRestartReason is the reason for suggestion status when experiment is restarting
+	SuggestionRestartReason = "Experiment is restarting"
+)
+
 func getCondition(suggestion *Suggestion, condType SuggestionConditionType) *SuggestionCondition {
 	if suggestion.Status.Conditions != nil {
 		for _, condition := range suggestion.Status.Conditions {
@@ -64,6 +69,15 @@ func (suggestion *Suggestion) IsRunning() bool {
 	return hasCondition(suggestion, SuggestionRunning)
 }
 
+// IsRestarting returns true if suggestion running status is false and reason = SuggestionRestartReason
+func (suggestion *Suggestion) IsRestarting() bool {
+	cond := getCondition(suggestion, SuggestionRunning)
+	if cond != nil && cond.Status == v1.ConditionFalse && cond.Reason == SuggestionRestartReason {
+		return true
+	}
+	return false
+}
+
 func (suggestion *Suggestion) IsDeploymentReady() bool {
 	return hasCondition(suggestion, SuggestionDeploymentReady)
 }

pkg/controller.v1beta1/experiment/experiment_controller.go

@@ -18,7 +18,9 @@ package experiment
 
 import (
 	"context"
+	"fmt"
 	"sort"
+	"time"
 
 	"github.com/spf13/viper"
 	corev1 "k8s.io/api/core/v1"
@@ -385,12 +387,66 @@ func (r *ReconcileExperiment) deleteTrials(instance *experimentsv1beta1.Experime
 			"expectedDeletions", expected, "trials", actual)
 		expected = actual
 	}
+	deletedNames := []string{}
 	for i := 0; i < expected; i++ {
 		if err := r.Delete(context.TODO(), &trialSlice[i]); err != nil {
 			logger.Error(err, "Trial Delete error")
 			return err
 		}
+		deletedNames = append(deletedNames, trialSlice[i].Name)
 	}
+
+	// Check if trials were deleted
+	timeout := 60 * time.Second
+	endTime := time.Now().Add(timeout)
+
+	for _, name := range deletedNames {
+		var err error
+		for !errors.IsNotFound(err) && time.Now().Before(endTime) {
+			err = r.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: instance.GetNamespace()}, &trialsv1beta1.Trial{})
+		}
+		// If trials were deleted, err == IsNotFound, return error if timeout is out
+		if !errors.IsNotFound(err) {
+			return fmt.Errorf("Unable to delete trials %v, error: %v", deletedNames, err)
+		}
+	}
+
+	// We have to delete trials from suggestion status and update SuggestionCount
+	suggestion := &suggestionsv1beta1.Suggestion{}
+	err := r.Get(context.TODO(), types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}, suggestion)
+	if err != nil {
+		logger.Error(err, "Suggestion Get error")
+		return err
+	}
+
+	deletedNamesMap := make(map[string]bool)
+	for _, name := range deletedNames {
+		deletedNamesMap[name] = true
+	}
+	// Create new Trial Assignment without deleted trials
+	newTrialAssignment := []suggestionsv1beta1.TrialAssignment{}
+	for _, ta := range suggestion.Status.Suggestions {
+		if _, ok := deletedNamesMap[ta.Name]; !ok {
+			newTrialAssignment = append(newTrialAssignment, ta)
+		}
+	}
+
+	// Update suggestion spec first.
+	// If Requests <= SuggestionCount suggestion controller returns nil.
+	suggestion.Spec.Requests = int32(len(newTrialAssignment))
+	if err := r.UpdateSuggestion(suggestion); err != nil {
+		return err
+	}
+
+	// Update suggestion status
+	suggestion.Status.Suggestions = newTrialAssignment
+	suggestion.Status.SuggestionCount = int32(len(newTrialAssignment))
+	if err := r.UpdateSuggestionStatus(suggestion); err != nil {
+		return err
+	}
+
+	logger.Info("Trials were successfully deleted", "trialNames", deletedNames)
+
 	return nil
 }