Fix: ttlSecondsAfterFinished not taking effect. (#15)

Signed-off-by: Syulin7 <[email protected]>
AliyunContainerService · Mar 9, 2023 · b81dfd7 · b81dfd7
1 parent 9c056b4
commit b81dfd7
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 9 deletions.
diff --git a/pkg/controller.v1/tensorflow/controller.go b/pkg/controller.v1/tensorflow/controller.go
@@ -536,7 +536,9 @@ func (tc *TFController) satisfiedExpectations(tfjob *tfv1.TFJob) bool {
 		satisfied = satisfied || tc.Expectations.SatisfiedExpectations(expectationServicesKey)
 	}
 
-	if util.CheckJobCompletedV1(tfjob.Status.Conditions) && tfjob.DeletionTimestamp == nil && tfjob.Annotations[TFCleanPodStatusLabel] == TFCleanStatusDone {
+	if util.CheckJobCompletedV1(tfjob.Status.Conditions) && tfjob.DeletionTimestamp == nil &&
+		tfjob.Annotations[TFCleanPodStatusLabel] == TFCleanStatusDone &&
+		tfjob.Spec.TTLSecondsAfterFinished == nil {
 		satisfied = false
 	}
 

diff --git a/pkg/controller.v1/tensorflow/job.go b/pkg/controller.v1/tensorflow/job.go
@@ -225,21 +225,28 @@ func (tc *TFController) cleanupTFJob(tfJob *tfv1.TFJob) error {
 		return nil
 	}
 	duration := time.Second * time.Duration(*ttl)
-	if currentTime.After(tfJob.Status.CompletionTime.Add(duration)) {
+	if tfJob.Status.CompletionTime == nil {
+		return fmt.Errorf("job completion time is nil, cannot cleanup")
+	}
+	finishTime := tfJob.Status.CompletionTime
+	expireTime := finishTime.Add(duration)
+	if currentTime.After(expireTime) {
 		err := tc.deleteTFJobHandler(tfJob)
 		if err != nil {
 			tflogger.LoggerForJob(tfJob).Warnf("Cleanup TFJob error: %v.", err)
 			return err
 		}
 		return nil
+	} else {
+		remaining := expireTime.Sub(currentTime)
+		key, err := KeyFunc(tfJob)
+		if err != nil {
+			tflogger.LoggerForJob(tfJob).Warnf("Couldn't get key for tfjob object: %v", err)
+			return err
+		}
+		tc.WorkQueue.AddAfter(key, remaining)
+		return nil
 	}
-	key, err := KeyFunc(tfJob)
-	if err != nil {
-		tflogger.LoggerForJob(tfJob).Warnf("Couldn't get key for tfjob object: %v", err)
-		return err
-	}
-	tc.WorkQueue.AddRateLimited(key)
-	return nil
 }
 
 // deleteTFJob deletes the given TFJob.