diff --git a/manager/orchestrator/update/updater.go b/manager/orchestrator/update/updater.go
index 2b1f55d3f2..8ba417c780 100644
--- a/manager/orchestrator/update/updater.go
+++ b/manager/orchestrator/update/updater.go
@@ -430,7 +430,7 @@ func (u *Updater) updateTask(ctx context.Context, slot orchestrator.Slot, update
 				u.updatedTasks[updated.ID] = time.Now()
 				u.updatedTasksMu.Unlock()
 
-				if startThenStop {
+				if startThenStop && updated.Status.State == api.TaskStateRunning {
 					err := u.store.Batch(func(batch *store.Batch) error {
 						_, err := u.removeOldTasks(ctx, batch, slot)
 						if err != nil {