refactor(api,ui): run retention policy with feature flipping (#5755)

Signed-off-by: richardlt <[email protected]>

Author: richardlt

docs/content/docs/concepts/workflow/retention.md

@@ -3,26 +3,62 @@ title: "Retention"
 weight: 10
 ---
 
-You can configure the workflow run retention in the workflow advanced section on the CDS UI.
+You can configure two options in Workflow advanced section on the CDS UI:
+* Workflow run retention policy. A lua rule to check if a run should be kept or not.
+* Maximum number of Workflow Runs. The maximum number of run to keep for the Workflow. 
 
 ![retention.png](../images/workflow_retention.png)
 
+The dry run button allows you to test your lua expression. The result is a table filled with all runs that would be kept
 
-* The first line defines the number maximum of runs that CDS can keep for this workflow. Only a CDS administrator can update this value.
+## Workflow run retention policy
 
-* On the second line, you will be able to define your retention policy through a lua condition.
-You will be able to use these variables:
-  * <b>run_days_before</b>: to identify runs older than x days
-  * <b>git_branch_exist</b>: to identify if the git branch used for this run still exists on the git repository
-  * <b>run_status</b>: to identidy run status
-  * <b>gerrit_change_merged</b>: to identify if the gerrit change has been merged
-  * <b>gerrit_change_abandoned</b>: to identify if the gerrit change has been abandoned
-  * <b>gerrit_change_days_before</b>: to identify gerrit change older than x days
-  * and all variables defined in your workflow payload
+{{% notice note %}}
+This feature is not currently enabled by default. However, you can try this feature on a CDS project using the feature flipping.
+To activate the feature you can create a file like the following:
+```sh
+cat <<EOF > workflow-retention-policy.yml
+name: workflow-retention-policy
+rule: return project_key == "KEY_FOR_PROJECT_THAT_YOU_WANT_TO_ACTIVATE"
+EOF
+cdsctl admin feature import workflow-retention-policy.yml
+```
+{{% /notice %}}
 
-For example, the rule defined above means:
+Retention policy is defined through a lua condition. This condition should be evaluated as **true** to keep a Workflow Run.
 
-Keep workflow run for 365 days, but if branch does not exist on repository, only keep the run for 2 days.
- 
+You will be able to use these variables in conditions:
+  * **run_days_before** (number): count of days between Workflow creation date and now.
+  * **git_branch_exist** (string: true|false): True if a *git.branch* variable is set and branch still exists on the git repository.
+  * **run_status** (string: Success|Fail|...): the Workflow Run status.
+  * **gerrit_change_merged** (string: true|false): to identify if the gerrit change has been merged.
+  * **gerrit_change_abandoned** (string: true|false): to identify if the gerrit change has been abandoned.
+  * **gerrit_change_days_before** (number): to identify gerrit change older than x days.
+  * All other variables from the Workflow Run payload (ex: cds_triggered_by_username, git_branch...).
 
-* The dry run button allows you to test your lua expression. The result is a table filled with all runs that would be kept
+Examples:
+```lua
+  -- Keep Run for 365 days
+  return run_days_before < 365
+````
+```lua
+  -- Keep Run for ever
+  return true
+```
+
+## Maximum number of Workflow Runs
+
+{{% notice note %}}
+This feature is not currently enabled by default. However, you can try this feature on a CDS project using the feature flipping.
+To activate the feature you can create a file like the following:
+```sh
+cat <<EOF > workflow-retention-maxruns.yml
+name: workflow-retention-maxruns
+rule: return project_key == "KEY_FOR_PROJECT_THAT_YOU_WANT_TO_ACTIVATE"
+EOF
+cdsctl admin feature import workflow-retention-maxruns.yml
+```
+{{% /notice %}}
+
+This value can be set only by a CDS administrator. In some case it prevent a Workflow to keep a lot of runs.
+When this feature is active, you'll not be able to start new Runs on a Workflow if the maximum count was reached.

engine/api/purge/purge.go

@@ -10,12 +10,14 @@ import (
 	"github.com/rockbears/log"
 	"go.opencensus.io/stats"
 
+	"github.com/ovh/cds/engine/api/database/gorpmapping"
 	"github.com/ovh/cds/engine/api/integration"
 	"github.com/ovh/cds/engine/api/objectstore"
 	"github.com/ovh/cds/engine/api/project"
 	"github.com/ovh/cds/engine/api/services"
 	"github.com/ovh/cds/engine/api/workflow"
 	"github.com/ovh/cds/engine/cache"
+	"github.com/ovh/cds/engine/featureflipping"
 	"github.com/ovh/cds/sdk"
 	"github.com/ovh/cds/sdk/telemetry"
 )
@@ -55,6 +57,11 @@ func WorkflowRuns(ctx context.Context, DBFunc func() *gorp.DbMap, sharedStorage
 				return
 			}
 		case <-tickPurge.C:
+			// Check all workflows to mark runs that should be deleted
+			if err := MarkWorkflowRuns(ctx, DBFunc(), workflowRunsMarkToDelete); err != nil {
+				log.Warn(ctx, "purge> Error: %v", err)
+			}
+
 			log.Debug(ctx, "purge> Deleting all workflow run marked to delete...")
 			if err := deleteWorkflowRunsHistory(ctx, DBFunc(), sharedStorage, workflowRunsDeleted); err != nil {
 				log.Warn(ctx, "purge> Error on deleteWorkflowRunsHistory : %v", err)
@@ -84,6 +91,41 @@ func Workflow(ctx context.Context, store cache.Store, DBFunc func() *gorp.DbMap,
 	}
 }
 
+// MarkWorkflowRuns Deprecated: old method to mark runs to delete
+func MarkWorkflowRuns(ctx context.Context, db *gorp.DbMap, workflowRunsMarkToDelete *stats.Int64Measure) error {
+	dao := new(workflow.WorkflowDAO)
+	dao.Filters.DisableFilterDeletedWorkflow = false
+	wfs, err := dao.LoadAll(ctx, db)
+	if err != nil {
+		return err
+	}
+	for _, wf := range wfs {
+		_, enabled := featureflipping.IsEnabled(ctx, gorpmapping.Mapper, db, sdk.FeaturePurgeName, map[string]string{"project_key": wf.ProjectKey})
+		if enabled {
+			continue
+		}
+		tx, err := db.Begin()
+		if err != nil {
+			log.Error(ctx, "workflow.PurgeWorkflowRuns> error %v", err)
+			tx.Rollback() // nolint
+			continue
+		}
+		if err := workflow.PurgeWorkflowRun(ctx, tx, wf); err != nil {
+			log.Error(ctx, "workflow.PurgeWorkflowRuns> error %v", err)
+			tx.Rollback() // nolint
+			continue
+		}
+		if err := tx.Commit(); err != nil {
+			log.Error(ctx, "workflow.PurgeWorkflowRuns> unable to commit transaction:  %v", err)
+			_ = tx.Rollback()
+			continue
+		}
+	}
+
+	workflow.CountWorkflowRunsMarkToDelete(ctx, db, workflowRunsMarkToDelete)
+	return nil
+}
+
 // workflows purges all marked workflows
 func workflows(ctx context.Context, db *gorp.DbMap, store cache.Store, workflowRunsMarkToDelete *stats.Int64Measure) error {
 	query := "SELECT id, project_id FROM workflow WHERE to_delete = true ORDER BY id ASC"

engine/api/purge/purge_run.go

@@ -11,10 +11,12 @@ import (
 	"github.com/rockbears/log"
 	"go.opencensus.io/stats"
 
+	"github.com/ovh/cds/engine/api/database/gorpmapping"
 	"github.com/ovh/cds/engine/api/event"
 	"github.com/ovh/cds/engine/api/repositoriesmanager"
 	"github.com/ovh/cds/engine/api/workflow"
 	"github.com/ovh/cds/engine/cache"
+	"github.com/ovh/cds/engine/featureflipping"
 	"github.com/ovh/cds/sdk"
 	"github.com/ovh/cds/sdk/luascript"
 )
@@ -33,7 +35,7 @@ const (
 	RunChangeDayBefore = "gerrit_change_days_before"
 )
 
-func GetRetetionPolicyVariables() []string {
+func GetRetentionPolicyVariables() []string {
 	return []string{RunDaysBefore, RunStatus, RunGitBranchExist, RunChangeMerged, RunChangeAbandoned, RunChangeDayBefore, RunChangeExist}
 }
 
@@ -44,6 +46,10 @@ func markWorkflowRunsToDelete(ctx context.Context, store cache.Store, db *gorp.D
 		return err
 	}
 	for _, wf := range wfs {
+		_, enabled := featureflipping.IsEnabled(ctx, gorpmapping.Mapper, db, sdk.FeaturePurgeName, map[string]string{"project_key": wf.ProjectKey})
+		if !enabled {
+			continue
+		}
 		if err := ApplyRetentionPolicyOnWorkflow(ctx, store, db, wf, MarkAsDeleteOptions{DryRun: false}, nil); err != nil {
 			ctx = sdk.ContextWithStacktrace(ctx, err)
 			log.Error(ctx, "%v", err)

engine/api/workflow.go

@@ -155,7 +155,7 @@ func (api *API) getRetentionPolicySuggestionHandler() service.Handler {
 			}
 		}
 
-		retentionPolicySuggestion := purge.GetRetetionPolicyVariables()
+		retentionPolicySuggestion := purge.GetRetentionPolicyVariables()
 		for k := range varsPayload {
 			retentionPolicySuggestion = append(retentionPolicySuggestion, k)
 		}

engine/api/workflow_run.go

@@ -1104,6 +1104,28 @@ func (api *API) initWorkflowRun(ctx context.Context, projKey string, wf *sdk.Wor
 	}
 	workflow.ResyncNodeRunsWithCommits(ctx, api.mustDB(), api.Cache, *p, report)
 
+	_, enabled := featureflipping.IsEnabled(ctx, gorpmapping.Mapper, api.mustDB(), sdk.FeaturePurgeName, map[string]string{"project_key": wf.ProjectKey})
+	if !enabled {
+		// Purge workflow run
+		api.GoRoutines.Exec(ctx, "workflow.PurgeWorkflowRun", func(ctx context.Context) {
+			tx, err := api.mustDB().Begin()
+			defer tx.Rollback() // nolint
+			if err != nil {
+				log.Error(ctx, "workflow.PurgeWorkflowRun> error %v", err)
+				return
+			}
+			if err := workflow.PurgeWorkflowRun(ctx, tx, *wf); err != nil {
+				log.Error(ctx, "workflow.PurgeWorkflowRun> error %v", err)
+				return
+			}
+			if err := tx.Commit(); err != nil {
+				log.Error(ctx, "workflow.PurgeWorkflowRun> unable to commit transaction:  %v", err)
+				return
+			}
+			workflow.CountWorkflowRunsMarkToDelete(ctx, api.mustDB(), api.Metrics.WorkflowRunsMarkToDelete)
+		})
+	}
+
 	// Update parent
 	for i := range report.WorkflowRuns() {
 		run := &report.WorkflowRuns()[i]

sdk/featureflipping.go

@@ -6,6 +6,7 @@ const (
 	FeatureCDNArtifact  FeatureName = "cdn-artifact"
 	FeatureCDNJobLogs   FeatureName = "cdn-job-logs"
 	FeatureMFARequired  FeatureName = "mfa_required"
+	FeaturePurgeName    FeatureName = "workflow-retention-policy"
 	FeaturePurgeMaxRuns FeatureName = "workflow-retention-maxruns"
 	FeatureTracing      FeatureName = "tracing"
 )

ui/src/app/service/feature/feature.service.ts

@@ -6,6 +6,7 @@ import { Observable } from 'rxjs';
 export enum FeatureNames {
     CDNJobLogs = 'cdn-job-logs',
     CDNArtifact = 'cdn-artifact',
+    WorkflowRetentionPolicy = 'workflow-retention-policy',
     WorkflowRetentionMaxRuns = 'workflow-retention-maxruns'
 }