Skip to content

Commit eb3c114

Browse files
sguiheuxsguiheux
authored
feat(sdk): add version 2 of workflow export entities (#5021)
1 parent b1b927a commit eb3c114

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+3031
-1493
lines changed

‎cli/cdsctl/workflow_init.go

+1-7
Original file line numberDiff line numberDiff line change
@@ -243,13 +243,7 @@ func interactiveChoosePipeline(pkey, defaultPipeline string) (string, *sdk.Pipel
243243

244244
func craftWorkflowFile(workflowName, appName, pipName, destinationDir string) (string, error) {
245245
// Crafting the workflow
246-
wkflw := exportentities.Workflow{
247-
Version: exportentities.WorkflowVersion1,
248-
Name: workflowName,
249-
ApplicationName: appName,
250-
PipelineName: pipName,
251-
}
252-
246+
wkflw := exportentities.InitWorkflow(workflowName, appName, pipName)
253247
b, err := exportentities.Marshal(wkflw, exportentities.FormatYAML)
254248
if err != nil {
255249
return "", fmt.Errorf("Unable to write workflow file format: %v", err)

‎engine/api/templates.go

+9-6
Original file line numberDiff line numberDiff line change
@@ -353,21 +353,24 @@ func (api *API) applyTemplate(ctx context.Context, u sdk.Identifiable, p *sdk.Pr
353353
// parse the generated workflow to find its name an update it in instance if not detached
354354
// also set the template path in generated workflow if not detached
355355
if !req.Detached {
356-
var wor exportentities.Workflow
357-
if err := yaml.Unmarshal([]byte(result.Workflow), &wor); err != nil {
356+
wor, err := exportentities.UnmarshalWorkflow([]byte(result.Workflow))
357+
if err != nil {
358358
return result, sdk.NewError(sdk.Error{
359359
ID: sdk.ErrWrongRequest.ID,
360360
Message: "Cannot parse generated workflow",
361361
}, err)
362362
}
363363

364-
wti.WorkflowName = wor.Name
364+
wti.WorkflowName = wor.GetName()
365365
if err := workflowtemplate.UpdateInstance(tx, wti); err != nil {
366366
return result, err
367367
}
368368

369369
templatePath := fmt.Sprintf("%s/%s", wt.Group.Name, wt.Slug)
370-
wor.Template = &templatePath
370+
wor, err = exportentities.SetTemplate(wor, templatePath)
371+
if err != nil {
372+
return result, err
373+
}
371374
b, err := yaml.Marshal(wor)
372375
if err != nil {
373376
return result, sdk.NewError(sdk.Error{
@@ -455,7 +458,7 @@ func (api *API) postTemplateApplyHandler() service.Handler {
455458
log.Debug("postTemplateApplyHandler> template %s applied (withImport=%v)", wt.Slug, withImport)
456459

457460
buf := new(bytes.Buffer)
458-
if err := workflowtemplate.Tar(ctx, wt, res, buf); err != nil {
461+
if err := workflowtemplate.Tar(ctx, res, buf); err != nil {
459462
return err
460463
}
461464

@@ -607,7 +610,7 @@ func (api *API) postTemplateBulkHandler() service.Handler {
607610
}
608611

609612
buf := new(bytes.Buffer)
610-
if err := workflowtemplate.Tar(ctx, wt, res, buf); err != nil {
613+
if err := workflowtemplate.Tar(ctx, res, buf); err != nil {
611614
if errD := errorDefer(err); errD != nil {
612615
log.Error(ctx, "%v", errD)
613616
return

‎engine/api/templates_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func generateTemplate(groupID int64, pipelineName string) *sdk.WorkflowTemplate
2727
Slug: slug.Convert(name),
2828
Workflow: base64.StdEncoding.EncodeToString([]byte(
2929
`name: [[.name]]
30-
version: v1.0
30+
version: v2.0
3131
workflow:
3232
Node-1:
3333
pipeline: ` + pipelineName,
@@ -150,7 +150,7 @@ func Test_postTemplateBulkHandler(t *testing.T) {
150150
Slug: slug.Convert(name),
151151
Workflow: base64.StdEncoding.EncodeToString([]byte(
152152
`name: [[.name]]
153-
version: v1.0
153+
version: v2.0
154154
workflow:
155155
Node-1:
156156
pipeline: ` + pipelineName,

‎engine/api/user_schema.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"encoding/json"
66
"fmt"
7+
v2 "github.com/ovh/cds/sdk/exportentities/v2"
78
"net/http"
89
"reflect"
910

@@ -30,7 +31,7 @@ func (api *API) getUserJSONSchema() service.Handler {
3031

3132
var sch *jsonschema.Schema
3233
if filter == "" || filter == "workflow" {
33-
sch = ref.ReflectFromType(reflect.TypeOf(exportentities.Workflow{}))
34+
sch = ref.ReflectFromType(reflect.TypeOf(v2.Workflow{}))
3435
buf, _ := json.Marshal(sch)
3536
res.Workflow = string(buf)
3637
}

‎engine/api/workflow.go

+5-7
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@ import (
1212

1313
"github.com/go-gorp/gorp"
1414
"github.com/gorilla/mux"
15-
yaml "gopkg.in/yaml.v2"
16-
1715
"github.com/ovh/cds/engine/api/application"
1816
"github.com/ovh/cds/engine/api/environment"
1917
"github.com/ovh/cds/engine/api/event"
@@ -210,8 +208,8 @@ func (api *API) postWorkflowRollbackHandler() service.Handler {
210208
return sdk.WrapError(err, "cannot load workflow audit %s/%s", key, workflowName)
211209
}
212210

213-
var exportWf exportentities.Workflow
214-
if err := yaml.Unmarshal([]byte(audit.DataBefore), &exportWf); err != nil {
211+
exportWf, err := exportentities.UnmarshalWorkflow([]byte(audit.DataBefore))
212+
if err != nil {
215213
return sdk.WrapError(err, "cannot unmarshal data before")
216214
}
217215

@@ -223,9 +221,9 @@ func (api *API) postWorkflowRollbackHandler() service.Handler {
223221
_ = tx.Rollback()
224222
}()
225223

226-
newWf, _, err := workflow.ParseAndImport(ctx, tx, api.Cache, *proj, wf, &exportWf, u, workflow.ImportOptions{Force: true, WorkflowName: workflowName})
227-
if err != nil {
228-
return sdk.WrapError(err, "cannot parse and import previous workflow")
224+
newWf, _, errP := workflow.ParseAndImport(ctx, tx, api.Cache, *proj, wf, exportWf, u, workflow.ImportOptions{Force: true, WorkflowName: workflowName})
225+
if errP != nil {
226+
return sdk.WrapError(errP, "cannot parse and import previous workflow")
229227
}
230228

231229
if err := tx.Commit(); err != nil {

‎engine/api/workflow/as_code.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"context"
66
"encoding/base64"
77
"fmt"
8+
v2 "github.com/ovh/cds/sdk/exportentities/v2"
89
"time"
910

1011
"github.com/go-gorp/gorp"
@@ -26,7 +27,7 @@ func UpdateWorkflowAsCode(ctx context.Context, store cache.Store, db gorp.SqlExe
2627

2728
var wp exportentities.WorkflowPulled
2829
buffw := new(bytes.Buffer)
29-
if _, err := exportWorkflow(ctx, wf, exportentities.FormatYAML, buffw, exportentities.WorkflowSkipIfOnlyOneRepoWebhook); err != nil {
30+
if _, err := exportWorkflow(ctx, wf, exportentities.FormatYAML, buffw, v2.WorkflowSkipIfOnlyOneRepoWebhook); err != nil {
3031
return nil, sdk.WrapError(err, "unable to export workflow")
3132
}
3233
wp.Workflow.Name = wf.Name
@@ -47,7 +48,7 @@ func MigrateAsCode(ctx context.Context, db *gorp.DbMap, store cache.Store, proj
4748
}
4849

4950
// Export workflow
50-
pull, err := Pull(ctx, db, store, proj, wf.Name, exportentities.FormatYAML, encryptFunc, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
51+
pull, err := Pull(ctx, db, store, proj, wf.Name, exportentities.FormatYAML, encryptFunc, v2.WorkflowSkipIfOnlyOneRepoWebhook)
5152
if err != nil {
5253
return nil, sdk.WrapError(err, "cannot pull workflow")
5354
}

‎engine/api/workflow/dao.go

+6-5
Original file line numberDiff line numberDiff line change
@@ -1329,7 +1329,7 @@ func checkApplication(store cache.Store, db gorp.SqlExecutor, proj sdk.Project,
13291329
if n.Context.ApplicationName != "" {
13301330
appDB, err := application.LoadByName(db, store, proj.Key, n.Context.ApplicationName, application.LoadOptions.WithDeploymentStrategies, application.LoadOptions.WithVariables)
13311331
if err != nil {
1332-
if sdk.ErrorIs(err, sdk.ErrPipelineNotFound) {
1332+
if sdk.ErrorIs(err, sdk.ErrApplicationNotFound) {
13331333
return sdk.WithStack(sdk.ErrorWithData(sdk.ErrApplicationNotFound, n.Context.ApplicationName))
13341334
}
13351335
return sdk.WrapError(err, "unable to load application %s", n.Context.ApplicationName)
@@ -1390,12 +1390,13 @@ func Push(ctx context.Context, db *gorp.DbMap, store cache.Store, proj *sdk.Proj
13901390
oldWf = opts.OldWorkflow
13911391
} else {
13921392
// load the workflow from database if exists
1393-
workflowExists, err = Exists(db, proj.Key, data.wrkflw.Name)
1393+
workflowExists, err = Exists(db, proj.Key, data.wrkflw.GetName())
13941394
if err != nil {
13951395
return nil, nil, nil, sdk.WrapError(err, "Cannot check if workflow exists")
13961396
}
13971397
if workflowExists {
1398-
oldWf, err = Load(ctx, db, store, *proj, data.wrkflw.Name, LoadOptions{WithIcon: true})
1398+
oldWf, err = Load(ctx, db, store, *proj, data.wrkflw.GetName(), LoadOptions{WithIcon: true})
1399+
13991400
if err != nil {
14001401
return nil, nil, nil, sdk.WrapError(err, "Unable to load existing workflow")
14011402
}
@@ -1477,9 +1478,9 @@ func Push(ctx context.Context, db *gorp.DbMap, store cache.Store, proj *sdk.Proj
14771478
importOptions.HookUUID = opts.HookUUID
14781479
}
14791480

1480-
wf, msgList, err := ParseAndImport(ctx, tx, store, *proj, oldWf, &data.wrkflw, u, importOptions)
1481+
wf, msgList, err := ParseAndImport(ctx, tx, store, *proj, oldWf, data.wrkflw, u, importOptions)
14811482
if err != nil {
1482-
return msgList, nil, nil, sdk.WrapError(err, "unable to import workflow %s", data.wrkflw.Name)
1483+
return msgList, nil, nil, sdk.WrapError(err, "unable to import workflow %s", data.wrkflw.GetName())
14831484
}
14841485

14851486
// If the workflow is "as-code", it should always be linked to a git repository

‎engine/api/workflow/dao_run.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ func UpdateWorkflowRun(ctx context.Context, db gorp.SqlExecutor, wr *sdk.Workflo
6363

6464
wr.LastModified = time.Now()
6565
for _, info := range wr.Infos {
66-
if info.IsError && info.SubNumber == wr.LastSubNumber {
67-
wr.Status = string(sdk.StatusFail)
66+
if info.Type == sdk.RunInfoTypeError && info.SubNumber == wr.LastSubNumber {
67+
wr.Status = sdk.StatusFail
6868
}
6969
}
7070

‎engine/api/workflow/execute_node_run.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -347,9 +347,10 @@ func executeNodeRun(ctx context.Context, db gorp.SqlExecutor, store cache.Store,
347347
log.Error(ctx, "workflow.execute> Unable to load mutex-locked workflow rnode un: %v", errWRun)
348348
return report, nil
349349
}
350-
AddWorkflowRunInfo(workflowRun, false, sdk.SpawnMsg{
350+
AddWorkflowRunInfo(workflowRun, sdk.SpawnMsg{
351351
ID: sdk.MsgWorkflowNodeMutexRelease.ID,
352352
Args: []interface{}{waitingRun.WorkflowNodeName},
353+
Type: sdk.MsgWorkflowNodeMutexRelease.Type,
353354
})
354355

355356
if err := UpdateWorkflowRun(ctx, db, workflowRun); err != nil {

‎engine/api/workflow/process.go

+6-4
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,10 @@ func checkCondition(ctx context.Context, wr *sdk.WorkflowRun, conditions sdk.Wor
5151
luacheck, err := luascript.NewCheck()
5252
if err != nil {
5353
log.Warning(ctx, "processWorkflowNodeRun> WorkflowCheckConditions error: %s", err)
54-
AddWorkflowRunInfo(wr, true, sdk.SpawnMsg{
54+
AddWorkflowRunInfo(wr, sdk.SpawnMsg{
5555
ID: sdk.MsgWorkflowError.ID,
5656
Args: []interface{}{fmt.Sprintf("Error init LUA System: %v", err)},
57+
Type: sdk.MsgWorkflowError.Type,
5758
})
5859
}
5960
luacheck.SetVariables(sdk.ParametersToMap(params))
@@ -62,22 +63,23 @@ func checkCondition(ctx context.Context, wr *sdk.WorkflowRun, conditions sdk.Wor
6263
}
6364
if errc != nil {
6465
log.Warning(ctx, "processWorkflowNodeRun> WorkflowCheckConditions error: %s", errc)
65-
AddWorkflowRunInfo(wr, true, sdk.SpawnMsg{
66+
AddWorkflowRunInfo(wr, sdk.SpawnMsg{
6667
ID: sdk.MsgWorkflowError.ID,
6768
Args: []interface{}{fmt.Sprintf("Error on LUA Condition: %v", errc)},
69+
Type: sdk.MsgWorkflowError.Type,
6870
})
6971
return false
7072
}
7173
return conditionsOK
7274
}
7375

7476
// AddWorkflowRunInfo add WorkflowRunInfo on a WorkflowRun
75-
func AddWorkflowRunInfo(run *sdk.WorkflowRun, isError bool, infos ...sdk.SpawnMsg) {
77+
func AddWorkflowRunInfo(run *sdk.WorkflowRun, infos ...sdk.SpawnMsg) {
7678
for _, i := range infos {
7779
run.Infos = append(run.Infos, sdk.WorkflowRunInfo{
7880
APITime: time.Now(),
7981
Message: i,
80-
IsError: isError,
82+
Type: i.Type,
8183
SubNumber: run.LastSubNumber,
8284
})
8385
}

‎engine/api/workflow/process_node.go

+10-6
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,10 @@ func processNodeTriggers(ctx context.Context, db gorp.SqlExecutor, store cache.S
3737
r1, _, errPwnr := processNodeRun(ctx, db, store, proj, wr, mapNodes, &t.ChildNode, int(parentSubNumber), parentNodeRun, nil, nil)
3838
if errPwnr != nil {
3939
log.Error(ctx, "processWorkflowRun> Unable to process node ID=%d: %s", t.ChildNode.ID, errPwnr)
40-
AddWorkflowRunInfo(wr, true, sdk.SpawnMsg{
40+
AddWorkflowRunInfo(wr, sdk.SpawnMsg{
4141
ID: sdk.MsgWorkflowError.ID,
4242
Args: []interface{}{errPwnr.Error()},
43+
Type: sdk.MsgWorkflowError.Type,
4344
})
4445
}
4546
_, _ = report.Merge(ctx, r1, nil)
@@ -240,9 +241,10 @@ func processNode(ctx context.Context, db gorp.SqlExecutor, store cache.Store, pr
240241
vcsServer := repositoriesmanager.GetProjectVCSServer(proj, app.VCSServer)
241242
vcsInf, errVcs = getVCSInfos(ctx, db, store, proj.Key, vcsServer, currentJobGitValues, app.Name, app.VCSServer, app.RepositoryFullname)
242243
if errVcs != nil {
243-
AddWorkflowRunInfo(wr, true, sdk.SpawnMsg{
244+
AddWorkflowRunInfo(wr, sdk.SpawnMsg{
244245
ID: sdk.MsgWorkflowError.ID,
245246
Args: []interface{}{errVcs.Error()},
247+
Type: sdk.MsgWorkflowError.Type,
246248
})
247249
return nil, false, sdk.WrapError(errVcs, "unable to get git informations")
248250
}
@@ -298,8 +300,8 @@ func processNode(ctx context.Context, db gorp.SqlExecutor, store cache.Store, pr
298300
}
299301

300302
for _, info := range wr.Infos {
301-
if info.IsError && info.SubNumber == wr.LastSubNumber {
302-
nr.Status = string(sdk.StatusFail)
303+
if info.Type == sdk.RunInfoTypeError && info.SubNumber == wr.LastSubNumber {
304+
nr.Status = sdk.StatusFail
303305
nr.Done = time.Now()
304306
break
305307
}
@@ -365,9 +367,10 @@ func processNode(ctx context.Context, db gorp.SqlExecutor, store cache.Store, pr
365367
}
366368
if nbMutex > 0 {
367369
log.Debug("Noderun %s processed but not executed because of mutex", n.Name)
368-
AddWorkflowRunInfo(wr, false, sdk.SpawnMsg{
370+
AddWorkflowRunInfo(wr, sdk.SpawnMsg{
369371
ID: sdk.MsgWorkflowNodeMutex.ID,
370372
Args: []interface{}{n.Name},
373+
Type: sdk.MsgWorkflowNodeMutex.Type,
371374
})
372375
if err := UpdateWorkflowRun(ctx, db, wr); err != nil {
373376
return nil, false, sdk.WrapError(err, "unable to update workflow run")
@@ -479,9 +482,10 @@ func computePayload(n *sdk.Node, hookEvent *sdk.WorkflowNodeRunHookEvent, manual
479482
func computeNodeContextBuildParameters(ctx context.Context, proj sdk.Project, wr *sdk.WorkflowRun, run *sdk.WorkflowNodeRun, n *sdk.Node, runContext nodeRunContext) {
480483
nodeRunParams, errParam := getNodeRunBuildParameters(ctx, proj, wr, run, runContext)
481484
if errParam != nil {
482-
AddWorkflowRunInfo(wr, true, sdk.SpawnMsg{
485+
AddWorkflowRunInfo(wr, sdk.SpawnMsg{
483486
ID: sdk.MsgWorkflowError.ID,
484487
Args: []interface{}{errParam.Error()},
488+
Type: sdk.MsgWorkflowError.Type,
485489
})
486490
// if there an error -> display it in workflowRunInfo and not stop the launch
487491
log.Error(ctx, "processNode> getNodeRunBuildParameters failed. Project:%s [#%d.%d]%s.%d with payload %v err:%v", proj.Name, wr.Number, run.SubNumber, wr.Workflow.Name, n.ID, run.Payload, errParam)

‎engine/api/workflow/process_start.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,13 @@ func processStartFromRootNode(ctx context.Context, db gorp.SqlExecutor, store ca
5555
log.Debug("processWorkflowRun> starting from the root: %d (pipeline %s)", wr.Workflow.WorkflowData.Node.ID, wr.Workflow.Pipelines[wr.Workflow.WorkflowData.Node.Context.PipelineID].Name)
5656
report := new(ProcessorReport)
5757
//Run the root: manual or from an event
58-
AddWorkflowRunInfo(wr, false, sdk.SpawnMsg{
58+
AddWorkflowRunInfo(wr, sdk.SpawnMsg{
5959
ID: sdk.MsgWorkflowStarting.ID,
6060
Args: []interface{}{
6161
wr.Workflow.Name,
6262
fmt.Sprintf("%d.%d", wr.Number, 0),
6363
},
64+
Type: sdk.MsgWorkflowStarting.Type,
6465
})
6566

6667
r1, conditionOK, errP := processNodeRun(ctx, db, store, proj, wr, mapNodes, &wr.Workflow.WorkflowData.Node, 0, nil, hookEvent, manual)

‎engine/api/workflow/repository.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,9 @@ func extractFromCDSFiles(ctx context.Context, tr *tar.Reader) (*exportedEntities
207207
mError.Append(fmt.Errorf("two workflows files found: %s and %s", workflowFileName, hdr.Name))
208208
break
209209
}
210-
if err := yaml.Unmarshal(b, &res.wrkflw); err != nil {
210+
var err error
211+
res.wrkflw, err = exportentities.UnmarshalWorkflow(b)
212+
if err != nil {
211213
log.Error(ctx, "Push> Unable to unmarshal workflow %s: %v", hdr.Name, err)
212214
mError.Append(fmt.Errorf("Unable to unmarshal workflow %s: %v", hdr.Name, err))
213215
continue

‎engine/api/workflow/resync_workflow.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ func ResyncWorkflowRunStatus(ctx context.Context, db gorp.SqlExecutor, wr *sdk.W
6161
var isInError bool
6262
var newStatus string
6363
for _, info := range wr.Infos {
64-
if info.IsError && info.SubNumber == wr.LastSubNumber {
64+
if info.Type == sdk.RunInfoTypeError && info.SubNumber == wr.LastSubNumber {
6565
isInError = true
6666
break
6767
}

‎engine/api/workflow/run_workflow.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func runFromHook(ctx context.Context, db gorp.SqlExecutor, store cache.Store, pr
5959

6060
// Add add code spawn info
6161
for _, msg := range asCodeMsg {
62-
AddWorkflowRunInfo(wr, false, sdk.SpawnMsg{ID: msg.ID, Args: msg.Args})
62+
AddWorkflowRunInfo(wr, sdk.SpawnMsg{ID: msg.ID, Args: msg.Args, Type: msg.Type})
6363
}
6464

6565
//Process it
@@ -107,7 +107,7 @@ func StartWorkflowRun(ctx context.Context, db *gorp.DbMap, store cache.Store, pr
107107
defer tx.Rollback() // nolint
108108

109109
for _, msg := range asCodeInfos {
110-
AddWorkflowRunInfo(wr, false, sdk.SpawnMsg{ID: msg.ID, Args: msg.Args})
110+
AddWorkflowRunInfo(wr, sdk.SpawnMsg{ID: msg.ID, Args: msg.Args, Type: msg.Type})
111111
}
112112

113113
wr.Status = sdk.StatusWaiting

0 commit comments

Comments
 (0)