Skip to content

Commit 521a739

Browse files
sguiheuxsguiheux
authored
feat(worker): add workflow command add run result (#5805)
1 parent 824c027 commit 521a739

28 files changed

+458
-78
lines changed

‎cli/cdsctl/workflow_run_result.go

+5-4
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@ var workflowRunResultCmd = cli.Command{
2020
}
2121

2222
type RunResultCli struct {
23-
ID string `cli:"id"`
24-
Type sdk.WorkflowRunResultType `cli:"type"`
25-
Name string `cli:"name"`
23+
ID string `cli:"id"`
24+
Type string `cli:"type"`
25+
Name string `cli:"name"`
2626
}
2727

2828
func workflowRunResult() *cobra.Command {
@@ -223,6 +223,7 @@ func workflowRunResultList(v cli.Values) (cli.ListResult, error) {
223223
func toCLIRunResult(results []sdk.WorkflowRunResult) ([]RunResultCli, error) {
224224
cliresults := make([]RunResultCli, 0, len(results))
225225
for _, r := range results {
226+
artiType := string(r.Type)
226227
var name string
227228
switch r.Type {
228229
case sdk.WorkflowRunResultTypeCoverage:
@@ -247,7 +248,7 @@ func toCLIRunResult(results []sdk.WorkflowRunResult) ([]RunResultCli, error) {
247248

248249
cliresults = append(cliresults, RunResultCli{
249250
ID: r.ID,
250-
Type: r.Type,
251+
Type: artiType,
251252
Name: name,
252253
})
253254
}

‎contrib/integrations/artifactory/artifactory.yml

-19
This file was deleted.

‎contrib/integrations/artifactory/plugin-artifactory-download-artifact/main.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,9 @@ func createArtifactoryClient(url, token string) (artifactory.ArtifactoryServices
7777
}
7878

7979
func (e *artifactoryDownloadArtifactPlugin) Run(_ context.Context, opts *integrationplugin.RunQuery) (*integrationplugin.RunResult, error) {
80-
cdsRepo := opts.GetOptions()["cds.integration.artifact_manager.artifactory.cds_repository"]
81-
artifactoryURL := opts.GetOptions()["cds.integration.artifact_manager.artifactory.url"]
82-
token := opts.GetOptions()["cds.integration.artifact_manager.artifactory.token"]
80+
cdsRepo := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigCdsRepository)]
81+
artifactoryURL := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigURL)]
82+
token := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigToken)]
8383

8484
filePath := opts.GetOptions()[sdk.ArtifactDownloadPluginInputFilePath]
8585
path := opts.GetOptions()[sdk.ArtifactDownloadPluginInputDestinationPath]

‎contrib/integrations/artifactory/plugin-artifactory-download-artifact/plugin-artifactory-download-artifact.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: artifactory-download-artifact-plugin
22
type: integration-download_artifact
3-
integration: Artifactory
3+
integration: ArtifactManager
44
author: "OVH SAS"
55
description: "OVH Artifactory Download Artifact Plugin"

‎contrib/integrations/artifactory/plugin-artifactory-upload-artifact/main.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -80,13 +80,13 @@ func (e *artifactoryUploadArtifactPlugin) createArtifactoryClient(url, token str
8080
}
8181

8282
func (e *artifactoryUploadArtifactPlugin) Run(_ context.Context, opts *integrationplugin.RunQuery) (*integrationplugin.RunResult, error) {
83-
cdsRepo := opts.GetOptions()["cds.integration.artifact_manager.artifactory.cds_repository"]
84-
artifactoryURL := opts.GetOptions()["cds.integration.artifact_manager.artifactory.url"]
85-
token := opts.GetOptions()["cds.integration.artifact_manager.artifactory.token"]
83+
cdsRepo := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigCdsRepository)]
84+
artifactoryURL := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigURL)]
85+
token := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigToken)]
8686
pathToUpload := opts.GetOptions()["cds.integration.artifact_manager.upload.path"]
8787
projectKey := opts.GetOptions()["cds.project"]
8888
workflowName := opts.GetOptions()["cds.workflow"]
89-
buildInfo := opts.GetOptions()["cds.integration.artifact_manager.build.info.path"]
89+
buildInfo := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigBuildInfoPath)]
9090
version := opts.GetOptions()["cds.version"]
9191

9292
artiClient, err := e.createArtifactoryClient(artifactoryURL, token)
@@ -104,7 +104,7 @@ func (e *artifactoryUploadArtifactPlugin) Run(_ context.Context, opts *integrati
104104

105105
summary, err := artiClient.UploadFilesWithSummary(params)
106106
if err != nil || summary.TotalFailed > 0 {
107-
return fail("unable to upload file %s into artifactory %s: %v", pathToUpload, params.Target, err)
107+
return fail("unable to upload file %s into artifactory[%s] %s: %v", pathToUpload, artifactoryURL, params.Target, err)
108108
}
109109
defer summary.Close()
110110

‎contrib/integrations/artifactory/plugin-artifactory-upload-artifact/plugin-artifactory-upload-artifact.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: artifactory-upload-artifact-plugin
22
type: integration-upload_artifact
3-
integration: Artifactory
3+
integration: ArtifactManager
44
author: "OVH SAS"
55
description: "OVH Artifactory Upload Artifact Plugin"

‎engine/api/integration/artifact_manager/artifactory/client.go

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package artifactory
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"strconv"
7+
"time"
8+
9+
"github.com/jfrog/jfrog-client-go/artifactory"
10+
"github.com/jfrog/jfrog-client-go/artifactory/services"
11+
"github.com/ovh/cds/sdk"
12+
)
13+
14+
type FileInfoResponse struct {
15+
Checksums *FileInfoChecksum `json:"checksums"`
16+
Created time.Time `json:"created"`
17+
CreatedBy string `json:"createdBy"`
18+
DownloadURI string `json:"downloadUri"`
19+
LastModified time.Time `json:"lastModified"`
20+
LastUpdated time.Time `json:"lastUpdated"`
21+
MimeType string `json:"mimeType"`
22+
ModifiedBy string `json:"modifiedBy"`
23+
OriginalChecksums *FileInfoChecksum `json:"originalChecksums"`
24+
Path string `json:"path"`
25+
RemoteURL string `json:"remoteUrl"`
26+
Repo string `json:"repo"`
27+
Size string `json:"size"`
28+
URI string `json:"uri"`
29+
}
30+
31+
type FileInfoChecksum struct {
32+
Md5 string `json:"md5"`
33+
Sha1 string `json:"sha1"`
34+
Sha256 string `json:"sha256"`
35+
}
36+
37+
type Client struct {
38+
Asm artifactory.ArtifactoryServicesManager
39+
}
40+
41+
func (c *Client) GetFileInfo(repoName string, filePath string) (sdk.FileInfo, error) {
42+
fi := sdk.FileInfo{}
43+
repoDetails := services.RepositoryDetails{}
44+
if err := c.Asm.GetRepository(repoName, &repoDetails); err != nil {
45+
return fi, sdk.NewErrorFrom(sdk.ErrUnknownError, "unable to get repository %s: %v", repoName, err)
46+
}
47+
fi.Type = repoDetails.PackageType
48+
49+
fileInfoURL := fmt.Sprintf("%sapi/storage/%s/%s", c.Asm.GetConfig().GetServiceDetails().GetUrl(), repoName, filePath)
50+
httpDetails := c.Asm.GetConfig().GetServiceDetails().CreateHttpClientDetails()
51+
re, body, _, err := c.Asm.Client().SendGet(fileInfoURL, true, &httpDetails)
52+
if err != nil {
53+
return fi, sdk.NewErrorFrom(sdk.ErrUnknownError, "unable to call artifactory: %v", err)
54+
}
55+
56+
if re.StatusCode >= 400 {
57+
return fi, sdk.NewErrorFrom(sdk.ErrUnknownError, "unable to call artifactory [HTTP: %d] %s", re.StatusCode, string(body))
58+
}
59+
60+
var resp FileInfoResponse
61+
if err := json.Unmarshal(body, &resp); err != nil {
62+
return fi, sdk.NewErrorFrom(sdk.ErrUnknownError, "unable to read artifactory response %s: %v", string(body), err)
63+
}
64+
65+
if resp.Size != "" {
66+
s, err := strconv.ParseInt(resp.Size, 10, 64)
67+
if err != nil {
68+
return fi, sdk.NewErrorFrom(sdk.ErrInvalidData, "size return by artifactory is not an integer %s: %v", resp.Size, err)
69+
}
70+
fi.Size = s
71+
}
72+
if resp.Checksums != nil {
73+
fi.Md5 = resp.Checksums.Md5
74+
}
75+
return fi, nil
76+
}

‎engine/api/integration/artifact_manager/interface.go

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package artifact_manager
2+
3+
import (
4+
"fmt"
5+
"os"
6+
7+
"github.com/jfrog/jfrog-client-go/artifactory"
8+
"github.com/jfrog/jfrog-client-go/artifactory/auth"
9+
"github.com/jfrog/jfrog-client-go/config"
10+
11+
"github.com/jfrog/jfrog-client-go/utils/log"
12+
arti "github.com/ovh/cds/engine/api/integration/artifact_manager/artifactory"
13+
"github.com/ovh/cds/sdk"
14+
)
15+
16+
type ArtifactManager interface {
17+
GetFileInfo(repoName string, filePath string) (sdk.FileInfo, error)
18+
}
19+
20+
func NewClient(managerType, url, token string) (ArtifactManager, error) {
21+
switch managerType {
22+
case "artifactory":
23+
return newArtifactoryClient(url, token)
24+
}
25+
return nil, fmt.Errorf("artifact Manager %s not implemented", managerType)
26+
}
27+
28+
func newArtifactoryClient(url string, token string) (ArtifactManager, error) {
29+
log.SetLogger(log.NewLogger(log.INFO, os.Stdout))
30+
rtDetails := auth.NewArtifactoryDetails()
31+
rtDetails.SetUrl(url)
32+
rtDetails.SetAccessToken(token)
33+
serviceConfig, err := config.NewConfigBuilder().
34+
SetServiceDetails(rtDetails).
35+
SetThreads(1).
36+
SetDryRun(false).
37+
Build()
38+
if err != nil {
39+
return nil, fmt.Errorf("unable to create service config: %v", err)
40+
}
41+
asm, err := artifactory.New(serviceConfig)
42+
if err != nil {
43+
return nil, sdk.WrapError(err, "unable to create artifactory client")
44+
}
45+
return &arti.Client{Asm: asm}, nil
46+
}

‎engine/api/integration/builtin.go

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ var (
1616
sdk.RabbitMQIntegration,
1717
sdk.OpenstackIntegration,
1818
sdk.AWSIntegration,
19+
sdk.ArtifactManagerIntegration,
1920
}
2021
)
2122

‎engine/api/workflow/dao.go

+8
Original file line numberDiff line numberDiff line change
@@ -984,6 +984,7 @@ func checkProjectIntegration(proj sdk.Project, w *sdk.Workflow, n *sdk.Node) err
984984

985985
// checkIntegration checks integration data
986986
func checkIntegration(proj sdk.Project, w *sdk.Workflow) error {
987+
countArtifactManagerIntegration := 0
987988
for i := range w.Integrations {
988989
workflowIntegration := &w.Integrations[i]
989990
found := false
@@ -1000,6 +1001,13 @@ func checkIntegration(proj sdk.Project, w *sdk.Workflow) error {
10001001
if !found {
10011002
return sdk.WithData(sdk.ErrIntegrationtNotFound, workflowIntegration.ProjectIntegration.Name)
10021003
}
1004+
if workflowIntegration.ProjectIntegration.Model.ArtifactManager {
1005+
countArtifactManagerIntegration++
1006+
}
1007+
}
1008+
1009+
if countArtifactManagerIntegration > 1 {
1010+
return sdk.NewErrorFrom(sdk.ErrInvalidData, "you can't have multiple artifact manager integrations on a workflow")
10031011
}
10041012
return nil
10051013
}

‎engine/api/workflow/dao_test.go

+9
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,15 @@ func TestUpdateWorkflowIntegration(t *testing.T) {
302302
wfDb, err := workflow.LoadByID(context.TODO(), db, cache, *proj, w.ID, workflow.LoadOptions{})
303303
require.NoError(t, err)
304304
require.Equal(t, "newValue", wfDb.Integrations[0].Config["BuildInfo"].Value)
305+
306+
w.Integrations = append(w.Integrations, sdk.WorkflowProjectIntegration{
307+
308+
ProjectIntegration: projInt,
309+
ProjectIntegrationID: projInt.ID,
310+
})
311+
errUpdate := workflow.Update(context.TODO(), db, cache, *proj, &w, workflow.UpdateOptions{})
312+
require.NotNil(t, errUpdate)
313+
require.Contains(t, errUpdate.Error(), "you can't have multiple artifact manager integrations on a workflow")
305314
}
306315

307316
func TestInsertComplexeWorkflowAndExport(t *testing.T) {

‎engine/api/workflow/execute_node_run.go

+12-7
Original file line numberDiff line numberDiff line change
@@ -588,18 +588,23 @@ func getIntegrationPlugins(db gorp.SqlExecutor, wr *sdk.WorkflowRun, nr *sdk.Wor
588588
plugins = append(plugins, *plugin)
589589
}
590590

591-
var artifactManagerModelID int64
592-
for _, int := range wr.Workflow.Integrations {
593-
if int.ProjectIntegration.Model.ArtifactManager {
594-
artifactManagerModelID = int.ProjectIntegration.Model.ID
591+
var artifactManagerInteg *sdk.WorkflowProjectIntegration
592+
for i := range wr.Workflow.Integrations {
593+
if wr.Workflow.Integrations[i].ProjectIntegration.Model.ArtifactManager {
594+
artifactManagerInteg = &wr.Workflow.Integrations[i]
595595
}
596596
}
597-
if artifactManagerModelID != 0 {
598-
plgs, err := plugin.LoadAllByIntegrationModelID(db, artifactManagerModelID)
597+
if artifactManagerInteg != nil {
598+
plgs, err := plugin.LoadAllByIntegrationModelID(db, artifactManagerInteg.ProjectIntegration.Model.ID)
599599
if err != nil {
600600
return nil, sdk.NewErrorFrom(sdk.ErrNotFound, "Cannot find plugin for integration model id %d, %v", projectIntegrationModelID, err)
601601
}
602-
plugins = append(plugins, plgs...)
602+
platform := artifactManagerInteg.ProjectIntegration.Config[sdk.ArtifactManagerConfigPlatform]
603+
for _, plg := range plgs {
604+
if strings.HasPrefix(plg.Name, fmt.Sprintf("%s-", platform.Value)) {
605+
plugins = append(plugins, plg)
606+
}
607+
}
603608
}
604609

605610
return plugins, nil

‎engine/api/workflow/workflow_run_results.go

+48-3
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,15 @@ package workflow
22

33
import (
44
"context"
5+
"encoding/json"
6+
"fmt"
57
"strconv"
68
"time"
79

810
"github.com/go-gorp/gorp"
911

1012
"github.com/ovh/cds/engine/api/database/gorpmapping"
13+
"github.com/ovh/cds/engine/api/integration/artifact_manager"
1114
"github.com/ovh/cds/engine/cache"
1215
"github.com/ovh/cds/engine/gorpmapper"
1316
"github.com/ovh/cds/sdk"
@@ -118,7 +121,7 @@ func CanUploadRunResult(ctx context.Context, db *gorp.DbMap, store cache.Store,
118121
return true, nil
119122
}
120123

121-
func AddResult(db *gorp.DbMap, store cache.Store, runResult *sdk.WorkflowRunResult) error {
124+
func AddResult(ctx context.Context, db *gorp.DbMap, store cache.Store, wr *sdk.WorkflowRun, runResult *sdk.WorkflowRunResult) error {
122125
var cacheKey string
123126
switch runResult.Type {
124127
case sdk.WorkflowRunResultTypeArtifact:
@@ -135,7 +138,7 @@ func AddResult(db *gorp.DbMap, store cache.Store, runResult *sdk.WorkflowRunResu
135138
}
136139
case sdk.WorkflowRunResultTypeArtifactManager:
137140
var err error
138-
cacheKey, err = verifyAddResultArtifactManager(store, runResult)
141+
cacheKey, err = verifyAddResultArtifactManager(ctx, db, store, wr, runResult)
139142
if err != nil {
140143
return err
141144
}
@@ -158,14 +161,56 @@ func AddResult(db *gorp.DbMap, store cache.Store, runResult *sdk.WorkflowRunResu
158161
return sdk.WithStack(store.Delete(cacheKey))
159162
}
160163

161-
func verifyAddResultArtifactManager(store cache.Store, runResult *sdk.WorkflowRunResult) (string, error) {
164+
// Check validity of the request + complete runResuklt with md5,size,type
165+
func verifyAddResultArtifactManager(ctx context.Context, db gorp.SqlExecutor, store cache.Store, wr *sdk.WorkflowRun, runResult *sdk.WorkflowRunResult) (string, error) {
162166
artResult, err := runResult.GetArtifactManager()
163167
if err != nil {
164168
return "", err
165169
}
170+
171+
// Check file in integration
172+
var artiInteg *sdk.WorkflowProjectIntegration
173+
for i := range wr.Workflow.Integrations {
174+
if !wr.Workflow.Integrations[i].ProjectIntegration.Model.ArtifactManager {
175+
continue
176+
}
177+
artiInteg = &wr.Workflow.Integrations[i]
178+
}
179+
if artiInteg == nil {
180+
return "", sdk.NewErrorFrom(sdk.ErrInvalidData, "you cannot add a artifact manager run result without an integration")
181+
}
182+
secrets, err := loadRunSecretWithDecryption(ctx, db, wr.ID, []string{fmt.Sprintf(SecretProjIntegrationContext, artiInteg.ProjectIntegrationID)})
183+
if err != nil {
184+
return "", err
185+
}
186+
var artifactManagerToken string
187+
for _, s := range secrets {
188+
if s.Name == fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigToken) {
189+
artifactManagerToken = s.Value
190+
break
191+
}
192+
}
193+
if artifactManagerToken == "" {
194+
return "", sdk.NewErrorFrom(sdk.ErrNotFound, "unable to find artifact manager token")
195+
}
196+
artifactClient, err := artifact_manager.NewClient(artiInteg.ProjectIntegration.Config[sdk.ArtifactManagerConfigPlatform].Value, artiInteg.ProjectIntegration.Config[sdk.ArtifactManagerConfigURL].Value, artifactManagerToken)
197+
if err != nil {
198+
return "", err
199+
}
200+
fileInfo, err := artifactClient.GetFileInfo(artResult.RepoName, artResult.Path)
201+
if err != nil {
202+
return "", err
203+
}
204+
artResult.Size = fileInfo.Size
205+
artResult.MD5 = fileInfo.Md5
206+
artResult.RepoType = fileInfo.Type
207+
166208
if err := artResult.IsValid(); err != nil {
167209
return "", err
168210
}
211+
dataUpdated, _ := json.Marshal(artResult)
212+
runResult.DataRaw = dataUpdated
213+
169214
cacheKey := GetRunResultKey(runResult.WorkflowRunID, runResult.Type, artResult.Name)
170215
b, err := store.Exist(cacheKey)
171216
if err != nil {

0 commit comments

Comments
 (0)