refactor(cdn,sdk,cdsctl): remove old artifacts engine (#6117)

Signed-off-by: Yvonnick Esnault <[email protected]>

Author: yesnault

cli/cdsctl/workflow_artifact.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"os"
 	"regexp"
@@ -12,7 +11,6 @@ import (
 
 	"github.com/ovh/cds/cli"
 	"github.com/ovh/cds/sdk"
-	"github.com/ovh/cds/sdk/cdn"
 )
 
 var workflowArtifactCmd = cli.Command{
@@ -23,141 +21,10 @@ var workflowArtifactCmd = cli.Command{
 
 func workflowArtifact() *cobra.Command {
 	return cli.NewCommand(workflowArtifactCmd, nil, []*cobra.Command{
-		cli.NewListCommand(workflowArtifactListCmd, workflowArtifactListRun, nil, withAllCommandModifiers()...),
 		cli.NewCommand(workflowArtifactDownloadCmd, workflowArtifactDownloadRun, nil, withAllCommandModifiers()...),
-		cli.NewCommand(workflowArtifactCDNMigrateCmd, workflowArtifactCDNMigrate, nil, withAllCommandModifiers()...),
 	})
 }
 
-var workflowArtifactCDNMigrateCmd = cli.Command{
-	Name:  "cdn-migrate",
-	Short: "Migrate artifact into CDN",
-	Long: `Migrate artifact from CDS to CDN
-cdsctl workflow artifact cdn-migrate <project_key> <workflow_name> <run_number>
-
-CDN does not manage artifact with same name inside a run. They will be renamed <num>_<artifact_name>
-Migrated artifacts will not be visible with cdsctl workflow result command
-`,
-	Ctx: []cli.Arg{
-		{Name: _ProjectKey},
-		{Name: _WorkflowName},
-	},
-	Args: []cli.Arg{
-		{Name: "number"},
-	},
-}
-
-func workflowArtifactCDNMigrate(v cli.Values) error {
-	number, err := strconv.ParseInt(v.GetString("number"), 10, 64)
-	if err != nil {
-		return cli.NewError("number parameter have to be an integer")
-	}
-	projKey := v.GetString(_ProjectKey)
-	wName := v.GetString(_WorkflowName)
-
-	run, err := client.WorkflowRunGet(projKey, wName, number)
-	if err != nil {
-		return err
-	}
-
-	workflowArtifacts, err := client.WorkflowRunArtifacts(projKey, wName, number)
-	if err != nil {
-		return err
-	}
-
-	artName := make(map[string]int64)
-	for _, art := range workflowArtifacts {
-		prefix := ""
-		nb, has := artName[art.Name]
-		if has {
-			prefix = fmt.Sprintf("%d_", nb)
-			fmt.Printf("%s/%s will be renamed to %s%s\n", art.Name, art.Tag, prefix, art.Name)
-
-		}
-		artName[art.Name] = nb + 1
-		artName := prefix + art.Name
-		// Call cdn to migrate
-		sign := cdn.Signature{
-			ProjectKey:   projKey,
-			WorkflowName: wName,
-			WorkflowID:   run.WorkflowID,
-			RunID:        art.WorkflowID,
-			RunNumber:    run.Number,
-			NodeRunID:    art.WorkflowNodeRunID,
-			JobName:      "",
-			JobID:        0,
-			Worker: &cdn.SignatureWorker{
-				FileName:      artName,
-				FilePerm:      art.Perm,
-				RunResultType: string(sdk.CDNTypeItemRunResult),
-			},
-		}
-
-		// call cdn
-		url := fmt.Sprintf("/migrate/artifact/%s/%s/%d", projKey, wName, art.ID)
-		bts, _ := json.Marshal(sign)
-		if bts, err := client.ServiceCallPOST(sdk.TypeCDN, url, bts); err != nil {
-			fmt.Printf("unable to migrate %s: %s: %v\n", art.Name, string(bts), err)
-			continue
-		}
-		fmt.Printf("artifact %s migrated\n", artName)
-
-	}
-	fmt.Printf("Migration done.")
-	return nil
-}
-
-var workflowArtifactListCmd = cli.Command{
-	Name:  "list",
-	Short: "List artifacts of one Workflow Run",
-	Ctx: []cli.Arg{
-		{Name: _ProjectKey},
-		{Name: _WorkflowName},
-	},
-	Args: []cli.Arg{
-		{Name: "number"},
-	},
-}
-
-func workflowArtifactListRun(v cli.Values) (cli.ListResult, error) {
-	number, err := strconv.ParseInt(v.GetString("number"), 10, 64)
-	if err != nil {
-		return nil, cli.NewError("number parameter have to be an integer")
-	}
-
-	workflowArtifacts, err := client.WorkflowRunArtifacts(v.GetString(_ProjectKey), v.GetString(_WorkflowName), number)
-	if err != nil {
-		return nil, err
-	}
-
-	results, err := client.WorkflowRunResultsList(context.Background(), v.GetString(_ProjectKey), v.GetString(_WorkflowName), number)
-	if err != nil {
-		return nil, err
-	}
-
-	type Artifact struct {
-		Name string `cli:"name"`
-		Md5  string `cli:"md5"`
-	}
-
-	artifacts := make([]Artifact, 0, len(workflowArtifacts))
-	for _, art := range workflowArtifacts {
-		artifacts = append(artifacts, Artifact{Name: art.Name, Md5: art.MD5sum})
-	}
-	for _, runResult := range results {
-		if runResult.Type != sdk.WorkflowRunResultTypeArtifact {
-			continue
-		}
-		artiData, err := runResult.GetArtifact()
-		if err != nil {
-			return nil, err
-		}
-		artifacts = append(artifacts, Artifact{Name: artiData.Name, Md5: artiData.MD5})
-	}
-
-	return cli.AsListResult(artifacts), nil
-}
-
 var workflowArtifactDownloadCmd = cli.Command{
 	Name:  "download",
 	Short: "Download artifacts of one Workflow Run",
@@ -200,14 +67,6 @@ func workflowArtifactDownloadRun(v cli.Values) error {
 		cdnURL = confCDN.HTTPURL
 	}
 
-	ok, err := downloadFromCDSAPI(v, number)
-	if err != nil {
-		return err
-	}
-	if ok {
-		return nil
-	}
-
 	// Search in result
 	results, err := client.WorkflowRunResultsList(context.Background(), v.GetString(_ProjectKey), v.GetString(_WorkflowName), number)
 	if err != nil {
@@ -222,6 +81,7 @@ func workflowArtifactDownloadRun(v cli.Values) error {
 			return cli.WrapError(err, "exclude parameter is not valid")
 		}
 	}
+	var ok bool
 	for _, runResult := range results {
 		if runResult.Type != sdk.WorkflowRunResultTypeArtifact {
 			continue
@@ -288,94 +148,3 @@ func workflowArtifactDownloadRun(v cli.Values) error {
 	}
 	return nil
 }
-
-func downloadFromCDSAPI(v cli.Values, number int64) (bool, error) {
-	artifacts, err := client.WorkflowRunArtifacts(v.GetString(_ProjectKey), v.GetString(_WorkflowName), number)
-	if err != nil {
-		return false, err
-	}
-
-	var reg *regexp.Regexp
-	if len(v.GetString("exclude")) > 0 {
-		var err error
-		reg, err = regexp.Compile(v.GetString("exclude"))
-		if err != nil {
-			return false, cli.WrapError(err, "exclude parameter is not valid")
-		}
-	}
-
-	var ok bool
-
-	for _, a := range artifacts {
-		if v.GetString("artifact-name") != "" && v.GetString("artifact-name") != a.Name {
-			continue
-		}
-		if v.GetString("exclude") != "" && reg.MatchString(a.Name) {
-			fmt.Printf("File %s is excluded from download\n", a.Name)
-			continue
-		}
-
-		var f *os.File
-		var toDownload bool
-		if _, err := os.Stat(a.Name); os.IsNotExist(err) {
-			toDownload = true
-		} else {
-			// file exists, check sha512
-			var errf error
-			f, errf = os.OpenFile(a.Name, os.O_RDWR|os.O_CREATE, os.FileMode(a.Perm))
-			if errf != nil {
-				return ok, errf
-			}
-			sha512sum, err512 := sdk.FileSHA512sum(a.Name)
-			if err512 != nil {
-				return ok, err512
-			}
-
-			if sha512sum != a.SHA512sum {
-				toDownload = true
-			}
-		}
-
-		if toDownload {
-			var errf error
-			f, errf = os.OpenFile(a.Name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.FileMode(a.Perm))
-			if errf != nil {
-				return ok, errf
-			}
-			fmt.Printf("Downloading %s...\n", a.Name)
-			if err := client.WorkflowNodeRunArtifactDownload(v.GetString(_ProjectKey), v.GetString(_WorkflowName), a, f); err != nil {
-				return ok, err
-			}
-			if err := f.Close(); err != nil {
-				return ok, err
-			}
-		}
-
-		sha512sum, err512 := sdk.FileSHA512sum(a.Name)
-		if err512 != nil {
-			return ok, err512
-		}
-
-		if sha512sum != a.SHA512sum {
-			return ok, cli.NewError("Invalid sha512sum \ndownloaded file:%s\n%s:%s", sha512sum, f.Name(), a.SHA512sum)
-		}
-
-		md5sum, errmd5 := sdk.FileMd5sum(a.Name)
-		if errmd5 != nil {
-			return ok, errmd5
-		}
-
-		if md5sum != a.MD5sum {
-			return ok, cli.NewError("Invalid md5sum \ndownloaded file:%s\n%s:%s", md5sum, f.Name(), a.MD5sum)
-		}
-
-		if toDownload {
-			fmt.Printf("File %s created, checksum OK\n", f.Name())
-		} else {
-			fmt.Printf("File %s already downloaded, checksum OK\n", f.Name())
-		}
-
-		ok = true
-	}
-	return ok, nil
-}

contrib/actions/cds-go-build.yml

@@ -92,7 +92,7 @@ steps:
   - "    \techo \"File ${GOPATH}/src/{{.package}}/{{.binary}} not found!\""
   - '        exit 1;'
   - "\tfi"
-  - "\tworker upload --tag={{.cds.version}} \"${GOPATH}/src/{{.package}}/{{.binary}}\""
+  - "\tworker upload \"${GOPATH}/src/{{.package}}/{{.binary}}\""
   - else
   - "\techo \"artifact upload: {{.artifactUpload}}. So, artifact is not uploaded\""
   - fi;

contrib/actions/cds-split-upload.yml

@@ -55,11 +55,11 @@ steps:
   - 'worker export ${SOURCE}_md5 $x'
   - ""
   - 'echo -e "\n$x > ${SOURCE}-md5"'
-  - 'worker upload --tag={{.cds.version}} ${SOURCE}-md5'
+  - 'worker upload ${SOURCE}-md5'
   - ""
   - echo -e "\n"
   - 'split --verbose --bytes={{.splitSize}} ${ARGS} ${SOURCE} ${PREFIX_HANDLE}-'
   - ""
   - echo -e "\n"
-  - 'worker upload --tag=${TAG} ./${PREFIX_HANDLE}-*'
+  - 'worker upload ./${PREFIX_HANDLE}-*'
   - ""

engine/cdn/cdn_router.go

@@ -38,8 +38,6 @@ func (s *Service) initRouter(ctx context.Context) {
 	r.Handle("/item/{type}/{apiRef}/download/{unit}", nil, r.GET(s.getItemDownloadInUnitHandler, service.OverrideAuth(s.itemAccessMiddleware)))
 	r.Handle("/item/{type}/{apiRef}/lines", nil, r.GET(s.getItemLogsLinesHandler, service.OverrideAuth(s.itemAccessMiddleware)))
 
-	r.Handle("/migrate/artifact/{projectKey}/{workflowName}/{artifactID}", nil, r.POST(s.migrateArtifactInCDNHandler))
-
 	r.Handle("/unit", nil, r.GET(s.getUnitsHandler))
 	r.Handle("/unit/{id}", nil, r.DELETE(s.deleteUnitHandler))
 	r.Handle("/unit/{id}/item", nil, r.DELETE(s.markItemUnitAsDeleteHandler))

engine/cdn/item_upload.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/ovh/cds/engine/service"
 	"github.com/ovh/cds/sdk"
-	_ "github.com/ovh/cds/sdk"
 	"github.com/ovh/cds/sdk/cdn"
 	"github.com/ovh/cds/sdk/jws"
 )

engine/cdn/migrate_handler.go

@@ -38,7 +38,7 @@ func InitMarathonMarathonTest(opts marathonJDD) *HatcheryMarathon {
 	if opts.WorkerSpawnTimeout > 0 {
 		h.Config.WorkerSpawnTimeout = opts.WorkerSpawnTimeout
 	}
-	h.Client = cdsclient.New(cdsclient.Config{Host: "http://lolcat.host", InsecureSkipVerifyTLS: false})
+	h.Client = cdsclient.New(cdsclient.Config{Host: "http://cds-api.local", InsecureSkipVerifyTLS: false})
 	gock.InterceptClient(h.Client.(cdsclient.Raw).HTTPClient())
 	return h
 }