feat: add git username and token for scripts (#786)

* feat: add git token field to script parameters

* feat: add secrets client to manage secrets

* feat: change scripts methods to use secrets

* feat: pass secret as env var to the k8s job

* feat: add username to repository model

* fix: check for nil repository

* feat: inject secret client dependency for api server

* style: rename package and methods

* style: rename secret package

* fix: add testkube-operator v0.9.1 dependency

* fix: use script name for secret references

Co-authored-by: Vladislav Sukhin <[email protected]>

api/v1/testkube.yaml

@@ -1404,6 +1404,12 @@ components:
         path:
           type: string
           description: if needed we can checkout particular path (dir or file) in case of BIG/mono repositories
+        username:
+          type: string
+          description: git auth username for private repositories
+        token:
+          type: string
+          description: git auth token for private repositories
 
     ExecutionRequest:
       description: scripts execution request body

cmd/api-server/main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/kubeshop/testkube/internal/pkg/api/repository/result"
 	"github.com/kubeshop/testkube/internal/pkg/api/repository/storage"
 	"github.com/kubeshop/testkube/internal/pkg/api/repository/testresult"
+	"github.com/kubeshop/testkube/pkg/secret"
 	"github.com/kubeshop/testkube/pkg/telemetry"
 	"github.com/kubeshop/testkube/pkg/ui"
 )
@@ -51,6 +52,9 @@ func main() {
 	kubeClient, err := kubeclient.GetClient()
 	ui.ExitOnError("Getting kubernetes client", err)
 
+	secretClient, err := secret.NewClient()
+	ui.ExitOnError("Getting secret client", err)
+
 	scriptsClient := scriptsclient.NewClient(kubeClient)
 	executorsClient := executorsclient.NewClient(kubeClient)
 	testsClient := testsclient.NewClient(kubeClient)
@@ -64,6 +68,7 @@ func main() {
 		scriptsClient,
 		executorsClient,
 		testsClient,
+		secretClient,
 	).Run()
 	ui.ExitOnError("Running API Server", err)
 }

cmd/kubectl-testkube/commands/scripts/create.go

@@ -12,6 +12,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
+// NewCreateScriptsCmd is a command tp create new scriptcustom resource
 func NewCreateScriptsCmd() *cobra.Command {
 
 	var (
@@ -21,6 +22,8 @@ func NewCreateScriptsCmd() *cobra.Command {
 		uri          string
 		gitBranch    string
 		gitPath      string
+		gitUsername  string
+		gitToken     string
 		tags         []string
 	)
 
@@ -58,10 +61,12 @@ func NewCreateScriptsCmd() *cobra.Command {
 			var repository *testkube.Repository
 			if uri != "" && gitBranch != "" {
 				repository = &testkube.Repository{
-					Type_:  "git",
-					Uri:    uri,
-					Branch: gitBranch,
-					Path:   gitPath,
+					Type_:    "git",
+					Uri:      uri,
+					Branch:   gitBranch,
+					Path:     gitPath,
+					Username: gitUsername,
+					Token:    gitToken,
 				}
 			}
 
@@ -103,6 +108,8 @@ func NewCreateScriptsCmd() *cobra.Command {
 	cmd.Flags().StringVarP(&uri, "uri", "", "", "if resource need to be loaded from URI")
 	cmd.Flags().StringVarP(&gitBranch, "git-branch", "", "", "if uri is git repository we can set additional branch parameter")
 	cmd.Flags().StringVarP(&gitPath, "git-path", "", "", "if repository is big we need to define additional path to directory/file to checkout partially")
+	cmd.Flags().StringVarP(&gitUsername, "git-username", "", "", "if git repository is private we can use username as an auth parameter")
+	cmd.Flags().StringVarP(&gitToken, "git-token", "", "", "if git repository is private we can use token as an auth parameter")
 	cmd.Flags().StringSliceVar(&tags, "tags", nil, "comma separated list of tags: --tags tag1,tag2,tag3")
 
 	return cmd

cmd/kubectl-testkube/commands/scripts/update.go

@@ -21,6 +21,8 @@ func NewUpdateScriptsCmd() *cobra.Command {
 		uri          string
 		gitBranch    string
 		gitPath      string
+		gitUsername  string
+		gitToken     string
 		tags         []string
 	)
 
@@ -56,10 +58,12 @@ func NewUpdateScriptsCmd() *cobra.Command {
 			var repository *testkube.Repository
 			if uri != "" && gitBranch != "" {
 				repository = &testkube.Repository{
-					Type_:  "git",
-					Uri:    uri,
-					Branch: gitBranch,
-					Path:   gitPath,
+					Type_:    "git",
+					Uri:      uri,
+					Branch:   gitBranch,
+					Path:     gitPath,
+					Username: gitUsername,
+					Token:    gitToken,
 				}
 			}
 
@@ -106,6 +110,8 @@ func NewUpdateScriptsCmd() *cobra.Command {
 	cmd.Flags().StringVarP(&uri, "uri", "", "", "if resource need to be loaded from URI")
 	cmd.Flags().StringVarP(&gitBranch, "git-branch", "", "", "if uri is git repository we can set additional branch parameter")
 	cmd.Flags().StringVarP(&gitPath, "git-path", "", "", "if repository is big we need to define additional path to directory/file to checkout partially")
+	cmd.Flags().StringVarP(&gitUsername, "git-username", "", "", "if git repository is private we can use username as an auth parameter")
+	cmd.Flags().StringVarP(&gitToken, "git-token", "", "", "if git repository is private we can use token as an auth parameter")
 	cmd.Flags().StringSliceVar(&tags, "tags", nil, "comma separated list of tags: --tags tag1,tag2,tag3 - Warning: by passing tags existing tags will be overwritten")
 
 	return cmd

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/gofiber/fiber/v2 v2.22.0
 	github.com/gookit/color v1.5.0
 	github.com/kelseyhightower/envconfig v1.4.0
-	github.com/kubeshop/testkube-operator v0.7.5
+	github.com/kubeshop/testkube-operator v0.9.1
 	github.com/minio/minio-go/v7 v7.0.14
 	github.com/moogar0880/problems v0.1.1
 	github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5

go.sum

@@ -370,6 +370,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubeshop/testkube-operator v0.7.5 h1:txf0QRc0R7EUb6clcvta4GEgcnp4QQzYCm4M8BRuR5k=
 github.com/kubeshop/testkube-operator v0.7.5/go.mod h1:P1LXcaJEt1eVVrdRlDZip9mNHEP7cN64USQ6u/aCOeo=
+github.com/kubeshop/testkube-operator v0.9.1 h1:rWMHuVYOJ/oItoBABG1b8vhx+kridO36JRm+8p17Lfc=
+github.com/kubeshop/testkube-operator v0.9.1/go.mod h1:P1LXcaJEt1eVVrdRlDZip9mNHEP7cN64USQ6u/aCOeo=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=

internal/app/api/v1/scripts.go

@@ -8,12 +8,14 @@ import (
 	scriptsv1 "github.com/kubeshop/testkube-operator/apis/script/v1"
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	scriptsMapper "github.com/kubeshop/testkube/pkg/mapper/scripts"
+	"github.com/kubeshop/testkube/pkg/secret"
 
+	"github.com/kubeshop/testkube/pkg/jobs"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// ListScripts for getting list of all available scripts
+// GetScriptHandler is method for getting an existing script
 func (s TestKubeAPI) GetScriptHandler() fiber.Handler {
 	return func(c *fiber.Ctx) error {
 		name := c.Params("id")
@@ -32,7 +34,7 @@ func (s TestKubeAPI) GetScriptHandler() fiber.Handler {
 	}
 }
 
-// ListScriptsHandler for getting list of all available scripts
+// ListScriptsHandler is a method for getting list of all available scripts
 func (s TestKubeAPI) ListScriptsHandler() fiber.Handler {
 	return func(c *fiber.Ctx) error {
 		namespace := c.Query("namespace", "testkube")
@@ -107,11 +109,22 @@ func (s TestKubeAPI) CreateScriptHandler() fiber.Handler {
 			return s.Error(c, http.StatusBadGateway, err)
 		}
 
+		// create secrets for script
+		stringData := map[string]string{jobs.GitUsernameSecretName: "", jobs.GitTokenSecretName: ""}
+		if request.Repository != nil {
+			stringData[jobs.GitUsernameSecretName] = request.Repository.Username
+			stringData[jobs.GitTokenSecretName] = request.Repository.Token
+		}
+
+		if err = s.SecretClient.Create(secret.GetMetadataName(request.Name), request.Namespace, stringData); err != nil {
+			return s.Error(c, http.StatusBadGateway, err)
+		}
+
 		return c.JSON(script)
 	}
 }
 
-// UpdateScriptHandler creates new script CR based on script content
+// UpdateScriptHandler updates an existing script CR based on script content
 func (s TestKubeAPI) UpdateScriptHandler() fiber.Handler {
 	return func(c *fiber.Ctx) error {
 
@@ -134,7 +147,7 @@ func (s TestKubeAPI) UpdateScriptHandler() fiber.Handler {
 			}
 		}
 
-		// we need to get resouece first and load its metadata.ResourceVersion
+		// we need to get resource first and load its metadata.ResourceVersion
 		script, err := s.ScriptsClient.Get(request.Namespace, request.Name)
 		if err != nil {
 			return s.Error(c, http.StatusBadGateway, err)
@@ -156,11 +169,26 @@ func (s TestKubeAPI) UpdateScriptHandler() fiber.Handler {
 			return s.Error(c, http.StatusBadGateway, err)
 		}
 
+		// update secrets for scipt
+		stringData := map[string]string{jobs.GitUsernameSecretName: "", jobs.GitTokenSecretName: ""}
+		if request.Repository != nil {
+			stringData[jobs.GitUsernameSecretName] = request.Repository.Username
+			stringData[jobs.GitTokenSecretName] = request.Repository.Token
+		}
+
+		if err = s.SecretClient.Update(secret.GetMetadataName(request.Name), request.Namespace, stringData); err != nil {
+			if errors.IsNotFound(err) {
+				return s.Warn(c, http.StatusNotFound, err)
+			}
+
+			return s.Error(c, http.StatusBadGateway, err)
+		}
+
 		return c.JSON(script)
 	}
 }
 
-// DeleteScriptHandler for deleting a script with id
+// DeleteScriptHandler is a method for deleting a script with id
 func (s TestKubeAPI) DeleteScriptHandler() fiber.Handler {
 	return func(c *fiber.Ctx) error {
 		name := c.Params("id")
@@ -174,6 +202,15 @@ func (s TestKubeAPI) DeleteScriptHandler() fiber.Handler {
 			return s.Error(c, http.StatusBadGateway, err)
 		}
 
+		// delete secrets for script
+		if err = s.SecretClient.Delete(secret.GetMetadataName(name), namespace); err != nil {
+			if errors.IsNotFound(err) {
+				return s.Warn(c, http.StatusNotFound, err)
+			}
+
+			return s.Error(c, http.StatusBadGateway, err)
+		}
+
 		return c.SendStatus(fiber.StatusNoContent)
 	}
 }
@@ -191,6 +228,15 @@ func (s TestKubeAPI) DeleteScriptsHandler() fiber.Handler {
 			return s.Error(c, http.StatusBadGateway, err)
 		}
 
+		// delete all secrets for scripts
+		if err = s.SecretClient.DeleteAll(namespace); err != nil {
+			if errors.IsNotFound(err) {
+				return s.Warn(c, http.StatusNotFound, err)
+			}
+
+			return s.Error(c, http.StatusBadGateway, err)
+		}
+
 		return c.SendStatus(fiber.StatusNoContent)
 	}
 }

internal/app/api/v1/server.go

@@ -17,6 +17,7 @@ import (
 	"github.com/kubeshop/testkube/internal/pkg/api/repository/testresult"
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/executor/client"
+	"github.com/kubeshop/testkube/pkg/secret"
 	"github.com/kubeshop/testkube/pkg/server"
 	"github.com/kubeshop/testkube/pkg/storage"
 	"github.com/kubeshop/testkube/pkg/storage/minio"
@@ -28,6 +29,7 @@ func NewServer(
 	scriptsClient *scriptscr.ScriptsClient,
 	executorsClient *executorscr.ExecutorsClient,
 	testsClient *testscr.TestsClient,
+	secretClient *secret.Client,
 ) TestKubeAPI {
 
 	var httpConfig server.Config
@@ -45,6 +47,7 @@ func NewServer(
 		Executor:             executor,
 		ScriptsClient:        scriptsClient,
 		ExecutorsClient:      executorsClient,
+		SecretClient:         secretClient,
 		TestsClient:          testsClient,
 		Metrics:              NewMetrics(),
 	}
@@ -61,6 +64,7 @@ type TestKubeAPI struct {
 	TestsClient          *testscr.TestsClient
 	ScriptsClient        *scriptscr.ScriptsClient
 	ExecutorsClient      *executorscr.ExecutorsClient
+	SecretClient         *secret.Client
 	Metrics              Metrics
 	Storage              storage.Client
 	storageParams        storageParams

pkg/api/v1/testkube/model_repository.go

@@ -19,4 +19,8 @@ type Repository struct {
 	Branch string `json:"branch"`
 	// if needed we can checkout particular path (dir or file) in case of BIG/mono repositories
 	Path string `json:"path,omitempty"`
+	// git auth username for private repositories
+	Username string `json:"username,omitempty"`
+	// git auth token for private repositories
+	Token string `json:"token,omitempty"`
 }