Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: [TKC-2683] run docker agent command #5921

Merged
merged 11 commits into from
Oct 14, 2024
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions build/kind/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -168,14 +168,14 @@ send_telenetry "docker_installation_started"

# Check if agent key is provided
if [ -z "$AGENT_KEY" ]; then
log "Please provide AGENT_KEY env var"
log "Testkube installation failed. Please provide AGENT_KEY env var"
send_telenetry "docker_installation_failed" "parameter_not_found" "agent key is empty"
exit 1
fi

# Check if cloud url is provided
if [ -z "$CLOUD_URL" ]; then
log "Please provide CLOUD_URL env var"
log "Testkube installation failed. Please provide CLOUD_URL env var"
send_telenetry "docker_installation_failed" "parameter_not_found" "cloud url is empty"
exit 1
fi
Expand Down Expand Up @@ -205,7 +205,7 @@ else
log "Creating Kubernetes cluster using Kind (Kubernetes v1.31.0)..."
kind create cluster --name testkube-cluster --image kindest/node:v1.31.0 --wait 5m
if [ $? -ne 0 ]; then
log "Failed to create Kind cluster."
log "Testkube installation failed. Couldn't create Kind cluster."
send_telenetry "docker_installation_failed" "kind_error" "Kind cluster was not created"
exit 1
fi
Expand All @@ -214,7 +214,7 @@ else
log "Verifying cluster is up..."
kubectl cluster-info
if [ $? -ne 0 ]; then
log "Failed to verify cluster."
log "Testkube installation failed. Couldn't verify cluster."
send_telenetry "docker_installation_failed" "kind_error" "Kind cluster is nor accessible"
exit 1
fi
Expand Down Expand Up @@ -244,7 +244,7 @@ else

# Check if there are any pods in the Testkube namespace
if [ -z "$pod_status" ]; then
log "No pods found in testkube namespace."
log "Testkube installation failed. No pods found in testkube namespace."
send_telenetry "docker_installation_failed" "tetkube_error" "No pods found in testkube namespace"
exit 1
fi
Expand Down Expand Up @@ -293,7 +293,7 @@ else
done

if [ $counter -eq 15 ]; then
log "Testkube validation failed."
log "Testkube installation failed."
send_telenetry "docker_installation_failed" "tetkube_error" "Testkube pods are not up and running"
exit 1
fi
Expand All @@ -303,7 +303,7 @@ else
log "Creating and running Testkube k6 Test Workflow..."
kubectl apply -f /examples/k6.yaml -n testkube

log "Testkube installation successful!"
log "Testkube installation succeed!"
log "You can now use Testkube in your Kind Kubernetes cluster."
send_telenetry "docker_installation_succeed"
fi
Expand Down
12 changes: 11 additions & 1 deletion cmd/kubectl-testkube/commands/common/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,23 @@ const (
TKErrConfigInitFailed ErrorCode = "TKERR-1201"
// TKErrInvalidInstallConfig is returned when invalid configuration is supplied when installing or upgrading.
TKErrInvalidInstallConfig ErrorCode = "TKERR-1202"
// TKErrInvalidDockerConfig is returned when docker client configuration is invalid.
TKErrInvalidDockerConfig ErrorCode = "TKERR-1203"

// TKERR-13xx errors are related to install operations.

// TKErrHelmCommandFailed is returned when a helm command fails.
TKErrHelmCommandFailed ErrorCode = "TKERR-1301"
// TKErrKubectlCommandFailed is returned when a kubectl command fail.
// TKErrKubectlCommandFailed is returned when a kubectl command fails.
TKErrKubectlCommandFailed ErrorCode = "TKERR-1302"
// TKErrDockerCommandFailed is returned when a docker command fails.
TKErrDockerCommandFailed ErrorCode = "TKERR-1303"
// TKErrDockerLogStreamingFailed is returned when a docker log streaming fails.
TKErrDockerLogStreamingFailed ErrorCode = "TKERR-1304"
// TKErrDockerLogReadingFailed is returned when a docker log reading fails.
TKErrDockerLogReadingFailed ErrorCode = "TKERR-1305"
// TKErrDockerInstallationFailed is returned when a docker installation fails.
TKErrDockerInstallationFailed ErrorCode = "TKERR-1306"

// TKErrCleanOldMigrationJobFailed is returned in case of issues with old migration jobs.
TKErrCleanOldMigrationJobFailed ErrorCode = "TKERR-1401"
Expand Down
219 changes: 216 additions & 3 deletions cmd/kubectl-testkube/commands/common/helper.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
package common

import (
"bufio"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os/exec"
"strings"
"time"

"github.com/docker/docker/api/types/container"
"github.com/docker/docker/client"
"github.com/pkg/errors"
"github.com/skratchdot/open-golang/open"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -38,8 +44,10 @@ type HelmOptions struct {
}

const (
github = "GitHub"
gitlab = "GitLab"
github = "GitHub"
gitlab = "GitLab"
dockerDaemonPrefixLen = 8
latestReleaseUrl = "https://api.github.com/repos/kubeshop/testkube/releases/latest"
)

func (o HelmOptions) GetApiURI() string {
Expand Down Expand Up @@ -292,7 +300,7 @@ func PopulateHelmFlags(cmd *cobra.Command, options *HelmOptions) {
cmd.Flags().BoolVar(&options.EmbeddedNATS, "embedded-nats", false, "embedded NATS server in agent")
}

func PopulateLoginDataToContext(orgID, envID, token, refreshToken string, options HelmOptions, cfg config.Data) error {
func PopulateLoginDataToContext(orgID, envID, token, refreshToken, dockerContainerName string, options HelmOptions, cfg config.Data) error {
if options.Master.AgentToken != "" {
cfg.CloudContext.AgentKey = options.Master.AgentToken
}
Expand All @@ -315,6 +323,7 @@ func PopulateLoginDataToContext(orgID, envID, token, refreshToken string, option
if refreshToken != "" {
cfg.CloudContext.RefreshToken = refreshToken
}
cfg.CloudContext.DockerContainerName = dockerContainerName

cfg, err := PopulateOrgAndEnvNames(cfg, orgID, envID, options.Master.URIs.Api)
if err != nil {
Expand Down Expand Up @@ -759,3 +768,207 @@ func UiGetNamespace(cmd *cobra.Command, defaultNamespace string) string {

return namespace
}

func RunDockerCommand(args []string) (output string, cliErr *CLIError) {
out, err := process.Execute("docker", args...)
if err != nil {
return "", NewCLIError(
TKErrDockerCommandFailed,
"Docker command failed",
"Check is the Docker service installed and running on your computer by executing 'docker info' ",
err,
)
}
return string(out), nil
}

func DockerRunTestkubeAgent(options HelmOptions, cfg config.Data, dockerContainerName, dockerImage string) *CLIError {
// use config if set
if cfg.CloudContext.AgentKey != "" && options.Master.AgentToken == "" {
options.Master.AgentToken = cfg.CloudContext.AgentKey
}

if options.Master.AgentToken == "" {
return NewCLIError(
TKErrInvalidInstallConfig,
"Invalid install config",
"Provide the agent token by setting the '--agent-token' flag",
errors.New("agent key is required"))
}

args := prepareTestkubeProDockerArgs(options, dockerContainerName, dockerImage)
output, err := RunDockerCommand(args)
if err != nil {
return err
}

ui.Debug("Docker command output:")
ui.Debug("Arguments", args...)

ui.Debug("Docker run testkube output", output)

return nil
}

// prepareTestkubeProDockerArgs prepares docker arguments for Testkube Pro running.
func prepareTestkubeProDockerArgs(options HelmOptions, dockerContainerName, dockerImage string) []string {
args := []string{
"run",
"--name", dockerContainerName,
"--privileged",
"-d",
"-e", "CLOUD_URL=" + options.Master.URIs.Agent,
"-e", "AGENT_KEY=" + options.Master.AgentToken,
dockerImage,
}

return args
}

// prepareTestkubeUpgradeDockerArgs prepares docker arguments for Testkube Upgrade running.
func prepareTestkubeUpgradeDockerArgs(options HelmOptions, dockerContainerName, latestVersion string) []string {
args := []string{
"exec",
dockerContainerName,
"helm",
"upgrade",
// These arguments are similar to Docker entrypoint script
"testkube",
"testkube/testkube",
"--namespace",
"testkube",
"--set",
"testkube-api.minio.enabled=false",
"--set",
"mongodb.enabled=false",
"--set",
"testkube-dashboard.enabled=false",
"--set",
"testkube-api.cloud.key=" + options.Master.AgentToken,
"--set",
"testkube-api.cloud.url=" + options.Master.URIs.Agent,
"--set",
"testkube-api.dockerImageVersion=" + latestVersion,
}

return args
}

func StreamDockerLogs(dockerContainerName string) *CLIError {
// Create a Docker client
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return NewCLIError(
TKErrInvalidDockerConfig,
"Invalid docker config",
"Check your environment variables used to connect to Docker daemon",
err)
}

ctx := context.Background()
// Set options to stream logs and show both stdout and stderr logs
opts := container.LogsOptions{
ShowStdout: true,
ShowStderr: true,
Follow: true, // Follow logs in real-time
Timestamps: false,
}

// Fetch logs from the container
logs, err := cli.ContainerLogs(ctx, dockerContainerName, opts)
if err != nil {
return NewCLIError(
TKErrDockerLogStreamingFailed,
"Docker log streaming failed",
"Check that your Testkube Docker Agent container is up and runnning",
err)
}
defer logs.Close()

// Use a buffered scanner to read the logs line by line
scanner := bufio.NewScanner(logs)
for scanner.Scan() {
line := scanner.Bytes()
if len(line) > dockerDaemonPrefixLen {
line = line[dockerDaemonPrefixLen:]
}

if ui.IsVerbose() {
fmt.Println(string(line)) // Optional: print logs to console
}

if strings.Contains(string(line), "Testkube installation succeed") {
break
}

if strings.Contains(string(line), "Testkube installation failed") {
return NewCLIError(
TKErrDockerInstallationFailed,
"Docker installation failed",
"Check logs of your Testkube Docker Agent container",
errors.New(string(line)))
}
}

if err := scanner.Err(); err != nil {
return NewCLIError(
TKErrDockerLogReadingFailed,
"Docker log reading failed",
"Check logs of your Testkube Docker Agent container",
err)
}

return nil
}

func DockerUpgradeTestkubeAgent(options HelmOptions, latestVersion string, cfg config.Data) *CLIError {
// use config if set
if cfg.CloudContext.AgentKey != "" && options.Master.AgentToken == "" {
options.Master.AgentToken = cfg.CloudContext.AgentKey
}

if options.Master.AgentToken == "" {
return NewCLIError(
TKErrInvalidInstallConfig,
"Invalid install config",
"Provide the agent token by setting the '--agent-token' flag",
errors.New("agent key is required"))
}

args := prepareTestkubeUpgradeDockerArgs(options, cfg.CloudContext.DockerContainerName, latestVersion)
output, err := RunDockerCommand(args)
if err != nil {
return err
}

ui.Debug("Docker command output:")
ui.Debug("Arguments", args...)

ui.Debug("Docker run testkube output", output)

return nil
}

type releaseMetadata struct {
TagName string `json:"tag_name"`
}

func GetLatestVersion() (string, error) {
resp, err := http.Get(latestReleaseUrl)
if err != nil {
return "", err
}
defer resp.Body.Close()

data, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}

var metadata releaseMetadata
if err := json.Unmarshal(data, &metadata); err != nil {
return "", err
}

return strings.TrimPrefix(metadata.TagName, "v"), nil
}
1 change: 1 addition & 0 deletions cmd/kubectl-testkube/commands/pro.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ func NewProCmd() *cobra.Command {
cmd.AddCommand(pro.NewDisconnectCmd())
cmd.AddCommand(pro.NewInitCmd())
cmd.AddCommand(pro.NewLoginCmd())
cmd.AddCommand(pro.NewDockerCmd())

return cmd
}
2 changes: 1 addition & 1 deletion cmd/kubectl-testkube/commands/pro/connect.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ func NewConnectCmd() *cobra.Command {
token, refreshToken, err = common.LoginUser(opts.Master.URIs.Auth)
ui.ExitOnError("user login", err)
}
err = common.PopulateLoginDataToContext(opts.Master.OrgId, opts.Master.EnvId, token, refreshToken, opts, cfg)
err = common.PopulateLoginDataToContext(opts.Master.OrgId, opts.Master.EnvId, token, refreshToken, "", opts, cfg)

ui.ExitOnError("Setting Pro environment context", err)

Expand Down
Loading
Loading