diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 623a0c91f7..7514981ed2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,15 +33,23 @@ repos: - id: check-unicode-non-breaking-spaces exclude: > (?x)^( - .*/?\.*.(gif|jpg|png) + .*/?\.*.(gif|jpg|png|tar.gz) )$ - id: remove-unicode-non-breaking-spaces exclude: > (?x)^( - .*/?\.*.(gif|jpg|png) + .*/?\.*.(gif|jpg|png|tar.gz) )$ - id: check-en-dashes + exclude: > + (?x)^( + .*/?\.*.(tar.gz) + )$ - id: remove-en-dashes + exclude: > + (?x)^( + .*/?\.*.(tar.gz) + )$ - id: check-jjbb - id: check-gherkin-lint args: [ diff --git a/cli/config/compose/services/centos-systemd/docker-compose.yml b/cli/config/compose/services/centos-systemd/docker-compose.yml index 6721740dcb..31ca6eb33d 100644 --- a/cli/config/compose/services/centos-systemd/docker-compose.yml +++ b/cli/config/compose/services/centos-systemd/docker-compose.yml @@ -7,5 +7,4 @@ services: platform: ${stackPlatform:-linux/amd64} privileged: true volumes: - - ${centos_systemdAgentBinarySrcPath:-.}:${centos_systemdAgentBinaryTargetPath:-/tmp} - /sys/fs/cgroup:/sys/fs/cgroup:ro diff --git a/cli/config/compose/services/centos/docker-compose.yml b/cli/config/compose/services/centos/docker-compose.yml index 88e76e0751..fa1c2f2162 100644 --- a/cli/config/compose/services/centos/docker-compose.yml +++ b/cli/config/compose/services/centos/docker-compose.yml @@ -4,5 +4,3 @@ services: image: centos:${centosTag:-7} container_name: ${centosContainerName} entrypoint: tail -f /dev/null - volumes: - - ${centosAgentBinarySrcPath:-.}:${centosAgentBinaryTargetPath:-/tmp} diff --git a/cli/config/compose/services/debian-systemd/docker-compose.yml b/cli/config/compose/services/debian-systemd/docker-compose.yml index ae47b83aac..1333569887 100644 --- a/cli/config/compose/services/debian-systemd/docker-compose.yml +++ b/cli/config/compose/services/debian-systemd/docker-compose.yml @@ -7,5 +7,4 @@ services: platform: ${stackPlatform:-linux/amd64} privileged: true volumes: - - ${debian_systemdAgentBinarySrcPath:-.}:${debian_systemdAgentBinaryTargetPath:-/tmp} - /sys/fs/cgroup:/sys/fs/cgroup:ro diff --git a/cli/config/compose/services/debian/docker-compose.yml b/cli/config/compose/services/debian/docker-compose.yml index 0aa23e816c..cb4491e840 100644 --- a/cli/config/compose/services/debian/docker-compose.yml +++ b/cli/config/compose/services/debian/docker-compose.yml @@ -4,5 +4,3 @@ services: image: debian:${debianTag:-9} container_name: ${debianContainerName} entrypoint: tail -f /dev/null - volumes: - - ${debianAgentBinarySrcPath:-.}:${debianAgentBinaryTargetPath:-/tmp} diff --git a/cli/config/compose/services/fleet-server-centos/docker-compose.yml b/cli/config/compose/services/fleet-server-centos/docker-compose.yml index 8492ce3947..942ad0be4b 100644 --- a/cli/config/compose/services/fleet-server-centos/docker-compose.yml +++ b/cli/config/compose/services/fleet-server-centos/docker-compose.yml @@ -6,5 +6,4 @@ services: entrypoint: "/usr/sbin/init" privileged: true volumes: - - ${fleet_server_centosAgentBinarySrcPath:-.}:${fleet_server_centosAgentBinaryTargetPath:-/tmp} - /sys/fs/cgroup:/sys/fs/cgroup:ro diff --git a/cli/config/compose/services/fleet-server-debian/docker-compose.yml b/cli/config/compose/services/fleet-server-debian/docker-compose.yml index bb86c67f83..8b2f3c3d09 100644 --- a/cli/config/compose/services/fleet-server-debian/docker-compose.yml +++ b/cli/config/compose/services/fleet-server-debian/docker-compose.yml @@ -6,5 +6,4 @@ services: entrypoint: "/sbin/init" privileged: true volumes: - - ${fleet_server_debianAgentBinarySrcPath:-.}:${fleet_server_debianAgentBinaryTargetPath:-/tmp} - /sys/fs/cgroup:/sys/fs/cgroup:ro diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 014f87f5d5..547edd5d20 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -1154,9 +1154,6 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe common.ProfileEnv[envVarsPrefix+"Tag"] = serviceTag // we are setting the container name because Centos service could be reused by any other test suite common.ProfileEnv[envVarsPrefix+"ContainerName"] = containerName - // define paths where the binary will be mounted - common.ProfileEnv[envVarsPrefix+"AgentBinarySrcPath"] = agentInstaller.BinaryPath - common.ProfileEnv[envVarsPrefix+"AgentBinaryTargetPath"] = "/" + agentInstaller.Name serviceManager := compose.NewServiceManager() @@ -1169,6 +1166,15 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe return nil, err } + isTar := (agentInstaller.InstallerType == "tar") + targetFile := "/" + + // copy downloaded agent to the root dir of the container + err = docker.CopyFileToContainer(context.Background(), containerName, agentInstaller.BinaryPath, targetFile, isTar) + if err != nil { + return nil, err + } + err = agentInstaller.PreInstallFn() if err != nil { return nil, err diff --git a/internal/_testresources/dockerCopy.txt b/internal/_testresources/dockerCopy.txt new file mode 100644 index 0000000000..d64066171a --- /dev/null +++ b/internal/_testresources/dockerCopy.txt @@ -0,0 +1 @@ +OK! diff --git a/internal/_testresources/sample.tar.gz b/internal/_testresources/sample.tar.gz new file mode 100644 index 0000000000..0abf0b55b9 Binary files /dev/null and b/internal/_testresources/sample.tar.gz differ diff --git a/internal/docker/docker.go b/internal/docker/docker.go index f99c550ce4..592702a7e0 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -5,11 +5,14 @@ package docker import ( + "archive/tar" + "bufio" "bytes" "compress/gzip" "context" "fmt" "io" + "io/ioutil" "os" "path/filepath" "strings" @@ -28,6 +31,36 @@ var instance *client.Client // OPNetworkName name of the network used by the tool const OPNetworkName = "elastic-dev-network" +func buildTarForDeployment(file *os.File) (bytes.Buffer, error) { + fileInfo, _ := file.Stat() + + var buffer bytes.Buffer + tarWriter := tar.NewWriter(&buffer) + err := tarWriter.WriteHeader(&tar.Header{ + Name: fileInfo.Name(), + Mode: 0777, + Size: int64(fileInfo.Size()), + }) + if err != nil { + log.WithFields(log.Fields{ + "fileInfoName": fileInfo.Name(), + "size": fileInfo.Size(), + "error": err, + }).Error("Could not build TAR header") + return bytes.Buffer{}, fmt.Errorf("could not build TAR header: %v", err) + } + + b, err := ioutil.ReadFile(file.Name()) + if err != nil { + return bytes.Buffer{}, err + } + + tarWriter.Write(b) + defer tarWriter.Close() + + return buffer, nil +} + // CheckProcessStateOnTheHost checks if a process is in the desired state in a container // name of the container for the service: // we are using the Docker client instead of docker-compose @@ -58,6 +91,72 @@ func CheckProcessStateOnTheHost(containerName string, process string, state stri return nil } +// CopyFileToContainer copies a file to the running container +func CopyFileToContainer(ctx context.Context, containerName string, srcPath string, parentDir string, isTar bool) error { + dockerClient := getDockerClient() + + log.WithFields(log.Fields{ + "container": containerName, + "src": srcPath, + "parentDir": parentDir, + }).Trace("Copying file to container") + + targetDirectory := filepath.Dir(parentDir) + + _, err := dockerClient.ContainerStatPath(ctx, containerName, targetDirectory) + if err != nil { + log.WithFields(log.Fields{ + "container": containerName, + "error": err, + "src": srcPath, + "target": targetDirectory, + }).Error("Could not get parent directory in the container") + return err + } + + file, err := os.Open(srcPath) + if err != nil { + log.WithFields(log.Fields{ + "container": containerName, + "error": err, + "src": srcPath, + "parentDir": parentDir, + }).Error("Could not open file to deploy") + return err + } + defer file.Close() + + // TODO: detect the file has TAR headers + var buffer bytes.Buffer + if !isTar { + buffer, err = buildTarForDeployment(file) + if err != nil { + return err + } + } else { + writer := bufio.NewWriter(&buffer) + b, err := ioutil.ReadFile(file.Name()) + if err != nil { + return err + } + + writer.Write(b) + } + + err = dockerClient.CopyToContainer(ctx, containerName, parentDir, &buffer, types.CopyToContainerOptions{AllowOverwriteDirWithFile: true}) + if err != nil { + log.WithFields(log.Fields{ + "container": containerName, + "error": err, + "src": srcPath, + "parentDir": parentDir, + }).Error("Could not copy file to container") + return err + } + + return nil +} + // ExecCommandIntoContainer executes a command, as a user, into a container func ExecCommandIntoContainer(ctx context.Context, containerName string, user string, cmd []string) (string, error) { return ExecCommandIntoContainerWithEnv(ctx, containerName, user, cmd, []string{}) diff --git a/internal/docker/docker_test.go b/internal/docker/docker_test.go new file mode 100644 index 0000000000..2451a8309e --- /dev/null +++ b/internal/docker/docker_test.go @@ -0,0 +1,83 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package docker + +import ( + "context" + "path" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + tc "github.com/testcontainers/testcontainers-go" +) + +func Test_CopyFile(t *testing.T) { + ctx := context.Background() + + containerName := "server" + c, err := tc.GenericContainer(ctx, tc.GenericContainerRequest{ + ContainerRequest: tc.ContainerRequest{ + Image: "busybox", + Name: containerName, + Entrypoint: []string{"sleep", "300"}, + }, + Started: true, + }) + assert.Nil(t, err) + + defer func() { + err := c.Terminate(ctx) + assert.Nil(t, err) + }() + + t.Run("Copy file succeded", func(t *testing.T) { + src := path.Join("..", "_testresources", "dockerCopy.txt") + target := "/tmp" + + err = CopyFileToContainer(ctx, containerName, src, target, false) + assert.Nil(t, err) + + output, err := ExecCommandIntoContainer(ctx, containerName, "root", []string{"cat", "/tmp/dockerCopy.txt"}) + assert.Nil(t, err) + assert.True(t, strings.HasSuffix(output, "OK!"), "File contains the 'OK!' string") + }) + + t.Run("Copy file raises error with invalid source path", func(t *testing.T) { + src := path.Join("..", "this", "path", "does", "not", "exist", "dockerCopy.txt") + target := "/tmp" + + err = CopyFileToContainer(ctx, containerName, src, target, false) + assert.NotNil(t, err) + }) + + t.Run("Copy file raises error with invalid target parent dir", func(t *testing.T) { + src := path.Join("..", "_testresources", "dockerCopy.txt") + target := "/this-path-does-not-exist" + + err = CopyFileToContainer(ctx, containerName, src, target, false) + assert.NotNil(t, err, "Parent path '/this-path-does-not-exist' should exist in the container") + }) + + t.Run("Copy file raises error with invalid target subdir", func(t *testing.T) { + src := path.Join("..", "_testresources", "dockerCopy.txt") + target := "/tmp/subdir/" + + err = CopyFileToContainer(ctx, containerName, src, target, false) + assert.NotNil(t, err, "Parent path '/tmp/subdir' should not exist in the container") + }) + + t.Run("Copy tar file", func(t *testing.T) { + src := path.Join("..", "_testresources", "sample.tar.gz") + target := "/" + + err = CopyFileToContainer(ctx, containerName, src, target, true) + assert.Nil(t, err) + + output, err := ExecCommandIntoContainer(ctx, containerName, "root", []string{"ls", "/project/txtr/kermit.jpg"}) + assert.Nil(t, err) + assert.True(t, strings.Contains(output, "/project/txtr/kermit.jpg"), "File '/project/txtr/kermit.jpg' should be present") + }) +} diff --git a/internal/installer/tar.go b/internal/installer/tar.go index 72cc698a08..759679120f 100644 --- a/internal/installer/tar.go +++ b/internal/installer/tar.go @@ -78,11 +78,6 @@ func (i *TARPackage) Postinstall() error { // Preinstall executes operations before installing a TAR package func (i *TARPackage) Preinstall() error { - err := i.extractPackage([]string{"tar", "-xvf", "/" + i.binaryName}) - if err != nil { - return err - } - // simplify layout cmds := [][]string{ {"rm", "-fr", "/elastic-agent"}, @@ -90,7 +85,7 @@ func (i *TARPackage) Preinstall() error { } for _, cmd := range cmds { sm := compose.NewServiceManager() - err = sm.ExecCommandInService(i.profile, i.image, i.service, cmd, common.ProfileEnv, false) + err := sm.ExecCommandInService(i.profile, i.image, i.service, cmd, common.ProfileEnv, false) if err != nil { log.WithFields(log.Fields{ "command": cmd, diff --git a/internal/utils/utils.go b/internal/utils/utils.go index e794e05f1f..c7a81d1bb0 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -7,7 +7,6 @@ package utils import ( "fmt" "io" - "io/ioutil" "math/rand" "net/http" "os" @@ -21,7 +20,9 @@ import ( backoff "github.com/cenkalti/backoff/v4" "github.com/elastic/e2e-testing/internal/common" curl "github.com/elastic/e2e-testing/internal/curl" + internalio "github.com/elastic/e2e-testing/internal/io" "github.com/elastic/e2e-testing/internal/shell" + "github.com/google/uuid" log "github.com/sirupsen/logrus" ) @@ -420,7 +421,10 @@ func GetObjectURLFromBucket(bucket string, prefix string, object string, maxtime // It writes to the destination file as it downloads it, without // loading the entire file into memory. func DownloadFile(url string) (string, error) { - tempFile, err := ioutil.TempFile(os.TempDir(), path.Base(url)) + tempParentDir := path.Join(os.TempDir(), uuid.NewString()) + internalio.MkdirAll(tempParentDir) + + tempFile, err := os.Create(path.Join(tempParentDir, path.Base(url))) if err != nil { log.WithFields(log.Fields{ "error": err, diff --git a/internal/utils/utils_test.go b/internal/utils/utils_test.go index 316e8b1fac..2df96cd062 100644 --- a/internal/utils/utils_test.go +++ b/internal/utils/utils_test.go @@ -4,6 +4,8 @@ import ( "io/ioutil" "os" "path" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -213,6 +215,14 @@ func TestCheckPRVersion(t *testing.T) { }) } +func TestDownloadFile(t *testing.T) { + f, err := DownloadFile("https://www.elastic.co/robots.txt") + assert.Nil(t, err) + defer os.Remove(filepath.Dir(f)) + + assert.True(t, strings.HasSuffix(f, "robots.txt")) +} + func TestGetBucketSearchNextPageParam_HasMorePages(t *testing.T) { expectedParam := "&pageToken=foo"