feat: use Docker copy to transfer binaries to containers (backport #1136)

.pre-commit-config.yaml

@@ -33,15 +33,23 @@ repos:
     -   id: check-unicode-non-breaking-spaces
         exclude: >
             (?x)^(
-                .*/?\.*.(gif|jpg|png)
+                .*/?\.*.(gif|jpg|png|tar.gz)
             )$
     -   id: remove-unicode-non-breaking-spaces
         exclude: >
             (?x)^(
-                .*/?\.*.(gif|jpg|png)
+                .*/?\.*.(gif|jpg|png|tar.gz)
             )$
     -   id: check-en-dashes
+        exclude: >
+            (?x)^(
+                .*/?\.*.(tar.gz)
+            )$
     -   id: remove-en-dashes
+        exclude: >
+            (?x)^(
+                .*/?\.*.(tar.gz)
+            )$
     -   id: check-jjbb
     -   id: check-gherkin-lint
         args: [

cli/config/compose/services/centos-systemd/docker-compose.yml

@@ -7,5 +7,4 @@ services:
     platform: ${stackPlatform:-linux/amd64}
     privileged: true
     volumes:
-      - ${centos_systemdAgentBinarySrcPath:-.}:${centos_systemdAgentBinaryTargetPath:-/tmp}
       - /sys/fs/cgroup:/sys/fs/cgroup:ro 

cli/config/compose/services/centos/docker-compose.yml

@@ -4,5 +4,3 @@ services:
     image: centos:${centosTag:-7}
     container_name: ${centosContainerName} 
     entrypoint: tail -f /dev/null
-    volumes:
-      - ${centosAgentBinarySrcPath:-.}:${centosAgentBinaryTargetPath:-/tmp}

cli/config/compose/services/debian-systemd/docker-compose.yml

@@ -7,5 +7,4 @@ services:
     platform: ${stackPlatform:-linux/amd64}
     privileged: true
     volumes:
-      - ${debian_systemdAgentBinarySrcPath:-.}:${debian_systemdAgentBinaryTargetPath:-/tmp}
       - /sys/fs/cgroup:/sys/fs/cgroup:ro

cli/config/compose/services/debian/docker-compose.yml

@@ -4,5 +4,3 @@ services:
     image: debian:${debianTag:-9}
     container_name: ${debianContainerName} 
     entrypoint: tail -f /dev/null
-    volumes:
-      - ${debianAgentBinarySrcPath:-.}:${debianAgentBinaryTargetPath:-/tmp}

cli/config/compose/services/fleet-server-centos/docker-compose.yml

@@ -6,5 +6,4 @@ services:
     entrypoint: "/usr/sbin/init"
     privileged: true
     volumes:
-      - ${fleet_server_centosAgentBinarySrcPath:-.}:${fleet_server_centosAgentBinaryTargetPath:-/tmp}
       - /sys/fs/cgroup:/sys/fs/cgroup:ro 

cli/config/compose/services/fleet-server-debian/docker-compose.yml

@@ -6,5 +6,4 @@ services:
     entrypoint: "/sbin/init"
     privileged: true
     volumes:
-      - ${fleet_server_debianAgentBinarySrcPath:-.}:${fleet_server_debianAgentBinaryTargetPath:-/tmp}
       - /sys/fs/cgroup:/sys/fs/cgroup:ro

e2e/_suites/fleet/fleet.go

@@ -1154,9 +1154,6 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe
 	common.ProfileEnv[envVarsPrefix+"Tag"] = serviceTag
 	// we are setting the container name because Centos service could be reused by any other test suite
 	common.ProfileEnv[envVarsPrefix+"ContainerName"] = containerName
-	// define paths where the binary will be mounted
-	common.ProfileEnv[envVarsPrefix+"AgentBinarySrcPath"] = agentInstaller.BinaryPath
-	common.ProfileEnv[envVarsPrefix+"AgentBinaryTargetPath"] = "/" + agentInstaller.Name
 
 	serviceManager := compose.NewServiceManager()
 
@@ -1169,6 +1166,15 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe
 		return nil, err
 	}
 
+	isTar := (agentInstaller.InstallerType == "tar")
+	targetFile := "/"
+
+	// copy downloaded agent to the root dir of the container
+	err = docker.CopyFileToContainer(context.Background(), containerName, agentInstaller.BinaryPath, targetFile, isTar)
+	if err != nil {
+		return nil, err
+	}
+
 	err = agentInstaller.PreInstallFn()
 	if err != nil {
 		return nil, err

internal/_testresources/dockerCopy.txt

@@ -0,0 +1 @@
+OK!

internal/docker/docker.go

@@ -5,11 +5,14 @@
 package docker
 
 import (
+	"archive/tar"
+	"bufio"
 	"bytes"
 	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -28,6 +31,36 @@ var instance *client.Client
 // OPNetworkName name of the network used by the tool
 const OPNetworkName = "elastic-dev-network"
 
+func buildTarForDeployment(file *os.File) (bytes.Buffer, error) {
+	fileInfo, _ := file.Stat()
+
+	var buffer bytes.Buffer
+	tarWriter := tar.NewWriter(&buffer)
+	err := tarWriter.WriteHeader(&tar.Header{
+		Name: fileInfo.Name(),
+		Mode: 0777,
+		Size: int64(fileInfo.Size()),
+	})
+	if err != nil {
+		log.WithFields(log.Fields{
+			"fileInfoName": fileInfo.Name(),
+			"size":         fileInfo.Size(),
+			"error":        err,
+		}).Error("Could not build TAR header")
+		return bytes.Buffer{}, fmt.Errorf("could not build TAR header: %v", err)
+	}
+
+	b, err := ioutil.ReadFile(file.Name())
+	if err != nil {
+		return bytes.Buffer{}, err
+	}
+
+	tarWriter.Write(b)
+	defer tarWriter.Close()
+
+	return buffer, nil
+}
+
 // CheckProcessStateOnTheHost checks if a process is in the desired state in a container
 // name of the container for the service:
 // we are using the Docker client instead of docker-compose
@@ -58,6 +91,72 @@ func CheckProcessStateOnTheHost(containerName string, process string, state stri
 	return nil
 }
 
+// CopyFileToContainer copies a file to the running container
+func CopyFileToContainer(ctx context.Context, containerName string, srcPath string, parentDir string, isTar bool) error {
+	dockerClient := getDockerClient()
+
+	log.WithFields(log.Fields{
+		"container": containerName,
+		"src":       srcPath,
+		"parentDir": parentDir,
+	}).Trace("Copying file to container")
+
+	targetDirectory := filepath.Dir(parentDir)
+
+	_, err := dockerClient.ContainerStatPath(ctx, containerName, targetDirectory)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"container": containerName,
+			"error":     err,
+			"src":       srcPath,
+			"target":    targetDirectory,
+		}).Error("Could not get parent directory in the container")
+		return err
+	}
+
+	file, err := os.Open(srcPath)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"container": containerName,
+			"error":     err,
+			"src":       srcPath,
+			"parentDir": parentDir,
+		}).Error("Could not open file to deploy")
+		return err
+	}
+	defer file.Close()
+
+	// TODO: detect the file has TAR headers
+	var buffer bytes.Buffer
+	if !isTar {
+		buffer, err = buildTarForDeployment(file)
+		if err != nil {
+			return err
+		}
+	} else {
+		writer := bufio.NewWriter(&buffer)
+		b, err := ioutil.ReadFile(file.Name())
+		if err != nil {
+			return err
+		}
+
+		writer.Write(b)
+	}
+
+	err = dockerClient.CopyToContainer(ctx, containerName, parentDir, &buffer, types.CopyToContainerOptions{AllowOverwriteDirWithFile: true})
+	if err != nil {
+		log.WithFields(log.Fields{
+			"container": containerName,
+			"error":     err,
+			"src":       srcPath,
+			"parentDir": parentDir,
+		}).Error("Could not copy file to container")
+		return err
+	}
+
+	return nil
+}
+
 // ExecCommandIntoContainer executes a command, as a user, into a container
 func ExecCommandIntoContainer(ctx context.Context, containerName string, user string, cmd []string) (string, error) {
 	return ExecCommandIntoContainerWithEnv(ctx, containerName, user, cmd, []string{})

internal/docker/docker_test.go

@@ -0,0 +1,83 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package docker
+
+import (
+	"context"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	tc "github.com/testcontainers/testcontainers-go"
+)
+
+func Test_CopyFile(t *testing.T) {
+	ctx := context.Background()
+
+	containerName := "server"
+	c, err := tc.GenericContainer(ctx, tc.GenericContainerRequest{
+		ContainerRequest: tc.ContainerRequest{
+			Image:      "busybox",
+			Name:       containerName,
+			Entrypoint: []string{"sleep", "300"},
+		},
+		Started: true,
+	})
+	assert.Nil(t, err)
+
+	defer func() {
+		err := c.Terminate(ctx)
+		assert.Nil(t, err)
+	}()
+
+	t.Run("Copy file succeded", func(t *testing.T) {
+		src := path.Join("..", "_testresources", "dockerCopy.txt")
+		target := "/tmp"
+
+		err = CopyFileToContainer(ctx, containerName, src, target, false)
+		assert.Nil(t, err)
+
+		output, err := ExecCommandIntoContainer(ctx, containerName, "root", []string{"cat", "/tmp/dockerCopy.txt"})
+		assert.Nil(t, err)
+		assert.True(t, strings.HasSuffix(output, "OK!"), "File contains the 'OK!' string")
+	})
+
+	t.Run("Copy file raises error with invalid source path", func(t *testing.T) {
+		src := path.Join("..", "this", "path", "does", "not", "exist", "dockerCopy.txt")
+		target := "/tmp"
+
+		err = CopyFileToContainer(ctx, containerName, src, target, false)
+		assert.NotNil(t, err)
+	})
+
+	t.Run("Copy file raises error with invalid target parent dir", func(t *testing.T) {
+		src := path.Join("..", "_testresources", "dockerCopy.txt")
+		target := "/this-path-does-not-exist"
+
+		err = CopyFileToContainer(ctx, containerName, src, target, false)
+		assert.NotNil(t, err, "Parent path '/this-path-does-not-exist' should exist in the container")
+	})
+
+	t.Run("Copy file raises error with invalid target subdir", func(t *testing.T) {
+		src := path.Join("..", "_testresources", "dockerCopy.txt")
+		target := "/tmp/subdir/"
+
+		err = CopyFileToContainer(ctx, containerName, src, target, false)
+		assert.NotNil(t, err, "Parent path '/tmp/subdir' should not exist in the container")
+	})
+
+	t.Run("Copy tar file", func(t *testing.T) {
+		src := path.Join("..", "_testresources", "sample.tar.gz")
+		target := "/"
+
+		err = CopyFileToContainer(ctx, containerName, src, target, true)
+		assert.Nil(t, err)
+
+		output, err := ExecCommandIntoContainer(ctx, containerName, "root", []string{"ls", "/project/txtr/kermit.jpg"})
+		assert.Nil(t, err)
+		assert.True(t, strings.Contains(output, "/project/txtr/kermit.jpg"), "File '/project/txtr/kermit.jpg' should be present")
+	})
+}

internal/installer/tar.go

@@ -78,19 +78,14 @@ func (i *TARPackage) Postinstall() error {
 
 // Preinstall executes operations before installing a TAR package
 func (i *TARPackage) Preinstall() error {
-	err := i.extractPackage([]string{"tar", "-xvf", "/" + i.binaryName})
-	if err != nil {
-		return err
-	}
-
 	// simplify layout
 	cmds := [][]string{
 		{"rm", "-fr", "/elastic-agent"},
 		{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, i.version, i.OS, i.arch), "/elastic-agent"},
 	}
 	for _, cmd := range cmds {
 		sm := compose.NewServiceManager()
-		err = sm.ExecCommandInService(i.profile, i.image, i.service, cmd, common.ProfileEnv, false)
+		err := sm.ExecCommandInService(i.profile, i.image, i.service, cmd, common.ProfileEnv, false)
 		if err != nil {
 			log.WithFields(log.Fields{
 				"command": cmd,

internal/utils/utils.go

@@ -7,7 +7,6 @@ package utils
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"os"
@@ -21,7 +20,9 @@ import (
 	backoff "github.com/cenkalti/backoff/v4"
 	"github.com/elastic/e2e-testing/internal/common"
 	curl "github.com/elastic/e2e-testing/internal/curl"
+	internalio "github.com/elastic/e2e-testing/internal/io"
 	"github.com/elastic/e2e-testing/internal/shell"
+	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -420,7 +421,10 @@ func GetObjectURLFromBucket(bucket string, prefix string, object string, maxtime
 // It writes to the destination file as it downloads it, without
 // loading the entire file into memory.
 func DownloadFile(url string) (string, error) {
-	tempFile, err := ioutil.TempFile(os.TempDir(), path.Base(url))
+	tempParentDir := path.Join(os.TempDir(), uuid.NewString())
+	internalio.MkdirAll(tempParentDir)
+
+	tempFile, err := os.Create(path.Join(tempParentDir, path.Base(url)))
 	if err != nil {
 		log.WithFields(log.Fields{
 			"error": err,

internal/utils/utils_test.go

@@ -4,6 +4,8 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -213,6 +215,14 @@ func TestCheckPRVersion(t *testing.T) {
 	})
 }
 
+func TestDownloadFile(t *testing.T) {
+	f, err := DownloadFile("https://www.elastic.co/robots.txt")
+	assert.Nil(t, err)
+	defer os.Remove(filepath.Dir(f))
+
+	assert.True(t, strings.HasSuffix(f, "robots.txt"))
+}
+
 func TestGetBucketSearchNextPageParam_HasMorePages(t *testing.T) {
 	expectedParam := "&pageToken=foo"