[Agent] Include beats in agent package

dev-tools/mage/pkg.go

@@ -60,17 +60,29 @@ func Package() error {
 					continue
 				}
 
+				agentPackageType := TarGz
+				if pkg.OS == "windows" {
+					agentPackageType = Zip
+				}
+
+				agentPackageArch, err := getOSArchName(target, agentPackageType)
+				if err != nil {
+					log.Printf("Skipping arch %v for package type %v: %v", target.Arch(), pkgType, err)
+					continue
+				}
+
 				spec := pkg.Spec.Clone()
 				spec.OS = target.GOOS()
 				spec.Arch = packageArch
 				spec.Snapshot = Snapshot
 				spec.evalContext = map[string]interface{}{
-					"GOOS":        target.GOOS(),
-					"GOARCH":      target.GOARCH(),
-					"GOARM":       target.GOARM(),
-					"Platform":    target,
-					"PackageType": pkgType.String(),
-					"BinaryExt":   binaryExtension(target.GOOS()),
+					"GOOS":          target.GOOS(),
+					"GOARCH":        target.GOARCH(),
+					"GOARM":         target.GOARM(),
+					"Platform":      target,
+					"AgentArchName": agentPackageArch,
+					"PackageType":   pkgType.String(),
+					"BinaryExt":     binaryExtension(target.GOOS()),
 				}
 				spec.packageDir = packageStagingDir + "/" + pkgType.AddFileExtension(spec.Name+"-"+target.GOOS()+"-"+target.Arch())
 				spec = spec.Evaluate()

dev-tools/mage/pkgspecs.go

@@ -42,6 +42,7 @@ func UseCommunityBeatPackaging() {
 // UseElasticBeatAgentPackaging configures the package target to build packages for
 // an Elastic Beat Agent.
 func UseElasticBeatAgentPackaging() {
+	// Prepare binaries so they can be packed into agent
 	MustUsePackaging("elastic_beat_agent_binaries", packageSpecFile)
 }
 
@@ -66,6 +67,12 @@ func UseElasticBeatXPackPackaging() {
 	MustUsePackaging("elastic_beat_xpack", packageSpecFile)
 }
 
+// UseElasticBeatXPackReducedPackaging configures the package target to build Elastic
+// licensed (X-Pack) packages for agent use.
+func UseElasticBeatXPackReducedPackaging() {
+	MustUsePackaging("elastic_beat_xpack_reduced", packageSpecFile)
+}
+
 // UseElasticBeatWithoutXPackPackaging configures the package target to build
 // packages for an Elastic Beat. This means it will generate two sets of
 // packages -- one that is purely OSS under Apache 2.0 and one that is licensed

dev-tools/packaging/packages.yml

@@ -54,6 +54,13 @@ shared:
       /etc/init.d/{{.BeatServiceName}}:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/init.sh.tmpl'
         mode: 0755
+      /etc/{{.BeatName}}/beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+      /etc/{{.BeatName}}/beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+
 
   # MacOS pkg spec for community beats.
   - &macos_agent_pkg_spec
@@ -90,6 +97,12 @@ shared:
         source: 'agent.yml'
         mode: 0600
         config: true
+      /etc/{{.BeatName}}/beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+      /etc/{{.BeatName}}/beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
 
   - &agent_binary_files
     '{{.BeatName}}{{.BinaryExt}}':
@@ -118,6 +131,12 @@ shared:
     <<: *common
     files:
       <<: *agent_binary_files
+      'beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+        source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+      'beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+        source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
 
   # Binary package spec (zip for windows) for community beats.
   - &agent_windows_binary_spec
@@ -130,6 +149,12 @@ shared:
       uninstall-service-{{.BeatName}}.ps1:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/uninstall-service.ps1.tmpl'
         mode: 0755
+      'beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
+        source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip'
+        mode: 0644
+      'beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
+        source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip'
+        mode: 0644
 
   - &agent_docker_spec
     <<: *agent_binary_spec
@@ -478,6 +503,30 @@ specs:
         <<: *elastic_docker_spec
         <<: *elastic_license_for_binaries
 
+  # Elastic Beat with Elastic License and binary taken the current directory.
+  elastic_beat_xpack_reduced:
+    ###
+    # Elastic Licensed Packages
+    ###
+    - os: windows
+      types: [zip]
+      spec:
+        <<: *windows_binary_spec
+        <<: *elastic_license_for_binaries
+
+    - os: darwin
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *elastic_license_for_binaries
+
+    - os: linux
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *elastic_license_for_binaries
+
+
   # Elastic Beat with Elastic License and binary taken from the x-pack dir.
   elastic_beat_xpack_separate_binaries:
     ###
@@ -550,7 +599,6 @@ specs:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
 
-
   # Elastic Beat with Elastic License and binary taken from the x-pack dir.
   elastic_beat_agent_binaries:
     ###

x-pack/agent/magefile.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
@@ -18,6 +19,7 @@ import (
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
 
+	"github.com/elastic/beats/agent/release"
 	devtools "github.com/elastic/beats/dev-tools/mage"
 	"github.com/elastic/beats/dev-tools/mage/target/common"
 	"github.com/elastic/beats/dev-tools/mage/target/unittest"
@@ -261,13 +263,62 @@ func Package() {
 	start := time.Now()
 	defer func() { fmt.Println("package ran for", time.Since(start)) }()
 
+	version, found := os.LookupEnv("BEAT_VERSION")
+	if !found {
+		version = release.Version()
+	}
+
+	packedBeats := []string{"filebeat", "metricbeat"}
+	requiredPackages := []string{
+		"darwin-x86_64.tar.gz",
+		"linux-x86.tar.gz",
+		"linux-x86_64.tar.gz",
+		"windows-x86.zip",
+		"windows-x86_64.zip",
+	}
+
+	for _, b := range packedBeats {
+		pwd, err := filepath.Abs(filepath.Join("..", b))
+		if err != nil {
+			panic(err)
+		}
+
+		if requiredPackagesPresent(pwd, b, version, requiredPackages) {
+			continue
+		}
+
+		cmd := exec.Command("mage", "package")
+		cmd.Dir = pwd
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd.Env = append(os.Environ(), fmt.Sprintf("PWD=%s", pwd), "AGENT_PACKAGING=on")
+
+		if err := cmd.Run(); err != nil {
+			panic(err)
+		}
+	}
+
+	// package agent
 	devtools.UseElasticBeatAgentPackaging()
 
 	mg.Deps(Update)
 	mg.Deps(CrossBuild, CrossBuildGoDaemon)
 	mg.SerialDeps(devtools.Package)
 }
 
+func requiredPackagesPresent(basePath, beat, version string, requiredPackages []string) bool {
+	for _, pkg := range requiredPackages {
+		packageName := fmt.Sprintf("%s-%s-%s", beat, version, pkg)
+		path := filepath.Join(basePath, "build", "distributions", packageName)
+
+		if _, err := os.Stat(path); err != nil {
+			fmt.Printf("Package '%s' does not exist on path: %s\n", packageName, path)
+			return false
+		}
+	}
+	return true
+}
+
 // TestPackages tests the generated packages (i.e. file modes, owners, groups).
 func TestPackages() error {
 	return devtools.TestPackages()

x-pack/agent/pkg/agent/application/stream.go

@@ -14,7 +14,7 @@ import (
 	"github.com/elastic/beats/x-pack/agent/pkg/agent/operation"
 	operatorCfg "github.com/elastic/beats/x-pack/agent/pkg/agent/operation/config"
 	"github.com/elastic/beats/x-pack/agent/pkg/agent/stateresolver"
-	downloader "github.com/elastic/beats/x-pack/agent/pkg/artifact/download/http"
+	downloader "github.com/elastic/beats/x-pack/agent/pkg/artifact/download/localremote"
 	"github.com/elastic/beats/x-pack/agent/pkg/artifact/install"
 	"github.com/elastic/beats/x-pack/agent/pkg/config"
 	"github.com/elastic/beats/x-pack/agent/pkg/core/logger"

x-pack/agent/pkg/artifact/download/fs/downloader.go

@@ -21,13 +21,15 @@ const (
 
 // Downloader is a downloader able to fetch artifacts from elastic.co web page.
 type Downloader struct {
-	config *artifact.Config
+	dropPath string
+	config   *artifact.Config
 }
 
 // NewDownloader creates and configures Elastic Downloader
 func NewDownloader(config *artifact.Config) *Downloader {
 	return &Downloader{
-		config: config,
+		config:   config,
+		dropPath: getDropPath(config),
 	}
 }
 
@@ -60,7 +62,7 @@ func (e *Downloader) download(operatingSystem, programName, version string) (str
 		return "", errors.New(err, "generating package path failed")
 	}
 
-	sourcePath := filepath.Join(e.getDropPath(), filename)
+	sourcePath := filepath.Join(e.dropPath, filename)
 	sourceFile, err := os.Open(sourcePath)
 	if err != nil {
 		return "", errors.New(err, fmt.Sprintf("package '%s' not found", sourcePath), errors.TypeFilesystem, errors.M(errors.MetaKeyPath, fullPath))
@@ -77,19 +79,17 @@ func (e *Downloader) download(operatingSystem, programName, version string) (str
 	return fullPath, nil
 }
 
-func (e *Downloader) getDropPath() string {
-	fmt.Println("drop path?")
-	fmt.Println(e.config.DropPath)
+func getDropPath(cfg *artifact.Config) string {
 	// if drop path is not provided fallback to beats subfolder
-	if e.config.DropPath == "" {
+	if cfg == nil || cfg.DropPath == "" {
 		return beatsSubfolder
 	}
 
 	// if droppath does not exist fallback to beats subfolder
-	stat, err := os.Stat(e.config.DropPath)
+	stat, err := os.Stat(cfg.DropPath)
 	if err != nil || !stat.IsDir() {
 		return beatsSubfolder
 	}
 
-	return e.config.DropPath
+	return cfg.DropPath
 }

x-pack/filebeat/magefile.go

@@ -9,6 +9,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/magefile/mage/mg"
@@ -67,7 +68,12 @@ func Package() {
 	start := time.Now()
 	defer func() { fmt.Println("package ran for", time.Since(start)) }()
 
-	devtools.UseElasticBeatXPackPackaging()
+	if v, found := os.LookupEnv("AGENT_PACKAGING"); found && v != "" {
+		devtools.UseElasticBeatXPackReducedPackaging()
+	} else {
+		devtools.UseElasticBeatXPackPackaging()
+	}
+
 	devtools.PackageKibanaDashboardsFromBuildDir()
 	filebeat.CustomizePackaging()
 

x-pack/metricbeat/magefile.go

@@ -75,7 +75,12 @@ func Package() {
 	start := time.Now()
 	defer func() { fmt.Println("package ran for", time.Since(start)) }()
 
-	devtools.UseElasticBeatXPackPackaging()
+	if v, found := os.LookupEnv("AGENT_PACKAGING"); found && v != "" {
+		devtools.UseElasticBeatXPackReducedPackaging()
+	} else {
+		devtools.UseElasticBeatXPackPackaging()
+	}
+
 	metricbeat.CustomizePackaging()
 	devtools.PackageKibanaDashboardsFromBuildDir()
 	packageLightModules()