[Agent] Make default config OS agnostic

dev-tools/packaging/packages.yml

@@ -54,10 +54,10 @@ shared:
       /etc/init.d/{{.BeatServiceName}}:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/init.sh.tmpl'
         mode: 0755
-      /etc/{{.BeatName}}/beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      /etc/{{.BeatName}}/beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
 
@@ -97,10 +97,10 @@ shared:
         source: 'agent.yml'
         mode: 0600
         config: true
-      /etc/{{.BeatName}}/beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      /etc/{{.BeatName}}/beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
 
@@ -131,10 +131,10 @@ shared:
     <<: *common
     files:
       <<: *agent_binary_files
-      'beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+      'data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
         source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      'beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+      'data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
         source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
 
@@ -149,10 +149,10 @@ shared:
       uninstall-service-{{.BeatName}}.ps1:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/uninstall-service.ps1.tmpl'
         mode: 0755
-      'beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
+      'data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
         source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
         mode: 0644
-      'beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
+      'data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
         source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
         mode: 0644
 

dev-tools/packaging/templates/docker/Dockerfile.agent.tmpl

@@ -26,7 +26,7 @@ RUN chmod 755 /usr/local/bin/docker-entrypoint
 
 RUN groupadd --gid 1000 {{ .BeatName }}
 
-RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \
+RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \
     chown -R root:{{ .BeatName }} {{ $beatHome }} && \
     find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
     find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \

x-pack/agent/CHANGELOG.asciidoc

@@ -17,3 +17,4 @@
 ==== New features
 
 - Generate index name in a format type-dataset-namespace {pull}16903[16903]
+- OS agnostic default configuration {pull}17016[17016]

x-pack/agent/_meta/agent.docker.yml

@@ -52,15 +52,15 @@ download:
   # e.g /windows-x86.zip
   sourceURI: "https://artifacts.elastic.co/downloads/beats/"
   # path to the directory containing downloaded packages
-  target_directory: "/home/elastic/downloads"
+  target_directory: "${path.data}/downloads"
   # timeout for downloading package
   timeout: 30s
   # file path to a public key used for verifying downloaded artifacts
   # if not file is present agent will try to load public key from elastic.co website.
-  pgpfile: "/home/elastic/elastic.pgp"
+  pgpfile: "${path.data}/elastic.pgp"
   # install_path describes the location of installed packages/programs. It is also used
   # for reading program specifications.
-  install_path: "/home/elastic/install"
+  install_path: "${path.data}/install"
 
 process:
   # minimal port number for spawned processes

x-pack/agent/_meta/agent.fleet.yml

@@ -10,15 +10,15 @@ download:
   # e.g /windows-x86.zip
   sourceURI: "https://artifacts.elastic.co/downloads/beats/"
   # path to the directory containing downloaded packages
-  target_directory: "/home/elastic/downloads"
+  target_directory: "${path.data}/downloads"
   # timeout for downloading package
   timeout: 30s
   # file path to a public key used for verifying downloaded artifacts
   # if not file is present agent will try to load public key from elastic.co website.
-  pgpfile: "/home/elastic/elastic.pgp"
+  pgpfile: "${path.data}/elastic.pgp"
   # install_path describes the location of installed packages/programs. It is also used
   # for reading program specifications.
-  install_path: "/home/elastic/install"
+  install_path: "${path.data}/install"
 
 process:
   # minimal port number for spawned processes

x-pack/agent/_meta/agent.yml

@@ -53,15 +53,15 @@ download:
   # e.g /windows-x86.zip
   sourceURI: "https://artifacts.elastic.co/downloads/beats/"
   # path to the directory containing downloaded packages
-  target_directory: "/home/elastic/downloads"
+  target_directory: "${path.data}/downloads"
   # timeout for downloading package
   timeout: 30s
   # file path to a public key used for verifying downloaded artifacts
   # if not file is present agent will try to load public key from elastic.co website.
-  pgpfile: "/home/elastic/elastic.pgp"
+  pgpfile: "${path.data}/elastic.pgp"
   # install_path describes the location of installed packages/programs. It is also used
   # for reading program specifications.
-  install_path: "/home/elastic/install"
+  install_path: "${path.data}/install"
 
 process:
   # minimal port number for spawned processes

x-pack/agent/_meta/common.p2.yml

@@ -78,15 +78,15 @@ download:
   # e.g /windows-x86.zip
   sourceURI: "https://artifacts.elastic.co/downloads/beats/"
   # path to the directory containing downloaded packages
-  target_directory: "/home/elastic/downloads"
+  target_directory: "${path.data}/downloads"
   # timeout for downloading package
   timeout: 30s
   # file path to a public key used for verifying downloaded artifacts
   # if not file is present agent will try to load public key from elastic.co website.
-  pgpfile: "/home/elastic/elastic.pgp"
+  pgpfile: "${path.data}/elastic.pgp"
   # install_path describes the location of installed packages/programs. It is also used
   # for reading program specifications.
-  install_path: "/home/elastic/install"
+  install_path: "${path.data}/install"
 
 process:
   # minimal port number for spawned processes

x-pack/agent/_meta/common.reference.p2.yml

@@ -78,15 +78,15 @@ download:
   # e.g /windows-x86.zip
   sourceURI: "https://artifacts.elastic.co/downloads/beats/"
   # path to the directory containing downloaded packages
-  target_directory: "/home/elastic/downloads"
+  target_directory: "${path.data}/downloads"
   # timeout for downloading package
   timeout: 30s
   # file path to a public key used for verifying downloaded artifacts
   # if not file is present agent will try to load public key from elastic.co website.
-  pgpfile: "/home/elastic/elastic.pgp"
+  pgpfile: "${path.data}/elastic.pgp"
   # install_path describes the location of installed packages/programs. It is also used
   # for reading program specifications.
-  install_path: "/home/elastic/install"
+  install_path: "${path.data}/install"
 
 process:
   # minimal port number for spawned processes

x-pack/agent/agent.docker.yml

@@ -52,15 +52,15 @@ download:
   # e.g /windows-x86.zip
   sourceURI: "https://artifacts.elastic.co/downloads/beats/"
   # path to the directory containing downloaded packages
-  target_directory: "/home/elastic/downloads"
+  target_directory: "${path.data}/downloads"
   # timeout for downloading package
   timeout: 30s
   # file path to a public key used for verifying downloaded artifacts
   # if not file is present agent will try to load public key from elastic.co website.
-  pgpfile: "/home/elastic/elastic.pgp"
+  pgpfile: "${path.data}/elastic.pgp"
   # install_path describes the location of installed packages/programs. It is also used
   # for reading program specifications.
-  install_path: "/home/elastic/install"
+  install_path: "${path.data}/install"
 
 process:
   # minimal port number for spawned processes

x-pack/agent/agent.reference.yml

@@ -83,15 +83,15 @@ download:
   # e.g /windows-x86.zip
   sourceURI: "https://artifacts.elastic.co/downloads/beats/"
   # path to the directory containing downloaded packages
-  target_directory: "/home/elastic/downloads"
+  target_directory: "${path.data}/downloads"
   # timeout for downloading package
   timeout: 30s
   # file path to a public key used for verifying downloaded artifacts
   # if not file is present agent will try to load public key from elastic.co website.
-  pgpfile: "/home/elastic/elastic.pgp"
+  pgpfile: "${path.data}/elastic.pgp"
   # install_path describes the location of installed packages/programs. It is also used
   # for reading program specifications.
-  install_path: "/home/elastic/install"
+  install_path: "${path.data}/install"
 
 process:
   # minimal port number for spawned processes

x-pack/agent/agent.yml

@@ -83,15 +83,15 @@ download:
   # e.g /windows-x86.zip
   sourceURI: "https://artifacts.elastic.co/downloads/beats/"
   # path to the directory containing downloaded packages
-  target_directory: "/home/elastic/downloads"
+  target_directory: "${path.data}/downloads"
   # timeout for downloading package
   timeout: 30s
   # file path to a public key used for verifying downloaded artifacts
   # if not file is present agent will try to load public key from elastic.co website.
-  pgpfile: "/home/elastic/elastic.pgp"
+  pgpfile: "${path.data}/elastic.pgp"
   # install_path describes the location of installed packages/programs. It is also used
   # for reading program specifications.
-  install_path: "/home/elastic/install"
+  install_path: "${path.data}/install"
 
 process:
   # minimal port number for spawned processes

x-pack/agent/pkg/agent/operation/operator.go

@@ -224,6 +224,7 @@ func (o *Operator) runFlow(p Descriptor, operations []operation) error {
 			continue
 		}
 
+		o.logger.Debugf("running operation '%s' for %s.%s", op.Name(), p.BinaryName(), p.Version())
 		if err := op.Run(o.bgContext, app); err != nil {
 			return err
 		}

x-pack/agent/pkg/artifact/download/fs/downloader.go

@@ -17,7 +17,10 @@ import (
 
 const (
 	packagePermissions = 0660
-	beatsSubfolder     = "beats"
+)
+
+var (
+	defaultDropSubdir = filepath.Join("data", "downloads")
 )
 
 // Downloader is a downloader able to fetch artifacts from elastic.co web page.
@@ -83,13 +86,13 @@ func (e *Downloader) download(operatingSystem, programName, version string) (str
 func getDropPath(cfg *artifact.Config) string {
 	// if drop path is not provided fallback to beats subfolder
 	if cfg == nil || cfg.DropPath == "" {
-		return beatsSubfolder
+		return defaultDropSubdir
 	}
 
 	// if droppath does not exist fallback to beats subfolder
 	stat, err := os.Stat(cfg.DropPath)
 	if err != nil || !stat.IsDir() {
-		return beatsSubfolder
+		return defaultDropSubdir
 	}
 
 	return cfg.DropPath

x-pack/agent/pkg/artifact/download/fs/verifier.go

@@ -79,7 +79,7 @@ func (v *Verifier) Verify(programName, version string) (bool, error) {
 
 func (v *Verifier) getPublicAsc(filename string) ([]byte, error) {
 	ascFile := fmt.Sprintf("%s%s", filename, ascSuffix)
-	fullPath := filepath.Join(beatsSubfolder, ascFile)
+	fullPath := filepath.Join(defaultDropSubdir, ascFile)
 
 	b, err := ioutil.ReadFile(fullPath)
 	if err != nil {