Cherry-pick #16927 to 7.x: Fix issue when building Agent snapshot and enable building the agent.

Makefile

@@ -1,6 +1,6 @@
 BUILD_DIR=$(CURDIR)/build
 COVERAGE_DIR=$(BUILD_DIR)/coverage
-BEATS?=auditbeat filebeat heartbeat journalbeat metricbeat packetbeat winlogbeat x-pack/functionbeat
+BEATS?=auditbeat filebeat heartbeat journalbeat metricbeat packetbeat winlogbeat x-pack/functionbeat x-pack/agent
 PROJECTS=libbeat $(BEATS)
 PROJECTS_ENV=libbeat filebeat metricbeat
 PYTHON_ENV?=$(BUILD_DIR)/python-env

dev-tools/mage/crossbuild.go

@@ -268,6 +268,7 @@ func (b GolangCrossBuilder) Build() error {
 		"--rm",
 		"--env", "MAGEFILE_VERBOSE="+verbose,
 		"--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""),
+		"--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot),
 		"-v", repoInfo.RootDir+":"+mountPoint,
 		"-w", workDir,
 		image,

dev-tools/mage/dmgbuilder.go

@@ -244,6 +244,7 @@ func (b *dmgBuilder) buildDMG() error {
 		"create",
 		"-volname", b.MustExpand("{{.BeatName | title}} {{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}"),
 		"-srcfolder", b.dmgDir,
+		"-size", "500m", // allowing extra space
 		"-ov",
 		createDir(dmgFile),
 	}

dev-tools/mage/dockerbuilder.go

@@ -120,8 +120,8 @@ func (b *dockerBuilder) prepareBuild() error {
 		"ModulesDirs": b.modulesDirs(),
 	}
 
-	return filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {
-		if !info.IsDir() {
+	err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {
+		if !info.IsDir() && !isDockerFile(path) {
 			target := strings.TrimSuffix(
 				filepath.Join(b.buildDir, filepath.Base(path)),
 				".tmpl",
@@ -134,6 +134,52 @@ func (b *dockerBuilder) prepareBuild() error {
 		}
 		return nil
 	})
+
+	if err != nil {
+		return err
+	}
+
+	return b.expandDockerfile(templatesDir, data)
+}
+
+func isDockerFile(path string) bool {
+	path = filepath.Base(path)
+	return strings.HasPrefix(path, "Dockerfile") || strings.HasPrefix(path, "docker-entrypoint")
+}
+
+func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]interface{}) error {
+	// has specific dockerfile
+	dockerfile := fmt.Sprintf("Dockerfile.%s.tmpl", b.imageName)
+	_, err := os.Stat(filepath.Join(templatesDir, dockerfile))
+	if err != nil {
+		// specific missing fallback to generic
+		dockerfile = "Dockerfile.tmpl"
+	}
+
+	entrypoint := fmt.Sprintf("docker-entrypoint.%s.tmpl", b.imageName)
+	_, err = os.Stat(filepath.Join(templatesDir, entrypoint))
+	if err != nil {
+		// specific missing fallback to generic
+		entrypoint = "docker-entrypoint.tmpl"
+	}
+
+	type fileExpansion struct {
+		source string
+		target string
+	}
+	for _, file := range []fileExpansion{{dockerfile, "Dockerfile.tmpl"}, {entrypoint, "docker-entrypoint.tmpl"}} {
+		target := strings.TrimSuffix(
+			filepath.Join(b.buildDir, file.target),
+			".tmpl",
+		)
+		path := filepath.Join(templatesDir, file.source)
+		err = b.ExpandFile(path, target, data)
+		if err != nil {
+			return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target)
+		}
+	}
+
+	return nil
 }
 
 func (b *dockerBuilder) dockerBuild() (string, error) {

dev-tools/packaging/packages.yml

@@ -54,11 +54,11 @@ shared:
       /etc/init.d/{{.BeatServiceName}}:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/init.sh.tmpl'
         mode: 0755
-      /etc/{{.BeatName}}/beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
-        source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+      /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      /etc/{{.BeatName}}/beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
-        source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+      /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
 
 
@@ -97,11 +97,11 @@ shared:
         source: 'agent.yml'
         mode: 0600
         config: true
-      /etc/{{.BeatName}}/beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
-        source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+      /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      /etc/{{.BeatName}}/beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
-        source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+      /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
 
   - &agent_binary_files
@@ -131,11 +131,11 @@ shared:
     <<: *common
     files:
       <<: *agent_binary_files
-      'beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
-        source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+      'data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+        source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      'beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
-        source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+      'data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+        source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
 
   # Binary package spec (zip for windows) for community beats.
@@ -149,18 +149,18 @@ shared:
       uninstall-service-{{.BeatName}}.ps1:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/uninstall-service.ps1.tmpl'
         mode: 0755
-      'beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
-        source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip'
+      'data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
+        source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
         mode: 0644
-      'beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
-        source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip'
+      'data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
+        source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
         mode: 0644
 
   - &agent_docker_spec
     <<: *agent_binary_spec
     extra_vars:
       from: 'centos:7'
-      user: '{{ .BeatName }}'
+      user: 'root'
       linux_capabilities: ''
     files:
       'agent.yml':

dev-tools/packaging/templates/docker/Dockerfile.agent.tmpl

@@ -0,0 +1,52 @@
+{{- $beatHome := printf "%s/%s" "/usr/share" .BeatName }}
+{{- $beatBinary := printf "%s/%s" $beatHome .BeatName }}
+{{- $repoInfo := repo }}
+
+FROM {{ .from }}
+
+LABEL \
+  org.label-schema.build-date="{{ date }}" \
+  org.label-schema.schema-version="1.0" \
+  org.label-schema.vendor="{{ .BeatVendor }}" \
+  org.label-schema.license="{{ .License }}" \
+  org.label-schema.name="{{ .BeatName }}" \
+  org.label-schema.version="{{ beat_version }}" \
+  org.label-schema.url="{{ .BeatURL }}" \
+  org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \
+  org.label-schema.vcs-ref="{{ commit }}" \
+  license="{{ .License }}" \
+  description="{{ .BeatDescription }}"
+
+ENV ELASTIC_CONTAINER "true"
+ENV PATH={{ $beatHome }}:$PATH
+
+COPY beat {{ $beatHome }}
+COPY docker-entrypoint /usr/local/bin/docker-entrypoint
+RUN chmod 755 /usr/local/bin/docker-entrypoint
+
+RUN groupadd --gid 1000 {{ .BeatName }}
+
+RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \
+    chown -R root:{{ .BeatName }} {{ $beatHome }} && \
+    find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
+    find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
+    chmod 0750 {{ $beatBinary }} && \
+{{- if .linux_capabilities }}
+    setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
+{{- end }}
+{{- range $i, $modulesd := .ModulesDirs }}
+    chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \
+{{- end }}
+    chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs
+
+{{- if ne .user "root" }}
+RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }}
+{{- end }}
+USER {{ .user }}
+
+{{- range $i, $port := .ExposePorts }}
+EXPOSE {{ $port }}
+{{- end }}
+
+WORKDIR {{ $beatHome }}
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]

dev-tools/packaging/templates/docker/docker-entrypoint.agent.tmpl

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Environment variables used
+# FLEET_ADMIN_PASSWORD - used for new fleet user [elastic]
+# FLEET_ADMIN_USERNAME - used for new fleet user [changeme]
+# FLEET_CONFIG_ID - config related to new token [defaul]
+# FLEET_ENROLLMENT_TOKEN - existing enrollment token to be used for enroll
+# FLEET_ENROLL - if set to 1 enroll will be performed
+# FLEET_SETUP - if  set to 1 fleet setup will be performed
+# FLEET_TOKEN_NAME - token name for a token to be created
+# KIBANA_HOST - actual kibana host [http://localhost:5601]
+# KIBANA_PASSWORD - password for accessing kibana API [changeme]
+# KIBANA_USERNAME - username for accessing kibana API [elastic]
+
+function setup(){
+  curl -X POST  ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/setup -H 'kbn-xsrf: true' -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme}
+  curl -X POST  ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/setup \
+    -H 'Content-Type: application/json' \
+    -H 'kbn-xsrf: true' \
+    -d '{"admin_username":"'"${FLEET_ADMIN_USERNAME:-elastic}"'","admin_password":"'"${FLEET_ADMIN_PASSWORD:-changeme}"'"}' \
+    -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme}
+}
+
+function enroll(){
+    local enrollResp
+    local apiKey
+
+    if [[ -n "${FLEET_ENROLLMENT_TOKEN}" ]] && [[ ${FLEET_ENROLLMENT_TOKEN} == 1 ]]; then
+      apikey = "${FLEET_ENROLLMENT_TOKEN}"
+    else
+      enrollResp=$(curl -X POST  ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/enrollment-api-keys \
+        -H 'Content-Type: application/json' \
+        -H 'kbn-xsrf: true' \
+        -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme} \
+        -d '{"name":"'"${FLEET_TOKEN_NAME:-demotoken}"'","config_id":"'"${FLEET_CONFIG_ID:-default}"'"}')
+
+      local exitCode=$?
+      if [ $exitCode -ne 0 ]; then
+        exit $exitCode
+      fi
+
+      apikey=$(echo $enrollResp | jq -r '.item.api_key')
+    fi
+
+    ./{{ .BeatName }} enroll ${KIBANA_HOST:-http://localhost:5601} $apikey -f
+}
+yum install -y epel-release
+yum install -y jq
+
+if [[ -n "${FLEET_SETUP}" ]] && [[ ${FLEET_SETUP} == 1 ]]; then setup; fi
+if [[ -n "${FLEET_ENROLL}" ]] && [[ ${FLEET_ENROLL} == 1 ]]; then enroll; fi
+
+exec {{ .BeatName }} run "$@"

x-pack/agent/CHANGELOG.asciidoc

@@ -0,0 +1,21 @@
+// Use these for links to issue and pulls. Note issues and pulls redirect one to
+// each other on Github, so don't worry too much on using the right prefix.
+:issue: https://github.com/elastic/beats/issues/
+:pull: https://github.com/elastic/beats/pull/
+
+
+[[release-notes-8.0.0]]
+=== Agent version 8.0.0
+
+
+==== Breaking changes
+
+==== Bugfixes
+
+- Fixed tests on windows {pull}16922[16922]
+- Fixed installers for SNAPSHOTs and windows {pull}17077[17077]
+
+==== New features
+
+- Generate index name in a format type-dataset-namespace {pull}16903[16903]
+- OS agnostic default configuration {pull}17016[17016]

x-pack/agent/_meta/agent.docker.yml

@@ -4,9 +4,9 @@
 outputs:
   default:
     type: elasticsearch
-    hosts: [127.0.0.1:9200, 127.0.0.1:9300]
-    username: elastic
-    password: changeme
+    hosts: '${ELASTICSEARCH_HOSTS:http://elasticsearch:9200}'
+    username: '${ELASTICSEARCH_USERNAME:elastic}'
+    password: '${ELASTICSEARCH_PASSWORD:changeme}'
 
 streams:
   - type: event/file
@@ -44,15 +44,15 @@ download:
   # e.g /windows-x86.zip
   sourceURI: "https://artifacts.elastic.co/downloads/beats/"
   # path to the directory containing downloaded packages
-  target_directory: "/home/elastic/downloads"
+  target_directory: "${path.data}/downloads"
   # timeout for downloading package
   timeout: 30s
   # file path to a public key used for verifying downloaded artifacts
   # if not file is present agent will try to load public key from elastic.co website.
-  pgpfile: "/home/elastic/elastic.pgp"
+  pgpfile: "${path.data}/elastic.pgp"
   # install_path describes the location of installed packages/programs. It is also used
   # for reading program specifications.
-  install_path: "/home/elastic/install"
+  install_path: "${path.data}/install"
 
 process:
   # minimal port number for spawned processes

x-pack/agent/_meta/agent.fleet.yml

@@ -10,15 +10,15 @@ download:
   # e.g /windows-x86.zip
   sourceURI: "https://artifacts.elastic.co/downloads/beats/"
   # path to the directory containing downloaded packages
-  target_directory: "/home/elastic/downloads"
+  target_directory: "${path.data}/downloads"
   # timeout for downloading package
   timeout: 30s
   # file path to a public key used for verifying downloaded artifacts
   # if not file is present agent will try to load public key from elastic.co website.
-  pgpfile: "/home/elastic/elastic.pgp"
+  pgpfile: "${path.data}/elastic.pgp"
   # install_path describes the location of installed packages/programs. It is also used
   # for reading program specifications.
-  install_path: "/home/elastic/install"
+  install_path: "${path.data}/install"
 
 process:
   # minimal port number for spawned processes

x-pack/agent/_meta/agent.yml

@@ -4,7 +4,7 @@
 outputs:
   default:
     type: elasticsearch
-    hosts: [127.0.0.1:9200, 127.0.0.1:9300]
+    hosts: [127.0.0.1:9200]
     username: elastic
     password: changeme
 
@@ -45,15 +45,15 @@ download:
   # e.g /windows-x86.zip
   sourceURI: "https://artifacts.elastic.co/downloads/beats/"
   # path to the directory containing downloaded packages
-  target_directory: "/home/elastic/downloads"
+  target_directory: "${path.data}/downloads"
   # timeout for downloading package
   timeout: 30s
   # file path to a public key used for verifying downloaded artifacts
   # if not file is present agent will try to load public key from elastic.co website.
-  pgpfile: "/home/elastic/elastic.pgp"
+  pgpfile: "${path.data}/elastic.pgp"
   # install_path describes the location of installed packages/programs. It is also used
   # for reading program specifications.
-  install_path: "/home/elastic/install"
+  install_path: "${path.data}/install"
 
 process:
   # minimal port number for spawned processes

x-pack/agent/_meta/common.p2.yml

@@ -4,7 +4,7 @@
 outputs:
   default:
     type: elasticsearch
-    hosts: [127.0.0.1:9200, 127.0.0.1:9300]
+    hosts: [127.0.0.1:9200]
     username: elastic
     password: changeme
 
@@ -70,15 +70,15 @@ download:
   # e.g /windows-x86.zip
   sourceURI: "https://artifacts.elastic.co/downloads/beats/"
   # path to the directory containing downloaded packages
-  target_directory: "/home/elastic/downloads"
+  target_directory: "${path.data}/downloads"
   # timeout for downloading package
   timeout: 30s
   # file path to a public key used for verifying downloaded artifacts
   # if not file is present agent will try to load public key from elastic.co website.
-  pgpfile: "/home/elastic/elastic.pgp"
+  pgpfile: "${path.data}/elastic.pgp"
   # install_path describes the location of installed packages/programs. It is also used
   # for reading program specifications.
-  install_path: "/home/elastic/install"
+  install_path: "${path.data}/install"
 
 process:
   # minimal port number for spawned processes