Skip to content

Agent required changes for building the agent artifacts #16624

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ph merged 5 commits into from
Mar 2, 2020
Merged

Agent required changes for building the agent artifacts #16624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3a15f38
Agent required changes for building the agent artifacts
ph Feb 26, 2020
bb66882
Update dev-tools/mage/pkgspecs.go
ph Feb 27, 2020
e3a175d
Update dev-tools/mage/pkgspecs.go
ph Feb 27, 2020
6034c9a
Update dev-tools/mage/pkgspecs.go
ph Feb 27, 2020
f146e39
update min version for macos to 1.11
ph Feb 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions dev-tools/mage/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -723,6 +723,14 @@ func OSSBeatDir(path ...string) string {
// XPackBeatDir returns the X-Pack beat directory. You can pass paths and they
// will be joined and appended to the X-Pack beat dir.
func XPackBeatDir(path ...string) string {
// Check if we have an X-Pack only beats
cur := CWD()

if parentDir := filepath.Base(filepath.Dir(cur)); parentDir == "x-pack" {
tmp := filepath.Join(filepath.Dir(cur), BeatName)
return filepath.Join(append([]string{tmp}, path...)...)
}

return OSSBeatDir(append([]string{XPackDir, BeatName}, path...)...)
}

Expand Down
24 changes: 18 additions & 6 deletions dev-tools/mage/pkg.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,17 +60,29 @@ func Package() error {
continue
}

agentPackageType := TarGz
if pkg.OS == "windows" {
agentPackageType = Zip
}

agentPackageArch, err := getOSArchName(target, agentPackageType)
if err != nil {
log.Printf("Skipping arch %v for package type %v: %v", target.Arch(), pkgType, err)
continue
}

spec := pkg.Spec.Clone()
spec.OS = target.GOOS()
spec.Arch = packageArch
spec.Snapshot = Snapshot
spec.evalContext = map[string]interface{}{
"GOOS": target.GOOS(),
"GOARCH": target.GOARCH(),
"GOARM": target.GOARM(),
"Platform": target,
"PackageType": pkgType.String(),
"BinaryExt": binaryExtension(target.GOOS()),
"GOOS": target.GOOS(),
"GOARCH": target.GOARCH(),
"GOARM": target.GOARM(),
"Platform": target,
"AgentArchName": agentPackageArch,
"PackageType": pkgType.String(),
"BinaryExt": binaryExtension(target.GOOS()),
}

spec.packageDir, err = pkgType.PackagingDir(packageStagingDir, target, spec)
Expand Down
13 changes: 13 additions & 0 deletions dev-tools/mage/pkgspecs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,13 @@ func UseCommunityBeatPackaging() {
MustUsePackaging("community_beat", packageSpecFile)
}

// UseElasticAgentPackaging configures the package target to build packages for
// an Elastic Agent.
func UseElasticAgentPackaging() {
// Prepare binaries so they can be packed into agent
MustUsePackaging("elastic_beat_agent_binaries", packageSpecFile)
}

// UseElasticBeatPackaging configures the package target to build packages for
// an Elastic Beat. This means it will generate two sets of packages -- one
// that is purely OSS under Apache 2.0 and one that is licensed under the
Expand All @@ -60,6 +67,12 @@ func UseElasticBeatXPackPackaging() {
MustUsePackaging("elastic_beat_xpack", packageSpecFile)
}

// UseElasticBeatXPackReducedPackaging configures the package target to build Elastic
// licensed (X-Pack) packages for agent use.
func UseElasticBeatXPackReducedPackaging() {
MustUsePackaging("elastic_beat_xpack_reduced", packageSpecFile)
}

// UseElasticBeatWithoutXPackPackaging configures the package target to build
// packages for an Elastic Beat. This means it will generate two sets of
// packages -- one that is purely OSS under Apache 2.0 and one that is licensed
Expand Down
3 changes: 2 additions & 1 deletion dev-tools/mage/pkgtypes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,8 @@ func PackageZip(spec PackageSpec) error {
// Add files to zip.
for _, pkgFile := range spec.Files {
if err := addFileToZip(w, baseDir, pkgFile); err != nil {
return errors.Wrapf(err, "failed adding file=%+v to zip", pkgFile)
p, _ := filepath.Abs(pkgFile.Source)
return errors.Wrapf(err, "failed adding file=%+v to zip", p)
}
}

Expand Down
237 changes: 237 additions & 0 deletions dev-tools/packaging/packages.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,158 @@ shared:
url: '{{.BeatURL}}'
description: '{{.BeatDescription}}'

# agent specific
# Deb/RPM spec for community beats.
- &deb_rpm_agent_spec
<<: *common
post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/systemd-daemon-reload.sh'
files:
/usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
mode: 0755
/usr/share/{{.BeatName}}/LICENSE.txt:
source: '{{ repo.RootDir }}/LICENSE.txt'
mode: 0644
/usr/share/{{.BeatName}}/README.md:
template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl'
Copy link
Copy Markdown
Member

@andrewkroh andrewkroh Feb 27, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This readme template is specific to a Beat. I think the agent will want to have its own.

mode: 0644
/usr/share/{{.BeatName}}/.build_hash.txt:
content: >
{{ commit }}
mode: 0644
/etc/{{.BeatName}}/agent.reference.yml:
source: 'agent.reference.yml'
mode: 0644
/etc/{{.BeatName}}/agent.yml:
source: 'agent.yml'
mode: 0600
config: true
/usr/share/{{.BeatName}}/bin/{{.BeatName}}-god:
source: build/golang-crossbuild/god-{{.GOOS}}-{{.Platform.Arch}}
mode: 0755
/usr/bin/{{.BeatName}}:
template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/linux/beatname.sh.tmpl'
mode: 0755
/lib/systemd/system/{{.BeatServiceName}}.service:
template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/linux/systemd.unit.tmpl'
mode: 0644
/etc/init.d/{{.BeatServiceName}}:
template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/init.sh.tmpl'
mode: 0755
/etc/{{.BeatName}}/beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644
/etc/{{.BeatName}}/beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644


# MacOS pkg spec for community beats.
- &macos_agent_pkg_spec
<<: *common
extra_vars:
# OS X 10.11 Mountain Lion is the oldest supported by Go 1.13.
# https://golang.org/doc/go1.13#ports
min_supported_osx_version: 10.11
identifier: 'co.{{.BeatVendor | tolower}}.beats.{{.BeatName}}'
install_path: /Library/Application Support
pre_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/scripts/preinstall.tmpl'
post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/scripts/postinstall.tmpl'
files:
/Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
mode: 0755
/Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/LICENSE.txt:
source: '{{ repo.RootDir }}/LICENSE.txt'
mode: 0644
/Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/README.md:
template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl'
mode: 0644
/Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/.build_hash.txt:
content: >
{{ commit }}
mode: 0644
/Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/{{.identifier}}.plist:
template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl'
mode: 0644
/etc/{{.BeatName}}/agent.reference.yml:
source: 'agent.reference.yml'
mode: 0644
/etc/{{.BeatName}}/agent.yml:
source: 'agent.yml'
mode: 0600
config: true
/etc/{{.BeatName}}/beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644
/etc/{{.BeatName}}/beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644

- &agent_binary_files
'{{.BeatName}}{{.BinaryExt}}':
source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
mode: 0755
LICENSE.txt:
source: '{{ repo.RootDir }}/LICENSE.txt'
mode: 0644
README.md:
template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl'
mode: 0644
.build_hash.txt:
content: >
{{ commit }}
mode: 0644
'agent.reference.yml':
source: 'agent.reference.yml'
mode: 0644
'agent.yml':
source: 'agent.yml'
mode: 0600
config: true

# Binary package spec (tar.gz for linux/darwin) for community beats.
- &agent_binary_spec
<<: *common
files:
<<: *agent_binary_files
'beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644
'beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644

# Binary package spec (zip for windows) for community beats.
- &agent_windows_binary_spec
<<: *common
files:
<<: *agent_binary_files
install-service-{{.BeatName}}.ps1:
template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/install-service.ps1.tmpl'
Copy link
Copy Markdown
Member

@andrewkroh andrewkroh Feb 27, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does the agent use the same CLI arguments as the beats? Maybe you need a customized install script?

Copy link
Copy Markdown
Contributor

@michalpristas michalpristas Mar 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we tried to mimic beat behavior here and we support args like -c or path.* i'm not sure about -E

mode: 0755
uninstall-service-{{.BeatName}}.ps1:
template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/uninstall-service.ps1.tmpl'
mode: 0755
'beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip'
mode: 0644
'beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip'
mode: 0644

- &agent_docker_spec
<<: *agent_binary_spec
extra_vars:
from: 'centos:7'
user: '{{ .BeatName }}'
linux_capabilities: ''
Copy link
Copy Markdown
Member

@andrewkroh andrewkroh Feb 27, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So that agent can run in a container as a non-root user without any special Linux capabilities added to the binary? 🤔

Copy link
Copy Markdown
Contributor

@michalpristas michalpristas Mar 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is a copy paste from some beat, we in fact need to update user to a full root probably

Copy link
Copy Markdown
Contributor

@michalpristas michalpristas Mar 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

root is used here in this PR: https://github.com/elastic/beats/pull/16898/files

files:
'agent.yml':
source: 'agent.docker.yml'
mode: 0600
config: true

# Deb/RPM spec for community beats.
- &deb_rpm_spec
<<: *common
Expand Down Expand Up @@ -338,6 +490,29 @@ specs:
<<: *elastic_docker_spec
<<: *elastic_license_for_binaries

# Elastic Beat with Elastic License and binary taken the current directory.
elastic_beat_xpack_reduced:
###
# Elastic Licensed Packages
###
- os: windows
types: [zip]
spec:
<<: *windows_binary_spec
<<: *elastic_license_for_binaries

- os: darwin
types: [tgz]
spec:
<<: *binary_spec
<<: *elastic_license_for_binaries

- os: linux
types: [tgz]
spec:
<<: *binary_spec
<<: *elastic_license_for_binaries

# Elastic Beat with Elastic License and binary taken from the x-pack dir.
elastic_beat_xpack_separate_binaries:
###
Expand Down Expand Up @@ -398,3 +573,65 @@ specs:
files:
'{{.BeatName}}{{.BinaryExt}}':
source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

# Elastic Beat with Elastic License and binary taken from the x-pack dir.
elastic_beat_agent_binaries:
###
# Elastic Licensed Packages
###
- os: windows
types: [zip]
spec:
<<: *agent_windows_binary_spec
<<: *elastic_license_for_binaries
files:
'{{.BeatName}}{{.BinaryExt}}':
source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

- os: darwin
types: [tgz]
spec:
<<: *agent_binary_spec
<<: *elastic_license_for_binaries
files:
'{{.BeatName}}{{.BinaryExt}}':
source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

- os: darwin
types: [dmg]
spec:
<<: *macos_agent_pkg_spec
<<: *elastic_license_for_macos_pkg
files:
/Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
mode: 0755
source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

- os: linux
types: [tgz]
spec:
<<: *agent_binary_spec
<<: *elastic_license_for_binaries
files:
'{{.BeatName}}{{.BinaryExt}}':
source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

- os: linux
types: [deb, rpm]
spec:
<<: *deb_rpm_agent_spec
<<: *elastic_license_for_deb_rpm
files:
/usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

- os: linux
types: [docker]
spec:
<<: *agent_docker_spec
<<: *elastic_docker_spec
<<: *elastic_license_for_binaries
files:
'{{.BeatName}}{{.BinaryExt}}':
source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}