From 3a15f38741c84a2e611ca969948a928e600649ef Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 26 Feb 2020 13:02:16 -0500 Subject: [PATCH 1/5] Agent required changes for building the agent artifacts The Agent has special requirements: - Only Elastic licensed - Bundle Metricbeat - Bundle Filebeat This PR isolate all the required changes for the devtools to correctly build the agent. This can be merged as is because it doesn't impact building any other beats. --- dev-tools/mage/common.go | 8 ++ dev-tools/mage/pkg.go | 24 +++- dev-tools/mage/pkgspecs.go | 13 ++ dev-tools/mage/pkgtypes.go | 3 +- dev-tools/packaging/packages.yml | 237 +++++++++++++++++++++++++++++++ 5 files changed, 278 insertions(+), 7 deletions(-) diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index 4b8215b7b439..2a173b8e3b48 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -723,6 +723,14 @@ func OSSBeatDir(path ...string) string { // XPackBeatDir returns the X-Pack beat directory. You can pass paths and they // will be joined and appended to the X-Pack beat dir. func XPackBeatDir(path ...string) string { + // Check if we have an X-Pack only beats + cur := CWD() + + if parentDir := filepath.Base(filepath.Dir(cur)); parentDir == "x-pack" { + tmp := filepath.Join(filepath.Dir(cur), BeatName) + return filepath.Join(append([]string{tmp}, path...)...) + } + return OSSBeatDir(append([]string{XPackDir, BeatName}, path...)...) } diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index 1c0ba566704e..7a9f4926bb45 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -60,17 +60,29 @@ func Package() error { continue } + agentPackageType := TarGz + if pkg.OS == "windows" { + agentPackageType = Zip + } + + agentPackageArch, err := getOSArchName(target, agentPackageType) + if err != nil { + log.Printf("Skipping arch %v for package type %v: %v", target.Arch(), pkgType, err) + continue + } + spec := pkg.Spec.Clone() spec.OS = target.GOOS() spec.Arch = packageArch spec.Snapshot = Snapshot spec.evalContext = map[string]interface{}{ - "GOOS": target.GOOS(), - "GOARCH": target.GOARCH(), - "GOARM": target.GOARM(), - "Platform": target, - "PackageType": pkgType.String(), - "BinaryExt": binaryExtension(target.GOOS()), + "GOOS": target.GOOS(), + "GOARCH": target.GOARCH(), + "GOARM": target.GOARM(), + "Platform": target, + "AgentArchName": agentPackageArch, + "PackageType": pkgType.String(), + "BinaryExt": binaryExtension(target.GOOS()), } spec.packageDir, err = pkgType.PackagingDir(packageStagingDir, target, spec) diff --git a/dev-tools/mage/pkgspecs.go b/dev-tools/mage/pkgspecs.go index ba8d403056b6..eb59e3349014 100644 --- a/dev-tools/mage/pkgspecs.go +++ b/dev-tools/mage/pkgspecs.go @@ -39,6 +39,13 @@ func UseCommunityBeatPackaging() { MustUsePackaging("community_beat", packageSpecFile) } +// UseElasticBeatAgentPackaging configures the package target to build packages for +// an Elastic Beat Agent. +func UseElasticBeatAgentPackaging() { + // Prepare binaries so they can be packed into agent + MustUsePackaging("elastic_beat_agent_binaries", packageSpecFile) +} + // UseElasticBeatPackaging configures the package target to build packages for // an Elastic Beat. This means it will generate two sets of packages -- one // that is purely OSS under Apache 2.0 and one that is licensed under the @@ -60,6 +67,12 @@ func UseElasticBeatXPackPackaging() { MustUsePackaging("elastic_beat_xpack", packageSpecFile) } +// UseElasticBeatXPackReducedPackaging configures the package target to build Elastic +// licensed (X-Pack) packages for agent use. +func UseElasticBeatXPackReducedPackaging() { + MustUsePackaging("elastic_beat_xpack_reduced", packageSpecFile) +} + // UseElasticBeatWithoutXPackPackaging configures the package target to build // packages for an Elastic Beat. This means it will generate two sets of // packages -- one that is purely OSS under Apache 2.0 and one that is licensed diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 1be456d2e139..82fea52376c5 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -539,7 +539,8 @@ func PackageZip(spec PackageSpec) error { // Add files to zip. for _, pkgFile := range spec.Files { if err := addFileToZip(w, baseDir, pkgFile); err != nil { - return errors.Wrapf(err, "failed adding file=%+v to zip", pkgFile) + p, _ := filepath.Abs(pkgFile.Source) + return errors.Wrapf(err, "failed adding file=%+v to zip", p) } } diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index a780f69c9bb6..7e66ae838e87 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -16,6 +16,158 @@ shared: url: '{{.BeatURL}}' description: '{{.BeatDescription}}' + # agent specific + # Deb/RPM spec for community beats. + - &deb_rpm_agent_spec + <<: *common + post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/systemd-daemon-reload.sh' + files: + /usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}: + source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + mode: 0755 + /usr/share/{{.BeatName}}/LICENSE.txt: + source: '{{ repo.RootDir }}/LICENSE.txt' + mode: 0644 + /usr/share/{{.BeatName}}/README.md: + template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' + mode: 0644 + /usr/share/{{.BeatName}}/.build_hash.txt: + content: > + {{ commit }} + mode: 0644 + /etc/{{.BeatName}}/agent.reference.yml: + source: 'agent.reference.yml' + mode: 0644 + /etc/{{.BeatName}}/agent.yml: + source: 'agent.yml' + mode: 0600 + config: true + /usr/share/{{.BeatName}}/bin/{{.BeatName}}-god: + source: build/golang-crossbuild/god-{{.GOOS}}-{{.Platform.Arch}} + mode: 0755 + /usr/bin/{{.BeatName}}: + template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/linux/beatname.sh.tmpl' + mode: 0755 + /lib/systemd/system/{{.BeatServiceName}}.service: + template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/linux/systemd.unit.tmpl' + mode: 0644 + /etc/init.d/{{.BeatServiceName}}: + template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/init.sh.tmpl' + mode: 0755 + /etc/{{.BeatName}}/beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: + source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + /etc/{{.BeatName}}/beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: + source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + + + # MacOS pkg spec for community beats. + - &macos_agent_pkg_spec + <<: *common + extra_vars: + # OS X 10.8 Mountain Lion is the oldest supported by Go 1.10. + # https://golang.org/doc/go1.10#ports + min_supported_osx_version: 10.8 + identifier: 'co.{{.BeatVendor | tolower}}.beats.{{.BeatName}}' + install_path: /Library/Application Support + pre_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/scripts/preinstall.tmpl' + post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/scripts/postinstall.tmpl' + files: + /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}: + source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + mode: 0755 + /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/LICENSE.txt: + source: '{{ repo.RootDir }}/LICENSE.txt' + mode: 0644 + /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/README.md: + template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' + mode: 0644 + /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/.build_hash.txt: + content: > + {{ commit }} + mode: 0644 + /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/{{.identifier}}.plist: + template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl' + mode: 0644 + /etc/{{.BeatName}}/agent.reference.yml: + source: 'agent.reference.yml' + mode: 0644 + /etc/{{.BeatName}}/agent.yml: + source: 'agent.yml' + mode: 0600 + config: true + /etc/{{.BeatName}}/beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: + source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + /etc/{{.BeatName}}/beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: + source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + + - &agent_binary_files + '{{.BeatName}}{{.BinaryExt}}': + source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + mode: 0755 + LICENSE.txt: + source: '{{ repo.RootDir }}/LICENSE.txt' + mode: 0644 + README.md: + template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' + mode: 0644 + .build_hash.txt: + content: > + {{ commit }} + mode: 0644 + 'agent.reference.yml': + source: 'agent.reference.yml' + mode: 0644 + 'agent.yml': + source: 'agent.yml' + mode: 0600 + config: true + + # Binary package spec (tar.gz for linux/darwin) for community beats. + - &agent_binary_spec + <<: *common + files: + <<: *agent_binary_files + 'beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz': + source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + 'beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz': + source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + + # Binary package spec (zip for windows) for community beats. + - &agent_windows_binary_spec + <<: *common + files: + <<: *agent_binary_files + install-service-{{.BeatName}}.ps1: + template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/install-service.ps1.tmpl' + mode: 0755 + uninstall-service-{{.BeatName}}.ps1: + template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/uninstall-service.ps1.tmpl' + mode: 0755 + 'beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip': + source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip' + mode: 0644 + 'beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip': + source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip' + mode: 0644 + + - &agent_docker_spec + <<: *agent_binary_spec + extra_vars: + from: 'centos:7' + user: '{{ .BeatName }}' + linux_capabilities: '' + files: + 'agent.yml': + source: 'agent.docker.yml' + mode: 0600 + config: true + # Deb/RPM spec for community beats. - &deb_rpm_spec <<: *common @@ -338,6 +490,29 @@ specs: <<: *elastic_docker_spec <<: *elastic_license_for_binaries + # Elastic Beat with Elastic License and binary taken the current directory. + elastic_beat_xpack_reduced: + ### + # Elastic Licensed Packages + ### + - os: windows + types: [zip] + spec: + <<: *windows_binary_spec + <<: *elastic_license_for_binaries + + - os: darwin + types: [tgz] + spec: + <<: *binary_spec + <<: *elastic_license_for_binaries + + - os: linux + types: [tgz] + spec: + <<: *binary_spec + <<: *elastic_license_for_binaries + # Elastic Beat with Elastic License and binary taken from the x-pack dir. elastic_beat_xpack_separate_binaries: ### @@ -398,3 +573,65 @@ specs: files: '{{.BeatName}}{{.BinaryExt}}': source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + # Elastic Beat with Elastic License and binary taken from the x-pack dir. + elastic_beat_agent_binaries: + ### + # Elastic Licensed Packages + ### + - os: windows + types: [zip] + spec: + <<: *agent_windows_binary_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + - os: darwin + types: [tgz] + spec: + <<: *agent_binary_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + - os: darwin + types: [dmg] + spec: + <<: *macos_agent_pkg_spec + <<: *elastic_license_for_macos_pkg + files: + /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}: + mode: 0755 + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + - os: linux + types: [tgz] + spec: + <<: *agent_binary_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + - os: linux + types: [deb, rpm] + spec: + <<: *deb_rpm_agent_spec + <<: *elastic_license_for_deb_rpm + files: + /usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}: + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + - os: linux + types: [docker] + spec: + <<: *agent_docker_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + From bb66882ea8f2513cc2819b880b77073da045e522 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 27 Feb 2020 08:14:03 -0500 Subject: [PATCH 2/5] Update dev-tools/mage/pkgspecs.go Co-Authored-By: Nicolas Ruflin --- dev-tools/mage/pkgspecs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/mage/pkgspecs.go b/dev-tools/mage/pkgspecs.go index eb59e3349014..a06bd16969d1 100644 --- a/dev-tools/mage/pkgspecs.go +++ b/dev-tools/mage/pkgspecs.go @@ -40,7 +40,7 @@ func UseCommunityBeatPackaging() { } // UseElasticBeatAgentPackaging configures the package target to build packages for -// an Elastic Beat Agent. +// an Elastic Agent. func UseElasticBeatAgentPackaging() { // Prepare binaries so they can be packed into agent MustUsePackaging("elastic_beat_agent_binaries", packageSpecFile) From e3a175d08a88dccb377e9d2e0757616513a8af45 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 27 Feb 2020 08:14:09 -0500 Subject: [PATCH 3/5] Update dev-tools/mage/pkgspecs.go Co-Authored-By: Nicolas Ruflin --- dev-tools/mage/pkgspecs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/mage/pkgspecs.go b/dev-tools/mage/pkgspecs.go index a06bd16969d1..c06fd276714f 100644 --- a/dev-tools/mage/pkgspecs.go +++ b/dev-tools/mage/pkgspecs.go @@ -41,7 +41,7 @@ func UseCommunityBeatPackaging() { // UseElasticBeatAgentPackaging configures the package target to build packages for // an Elastic Agent. -func UseElasticBeatAgentPackaging() { +func UseElasticAgentPackaging() { // Prepare binaries so they can be packed into agent MustUsePackaging("elastic_beat_agent_binaries", packageSpecFile) } From 6034c9ab63c41848a36a528a7d27165c1419ea34 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 27 Feb 2020 08:14:15 -0500 Subject: [PATCH 4/5] Update dev-tools/mage/pkgspecs.go Co-Authored-By: Nicolas Ruflin --- dev-tools/mage/pkgspecs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/mage/pkgspecs.go b/dev-tools/mage/pkgspecs.go index c06fd276714f..2500230c55e4 100644 --- a/dev-tools/mage/pkgspecs.go +++ b/dev-tools/mage/pkgspecs.go @@ -39,7 +39,7 @@ func UseCommunityBeatPackaging() { MustUsePackaging("community_beat", packageSpecFile) } -// UseElasticBeatAgentPackaging configures the package target to build packages for +// UseElasticAgentPackaging configures the package target to build packages for // an Elastic Agent. func UseElasticAgentPackaging() { // Prepare binaries so they can be packed into agent From f146e390a897685a2aa80ff028d20f2da7338daa Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 27 Feb 2020 09:04:55 -0500 Subject: [PATCH 5/5] update min version for macos to 1.11 --- dev-tools/packaging/packages.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 7e66ae838e87..990751b3e789 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -66,9 +66,9 @@ shared: - &macos_agent_pkg_spec <<: *common extra_vars: - # OS X 10.8 Mountain Lion is the oldest supported by Go 1.10. - # https://golang.org/doc/go1.10#ports - min_supported_osx_version: 10.8 + # OS X 10.11 Mountain Lion is the oldest supported by Go 1.13. + # https://golang.org/doc/go1.13#ports + min_supported_osx_version: 10.11 identifier: 'co.{{.BeatVendor | tolower}}.beats.{{.BeatName}}' install_path: /Library/Application Support pre_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/scripts/preinstall.tmpl'