x-pack/packetbeat: install Npcap at start-up when required

CHANGELOG.next.asciidoc

@@ -195,6 +195,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Packetbeat*
 
+- Add automated OEM Npcap installation handling. {pull}29112[29112]
+
 *Functionbeat*
 
 

Jenkinsfile

@@ -51,6 +51,8 @@ pipeline {
     stage('Checkout') {
       options { skipDefaultCheckout() }
       steps {
+        // For testing purposes, let's print the env variable CI
+        sh 'env | grep CI'
         pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ])
         deleteDir()
         // Here we do a checkout into a temporary directory in order to have the
@@ -378,6 +380,8 @@ def packagingLinux(Map args = [:]) {
   withEnv([
     "PLATFORMS=${PLATFORMS}"
   ]) {
+    // for testing purposes
+    echo "CI=${env.CI}"
     target(args)
   }
 }
@@ -997,6 +1001,7 @@ def dumpVariables(){
   BEAT_VERSION: ${env.BEAT_VERSION}
   BEATS: ${env.BEATS}
   BUILD_DIR: ${env.BUILD_DIR}
+  CI: ${env.CI}
   COMMIT_ID: ${env.COMMIT_ID}
   COVERAGE_DIR: ${env.COVERAGE_DIR}
   COVERAGE_TOOL: ${env.COVERAGE_TOOL}

NOTICE.txt

@@ -16728,6 +16728,43 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+--------------------------------------------------------------------------------
+Dependency : golang.org/x/mod
+Version: v0.5.1
+Licence type (autodetected): BSD-3-Clause
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]/LICENSE:
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/net
 Version: v0.0.0-20211020060615-d418f374d309
@@ -34519,43 +34556,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
 
---------------------------------------------------------------------------------
-Dependency : golang.org/x/mod
-Version: v0.5.1
-Licence type (autodetected): BSD-3-Clause
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]/LICENSE:
-
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/term
 Version: v0.0.0-20210615171337-6886f2dfbf5b
@@ -37624,4 +37624,3 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-

dev-tools/mage/config.go

@@ -149,6 +149,7 @@ func makeConfigTemplate(destination string, mode os.FileMode, confParams ConfigF
 	params := map[string]interface{}{
 		"GOOS":                           EnvOr("DEV_OS", "linux"),
 		"GOARCH":                         EnvOr("DEV_ARCH", "amd64"),
+		"BeatLicense":                    BeatLicense,
 		"Reference":                      false,
 		"Docker":                         false,
 		"ExcludeConsole":                 false,

dev-tools/mage/crossbuild.go

@@ -183,7 +183,7 @@ func CrossBuild(options ...CrossBuildOption) error {
 		builder := GolangCrossBuilder{buildPlatform.Name, params.Target, params.InDir, params.ImageSelector}
 		if params.Serial {
 			if err := builder.Build(); err != nil {
-				return errors.Wrapf(err, "failed cross-building target=%v for platform=%v %v", params.ImageSelector,
+				return errors.Wrapf(err, "failed cross-building target=%s for platform=%s",
 					params.Target, buildPlatform.Name)
 			}
 		} else {
@@ -321,8 +321,11 @@ func (b GolangCrossBuilder) Build() error {
 		"-v", repoInfo.RootDir+":"+mountPoint,
 		"-w", workDir,
 		image,
+
+		// Arguments for docker crossbuild entrypoint. For details see
+		// https://github.com/elastic/golang-crossbuild/blob/main/go1.17/base/rootfs/entrypoint.go.
 		"--build-cmd", buildCmd+" "+b.Target,
-		"-p", b.Platform,
+		"--platforms", b.Platform,
 	)
 
 	return dockerRun(args...)

dev-tools/notice/NOTICE.txt.tmpl

@@ -26,5 +26,5 @@ Third party libraries used by the Elastic Beats project:
 {{ "=" | line }}
 Indirect dependencies
 
-{{ template "depInfo" .Indirect }}
-{{ end }}
+{{ template "depInfo" .Indirect -}}
+{{- end}}

dev-tools/packaging/package_test.go

@@ -23,6 +23,7 @@ package dev_tools
 import (
 	"archive/tar"
 	"archive/zip"
+	"bufio"
 	"bytes"
 	"compress/gzip"
 	"encoding/json"
@@ -169,7 +170,7 @@ func checkTar(t *testing.T, file string) {
 }
 
 func checkZip(t *testing.T, file string) {
-	p, err := readZip(file)
+	p, err := readZip(t, file, checkNpcapNotices)
 	if err != nil {
 		t.Error(err)
 		return
@@ -183,6 +184,62 @@ func checkZip(t *testing.T, file string) {
 	checkLicensesPresent(t, "", p)
 }
 
+const (
+	npcapSettings   = "Windows Npcap installation settings"
+	npcapGrant      = `Insecure.Com LLC \(“The Nmap Project”\) has granted Elasticsearch`
+	npcapLicense    = `Dependency : Npcap \(https://nmap.org/npcap/\)`
+	libpcapLicense  = `Dependency : Libpcap \(http://www.tcpdump.org/\)`
+	winpcapLicense  = `Dependency : Winpcap \(https://www.winpcap.org/\)`
+	radiotapLicense = `Dependency : ieee80211_radiotap.h Header File`
+)
+
+var (
+	// These reflect the order that the licenses and notices appear in the relevant files.
+	npcapConfigPattern = regexp.MustCompile(
+		"(?s)" + npcapSettings +
+			".*" + npcapGrant,
+	)
+	npcapLicensePattern = regexp.MustCompile(
+		"(?s)" + npcapLicense +
+			".*" + libpcapLicense +
+			".*" + winpcapLicense +
+			".*" + radiotapLicense,
+	)
+)
+
+func checkNpcapNotices(pkg, file string, contents io.Reader) error {
+	if !strings.Contains(pkg, "packetbeat") {
+		return nil
+	}
+
+	wantNotices := strings.Contains(pkg, "windows") && !strings.Contains(pkg, "oss")
+
+	// If the packetbeat README.md is made to be generated
+	// conditionally then it should also be checked here.
+	pkg = filepath.Base(pkg)
+	file, err := filepath.Rel(pkg[:len(pkg)-len(filepath.Ext(pkg))], file)
+	if err != nil {
+		return err
+	}
+	switch file {
+	case "packetbeat.yml", "packetbeat.reference.yml":
+		if npcapConfigPattern.MatchReader(bufio.NewReader(contents)) != wantNotices {
+			if wantNotices {
+				return fmt.Errorf("Npcap config section not found in config file %s in %s", file, pkg)
+			}
+			return fmt.Errorf("unexpected Npcap config section found in config file %s in %s", file, pkg)
+		}
+	case "NOTICE.txt":
+		if npcapLicensePattern.MatchReader(bufio.NewReader(contents)) != wantNotices {
+			if wantNotices {
+				return fmt.Errorf("Npcap license section not found in %s file in %s", file, pkg)
+			}
+			return fmt.Errorf("unexpected Npcap license section found in %s file in %s", file, pkg)
+		}
+	}
+	return nil
+}
+
 func checkDocker(t *testing.T, file string) {
 	p, info, err := readDocker(file)
 	if err != nil {
@@ -623,7 +680,11 @@ func readTarContents(tarName string, data io.Reader) (*packageFile, error) {
 	return p, nil
 }
 
-func readZip(zipFile string) (*packageFile, error) {
+// inspector is a file contents inspector. It vets the contents of the file
+// within a package for a requirement and returns an error if it is not met.
+type inspector func(pkg, file string, contents io.Reader) error
+
+func readZip(t *testing.T, zipFile string, inspectors ...inspector) (*packageFile, error) {
 	r, err := zip.OpenReader(zipFile)
 	if err != nil {
 		return nil, err
@@ -636,6 +697,18 @@ func readZip(zipFile string) (*packageFile, error) {
 			File: f.Name,
 			Mode: f.Mode(),
 		}
+		for _, inspect := range inspectors {
+			r, err := f.Open()
+			if err != nil {
+				t.Errorf("failed to open %s in %s: %v", f.Name, zipFile, err)
+				break
+			}
+			err = inspect(zipFile, f.Name, r)
+			if err != nil {
+				t.Error(err)
+			}
+			r.Close()
+		}
 	}
 
 	return p, nil
@@ -740,7 +813,6 @@ func readDockerManifest(r io.Reader) (*dockerManifest, error) {
 	err = json.Unmarshal(data, &manifests)
 	if err != nil {
 		return nil, err
-
 	}
 
 	if len(manifests) != 1 {

go.mod

@@ -164,6 +164,7 @@ require (
 	go.uber.org/zap v1.14.1
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616
+	golang.org/x/mod v0.5.1
 	golang.org/x/net v0.0.0-20211020060615-d418f374d309
 	golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
@@ -268,7 +269,6 @@ require (
 	github.com/xdg/stringprep v1.0.3 // indirect
 	go.elastic.co/fastjson v1.1.0 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/mod v0.5.1 // indirect
 	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

packetbeat/README.md

@@ -39,3 +39,17 @@ If you are sure you found a bug or have a feature request, open an issue on
 
 We love contributions from our community! Please read the
 [CONTRIBUTING.md](../CONTRIBUTING.md) file.
+
+## LICENSE NOTICE for Windows users of Packetbeat
+
+The default distribution of Packetbeat for Windows comes bundled with the Npcap
+library. This is not available in the OSS-only distribution of Packetbeat.
+
+**Restrictions on Distribution**
+
+Insecure.Com LLC (“The Nmap Project”) has granted Elasticsearch BV and its
+affiliates the right to include Npcap with this distribution of Packetbeat.
+You may not distribute this version of Packetbeat or any other package from
+Elastic that includes Npcap. If you wish to distribute Npcap, or any package
+that includes Npcap, you should reach out to The Nmap Project to obtain a
+distribution license. See https://nmap.org/npcap/ for more details.

packetbeat/_meta/config/beat.reference.yml.tmpl

@@ -57,6 +57,8 @@ packetbeat.interfaces.internal_networks:
 # can stay enabled even after beat is shut down.
 #packetbeat.interfaces.auto_promisc_mode: true
 
+{{- template "windows_npcap.yml.tmpl" .}}
+
 {{header "Flows"}}
 
 packetbeat.flows:

packetbeat/_meta/config/beat.yml.tmpl

@@ -23,6 +23,8 @@ packetbeat.interfaces.device: {{ call .device .GOOS }}
 packetbeat.interfaces.internal_networks:
   - private
 
+{{- template "windows_npcap.yml.tmpl" .}}
+
 {{header "Flows"}}
 
 # Set `enabled: false` or comment out all options to disable flows reporting.

packetbeat/_meta/config/windows_npcap.yml.tmpl

@@ -0,0 +1,36 @@
+{{if and (eq .BeatLicense "Elastic License") (eq .GOOS "windows")}}
+
+{{header "Windows Npcap installation settings"}}
+
+# Windows Npcap installation options. These options specify how the Npcap packet
+# capture library for Windows should be obtained and installed.
+# Npcap installation is only available in the default distribution of Packetbeat
+# for Windows and is not available in the OSS-only distribution of Packetbeat.
+#
+# LICENSE NOTICE
+#
+# Restrictions on Distribution
+#
+# Insecure.Com LLC (“The Nmap Project”) has granted Elasticsearch BV and its
+# affiliates the right to include Npcap with this distribution of Packetbeat.
+# You may not distribute this version of Packetbeat or any other package from
+# Elastic that includes Npcap. If you wish to distribute Npcap, or any package
+# that includes Npcap, you should reach out to The Nmap Project to obtain a
+# distribution license. See https://nmap.org/npcap/ for more details.
+#
+#npcap:
+#  # install_destination allows configuration of the location that the Npcap will
+#  # place the Npcap library and associated files. See https://nmap.org/npcap/guide/npcap-users-guide.html#npcap-installation-uninstall-options.
+#  install_destination: ""
+#  install_timeout: 120s
+#  # ignore_missing_registry specifies that failure to query the registry server
+#  # will be ignored with a logged warning.
+#  ignore_missing_registry: false
+#  # By default Npcap will be installed only when a newer version of Npcap is available.
+#  # force_reinstall forces a new installation of Npcap in all cases.
+#  force_reinstall: false
+#  # If a specific local version of Npcap is required installation by packetbeat
+#  # can be blocked by setting never_install to true. No action is taken if this
+#  # option is set to true.
+#  never_install: false
+{{- end -}}