diff --git a/auditbeat/tests/system/auditbeat.py b/auditbeat/tests/system/auditbeat.py index 1c0f4e816b89..8892afbe1027 100644 --- a/auditbeat/tests/system/auditbeat.py +++ b/auditbeat/tests/system/auditbeat.py @@ -15,8 +15,11 @@ class BaseTest(MetricbeatTest): @classmethod def setUpClass(self): self.beat_name = "auditbeat" - self.beat_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), "../../")) + + if not hasattr(self, 'beat_path'): + self.beat_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../")) + super(MetricbeatTest, self).setUpClass() def create_file(self, path, contents): diff --git a/libbeat/generator/fields/fields.go b/libbeat/generator/fields/fields.go index 9449f55de907..87c2cd46c818 100644 --- a/libbeat/generator/fields/fields.go +++ b/libbeat/generator/fields/fields.go @@ -31,6 +31,27 @@ type YmlFile struct { Indent int } +// NewYmlFile performs some checks and then creates and returns a YmlFile struct +func NewYmlFile(path string, indent int) (*YmlFile, error) { + _, err := os.Stat(path) + + if os.IsNotExist(err) { + // skip + return nil, nil + } + + if err != nil { + // return error + return nil, err + } + + // All good, return file + return &YmlFile{ + Path: path, + Indent: indent, + }, nil +} + func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) { commonFields := []string{ filepath.Join(beatPath, "_meta/fields.common.yml"), @@ -52,16 +73,13 @@ func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([] var files []*YmlFile for _, cf := range commonFields { - _, err := os.Stat(cf) - if os.IsNotExist(err) { - continue - } else if err != nil { + ymlFile, err := NewYmlFile(cf, 0) + + if err != nil { return nil, err + } else if ymlFile != nil { + files = append(files, ymlFile) } - files = append(files, &YmlFile{ - Path: cf, - Indent: 0, - }) } files = append(files, libbeatFieldFiles...) diff --git a/libbeat/generator/fields/module_fields_collector.go b/libbeat/generator/fields/module_fields_collector.go index f9feeefc4b6e..a095b16aae19 100644 --- a/libbeat/generator/fields/module_fields_collector.go +++ b/libbeat/generator/fields/module_fields_collector.go @@ -19,7 +19,6 @@ package fields import ( "io/ioutil" - "os" "path/filepath" ) @@ -69,16 +68,15 @@ func CollectModuleFiles(modulesDir string) ([]*YmlFile, error) { // CollectFiles collects all files for the given module including filesets func CollectFiles(module string, modulesPath string) ([]*YmlFile, error) { - var files []*YmlFile + fieldsYmlPath := filepath.Join(modulesPath, module, "_meta/fields.yml") - if _, err := os.Stat(fieldsYmlPath); !os.IsNotExist(err) { - files = append(files, &YmlFile{ - Path: fieldsYmlPath, - Indent: 0, - }) - } else if !os.IsNotExist(err) && err != nil { + ymlFile, err := NewYmlFile(fieldsYmlPath, 0) + + if err != nil { return nil, err + } else if ymlFile != nil { + files = append(files, ymlFile) } modulesRoot := filepath.Base(modulesPath) @@ -91,14 +89,14 @@ func CollectFiles(module string, modulesPath string) ([]*YmlFile, error) { if !s.IsDir() { continue } + fieldsYmlPath = filepath.Join(modulesPath, module, s.Name(), "_meta/fields.yml") - if _, err = os.Stat(fieldsYmlPath); !os.IsNotExist(err) { - files = append(files, &YmlFile{ - Path: fieldsYmlPath, - Indent: indentByModule[modulesRoot], - }) - } else if !os.IsNotExist(err) && err != nil { + ymlFile, err := NewYmlFile(fieldsYmlPath, indentByModule[modulesRoot]) + + if err != nil { return nil, err + } else if ymlFile != nil { + files = append(files, ymlFile) } } return files, nil diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 916cd6e4ee02..286e0b712065 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -80,7 +80,7 @@ PIP_INSTALL_PARAMS?= BUILDID?=$(shell git rev-parse HEAD) ## @Building The build ID VIRTUALENV_PARAMS?= INTEGRATION_TESTS?= -FIND=. ${PYTHON_ENV}/bin/activate; find . -type f -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*" +FIND?=. ${PYTHON_ENV}/bin/activate; find . -type f -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*" PERM_EXEC?=$(shell [ `uname -s` = "Darwin" ] && echo "+111" || echo "/a+x") ifeq ($(DOCKER_CACHE),0) @@ -201,7 +201,7 @@ integration-tests-environment: prepare-tests build-image system-tests: ## @testing Runs the system tests system-tests: prepare-tests ${BEAT_NAME}.test python-env . ${PYTHON_ENV}/bin/activate; INTEGRATION_TESTS=${INTEGRATION_TESTS} TESTING_ENVIRONMENT=${TESTING_ENVIRONMENT} DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} nosetests ${PYTHON_TEST_FILES} ${NOSETESTS_OPTIONS} - python ${ES_BEATS}/dev-tools/aggregate_coverage.py -o ${COVERAGE_DIR}/system.cov ./build/system-tests/run + python ${ES_BEATS}/dev-tools/aggregate_coverage.py -o ${COVERAGE_DIR}/system.cov ${BUILD_DIR}/system-tests/run # Runs the system tests .PHONY: system-tests-environment diff --git a/libbeat/scripts/cmd/global_fields/main.go b/libbeat/scripts/cmd/global_fields/main.go index 238ce040b8b8..2b77b6a9e85a 100644 --- a/libbeat/scripts/cmd/global_fields/main.go +++ b/libbeat/scripts/cmd/global_fields/main.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/elastic/beats/libbeat/generator/fields" ) @@ -78,15 +79,29 @@ func main() { var fieldsFiles []*fields.YmlFile for _, fieldsFilePath := range beatFieldsPaths { - pathToModules := filepath.Join(beatPath, fieldsFilePath) + fullPath := filepath.Join(beatPath, fieldsFilePath) - fieldsFile, err := fields.CollectModuleFiles(pathToModules) - if err != nil { - fmt.Fprintf(os.Stderr, "Cannot collect fields.yml files: %+v\n", err) - os.Exit(2) - } + isFile := strings.HasSuffix(fullPath, ".yml") + if isFile { + file, err := fields.NewYmlFile(fullPath, 0) + + if err != nil || file == nil { + fmt.Fprintf(os.Stderr, "Cannot collect fields.yml file: %+v\n", err) + os.Exit(2) + } + + fieldsFiles = append(fieldsFiles, file) + } else { + // fullPath is a directory + files, err := fields.CollectModuleFiles(fullPath) - fieldsFiles = append(fieldsFiles, fieldsFile...) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot collect fields.yml files: %+v\n", err) + os.Exit(2) + } + + fieldsFiles = append(fieldsFiles, files...) + } } err = fields.Generate(esBeatsPath, beatPath, fieldsFiles, output) diff --git a/metricbeat/tests/system/metricbeat.py b/metricbeat/tests/system/metricbeat.py index 8308a9c0e765..dc80d7060656 100644 --- a/metricbeat/tests/system/metricbeat.py +++ b/metricbeat/tests/system/metricbeat.py @@ -19,8 +19,12 @@ class BaseTest(TestCase): @classmethod def setUpClass(self): - self.beat_name = "metricbeat" - self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../")) + if not hasattr(self, 'beat_name'): + self.beat_name = "metricbeat" + + if not hasattr(self, 'beat_path'): + self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../")) + super(BaseTest, self).setUpClass() def de_dot(self, existing_fields): diff --git a/x-pack/auditbeat/.gitignore b/x-pack/auditbeat/.gitignore new file mode 100644 index 000000000000..1a3313209869 --- /dev/null +++ b/x-pack/auditbeat/.gitignore @@ -0,0 +1,4 @@ +/auditbeat +/auditbeat.test +/data +/fields.yml diff --git a/x-pack/auditbeat/Makefile b/x-pack/auditbeat/Makefile new file mode 100644 index 000000000000..00efca3d7e55 --- /dev/null +++ b/x-pack/auditbeat/Makefile @@ -0,0 +1,28 @@ +BEAT_NAME=auditbeat +ES_BEATS=../.. +XPACK_BEAT_PATH?=github.com/elastic/beats/x-pack/${BEAT_NAME} +GOPACKAGES=$(shell go list ${BEAT_PATH}/... ${XPACK_BEAT_PATH}/... | grep -v /vendor/ | grep -v /scripts/cmd/ ) +FIND=. ${PYTHON_ENV}/bin/activate; find . ${ES_BEATS}/${BEAT_NAME} -type f -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*" + +# Include main auditbeat Makefile +include ${ES_BEATS}/${BEAT_NAME}/Makefile + +# Overwrite check-headers - check for Apache license in +# auditbeat/ and Elastic license in xpack/auditbeat/ +.PHONY: check-headers +check-headers: +ifndef CHECK_HEADERS_DISABLED + @go get -u github.com/elastic/go-licenser + @go-licenser -d -license ${LICENSE} ${ES_BEATS}/${BEAT_NAME} + @go-licenser -d -license Elastic . +endif + +# Overwrite check-headers - insert Apache license in +# auditbeat/ and Elastic license in xpack/auditbeat/ +.PHONY: add-headers +add-headers: +ifndef CHECK_HEADERS_DISABLED + @go get github.com/elastic/go-licenser + @go-licenser -license ${LICENSE} ${ES_BEATS}/${BEAT_NAME} + @go-licenser -license Elastic . +endif \ No newline at end of file diff --git a/x-pack/auditbeat/cache/cache.go b/x-pack/auditbeat/cache/cache.go new file mode 100644 index 000000000000..6f213559b184 --- /dev/null +++ b/x-pack/auditbeat/cache/cache.go @@ -0,0 +1,59 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cache + +// Cache is just a map being used as a cache. +type Cache struct { + hashMap map[uint64]Cacheable +} + +// Cacheable is the interface items stored in Cache need to implement. +type Cacheable interface { + Hash() uint64 +} + +// New creates a new cache. +func New() *Cache { + return &Cache{ + hashMap: make(map[uint64]Cacheable), + } +} + +// IsEmpty checks if the cache is empty. +func (cache *Cache) IsEmpty() bool { + return len(cache.hashMap) == 0 +} + +// DiffAndUpdateCache takes a list of new items to cache, compares them to the current +// cache contents, and returns both items new to the cache and items that are in the cache +// but missing in the new data. +func (cache *Cache) DiffAndUpdateCache(current []Cacheable) (new, missing []interface{}) { + // Create hashmap of incoming Cacheables to avoid calling Hash() on each one many times + currentMap := make(map[uint64]Cacheable, len(current)) + + for _, currentValue := range current { + currentMap[currentValue.Hash()] = currentValue + } + + // Check for and delete missing - what is no longer in current that was in the cache + for cacheHash, cacheValue := range cache.hashMap { + _, found := currentMap[cacheHash] + + if !found { + missing = append(missing, cacheValue) + delete(cache.hashMap, cacheHash) + } + } + + // Check for new - what is in current but not in cache + for currentHash, currentValue := range currentMap { + if _, contains := cache.hashMap[currentHash]; !contains { + new = append(new, currentValue) + cache.hashMap[currentHash] = currentValue + } + } + + return +} diff --git a/x-pack/auditbeat/cache/cache_test.go b/x-pack/auditbeat/cache/cache_test.go new file mode 100644 index 000000000000..5e921a134b58 --- /dev/null +++ b/x-pack/auditbeat/cache/cache_test.go @@ -0,0 +1,55 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cache + +import ( + "testing" + + "github.com/OneOfOne/xxhash" + "github.com/stretchr/testify/assert" +) + +type CacheTestItem struct { + s string +} + +func (item CacheTestItem) Hash() uint64 { + h := xxhash.New64() + h.WriteString(item.s) + return h.Sum64() +} + +func TestCache(t *testing.T) { + c := New() + + assert.True(t, c.IsEmpty()) + + oldItems := []Cacheable{ + CacheTestItem{"item1"}, + CacheTestItem{"item2"}, + } + + newItems := []Cacheable{ + CacheTestItem{"item1"}, + CacheTestItem{"item3"}, + } + + new, missing := c.DiffAndUpdateCache(oldItems) + + assert.Equal(t, 2, len(new)) + assert.Equal(t, 0, len(missing)) + assert.False(t, c.IsEmpty()) + + new, missing = c.DiffAndUpdateCache(newItems) + + assert.Equal(t, 1, len(new)) + assert.Equal(t, 1, len(missing)) + + new, missing = c.DiffAndUpdateCache([]Cacheable{}) + + assert.Equal(t, 0, len(new)) + assert.Equal(t, 2, len(missing)) + assert.True(t, c.IsEmpty()) +} diff --git a/x-pack/auditbeat/include/list.go b/x-pack/auditbeat/include/list.go index 1820d2ce9a0b..a75d8d536c3f 100644 --- a/x-pack/auditbeat/include/list.go +++ b/x-pack/auditbeat/include/list.go @@ -8,4 +8,6 @@ import ( // Include all Auditbeat modules so that they register their // factories with the global registry. _ "github.com/elastic/beats/x-pack/auditbeat/module/system/host" + _ "github.com/elastic/beats/x-pack/auditbeat/module/system/packages" + _ "github.com/elastic/beats/x-pack/auditbeat/module/system/processes" ) diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go new file mode 100644 index 000000000000..12b8c957711c --- /dev/null +++ b/x-pack/auditbeat/magefile.go @@ -0,0 +1,258 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build mage + +package main + +import ( + "context" + "fmt" + "regexp" + "time" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + "github.com/pkg/errors" + + "github.com/elastic/beats/dev-tools/mage" +) + +func init() { + mage.BeatDescription = "Audit the activities of users and processes on your system." +} + +// Build builds the Beat binary. +func Build() error { + return mage.Build(mage.DefaultBuildArgs()) +} + +// GolangCrossBuild build the Beat binary inside of the golang-builder. +// Do not use directly, use crossBuild instead. +func GolangCrossBuild() error { + return mage.GolangCrossBuild(mage.DefaultGolangCrossBuildArgs()) +} + +// BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon). +func BuildGoDaemon() error { + return mage.BuildGoDaemon() +} + +// CrossBuild cross-builds the beat for all target platforms. +func CrossBuild() error { + return mage.CrossBuild() +} + +// CrossBuildXPack cross-builds the beat with XPack for all target platforms. +func CrossBuildXPack() error { + return mage.CrossBuildXPack() +} + +// CrossBuildGoDaemon cross-builds the go-daemon binary using Docker. +func CrossBuildGoDaemon() error { + return mage.CrossBuildGoDaemon() +} + +// Clean cleans all generated files and build artifacts. +func Clean() error { + return mage.Clean() +} + +// Package packages the Beat for distribution. +// Use SNAPSHOT=true to build snapshots. +// Use PLATFORMS to control the target platforms. +func Package() { + start := time.Now() + defer func() { fmt.Println("package ran for", time.Since(start)) }() + + mage.UseElasticBeatPackaging() + customizePackaging() + + mg.Deps(Update) + mg.Deps(makeConfigTemplates, CrossBuild, CrossBuildXPack, CrossBuildGoDaemon) + mg.SerialDeps(mage.Package, TestPackages) +} + +// TestPackages tests the generated packages (i.e. file modes, owners, groups). +func TestPackages() error { + return mage.TestPackages() +} + +// Update updates the generated files (aka make update). +func Update() error { + return sh.Run("make", "update") +} + +// Fields generates a fields.yml for the Beat. +func Fields() error { + return mage.GenerateFieldsYAML("module", "../../auditbeat/module", "../../auditbeat/_meta/fields.common.yml") +} + +// GoTestUnit executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestUnit(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) +} + +// GoTestIntegration executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestIntegration(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) +} + +// ----------------------------------------------------------------------------- +// Customizations specific to Auditbeat. +// - Config files are Go templates. + +const ( + configTemplateGlob = "module/*/_meta/config*.yml.tmpl" + shortConfigTemplate = "build/auditbeat.yml.tmpl" + referenceConfigTemplate = "build/auditbeat.reference.yml.tmpl" +) + +func makeConfigTemplates() error { + configFiles, err := mage.FindFiles(configTemplateGlob) + if err != nil { + return errors.Wrap(err, "failed to find config templates") + } + + var shortIn []string + shortIn = append(shortIn, "_meta/common.p1.yml") + shortIn = append(shortIn, configFiles...) + shortIn = append(shortIn, "_meta/common.p2.yml") + shortIn = append(shortIn, "../libbeat/_meta/config.yml") + if !mage.IsUpToDate(shortConfigTemplate, shortIn...) { + fmt.Println(">> Building", shortConfigTemplate) + mage.MustFileConcat(shortConfigTemplate, 0600, shortIn...) + mage.MustFindReplace(shortConfigTemplate, regexp.MustCompile("beatname"), "{{.BeatName}}") + mage.MustFindReplace(shortConfigTemplate, regexp.MustCompile("beat-index-prefix"), "{{.BeatIndexPrefix}}") + } + + var referenceIn []string + referenceIn = append(referenceIn, "_meta/common.reference.yml") + referenceIn = append(referenceIn, configFiles...) + referenceIn = append(referenceIn, "../libbeat/_meta/config.reference.yml") + if !mage.IsUpToDate(referenceConfigTemplate, referenceIn...) { + fmt.Println(">> Building", referenceConfigTemplate) + mage.MustFileConcat(referenceConfigTemplate, 0644, referenceIn...) + mage.MustFindReplace(referenceConfigTemplate, regexp.MustCompile("beatname"), "{{.BeatName}}") + mage.MustFindReplace(referenceConfigTemplate, regexp.MustCompile("beat-index-prefix"), "{{.BeatIndexPrefix}}") + } + + return nil +} + +// customizePackaging modifies the package specs to use templated config files +// instead of the defaults. +// +// Customizations specific to Auditbeat: +// - Include audit.rules.d directory in packages. +func customizePackaging() { + var ( + shortConfig = mage.PackageFile{ + Mode: 0600, + Source: "{{.PackageDir}}/auditbeat.yml", + Dep: generateShortConfig, + Config: true, + } + referenceConfig = mage.PackageFile{ + Mode: 0644, + Source: "{{.PackageDir}}/auditbeat.reference.yml", + Dep: generateReferenceConfig, + } + ) + + archiveRulesDir := "audit.rules.d" + linuxPkgRulesDir := "/etc/{{.BeatName}}/audit.rules.d" + rulesSrcDir := "module/auditd/_meta/audit.rules.d" + sampleRules := mage.PackageFile{ + Mode: 0644, + Source: rulesSrcDir, + Dep: func(spec mage.PackageSpec) error { + if spec.OS == "linux" { + params := map[string]interface{}{ + "ArchBits": archBits, + } + rulesFile := spec.MustExpand(rulesSrcDir+"/sample-rules-linux-{{call .ArchBits .GOARCH}}bit.conf", params) + if err := mage.Copy(rulesFile, spec.MustExpand("{{.PackageDir}}/audit.rules.d/sample-rules.conf.disabled")); err != nil { + return errors.Wrap(err, "failed to copy sample rules") + } + } + return nil + }, + } + + for _, args := range mage.Packages { + pkgType := args.Types[0] + switch pkgType { + case mage.TarGz, mage.Zip: + args.Spec.ReplaceFile("{{.BeatName}}.yml", shortConfig) + args.Spec.ReplaceFile("{{.BeatName}}.reference.yml", referenceConfig) + case mage.Deb, mage.RPM, mage.DMG: + args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.yml", shortConfig) + args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfig) + default: + panic(errors.Errorf("unhandled package type: %v", pkgType)) + } + if args.OS == "linux" { + rulesDest := archiveRulesDir + if pkgType != mage.TarGz { + rulesDest = linuxPkgRulesDir + } + args.Spec.Files[rulesDest] = sampleRules + } + } +} + +func generateReferenceConfig(spec mage.PackageSpec) error { + params := map[string]interface{}{ + "Reference": true, + "ArchBits": archBits, + } + return spec.ExpandFile(referenceConfigTemplate, + "{{.PackageDir}}/auditbeat.reference.yml", params) +} + +func generateShortConfig(spec mage.PackageSpec) error { + params := map[string]interface{}{ + "Reference": false, + "ArchBits": archBits, + } + return spec.ExpandFile(shortConfigTemplate, + "{{.PackageDir}}/auditbeat.yml", params) +} + +// archBits returns the number of bit width of the GOARCH architecture value. +// This function is used by the auditd module configuration templates to +// generate architecture specific audit rules. +func archBits(goarch string) int { + switch goarch { + case "386", "arm": + return 32 + default: + return 64 + } +} + +// Configs generates the auditbeat.yml and auditbeat.reference.yml config files. +// Set DEV_OS and DEV_ARCH to change the target host for the generated configs. +// Defaults to linux/amd64. +func Configs() { + mg.Deps(makeConfigTemplates) + + params := map[string]interface{}{ + "GOOS": mage.EnvOr("DEV_OS", "linux"), + "GOARCH": mage.EnvOr("DEV_ARCH", "amd64"), + "ArchBits": archBits, + "Reference": false, + } + fmt.Printf(">> Building auditbeat.yml for %v/%v\n", params["GOOS"], params["GOARCH"]) + mage.MustExpandFile(shortConfigTemplate, "auditbeat.yml", params) + + params["Reference"] = true + fmt.Printf(">> Building auditbeat.reference.yml for %v/%v\n", params["GOOS"], params["GOARCH"]) + mage.MustExpandFile(referenceConfigTemplate, "auditbeat.reference.yml", params) +} diff --git a/x-pack/auditbeat/main_test.go b/x-pack/auditbeat/main_test.go new file mode 100644 index 000000000000..1eca7b4f54df --- /dev/null +++ b/x-pack/auditbeat/main_test.go @@ -0,0 +1,30 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +// This file is mandatory as otherwise the auditbeat.test binary is not generated correctly. + +import ( + "flag" + "testing" + + "github.com/elastic/beats/auditbeat/cmd" +) + +var systemTest *bool + +func init() { + systemTest = flag.Bool("systemTest", false, "Set to true when running system tests") + + cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest")) + cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile")) +} + +// Test started when the test binary is started. Only calls main. +func TestSystem(t *testing.T) { + if *systemTest { + main() + } +} diff --git a/x-pack/auditbeat/module/system/_meta/config.yml.tmpl b/x-pack/auditbeat/module/system/_meta/config.yml.tmpl index 80e133ae740b..4e00bdeb6e99 100644 --- a/x-pack/auditbeat/module/system/_meta/config.yml.tmpl +++ b/x-pack/auditbeat/module/system/_meta/config.yml.tmpl @@ -1,15 +1,17 @@ {{ if .Reference -}} {{ end -}} - module: system - {{ if eq .GOOS "darwin" -}} + metricsets: - host + - packages + - processes + + report_changes: true + + {{ if eq .GOOS "darwin" -}} {{ else if eq .GOOS "windows" -}} - metricsets: - - host {{ else -}} - metricsets: - - host {{- end }} {{ if .Reference }} {{- end }} diff --git a/x-pack/auditbeat/module/system/_meta/fields.yml b/x-pack/auditbeat/module/system/_meta/fields.yml index 36667c7b0025..b5671726ab26 100644 --- a/x-pack/auditbeat/module/system/_meta/fields.yml +++ b/x-pack/auditbeat/module/system/_meta/fields.yml @@ -1,4 +1,11 @@ - key: system - title: System - description: These are the fields generated by the system module. + title: "System" + description: > + These are the fields generated by the system module. + release: experimental fields: + - name: system + type: group + description: > + fields: + diff --git a/x-pack/auditbeat/module/system/host/_meta/data.json b/x-pack/auditbeat/module/system/host/_meta/data.json new file mode 100644 index 000000000000..0cd55dd04b46 --- /dev/null +++ b/x-pack/auditbeat/module/system/host/_meta/data.json @@ -0,0 +1,46 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "module": "system", + "name": "host", + "rtt": 115 + }, + "system": { + "host": { + "architecture": "x86_64", + "boottime": "2018-10-01T13:33:02Z", + "containerized": false, + "id": "87778e62461b4d609aee5a20f2ec4be6", + "name": "ubuntu-bionic", + "network": { + "interfaces": [ + { + "flags": "up|broadcast|multicast", + "index": 2, + "ip": [ + "10.0.2.15", + "fe80::2d:fdff:fe81:e747" + ], + "mac": "02:2d:fd:81:e7:47", + "mtu": 1500, + "name": "enp0s3" + } + ] + }, + "os": { + "family": "debian", + "kernel": "4.15.0-34-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "version": "18.04.1 LTS (Bionic Beaver)" + }, + "timezone.name": "UTC", + "timezone.offset.sec": 0, + "uptime": 222611977726 + } + } +} diff --git a/x-pack/auditbeat/module/system/host/_meta/fields.yml b/x-pack/auditbeat/module/system/host/_meta/fields.yml index 5cade6d671b6..2e9f2f479722 100644 --- a/x-pack/auditbeat/module/system/host/_meta/fields.yml +++ b/x-pack/auditbeat/module/system/host/_meta/fields.yml @@ -1,6 +1,97 @@ - name: host type: group description: > - `host` contains TODO. + `host` contains general host information. release: experimental fields: + - name: uptime + type: long + description: > + Uptime in nanoseconds. + - name: boottime + type: date + description: > + Boot time. + - name: containerized + type: boolean + description: > + Set if host is a container. + - name: timezone.name + type: keyword + description: > + Name of the timezone of the host, e.g. BST. + - name: timezone.offset.sec + type: long + description: > + Timezone offset in seconds. + - name: name + type: keyword + description: > + Hostname. + - name: id + type: keyword + description: > + Host ID. + - name: architecture + type: keyword + description: > + Host architecture (e.g. x86_64). + - name: os + type: group + description: > + `os` contains information about the operating system. + fields: + - name: platform + type: keyword + description: > + OS platform (e.g. centos, ubuntu, windows). + - name: name + type: keyword + description: > + OS name (e.g. Mac OS X). + - name: family + type: keyword + description: > + OS family (e.g. redhat, debian, freebsd, windows). + - name: version + type: keyword + description: > + OS version. + - name: kernel + type: keyword + description: > + The operating system's kernel version. + - name: network + type: group + description: > + `network` contains network information from the system. + fields: + - name: interfaces + type: array + description: > + `interfaces` contains information about network interfaces. + fields: + - name: index + type: integer + description: > + Index of the interface. + - name: mtu + type: integer + description: > + Maximum transmission unit. + - name: name + type: keyword + description: > + Interface name. + - name: mac + type: keyword + description: > + MAC address. + - name: flags + type: text + description: > + Interface flags. + - name: ip + type: ip + description: > + IP addresses. diff --git a/x-pack/auditbeat/module/system/host/host.go b/x-pack/auditbeat/module/system/host/host.go index 1b630a7e4ea5..94714d737ca0 100644 --- a/x-pack/auditbeat/module/system/host/host.go +++ b/x-pack/auditbeat/module/system/host/host.go @@ -5,11 +5,16 @@ package host import ( + "net" + "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/metricbeat/mb" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/go-sysinfo" ) const ( @@ -26,25 +31,123 @@ func init() { // MetricSet collects data about the host. type MetricSet struct { mb.BaseMetricSet + log *logp.Logger } // New constructs a new MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Experimental("The %v/%v dataset is experimental", moduleName, metricsetName) - config := defaultConfig - if err := base.Module().UnpackConfig(&config); err != nil { - return nil, errors.Wrapf(err, "failed to unpack the %v/%v config", moduleName, metricsetName) - } - - return &MetricSet{base}, nil + return &MetricSet{ + BaseMetricSet: base, + log: logp.NewLogger(moduleName), + }, nil } // Fetch collects data about the host. It is invoked periodically. func (ms *MetricSet) Fetch(report mb.ReporterV2) { + host, err := sysinfo.Host() + if err != nil { + errW := errors.Wrap(err, "Failed to load host information") + ms.log.Error(errW) + report.Error(errW) + return + } + + networkInterfaces, err := getNetworkInterfaces() + if err != nil { + errW := errors.Wrap(err, "Failed to load network interface information") + ms.log.Error(errW) + report.Error(errW) + return + } + + var networkInterfaceMapStr []common.MapStr + for _, ifc := range networkInterfaces { + networkInterfaceMapStr = append(networkInterfaceMapStr, ifc.toMapStr()) + } + report.Event(mb.Event{ - RootFields: common.MapStr{ - "hello": "world", + MetricSetFields: common.MapStr{ + // https://github.com/elastic/ecs#-host-fields + "uptime": host.Info().Uptime(), + "boottime": host.Info().BootTime, + "containerized": host.Info().Containerized, + "timezone.name": host.Info().Timezone, + "timezone.offset.sec": host.Info().TimezoneOffsetSec, + "name": host.Info().Hostname, + "id": host.Info().UniqueID, + // TODO "host.type": ? + "architecture": host.Info().Architecture, + + // https://github.com/elastic/ecs#-operating-system-fields + "os": common.MapStr{ + "platform": host.Info().OS.Platform, + "name": host.Info().OS.Name, + "family": host.Info().OS.Family, + "version": host.Info().OS.Version, + "kernel": host.Info().KernelVersion, + }, + + "network": common.MapStr{ + "interfaces": networkInterfaceMapStr, + }, }, }) } + +// NetworkInterface represent information on a network interface. +type NetworkInterface struct { + net.Interface + + ips []net.IP +} + +func (ifc NetworkInterface) toMapStr() common.MapStr { + return common.MapStr{ + "index": ifc.Index, + "mtu": ifc.MTU, + "name": ifc.Name, + "mac": ifc.HardwareAddr.String(), + "flags": ifc.Flags.String(), + "ip": ifc.ips, + } +} + +// getInterfaces fetches information about the system's network interfaces. +// TODO: Move to go-sysinfo? +func getNetworkInterfaces() ([]NetworkInterface, error) { + ifcs, err := net.Interfaces() + if err != nil { + return nil, err + } + + var networkInterfaces []NetworkInterface + + for _, ifc := range ifcs { + addrs, err := ifc.Addrs() + if err != nil { + return nil, err + } + + var ips []net.IP + for _, addr := range addrs { + ip, _, err := net.ParseCIDR(addr.String()) + if err != nil { + return nil, err + } + + ips = append(ips, ip) + } + + isLoopback := ifc.Flags&net.FlagLoopback != 0 + if !isLoopback { + networkInterfaces = append(networkInterfaces, NetworkInterface{ + ifc, + ips, + }) + } + } + + return networkInterfaces, nil +} diff --git a/x-pack/auditbeat/module/system/host/host_test.go b/x-pack/auditbeat/module/system/host/host_test.go new file mode 100644 index 000000000000..bd356379c992 --- /dev/null +++ b/x-pack/auditbeat/module/system/host/host_test.go @@ -0,0 +1,26 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package host + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + err := mbtest.WriteEventsReporterV2(f, t, "") + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "system", + "metricsets": []string{"host"}, + } +} diff --git a/x-pack/auditbeat/module/system/packages/_meta/data.json b/x-pack/auditbeat/module/system/packages/_meta/data.json new file mode 100644 index 000000000000..36bbb2fd0bac --- /dev/null +++ b/x-pack/auditbeat/module/system/packages/_meta/data.json @@ -0,0 +1,30 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "module": "system", + "name": "packages", + "rtt": 115 + }, + "system": { + "packages": { + "package": [ + { + "arch": "amd64", + "installtime": "0001-01-01T00:00:00Z", + "license": "", + "name": "vim-tiny", + "release": "", + "size": 1271, + "status": "installed", + "summary": "Vi IMproved - enhanced vi editor - compact version", + "url": "", + "version": "2:8.0.1453-1ubuntu1" + } + ] + } + } +} diff --git a/x-pack/auditbeat/module/system/packages/_meta/docs.asciidoc b/x-pack/auditbeat/module/system/packages/_meta/docs.asciidoc new file mode 100644 index 000000000000..a2f70162151f --- /dev/null +++ b/x-pack/auditbeat/module/system/packages/_meta/docs.asciidoc @@ -0,0 +1,8 @@ +The System `packages` metricset provides ... TODO. + +The module is implemented for Linux, macOS (Darwin), and Windows. + +[float] +=== Configuration options + +TODO diff --git a/x-pack/auditbeat/module/system/packages/_meta/fields.yml b/x-pack/auditbeat/module/system/packages/_meta/fields.yml new file mode 100644 index 000000000000..f57093bc150d --- /dev/null +++ b/x-pack/auditbeat/module/system/packages/_meta/fields.yml @@ -0,0 +1,50 @@ +- name: packages + type: group + description: > + `packages` contains information about installed packages. + release: experimental + fields: + - name: package + type: array + description: > + One or more packages. + fields: + - name: status + type: keyword + description: > + Package change - `new`, `installed` or `removed`. + - name: package.name + type: keyword + description: > + Package name. + - name: package.version + type: keyword + description: > + Package version. + - name: package.release + type: keyword + description: > + Package release. + - name: package.arch + type: keyword + description: > + Package architecture. + - name: package.license + type: keyword + description: > + Package license. + - name: package.installtime + type: date + description: > + Package install time. + - name: package.size + type: long + description: > + Package size. + - name: package.summary + description: > + Package summary. + - name: package.url + type: keyword + description: > + Package URL. diff --git a/x-pack/auditbeat/module/system/host/config.go b/x-pack/auditbeat/module/system/packages/config.go similarity index 77% rename from x-pack/auditbeat/module/system/host/config.go rename to x-pack/auditbeat/module/system/packages/config.go index 3cfaf464aff3..05fdce2fcb16 100644 --- a/x-pack/auditbeat/module/system/host/config.go +++ b/x-pack/auditbeat/module/system/packages/config.go @@ -2,11 +2,11 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package host +package packages // Config defines the host metricset's configuration options. type Config struct { - // TODO: Add config options. + ReportChanges bool `config:"packages.report_changes"` } // Validate validates the host metricset config. @@ -14,4 +14,6 @@ func (c *Config) Validate() error { return nil } -var defaultConfig = Config{} +var defaultConfig = Config{ + ReportChanges: true, +} diff --git a/x-pack/auditbeat/module/system/packages/packages.go b/x-pack/auditbeat/module/system/packages/packages.go new file mode 100644 index 000000000000..d73ca70ba97f --- /dev/null +++ b/x-pack/auditbeat/module/system/packages/packages.go @@ -0,0 +1,336 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package packages + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "path" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/auditbeat/cache" + "github.com/elastic/go-sysinfo/types" + + "github.com/OneOfOne/xxhash" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/go-sysinfo" +) + +const ( + moduleName = "system" + metricsetName = "packages" + + redhat = "redhat" + debian = "debian" + darwin = "darwin" +) + +func init() { + mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +// MetricSet collects data about the host. +type MetricSet struct { + mb.BaseMetricSet + config Config + osFamily string + cache *cache.Cache + log *logp.Logger +} + +// Package represents information for a package. +type Package struct { + Name string + Version string + Release string + Arch string + License string + InstallTime time.Time + Size uint64 + Summary string + URL string +} + +// Hash creates a hash for Package. +func (pkg Package) Hash() uint64 { + h := xxhash.New64() + h.WriteString(pkg.Name) + h.WriteString(pkg.InstallTime.String()) + return h.Sum64() +} + +func (pkg Package) toMapStr() common.MapStr { + return common.MapStr{ + "name": pkg.Name, + "version": pkg.Version, + "release": pkg.Release, + "arch": pkg.Arch, + "license": pkg.License, + "installtime": pkg.InstallTime, + "size": pkg.Size, + "summary": pkg.Summary, + "url": pkg.URL, + } +} + +func getOS() (*types.OSInfo, error) { + host, err := sysinfo.Host() + if err != nil { + return nil, errors.Wrap(err, "error getting the OS") + } + + hostInfo := host.Info() + if hostInfo.OS == nil { + return nil, errors.New("no host info") + } + + return hostInfo.OS, nil +} + +// New constructs a new MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Experimental("The %v/%v dataset is experimental", moduleName, metricsetName) + + config := defaultConfig + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, errors.Wrapf(err, "failed to unpack the %v/%v config", moduleName, metricsetName) + } + + ms := &MetricSet{ + BaseMetricSet: base, + config: config, + log: logp.NewLogger(moduleName), + } + + if os, err := getOS(); err == nil { + switch os.Family { + case redhat, debian, darwin: + ms.osFamily = os.Family + default: + return nil, fmt.Errorf("this metricset does not support OS family %v", os.Family) + } + } else if err != nil { + return nil, err + } + + if config.ReportChanges { + ms.cache = cache.New() + } + + return ms, nil +} + +// Fetch collects data about the host. It is invoked periodically. +func (ms *MetricSet) Fetch(report mb.ReporterV2) { + packages, err := getPackages(ms.osFamily) + if err != nil { + ms.log.Error(err) + report.Error(err) + return + } + + if ms.cache != nil && !ms.cache.IsEmpty() { + installed, removed := ms.cache.DiffAndUpdateCache(convertToCacheable(packages)) + + for _, pkgInfo := range installed { + pkgInfoMapStr := pkgInfo.(*Package).toMapStr() + pkgInfoMapStr.Put("status", "new") + + report.Event(mb.Event{ + MetricSetFields: common.MapStr{ + "package": pkgInfoMapStr, + }, + }) + } + + for _, pkgInfo := range removed { + pkgInfoMapStr := pkgInfo.(*Package).toMapStr() + pkgInfoMapStr.Put("status", "removed") + + report.Event(mb.Event{ + MetricSetFields: common.MapStr{ + "package": pkgInfoMapStr, + }, + }) + } + } else { + // Report all installed packages + var pkgInfos []common.MapStr + + for _, pkgInfo := range packages { + pkgInfoMapStr := pkgInfo.toMapStr() + pkgInfoMapStr.Put("status", "installed") + + pkgInfos = append(pkgInfos, pkgInfoMapStr) + } + + report.Event(mb.Event{ + MetricSetFields: common.MapStr{ + "package": pkgInfos, + }, + }) + + if ms.cache != nil { + // This will initialize the cache with the current packages + ms.cache.DiffAndUpdateCache(convertToCacheable(packages)) + } + } +} + +func convertToCacheable(packages []*Package) []cache.Cacheable { + c := make([]cache.Cacheable, 0, len(packages)) + + for _, p := range packages { + c = append(c, p) + } + + return c +} + +func getPackages(osFamily string) (packages []*Package, err error) { + switch osFamily { + case redhat: + // TODO: Implement RPM + err = errors.New("RPM not yet supported") + case debian: + packages, err = listDebPackages() + if err != nil { + err = errors.Wrap(err, "error getting DEB packages") + } + case darwin: + packages, err = listBrewPackages() + if err != nil { + err = errors.Wrap(err, "error getting Homebrew packages") + } + default: + panic("unknown OS - this should not have happened") + } + + return +} + +func listDebPackages() ([]*Package, error) { + const statusFile = "/var/lib/dpkg/status" + file, err := os.Open(statusFile) + if err != nil { + return nil, errors.Wrapf(err, "error opening '%s'", statusFile) + } + defer file.Close() + + var packages []*Package + pkg := &Package{} + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(strings.TrimSpace(line)) == 0 { + // empty line signals new package + packages = append(packages, pkg) + pkg = &Package{} + continue + } + if strings.HasPrefix(line, " ") { + // not interested in multi-lines for now + continue + } + words := strings.SplitN(line, ":", 2) + if len(words) != 2 { + return nil, fmt.Errorf("the following line was unexpected (no ':' found): '%s'", line) + } + value := strings.TrimSpace(words[1]) + switch strings.ToLower(words[0]) { + case "package": + pkg.Name = value + case "architecture": + pkg.Arch = value + case "version": + pkg.Version = value + case "description": + pkg.Summary = value + case "installed-size": + pkg.Size, err = strconv.ParseUint(value, 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error converting %s to int", value) + } + default: + continue + } + } + if err = scanner.Err(); err != nil { + return nil, errors.Wrap(err, "error scanning file") + } + return packages, nil +} + +func listBrewPackages() ([]*Package, error) { + const cellarPath = "/usr/local/Cellar" + + packageDirs, err := ioutil.ReadDir(cellarPath) + if os.IsNotExist(err) { + return nil, errors.Wrapf(err, "%s does not exist - is Homebrew installed?", cellarPath) + } else if err != nil { + return nil, errors.Wrapf(err, "error reading directory %s", cellarPath) + } + + var packages []*Package + for _, packageDir := range packageDirs { + if !packageDir.IsDir() { + continue + } + pkgPath := path.Join(cellarPath, packageDir.Name()) + versions, err := ioutil.ReadDir(pkgPath) + if err != nil { + return nil, errors.Wrapf(err, "error reading directory: %s", pkgPath) + } + + for _, version := range versions { + if !version.IsDir() { + continue + } + pkg := &Package{ + Name: packageDir.Name(), + Version: version.Name(), + InstallTime: version.ModTime(), + } + + // read formula + formulaPath := path.Join(cellarPath, pkg.Name, pkg.Version, ".brew", pkg.Name+".rb") + file, err := os.Open(formulaPath) + if err != nil { + //fmt.Printf("WARNING: Can't get formula for package %s-%s\n", pkg.Name, pkg.Version) + // TODO: follow the path from INSTALL_RECEIPT.json to find the formula + continue + } + scanner := bufio.NewScanner(file) + count := 15 // only look into the first few lines of the formula + for scanner.Scan() { + count-- + if count == 0 { + break + } + line := scanner.Text() + if strings.HasPrefix(line, " desc ") { + pkg.Summary = strings.Trim(line[7:], " \"") + } else if strings.HasPrefix(line, " homepage ") { + pkg.URL = strings.Trim(line[11:], " \"") + } + } + + packages = append(packages, pkg) + } + } + return packages, nil +} diff --git a/x-pack/auditbeat/module/system/packages/packages_test.go b/x-pack/auditbeat/module/system/packages/packages_test.go new file mode 100644 index 000000000000..f8078d078cc1 --- /dev/null +++ b/x-pack/auditbeat/module/system/packages/packages_test.go @@ -0,0 +1,26 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package packages + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + err := mbtest.WriteEventsReporterV2(f, t, "") + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "system", + "metricsets": []string{"packages"}, + } +} diff --git a/x-pack/auditbeat/module/system/processes/_meta/data.json b/x-pack/auditbeat/module/system/processes/_meta/data.json new file mode 100644 index 000000000000..183bc0ada59b --- /dev/null +++ b/x-pack/auditbeat/module/system/processes/_meta/data.json @@ -0,0 +1,30 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "module": "system", + "name": "processes", + "rtt": 115 + }, + "system": { + "processes": { + "process": [ + { + "args": [ + "/sbin/init" + ], + "cwd": "/", + "exe": "/lib/systemd/systemd", + "name": "systemd", + "pid": 1, + "ppid": 0, + "starttime": "2018-10-01T14:21:49.06Z", + "status": "running" + } + ] + } + } +} diff --git a/x-pack/auditbeat/module/system/processes/_meta/docs.asciidoc b/x-pack/auditbeat/module/system/processes/_meta/docs.asciidoc new file mode 100644 index 000000000000..83d4a74159f2 --- /dev/null +++ b/x-pack/auditbeat/module/system/processes/_meta/docs.asciidoc @@ -0,0 +1,8 @@ +The System `processes` metricset provides ... TODO. + +The module is implemented for Linux, macOS (Darwin), and Windows. + +[float] +=== Configuration options + +TODO diff --git a/x-pack/auditbeat/module/system/processes/_meta/fields.yml b/x-pack/auditbeat/module/system/processes/_meta/fields.yml new file mode 100644 index 000000000000..2778e2722051 --- /dev/null +++ b/x-pack/auditbeat/module/system/processes/_meta/fields.yml @@ -0,0 +1,43 @@ +- name: processes + type: group + description: > + `processes` contains process information. + release: experimental + fields: + - name: process + type: array + description: > + One or more processes. + fields: + - name: status + type: keyword + description: > + Process status - `started`, `running`, or `stopped`. + - name: process.name + type: keyword + description: > + Process name. + - name: process.args + type: text + description: > + List of process arguments. + - name: process.pid + type: keyword + description: > + Process PID. + - name: process.ppid + type: keyword + description: > + Process Parent PID. + - name: process.cwd + type: text + description: > + Directory from which the process was started. + - name: process.exe + type: text + description: > + Full path of the process executable. + - name: process.starttime + type: date + description: > + Start time of the process. diff --git a/x-pack/auditbeat/module/system/processes/config.go b/x-pack/auditbeat/module/system/processes/config.go new file mode 100644 index 000000000000..4eff02a211ef --- /dev/null +++ b/x-pack/auditbeat/module/system/processes/config.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package processes + +// Config defines the host metricset's configuration options. +type Config struct { + ReportChanges bool `config:"processes.report_changes"` +} + +// Validate validates the host metricset config. +func (c *Config) Validate() error { + return nil +} + +var defaultConfig = Config{ + ReportChanges: true, +} diff --git a/x-pack/auditbeat/module/system/processes/processes.go b/x-pack/auditbeat/module/system/processes/processes.go new file mode 100644 index 000000000000..76bf7bc24698 --- /dev/null +++ b/x-pack/auditbeat/module/system/processes/processes.go @@ -0,0 +1,187 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package processes + +import ( + "strconv" + + "github.com/OneOfOne/xxhash" + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/auditbeat/cache" + "github.com/elastic/go-sysinfo/types" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/metric/system/process" + "github.com/elastic/go-sysinfo" +) + +const ( + moduleName = "system" + metricsetName = "processes" +) + +func init() { + mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +// MetricSet collects data about the host. +type MetricSet struct { + mb.BaseMetricSet + config Config + cache *cache.Cache + log *logp.Logger +} + +// ProcessInfo wraps the process information and implements cache.Cacheable. +type ProcessInfo struct { + types.ProcessInfo +} + +// Hash creates a hash for ProcessInfo. +func (pInfo ProcessInfo) Hash() uint64 { + h := xxhash.New64() + h.WriteString(strconv.Itoa(pInfo.PID)) + h.WriteString(pInfo.StartTime.String()) + return h.Sum64() +} + +func (pInfo ProcessInfo) toMapStr() common.MapStr { + return common.MapStr{ + // https://github.com/elastic/ecs#-process-fields + "name": pInfo.Name, + "args": pInfo.Args, + "pid": pInfo.PID, + "ppid": pInfo.PPID, + "cwd": pInfo.CWD, + "exe": pInfo.Exe, + "starttime": pInfo.StartTime, + } +} + +// New constructs a new MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Experimental("The %v/%v dataset is experimental", moduleName, metricsetName) + + config := defaultConfig + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, errors.Wrapf(err, "failed to unpack the %v/%v config", moduleName, metricsetName) + } + + ms := &MetricSet{ + BaseMetricSet: base, + config: config, + log: logp.NewLogger(moduleName), + } + + if config.ReportChanges { + ms.cache = cache.New() + } + + return ms, nil +} + +// Fetch checks which processes are running on the host and reports them. +// It is invoked periodically. +func (ms *MetricSet) Fetch(report mb.ReporterV2) { + processInfos, errorList := ms.getProcessInfos() + if len(errorList) != 0 { + for _, err := range errorList { + ms.log.Error(err) + report.Error(err) + } + } + if processInfos == nil { + return + } + + if ms.cache != nil && !ms.cache.IsEmpty() { + started, stopped := ms.cache.DiffAndUpdateCache(convertToCacheable(processInfos)) + + for _, pInfo := range started { + pInfoMapStr := pInfo.(*ProcessInfo).toMapStr() + pInfoMapStr.Put("status", "started") + + report.Event(mb.Event{ + MetricSetFields: common.MapStr{ + "process": pInfoMapStr, + }, + }) + } + + for _, pInfo := range stopped { + pInfoMapStr := pInfo.(*ProcessInfo).toMapStr() + pInfoMapStr.Put("status", "stopped") + + report.Event(mb.Event{ + MetricSetFields: common.MapStr{ + "process": pInfoMapStr, + }, + }) + } + } else { + // Report all running processes + var processEvents []common.MapStr + + for _, pInfo := range processInfos { + pInfoMapStr := pInfo.toMapStr() + pInfoMapStr.Put("status", "running") + + processEvents = append(processEvents, pInfoMapStr) + } + + report.Event(mb.Event{ + MetricSetFields: common.MapStr{ + "process": processEvents, + }, + }) + + if ms.cache != nil { + // This will initialize the cache with the current processes + ms.cache.DiffAndUpdateCache(convertToCacheable(processInfos)) + } + } +} + +func convertToCacheable(processInfos []*ProcessInfo) []cache.Cacheable { + c := make([]cache.Cacheable, 0, len(processInfos)) + + for _, p := range processInfos { + c = append(c, p) + } + + return c +} + +func (ms *MetricSet) getProcessInfos() ([]*ProcessInfo, []error) { + // TODO: Implement Processes() in go-sysinfo + // e.g. https://github.com/elastic/go-sysinfo/blob/master/providers/darwin/process_darwin_amd64.go#L41 + pids, err := process.Pids() + if err != nil { + return nil, []error{errors.Wrap(err, "Failed to fetch the list of PIDs")} + } + + var processInfos []*ProcessInfo + var errorList []error + + for _, pid := range pids { + if p, err := sysinfo.Process(pid); err == nil { + if pInfo, err := p.Info(); err == nil { + processInfos = append(processInfos, &ProcessInfo{pInfo}) + } else { + errorList = append(errorList, errors.Wrap(err, "Failed to load process information")) + } + } else { + errorList = append(errorList, errors.Wrap(err, "Failed to load process")) + } + } + + return processInfos, errorList +} diff --git a/x-pack/auditbeat/module/system/processes/processes_test.go b/x-pack/auditbeat/module/system/processes/processes_test.go new file mode 100644 index 000000000000..c14c0afb9958 --- /dev/null +++ b/x-pack/auditbeat/module/system/processes/processes_test.go @@ -0,0 +1,26 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package processes + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + err := mbtest.WriteEventsReporterV2(f, t, "") + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "system", + "metricsets": []string{"processes"}, + } +} diff --git a/x-pack/auditbeat/tests/system/auditbeat_xpack.py b/x-pack/auditbeat/tests/system/auditbeat_xpack.py new file mode 100644 index 000000000000..ede9115b67ac --- /dev/null +++ b/x-pack/auditbeat/tests/system/auditbeat_xpack.py @@ -0,0 +1,55 @@ +import jinja2 +import os + +from auditbeat import BaseTest as AuditbeatTest + + +class AuditbeatXPackTest(AuditbeatTest): + + @classmethod + def setUpClass(self): + self.beat_name = "auditbeat" + self.beat_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../")) + + super(AuditbeatTest, self).setUpClass() + + def setUp(self): + super(AuditbeatTest, self).setUp() + + # Hack to make jinja2 have the right paths + self.template_env = jinja2.Environment( + loader=jinja2.FileSystemLoader([ + os.path.abspath(os.path.join(self.beat_path, "../../auditbeat")), + os.path.abspath(os.path.join(self.beat_path, "../../libbeat")) + ]) + ) + + # Adapted from metricbeat.py + def check_metricset(self, module, metricset, fields=[], warnings_allowed=False): + """ + Method to test a metricset for its fields + """ + self.render_config_template(modules=[{ + "name": module, + "metricsets": [metricset], + "period": "10s", + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + + if not warnings_allowed: + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertTrue(len(output) >= 1) + evt = output[0] + print(evt) + + flattened = self.flatten_object(evt, {}) + for f in fields: + if not f in flattened: + raise Exception("Field '{}' not found in event.".format(f)) + + self.assert_fields_are_documented(evt) diff --git a/x-pack/auditbeat/tests/system/test_metricsets.py b/x-pack/auditbeat/tests/system/test_metricsets.py new file mode 100644 index 000000000000..c7bd108ca80e --- /dev/null +++ b/x-pack/auditbeat/tests/system/test_metricsets.py @@ -0,0 +1,45 @@ +import jinja2 +import os +import sys +import time +import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../../../auditbeat/tests/system')) + +from auditbeat_xpack import * + +COMMON_FIELDS = ["@timestamp", "beat.version", "host.name", "event.module", "event.dataset"] + + +class Test(AuditbeatXPackTest): + + def test_metricset_host(self): + """ + host metricset collects general information about a server. + """ + + fields = ["system.host.uptime", "system.host.network.interfaces", "system.host.os.name"] + + # Metricset is experimental and that generates a warning, TODO: remove later + self.check_metricset("system", "host", COMMON_FIELDS + fields, warnings_allowed=True) + + def test_metricset_packages(self): + """ + packages metricset collects information about installed packages on a system. + """ + + fields = ["system.packages.package"] + + # Metricset is experimental and that generates a warning, TODO: remove later + self.check_metricset("system", "packages", COMMON_FIELDS + fields, warnings_allowed=True) + + @unittest.skipIf(sys.platform == "darwin" and os.geteuid != 0, "Requires root on macOS") + def test_metricset_processes(self): + """ + processes metricset collects information about processes running on a system. + """ + + fields = ["system.processes.process"] + + # Metricset is experimental and that generates a warning, TODO: remove later + self.check_metricset("system", "processes", COMMON_FIELDS + fields, warnings_allowed=True)