[Auditbeat] Add system module

.travis.yml

@@ -62,6 +62,10 @@ jobs:
       env: TARGETS="-C auditbeat crosscompile"
       go: $GO_VERSION
       stage: test
+    - os: linux
+      env: TARGETS="-C x-pack/auditbeat testsuite"
+      go: $GO_VERSION
+      stage: test
 
     # Libbeat
     - os: linux

CHANGELOG.asciidoc

@@ -97,6 +97,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 
 *Auditbeat*
 
+- Add system module. {pull}9546[9546]
+
 *Filebeat*
 
 - Added `detect_null_bytes` selector to detect null bytes from a io.reader. {pull}9210[9210]

Makefile

@@ -13,11 +13,15 @@ REVIEWDOG_OPTIONS?=-diff "git diff master"
 REVIEWDOG_REPO=github.com/haya14busa/reviewdog/cmd/reviewdog
 XPACK_SUFFIX=x-pack/
 
+# PROJECTS_XPACK_PKG is a list of Beats that have independent packaging support
+# in the x-pack directory (rather than having the OSS build produce both sets
+# of artifacts). This will be removed once we complete the transition.
+PROJECTS_XPACK_PKG=x-pack/auditbeat
 # PROJECTS_XPACK_MAGE is a list of Beats whose primary build logic is based in
 # Mage. For compatibility with CI testing these projects support a subset of the
 # makefile targets. After all Beats converge to primarily using Mage we can
 # remove this and treat all sub-projects the same.
-PROJECTS_XPACK_MAGE=x-pack/filebeat x-pack/metricbeat
+PROJECTS_XPACK_MAGE=x-pack/filebeat x-pack/metricbeat $(PROJECTS_XPACK_PKG)
 
 # Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection.
 # Also it builds the docs and the generators
@@ -156,8 +160,8 @@ snapshot:
 # Builds a release.
 .PHONY: release
 release: beats-dashboards
-	@$(foreach var,$(BEATS),$(MAKE) -C $(var) release || exit 1;)
-	@$(foreach var,$(BEATS), \
+	@$(foreach var,$(BEATS) $(PROJECTS_XPACK_PKG),$(MAKE) -C $(var) release || exit 1;)
+	@$(foreach var,$(BEATS) $(PROJECTS_XPACK_PKG), \
       test -d $(var)/build/distributions && test -n "$$(ls $(var)/build/distributions)" || exit 0; \
       mkdir -p build/distributions/$(subst $(XPACK_SUFFIX),'',$(var)) && mv -f $(var)/build/distributions/* build/distributions/$(subst $(XPACK_SUFFIX),'',$(var))/ || exit 1;)
 

Vagrantfile

@@ -37,7 +37,7 @@ cmd /c mklink /d C:\\Gopath\\src\\github.com\\elastic\\beats \\\\vboxsvr\\vagran
 
 echo "Installing gvm to manage go version"
 [Net.ServicePointManager]::SecurityProtocol = "tls12"
-Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.0.5/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
+Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.1.0/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
 C:\Windows\System32\gvm.exe --format=powershell #{GO_VERSION} | Invoke-Expression
 go version
 
@@ -72,8 +72,9 @@ SCRIPT
 $linuxGvmProvision = <<SCRIPT
 mkdir -p ~/bin
 if [ ! -e "~/bin/gvm" ]; then
-  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.0.5/gvm-linux-amd64
+  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.1.0/gvm-linux-amd64
   chmod +x ~/bin/gvm
+  ~/bin/gvm $GO_VERSION
   echo 'export GOPATH=$HOME/go' >> ~/.bash_profile
   echo 'export PATH=$HOME/bin:$GOPATH/bin:$PATH' >> ~/.bash_profile
   echo 'eval "$(gvm #{GO_VERSION})"' >> ~/.bash_profile

auditbeat/Dockerfile

@@ -1,17 +1,12 @@
 FROM golang:1.11.3
-MAINTAINER Nicolas Ruflin <ruflin@elastic.co>
 
-RUN set -x && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends \
-         netcat python-pip virtualenv && \
-    apt-get clean
+RUN \
+    apt-get update \
+      && apt-get install -y --no-install-recommends \
+         python-pip \
+         virtualenv \
+      && rm -rf /var/lib/apt/lists/*
 
+RUN pip install --upgrade pip
 RUN pip install --upgrade setuptools
-
-# Setup work environment
-ENV AUDITBEAT_PATH /go/src/github.com/elastic/beats/auditbeat
-
-RUN mkdir -p $AUDITBEAT_PATH/build/coverage
-WORKDIR $AUDITBEAT_PATH
-HEALTHCHECK CMD exit 0
+RUN pip install --upgrade docker-compose==1.21.0

auditbeat/Makefile

@@ -3,36 +3,12 @@ BEAT_TITLE=Auditbeat
 SYSTEM_TESTS=true
 TEST_ENVIRONMENT?=true
 GOX_OS?=linux windows ## @Building List of all OS to be supported by "make crosscompile".
-DEV_OS?=linux
 ES_BEATS?=..
+EXCLUDE_COMMON_UPDATE_TARGET=true
 
 # Path to the libbeat Makefile
 include ${ES_BEATS}/libbeat/scripts/Makefile
 
-# Collects all dependencies and then calls update
-.PHONY: collect
-collect: collect-docs configs kibana
-
-# Collects all module configs
-.PHONY: configs
-configs: python-env
-	@cat ${ES_BEATS}/auditbeat/_meta/common.p1.yml \
-		<(go run scripts/generate_config.go -os ${DEV_OS} -concat) \
-		${ES_BEATS}/auditbeat/_meta/common.p2.yml > _meta/beat.yml
-	@cat ${ES_BEATS}/auditbeat/_meta/common.reference.yml \
-		<(go run scripts/generate_config.go -os ${DEV_OS} -ref -concat) > _meta/beat.reference.yml
-
-# Collects all module docs
-.PHONY: collect-docs
-collect-docs: python-env
-	@rm -rf docs/modules
-	@mkdir -p docs/modules
-	@go run scripts/generate_config.go -os linux
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/auditbeat/scripts/docs_collector.py --beat ${BEAT_NAME}
-
-# Collects all module dashboards
-.PHONY: kibana
-kibana:
-	@-rm -rf _meta/kibana.generated
-	@mkdir -p _meta/kibana.generated
-	@-cp -pr module/*/_meta/kibana/* _meta/kibana.generated
+.PHONY: update
+update: mage
+	mage update

auditbeat/auditbeat.yml

@@ -48,7 +48,6 @@ auditbeat.modules:
   - /etc
 
 
-
 #==================== Elasticsearch template setting ==========================
 setup.template.settings:
   index.number_of_shards: 3

auditbeat/core/eventmod.go

@@ -30,4 +30,10 @@ func AddDatasetToEvent(module, metricSet string, event *mb.Event) {
 	}
 
 	event.RootFields.Put("event.module", module)
+
+	// Modules without "datasets" should set their module and metricset names
+	// to the same value then this will omit the event.dataset field.
+	if module != metricSet {
+		event.RootFields.Put("event.dataset", metricSet)
+	}
 }

auditbeat/docker-compose.yml

@@ -4,10 +4,12 @@ services:
     build: ${PWD}/.
     depends_on:
       - proxy_dep
-    env_file:
-      - ${PWD}/build/test.env
     working_dir: /go/src/github.com/elastic/beats/auditbeat
     environment:
+      - ES_HOST=elasticsearch
+      - ES_PORT=9200
+      - ES_USER=beats
+      - ES_PASS=testing
       - KIBANA_HOST=kibana
       - KIBANA_PORT=5601
     volumes: