[Auditbeat] Backport #9546 to 6.x: Add system module

.travis.yml

@@ -58,6 +58,10 @@ jobs:
       env: TARGETS="-C auditbeat crosscompile"
       go: $GO_VERSION
       stage: test
+    - os: linux
+      env: TARGETS="-C x-pack/auditbeat testsuite"
+      go: $GO_VERSION
+      stage: test
 
     # Libbeat
     - os: linux

CHANGELOG.asciidoc

@@ -101,6 +101,8 @@ https://github.com/elastic/beats/compare/v6.5.0...6.x[Check the HEAD diff]
 
 *Auditbeat*
 
+- Add system module. {pull}9546[9546]
+
 *Filebeat*
 - Added `detect_null_bytes` selector to detect null bytes from a io.reader. {pull}9210[9210]
 - Added `syslog_host` variable to HAProxy module to allow syslog listener to bind to configured host. {pull}9366[9366]

Makefile

@@ -13,6 +13,11 @@ REVIEWDOG_OPTIONS?=-diff "git diff master"
 REVIEWDOG_REPO=github.com/haya14busa/reviewdog/cmd/reviewdog
 XPACK_SUFFIX=x-pack/
 
+# PROJECTS_XPACK_PKG is a list of Beats that have independent packaging support
+# in the x-pack directory (rather than having the OSS build produce both sets
+# of artifacts). This will be removed once we complete the transition.
+PROJECTS_XPACK_PKG=x-pack/auditbeat
+
 # Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection.
 # Also it builds the docs and the generators
 
@@ -150,8 +155,8 @@ snapshot:
 # Builds a release.
 .PHONY: release
 release: beats-dashboards
-	@$(foreach var,$(BEATS),$(MAKE) -C $(var) release || exit 1;)
-	@$(foreach var,$(BEATS), \
+	@$(foreach var,$(BEATS) $(PROJECTS_XPACK_PKG),$(MAKE) -C $(var) release || exit 1;)
+	@$(foreach var,$(BEATS) $(PROJECTS_XPACK_PKG), \
       test -d $(var)/build/distributions && test -n "$$(ls $(var)/build/distributions)" || exit 0; \
       mkdir -p build/distributions/$(subst $(XPACK_SUFFIX),'',$(var)) && mv -f $(var)/build/distributions/* build/distributions/$(subst $(XPACK_SUFFIX),'',$(var))/ || exit 1;)
 

Vagrantfile

@@ -37,7 +37,7 @@ cmd /c mklink /d C:\\Gopath\\src\\github.com\\elastic\\beats \\\\vboxsvr\\vagran
 
 echo "Installing gvm to manage go version"
 [Net.ServicePointManager]::SecurityProtocol = "tls12"
-Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.0.5/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
+Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.1.0/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
 C:\Windows\System32\gvm.exe --format=powershell #{GO_VERSION} | Invoke-Expression
 go version
 
@@ -72,8 +72,9 @@ SCRIPT
 $linuxGvmProvision = <<SCRIPT
 mkdir -p ~/bin
 if [ ! -e "~/bin/gvm" ]; then
-  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.0.5/gvm-linux-amd64
+  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.1.0/gvm-linux-amd64
   chmod +x ~/bin/gvm
+  ~/bin/gvm $GO_VERSION
   echo 'export GOPATH=$HOME/go' >> ~/.bash_profile
   echo 'export PATH=$HOME/bin:$GOPATH/bin:$PATH' >> ~/.bash_profile
   echo 'eval "$(gvm #{GO_VERSION})"' >> ~/.bash_profile

auditbeat/Dockerfile

@@ -1,17 +1,13 @@
 FROM golang:1.10.6
 MAINTAINER Nicolas Ruflin <ruflin@elastic.co>
 
-RUN set -x && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends \
-         netcat python-pip virtualenv && \
-    apt-get clean
-
+RUN \
+    apt-get update \
+      && apt-get install -y --no-install-recommends \
+         python-pip \
+         virtualenv \
+      && rm -rf /var/lib/apt/lists/*
+
+RUN pip install --upgrade pip
 RUN pip install --upgrade setuptools
-
-# Setup work environment
-ENV AUDITBEAT_PATH /go/src/github.com/elastic/beats/auditbeat
-
-RUN mkdir -p $AUDITBEAT_PATH/build/coverage
-WORKDIR $AUDITBEAT_PATH
-HEALTHCHECK CMD exit 0
+RUN pip install --upgrade docker-compose==1.21.0

auditbeat/Makefile

@@ -3,36 +3,12 @@ BEAT_TITLE=Auditbeat
 SYSTEM_TESTS=true
 TEST_ENVIRONMENT?=true
 GOX_OS?=linux windows ## @Building List of all OS to be supported by "make crosscompile".
-DEV_OS?=linux
 ES_BEATS?=..
+EXCLUDE_COMMON_UPDATE_TARGET=true
 
 # Path to the libbeat Makefile
 include ${ES_BEATS}/libbeat/scripts/Makefile
 
-# Collects all dependencies and then calls update
-.PHONY: collect
-collect: collect-docs configs kibana
-
-# Collects all module configs
-.PHONY: configs
-configs: python-env
-	@cat ${ES_BEATS}/auditbeat/_meta/common.p1.yml \
-		<(go run scripts/generate_config.go -os ${DEV_OS} -concat) \
-		${ES_BEATS}/auditbeat/_meta/common.p2.yml > _meta/beat.yml
-	@cat ${ES_BEATS}/auditbeat/_meta/common.reference.yml \
-		<(go run scripts/generate_config.go -os ${DEV_OS} -ref -concat) > _meta/beat.reference.yml
-
-# Collects all module docs
-.PHONY: collect-docs
-collect-docs: python-env
-	@rm -rf docs/modules
-	@mkdir -p docs/modules
-	@go run scripts/generate_config.go -os linux
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/auditbeat/scripts/docs_collector.py --beat ${BEAT_NAME}
-
-# Collects all module dashboards
-.PHONY: kibana
-kibana:
-	@-rm -rf _meta/kibana.generated
-	@mkdir -p _meta/kibana.generated
-	@-cp -pr module/*/_meta/kibana/* _meta/kibana.generated
+.PHONY: update
+update: mage
+	mage update

auditbeat/_meta/fields.common.yml

@@ -7,6 +7,10 @@
     description: >
       The name of the module that generated the event.
 
+  - name: event.dataset
+    description: >
+      The name of the module's dataset that generated the event.
+
   - name: event.action
     type: keyword
     example: logged-in
@@ -16,6 +20,85 @@
       For the file integrity module the possible values are:
       attributes_modified, created, deleted, updated, moved, and config_change.
 
+  - name: event.id
+    type: keyword
+    description: >
+      Unique ID to describe the event.
+    example: 8a4f500d
+
+  - name: event.kind
+    type: keyword
+    description: >
+      The kind of the event.
+
+      This gives information about what type of information the event
+      contains, without being specific to the contents of the event.  Examples
+      are `event`, `state`, `alarm`. Warning: In future versions of ECS, we
+      plan to provide a list of acceptable values for this field, please use
+      with caution.
+    example: state
+
+  - name: message
+    type: text
+    example: "Hello World"
+    description: >
+      For log events the message field contains the log message.
+
+      In other use cases the message field can be used to concatenate
+      different values which are then freely searchable. If multiple
+      messages exist, they can be combined into one message.
+
+  - name: process
+    title: Process
+    group: 2
+    description: >
+      These fields contain information about a process.
+      These fields can help you correlate metrics information with a process id/name
+      from a log message.  The `process.pid` often stays in the metric itself and is
+      copied to the global field for correlation.
+    type: group
+    fields:
+      - name: start
+        type: date
+        example: "2016-05-23T08:05:34.853Z"
+        description: >
+          The time the process started.
+      - name: working_directory
+        type: keyword
+        example: /home/alice
+        description: >
+          The working directory of the process.
+      - name: executable
+        type: keyword
+        description: >
+          Absolute path to the process executable.
+        example: /usr/bin/ssh
+
+  - name: network.type
+    type: keyword
+    description: >
+      In the OSI Model this would be the Network Layer. IPv4, IPv6, IPSec, PIM, etc
+    example: IPv4
+
+  - name: user
+    title: User
+    description: >
+      The user fields describe information about the user that is relevant
+      to  the event. Fields can have one entry or multiple entries. If a
+      user has more than one id, provide an array that includes all of
+      them.
+    type: group
+    fields:
+      - name: id
+        type: keyword
+        description: >
+          One or multiple unique identifiers of the user.
+      - name: name
+        type: keyword
+        example: albert
+        description: >
+          Short name or login of the user.
+
   - name: file
     type: group
     description: File attributes.

auditbeat/auditbeat.yml

@@ -48,7 +48,6 @@ auditbeat.modules:
   - /etc
 
 
-
 #==================== Elasticsearch template setting ==========================
 setup.template.settings:
   index.number_of_shards: 3

auditbeat/core/eventmod.go

@@ -30,4 +30,10 @@ func AddDatasetToEvent(module, metricSet string, event *mb.Event) {
 	}
 
 	event.RootFields.Put("event.module", module)
+
+	// Modules without "datasets" should set their module and metricset names
+	// to the same value then this will omit the event.dataset field.
+	if module != metricSet {
+		event.RootFields.Put("event.dataset", metricSet)
+	}
 }

auditbeat/docker-compose.yml

@@ -4,10 +4,12 @@ services:
     build: ${PWD}/.
     depends_on:
       - proxy_dep
-    env_file:
-      - ${PWD}/build/test.env
     working_dir: /go/src/github.com/elastic/beats/auditbeat
     environment:
+      - ES_HOST=elasticsearch
+      - ES_PORT=9200
+      - ES_USER=beats
+      - ES_PASS=testing
       - KIBANA_HOST=kibana
       - KIBANA_PORT=5601
     volumes: