Add File Integrity Monitoring package

.github/CODEOWNERS

@@ -54,6 +54,7 @@
 /packages/elasticsearch @elastic/infra-monitoring-ui
 /packages/etcd @elastic/integrations
 /packages/f5 @elastic/security-external-integrations
+/packages/fim @elastic/security-external-integrations
 /packages/fireeye @elastic/security-external-integrations
 /packages/fleet_server @elastic/elastic-agent-control-plane
 /packages/fortinet @elastic/security-external-integrations

packages/fim/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@8.2

packages/fim/_dev/build/docs/README.md

@@ -0,0 +1,26 @@
+# File Integrity Monitoring Integration
+
+This integration sends events when a file is changed (created, updated, or deleted) on disk. The events contain file metadata and hashes.
+
+The integration is implemented for Linux, macOS (Darwin), and Windows.
+
+
+| ⚠️ This integration should not be used to monitor paths on network file systems. |
+| ---- |
+
+## How it works
+
+This integration uses features of the operating system to monitor file changes in realtime. When the integration starts it creates a subscription with the OS to receive notifications of changes to the specified files or directories. Upon receiving notification of a change the integration will read the file’s metadata and then compute a hash of the file’s contents.
+
+At startup this integration will perform an initial scan of the configured files and directories to generate baseline data for the monitored paths and detect changes since the last time it was run. It uses locally persisted data in order to only send events for new or modified files.
+
+## Compatibility
+
+The operating system features that power this feature are as follows:
+- **Linux** - inotify is used, and therefore the kernel must have inotify support. Inotify was initially merged into the 2.6.13 Linux kernel.
+- **macOS (Darwin)** - Uses the FSEvents API, present since macOS 10.5. This API coalesces multiple changes to a file into a single event. Auditbeat translates this coalesced changes into a meaningful sequence of actions. However, in rare situations the reported events may have a different ordering than what actually happened.
+- **Windows** - ReadDirectoryChangesW is used.
+
+{{ event "event" }}
+
+{{ fields "event" }}

packages/fim/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3.0'
+services:
+  fim_events_generator:
+    image: alpine
+    volumes:
+      - ./:/pwd/
+      - ${SERVICE_LOGS_DIR}:/shared/
+    command:
+      - /pwd/run.sh

packages/fim/_dev/deploy/docker/run.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+log() {
+  echo "$@" >&2
+}
+signal_fired() {
+  done=1
+}
+wait_for_signal() {
+  while test $done -eq 0
+  do
+    sleep 1
+  done
+}
+done=0
+trap signal_fired HUP
+log Waiting for signal
+wait_for_signal
+log Performing test actions
+echo hello world > shared/hello
+sleep 2
+rm shared/hello
+touch shared/done

packages/fim/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.1.0"
+  changes:
+    - description: Initial version
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3143

packages/fim/data_stream/event/_dev/test/system/test-common-config.yml

@@ -0,0 +1,9 @@
+service: fim_events_generator
+service_notify_signal: SIGHUP
+input: audit/file_integrity
+wait_for_data_timeout: 2m
+vars:
+  paths:
+    - "{{SERVICE_LOGS_DIR}}"
+  recursive: true
+  scan_at_start: false

packages/fim/data_stream/event/agent/stream/file_integrity.yml.hbs

@@ -0,0 +1,33 @@
+type: audit/file_integrity
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+recursive: {{recursive}}
+scan_at_start: {{scan_at_start}}
+hash_types:
+{{#each hash_types as |hash i|}}
+  - {{hash}}
+{{/each}}
+max_file_size: {{max_file_size}}
+scan_rate_per_sec: {{scan_rate_per_sec}}
+exclude_files:
+{{#each exclude_files as |expr i|}}
+  - {{expr}}
+{{/each}}
+include_files:
+{{#each include_files as |expr i|}}
+  - {{expr}}
+{{/each}}
+keep_null: {{keep_null}}
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/fim/data_stream/event/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,10 @@
+---
+description: Pipeline for processing auditd events
+processors:
+  - set:
+      field: ecs.version
+      value: 8.2.0
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

packages/fim/data_stream/event/fields/agent.yml

@@ -0,0 +1,204 @@
+- name: cloud
+  title: Cloud
+  group: 2
+  description: Fields related to the cloud or infrastructure the events are coming from.
+  footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.'
+  type: group
+  fields:
+    - name: account.id
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
+
+        Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.'
+      example: 666777888999
+    - name: availability_zone
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Availability zone in which this host is running.
+      example: us-east-1c
+    - name: instance.id
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Instance ID of the host machine.
+      example: i-1234567890abcdef0
+    - name: instance.name
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Instance name of the host machine.
+    - name: machine.type
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Machine type of the host machine.
+      example: t2.medium
+    - name: provider
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
+      example: aws
+    - name: region
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Region in which this host is running.
+      example: us-east-1
+    - name: project.id
+      type: keyword
+      description: Name of the project in Google Cloud.
+    - name: image.id
+      type: keyword
+      description: Image ID for the cloud instance.
+- name: container
+  title: Container
+  group: 2
+  description: 'Container fields are used for meta information about the specific container that is the source of information.
+
+    These fields help correlate data based containers from any runtime.'
+  type: group
+  fields:
+    - name: id
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: Unique container id.
+    - name: image.name
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Name of the image the container was built on.
+    - name: labels
+      level: extended
+      type: object
+      object_type: keyword
+      description: Image labels.
+    - name: name
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Container name.
+- name: host
+  title: Host
+  group: 2
+  description: 'A host is defined as a general computing instance.
+
+    ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.'
+  type: group
+  fields:
+    - name: architecture
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: Operating system architecture.
+      example: x86_64
+    - name: domain
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'Name of the domain of which the host is a member.
+
+        For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.'
+      example: CONTOSO
+      default_field: false
+    - name: hostname
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: 'Hostname of the host.
+
+        It normally contains what the `hostname` command returns on the host machine.'
+    - name: id
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: 'Unique host id.
+
+        As hostname is not always unique, use values that are meaningful in your environment.
+
+        Example: The current usage of `beat.name`.'
+    - name: ip
+      level: core
+      type: ip
+      description: Host ip addresses.
+    - name: mac
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: Host mac addresses.
+    - name: name
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: 'Name of the host.
+
+        It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.'
+    - name: os.family
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: OS family (such as redhat, debian, freebsd, windows).
+      example: debian
+    - name: os.kernel
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Operating system kernel version as a raw string.
+      example: 4.4.0-112-generic
+    - name: os.name
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      multi_fields:
+        - name: text
+          type: text
+          norms: false
+          default_field: false
+      description: Operating system name, without the version.
+      example: Mac OS X
+    - name: os.platform
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Operating system platform (such centos, ubuntu, windows).
+      example: darwin
+    - name: os.version
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Operating system version as a raw string.
+      example: 10.14.1
+    - name: type
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: 'Type of host.
+
+        For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.'
+    - name: containerized
+      type: boolean
+      description: >
+        If the host is a container.
+
+    - name: os.build
+      type: keyword
+      example: "18D109"
+      description: >
+        OS build information.
+
+    - name: os.codename
+      type: keyword
+      example: "stretch"
+      description: >
+        OS codename, if any.
+
+- name: input.type
+  type: keyword
+  description: Input type
+- name: log.offset
+  type: long
+  description: Log offset

packages/fim/data_stream/event/fields/base-fields.yml

@@ -0,0 +1,25 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: event.module
+  type: constant_keyword
+  description: Event module
+  value: file_integrity
+- name: event.dataset
+  type: constant_keyword
+  description: Event dataset
+  value: fim.event
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.
+- name: tags
+  description: List of keywords used to tag each event.
+  example: '["production", "env2"]'
+  ignore_above: 1024
+  type: keyword