Add Splunk httpjson input to winlog package

packages/winlog/_dev/build/docs/README.md

@@ -6,5 +6,11 @@ log channels by running Get-EventLog * in PowerShell.  Custom ingest
 pipelines may be added by setting one up in
 [Ingest Node Pipelines](/app/management/ingest/ingest_pipelines/).
 
+## Configuration
+
+### Splunk Enterprise
+
+To configure Splunk Enterprise to be able to pull events from it, please visit
+[Splunk docs](https://docs.splunk.com/Documentation/SplunkCloud/latest/Data/MonitorWindowseventlogdata) for details. **The integration requires events in XML format, for this `renderXml` option needs to be set to `1` in your `inputs.conf`.**
 
 {{fields "winlog"}}

packages/winlog/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.2.0"
+  changes:
+    - description: Add Splunk httpjson input
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/integrations/pull/884
 - version: "0.1.0"
   changes:
     - description: initial release

packages/winlog/data_stream/winlog/agent/stream/httpjson.yml.hbs

@@ -0,0 +1,88 @@
+data_stream:
+  dataset: {{data_stream.dataset}}
+config_version: "2"
+interval: {{interval}}
+auth.basic.user: {{username}}
+auth.basic.password: {{password}}
+cursor:
+  index_earliest:
+    value: '[[.last_event.result.max_indextime]]'
+request.url: {{url}}/services/search/jobs/export
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+request.method: POST
+request.transforms:
+  - set:
+      target: url.params.search
+      value: |-
+        {{search}} | streamstats max(_indextime) AS max_indextime
+  - set:
+      target: url.params.output_mode
+      value: "json"
+  - set:
+      target: url.params.index_earliest
+      value: '[[ .cursor.index_earliest ]]'
+      default: '[[(now (parseDuration "-{{interval}}")).Unix]]'
+  - set:
+      target: url.params.index_latest
+      value: '[[(now).Unix]]'
+  - set:
+      target: header.Content-Type
+      value: application/x-www-form-urlencoded
+response.decode_as: application/x-ndjson
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+  - decode_json_fields:
+      fields: message
+      target: json
+      add_error_key: true
+  - drop_event:
+      when:
+        not:
+          has_fields: ['json.result']
+  - fingerprint:
+      fields:
+        - json.result._cd
+        - json.result._indextime
+        - json.result._raw
+        - json.result._time
+        - json.result.host
+        - json.result.source
+      target_field: "@metadata._id"
+  - drop_fields:
+      fields: message
+  - rename:
+      fields:
+        - from: json.result._raw
+          to: event.original
+        - from: json.result.host
+          to: host.name
+        - from: json.result.source
+          to: event.provider
+      ignore_missing: true
+      fail_on_error: false
+  - drop_fields:
+      fields: json
+  - decode_xml:
+      field: event.original
+      target_field: winlog
+      schema: wineventlog
+      ignore_missing: true
+      ignore_failure: true
+  - timestamp:
+      field: winlog.time_created
+      layouts:
+        - '2006-01-02T15:04:05Z'
+        - '2006-01-02T15:04:05.999Z'
+        - '2006-01-02T15:04:05.999-07:00'
+      test:
+        - '2019-06-22T16:33:51Z'
+        - '2019-11-18T04:59:51.123Z'
+        - '2020-08-03T07:10:20.123456+02:00'

packages/winlog/data_stream/winlog/fields/winlog.yml

@@ -293,6 +293,12 @@
       description: >
         A globally unique identifier that identifies the activity to which control was transferred to. The related events would then have this identifier as their `activity_id` identifier.
 
+    - name: time_created
+      type: date
+      required: false
+      description: >
+        The event creation time.
+
     - name: opcode
       type: keyword
       required: false

packages/winlog/data_stream/winlog/manifest.yml

@@ -40,3 +40,37 @@ streams:
           #  - drop_event.when.not.or:
           #    - equals.winlog.event_id: 903
           #    - equals.winlog.event_id: 1024
+  - input: httpjson
+    title: Windows ForwardedEvents via Splunk Enterprise REST API
+    description: Collect ForwardedEvents via Splunk Enterprise REST API
+    enabled: false
+    template_path: httpjson.yml.hbs
+    vars:
+      - name: interval
+        type: text
+        title: Interval to query Splunk Enterprise REST API
+        description: Go Duration syntax (eg. 10s)
+        show_user: true
+        required: true
+        default: 10s
+      - name: search
+        type: text
+        title: Splunk search string
+        show_user: true
+        required: true
+        default: "search sourcetype=\"XmlWinEventLog:ChannelName\""
+      - name: data_stream.dataset
+        type: text
+        title: Dataset name
+        description: >-
+          Dataset to write data to. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html).
+        default: windows_custom
+        required: true
+        show_user: true
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+        default:
+          - forwarded

packages/winlog/docs/README.md

@@ -6,6 +6,12 @@ log channels by running Get-EventLog * in PowerShell.  Custom ingest
 pipelines may be added by setting one up in
 [Ingest Node Pipelines](/app/management/ingest/ingest_pipelines/).
 
+## Configuration
+
+### Splunk Enterprise
+
+To configure Splunk Enterprise to be able to pull events from it, please visit
+[Splunk docs](https://docs.splunk.com/Documentation/SplunkCloud/latest/Data/MonitorWindowseventlogdata) for details. **The integration requires events in XML format, for this `renderXml` option needs to be set to `1` in your `inputs.conf`.**
 
 **Exported fields**
 
@@ -143,6 +149,7 @@ pipelines may be added by setting one up in
 | winlog.record_id | The record ID of the event log record. The first record written to an event log is record number 1, and other records are numbered sequentially. If the record number reaches the maximum value (2^32^ for the Event Logging API and 2^64^ for the Windows Event Log API), the next record number will be 0. | keyword |
 | winlog.related_activity_id | A globally unique identifier that identifies the activity to which control was transferred to. The related events would then have this identifier as their `activity_id` identifier. | keyword |
 | winlog.task | The task defined in the event. Task and opcode are typically used to identify the location in the application from where the event was logged. The category used by the Event Logging API (on pre Windows Vista operating systems) is written to this field. | keyword |
+| winlog.time_created | The event creation time. | date |
 | winlog.user.domain | The domain that the account associated with this event is a member of. | keyword |
 | winlog.user.identifier | The Windows security identifier (SID) of the account associated with this event. If Winlogbeat cannot resolve the SID to a name, then the `user.name`, `user.domain`, and `user.type` fields will be omitted from the event. If you discover Winlogbeat not resolving SIDs, review the log for clues as to what the problem may be. | keyword |
 | winlog.user.name | Name of the user associated with this event. | keyword |

packages/winlog/manifest.yml

@@ -4,10 +4,10 @@ title: Custom Windows event logs
 description: |-
   Collect your custom Windows event logs.
 type: integration
-version: 0.1.0
+version: 0.2.0
 release: experimental
 conditions:
-  kibana.version: '^7.12.0'
+  kibana.version: '^7.13.0'
 license: basic
 categories:
   - custom
@@ -20,6 +20,34 @@ policy_templates:
         title: Custom Windows event logs
         description: Collect events from custom Windows event log channels.
         multi: true
+      - type: httpjson
+        title: Collect logs from third-party REST API (experimental)
+        description: Collect logs from third-party REST API (experimental)
+        vars:
+          - name: url
+            type: text
+            title: URL of Splunk Enterprise Server
+            description: i.e. scheme://host:port,  path is automatic
+            show_user: true
+            required: true
+            default: https://server.example.com:8089
+          - name: username
+            type: text
+            title: Splunk REST API Username
+            show_user: true
+            required: true
+          - name: password
+            type: password
+            title: Splunk REST API Password
+            required: true
+            show_user: true
+          - name: ssl
+            type: yaml
+            title: SSL Configuration
+            description: i.e. certificate_authorities, supported_protocols, verification_mode etc.
+            multi: false
+            required: false
+            show_user: false
 icons:
   - src: "/img/logo_windows.svg"
     type: "image/svg+xml"