Manage custom Elasticsearch and Logstash pipelines in UI

salt/elasticsearch/config.sls

@@ -118,6 +118,19 @@ esingestconf:
     - user: 930
     - group: 939
 
+# Auto-generate Elasticsearch ingest node pipelines from pillar
+{% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %}
+es_ingest_conf_{{pipeline}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/ingest/{{ pipeline }}
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+      TEMPLATE_CONFIG: {{ config }}
+    - template: jinja
+    - onchanges_in:
+      - file: so-pipelines-reload
+{%     endfor %}
+
 eslog4jfile:
   file.managed:
     - name: /opt/so/conf/elasticsearch/log4j2.properties

salt/elasticsearch/defaults.yaml

@@ -55,6 +55,87 @@ elasticsearch:
             key: /usr/share/elasticsearch/config/elasticsearch.key
             verification_mode: none
   enabled: false
+  pipelines:
+    custom001:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom001
+      - pipeline:
+          name: common
+    custom002:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom002
+      - pipeline:
+          name: common
+    custom003:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom003
+      - pipeline:
+          name: common
+    custom004:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom004
+      - pipeline:
+          name: common
+    custom005:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom005
+      - pipeline:
+          name: common
+    custom006:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom006
+      - pipeline:
+          name: common
+    custom007:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom007
+      - pipeline:
+          name: common
+    custom008:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom008
+      - pipeline:
+          name: common
+    custom009:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom009
+      - pipeline:
+          name: common
+    custom010:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom010
+      - pipeline:
+          name: common
   index_settings:
     global_overrides:
       index_template:

salt/elasticsearch/soc_elasticsearch.yaml

@@ -45,6 +45,28 @@ elasticsearch:
             description: Max number of boolean clauses per query.
             global: True
             helpLink: elasticsearch.html
+  pipelines:
+    custom001: &pipelines
+      description:
+        description: Description of the ingest node pipeline
+        global: True
+        advanced: True
+        helpLink: elasticsearch.html
+      processors:
+        description: Processors for the ingest node pipeline
+        global: True
+        advanced: True
+        multiline: True
+        helpLink: elasticsearch.html
+    custom002: *pipelines
+    custom003: *pipelines
+    custom004: *pipelines
+    custom005: *pipelines
+    custom006: *pipelines
+    custom007: *pipelines
+    custom008: *pipelines
+    custom009: *pipelines
+    custom010: *pipelines
   index_settings:
     global_overrides:
       index_template:

salt/logstash/config.sls

@@ -63,6 +63,20 @@ lspipelinedir:
     - user: 931
     - group: 939
 
+# Auto-generate Logstash pipeline config
+{% for pipeline, config in LOGSTASH_MERGED.pipeline_config.items() %}
+{% for assigned_pipeline in ASSIGNED_PIPELINES %}
+{% set custom_pipeline = 'custom/' + pipeline + '.conf' %}
+{% if custom_pipeline in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
+ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}:
+  file.managed:
+    - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{ pipeline }}.conf
+    - contents: LOGSTASH_MERGED.pipeline_config.{{pipeline}}
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+
 {% for assigned_pipeline in ASSIGNED_PIPELINES %}
     {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
 ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:

salt/logstash/defaults.yaml

@@ -42,6 +42,24 @@ logstash:
     custom2: []
     custom3: []
     custom4: []
+  pipeline_config:
+    custom001: |-
+      filter {
+        if [event][module] =~ "zeek" {
+          mutate {
+            add_tag => ["network_stuff"]
+          }
+        }
+      }
+    custom002: PLACEHOLDER
+    custom003: PLACEHOLDER
+    custom004: PLACEHOLDER
+    custom005: PLACEHOLDER
+    custom006: PLACEHOLDER
+    custom007: PLACEHOLDER
+    custom008: PLACEHOLDER
+    custom009: PLACEHOLDER
+    custom010: PLACEHOLDER
   settings:
     lsheap: 500m
   config:

salt/logstash/soc_logstash.yaml

@@ -31,6 +31,22 @@ logstash:
     custom2: *defined_pipelines
     custom3: *defined_pipelines
     custom4: *defined_pipelines
+  pipeline_config:
+    custom001: &pipeline_config
+      description: Pipeline configuration for Logstash
+      advanced: True
+      multiline: True
+      forcedType: string
+      helpLink: logstash.html
+    custom002: *pipeline_config
+    custom003: *pipeline_config
+    custom004: *pipeline_config
+    custom005: *pipeline_config
+    custom006: *pipeline_config
+    custom007: *pipeline_config
+    custom008: *pipeline_config
+    custom009: *pipeline_config
+    custom010: *pipeline_config
   settings:
     lsheap: 
       description: Heap size to use for logstash