Generator for a Filebeat module/fileset (#3248)

Following the model of the Metricbeat module generator, this creates
templates for most of the required files of a fileset, making it
easier to get started.

To use, run as:

MODULE=apache FILESET=access make create-fileset

The PR includes some minor cleanups to the Metricbeat version.

Part of #3159.

filebeat/Makefile

@@ -31,3 +31,10 @@ fields:
 # Runs all collection steps and updates afterwards
 .PHONY: collect
 collect: fields kibana
+
+
+# Creates a new fileset. Requires the params MODULE and FILESET
+.PHONY: create-fileset
+create-fileset: python-env
+	. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/filebeat/scripts/create_fileset.py --path=$(PWD) --es_beats=$(ES_BEATS) --module=$(MODULE) --fileset=$(FILESET)
+

filebeat/scripts/create_fileset.py

@@ -0,0 +1,103 @@
+import os
+import argparse
+
+# Creates a new fileset with all the necessary files.
+# In case the module does not exist, also the module is created.
+
+
+def generate_fileset(base_path, metricbeat_path, module, fileset):
+
+    generate_module(base_path, metricbeat_path, module, fileset)
+    fileset_path = base_path + "/module/" + module + "/" + fileset
+    meta_path = fileset_path + "/_meta"
+
+    if os.path.isdir(fileset_path):
+        print("Fileset already exists. Skipping creating fileset {}"
+              .format(fileset))
+        return
+
+    os.makedirs(meta_path)
+    os.makedirs(os.path.join(fileset_path, "test"))
+
+    templates = metricbeat_path + "/scripts/module/fileset/"
+
+    content = load_file(templates + "fields.yml", module, fileset)
+    with open(meta_path + "/fields.yml", "w") as f:
+        f.write(content)
+
+    os.makedirs(os.path.join(fileset_path, "config"))
+    content = load_file(templates + "/config/config.yml", module, fileset)
+    with open("{}/config/{}.yml".format(fileset_path, fileset), "w") as f:
+        f.write(content)
+
+    os.makedirs(os.path.join(fileset_path, "ingest"))
+    content = load_file(templates + "/ingest/pipeline.json", module, fileset)
+    with open("{}/ingest/pipeline.json".format(fileset_path), "w") as f:
+        f.write(content)
+
+    content = load_file(templates + "/manifest.yml", module, fileset)
+    with open("{}/manifest.yml".format(fileset_path), "w") as f:
+        f.write(content)
+
+    print("Fileset {} created.".format(fileset))
+
+
+def generate_module(base_path, metricbeat_path, module, fileset):
+
+    module_path = base_path + "/module/" + module
+    meta_path = module_path + "/_meta"
+
+    if os.path.isdir(module_path):
+        print("Module already exists. Skipping creating module {}"
+              .format(module))
+        return
+
+    os.makedirs(meta_path)
+
+    templates = metricbeat_path + "/scripts/module/"
+
+    content = load_file(templates + "fields.yml", module, "")
+    with open(meta_path + "/fields.yml", "w") as f:
+        f.write(content)
+
+    print("Module {} created.".format(module))
+
+
+def load_file(file, module, fileset):
+    content = ""
+    with open(file) as f:
+        content = f.read()
+
+    return content.replace("{module}", module).replace("{fileset}", fileset)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Creates a fileset")
+    parser.add_argument("--module", help="Module name")
+    parser.add_argument("--fileset", help="Fileset name")
+
+    parser.add_argument("--path", help="Beat path")
+    parser.add_argument("--es_beats",
+                        help="The path to the general beats folder")
+
+    args = parser.parse_args()
+
+    if args.path is None:
+        args.path = './'
+        print "Set default path for beat path: " + args.path
+
+    if args.es_beats is None:
+        args.es_beats = '../'
+        print "Set default path for es_beats path: " + args.es_beats
+
+    if args.module is None or args.module == '':
+        args.module = raw_input("Module name: ")
+
+    if args.fileset is None or args.fileset == '':
+        args.fileset = raw_input("Fileset name: ")
+
+    path = os.path.abspath(args.path)
+    filebeat_path = os.path.abspath(args.es_beats + "/filebeat")
+
+    generate_fileset(path, filebeat_path, args.module.lower(),
+                     args.fileset.lower())

filebeat/scripts/module/fields.yml

@@ -0,0 +1,9 @@
+- key: {module}
+  title: "{module}"
+  description: >
+    {module} Module
+  fields:
+    - name: {module}
+      type: group
+      description: >
+      fields:

filebeat/scripts/module/fileset/config/config.yml

@@ -0,0 +1,9 @@
+- input_type: log
+  paths:
+  {%- for path in paths %}
+   - {{path}}
+  {%- endfor %}
+  exclude_files: [".gz$"]
+  fields:
+    source_type: {module}-{fileset}
+    pipeline_id: {{beat.pipeline_id}}

filebeat/scripts/module/fileset/fields.yml

@@ -0,0 +1,9 @@
+- name: {fileset}
+  type: group
+  description: >
+    {fileset}
+  fields:
+    - name: example
+      type: keyword
+      description: >
+        Example field

filebeat/scripts/module/fileset/ingest/pipeline.json

@@ -0,0 +1,11 @@
+{
+  "description": "Pipeline for parsing {module} {fileset} logs",
+  "processors": [
+    ],
+  "on_failure" : [{
+    "set" : {
+      "field" : "error",
+      "value" : "{{ _ingest.on_failure_message }}"
+    }
+  }]
+}

filebeat/scripts/module/fileset/manifest.yml

@@ -0,0 +1,14 @@
+module_version: 1.0
+
+vars:
+  paths:
+    default:
+      - /example/path*
+    os.darwin:
+      - /usr/local/example/path*
+    os.windows:
+      - "c:/example/path*"
+
+ingest_pipeline: ingest/pipeline.json
+prospectors:
+  - config/{fileset}.yml

metricbeat/scripts/create_metricset.py

@@ -12,14 +12,14 @@ def generate_metricset(base_path, metricbeat_path, module, metricset):
     meta_path = metricset_path + "/_meta"
 
     if os.path.isdir(metricset_path):
-        print "MericSet already exists. Skipping creating metricset " + metricset + ".\n"
+        print("Metricset already exists. Skipping creating metricset {}"
+              .format(metricset))
         return
 
     os.makedirs(meta_path)
 
     templates = metricbeat_path + "/scripts/module/metricset/"
 
-
     content = load_file(templates + "metricset.go.tmpl", module, metricset)
     with open(metricset_path + "/" + metricset + ".go", "w") as f:
         f.write(content)
@@ -36,7 +36,7 @@ def generate_metricset(base_path, metricbeat_path, module, metricset):
     with open(meta_path + "/data.json", "w") as f:
         f.write(content)
 
-    print "Metricset " + metricset + " created."
+    print("Metricset {} created.".format(metricset))
 
 
 def generate_module(base_path, metricbeat_path, module, metricset):
@@ -45,7 +45,8 @@ def generate_module(base_path, metricbeat_path, module, metricset):
     meta_path = module_path + "/_meta"
 
     if os.path.isdir(module_path):
-        print "Module already exists. Skipping creating module " + module + ".\n"
+        print("Module already exists. Skipping creating module {}"
+              .format(module))
         return
 
     os.makedirs(meta_path)
@@ -68,15 +69,16 @@ def generate_module(base_path, metricbeat_path, module, metricset):
     with open(module_path + "/doc.go", "w") as f:
         f.write(content)
 
-    print "Module " + module + " created."
+    print("Module {} created.".format(module))
 
 
 def load_file(file, module, metricset):
     content = ""
     with open(file) as f:
         content = f.read()
 
-    return content.replace("{module}", module).replace("{metricset}", metricset)
+    return content.replace("{module}", module).replace("{metricset}",
+                                                       metricset)
 
 if __name__ == "__main__":
 
@@ -85,7 +87,8 @@ def load_file(file, module, metricset):
     parser.add_argument("--metricset", help="Metricset name")
 
     parser.add_argument("--path", help="Beat path")
-    parser.add_argument("--es_beats", help="The path to the general beats folder")
+    parser.add_argument("--es_beats",
+                        help="The path to the general beats folder")
 
     args = parser.parse_args()
 
@@ -106,4 +109,5 @@ def load_file(file, module, metricset):
     path = os.path.abspath(args.path)
     metricbeat_path = os.path.abspath(args.es_beats + "/metricbeat")
 
-    generate_metricset(path, metricbeat_path, args.module.lower(), args.metricset.lower())
+    generate_metricset(path, metricbeat_path, args.module.lower(),
+                       args.metricset.lower())