From 72823f8d29881e3561922fa85452f49c3779ab1b Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Tue, 3 Jan 2017 08:59:39 +0200 Subject: [PATCH] Generator for a Filebeat module/fileset (#3248) Following the model of the Metricbeat module generator, this creates templates for most of the required files of a fileset, making it easier to get started. To use, run as: MODULE=apache FILESET=access make create-fileset The PR includes some minor cleanups to the Metricbeat version. Part of #3159. --- filebeat/Makefile | 7 ++ filebeat/scripts/create_fileset.py | 103 ++++++++++++++++++ filebeat/scripts/module/fields.yml | 9 ++ .../scripts/module/fileset/config/config.yml | 9 ++ filebeat/scripts/module/fileset/fields.yml | 9 ++ .../module/fileset/ingest/pipeline.json | 11 ++ filebeat/scripts/module/fileset/manifest.yml | 14 +++ metricbeat/scripts/create_metricset.py | 20 ++-- 8 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 filebeat/scripts/create_fileset.py create mode 100644 filebeat/scripts/module/fields.yml create mode 100644 filebeat/scripts/module/fileset/config/config.yml create mode 100644 filebeat/scripts/module/fileset/fields.yml create mode 100644 filebeat/scripts/module/fileset/ingest/pipeline.json create mode 100644 filebeat/scripts/module/fileset/manifest.yml diff --git a/filebeat/Makefile b/filebeat/Makefile index 8b820466439..afce82109e3 100644 --- a/filebeat/Makefile +++ b/filebeat/Makefile @@ -31,3 +31,10 @@ fields: # Runs all collection steps and updates afterwards .PHONY: collect collect: fields kibana + + +# Creates a new fileset. Requires the params MODULE and FILESET +.PHONY: create-fileset +create-fileset: python-env + . ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/filebeat/scripts/create_fileset.py --path=$(PWD) --es_beats=$(ES_BEATS) --module=$(MODULE) --fileset=$(FILESET) + diff --git a/filebeat/scripts/create_fileset.py b/filebeat/scripts/create_fileset.py new file mode 100644 index 00000000000..31c9c24c70c --- /dev/null +++ b/filebeat/scripts/create_fileset.py @@ -0,0 +1,103 @@ +import os +import argparse + +# Creates a new fileset with all the necessary files. +# In case the module does not exist, also the module is created. + + +def generate_fileset(base_path, metricbeat_path, module, fileset): + + generate_module(base_path, metricbeat_path, module, fileset) + fileset_path = base_path + "/module/" + module + "/" + fileset + meta_path = fileset_path + "/_meta" + + if os.path.isdir(fileset_path): + print("Fileset already exists. Skipping creating fileset {}" + .format(fileset)) + return + + os.makedirs(meta_path) + os.makedirs(os.path.join(fileset_path, "test")) + + templates = metricbeat_path + "/scripts/module/fileset/" + + content = load_file(templates + "fields.yml", module, fileset) + with open(meta_path + "/fields.yml", "w") as f: + f.write(content) + + os.makedirs(os.path.join(fileset_path, "config")) + content = load_file(templates + "/config/config.yml", module, fileset) + with open("{}/config/{}.yml".format(fileset_path, fileset), "w") as f: + f.write(content) + + os.makedirs(os.path.join(fileset_path, "ingest")) + content = load_file(templates + "/ingest/pipeline.json", module, fileset) + with open("{}/ingest/pipeline.json".format(fileset_path), "w") as f: + f.write(content) + + content = load_file(templates + "/manifest.yml", module, fileset) + with open("{}/manifest.yml".format(fileset_path), "w") as f: + f.write(content) + + print("Fileset {} created.".format(fileset)) + + +def generate_module(base_path, metricbeat_path, module, fileset): + + module_path = base_path + "/module/" + module + meta_path = module_path + "/_meta" + + if os.path.isdir(module_path): + print("Module already exists. Skipping creating module {}" + .format(module)) + return + + os.makedirs(meta_path) + + templates = metricbeat_path + "/scripts/module/" + + content = load_file(templates + "fields.yml", module, "") + with open(meta_path + "/fields.yml", "w") as f: + f.write(content) + + print("Module {} created.".format(module)) + + +def load_file(file, module, fileset): + content = "" + with open(file) as f: + content = f.read() + + return content.replace("{module}", module).replace("{fileset}", fileset) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Creates a fileset") + parser.add_argument("--module", help="Module name") + parser.add_argument("--fileset", help="Fileset name") + + parser.add_argument("--path", help="Beat path") + parser.add_argument("--es_beats", + help="The path to the general beats folder") + + args = parser.parse_args() + + if args.path is None: + args.path = './' + print "Set default path for beat path: " + args.path + + if args.es_beats is None: + args.es_beats = '../' + print "Set default path for es_beats path: " + args.es_beats + + if args.module is None or args.module == '': + args.module = raw_input("Module name: ") + + if args.fileset is None or args.fileset == '': + args.fileset = raw_input("Fileset name: ") + + path = os.path.abspath(args.path) + filebeat_path = os.path.abspath(args.es_beats + "/filebeat") + + generate_fileset(path, filebeat_path, args.module.lower(), + args.fileset.lower()) diff --git a/filebeat/scripts/module/fields.yml b/filebeat/scripts/module/fields.yml new file mode 100644 index 00000000000..50d767ec5aa --- /dev/null +++ b/filebeat/scripts/module/fields.yml @@ -0,0 +1,9 @@ +- key: {module} + title: "{module}" + description: > + {module} Module + fields: + - name: {module} + type: group + description: > + fields: diff --git a/filebeat/scripts/module/fileset/config/config.yml b/filebeat/scripts/module/fileset/config/config.yml new file mode 100644 index 00000000000..20122796dbe --- /dev/null +++ b/filebeat/scripts/module/fileset/config/config.yml @@ -0,0 +1,9 @@ +- input_type: log + paths: + {%- for path in paths %} + - {{path}} + {%- endfor %} + exclude_files: [".gz$"] + fields: + source_type: {module}-{fileset} + pipeline_id: {{beat.pipeline_id}} diff --git a/filebeat/scripts/module/fileset/fields.yml b/filebeat/scripts/module/fileset/fields.yml new file mode 100644 index 00000000000..d237ec1b573 --- /dev/null +++ b/filebeat/scripts/module/fileset/fields.yml @@ -0,0 +1,9 @@ +- name: {fileset} + type: group + description: > + {fileset} + fields: + - name: example + type: keyword + description: > + Example field diff --git a/filebeat/scripts/module/fileset/ingest/pipeline.json b/filebeat/scripts/module/fileset/ingest/pipeline.json new file mode 100644 index 00000000000..90fde13d235 --- /dev/null +++ b/filebeat/scripts/module/fileset/ingest/pipeline.json @@ -0,0 +1,11 @@ +{ + "description": "Pipeline for parsing {module} {fileset} logs", + "processors": [ + ], + "on_failure" : [{ + "set" : { + "field" : "error", + "value" : "{{ _ingest.on_failure_message }}" + } + }] +} diff --git a/filebeat/scripts/module/fileset/manifest.yml b/filebeat/scripts/module/fileset/manifest.yml new file mode 100644 index 00000000000..2982d637b16 --- /dev/null +++ b/filebeat/scripts/module/fileset/manifest.yml @@ -0,0 +1,14 @@ +module_version: 1.0 + +vars: + paths: + default: + - /example/path* + os.darwin: + - /usr/local/example/path* + os.windows: + - "c:/example/path*" + +ingest_pipeline: ingest/pipeline.json +prospectors: + - config/{fileset}.yml diff --git a/metricbeat/scripts/create_metricset.py b/metricbeat/scripts/create_metricset.py index 2ad710d6adb..1f8425eb1b3 100644 --- a/metricbeat/scripts/create_metricset.py +++ b/metricbeat/scripts/create_metricset.py @@ -12,14 +12,14 @@ def generate_metricset(base_path, metricbeat_path, module, metricset): meta_path = metricset_path + "/_meta" if os.path.isdir(metricset_path): - print "MericSet already exists. Skipping creating metricset " + metricset + ".\n" + print("Metricset already exists. Skipping creating metricset {}" + .format(metricset)) return os.makedirs(meta_path) templates = metricbeat_path + "/scripts/module/metricset/" - content = load_file(templates + "metricset.go.tmpl", module, metricset) with open(metricset_path + "/" + metricset + ".go", "w") as f: f.write(content) @@ -36,7 +36,7 @@ def generate_metricset(base_path, metricbeat_path, module, metricset): with open(meta_path + "/data.json", "w") as f: f.write(content) - print "Metricset " + metricset + " created." + print("Metricset {} created.".format(metricset)) def generate_module(base_path, metricbeat_path, module, metricset): @@ -45,7 +45,8 @@ def generate_module(base_path, metricbeat_path, module, metricset): meta_path = module_path + "/_meta" if os.path.isdir(module_path): - print "Module already exists. Skipping creating module " + module + ".\n" + print("Module already exists. Skipping creating module {}" + .format(module)) return os.makedirs(meta_path) @@ -68,7 +69,7 @@ def generate_module(base_path, metricbeat_path, module, metricset): with open(module_path + "/doc.go", "w") as f: f.write(content) - print "Module " + module + " created." + print("Module {} created.".format(module)) def load_file(file, module, metricset): @@ -76,7 +77,8 @@ def load_file(file, module, metricset): with open(file) as f: content = f.read() - return content.replace("{module}", module).replace("{metricset}", metricset) + return content.replace("{module}", module).replace("{metricset}", + metricset) if __name__ == "__main__": @@ -85,7 +87,8 @@ def load_file(file, module, metricset): parser.add_argument("--metricset", help="Metricset name") parser.add_argument("--path", help="Beat path") - parser.add_argument("--es_beats", help="The path to the general beats folder") + parser.add_argument("--es_beats", + help="The path to the general beats folder") args = parser.parse_args() @@ -106,4 +109,5 @@ def load_file(file, module, metricset): path = os.path.abspath(args.path) metricbeat_path = os.path.abspath(args.es_beats + "/metricbeat") - generate_metricset(path, metricbeat_path, args.module.lower(), args.metricset.lower()) + generate_metricset(path, metricbeat_path, args.module.lower(), + args.metricset.lower())