diff --git a/filebeat/cmd/generate.go b/filebeat/cmd/generate.go new file mode 100644 index 00000000000..9b081ff6eef --- /dev/null +++ b/filebeat/cmd/generate.go @@ -0,0 +1,121 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/elastic/beats/filebeat/generator/fields" + "github.com/elastic/beats/filebeat/generator/fileset" + "github.com/elastic/beats/filebeat/generator/module" + "github.com/elastic/beats/libbeat/common/cli" + "github.com/elastic/beats/libbeat/paths" +) + +var ( + defaultHomePath = paths.Resolve(paths.Home, "") +) + +func genGenerateCmd() *cobra.Command { + generateCmd := cobra.Command{ + Use: "generate", + Short: "Generate Filebeat modules, filesets and fields.yml", + } + generateCmd.AddCommand(genGenerateModuleCmd()) + generateCmd.AddCommand(genGenerateFilesetCmd()) + generateCmd.AddCommand(genGenerateFieldsCmd()) + + return &generateCmd +} + +func genGenerateModuleCmd() *cobra.Command { + genModuleCmd := &cobra.Command{ + Use: "module [module]", + Short: "Generates a new module", + Run: cli.RunWith(func(cmd *cobra.Command, args []string) error { + modulesPath, _ := cmd.Flags().GetString("modules-path") + esBeatsPath, _ := cmd.Flags().GetString("es-beats") + + if len(args) != 1 { + fmt.Fprintf(os.Stderr, "Exactly one parameter is required: module name\n") + os.Exit(1) + } + name := args[0] + + return module.Generate(name, modulesPath, esBeatsPath) + }), + } + + genModuleCmd.Flags().String("modules-path", defaultHomePath, "Path to modules directory") + genModuleCmd.Flags().String("es-beats", defaultHomePath, "Path to Elastic Beats") + + return genModuleCmd +} + +func genGenerateFilesetCmd() *cobra.Command { + genFilesetCmd := &cobra.Command{ + Use: "fileset [module] [fileset]", + Short: "Generates a new fileset", + Run: cli.RunWith(func(cmd *cobra.Command, args []string) error { + modulesPath, _ := cmd.Flags().GetString("modules-path") + esBeatsPath, _ := cmd.Flags().GetString("es-beats") + + if len(args) != 2 { + fmt.Fprintf(os.Stderr, "Two parameters are required: module name, fileset name\n") + os.Exit(1) + } + moduleName := args[0] + filesetName := args[1] + + return fileset.Generate(moduleName, filesetName, modulesPath, esBeatsPath) + }), + } + + genFilesetCmd.Flags().String("modules-path", defaultHomePath, "Path to modules directory") + genFilesetCmd.Flags().String("es-beats", defaultHomePath, "Path to Elastic Beats") + + return genFilesetCmd +} + +func genGenerateFieldsCmd() *cobra.Command { + genFieldsCmd := &cobra.Command{ + Use: "fields [module] [fileset]", + Short: "Generates a new fields.yml file for fileset", + Run: cli.RunWith(func(cmd *cobra.Command, args []string) error { + esBeatsPath, _ := cmd.Flags().GetString("es-beats") + noDoc, _ := cmd.Flags().GetBool("without-documentation") + + if len(args) != 2 { + fmt.Fprintf(os.Stderr, "Two parameters are required: module name, fileset name\n") + os.Exit(1) + } + moduleName := args[0] + filesetName := args[1] + + return fields.Generate(esBeatsPath, moduleName, filesetName, noDoc) + }), + } + + genFieldsCmd.Flags().String("es-beats", defaultHomePath, "Path to Elastic Beats") + genFieldsCmd.Flags().Bool("without-documentation", false, "Do not add description fields") + + return genFieldsCmd +} diff --git a/filebeat/cmd/root.go b/filebeat/cmd/root.go index c426c5012d9..31618aee1a1 100644 --- a/filebeat/cmd/root.go +++ b/filebeat/cmd/root.go @@ -43,4 +43,5 @@ func init() { RootCmd.TestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules")) RootCmd.SetupCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules")) RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager)) + RootCmd.AddCommand(genGenerateCmd()) } diff --git a/filebeat/tests/system/input/my-module-pipeline.json b/filebeat/tests/system/input/my-module-pipeline.json new file mode 100644 index 00000000000..c492cc56375 --- /dev/null +++ b/filebeat/tests/system/input/my-module-pipeline.json @@ -0,0 +1,27 @@ +{ + "description": "Pipeline for parsing PostgreSQL logs.", + "processors": [ + { + "grok": { + "field": "message", + "ignore_missing": true, + "patterns": [ + "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] ((\\[%{USERNAME:postgresql.log.user}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:postgresql.log.user}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:postgresql.log.level}: (duration: %{NUMBER:postgresql.log.duration} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:postgresql.log.message})" + ], + "pattern_definitions": { + "LOCALDATETIME": "[-0-9]+ %{TIME}", + "GREEDYDATA": "(.|\n|\t)*", + "POSTGRESQL_DB_NAME": "[a-zA-Z0-9_]+[a-zA-Z0-9_\\$]*" + } + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/filebeat/tests/system/test_generate.py b/filebeat/tests/system/test_generate.py new file mode 100644 index 00000000000..e85c8cdcd55 --- /dev/null +++ b/filebeat/tests/system/test_generate.py @@ -0,0 +1,127 @@ +import os +import shutil +import filebeat + + +class Test(filebeat.BaseTest): + def test_generate_module(self): + """ + Test filebeat generate module my_module generates a new module + """ + + self._create_clean_test_modules() + exit_code = self.run_beat( + extra_args=["generate", "module", "my_module", "-modules-path", "test_modules", "-es-beats", self.beat_path]) + assert exit_code == 0 + + module_root = os.path.join("test_modules", "module", "my_module") + module_meta_root = os.path.join(module_root, "_meta") + self._assert_required_module_directories_are_created(module_root, module_meta_root) + self._assert_required_module_files_are_created_and_substitution_is_done(module_root, module_meta_root) + + shutil.rmtree("test_modules") + + def _assert_required_module_directories_are_created(self, module_root, module_meta_root): + expected_created_directories = [ + module_root, + module_meta_root, + ] + for expected_dir in expected_created_directories: + assert os.path.isdir(expected_dir) + + def _assert_required_module_files_are_created_and_substitution_is_done(self, module_root, module_meta_root): + expected_created_template_files = [ + os.path.join(module_root, "module.yml"), + os.path.join(module_meta_root, "config.yml"), + os.path.join(module_meta_root, "docs.asciidoc"), + os.path.join(module_meta_root, "fields.yml"), + ] + for template_file in expected_created_template_files: + assert os.path.isfile(template_file) + assert '{module}' not in open(template_file) + + def test_generate_fileset(self): + """ + Test filebeat generate fileset my_module my_fileset generates a new fileset + """ + + self._create_clean_test_modules() + exit_code = self.run_beat( + extra_args=["generate", "module", "my_module", "-modules-path", "test_modules", "-es-beats", self.beat_path]) + assert exit_code == 0 + + exit_code = self.run_beat( + extra_args=["generate", "fileset", "my_module", "my_fileset", "-modules-path", "test_modules", "-es-beats", self.beat_path]) + assert exit_code == 0 + + fileset_root = os.path.join("test_modules", "module", "my_module", "my_fileset") + self._assert_required_fileset_directories_are_created(fileset_root) + self._assert_required_fileset_files_are_created_and_substitution_is_done(fileset_root) + + shutil.rmtree("test_modules") + + def _assert_required_fileset_directories_are_created(self, fileset_root): + expected_created_directories = [ + fileset_root, + os.path.join(fileset_root, "config"), + os.path.join(fileset_root, "ingest"), + os.path.join(fileset_root, "_meta"), + os.path.join(fileset_root, "test"), + ] + for expected_dir in expected_created_directories: + assert os.path.isdir(expected_dir) + + def _assert_required_fileset_files_are_created_and_substitution_is_done(self, fileset_root): + expected_created_template_files = [ + os.path.join(fileset_root, "config", "my_fileset.yml"), + os.path.join(fileset_root, "ingest", "pipeline.json"), + os.path.join(fileset_root, "manifest.yml"), + ] + for template_file in expected_created_template_files: + assert os.path.isfile(template_file) + assert '{fileset}' not in open(template_file) + + def test_generate_fields_yml(self): + """ + Test filebeat generate fields my_module my_fileset generates a new fields.yml for my_module/my_fileset + """ + + self._create_clean_test_modules() + exit_code = self.run_beat( + extra_args=["generate", "module", "my_module", "-modules-path", "test_modules", "-es-beats", self.beat_path]) + assert exit_code == 0 + + exit_code = self.run_beat( + extra_args=["generate", "fileset", "my_module", "my_fileset", "-modules-path", "test_modules", "-es-beats", self.beat_path]) + assert exit_code == 0 + + test_pipeline_path = os.path.join(self.beat_path, "tests", "system", "input", "my-module-pipeline.json") + fileset_pipeline = os.path.join("test_modules", "module", + "my_module", "my_fileset", "ingest", "pipeline.json") + + print(os.path.isdir("test_modules")) + print(os.path.isdir(os.path.join("test_modules", "module"))) + print(os.path.isdir(os.path.join("test_modules", "module", "my_module"))) + print(os.path.isdir(os.path.join("test_modules", "module", "my_module", "my_fileset"))) + print(os.path.isdir(os.path.join("test_modules", "module", "my_module", "my_fileset", "ingest"))) + print(os.path.isdir(os.path.join("test_modules", "module", "my_module", "my_fileset", "ingest"))) + print(os.path.exists(os.path.join("test_modules", "module", "my_module", "my_fileset", "ingest", "pipeline.json"))) + print(os.path.exists(fileset_pipeline)) + print(os.path.exists(test_pipeline_path)) + shutil.copyfile(test_pipeline_path, fileset_pipeline) + + print(fileset_pipeline) + print(os.path.abspath(fileset_pipeline)) + exit_code = self.run_beat( + extra_args=["generate", "fields", "my_module", "my_fileset", "-es-beats", "test_modules", "-without-documentation"]) + assert exit_code == 0 + + fields_yml_path = os.path.join("test_modules", "module", "my_module", "my_fileset", "_meta", "fields.yml") + assert os.path.isfile(fields_yml_path) + + shutil.rmtree("test_modules") + + def _create_clean_test_modules(self): + if os.path.isdir("test_modules"): + shutil.rmtree("test_modules") + os.mkdir("test_modules")