From e61c84b901a6c42699584ed51dcde4287b0ab761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 27 Nov 2018 12:36:37 +0100 Subject: [PATCH] Unify dashboard exporter tools (#9097) The existing `export_dashboards.go` and the command `export dashboards` are unified, using the same code and slightly different logic. Configuration of `export_dashboards` is done using the following command line options: `-kibana`, `-space`, `-output`, `-quiet` and `-index-pattern`. The last flag is an odd one out, because it is part of the script, but its value is never used. So far no one complained, so I did not port it. The default value of that flag is the same in case of both methods. Configuration the Kibana client of `export dashboard` is read from the config of the Beat. Thus, Kibana-related flags are not part of its CLI. By default `export dashboard` does not decode the exported dashboard. If flag `-decode` passed to the command, the dashboard is decoded. export dashboards from a dashboards.yml ``` $ ./filebeat export dashboard -yml path/to/dashboards.yml -decode $ go run dev-tools/cmd/dashboards/export_dashboards.go -yml path/to/dashboards.yml ``` export a dashboard with an id and print to stdout ``` $ ./filebeat export dashboard -id {uuid} -decode $ go run dev-tools/cmd/dashboards/export_dashboards.go -dashboard {uuid} ``` (cherry picked from commit 1411852) --- CHANGELOG.asciidoc | 1 + dev-tools/cmd/dashboards/export_dashboards.go | 171 +++++------------ docs/devguide/newdashboards.asciidoc | 17 +- filebeat/scripts/generator/fileset/main.go | 2 +- filebeat/scripts/generator/generator.go | 2 +- filebeat/scripts/generator/module/main.go | 11 +- libbeat/cmd/export/dashboard.go | 44 ++++- libbeat/dashboards/decode.go | 70 +++++++ libbeat/dashboards/export.go | 106 +++++++++++ libbeat/kibana/client.go | 1 + libbeat/tests/files/dashboards.yml | 3 + libbeat/tests/system/test_dashboard.py | 176 +++++++++++++++++- 12 files changed, 461 insertions(+), 143 deletions(-) create mode 100644 libbeat/dashboards/decode.go create mode 100644 libbeat/dashboards/export.go create mode 100644 libbeat/tests/files/dashboards.yml diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 900a3b92015..a5e55ca4a7d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -70,6 +70,7 @@ https://github.com/elastic/beats/compare/v6.5.0...6.x[Check the HEAD diff] ==== Added *Affecting all Beats* +- Unify dashboard exporter tools. {pull}9097[9097] - Dissect will now flag event on parsing error. {pull}8751[8751] - Added the `redirect_stderr` option that allows panics to be logged to log files. {pull}8430[8430] diff --git a/dev-tools/cmd/dashboards/export_dashboards.go b/dev-tools/cmd/dashboards/export_dashboards.go index a5338099f5f..22749237918 100644 --- a/dev-tools/cmd/dashboards/export_dashboards.go +++ b/dev-tools/cmd/dashboards/export_dashboards.go @@ -18,127 +18,27 @@ package main import ( - "crypto/tls" "encoding/json" "flag" - "fmt" "io/ioutil" "log" - "net/http" "net/url" "os" - "path" "path/filepath" - "strings" + "time" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/dashboards" "github.com/elastic/beats/libbeat/kibana" ) -var exportAPI = "/api/kibana/dashboards/export" - -type manifest struct { - Dashboards []map[string]string `config:"dashboards"` -} - -func makeURL(url, path string, params url.Values) string { - if len(params) == 0 { - return url + path - } - - return strings.Join([]string{url, path, "?", params.Encode()}, "") -} - -func Export(client *http.Client, conn string, spaceID string, dashboard string, out string) error { - params := url.Values{} - - params.Add("dashboard", dashboard) - - if spaceID != "" { - exportAPI = path.Join("/s", spaceID, exportAPI) - } - fullURL := makeURL(conn, exportAPI, params) - if !quiet { - log.Printf("Calling HTTP GET %v\n", fullURL) - } - - req, err := http.NewRequest("GET", fullURL, nil) - - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("GET HTTP request fails with: %v", err) - } - - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("fail to read response %s", err) - } - - if resp.StatusCode != 200 { - return fmt.Errorf("HTTP GET %s fails with %s, %s", fullURL, resp.Status, body) - } - - data, err := kibana.RemoveIndexPattern(body) - if err != nil { - return fmt.Errorf("fail to extract the index pattern: %v", err) - } - - objects := data["objects"].([]interface{}) - for _, obj := range objects { - o := obj.(common.MapStr) - - decodeValue(o, "attributes.uiStateJSON") - decodeValue(o, "attributes.visState") - decodeValue(o, "attributes.optionsJSON") - decodeValue(o, "attributes.panelsJSON") - decodeValue(o, "attributes.kibanaSavedObjectMeta.searchSourceJSON") - } - - data["objects"] = objects - - // Create all missing directories - err = os.MkdirAll(filepath.Dir(out), 0755) - if err != nil { - return err - } - - err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0666) - if !quiet { - log.Printf("The dashboard %s was exported under the %s file\n", dashboard, out) - } - return err -} - -func decodeValue(data common.MapStr, key string) { - v, err := data.GetValue(key) - if err != nil { - return - } - s := v.(string) - var d interface{} - json.Unmarshal([]byte(s), &d) - - data.Put(key, d) -} - -func ReadManifest(file string) ([]map[string]string, error) { - cfg, err := common.LoadFile(file) - if err != nil { - return nil, fmt.Errorf("error reading manifest file: %v", err) - } - - var manifest manifest - err = cfg.Unpack(&manifest) - if err != nil { - return nil, fmt.Errorf("error unpacking manifest: %v", err) - } - return manifest.Dashboards, nil -} +var ( + indexPattern = false + quiet = false +) -var indexPattern = false -var quiet = false +const ( + kibanaTimeout = 90 * time.Second +) func main() { kibanaURL := flag.String("kibana", "http://localhost:5601", "Kibana URL") @@ -152,11 +52,20 @@ func main() { flag.Parse() log.SetFlags(0) - transCfg := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // ignore expired SSL certificates + u, err := url.Parse(*kibanaURL) + if err != nil { + log.Fatalf("Error parsing Kibana URL: %v", err) } - client := &http.Client{Transport: transCfg} + client, err := kibana.NewClientWithConfig(&kibana.ClientConfig{ + Protocol: u.Scheme, + Host: u.Host, + SpaceID: *spaceID, + Timeout: kibanaTimeout, + }) + if err != nil { + log.Fatalf("Error while connecting to Kibana: %+v", err) + } if len(*ymlFile) == 0 && len(*dashboard) == 0 { flag.Usage() @@ -164,30 +73,36 @@ func main() { } if len(*ymlFile) > 0 { - dashboards, err := ReadManifest(*ymlFile) - if err != nil { - log.Fatalf("%s", err) - } - - for _, dashboard := range dashboards { - log.Printf("id=%s, name=%s\n", dashboard["id"], dashboard["file"]) - directory := filepath.Join(filepath.Dir(*ymlFile), "_meta/kibana/6/dashboard") - err := os.MkdirAll(directory, 0755) - if err != nil { - log.Fatalf("fail to create directory %s: %v", directory, err) - } - err = Export(client, *kibanaURL, *spaceID, dashboard["id"], filepath.Join(directory, dashboard["file"])) + results, info, err := dashboards.ExportAllFromYml(client, *ymlFile) + for i, r := range results { + log.Printf("id=%s, name=%s\n", info.Dashboards[i].ID, info.Dashboards[i].File) + r = dashboards.DecodeExported(r) + err = dashboards.SaveToFile(r, info.Dashboards[i].File, filepath.Dir(*ymlFile), client.GetVersion()) if err != nil { - log.Fatalf("fail to export the dashboards: %s", err) + log.Fatalf("failed to export the dashboards: %s", err) } } os.Exit(0) } if len(*dashboard) > 0 { - err := Export(client, *kibanaURL, *spaceID, *dashboard, *fileOutput) + result, err := dashboards.Export(client, *dashboard) if err != nil { - log.Fatalf("fail to export the dashboards: %s", err) + log.Fatalf("Failed to export the dashboard: %s", err) + } + result = dashboards.DecodeExported(result) + bytes, err := json.Marshal(result) + if err != nil { + log.Fatalf("Failed to save the dashboard: %s", err) + } + + err = ioutil.WriteFile(*fileOutput, bytes, 0644) + if err != nil { + log.Fatalf("Failed to save the dashboard: %s", err) + + } + if !quiet { + log.Printf("The dashboard %s was exported under the %s file\n", *dashboard, *fileOutput) } } } diff --git a/docs/devguide/newdashboards.asciidoc b/docs/devguide/newdashboards.asciidoc index b532655fc20..a203c5e4217 100644 --- a/docs/devguide/newdashboards.asciidoc +++ b/docs/devguide/newdashboards.asciidoc @@ -232,10 +232,13 @@ ES_URL="http://192.168.3.206:9200" make export-dashboards ///////////////////// +Alternatively, if the scripts above are not available, you can use your Beat binary to export Kibana 6.0 dashboards or later. + ==== Exporting Kibana 6.0 dashboards and newer The `dev-tools/cmd/export_dashboards.go` script helps you export your customized Kibana 6.0 dashboards and newer. You might need to export a single dashboard or all the dashboards available for a module or Beat. +It is also possible to use a Beat binary to export. ===== Export a single Kibana dashboard @@ -250,6 +253,11 @@ cd filebeat/module/redis/_meta/kibana/default/dashboard go run ../../../../../../../dev-tools/cmd/dashboards/export_dashboards.go -id 7fea2930-478e-11e7-b1f0-cb29bac6bf8b -output Filebeat-redis.json --------------- +[source,shell] +--------------- +./filebeat export dashboard -id 7fea2930-478e-11e7-b1f0-cb29bac6bf8b >> Filebeat-redis.json +--------------- + This generates the `Filebeat-redis.json` file that contains the dashboard for the Redis module of Filebeat, including the dependencies (visualizations and searches). ===== Export all module/Beat dashboards @@ -275,13 +283,18 @@ dashboards: Each dashboard is defined by an `id` and the name of json `file` where the dashboard is saved locally. -By passing the yml file to the `export_dashboards.go` script, you can export all the dashboards defined: +By passing the yml file to the `export_dashboards.go` script or to the Beat, you can export all the dashboards defined: [source,shell] ------------------- go run dev-tools/cmd/dashboards/export_dashboards.go -yml filebeat/module/system/module.yml ------------------- +[source,shell] +------------------- +./filebeat export dashboard -yml filebeat/module/system/module.yml +------------------- + ===== Export dashboards from a Kibana Space @@ -292,6 +305,8 @@ If you are using the Kibana Spaces feature and want to export dashboards from a go run dev-tools/cmd/dashboards/export_dashboards.go -space-id my-space [other-options] ------------------- +In case of running `export dashboard` of a Beat, you need to set the Space ID in `setup.kibana.space.id`. + ==== Exporting Kibana 5.x dashboards diff --git a/filebeat/scripts/generator/fileset/main.go b/filebeat/scripts/generator/fileset/main.go index 94d0ac6b6a2..76de5168d05 100644 --- a/filebeat/scripts/generator/fileset/main.go +++ b/filebeat/scripts/generator/fileset/main.go @@ -32,7 +32,7 @@ func generateFileset(module, fileset, modulesPath, beatsPath string) error { return fmt.Errorf("fileset already exists: %s", fileset) } - err := generator.CreateDirectories(filesetPath, []string{"", "_meta", "test", "config", "ingest"}) + err := generator.CreateDirectories(filesetPath, "_meta", "test", "config", "ingest") if err != nil { return err } diff --git a/filebeat/scripts/generator/generator.go b/filebeat/scripts/generator/generator.go index e5aaf771ab4..40df8311ede 100644 --- a/filebeat/scripts/generator/generator.go +++ b/filebeat/scripts/generator/generator.go @@ -35,7 +35,7 @@ func DirExists(dir string) bool { } // CreateDirectories create directories in baseDir -func CreateDirectories(baseDir string, directories []string) error { +func CreateDirectories(baseDir string, directories ...string) error { for _, d := range directories { p := path.Join(baseDir, d) err := os.MkdirAll(p, 0750) diff --git a/filebeat/scripts/generator/module/main.go b/filebeat/scripts/generator/module/main.go index dedf1f3e279..b88ed2d207b 100644 --- a/filebeat/scripts/generator/module/main.go +++ b/filebeat/scripts/generator/module/main.go @@ -32,15 +32,20 @@ func generateModule(module, modulesPath, beatsPath string) error { return fmt.Errorf("module already exists: %s", module) } - err := generator.CreateDirectories(modulePath, []string{path.Join("_meta", "kibana", "6")}) + err := generator.CreateDirectories(modulePath, "_meta") if err != nil { return err } replace := map[string]string{"module": module} templatesPath := path.Join(beatsPath, "scripts", "module") - filesToCopy := []string{path.Join("_meta", "fields.yml"), path.Join("_meta", "docs.asciidoc"), path.Join("_meta", "config.yml"), path.Join("module.yml")} - generator.CopyTemplates(templatesPath, modulePath, filesToCopy, replace) + filesToCopy := []string{ + path.Join("_meta", "fields.yml"), + path.Join("_meta", "docs.asciidoc"), + path.Join("_meta", "config.yml"), + "module.yml", + } + err = generator.CopyTemplates(templatesPath, modulePath, filesToCopy, replace) if err != nil { return err } diff --git a/libbeat/cmd/export/dashboard.go b/libbeat/cmd/export/dashboard.go index 59a6fa04cc3..697da10061e 100644 --- a/libbeat/cmd/export/dashboard.go +++ b/libbeat/cmd/export/dashboard.go @@ -20,11 +20,13 @@ package export import ( "fmt" "os" + "path/filepath" "github.com/spf13/cobra" "github.com/elastic/beats/libbeat/cmd/instance" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/dashboards" "github.com/elastic/beats/libbeat/kibana" ) @@ -35,6 +37,8 @@ func GenDashboardCmd(name, idxPrefix, beatVersion string) *cobra.Command { Short: "Export defined dashboard to stdout", Run: func(cmd *cobra.Command, args []string) { dashboard, _ := cmd.Flags().GetString("id") + yml, _ := cmd.Flags().GetString("yml") + decode, _ := cmd.Flags().GetBool("decode") b, err := instance.NewBeat(name, idxPrefix, beatVersion) if err != nil { @@ -58,16 +62,46 @@ func GenDashboardCmd(name, idxPrefix, beatVersion string) *cobra.Command { os.Exit(1) } - result, err := client.GetDashboard(dashboard) - if err != nil { - fmt.Fprintf(os.Stderr, "Error getting dashboard: %+v\n", err) - os.Exit(1) + // Export dashboards from yml file + if yml != "" { + results, info, err := dashboards.ExportAllFromYml(client, yml) + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting dashboards from yml: %+v\n", err) + os.Exit(1) + } + for i, r := range results { + if decode { + r = dashboards.DecodeExported(r) + } + err = dashboards.SaveToFile(r, info.Dashboards[i].File, filepath.Dir(yml), client.GetVersion()) + if err != nil { + fmt.Fprintf(os.Stderr, "Error saving dashboard '%s' to file '%s' : %+v\n", + info.Dashboards[i].ID, info.Dashboards[i].File, err) + os.Exit(1) + } + } + return + } + + // Export single dashboard + if dashboard != "" { + result, err := dashboards.Export(client, dashboard) + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting dashboard: %+v\n", err) + os.Exit(1) + } + + if decode { + result = dashboards.DecodeExported(result) + } + fmt.Println(result.StringToPrint()) } - fmt.Println(result.StringToPrint()) }, } genTemplateConfigCmd.Flags().String("id", "", "Dashboard id") + genTemplateConfigCmd.Flags().String("yml", "", "Yaml file containing list of dashboard ID and filename pairs") + genTemplateConfigCmd.Flags().Bool("decode", false, "Decode exported dashboard") return genTemplateConfigCmd } diff --git a/libbeat/dashboards/decode.go b/libbeat/dashboards/decode.go new file mode 100644 index 00000000000..2b659d7251e --- /dev/null +++ b/libbeat/dashboards/decode.go @@ -0,0 +1,70 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package dashboards + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" +) + +var ( + responseToDecode = []string{ + "attributes.uiStateJSON", + "attributes.visState", + "attributes.optionsJSON", + "attributes.panelsJSON", + "attributes.kibanaSavedObjectMeta.searchSourceJSON", + } +) + +// DecodeExported decodes an exported dashboard +func DecodeExported(result common.MapStr) common.MapStr { + // remove unsupported chars + objects := result["objects"].([]interface{}) + for _, obj := range objects { + o := obj.(common.MapStr) + for _, key := range responseToDecode { + // All fields are optional, so errors are not caught + err := decodeValue(o, key) + if err != nil { + logp.Debug("dashboards", "Error while decoding dashboard objects: %+v", err) + } + } + } + result["objects"] = objects + return result +} + +func decodeValue(data common.MapStr, key string) error { + v, err := data.GetValue(key) + if err != nil { + return err + } + s := v.(string) + var d interface{} + err = json.Unmarshal([]byte(s), &d) + if err != nil { + return fmt.Errorf("error decoding %s: %v", key, err) + } + + data.Put(key, d) + return nil +} diff --git a/libbeat/dashboards/export.go b/libbeat/dashboards/export.go new file mode 100644 index 00000000000..d74a1d84b7b --- /dev/null +++ b/libbeat/dashboards/export.go @@ -0,0 +1,106 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package dashboards + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "strconv" + + yaml "gopkg.in/yaml.v2" + + "github.com/elastic/beats/filebeat/scripts/generator" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/kibana" +) + +const ( + dashboardPerm = 0644 +) + +// ListYML is the yaml file which contains list of available dashboards. +type ListYML struct { + Dashboards []YMLElement `yaml:"dashboards"` +} + +// YMLElement contains the data of a dashboard: +// * its uuid in Kibana +// * filename to be saved as +type YMLElement struct { + ID string `yaml:"id"` + File string `yaml:"file"` +} + +// Export wraps GetDashboard call to provide a more descriptive API +func Export(client *kibana.Client, id string) (common.MapStr, error) { + return client.GetDashboard(id) +} + +// ExportAllFromYml exports all dashboards found in the YML file +func ExportAllFromYml(client *kibana.Client, ymlPath string) ([]common.MapStr, ListYML, error) { + b, err := ioutil.ReadFile(ymlPath) + if err != nil { + return nil, ListYML{}, fmt.Errorf("error opening the list of dashboards: %+v", err) + } + var list ListYML + err = yaml.Unmarshal(b, &list) + if err != nil { + return nil, ListYML{}, fmt.Errorf("error reading the list of dashboards: %+v", err) + } + + results, err := ExportAll(client, list) + + return results, list, err +} + +// ExportAll exports all dashboards from an opened and parsed dashboards YML. +func ExportAll(client *kibana.Client, list ListYML) ([]common.MapStr, error) { + var results []common.MapStr + for _, e := range list.Dashboards { + result, err := Export(client, e.ID) + if err != nil { + return nil, err + } + results = append(results, result) + } + return results, nil +} + +// SaveToFile creates the required directories if needed and saves dashboard. +func SaveToFile(dashboard common.MapStr, filename, root, versionStr string) error { + version, err := common.NewVersion(versionStr) + if err != nil { + return err + } + + dashboardsPath := "_meta/kibana/" + strconv.Itoa(version.Major) + "/dashboard" + err = generator.CreateDirectories(root, dashboardsPath) + if err != nil { + return err + } + + out := filepath.Join(root, dashboardsPath, filename) + bytes, err := json.Marshal(dashboard) + if err != nil { + return err + } + + return ioutil.WriteFile(out, bytes, dashboardPerm) +} diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index 75323a3b655..fe21af40dcb 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -283,6 +283,7 @@ func (client *Client) GetDashboard(id string) (common.MapStr, error) { if err != nil { return nil, fmt.Errorf("error removing index pattern: %+v", err) } + return result, nil } diff --git a/libbeat/tests/files/dashboards.yml b/libbeat/tests/files/dashboards.yml new file mode 100644 index 00000000000..8278dd7b319 --- /dev/null +++ b/libbeat/tests/files/dashboards.yml @@ -0,0 +1,3 @@ +dashboards: +- id: Metricbeat-system-overview + file: Metricbeat-system-test-overview.json diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index 7386334bf82..ad445d1779f 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -125,9 +125,130 @@ def test_load_only_index_patterns(self): @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') - def test_export_dashboard(self): + def test_export_dashboard_cmd_export_dashboard_by_id_and_decoding(self): """ - Test export dashboards and remove unsupported characters + Test testbeat export dashboard can export dashboards + and removes unsupported characters + """ + self.render_config_template() + self.test_load_dashboard() + beat = self.start_beat( + logging_args=["-e", "-d", "*"], + extra_args=["export", + "dashboard", + "-E", "setup.kibana.protocol=http", + "-E", "setup.kibana.host=" + self.get_kibana_host(), + "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-decode", + "-id", "Metricbeat-system-overview"] + ) + + beat.check_wait(exit_code=0) + + assert self.log_contains("\"id\": \"Metricbeat-system-overview\",") is True + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_export_dashboard_cmd_export_dashboard_by_id(self): + """ + Test testbeat export dashboard can export dashboards + """ + self.render_config_template() + self.test_load_dashboard() + beat = self.start_beat( + logging_args=["-e", "-d", "*"], + extra_args=["export", + "dashboard", + "-E", "setup.kibana.protocol=http", + "-E", "setup.kibana.host=" + self.get_kibana_host(), + "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-id", "Metricbeat-system-overview"] + ) + + beat.check_wait(exit_code=0) + + assert self.log_contains("\"id\": \"Metricbeat-system-overview\",") is True + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_export_dashboard_cmd_export_dashboard_by_id_unknown_id(self): + """ + Test testbeat export dashboard fails gracefully when dashboard with unknown ID is requested + """ + self.render_config_template() + beat = self.start_beat( + logging_args=["-e", "-d", "*"], + extra_args=["export", + "dashboard", + "-E", "setup.kibana.protocol=http", + "-E", "setup.kibana.host=" + self.get_kibana_host(), + "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-id", "No-such-dashboard"] + ) + + beat.check_wait(exit_code=1) + + assert self.log_contains("Error getting dashboard: error exporting dashboard: Not found") is True + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_export_dashboard_cmd_export_dashboard_from_yml(self): + """ + Test testbeat export dashboard can export dashboards from dashboards YAML file + and removes unsupported characters + """ + + self.render_config_template() + self.test_load_dashboard() + beat = self.start_beat( + logging_args=["-e", "-d", "*"], + extra_args=["export", + "dashboard", + "-E", "setup.kibana.protocol=http", + "-E", "setup.kibana.host=" + self.get_kibana_host(), + "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-yml", os.path.join(self.beat_path, "tests", "files", "dashboards.yml")] + ) + + beat.check_wait(exit_code=0) + + version = self.get_version() + kibana_semver = semver.VersionInfo.parse(version) + exported_dashboard_path = os.path.join(self.beat_path, "tests", "files", "_meta", + "kibana", str(kibana_semver.major), "dashboard", "Metricbeat-system-test-overview.json") + + with open(exported_dashboard_path) as f: + content = f.read() + assert "Metricbeat-system-overview" in content + + os.remove(exported_dashboard_path) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_export_dashboard_cmd_export_dashboard_from_not_existent_yml(self): + """ + Test testbeat export dashboard fails gracefully when cannot find YAML file + """ + + self.render_config_template() + beat = self.start_beat( + logging_args=["-e", "-d", "*"], + extra_args=["export", + "dashboard", + "-E", "setup.kibana.protocol=http", + "-E", "setup.kibana.host=" + self.get_kibana_host(), + "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-yml", os.path.join(self.beat_path, "tests", "files", "no-such-file.yml")] + ) + + beat.check_wait(exit_code=1) + assert self.log_contains("Error getting dashboards from yml: error opening the list of dashboards:") is True + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_dev_tool_export_dashboard_by_id(self): + """ + Test dev-tools/cmd/dashboards exports dashboard and removes unsupported characters """ self.test_load_dashboard() @@ -151,9 +272,26 @@ def test_export_dashboard(self): @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') - def test_export_dashboard_from_space(self): + def test_dev_tool_export_dashboard_by_id_unknown_id(self): """ - Test export dashboards from Kibana space and remove unsupported characters + Test dev-tools/cmd/dashboards fails gracefully when dashboard with unknown ID is requested + """ + + path = os.path.normpath(self.beat_path + "/../dev-tools/cmd/dashboards/export_dashboards.go") + command = path + " -kibana http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + command = "go run " + command + " -dashboard No-such-dashboard" + + p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + content, err = p.communicate() + + assert p.returncode != 0 + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_dev_tool_export_dashboard_by_id_from_space(self): + """ + Test dev-tools/cmd/dashboards exports dashboard from Kibana space + and removes unsupported characters """ version = self.get_version() if semver.compare(version, "6.5.0") == -1: @@ -179,6 +317,36 @@ def test_export_dashboard_from_space(self): os.remove("output.json") + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_dev_tool_export_dashboard_from_yml(self): + """ + Test dev-tools/cmd/dashboards exports dashboard from dashboards YAML file + and removes unsupported characters + """ + + self.test_load_dashboard() + + path = os.path.normpath(self.beat_path + "/../dev-tools/cmd/dashboards/export_dashboards.go") + command = path + " -kibana http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + command = "go run " + command + " -yml " + os.path.join(self.beat_path, "tests", "files", "dashboards.yml") + + p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + content, err = p.communicate() + + assert p.returncode == 0 + + version = self.get_version() + kibana_semver = semver.VersionInfo.parse(version) + exported_dashboard_path = os.path.join(self.beat_path, "tests", "files", "_meta", + "kibana", str(kibana_semver.major), "dashboard", "Metricbeat-system-test-overview.json") + + with open(exported_dashboard_path) as f: + content = f.read() + assert "Metricbeat-system-overview" in content + + os.remove(exported_dashboard_path) + def get_host(self): return os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200')