Unify dashboard exporter tools (#9097)

The existing `export_dashboards.go` and the command `export dashboards` are unified, using the same code and slightly different logic. Configuration of `export_dashboards` is done using the following command line options: `-kibana`, `-space`, `-output`, `-quiet` and `-index-pattern`. The last flag is an odd one out, because it is part of the script, but its value is never used. So far no one complained, so I did not port it. The default value of that flag is the same in case of both methods.

Configuration the Kibana client of `export dashboard` is read from the config of the Beat. Thus, Kibana-related flags are not part of its CLI.

By default `export dashboard` does not decode the exported dashboard. If flag `-decode` passed to the command, the dashboard is decoded.

### Equivalent commands

export dashboards from a dashboards.yml
```
$ ./filebeat export dashboard -yml path/to/dashboards.yml -decode
$ go run dev-tools/cmd/dashboards/export_dashboards.go -yml path/to/dashboards.yml
```
export a dashboard with an id and print to stdout
```
$ ./filebeat export dashboard -id {uuid} -decode
$ go run dev-tools/cmd/dashboards/export_dashboards.go -dashboard {uuid}
```

CHANGELOG.asciidoc

@@ -56,6 +56,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 ==== Added
 
 *Affecting all Beats*
+- Unify dashboard exporter tools. {pull}9097[9097]
 
 *Auditbeat*
 

dev-tools/cmd/dashboards/export_dashboards.go

@@ -18,132 +18,27 @@
 package main
 
 import (
-	"crypto/tls"
 	"encoding/json"
 	"flag"
-	"fmt"
 	"io/ioutil"
 	"log"
-	"net/http"
 	"net/url"
 	"os"
-	"path"
 	"path/filepath"
-	"strings"
+	"time"
 
-	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/dashboards"
 	"github.com/elastic/beats/libbeat/kibana"
 )
 
-var exportAPI = "/api/kibana/dashboards/export"
-
-type manifest struct {
-	Dashboards []map[string]string `config:"dashboards"`
-}
-
-func makeURL(url, path string, params url.Values) string {
-	if len(params) == 0 {
-		return url + path
-	}
-
-	return strings.Join([]string{url, path, "?", params.Encode()}, "")
-}
-
-func Export(client *http.Client, conn string, spaceID string, dashboard string, out string) error {
-	params := url.Values{}
-
-	params.Add("dashboard", dashboard)
-
-	if spaceID != "" {
-		exportAPI = path.Join("/s", spaceID, exportAPI)
-	}
-	fullURL := makeURL(conn, exportAPI, params)
-	if !quiet {
-		log.Printf("Calling HTTP GET %v\n", fullURL)
-	}
-
-	req, err := http.NewRequest("GET", fullURL, nil)
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("GET HTTP request fails with: %v", err)
-	}
-
-	defer resp.Body.Close()
-
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return fmt.Errorf("fail to read response %s", err)
-	}
-
-	if resp.StatusCode != 200 {
-		return fmt.Errorf("HTTP GET %s fails with %s, %s", fullURL, resp.Status, body)
-	}
-
-	data, err := kibana.RemoveIndexPattern(body)
-	if err != nil {
-		return fmt.Errorf("fail to extract the index pattern: %v", err)
-	}
-
-	objects := data["objects"].([]interface{})
-	for _, obj := range objects {
-		o := obj.(common.MapStr)
-
-		// All fields are optional, so errors are not catched
-		decodeValue(o, "attributes.uiStateJSON")
-		decodeValue(o, "attributes.visState")
-		decodeValue(o, "attributes.optionsJSON")
-		decodeValue(o, "attributes.panelsJSON")
-		decodeValue(o, "attributes.kibanaSavedObjectMeta.searchSourceJSON")
-	}
-
-	data["objects"] = objects
-
-	// Create all missing directories
-	err = os.MkdirAll(filepath.Dir(out), 0755)
-	if err != nil {
-		return err
-	}
-
-	err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0644)
-	if !quiet {
-		log.Printf("The dashboard %s was exported under the %s file\n", dashboard, out)
-	}
-	return err
-}
-
-func decodeValue(data common.MapStr, key string) error {
-	v, err := data.GetValue(key)
-	if err != nil {
-		return err
-	}
-	s := v.(string)
-	var d interface{}
-	err = json.Unmarshal([]byte(s), &d)
-	if err != nil {
-		return fmt.Errorf("error decoding %s: %v", key, err)
-	}
-
-	data.Put(key, d)
-	return nil
-}
-
-func ReadManifest(file string) ([]map[string]string, error) {
-	cfg, err := common.LoadFile(file)
-	if err != nil {
-		return nil, fmt.Errorf("error reading manifest file: %v", err)
-	}
-
-	var manifest manifest
-	err = cfg.Unpack(&manifest)
-	if err != nil {
-		return nil, fmt.Errorf("error unpacking manifest: %v", err)
-	}
-	return manifest.Dashboards, nil
-}
+var (
+	indexPattern = false
+	quiet        = false
+)
 
-var indexPattern = false
-var quiet = false
+const (
+	kibanaTimeout = 90 * time.Second
+)
 
 func main() {
 	kibanaURL := flag.String("kibana", "http://localhost:5601", "Kibana URL")
@@ -157,42 +52,57 @@ func main() {
 	flag.Parse()
 	log.SetFlags(0)
 
-	transCfg := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // ignore expired SSL certificates
+	u, err := url.Parse(*kibanaURL)
+	if err != nil {
+		log.Fatalf("Error parsing Kibana URL: %v", err)
 	}
 
-	client := &http.Client{Transport: transCfg}
+	client, err := kibana.NewClientWithConfig(&kibana.ClientConfig{
+		Protocol: u.Scheme,
+		Host:     u.Host,
+		SpaceID:  *spaceID,
+		Timeout:  kibanaTimeout,
+	})
+	if err != nil {
+		log.Fatalf("Error while connecting to Kibana: %+v", err)
+	}
 
 	if len(*ymlFile) == 0 && len(*dashboard) == 0 {
 		flag.Usage()
 		log.Fatalf("Please specify a dashboard ID (-dashboard) or a manifest file (-yml)")
 	}
 
 	if len(*ymlFile) > 0 {
-		dashboards, err := ReadManifest(*ymlFile)
-		if err != nil {
-			log.Fatalf("%s", err)
-		}
-
-		for _, dashboard := range dashboards {
-			log.Printf("id=%s, name=%s\n", dashboard["id"], dashboard["file"])
-			directory := filepath.Join(filepath.Dir(*ymlFile), "_meta/kibana/6/dashboard")
-			err := os.MkdirAll(directory, 0755)
-			if err != nil {
-				log.Fatalf("fail to create directory %s: %v", directory, err)
-			}
-			err = Export(client, *kibanaURL, *spaceID, dashboard["id"], filepath.Join(directory, dashboard["file"]))
+		results, info, err := dashboards.ExportAllFromYml(client, *ymlFile)
+		for i, r := range results {
+			log.Printf("id=%s, name=%s\n", info.Dashboards[i].ID, info.Dashboards[i].File)
+			r = dashboards.DecodeExported(r)
+			err = dashboards.SaveToFile(r, info.Dashboards[i].File, filepath.Dir(*ymlFile), client.GetVersion())
 			if err != nil {
-				log.Fatalf("fail to export the dashboards: %s", err)
+				log.Fatalf("failed to export the dashboards: %s", err)
 			}
 		}
 		os.Exit(0)
 	}
 
 	if len(*dashboard) > 0 {
-		err := Export(client, *kibanaURL, *spaceID, *dashboard, *fileOutput)
+		result, err := dashboards.Export(client, *dashboard)
+		if err != nil {
+			log.Fatalf("Failed to export the dashboard: %s", err)
+		}
+		result = dashboards.DecodeExported(result)
+		bytes, err := json.Marshal(result)
+		if err != nil {
+			log.Fatalf("Failed to save the dashboard: %s", err)
+		}
+
+		err = ioutil.WriteFile(*fileOutput, bytes, 0644)
 		if err != nil {
-			log.Fatalf("fail to export the dashboards: %s", err)
+			log.Fatalf("Failed to save the dashboard: %s", err)
+
+		}
+		if !quiet {
+			log.Printf("The dashboard %s was exported under the %s file\n", *dashboard, *fileOutput)
 		}
 	}
 }

docs/devguide/newdashboards.asciidoc

@@ -213,10 +213,14 @@ https://github.com/elastic/beats/tree/master/dev-tools/cmd/dashboards[dev-tools]
 for exporting Kibana 5.x dashboards. See the dev-tools
 https://github.com/elastic/beats/tree/master/dev-tools/README.md[readme] for more info.
 
+Alternatively, if the scripts above are not available, you can use your Beat binary to export Kibana 6.0 dashboards or later.
+
 ==== Exporting Kibana 6.0 dashboards and newer
 
 The `dev-tools/cmd/export_dashboards.go` script helps you export your customized Kibana 6.0 dashboards and newer. You might need to export a single dashboard or all the dashboards available for a module or Beat.
 
+It is also possible to use a Beat binary to export.
+
 ===== Export a single Kibana dashboard
 
 To export a single dashboard for a module you can use the following command inside a Beat with modules:
@@ -226,6 +230,11 @@ To export a single dashboard for a module you can use the following command insi
 MODULE=redis ID=AV4REOpp5NkDleZmzKkE mage exportDashboard
 ---------------
 
+[source,shell]
+---------------
+./filebeat export dashboard -id 7fea2930-478e-11e7-b1f0-cb29bac6bf8b >> Filebeat-redis.json
+---------------
+
 This generates a `AV4REOpp5NkDleZmzKkE.json` file inside dashboard directory in the redis module.
 It contains all dependencies like visualizations and searches.
 
@@ -255,13 +264,18 @@ dashboards:
 
 Each dashboard is defined by an `id` and the name of json `file` where the dashboard is saved locally.
 
-By passing the yml file to the `export_dashboards.go` script, you can export all the dashboards defined:
+By passing the yml file to the `export_dashboards.go` script or to the Beat, you can export all the dashboards defined:
 
 [source,shell]
 -------------------
 go run dev-tools/cmd/dashboards/export_dashboards.go -yml filebeat/module/system/module.yml
 -------------------
 
+[source,shell]
+-------------------
+./filebeat export dashboard -yml filebeat/module/system/module.yml
+-------------------
+
 
 ===== Export dashboards from a Kibana Space
 
@@ -272,6 +286,8 @@ If you are using the Kibana Spaces feature and want to export dashboards from a
 go run dev-tools/cmd/dashboards/export_dashboards.go -space-id my-space [other-options]
 -------------------
 
+In case of running `export dashboard` of a Beat, you need to set the Space ID in `setup.kibana.space.id`.
+
 
 ==== Exporting Kibana 5.x dashboards
 

filebeat/scripts/generator/fileset/main.go

@@ -32,7 +32,7 @@ func generateFileset(module, fileset, modulesPath, beatsPath string) error {
 		return fmt.Errorf("fileset already exists: %s", fileset)
 	}
 
-	err := generator.CreateDirectories(filesetPath, []string{"", "_meta", "test", "config", "ingest"})
+	err := generator.CreateDirectories(filesetPath, "_meta", "test", "config", "ingest")
 	if err != nil {
 		return err
 	}

filebeat/scripts/generator/generator.go

@@ -35,7 +35,7 @@ func DirExists(dir string) bool {
 }
 
 // CreateDirectories create directories in baseDir
-func CreateDirectories(baseDir string, directories []string) error {
+func CreateDirectories(baseDir string, directories ...string) error {
 	for _, d := range directories {
 		p := path.Join(baseDir, d)
 		err := os.MkdirAll(p, 0750)

filebeat/scripts/generator/module/main.go

@@ -32,15 +32,20 @@ func generateModule(module, modulesPath, beatsPath string) error {
 		return fmt.Errorf("module already exists: %s", module)
 	}
 
-	err := generator.CreateDirectories(modulePath, []string{path.Join("_meta", "kibana", "6")})
+	err := generator.CreateDirectories(modulePath, "_meta")
 	if err != nil {
 		return err
 	}
 
 	replace := map[string]string{"module": module}
 	templatesPath := path.Join(beatsPath, "scripts", "module")
-	filesToCopy := []string{path.Join("_meta", "fields.yml"), path.Join("_meta", "docs.asciidoc"), path.Join("_meta", "config.yml"), path.Join("module.yml")}
-	generator.CopyTemplates(templatesPath, modulePath, filesToCopy, replace)
+	filesToCopy := []string{
+		path.Join("_meta", "fields.yml"),
+		path.Join("_meta", "docs.asciidoc"),
+		path.Join("_meta", "config.yml"),
+		"module.yml",
+	}
+	err = generator.CopyTemplates(templatesPath, modulePath, filesToCopy, replace)
 	if err != nil {
 		return err
 	}

libbeat/cmd/export/dashboard.go

@@ -20,11 +20,13 @@ package export
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/spf13/cobra"
 
 	"github.com/elastic/beats/libbeat/cmd/instance"
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/dashboards"
 	"github.com/elastic/beats/libbeat/kibana"
 )
 
@@ -35,6 +37,8 @@ func GenDashboardCmd(name, idxPrefix, beatVersion string) *cobra.Command {
 		Short: "Export defined dashboard to stdout",
 		Run: func(cmd *cobra.Command, args []string) {
 			dashboard, _ := cmd.Flags().GetString("id")
+			yml, _ := cmd.Flags().GetString("yml")
+			decode, _ := cmd.Flags().GetBool("decode")
 
 			b, err := instance.NewBeat(name, idxPrefix, beatVersion)
 			if err != nil {
@@ -58,16 +62,46 @@ func GenDashboardCmd(name, idxPrefix, beatVersion string) *cobra.Command {
 				os.Exit(1)
 			}
 
-			result, err := client.GetDashboard(dashboard)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "Error getting dashboard: %+v\n", err)
-				os.Exit(1)
+			// Export dashboards from yml file
+			if yml != "" {
+				results, info, err := dashboards.ExportAllFromYml(client, yml)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "Error getting dashboards from yml: %+v\n", err)
+					os.Exit(1)
+				}
+				for i, r := range results {
+					if decode {
+						r = dashboards.DecodeExported(r)
+					}
+					err = dashboards.SaveToFile(r, info.Dashboards[i].File, filepath.Dir(yml), client.GetVersion())
+					if err != nil {
+						fmt.Fprintf(os.Stderr, "Error saving dashboard '%s' to file '%s' : %+v\n",
+							info.Dashboards[i].ID, info.Dashboards[i].File, err)
+						os.Exit(1)
+					}
+				}
+				return
+			}
+
+			// Export single dashboard
+			if dashboard != "" {
+				result, err := dashboards.Export(client, dashboard)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "Error getting dashboard: %+v\n", err)
+					os.Exit(1)
+				}
+
+				if decode {
+					result = dashboards.DecodeExported(result)
+				}
+				fmt.Println(result.StringToPrint())
 			}
-			fmt.Println(result.StringToPrint())
 		},
 	}
 
 	genTemplateConfigCmd.Flags().String("id", "", "Dashboard id")
+	genTemplateConfigCmd.Flags().String("yml", "", "Yaml file containing list of dashboard ID and filename pairs")
+	genTemplateConfigCmd.Flags().Bool("decode", false, "Decode exported dashboard")
 
 	return genTemplateConfigCmd
 }