Setup an independent build for x-pack/winlogbeat

.gitignore

@@ -3,12 +3,13 @@
 /.idea
 /.vscode
 /build
-/*/data
-/*/logs
-/*/fields.yml
 /*/*.template*.json
 **/html_docs
-/*/_meta/kibana.generated
+*beat/fields.yml
+*beat/_meta/kibana.generated
+*beat/build
+*beat/logs
+*beat/data
 
 # Files
 .DS_Store
@@ -19,6 +20,7 @@ coverage.out
 .python-version
 beat.db
 *.keystore
+mage_output_file.go
 
 # Editor swap files
 *.swp

Makefile

@@ -16,7 +16,7 @@ XPACK_SUFFIX=x-pack/
 # PROJECTS_XPACK_PKG is a list of Beats that have independent packaging support
 # in the x-pack directory (rather than having the OSS build produce both sets
 # of artifacts). This will be removed once we complete the transition.
-PROJECTS_XPACK_PKG=x-pack/auditbeat x-pack/filebeat x-pack/metricbeat
+PROJECTS_XPACK_PKG=x-pack/auditbeat x-pack/filebeat x-pack/metricbeat x-pack/winlogbeat
 # PROJECTS_XPACK_MAGE is a list of Beats whose primary build logic is based in
 # Mage. For compatibility with CI testing these projects support a subset of the
 # makefile targets. After all Beats converge to primarily using Mage we can

dev-tools/.gitignore

@@ -1 +1 @@
-build
+/build

dev-tools/cmd/module_fields/module_fields.go

@@ -23,7 +23,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"path"
+	"path/filepath"
 
 	"github.com/elastic/beats/libbeat/asset"
 	"github.com/elastic/beats/libbeat/generator/fields"
@@ -73,6 +73,11 @@ func main() {
 		log.Fatalf("Error fetching modules: %v", err)
 	}
 
+	wd, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("Failed to determine working directory: %v", err)
+	}
+
 	for _, module := range modules {
 		files, err := fields.CollectFiles(module, dir)
 		if err != nil {
@@ -89,12 +94,17 @@ func main() {
 			log.Fatalf("error fetching files for package %v: %v", module, err)
 		}
 
-		bs, err := asset.CreateAsset(license, beatName, module, module, data, "asset.ModuleFieldsPri", dir+"/"+module)
+		p, err := filepath.Rel(wd, filepath.Join(dir, module))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		bs, err := asset.CreateAsset(license, beatName, module, module, data, "asset.ModuleFieldsPri", filepath.ToSlash(p))
 		if err != nil {
 			log.Fatalf("Error creating golang file from template: %v", err)
 		}
 
-		err = ioutil.WriteFile(path.Join(dir, module, "fields.go"), bs, 0644)
+		err = ioutil.WriteFile(filepath.Join(dir, module, "fields.go"), bs, 0644)
 		if err != nil {
 			log.Fatalf("Error writing fields.go: %v", err)
 		}

dev-tools/mage/.gitignore

@@ -1 +1 @@
-build
+/build

dev-tools/mage/dashboard.go

@@ -19,8 +19,11 @@ package mage
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
+	"strconv"
 
+	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
 )
 
@@ -33,11 +36,9 @@ func ExportDashboard() error {
 
 	id := EnvOr("ID", "")
 	if id == "" {
-		return fmt.Errorf("Dashboad ID must be specified")
+		return fmt.Errorf("Dashboard ID must be specified")
 	}
 
-	kibanaURL := EnvOr("KIBANA_URL", "")
-
 	beatsDir, err := ElasticBeatsDir()
 	if err != nil {
 		return err
@@ -51,9 +52,37 @@ func ExportDashboard() error {
 		"-output", file, "-dashboard", id,
 	)
 
-	if kibanaURL != "" {
-		return dashboardCmd("-kibana", kibanaURL)
-	} else {
-		return dashboardCmd()
+	return dashboardCmd()
+}
+
+// ImportDashboards imports dashboards to Kibana using the Beat setup command.
+//
+// Depends on: build, dashboard
+//
+// Optional environment variables:
+// - KIBANA_URL: URL of Kibana
+// - KIBANA_ALWAYS: Connect to Kibana without checking ES version. Default true.
+// - ES_URL: URL of Elasticsearch (only used with KIBANA_ALWAYS=false).
+func ImportDashboards(buildDep, dashboardDep interface{}) error {
+	mg.Deps(buildDep, dashboardDep)
+
+	setupDashboards := sh.RunCmd(CWD(BeatName+binaryExtension(GOOS)),
+		"setup", "--dashboards",
+		"-E", "setup.dashboards.directory="+kibanaBuildDir)
+
+	kibanaAlways := true
+	if b, err := strconv.ParseBool(os.Getenv("KIBANA_ALWAYS")); err == nil {
+		kibanaAlways = b
+	}
+
+	var args []string
+	if kibanaURL := EnvOr("KIBANA_URL", ""); kibanaURL != "" {
+		args = append(args, "-E", "setup.kibana.host="+kibanaURL)
 	}
+	if esURL := EnvOr("ES_URL", ""); !kibanaAlways && esURL != "" {
+		args = append(args, "-E", "setup.elasticsearch.host="+esURL)
+	}
+	args = append(args, "-E", "setup.dashboards.always_kibana="+strconv.FormatBool(kibanaAlways))
+
+	return setupDashboards(args...)
 }

dev-tools/mage/docs.go

@@ -18,18 +18,57 @@
 package mage
 
 import (
+	"context"
+	"fmt"
 	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"syscall"
 
 	"github.com/magefile/mage/sh"
+	"github.com/pkg/errors"
+)
+
+const (
+	elasticDocsRepoURL = "https://github.com/elastic/docs.git"
 )
 
 type docsBuilder struct{}
 
+type asciidocParams struct {
+	name      string
+	indexFile string
+}
+
+// DocsOption is a documentation generation option for controlling how the docs
+// are built.
+type DocsOption func(params *asciidocParams)
+
+// DocsName specifies the documentation's name (default to BeatName).
+func DocsName(name string) DocsOption {
+	return func(params *asciidocParams) {
+		params.name = name
+	}
+}
+
+// DocsIndexFile specifies the index file (defaults to docs/index.asciidoc).
+func DocsIndexFile(file string) DocsOption {
+	return func(params *asciidocParams) {
+		params.indexFile = file
+	}
+}
+
 // Docs holds the utilities for building documentation.
 var Docs = docsBuilder{}
 
 // FieldDocs generates docs/fields.asciidoc from the specified fields.yml file.
-func (b docsBuilder) FieldDocs(fieldsYML string) error {
+func (docsBuilder) FieldDocs(fieldsYML string) error {
 	// Run the docs_collector.py script.
 	ve, err := PythonVirtualenv()
 	if err != nil {
@@ -46,10 +85,100 @@ func (b docsBuilder) FieldDocs(fieldsYML string) error {
 		return err
 	}
 
+	// TODO: Port this script to Go.
 	log.Println(">> Generating docs/fields.asciidoc for", BeatName)
 	return sh.Run(python, LibbeatDir("scripts/generate_fields_docs.py"),
 		fieldsYML,                     // Path to fields.yml.
 		BeatName,                      // Beat title.
 		esBeats,                       // Path to general beats folder.
 		"--output_path", OSSBeatDir()) // It writes to {output_path}/docs/fields.asciidoc.
 }
+
+func (b docsBuilder) AsciidocBook(opts ...DocsOption) error {
+	params := asciidocParams{
+		name:      BeatName,
+		indexFile: CWD("docs/index.asciidoc"),
+	}
+	for _, opt := range opts {
+		opt(&params)
+	}
+
+	repo, err := GetProjectRepoInfo()
+	if err != nil {
+		return err
+	}
+
+	cloneDir := CreateDir(filepath.Join(repo.RootDir, "build/elastic_docs_repo"))
+
+	// Clone if elastic_docs_repo does not exist.
+	if _, err := os.Stat(cloneDir); err != nil {
+		log.Println("Cloning elastic/docs to", cloneDir)
+		if err = sh.Run("git", "clone", "--depth=1", elasticDocsRepoURL, cloneDir); err != nil {
+			return err
+		}
+	} else {
+		log.Println("Using existing elastic/docs at", cloneDir)
+	}
+
+	// Render HTML.
+	htmlDir := CWD("build/html_docs", params.name)
+	args := []string{
+		filepath.Join(cloneDir, "build_docs.pl"),
+		"--chunk=1",
+		"--doc", params.indexFile,
+		"--out", htmlDir,
+	}
+	fmt.Println(">> Building HTML docs at", filepath.Join(htmlDir, "index.html"))
+	if err := sh.Run("perl", args...); err != nil {
+		return err
+	}
+
+	// Serve docs with and HTTP server and open the browser.
+	if preview, _ := strconv.ParseBool(os.Getenv("PREVIEW")); preview {
+		srv := b.servePreview(htmlDir)
+		url := "http://" + srv.Addr
+		fmt.Println("Serving docs preview at", url)
+		b.openBrowser(url)
+
+		// Wait
+		fmt.Println("Ctrl+C to stop")
+		sigs := make(chan os.Signal, 1)
+		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+		<-sigs
+		srv.Shutdown(context.Background())
+	}
+	return nil
+}
+
+// open opens the specified URL in the default browser.
+func (docsBuilder) openBrowser(url string) error {
+	var cmd string
+	var args []string
+
+	switch runtime.GOOS {
+	case "darwin":
+		cmd = "open"
+	case "windows":
+		cmd = "cmd"
+		args = []string{"/c", "start"}
+	default:
+		cmd = "xdg-open"
+	}
+	args = append(args, url)
+	return exec.Command(cmd, args...).Start()
+}
+
+func (docsBuilder) servePreview(dir string) *http.Server {
+	server := &http.Server{
+		Addr:    net.JoinHostPort("localhost", EnvOr("PREVIEW_PORT", "8000")),
+		Handler: http.FileServer(http.Dir(dir)),
+	}
+
+	go func() {
+		if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			panic(errors.Wrap(err, "failed to start docs preview"))
+		}
+	}()
+
+	return server
+}