[WIP] Introduce local fields generation

auditbeat/magefile.go

@@ -107,17 +107,11 @@ func Config() error {
 
 // Fields generates fields.yml and fields.go files for the Beat.
 func Fields() {
-	mg.Deps(libbeatAndAuditbeatCommonFieldsGo, moduleFieldsGo)
-	mg.Deps(fieldsYML)
+	mg.Deps(beatFieldsGo, moduleFieldsGo, fieldsYML)
 }
 
-// libbeatAndAuditbeatCommonFieldsGo generates a fields.go containing both
-// libbeat and auditbeat's common fields.
-func libbeatAndAuditbeatCommonFieldsGo() error {
-	if err := mage.GenerateFieldsYAML(); err != nil {
-		return err
-	}
-	return mage.GenerateAllInOneFieldsGo()
+func beatFieldsGo() error {
+	return mage.GenerateFieldsGo("beat")
 }
 
 // moduleFieldsGo generates a fields.go for each module.

dev-tools/mage/clean.go

@@ -35,6 +35,11 @@ var DefaultCleanPaths = []string{
 	"_meta/kibana.generated",
 	"_meta/kibana/6/index-pattern/{{.BeatName}}.json",
 	"_meta/kibana/7/index-pattern/{{.BeatName}}.json",
+	// Generated fields files
+	"include/fields.go",
+	"include/module.go",
+	"include/beat.go",
+	"include/libbeat.go",
 }
 
 // Clean clean generated build artifacts.

dev-tools/mage/fields.go

@@ -19,6 +19,7 @@ package mage
 
 import (
 	"path/filepath"
+	"strings"
 
 	"github.com/pkg/errors"
 
@@ -60,13 +61,8 @@ func generateFieldsYAML(baseDir, output string, moduleDirs ...string) error {
 	return globalFieldsCmd(moduleDirs...)
 }
 
-// GenerateAllInOneFieldsGo generates an all-in-one fields.go file.
-func GenerateAllInOneFieldsGo() error {
-	return GenerateFieldsGo("fields.yml", "include/fields.go")
-}
-
-// GenerateFieldsGo generates a .go file containing the fields.yml data.
-func GenerateFieldsGo(fieldsYML, out string) error {
+// GenerateFieldsGo generates a *.go file from the related *.yml file in the build directory.
+func GenerateFieldsGo(name string) error {
 	const assetCmdPath = "dev-tools/cmd/asset/asset.go"
 
 	beatsDir, err := ElasticBeatsDir()
@@ -76,10 +72,11 @@ func GenerateFieldsGo(fieldsYML, out string) error {
 
 	assetCmd := sh.RunCmd("go", "run",
 		filepath.Join(beatsDir, assetCmdPath),
-		"-pkg", "include",
-		"-in", fieldsYML,
-		"-out", createDir(out),
 		"-license", toLibbeatLicenseName(BeatLicense),
+		"-pkg", "include",
+		"-in", "./build/fields/"+name+".yml",
+		"-out", "include/"+name+".go",
+		"-name", strings.Title(name),
 		BeatName,
 	)
 

filebeat/magefile.go

@@ -117,19 +117,11 @@ func includeList() error {
 
 // Fields generates fields.yml and fields.go files for the Beat.
 func Fields() {
-	mg.Deps(libbeatAndFilebeatCommonFieldsGo, moduleFieldsGo)
+	mg.Deps(moduleFieldsGo)
+	mage.GenerateFieldsGo("beat")
 	mg.Deps(fieldsYML)
 }
 
-// libbeatAndFilebeatCommonFieldsGo generates a fields.go containing both
-// libbeat and filebeat's common fields.
-func libbeatAndFilebeatCommonFieldsGo() error {
-	if err := mage.GenerateFieldsYAML(); err != nil {
-		return err
-	}
-	return mage.GenerateAllInOneFieldsGo()
-}
-
 // moduleFieldsGo generates a fields.go for each module.
 func moduleFieldsGo() error {
 	return mage.GenerateModuleFieldsGo("module")

heartbeat/magefile.go

@@ -100,6 +100,8 @@ func Update() error {
 
 // Fields generates a fields.yml for the Beat.
 func Fields() error {
+	mage.GenerateFieldsGo("beat")
+	mage.GenerateFieldsGo("module")
 	return mage.GenerateFieldsYAML("monitors/active")
 }
 

journalbeat/magefile.go

@@ -107,6 +107,8 @@ func Update() error {
 
 // Fields generates a fields.yml for the Beat.
 func Fields() error {
+	mage.GenerateFieldsGo("beat")
+
 	return mage.GenerateFieldsYAML()
 }
 

libbeat/beat/beat.go

@@ -20,6 +20,9 @@ package beat
 import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/management"
+
+	// Include basic fields
+	_ "github.com/elastic/beats/libbeat/include"
 )
 
 // Creator initializes and configures a new Beater instance used to execute

libbeat/cmd/instance/beat.go

@@ -193,11 +193,19 @@ func NewBeat(name, indexPrefix, v string) (*Beat, error) {
 		return nil, err
 	}
 
+	// All Beat need the libbeat fields
+	libbeatFields, err := asset.GetFields("libbeat")
+	if err != nil {
+		return nil, err
+	}
+
 	fields, err := asset.GetFields(name)
 	if err != nil {
 		return nil, err
 	}
 
+	fields = append(libbeatFields[:], fields[:]...)
+
 	id, err := uuid.NewV4()
 	if err != nil {
 		return nil, err