port fields.yml collector to Golang

elastic · Jul 3, 2018 · 789170a · 789170a
1 parent 6544fb1
commit 789170a
Show file tree

Hide file tree

Showing 6 changed files with 64 additions and 3 deletions.
diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go
@@ -30,6 +30,7 @@ import (
 	"github.com/elastic/beats/libbeat/cfgfile"
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/common/cfgwarn"
+	"github.com/elastic/beats/libbeat/generator/fields"
 	"github.com/elastic/beats/libbeat/kibana"
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/monitoring"
@@ -149,6 +150,10 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
 		return nil, err
 	}
 
+	b.CollectFieldsYmlCallback = func() ([]*fields.YmlFile, error) {
+		return fields.CollectModuleFiles("module")
+	}
+
 	return fb, nil
 }
 

diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go
@@ -19,10 +19,12 @@ package beater
 
 import (
 	"fmt"
+	"path/filepath"
 	"time"
 
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/generator/fields"
 	"github.com/elastic/beats/libbeat/logp"
 
 	"github.com/elastic/beats/heartbeat/config"
@@ -66,6 +68,12 @@ func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) {
 		scheduler: sched,
 		manager:   manager,
 	}
+
+	pathToFields := filepath.Join("monitors", "active")
+	b.CollectFieldsYmlCallback = func() ([]*fields.YmlFile, error) {
+		return fields.CollectModuleFiles(pathToFields)
+	}
+
 	return bt, nil
 }
 

diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go
@@ -19,6 +19,7 @@ package beat
 
 import (
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/generator/fields"
 )
 
 // Creator initializes and configures a new Beater instance used to execute
@@ -52,10 +53,11 @@ type Beat struct {
 	Info      Info     // beat metadata.
 	Publisher Pipeline // Publisher pipeline
 
-	SetupMLCallback SetupMLCallback // setup callback for ML job configs
-	InSetupCmd      bool            // this is set to true when the `setup` command is called
-
+	SetupMLCallback            SetupMLCallback            // setup callback for ML job configs
 	OverwritePipelinesCallback OverwritePipelinesCallback // ingest pipeline loader callback
+	CollectFieldsYmlCallback   CollectFieldsYmlCallback   // fields.yml collector of the Beat
+	InSetupCmd                 bool                       // this is set to true when the `setup` command is called
+
 	// XXX: remove Config from public interface.
 	//      It's currently used by filebeat modules to setup the Ingest Node
 	//      pipeline and ML jobs.
@@ -79,3 +81,6 @@ type SetupMLCallback func(*Beat, *common.Config) error
 // OverwritePipelinesCallback can be used by the Beat to register Ingest pipeline loader
 // for the enabled modules.
 type OverwritePipelinesCallback func(*common.Config) error
+
+// CollectFieldsYmlCallback TODO
+type CollectFieldsYmlCallback func() ([]*fields.YmlFile, error)
diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go
@@ -45,6 +45,7 @@ import (
 	"github.com/elastic/beats/libbeat/common/file"
 	"github.com/elastic/beats/libbeat/common/seccomp"
 	"github.com/elastic/beats/libbeat/dashboards"
+	"github.com/elastic/beats/libbeat/generator/fields"
 	"github.com/elastic/beats/libbeat/keystore"
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/logp/configure"
@@ -459,6 +460,38 @@ func (b *Beat) Setup(bt beat.Creator, template, dashboards, machineLearning, pip
 	}())
 }
 
+// Setup registers ES index template and kibana dashboards
+func (b *Beat) GenerateGlobalFields(bt beat.Creator, beatsPath string) error {
+	return handleError(func() error {
+		err := b.Init()
+		if err != nil {
+			return err
+		}
+
+		_, err = b.createBeater(bt)
+		if err != nil {
+			return err
+		}
+
+		if b.CollectFieldsYmlCallback == nil {
+			return fmt.Errorf("fields.yml collector callback is not set")
+		}
+
+		fieldFiles, err := b.CollectFieldsYmlCallback()
+		if err != nil {
+			return err
+		}
+
+		err = fields.Generate(beatsPath, b.Beat.Info.Beat, fieldFiles)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("Generated global fields.yml under _meta/fields.generated.yml")
+		return nil
+	}())
+}
+
 // handleFlags parses the command line flags. It handles the '-version' flag
 // and invokes the HandleFlags callback if implemented by the Beat.
 func (b *Beat) handleFlags() error {

diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go
@@ -28,6 +28,7 @@ import (
 	"github.com/elastic/beats/libbeat/cfgfile"
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/common/cfgwarn"
+	"github.com/elastic/beats/libbeat/generator/fields"
 	"github.com/elastic/beats/libbeat/logp"
 	mbautodiscover "github.com/elastic/beats/metricbeat/autodiscover"
 	"github.com/elastic/beats/metricbeat/mb"
@@ -176,6 +177,10 @@ func newMetricbeat(b *beat.Beat, c *common.Config, options ...Option) (*Metricbe
 		}
 	}
 
+	b.CollectFieldsYmlCallback = func() ([]*fields.YmlFile, error) {
+		return fields.CollectModuleFiles("module")
+	}
+
 	return metricbeat, nil
 }
 

diff --git a/packetbeat/beater/packetbeat.go b/packetbeat/beater/packetbeat.go
@@ -28,6 +28,7 @@ import (
 
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/generator/fields"
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/processors"
 	"github.com/elastic/beats/libbeat/service"
@@ -104,6 +105,10 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
 		return nil, err
 	}
 
+	b.CollectFieldsYmlCallback = func() ([]*fields.YmlFile, error) {
+		return fields.CollectModuleFiles("protos")
+	}
+
 	return pb, nil
 }