Replace generate_imports python script with mage imports

dev-tools/cmd/module_include_list/module_include_list.go

@@ -50,18 +50,22 @@ Options:
 `[1:]
 
 var (
-	license    string
-	pkg        string
-	outFile    string
-	moduleDirs stringSliceFlag
-	importDirs stringSliceFlag
+	license           string
+	pkg               string
+	outFile           string
+	buildTags         string
+	moduleDirs        stringSliceFlag
+	moduleExcludeDirs stringSliceFlag
+	importDirs        stringSliceFlag
 )
 
 func init() {
 	flag.StringVar(&license, "license", "ASL2", "License header for generated file (ASL2 or Elastic).")
 	flag.StringVar(&pkg, "pkg", "include", "Package name.")
 	flag.StringVar(&outFile, "out", "include/list.go", "Output file.")
+	flag.StringVar(&buildTags, "buildTags", "", "Build Tags.")
 	flag.Var(&moduleDirs, "moduleDir", "Directory to search for modules to include")
+	flag.Var(&moduleExcludeDirs, "moduleExcludeDirs", "Directory to exclude from the list")
 	flag.Var(&importDirs, "import", "Directory to include")
 	flag.Usage = usageFlag
 }
@@ -138,9 +142,10 @@ func main() {
 	// Populate the template.
 	var buf bytes.Buffer
 	err = Template.Execute(&buf, Data{
-		License: license,
-		Package: pkg,
-		Imports: imports,
+		License:   license,
+		Package:   pkg,
+		BuildTags: buildTags,
+		Imports:   imports,
 	})
 	if err != nil {
 		log.Fatalf("Failed executing template: %v", err)
@@ -168,7 +173,7 @@ var Template = template.Must(template.New("normalizations").Funcs(map[string]int
 {{ .License | trim }}
 
 // Code generated by beats/dev-tools/cmd/module_include_list/module_include_list.go - DO NOT EDIT.
-
+{{ .BuildTags }}
 package {{ .Package }}
 
 import (
@@ -180,9 +185,10 @@ import (
 `[1:]))
 
 type Data struct {
-	License string
-	Package string
-	Imports []string
+	License   string
+	Package   string
+	BuildTags string
+	Imports   []string
 }
 
 //stringSliceFlag is a flag type that allows more than one value to be specified.
@@ -211,6 +217,16 @@ func findModuleAndDatasets() ([]string, error) {
 
 		for _, metaDir := range metaDirs {
 			// Strip off _meta.
+			skipDir := false
+			for _, excludeModule := range moduleExcludeDirs {
+				if strings.Contains(metaDir, excludeModule) {
+					skipDir = true
+					break
+				}
+			}
+			if skipDir {
+				continue
+			}
 			dirs = append(dirs, filepath.Dir(metaDir))
 		}
 	}

dev-tools/mage/fields.go

@@ -143,13 +143,33 @@ func GenerateModuleFieldsGo(moduleDir string) error {
 // GenerateModuleIncludeListGo generates an include/list.go file containing
 // a import statement for each module and dataset.
 func GenerateModuleIncludeListGo() error {
-	return GenerateIncludeListGo(nil, []string{"module"})
+	return GenerateIncludeListGo(nil, []string{"module"}, nil, "include/list.go", "")
+}
+
+// GenerateOSSMetricbeatModuleIncludeListGo generates include/list_{suffix}.go files containing
+// a import statement for each module and dataset.
+func GenerateOSSMetricbeatModuleIncludeListGo() error {
+	err := GenerateIncludeListGo(
+		nil, []string{"module"},
+		[]string{"module/docker", "module/kubernetes"},
+		"include/list_common.go", "")
+	if err != nil {
+		return err
+	}
+	err = GenerateIncludeListGo(
+		nil, []string{"module/docker", "module/kubernetes"},
+		nil,
+		"include/list_docker.go", "\n// +build linux darwin windows\n")
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 // GenerateIncludeListGo generates an include/list.go file containing imports
 // for the packages that match the paths (or globs) in importDirs (optional)
 // and moduleDirs (optional).
-func GenerateIncludeListGo(importDirs []string, moduleDirs []string) error {
+func GenerateIncludeListGo(importDirs []string, moduleDirs []string, modulesToExclude []string, outfile string, buildTags string) error {
 	const moduleIncludeListCmdPath = "dev-tools/cmd/module_include_list/module_include_list.go"
 
 	beatsDir, err := ElasticBeatsDir()
@@ -160,6 +180,7 @@ func GenerateIncludeListGo(importDirs []string, moduleDirs []string) error {
 	includeListCmd := sh.RunCmd("go", "run",
 		filepath.Join(beatsDir, moduleIncludeListCmdPath),
 		"-license", toLibbeatLicenseName(BeatLicense),
+		"-out", outfile, "-buildTags", buildTags,
 	)
 
 	var args []string
@@ -175,7 +196,12 @@ func GenerateIncludeListGo(importDirs []string, moduleDirs []string) error {
 		}
 		args = append(args, "-moduleDir", dir)
 	}
-
+	for _, dir := range modulesToExclude {
+		if !filepath.IsAbs(dir) {
+			dir = CWD(dir)
+		}
+		args = append(args, "-moduleExcludeDirs", dir)
+	}
 	return includeListCmd(args...)
 }
 

filebeat/magefile.go

@@ -112,7 +112,7 @@ func configYML() error {
 
 // includeList generates include/list.go with imports for inputs.
 func includeList() error {
-	return devtools.GenerateIncludeListGo([]string{"input/*"}, []string{"module"})
+	return devtools.GenerateIncludeListGo([]string{"input/*"}, []string{"module"}, nil, "include/list.go", "")
 }
 
 // Fields generates fields.yml and fields.go files for the Beat.

metricbeat/Makefile

@@ -41,9 +41,9 @@ configs: python-env
 
 # Generates imports for all modules and metricsets
 .PHONY: imports
-imports: python-env
+imports:
 	@mkdir -p include
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/generate_imports.py ${BEAT_PATH}
+	mage imports
 
 # Runs all collection steps and updates afterwards
 .PHONY: collect

metricbeat/magefile.go

@@ -91,6 +91,12 @@ func Config() {
 	mg.Deps(configYML, metricbeat.GenerateDirModulesD)
 }
 
+// Imports generates an include/list_{suffix}.go file containing
+// a import statement for each module and dataset.
+func Imports() error {
+	return devtools.GenerateOSSMetricbeatModuleIncludeListGo()
+}
+
 func configYML() error {
 	return devtools.Config(devtools.AllConfigTypes, metricbeat.OSSConfigFileParams(), ".")
 }

packetbeat/magefile.go

@@ -150,7 +150,7 @@ func Config() error {
 }
 
 func includeList() error {
-	return devtools.GenerateIncludeListGo([]string{"protos/*"}, nil)
+	return devtools.GenerateIncludeListGo([]string{"protos/*"}, nil, nil, "include/list.go", "")
 }
 
 // Fields generates fields.yml and fields.go files for the Beat.

x-pack/filebeat/magefile.go

@@ -134,7 +134,7 @@ func Update() {
 }
 
 func includeList() error {
-	return devtools.GenerateIncludeListGo([]string{"input/*", "processors/*"}, []string{"module"})
+	return devtools.GenerateIncludeListGo([]string{"input/*", "processors/*"}, []string{"module"}, nil, "include/list.go", "")
 }
 
 // IntegTest executes integration tests (it uses Docker to run the tests).