Added path.logs and made logging to files the default (#1437)

This adds the `-path.logs` CLI flag and corresponding config file
setting. By default it's set to `{path.home}/logs`. The rotating
file writer now writes to this path by default.

For DEB/RPM, the logs path is set to `/var/log/{beatname}` which
matches the previous default location. However, this is a breaking
change because the previous default was to send the logs to syslog.

* Added path.logs and made logging to files the default
* Updated CHANGELOG
* Removed unreachable code

Part of #1371.

CHANGELOG.asciidoc

@@ -16,6 +16,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha1...master[Check the HEAD d
 
 - Add scripts for managing the dashboards of a single Beat {pull}1359[1359]
 - On DEB/RPM installations, the binary files are now found under `/usr/share/{{beat_name}}/bin`, not in `/usr/bin`. {pull}1385[1385]
+- The logs are written by default to self rotating files, instead of syslog. {pull}1371[1371]
 
 *Packetbeat*
 
@@ -60,11 +61,15 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha1...master[Check the HEAD d
 - Failover and load balancing support in redis output. {pull}1353[1353]
 - Multiple-worker per host support for redis output. {pull}1353[1353]
 - Added ability to escape `${x}` in config files to avoid environment variable expansion {pull}1389[1389]
+- Configuration options and CLI flags for setting the home, data and config paths. {pull}1373[1373]
+- Configuration options and CLI flags for setting the default logs path. {pull}1437[1437]
 
 *Packetbeat*
+
 - Add support for OpenBSD. {pull}1347[1347]
 
 *Topbeat*
+
 - Add support for OpenBSD. {pull}1403[1403]
 
 *Filebeat*
@@ -76,7 +81,6 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha1...master[Check the HEAD d
 
 *Affecting all Beats*
 
-* Configuration options and CLI flags for setting the home, data and config paths. {pull}1373[1373]
 
 *Packetbeat*
 

dev-tools/packer/platforms/centos/init.j2

@@ -24,7 +24,7 @@ export PATH
 [ -f /etc/sysconfig/{{.beat_name}} ] && . /etc/sysconfig/{{.beat_name}}
 pidfile=${PIDFILE-/var/run/{{.beat_name}}.pid}
 agent=${BEATS_AGENT-/usr/share/{{.beat_name}}/bin/{{.beat_name}}}
-args="-c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}}"
+args="-c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}} -path.logs /var/log/{{.beat_name}}"
 test_args="-e -configtest"
 wrapper="/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god"
 wrapperopts="-r / -n -p $pidfile"

dev-tools/packer/platforms/centos/systemd.j2

@@ -5,7 +5,7 @@ Wants=network-online.target
 After=network-online.target
 
 [Service]
-ExecStart=/usr/share/{{.beat_name}}/bin/{{.beat_name}} -c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}}
+ExecStart=/usr/share/{{.beat_name}}/bin/{{.beat_name}} -c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}} -path.logs /var/log/{{.beat_name}}
 Restart=always
 
 [Install]

dev-tools/packer/platforms/debian/init.j2

@@ -17,7 +17,7 @@ PATH=/sbin:/usr/sbin:/bin:/usr/bin
 DESC="{{.beat_description}}"
 NAME="{{.beat_name}}"
 DAEMON=/usr/share/{{.beat_name}}/bin/$NAME
-DAEMON_ARGS="-c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}}"
+DAEMON_ARGS="-c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}} -path.logs /var/log/{{.beat_name}}"
 TEST_ARGS="-e -configtest"
 PIDFILE=/var/run/{{.beat_name}}.pid
 WRAPPER="/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god"

dev-tools/packer/platforms/debian/systemd.j2

@@ -5,7 +5,7 @@ Wants=network-online.target
 After=network-online.target
 
 [Service]
-ExecStart=/usr/share/{{.beat_name}}/bin/{{.beat_name}} -c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}}
+ExecStart=/usr/share/{{.beat_name}}/bin/{{.beat_name}} -c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}} -path.logs /var/log/{{.beat_name}}
 Restart=always
 
 [Install]

libbeat/logp/logp.go

@@ -5,8 +5,9 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
-	"runtime"
 	"strings"
+
+	"github.com/elastic/beats/libbeat/paths"
 )
 
 // cmd line flags
@@ -57,29 +58,19 @@ func Init(name string, config *Logging) error {
 		logLevel = LOG_DEBUG
 	}
 
-	var defaultToFiles, defaultToSyslog bool
-	var defaultFilePath string
-	if runtime.GOOS == "windows" {
-		// always disabled on windows
-		defaultToSyslog = false
-		defaultToFiles = true
-		defaultFilePath = fmt.Sprintf("C:\\ProgramData\\%s\\Logs", name)
-	} else {
-		defaultToSyslog = true
-		defaultToFiles = false
-		defaultFilePath = fmt.Sprintf("/var/log/%s", name)
-	}
+	// default log location is in the logs path
+	defaultFilePath := paths.Resolve(paths.Logs, "")
 
 	var toSyslog, toFiles bool
 	if config.ToSyslog != nil {
 		toSyslog = *config.ToSyslog
 	} else {
-		toSyslog = defaultToSyslog
+		toSyslog = false
 	}
 	if config.ToFiles != nil {
 		toFiles = *config.ToFiles
 	} else {
-		toFiles = defaultToFiles
+		toFiles = true
 	}
 
 	// toStderr disables logging to syslog/files

libbeat/paths/paths.go

@@ -25,20 +25,20 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-
-	"github.com/elastic/beats/libbeat/logp"
 )
 
 var (
 	homePath   = flag.String("path.home", "", "Home path")
 	configPath = flag.String("path.config", "", "Configuration path")
 	dataPath   = flag.String("path.data", "", "Data path")
+	logsPath   = flag.String("path.logs", "", "Logs path")
 )
 
 type Path struct {
 	Home   string
 	Config string
 	Data   string
+	Logs   string
 }
 
 // FileType is an enumeration type representing the file types.
@@ -49,6 +49,7 @@ const (
 	Home   FileType = "home"
 	Config FileType = "config"
 	Data   FileType = "data"
+	Logs   FileType = "logs"
 )
 
 // Paths is the Path singleton on which the top level functions from this
@@ -91,6 +92,7 @@ func (paths *Path) initPaths(cfg *Path) error {
 	paths.Home = cfg.Home
 	paths.Config = cfg.Config
 	paths.Data = cfg.Data
+	paths.Logs = cfg.Logs
 
 	// overwrite paths from CLI flags
 	if homePath != nil && len(*homePath) > 0 {
@@ -102,6 +104,9 @@ func (paths *Path) initPaths(cfg *Path) error {
 	if dataPath != nil && len(*dataPath) > 0 {
 		paths.Data = *dataPath
 	}
+	if logsPath != nil && len(*logsPath) > 0 {
+		paths.Logs = *logsPath
+	}
 
 	// default for the home path is the binary location
 	if len(paths.Home) == 0 {
@@ -123,6 +128,11 @@ func (paths *Path) initPaths(cfg *Path) error {
 		paths.Data = filepath.Join(paths.Home, "data")
 	}
 
+	// default for logs path
+	if len(paths.Logs) == 0 {
+		paths.Logs = filepath.Join(paths.Home, "logs")
+	}
+
 	return nil
 }
 
@@ -142,9 +152,10 @@ func (paths *Path) Resolve(fileType FileType, path string) string {
 		return filepath.Join(paths.Config, path)
 	case Data:
 		return filepath.Join(paths.Data, path)
+	case Logs:
+		return filepath.Join(paths.Logs, path)
 	default:
-		logp.WTF("Unknown file type: %s", fileType)
-		return ""
+		panic(fmt.Sprintf("Unknown file type: %s", fileType))
 	}
 }
 
@@ -157,6 +168,6 @@ func Resolve(fileType FileType, path string) string {
 
 // String returns a textual representation
 func (paths *Path) String() string {
-	return fmt.Sprintf("Home path: [%s] Config path: [%s] Data path: [%s]",
-		paths.Home, paths.Config, paths.Data)
+	return fmt.Sprintf("Home path: [%s] Config path: [%s] Data path: [%s] Logs path: [%s]",
+		paths.Home, paths.Config, paths.Data, paths.Logs)
 }

libbeat/paths/paths_test.go

@@ -60,6 +60,7 @@ func TestHomePath(t *testing.T) {
 	}
 
 	for _, test := range tests {
+		t.Log("Executing test", test)
 		homePath = test.CLIHome
 		cfg := Path{Home: test.CfgHome}
 		assert.NoError(t, Paths.initPaths(&cfg))
@@ -126,6 +127,7 @@ func TestDataPath(t *testing.T) {
 	}
 
 	for _, test := range tests {
+		t.Log("Executing test", test)
 		homePath = test.CLIHome
 		dataPath = test.CLIData
 		cfg := Path{Home: test.CfgHome, Data: test.CfgData}
@@ -135,3 +137,73 @@ func TestDataPath(t *testing.T) {
 	}
 
 }
+
+func TestLogsPath(t *testing.T) {
+	type io struct {
+		CLIHome    *string // cli flag home setting
+		CfgHome    string  // config file home setting
+		CLILogs    *string // cli flag for data setting
+		CfgLogs    string  // config file data setting
+		Path       string  // requested path
+		ResultLogs string  // expected logs path
+	}
+
+	binDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
+	assert.NoError(t, err)
+	tmp := "/tmp/"
+	root := "/root/"
+
+	tests := []io{
+		{
+			CLIHome:    nil,
+			CfgHome:    "",
+			CLILogs:    nil,
+			CfgLogs:    "",
+			Path:       "",
+			ResultLogs: filepath.Join(binDir, "logs"),
+		},
+		{
+			CLIHome:    nil,
+			CfgHome:    "",
+			CLILogs:    nil,
+			CfgLogs:    "",
+			Path:       "test",
+			ResultLogs: filepath.Join(binDir, "logs", "test"),
+		},
+		{
+			CLIHome:    nil,
+			CfgHome:    "/tmp/",
+			CLILogs:    nil,
+			CfgLogs:    "/root/logs",
+			Path:       "",
+			ResultLogs: "/root/logs",
+		},
+		{
+			CLIHome:    &tmp,
+			CfgHome:    "",
+			CLILogs:    nil,
+			CfgLogs:    "/root/logs",
+			Path:       "",
+			ResultLogs: "/root/logs",
+		},
+		{
+			CLIHome:    &tmp,
+			CfgHome:    "",
+			CLILogs:    &root,
+			CfgLogs:    "/root/logs",
+			Path:       "",
+			ResultLogs: "/root",
+		},
+	}
+
+	for _, test := range tests {
+		t.Log("Executing test", test)
+		homePath = test.CLIHome
+		logsPath = test.CLILogs
+		cfg := Path{Home: test.CfgHome, Logs: test.CfgLogs}
+		assert.NoError(t, Paths.initPaths(&cfg))
+
+		assert.Equal(t, test.ResultLogs, Resolve(Logs, test.Path))
+	}
+
+}