Snapshot expvar maps for periodic logging

Part of elastic#1931, this adds logic to descend into maps. This is useful especially
to support the metricbeat fetches stats. Locking should be safe as the expvar.Map
Do function takes a read lock on the map.

Note that expvars of type Func are still not included, which is why
the memstats are not logged.

libbeat/logp/logp.go

@@ -157,14 +157,28 @@ func getLogLevel(config *Logging) (Priority, error) {
 	return level, nil
 }
 
-// snapshotExpvars iterates through all the defined expvars, and for the top
-// level vars that are integers it snapshots the name and value in a separate
-// map.
+// snapshotMap recursively walks expvar Maps and records their integer expvars
+// in a separate flat map.
+func snapshotMap(varsMap map[string]int64, path string, mp *expvar.Map) {
+	mp.Do(func(kv expvar.KeyValue) {
+		switch kv.Value.(type) {
+		case *expvar.Int:
+			varsMap[path+"."+kv.Key], _ = strconv.ParseInt(kv.Value.String(), 10, 64)
+		case *expvar.Map:
+			snapshotMap(varsMap, path+"."+kv.Key, kv.Value.(*expvar.Map))
+		}
+	})
+}
+
+// snapshotExpvars iterates through all the defined expvars, and for the vars
+// that are integers it snapshots the name and value in a separate (flat) map.
 func snapshotExpvars(varsMap map[string]int64) {
 	expvar.Do(func(kv expvar.KeyValue) {
 		switch kv.Value.(type) {
 		case *expvar.Int:
 			varsMap[kv.Key], _ = strconv.ParseInt(kv.Value.String(), 10, 64)
+		case *expvar.Map:
+			snapshotMap(varsMap, kv.Key, kv.Value.(*expvar.Map))
 		}
 	})
 }

libbeat/logp/logp_test.go

@@ -19,6 +19,21 @@ func TestSnapshotExpvars(t *testing.T) {
 	assert.Equal(t, vals["test"], int64(42))
 }
 
+func TestSnapshotExpvarsMap(t *testing.T) {
+	test := expvar.NewMap("testMap")
+	test.Add("hello", 42)
+
+	map2 := new(expvar.Map).Init()
+	map2.Add("test", 5)
+	test.Set("map2", map2)
+
+	vals := map[string]int64{}
+	snapshotExpvars(vals)
+
+	assert.Equal(t, vals["testMap.hello"], int64(42))
+	assert.Equal(t, vals["testMap.map2.test"], int64(5))
+}
+
 func TestBuildMetricsOutput(t *testing.T) {
 	test := expvar.NewInt("testLog")
 	test.Add(1)