govppmux stats REST handler (#1396)

* Add govppmux stats handler for REST and improve stats

Signed-off-by: Ondrej Fabry <[email protected]>

* Allow disabling status publishing and interface stats collection

Signed-off-by: Ondrej Fabry <[email protected]>

Author: ondrej-fabry

pkg/metrics/metrics.go

@@ -16,12 +16,13 @@ package metrics
 
 import (
 	"encoding/json"
+	"math"
 	"sort"
 	"time"
 )
 
 // RoundDuration is the default value used for rounding durations.
-var RoundDuration = time.Microsecond * 10
+var RoundDuration = time.Millisecond * 1
 
 type Calls map[string]*CallStats
 
@@ -32,27 +33,39 @@ func (m Calls) MarshalJSON() ([]byte, error) {
 		calls = append(calls, s)
 	}
 	sort.Slice(calls, func(i, j int) bool {
-		return calls[i].Total > calls[j].Total
+		return calls[i].Count > calls[j].Count
 	})
+
+	/*var buf bytes.Buffer
+	buf.WriteByte('{')
+	for i, c := range calls {
+		if i > 0 {
+			buf.WriteByte(',')
+		}
+		buf.WriteString(fmt.Sprintf(`"%s":{"count":%d}`, c.Name, c.Count))
+	}
+	buf.WriteByte('}')
+	return buf.Bytes(), nil*/
+
 	return json.Marshal(calls)
 }
 
 // CallStats represents generic stats for call metrics.
 type CallStats struct {
-	Name  string `json:",omitempty"`
-	Count uint64
-	Total Duration
-	Avg   Duration
-	Min   Duration
-	Max   Duration
+	Name  string  `json:"name,omitempty"`
+	Count uint64  `json:"count"`
+	Total float64 `json:"total,omitempty"`
+	Avg   float64 `json:"avg,omitempty"`
+	Min   float64 `json:"min,omitempty"`
+	Max   float64 `json:"max,omitempty"`
 }
 
 // Increment increments call count and recalculates durations
 func (m *CallStats) Increment(d time.Duration) {
-	took := Duration(d)
+	took := d.Round(RoundDuration).Seconds()
 	m.Count++
-	m.Total += took
-	m.Avg = m.Total / Duration(m.Count)
+	m.Total = round(m.Total + took)
+	m.Avg = round(m.Total / float64(m.Count))
 	if took > m.Max {
 		m.Max = took
 	}
@@ -61,23 +74,17 @@ func (m *CallStats) Increment(d time.Duration) {
 	}
 }
 
-/*
+func round(n float64) float64 {
+	return math.Round(n*1000) / 1000
+}
+
 // MarshalJSON implements json.Marshaler interface
-func (m *CallStats) MarshalJSON() ([]byte, error) {
+/*func (m *CallStats) MarshalJSON() ([]byte, error) {
 	var d string
 	d = fmt.Sprintf(
-		"count: %d, total: %s (avg/min/max: %s/%s/%s)",
-		m.Count, durStr(m.TotalDur),
-		durStr(m.AvgDur), durStr(m.MinDur), durStr(m.MaxDur),
+		"%s - count: %d, total: %s (avg/min/max: %s/%s/%s)",
+		m.Name, m.Count, durStr(m.Total),
+		durStr(m.Avg), durStr(m.Min), durStr(m.Max),
 	)
 	return json.Marshal(d)
-}
-*/
-
-type Duration time.Duration
-
-// MarshalJSON implements json.Marshaler interface
-func (m *Duration) MarshalJSON() ([]byte, error) {
-	s := time.Duration(*m).Round(RoundDuration).String()
-	return json.Marshal(s)
-}
+}*/

plugins/govppmux/govpp_channel.go

@@ -148,8 +148,10 @@ func (r *govppRequestCtx) ReceiveReply(reply govppapi.Message) error {
 		err = r.sendRequest(r.requestMsg).ReceiveReply(reply)
 	}
 
+	atomic.AddUint64(&stats.RequestsDone, 1)
 	if err != nil {
-		atomic.AddUint64(&stats.RequestsFailed, 1)
+		trackError(err.Error())
+		atomic.AddUint64(&stats.RequestsErrors, 1)
 	}
 
 	took := time.Since(r.start)
@@ -184,18 +186,25 @@ func (c *goVppChan) SendMultiRequest(request govppapi.Message) govppapi.MultiReq
 func (r *govppMultirequestCtx) ReceiveReply(reply govppapi.Message) (bool, error) {
 	// Receive reply from original send
 	last, err := r.requestCtx.ReceiveReply(reply)
-	if last {
+	if last || err != nil {
 		took := time.Since(r.start)
 		trackMsgRequestDur(r.requestMsg.GetMessageName(), took)
+
+		atomic.AddUint64(&stats.RequestsDone, 1)
 		if err != nil {
-			atomic.AddUint64(&stats.RequestsFailed, 1)
+			trackError(err.Error())
+			atomic.AddUint64(&stats.RequestsErrors, 1)
 		}
+
 		defer func() {
 			r.task.End()
 			if r.tracer != nil {
 				r.tracer.LogTime(r.requestMsg.GetMessageName(), r.start)
 			}
 		}()
+	} else {
+		atomic.AddUint64(&stats.RequestReplies, 1)
+		trackMsgReply(reply.GetMessageName())
 	}
 	return last, err
 }

plugins/govppmux/options.go

@@ -17,6 +17,7 @@ package govppmux
 import (
 	"github.com/ligato/cn-infra/datasync/resync"
 	"github.com/ligato/cn-infra/health/statuscheck"
+	"github.com/ligato/cn-infra/rpc/rest"
 )
 
 // DefaultPlugin is default instance of Plugin
@@ -27,6 +28,7 @@ func NewPlugin(opts ...Option) *Plugin {
 	p := &Plugin{}
 
 	p.PluginName = "govpp"
+	p.HTTPHandlers = &rest.DefaultPlugin
 	p.StatusCheck = &statuscheck.DefaultPlugin
 	p.Resync = &resync.DefaultPlugin
 

plugins/govppmux/plugin_impl_govppmux.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ligato/cn-infra/logging"
 	"github.com/ligato/cn-infra/logging/measure"
 	"github.com/ligato/cn-infra/logging/measure/model/apitrace"
+	"github.com/ligato/cn-infra/rpc/rest"
 	"github.com/pkg/errors"
 
 	"github.com/ligato/vpp-agent/plugins/govppmux/vppcalls"
@@ -79,8 +80,9 @@ type Plugin struct {
 // so that they do not mix with other plugin fields.
 type Deps struct {
 	infra.PluginDeps
-	StatusCheck statuscheck.PluginStatusWriter
-	Resync      *resync.Plugin
+	HTTPHandlers rest.HTTPHandlers
+	StatusCheck  statuscheck.PluginStatusWriter
+	Resync       *resync.Plugin
 }
 
 // Config groups the configurable parameter of GoVpp.
@@ -152,6 +154,9 @@ func (p *Plugin) Init() error {
 		p.Log.Info("VPP API trace enabled")
 	}
 
+	// register REST API handlers
+	p.registerHandlers(p.HTTPHandlers)
+
 	if p.vppAdapter == nil {
 		address := p.config.BinAPISocketPath
 		useShm := disabledSocketClient || p.config.ConnectViaShm

plugins/govppmux/rest.go

@@ -0,0 +1,39 @@
+//  Copyright (c) 2019 Cisco and/or its affiliates.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at:
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package govppmux
+
+import (
+	"net/http"
+
+	"github.com/ligato/cn-infra/rpc/rest"
+	"github.com/unrolled/render"
+)
+
+// registerHandlers registers all supported REST APIs.
+func (s *Plugin) registerHandlers(http rest.HTTPHandlers) {
+	if http == nil {
+		s.Log.Debug("No http handler provided, skipping registration of REST handlers")
+		return
+	}
+	http.RegisterHTTPHandler("/govppmux/stats", s.statsHandler, "GET")
+}
+
+func (s *Plugin) statsHandler(formatter *render.Render) http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		if err := formatter.JSON(w, http.StatusOK, GetStats()); err != nil {
+			s.Log.Warnf("stats handler errored: %v", err)
+		}
+	}
+}

plugins/govppmux/stats.go

@@ -28,7 +28,9 @@ var (
 )
 
 func init() {
+	stats.Errors = make(metrics.Calls)
 	stats.Messages = make(metrics.Calls)
+	stats.Replies = make(metrics.Calls)
 }
 
 func GetStats() *Stats {
@@ -42,10 +44,18 @@ func GetStats() *Stats {
 type Stats struct {
 	ChannelsCreated uint64
 	ChannelsOpen    uint64
-	RequestsSent    uint64
-	RequestsFailed  uint64
-	AllMessages     metrics.CallStats
-	Messages        metrics.Calls
+
+	RequestsSent   uint64
+	RequestsDone   uint64
+	RequestsErrors uint64
+	RequestReplies uint64
+
+	Errors metrics.Calls
+
+	AllMessages metrics.CallStats
+	Messages    metrics.Calls
+
+	Replies metrics.Calls
 }
 
 func (s *Stats) getOrCreateMessage(msg string) *metrics.CallStats {
@@ -69,6 +79,46 @@ func trackMsgRequestDur(m string, d time.Duration) {
 	statsMu.Unlock()
 }
 
+func (s *Stats) getOrCreateReply(msg string) *metrics.CallStats {
+	statsMu.RLock()
+	ms, ok := s.Replies[msg]
+	statsMu.RUnlock()
+	if !ok {
+		ms = &metrics.CallStats{Name: msg}
+		statsMu.Lock()
+		s.Replies[msg] = ms
+		statsMu.Unlock()
+	}
+	return ms
+}
+
+func trackMsgReply(m string) {
+	ms := stats.getOrCreateReply(m)
+	statsMu.Lock()
+	ms.Increment(0)
+	statsMu.Unlock()
+}
+
+func (s *Stats) getOrCreateError(msg string) *metrics.CallStats {
+	statsMu.RLock()
+	ms, ok := s.Errors[msg]
+	statsMu.RUnlock()
+	if !ok {
+		ms = &metrics.CallStats{Name: msg}
+		statsMu.Lock()
+		s.Errors[msg] = ms
+		statsMu.Unlock()
+	}
+	return ms
+}
+
+func trackError(m string) {
+	ms := stats.getOrCreateError(m)
+	statsMu.Lock()
+	ms.Increment(0)
+	statsMu.Unlock()
+}
+
 func init() {
 	expvar.Publish("govppstats", expvar.Func(func() interface{} {
 		return GetStats()

plugins/vpp/ifplugin/interface_state.go

@@ -39,7 +39,8 @@ var (
 	// StateUpdateDelay defines delay before dumping states
 	StateUpdateDelay = time.Second * 3
 
-	disableInterfaceStats = os.Getenv("DISABLE_INTERFACE_STATS") != ""
+	disableInterfaceStats   = os.Getenv("DISABLE_INTERFACE_STATS") != ""
+	disableStatusPublishing = os.Getenv("DISABLE_STATUS_PUBLISHING") != ""
 )
 
 // InterfaceStateUpdater holds state data of all VPP interfaces.
@@ -113,14 +114,18 @@ func (c *InterfaceStateUpdater) Init(ctx context.Context, logger logging.PluginL
 
 	// Periodically read VPP counters and combined counters for VPP statistics
 	if disableInterfaceStats {
-		c.log.Warnf("reading interface stats is disabled!")
+		c.log.Warnf("reading interface stats is DISABLED!")
 	} else if readCounters {
 		c.wg.Add(1)
 		go c.startReadingCounters(childCtx)
 	}
 
-	c.wg.Add(1)
-	go c.startUpdatingIfStateDetails(childCtx)
+	if disableStatusPublishing {
+		c.log.Warnf("publishing interface status is DISABLED!")
+	} else {
+		c.wg.Add(1)
+		go c.startUpdatingIfStateDetails(childCtx)
+	}
 
 	return nil
 }