Add support for collecting CPU usage for SystemRule and polish related code (#57)

* Add support for collecting CPU usage for SystemRule

Signed-off-by: Eric Zhao <[email protected]>

* Improve system metric collector

Signed-off-by: Eric Zhao <[email protected]>

* Add stat.system.collectIntervalMs conf item and polish related code

Signed-off-by: Eric Zhao <[email protected]>

Author: sczyh30

api/init.go

@@ -55,7 +55,7 @@ func initSentinel(configPath string, logDir string) (err error) {
 	}
 
 	metric.InitTask()
-	system.InitCollector()
+	system.InitCollector(config.SystemStatCollectIntervalMs())
 
 	return err
 }

core/config/entity.go

@@ -2,28 +2,25 @@ package config
 
 import "github.com/pkg/errors"
 
-type Config struct {
-	Version  string
-	Sentinel struct {
-		App struct {
-			// Name represents the name of current running service.
-			Name string
-			// Type indicates the classification of the service (e.g. web service, API gateway).
-			Type int32
-		}
-		Log struct {
-			// Dir represents the log directory path.
-			Dir string
-			// UsePid indicates whether the filename ends with the process ID (PID).
-			UsePid bool
-			// Metric represents the configuration items of the metric log.
-			Metric struct {
-				SingleFileMaxSize uint64
-				MaxFileCount      uint32
-				FlushIntervalSec  uint32
-			}
-		}
+type Entity struct {
+	// Version represents the format version of the entity.
+	Version string
+
+	Sentinel SentinelConfig
+}
+
+// SentinelConfig represent the general configuration of Sentinel.
+type SentinelConfig struct {
+	App struct {
+		// Name represents the name of current running service.
+		Name string
+		// Type indicates the classification of the service (e.g. web service, API gateway).
+		Type int32
 	}
+	// Log represents configuration items related to logging.
+	Log LogConfig
+	// Stat represents configuration items related to statistics.
+	Stat StatConfig
 }
 
 // LogConfig represent the configuration of logging in Sentinel.
@@ -43,23 +40,15 @@ type MetricLogConfig struct {
 	FlushIntervalSec  uint32 `yaml:"flushIntervalSec"`
 }
 
-// SentinelConfig represent the general configuration of Sentinel.
-type SentinelConfig struct {
-	App struct {
-		// Name represents the name of current running service.
-		Name string
-		// Type indicates the classification of the service (e.g. web service, API gateway).
-		Type int32
-	}
-	// Log represents configuration items related to logging.
-	Log LogConfig
+// StatConfig represents the configuration items of statistics.
+type StatConfig struct {
+	System SystemStatConfig `yaml:"system"`
 }
 
-type Entity struct {
-	// Version represents the format version of the entity.
-	Version string
-
-	Sentinel SentinelConfig
+// SystemStatConfig represents the configuration items of system statistics.
+type SystemStatConfig struct {
+	// CollectIntervalMs represents the collecting interval of the system metrics collector.
+	CollectIntervalMs uint32 `yaml:"collectIntervalMs"`
 }
 
 func NewDefaultConfig() *Entity {
@@ -73,7 +62,11 @@ func NewDefaultConfig() *Entity {
 				Name: UnknownProjectName,
 				Type: DefaultAppType,
 			},
-			Log: LogConfig{Metric: MetricLogConfig{SingleFileMaxSize: DefaultMetricLogSingleFileMaxSize, MaxFileCount: DefaultMetricLogMaxFileAmount, FlushIntervalSec: DefaultMetricLogFlushIntervalSec}},
+			Log: LogConfig{Metric: MetricLogConfig{SingleFileMaxSize: DefaultMetricLogSingleFileMaxSize,
+				MaxFileCount: DefaultMetricLogMaxFileAmount, FlushIntervalSec: DefaultMetricLogFlushIntervalSec}},
+			Stat: StatConfig{
+				System: SystemStatConfig{CollectIntervalMs: DefaultSystemStatCollectIntervalMs},
+			},
 		},
 	}
 }
@@ -92,5 +85,8 @@ func checkValid(conf *SentinelConfig) error {
 	if mc.SingleFileMaxSize <= 0 {
 		return errors.New("Bad metric log config: singleFileMaxSize <= 0")
 	}
+	if conf.Stat.System.CollectIntervalMs == 0 {
+		return errors.New("Bad system stat config: collectIntervalMs = 0")
+	}
 	return nil
 }

core/config/local_storage.go

@@ -17,11 +17,12 @@ const (
 	AppNameEnvKey = "SENTINEL_APP_NAME"
 	AppTypeEnvKey = "SENTINEL_APP_TYPE"
 
-	DefaultConfigFilename                    = "sentinel.yml"
-	DefaultAppType                    int32  = 0
-	DefaultMetricLogFlushIntervalSec  uint32 = 1
-	DefaultMetricLogSingleFileMaxSize uint64 = 1024 * 1024 * 50
-	DefaultMetricLogMaxFileAmount     uint32 = 8
+	DefaultConfigFilename                     = "sentinel.yml"
+	DefaultAppType                     int32  = 0
+	DefaultMetricLogFlushIntervalSec   uint32 = 1
+	DefaultMetricLogSingleFileMaxSize  uint64 = 1024 * 1024 * 50
+	DefaultMetricLogMaxFileAmount      uint32 = 8
+	DefaultSystemStatCollectIntervalMs uint32 = 1000
 )
 
 var localConf = NewDefaultConfig()
@@ -44,6 +45,10 @@ func InitConfigFromFile(filePath string) error {
 	}
 	loadFromSystemEnv()
 
+	if err = checkValid(&localConf.Sentinel); err != nil {
+		return err
+	}
+
 	logger := logging.GetDefaultLogger()
 	logger.Infof("App name resolved: %s", AppName())
 
@@ -74,7 +79,7 @@ func loadFromSystemEnv() {
 	appTypeStr := os.Getenv(AppTypeEnvKey)
 	appType, err := strconv.ParseInt(appTypeStr, 10, 32)
 	if err != nil {
-
+		logging.GetDefaultLogger().Warnf("Ignoring bad appType from system env: %s", appTypeStr)
 	} else {
 		localConf.Sentinel.App.Type = int32(appType)
 	}
@@ -99,3 +104,7 @@ func MetricLogSingleFileMaxSize() uint64 {
 func MetricLogMaxFileAmount() uint32 {
 	return localConf.Sentinel.Log.Metric.MaxFileCount
 }
+
+func SystemStatCollectIntervalMs() uint32 {
+	return localConf.Sentinel.Stat.System.CollectIntervalMs
+}

core/system/sys_stat.go

@@ -2,6 +2,8 @@ package system
 
 import (
 	"github.com/alibaba/sentinel-golang/util"
+	"math"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -17,25 +19,38 @@ var (
 	currentLoad     atomic.Value
 	currentCpuUsage atomic.Value
 
+	prevCpuStat *cpu.TimesStat
+	initOnce    sync.Once
+
 	ssStopChan = make(chan struct{})
 )
 
-func InitCollector() {
+func init() {
 	currentLoad.Store(notRetrievedValue)
 	currentCpuUsage.Store(notRetrievedValue)
+}
+
+func InitCollector(intervalMs uint32) {
+	if intervalMs == 0 {
+		return
+	}
+	initOnce.Do(func() {
+		// Initial retrieval.
+		retrieveAndUpdateSystemStat()
 
-	ticker := time.NewTicker(1 * time.Second)
-	go util.RunWithRecover(func() {
-		for {
-			select {
-			case <-ticker.C:
-				retrieveAndUpdateSystemStat()
-			case <-ssStopChan:
-				ticker.Stop()
-				return
+		ticker := time.NewTicker(time.Duration(intervalMs) * time.Millisecond)
+		go util.RunWithRecover(func() {
+			for {
+				select {
+				case <-ticker.C:
+					retrieveAndUpdateSystemStat()
+				case <-ssStopChan:
+					ticker.Stop()
+					return
+				}
 			}
-		}
-	}, logger)
+		}, logger)
+	})
 }
 
 func retrieveAndUpdateSystemStat() {
@@ -48,15 +63,49 @@ func retrieveAndUpdateSystemStat() {
 		logger.Warnf("Failed to retrieve current system load: %+v", err)
 	}
 	if len(cpuStats) > 0 {
-		// TODO: calculate the real CPU usage
-		// cpuStat := cpuStats[0]
-		// currentCpuUsage.Store(cpuStat.User)
+		curCpuStat := &cpuStats[0]
+		recordCpuUsage(prevCpuStat, curCpuStat)
+		// Cache the latest CPU stat info.
+		prevCpuStat = curCpuStat
 	}
 	if loadStat != nil {
 		currentLoad.Store(loadStat.Load1)
 	}
 }
 
+func recordCpuUsage(prev, curCpuStat *cpu.TimesStat) {
+	if prev != nil && curCpuStat != nil {
+		prevTotal := calculateTotalCpuTick(prev)
+		curTotal := calculateTotalCpuTick(curCpuStat)
+
+		tDiff := curTotal - prevTotal
+		var cpuUsage float64
+		if tDiff == 0 {
+			cpuUsage = 0
+		} else {
+			prevUsed := calculateUserCpuTick(prev) + calculateKernelCpuTick(prev)
+			curUsed := calculateUserCpuTick(curCpuStat) + calculateKernelCpuTick(curCpuStat)
+			cpuUsage = (curUsed - prevUsed) / tDiff
+			cpuUsage = math.Max(0.0, cpuUsage)
+			cpuUsage = math.Min(1.0, cpuUsage)
+		}
+		currentCpuUsage.Store(cpuUsage)
+	}
+}
+
+func calculateTotalCpuTick(stat *cpu.TimesStat) float64 {
+	return stat.User + stat.Nice + stat.System + stat.Idle +
+		stat.Iowait + stat.Irq + stat.Softirq + stat.Steal
+}
+
+func calculateUserCpuTick(stat *cpu.TimesStat) float64 {
+	return stat.User + stat.Nice
+}
+
+func calculateKernelCpuTick(stat *cpu.TimesStat) float64 {
+	return stat.System + stat.Irq + stat.Softirq
+}
+
 func CurrentLoad() float64 {
 	r, ok := currentLoad.Load().(float64)
 	if !ok {

core/system/sys_stat_test.go

@@ -0,0 +1,45 @@
+package system
+
+import (
+	"github.com/shirou/gopsutil/cpu"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func Test_recordCpuUsage(t *testing.T) {
+	var emptyStat *cpu.TimesStat = nil
+	// total: 2260, user+nice: 950, system+irqs=210
+	prev := &cpu.TimesStat{
+		CPU:     "all",
+		User:    900,
+		System:  200,
+		Idle:    300,
+		Nice:    50,
+		Iowait:  100,
+		Irq:     5,
+		Softirq: 5,
+		Steal:   700,
+	}
+	// total: 4180, user+nice: 1600, system+irqs=430
+	cur := &cpu.TimesStat{
+		CPU:     "all",
+		User:    1500,
+		System:  400,
+		Idle:    400,
+		Nice:    100,
+		Iowait:  150,
+		Irq:     15,
+		Softirq: 15,
+		Steal:   1600,
+	}
+	expected := float64(1600+430-950-210) / (4180 - 2260)
+
+	recordCpuUsage(emptyStat, cur)
+	assert.Equal(t, notRetrievedValue, CurrentCpuUsage())
+
+	recordCpuUsage(prev, prev)
+	assert.Equal(t, 0.0, CurrentCpuUsage())
+
+	recordCpuUsage(prev, cur)
+	assert.InEpsilon(t, expected, CurrentCpuUsage(), 0.001)
+}