diff --git a/api/v1alpha1/worker_types.go b/api/v1alpha1/worker_types.go index 556588a..4bfea46 100644 --- a/api/v1alpha1/worker_types.go +++ b/api/v1alpha1/worker_types.go @@ -28,6 +28,8 @@ type WorkerStatus struct { type WorkerInfo struct { ExternalIPAddress string `json:"externalIPAddress"` Hostname string `json:"hostname"` + MemoryUsedPercent string `json:"memoryUsedPercent"` + CPUUsedPercent string `json:"cpuUsedPercent"` } //+kubebuilder:object:root=true diff --git a/config/crd/bases/netcon.janog.gr.jp_workers.yaml b/config/crd/bases/netcon.janog.gr.jp_workers.yaml index 5d1b510..f762bbb 100644 --- a/config/crd/bases/netcon.janog.gr.jp_workers.yaml +++ b/config/crd/bases/netcon.janog.gr.jp_workers.yaml @@ -37,13 +37,19 @@ spec: properties: workerInfo: properties: + cpuUsedPercent: + type: string externalIPAddress: type: string hostname: type: string + memoryUsedPercent: + type: string required: + - cpuUsedPercent - externalIPAddress - hostname + - memoryUsedPercent type: object required: - workerInfo diff --git a/controllers/controller-manager/problemenvironment_controller.go b/controllers/controller-manager/problemenvironment_controller.go index bae6b1d..04e2b6e 100644 --- a/controllers/controller-manager/problemenvironment_controller.go +++ b/controllers/controller-manager/problemenvironment_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "sort" + "strconv" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,6 +39,10 @@ type ProblemEnvironmentReconciler struct { Scheme *runtime.Scheme } +const ( + MAX_USED_PERCENT float64 = 100.0 +) + //+kubebuilder:rbac:groups=netcon.janog.gr.jp,resources=problemenvironments,verbs=get;list;watch;update;patch //+kubebuilder:rbac:groups=netcon.janog.gr.jp,resources=problemenvironments/status,verbs=get;update;patch //+kubebuilder:rbac:groups=netcon.janog.gr.jp,resources=problems,verbs=get;list;watch @@ -100,6 +105,51 @@ func (r *ProblemEnvironmentReconciler) updateStatus( return ctrl.Result{}, nil } +func (r *ProblemEnvironmentReconciler) electWorker( + ctx context.Context, + workers netconv1alpha1.WorkerList, + problemEnvironments netconv1alpha1.ProblemEnvironmentList, +) string { + log := log.FromContext(ctx) + + workerLength := len(workers.Items) + if workerLength == 0 { + return "" + } else if workerLength < 2 { + return workers.Items[0].Name + } + + type WorkerResource struct { + Name string + // CPUUsedPercent float64 + // MemoryUsedPercent float64 + SumOfResourcesUsedPercent float64 + } + arr := make([]WorkerResource, 0, workerLength) + for i := 0; i < workerLength; i++ { + cpuUsedPct, err := strconv.ParseFloat(workers.Items[i].Status.WorkerInfo.CPUUsedPercent, 64) + if err != nil { + log.Error(err, "failed to parse CPUUsedPercent for worker election") + cpuUsedPct = MAX_USED_PERCENT + } + memoryUsedPercent, err := strconv.ParseFloat(workers.Items[i].Status.WorkerInfo.MemoryUsedPercent, 64) + if err != nil { + log.Error(err, "failed to parse MemoryUsedPercent for worker election") + memoryUsedPercent = MAX_USED_PERCENT + } + sumOfResourcesUsedPercent := cpuUsedPct + memoryUsedPercent + + arr = append(arr, WorkerResource{workers.Items[i].Name, sumOfResourcesUsedPercent}) + } + sort.Slice(arr, func(i, j int) bool { + return arr[i].SumOfResourcesUsedPercent < arr[j].SumOfResourcesUsedPercent + }) + + log.Info("electWorker : " + arr[0].Name) + + return arr[0].Name +} + func (r *ProblemEnvironmentReconciler) schedule( ctx context.Context, problemEnvironment *netconv1alpha1.ProblemEnvironment, @@ -145,33 +195,17 @@ func (r *ProblemEnvironmentReconciler) schedule( // TODO: handle error } - workerNameProbEnvCountsMap := make(map[string]int) - for i := 0; i < len(problemEnvironments.Items); i++ { - if problemEnvironments.Items[i].Spec.WorkerName != "" { - key := problemEnvironments.Items[i].Spec.WorkerName - workerNameProbEnvCountsMap[key] = workerNameProbEnvCountsMap[key] + 1 - } - } - // there is no key-value pairs when getting started - if len(workerNameProbEnvCountsMap) == 0 { - problemEnvironment.Spec.WorkerName = workers.Items[0].Name + electedWorkerName := r.electWorker(ctx, workers, problemEnvironments) + + if electedWorkerName != "" { + problemEnvironment.Spec.WorkerName = electedWorkerName } else { - type kv struct { - Key string - Value int - } - ss := make([]kv, 0, len(workerNameProbEnvCountsMap)) - for k, v := range workerNameProbEnvCountsMap { - ss = append(ss, kv{k, v}) - } - sort.Slice(ss, func(i, j int) bool { - return ss[i].Value < ss[j].Value - }) - problemEnvironment.Spec.WorkerName = ss[0].Key - log.V(1).Info("elected workerName " + problemEnvironment.Spec.WorkerName) + // TODO: statusの Scheduled = Falseに更新する + // 更新せずとも、再reconcileが走るので問題はないはず + message := "failed to elect worker for scheduling" + log.Info(message) } - log.Info("scheduled", "newWorkerName", problemEnvironment.Spec.WorkerName) return r.update(ctx, problemEnvironment, ctrl.Result{}) } diff --git a/controllers/nclet/heartbeat.go b/controllers/nclet/heartbeat.go index 369e171..1b66a52 100644 --- a/controllers/nclet/heartbeat.go +++ b/controllers/nclet/heartbeat.go @@ -3,6 +3,7 @@ package controllers import ( "context" "os" + "strconv" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -15,6 +16,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/runtime/inject" netconv1alpha1 "github.com/janog-netcon/netcon-problem-management-subsystem/api/v1alpha1" + cpu "github.com/shirou/gopsutil/v3/cpu" + mem "github.com/shirou/gopsutil/v3/mem" ) type HeartbeatAgent struct { @@ -104,9 +107,23 @@ func (a *HeartbeatAgent) Start(ctx context.Context) error { // TODO: resolve external IP address used by users to access externalIPAddress := "..." + virtualMemory, err := mem.VirtualMemory() + if err != nil { + log.Error(err, "failed to get memoryUsedPercent") + continue + } + + cpuUsedPercents, err := cpu.Percent(time.Minute, false) + if err != nil { + log.Error(err, "failed to get CPUUsedPercent") + continue + } + worker.Status.WorkerInfo = netconv1alpha1.WorkerInfo{ Hostname: hostname, ExternalIPAddress: externalIPAddress, + MemoryUsedPercent: strconv.FormatFloat(virtualMemory.UsedPercent, 'f', -1, 64), + CPUUsedPercent: strconv.FormatFloat(cpuUsedPercents[0], 'f', -1, 64), } if err := a.Status().Update(ctx, &worker); err != nil { diff --git a/go.mod b/go.mod index 51ccebc..8b4aacf 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,15 @@ require ( cloud.google.com/go/compute v1.7.0 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/benbjohnson/clock v1.3.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect - github.com/stretchr/testify v1.8.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shirou/gopsutil/v3 v3.22.10 // indirect + github.com/stretchr/testify v1.8.1 // indirect + github.com/tklauser/go-sysconf v0.3.10 // indirect + github.com/tklauser/numcpus v0.4.0 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index cb01d18..366e6cb 100644 --- a/go.sum +++ b/go.sum @@ -160,6 +160,8 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -299,6 +301,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= @@ -330,6 +334,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -357,6 +363,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil/v3 v3.22.10 h1:4KMHdfBRYXGF9skjDWiL4RA2N+E8dRdodU/bOZpPoVg= +github.com/shirou/gopsutil/v3 v3.22.10/go.mod h1:QNza6r4YQoydyCfo6rH0blGfKahgibh4dQmV5xdFkQk= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -370,6 +378,7 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -379,6 +388,12 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= +github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= +github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= +github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -387,6 +402,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -549,6 +566,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -572,6 +590,7 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=