Skip to content

Commit

Permalink
[horus] Added self-healing manager (#345)
Browse files Browse the repository at this point in the history
fix typo
  • Loading branch information
mfordjody authored Sep 15, 2024
1 parent cba3490 commit c10e540
Show file tree
Hide file tree
Showing 5 changed files with 62 additions and 1 deletion.
7 changes: 7 additions & 0 deletions app/horus/basic/db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,3 +118,10 @@ func (n *NodeDataInfo) AddOrGet() (int64, error) {
row, err := n.Add()
return row, err
}

func GetRecoveryNodeDataInfoDate(day int) ([]*NodeDataInfo, error) {
var ndi []*NodeDataInfo
session := db.Where(fmt.Sprintf("first_date > DATE_SUB(CURDATE(),INTERVAL %d DAY)", day))
err := session.Find(&ndi)
return nil, err
}
10 changes: 10 additions & 0 deletions app/horus/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"flag"
"github.com/apache/dubbo-kubernetes/app/horus/basic/config"
"github.com/apache/dubbo-kubernetes/app/horus/basic/db"
"github.com/apache/dubbo-kubernetes/app/horus/core/horuser"
"github.com/apache/dubbo-kubernetes/app/horus/core/ticker"
"github.com/prometheus/client_golang/prometheus/promhttp"
"k8s.io/klog"
Expand Down Expand Up @@ -56,6 +57,7 @@ func main() {
} else {
klog.Infof("horus db initial success.")
}
horus := horuser.NewHoruser(c)
group, stopChan := setupStopChanWithContext()
ctx, cancel := context.WithCancel(context.Background())
group.Add(func() error {
Expand Down Expand Up @@ -87,6 +89,14 @@ func main() {
}
return nil
})
group.Add(func() error {
klog.Info("horus recovery manager start success.")
err := horus.RecoveryManager(ctx)
if err != nil {
klog.Errorf("horus recovery manager start failed error:%v", err)
}
return nil
})
group.Wait()
}

Expand Down
42 changes: 42 additions & 0 deletions app/horus/core/horuser/recovery.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package horuser

import (
"context"
"github.com/apache/dubbo-kubernetes/app/horus/basic/db"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog/v2"
"time"
)

func (h *Horuser) RecoveryManager(ctx context.Context) error {
go wait.UntilWithContext(ctx, h.recoveryCheck, time.Duration(h.cc.NodeRecovery.CheckIntervalSecond)*time.Second)
<-ctx.Done()
return nil
}

func (h *Horuser) recoveryCheck(ctx context.Context) {
data, err := db.GetRecoveryNodeDataInfoDate(h.cc.NodeRecovery.DayNumber)
if err != nil {
klog.Errorf("recovery check GetRecoveryNodeDataInfoDate err:%v", err)
return
}
if len(data) == 0 {
klog.Errorf("recovery check GetRecoveryNodeDataInfoDate zero.")
return
}
}
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ require (
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.20.3
github.com/prometheus/common v0.55.0
github.com/sethvargo/go-retry v0.2.4
github.com/slok/go-http-metrics v0.11.0
github.com/spf13/cobra v1.8.0
Expand Down Expand Up @@ -306,7 +307,6 @@ require (
github.com/polarismesh/polaris-go v1.3.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/rivo/tview v0.0.0-20220307222120-9994674d60a8 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -861,6 +861,7 @@ github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUB
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
Expand Down Expand Up @@ -1053,6 +1054,7 @@ github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOl
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
Expand Down

0 comments on commit c10e540

Please sign in to comment.