diff --git a/Gopkg.lock b/Gopkg.lock index 62ba63981e8e8..9bdb0a9430ca9 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -36,8 +36,7 @@ name = "github.com/argoproj/argo" packages = [ "pkg/apis/workflow", - "pkg/apis/workflow/v1alpha1", - "util/stats" + "pkg/apis/workflow/v1alpha1" ] revision = "ac241c95c13f08e868cd6f5ee32c9ce273e239ff" version = "v2.1.1" @@ -1037,6 +1036,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "5e81fc283a0f5288421e4db14372b1921fb812837f44efec608ca4ab23005877" + inputs-digest = "2eaad7537e4d4bc23be336a9e96c3e589cefccf6913125a37bf9b069bf02b5e9" solver-name = "gps-cdcl" solver-version = 1 diff --git a/cmd/argocd-application-controller/main.go b/cmd/argocd-application-controller/main.go index 4f71ec763eab1..4cdb49519e253 100644 --- a/cmd/argocd-application-controller/main.go +++ b/cmd/argocd-application-controller/main.go @@ -8,7 +8,6 @@ import ( "strconv" "time" - "github.com/argoproj/argo/util/stats" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "k8s.io/client-go/kubernetes" @@ -25,6 +24,7 @@ import ( "github.com/argoproj/argo-cd/reposerver" "github.com/argoproj/argo-cd/util/cli" "github.com/argoproj/argo-cd/util/db" + "github.com/argoproj/argo-cd/util/stats" ) const ( @@ -93,6 +93,7 @@ func newCommand() *cobra.Command { log.Infof("Application Controller (version: %s) starting (namespace: %s)", argocd.GetVersion(), namespace) stats.RegisterStackDumper() stats.StartStatsTicker(10 * time.Minute) + stats.RegisterHeapDumper("memprofile") go secretController.Run(ctx) go appController.Run(ctx, statusProcessors, operationProcessors) diff --git a/cmd/argocd-repo-server/main.go b/cmd/argocd-repo-server/main.go index 157ea0f8dd0ce..40eb94b959065 100644 --- a/cmd/argocd-repo-server/main.go +++ b/cmd/argocd-repo-server/main.go @@ -6,7 +6,6 @@ import ( "os" "time" - "github.com/argoproj/argo/util/stats" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -17,6 +16,7 @@ import ( "github.com/argoproj/argo-cd/util/cache" "github.com/argoproj/argo-cd/util/git" "github.com/argoproj/argo-cd/util/ksonnet" + "github.com/argoproj/argo-cd/util/stats" ) const ( @@ -49,6 +49,7 @@ func newCommand() *cobra.Command { log.Infof("ksonnet version: %s", ksVers) stats.RegisterStackDumper() stats.StartStatsTicker(10 * time.Minute) + stats.RegisterHeapDumper("memprofile") err = grpc.Serve(listener) errors.CheckError(err) return nil diff --git a/cmd/argocd-server/commands/root.go b/cmd/argocd-server/commands/root.go index b323578d5d39e..22fdcf6b0342a 100644 --- a/cmd/argocd-server/commands/root.go +++ b/cmd/argocd-server/commands/root.go @@ -6,7 +6,6 @@ import ( "strconv" "time" - "github.com/argoproj/argo/util/stats" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "k8s.io/client-go/kubernetes" @@ -17,6 +16,7 @@ import ( "github.com/argoproj/argo-cd/reposerver" "github.com/argoproj/argo-cd/server" "github.com/argoproj/argo-cd/util/cli" + "github.com/argoproj/argo-cd/util/stats" ) // NewCommand returns a new instance of an argocd command @@ -66,6 +66,7 @@ func NewCommand() *cobra.Command { stats.RegisterStackDumper() stats.StartStatsTicker(10 * time.Minute) + stats.RegisterHeapDumper("memprofile") for { argocd := server.NewServer(argoCDOpts) diff --git a/util/stats/stats.go b/util/stats/stats.go new file mode 100644 index 0000000000000..3cf6374719246 --- /dev/null +++ b/util/stats/stats.go @@ -0,0 +1,82 @@ +package stats + +import ( + "os" + "os/signal" + "runtime" + "runtime/pprof" + "syscall" + "time" + + log "github.com/sirupsen/logrus" +) + +// StartStatsTicker starts a goroutine which dumps stats at a specified interval +func StartStatsTicker(d time.Duration) { + ticker := time.NewTicker(d) + go func() { + for { + <-ticker.C + LogStats() + } + }() +} + +// RegisterStackDumper spawns a goroutine which dumps stack trace upon a SIGUSR1 +func RegisterStackDumper() { + go func() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGUSR1) + for { + <-sigs + LogStack() + } + }() +} + +// RegisterHeapDumper spawns a goroutine which dumps heap profile upon a SIGUSR2 +func RegisterHeapDumper(filePath string) { + go func() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGUSR2) + for { + <-sigs + runtime.GC() + if _, err := os.Stat(filePath); err == nil { + err = os.Remove(filePath) + if err != nil { + log.Warnf("could not delete heap profile file: ", err) + return + } + } + f, err := os.Create(filePath) + if err != nil { + log.Warnf("could not create heap profile file: ", err) + return + } + + if err := pprof.WriteHeapProfile(f); err != nil { + log.Warnf("could not write heap profile: ", err) + return + } else { + log.Infof("dumped heap profile to %s", filePath) + } + } + }() +} + +// LogStats logs runtime statistics +func LogStats() { + var m runtime.MemStats + runtime.ReadMemStats(&m) + log.Infof("Alloc=%v TotalAlloc=%v Sys=%v NumGC=%v Goroutines=%d", + m.Alloc/1024, m.TotalAlloc/1024, m.Sys/1024, m.NumGC, runtime.NumGoroutine()) + +} + +// LogStack will log the current stack +func LogStack() { + buf := make([]byte, 1<<20) + stacklen := runtime.Stack(buf, true) + log.Infof("*** goroutine dump...\n%s\n*** end\n", buf[:stacklen]) +}