Issue #419 - Add ability to dump heap profile by sending SIGUSR2

cmd/argocd-application-controller/main.go

@@ -8,7 +8,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/argoproj/argo/util/stats"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"k8s.io/client-go/kubernetes"
@@ -25,6 +24,7 @@ import (
 	"github.com/argoproj/argo-cd/reposerver"
 	"github.com/argoproj/argo-cd/util/cli"
 	"github.com/argoproj/argo-cd/util/db"
+	"github.com/argoproj/argo-cd/util/stats"
 )
 
 const (
@@ -93,6 +93,7 @@ func newCommand() *cobra.Command {
 			log.Infof("Application Controller (version: %s) starting (namespace: %s)", argocd.GetVersion(), namespace)
 			stats.RegisterStackDumper()
 			stats.StartStatsTicker(10 * time.Minute)
+			stats.RegisterHeapDumper("memprofile")
 
 			go secretController.Run(ctx)
 			go appController.Run(ctx, statusProcessors, operationProcessors)

cmd/argocd-repo-server/main.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"time"
 
-	"github.com/argoproj/argo/util/stats"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
@@ -17,6 +16,7 @@ import (
 	"github.com/argoproj/argo-cd/util/cache"
 	"github.com/argoproj/argo-cd/util/git"
 	"github.com/argoproj/argo-cd/util/ksonnet"
+	"github.com/argoproj/argo-cd/util/stats"
 )
 
 const (
@@ -49,6 +49,7 @@ func newCommand() *cobra.Command {
 			log.Infof("ksonnet version: %s", ksVers)
 			stats.RegisterStackDumper()
 			stats.StartStatsTicker(10 * time.Minute)
+			stats.RegisterHeapDumper("memprofile")
 			err = grpc.Serve(listener)
 			errors.CheckError(err)
 			return nil

cmd/argocd-server/commands/root.go

@@ -6,7 +6,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/argoproj/argo/util/stats"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"k8s.io/client-go/kubernetes"
@@ -17,6 +16,7 @@ import (
 	"github.com/argoproj/argo-cd/reposerver"
 	"github.com/argoproj/argo-cd/server"
 	"github.com/argoproj/argo-cd/util/cli"
+	"github.com/argoproj/argo-cd/util/stats"
 )
 
 // NewCommand returns a new instance of an argocd command
@@ -66,6 +66,7 @@ func NewCommand() *cobra.Command {
 
 			stats.RegisterStackDumper()
 			stats.StartStatsTicker(10 * time.Minute)
+			stats.RegisterHeapDumper("memprofile")
 
 			for {
 				argocd := server.NewServer(argoCDOpts)

util/stats/stats.go

@@ -0,0 +1,82 @@
+package stats
+
+import (
+	"os"
+	"os/signal"
+	"runtime"
+	"runtime/pprof"
+	"syscall"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// StartStatsTicker starts a goroutine which dumps stats at a specified interval
+func StartStatsTicker(d time.Duration) {
+	ticker := time.NewTicker(d)
+	go func() {
+		for {
+			<-ticker.C
+			LogStats()
+		}
+	}()
+}
+
+// RegisterStackDumper spawns a goroutine which dumps stack trace upon a SIGUSR1
+func RegisterStackDumper() {
+	go func() {
+		sigs := make(chan os.Signal, 1)
+		signal.Notify(sigs, syscall.SIGUSR1)
+		for {
+			<-sigs
+			LogStack()
+		}
+	}()
+}
+
+// RegisterHeapDumper spawns a goroutine which dumps heap profile upon a SIGUSR2
+func RegisterHeapDumper(filePath string) {
+	go func() {
+		sigs := make(chan os.Signal, 1)
+		signal.Notify(sigs, syscall.SIGUSR2)
+		for {
+			<-sigs
+			runtime.GC()
+			if _, err := os.Stat(filePath); err == nil {
+				err = os.Remove(filePath)
+				if err != nil {
+					log.Warnf("could not delete heap profile file: ", err)
+					return
+				}
+			}
+			f, err := os.Create(filePath)
+			if err != nil {
+				log.Warnf("could not create heap profile file: ", err)
+				return
+			}
+
+			if err := pprof.WriteHeapProfile(f); err != nil {
+				log.Warnf("could not write heap profile: ", err)
+				return
+			} else {
+				log.Infof("dumped heap profile to %s", filePath)
+			}
+		}
+	}()
+}
+
+// LogStats logs runtime statistics
+func LogStats() {
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	log.Infof("Alloc=%v TotalAlloc=%v Sys=%v NumGC=%v Goroutines=%d",
+		m.Alloc/1024, m.TotalAlloc/1024, m.Sys/1024, m.NumGC, runtime.NumGoroutine())
+
+}
+
+// LogStack will log the current stack
+func LogStack() {
+	buf := make([]byte, 1<<20)
+	stacklen := runtime.Stack(buf, true)
+	log.Infof("*** goroutine dump...\n%s\n*** end\n", buf[:stacklen])
+}