Add monitoring of heap and save heap profile if size is over some lim…

…it (kaspanet#1932)

* Add monitoring of heap and save heap profile if size is over some limit

* Exported function

* Extract dump logic to a function (for defer close)

* Change trackHeapSize ticker interval to 10 seconds

* Add timestamp to dump file name

Co-authored-by: Ori Newman <>

app/app.go

@@ -87,6 +87,7 @@ func (app *kaspadApp) main(startedChan chan<- struct{}) error {
 	if app.cfg.Profile != "" {
 		profiling.Start(app.cfg.Profile, log)
 	}
+	profiling.TrackHeap(app.cfg.AppDir, log)
 
 	// Return now if an interrupt signal was triggered.
 	if signal.InterruptRequested(interrupt) {

util/profiling/profiling.go

@@ -1,16 +1,26 @@
 package profiling
 
 import (
+	"fmt"
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 	"net"
 	"net/http"
+	"os"
+	"path/filepath"
+	"time"
 
 	// Required for profiling
 	_ "net/http/pprof"
 
 	"github.com/kaspanet/kaspad/util/panics"
+	"runtime"
+	"runtime/pprof"
 )
 
+// heapDumpFileName is the name of the heap dump file. We want every run to have its own
+// file, so we append the timestamp of the program launch time to the file name.
+var heapDumpFileName = fmt.Sprintf("heap-%s.pprof", time.Now().Format(time.RFC3339))
+
 // Start starts the profiling server
 func Start(port string, log *logger.Logger) {
 	spawn := panics.GoroutineWrapperFunc(log)
@@ -22,3 +32,45 @@ func Start(port string, log *logger.Logger) {
 		log.Error(http.ListenAndServe(listenAddr, nil))
 	})
 }
+
+// TrackHeap tracks the size of the heap and dumps a profile if it passes a limit
+func TrackHeap(appDir string, log *logger.Logger) {
+	spawn := panics.GoroutineWrapperFunc(log)
+	spawn("profiling.TrackHeap", func() {
+		dumpFolder := filepath.Join(appDir, "dumps")
+		err := os.MkdirAll(dumpFolder, 0700)
+		if err != nil {
+			log.Errorf("Could not create heap dumps folder at %s", dumpFolder)
+			return
+		}
+		const limitInGigabytes = 8
+		trackHeapSize(limitInGigabytes*1024*1024*1024, dumpFolder, log)
+	})
+}
+
+func trackHeapSize(heapLimit uint64, dumpFolder string, log *logger.Logger) {
+	ticker := time.NewTicker(10 * time.Second)
+	defer ticker.Stop()
+	for range ticker.C {
+		memStats := &runtime.MemStats{}
+		runtime.ReadMemStats(memStats)
+		// If we passed the expected heap limit, dump the heap profile to a file
+		if memStats.HeapAlloc > heapLimit {
+			dumpHeapProfile(heapLimit, dumpFolder, memStats, log)
+		}
+	}
+}
+
+func dumpHeapProfile(heapLimit uint64, dumpFolder string, memStats *runtime.MemStats, log *logger.Logger) {
+	heapFile := filepath.Join(dumpFolder, heapDumpFileName)
+	log.Infof("Saving heap statistics into %s (HeapAlloc=%d > %d=heapLimit)", heapFile, memStats.HeapAlloc, heapLimit)
+	f, err := os.Create(heapFile)
+	defer f.Close()
+	if err != nil {
+		log.Infof("Could not create heap profile: %s", err)
+		return
+	}
+	if err := pprof.WriteHeapProfile(f); err != nil {
+		log.Infof("Could not write heap profile: %s", err)
+	}
+}