node: collect goroutines stacks before SIGKILL

nodecontrol/NodeController.go

@@ -17,7 +17,11 @@
 package nodecontrol
 
 import (
+	"context"
+	"fmt"
+	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"time"
 
@@ -111,7 +115,7 @@ func (nc NodeController) stopProcesses() (kmdAlreadyStopped bool, err error) {
 	return
 }
 
-func killPID(pid int) (killed bool, err error) {
+func killPID(pid int, beforeKill func()) (killed bool, err error) {
 	process, err := util.FindProcess(pid)
 	if process == nil || err != nil {
 		return false, err
@@ -130,8 +134,42 @@ func killPID(pid int) (killed bool, err error) {
 		}
 		select {
 		case <-waitLong:
+			if beforeKill != nil {
+				beforeKill()
+			}
 			return true, util.KillProcess(pid, syscall.SIGKILL)
 		case <-time.After(time.Millisecond * 100):
 		}
 	}
 }
+
+// collectGoroutineStacks fetches goroutine stacks from the node's pprof endpoint
+// and saves them to a file in the data directory.
+func (nc *NodeController) collectGoroutineStacks() {
+	algodClient, err := nc.AlgodClient()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create algod client for goroutine dump: %v\n", err)
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	goRoutines, err := algodClient.GetGoRoutines(ctx)
+	if err != nil {
+		msg := err.Error()
+		if strings.Contains(msg, "404 Not Found") {
+			msg = "pprof most likely disabled, consider setting EnableProfiler=true for further debugging"
+		}
+		fmt.Fprintf(os.Stderr, "cannot fetch goroutine stacks: %s\n", msg)
+		return
+	}
+
+	dumpFile := filepath.Join(nc.algodDataDir, fmt.Sprintf("goroutine-dump-%s.txt", time.Now().Format("20060102-150405")))
+	err = os.WriteFile(dumpFile, []byte(goRoutines), 0600)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to write goroutine dump to %s: %v\n", dumpFile, err)
+		return
+	}
+	fmt.Fprintf(os.Stderr, "goroutine dump saved to %s\n", dumpFile)
+}

nodecontrol/algodControl.go

@@ -175,7 +175,7 @@ func (nc *NodeController) StopAlgod() (err error) {
 	algodPID, err := nc.GetAlgodPID()
 	if err == nil {
 		// Kill algod by PID
-		killed, killErr := killPID(int(algodPID))
+		killed, killErr := killPID(int(algodPID), nc.collectGoroutineStacks)
 		if killErr != nil {
 			return killErr
 		}

nodecontrol/kmdControl.go

@@ -121,7 +121,7 @@ func (kc *KMDController) StopKMD() (alreadyStopped bool, err error) {
 	kmdPID, err := kc.GetKMDPID()
 	if err == nil {
 		// Kill kmd by PID
-		killed, killErr := killPID(int(kmdPID))
+		killed, killErr := killPID(int(kmdPID), nil)
 		if killErr != nil {
 			return false, killErr
 		}