Add pipes for parent process IPC. (#311)

Rewrite startup/shutdown logic to simplify shutdown signaling.  All
cleanup is now run from deferred functions in the main function.

Add two new config options to set the read and write ends of a pair of
pipes.  This is used as a simple mechanism for a parent process to
communicate with, observe, and manage the lifetime of a child dcrd
process.  When the RX (read end) pipe is closed, clean shutdown
automatically begins.

Add a new flag --lifetimeevents to create and send lifetime event
notifications over the TX (write end) pipe during bringup and
shutdown.  This allows the parent process to observe which subsystems
are currently starting, running, and stopping.

Fixes #297.
Fixes #298.

config.go

@@ -155,6 +155,9 @@ type config struct {
 	DropAddrIndex       bool          `long:"dropaddrindex" description:"Deletes the address-based transaction index from the database on start up and then exits."`
 	NoExistsAddrIndex   bool          `long:"noexistsaddrindex" description:"Disable the exists address index, which tracks whether or not an address has even been used."`
 	DropExistsAddrIndex bool          `long:"dropexistsaddrindex" description:"Deletes the exists address index from the database on start up and then exits."`
+	PipeRx              uint          `long:"piperx" description:"File descriptor of read end pipe to enable parent -> child process communication"`
+	PipeTx              uint          `long:"pipetx" description:"File descriptor of write end pipe to enable parent <- child process communication"`
+	LifetimeEvents      bool          `long:"lifetimeevents" description:"Send lifetime notifications over the TX pipe"`
 	onionlookup         func(string) ([]net.IP, error)
 	lookup              func(string) ([]net.IP, error)
 	oniondial           func(string, string) (net.Conn, error)

dcrd.go

@@ -17,13 +17,11 @@ import (
 	"time"
 
 	"github.com/decred/dcrd/blockchain/indexers"
+	"github.com/decred/dcrd/blockchain/stake"
 	"github.com/decred/dcrd/limits"
 )
 
-var (
-	cfg             *config
-	shutdownChannel = make(chan struct{})
-)
+var cfg *config
 
 // winServiceMain is only invoked on Windows.  It detects when dcrd is running
 // as a service and reacts accordingly.
@@ -44,6 +42,9 @@ func dcrdMain(serverChan chan<- *server) error {
 	cfg = tcfg
 	defer backendLog.Flush()
 
+	interrupted := interruptListener()
+	defer dcrdLog.Info("Shutdown complete")
+
 	// Show version at startup.
 	dcrdLog.Infof("Version %s", version())
 	// Show dcrd home dir location
@@ -92,38 +93,50 @@ func dcrdMain(serverChan chan<- *server) error {
 		}()
 	}
 
+	var lifetimeNotifier lifetimeEventServer
+	if cfg.LifetimeEvents {
+		lifetimeNotifier = newLifetimeEventServer(outgoingPipeMessages)
+	}
+
+	if cfg.PipeRx != 0 {
+		go serviceControlPipeRx(uintptr(cfg.PipeRx))
+	}
+	if cfg.PipeTx != 0 {
+		go serviceControlPipeTx(uintptr(cfg.PipeTx))
+	} else {
+		go drainOutgoingPipeMessages()
+	}
+
+	if interruptRequested(interrupted) {
+		return nil
+	}
+
 	// Perform upgrades to dcrd as new versions require it.
 	if err := doUpgrades(); err != nil {
 		dcrdLog.Errorf("%v", err)
 		return err
 	}
 
-	// Load the block database.
-	db, err := loadBlockDB()
-	if err != nil {
-		dcrdLog.Errorf("%v", err)
-		return err
+	if interruptRequested(interrupted) {
+		return nil
 	}
-	defer db.Close()
 
-	tmdb, err := loadTicketDB(db, activeNetParams.Params)
+	// Load the block database.
+	lifetimeNotifier.notifyStartupEvent(lifetimeEventDBOpen)
+	db, err := loadBlockDB()
 	if err != nil {
 		dcrdLog.Errorf("%v", err)
 		return err
 	}
 	defer func() {
-		err := tmdb.Store(cfg.DataDir, "ticketdb.gob")
-		if err != nil {
-			dcrdLog.Errorf("Failed to store ticket database: %v", err.Error())
-		}
-	}()
-	defer tmdb.Close()
-
-	// Ensure the databases are sync'd and closed on Ctrl+C.
-	addInterruptHandler(func() {
+		lifetimeNotifier.notifyShutdownEvent(lifetimeEventDBOpen)
 		dcrdLog.Infof("Gracefully shutting down the database...")
 		db.Close()
-	})
+	}()
+
+	if interruptRequested(interrupted) {
+		return nil
+	}
 
 	// Drop indexes and exit if requested.
 	//
@@ -154,40 +167,70 @@ func dcrdMain(serverChan chan<- *server) error {
 		return nil
 	}
 
+	// The ticket "DB" takes ages to load and serialize back out to a file.
+	// Load it asynchronously and if the process is interrupted during the
+	// load, discard the result since no cleanup is necessary.
+	lifetimeNotifier.notifyStartupEvent(lifetimeEventTicketDB)
+	type ticketDBResult struct {
+		ticketDB *stake.TicketDB
+		err      error
+	}
+	ticketDBResultChan := make(chan ticketDBResult)
+	go func() {
+		tmdb, err := loadTicketDB(db, activeNetParams.Params)
+		ticketDBResultChan <- ticketDBResult{tmdb, err}
+	}()
+	var tmdb *stake.TicketDB
+	select {
+	case <-interrupted:
+		return nil
+	case r := <-ticketDBResultChan:
+		if r.err != nil {
+			dcrdLog.Errorf("%v", err)
+			return err
+		}
+		tmdb = r.ticketDB
+	}
+	defer func() {
+		lifetimeNotifier.notifyShutdownEvent(lifetimeEventTicketDB)
+		tmdb.Close()
+		err := tmdb.Store(cfg.DataDir, "ticketdb.gob")
+		if err != nil {
+			dcrdLog.Errorf("Failed to store ticket database: %v", err.Error())
+		}
+	}()
+
 	// Create server and start it.
+	lifetimeNotifier.notifyStartupEvent(lifetimeEventP2PServer)
 	server, err := newServer(cfg.Listeners, db, tmdb, activeNetParams.Params)
 	if err != nil {
 		// TODO(oga) this logging could do with some beautifying.
 		dcrdLog.Errorf("Unable to start server on %v: %v",
 			cfg.Listeners, err)
 		return err
 	}
-	addInterruptHandler(func() {
+	defer func() {
+		lifetimeNotifier.notifyShutdownEvent(lifetimeEventP2PServer)
 		dcrdLog.Infof("Gracefully shutting down the server...")
 		server.Stop()
 		server.WaitForShutdown()
-	})
+		srvrLog.Infof("Server shutdown complete")
+	}()
+
 	server.Start()
 	if serverChan != nil {
 		serverChan <- server
 	}
 
-	// Monitor for graceful server shutdown and signal the main goroutine
-	// when done.  This is done in a separate goroutine rather than waiting
-	// directly so the main goroutine can be signaled for shutdown by either
-	// a graceful shutdown or from the main interrupt handler.  This is
-	// necessary since the main goroutine must be kept running long enough
-	// for the interrupt handler goroutine to finish.
-	go func() {
-		server.WaitForShutdown()
-		srvrLog.Infof("Server shutdown complete")
-		shutdownChannel <- struct{}{}
-	}()
+	if interruptRequested(interrupted) {
+		return nil
+	}
+
+	lifetimeNotifier.notifyStartupComplete()
 
-	// Wait for shutdown signal from either a graceful server stop or from
-	// the interrupt handler.
-	<-shutdownChannel
-	dcrdLog.Info("Shutdown complete")
+	// Wait until the interrupt signal is received from an OS signal or
+	// shutdown is requested through the RPC server.
+	<-interrupted
 	return nil
 }