diff --git a/dgraph/cmd/alpha/run.go b/dgraph/cmd/alpha/run.go index eee5b611152..cb9f281ee7a 100644 --- a/dgraph/cmd/alpha/run.go +++ b/dgraph/cmd/alpha/run.go @@ -677,5 +677,6 @@ func run() { adminCloser.SignalAndWait() glog.Info("Disposing server state.") worker.State.Dispose() + x.RemoveCidFile() glog.Infoln("Server shutdown. Bye!") } diff --git a/dgraph/cmd/zero/raft.go b/dgraph/cmd/zero/raft.go index 395c80189ed..120cb70bab4 100644 --- a/dgraph/cmd/zero/raft.go +++ b/dgraph/cmd/zero/raft.go @@ -508,6 +508,7 @@ func (n *node) initAndStartNode() error { err := n.proposeAndWait(context.Background(), &pb.ZeroProposal{Cid: id}) if err == nil { glog.Infof("CID set for cluster: %v", id) + x.WriteCidFile(id) break } if err == errInvalidProposal { diff --git a/dgraph/cmd/zero/run.go b/dgraph/cmd/zero/run.go index b3873391355..0f4aaa47723 100644 --- a/dgraph/cmd/zero/run.go +++ b/dgraph/cmd/zero/run.go @@ -306,6 +306,8 @@ func run() { store.Closer.SignalAndWait() // Stop all internal requests. _ = grpcListener.Close() + + x.RemoveCidFile() }() glog.Infoln("Running Dgraph Zero...") diff --git a/worker/groups.go b/worker/groups.go index 7cea7cb2a7d..d285a8903df 100644 --- a/worker/groups.go +++ b/worker/groups.go @@ -811,6 +811,7 @@ START: } if i == 0 { glog.Infof("Received first state update from Zero: %+v", state) + x.WriteCidFile(state.Cid) } select { case stateCh <- state: diff --git a/x/sentry_integration.go b/x/sentry_integration.go index 9718e00de8c..16bfe3634e1 100644 --- a/x/sentry_integration.go +++ b/x/sentry_integration.go @@ -18,6 +18,7 @@ package x import ( "errors" + "io/ioutil" "os" "strings" "time" @@ -28,8 +29,9 @@ import ( ) var ( - env string - dsn string // API KEY to use + env string + dsn string // API KEY to use + cidPath string ) // Sentry API KEYs to use. @@ -99,6 +101,41 @@ func ConfigureSentryScope(subcmd string) { scope.SetTag("dgraph", subcmd) scope.SetLevel(sentry.LevelFatal) }) + + // e.g. /tmp/dgraph-alpha-cid-sentry + cidPath = os.TempDir() + "/" + "dgraph-" + subcmd + "-cid-sentry" +} + +// WriteCidFile writes the CID to a well-known location so it can be read and +// sent to Sentry on panic. +func WriteCidFile(cid string) { + if cid == "" { + return + } + if err := ioutil.WriteFile(cidPath, []byte(cid), 0644); err != nil { + glog.Warningf("unable to write CID to file %v %v", cidPath, err) + return + } +} + +// readAndRemoveCidFile reads the file from a well-known location so +// it can be read and sent to Sentry on panic. +func readAndRemoveCidFile() string { + cid, err := ioutil.ReadFile(cidPath) + if err != nil { + glog.Warningf("unable to read CID from file %v %v. Skip", cidPath, err) + return "" + } + RemoveCidFile() + return string(cid) +} + +// RemoveCidFile removes the file. +func RemoveCidFile() { + if err := os.RemoveAll(cidPath); err != nil { + glog.Warningf("unable to remove the CID file at %v %v. Skip", cidPath, err) + return + } } // CaptureSentryException sends the error report to Sentry. @@ -111,6 +148,12 @@ func CaptureSentryException(err error) { // PanicHandler is the callback function when a panic happens. It does not recover and is // only used to log panics (in our case send an event to sentry). func PanicHandler(out string) { + if cid := readAndRemoveCidFile(); cid != "" { + // re-configure sentry scope to include cid if found. + sentry.ConfigureScope(func(scope *sentry.Scope) { + scope.SetTag("CID", cid) + }) + } // Output contains the full output (including stack traces) of the panic. sentry.CaptureException(errors.New(out)) FlushSentry() // Need to flush asap. Don't defer here. @@ -119,7 +162,6 @@ func PanicHandler(out string) { } // WrapPanics is a wrapper on panics. We use it to send sentry events about panics -// and crash right after. func WrapPanics() { exitStatus, err := panicwrap.BasicWrap(PanicHandler) if err != nil {