Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry pick for v20.03 Send CID for sentry events. (#5625) #5633

Merged
merged 1 commit into from
Jun 11, 2020
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Send CID for sentry events. (#5625)
(cherry picked from commit 9765e3f)
parasssh committed Jun 10, 2020
commit 964ec7543356145fccd66c5a04a545d57e3f3c5c
1 change: 1 addition & 0 deletions dgraph/cmd/alpha/run.go
Original file line number Diff line number Diff line change
@@ -677,5 +677,6 @@ func run() {
adminCloser.SignalAndWait()
glog.Info("Disposing server state.")
worker.State.Dispose()
x.RemoveCidFile()
glog.Infoln("Server shutdown. Bye!")
}
1 change: 1 addition & 0 deletions dgraph/cmd/zero/raft.go
Original file line number Diff line number Diff line change
@@ -508,6 +508,7 @@ func (n *node) initAndStartNode() error {
err := n.proposeAndWait(context.Background(), &pb.ZeroProposal{Cid: id})
if err == nil {
glog.Infof("CID set for cluster: %v", id)
x.WriteCidFile(id)
break
}
if err == errInvalidProposal {
2 changes: 2 additions & 0 deletions dgraph/cmd/zero/run.go
Original file line number Diff line number Diff line change
@@ -306,6 +306,8 @@ func run() {
store.Closer.SignalAndWait()
// Stop all internal requests.
_ = grpcListener.Close()

x.RemoveCidFile()
}()

glog.Infoln("Running Dgraph Zero...")
1 change: 1 addition & 0 deletions worker/groups.go
Original file line number Diff line number Diff line change
@@ -811,6 +811,7 @@ START:
}
if i == 0 {
glog.Infof("Received first state update from Zero: %+v", state)
x.WriteCidFile(state.Cid)
}
select {
case stateCh <- state:
48 changes: 45 additions & 3 deletions x/sentry_integration.go
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@ package x

import (
"errors"
"io/ioutil"
"os"
"strings"
"time"
@@ -28,8 +29,9 @@ import (
)

var (
env string
dsn string // API KEY to use
env string
dsn string // API KEY to use
cidPath string
)

// Sentry API KEYs to use.
@@ -99,6 +101,41 @@ func ConfigureSentryScope(subcmd string) {
scope.SetTag("dgraph", subcmd)
scope.SetLevel(sentry.LevelFatal)
})

// e.g. /tmp/dgraph-alpha-cid-sentry
cidPath = os.TempDir() + "/" + "dgraph-" + subcmd + "-cid-sentry"
}

// WriteCidFile writes the CID to a well-known location so it can be read and
// sent to Sentry on panic.
func WriteCidFile(cid string) {
if cid == "" {
return
}
if err := ioutil.WriteFile(cidPath, []byte(cid), 0644); err != nil {
glog.Warningf("unable to write CID to file %v %v", cidPath, err)
return
}
}

// readAndRemoveCidFile reads the file from a well-known location so
// it can be read and sent to Sentry on panic.
func readAndRemoveCidFile() string {
cid, err := ioutil.ReadFile(cidPath)
if err != nil {
glog.Warningf("unable to read CID from file %v %v. Skip", cidPath, err)
return ""
}
RemoveCidFile()
return string(cid)
}

// RemoveCidFile removes the file.
func RemoveCidFile() {
if err := os.RemoveAll(cidPath); err != nil {
glog.Warningf("unable to remove the CID file at %v %v. Skip", cidPath, err)
return
}
}

// CaptureSentryException sends the error report to Sentry.
@@ -111,6 +148,12 @@ func CaptureSentryException(err error) {
// PanicHandler is the callback function when a panic happens. It does not recover and is
// only used to log panics (in our case send an event to sentry).
func PanicHandler(out string) {
if cid := readAndRemoveCidFile(); cid != "" {
// re-configure sentry scope to include cid if found.
sentry.ConfigureScope(func(scope *sentry.Scope) {
scope.SetTag("CID", cid)
})
}
// Output contains the full output (including stack traces) of the panic.
sentry.CaptureException(errors.New(out))
FlushSentry() // Need to flush asap. Don't defer here.
@@ -119,7 +162,6 @@ func PanicHandler(out string) {
}

// WrapPanics is a wrapper on panics. We use it to send sentry events about panics
// and crash right after.
func WrapPanics() {
exitStatus, err := panicwrap.BasicWrap(PanicHandler)
if err != nil {