Skip to content

Commit

Permalink
Fix Manager.Run stop deadlock
Browse files Browse the repository at this point in the history
Fix open-telemetry#1218

Close probes in a different goroutine than the one draining their event
channel.
  • Loading branch information
MrAlias committed Oct 25, 2024
1 parent 098f594 commit 91cc068
Showing 1 changed file with 21 additions and 12 deletions.
33 changes: 21 additions & 12 deletions internal/pkg/instrumentation/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -277,21 +277,30 @@ func (m *Manager) Run(ctx context.Context, target *process.TargetDetails) error

go m.ConfigLoop(ctx)

for {
select {
case <-ctx.Done():
m.probeMu.Lock()
done := make(chan error, 1)
go func() {
defer close(done)
<-ctx.Done()

m.probeMu.Lock()

m.logger.Debug("Shutting down all probes")
err := m.cleanup(target)

m.logger.Debug("Shutting down all probes")
err := m.cleanup(target)
// Wait for all probes to stop before closing the chan they send on.
m.runningProbesWG.Wait()
close(m.eventCh)

// Wait for all probes to stop before closing the chan they send on.
m.runningProbesWG.Wait()
close(m.eventCh)
m.state = managerStateStopped
m.probeMu.Unlock()

m.state = managerStateStopped
m.probeMu.Unlock()
return errors.Join(err, ctx.Err())
done <- errors.Join(err, ctx.Err())
}()

for {
select {
case err := <-done:
return err
case e := <-m.eventCh:
m.otelController.Trace(e)
}
Expand Down

0 comments on commit 91cc068

Please sign in to comment.