diff --git a/signals.go b/signals.go
index e0bc7c61cab..a992dc4abeb 100644
--- a/signals.go
+++ b/signals.go
@@ -18,22 +18,30 @@ const signalBufferSize = 2048
 // while still forwarding all other signals to the process.
 // If notifySocket is present, use it to read systemd notifications from the container and
 // forward them to notifySocketHost.
-func newSignalHandler(enableSubreaper bool, notifySocket *notifySocket) *signalHandler {
+func newSignalHandler(enableSubreaper bool, notifySocket *notifySocket) chan *signalHandler {
 	if enableSubreaper {
 		// set us as the subreaper before registering the signal handler for the container
 		if err := system.SetSubreaper(1); err != nil {
 			logrus.Warn(err)
 		}
 	}
-	// ensure that we have a large buffer size so that we do not miss any signals
-	// in case we are not processing them fast enough.
-	s := make(chan os.Signal, signalBufferSize)
-	// handle all signals for the process.
-	signal.Notify(s)
-	return &signalHandler{
-		signals:      s,
-		notifySocket: notifySocket,
-	}
+	handler := make(chan *signalHandler)
+	// signal.Notify is actually quite expensive, as it has to configure the
+	// signal mask and add signal handlers for all signals (all ~65 of them).
+	// So, defer this to a background thread while doing the rest of the io/tty
+	// setup.
+	go func() {
+		// ensure that we have a large buffer size so that we do not miss any
+		// signals in case we are not processing them fast enough.
+		s := make(chan os.Signal, signalBufferSize)
+		// handle all signals for the process.
+		signal.Notify(s)
+		handler <- &signalHandler{
+			signals:      s,
+			notifySocket: notifySocket,
+		}
+	}()
+	return handler
 }
 
 // exit models a process exit status with the pid and
diff --git a/utils_linux.go b/utils_linux.go
index c13ed585bfa..d1ac93f638b 100644
--- a/utils_linux.go
+++ b/utils_linux.go
@@ -246,7 +246,7 @@ func (r *runner) run(config *specs.Process) (int, error) {
 	// Setting up IO is a two stage process. We need to modify process to deal
 	// with detaching containers, and then we get a tty after the container has
 	// started.
-	handler := newSignalHandler(r.enableSubreaper, r.notifySocket)
+	handlerCh := newSignalHandler(r.enableSubreaper, r.notifySocket)
 	tty, err := setupIO(process, r.container, config.Terminal, detach, r.consoleSocket)
 	if err != nil {
 		return -1, err
@@ -285,6 +285,7 @@ func (r *runner) run(config *specs.Process) (int, error) {
 			return -1, err
 		}
 	}
+	handler := <-handlerCh
 	status, err := handler.forward(process, tty, detach)
 	if err != nil {
 		r.terminate(process)