Fix #1206, dispose ingester while server quiting. 3.0.111

Author: winlinvip

README.md

@@ -146,6 +146,7 @@ For previous versions, please read:
 
 ## V3 changes
 
+* v3.0, 2020-01-29, Fix [#1206][bug #1206], dispose ingester while server quiting. 3.0.111
 * v3.0, 2020-01-28, Fix [#1230][bug #1230], racing condition in source fetch or create. 3.0.110
 * v3.0, 2020-01-27, Fix [#1303][bug #1303], do not dispatch previous meta when not publishing. 3.0.109
 * v3.0, 2020-01-26, Allow use libst.so for ST is MPL license.
@@ -1632,6 +1633,7 @@ Winlin
 [bug #607]: https://github.com/ossrs/srs/issues/607
 [bug #1303]: https://github.com/ossrs/srs/issues/1303
 [bug #1230]: https://github.com/ossrs/srs/issues/1230
+[bug #1206]: https://github.com/ossrs/srs/issues/1206
 [bug #xxxxxxxxxxxxx]: https://github.com/ossrs/srs/issues/xxxxxxxxxxxxx
 
 [exo #828]: https://github.com/google/ExoPlayer/pull/828

trunk/src/app/srs_app_ffmpeg.cpp

@@ -418,6 +418,11 @@ void SrsFFMPEG::fast_stop()
     process->fast_stop();
 }
 
+void SrsFFMPEG::fast_kill()
+{
+    process->fast_kill();
+}
+
 #endif
 
 

trunk/src/app/srs_app_ffmpeg.hpp

@@ -83,6 +83,7 @@ class SrsFFMPEG
     virtual void stop();
 public:
     virtual void fast_stop();
+    virtual void fast_kill();
 };
 
 #endif

trunk/src/app/srs_app_ingest.cpp

@@ -97,11 +97,17 @@ void SrsIngesterFFMPEG::fast_stop()
     ffmpeg->fast_stop();
 }
 
+void SrsIngesterFFMPEG::fast_kill()
+{
+    ffmpeg->fast_kill();
+}
+
 SrsIngester::SrsIngester()
 {
     _srs_config->subscribe(this);
 
     expired = false;
+    disposed = false;
 
     trd = new SrsDummyCoroutine();
     pprint = SrsPithyPrint::create_ingester();
@@ -117,11 +123,18 @@ SrsIngester::~SrsIngester()
 
 void SrsIngester::dispose()
 {
+    if (disposed) {
+        return;
+    }
+    disposed = true;
+
     // first, use fast stop to notice all FFMPEG to quit gracefully.
     fast_stop();
+
+    srs_usleep(100 * SRS_UTIME_MILLISECONDS);
 
-    // then, use stop to wait FFMPEG quit one by one and send SIGKILL if needed.
-    stop();
+    // then, use fast kill to ensure FFMPEG quit.
+    fast_kill();
 }
 
 srs_error_t SrsIngester::start()
@@ -166,6 +179,19 @@ void SrsIngester::fast_stop()
     }
 }
 
+void SrsIngester::fast_kill()
+{
+    std::vector<SrsIngesterFFMPEG*>::iterator it;
+    for (it = ingesters.begin(); it != ingesters.end(); ++it) {
+        SrsIngesterFFMPEG* ingester = *it;
+        ingester->fast_kill();
+    }
+
+    if (!ingesters.empty()) {
+        srs_trace("fast kill all ingesters ok.");
+    }
+}
+
 // when error, ingester sleep for a while and retry.
 // ingest never sleep a long time, for we must start the stream ASAP.
 #define SRS_AUTO_INGESTER_CIMS (3 * SRS_UTIME_SECONDS)
@@ -174,7 +200,7 @@ srs_error_t SrsIngester::cycle()
 {
     srs_error_t err = srs_success;
 
-    while (true) {
+    while (!disposed) {
         if ((err = do_cycle()) != srs_success) {
             srs_warn("Ingester: Ignore error, %s", srs_error_desc(err).c_str());
             srs_freep(err);

trunk/src/app/srs_app_ingest.hpp

@@ -60,6 +60,7 @@ class SrsIngesterFFMPEG
     virtual srs_error_t cycle();
     // @see SrsFFMPEG.fast_stop().
     virtual void fast_stop();
+    virtual void fast_kill();
 };
 
 // Ingest file/stream/device,
@@ -75,6 +76,8 @@ class SrsIngester : public ISrsCoroutineHandler, public ISrsReloadHandler
     // Whether the ingesters are expired, for example, the listen port changed,
     // all ingesters must be restart.
     bool expired;
+    // Whether already disposed.
+    bool disposed;
 public:
     SrsIngester();
     virtual ~SrsIngester();
@@ -84,7 +87,10 @@ class SrsIngester : public ISrsCoroutineHandler, public ISrsReloadHandler
     virtual srs_error_t start();
     virtual void stop();
 private:
+    // Notify FFMPEG to fast stop.
     virtual void fast_stop();
+    // When SRS quit, directly kill FFMPEG after fast stop.
+    virtual void fast_kill();
 // Interface ISrsReusableThreadHandler.
 public:
     virtual srs_error_t cycle();

trunk/src/app/srs_app_process.cpp

@@ -327,3 +327,28 @@ void SrsProcess::fast_stop()
     return;
 }
 
+void SrsProcess::fast_kill()
+{
+    int ret = ERROR_SUCCESS;
+
+    if (!is_started) {
+        return;
+    }
+
+    if (pid <= 0) {
+        return;
+    }
+
+    if (kill(pid, SIGKILL) < 0) {
+        ret = ERROR_SYSTEM_KILL;
+        srs_warn("ignore fast kill process failed, pid=%d. ret=%d", pid, ret);
+        return;
+    }
+
+    // Try to wait pid to avoid zombie FFMEPG.
+    int status = 0;
+    waitpid(pid, &status, WNOHANG);
+
+    return;
+}
+

trunk/src/app/srs_app_process.hpp

@@ -91,6 +91,8 @@ class SrsProcess
     //      when use stop without fast_stop, we spend maybe [0, SRS_PROCESS_QUIT_TIMEOUT_MS * N]
     //      but use fast_stop then stop, the time is almost [0, SRS_PROCESS_QUIT_TIMEOUT_MS].
     virtual void fast_stop();
+    // Directly kill process, never use it except server quiting.
+    virtual void fast_kill();
 };
 
 #endif

trunk/src/app/srs_app_server.cpp

@@ -520,7 +520,8 @@ void SrsServer::dispose()
     close_listeners(SrsListenerRtsp);
     close_listeners(SrsListenerFlv);
 
-    // @remark don't dispose ingesters, for too slow.
+    // Fast stop to notify FFMPEG to quit, wait for a while then fast kill.
+    ingester->dispose();
 
     // dispose the source for hls and dvr.
     _srs_sources->dispose();
@@ -856,17 +857,14 @@ void SrsServer::on_signal(int signo)
         srs_trace("gmc is on, main cycle will terminate normally.");
         signal_gmc_stop = true;
 #else
-        srs_trace("user terminate program");
-#ifdef SRS_AUTO_MEM_WATCH
+        #ifdef SRS_AUTO_MEM_WATCH
         srs_memory_report();
+        #endif
 #endif
-        exit(0);
-#endif
-        return;
     }
 
-    if (signo == SRS_SIGNAL_GRACEFULLY_QUIT && !signal_gracefully_quit) {
-        srs_trace("user terminate program, gracefully quit.");
+    if ((signo == SIGINT || signo == SRS_SIGNAL_GRACEFULLY_QUIT) && !signal_gracefully_quit) {
+        srs_trace("sig=%d, user terminate program, gracefully quit", signo);
         signal_gracefully_quit = true;
         return;
     }

trunk/src/core/srs_core.hpp

@@ -27,7 +27,7 @@
 // The version config.
 #define VERSION_MAJOR       3
 #define VERSION_MINOR       0
-#define VERSION_REVISION    110
+#define VERSION_REVISION    111
 
 // The macros generated by configure script.
 #include <srs_auto_headers.hpp>