Detect dead jobs and set to 'failed' status

src/Controller.php

@@ -566,6 +566,9 @@ public static function wp2staticManuallyEnqueueJobs() : void {
         earlier jobs of the same type having been "squashed" first
     */
     public static function wp2staticProcessQueue() : void {
+        global $wpdb;
+
+        JobQueue::markFailedJobs();
         // skip any earlier jobs of same type still in 'waiting' status
         JobQueue::squashQueue();
 
@@ -609,11 +612,21 @@ public static function wp2staticProcessQueue() : void {
                         WsLog::l( 'No deployment add-ons are enabled, skipping deployment.' );
                     } else {
                         WsLog::l( 'Starting deployment' );
-                        do_action(
-                            'wp2static_deploy',
-                            ProcessedSite::getPath(),
-                            $deployer
-                        );
+                        $query = "SELECT GET_LOCK('wp2static_jobs_deploying', 30) AS lck";
+                        $locked = intval( $wpdb->get_row( $query )->lck );
+                        if ( ! $locked ) {
+                            WsLog::l( 'Failed to acquire "wp2static_jobs_deploying" lock.' );
+                            return;
+                        }
+                        try {
+                            do_action(
+                                'wp2static_deploy',
+                                ProcessedSite::getPath(),
+                                $deployer
+                            );
+                        } finally {
+                            $wpdb->query( "DO RELEASE_LOCK('wp2static_jobs_deploying')" );
+                        }
                     }
                     WsLog::l( 'Starting post-deployment actions' );
                     do_action( 'wp2static_post_deploy_trigger', $deployer );

src/JobQueue.php

@@ -231,4 +231,39 @@ public static function truncate() : void {
             WsLog::l( 'failed to truncate JobQueue: try deleting instead' );
         }
     }
+
+    /**
+     *  Detect any 'processing' jobs that are not running and change status to 'failed'.
+     *
+     *  @throws \Throwable
+     */
+    public static function markFailedJobs() : void {
+        global $wpdb;
+
+        $table_name = $wpdb->prefix . 'wp2static_jobs';
+
+        $wpdb->query( 'START TRANSACTION' );
+
+        try {
+            $query = "SELECT IS_FREE_LOCK('wp2static_jobs_deploying') AS free";
+            $deploy_free = intval( $wpdb->get_row( $query )->free );
+
+            if ( $deploy_free ) {
+                $failed_jobs = $wpdb->query(
+                    "UPDATE $table_name
+                    SET status = 'failed'
+                    WHERE job_type = 'deploy' AND status = 'processing'"
+                );
+                if ( $failed_jobs ) {
+                    $s = $failed_jobs === 1 ? '' : 's';
+                    WsLog::l( "$failed_jobs processing deploy job$s marked as failed." );
+                }
+            }
+
+            $wpdb->query( 'COMMIT' );
+        } catch ( \Throwable $e ) {
+            $wpdb->query( 'ROLLBACK' );
+            throw $e;
+        }
+    }
 }

src/ViewRenderer.php

@@ -217,6 +217,9 @@ function ( $path ) use ( $s ) {
     }
 
     public static function renderJobsPage() : void {
+        JobQueue::markFailedJobs();
+        JobQueue::squashQueue();
+
         $view = [];
         $view['nonce_action'] = 'wp2static-ui-job-options';
         $view['jobs'] = JobQueue::getJobs();