feat: Fill all available slots (max_proc) when polling, closes #173

docs/news.rst

@@ -28,6 +28,7 @@ Documentation
 Changed
 ~~~~~~~
 
+- Every :ref:`poll_interval`, up to :ref:`max_proc` processes are started by the default :ref:`poller`, instead of only one process. (The number of running jobs will not exceed :ref:`max_proc`.)
 - Drop support for end-of-life Python version 3.7.
 
 Web UI

scrapyd/poller.py

@@ -15,17 +15,17 @@ def __init__(self, config):
     @inlineCallbacks
     def poll(self):
         for project, queue in self.queues.items():
-            # If the "waiting" backlog is empty (that is, if the maximum number of Scrapy processes are running):
-            if not self.dq.waiting:
-                return
-            if (yield maybeDeferred(queue.count)):
+            while (yield maybeDeferred(queue.count)):
+                # If the "waiting" backlog is empty (that is, if the maximum number of Scrapy processes are running):
+                if not self.dq.waiting:
+                    return
                 message = (yield maybeDeferred(queue.pop)).copy()
                 # The message can be None if, for example, two Scrapyd instances share a spider queue database.
                 if message is not None:
                     message["_project"] = project
                     message["_spider"] = message.pop("name")
                     # Pop a dummy item from the "waiting" backlog. and fire the message's callbacks.
-                    return self.dq.put(message)
+                    self.dq.put(message)
 
     def next(self):
         """