initial commit

LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) w3lib and Scrapy developers.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice, 
+       this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright 
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+
+    3. Neither the name of Scrapy nor the names of its contributors may be used
+       to endorse or promote products derived from this software without
+       specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

MANIFEST

@@ -0,0 +1,11 @@
+# file GENERATED by distutils, do NOT edit
+LICENSE
+NEWS
+README.rst
+setup.py
+queuelib/__init__.py
+queuelib/pqueue.py
+queuelib/queue.py
+queuelib/tests/__init__.py
+queuelib/tests/test_pqueue.py
+queuelib/tests/test_queue.py

MANIFEST.in

@@ -0,0 +1,3 @@
+include README.rst
+include LICENSE
+include NEWS

NEWS

@@ -0,0 +1,8 @@
+Queuelib release notes
+======================
+
+Version 1.0
+-----------
+(released on April 23rd, 2013)
+
+First release of Queuelib.

README.rst

@@ -0,0 +1,146 @@
+========
+queuelib
+========
+
+Queuelib is a collection of persistent (disk-based) queues for Python.
+
+Queuelib goals are speed and simplicity. It was originally part of the `Scrapy
+framework`_ and stripped out on its own library.
+
+Requirements
+============
+
+* Python 2.6
+* no external library requirements
+
+Installation
+
+Installation
+============
+
+You can install Queuelib either via the Python Package Index (PyPI) or from
+source.
+
+To install using pip::
+
+    $ pip install queuelib
+
+To install using easy_install::
+
+    $ easy_install queuelib
+
+If you have downloaded a source tarball you can install it by running the
+following (as root)::
+
+    # python setup.py install
+
+FIFO/LIFO disk queues
+=====================
+
+Queuelib provides FIFO and LIFO queue implementations.
+
+Here is an example usage of the FIFO queue::
+
+    >>> from queuelib import FifoDiskQueue
+    >>> q = FifoDiskQueue("queuefile")
+    >>> q.push('a')
+    >>> q.push('b')
+    >>> q.push('c')
+    >>> q.pop()
+    'c'
+    >>> q.close()
+    >>> q = FifoDiskQueue("queuefile")
+    >>> q.pop()
+    'b'
+    >>> q.pop()
+    'a'
+    >>> q.pop()
+    >>>
+
+The LIFO queue is identical (API-wise), but importing ``LifoDiskQueue``
+instead.
+
+PriorityQueue
+=============
+
+A discrete-priority queue implemented by combining multiple FIFO/LIFO queues
+(one per priority).
+
+First, select the type of QUEUE (FIFO or LIFO)::
+
+    >>> from queuelib import FifoDiskQueue
+    >>> q = FifoDiskQueue("somedir")
+
+Then instantiate the Priority Queue with it::
+
+    >>> from queuelib import PriorityQueue
+    >>> pq = PriorityQueue(q)
+
+And use it::
+
+    >>> pq.push('a', 3)
+    >>> pq.push('b', 1)
+    >>> pq.push('c', 2)
+    >>> pq.push('d', 2)
+    >>> pq.pop()
+    'b'
+    >>> pq.pop()
+    'c'
+    >>> pq.pop()
+    'd'
+    >>> pq.pop()
+    'a'
+
+Mailing list
+============
+
+Use the `scrapy-users`_ mailing list for questions about Queuelib.
+
+Bug tracker
+===========
+
+If you have any suggestions, bug reports or annoyances please report them to
+our issue tracker at: https://github.com/scrapy/queuelib/issues/
+
+Contributing
+============
+
+Development of Queuelib happens at GitHub: https://github.com/scrapy/queuelib
+
+You are highly encouraged to participate in the development. If you don't like
+GitHub (for some reason) you're welcome to send regular patches.
+
+All changes require tests to be merged.
+
+Tests
+=====
+
+Tests are located in `queuelib/tests` directory. They can be run using
+`nosetests`_ with the following command::
+
+    nosetests
+
+The output should be something like the following::
+
+    $ nosetests
+    .............................................................................
+    ----------------------------------------------------------------------
+    Ran 77 tests in 0.145s
+
+    OK
+
+License
+=======
+
+This software is licensed under the BSD License. See the LICENSE file in the
+top distribution directory for the full license text.
+
+Versioning
+==========
+
+This software follows `Semantic Versioning`_
+
+.. _Scrapy framework: http://scrapy.org
+.. _scrapy-users: http://groups.google.com/group/scrapy-users
+.. _Semantic Versioning: http://semver.org/
+.. _nosetests: https://nose.readthedocs.org/en/latest/

queuelib/__init__.py

@@ -0,0 +1,2 @@
+from queuelib.queue import FifoDiskQueue, LifoDiskQueue
+from queuelib.pqueue import PriorityQueue

queuelib/pqueue.py

@@ -0,0 +1,60 @@
+class PriorityQueue(object):
+    """A priority queue implemented using multiple internal queues (typically,
+    FIFO queues). The internal queue must implement the following methods:
+
+        * push(obj)
+        * pop()
+        * close()
+        * __len__()
+
+    The constructor receives a qfactory argument, which is a callable used to
+    instantiate a new (internal) queue when a new priority is allocated. The
+    qfactory function is called with the priority number as first and only
+    argument.
+
+    Only integer priorities should be used. Lower numbers are higher
+    priorities.
+
+    startprios is a sequence of priorities to start with. If the queue was
+    previously closed leaving some priority buckets non-empty, those priorities
+    should be passed in startprios.
+
+    """
+
+    def __init__(self, qfactory, startprios=()):
+        self.queues = {}
+        self.qfactory = qfactory
+        for p in startprios:
+            self.queues[p] = self.qfactory(p)
+        self.curprio = min(startprios) if startprios else None
+
+    def push(self, obj, priority=0):
+        if priority not in self.queues:
+            self.queues[priority] = self.qfactory(priority)
+        q = self.queues[priority]
+        q.push(obj) # this may fail (eg. serialization error)
+        if priority < self.curprio or self.curprio is None:
+            self.curprio = priority
+
+    def pop(self):
+        if self.curprio is None:
+            return
+        q = self.queues[self.curprio]
+        m = q.pop()
+        if len(q) == 0:
+            del self.queues[self.curprio]
+            q.close()
+            prios = [p for p, q in self.queues.items() if len(q) > 0]
+            self.curprio = min(prios) if prios else None
+        return m
+
+    def close(self):
+        active = []
+        for p, q in self.queues.items():
+            if len(q):
+                active.append(p)
+            q.close()
+        return active
+
+    def __len__(self):
+        return sum(len(x) for x in self.queues.values()) if self.queues else 0