diff --git a/CHANGES.rst b/CHANGES.rst index fab624301..4d92dcaaf 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,14 @@ upstream changes. .. begin_changelog_body +4.0.0a1 +------- + +**Removed** + +* Removed `praw-multiprocess` and its associated ``MultiprocessHandler``. This + functionality is no longer needed with PRAW4. + 3.4.0 (2016-02-21) ------------------ diff --git a/docs/index.rst b/docs/index.rst index 8401e6774..65562ad86 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,7 +19,6 @@ Content Pages pages/comment_parsing pages/oauth pages/lazy-loading - pages/multiprocess pages/contributor_guidelines pages/configuration_files pages/faq diff --git a/docs/pages/multiprocess.rst b/docs/pages/multiprocess.rst deleted file mode 100644 index ace7c2944..000000000 --- a/docs/pages/multiprocess.rst +++ /dev/null @@ -1,85 +0,0 @@ -.. _multiprocess: - -Concurrent PRAW Instances -========================= - -By default, PRAW works great when there is a one-to-one mapping between running -PRAW processes, and the IP address / user account that you make requests -from. In fact, as of version 2.1.0, PRAW has multithreaded support as long as -there is a one-to-one mapping between thread and PRAW :class:`.Reddit` object -instance. That is, in order to be thread safe, each thread needs to have its -own :class:`.Reddit` instance [#]_. In addition to multithreaded rate-limiting -support, PRAW 2.1.0 added multiprocess rate limiting support. - -.. [#] It is undetermined at this time if the same authentication credentials - can be used on multiple instances where the modhash is concerned. - - -praw-multiprocess ------------------ - -PRAW version 2.1.0 and later install with a program **praw-multiprocess**. This -program provides a request handling server that manages the rate-limiting and -caching of any PRAW process which directs requests toward it. Starting -**praw-multiprocess** is as simple as running ``praw-multiprocess`` from your -terminal / command line assuming you :ref:`installed PRAW ` -properly. - -By default **praw-multiprocess** will listen only on *localhost* port -*10101*. You can adjust those settings by passing in ``--addr`` and ``--port`` -arguments respectively. For instance to have **praw-multiprocess** listen on -all valid addresses on port 65000, execute via: ``praw-multiprocess --addr -0.0.0.0 --port 65000``. For a full list of options execute ``praw-multiprocess ---help``. - - -PRAW's MultiprocessingHandler ------------------------------ - -In order to interact with a **praw-multiprocess** server, PRAW needs to be -instructed to use the :class:`.MultiprocessHandler` rather than the -:class:`.DefaultHandler`. In your program you need to pass an instance of -:class:`.MultiprocessHandler` into the ``handler`` keyword argument when -creating the :class:`.Reddit` instance. Below is an example to connect to a -**praw-multiprocess** server running with the default arguments: - -.. code-block:: python - - import praw - from praw.handlers import MultiprocessHandler - - handler = MultiprocessHandler() - r = praw.Reddit(user_agent='a descriptive user-agent', handler=handler) - -With this configuration, all network requests made from your program(s) that -include the above code will be *proxied* through the `praw-multiprocess` -server. All requests made through the same **praw-multiprocess** server will -respect reddit's API rate-limiting rules - -If instead, you wish to connect to a **praw-multiprocess** server running at -address ``10.0.0.1`` port 65000 then you would create the PRAW instance via: - -.. code-block:: python - - import praw - from praw.handlers import MultiprocessHandler - - handler = MultiprocessHandler('10.0.0.1', 65000) - r = praw.Reddit(user_agent='a descriptive user-agent', handler=handler) - - -PRAW Multiprocess Resiliency ----------------------------- - -With all client/server type programs there is the possibility of network issues -or simply a lack of an available server. PRAW's :class:`.MultiprocessHandler` -was created to be quite resilient to such issues. PRAW will retry indefinitely -to connect to **praw-multiprocess** server. This means that a -**praw-multiprocess** server can be stopped and restarted without any effect on -programs utilizing it. - -On the other hand, consecutive network failures where the -:class:`.MultiprocessHandler` has no issue establishing a connection to a -**praw-multiprocess** server will result in :class:`.ClientException` after -three failures. Such failures are **not** expected to occur and if -reproducable should be :ref:`reported `. diff --git a/docs/pages/oauth.rst b/docs/pages/oauth.rst index 8455d22d1..1f3e3c520 100644 --- a/docs/pages/oauth.rst +++ b/docs/pages/oauth.rst @@ -50,9 +50,7 @@ Step 2: Setting up PRAW. If you want to persist instances of PRAW across multiple requests in a web application, we recommend that you create an new instance per distinct - authentication. Furthermore, if your web application spawns multiple - processes, it is highly recommended that you utilize PRAW's - :ref:`multiprocess ` functionality. + authentication. We start as usual by importing the PRAW package and creating a :class:`.Reddit` object with a clear and descriptive useragent that follows the `api rules diff --git a/praw/handlers.py b/praw/handlers.py index 85e9fe68a..1e356fdaf 100644 --- a/praw/handlers.py +++ b/praw/handlers.py @@ -1,16 +1,11 @@ """Provides classes that handle request dispatching.""" from __future__ import print_function, unicode_literals - -import socket -import sys import time from functools import wraps -from praw.errors import ClientException from praw.helpers import normalize_url from requests import Session from six import text_type -from six.moves import cPickle # pylint: disable=F0401 from threading import Lock from timeit import default_timer as timer @@ -181,59 +176,3 @@ def evict(cls, urls): del cls.timeouts[key] return retval DefaultHandler.request = DefaultHandler.with_cache(RateLimitHandler.request) - - -class MultiprocessHandler(object): - """A PRAW handler to interact with the PRAW multi-process server.""" - - def __init__(self, host='localhost', port=10101): - """Construct an instance of the MultiprocessHandler.""" - self.host = host - self.port = port - - def _relay(self, **kwargs): - """Send the request through the server and return the HTTP response.""" - retval = None - delay_time = 2 # For connection retries - read_attempts = 0 # For reading from socket - while retval is None: # Evict can return False - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock_fp = sock.makefile('rwb') # Used for pickle - try: - sock.connect((self.host, self.port)) - cPickle.dump(kwargs, sock_fp, cPickle.HIGHEST_PROTOCOL) - sock_fp.flush() - retval = cPickle.load(sock_fp) - except: # pylint: disable=W0702 - exc_type, exc, _ = sys.exc_info() - socket_error = exc_type is socket.error - if socket_error and exc.errno == 111: # Connection refused - sys.stderr.write('Cannot connect to multiprocess server. I' - 's it running? Retrying in {0} seconds.\n' - .format(delay_time)) - time.sleep(delay_time) - delay_time = min(64, delay_time * 2) - elif exc_type is EOFError or socket_error and exc.errno == 104: - # Failure during socket READ - if read_attempts >= 3: - raise ClientException('Successive failures reading ' - 'from the multiprocess server.') - sys.stderr.write('Lost connection with multiprocess server' - ' during read. Trying again.\n') - read_attempts += 1 - else: - raise - finally: - sock_fp.close() - sock.close() - if isinstance(retval, Exception): - raise retval # pylint: disable=E0702 - return retval - - def evict(self, urls): - """Forward the eviction to the server and return its response.""" - return self._relay(method='evict', urls=urls) - - def request(self, **kwargs): - """Forward the request to the server and return its HTTP response.""" - return self._relay(method='request', **kwargs) diff --git a/praw/multiprocess.py b/praw/multiprocess.py deleted file mode 100644 index 3d5aad2d4..000000000 --- a/praw/multiprocess.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Provides a request server to be used with the multiprocess handler.""" - -from __future__ import print_function, unicode_literals - -import socket -import sys -from optparse import OptionParser -from praw import __version__ -from praw.handlers import DefaultHandler -from requests import Session -from six.moves import cPickle, socketserver # pylint: disable=F0401 -from threading import Lock - - -class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): - # pylint: disable=R0903,W0232 - """A TCP server that creates new threads per connection.""" - - allow_reuse_address = True - - @staticmethod - def handle_error(_, client_addr): - """Mute tracebacks of common errors.""" - exc_type, exc_value, _ = sys.exc_info() - if exc_type is socket.error and exc_value[0] == 32: - pass - elif exc_type is cPickle.UnpicklingError: - sys.stderr.write('Invalid connection from {0}\n' - .format(client_addr[0])) - else: - raise - - -class RequestHandler(socketserver.StreamRequestHandler): - # pylint: disable=W0232 - """A class that handles incoming requests. - - Requests to the same domain are cached and rate-limited. - - """ - - ca_lock = Lock() # lock around cache and timeouts - cache = {} # caches requests - http = Session() # used to make requests - last_call = {} # Stores a two-item list: [lock, previous_call_time] - rl_lock = Lock() # lock used for adding items to last_call - timeouts = {} # store the time items in cache were entered - - do_evict = DefaultHandler.evict # Add in the evict method - - @staticmethod - def cache_hit_callback(key): - """Output when a cache hit occurs.""" - print('HIT {0} {1}'.format('POST' if key[1][1] else 'GET', key[0])) - - @DefaultHandler.with_cache - @DefaultHandler.rate_limit - def do_request(self, request, proxies, timeout, **_): - """Dispatch the actual request and return the result.""" - print('{0} {1}'.format(request.method, request.url)) - response = self.http.send(request, proxies=proxies, timeout=timeout, - allow_redirects=False) - response.raw = None # Make pickleable - return response - - def handle(self): - """Parse the RPC, make the call, and pickle up the return value.""" - data = cPickle.load(self.rfile) # pylint: disable=E1101 - method = data.pop('method') - try: - retval = getattr(self, 'do_{0}'.format(method))(**data) - except Exception as e: - # All exceptions should be passed to the client - retval = e - cPickle.dump(retval, self.wfile, # pylint: disable=E1101 - cPickle.HIGHEST_PROTOCOL) - - -def run(): - """The entry point from the praw-multiprocess utility.""" - parser = OptionParser(version='%prog {0}'.format(__version__)) - parser.add_option('-a', '--addr', default='localhost', - help=('The address or host to listen on. Specify -a ' - '0.0.0.0 to listen on all addresses. ' - 'Default: localhost')) - parser.add_option('-p', '--port', type='int', default='10101', - help=('The port to listen for requests on. ' - 'Default: 10101')) - options, _ = parser.parse_args() - try: - server = ThreadingTCPServer((options.addr, options.port), - RequestHandler) - except (socket.error, socket.gaierror) as exc: # Handle bind errors - print(exc) - sys.exit(1) - print('Listening on {0} port {1}'.format(options.addr, options.port)) - try: - server.serve_forever() # pylint: disable=E1101 - except KeyboardInterrupt: - server.socket.close() # pylint: disable=E1101 - RequestHandler.http.close() - print('Goodbye!') diff --git a/setup.py b/setup.py index dc3f3e209..943308c4d 100644 --- a/setup.py +++ b/setup.py @@ -36,8 +36,6 @@ description=('PRAW, an acronym for `Python Reddit API Wrapper`, is a ' 'python package that allows for simple access to ' 'reddit\'s API.'), - entry_points={'console_scripts': [ - 'praw-multiprocess = praw.multiprocess:run']}, install_requires=['decorator >=4.0.9, <4.1', 'requests >=2.3.0', 'six ==1.10',