diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..d0e2fa2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,37 @@ +name: release +on: + push: + tags: + - 'v*.*.*' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: build + shell: bash + run: | + python -m pip install --upgrade wheel gevent + python setup.py sdist bdist_wheel --universal + - name: Release PyPI + shell: bash + env: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + pip install --upgrade twine + twine upload dist/* + + - name: Release GitHub + uses: softprops/action-gh-release@v1 + with: + files: "dist/*" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..3c7b97a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,20 @@ +on: [ push, pull_request ] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install gevent requests pytest nose + - name: Test with pytest + run: | + pytest tests.py diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c4f010d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,27 +0,0 @@ -dist: xenial -language: python - - -python: - - "2.7" - - "3.6" - - "3.7" - - -install: - - pip install -r requirements.txt - - pip install pytest - -script: - - pytest tests.py - - -deploy: - provider: pypi - distributions: "sdist bdist_wheel" - user: "spyoungtech" - password: - secure: "QtuuH0X/A/iQI23MxvqsnxUy63XD5awJHDkeQNmUDIGGQqIox2DTYKoc6x354I5wpqprtODQRYRqIsA9+2cpRcF49Ft50cvi3cmuoeozkID3ybQyLHCIcJ4CKt6X+h2LFbrgqyyBcny7tKQlYr4/nsjeQegPblnJ6OTljJgJyE0=" - on: - tags: true - python: 3.6 \ No newline at end of file diff --git a/README.rst b/README.rst index 20db323..d59640e 100644 --- a/README.rst +++ b/README.rst @@ -80,6 +80,22 @@ For some speed/performance gains, you may also want to use ``imap`` instead of ` print(resp) +There is also an enumerated version of ``imap`` which yields the index and response. However, unlike ``imap`` the ``requests`` parameter for ``imap_enumerated`` must be a sequence. Additionally, +failed requests and exception handler results that return None will also be yielded (whereas in ``imap`` they are ignored). Like in ``imap``, the order in which requests are sent and received should be +considered arbitrary. + +.. code-block:: python + + >>> rs = [grequests.get(f'https://httpbin.org/status/{code}') for code in range(200, 206)] + >>> for index, response in grequests.imap_enumerated(rs, size=5): + ... print(index, response) + 1 + 0 + 4 + 2 + 5 + 3 + NOTE: because ``grequests`` leverages ``gevent`` (which in turn uses monkeypatching for enabling concurrency), you will often need to make sure ``grequests`` is imported before other libraries, especially ``requests``, to avoid problems. See `grequests gevent issues `_ for additional information. diff --git a/grequests.py b/grequests.py index f4ec5c7..acf1933 100755 --- a/grequests.py +++ b/grequests.py @@ -163,3 +163,51 @@ def send(r): yield ex_result pool.join() + + +def imap_enumerated(requests, stream=False, size=2, exception_handler=None): + """ + Like imap, but yields tuple of original request index and response object + + Unlike imap, failed results and responses from exception handlers that return None are not ignored. Instead, a + tuple of (index, None) is yielded. Additionally, the ``requests`` parameter must be a sequence of Request objects + (generators or other non-sequence iterables are not allowed) + + The index is merely the original index of the original request in the requests list and does NOT provide any + indication of the order in which requests or responses are sent or received. Responses are still in arbitrary order. + + :: + >>> rs = [grequests.get(f'https://httpbin.org/status/{i}') for i in range(200, 206)] + >>> for index, response in grequests.imap_enumerated(rs, size=5): + ... print(index, response) + 1 + 0 + 4 + 2 + 5 + 3 + + + :param requests: a sequence of Request objects. + :param stream: If True, the content will not be downloaded immediately. + :param size: Specifies the number of requests to make at a time. default is 2 + :param exception_handler: Callback function, called when exception occurred. Params: Request, Exception + """ + + pool = Pool(size) + + def send(r): + return r._index, r.send(stream=stream) + + requests = list(requests) + for index, req in enumerate(requests): + req._index = index + + for index, request in pool.imap_unordered(send, requests): + if request.response is not None: + yield index, request.response + elif exception_handler: + ex_result = exception_handler(request, request.exception) + yield index, ex_result + else: + yield index, None diff --git a/setup.py b/setup.py index 9122abd..0343841 100644 --- a/setup.py +++ b/setup.py @@ -33,8 +33,8 @@ setup( name='grequests', - version='0.6.0', - url='https://github.com/kennethreitz/grequests', + version='0.7.0', + url='https://github.com/spyoungtech/grequests', license='BSD', author='Kenneth Reitz', author_email='me@kennethreitz.com',