rename

Author: nicoloboschi

.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+name: CI
+on:
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: "Setup: Python 3.11"
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install poetry
+
+      - name: Run unit tests
+        run: |
+          poetry install
+          poetry run pytest

README.md

@@ -1,37 +1,37 @@
-# `requests-anon`
+# Protatoquests (Proxy Rotation Requests)
 
 <p align="center">
-  <a href="https://pypi.org/project/requests-anon/">
-    <img src="https://img.shields.io/pypi/v/requests-anon?color=green&amp;label=pypi%20package" alt="PyPI">
+  <a href="https://pypi.org/project/protatoquests/">
+    <img src="https://img.shields.io/pypi/v/protatoquests?color=green&amp;label=pypi%20package" alt="PyPI">
   </a>
-  <a href="https://pepy.tech/project/requests-anon">
-    <img src="https://static.pepy.tech/badge/requests-anon" alt="Downloads">
+  <a href="https://pepy.tech/project/protatoquests">
+    <img src="https://static.pepy.tech/badge/protatoquests" alt="Downloads">
   </a>
   <a href="">
-    <img src="https://img.shields.io/pypi/pyversions/requests-anon?color=green" alt="Py versions">
+    <img src="https://img.shields.io/pypi/pyversions/protatoquests?color=green" alt="Py versions">
   </a>
 </p>
 
-Execute HTTPS requests to a server using anonymous proxies.
-The intended usage is to by-pass server's IP Blocking while scraping data from API or webpages.
+Automatic proxy rotation for anonymous web requests.
+The intended usage is to by-pass server's IP blocking by using a pool of free proxies to perform requests and web scraping.
 It's important to note that using anonymous proxies is **risky**, **unsafe** and will cause **credentials leaks**.
 
 ## Installation
 
 ```bash
-pip install requests-anon
+pip install protatoquests
 ```
 
 ## Usage
 
 ```python
 import requests
-from requests_anon import anon_request
+import protatoquests
 
 # this one will contact the server directly
 response = requests.get("https://google.com")
 # this one will contact the server using an anonymous proxy 
-response = anon_request("get", "https://google.com")
+response = protatoquests.get("https://google.com")
 ```
 
 ## Considerations

protatoquests/__init__.py

@@ -0,0 +1,50 @@
+import requests
+import logging
+
+from protatoquests.proxy import download_proxies, get_cached_proxies
+
+logger = logging.getLogger(__name__)
+
+def get(url, **kwargs):
+    return request("GET", url, **kwargs)
+
+
+def post(url, **kwargs):
+    return request("POST", url, **kwargs)
+
+def put(url, **kwargs):
+    return request("PUT", url, **kwargs)
+
+
+def delete(url, **kwargs):
+    return request("DELETE", url, **kwargs)
+
+
+def patch(url, **kwargs):
+    return request("PATCH", url, **kwargs)
+
+
+def request(method, url, **kwargs):
+    return _request_with_proxy(False, method, url, **kwargs)
+
+
+def _request_with_proxy(ssl_verify, method, url, **kwargs):
+    proxies = get_cached_proxies()
+    for proxy in proxies:
+        logger.debug(f"Using proxy: {proxy}")
+        try:
+            args = {**kwargs, "proxies": {"https": "socks5://" + proxy}}
+            if not ssl_verify:
+                args["verify"] = False
+            response = requests.request(method, url, **args)
+        except Exception as e:
+            logger.debug(f"Proxy {proxy} failed: {e}")
+            continue
+        if response.status_code < 400:
+            logger.debug(f"Proxy {proxy} success")
+            return response
+        else:
+            logger.debug(f"Proxy {proxy} failed: {response.status_code}: {response.text}")
+
+    logger.warning("Failed to execute request with proxies, falling back to not using proxy")
+    return requests.request(method, url, **kwargs)

protatoquests/proxy.py

@@ -0,0 +1,38 @@
+import base64
+import logging
+import time
+from functools import lru_cache
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_CACHE_TTL = 300
+cache_ttl_seconds = DEFAULT_CACHE_TTL
+
+def set_cache_ttl(ttl: int):
+    global cache_ttl_seconds
+    cache_ttl_seconds = ttl
+
+def get_cached_proxies():
+    global cache_ttl_seconds
+    cache_hash = round(time.time() / cache_ttl_seconds)
+    return download_proxies(cache_hash=cache_hash)
+
+@lru_cache
+def download_proxies(cache_hash: int):
+    response = requests.get("https://advanced.name/freeproxy?type=socks5")
+    if response.status_code != 200:
+        logger.error(f"Unable to scrape proxies: {response.status_code} {response.text})")
+        return []
+    from bs4 import BeautifulSoup
+    parsed_html = BeautifulSoup(response.text)
+    table = parsed_html.body.find("table", id="table_proxies")
+    proxies = []
+    for row in table.find_all("tr"):
+        cols = row.find_all("td")
+        if cols:
+            ip = base64.b64decode(cols[1].get("data-ip")).decode("utf-8")
+            port = base64.b64decode(cols[2].get("data-port")).decode("utf-8")
+            proxies.append(f"{ip}:{port}")
+    return proxies

pyproject.toml

@@ -1,16 +1,25 @@
 [tool.poetry]
-name = "requests-anon"
-version = "0.1.1"
-description = "Execute HTTPS requests to a server using anonymous proxies."
+name = "protatoquests"
+version = "1.0.0"
+description = "Automatic proxy rotation for anonymous web requests."
 authors = ["Nicolò Boschi <[email protected]>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.9"
 requests = {extras = ["socks"], version = "^2.32.3"}
+beautifulsoup4 = "^4"
 
 
+[tool.poetry.group.test.dependencies]
+pytest = "^8.3.2"
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
+[tool.pytest.ini_options]
+log_cli = true
+log_cli_level = "INFO"
+log_cli_format = "%(asctime)s %(levelname)s %(message)s"
+log_cli_date_format = "%Y-%m-%d %H:%M:%S"

requests_anon/__init__.py

@@ -0,0 +1,7 @@
+import protatoquests
+
+def test():
+    response = protatoquests.get("https://www.google.com")
+    response.raise_for_status()
+    print(response.text)
+    assert "https://www.google.com" in response.text