Issue #78: Random user agent by default

dirhunt/agents.py

@@ -0,0 +1,16 @@
+import random
+
+USER_AGENTS = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 '
+    'Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36',
+    'Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 '
+    'Safari/537.36 Edge/17.17134',
+    'Mozilla/5.0 (Linux; U; Android 2.2) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1',
+]
+
+
+def get_random_user_agent():
+    return random.choice(USER_AGENTS)

dirhunt/sessions.py

@@ -13,6 +13,7 @@
 from proxy_db.proxies import ProxiesList
 from proxy_db.models import Proxy
 
+from dirhunt.agents import get_random_user_agent
 
 if sys.version_info < (3, 0):
     ConnectionError = IOError
@@ -82,6 +83,9 @@ def __init__(self, sessions, proxy):
         self.proxy_name = proxy
         self.proxy = normalize_proxy(self.proxy_name, sessions)
         self.session = requests.Session()
+        self.session.headers = {
+            'User-Agent': get_random_user_agent(),
+        }
         adapter = HTTPAdapter(pool_connections=POOL_CONNECTIONS, pool_maxsize=POOL_CONNECTIONS)
         self.session.mount('http://', adapter)
         self.session.mount('https://', adapter)