Merge pull request Anorov#239 from pro-src/tests

Add tests

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+addopts = -p no:warnings
+timeout = 2000

tests/__init__.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+
+import responses
+import pytest
+import re
+
+from requests.compat import urlencode
+from collections import OrderedDict
+from os import path
+from io import open
+
+# Fake URL, network requests are not allowed by default when using the decorator
+url = 'http://example-site.dev'
+# These kwargs will be passed to tests by the decorator
+cfscrape_kwargs = dict(
+    delay=0.01
+)
+# Cloudflare challenge fixtures are only read from the FS once
+cache = {}
+
+
+class ChallengeResponse(responses.Response):
+    """Simulates a standard IUAM JS challenge response from Cloudflare
+
+    This would be the first response in a test.
+
+    Kwargs:
+        Keyword arguments used to override the defaults.
+        The request will error if it doesn't match a defined response.
+    """
+
+    def __init__(self, **kwargs):
+        defaults = (('method', 'GET'),
+                    ('status', 503),
+                    ('headers', {'Server': 'cloudflare'}),
+                    ('content_type', 'text/html'))
+
+        for k, v in defaults:
+            kwargs.setdefault(k, v)
+
+        super(ChallengeResponse, self).__init__(**kwargs)
+
+
+class RedirectResponse(responses.CallbackResponse):
+    """Simulate the redirect response that occurs after sending a correct answer
+
+    This would be the second response in a test.
+    It will call the provided callback when a matching request is received.
+    Afterwards, the default is to redirect to the index page "/" aka fake URL.
+
+    Kwargs:
+        Keyword arguments used to override the defaults.
+        The request will error if it doesn't match a defined response.
+    """
+
+    def __init__(self, callback=lambda request: None, **kwargs):
+        defaults = (('method', 'GET'),
+                    ('status', 302),
+                    ('headers', {'Location': '/'}),
+                    ('content_type', 'text/html'),
+                    ('body', ''))
+
+        for k, v in defaults:
+            kwargs.setdefault(k, v)
+
+        args = tuple(kwargs.pop(k) for k in ('status', 'headers', 'body'))
+        kwargs['callback'] = lambda request: callback(request) or args
+
+        super(RedirectResponse, self).__init__(**kwargs)
+
+
+class DefaultResponse(responses.Response):
+    """Simulate the final response after the challenge is solved
+
+    This would be the last response in a test and normally occurs after a redirect.
+
+    Kwargs:
+        Keyword arguments used to override the defaults.
+        The request will error if it doesn't match a defined response.
+    """
+
+    def __init__(self, **kwargs):
+        defaults = (('method', 'GET'),
+                    ('status', 200),
+                    ('content_type', 'text/html'))
+
+        for k, v in defaults:
+            kwargs.setdefault(k, v)
+
+        super(DefaultResponse, self).__init__(**kwargs)
+
+
+def fixtures(filename):
+    """Read and cache a challenge fixture
+
+    Returns: HTML (bytes): The HTML challenge fixture
+    """
+    if not cache.get(filename):
+        with open(path.join(path.dirname(__file__), 'fixtures', filename), 'rb') as fp:
+            cache[filename] = fp.read()
+    return cache[filename]
+
+
+# This is the page that should be received after bypassing the JS challenge.
+requested_page = fixtures('requested_page.html')
+
+
+# This fancy decorator wraps tests so the responses will be mocked.
+# It could be called directly e.g. challenge_responses(*args)(test_func) -> wrapper
+def challenge_responses(filename, jschl_answer):
+    # This function is called with the test_func and returns a new wrapper.
+    def challenge_responses_decorator(test):
+        @responses.activate
+        def wrapper(self):
+            html = fixtures(filename).decode('utf-8')
+
+            params = OrderedDict()
+
+            s = re.search(r'name="s"\svalue="(?P<s_value>[^"]+)', html)
+            if s:
+                params['s'] = s.group('s_value')
+            params['jschl_vc'] = re.search(r'name="jschl_vc" value="(\w+)"', html).group(1)
+            params['pass'] = re.search(r'name="pass" value="(.+?)"', html).group(1)
+            params['jschl_answer'] = jschl_answer
+
+            submit_uri = '{}/cdn-cgi/l/chk_jschl?{}'.format(url, urlencode(params))
+
+            responses.add(ChallengeResponse(url=url, body=fixtures(filename)))
+
+            def onRedirect(request):
+                # We don't register the last response unless the redirect occurs
+                responses.add(DefaultResponse(url=url, body=requested_page))
+
+            responses.add(RedirectResponse(url=submit_uri, callback=onRedirect))
+
+            return test(self, **cfscrape_kwargs)
+        # The following causes pytest to call the test wrapper once for each interpreter.
+        return wrapper
+
+    return challenge_responses_decorator

tests/fixtures/cf_recaptcha_15_04_2019.html

@@ -0,0 +1,162 @@
+<!DOCTYPE html>
+<!--[if lt IE 7]> <html class="no-js ie6 oldie" lang="en-US"> <![endif]-->
+<!--[if IE 7]>    <html class="no-js ie7 oldie" lang="en-US"> <![endif]-->
+<!--[if IE 8]>    <html class="no-js ie8 oldie" lang="en-US"> <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en-US"> <!--<![endif]-->
+<head>
+<title>Attention Required! | Cloudflare</title>
+<meta name="captcha-bypass" id="captcha-bypass" />
+<meta charset="UTF-8" />
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<meta http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1" />
+<meta name="robots" content="noindex, nofollow" />
+<meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1" />
+<link rel="stylesheet" id="cf_styles-css" href="/cdn-cgi/styles/cf.errors.css" type="text/css" media="screen,projection" />
+<!--[if lt IE 9]><link rel="stylesheet" id='cf_styles-ie-css' href="/cdn-cgi/styles/cf.errors.ie.css" type="text/css" media="screen,projection" /><![endif]-->
+<style type="text/css">body{margin:0;padding:0}</style>
+
+
+<!--[if gte IE 10]><!--><script type="text/javascript" src="/cdn-cgi/scripts/zepto.min.js"></script><!--<![endif]-->
+<!--[if gte IE 10]><!--><script type="text/javascript" src="/cdn-cgi/scripts/cf.common.js"></script><!--<![endif]-->
+
+
+
+
+</head>
+<body>
+  <div id="cf-wrapper">
+    <div class="cf-alert cf-alert-error cf-cookie-error" id="cookie-alert" data-translate="enable_cookies">Please enable cookies.</div>
+    <div id="cf-error-details" class="cf-error-details-wrapper">
+      <div class="cf-wrapper cf-header cf-error-overview">
+        <h1 data-translate="challenge_headline">One more step</h1>
+        <h2 class="cf-subheadline"><span data-translate="complete_sec_check">Please complete the security check to access</span> example-site.dev</h2>
+      </div><!-- /.header -->
+
+      <div class="cf-section cf-highlight cf-captcha-container">
+        <div class="cf-wrapper">
+          <div class="cf-columns two">
+            <div class="cf-column">
+            <div style="position: absolute; top: -250px; left: -250px;"><a href="https://macinline.com/answeradenoidal.php?day=688">table</a></div>
+              <div class="cf-highlight-inverse cf-form-stacked">
+                <form class="challenge-form" id="challenge-form" action="/cdn-cgi/l/chk_captcha" method="get">
+                  <input type="hidden" name="s" value="6b132d85d185a8255f2451d48fe6a8bee7154ea2-1555377580-1800-AQ1azEkeDOnQP5ByOpwUU/RdbKrmMwHYpkaenRvjPXtB0w8Vbjn/Ceg62tfpp/lT799kjDLEMMuDkEMqQ7iO51kniWCQm00BQvDGl+D0h/WvXDWO96YXOUD3qrqUTuzO7QbUOinc8y8kedvOQkr4c0o="></input>
+  <script type="text/javascript" src="/cdn-cgi/scripts/cf.challenge.js" data-type="normal"  data-ray="0000000000000000" async data-sitekey="6LfBixYUAAAAABhdHynFUIMA_sa4s-XsJvnjtgB0"></script>
+  <div class="g-recaptcha"></div>
+  <noscript id="cf-captcha-bookmark" class="cf-captcha-info">
+    <div><div style="width: 302px">
+      <div>
+        <iframe src="https://www.google.com/recaptcha/api/fallback?k=6LfBixYUAAAAABhdHynFUIMA_sa4s-XsJvnjtgB0" frameborder="0" scrolling="no" style="width: 302px; height:422px; border-style: none;"></iframe>
+      </div>
+      <div style="width: 300px; border-style: none; bottom: 12px; left: 25px; margin: 0px; padding: 0px; right: 25px; background: #f9f9f9; border: 1px solid #c1c1c1; border-radius: 3px;">
+        <textarea id="g-recaptcha-response" name="g-recaptcha-response" class="g-recaptcha-response" style="width: 250px; height: 40px; border: 1px solid #c1c1c1; margin: 10px 25px; padding: 0px; resize: none;"></textarea>
+        <input type="submit" value="Submit"></input>
+      </div>
+    </div></div>
+  </noscript>
+</form>
+
+                <script type="text/javascript">
+  (function(){
+    var a = function() {try{return !!window.addEventListener} catch(e) {return !1} },
+    b = function(b, c) {a() ? document.addEventListener("DOMContentLoaded", b, c) : document.attachEvent("onreadystatechange", b)};
+    b(function(){
+      if (!a()) return;
+
+      window.addEventListener("message", handleMessage, false)
+
+      function handleMessage(event) {
+        if (event.data && event.data.type === 'results') {
+          var f = document.getElementById('challenge-form');
+
+          if (f) {
+            addInput(f, 'bf_challenge_id', '342');
+            addInput(f, 'bf_execution_time', event.data.executionTimeMs);
+            addInput(f, 'bf_result_hash', event.data.resultHash);
+          }
+
+          window.removeEventListener("message", handleMessage, false)
+        }
+      }
+
+      function addInput(parent, name, value) {
+        var input = document.createElement('input');
+        input.type = 'hidden';
+        input.name = name;
+        input.value = value;
+        parent.appendChild(input);
+      }
+
+      function withIframe(iframeContent) {
+        var iframe = document.createElement('iframe');
+        iframe.id = 'bf_test_iframe';
+        iframe.style.visibility = 'hidden';
+        document.body.appendChild(iframe);
+        var doc = (iframe.contentWindow || iframe.contentDocument).document;
+        doc.write(iframeContent);
+        doc.close();
+      }
+
+      withIframe("<!DOCTYPE HTML>\n<meta charset=utf-8>\n<html>\n <head>\n  <title><\/title>\n  <script src=\"https:\/\/ajax.cloudflare.com\/cdn-cgi\/scripts\/697236fc\/cloudflare-static\/bot-filter.js\"><\/__script__>\n  \n <\/head>\n <body>\n    <h1><\/h1>\n  \n <\/body>\n<\/html>\n<script>function r(){var r='<?xml version=\"1.0\" encoding=\"UTF-8\"?><root><child1>value1<\/child1><\/root>',e;return(new DOMParser).parseFromString(r,\"text\/xml\")}test(function(){var r='<root xmlns=\"urn:bar\"><outer xmlns=\"\"><inner>value1<\/inner><\/outer><\/root>',root=(new DOMParser).parseFromString(r,\"text\/xml\").documentElement,e=(new XMLSerializer).serializeToString(root);__c$1(e)});<\/__script__>".replace(/\/__script__/g, '/script'));
+
+    }, false);
+  })();
+  </script>
+
+              </div>
+            </div>
+
+            <div class="cf-column">
+              <div class="cf-screenshot-container">
+
+                <span class="cf-no-screenshot"></span>
+
+              </div>
+            </div>
+          </div><!-- /.columns -->
+        </div>
+      </div><!-- /.captcha-container -->
+
+      <div class="cf-section cf-wrapper">
+        <div class="cf-columns two">
+          <div class="cf-column">
+            <h2 data-translate="why_captcha_headline">Why do I have to complete a CAPTCHA?</h2>
+
+            <p data-translate="why_captcha_detail">Completing the CAPTCHA proves you are a human and gives you temporary access to the web property.</p>
+          </div>
+
+          <div class="cf-column">
+            <h2 data-translate="resolve_captcha_headline">What can I do to prevent this in the future?</h2>
+
+
+            <p data-translate="resolve_captcha_antivirus">If you are on a personal connection, like at home, you can run an anti-virus scan on your device to make sure it is not infected with malware.</p>
+
+            <p data-translate="resolve_captcha_network">If you are at an office or shared network, you can ask the network administrator to run a scan across the network looking for misconfigured or infected devices.</p>
+
+          </div>
+        </div>
+      </div><!-- /.section -->
+
+
+      <div class="cf-error-footer cf-wrapper">
+  <p>
+    <span class="cf-footer-item">Cloudflare Ray ID: <strong>0000000000000000</strong></span>
+    <span class="cf-footer-separator">&bull;</span>
+    <span class="cf-footer-item"><span>Your IP</span>: 000.00.000.00</span>
+    <span class="cf-footer-separator">&bull;</span>
+    <span class="cf-footer-item"><span>Performance &amp; security by</span> <a href="https://www.cloudflare.com/5xx-error-landing?utm_source=error_footer" id="brand_link" target="_blank">Cloudflare</a></span>
+
+  </p>
+</div><!-- /.error-footer -->
+
+
+    </div><!-- /#cf-error-details -->
+  </div><!-- /#cf-wrapper -->
+
+  <script type="text/javascript">
+  window._cf_translation = {};
+
+
+</script>
+
+</body>
+</html>

tests/fixtures/js_challenge_03_12_2018.html

@@ -0,0 +1,84 @@
+<!DOCTYPE HTML>
+<html lang="en-US">
+<head>
+  <meta charset="UTF-8" />
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+  <meta http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1" />
+  <meta name="robots" content="noindex, nofollow" />
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
+  <title>Just a moment...</title>
+  <style type="text/css">
+    html, body {width: 100%; height: 100%; margin: 0; padding: 0;}
+    body {background-color: #ffffff; font-family: Helvetica, Arial, sans-serif; font-size: 100%;}
+    h1 {font-size: 1.5em; color: #404040; text-align: center;}
+    p {font-size: 1em; color: #404040; text-align: center; margin: 10px 0 0 0;}
+    #spinner {margin: 0 auto 30px auto; display: block;}
+    .attribution {margin-top: 20px;}
+    @-webkit-keyframes bubbles { 33%: { -webkit-transform: translateY(10px); transform: translateY(10px); } 66% { -webkit-transform: translateY(-10px); transform: translateY(-10px); } 100% { -webkit-transform: translateY(0); transform: translateY(0); } }
+    @keyframes bubbles { 33%: { -webkit-transform: translateY(10px); transform: translateY(10px); } 66% { -webkit-transform: translateY(-10px); transform: translateY(-10px); } 100% { -webkit-transform: translateY(0); transform: translateY(0); } }
+    .bubbles { background-color: #404040; width:15px; height: 15px; margin:2px; border-radius:100%; -webkit-animation:bubbles 0.6s 0.07s infinite ease-in-out; animation:bubbles 0.6s 0.07s infinite ease-in-out; -webkit-animation-fill-mode:both; animation-fill-mode:both; display:inline-block; }
+  </style>
+
+    <script type="text/javascript">
+  //<![CDATA[
+  (function(){
+    var a = function() {try{return !!window.addEventListener} catch(e) {return !1} },
+    b = function(b, c) {a() ? document.addEventListener("DOMContentLoaded", b, c) : document.attachEvent("onreadystatechange", b)};
+    b(function(){
+      var a = document.getElementById('cf-content');a.style.display = 'block';
+      setTimeout(function(){
+        var s,t,o,p,b,r,e,a,k,i,n,g,f, zoqqEUY={"xzWMiyQ":+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]))};
+        t = document.createElement('div');
+        t.innerHTML="<a href='/'>x</a>";
+        t = t.firstChild.href;r = t.match(/https?:\/\//)[0];
+        t = t.substr(r.length); t = t.substr(0,t.length-1);
+        a = document.getElementById('jschl-answer');
+        f = document.getElementById('challenge-form');
+        ;zoqqEUY.xzWMiyQ-=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]))/+((!+[]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(+!![]));zoqqEUY.xzWMiyQ-=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(+!![]))/+((!+[]+!![]+[])+(+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]));a.value = +zoqqEUY.xzWMiyQ.toFixed(10) + t.length; '; 121'
+        f.action += location.hash;
+        f.submit();
+      }, 4000);
+    }, false);
+  })();
+  //]]>
+</script>
+
+
+</head>
+<body>
+  <table width="100%" height="100%" cellpadding="20">
+    <tr>
+      <td align="center" valign="middle">
+          <div class="cf-browser-verification cf-im-under-attack">
+  <noscript><h1 data-translate="turn_on_js" style="color:#bd2426;">Please turn JavaScript on and reload the page.</h1></noscript>
+  <div id="cf-content" style="display:none">
+
+    <div>
+      <div class="bubbles"></div>
+      <div class="bubbles"></div>
+      <div class="bubbles"></div>
+    </div>
+    <h1><span data-translate="checking_browser">Checking your browser before accessing</span> iload.to.</h1>
+    <p data-translate="process_is_automatic">This process is automatic. Your browser will redirect to your requested content shortly.</p>
+    <p data-translate="allow_5_secs">Please allow up to 5 seconds&hellip;</p>
+  </div>
+
+  <form id="challenge-form" action="/cdn-cgi/l/chk_jschl" method="get">
+    <input type="hidden" name="jschl_vc" value="427c2b1cd4fba29608ee81b200e94bfa"/>
+    <input type="hidden" name="pass" value="1543827239.915-44n9IE20mS"/>
+    <input type="hidden" id="jschl-answer" name="jschl_answer"/>
+  </form>
+</div>
+
+
+          <div class="attribution">
+            <a href="https://www.cloudflare.com/5xx-error-landing?utm_source=iuam" target="_blank" style="font-size: 12px;">DDoS protection by Cloudflare</a>
+            <br>
+            Ray ID: 4834ce407815974a
+          </div>
+      </td>
+
+    </tr>
+  </table>
+</body>
+</html>