diff --git a/.travis.yml b/.travis.yml index ee28ff2..add7187 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ -# Config file for automatic testing at travis-ci.org - language: python +sudo: false python: - "3.4" @@ -9,12 +8,13 @@ python: - "2.6" - "pypy" -# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: - - pip install -r requirements.txt +install: + - python setup.py install + - pip install -U -r dev-requirements.txt - pip install coverage coveralls nose responses -# command to run tests, e.g. python setup.py test +before_script: flake8 robobrowser + script: nosetests --with-coverage --cover-package=robobrowser after_success: coveralls diff --git a/HISTORY.rst b/HISTORY.rst index 6d3885f..dd1a5fd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,35 @@ History ------- +0.5.3 +++++++++++++++++++ +* Improve documentation. Thanks tpugsley and rcutmore for improvements! +* Improve messages in error handling. Thanks again rcutmore! +* Fix default values for - -
' - -
- ''' - ), - ArgCatcher( - responses.GET, 'http://robobrowser.com/post_form/', - body=b''' -
' - -
-
' - -
- ''' - ), - ArgCatcher( - responses.GET, 'http://robobrowser.com/noname/', - body=b''' -
- - I have a bike
- I have a car -

- -
- ''' - ), - ArgCatcher( - responses.POST, 'http://robobrowser.com/submit/', - ), - ] -) - -mock_urls = mock_responses( - [ - ArgCatcher(responses.GET, 'http://robobrowser.com/page1/'), - ArgCatcher(responses.GET, 'http://robobrowser.com/page2/'), - ArgCatcher(responses.GET, 'http://robobrowser.com/page3/'), - ArgCatcher(responses.GET, 'http://robobrowser.com/page4/'), - ] -) +from robobrowser import exceptions +from tests.fixtures import mock_links, mock_urls, mock_forms -class TestHeaders(unittest.TestCase): - def test_headers(self): - headers = { - 'X-Song': 'Innuendo', - 'X-Writer': 'Freddie', - } - browser = RoboBrowser(headers=headers) - browser.open('http://robobrowser.com/links/') - for key, value in headers.items(): - assert_equal(browser.session.headers[key], value) +class TestHeaders(unittest.TestCase): + @mock_links def test_user_agent(self): browser = RoboBrowser(user_agent='freddie') browser.open('http://robobrowser.com/links/') @@ -120,6 +28,28 @@ def test_default_headers(self): assert_equal(browser.session.headers, requests.Session().headers) +class TestOpen(unittest.TestCase): + + def setUp(self): + self.browser = RoboBrowser() + + @mock.patch('requests.Session.request') + def test_open_default_method(self, mock_request): + url = 'http://robobrowser.com' + self.browser.open(url) + assert_true(mock_request.called) + args = mock_request.mock_calls[0][1] + assert_equal(args, ('get', url)) + + @mock.patch('requests.Session.request') + def test_open_custom_method(self, mock_request): + url = 'http://robobrowser.com' + self.browser.open(url, method='post') + assert_true(mock_request.called) + args = mock_request.mock_calls[0][1] + assert_equal(args, ('post', url)) + + class TestLinks(unittest.TestCase): @mock_links @@ -135,12 +65,7 @@ def test_get_link(self): @mock_links def test_get_links(self): links = self.browser.get_links() - assert_equal(len(links), 2) - - @mock_links - def test_get_link_by_text(self): - link = self.browser.get_link('opera') - assert_equal(link.get('href'), '/link2/') + assert_equal(len(links), 3) @mock_links def test_follow_link_tag(self): @@ -149,19 +74,12 @@ def test_follow_link_tag(self): assert_equal(self.browser.url, 'http://robobrowser.com/link1/') @mock_links - def test_follow_link_text(self): - self.browser.follow_link('heart attack') - assert_equal(self.browser.url, 'http://robobrowser.com/link1/') - - @mock_links - def test_follow_link_regex(self): - self.browser.follow_link(re.compile(r'opera')) - assert_equal(self.browser.url, 'http://robobrowser.com/link2/') - - @mock_links - def test_follow_link_bs_args(self): - self.browser.follow_link(class_=re.compile(r'song')) - assert_equal(self.browser.url, 'http://robobrowser.com/link2/') + def test_follow_link_no_href(self): + link = BeautifulSoup('nohref').find('a') + assert_raises( + exceptions.RoboError, + lambda: self.browser.follow_link(link) + ) class TestForms(unittest.TestCase): @@ -192,6 +110,18 @@ def test_submit_form_get(self): ) assert_true(self.browser.state.response.request.body is None) + @mock_forms + def test_submit_form_multi_submit(self): + self.browser.open('http://robobrowser.com/multi_submit_form/') + form = self.browser.get_form() + submit = form.submit_fields['submit2'] + self.browser.submit_form(form, submit=submit) + assert_equal( + self.browser.url, + 'http://robobrowser.com/multi_submit_form/' + '?deacon=john&submit2=value2' + ) + @mock_forms def test_submit_form_post(self): self.browser.open('http://robobrowser.com/post_form/') @@ -317,67 +247,99 @@ def test_open_clears_forward(self): len(self.browser._states) - 1 ) assert_raises( - RoboError, + exceptions.RoboError, self.browser.forward ) def test_back_error(self): assert_raises( - RoboError, + exceptions.RoboError, self.browser.back, 5 ) +class TestCustomSession(unittest.TestCase): + + @mock_links + def test_custom_headers(self): + session = requests.Session() + session.headers.update({ + 'Content-Encoding': 'gzip', + }) + browser = RoboBrowser(session=session) + browser.open('http://robobrowser.com/links/') + assert_equal( + browser.response.request.headers.get('Content-Encoding'), + 'gzip' + ) + + @mock_links + def test_custom_headers_override(self): + session = requests.Session() + session.headers.update({ + 'Content-Encoding': 'gzip', + }) + browser = RoboBrowser(session=session) + browser.open( + 'http://robobrowser.com/links/', + headers={'Content-Encoding': 'identity'} + ) + assert_equal( + browser.response.request.headers.get('Content-Encoding'), + 'identity' + ) + + class TestTimeout(unittest.TestCase): - @mock.patch('requests.Session.get') - def test_no_timeout(self, mock_get): + @mock.patch('requests.Session.request') + def test_no_timeout(self, mock_request): browser = RoboBrowser() browser.open('http://robobrowser.com/') - mock_get.assert_called_once_with( - 'http://robobrowser.com/', timeout=None, verify=True - ) + assert_true(mock_request.called) + kwargs = mock_request.mock_calls[0][2] + assert_true(kwargs.get('timeout') is None) - @mock.patch('requests.Session.get') - def test_instance_timeout(self, mock_get): + @mock.patch('requests.Session.request') + def test_instance_timeout(self, mock_request): browser = RoboBrowser(timeout=5) browser.open('http://robobrowser.com/') - mock_get.assert_called_once_with( - 'http://robobrowser.com/', timeout=5, verify=True - ) + assert_true(mock_request.called) + kwargs = mock_request.mock_calls[0][2] + assert_equal(kwargs.get('timeout'), 5) - @mock.patch('requests.Session.get') - def test_call_timeout(self, mock_get): + @mock.patch('requests.Session.request') + def test_call_timeout(self, mock_request): browser = RoboBrowser(timeout=5) browser.open('http://robobrowser.com/', timeout=10) - mock_get.assert_called_once_with( - 'http://robobrowser.com/', timeout=10, verify=True - ) + assert_true(mock_request.called) + kwargs = mock_request.mock_calls[0][2] + assert_equal(kwargs.get('timeout'), 10) -class TestVerify(unittest.TestCase): +class TestAllowRedirects(unittest.TestCase): - @mock.patch('requests.Session.get') - def test_no_verify(self, mock_get): + @mock.patch('requests.Session.request') + def test_no_allow_redirects(self, mock_request): browser = RoboBrowser() browser.open('http://robobrowser.com/') - mock_get.assert_called_once_with( - 'http://robobrowser.com/', verify=True, timeout=None - ) + assert_true(mock_request.called) + kwargs = mock_request.mock_calls[0][2] + assert_true(kwargs.get('allow_redirects') is True) - @mock.patch('requests.Session.get') - def test_instance_verify(self, mock_get): - browser = RoboBrowser(verify=True) + @mock.patch('requests.Session.request') + def test_instance_allow_redirects(self, mock_request): + browser = RoboBrowser(allow_redirects=False) browser.open('http://robobrowser.com/') - mock_get.assert_called_once_with( - 'http://robobrowser.com/', verify=True, timeout=None - ) - - @mock.patch('requests.Session.get') - def test_call_verify(self, mock_get): - browser = RoboBrowser(verify=True) - browser.open('http://robobrowser.com/', verify=False) - mock_get.assert_called_once_with( - 'http://robobrowser.com/', verify=False, timeout=None - ) + assert_true(mock_request.called) + kwargs = mock_request.mock_calls[0][2] + assert_true(kwargs.get('allow_redirects') is False) + + @mock.patch('requests.Session.request') + def test_call_allow_redirects(self, mock_request): + browser = RoboBrowser(allow_redirects=True) + browser.open('http://robobrowser.com/', allow_redirects=False) + assert_true(mock_request.called) + kwargs = mock_request.mock_calls[0][2] + assert_true(kwargs.get('allow_redirects') is False) diff --git a/tests/test_cache.py b/tests/test_cache.py index c29c5f2..92967c5 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -7,6 +7,7 @@ from robobrowser.cache import RoboCache from tests.utils import KwargSetter + class TestAdapter(unittest.TestCase): def test_cache_on(self): @@ -25,6 +26,7 @@ def test_cache_off(self): resp2 = self.browser.state.response assert_true(resp1 is not resp2) + class TestCache(unittest.TestCase): def setUp(self): diff --git a/tests/test_forms.py b/tests/test_forms.py index 7dbfac4..51128a8 100644 --- a/tests/test_forms.py +++ b/tests/test_forms.py @@ -1,45 +1,47 @@ +# -*- coding: utf-8 -*- + import mock import unittest -from nose.tools import * +from nose.tools import * # noqa import tempfile from bs4 import BeautifulSoup from robobrowser.compat import builtin_name -from robobrowser.forms.form import Form, FormData, fields, _parse_fields +from robobrowser.forms.form import Form, Payload, fields, _parse_fields from robobrowser import exceptions -class TestFormData(unittest.TestCase): +class TestPayload(unittest.TestCase): def setUp(self): - self.form_data = FormData() - self.form_data.add({'red': 'special'}) + self.payload = Payload() + self.payload.add({'red': 'special'}) def test_add_payload(self): - self.form_data.add({'lazing': 'sunday'}) - assert_true('lazing' in self.form_data.payload) - assert_equal(self.form_data.payload['lazing'], 'sunday') + self.payload.add({'lazing': 'sunday'}) + assert_true('lazing' in self.payload.data) + assert_equal(self.payload.data['lazing'], 'sunday') def test_add_by_key(self): - self.form_data.add({'lazing': 'sunday'}, 'afternoon') - assert_false('lazing' in self.form_data.payload) - assert_true('afternoon' in self.form_data.options) - assert_true('lazing' in self.form_data.options['afternoon']) + self.payload.add({'lazing': 'sunday'}, 'afternoon') + assert_false('lazing' in self.payload.data) + assert_true('afternoon' in self.payload.options) + assert_true('lazing' in self.payload.options['afternoon']) assert_equal( - self.form_data.options['afternoon']['lazing'], + self.payload.options['afternoon']['lazing'], 'sunday' ) def test_requests_get(self): - out = self.form_data.to_requests('get') + out = self.payload.to_requests('get') assert_true('params' in out) - assert_equal(out['params'], {'red': 'special'}) + assert_equal(list(out['params']), [('red', 'special')]) def test_requests_post(self): - out = self.form_data.to_requests('post') + out = self.payload.to_requests('post') assert_true('data' in out) - assert_equal(out['data'], {'red': 'special'}) + assert_equal(list(out['data']), [('red', 'special')]) class TestForm(unittest.TestCase): @@ -55,12 +57,15 @@ def setUp(self): Roger
John
+ + + ''' self.form = Form(self.html) def test_fields(self): - keys = set(('vocals', 'guitar', 'drums', 'bass')) + keys = set(('vocals', 'guitar', 'drums', 'bass', 'multi', 'submit')) assert_equal(set(self.form.fields.keys()), keys) assert_equal(set(self.form.keys()), keys) @@ -76,7 +81,8 @@ def test_add_field_wrong_type(self): def test_repr(self): assert_equal( repr(self.form), - '' + '' ) def test_repr_empty(self): @@ -85,6 +91,61 @@ def test_repr_empty(self): '' ) + def test_repr_unicode(self): + form = Form(u'
') + assert_equal( + repr(form), + '' + ) + + def test_serialize(self): + serialized = self.form.serialize() + assert_equal(serialized.data.getlist('multi'), ['multi1', 'multi2']) + assert_equal(serialized.data['submit'], 'submit') + + def test_serialize_skips_disabled(self): + html = ''' +
+ + + +
+ ''' + form = Form(html) + serialized = form.serialize() + assert_false('guitar' in serialized.data) + + +class TestFormMultiSubmit(unittest.TestCase): + + def setUp(self): + self.html = ''' +
+ + +
+ ''' + self.form = Form(self.html) + + def test_serialize_multi_no_submit_specified(self): + assert_raises( + exceptions.InvalidSubmitError, + lambda: self.form.serialize() + ) + + def test_serialize_multi_wrong_submit_specified(self): + fake_submit = fields.Submit('') + assert_raises( + exceptions.InvalidSubmitError, + lambda: self.form.serialize(submit=fake_submit) + ) + + def test_serialize_multi(self): + submit = self.form.submit_fields['submit1'] + serialized = self.form.serialize(submit) + assert_equal(serialized.data['submit1'], 'value1') + assert_false('submit2' in serialized.data) + class TestParser(unittest.TestCase): @@ -169,6 +230,16 @@ def test_parse_select(self): assert_equal(len(_fields), 1) assert_true(isinstance(_fields[0], fields.Select)) + def test_parse_empty_select(self): + html = ''' + + ''' + _fields = _parse_fields(BeautifulSoup(html)) + assert_equal(len(_fields), 1) + assert_true(isinstance(_fields[0], fields.Select)) + assert_equal(_fields[0].value, '') + assert_equal(_fields[0].options, []) + def test_parse_select_multi(self): html = ''' ' + input = fields.Input(BeautifulSoup(html).find('input')) + assert_false(input.disabled) + + def test_input_disabled(self): + html = '' + input = fields.Input(BeautifulSoup(html).find('input')) + assert_true(input.disabled) + + def test_checkbox_enabled(self): + html = ''' + vocals
+ guitar
+ drums
+ bass
+ ''' + input = fields.Checkbox(BeautifulSoup(html).find_all('input')) + assert_false(input.disabled) + + def test_checkbox_disabled(self): + html = ''' + vocals
+ guitar
+ drums
+ bass
+ ''' + input = fields.Checkbox(BeautifulSoup(html).find_all('input')) + assert_true(input.disabled) + + def test_select_enabled(self): + html = ''' + + ''' + input = fields.Select(BeautifulSoup(html).find('select')) + assert_false(input.disabled) + + def test_select_disabled_root(self): + html = ''' + + ''' + input = fields.Select(BeautifulSoup(html).find('select')) + assert_true(input.disabled) + + def test_select_disabled_options(self): + html = ''' + + ''' + input = fields.Select(BeautifulSoup(html).find('select')) + assert_true(input.disabled) + + +class TestDefaultValues(unittest.TestCase): + + def test_checkbox_default(self): + inputs = BeautifulSoup(''' + + ''').find_all('input') + checkbox = fields.Checkbox(inputs) + assert_equal(checkbox.options, ['on']) + + def test_radio_default(self): + inputs = BeautifulSoup(''' + + ''').find_all('input') + radio = fields.Radio(inputs) + assert_equal(radio.options, ['on']) + + def test_select_default(self): + parsed = BeautifulSoup(''' + + ''', 'html.parser') + select = fields.Select(parsed) + assert_equal(select.options, ['opt']) + + def test_multi_select_default(self): + parsed = BeautifulSoup(''' + + ''', 'html.parser') + select = fields.Select(parsed) + assert_equal(select.options, ['opt']) diff --git a/tests/utils.py b/tests/utils.py index 51aaf53..0a567f7 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,9 @@ +import functools + +from robobrowser import responses from robobrowser.compat import iteritems + class ArgCatcher(object): """Simple class for memorizing positional and keyword arguments. Used to capture responses for mock_responses. @@ -9,6 +13,7 @@ def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs + class KwargSetter(object): """Simple class for memorizing keyword arguments as instance attributes. Used to mock requests and responses for testing. @@ -17,3 +22,21 @@ class KwargSetter(object): def __init__(self, **kwargs): for key, value in iteritems(kwargs): setattr(self, key, value) + + +def mock_responses(resps): + """Decorator factory to make tests more DRY. Bundles responses.activate + with a collection of response rules. + + :param list resps: List of response-formatted ArgCatcher arguments. + + """ + def wrapper(func): + @responses.activate + @functools.wraps(func) + def wrapped(*args, **kwargs): + for resp in resps: + responses.add(*resp.args, **resp.kwargs) + return func(*args, **kwargs) + return wrapped + return wrapper