From 38ba4e9f65618aafb076cd77ce714fe618b8d09b Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Thu, 11 May 2017 17:30:52 +0200 Subject: [PATCH 1/2] Sort command list in shub.tool --- shub/tool.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/shub/tool.py b/shub/tool.py index 423824f3..1d559196 100644 --- a/shub/tool.py +++ b/shub/tool.py @@ -35,20 +35,20 @@ def cli(): commands = [ + "copy_eggs", "deploy", - "login", "deploy_egg", - "fetch_eggs", "deploy_reqs", - "logout", - "version", + "fetch_eggs", + "image", "items", - "schedule", "log", - "requests", - "copy_eggs", + "login", + "logout", "migrate_eggs", - "image", + "requests", + "schedule", + "version", ] for command in commands: From df3805ff94186567b58f970b21614b0724b87808 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Thu, 11 May 2017 17:31:15 +0200 Subject: [PATCH 2/2] Add 'list-stacks' command --- shub/list_stacks.py | 84 +++++++++++++++++++++++++++++++++++++++ shub/tool.py | 1 + tests/test_list_stacks.py | 72 +++++++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 shub/list_stacks.py create mode 100644 tests/test_list_stacks.py diff --git a/shub/list_stacks.py b/shub/list_stacks.py new file mode 100644 index 00000000..773efc97 --- /dev/null +++ b/shub/list_stacks.py @@ -0,0 +1,84 @@ +from __future__ import absolute_import + +import re + +import click +import requests + +from shub.exceptions import RemoteErrorException + + +RELEASES_WEB_URL = "https://github.com/{repo}/releases" +TAGS_API_URL = "https://api.github.com/repos/{repo}/tags" +STACK_REPOSITORIES = [ + # (stack type, prefix, repository) + ('Scrapy', 'scrapy', 'scrapinghub/scrapinghub-stack-scrapy'), + ('Portia', 'portia', 'scrapinghub/scrapinghub-stack-portia'), +] + +HELP = """ +List the available stacks that you can run your project in. Use --all to +include regular releases. + +\b +See +https://helpdesk.scrapinghub.com/support/solutions/articles/22000200402-scrapy-cloud-stacks +for a general introduction to stacks, and +https://shub.readthedocs.io/en/stable/configuration.html#choosing-a-scrapy-cloud-stack +for information on how to configure a stack for your project. +""" + +SHORT_HELP = "List available stacks" + + +@click.command(help=HELP, short_help=SHORT_HELP) +@click.option('-a', '--all', 'print_all', help='include regular releases', + is_flag=True) +def cli(print_all): + for i, (stack_type, prefix, repo) in enumerate(STACK_REPOSITORIES): + if i: + click.echo('') + tags = get_repository_tags(repo) + click.echo("%s stacks:" % stack_type) + click.echo(_format_list( + prefix + ':' + tag + for tag in filter_tags(tags, include_regular=print_all))) + + +def _format_list(l): + return '\n'.join(' %s' % x for x in l) + + +def get_repository_tags(repo): + try: + resp = requests.get(TAGS_API_URL.format(repo=repo)) + resp.raise_for_status() + tags = resp.json() + while 'next' in resp.links: + resp = requests.get(resp.links['next']['url']) + resp.raise_for_status() + tags.extend(resp.json()) + except (requests.HTTPError, requests.ConnectionError) as e: + if isinstance(e, requests.HTTPError): + msg = resp.json()['message'] + else: + msg = e.args[0] + repo_url_list = _format_list( + '%s: %s' % (desc, RELEASES_WEB_URL.format(repo=repo)) + for desc, _, repo in STACK_REPOSITORIES) + raise RemoteErrorException( + "Error while retrieving the list of stacks from GitHub: %s\n\n" + "Please visit the following URLs to see the available stacks: \n%s" + "" % (msg, repo_url_list)) + else: + return [tag['name'] for tag in tags] + + +def _is_regular_release(tag): + return re.search('\d{8}', tag) + + +def filter_tags(tags, include_regular=False): + if include_regular: + return tags + return [t for t in tags if not _is_regular_release(t)] diff --git a/shub/tool.py b/shub/tool.py index 1d559196..9b372d1e 100644 --- a/shub/tool.py +++ b/shub/tool.py @@ -42,6 +42,7 @@ def cli(): "fetch_eggs", "image", "items", + "list_stacks", "log", "login", "logout", diff --git a/tests/test_list_stacks.py b/tests/test_list_stacks.py new file mode 100644 index 00000000..c5f15440 --- /dev/null +++ b/tests/test_list_stacks.py @@ -0,0 +1,72 @@ +import mock +import pytest +import requests +from click.testing import CliRunner + +from shub.exceptions import RemoteErrorException +from shub.list_stacks import ( + cli, filter_tags, get_repository_tags, RELEASES_WEB_URL, TAGS_API_URL, + STACK_REPOSITORIES) + + +TAGS = ['1.3', '1.3-py3', '1.3-20170421', '1.3-py3-20170421', + '1.2-20161201-forwarder', '1.1-test', '1.1-20160726.1'] + + +@pytest.fixture +def requests_get_mock(): + with mock.patch('shub.list_stacks.requests.get') as m: + yield m + + +def test_get_tags_extracts_tags(requests_get_mock): + repo = 'scrapinghub/stack-repo' + requests_get_mock.return_value.json.return_value = [ + {'name': tag} for tag in TAGS] + assert get_repository_tags(repo) == TAGS + requests_get_mock.assert_called_once_with(TAGS_API_URL.format(repo=repo)) + + +def _assert_repo_urls_in_str(x): + for _, _, repo in STACK_REPOSITORIES: + assert RELEASES_WEB_URL.format(repo=repo) in x + + +def test_get_tags_prints_urls_on_connection_error(requests_get_mock): + requests_get_mock.side_effect = requests.ConnectionError( + "ConnectionError description") + with pytest.raises(RemoteErrorException) as e: + get_repository_tags('some_repo') + assert "ConnectionError description" in e.args[0] + _assert_repo_urls_in_str(e.args[0]) + + +def test_get_tags_prints_urls_on_http_error(requests_get_mock): + requests_get_mock.return_value.raise_for_status.side_effect = ( + requests.HTTPError) + requests_get_mock.return_value.json.return_value = { + 'message': 'HTTPError description'} + with pytest.raises(RemoteErrorException) as e: + get_repository_tags('some_repo') + assert "HTTPError description" in e.args[0] + _assert_repo_urls_in_str(e.args[0]) + + +def test_filter_tags(): + assert filter_tags(TAGS) == ['1.3', '1.3-py3', '1.1-test'] + assert filter_tags(TAGS, include_regular=True) == TAGS + + +def test_cli_prints_tags(): + runner = CliRunner() + with mock.patch('shub.list_stacks.get_repository_tags') as mock_tags: + mock_tags.return_value = TAGS + result = runner.invoke(cli) + for desc, prefix, repo in STACK_REPOSITORIES: + assert desc in result.output + for tag in filter_tags(TAGS): + assert prefix + ':' + tag in result.output + result = runner.invoke(cli, ('--all', )) + for desc, prefix, repo in STACK_REPOSITORIES: + for tag in TAGS: + assert prefix + ':' + tag in result.output