Add 'list-stacks' command

shub/list_stacks.py

@@ -0,0 +1,84 @@
+from __future__ import absolute_import
+
+import re
+
+import click
+import requests
+
+from shub.exceptions import RemoteErrorException
+
+
+RELEASES_WEB_URL = "https://github.com/{repo}/releases"
+TAGS_API_URL = "https://api.github.com/repos/{repo}/tags"
+STACK_REPOSITORIES = [
+    # (stack type, prefix, repository)
+    ('Scrapy', 'scrapy', 'scrapinghub/scrapinghub-stack-scrapy'),
+    ('Portia', 'portia', 'scrapinghub/scrapinghub-stack-portia'),
+]
+
+HELP = """
+List the available stacks that you can run your project in. Use --all to
+include regular releases.
+
+\b
+See
+https://helpdesk.scrapinghub.com/support/solutions/articles/22000200402-scrapy-cloud-stacks
+for a general introduction to stacks, and
+https://shub.readthedocs.io/en/stable/configuration.html#choosing-a-scrapy-cloud-stack
+for information on how to configure a stack for your project.
+"""
+
+SHORT_HELP = "List available stacks"
+
+
+@click.command(help=HELP, short_help=SHORT_HELP)
+@click.option('-a', '--all', 'print_all', help='include regular releases',
+              is_flag=True)
+def cli(print_all):
+    for i, (stack_type, prefix, repo) in enumerate(STACK_REPOSITORIES):
+        if i:
+            click.echo('')
+        tags = get_repository_tags(repo)
+        click.echo("%s stacks:" % stack_type)
+        click.echo(_format_list(
+            prefix + ':' + tag
+            for tag in filter_tags(tags, include_regular=print_all)))
+
+
+def _format_list(l):
+    return '\n'.join('  %s' % x for x in l)
+
+
+def get_repository_tags(repo):
+    try:
+        resp = requests.get(TAGS_API_URL.format(repo=repo))
+        resp.raise_for_status()
+        tags = resp.json()
+        while 'next' in resp.links:
+            resp = requests.get(resp.links['next']['url'])
+            resp.raise_for_status()
+            tags.extend(resp.json())
+    except (requests.HTTPError, requests.ConnectionError) as e:
+        if isinstance(e, requests.HTTPError):
+            msg = resp.json()['message']
+        else:
+            msg = e.args[0]
+        repo_url_list = _format_list(
+            '%s: %s' % (desc, RELEASES_WEB_URL.format(repo=repo))
+            for desc, _, repo in STACK_REPOSITORIES)
+        raise RemoteErrorException(
+            "Error while retrieving the list of stacks from GitHub: %s\n\n"
+            "Please visit the following URLs to see the available stacks: \n%s"
+            "" % (msg, repo_url_list))
+    else:
+        return [tag['name'] for tag in tags]
+
+
+def _is_regular_release(tag):
+    return re.search('\d{8}', tag)
+
+
+def filter_tags(tags, include_regular=False):
+    if include_regular:
+        return tags
+    return [t for t in tags if not _is_regular_release(t)]

shub/tool.py

@@ -35,20 +35,21 @@ def cli():
 
 
 commands = [
+    "copy_eggs",
     "deploy",
-    "login",
     "deploy_egg",
-    "fetch_eggs",
     "deploy_reqs",
-    "logout",
-    "version",
+    "fetch_eggs",
+    "image",
     "items",
-    "schedule",
+    "list_stacks",
     "log",
-    "requests",
-    "copy_eggs",
+    "login",
+    "logout",
     "migrate_eggs",
-    "image",
+    "requests",
+    "schedule",
+    "version",
 ]
 
 for command in commands:

tests/test_list_stacks.py

@@ -0,0 +1,72 @@
+import mock
+import pytest
+import requests
+from click.testing import CliRunner
+
+from shub.exceptions import RemoteErrorException
+from shub.list_stacks import (
+    cli, filter_tags, get_repository_tags, RELEASES_WEB_URL, TAGS_API_URL,
+    STACK_REPOSITORIES)
+
+
+TAGS = ['1.3', '1.3-py3', '1.3-20170421', '1.3-py3-20170421',
+        '1.2-20161201-forwarder', '1.1-test', '1.1-20160726.1']
+
+
+@pytest.fixture
+def requests_get_mock():
+    with mock.patch('shub.list_stacks.requests.get') as m:
+        yield m
+
+
+def test_get_tags_extracts_tags(requests_get_mock):
+    repo = 'scrapinghub/stack-repo'
+    requests_get_mock.return_value.json.return_value = [
+        {'name': tag} for tag in TAGS]
+    assert get_repository_tags(repo) == TAGS
+    requests_get_mock.assert_called_once_with(TAGS_API_URL.format(repo=repo))
+
+
+def _assert_repo_urls_in_str(x):
+    for _, _, repo in STACK_REPOSITORIES:
+        assert RELEASES_WEB_URL.format(repo=repo) in x
+
+
+def test_get_tags_prints_urls_on_connection_error(requests_get_mock):
+    requests_get_mock.side_effect = requests.ConnectionError(
+        "ConnectionError description")
+    with pytest.raises(RemoteErrorException) as e:
+        get_repository_tags('some_repo')
+        assert "ConnectionError description" in e.args[0]
+        _assert_repo_urls_in_str(e.args[0])
+
+
+def test_get_tags_prints_urls_on_http_error(requests_get_mock):
+    requests_get_mock.return_value.raise_for_status.side_effect = (
+        requests.HTTPError)
+    requests_get_mock.return_value.json.return_value = {
+        'message': 'HTTPError description'}
+    with pytest.raises(RemoteErrorException) as e:
+        get_repository_tags('some_repo')
+        assert "HTTPError description" in e.args[0]
+        _assert_repo_urls_in_str(e.args[0])
+
+
+def test_filter_tags():
+    assert filter_tags(TAGS) == ['1.3', '1.3-py3', '1.1-test']
+    assert filter_tags(TAGS, include_regular=True) == TAGS
+
+
+def test_cli_prints_tags():
+    runner = CliRunner()
+    with mock.patch('shub.list_stacks.get_repository_tags') as mock_tags:
+        mock_tags.return_value = TAGS
+        result = runner.invoke(cli)
+        for desc, prefix, repo in STACK_REPOSITORIES:
+            assert desc in result.output
+            for tag in filter_tags(TAGS):
+                assert prefix + ':' + tag in result.output
+        result = runner.invoke(cli, ('--all', ))
+        for desc, prefix, repo in STACK_REPOSITORIES:
+            for tag in TAGS:
+                assert prefix + ':' + tag in result.output