From c68e9344b08a1f70ef2e337f3dfd97258f7ec8bd Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Fri, 11 Oct 2024 02:46:52 -0400 Subject: [PATCH] feat: --target accepts a target name not a URL (closes #89). Move --target to relevant subcommands. Remove --username and --password. --- CHANGES.rst | 13 ++++++++++--- scrapyd_client/__main__.py | 28 +++++++--------------------- scrapyd_client/pyclient.py | 3 ++- scrapyd_client/utils.py | 2 -- tests/test_errors.py | 8 ++++---- tests/test_projects.py | 2 +- tests/test_schedule.py | 2 +- tests/test_spiders.py | 4 ++-- tests/test_targets.py | 3 +-- 9 files changed, 28 insertions(+), 37 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2ae51a9..b8a71a4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,14 +14,20 @@ Added Changed ^^^^^^^ -- **BREAKING CHANGE:** Move exceptions from ``scrapyd_client.utils`` to ``scrapyd_client.exceptions``. -- **BREAKING CHANGE:** Move ``HEADERS`` from ``scrapyd_client.utils`` to ``scrapyd_client.pyclient``. -- **BREAKING CHANGE:** Merge ``scrapyd_client.commands`` and ``scrapyd_client.cli`` into ``scrapyd_client.__main__``. +Changes to commands: + +- **BREAKING CHANGE:** ``scrapyd-client`` subcommands accept the ``--target`` (``-t``) option, instead of the ``scrapyd-client`` command itself. The target is the name of a target in the ``scrapy.cfg`` file, like ``scrapyd-deploy``, instead of a URL. - **BREAKING CHANGE:** ``scrapyd-client`` raises an error if no Scrapy project is found, like ``scrapyd-deploy``, instead of assuming a target at ``http://localhost:6800``. - The ``scrapyd-client schedule`` subcommand accepts multiple ``--arg setting=...`` arguments. (@mxdev88) - The ``scrapyd_client.ScrapyClient.schedule`` methods accept ``args`` as a list, instead of as a dict. - The ``scrapyd-deploy --debug`` option prints the subprocess' standard output and standard error, instead of writing to ``stdout`` and ``stderr`` files. +Changes to modules: + +- **BREAKING CHANGE:** Move exceptions from ``scrapyd_client.utils`` to ``scrapyd_client.exceptions``. +- **BREAKING CHANGE:** Move ``DEFAULT_TARGET_URL`` and ``HEADERS`` from ``scrapyd_client.utils`` to ``scrapyd_client.pyclient``. +- **BREAKING CHANGE:** Merge ``scrapyd_client.commands`` and ``scrapyd_client.cli`` into ``scrapyd_client.__main__``. + Fixed ^^^^^ @@ -31,6 +37,7 @@ Fixed Removed ^^^^^^^ +- **BREAKING CHANGE:** Remove the ``scrapyd-client --username`` (``-u``) and ``--password`` (``-p``) options, in favor of using the ``scrapy.cfg`` file. - **BREAKING CHANGE:** Remove the ``scrapyd-deploy --list-targets`` (``-l``) option, in favor of ``scrapyd-client targets``. - **BREAKING CHANGE:** Remove the ``scrapyd-deploy --list-projects`` (``-L``) option, in favor of ``scrapyd-client projects``. - **BREAKING CHANGE:** Remove the ``get_request`` and ``post_request`` functions from ``scrapyd_client.utils``. diff --git a/scrapyd_client/__main__.py b/scrapyd_client/__main__.py index b222c7b..41bf5cc 100644 --- a/scrapyd_client/__main__.py +++ b/scrapyd_client/__main__.py @@ -9,13 +9,15 @@ import scrapyd_client.deploy from scrapyd_client.exceptions import ErrorResponse, MalformedResponse from scrapyd_client.pyclient import ScrapydClient -from scrapyd_client.utils import DEFAULT_TARGET_URL, _get_targets, get_config +from scrapyd_client.utils import _get_targets, get_config ISSUE_TRACKER_URL = "https://github.com/scrapy/scrapyd-client/issues" def _get_client(args): - return ScrapydClient(args.target, username=args.username, password=args.password) + target = _get_targets()[args.target] + + return ScrapydClient(args.target, target.get("username"), password=target.get("password", "")) def deploy(args): # noqa: ARG001 @@ -80,25 +82,6 @@ def parse_cli_args(args): mainparser = ArgumentParser(description=description) subparsers = mainparser.add_subparsers() - mainparser.add_argument( - "-t", - "--target", - default=cfg.get("deploy", "url", fallback=DEFAULT_TARGET_URL).rstrip("/"), - help="Specifies the Scrapyd's API base URL.", - ) - mainparser.add_argument( - "-u", - "--username", - default=cfg.get("deploy", "username", fallback=None), - help="Specifies the username to connect to the Scrapyd target.", - ) - mainparser.add_argument( - "-p", - "--password", - default=cfg.get("deploy", "password", fallback=None), - help="Specifies the password to connect to the Scrapyd target.", - ) - parser = subparsers.add_parser("deploy", description=deploy.__doc__) parser.set_defaults(action=deploy) @@ -107,9 +90,11 @@ def parse_cli_args(args): parser = subparsers.add_parser("projects", description=projects.__doc__) parser.set_defaults(action=projects) + parser.add_argument("-t", "--target", default="default", help="Specifies the target Scrapyd server by name.") parser = subparsers.add_parser("schedule", description=schedule.__doc__) parser.set_defaults(action=schedule) + parser.add_argument("-t", "--target", default="default", help="Specifies the target Scrapyd server by name.") parser.add_argument("-p", "--project", **project_kwargs) parser.add_argument( "spider", @@ -125,6 +110,7 @@ def parse_cli_args(args): parser = subparsers.add_parser("spiders", description=spiders.__doc__) parser.set_defaults(action=spiders) + parser.add_argument("-t", "--target", default="default", help="Specifies the target Scrapyd server by name.") parser.add_argument("-p", "--project", **project_kwargs) parser.add_argument( "-v", diff --git a/scrapyd_client/pyclient.py b/scrapyd_client/pyclient.py index 67692c3..6ecc89d 100644 --- a/scrapyd_client/pyclient.py +++ b/scrapyd_client/pyclient.py @@ -6,8 +6,9 @@ import requests from scrapyd_client.exceptions import ErrorResponse, MalformedResponse -from scrapyd_client.utils import DEFAULT_TARGET_URL, get_auth +from scrapyd_client.utils import get_auth +DEFAULT_TARGET_URL = "http://localhost:6800" HEADERS = requests.utils.default_headers().copy() HEADERS["User-Agent"] = "Scrapyd-client/2.0.0" diff --git a/scrapyd_client/utils.py b/scrapyd_client/utils.py index fee52a6..b87c0af 100644 --- a/scrapyd_client/utils.py +++ b/scrapyd_client/utils.py @@ -8,8 +8,6 @@ from requests.auth import HTTPBasicAuth from scrapy.utils import conf -DEFAULT_TARGET_URL = "http://localhost:6800" - class EnvInterpolation(BasicInterpolation): """Interpolation which expands environment variables in values.""" diff --git a/tests/test_errors.py b/tests/test_errors.py index 7f60c7a..3237f1b 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -3,7 +3,7 @@ import requests -def test_decode_error(mocker, script_runner, project): +def test_decode_error(mocker, script_runner, conf_default_target): mock_response = mocker.Mock() mock_response.json.side_effect = json.decoder.JSONDecodeError("", "", 0) mock_get = mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True) @@ -14,7 +14,7 @@ def test_decode_error(mocker, script_runner, project): assert result.stdout.startswith("Received a malformed response:\n") -def test_projects(mocker, script_runner, project): +def test_projects(mocker, script_runner, conf_default_target): mock_response = mocker.Mock() mock_response.json.return_value = { "status": "error", @@ -28,10 +28,10 @@ def test_projects(mocker, script_runner, project): assert result.stdout == "Scrapyd responded with an error:\nSomething went wrong.\n" -def test_connection_error(mocker, script_runner, project): +def test_connection_error(mocker, script_runner, conf_default_target): mock_get = mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True) mock_get.side_effect = requests.ConnectionError() result = script_runner.run(["scrapyd-client", "projects"]) assert not result.success - assert result.stdout == "Failed to connect to target (http://localhost:6800):\n\n" + assert result.stdout == "Failed to connect to target (default):\n\n" diff --git a/tests/test_projects.py b/tests/test_projects.py index 884cb11..0535b33 100644 --- a/tests/test_projects.py +++ b/tests/test_projects.py @@ -1,4 +1,4 @@ -def test_projects(mocker, script_runner, project): +def test_projects(mocker, script_runner, conf_default_target): projects = ["foo", "bar"] mock_response = mocker.Mock() mock_response.json.return_value = {"projects": ["foo", "bar"], "status": "ok"} diff --git a/tests/test_schedule.py b/tests/test_schedule.py index 74e8d1e..6e425c1 100644 --- a/tests/test_schedule.py +++ b/tests/test_schedule.py @@ -1,7 +1,7 @@ from itertools import chain -def test_schedule(mocker, script_runner, project): +def test_schedule(mocker, script_runner, conf_default_target): get_responses = [ {"projects": ["foo"]}, {"spiders": ["bar"]}, diff --git a/tests/test_spiders.py b/tests/test_spiders.py index 351dd2d..50ec8c8 100644 --- a/tests/test_spiders.py +++ b/tests/test_spiders.py @@ -8,7 +8,7 @@ response["status"] = "ok" -def test_spiders(mocker, script_runner, project): +def test_spiders(mocker, script_runner, conf_default_target): mock_response = mocker.Mock() mock_response.json.side_effect = responses mock_get = mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True) @@ -33,7 +33,7 @@ def test_spiders(mocker, script_runner, project): ) -def test_spiders_verbose(mocker, script_runner, project): +def test_spiders_verbose(mocker, script_runner, conf_default_target): mock_response = mocker.Mock() mock_response.json.side_effect = responses mock_get = mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True) diff --git a/tests/test_targets.py b/tests/test_targets.py index cac5e4e..13528cf 100644 --- a/tests/test_targets.py +++ b/tests/test_targets.py @@ -19,8 +19,7 @@ def test_too_many_arguments(script_runner, project): ret.stderr, dedent( """\ - usage: scrapyd-client [-h] [-t TARGET] [-u USERNAME] [-p PASSWORD] - {deploy,targets,projects,schedule,spiders} ... + usage: scrapyd-client [-h] {deploy,targets,projects,schedule,spiders} ... scrapyd-client: error: unrecognized arguments: extra """ ),