From 1ed2b83c7176695c6cc4ed606e3d8e8525fbe2e1 Mon Sep 17 00:00:00 2001 From: Bart Kamphorst Date: Mon, 24 Oct 2022 22:13:49 +0200 Subject: [PATCH] sources: introduce priority "explicit" (#7658) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Explicit sources are considered only for packages that explicitly indicate their source. Co-authored-by: Randy Döring <30527984+radoering@users.noreply.github.com> --- docs/cli.md | 2 +- docs/repositories.md | 19 +++++- src/poetry/json/schemas/poetry.json | 3 +- src/poetry/repositories/repository_pool.py | 28 ++++++-- tests/console/commands/source/conftest.py | 9 +++ tests/console/commands/source/test_add.py | 10 +++ tests/console/commands/source/test_show.py | 1 + .../with_explicit_source/pyproject.toml | 19 ++++++ .../json/fixtures/source/complete_valid.toml | 2 +- tests/json/test_schema_sources.py | 2 +- tests/puzzle/test_solver.py | 65 +++++++++++++++++++ tests/repositories/test_repository_pool.py | 19 +++++- tests/test_factory.py | 13 ++++ tests/utils/test_source.py | 4 +- 14 files changed, 184 insertions(+), 12 deletions(-) create mode 100644 tests/fixtures/with_explicit_source/pyproject.toml diff --git a/docs/cli.md b/docs/cli.md index ae161d1d7ac..e57b1548e2e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -786,7 +786,7 @@ You cannot use the name `pypi` as it is reserved for use by the default PyPI sou * `--default`: Set this source as the [default]({{< relref "repositories#default-package-source" >}}) (disable PyPI). Deprecated in favor of `--priority`. * `--secondary`: Set this source as a [secondary]({{< relref "repositories#secondary-package-sources" >}}) source. Deprecated in favor of `--priority`. -* `--priority`: Set the priority of this source. Accepted values are: [`default`]({{< relref "repositories#default-package-source" >}}), and [`secondary`]({{< relref "repositories#secondary-package-sources" >}}). Refer to the dedicated sections in [Repositories]({{< relref "repositories" >}}) for more information. +* `--priority`: Set the priority of this source. Accepted values are: [`default`]({{< relref "repositories#default-package-source" >}}), [`secondary`]({{< relref "repositories#secondary-package-sources" >}}), and [`explicit`]({{< relref "repositories#explicit-package-sources" >}}). Refer to the dedicated sections in [Repositories]({{< relref "repositories" >}}) for more information. {{% note %}} At most one of the options above can be provided. See [package sources]({{< relref "repositories#package-sources" >}}) for more information. diff --git a/docs/repositories.md b/docs/repositories.md index c8d92301532..38f993892f4 100644 --- a/docs/repositories.md +++ b/docs/repositories.md @@ -123,7 +123,7 @@ url = "https://foo.bar/simple/" priority = "primary" ``` -If `priority` is undefined, the source is considered a primary source that takes precedence over PyPI and secondary sources. +If `priority` is undefined, the source is considered a primary source that takes precedence over PyPI, secondary and explicit sources. Package sources are considered in the following order: 1. [default source](#default-package-source), @@ -131,6 +131,8 @@ Package sources are considered in the following order: 3. PyPI (unless disabled by another default source), 4. [secondary sources](#secondary-package-sources), +[Explicit sources](#explicit-package-sources) are considered only for packages that explicitly [indicate their source](#package-source-constraint). + Within each priority class, package sources are considered in order of appearance in `pyproject.toml`. {{% note %}} @@ -181,6 +183,20 @@ poetry source add --priority=secondary https://foo.bar/simple/ There can be more than one secondary package source. +#### Explicit Package Sources + +*Introduced in 1.5.0* + +If package sources are configured as explicit, these sources are only searched when a package configuration [explicitly indicates](#package-source-constraint) that it should be found on this package source. + +You can configure a package source as an explicit source with `priority = "explicit` in your package source configuration. + +```bash +poetry source add --priority=explicit foo https://foo.bar/simple/ +``` + +There can be more than one explicit package source. + #### Package Source Constraint All package sources (including secondary sources) will be searched during the package lookup @@ -209,6 +225,7 @@ priority = ... {{% note %}} A repository that is configured to be the only source for retrieving a certain package can itself have any priority. +In particular, it does not need to have priority `"explicit"`. If a repository is configured to be the source of a package, it will be the only source that is considered for that package and the repository priority will have no effect on the resolution. diff --git a/src/poetry/json/schemas/poetry.json b/src/poetry/json/schemas/poetry.json index 04d0ff340d7..c9191b03d23 100644 --- a/src/poetry/json/schemas/poetry.json +++ b/src/poetry/json/schemas/poetry.json @@ -45,7 +45,8 @@ "enum": [ "primary", "default", - "secondary" + "secondary", + "explicit" ], "description": "Declare the priority of this repository." }, diff --git a/src/poetry/repositories/repository_pool.py b/src/poetry/repositories/repository_pool.py index a4dffbbff37..304f7e9ed33 100644 --- a/src/poetry/repositories/repository_pool.py +++ b/src/poetry/repositories/repository_pool.py @@ -26,6 +26,7 @@ class Priority(IntEnum): DEFAULT = enum.auto() PRIMARY = enum.auto() SECONDARY = enum.auto() + EXPLICIT = enum.auto() @dataclass(frozen=True) @@ -51,11 +52,30 @@ def __init__( @property def repositories(self) -> list[Repository]: - unsorted_repositories = self._repositories.values() - sorted_repositories = sorted( - unsorted_repositories, key=lambda prio_repo: prio_repo.priority + """ + Returns the repositories in the pool, + in the order they will be searched for packages. + + ATTENTION: For backwards compatibility and practical reasons, + repositories with priority EXPLICIT are NOT included, + because they will not be searched. + """ + sorted_repositories = self._sorted_repositories + return [ + prio_repo.repository + for prio_repo in sorted_repositories + if prio_repo.priority is not Priority.EXPLICIT + ] + + @property + def all_repositories(self) -> list[Repository]: + return [prio_repo.repository for prio_repo in self._sorted_repositories] + + @property + def _sorted_repositories(self) -> list[PrioritizedRepository]: + return sorted( + self._repositories.values(), key=lambda prio_repo: prio_repo.priority ) - return [prio_repo.repository for prio_repo in sorted_repositories] def has_default(self) -> bool: return self._contains_priority(Priority.DEFAULT) diff --git a/tests/console/commands/source/conftest.py b/tests/console/commands/source/conftest.py index f9db68a058a..5ec79df2c9e 100644 --- a/tests/console/commands/source/conftest.py +++ b/tests/console/commands/source/conftest.py @@ -51,6 +51,13 @@ def source_secondary() -> Source: ) +@pytest.fixture +def source_explicit() -> Source: + return Source( + name="explicit", url="https://explicit.com", priority=Priority.EXPLICIT + ) + + _existing_source = Source(name="existing", url="https://existing.com") @@ -110,11 +117,13 @@ def add_all_source_types( source_primary: Source, source_default: Source, source_secondary: Source, + source_explicit: Source, ) -> None: add = command_tester_factory("source add", poetry=poetry_with_source) for source in [ source_primary, source_default, source_secondary, + source_explicit, ]: add.execute(f"{source.name} {source.url} --priority={source.name}") diff --git a/tests/console/commands/source/test_add.py b/tests/console/commands/source/test_add.py index d25239c052c..468e9dec271 100644 --- a/tests/console/commands/source/test_add.py +++ b/tests/console/commands/source/test_add.py @@ -136,6 +136,16 @@ def test_source_add_secondary( assert_source_added(tester, poetry_with_source, source_existing, source_secondary) +def test_source_add_explicit( + tester: CommandTester, + source_existing: Source, + source_explicit: Source, + poetry_with_source: Poetry, +) -> None: + tester.execute(f"--priority=explicit {source_explicit.name} {source_explicit.url}") + assert_source_added(tester, poetry_with_source, source_existing, source_explicit) + + def test_source_add_error_default_and_secondary_legacy(tester: CommandTester) -> None: tester.execute("--default --secondary error https://error.com") assert ( diff --git a/tests/console/commands/source/test_show.py b/tests/console/commands/source/test_show.py index b636d1bd30a..d3c94682650 100644 --- a/tests/console/commands/source/test_show.py +++ b/tests/console/commands/source/test_show.py @@ -101,6 +101,7 @@ def test_source_show_two( "source_primary", "source_default", "source_secondary", + "source_explicit", ), ) def test_source_show_given_priority( diff --git a/tests/fixtures/with_explicit_source/pyproject.toml b/tests/fixtures/with_explicit_source/pyproject.toml new file mode 100644 index 00000000000..e19ad99bad4 --- /dev/null +++ b/tests/fixtures/with_explicit_source/pyproject.toml @@ -0,0 +1,19 @@ +[tool.poetry] +name = "my-package" +version = "1.2.3" +description = "Some description." +authors = [ + "Your Name " +] +license = "MIT" + +# Requirements +[tool.poetry.dependencies] +python = "~2.7 || ^3.6" + +[tool.poetry.dev-dependencies] + +[[tool.poetry.source]] +name = "explicit" +url = "https://explicit.com/simple/" +priority = "explicit" diff --git a/tests/json/fixtures/source/complete_valid.toml b/tests/json/fixtures/source/complete_valid.toml index 46186e1a7a3..fedab7bed39 100644 --- a/tests/json/fixtures/source/complete_valid.toml +++ b/tests/json/fixtures/source/complete_valid.toml @@ -10,7 +10,7 @@ python = "^3.10" [[tool.poetry.source]] name = "pypi-simple" url = "https://pypi.org/simple/" -priority = "primary" +priority = "explicit" [build-system] requires = ["poetry-core"] diff --git a/tests/json/test_schema_sources.py b/tests/json/test_schema_sources.py index 410b85a1183..78e446bc6b3 100644 --- a/tests/json/test_schema_sources.py +++ b/tests/json/test_schema_sources.py @@ -36,7 +36,7 @@ def test_pyproject_toml_invalid_priority() -> None: assert Factory.validate(content) == { "errors": [ "[source.0.priority] 'arbitrary' is not one of ['primary', 'default'," - " 'secondary']" + " 'secondary', 'explicit']" ], "warnings": [], } diff --git a/tests/puzzle/test_solver.py b/tests/puzzle/test_solver.py index 3eaed889351..080fa975c63 100644 --- a/tests/puzzle/test_solver.py +++ b/tests/puzzle/test_solver.py @@ -2998,6 +2998,71 @@ def test_solver_chooses_from_secondary_if_explicit( assert ops[2].package.source_url is None +def test_solver_does_not_choose_from_explicit_repository( + package: ProjectPackage, io: NullIO +) -> None: + package.python_versions = "^3.7" + package.add_dependency(Factory.create_dependency("attrs", {"version": "^17.4.0"})) + + pool = RepositoryPool() + pool.add_repository(MockPyPIRepository(), priority=Priority.EXPLICIT) + pool.add_repository(MockLegacyRepository()) + + solver = Solver(package, pool, [], [], io) + + with pytest.raises(SolverProblemError): + solver.solve() + + +def test_solver_chooses_direct_dependency_from_explicit_if_explicit( + package: ProjectPackage, + io: NullIO, +) -> None: + package.python_versions = "^3.7" + package.add_dependency( + Factory.create_dependency("pylev", {"version": "^1.2.0", "source": "PyPI"}) + ) + + pool = RepositoryPool() + pool.add_repository(MockPyPIRepository(), priority=Priority.EXPLICIT) + pool.add_repository(MockLegacyRepository()) + + solver = Solver(package, pool, [], [], io) + + transaction = solver.solve() + + ops = check_solver_result( + transaction, + [ + {"job": "install", "package": get_package("pylev", "1.3.0")}, + ], + ) + + assert ops[0].package.source_type is None + assert ops[0].package.source_url is None + + +def test_solver_ignores_explicit_repo_for_transient_dependencies( + package: ProjectPackage, + io: NullIO, +) -> None: + # clikit depends on pylev, which is in MockPyPIRepository (explicit) but not in + # MockLegacyRepository + package.python_versions = "^3.7" + package.add_dependency( + Factory.create_dependency("clikit", {"version": "^0.2.0", "source": "PyPI"}) + ) + + pool = RepositoryPool() + pool.add_repository(MockPyPIRepository(), priority=Priority.EXPLICIT) + pool.add_repository(MockLegacyRepository()) + + solver = Solver(package, pool, [], [], io) + + with pytest.raises(SolverProblemError): + solver.solve() + + def test_solver_discards_packages_with_empty_markers( package: ProjectPackage, repo: Repository, diff --git a/tests/repositories/test_repository_pool.py b/tests/repositories/test_repository_pool.py index 7bc77331eec..21c5941d73d 100644 --- a/tests/repositories/test_repository_pool.py +++ b/tests/repositories/test_repository_pool.py @@ -81,26 +81,43 @@ def test_repository_from_single_repo_pool_legacy( assert pool.get_priority("foo") == expected_priority -def test_repository_with_normal_default_and_secondary_repositories() -> None: +def test_repository_with_normal_default_secondary_and_explicit_repositories(): secondary = LegacyRepository("secondary", "https://secondary.com") default = LegacyRepository("default", "https://default.com") repo1 = LegacyRepository("foo", "https://foo.bar") repo2 = LegacyRepository("bar", "https://bar.baz") + explicit = LegacyRepository("explicit", "https://bar.baz") pool = RepositoryPool() pool.add_repository(repo1) pool.add_repository(secondary, priority=Priority.SECONDARY) pool.add_repository(repo2) + pool.add_repository(explicit, priority=Priority.EXPLICIT) pool.add_repository(default, priority=Priority.DEFAULT) assert pool.repository("secondary") is secondary assert pool.repository("default") is default assert pool.repository("foo") is repo1 assert pool.repository("bar") is repo2 + assert pool.repository("explicit") is explicit assert pool.has_default() assert pool.has_primary_repositories() +def test_repository_explicit_repositories_do_not_show() -> None: + explicit = LegacyRepository("explicit", "https://explicit.com") + default = LegacyRepository("default", "https://default.com") + + pool = RepositoryPool() + pool.add_repository(explicit, priority=Priority.EXPLICIT) + pool.add_repository(default, priority=Priority.DEFAULT) + + assert pool.repository("explicit") is explicit + assert pool.repository("default") is default + assert pool.repositories == [default] + assert pool.all_repositories == [default, explicit] + + def test_remove_non_existing_repository_raises_indexerror() -> None: pool = RepositoryPool() diff --git a/tests/test_factory.py b/tests/test_factory.py index 356d0b24142..2ba1104e768 100644 --- a/tests/test_factory.py +++ b/tests/test_factory.py @@ -338,6 +338,19 @@ def test_poetry_with_no_default_source(): assert {repo.name for repo in poetry.pool.repositories} == {"PyPI"} +def test_poetry_with_explicit_source(with_simple_keyring: None) -> None: + poetry = Factory().create_poetry(fixtures_dir / "with_explicit_source") + + assert len(poetry.pool.repositories) == 1 + assert len(poetry.pool.all_repositories) == 2 + assert poetry.pool.has_repository("PyPI") + assert poetry.pool.get_priority("PyPI") is Priority.DEFAULT + assert isinstance(poetry.pool.repository("PyPI"), PyPiRepository) + assert poetry.pool.has_repository("explicit") + assert isinstance(poetry.pool.repository("explicit"), LegacyRepository) + assert [repo.name for repo in poetry.pool.repositories] == ["PyPI"] + + def test_poetry_with_two_default_sources_legacy(with_simple_keyring: None): with pytest.raises(ValueError) as e: Factory().create_poetry(fixtures_dir / "with_two_default_sources_legacy") diff --git a/tests/utils/test_source.py b/tests/utils/test_source.py index a5093324d04..4908a6e0978 100644 --- a/tests/utils/test_source.py +++ b/tests/utils/test_source.py @@ -23,10 +23,10 @@ }, ), ( - Source("bar", "https://example.com/bar", priority=Priority.SECONDARY), + Source("bar", "https://example.com/bar", priority=Priority.EXPLICIT), { "name": "bar", - "priority": "secondary", + "priority": "explicit", "url": "https://example.com/bar", }, ),