Allow for installation of the latest pypi version

Uses `pip install foo==` which will output all available versions to
STDERR. The last package is then chosen.

This API is deprecated ref pypa/pip#9139 but
is seemingly kept around until a replacement is made.

komodo/build.py

@@ -7,7 +7,7 @@
 import sys
 from distutils.dir_util import mkpath
 
-from komodo.package_version import strip_version
+from komodo.package_version import LATEST_PACKAGE_ALIAS, latest_pypi_version, strip_version
 from komodo.shell import pushd, shell
 
 flatten = itr.chain.from_iterable
@@ -117,6 +117,9 @@ def rsync(pkg, ver, pkgpath, data, prefix, *args, **kwargs):
 
 
 def pip_install(pkg, ver, pkgpath, data, prefix, dlprefix, pip="pip", *args, **kwargs):
+    ver = strip_version(ver)
+    if ver == LATEST_PACKAGE_ALIAS:
+        ver = latest_pypi_version(ver)
     cmd = [
         pip,
         "install {}=={}".format(pkg, strip_version(ver)),

komodo/cli.py

@@ -11,7 +11,7 @@
 from komodo.build import make
 from komodo.data import Data
 from komodo.fetch import fetch
-from komodo.package_version import strip_version
+from komodo.package_version import LATEST_PACKAGE_ALIAS, latest_pypi_version, strip_version
 from komodo.shebang import fixup_python_shebangs
 from komodo.shell import pushd, shell
 from komodo.yaml_file_type import YamlFile
@@ -128,6 +128,8 @@ def _main(args):
             continue
 
         package_name = current.get("pypi_package_name", pkg)
+        if ver == LATEST_PACKAGE_ALIAS:
+            ver = latest_pypi_version(package_name)
         shell_input = [
             args.pip,
             "install {}=={}".format(package_name, strip_version(ver)),

komodo/fetch.py

@@ -6,7 +6,7 @@
 import os
 import sys
 
-from komodo.package_version import strip_version
+from komodo.package_version import LATEST_PACKAGE_ALIAS, strip_version, latest_pypi_version
 from komodo.shell import pushd, shell
 from komodo.yaml_file_type import YamlFile
 
@@ -91,6 +91,10 @@ def fetch(pkgs, repo, outdir=".", pip="pip", git="git"):
             url = current.get("source")
             protocol = current.get("fetch")
             pkg_alias = current.get("pypi_package_name", pkg)
+
+            if url == "pypi" and ver == LATEST_PACKAGE_ALIAS:
+                ver = latest_pypi_version(pkg_alias)
+
             name = "{} ({}): {}".format(pkg_alias, ver, url)
             pkgname = "{}-{}".format(pkg_alias, ver)
 

komodo/package_version.py

@@ -1,3 +1,15 @@
+import re
+import subprocess
+import sys
+
+
+LATEST_PACKAGE_ALIAS = "*"
+_PYPI_LATEST_VERSION_RE = r".+from\ versions\:\ (.+)\)"
+# This command is deprecated. Hopefully it is not removed until a replacement
+# is made. For updates on this, see https://github.com/pypa/pip/issues/9139
+_PYPI_LATEST_VERSION_CMD = "python -m pip install --use-deprecated=legacy-resolver {}=="
+
+
 def strip_version(version):
     """
     In order to be able to support both py2 and py3 we need to be able
@@ -8,3 +20,22 @@ def strip_version(version):
     able to install.
     """
     return version.split("+")[0]
+
+
+def latest_pypi_version(package):
+    cmd = _PYPI_LATEST_VERSION_CMD.format(package)
+    try:
+        subprocess.check_output(cmd.split(" "), stderr=subprocess.PIPE)
+    except subprocess.CalledProcessError as e:
+        stderr = e.stderr.decode(sys.getfilesystemencoding())
+        matches = re.match(_PYPI_LATEST_VERSION_RE, stderr)
+        if matches.lastindex == 0:
+            raise ValueError(
+                f"got unexpected output from {cmd} using {_PYPI_LATEST_VERSION_RE}: {stderr}"
+            )
+        versions = matches.group(1).split(",")
+        version = versions[len(versions) - 1].strip()
+        if version == "none":
+            return None
+        return version
+    raise ValueError(f"{cmd} did not raise CalledProcessError")

tests/test_build.py

@@ -1,6 +1,8 @@
-import pytest
+from unittest.mock import patch
 
+import pytest
 from komodo.build import make
+from komodo.package_version import LATEST_PACKAGE_ALIAS
 
 
 @pytest.fixture
@@ -58,6 +60,29 @@ def test_make_one_pip_package_different_name(captured_shell_commands, tmpdir):
     assert "pip install PyYaml==20.4.0" in " ".join(captured_shell_commands[1])
 
 
+def test_make_pip_package_from_latest(captured_shell_commands, tmpdir):
+    packages = {"yaml": LATEST_PACKAGE_ALIAS}
+    repositories = {
+        "yaml": {
+            LATEST_PACKAGE_ALIAS: {
+                "source": "pypi",
+                "pypi_package_name": "PyYaml",
+                "make": "pip",
+                "maintainer": "someone",
+                "depends": [],
+            }
+        }
+    }
+
+    with patch("komodo.build.latest_pypi_version") as mock_latest_version:
+        mock_latest_version.return_value = "1.0.0"
+        make(packages, repositories, {}, str(tmpdir))
+
+    assert len(captured_shell_commands) == 2
+    assert "mkdir" in " ".join(captured_shell_commands[0])
+    assert "pip install PyYaml==1.0.0" in " ".join(captured_shell_commands[1])
+
+
 def test_make_sh_does_not_accept_pypi_package_name(captured_shell_commands, tmpdir):
     packages = {"ert": "2.16.0"}
     repositories = {

tests/test_fetch.py

@@ -1,6 +1,8 @@
-import pytest
+from unittest.mock import patch
 
+import pytest
 from komodo.fetch import fetch
+from komodo.package_version import LATEST_PACKAGE_ALIAS
 
 
 @pytest.fixture
@@ -74,3 +76,24 @@ def test_fetch_git_does_not_accept_pypi_package_name(captured_shell_commands, tm
 
     with pytest.raises(ValueError, match="pypi_package_name"):
         fetch(packages, repositories, str(tmpdir))
+
+
+def test_fetch_pip_with_latest_version(captured_shell_commands, tmpdir):
+    packages = {"ert": LATEST_PACKAGE_ALIAS}
+    repositories = {
+        "ert": {
+            LATEST_PACKAGE_ALIAS: {
+                "source": "pypi",
+                "pypi_package_name": "ert3",
+                "fetch": "pip",
+                "make": "pip",
+                "maintainer": "someone",
+                "depends": [],
+            }
+        }
+    }
+
+    with patch("komodo.fetch.latest_pypi_version") as mock_latest_ver:
+        mock_latest_ver.return_value = "1.0.0"
+        fetch(packages, repositories, str(tmpdir))
+        assert "ert3==1.0.0" in captured_shell_commands[0]

tests/test_package_version.py

@@ -0,0 +1,37 @@
+import sys
+from subprocess import CalledProcessError
+from unittest.mock import patch
+
+import pytest
+from komodo.package_version import latest_pypi_version
+
+
+@pytest.mark.parametrize(
+    "cmd_stderr,expected_version",
+    [
+        (
+            """ERROR: Could not find a version that satisfies the requirement equinor_libres== (from versions: 3.3.1a9, 7.2.0b0, 7.2.0rc0, 8.0.0rc0, 8.0.0, 8.0.1, 8.1.0rc0, 9.0.0b0, 9.0.0b1)
+    ERROR: No matching distribution found for equinor_libres==""",
+            "9.0.0b1",
+        ),
+        (
+            """ERROR: Could not find a version that satisfies the requirement equinor_libres== (from versions: 0.1.0)
+ERROR: No matching distribution found for equinor_libres==""",
+            "0.1.0",
+        ),
+        (
+            """ERROR: Could not find a version that satisfies the requirement equinor_libres== (from versions: none)
+ERROR: No matching distribution found for equinor_libres==""",
+            None,
+        ),
+    ],
+)
+def test_latest_pypi_version(cmd_stderr, expected_version):
+    def _raise(*args, **kwargs):
+        raise CalledProcessError(
+            1, "", stderr=bytes(cmd_stderr.encode(sys.getfilesystemencoding()))
+        )
+
+    with patch("subprocess.check_output") as mock_check_output:
+        mock_check_output.side_effect = _raise
+        assert latest_pypi_version("equinor_libres") == expected_version