[Backport release-22.05] poetry2nix: 1.31.0 -> 1.36.0

pkgs/development/tools/poetry2nix/poetry2nix/default.nix

@@ -5,9 +5,9 @@
 }:
 let
   # Poetry2nix version
-  version = "1.31.0";
+  version = "1.36.0";
 
-  inherit (poetryLib) isCompatible readTOML moduleName;
+  inherit (poetryLib) isCompatible readTOML normalizePackageName normalizePackageSet;
 
   # Map SPDX identifiers to license names
   spdxLicenses = lib.listToAttrs (lib.filter (pair: pair.name != null) (builtins.map (v: { name = if lib.hasAttr "spdxId" v then v.spdxId else null; value = v; }) (lib.attrValues lib.licenses)));
@@ -17,29 +17,34 @@ let
   # Experimental withPlugins functionality
   toPluginAble = (import ./plugins.nix { inherit pkgs lib; }).toPluginAble;
 
+  # List of known build systems that are passed through from nixpkgs unmodified
+  knownBuildSystems = builtins.fromJSON (builtins.readFile ./known-build-systems.json);
+  nixpkgsBuildSystems = lib.subtractLists [ "poetry" "poetry-core" ] knownBuildSystems;
+
   mkInputAttrs =
     { py
     , pyProject
     , attrs
     , includeBuildSystem ? true
+    , groups ? [ ]
+    , checkGroups ? [ "dev" ]
     }:
     let
       getInputs = attr: attrs.${attr} or [ ];
 
       # Get dependencies and filter out depending on interpreter version
-      getDeps = depAttr:
+      getDeps = depSet:
         let
           compat = isCompatible (poetryLib.getPythonVersion py);
-          deps = pyProject.tool.poetry.${depAttr} or { };
-          depAttrs = builtins.map (d: lib.toLower d) (builtins.attrNames deps);
+          depAttrs = builtins.map (d: lib.toLower d) (builtins.attrNames depSet);
         in
         (
           builtins.map
             (
               dep:
               let
-                pkg = py.pkgs."${moduleName dep}";
-                constraints = deps.${dep}.python or "";
+                pkg = py.pkgs."${normalizePackageName dep}";
+                constraints = depSet.${dep}.python or "";
                 isCompat = compat constraints;
               in
               if isCompat then pkg else null
@@ -57,9 +62,21 @@ let
     in
     {
       buildInputs = mkInput "buildInputs" (if includeBuildSystem then buildSystemPkgs else [ ]);
-      propagatedBuildInputs = mkInput "propagatedBuildInputs" (getDeps "dependencies") ++ ([ py.pkgs.setuptools ]);
+      propagatedBuildInputs = mkInput "propagatedBuildInputs" (
+        (getDeps pyProject.tool.poetry."dependencies" or { })
+        ++ (
+          # >=poetry-1.2.0 dependency groups
+          if pyProject.tool.poetry.group or { } != { }
+          then lib.flatten (map (g: getDeps pyProject.tool.poetry.group.${g}.dependencies) groups)
+          else [ ]
+        )
+      );
       nativeBuildInputs = mkInput "nativeBuildInputs" [ ];
-      checkInputs = mkInput "checkInputs" (getDeps "dev-dependencies");
+      checkInputs = mkInput "checkInputs" (
+        getDeps (pyProject.tool.poetry."dev-dependencies" or { })  # <poetry-1.2.0
+        # >=poetry-1.2.0 dependency groups
+        ++ lib.flatten (map (g: getDeps (pyProject.tool.poetry.group.${g}.dependencies or { })) checkGroups)
+      );
     };
 
 
@@ -115,7 +132,9 @@ lib.makeScope pkgs.newScope (self: {
       # Example: { my-app = ./src; }
     , editablePackageSources ? { }
     , pyProject ? readTOML pyproject
-    }@attrs:
+    , groups ? [ ]
+    , checkGroups ? [ "dev" ]
+    }:
     let
       /* The default list of poetry2nix override overlays */
       mkEvalPep508 = import ./pep508.nix {
@@ -140,19 +159,14 @@ lib.makeScope pkgs.newScope (self: {
       };
 
       poetryLock = readTOML poetrylock;
+
+      # Lock file version 1.1 files
       lockFiles =
         let
           lockfiles = lib.getAttrFromPath [ "metadata" "files" ] poetryLock;
         in
-        lib.listToAttrs (lib.mapAttrsToList (n: v: { name = moduleName n; value = v; }) lockfiles);
-      specialAttrs = [
-        "overrides"
-        "poetrylock"
-        "projectDir"
-        "pwd"
-        "preferWheels"
-      ];
-      passedAttrs = builtins.removeAttrs attrs specialAttrs;
+        lib.listToAttrs (lib.mapAttrsToList (n: v: { name = normalizePackageName n; value = v; }) lockfiles);
+
       evalPep508 = mkEvalPep508 python;
 
       # Filter packages by their PEP508 markers & pyproject interpreter version
@@ -170,27 +184,36 @@ lib.makeScope pkgs.newScope (self: {
       # closure as python can only ever have one version of a dependency
       baseOverlay = self: super:
         let
-          getDep = depName: self.${depName};
           lockPkgs = builtins.listToAttrs (
             builtins.map
               (
-                pkgMeta: rec {
-                  name = moduleName pkgMeta.name;
+                pkgMeta:
+                let normalizedName = normalizePackageName pkgMeta.name; in
+                {
+                  name = normalizedName;
                   value = self.mkPoetryDep (
                     pkgMeta // {
                       inherit pwd preferWheels;
                       source = pkgMeta.source or null;
-                      files = lockFiles.${name};
+                      # Default to files from lock file version 2.0 and fall back to 1.1
+                      files = pkgMeta.files or lockFiles.${normalizedName};
                       pythonPackages = self;
-                      sourceSpec = pyProject.tool.poetry.dependencies.${name} or pyProject.tool.poetry.dev-dependencies.${name} or { };
+
+                      sourceSpec = (
+                        (normalizePackageSet pyProject.tool.poetry.dependencies or { }).${normalizedName}
+                          or (normalizePackageSet pyProject.tool.poetry.dev-dependencies or { }).${normalizedName}
+                          or (normalizePackageSet pyProject.tool.poetry.group.dev.dependencies or { }).${normalizedName} # Poetry 1.2.0+
+                          or { }
+                      );
                     }
                   );
                 }
               )
               (lib.reverseList compatible)
           );
+          buildSystems = builtins.listToAttrs (builtins.map (x: { name = x; value = super.${x}; }) nixpkgsBuildSystems);
         in
-        lockPkgs // {
+        lockPkgs // buildSystems // {
           # Create a dummy null package for the current project in case any dependencies depend on the root project (issue #307)
           ${pyProject.tool.poetry.name} = null;
         };
@@ -200,9 +223,6 @@ lib.makeScope pkgs.newScope (self: {
           [
             (
               self: super:
-                let
-                  hooks = self.callPackage ./hooks { };
-                in
                 {
                   mkPoetryDep = self.callPackage ./mk-poetry-dep.nix {
                     inherit lib python poetryLib evalPep508;
@@ -213,8 +233,6 @@ lib.makeScope pkgs.newScope (self: {
                   poetry = poetryPkg;
 
                   __toPluginAble = toPluginAble self;
-
-                  inherit (hooks) pipBuildHook removePathDependenciesHook removeGitDependenciesHook poetry2nixFixupHook wheelUnpackHook;
                 } // lib.optionalAttrs (! super ? setuptools-scm) {
                   # The canonical name is setuptools-scm
                   setuptools-scm = super.setuptools_scm;
@@ -231,7 +249,7 @@ lib.makeScope pkgs.newScope (self: {
               super)
 
             # Null out any filtered packages, we don't want python.pkgs from nixpkgs
-            (self: super: builtins.listToAttrs (builtins.map (x: { name = moduleName x.name; value = null; }) incompatible))
+            (self: super: builtins.listToAttrs (builtins.map (x: { name = normalizePackageName x.name; value = null; }) incompatible))
             # Create poetry2nix layer
             baseOverlay
 
@@ -241,7 +259,7 @@ lib.makeScope pkgs.newScope (self: {
       packageOverrides = lib.foldr lib.composeExtensions (self: super: { }) overlays;
       py = python.override { inherit packageOverrides; self = py; };
 
-      inputAttrs = mkInputAttrs { inherit py pyProject; attrs = { }; includeBuildSystem = false; };
+      inputAttrs = mkInputAttrs { inherit py pyProject groups checkGroups; attrs = { }; includeBuildSystem = false; };
 
       requiredPythonModules = python.pkgs.requiredPythonModules;
       /* Include all the nested dependencies which are required for each package.
@@ -276,9 +294,10 @@ lib.makeScope pkgs.newScope (self: {
     , preferWheels ? false
     , editablePackageSources ? { }
     , extraPackages ? ps: [ ]
+    , groups ? [ "dev" ]
     }:
     let
-      inherit (lib) elem hasAttr;
+      inherit (lib) hasAttr;
 
       pyProject = readTOML pyproject;
 
@@ -294,6 +313,12 @@ lib.makeScope pkgs.newScope (self: {
       allEditablePackageSources = (
         (getEditableDeps (pyProject.tool.poetry."dependencies" or { }))
         // (getEditableDeps (pyProject.tool.poetry."dev-dependencies" or { }))
+        // (
+          # Poetry>=1.2.0
+          if pyProject.tool.poetry.group or { } != { } then
+            builtins.foldl' (acc: g: acc // getEditableDeps pyProject.tool.poetry.group.${g}.dependencies) { } groups
+          else { }
+        )
         // editablePackageSources
       );
 
@@ -302,7 +327,7 @@ lib.makeScope pkgs.newScope (self: {
         excludedEditablePackageNames;
 
       poetryPython = self.mkPoetryPackages {
-        inherit pyproject poetrylock overrides python pwd preferWheels pyProject;
+        inherit pyproject poetrylock overrides python pwd preferWheels pyProject groups;
         editablePackageSources = editablePackageSources';
       };
 
@@ -335,14 +360,18 @@ lib.makeScope pkgs.newScope (self: {
     , python ? pkgs.python3
     , pwd ? projectDir
     , preferWheels ? false
+    , groups ? [ ]
+    , checkGroups ? [ "dev" ]
     , ...
     }@attrs:
     let
       poetryPython = self.mkPoetryPackages {
-        inherit pyproject poetrylock overrides python pwd preferWheels;
+        inherit pyproject poetrylock overrides python pwd preferWheels groups checkGroups;
       };
       py = poetryPython.python;
 
+      hooks = py.pkgs.callPackage ./hooks { };
+
       inherit (poetryPython) pyProject;
       specialAttrs = [
         "overrides"
@@ -354,16 +383,16 @@ lib.makeScope pkgs.newScope (self: {
       ];
       passedAttrs = builtins.removeAttrs attrs specialAttrs;
 
-      inputAttrs = mkInputAttrs { inherit py pyProject attrs; };
+      inputAttrs = mkInputAttrs { inherit py pyProject attrs groups checkGroups; };
 
       app = py.pkgs.buildPythonPackage (
         passedAttrs // inputAttrs // {
           nativeBuildInputs = inputAttrs.nativeBuildInputs ++ [
-            py.pkgs.removePathDependenciesHook
-            py.pkgs.removeGitDependenciesHook
+            hooks.removePathDependenciesHook
+            hooks.removeGitDependenciesHook
           ];
         } // {
-          pname = moduleName pyProject.tool.poetry.name;
+          pname = normalizePackageName pyProject.tool.poetry.name;
           version = pyProject.tool.poetry.version;
 
           inherit src;
@@ -445,7 +474,7 @@ lib.makeScope pkgs.newScope (self: {
 
     Can be overriden by calling defaultPoetryOverrides.overrideOverlay which takes an overlay function
   */
-  defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides { inherit pkgs lib; });
+  defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides { inherit pkgs lib poetryLib; });
 
   /*
     Convenience functions for specifying overlays with or without the poerty2nix default overrides

pkgs/development/tools/poetry2nix/poetry2nix/editable.nix

@@ -6,7 +6,7 @@
 , editablePackageSources
 }:
 let
-  name = poetryLib.moduleName pyProject.tool.poetry.name;
+  name = poetryLib.normalizePackageName pyProject.tool.poetry.name;
 
   # Just enough standard PKG-INFO fields for an editable installation
   pkgInfoFields = {

pkgs/development/tools/poetry2nix/poetry2nix/fetch_from_legacy.py

@@ -80,14 +80,23 @@ def handle_endtag(self, tag):
     exit(1)
 
 package_file = open(package_filename, "wb")
-# Sometimes the href is a relative path
-if urlparse(parser.sources[package_filename]).netloc == "":
+# Sometimes the href is a relative or absolute path within the index's domain.
+indicated_url = urlparse(parser.sources[package_filename])
+if indicated_url.netloc == "":
     parsed_url = urlparse(index_url)
+
+    if indicated_url.path.startswith("/"):
+        # An absolute path within the index's domain.
+        path = parser.sources[package_filename]
+    else:
+        # A relative path.
+        path = parsed_url.path + "/" + parser.sources[package_filename]
+
     package_url = urlunparse(
         (
             parsed_url.scheme,
             parsed_url.netloc,
-            parsed_url.path + "/" + parser.sources[package_filename],
+            path,
             None,
             None,
             None,

pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix

@@ -70,6 +70,7 @@ in
             filenames = builtins.concatStringsSep " " [
               "pyproject.toml"
               "README.md"
+              "LICENSE"
             ];
           };
         } ./fixup-hook.sh

pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-special-deps.py

@@ -22,6 +22,7 @@ def main(input, output, fields_to_remove):
                     any_removed |= dep.pop(field, None) is not None
                 if any_removed:
                     dep["version"] = "*"
+                    dep.pop("develop", None)
 
     output.write(tomlkit.dumps(data))
 

pkgs/development/tools/poetry2nix/poetry2nix/known-build-systems.json

@@ -0,0 +1,13 @@
+[
+    "poetry",
+    "poetry-core",
+    "flit",
+    "flit-core",
+    "pbr",
+    "flitBuildHook",
+    "cython",
+    "hatchling",
+    "hatch-vcs",
+    "setuptools",
+    "setuptools-scm"
+]

pkgs/development/tools/poetry2nix/poetry2nix/lib.nix

@@ -8,8 +8,16 @@ let
     genList (i: if i == idx then value else (builtins.elemAt list i)) (length list)
   );
 
-  # Do some canonicalisation of module names
-  moduleName = name: lib.toLower (lib.replaceStrings [ "_" "." ] [ "-" "-" ] name);
+  # Normalize package names as per PEP 503
+  normalizePackageName = name:
+    let
+      parts = builtins.split "[-_.]+" name;
+      partsWithoutSeparator = builtins.filter (x: builtins.typeOf x == "string") parts;
+    in
+    lib.strings.toLower (lib.strings.concatStringsSep "-" partsWithoutSeparator);
+
+  # Normalize an entire attrset of packages
+  normalizePackageSet = lib.attrsets.mapAttrs' (name: value: lib.attrsets.nameValuePair (normalizePackageName name) value);
 
   # Get a full semver pythonVersion from a python derivation
   getPythonVersion = python:
@@ -233,7 +241,8 @@ in
     getBuildSystemPkgs
     satisfiesSemver
     cleanPythonSources
-    moduleName
+    normalizePackageName
+    normalizePackageSet
     getPythonVersion
     getTargetMachine
     ;