feat: big messy refactoring

- normalize package names to match PEP-503 - don't assume build dependencies on setuptools{,-scm} - automatically generate overrides for setuptools{,-scm} - pull known build systems directly from nixpkgs - fix a bunch of mostly unrelated warnings Co-authored-by: Phillip Cloud <[email protected]>
nix-community · Oct 6, 2022 · 33db1f3 · 33db1f3
1 parent 8be90b6
commit 33db1f3
Show file tree

Hide file tree

Showing 15 changed files with 17,324 additions and 1,350 deletions.
diff --git a/default.nix b/default.nix
@@ -7,7 +7,7 @@ let
   # Poetry2nix version
   version = "1.31.0";
 
-  inherit (poetryLib) isCompatible readTOML moduleName underscorify;
+  inherit (poetryLib) isCompatible readTOML normalizePackageName normalizePackageSet;
 
   # Map SPDX identifiers to license names
   spdxLicenses = lib.listToAttrs (lib.filter (pair: pair.name != null) (builtins.map (v: { name = if lib.hasAttr "spdxId" v then v.spdxId else null; value = v; }) (lib.attrValues lib.licenses)));
@@ -17,6 +17,10 @@ let
   # Experimental withPlugins functionality
   toPluginAble = (import ./plugins.nix { inherit pkgs lib; }).toPluginAble;
 
+  # List of known build systems that are passed through from nixpkgs unmodified
+  knownBuildSystems = builtins.fromJSON (builtins.readFile ./known-build-systems.json);
+  nixpkgsBuildSystems = lib.subtractLists [ "poetry" "poetry-core" ] knownBuildSystems;
+
   mkInputAttrs =
     { py
     , pyProject
@@ -38,7 +42,7 @@ let
             (
               dep:
               let
-                pkg = py.pkgs."${moduleName dep}";
+                pkg = py.pkgs."${normalizePackageName dep}";
                 constraints = depSet.${dep}.python or "";
                 isCompat = compat constraints;
               in
@@ -59,7 +63,6 @@ let
       buildInputs = mkInput "buildInputs" (if includeBuildSystem then buildSystemPkgs else [ ]);
       propagatedBuildInputs = mkInput "propagatedBuildInputs" (
         (getDeps pyProject.tool.poetry."dependencies" or { })
-        ++ ([ py.pkgs.setuptools ])
         ++ (
           # >=poetry-1.2.0 dependency groups
           if pyProject.tool.poetry.group or { } != { }
@@ -128,7 +131,7 @@ lib.makeScope pkgs.newScope (self: {
     , editablePackageSources ? { }
     , pyProject ? readTOML pyproject
     , groups ? [ ]
-    }@attrs:
+    }:
     let
       /* The default list of poetry2nix override overlays */
       mkEvalPep508 = import ./pep508.nix {
@@ -157,15 +160,7 @@ lib.makeScope pkgs.newScope (self: {
         let
           lockfiles = lib.getAttrFromPath [ "metadata" "files" ] poetryLock;
         in
-        lib.listToAttrs (lib.mapAttrsToList (n: v: { name = moduleName n; value = v; }) lockfiles);
-      specialAttrs = [
-        "overrides"
-        "poetrylock"
-        "projectDir"
-        "pwd"
-        "preferWheels"
-      ];
-      passedAttrs = builtins.removeAttrs attrs specialAttrs;
+        lib.listToAttrs (lib.mapAttrsToList (n: v: { name = normalizePackageName n; value = v; }) lockfiles);
       evalPep508 = mkEvalPep508 python;
 
       # Filter packages by their PEP508 markers & pyproject interpreter version
@@ -183,30 +178,32 @@ lib.makeScope pkgs.newScope (self: {
       # closure as python can only ever have one version of a dependency
       baseOverlay = self: super:
         let
-          getDep = depName: self.${depName};
           lockPkgs = builtins.listToAttrs (
             builtins.map
               (
-                pkgMeta: rec {
-                  name = moduleName pkgMeta.name;
+                pkgMeta:
+                if builtins.elem pkgMeta.name nixpkgsBuildSystems then {
+                  name = pkgMeta.name;
+                  value = super."${pkgMeta.name}";
+                } else rec {
+                  name = normalizePackageName pkgMeta.name;
                   value = self.mkPoetryDep (
                     pkgMeta // {
                       inherit pwd preferWheels;
                       source = pkgMeta.source or null;
                       files = lockFiles.${name};
                       pythonPackages = self;
 
-                      # Packages can be specified with underscores in pyproject.toml; check for
-                      # both possibilities.
-                      sourceSpec = with pyProject.tool.poetry; (
-                        dependencies.${pkgMeta.name} or
-                          dependencies.${underscorify pkgMeta.name} or
-                            dev-dependencies.${pkgMeta.name} or
-                              dev-dependencies.${underscorify pkgMeta.name} or
-                                group.dev.dependencies.${underscorify pkgMeta.name} or # Poetry 1.2.0+
-                                  { }
-
-                      );
+                      sourceSpec =
+                        let
+                          normalizedName = normalizePackageName pkgMeta.name;
+                        in
+                        with pyProject.tool.poetry; (
+                          (normalizePackageSet dependencies).${normalizedName}
+                            or (normalizePackageSet dev-dependencies).${normalizedName}
+                            or (normalizePackageSet group.dev.dependencies).${normalizedName} # Poetry 1.2.0+
+                            or { }
+                        );
                     }
                   );
                 }
@@ -255,7 +252,7 @@ lib.makeScope pkgs.newScope (self: {
               super)
 
             # Null out any filtered packages, we don't want python.pkgs from nixpkgs
-            (self: super: builtins.listToAttrs (builtins.map (x: { name = moduleName x.name; value = null; }) incompatible))
+            (self: super: builtins.listToAttrs (builtins.map (x: { name = normalizePackageName x.name; value = null; }) incompatible))
             # Create poetry2nix layer
             baseOverlay
 
@@ -303,7 +300,7 @@ lib.makeScope pkgs.newScope (self: {
     , groups ? [ "dev" ]
     }:
     let
-      inherit (lib) elem hasAttr;
+      inherit (lib) hasAttr;
 
       pyProject = readTOML pyproject;
 
@@ -395,7 +392,7 @@ lib.makeScope pkgs.newScope (self: {
             py.pkgs.removeGitDependenciesHook
           ];
         } // {
-          pname = moduleName pyProject.tool.poetry.name;
+          pname = normalizePackageName pyProject.tool.poetry.name;
           version = pyProject.tool.poetry.version;
 
           inherit src;
@@ -477,7 +474,7 @@ lib.makeScope pkgs.newScope (self: {
 
     Can be overriden by calling defaultPoetryOverrides.overrideOverlay which takes an overlay function
   */
-  defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides { inherit pkgs lib; });
+  defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides { inherit pkgs lib poetryLib; });
 
   /*
     Convenience functions for specifying overlays with or without the poerty2nix default overrides

diff --git a/editable.nix b/editable.nix
@@ -6,7 +6,7 @@
 , editablePackageSources
 }:
 let
-  name = poetryLib.moduleName pyProject.tool.poetry.name;
+  name = poetryLib.normalizePackageName pyProject.tool.poetry.name;
 
   # Just enough standard PKG-INFO fields for an editable installation
   pkgInfoFields = {

diff --git a/known-build-systems.json b/known-build-systems.json
@@ -0,0 +1,12 @@
+[
+    "poetry",
+    "poetry-core",
+    "flit",
+    "flit-core",
+    "pbr",
+    "flitBuildHook",
+    "cython",
+    "hatchling",
+    "setuptools",
+    "setuptools-scm"
+]
diff --git a/lib.nix b/lib.nix
@@ -8,12 +8,16 @@ let
     genList (i: if i == idx then value else (builtins.elemAt list i)) (length list)
   );
 
-  # Do some canonicalisation of module names
-  moduleName = name: lib.toLower (lib.replaceStrings [ "_" "." ] [ "-" "-" ] name);
+  # Normalize package names as per PEP 503
+  normalizePackageName = name:
+    let
+      parts = builtins.split "[-_.]+" name;
+      partsWithoutSeparator = builtins.filter (x: builtins.typeOf x == "string") parts;
+    in
+    lib.strings.toLower (lib.strings.concatStringsSep "-" partsWithoutSeparator);
 
-  # For some reason, poetry replaces underscores with dashes in module
-  # names, this has to be reversed sometimes
-  underscorify = name: (lib.replaceStrings [ "-" ] [ "_" ] name);
+  # Normalize an entire attrset of packages
+  normalizePackageSet = lib.attrsets.mapAttrs' (name: value: lib.attrsets.nameValuePair (normalizePackageName name) value);
 
   # Get a full semver pythonVersion from a python derivation
   getPythonVersion = python:
@@ -237,8 +241,8 @@ in
     getBuildSystemPkgs
     satisfiesSemver
     cleanPythonSources
-    moduleName
-    underscorify
+    normalizePackageName
+    normalizePackageSet
     getPythonVersion
     getTargetMachine
     ;

diff --git a/mk-poetry-dep.nix b/mk-poetry-dep.nix
@@ -26,7 +26,7 @@ pythonPackages.callPackage
     }@args:
     let
       inherit (python) stdenv;
-      inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi moduleName;
+      inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi normalizePackageName;
 
       inherit (import ./pep425.nix {
         inherit lib poetryLib python stdenv;
@@ -88,23 +88,10 @@ pythonPackages.callPackage
             else (builtins.elemAt (lib.strings.splitString "-" name) 2);
         };
 
-      # Prevent infinite recursion
-      skipSetupToolsSCM = [
-        "setuptools_scm"
-        "setuptools-scm"
-        "toml" # Toml is an extra for setuptools-scm
-        "tomli" # tomli is an extra for later versions of setuptools-scm
-        "flit-core"
-        "packaging"
-        "six"
-        "pyparsing"
-        "typing-extensions"
-      ];
-      baseBuildInputs = lib.optional (! lib.elem name skipSetupToolsSCM) pythonPackages.setuptools-scm;
       format = if isDirectory || isGit || isUrl then "pyproject" else fileInfo.format;
     in
     buildPythonPackage {
-      pname = moduleName name;
+      pname = normalizePackageName name;
       version = version;
 
       inherit format;
@@ -124,10 +111,9 @@ pythonPackages.callPackage
       ];
 
       buildInputs = (
-        baseBuildInputs
-        ++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) pythonPackages.setuptools
-        ++ lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg
+        lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg
         ++ lib.optional isDirectory buildSystemPkgs
+        ++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) pythonPackages.setuptools
       );
 
       propagatedBuildInputs =
@@ -149,7 +135,7 @@ pythonPackages.callPackage
             );
           depAttrs = lib.attrNames deps;
         in
-        builtins.map (n: pythonPackages.${moduleName n}) depAttrs;
+        builtins.map (n: pythonPackages.${normalizePackageName n}) depAttrs;
 
       meta = {
         broken = ! isCompatible (poetryLib.getPythonVersion python) python-versions;