From fcda7d468272a1ef86b1a93284ad8410d70cc92f Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 29 Jan 2017 11:47:20 +0100 Subject: [PATCH 1/7] fetchurl: add pypi.io base url under new name --- pkgs/build-support/fetchurl/mirrors.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/build-support/fetchurl/mirrors.nix b/pkgs/build-support/fetchurl/mirrors.nix index 5ae5eb105e791..99ab5d9051bb2 100644 --- a/pkgs/build-support/fetchurl/mirrors.nix +++ b/pkgs/build-support/fetchurl/mirrors.nix @@ -386,6 +386,10 @@ rec { ]; # Python PyPI mirrors + + pypiio = [ + https://pypi.io/packages/ + ]; pypi = [ https://pypi.io/packages/source/ ]; From c142e12afa5fed87f814933af0d00c4e9e490f0d Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 29 Jan 2017 12:26:04 +0100 Subject: [PATCH 2/7] introduce python.fetchSource, test it on acoustics --- pkgs/top-level/python-packages.nix | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index aee320cea1fe9..deb007b0a3389 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -37,6 +37,17 @@ let graphiteVersion = "0.9.15"; + fetchSource = ({ name ? null , pname ? null, version ? null, type ? "tar.gz", sha256}: + let + _name = if name == null then "${pname}-${version}" else name; + _pdrv = builtins.parseDrvName _name; + in + pkgs.fetchurl { + + url = "mirror://pypiio/source/${builtins.substring 0 1 _name}/${_pdrv.name}/${_pdrv.name}-${_pdrv.version}.${type}"; + sha256 = sha256; + }); + in { inherit python bootstrapped-pip pythonAtLeast pythonOlder isPy26 isPy27 isPy33 isPy34 isPy35 isPy36 isPyPy isPy3k mkPythonDerivation buildPythonPackage buildPythonApplication; @@ -59,8 +70,8 @@ in { buildInputs = with self; [ cython pytest ]; propagatedBuildInputs = with self; [ numpy scipy matplotlib pandas tabulate ]; - src = pkgs.fetchurl { - url = "mirror://pypi/${builtins.substring 0 1 pname}/${pname}/${name}.tar.gz"; + src = fetchSource { + inherit pname version; sha256 = "b75a47de700d01e704de95953a6e969922b2f510d7eefe59f7f8980ad44ad1b7"; }; From a87167cc58fc5a9d7c5b0d2de242e2ab3750f04f Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 29 Jan 2017 14:43:22 +0100 Subject: [PATCH 3/7] python: introduce fetchers for wheel and sdists --- pkgs/top-level/python-packages.nix | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index deb007b0a3389..39b01c5d6a8fe 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -37,17 +37,20 @@ let graphiteVersion = "0.9.15"; - fetchSource = ({ name ? null , pname ? null, version ? null, type ? "tar.gz", sha256}: + fetchPypiMirror = ({ name ? null , pname ? null, version ? null, kind, ext, sha256}: let _name = if name == null then "${pname}-${version}" else name; _pdrv = builtins.parseDrvName _name; in pkgs.fetchurl { - - url = "mirror://pypiio/source/${builtins.substring 0 1 _name}/${_pdrv.name}/${_pdrv.name}-${_pdrv.version}.${type}"; - sha256 = sha256; + inherit sha256; + url = "mirror://pypiio/${kind}/${builtins.substring 0 1 _name}/${_pdrv.name}/${_name}${ext}"; }); + fetchSource = args: fetchPypiMirror (args // { kind = "source"; ext=".tar.gz"; }); + fetchSourceZip = args: fetchPypiMirror (args // { kind = "source"; ext = ".zip";}); + fetchWheel = args: fetchPypiMirror (args // rec { kind = if "source" then "py3" else "py2"; ext = "-${kind}-none-any.whl";}); + fetchUniversalWheel = args: fetchPypiMirror (args // rec { kind = "py2.py3"; ext = "-${kind}-none-any.whl";}); in { inherit python bootstrapped-pip pythonAtLeast pythonOlder isPy26 isPy27 isPy33 isPy34 isPy35 isPy36 isPyPy isPy3k mkPythonDerivation buildPythonPackage buildPythonApplication; @@ -64,14 +67,14 @@ in { acoustics = buildPythonPackage rec { pname = "acoustics"; + name = "acoustics-${version}"; version = "0.1.2"; - name = pname + "-" + version; buildInputs = with self; [ cython pytest ]; propagatedBuildInputs = with self; [ numpy scipy matplotlib pandas tabulate ]; src = fetchSource { - inherit pname version; + inherit name; sha256 = "b75a47de700d01e704de95953a6e969922b2f510d7eefe59f7f8980ad44ad1b7"; }; @@ -99,8 +102,8 @@ in { propagatedBuildInputs = with self; [ discid six parsedatetime isodate Babel pytimeparse ]; - src = pkgs.fetchurl { - url = "mirror://pypi/a/agate/${name}.tar.gz"; + src = fetchSource { + inherit name; sha256 = "0h2w30a0zhylivz86d823a05hvg8w8p61lmm855z1wwkgml9l9d4"; }; }; @@ -766,8 +769,8 @@ in { almir = buildPythonPackage rec { name = "almir-0.1.8"; - src = pkgs.fetchurl { - url = "mirror://pypi/a/almir/${name}.zip"; + src = fetchSourceZip { + inherit name; sha256 = "5dc0b8a5071f3ff46cd2d92608f567ba446e4c733c063b17d89703caeb9868fe"; }; From 0516fd21bc7831d372750e7e724201879a232777 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 2 Feb 2017 07:55:22 +0100 Subject: [PATCH 4/7] python: simplify fetchPypiMirror as suggested --- pkgs/top-level/python-packages.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 39b01c5d6a8fe..5fdfeec94417d 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -37,9 +37,9 @@ let graphiteVersion = "0.9.15"; - fetchPypiMirror = ({ name ? null , pname ? null, version ? null, kind, ext, sha256}: + fetchPypiMirror = ({ kind, ext, sha256, ...} @ args: let - _name = if name == null then "${pname}-${version}" else name; + _name = args.name or "${args.pname}-${args.version}"; _pdrv = builtins.parseDrvName _name; in pkgs.fetchurl { From 5ba74e20ccca7cd9f742b023909cb92a77501708 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 2 Feb 2017 08:01:51 +0100 Subject: [PATCH 5/7] fetchPypi: add url variations with name normalization --- pkgs/top-level/python-packages.nix | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 5fdfeec94417d..065b6db0ea632 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -10,6 +10,8 @@ let packages = ( self: let + + _firstChar = strings.substring 0 1; pythonAtLeast = versionAtLeast python.pythonVersion; pythonOlder = versionOlder python.pythonVersion; isPy26 = python.majorVersion == "2.6"; @@ -19,7 +21,7 @@ let isPy35 = python.majorVersion == "3.5"; isPy36 = python.majorVersion == "3.6"; isPyPy = python.executable == "pypy"; - isPy3k = strings.substring 0 1 python.majorVersion == "3"; + isPy3k = _firstChar python.majorVersion == "3"; callPackage = pkgs.newScope self; @@ -36,21 +38,30 @@ let buildPythonApplication = args: buildPythonPackage ({namePrefix="";} // args ); graphiteVersion = "0.9.15"; + - fetchPypiMirror = ({ kind, ext, sha256, ...} @ args: + mkurls = ({name, kind, ext}: + let + drvs = map builtins.parseDrvName + [ + name + (toLower name) + ]; + in map ({name, version}: "mirror://pypiio/${kind}/${_firstChar name}/${name}/${name}-${version}${ext}") drvs + ); + + fetchPypiMirror = ({ kind, exts, sha256, ...} @ args: let - _name = args.name or "${args.pname}-${args.version}"; - _pdrv = builtins.parseDrvName _name; + name = args.name or "${args.pname}-${args.version}"; in pkgs.fetchurl { inherit sha256; - url = "mirror://pypiio/${kind}/${builtins.substring 0 1 _name}/${_pdrv.name}/${_name}${ext}"; + urls = concatMap (ext: mkurls { inherit name kind ext;}) exts; }); - fetchSource = args: fetchPypiMirror (args // { kind = "source"; ext=".tar.gz"; }); - fetchSourceZip = args: fetchPypiMirror (args // { kind = "source"; ext = ".zip";}); - fetchWheel = args: fetchPypiMirror (args // rec { kind = if "source" then "py3" else "py2"; ext = "-${kind}-none-any.whl";}); - fetchUniversalWheel = args: fetchPypiMirror (args // rec { kind = "py2.py3"; ext = "-${kind}-none-any.whl";}); + fetchSource = args: fetchPypiMirror (args // { kind = "source"; exts=[".tar.gz" ".zip"]; }); + fetchWheel = args: fetchPypiMirror (args // rec { kind ="py${_firstChar python.majorVersion}"; exts = ["-${kind}-none-any.whl"];}); + fetchUniversalWheel = args: fetchPypiMirror (args // rec { kind = "py2.py3"; exts = ["-${kind}-none-any.whl"];}); in { inherit python bootstrapped-pip pythonAtLeast pythonOlder isPy26 isPy27 isPy33 isPy34 isPy35 isPy36 isPyPy isPy3k mkPythonDerivation buildPythonPackage buildPythonApplication; From 8da067eb584ea9011f382fa1597c0a2de2154f8a Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 2 Feb 2017 09:29:14 +0100 Subject: [PATCH 6/7] fechPypi: style fix --- pkgs/top-level/python-packages.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 065b6db0ea632..ed36cfcd7f747 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -42,11 +42,7 @@ let mkurls = ({name, kind, ext}: let - drvs = map builtins.parseDrvName - [ - name - (toLower name) - ]; + drvs = map builtins.parseDrvName [ name (toLower name) ]; in map ({name, version}: "mirror://pypiio/${kind}/${_firstChar name}/${name}/${name}-${version}${ext}") drvs ); From df6185bf7aa83cb1c5f8f0c9b313e291d05eb743 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 2 Feb 2017 09:32:01 +0100 Subject: [PATCH 7/7] fetchpypi: add forgotten .tgz extension --- pkgs/top-level/python-packages.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index ed36cfcd7f747..ae725532c5d4b 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -55,7 +55,7 @@ let urls = concatMap (ext: mkurls { inherit name kind ext;}) exts; }); - fetchSource = args: fetchPypiMirror (args // { kind = "source"; exts=[".tar.gz" ".zip"]; }); + fetchSource = args: fetchPypiMirror (args // { kind = "source"; exts=[".tar.gz" ".zip" ".tgz"]; }); fetchWheel = args: fetchPypiMirror (args // rec { kind ="py${_firstChar python.majorVersion}"; exts = ["-${kind}-none-any.whl"];}); fetchUniversalWheel = args: fetchPypiMirror (args // rec { kind = "py2.py3"; exts = ["-${kind}-none-any.whl"];}); in {