From e61656f55bacc244cb06755c3922ffa53eaf0391 Mon Sep 17 00:00:00 2001 From: Mario Rodas Date: Thu, 7 Mar 2024 04:20:00 +0000 Subject: [PATCH 1/7] re-flex: 4.0.1 -> 4.1.0 Changelog: https://github.com/Genivia/RE-flex/releases/tag/v4.1.0 --- .../tools/parsing/re-flex/default.nix | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pkgs/development/tools/parsing/re-flex/default.nix b/pkgs/development/tools/parsing/re-flex/default.nix index 645d4e2802885..c75245541287e 100644 --- a/pkgs/development/tools/parsing/re-flex/default.nix +++ b/pkgs/development/tools/parsing/re-flex/default.nix @@ -1,30 +1,32 @@ { lib , stdenv , fetchFromGitHub -, autoreconfHook -, boost -, autoconf -, automake +, cmake }: stdenv.mkDerivation rec { pname = "re-flex"; - version = "4.0.1"; + version = "4.1.0"; src = fetchFromGitHub { owner = "Genivia"; repo = "RE-flex"; rev = "v${version}"; - sha256 = "sha256-eQ2+RthvOKCd2Dl6i+9DahJArFfOhPJkn6PI/yuaqos="; + hash = "sha256-pjYiCRKaskJg1IuCxNBUQ9FY2abGi4HEZxsfZ5ctjNY="; }; - nativeBuildInputs = [ boost autoconf automake ]; + outputs = [ "out" "bin" "dev" ]; + + nativeBuildInputs = [ + cmake + ]; meta = with lib; { - homepage = "https://github.com/Genivia/RE-flex"; + homepage = "https://www.genivia.com/doc/reflex/html"; description = "The regex-centric, fast lexical analyzer generator for C++ with full Unicode support"; license = licenses.bsd3; - platforms = platforms.unix; + platforms = platforms.all; maintainers = with lib.maintainers; [ prrlvr ]; + mainProgram = "reflex"; }; } From 870f58ffa94d8dd44246351aec220c77399ed4f8 Mon Sep 17 00:00:00 2001 From: matthewcroughan Date: Tue, 2 Jan 2024 04:19:11 +0000 Subject: [PATCH 2/7] python3Packages.gpuctypes: init at unstable-2023-11-26 --- .../gpuctypes/0001-fix-dlopen-cuda.patch | 44 +++++++++ .../python-modules/gpuctypes/default.nix | 91 +++++++++++++++++++ .../gpuctypes/fix-dlopen-cuda.patch | 32 +++++++ pkgs/top-level/python-packages.nix | 2 + 4 files changed, 169 insertions(+) create mode 100644 pkgs/development/python-modules/gpuctypes/0001-fix-dlopen-cuda.patch create mode 100644 pkgs/development/python-modules/gpuctypes/default.nix create mode 100644 pkgs/development/python-modules/gpuctypes/fix-dlopen-cuda.patch diff --git a/pkgs/development/python-modules/gpuctypes/0001-fix-dlopen-cuda.patch b/pkgs/development/python-modules/gpuctypes/0001-fix-dlopen-cuda.patch new file mode 100644 index 0000000000000..bc9f6c7ec64b9 --- /dev/null +++ b/pkgs/development/python-modules/gpuctypes/0001-fix-dlopen-cuda.patch @@ -0,0 +1,44 @@ +From d448321436e8314d3e2a6a09d4017c4bc10f612d Mon Sep 17 00:00:00 2001 +From: Gaetan Lepage +Date: Sat, 17 Feb 2024 17:37:22 +0100 +Subject: [PATCH] fix-dlopen-cuda + +--- + gpuctypes/cuda.py | 20 ++++++++++++++++++-- + 1 file changed, 18 insertions(+), 2 deletions(-) + +diff --git a/gpuctypes/cuda.py b/gpuctypes/cuda.py +index acba81c..091f7f7 100644 +--- a/gpuctypes/cuda.py ++++ b/gpuctypes/cuda.py +@@ -143,9 +143,25 @@ def char_pointer_cast(string, encoding='utf-8'): + + + ++NAME_TO_PATHS = { ++ "libcuda.so": ["@driverLink@/lib/libcuda.so"], ++ "libnvrtc.so": ["@libnvrtc@"], ++} ++def _try_dlopen(name): ++ try: ++ return ctypes.CDLL(name) ++ except OSError: ++ pass ++ for candidate in NAME_TO_PATHS.get(name, []): ++ try: ++ return ctypes.CDLL(candidate) ++ except OSError: ++ pass ++ raise RuntimeError(f"{name} not found") ++ + _libraries = {} +-_libraries['libcuda.so'] = ctypes.CDLL(ctypes.util.find_library('cuda')) +-_libraries['libnvrtc.so'] = ctypes.CDLL(ctypes.util.find_library('nvrtc')) ++_libraries['libcuda.so'] = _try_dlopen('libcuda.so') ++_libraries['libnvrtc.so'] = _try_dlopen('libnvrtc.so') + + + cuuint32_t = ctypes.c_uint32 +-- +2.43.0 + diff --git a/pkgs/development/python-modules/gpuctypes/default.nix b/pkgs/development/python-modules/gpuctypes/default.nix new file mode 100644 index 0000000000000..4b1d8560fa4de --- /dev/null +++ b/pkgs/development/python-modules/gpuctypes/default.nix @@ -0,0 +1,91 @@ +{ lib +, config +, buildPythonPackage +, fetchFromGitHub +, substituteAll +, addDriverRunpath +, cudaSupport ? config.cudaSupport +, rocmSupport ? config.rocmSupport +, cudaPackages +, setuptools +, ocl-icd +, rocmPackages +, pytestCheckHook +}: +buildPythonPackage rec { + pname = "gpuctypes"; + version = "0.3.0"; + pyproject = true; + + src = fetchFromGitHub { + repo = "gpuctypes"; + owner = "tinygrad"; + rev = "refs/tags/${version}"; + hash = "sha256-xUMvMBK1UhZaMZfik0Ia6+siyZGpCkBV+LTnQvzt/rw="; + }; + + patches = [ + (substituteAll { + src = ./0001-fix-dlopen-cuda.patch; + inherit (addDriverRunpath) driverLink; + libnvrtc = + if cudaSupport + then "${lib.getLib cudaPackages.cuda_nvrtc}/lib/libnvrtc.so" + else "Please import nixpkgs with `config.cudaSupport = true`"; + }) + ]; + + nativeBuildInputs = [ + setuptools + ]; + + postPatch = '' + substituteInPlace gpuctypes/opencl.py \ + --replace "ctypes.util.find_library('OpenCL')" "'${ocl-icd}/lib/libOpenCL.so'" + '' + # hipGetDevicePropertiesR0600 is a symbol from rocm-6. We are currently at rocm-5. + # We are not sure that this works. Remove when rocm gets updated to version 6. + + lib.optionalString rocmSupport '' + substituteInPlace gpuctypes/hip.py \ + --replace "/opt/rocm/lib/libamdhip64.so" "${rocmPackages.clr}/lib/libamdhip64.so" \ + --replace "/opt/rocm/lib/libhiprtc.so" "${rocmPackages.clr}/lib/libhiprtc.so" \ + --replace "hipGetDevicePropertiesR0600" "hipGetDeviceProperties" + + substituteInPlace gpuctypes/comgr.py \ + --replace "/opt/rocm/lib/libamd_comgr.so" "${rocmPackages.rocm-comgr}/lib/libamd_comgr.so" + ''; + + pythonImportsCheck = [ "gpuctypes" ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + disabledTestPaths = [ + "test/test_opencl.py" + ] ++ lib.optionals (!rocmSupport) [ + "test/test_hip.py" + ] ++ lib.optionals (!cudaSupport) [ + "test/test_cuda.py" + ]; + + # Require GPU access to run (not available in the sandbox) + pytestFlagsArray = [ + "-k" "'not TestCUDADevice'" + "-k" "'not TestHIPDevice'" + ]; + + preCheck = lib.optionalString cudaSupport '' + addToSearchPath LD_LIBRARY_PATH ${lib.getLib cudaPackages.cuda_cudart}/lib/stubs + ''; + + # If neither rocmSupport or cudaSupport is enabled, no tests are selected + dontUsePytestCheck = !(rocmSupport || cudaSupport); + + meta = with lib; { + description = "Ctypes wrappers for HIP, CUDA, and OpenCL"; + homepage = "https://github.com/tinygrad/gpuctypes"; + license = licenses.mit; + maintainers = with maintainers; [ GaetanLepage matthewcroughan wozeparrot ]; + }; +} diff --git a/pkgs/development/python-modules/gpuctypes/fix-dlopen-cuda.patch b/pkgs/development/python-modules/gpuctypes/fix-dlopen-cuda.patch new file mode 100644 index 0000000000000..8d3b69e35e110 --- /dev/null +++ b/pkgs/development/python-modules/gpuctypes/fix-dlopen-cuda.patch @@ -0,0 +1,32 @@ +diff --git a/gpuctypes/cuda.py b/gpuctypes/cuda.py +index acba81c..aac5fc7 100644 +--- a/gpuctypes/cuda.py ++++ b/gpuctypes/cuda.py +@@ -143,9 +143,25 @@ def char_pointer_cast(string, encoding='utf-8'): + + + ++NAME_TO_PATHS = { ++ "libcuda.so": ["@driverLink@/lib/libcuda.so"], ++ "libnvrtc.so": ["@libnvrtc@"], ++} ++def _try_dlopen(name): ++ try: ++ return ctypes.CDLL(name) ++ except OSError: ++ pass ++ for candidate in NAME_TO_PATHS.get(name, []): ++ try: ++ return ctypes.CDLL(candidate) ++ except OSError: ++ pass ++ raise RuntimeError(f"{name} not found") ++ + _libraries = {} +-_libraries['libcuda.so'] = ctypes.CDLL(ctypes.util.find_library('cuda')) +-_libraries['libnvrtc.so'] = ctypes.CDLL(ctypes.util.find_library('nvrtc')) ++_libraries['libcuda.so'] = _try_dlopen('libcuda.so') ++_libraries['libnvrtc.so'] = _try_dlopen('libnvrtc.so') + + + cuuint32_t = ctypes.c_uint32 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index b90813500e443..1da6f92cce85c 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -4933,6 +4933,8 @@ self: super: with self; { gpsoauth = callPackage ../development/python-modules/gpsoauth { }; + gpuctypes = callPackage ../development/python-modules/gpuctypes { }; + gpustat = callPackage ../development/python-modules/gpustat { }; gpxpy = callPackage ../development/python-modules/gpxpy { }; From b7bcf94a708ce799a8586ce6255f80a27615731c Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 21 Feb 2024 00:15:11 +0100 Subject: [PATCH 3/7] python311Packages.gpuctypes: add gpu tests Co-authored-by: SomeoneSerge --- .../python-modules/gpuctypes/default.nix | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/pkgs/development/python-modules/gpuctypes/default.nix b/pkgs/development/python-modules/gpuctypes/default.nix index 4b1d8560fa4de..e5987f78ee07e 100644 --- a/pkgs/development/python-modules/gpuctypes/default.nix +++ b/pkgs/development/python-modules/gpuctypes/default.nix @@ -11,7 +11,14 @@ , ocl-icd , rocmPackages , pytestCheckHook +, gpuctypes +, testCudaRuntime ? false +, testOpenclRuntime ? false +, testRocmRuntime ? false }: +assert testCudaRuntime -> cudaSupport; +assert testRocmRuntime -> rocmSupport; + buildPythonPackage rec { pname = "gpuctypes"; version = "0.3.0"; @@ -61,7 +68,7 @@ buildPythonPackage rec { pytestCheckHook ]; - disabledTestPaths = [ + disabledTestPaths = lib.optionals (!testOpenclRuntime) [ "test/test_opencl.py" ] ++ lib.optionals (!rocmSupport) [ "test/test_hip.py" @@ -70,17 +77,46 @@ buildPythonPackage rec { ]; # Require GPU access to run (not available in the sandbox) - pytestFlagsArray = [ + pytestFlagsArray = lib.optionals (!testCudaRuntime) [ "-k" "'not TestCUDADevice'" + ] ++ lib.optionals (!testRocmRuntime) [ "-k" "'not TestHIPDevice'" + ] ++ lib.optionals (testCudaRuntime || testOpenclRuntime || testRocmRuntime) [ + "-v" + ]; + + # Running these tests requires special configuration on the builder. + # e.g. https://github.com/NixOS/nixpkgs/pull/256230 implements a nix + # pre-build hook which exposes the devices and the drivers in the sandbox + # based on requiredSystemFeatures: + requiredSystemFeatures = lib.optionals testCudaRuntime [ + "cuda" + ] ++ lib.optionals testOpenclRuntime [ + "opencl" + ] ++ lib.optionals testRocmRuntime [ + "rocm" ]; - preCheck = lib.optionalString cudaSupport '' + passthru.gpuChecks = { + cuda = gpuctypes.override { + cudaSupport = true; + testCudaRuntime = true; + }; + opencl = gpuctypes.override { + testOpenclRuntime = true; + }; + rocm = gpuctypes.override { + rocmSupport = true; + testRocmRuntime = true; + }; + }; + + preCheck = lib.optionalString (cudaSupport && !testCudaRuntime) '' addToSearchPath LD_LIBRARY_PATH ${lib.getLib cudaPackages.cuda_cudart}/lib/stubs ''; # If neither rocmSupport or cudaSupport is enabled, no tests are selected - dontUsePytestCheck = !(rocmSupport || cudaSupport); + dontUsePytestCheck = !(rocmSupport || cudaSupport) && (!testOpenclRuntime); meta = with lib; { description = "Ctypes wrappers for HIP, CUDA, and OpenCL"; From 24833cd162babe2c4d50b910d439d714fa8897d2 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Sun, 11 Feb 2024 02:35:10 +0100 Subject: [PATCH 4/7] python3Packages.tinygrad: init at 0.8.0 Co-authored-by: matthewcroughan --- .../python-modules/tinygrad/default.nix | 111 ++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 113 insertions(+) create mode 100644 pkgs/development/python-modules/tinygrad/default.nix diff --git a/pkgs/development/python-modules/tinygrad/default.nix b/pkgs/development/python-modules/tinygrad/default.nix new file mode 100644 index 0000000000000..25b1f049bf697 --- /dev/null +++ b/pkgs/development/python-modules/tinygrad/default.nix @@ -0,0 +1,111 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, setuptools +, wheel +, gpuctypes +, numpy +, tqdm +, hypothesis +, librosa +, onnx +, pillow +, pytest-xdist +, pytestCheckHook +, safetensors +, sentencepiece +, tiktoken +, torch +, transformers +}: + +buildPythonPackage rec { + pname = "tinygrad"; + version = "0.8.0"; + pyproject = true; + + src = fetchFromGitHub { + owner = "tinygrad"; + repo = "tinygrad"; + rev = "refs/tags/v${version}"; + hash = "sha256-QAccZ79qUbe27yUykIf22WdkxYUlOffnMlShakKfp60="; + }; + + nativeBuildInputs = [ + setuptools + wheel + ]; + + propagatedBuildInputs = [ + gpuctypes + numpy + tqdm + ]; + + pythonImportsCheck = [ "tinygrad" ]; + + nativeCheckInputs = [ + hypothesis + librosa + onnx + pillow + pytest-xdist + pytestCheckHook + safetensors + sentencepiece + tiktoken + torch + transformers + ]; + + preCheck = '' + export HOME=$(mktemp -d) + ''; + + disabledTests = [ + # Require internet access + "test_benchmark_openpilot_model" + "test_bn_alone" + "test_bn_linear" + "test_bn_mnist" + "test_car" + "test_chicken" + "test_chicken_bigbatch" + "test_conv_mnist" + "testCopySHMtoDefault" + "test_data_parallel_resnet" + "test_e2e_big" + "test_fetch_small" + "test_huggingface_enet_safetensors" + "test_linear_mnist" + "test_load_convnext" + "test_load_enet" + "test_load_enet_alt" + "test_load_llama2bfloat" + "test_load_resnet" + "test_openpilot_model" + "test_resnet" + "test_shufflenet" + "test_transcribe_batch12" + "test_transcribe_batch21" + "test_transcribe_file1" + "test_transcribe_file2" + "test_transcribe_long" + "test_transcribe_long_no_batch" + "test_vgg7" + ]; + + disabledTestPaths = [ + "test/extra/test_lr_scheduler.py" + "test/models/test_mnist.py" + "test/models/test_real_world.py" + ]; + + meta = with lib; { + description = "A simple and powerful neural network framework"; + homepage = "https://github.com/tinygrad/tinygrad"; + changelog = "https://github.com/tinygrad/tinygrad/releases/tag/v${version}"; + license = licenses.mit; + maintainers = with maintainers; [ GaetanLepage ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 1da6f92cce85c..8bcdbe939dc9a 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -14761,6 +14761,8 @@ self: super: with self; { tinydb = callPackage ../development/python-modules/tinydb { }; + tinygrad = callPackage ../development/python-modules/tinygrad { }; + tinyobjloader-py = callPackage ../development/python-modules/tinyobjloader-py { }; tinyrecord = callPackage ../development/python-modules/tinyrecord { }; From 01a7b9d8abc7947bff4e9da34d101d976bb35ffd Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Wed, 13 Mar 2024 06:42:38 +0000 Subject: [PATCH 5/7] python311Packages.autoflake: 2.3.0 -> 2.3.1 --- pkgs/development/python-modules/autoflake/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/autoflake/default.nix b/pkgs/development/python-modules/autoflake/default.nix index 64d429629276a..e6f9c94c3f8f1 100644 --- a/pkgs/development/python-modules/autoflake/default.nix +++ b/pkgs/development/python-modules/autoflake/default.nix @@ -9,12 +9,12 @@ }: buildPythonPackage rec { pname = "autoflake"; - version = "2.3.0"; + version = "2.3.1"; format = "pyproject"; src = fetchPypi { inherit pname version; - hash = "sha256-jCAR+jRwG519zwW5hzvEhZ1Pzk5i3+qQ3/79FXb18B0="; + hash = "sha256-yYt13FsKhkWcTwGh0yrH60M47EMXpEaVFf8eaH7NkJ4="; }; nativeBuildInputs = [ From b622ed3f841ba8b0ef36c56fe1197c38fb2787bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Sun, 10 Mar 2024 10:51:41 -0700 Subject: [PATCH 6/7] plfit: 0.9.4 -> 0.9.6 Diff: https://github.com/ntamas/plfit/compare/0.9.4...0.9.6 Changelog: https://github.com/ntamas/plfit/blob/0.9.6/CHANGELOG.md --- pkgs/tools/misc/plfit/default.nix | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pkgs/tools/misc/plfit/default.nix b/pkgs/tools/misc/plfit/default.nix index 78e7c3572b6f0..8b1726666bccb 100644 --- a/pkgs/tools/misc/plfit/default.nix +++ b/pkgs/tools/misc/plfit/default.nix @@ -7,17 +7,23 @@ , llvmPackages }: -stdenv.mkDerivation rec { +stdenv.mkDerivation (finalAttrs: { pname = "plfit"; - version = "0.9.4"; + version = "0.9.6"; src = fetchFromGitHub { owner = "ntamas"; repo = "plfit"; - rev = version; - hash = "sha256-hnmP/56P2anR0S8zQyQqN1lbge5GgK+P8Lx8bRkwSxA="; + rev = finalAttrs.version; + hash = "sha256-XRl6poEdgPNorFideQmEJHCU+phs4rIhMYa8iAOtL1A="; }; + postPatch = lib.optionalString (python != null) '' + substituteInPlace src/CMakeLists.txt \ + --replace-fail ' ''${Python3_SITEARCH}' ' ${placeholder "out"}/${python.sitePackages}' \ + --replace-fail ' ''${Python3_SITELIB}' ' ${placeholder "out"}/${python.sitePackages}' + ''; + nativeBuildInputs = [ cmake ] ++ lib.optionals (python != null) [ @@ -35,11 +41,13 @@ stdenv.mkDerivation rec { llvmPackages.openmp ]; + doCheck = true; + meta = with lib; { description = "Fitting power-law distributions to empirical data"; homepage = "https://github.com/ntamas/plfit"; - changelog = "https://github.com/ntamas/plfit/blob/${src.rev}/CHANGELOG.md"; + changelog = "https://github.com/ntamas/plfit/blob/${finalAttrs.src.rev}/CHANGELOG.md"; license = licenses.gpl2Plus; maintainers = with maintainers; [ dotlambda ]; }; -} +}) From e3abe2987d47aee9ceb34a850da824471b53a39c Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Wed, 13 Mar 2024 19:10:14 +0000 Subject: [PATCH 7/7] clash-geoip: 20240212 -> 20240312 --- pkgs/data/misc/clash-geoip/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/data/misc/clash-geoip/default.nix b/pkgs/data/misc/clash-geoip/default.nix index 1ad3226080a40..0ec9663cbac31 100644 --- a/pkgs/data/misc/clash-geoip/default.nix +++ b/pkgs/data/misc/clash-geoip/default.nix @@ -2,11 +2,11 @@ stdenvNoCC.mkDerivation rec { pname = "clash-geoip"; - version = "20240212"; + version = "20240312"; src = fetchurl { url = "https://github.com/Dreamacro/maxmind-geoip/releases/download/${version}/Country.mmdb"; - sha256 = "sha256-cNVEWdIRo2Z2FluZIR0O5o3Aso4tDcVyHAG3DkNmpSQ="; + sha256 = "sha256-h6nrlzFBRrvL+hUOnpWi/aixKDOlRoTV4zQYIHGslIY="; }; dontUnpack = true;