From 640cdcb814ce7476d002a306816ed9d88baaf4f3 Mon Sep 17 00:00:00 2001 From: Matthieu Coudron <886074+teto@users.noreply.github.com> Date: Wed, 6 Mar 2024 22:01:08 +0100 Subject: [PATCH 1/5] lua: smarter/more correct patching In order to have the 'reset' LUA_PATH (aka `;;`) work, and for purity reasons(removing /usr paths) we(I) decided to patch the lua interpreters default LUA_PATH. Turns out the interpreters have different defaults and the patch was too coarse. There is smarter patching that can be done via LUA_ROOT. Also luajit doesn't need patching at all since LUA_ROOT is set via the build system. --- .../interpreters/lua-5/hooks/default.nix | 1 - .../interpreters/lua-5/interpreter.nix | 25 ++++++++----------- .../interpreters/luajit/default.nix | 16 +++--------- 3 files changed, 14 insertions(+), 28 deletions(-) diff --git a/pkgs/development/interpreters/lua-5/hooks/default.nix b/pkgs/development/interpreters/lua-5/hooks/default.nix index 6c303f770decc..ca9c15e8a3b1e 100644 --- a/pkgs/development/interpreters/lua-5/hooks/default.nix +++ b/pkgs/development/interpreters/lua-5/hooks/default.nix @@ -8,7 +8,6 @@ let callPackage = lua.pkgs.callPackage; - luaInterpreter = lua.interpreter; in { lua-setup-hook = LuaPathSearchPaths: LuaCPathSearchPaths: diff --git a/pkgs/development/interpreters/lua-5/interpreter.nix b/pkgs/development/interpreters/lua-5/interpreter.nix index 4091fdd49e0e4..2856951d9fbdf 100644 --- a/pkgs/development/interpreters/lua-5/interpreter.nix +++ b/pkgs/development/interpreters/lua-5/interpreter.nix @@ -22,7 +22,7 @@ stdenv.mkDerivation (finalAttrs: let luaPackages = self.pkgs; - luaversion = lib.versions.majorMinor version; + luaversion = lib.versions.majorMinor finalAttrs.version; plat = if (stdenv.isLinux && lib.versionOlder self.luaversion "5.4") then "linux" else if (stdenv.isLinux && lib.versionAtLeast self.luaversion "5.4") then "linux-readline" @@ -45,7 +45,7 @@ stdenv.mkDerivation (finalAttrs: outputs = [ "out" "doc" ]; src = fetchurl { - url = "https://www.lua.org/ftp/${finalAttrs.pname}-${finalAttrs.version}.tar.gz"; + url = "https://www.lua.org/ftp/lua-${finalAttrs.version}.tar.gz"; sha256 = hash; }; @@ -60,16 +60,11 @@ stdenv.mkDerivation (finalAttrs: inherit patches; - # we can't pass flags to the lua makefile because for portability, everything is hardcoded postPatch = '' - { - echo -e ' - #undef LUA_PATH_DEFAULT - #define LUA_PATH_DEFAULT "./share/lua/${luaversion}/?.lua;./?.lua;./?/init.lua" - #undef LUA_CPATH_DEFAULT - #define LUA_CPATH_DEFAULT "./lib/lua/${luaversion}/?.so;./?.so;./lib/lua/${luaversion}/loadall.so" - ' - } >> src/luaconf.h + sed -i "s@#define LUA_ROOT[[:space:]]*\"/usr/local/\"@#define LUA_ROOT \"$out/\"@g" src/luaconf.h + + # abort if patching didn't work + grep $out src/luaconf.h '' + lib.optionalString (!stdenv.isDarwin && !staticOnly) '' # Add a target for a shared library to the Makefile. sed -e '1s/^/LUA_SO = liblua.so/' \ @@ -102,8 +97,8 @@ stdenv.mkDerivation (finalAttrs: makeFlagsArray+=(${lib.optionalString stdenv.isDarwin "CC=\"$CC\""}${lib.optionalString (stdenv.buildPlatform != stdenv.hostPlatform) " 'AR=${stdenv.cc.targetPrefix}ar rcu'"}) installFlagsArray=( TO_BIN="lua luac" INSTALL_DATA='cp -d' \ - TO_LIB="${if stdenv.isDarwin then "liblua.${version}.dylib" - else ("liblua.a" + lib.optionalString (!staticOnly) " liblua.so liblua.so.${luaversion} liblua.so.${version}" )}" ) + TO_LIB="${if stdenv.isDarwin then "liblua.${finalAttrs.version}.dylib" + else ("liblua.a" + lib.optionalString (!staticOnly) " liblua.so liblua.so.${luaversion} liblua.so.${finalAttrs.version}" )}" ) runHook postConfigure ''; @@ -128,7 +123,7 @@ stdenv.mkDerivation (finalAttrs: Name: Lua Description: An Extensible Extension Language - Version: ${version} + Version: ${finalAttrs.version} Requires: Libs: -L$out/lib -llua Cflags: -I$out/include @@ -138,7 +133,7 @@ stdenv.mkDerivation (finalAttrs: ln -s "$out/lib/pkgconfig/lua.pc" "$out/lib/pkgconfig/lua${lib.replaceStrings [ "." ] [ "" ] luaversion}.pc" # Make documentation outputs of different versions co-installable. - mv $out/share/doc/lua $out/share/doc/lua-${version} + mv $out/share/doc/lua $out/share/doc/lua-${finalAttrs.version} ''; # copied from python diff --git a/pkgs/development/interpreters/luajit/default.nix b/pkgs/development/interpreters/luajit/default.nix index 211fa56e91197..15bcfee3a44cc 100644 --- a/pkgs/development/interpreters/luajit/default.nix +++ b/pkgs/development/interpreters/luajit/default.nix @@ -62,7 +62,7 @@ let else buildPackages.stdenv; in -stdenv.mkDerivation rec { +stdenv.mkDerivation (finalAttrs: { pname = "luajit"; inherit version src; @@ -75,15 +75,6 @@ stdenv.mkDerivation rec { # passed by nixpkgs CC wrapper is insufficient on its own substituteInPlace src/Makefile --replace "#CCDEBUG= -g" "CCDEBUG= -g" fi - - { - echo -e ' - #undef LUA_PATH_DEFAULT - #define LUA_PATH_DEFAULT "./share/lua/${luaversion}/?.lua;./?.lua;./?/init.lua" - #undef LUA_CPATH_DEFAULT - #define LUA_CPATH_DEFAULT "./lib/lua/${luaversion}/?.so;./?.so;./lib/lua/${luaversion}/loadall.so" - ' - } >> src/luaconf.h ''; dontConfigure = true; @@ -122,7 +113,8 @@ stdenv.mkDerivation rec { inputs' = lib.filterAttrs (n: v: ! lib.isDerivation v && n != "passthruFun") inputs; override = attr: let lua = attr.override (inputs' // { self = lua; }); in lua; in passthruFun rec { - inherit self luaversion packageOverrides luaAttr; + inherit self packageOverrides luaAttr; + inherit (finalAttrs) luaversion; executable = "lua"; luaOnBuildForBuild = override pkgsBuildBuild.${luaAttr}; luaOnBuildForHost = override pkgsBuildHost.${luaAttr}; @@ -142,4 +134,4 @@ stdenv.mkDerivation rec { ]; maintainers = with maintainers; [ thoughtpolice smironov vcunat lblasc ]; } // extraMeta; -} +}) From 815d3683f70470e4c1b6c72535576e5e41d731ae Mon Sep 17 00:00:00 2001 From: Matthieu Coudron <886074+teto@users.noreply.github.com> Date: Wed, 6 Mar 2024 22:40:01 +0100 Subject: [PATCH 2/5] lua.tests: update golden values for default LUA_PATH when LUA_PATH contains `;;`, it is replaced by `;LUA_PATH_DEFAULT;` in lua 5.1: Updated https://github.com/lua/lua/blob/69ea087dff1daba25a2000dfb8f1883c17545b7a/loadlib.c#L599 More recent versions might be smarter ? https://github.com/lua/lua/blob/65b07dd53d7938a60112fc4473f5cad3473e3534/loadlib.c#L301 --- .../interpreters/lua-5/tests/assert.sh | 4 +- .../interpreters/lua-5/tests/default.nix | 56 ++++++++++++------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/pkgs/development/interpreters/lua-5/tests/assert.sh b/pkgs/development/interpreters/lua-5/tests/assert.sh index c5783a24b2d7b..b0aa3825ef93a 100644 --- a/pkgs/development/interpreters/lua-5/tests/assert.sh +++ b/pkgs/development/interpreters/lua-5/tests/assert.sh @@ -3,14 +3,14 @@ # Example: # fail "It should have been but it wasn't to be" function fail() { - echo "$1" + echo -e "$1" exit 1 } function assertStringEqual() { if ! diff <(echo "$1") <(echo "$2") ; then - fail "expected \"$1\" to be equal to \"$2\"" + fail "Actual value: \"$1\"\nExpected value: \"$2\"" fi } diff --git a/pkgs/development/interpreters/lua-5/tests/default.nix b/pkgs/development/interpreters/lua-5/tests/default.nix index 7351fb7cd6f4d..6ca6b153c0b62 100644 --- a/pkgs/development/interpreters/lua-5/tests/default.nix +++ b/pkgs/development/interpreters/lua-5/tests/default.nix @@ -1,12 +1,10 @@ { lua , hello , wrapLua -, lib, fetchFromGitHub -, fetchFromGitLab +, lib , pkgs }: let - runTest = lua: { name, command }: pkgs.runCommandLocal "test-${lua.name}-${name}" ({ nativeBuildInputs = [lua]; @@ -18,29 +16,47 @@ let + "touch $out" ); - wrappedHello = hello.overrideAttrs(oa: { - propagatedBuildInputs = [ - wrapLua - lua.pkgs.cjson - ]; - postFixup = '' - wrapLuaPrograms - ''; - }); + wrappedHello = hello.overrideAttrs(oa: { + propagatedBuildInputs = [ + wrapLua + lua.pkgs.cjson + ]; + postFixup = '' + wrapLuaPrograms + ''; + }); + + luaWithModule = lua.withPackages(ps: [ + ps.lua-cjson + ]); + + golden_LUA_PATHS = { - luaWithModule = lua.withPackages(ps: [ - ps.lua-cjson - ]); + # Looking at lua interpreter 'setpath' code + # for instance https://github.com/lua/lua/blob/69ea087dff1daba25a2000dfb8f1883c17545b7a/loadlib.c#L599 + # replace ";;" by ";LUA_PATH_DEFAULT;" + "5.1" = ";./?.lua;${lua}/share/lua/5.1/?.lua;${lua}/share/lua/5.1/?/init.lua;${lua}/lib/lua/5.1/?.lua;${lua}/lib/lua/5.1/?/init.lua;"; + "5.2" = ";${lua}/share/lua/5.2/?.lua;${lua}/share/lua/5.2/?/init.lua;${lua}/lib/lua/5.2/?.lua;${lua}/lib/lua/5.2/?/init.lua;./?.lua;"; + "5.3" = ";${lua}/share/lua/5.3/?.lua;${lua}/share/lua/5.3/?/init.lua;${lua}/lib/lua/5.3/?.lua;${lua}/lib/lua/5.3/?/init.lua;./?.lua;./?/init.lua;"; + # lua5.4 seems to be smarter about it and dont add the lua separators when nothing left or right + "5.4" = "${lua}/share/lua/5.4/?.lua;${lua}/share/lua/5.4/?/init.lua;${lua}/lib/lua/5.4/?.lua;${lua}/lib/lua/5.4/?/init.lua;./?.lua;./?/init.lua"; + + # luajit versions + "2.0" = ";./?.lua;${lua}/share/luajit-2.0/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua;${lua}/share/lua/5.1/?.lua;${lua}/share/lua/5.1/?/init.lua;"; + "2.1" = ";./?.lua;${lua}/share/luajit-2.1/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua;${lua}/share/lua/5.1/?.lua;${lua}/share/lua/5.1/?/init.lua;"; + }; in pkgs.recurseIntoAttrs ({ - checkAliases = runTest lua { - name = "check-aliases"; + checkInterpreterPatch = let + golden_LUA_PATH = golden_LUA_PATHS.${lib.versions.majorMinor lua.version}; + in + runTest lua { + name = "check-default-lua-path"; command = '' + export LUA_PATH=";;" generated=$(lua -e 'print(package.path)') - golden_LUA_PATH='./share/lua/${lua.luaversion}/?.lua;./?.lua;./?/init.lua' - - assertStringContains "$generated" "$golden_LUA_PATH" + assertStringEqual "$generated" "${golden_LUA_PATH}" ''; }; From 0c9417100fb3172cfa874f0d8641471d91a94c76 Mon Sep 17 00:00:00 2001 From: Matthieu Coudron <886074+teto@users.noreply.github.com> Date: Wed, 6 Mar 2024 22:41:35 +0100 Subject: [PATCH 3/5] lua: update setup-hook to limit LUA_PATH size now that the lua interpreters include working directories with `./?.lua` in LUA_PATH, the current test includes every derivation which quickly becomes unreadable and unpractical. The test is adapted to add a folder only if it can find lua files in the subfolder. --- .../interpreters/lua-5/hooks/setup-hook.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkgs/development/interpreters/lua-5/hooks/setup-hook.sh b/pkgs/development/interpreters/lua-5/hooks/setup-hook.sh index 7b2d2a4d83d86..3041b7f1c3f71 100644 --- a/pkgs/development/interpreters/lua-5/hooks/setup-hook.sh +++ b/pkgs/development/interpreters/lua-5/hooks/setup-hook.sh @@ -13,11 +13,6 @@ nix_debug() { addToLuaSearchPathWithCustomDelimiter() { local varName="$1" local absPattern="$2" - # delete longest match starting from the lua placeholder '?' - local topDir="${absPattern%%\?*}" - - # export only if the folder exists else LUA_PATH/LUA_CPATH grow too large - if [[ ! -d "$topDir" ]]; then return; fi # export only if we haven't already got this dir in the search path if [[ ${!varName-} == *"$absPattern"* ]]; then return; fi @@ -27,7 +22,15 @@ addToLuaSearchPathWithCustomDelimiter() { # allowing relative modules to be used even when there are system modules. if [[ ! -v "${varName}" ]]; then export "${varName}=;;"; fi - export "${varName}=${!varName:+${!varName};}${absPattern}" + # export only if the folder contains lua files + shopt -s globstar + + for _file in ${absPattern/\?/\*\*}; do + export "${varName}=${!varName:+${!varName};}${absPattern}" + shopt -u globstar + return; + done + shopt -u globstar } addToLuaPath() { From 545c14d5e4d65d8ac8aa282bf5720e83695cd1f8 Mon Sep 17 00:00:00 2001 From: Matthieu Coudron <886074+teto@users.noreply.github.com> Date: Wed, 6 Mar 2024 23:32:41 +0100 Subject: [PATCH 4/5] doc/lua: mention the patching of interpreters explain why they are patched in nixpkgs. --- doc/languages-frameworks/lua.section.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/languages-frameworks/lua.section.md b/doc/languages-frameworks/lua.section.md index a6577a56a436c..87cd0c4c90d7b 100644 --- a/doc/languages-frameworks/lua.section.md +++ b/doc/languages-frameworks/lua.section.md @@ -17,6 +17,9 @@ The main package set contains aliases to these package sets, e.g. `luaPackages` refers to `lua5_1.pkgs` and `lua52Packages` to `lua5_2.pkgs`. +Note that nixpkgs patches the non-luajit interpreters to avoid referring to +`/usr` and have `;;` (a [placeholder](https://www.lua.org/manual/5.1/manual.html#pdf-package.path) replaced with the default LUA_PATH) work correctly. + ### Installing Lua and packages {#installing-lua-and-packages} #### Lua environment defined in separate `.nix` file {#lua-environment-defined-in-separate-.nix-file} From 99bb198cddd7c6d39d86736d5dbb27c9b0d21cd4 Mon Sep 17 00:00:00 2001 From: Matthieu Coudron <886074+teto@users.noreply.github.com> Date: Sat, 9 Mar 2024 18:39:17 +0100 Subject: [PATCH 5/5] doc: update release notes for 24.05 with the lua patching --- nixos/doc/manual/release-notes/rl-2405.section.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2405.section.md b/nixos/doc/manual/release-notes/rl-2405.section.md index b4eea551d2703..6c72ef30b87e7 100644 --- a/nixos/doc/manual/release-notes/rl-2405.section.md +++ b/nixos/doc/manual/release-notes/rl-2405.section.md @@ -58,6 +58,9 @@ In addition to numerous new and upgraded packages, this release has the followin } ``` +- lua interpreters default LUA_PATH and LUA_CPATH are not overriden by nixpkgs + anymore, we patch LUA_ROOT instead which is more respectful to upstream. + - Plasma 6 is now available and can be installed with `services.xserver.desktopManager.plasma6.enable = true;`. Plasma 5 will likely be deprecated in the next release (24.11). Note that Plasma 6 runs as Wayland by default, and the X11 session needs to be explicitly selected if necessary. ## New Services {#sec-release-24.05-new-services}