From cbd09efda26506844dbf4e72a9b8a2d250d8c5a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Tue, 6 Aug 2024 17:39:49 +0200 Subject: [PATCH 01/10] zopfli: set meta.mainProgram --- pkgs/tools/compression/zopfli/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/tools/compression/zopfli/default.nix b/pkgs/tools/compression/zopfli/default.nix index 2c844cffc21b6..8d8813e2eda4f 100644 --- a/pkgs/tools/compression/zopfli/default.nix +++ b/pkgs/tools/compression/zopfli/default.nix @@ -34,6 +34,7 @@ stdenv.mkDerivation rec { ''; platforms = platforms.unix; license = licenses.asl20; + mainProgram = "zopfli"; maintainers = with maintainers; [ bobvanderlinden edef ]; }; } From d3a139bf97854d493821f5390dd8da6c034cac2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Tue, 6 Aug 2024 17:39:58 +0200 Subject: [PATCH 02/10] zstd: set meta.mainProgram --- pkgs/tools/compression/zstd/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/tools/compression/zstd/default.nix b/pkgs/tools/compression/zstd/default.nix index d42866cdda9f9..ba5fb90c1360e 100644 --- a/pkgs/tools/compression/zstd/default.nix +++ b/pkgs/tools/compression/zstd/default.nix @@ -122,7 +122,7 @@ stdenv.mkDerivation rec { homepage = "https://facebook.github.io/zstd/"; changelog = "https://github.com/facebook/zstd/blob/v${version}/CHANGELOG"; license = with licenses; [ bsd3 ]; # Or, at your opinion, GPL-2.0-only. - + mainProgram = "zstd"; platforms = platforms.all; maintainers = with maintainers; [ orivej ]; }; From 7ebbf278326dd25d5ba921cbb705da6c11ff5984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Tue, 6 Aug 2024 16:29:27 +0200 Subject: [PATCH 03/10] compress-drv: sort formats --- pkgs/build-support/compress-drv/web.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/build-support/compress-drv/web.nix b/pkgs/build-support/compress-drv/web.nix index 86ed99e26fa7f..d60a67b0b75de 100644 --- a/pkgs/build-support/compress-drv/web.nix +++ b/pkgs/build-support/compress-drv/web.nix @@ -108,16 +108,16 @@ drv: { formats ? [ "css" + "eot" + "html" "js" + "json" + "map" "svg" "ttf" - "eot" "txt" - "xml" - "map" - "html" - "json" "webmanifest" + "xml" ], extraFormats ? [ ], compressors ? { From 6449d32b0a0f94bd6e6bc057acecb16ce00a0ffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Tue, 6 Aug 2024 16:31:18 +0200 Subject: [PATCH 04/10] compress-drv: add htm, otf formats to default --- pkgs/build-support/compress-drv/web.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/build-support/compress-drv/web.nix b/pkgs/build-support/compress-drv/web.nix index d60a67b0b75de..55942e327cee2 100644 --- a/pkgs/build-support/compress-drv/web.nix +++ b/pkgs/build-support/compress-drv/web.nix @@ -109,10 +109,12 @@ drv: formats ? [ "css" "eot" + "htm" "html" "js" "json" "map" + "otf" "svg" "ttf" "txt" From 9581a2bdcaa97a17f1147deab25e12c3e6d6ccf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Tue, 6 Aug 2024 16:46:49 +0200 Subject: [PATCH 05/10] compress-drv: misc cleanup --- pkgs/build-support/compress-drv/web.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/compress-drv/web.nix b/pkgs/build-support/compress-drv/web.nix index 55942e327cee2..6bf819323886e 100644 --- a/pkgs/build-support/compress-drv/web.nix +++ b/pkgs/build-support/compress-drv/web.nix @@ -1,7 +1,8 @@ { - zopfli, brotli, compressDrv, + lib, + zopfli, }: /** compressDrvWeb compresses a derivation for common web server use. @@ -123,8 +124,8 @@ drv: ], extraFormats ? [ ], compressors ? { - "gz" = "${zopfli}/bin/zopfli --keep {}"; - "br" = "${brotli}/bin/brotli --keep --no-copy-stat {}"; + br = "${lib.getExe brotli} --keep --no-copy-stat {}"; + gz = "${lib.getExe zopfli} --keep {}"; }, }: compressDrv drv { From 0ca4bfa7002996b2207431d791dd66a8dd9acd84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Tue, 6 Aug 2024 16:46:55 +0200 Subject: [PATCH 06/10] compress-drv: add zstd --- pkgs/build-support/compress-drv/web.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/build-support/compress-drv/web.nix b/pkgs/build-support/compress-drv/web.nix index 6bf819323886e..462cd0811bb4a 100644 --- a/pkgs/build-support/compress-drv/web.nix +++ b/pkgs/build-support/compress-drv/web.nix @@ -3,6 +3,7 @@ compressDrv, lib, zopfli, + zstd, }: /** compressDrvWeb compresses a derivation for common web server use. @@ -126,6 +127,10 @@ drv: compressors ? { br = "${lib.getExe brotli} --keep --no-copy-stat {}"; gz = "${lib.getExe zopfli} --keep {}"; + # --force is required to not fail on symlinks + # for details on the compression level see + # https://github.com/NixOS/nixpkgs/pull/332752#issuecomment-2275110390 + zstd = "${lib.getExe zstd} --force --keep --quiet -19 {}"; }, }: compressDrv drv { From 4cc5dee048cb5e5e61db4fe74bf21812add9e028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Tue, 6 Aug 2024 17:58:13 +0200 Subject: [PATCH 07/10] compress-drv: allow passing extra arguments to find This is useful for eg. nextcloud to prevent compressing thousands of later unused files which are actually not used by the web server. --- pkgs/build-support/compress-drv/default.nix | 16 +++++++++++++--- pkgs/build-support/compress-drv/web.nix | 6 ++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/compress-drv/default.nix b/pkgs/build-support/compress-drv/default.nix index 0773ab9e0554e..f916e9e89a081 100644 --- a/pkgs/build-support/compress-drv/default.nix +++ b/pkgs/build-support/compress-drv/default.nix @@ -9,6 +9,12 @@ : List of file extensions to compress. Example: `["txt" "svg" "xml"]`. + `extraFindOperands` (String) + + : Extra command line parameters to pass to the find command. + This can be used to exclude certain files. + For example: `-not -iregex ".*(\/apps\/.*\/l10n\/).*"` + `compressors` ( { ${fileExtension} :: String }) : Map a desired extension (e.g. `gz`) to a compress program. @@ -47,7 +53,11 @@ ::: */ drv: -{ formats, compressors }: +{ + formats, + compressors, + extraFindOperands ? "", +}: let validProg = ext: prog: @@ -61,10 +71,10 @@ let ext: prog: assert validProg ext prog; '' - find -L $out -type f -regextype posix-extended -iregex '.*\.(${formatsPipe})' -print0 \ + find -L $out -type f -regextype posix-extended -iregex '.*\.(${formatsPipe})' ${extraFindOperands} -print0 \ | xargs -0 -P$NIX_BUILD_CORES -I{} ${prog} ''; - formatsPipe = builtins.concatStringsSep "|" formats; + formatsPipe = lib.concatStringsSep "|" formats; in runCommand "${drv.name}-compressed" { } '' mkdir $out diff --git a/pkgs/build-support/compress-drv/web.nix b/pkgs/build-support/compress-drv/web.nix index 462cd0811bb4a..17deb1c0e3bd0 100644 --- a/pkgs/build-support/compress-drv/web.nix +++ b/pkgs/build-support/compress-drv/web.nix @@ -19,6 +19,10 @@ Defaults to common formats that compress well. + `extraFindOperands` (String) + + : See compressDrv for details. + `extraFormats` ([ String ]) : Extra extensions to compress in addition to `formats`. @@ -132,8 +136,10 @@ drv: # https://github.com/NixOS/nixpkgs/pull/332752#issuecomment-2275110390 zstd = "${lib.getExe zstd} --force --keep --quiet -19 {}"; }, + extraFindOperands ? "", }: compressDrv drv { formats = formats ++ extraFormats; compressors = compressors; + inherit extraFindOperands; } From 0654f81d56c56b455274be2553e374df2ca7a458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Tue, 6 Aug 2024 17:59:18 +0200 Subject: [PATCH 08/10] compress-drv: carry pname, version forward This is required to overwrite to use a compressed version of nextcloud in services.nextcloud.package because the module accesses version --- pkgs/build-support/compress-drv/default.nix | 25 ++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/pkgs/build-support/compress-drv/default.nix b/pkgs/build-support/compress-drv/default.nix index f916e9e89a081..b164fa76ada50 100644 --- a/pkgs/build-support/compress-drv/default.nix +++ b/pkgs/build-support/compress-drv/default.nix @@ -76,13 +76,18 @@ let ''; formatsPipe = lib.concatStringsSep "|" formats; in -runCommand "${drv.name}-compressed" { } '' - mkdir $out - - # cannot use lndir here, because it also symlinks directories, - # which we do not need; we only need to symlink files. - (cd ${drv}; find -L -type d -exec mkdir -p $out/{} ';') - (cd ${drv}; find -L -type f -exec ln -s ${drv}/{} $out/{} ';') - - ${lib.concatStringsSep "\n\n" (lib.mapAttrsToList mkCmd compressors)} -'' +runCommand "${drv.name}-compressed" + ( + (lib.optionalAttrs (drv ? pname) { inherit (drv) pname; }) + // (lib.optionalAttrs (drv ? version) { inherit (drv) version; }) + ) + '' + mkdir $out + + # cannot use lndir here, because it also symlinks directories, + # which we do not need; we only need to symlink files. + (cd ${drv}; find -L -type d -exec mkdir -p $out/{} ';') + (cd ${drv}; find -L -type f -exec ln -s ${drv}/{} $out/{} ';') + + ${lib.concatStringsSep "\n\n" (lib.mapAttrsToList mkCmd compressors)} + '' From a1d5c9d11af37c06a1bda4894699c3e5dceeb2b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Tue, 6 Aug 2024 18:01:02 +0200 Subject: [PATCH 09/10] paperless-ngx: change symlink to work with compressDrvWeb Before compressDrvWeb would follow the frontend symlink into the nix store and couldn't write it's file into the directories below the symlink. lndir creates all dirs and then symlinks the actual files into them which allows compressDrvWeb to place the compressed variants next to them. --- pkgs/applications/office/paperless-ngx/default.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkgs/applications/office/paperless-ngx/default.nix b/pkgs/applications/office/paperless-ngx/default.nix index 391569825b183..be9b11f5da015 100644 --- a/pkgs/applications/office/paperless-ngx/default.nix +++ b/pkgs/applications/office/paperless-ngx/default.nix @@ -21,6 +21,7 @@ , pango , pkg-config , nltk-data +, xorg }: let @@ -121,6 +122,7 @@ python.pkgs.buildPythonApplication rec { nativeBuildInputs = [ gettext + xorg.lndir ]; propagatedBuildInputs = with python.pkgs; [ @@ -194,9 +196,9 @@ python.pkgs.buildPythonApplication rec { in '' runHook preInstall - mkdir -p $out/lib/paperless-ngx + mkdir -p $out/lib/paperless-ngx/static/frontend cp -r {src,static,LICENSE,gunicorn.conf.py} $out/lib/paperless-ngx - ln -s ${frontend}/lib/paperless-ui/frontend $out/lib/paperless-ngx/static/ + lndir -silent ${frontend}/lib/paperless-ui/frontend $out/lib/paperless-ngx/static/frontend chmod +x $out/lib/paperless-ngx/src/manage.py makeWrapper $out/lib/paperless-ngx/src/manage.py $out/bin/paperless-ngx \ --prefix PYTHONPATH : "${pythonPath}" \ From 0505523e987e9c2e2ad4a7690583f86c5485cabf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Mon, 19 Aug 2024 13:19:27 +0200 Subject: [PATCH 10/10] compress-drv: correct comment --- pkgs/build-support/compress-drv/default.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkgs/build-support/compress-drv/default.nix b/pkgs/build-support/compress-drv/default.nix index b164fa76ada50..f1ace8c7a7aa2 100644 --- a/pkgs/build-support/compress-drv/default.nix +++ b/pkgs/build-support/compress-drv/default.nix @@ -84,8 +84,7 @@ runCommand "${drv.name}-compressed" '' mkdir $out - # cannot use lndir here, because it also symlinks directories, - # which we do not need; we only need to symlink files. + # cannot use lndir here, because it stop recursing at symlinks that point to directories (cd ${drv}; find -L -type d -exec mkdir -p $out/{} ';') (cd ${drv}; find -L -type f -exec ln -s ${drv}/{} $out/{} ';')