From bbd0655ae828b9f1cf39d891b52aa6506394ef46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= Date: Thu, 29 Feb 2024 13:53:30 +0200 Subject: [PATCH 1/2] add compressDrv and compressDrvWeb *compressDrv* compresses files in a given derivation. *compressDrvWeb* compresses a derivation for a loosely-defined pre-compressed "web server" usage. This intends to replace the `passthru.data-compressed` derivations that have accumulated in nixpkgs with something more reusable. --- .../manual/release-notes/rl-2411.section.md | 2 + pkgs/build-support/compress-drv/default.nix | 63 +++++++++++ pkgs/build-support/compress-drv/test.nix | 42 +++++++ pkgs/build-support/compress-drv/web.nix | 103 ++++++++++++++++++ pkgs/test/default.nix | 2 + pkgs/top-level/all-packages.nix | 4 + 6 files changed, 216 insertions(+) create mode 100644 pkgs/build-support/compress-drv/default.nix create mode 100644 pkgs/build-support/compress-drv/test.nix create mode 100644 pkgs/build-support/compress-drv/web.nix diff --git a/nixos/doc/manual/release-notes/rl-2411.section.md b/nixos/doc/manual/release-notes/rl-2411.section.md index 4a4dba0fdafb4..63287161f084c 100644 --- a/nixos/doc/manual/release-notes/rl-2411.section.md +++ b/nixos/doc/manual/release-notes/rl-2411.section.md @@ -13,6 +13,8 @@ - `authelia` has been upgraded to version 4.38. This version brings several features and improvements which are detailed in the [release blog post](https://www.authelia.com/blog/4.38-release-notes/). This release also deprecates some configuration keys, which are likely to be removed in future version 5.0, but they are still supported and expected to be working in the current version. +- `compressDrv` can compress selected files in a derivation. `compressDrvWeb` compresses files for common web server usage (`.gz` with `zopfli`, `.br` with `brotli`). + - `hardware.display` is a new module implementing workarounds for misbehaving monitors through setting up custom EDID files and forcing kernel/framebuffer modes. diff --git a/pkgs/build-support/compress-drv/default.nix b/pkgs/build-support/compress-drv/default.nix new file mode 100644 index 0000000000000..f5507cb3d3ed0 --- /dev/null +++ b/pkgs/build-support/compress-drv/default.nix @@ -0,0 +1,63 @@ +{ + lib, + xorg, + runCommand, +}: +/** + # compressDrv compresses files in a given derivation. + + ## Inputs: + + `formats` ([String]) + + : List of file extensions to compress. Example: `["txt" "svg" "xml"]`. + + `compressors` (String -> String) + + : Map a desired extension (e.g. `gz`) to a compress program. + + The compressor program that will be executed to get the `COMPRESSOR` extension. + The program should have a single " {}", which will be the replaced with the + target filename. + + Compressor must: + - read symlinks (thus --force is needed to gzip, zstd, xz). + - keep the original file in place (--keep). + + Example: + + ``` + { + xz = "${xz}/bin/xz --force --keep {}"; + } + ``` + + See compressDrvWeb, which is a wrapper on top of compressDrv, for broader use + examples. +*/ +drv: +{ formats, compressors, ... }: +let + validProg = + ext: prog: + let + matches = (builtins.length (builtins.split "\\{}" prog) - 1) / 2; + in + lib.assertMsg ( + matches == 1 + ) "compressor ${ext} needs to have exactly one '{}', found ${builtins.toString matches}"; + mkCmd = + ext: prog: + assert validProg ext prog; + '' + find -L $out -type f -regextype posix-extended -iregex '.*\.(${formatsPipe})' -print0 \ + | xargs -0 -P$NIX_BUILD_CORES -I{} ${prog} + ''; + formatsPipe = builtins.concatStringsSep "|" formats; +in +runCommand "${drv.name}-compressed" { } '' + mkdir $out + (cd $out; ${xorg.lndir}/bin/lndir ${drv}) + + ${lib.concatStringsSep "\n\n" (lib.mapAttrsToList mkCmd compressors)} +'' diff --git a/pkgs/build-support/compress-drv/test.nix b/pkgs/build-support/compress-drv/test.nix new file mode 100644 index 0000000000000..74cc8c4b463b0 --- /dev/null +++ b/pkgs/build-support/compress-drv/test.nix @@ -0,0 +1,42 @@ +{ + gzip, + runCommand, + compressDrv, +}: +let + example = runCommand "sample-drv" { } '' + mkdir $out + echo 42 > $out/1.txt + echo 43 > $out/1.md + touch $out/2.png + ''; + drv = compressDrv example { + formats = [ "txt" ]; + compressors.gz = "${gzip}/bin/gzip --force --keep --fast {}"; + }; + wrapped = compressDrv drv { + formats = [ "md" ]; + compressors.gz = "${gzip}/bin/gzip --force --keep --fast {}"; + }; +in +runCommand "test-compressDrv" { } '' + set -ex + + ls -l ${drv} + test -h ${drv}/1.txt + test -f ${drv}/1.txt.gz + cmp ${drv}/1.txt <(${gzip}/bin/zcat ${drv}/1.txt.gz) + + test -h ${drv}/2.png + test ! -a ${drv}/2.png.gz + + # compressDrv always points to the final file, no matter how many times + # it's been wrapped + cmp <(readlink -e ${drv}/1.txt) <(readlink -e ${wrapped}/1.txt) + + test -f ${wrapped}/1.txt.gz + test -f ${wrapped}/1.md.gz + test ! -f ${drv}/1.md.gz + + mkdir $out +'' diff --git a/pkgs/build-support/compress-drv/web.nix b/pkgs/build-support/compress-drv/web.nix new file mode 100644 index 0000000000000..6ee15e87d13b1 --- /dev/null +++ b/pkgs/build-support/compress-drv/web.nix @@ -0,0 +1,103 @@ +{ + zopfli, + brotli, + compressDrv, +}: +/** + # compressDrvWeb compresses a derivation for common web server use. + + Useful when one wants to pre-compress certain static assets and pass them to + the web server. For example, `pkgs.gamja` creates this derivation: + + /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/ + ├── index.2fd01148.js + ├── index.2fd01148.js.map + ├── index.37aa9a8a.css + ├── index.37aa9a8a.css.map + ├── index.html + └── manifest.webmanifest + + `pkgs.compressDrvWeb pkgs.gamja`: + + /nix/store/f5ryid7zrw2hid7h9kil5g5j29q5r2f7-gamja-1.0.0-beta.9-compressed + ├── index.2fd01148.js -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.2fd01148.js + ├── index.2fd01148.js.br + ├── index.2fd01148.js.gz + ├── index.2fd01148.js.map -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.2fd01148.js.map + ├── index.2fd01148.js.map.br + ├── index.2fd01148.js.map.gz + ├── index.37aa9a8a.css -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.37aa9a8a.css + ├── index.37aa9a8a.css.br + ├── index.37aa9a8a.css.gz + ├── index.37aa9a8a.css.map -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.37aa9a8a.css.map + ├── index.37aa9a8a.css.map.br + ├── index.37aa9a8a.css.map.gz + ├── index.html -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.html + ├── index.html.br + ├── index.html.gz + ├── manifest.webmanifest -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/manifest.webmanifest + ├── manifest.webmanifest.br + └── manifest.webmanifest.gz + + When this `-compressed` directory is passed to a properly configured web + server, it will serve those pre-compressed files: + + $ curl -I -H 'Accept-Encoding: br' https://irc.example.org/ + <...> + content-encoding: br + <...> + + For example, a caddy configuration snippet for gamja to serve + the static assets (JS, CSS files) pre-compressed: + + virtualHosts."irc.example.org".extraConfig = '' + root * ${pkgs.compressDrvWeb pkgs.gamja {}} + file_server browse { + precompressed br gzip + } + ''; + + This feature is also available in nginx via `ngx_brotli` and + `ngx_http_gzip_static_module`. + + ## Inputs + + `formats` ([String]) + + : List of file extensions to compress. Default is common formats that compress + well. The list may be expanded. + + `extraFormats` ([String]) + + : Extra extensions to compress in addition to `formats`. + + `compressors` (String -> String) + + : See parameter `compressors` of compressDrv. +*/ +drv: +{ + formats ? [ + "css" + "js" + "svg" + "ttf" + "eot" + "txt" + "xml" + "map" + "html" + "json" + "webmanifest" + ], + extraFormats ? [ ], + compressors ? { + "gz" = "${zopfli}/bin/zopfli --keep {}"; + "br" = "${brotli}/bin/brotli --keep --no-copy-stat {}"; + }, + ... +}: +compressDrv drv { + formats = formats ++ extraFormats; + compressors = compressors; +} diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix index 3612bbfd1ef83..2d8d1c1c68426 100644 --- a/pkgs/test/default.nix +++ b/pkgs/test/default.nix @@ -106,6 +106,8 @@ with pkgs; cc-multilib-gcc = callPackage ./cc-wrapper/multilib.nix { stdenv = gccMultiStdenv; }; cc-multilib-clang = callPackage ./cc-wrapper/multilib.nix { stdenv = clangMultiStdenv; }; + compress-drv = callPackage ../build-support/compress-drv/test.nix { }; + fetchurl = callPackages ../build-support/fetchurl/tests.nix { }; fetchtorrent = callPackages ../build-support/fetchtorrent/tests.nix { }; fetchpatch = callPackages ../build-support/fetchpatch/tests.nix { }; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 679af721d1751..c7b3ed2174378 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -39878,6 +39878,10 @@ with pkgs; }; }; + compressDrv = callPackage ../build-support/compress-drv { }; + + compressDrvWeb = callPackage ../build-support/compress-drv/web.nix { }; + duti = callPackage ../os-specific/darwin/duti { inherit (darwin.apple_sdk.frameworks) ApplicationServices; }; From d97365e05e847554a7c51146a059a750cc9ceed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= Date: Thu, 29 Feb 2024 14:14:38 +0200 Subject: [PATCH 2/2] gitea.passthru.data-compressed: switch to compressDrvWeb --- pkgs/by-name/gi/gitea/package.nix | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/pkgs/by-name/gi/gitea/package.nix b/pkgs/by-name/gi/gitea/package.nix index 4bb7c0014c0db..81625263a3566 100644 --- a/pkgs/by-name/gi/gitea/package.nix +++ b/pkgs/by-name/gi/gitea/package.nix @@ -6,15 +6,13 @@ , git , bash , coreutils +, compressDrvWeb , gitea , gzip , openssh , pam , sqliteSupport ? true , pamSupport ? stdenv.hostPlatform.isLinux -, runCommand -, brotli -, xorg , nixosTests , buildNpmPackage }: @@ -90,19 +88,7 @@ in buildGoModule rec { ''; passthru = { - data-compressed = runCommand "gitea-data-compressed" { - nativeBuildInputs = [ brotli xorg.lndir ]; - } '' - mkdir -p $out/{options,public,templates} - lndir ${frontend}/public $out/public - lndir ${gitea.data}/options $out/options - lndir ${gitea.data}/templates $out/templates - - # Create static gzip and brotli files - find -L $out -type f -regextype posix-extended -iregex '.*\.(css|html|js|svg|ttf|txt)' \ - -exec gzip --best --keep --force {} ';' \ - -exec brotli --best --keep --no-copy-stat {} ';' - ''; + data-compressed = lib.warn "gitea.passthru.data-compressed is deprecated. Use \"compressDrvWeb gitea.data\"." (compressDrvWeb gitea.data); tests = nixosTests.gitea; };