From 9e406d260f0a787c3991491b9a8a1d031eb957d3 Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 10:57:07 -0500 Subject: [PATCH 01/13] Reduce closure size of executable --- CHANGELOG.md | 5 +++++ emanote.cabal | 2 +- flake.nix | 23 +++++++++++++++++++++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffb03e927..82a12b8e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Revision history for emanote +## Unreleased + +- Nix + - Reduce Emanote's Nix runtime closure size + ## 1.0.0.0 (2022-12-04) - UI diff --git a/emanote.cabal b/emanote.cabal index b6e198202..8b4fdc438 100644 --- a/emanote.cabal +++ b/emanote.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: emanote -version: 1.0.0.0 +version: 1.0.1.0 license: AGPL-3.0-only copyright: 2022 Sridhar Ratnakumar maintainer: srid@srid.ca diff --git a/flake.nix b/flake.nix index e9298eac4..428fb8a81 100644 --- a/flake.nix +++ b/flake.nix @@ -27,7 +27,7 @@ ./nix/stork.nix ./nix/tailwind.nix ]; - perSystem = { pkgs, config, ... }: { + perSystem = { pkgs, lib, config, ... }: { haskellProjects.default = { packages.emanote.root = ./.; buildTools = hp: { @@ -52,7 +52,24 @@ emanote = addBuildDepends super.emanote [ config.packages.stork ]; }; }; - packages.default = config.packages.emanote; + packages.default = + let + haskellExeSansDependencyBloat = pkg: get-deps: with pkgs.haskell.lib; + (justStaticExecutables pkg).overrideAttrs (old: rec { + disallowedReferences = get-deps config.haskellProjects.default.haskellPackages; + # Ditch data dependencies that are not needed at runtime. + # cf. https://github.com/NixOS/nixpkgs/pull/204675 + postInstall = (old.postInstall or "") + '' + ${lib.concatStrings (map (e: "remove-references-to -t ${e} $out/bin/emanote\n") disallowedReferences)} + ''; + }); + in + haskellExeSansDependencyBloat config.packages.emanote + (hp: with hp; [ + pandoc + pandoc-types + warp + ]); emanote = { package = config.packages.emanote; sites = { @@ -70,3 +87,5 @@ }; }; } + + From 3982dae8e229ce80ef8a00aa31f25fe1d8780e4f Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 11:05:18 -0500 Subject: [PATCH 02/13] remove newlines --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 428fb8a81..37a2a734b 100644 --- a/flake.nix +++ b/flake.nix @@ -87,5 +87,3 @@ }; }; } - - From 09531b1332accc9f6f43f6ce87b3eef6efd298b5 Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 11:05:24 -0500 Subject: [PATCH 03/13] Make docs use the final emanote exe --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 37a2a734b..6081cbedd 100644 --- a/flake.nix +++ b/flake.nix @@ -71,7 +71,7 @@ warp ]); emanote = { - package = config.packages.emanote; + package = config.packages.default; sites = { "docs" = { layers = [ ./docs ]; From 0c4c24bd209ff1d1be100f37877830eb4b5aa213 Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 11:29:00 -0500 Subject: [PATCH 04/13] Log what got removed --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 6081cbedd..3c07a0941 100644 --- a/flake.nix +++ b/flake.nix @@ -60,6 +60,7 @@ # Ditch data dependencies that are not needed at runtime. # cf. https://github.com/NixOS/nixpkgs/pull/204675 postInstall = (old.postInstall or "") + '' + ${lib.concatStrings (map (e: "echo Removing reference to: ${e}\n") disallowedReferences)} ${lib.concatStrings (map (e: "remove-references-to -t ${e} $out/bin/emanote\n") disallowedReferences)} ''; }); From f027efdb4e2120c8c88e38a79970f80d71c09830 Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 11:30:35 -0500 Subject: [PATCH 05/13] Move the bloat fixing to overrides --- flake.nix | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/flake.nix b/flake.nix index 3c07a0941..2be620a34 100644 --- a/flake.nix +++ b/flake.nix @@ -49,28 +49,28 @@ heist = dontCheck super.heist; # Tests are broken. tailwind = addBuildDepends (unmarkBroken super.tailwind) [ config.packages.tailwind ]; commonmark-extensions = self.callHackage "commonmark-extensions" "0.2.3.2" { }; - emanote = addBuildDepends super.emanote [ config.packages.stork ]; + emanote = + let + haskellExeSansDependencyBloat = pkg: get-deps: with pkgs.haskell.lib; + (justStaticExecutables pkg).overrideAttrs (old: rec { + disallowedReferences = get-deps config.haskellProjects.default.haskellPackages; + # Ditch data dependencies that are not needed at runtime. + # cf. https://github.com/NixOS/nixpkgs/pull/204675 + postInstall = (old.postInstall or "") + '' + ${lib.concatStrings (map (e: "echo Removing reference to: ${e}\n") disallowedReferences)} + ${lib.concatStrings (map (e: "remove-references-to -t ${e} $out/bin/emanote\n") disallowedReferences)} + ''; + }); + in + haskellExeSansDependencyBloat (addBuildDepends super.emanote [ config.packages.stork ]) + (hp: with hp; [ + pandoc + pandoc-types + warp + ]); }; }; - packages.default = - let - haskellExeSansDependencyBloat = pkg: get-deps: with pkgs.haskell.lib; - (justStaticExecutables pkg).overrideAttrs (old: rec { - disallowedReferences = get-deps config.haskellProjects.default.haskellPackages; - # Ditch data dependencies that are not needed at runtime. - # cf. https://github.com/NixOS/nixpkgs/pull/204675 - postInstall = (old.postInstall or "") + '' - ${lib.concatStrings (map (e: "echo Removing reference to: ${e}\n") disallowedReferences)} - ${lib.concatStrings (map (e: "remove-references-to -t ${e} $out/bin/emanote\n") disallowedReferences)} - ''; - }); - in - haskellExeSansDependencyBloat config.packages.emanote - (hp: with hp; [ - pandoc - pandoc-types - warp - ]); + packages.default = config.packages.emanote; emanote = { package = config.packages.default; sites = { From ddde55a359dfaeac2431629f934346aa487f07b7 Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 11:32:59 -0500 Subject: [PATCH 06/13] Use the overriden package set --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 2be620a34..34e726e14 100644 --- a/flake.nix +++ b/flake.nix @@ -53,7 +53,7 @@ let haskellExeSansDependencyBloat = pkg: get-deps: with pkgs.haskell.lib; (justStaticExecutables pkg).overrideAttrs (old: rec { - disallowedReferences = get-deps config.haskellProjects.default.haskellPackages; + disallowedReferences = get-deps self; # Ditch data dependencies that are not needed at runtime. # cf. https://github.com/NixOS/nixpkgs/pull/204675 postInstall = (old.postInstall or "") + '' From 139e2ace3c1330a0408d76b4255a63d395423453 Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 11:36:58 -0500 Subject: [PATCH 07/13] Simplify --- flake.nix | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index 34e726e14..453e3d57e 100644 --- a/flake.nix +++ b/flake.nix @@ -51,9 +51,9 @@ commonmark-extensions = self.callHackage "commonmark-extensions" "0.2.3.2" { }; emanote = let - haskellExeSansDependencyBloat = pkg: get-deps: with pkgs.haskell.lib; + haskellExeSansDependencyBloat = pkg: disallowedReferences: with pkgs.haskell.lib; (justStaticExecutables pkg).overrideAttrs (old: rec { - disallowedReferences = get-deps self; + inherit disallowedReferences; # Ditch data dependencies that are not needed at runtime. # cf. https://github.com/NixOS/nixpkgs/pull/204675 postInstall = (old.postInstall or "") + '' @@ -62,12 +62,11 @@ ''; }); in - haskellExeSansDependencyBloat (addBuildDepends super.emanote [ config.packages.stork ]) - (hp: with hp; [ - pandoc - pandoc-types - warp - ]); + haskellExeSansDependencyBloat (addBuildDepends super.emanote [ config.packages.stork ]) [ + self.pandoc + self.pandoc-types + self.warp + ]; }; }; packages.default = config.packages.emanote; From 9776ebb46add5ae1cbc2505168646f4baa98982f Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 11:37:16 -0500 Subject: [PATCH 08/13] Make general --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 453e3d57e..0c092e84e 100644 --- a/flake.nix +++ b/flake.nix @@ -58,7 +58,7 @@ # cf. https://github.com/NixOS/nixpkgs/pull/204675 postInstall = (old.postInstall or "") + '' ${lib.concatStrings (map (e: "echo Removing reference to: ${e}\n") disallowedReferences)} - ${lib.concatStrings (map (e: "remove-references-to -t ${e} $out/bin/emanote\n") disallowedReferences)} + ${lib.concatStrings (map (e: "remove-references-to -t ${e} $out/bin/*\n") disallowedReferences)} ''; }); in From 25fba1b1a8c016c4e449fb72c5df72ff556b4cec Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 11:39:01 -0500 Subject: [PATCH 09/13] move around --- flake.nix | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/flake.nix b/flake.nix index 0c092e84e..006ecd52d 100644 --- a/flake.nix +++ b/flake.nix @@ -45,29 +45,30 @@ heist-extra heist; ema = inputs.ema + /ema; }; - overrides = self: super: with pkgs.haskell.lib; { - heist = dontCheck super.heist; # Tests are broken. - tailwind = addBuildDepends (unmarkBroken super.tailwind) [ config.packages.tailwind ]; - commonmark-extensions = self.callHackage "commonmark-extensions" "0.2.3.2" { }; - emanote = - let - haskellExeSansDependencyBloat = pkg: disallowedReferences: with pkgs.haskell.lib; - (justStaticExecutables pkg).overrideAttrs (old: rec { - inherit disallowedReferences; - # Ditch data dependencies that are not needed at runtime. - # cf. https://github.com/NixOS/nixpkgs/pull/204675 - postInstall = (old.postInstall or "") + '' - ${lib.concatStrings (map (e: "echo Removing reference to: ${e}\n") disallowedReferences)} - ${lib.concatStrings (map (e: "remove-references-to -t ${e} $out/bin/*\n") disallowedReferences)} - ''; - }); - in - haskellExeSansDependencyBloat (addBuildDepends super.emanote [ config.packages.stork ]) [ - self.pandoc - self.pandoc-types - self.warp - ]; - }; + overrides = with pkgs.haskell.lib; + let + haskellExeSansDependencyBloat = pkg: disallowedReferences: + (justStaticExecutables pkg).overrideAttrs (old: rec { + inherit disallowedReferences; + # Ditch data dependencies that are not needed at runtime. + # cf. https://github.com/NixOS/nixpkgs/pull/204675 + postInstall = (old.postInstall or "") + '' + ${lib.concatStrings (map (e: "echo Removing reference to: ${e}\n") disallowedReferences)} + ${lib.concatStrings (map (e: "remove-references-to -t ${e} $out/bin/*\n") disallowedReferences)} + ''; + }); + in + self: super: { + heist = dontCheck super.heist; # Tests are broken. + tailwind = addBuildDepends (unmarkBroken super.tailwind) [ config.packages.tailwind ]; + commonmark-extensions = self.callHackage "commonmark-extensions" "0.2.3.2" { }; + emanote = + haskellExeSansDependencyBloat (addBuildDepends super.emanote [ config.packages.stork ]) [ + self.pandoc + self.pandoc-types + self.warp + ]; + }; }; packages.default = config.packages.emanote; emanote = { From 051188f4962b7da0c0a42443524d4eea8f49c8c2 Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 11:40:28 -0500 Subject: [PATCH 10/13] Add path-info log to CI --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8e25dde6f..7707e35c2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -30,6 +30,7 @@ jobs: id: build-nix run: | nix build -j auto --no-link + nix path-info -rSh .#default | sort -rhk2 | head - name: Build as docker img id: build run: | From 4d7aac251dfc3f1bfbcf2781a8dd09322bebeeca Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 11:44:04 -0500 Subject: [PATCH 11/13] make the function do one thing only --- flake.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 006ecd52d..b767a647d 100644 --- a/flake.nix +++ b/flake.nix @@ -47,11 +47,13 @@ }; overrides = with pkgs.haskell.lib; let - haskellExeSansDependencyBloat = pkg: disallowedReferences: - (justStaticExecutables pkg).overrideAttrs (old: rec { + # We shouldn't need this after https://github.com/haskell/cabal/pull/8534 + haskellExeSansDependencyBloat = drv: disallowedReferences: + drv.overrideAttrs (old: rec { inherit disallowedReferences; # Ditch data dependencies that are not needed at runtime. # cf. https://github.com/NixOS/nixpkgs/pull/204675 + # cf. https://srid.ca/remove-references-to postInstall = (old.postInstall or "") + '' ${lib.concatStrings (map (e: "echo Removing reference to: ${e}\n") disallowedReferences)} ${lib.concatStrings (map (e: "remove-references-to -t ${e} $out/bin/*\n") disallowedReferences)} @@ -63,7 +65,7 @@ tailwind = addBuildDepends (unmarkBroken super.tailwind) [ config.packages.tailwind ]; commonmark-extensions = self.callHackage "commonmark-extensions" "0.2.3.2" { }; emanote = - haskellExeSansDependencyBloat (addBuildDepends super.emanote [ config.packages.stork ]) [ + haskellExeSansDependencyBloat (justStaticExecutables (addBuildDepends super.emanote [ config.packages.stork ])) [ self.pandoc self.pandoc-types self.warp From b3d01346f422cf1775230ee42045a173e1365f04 Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 11:47:42 -0500 Subject: [PATCH 12/13] refactor using lib.pipe --- flake.nix | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index b767a647d..3571e2df7 100644 --- a/flake.nix +++ b/flake.nix @@ -48,7 +48,7 @@ overrides = with pkgs.haskell.lib; let # We shouldn't need this after https://github.com/haskell/cabal/pull/8534 - haskellExeSansDependencyBloat = drv: disallowedReferences: + haskellExeSansDependencyBloat = disallowedReferences: drv: drv.overrideAttrs (old: rec { inherit disallowedReferences; # Ditch data dependencies that are not needed at runtime. @@ -65,10 +65,14 @@ tailwind = addBuildDepends (unmarkBroken super.tailwind) [ config.packages.tailwind ]; commonmark-extensions = self.callHackage "commonmark-extensions" "0.2.3.2" { }; emanote = - haskellExeSansDependencyBloat (justStaticExecutables (addBuildDepends super.emanote [ config.packages.stork ])) [ - self.pandoc - self.pandoc-types - self.warp + lib.pipe super.emanote [ + (lib.flip addBuildDepends [ config.packages.stork ]) + justStaticExecutables + (haskellExeSansDependencyBloat [ + self.pandoc + self.pandoc-types + self.warp + ]) ]; }; }; From 24c7e5e29a91ec201a48fad1ac028a123b82a402 Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 6 Dec 2022 11:48:29 -0500 Subject: [PATCH 13/13] rename for clarity --- flake.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 3571e2df7..34585ab2e 100644 --- a/flake.nix +++ b/flake.nix @@ -47,8 +47,9 @@ }; overrides = with pkgs.haskell.lib; let + # Remove the given references from drv's executables. # We shouldn't need this after https://github.com/haskell/cabal/pull/8534 - haskellExeSansDependencyBloat = disallowedReferences: drv: + removeReferencesTo = disallowedReferences: drv: drv.overrideAttrs (old: rec { inherit disallowedReferences; # Ditch data dependencies that are not needed at runtime. @@ -68,7 +69,7 @@ lib.pipe super.emanote [ (lib.flip addBuildDepends [ config.packages.stork ]) justStaticExecutables - (haskellExeSansDependencyBloat [ + (removeReferencesTo [ self.pandoc self.pandoc-types self.warp