WIP: Switch to ghc (js) 9.12 via nixpkgs

default.nix

@@ -7,6 +7,7 @@
   }
 , reflex-platform-func ? import ./dep/reflex-platform
 , useGHC810 ? true # false if one wants to use ghc 8.6.5
+, zopfli ? true
 }:
 let
   reflex-platform = getReflexPlatform { inherit system; };
@@ -38,7 +39,8 @@ let
   # Development environments for obelisk packages.
   ghcObeliskEnvs = pkgs.lib.mapAttrs (n: v: reflex-platform.workOn ghcObelisk v) ghcObelisk;
 
-  inherit (import ./lib/asset/assets.nix { inherit nixpkgs; }) mkAssets;
+  assets = import ./lib/asset/assets.nix { inherit nixpkgs; };
+  mkAssets = assets.mkAssetsWith (if zopfli then assets.defaultEncodings else assets.gzipEncodings);
 
   haskellLib = pkgs.haskell.lib;
 
@@ -91,7 +93,7 @@ in rec {
         ln -s "$dir/all.unminified.js" "$dir/all.js"
       '' else ''
         # NOTE: "--error_format JSON" avoids closurecompiler crashes when trying to report errors.
-        '${pkgs.closurecompiler}/bin/closure-compiler' --error_format JSON ${if externs == null then "" else "--externs '${externs}'"} --externs '${reflex-platform.ghcjsExternsJs}' -O '${optimizationLevel}' --jscomp_warning=checkVars --warning_level=QUIET --create_source_map="$dir/all.js.map" --source_map_format=V3 --js_output_file="$dir/all.js" "$dir/all.unminified.js"
+        '${pkgs.closurecompiler}/bin/closure-compiler' --language_in UNSTABLE --error_format JSON ${if externs == null then "" else "--externs '${externs}'"} --externs '${reflex-platform.ghcjsExternsJs}' -O '${optimizationLevel}' --jscomp_warning=checkVars --warning_level=QUIET --create_source_map="$dir/all.js.map" --source_map_format=V3 --js_output_file="$dir/all.js" "$dir/all.unminified.js"
         echo '//# sourceMappingURL=all.js.map' >> "$dir/all.js"
       ''}
     done
@@ -210,7 +212,7 @@ in rec {
 
   serverExe = backend: frontend: assets: optimizationLevel: externjs: version:
     let
-      exeBackend = if profiling then backend else haskellLib.justStaticExecutables backend;
+      exeBackend = lib.getBin backend;
       exeFrontend = compressedJs frontend optimizationLevel externjs;
       exeFrontendAssets = mkAssets exeFrontend;
       exeAssets = mkAssets assets;
@@ -260,7 +262,9 @@ in rec {
             , shellToolOverrides ? _: _: {}
             , withHoogle ? false # Setting this to `true` makes shell reloading far slower
             , externjs ? null
-            , __closureCompilerOptimizationLevel ? "ADVANCED" # Set this to `null` to skip the closure-compiler step
+            # TODO: Need to figure if we can reset this to ADVANCED or figure out better compression via
+            # https://blog.haskell.org/case-study-foreign-integration-js-browser/
+            , __closureCompilerOptimizationLevel ? "SIMPLE" # Set this to `null` to skip the closure-compiler step
             , __withGhcide ? false
             , __deprecated ? {}
             }:
@@ -293,7 +297,20 @@ in rec {
                 combinedPackages = self.predefinedPackages // self.userSettings.packages // self.shellPackages;
                 projectOverrides = self': super': {
                   ${self.staticName} = haskellLib.dontHaddock (self'.callCabal2nix self.staticName self.processedStatic.haskellManifest {});
-                  ${self.backendName} = haskellLib.addBuildDepend super'.${self.backendName} self'.obelisk-run;
+                  ${self.backendName} = lib.pipe super'.${self.backendName} [
+                    (haskellLib.compose.addBuildDepend self'.obelisk-run)
+                    haskellLib.enableSeparateBinOutput
+                    (haskellLib.compose.overrideCabal
+                      (old: {
+                        # Newer nixpkgs version make sure that static executables don’t pull in GHC via their closure.
+                        # This remove-references-to fixes that for normal obelisk backends.
+                        postInstall = ''
+                          ${old.postInstall or ""}
+                          ${lib.getExe pkgs.removeReferencesTo} -t ${obelisk.snap-server} "$bin/bin/backend"
+                        '';
+                      })
+                    )
+                  ];
                 };
                 totalOverrides = lib.composeExtensions self.projectOverrides self.userSettings.overrides;
                 privateConfigDirs = ["config/backend"];

dep/reflex-platform/github.json

@@ -1,8 +1,8 @@
 {
   "owner": "reflex-frp",
   "repo": "reflex-platform",
-  "branch": "release/1.2.0.0",
+  "branch": "maralorn/new-backend",
   "private": false,
-  "rev": "f231e2425ac92339b8491cdd970930d63d9ad1ad",
-  "sha256": "0b042x423p04shhidni08f47ydgpfj0rpqhb0m6gj2lg8b3s9l8k"
+  "rev": "88792de686f9f0426e6e01575cf800b6d9f71cda",
+  "sha256": "1nvk11qfi1rzffpvmn4nrah2a75liis2is8pbkydj7sb352r91l5"
 }

haskell-overlays/misc-deps.nix

@@ -5,112 +5,30 @@ self: super:
 
 let
   pkgs = self.callPackage ({ pkgs }: pkgs) { };
-  haskellLib = pkgs.haskell.lib;
-  mkVersionset = v: p: q: if __useNewerCompiler then q else p;
+  haskellLib = pkgs.haskell.lib.compose;
+  inherit (haskellLib) dontCheck doJailbreak;
 in
 
-rec {
-  resolv = haskellLib.dontCheck (self.callHackage "resolv" "0.1.2.0" {});
-  cabal-install = haskellLib.doJailbreak ((self.callHackage "cabal-install" "3.4.1.0" {}).overrideScope (self: super: { Cabal = self.Cabal_3_4_1_0; }));
-
-  # hpack requires cabal >= 3.0 but the ghc865 package set builds it with 2.4 by default
-  hpack = super.hpack.overrideScope (self: super: { Cabal = self.Cabal_3_2_1_0; });
-
-  # These versions work with both the ghc865 and ghc8107 package sets
-  git = self.callCabal2nix "git" (hackGet ../dep/hs-git) { };
-  http-link-header = haskellLib.doJailbreak super.http-link-header;
-  universe-base-810 = haskellLib.doJailbreak (self.callHackage "universe-base" "1.1" {});
-  universe-dependent-sum-810 = self.callHackage "universe-dependent-sum" "1.3" {};
-  universe-some-810 = haskellLib.dontHaddock (haskellLib.appendBuildFlags (haskellLib.doJailbreak (self.callHackage "universe-some" "1.2" { })) [ "--ghc-option=-Wno-inferred-safe-imports" "--ghc-option=-Wno-missing-safe-haskell-mode" ]);
-
-  stylish-haskell = null; # FIXME
-  beam-migrate = self.callHackageDirect {
-    pkg = "beam-migrate";
-    ver = "0.5.1.2";
-    sha256 = "sha256-vEv/6DCvuEq6cmxoPKxZNIm5g6YUgrdvAK4YAoZQr/E=";
-  } {};
-
-  universe-810 = self.callHackage "universe" "1.2" {};
-  universe-instances-extended-810 = self.callHackage "universe-instances-extended" "1.1.1" {};
-  universe-reverse-instances-810 = self.callHackage "universe-reverse-instances" "1.1" {};
-
-  # We use our fork of hnix which has some compatibility patches on top of 0.12 from hackage
-  hnix = haskellLib.doJailbreak (haskellLib.dontHaddock (haskellLib.dontCheck (self.callCabal2nix "hnix" (hackGet ../dep/hnix) {})));
-
-  universe-86 = haskellLib.dontCheck (self.callHackage "universe" "1.2" {});
-  universe-instances-extended-86 = self.callHackage "universe-instances-extended" "1.1.1" {};
-  hnix-86 = haskellLib.dontCheck super.hnix;
-
-  universe = mkVersionset __useNewerCompiler universe-86 universe-810;
-  universe-instances-extended = mkVersionset __useNewerCompiler universe-instances-extended-86 universe-instances-extended-810;
-  universe-reverse-instances = mkVersionset __useNewerCompiler super.universe-reverse-instances universe-reverse-instances-810;
-  universe-base = haskellLib.dontCheck (mkVersionset __useNewerCompiler super.universe-base universe-base-810);
-  universe-dependent-sum = mkVersionset __useNewerCompiler super.universe-dependent-sum universe-dependent-sum-810;
-  universe-some-86 = self.callHackage "universe-some" "1.2" {};
-  universe-some = mkVersionset __useNewerCompiler universe-some-86 universe-some-810;
-
-  regex-base = self.callHackage "regex-base" "0.94.0.0" { };
-  regex-posix = self.callHackage "regex-posix" "0.96.0.0" { };
-  regex-tdfa = self.callHackage "regex-tdfa" "1.3.1.0" { };
-  test-framework = haskellLib.dontCheck (self.callHackage "test-framework" "0.8.2.0" { });
-
-  hnix-store-core = haskellLib.doJailbreak (haskellLib.dontCheck super.hnix-store-core);
-  hnix-store = haskellLib.doJailbreak (haskellLib.dontCheck super.hnix-store);
-
-  # https://github.com/haskell/hackage-security/issues/247
-  hackage-security = haskellLib.dontCheck super.hackage-security; # only tests use aeson and are not compat with 1.5;
-  heist = haskellLib.dontCheck (self.callHackage "heist" "1.1.1.0" {});
-  aeson-gadt-th = haskellLib.doJailbreak super.aeson-gadt-th; # requires aeson 1.5 for ghc8.10 support?
-  deriving-compat = self.callHackage "deriving-compat" "0.6" { };
-  http-api-data = haskellLib.doJailbreak super.http-api-data;
-  nix-derivation = haskellLib.doJailbreak super.nix-derivation;
-  algebraic-graphs = haskellLib.doJailbreak super.algebraic-graphs;
-  snap = haskellLib.doJailbreak super.snap;
-
-  snap-core = self.callHackage "snap-core" "1.0.5.0" {};
-  snap-server = haskellLib.doJailbreak super.snap-server;
-
-  logging-effect = self.callHackageDirect {
-    pkg = "logging-effect";
-    ver = "1.4.0";
-    sha256 = "0xxw21h406xybpj04hpx8vjfdbszv5ymli4vll88lssk6jpc3pfg";
-  } {};
-
-  resourcet = self.callHackage "resourcet" "1.2.4.2" { };
-  unliftio-core = self.callHackage "unliftio-core" "0.2.0.1" { };
-  shelly = self.callHackage "shelly" "1.9.0" { };
-  # version >= 0.2.5.2 has a Cabal version of 3.0, which nix doesn't like
-  vector-binary-instances = self.callHackage "vector-binary-instances" "0.2.5.1" {};
-  binary-instances = self.callHackage "binary-instances" "1.0.2" {};
-  modern-uri = haskellLib.doJailbreak super.modern-uri;
-  monad-logger = self.callHackage "monad-logger" "0.3.36" { };
-  neat-interpolation = haskellLib.doJailbreak super.neat-interpolation;
-  nix-thunk = (import ../dep/nix-thunk { inherit pkgs; }).makeRunnableNixThunk (haskellLib.doJailbreak (self.callCabal2nix "nix-thunk" (hackGet ../dep/nix-thunk) { }));
-  cli-extras = haskellLib.doJailbreak (self.callCabal2nix "cli-extras" (hackGet ../dep/cli-extras) { });
-  cli-git = haskellLib.doJailbreak (haskellLib.overrideCabal (self.callCabal2nix "cli-git" (hackGet ../dep/cli-git) { }) {
-    librarySystemDepends = with pkgs; [
-      gitMinimal
-    ];
-  });
-  cli-nix = haskellLib.doJailbreak (haskellLib.overrideCabal (self.callCabal2nix "cli-nix" (hackGet ../dep/cli-nix) { }) {
-    librarySystemDepends = with pkgs; [
-      gitMinimal
-      nix
-      nix-prefetch-git
-    ];
-  });
-
-  haddock-library = haskellLib.doJailbreak (self.callHackage "haddock-library" "1.10.0" {});
-  io-streams = self.callHackage "io-streams" "1.5.2.1" {};
-  io-streams-haproxy = haskellLib.doJailbreak super.io-streams-haproxy;
-
-  semialign-indexed = haskellLib.doJailbreak super.semialign-indexed;
-  cborg = haskellLib.dontCheck super.cborg;
-  github = self.callHackage "github" "0.28" {};
-  http2 = haskellLib.dontCheck super.http2;
-  http-streams = haskellLib.dontCheck super.http-streams;
-  ghc-lib-parser = super.ghc-lib-parser_8_10_7_20220219;
-  ghc-lib-parser-ex = super.ghc-lib-parser-ex_8_10_0_24;
-  aeson-qq = self.callHackage "aeson-qq" "0.8.4" {};
-  fsnotify = haskellLib.dontCheck super.fsnotify;
+{
+  heist = dontCheck super.heist;
+  crypton = dontCheck super.crypton;
+  cryptonite = dontCheck super.cryptonite;
+  hashing = doJailbreak super.hashing;
+  lens-family-th = doJailbreak super.lens-family-th;
+  repline = doJailbreak super.repline;
+  string-interpolate = doJailbreak super.string-interpolate;
+  interpolate = dontCheck super.interpolate;
+  logging-effect = doJailbreak super.logging-effect;
+  hpack = doJailbreak super.hpack;
+  nix-derivation = doJailbreak super.nix-derivation;
+  brick = doJailbreak super.brick;
+  cli-extras = doJailbreak super.cli-extras;
+  http-link-header = self.callHackageDirect {
+      pkg = "http-link-header";
+      ver = "1.2.3";
+      sha256 = "sha256-0oopfwTs3lHvt6D5R/ZJ5FJt/OmVDkwRsRtYYAty87E=";
+    } {}; # build error is fixed upstream
+  cli-nix = doJailbreak super.cli-nix;
+  cli-git = doJailbreak super.cli-git;
+  nix-thunk = doJailbreak super.nix-thunk;
 }

haskell-overlays/tighten-ob-exes.nix

@@ -16,7 +16,7 @@ in
   # Dynamic linking with split objects dramatically increases startup time (about
   # 0.5 seconds on a decent machine with SSD), so we do `justStaticExecutables`.
   obelisk-command = haskellLib.overrideCabal
-    (haskellLib.generateOptparseApplicativeCompletion "ob"
+    (self.generateOptparseApplicativeCompletions ["ob"]
       (haskellLib.justStaticExecutables super.obelisk-command))
     (drv: {
       buildTools = (drv.buildTools or []) ++ [ pkgs.buildPackages.makeWrapper ];

lib/asset/assets.nix

@@ -262,9 +262,7 @@ mkPath = path:
 # build a DirEntry for dirToPath with the various encodings of the asset for dirToPath to build into a final directory tree.
 mkAsset = encodings: {name, value}:
   let hashD = hashFileD value.path;
-      nameWithHash = "${delay "1" (builtins.trace
-        "importing IFD asset hash"
-        (builtins.readFile hashD))}-${name}";
+      nameWithHash = "${delay "1" (builtins.readFile hashD)}-${name}";
   in {
     toDo = hashD;
     res = delay "2" {

lib/asset/manifest/obelisk-asset-manifest.cabal

@@ -20,44 +20,41 @@ library
     , containers
     , deepseq
     , directory
-    , SHA
     , filepath
+    , SHA
     , template-haskell
     , text
     , th-abstraction >= 0.4
     , transformers
-    , unix-compat
     , vector
   exposed-modules:
-    Obelisk.Asset.Cabal
     Obelisk.Asset.Gather
-    Obelisk.Asset.Promoted
-    Obelisk.Asset.Symlink
     Obelisk.Asset.TH
 
   other-extensions: TemplateHaskell
   ghc-options:
     -Wall -Werror -Wredundant-constraints -Wincomplete-uni-patterns -Wincomplete-record-updates -O2
     -fno-warn-unused-do-bind -funbox-strict-fields -fprof-auto-calls
 
-  if arch(javascript)
-    buildable: False
-
-executable obelisk-asset-manifest-generate
-  default-language: Haskell2010
-  hs-source-dirs: src-bin
-  main-is: generate.hs
-  build-depends:
-      base
-    , obelisk-asset-manifest
-    , text
-
 executable obelisk-asset-th-generate
   default-language: Haskell2010
   hs-source-dirs: src-bin
   main-is: static-th.hs
+  other-modules:
+    Obelisk.Asset.Cabal
+    Obelisk.Asset.Promoted
+    Obelisk.Asset.Symlink
   build-depends:
       base
+    , containers
+    , directory
+    , filepath
     , obelisk-asset-manifest
+    , template-haskell
+    , th-abstraction >= 0.4
     , filepath
     , text
+    , transformers
+    , unix-compat
+  if arch(javascript)
+    buildable: False

lib/command/src/Obelisk/Command/Run.hs

@@ -61,7 +61,7 @@ import Distribution.Fields.ParseResult (runParseResult)
 import Distribution.PackageDescription.Parsec (parseGenericPackageDescription, runParseResult)
 #endif
 import Distribution.Pretty (prettyShow)
-import Distribution.Simple.Compiler (PackageDB (GlobalPackageDB))
+import Distribution.Simple.Compiler (PackageDB, PackageDBX (GlobalPackageDB))
 import Distribution.Simple.Configure (configCompilerEx, getInstalledPackages)
 import Distribution.Simple.PackageIndex (InstalledPackageIndex, lookupDependency)
 import Distribution.Simple.Program.Db (defaultProgramDb)
@@ -234,9 +234,9 @@ run certDir portOverride root interpretPaths = do
       , "Backend.backend"
       , "Frontend.frontend"
       , "(Obelisk.Run.runServeAsset " ++ show assets ++ ")"
-      , ") { Obelisk.Run._runApp_backendPort =", show freePort
-      ,   ", Obelisk.Run._runApp_forceFrontendPort =", show portOverride
-      ,   ", Obelisk.Run._runApp_tlsCertDirectory =", show certDir
+      , ") { _runApp_backendPort =", show freePort
+      ,   ", _runApp_forceFrontendPort =", show portOverride
+      ,   ", _runApp_tlsCertDirectory =", show certDir
       , "}"
       ]
 
@@ -568,7 +568,7 @@ loadPackageIndex packageInfos root = do
   ghcPkgPath <- getPathInNixEnvironment "bash -c 'type -p ghc-pkg'"
   (compiler, _platform, programDb) <- liftIO
     $ configCompilerEx (Just GHC) (Just ghcPath) (Just ghcPkgPath) defaultProgramDb Verbosity.silent
-  liftIO $ getInstalledPackages Verbosity.silent compiler [GlobalPackageDB] programDb
+  liftIO $ getInstalledPackages Verbosity.silent compiler Nothing [GlobalPackageDB] programDb
   where
     getPathInNixEnvironment cmd = do
       path <- readProcessAndLogStderr Debug =<< mkObNixShellProc root False True (packageInfoToNamePathMap packageInfos) "ghc" (Just cmd)

lib/route/src/Obelisk/Route/Frontend.hs

@@ -82,6 +82,7 @@ import Data.Dependent.Sum (DSum (..))
 import Data.Functor.Compose
 import Data.Functor.Misc
 import Data.GADT.Compare
+import Data.Kind (Type)
 import qualified Data.List as L
 import Data.Map as Map (Map, lookup)
 import Data.Maybe (fromMaybe)
@@ -118,7 +119,7 @@ instance Monad m => Routed t r (RoutedT t r m) where
 
 instance (Monad m, Routed t r m) => Routed t r (ReaderT r' m)
 
-newtype RoutedT t r m a = RoutedT { unRoutedT :: ReaderT (Dynamic t r) m a }
+newtype RoutedT (t :: Type) r m a = RoutedT { unRoutedT :: ReaderT (Dynamic t r) m a }
   deriving
     ( Functor
     , Applicative

skeleton/frontend/frontend.cabal

@@ -43,8 +43,9 @@ executable frontend
   if impl(ghc >= 8.8)
     ghc-options:
                -Wmissing-deriving-strategies
-  if impl(ghcjs)
-    ghc-options: -dedupe
-    cpp-options: -DGHCJS_BROWSER
+  if arch(javascript)
+    -- Workaround for a runtime error ("HEAP8 not found"). Only necessary until
+    -- https://gitlab.haskell.org/ghc/ghc/-/merge_requests/14362 is resolved.
+    ld-options: -sEXPORTED_RUNTIME_METHODS=HEAP8 
   if os(darwin)
     ghc-options: -dynamic