chromium: fetch src from git instead of using release tarball, {ungoogled-,}chromium,chromedriver: 130.0.6723.116 -> 131.0.6778.69/85

pkgs/applications/networking/browsers/chromium/browser.nix

@@ -1,5 +1,5 @@
 { lib, mkChromiumDerivation
-, channel, chromiumVersionAtLeast
+, chromiumVersionAtLeast
 , enableWideVine, ungoogled
 }:
 
@@ -90,7 +90,7 @@ mkChromiumDerivation (base: rec {
     license = if enableWideVine then lib.licenses.unfree else lib.licenses.bsd3;
     platforms = lib.platforms.linux;
     mainProgram = "chromium";
-    hydraPlatforms = lib.optionals (channel == "stable" || channel == "ungoogled-chromium") ["aarch64-linux" "x86_64-linux"];
+    hydraPlatforms = ["aarch64-linux" "x86_64-linux"];
     timeout = 172800; # 48 hours (increased from the Hydra default of 10h)
   };
 })

pkgs/applications/networking/browsers/chromium/common.nix

@@ -1,15 +1,19 @@
 { stdenv, lib, fetchpatch
-, recompressTarball
+, zstd
+, fetchFromGitiles
+, fetchNpmDeps
 , buildPackages
 , pkgsBuildBuild
 # Channel data:
-, channel, upstream-info
+, upstream-info
 # Helper functions:
 , chromiumVersionAtLeast, versionRange
 
 # Native build inputs:
 , ninja, pkg-config
 , python3, perl
+, nodejs
+, npmHooks
 , which
 , libuuid
 , overrideCC
@@ -145,19 +149,74 @@ let
       else throw "no chromium Rosetta Stone entry for os: ${platform.config}";
   };
 
+  isElectron = packageName == "electron";
+
+  chromiumDeps = lib.mapAttrs (path: args: fetchFromGitiles (removeAttrs args [ "recompress" ] // lib.optionalAttrs args.recompress or false {
+    name = "source.tar.zstd";
+    downloadToTemp = false;
+    passthru.unpack = true;
+    postFetch = ''
+      tar \
+        --use-compress-program="${lib.getExe zstd} -T$NIX_BUILD_CORES" \
+        --sort=name \
+        --mtime="1970-01-01" \
+        --owner=root --group=root \
+        --numeric-owner --mode=go=rX,u+rw,a-s \
+        --remove-files \
+        --directory="$out" \
+        -cf "$TMPDIR/source.zstd" .
+      mv "$TMPDIR/source.zstd" "$out"
+    '';
+  })) upstream-info.DEPS;
+
+  unpackPhaseSnippet = lib.concatStrings (lib.mapAttrsToList (path: dep:
+    (if dep.unpack or false
+      then ''
+        mkdir -p ${path}
+        pushd ${path}
+        unpackFile ${dep}
+        popd
+      ''
+      else ''
+        mkdir -p ${builtins.dirOf path}
+        cp -r ${dep}/. ${path}
+      ''
+    ) + ''
+      chmod u+w -R ${path}
+    '') chromiumDeps);
+
   base = rec {
     pname = "${lib.optionalString ungoogled "ungoogled-"}${packageName}-unwrapped";
     inherit (upstream-info) version;
     inherit packageName buildType buildPath;
 
-    src = recompressTarball { inherit version; inherit (upstream-info) hash; };
+    unpackPhase = ''
+      runHook preUnpack
+
+      ${unpackPhaseSnippet}
+      sourceRoot=src
+
+      runHook postUnpack
+    '';
+
+    npmRoot = "third_party/node";
+    npmDeps = (fetchNpmDeps {
+      src = chromiumDeps."src";
+      sourceRoot = npmRoot;
+      hash = upstream-info.deps.npmHash;
+    }).overrideAttrs (p: {
+      nativeBuildInputs = p.nativeBuildInputs or [ ] ++ [ zstd ];
+    });
 
     nativeBuildInputs = [
       ninja pkg-config
       python3WithPackages perl
       which
       buildPackages.rustc.llvmPackages.bintools
       bison gperf
+    ] ++ lib.optionals (!isElectron) [
+      nodejs
+      npmHooks.npmConfigHook
     ];
 
     depsBuildBuild = [
@@ -317,7 +376,32 @@ let
       })
     ];
 
-    postPatch = ''
+    postPatch =  lib.optionalString (!isElectron) ''
+      ln -s ${./files/gclient_args.gni} build/config/gclient_args.gni
+
+      echo 'LASTCHANGE=${upstream-info.DEPS."src".rev}-refs/heads/master@{#0}' > build/util/LASTCHANGE
+      echo "$SOURCE_DATE_EPOCH" > build/util/LASTCHANGE.committime
+
+      cat << EOF > gpu/config/gpu_lists_version.h
+      /* Generated by lastchange.py, do not edit.*/
+      #ifndef GPU_CONFIG_GPU_LISTS_VERSION_H_
+      #define GPU_CONFIG_GPU_LISTS_VERSION_H_
+      #define GPU_LISTS_VERSION "${upstream-info.DEPS."src".rev}"
+      #endif  // GPU_CONFIG_GPU_LISTS_VERSION_H_
+      EOF
+
+      cat << EOF > skia/ext/skia_commit_hash.h
+      /* Generated by lastchange.py, do not edit.*/
+      #ifndef SKIA_EXT_SKIA_COMMIT_HASH_H_
+      #define SKIA_EXT_SKIA_COMMIT_HASH_H_
+      #define SKIA_COMMIT_HASH "${upstream-info.DEPS."src/third_party/skia".rev}-"
+      #endif  // SKIA_EXT_SKIA_COMMIT_HASH_H_
+      EOF
+
+      echo -n '${upstream-info.DEPS."src/third_party/dawn".rev}' > gpu/webgpu/DAWN_VERSION
+
+      mkdir -p third_party/jdk/current/bin
+    '' + ''
       # Workaround/fix for https://bugs.chromium.org/p/chromium/issues/detail?id=1313361:
       substituteInPlace BUILD.gn \
         --replace '"//infra/orchestrator:orchestrator_all",' ""
@@ -513,6 +597,11 @@ let
     # enable those features in our stable builds.
     preConfigure = ''
       export RUSTC_BOOTSTRAP=1
+    '' + lib.optionalString (!isElectron) ''
+      (
+        cd third_party/node
+        grep patch update_npm_deps | sh
+      )
     '';
 
     configurePhase = ''
@@ -570,11 +659,9 @@ let
     '';
 
     passthru = {
-      updateScript = ./update.py;
-      chromiumDeps = {
-        gn = gnChromium;
-      };
-      inherit recompressTarball;
+      updateScript = ./update.mjs;
+    } // lib.optionalAttrs (!isElectron) {
+      inherit chromiumDeps npmDeps;
     };
   }
   # overwrite `version` with the exact same `version` from the same source,

pkgs/applications/networking/browsers/chromium/default.nix

@@ -10,8 +10,7 @@
 
 # package customization
 # Note: enable* flags should not require full rebuilds (i.e. only affect the wrapper)
-, channel ? "stable"
-, upstream-info ? (import ./upstream-info.nix).${channel}
+, upstream-info ? (lib.importJSON ./info.json).${if !ungoogled then "chromium" else "ungoogled-chromium"}
 , proprietaryCodecs ? true
 , enableWideVine ? false
 , ungoogled ? false # Whether to build chromium or ungoogled-chromium
@@ -46,13 +45,14 @@ let
     inherit stdenv upstream-info;
 
     mkChromiumDerivation = callPackage ./common.nix ({
-      inherit channel chromiumVersionAtLeast versionRange;
+      inherit chromiumVersionAtLeast versionRange;
       inherit proprietaryCodecs
               cupsSupport pulseSupport ungoogled;
       gnChromium = buildPackages.gn.overrideAttrs (oldAttrs: {
-        inherit (upstream-info.deps.gn) version;
+        version = if (upstream-info.deps.gn ? "version") then upstream-info.deps.gn.version else "0";
         src = fetchgit {
-          inherit (upstream-info.deps.gn) url rev hash;
+          url = "https://gn.googlesource.com/gn";
+          inherit (upstream-info.deps.gn) rev hash;
         };
       } // lib.optionalAttrs (chromiumVersionAtLeast "127") {
         # Relax hardening as otherwise gn unstable 2024-06-06 and later fail with:
@@ -65,11 +65,10 @@ let
         # As a work around until gn is updated again, we filter specifically that patch out.
         patches = lib.filter (e: lib.getName e != "LFS64.patch") oldAttrs.patches;
       });
-      recompressTarball = callPackage ./recompress-tarball.nix { inherit chromiumVersionAtLeast; };
     });
 
     browser = callPackage ./browser.nix {
-      inherit channel chromiumVersionAtLeast enableWideVine ungoogled;
+      inherit chromiumVersionAtLeast enableWideVine ungoogled;
     };
 
     # ungoogled-chromium is, contrary to its name, not a build of
@@ -80,8 +79,6 @@ let
     ungoogled-chromium = pkgsBuildBuild.callPackage ./ungoogled.nix {};
   };
 
-  suffix = lib.optionalString (channel != "stable" && channel != "ungoogled-chromium") ("-" + channel);
-
   sandboxExecutableName = chromium.browser.passthru.sandboxExecutableName;
 
   # We want users to be able to enableWideVine without rebuilding all of
@@ -99,7 +96,7 @@ let
 
 in stdenv.mkDerivation {
   pname = lib.optionalString ungoogled "ungoogled-"
-    + "chromium${suffix}";
+    + "chromium";
   inherit (chromium.browser) version;
 
   nativeBuildInputs = [

pkgs/applications/networking/browsers/chromium/depot_tools.py

@@ -0,0 +1,122 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i python -p python3
+"""
+This is a heavily simplified variant of electron's update.py
+for use in ./update.mjs and should not be called manually.
+
+It resolves chromium's DEPS file recursively when called with
+a working depot_tools checkout and a ref to fetch and prints
+the result as JSON to stdout.
+"""
+import base64
+import json
+from typing import Optional
+from urllib.request import urlopen
+
+import sys
+
+if len(sys.argv) != 3:
+    print("""This internal script has been called with the wrong amount of parameters.
+This script is not supposed to be called manually.
+Refer to ./update.mjs instead.""")
+    exit(1)
+
+_, depot_tools_checkout, chromium_version = sys.argv
+
+sys.path.append(depot_tools_checkout)
+import gclient_eval
+import gclient_utils
+
+
+class Repo:
+    fetcher: str
+    args: dict
+
+    def __init__(self) -> None:
+        self.deps: dict = {}
+        self.hash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+
+    def get_deps(self, repo_vars: dict, path: str) -> None:
+        print(
+            "evaluating " + json.dumps(self, default=vars, sort_keys=True),
+            file=sys.stderr,
+        )
+
+        deps_file = self.get_file("DEPS")
+        evaluated = gclient_eval.Parse(deps_file, vars_override=repo_vars, filename="DEPS")
+
+        repo_vars = dict(evaluated.get("vars", {})) | repo_vars
+
+        prefix = f"{path}/" if evaluated.get("use_relative_paths", False) else ""
+
+        self.deps = {
+            prefix + dep_name: repo_from_dep(dep)
+            for dep_name, dep in evaluated.get("deps", {}).items()
+            if (
+                gclient_eval.EvaluateCondition(dep["condition"], repo_vars)
+                if "condition" in dep
+                else True
+            )
+            and repo_from_dep(dep) != None
+        }
+
+        for key in evaluated.get("recursedeps", []):
+            dep_path = prefix + key
+            if dep_path in self.deps and dep_path != "src/third_party/squirrel.mac":
+                self.deps[dep_path].get_deps(repo_vars, dep_path)
+
+    def flatten_repr(self) -> dict:
+        return {"fetcher": self.fetcher, "hash": self.hash, **self.args}
+
+    def flatten(self, path: str) -> dict:
+        out = {path: self.flatten_repr()}
+        for dep_path, dep in self.deps.items():
+            out |= dep.flatten(dep_path)
+        return out
+
+    def get_file(self, filepath: str) -> str:
+        raise NotImplementedError
+
+
+class GitilesRepo(Repo):
+    def __init__(self, url: str, rev: str) -> None:
+        super().__init__()
+        self.fetcher = "fetchFromGitiles"
+        self.args = {
+            "url": url,
+            "rev": rev,
+        }
+
+    def get_file(self, filepath: str) -> str:
+        return base64.b64decode(
+            urlopen(
+                f"{self.args['url']}/+/{self.args['rev']}/{filepath}?format=TEXT"
+            ).read()
+        ).decode("utf-8")
+
+
+def repo_from_dep(dep: dict) -> Optional[Repo]:
+    if "url" in dep:
+        url, rev = gclient_utils.SplitUrlRevision(dep["url"])
+        return GitilesRepo(url, rev)
+    else:
+        # Not a git dependency; skip
+        return None
+
+
+
+chromium = GitilesRepo("https://chromium.googlesource.com/chromium/src.git", chromium_version)
+chromium.get_deps(
+    {
+        **{
+        f"checkout_{platform}": platform == "linux" or platform == "x64" or platform == "arm64" or platform == "arm"
+        for platform in ["ios", "chromeos", "android", "mac", "win", "linux"]
+        },
+        **{
+        f"checkout_{arch}": True
+        for arch in ["x64", "arm64", "arm", "x86", "mips", "mips64"]
+        },
+    },
+    "",
+)
+print(json.dumps(chromium.flatten("src")))

pkgs/applications/networking/browsers/chromium/files/gclient_args.gni

@@ -0,0 +1,12 @@
+build_with_chromium = true
+checkout_android = false
+checkout_android_prebuilts_build_tools = false
+checkout_clang_coverage_tools = false
+checkout_copybara = false
+checkout_ios_webkit = false
+checkout_nacl = false
+checkout_openxr = false
+checkout_src_internal = false
+cros_boards = ""
+cros_boards_with_qemu_images = ""
+generate_location_tags = true