From 46da6473780f2b3914a5ad258be2595dd3451ef9 Mon Sep 17 00:00:00 2001 From: Konstantin Alekseev Date: Sat, 1 Jun 2024 17:13:03 +0300 Subject: [PATCH 1/2] playwright: apply nixfmt --- pkgs/development/web/playwright/driver.nix | 221 +++++++++++---------- 1 file changed, 117 insertions(+), 104 deletions(-) diff --git a/pkgs/development/web/playwright/driver.nix b/pkgs/development/web/playwright/driver.nix index a4fe8272be4c9..abf2f43c0948a 100644 --- a/pkgs/development/web/playwright/driver.nix +++ b/pkgs/development/web/playwright/driver.nix @@ -1,85 +1,94 @@ -{ lib -, stdenv -, chromium -, ffmpeg -, git -, jq -, nodejs -, fetchurl -, makeFontsConf -, makeWrapper -, runCommand -, unzip -, cacert +{ + lib, + stdenv, + chromium, + ffmpeg, + git, + jq, + nodejs, + fetchurl, + makeFontsConf, + makeWrapper, + runCommand, + unzip, + cacert, }: let inherit (stdenv.hostPlatform) system; throwSystem = throw "Unsupported system: ${system}"; - driver = stdenv.mkDerivation (finalAttrs: + driver = stdenv.mkDerivation ( + finalAttrs: let - suffix = { - x86_64-linux = "linux"; - aarch64-linux = "linux-arm64"; - x86_64-darwin = "mac"; - aarch64-darwin = "mac-arm64"; - }.${system} or throwSystem; + suffix = + { + x86_64-linux = "linux"; + aarch64-linux = "linux-arm64"; + x86_64-darwin = "mac"; + aarch64-darwin = "mac-arm64"; + } + .${system} or throwSystem; filename = "playwright-${finalAttrs.version}-${suffix}.zip"; in { - pname = "playwright-driver"; - # run ./pkgs/development/python-modules/playwright/update.sh to update - version = "1.40.0"; - - src = fetchurl { - url = "https://playwright.azureedge.net/builds/driver/${filename}"; - sha256 = { - x86_64-linux = "0y9n23r4yfcgm4a50rfgicl91vrllak0d8h26yagh6h8hl0r3nhh"; - aarch64-linux = "0zd456klidi4sg7wahfrdbs2bwiq3q6ngxd4iv3vi9f9w9nq2p2k"; - x86_64-darwin = "0yaiwg9821w9nszzkrp5skzf5792nahvfqnr4axk84dcngslxvmk"; - aarch64-darwin = "1b1jmv6l97ss8c4sc3n1xckn05fpq3fihjbjxr2qz6i9dsy3xj57"; - }.${system} or throwSystem; - }; - - sourceRoot = "."; - - nativeBuildInputs = [ unzip ]; - - postPatch = '' - # Use Nix's NodeJS instead of the bundled one. - substituteInPlace playwright.sh --replace '"$SCRIPT_PATH/node"' '"${nodejs}/bin/node"' - rm node - - # Hard-code the script path to $out directory to avoid a dependency on coreutils - substituteInPlace playwright.sh \ - --replace 'SCRIPT_PATH="$(cd "$(dirname "$0")" ; pwd -P)"' "SCRIPT_PATH=$out" - - patchShebangs playwright.sh package/bin/*.sh - ''; - - installPhase = '' - runHook preInstall - - mkdir -p $out/bin - mv playwright.sh $out/bin/playwright - mv package $out/ - - runHook postInstall - ''; - - passthru = { - inherit filename; - browsers = { - x86_64-linux = browsers-linux { }; - aarch64-linux = browsers-linux { }; - x86_64-darwin = browsers-mac; - aarch64-darwin = browsers-mac; - }.${system} or throwSystem; - browsers-chromium = browsers-linux {}; - }; - meta.mainProgram = "playwright"; - }); + pname = "playwright-driver"; + # run ./pkgs/development/python-modules/playwright/update.sh to update + version = "1.40.0"; + + src = fetchurl { + url = "https://playwright.azureedge.net/builds/driver/${filename}"; + sha256 = + { + x86_64-linux = "0y9n23r4yfcgm4a50rfgicl91vrllak0d8h26yagh6h8hl0r3nhh"; + aarch64-linux = "0zd456klidi4sg7wahfrdbs2bwiq3q6ngxd4iv3vi9f9w9nq2p2k"; + x86_64-darwin = "0yaiwg9821w9nszzkrp5skzf5792nahvfqnr4axk84dcngslxvmk"; + aarch64-darwin = "1b1jmv6l97ss8c4sc3n1xckn05fpq3fihjbjxr2qz6i9dsy3xj57"; + } + .${system} or throwSystem; + }; + + sourceRoot = "."; + + nativeBuildInputs = [ unzip ]; + + postPatch = '' + # Use Nix's NodeJS instead of the bundled one. + substituteInPlace playwright.sh --replace '"$SCRIPT_PATH/node"' '"${nodejs}/bin/node"' + rm node + + # Hard-code the script path to $out directory to avoid a dependency on coreutils + substituteInPlace playwright.sh \ + --replace 'SCRIPT_PATH="$(cd "$(dirname "$0")" ; pwd -P)"' "SCRIPT_PATH=$out" + + patchShebangs playwright.sh package/bin/*.sh + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out/bin + mv playwright.sh $out/bin/playwright + mv package $out/ + + runHook postInstall + ''; + + passthru = { + inherit filename; + browsers = + { + x86_64-linux = browsers-linux { }; + aarch64-linux = browsers-linux { }; + x86_64-darwin = browsers-mac; + aarch64-darwin = browsers-mac; + } + .${system} or throwSystem; + browsers-chromium = browsers-linux { }; + }; + meta.mainProgram = "playwright"; + } + ); browsers-mac = stdenv.mkDerivation { pname = "playwright-browsers"; @@ -87,9 +96,7 @@ let dontUnpack = true; - nativeBuildInputs = [ - cacert - ]; + nativeBuildInputs = [ cacert ]; installPhase = '' runHook preInstall @@ -104,33 +111,39 @@ let meta.platforms = lib.platforms.darwin; }; - browsers-linux = { withChromium ? true }: let - fontconfig = makeFontsConf { - fontDirectories = []; - }; - in - runCommand ("playwright-browsers" - + lib.optionalString withChromium "-chromium") - { - nativeBuildInputs = [ - makeWrapper - jq - ]; - } ('' - BROWSERS_JSON=${driver}/package/browsers.json - '' + lib.optionalString withChromium '' - CHROMIUM_REVISION=$(jq -r '.browsers[] | select(.name == "chromium").revision' $BROWSERS_JSON) - mkdir -p $out/chromium-$CHROMIUM_REVISION/chrome-linux - - # See here for the Chrome options: - # https://github.com/NixOS/nixpkgs/issues/136207#issuecomment-908637738 - makeWrapper ${chromium}/bin/chromium $out/chromium-$CHROMIUM_REVISION/chrome-linux/chrome \ - --set SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt \ - --set FONTCONFIG_FILE ${fontconfig} - '' + '' - FFMPEG_REVISION=$(jq -r '.browsers[] | select(.name == "ffmpeg").revision' $BROWSERS_JSON) - mkdir -p $out/ffmpeg-$FFMPEG_REVISION - ln -s ${ffmpeg}/bin/ffmpeg $out/ffmpeg-$FFMPEG_REVISION/ffmpeg-linux - ''); + browsers-linux = + { + withChromium ? true, + }: + let + fontconfig = makeFontsConf { fontDirectories = [ ]; }; + in + runCommand ("playwright-browsers" + lib.optionalString withChromium "-chromium") + { + nativeBuildInputs = [ + makeWrapper + jq + ]; + } + ( + '' + BROWSERS_JSON=${driver}/package/browsers.json + '' + + lib.optionalString withChromium '' + CHROMIUM_REVISION=$(jq -r '.browsers[] | select(.name == "chromium").revision' $BROWSERS_JSON) + mkdir -p $out/chromium-$CHROMIUM_REVISION/chrome-linux + + # See here for the Chrome options: + # https://github.com/NixOS/nixpkgs/issues/136207#issuecomment-908637738 + makeWrapper ${chromium}/bin/chromium $out/chromium-$CHROMIUM_REVISION/chrome-linux/chrome \ + --set SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt \ + --set FONTCONFIG_FILE ${fontconfig} + '' + + '' + FFMPEG_REVISION=$(jq -r '.browsers[] | select(.name == "ffmpeg").revision' $BROWSERS_JSON) + mkdir -p $out/ffmpeg-$FFMPEG_REVISION + ln -s ${ffmpeg}/bin/ffmpeg $out/ffmpeg-$FFMPEG_REVISION/ffmpeg-linux + '' + ); in - driver +driver From 4519015fd06f5c2dda1a7bfe0b632fc8f5a9656b Mon Sep 17 00:00:00 2001 From: Konstantin Alekseev Date: Mon, 29 Apr 2024 16:08:07 +0300 Subject: [PATCH 2/2] playwright: build from source playwright-driver: 1.40.0 -> 1.46.0 python3Packages.playwright: 1.44.0 -> 1.46.0 --- .../python-modules/playwright/default.nix | 8 +- .../playwright/driver-location.patch | 19 +- .../python-modules/playwright/update.sh | 59 +- .../web/playwright-test/default.nix | 17 - .../web/playwright-test/node-env.nix | 689 ------------------ .../web/playwright-test/node-packages.json | 3 - .../web/playwright-test/node-packages.nix | 50 -- .../web/playwright-test/wrapped.nix | 29 - pkgs/development/web/playwright/driver.nix | 248 +++++-- pkgs/top-level/all-packages.nix | 4 +- 10 files changed, 229 insertions(+), 897 deletions(-) delete mode 100644 pkgs/development/web/playwright-test/default.nix delete mode 100644 pkgs/development/web/playwright-test/node-env.nix delete mode 100644 pkgs/development/web/playwright-test/node-packages.json delete mode 100644 pkgs/development/web/playwright-test/node-packages.nix delete mode 100644 pkgs/development/web/playwright-test/wrapped.nix diff --git a/pkgs/development/python-modules/playwright/default.nix b/pkgs/development/python-modules/playwright/default.nix index 42472024e0201..217e8d7cd292a 100644 --- a/pkgs/development/python-modules/playwright/default.nix +++ b/pkgs/development/python-modules/playwright/default.nix @@ -21,7 +21,7 @@ in buildPythonPackage rec { pname = "playwright"; # run ./pkgs/development/python-modules/playwright/update.sh to update - version = "1.44.0"; + version = "1.46.0"; pyproject = true; disabled = pythonOlder "3.7"; @@ -29,7 +29,7 @@ buildPythonPackage rec { owner = "microsoft"; repo = "playwright-python"; rev = "refs/tags/v${version}"; - hash = "sha256-RM04I1QiyJhPvKdAdy8w2GmOOR+BWilxrZ5QUrwxBWA="; + hash = "sha256-88ZFhP8Bd10czoW71ltelWStypX4z4g18LT3Zo5ACMg="; }; patches = [ @@ -51,11 +51,11 @@ buildPythonPackage rec { git commit -m "workaround setuptools-scm" substituteInPlace setup.py \ - --replace "setuptools-scm==8.0.4" "setuptools-scm" \ + --replace "setuptools-scm==8.1.0" "setuptools-scm" \ --replace-fail "wheel==0.42.0" "wheel" substituteInPlace pyproject.toml \ - --replace 'requires = ["setuptools==68.2.2", "setuptools-scm==8.0.4", "wheel==0.42.0", "auditwheel==5.4.0"]' \ + --replace 'requires = ["setuptools==68.2.2", "setuptools-scm==8.1.0", "wheel==0.42.0", "auditwheel==5.4.0"]' \ 'requires = ["setuptools", "setuptools-scm", "wheel"]' # Skip trying to download and extract the driver. diff --git a/pkgs/development/python-modules/playwright/driver-location.patch b/pkgs/development/python-modules/playwright/driver-location.patch index 033b785e4eed3..87e193f6812e8 100644 --- a/pkgs/development/python-modules/playwright/driver-location.patch +++ b/pkgs/development/python-modules/playwright/driver-location.patch @@ -1,19 +1,22 @@ diff --git a/playwright/_impl/_driver.py b/playwright/_impl/_driver.py -index 9e8cdc1..7ee382d 100644 +index 22b53b8..2d86626 100644 --- a/playwright/_impl/_driver.py +++ b/playwright/_impl/_driver.py -@@ -23,11 +23,7 @@ from playwright._repo_version import version - - +@@ -23,14 +23,7 @@ from playwright._repo_version import version + + def compute_driver_executable() -> Tuple[str, str]: - driver_path = Path(inspect.getfile(playwright)).parent / "driver" - cli_path = str(driver_path / "package" / "cli.js") - if sys.platform == "win32": -- return (str(driver_path / "node.exe"), cli_path) +- return ( +- os.getenv("PLAYWRIGHT_NODEJS_PATH", str(driver_path / "node.exe")), +- cli_path, +- ) - return (os.getenv("PLAYWRIGHT_NODEJS_PATH", str(driver_path / "node")), cli_path) -+ return "@node@", "@driver@" - - ++ return "@node@", "@driver@" + + def get_driver_env() -> dict: diff --git a/setup.py b/setup.py index 8709e52..59784dd 100644 diff --git a/pkgs/development/python-modules/playwright/update.sh b/pkgs/development/python-modules/playwright/update.sh index 23c3276088659..eb04e619feb62 100755 --- a/pkgs/development/python-modules/playwright/update.sh +++ b/pkgs/development/python-modules/playwright/update.sh @@ -1,35 +1,54 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p curl gnused nix-prefetch common-updater-scripts node2nix jq +#!nix-shell -i bash -p curl gnused common-updater-scripts jq prefetch-npm-deps set -euo pipefail root="$(dirname "$(readlink -f "$0")")" -driver_file="$root/../../web/playwright/driver.nix" -playwright_test="$root/../../web/playwright-test" version=$(curl ${GITHUB_TOKEN:+" -u \":$GITHUB_TOKEN\""} -s https://api.github.com/repos/microsoft/playwright-python/releases/latest | jq -r '.tag_name | sub("^v"; "")') - # Most of the time, this should be the latest stable release of the Node-based # Playwright version, but that isn't a guarantee, so this needs to be specified # as well: setup_py_url="https://github.com/microsoft/playwright-python/raw/v${version}/setup.py" driver_version=$(curl -Ls "$setup_py_url" | grep '^driver_version =' | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+') -fetch_driver_arch() { - nix-prefetch-url "https://playwright.azureedge.net/builds/driver/playwright-${driver_version}-${1}.zip" -} +update-source-version playwright-driver "$driver_version" +update-source-version python3Packages.playwright "$version" + +# Update package-lock.json files for all npm deps that are built in playwright +# TODO: skip if update-source-version reported the same version +driver_file="$root/../../web/playwright/driver.nix" +repo_url_prefix="https://github.com/microsoft/playwright/raw" +temp_dir=$(mktemp -d) + +trap 'rm -rf "$temp_dir"' EXIT + -replace_sha() { - sed -i "s|$1 = \".\{44,52\}\"|$1 = \"$2\"|" "$driver_file" +# Function to download `package-lock.json` for a given source path and update hash +update_hash() { + local source_root_path="$1" + local existing_hash="$2" + + # Formulate download URL + local download_url="${repo_url_prefix}/v${driver_version}${source_root_path}/package-lock.json" + + # Download package-lock.json to temporary directory + curl -fsSL -o "${temp_dir}/package-lock.json" "$download_url" + + # Calculate the new hash + local new_hash + new_hash=$(prefetch-npm-deps "${temp_dir}/package-lock.json") + + # Update npmDepsHash in the original file + sed -i "s|$existing_hash|${new_hash}|" "$driver_file" } -# Replace SHAs for the driver downloads -replace_sha "x86_64-linux" "$(fetch_driver_arch "linux")" -replace_sha "x86_64-darwin" "$(fetch_driver_arch "mac")" -replace_sha "aarch64-linux" "$(fetch_driver_arch "linux-arm64")" -replace_sha "aarch64-darwin" "$(fetch_driver_arch "mac-arm64")" - -# Update the version stamps -sed -i "s/version =\s*\"[^\$]*\"/version = \"$driver_version\"/" "$driver_file" -sed -i "s/\"@playwright\/test\": \"[^\$]*\"/\"@playwright\/test\": \"$driver_version\"/" "$playwright_test/node-packages.json" -(cd "$playwright_test"; node2nix -i node-packages.json) -update-source-version playwright "$version" --rev="v$version" +while IFS= read -r source_root_line; do + [[ "$source_root_line" =~ sourceRoot ]] || continue + source_root_path=$(echo "$source_root_line" | sed -e 's/^.*"${src.name}\(.*\)";.*$/\1/') + + # Extract the current npmDepsHash for this sourceRoot + existing_hash=$(grep -A1 "$source_root_line" "$driver_file" | grep 'npmDepsHash' | sed -e 's/^.*npmDepsHash = "\(.*\)";$/\1/') + + # Call the function to download and update the hash + update_hash "$source_root_path" "$existing_hash" +done < "$driver_file" diff --git a/pkgs/development/web/playwright-test/default.nix b/pkgs/development/web/playwright-test/default.nix deleted file mode 100644 index e59fd8a77f515..0000000000000 --- a/pkgs/development/web/playwright-test/default.nix +++ /dev/null @@ -1,17 +0,0 @@ -# This file has been generated by node2nix 1.11.1. Do not edit! - -{pkgs ? import { - inherit system; - }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs_18"}: - -let - nodeEnv = import ./node-env.nix { - inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript; - inherit pkgs nodejs; - libtool = if pkgs.stdenv.isDarwin then pkgs.darwin or pkgs.darwin.cctools else null; - }; -in -import ./node-packages.nix { - inherit (pkgs) fetchurl nix-gitignore stdenv lib fetchgit; - inherit nodeEnv; -} diff --git a/pkgs/development/web/playwright-test/node-env.nix b/pkgs/development/web/playwright-test/node-env.nix deleted file mode 100644 index bc1e36628ac8a..0000000000000 --- a/pkgs/development/web/playwright-test/node-env.nix +++ /dev/null @@ -1,689 +0,0 @@ -# This file originates from node2nix - -{lib, stdenv, nodejs, python2, pkgs, libtool, runCommand, writeTextFile, writeShellScript}: - -let - # Workaround to cope with utillinux in Nixpkgs 20.09 and util-linux in Nixpkgs master - utillinux = if pkgs ? utillinux then pkgs.utillinux else pkgs.util-linux; - - python = if nodejs ? python then nodejs.python else python2; - - # Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise - tarWrapper = runCommand "tarWrapper" {} '' - mkdir -p $out/bin - - cat > $out/bin/tar <> $out/nix-support/hydra-build-products - ''; - }; - - # Common shell logic - installPackage = writeShellScript "install-package" '' - installPackage() { - local packageName=$1 src=$2 - - local strippedName - - local DIR=$PWD - cd $TMPDIR - - unpackFile $src - - # Make the base dir in which the target dependency resides first - mkdir -p "$(dirname "$DIR/$packageName")" - - if [ -f "$src" ] - then - # Figure out what directory has been unpacked - packageDir="$(find . -maxdepth 1 -type d | tail -1)" - - # Restore write permissions to make building work - find "$packageDir" -type d -exec chmod u+x {} \; - chmod -R u+w "$packageDir" - - # Move the extracted tarball into the output folder - mv "$packageDir" "$DIR/$packageName" - elif [ -d "$src" ] - then - # Get a stripped name (without hash) of the source directory. - # On old nixpkgs it's already set internally. - if [ -z "$strippedName" ] - then - strippedName="$(stripHash $src)" - fi - - # Restore write permissions to make building work - chmod -R u+w "$strippedName" - - # Move the extracted directory into the output folder - mv "$strippedName" "$DIR/$packageName" - fi - - # Change to the package directory to install dependencies - cd "$DIR/$packageName" - } - ''; - - # Bundle the dependencies of the package - # - # Only include dependencies if they don't exist. They may also be bundled in the package. - includeDependencies = {dependencies}: - lib.optionalString (dependencies != []) ( - '' - mkdir -p node_modules - cd node_modules - '' - + (lib.concatMapStrings (dependency: - '' - if [ ! -e "${dependency.packageName}" ]; then - ${composePackage dependency} - fi - '' - ) dependencies) - + '' - cd .. - '' - ); - - # Recursively composes the dependencies of a package - composePackage = { name, packageName, src, dependencies ? [], ... }@args: - builtins.addErrorContext "while evaluating node package '${packageName}'" '' - installPackage "${packageName}" "${src}" - ${includeDependencies { inherit dependencies; }} - cd .. - ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} - ''; - - pinpointDependencies = {dependencies, production}: - let - pinpointDependenciesFromPackageJSON = writeTextFile { - name = "pinpointDependencies.js"; - text = '' - var fs = require('fs'); - var path = require('path'); - - function resolveDependencyVersion(location, name) { - if(location == process.env['NIX_STORE']) { - return null; - } else { - var dependencyPackageJSON = path.join(location, "node_modules", name, "package.json"); - - if(fs.existsSync(dependencyPackageJSON)) { - var dependencyPackageObj = JSON.parse(fs.readFileSync(dependencyPackageJSON)); - - if(dependencyPackageObj.name == name) { - return dependencyPackageObj.version; - } - } else { - return resolveDependencyVersion(path.resolve(location, ".."), name); - } - } - } - - function replaceDependencies(dependencies) { - if(typeof dependencies == "object" && dependencies !== null) { - for(var dependency in dependencies) { - var resolvedVersion = resolveDependencyVersion(process.cwd(), dependency); - - if(resolvedVersion === null) { - process.stderr.write("WARNING: cannot pinpoint dependency: "+dependency+", context: "+process.cwd()+"\n"); - } else { - dependencies[dependency] = resolvedVersion; - } - } - } - } - - /* Read the package.json configuration */ - var packageObj = JSON.parse(fs.readFileSync('./package.json')); - - /* Pinpoint all dependencies */ - replaceDependencies(packageObj.dependencies); - if(process.argv[2] == "development") { - replaceDependencies(packageObj.devDependencies); - } - else { - packageObj.devDependencies = {}; - } - replaceDependencies(packageObj.optionalDependencies); - replaceDependencies(packageObj.peerDependencies); - - /* Write the fixed package.json file */ - fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2)); - ''; - }; - in - '' - node ${pinpointDependenciesFromPackageJSON} ${if production then "production" else "development"} - - ${lib.optionalString (dependencies != []) - '' - if [ -d node_modules ] - then - cd node_modules - ${lib.concatMapStrings (dependency: pinpointDependenciesOfPackage dependency) dependencies} - cd .. - fi - ''} - ''; - - # Recursively traverses all dependencies of a package and pinpoints all - # dependencies in the package.json file to the versions that are actually - # being used. - - pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args: - '' - if [ -d "${packageName}" ] - then - cd "${packageName}" - ${pinpointDependencies { inherit dependencies production; }} - cd .. - ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} - fi - ''; - - # Extract the Node.js source code which is used to compile packages with - # native bindings - nodeSources = runCommand "node-sources" {} '' - tar --no-same-owner --no-same-permissions -xf ${nodejs.src} - mv node-* $out - ''; - - # Script that adds _integrity fields to all package.json files to prevent NPM from consulting the cache (that is empty) - addIntegrityFieldsScript = writeTextFile { - name = "addintegrityfields.js"; - text = '' - var fs = require('fs'); - var path = require('path'); - - function augmentDependencies(baseDir, dependencies) { - for(var dependencyName in dependencies) { - var dependency = dependencies[dependencyName]; - - // Open package.json and augment metadata fields - var packageJSONDir = path.join(baseDir, "node_modules", dependencyName); - var packageJSONPath = path.join(packageJSONDir, "package.json"); - - if(fs.existsSync(packageJSONPath)) { // Only augment packages that exist. Sometimes we may have production installs in which development dependencies can be ignored - console.log("Adding metadata fields to: "+packageJSONPath); - var packageObj = JSON.parse(fs.readFileSync(packageJSONPath)); - - if(dependency.integrity) { - packageObj["_integrity"] = dependency.integrity; - } else { - packageObj["_integrity"] = "sha1-000000000000000000000000000="; // When no _integrity string has been provided (e.g. by Git dependencies), add a dummy one. It does not seem to harm and it bypasses downloads. - } - - if(dependency.resolved) { - packageObj["_resolved"] = dependency.resolved; // Adopt the resolved property if one has been provided - } else { - packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories. - } - - if(dependency.from !== undefined) { // Adopt from property if one has been provided - packageObj["_from"] = dependency.from; - } - - fs.writeFileSync(packageJSONPath, JSON.stringify(packageObj, null, 2)); - } - - // Augment transitive dependencies - if(dependency.dependencies !== undefined) { - augmentDependencies(packageJSONDir, dependency.dependencies); - } - } - } - - if(fs.existsSync("./package-lock.json")) { - var packageLock = JSON.parse(fs.readFileSync("./package-lock.json")); - - if(![1, 2].includes(packageLock.lockfileVersion)) { - process.stderr.write("Sorry, I only understand lock file versions 1 and 2!\n"); - process.exit(1); - } - - if(packageLock.dependencies !== undefined) { - augmentDependencies(".", packageLock.dependencies); - } - } - ''; - }; - - # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes - reconstructPackageLock = writeTextFile { - name = "reconstructpackagelock.js"; - text = '' - var fs = require('fs'); - var path = require('path'); - - var packageObj = JSON.parse(fs.readFileSync("package.json")); - - var lockObj = { - name: packageObj.name, - version: packageObj.version, - lockfileVersion: 2, - requires: true, - packages: { - "": { - name: packageObj.name, - version: packageObj.version, - license: packageObj.license, - bin: packageObj.bin, - dependencies: packageObj.dependencies, - engines: packageObj.engines, - optionalDependencies: packageObj.optionalDependencies - } - }, - dependencies: {} - }; - - function augmentPackageJSON(filePath, packages, dependencies) { - var packageJSON = path.join(filePath, "package.json"); - if(fs.existsSync(packageJSON)) { - var packageObj = JSON.parse(fs.readFileSync(packageJSON)); - packages[filePath] = { - version: packageObj.version, - integrity: "sha1-000000000000000000000000000=", - dependencies: packageObj.dependencies, - engines: packageObj.engines, - optionalDependencies: packageObj.optionalDependencies - }; - dependencies[packageObj.name] = { - version: packageObj.version, - integrity: "sha1-000000000000000000000000000=", - dependencies: {} - }; - processDependencies(path.join(filePath, "node_modules"), packages, dependencies[packageObj.name].dependencies); - } - } - - function processDependencies(dir, packages, dependencies) { - if(fs.existsSync(dir)) { - var files = fs.readdirSync(dir); - - files.forEach(function(entry) { - var filePath = path.join(dir, entry); - var stats = fs.statSync(filePath); - - if(stats.isDirectory()) { - if(entry.substr(0, 1) == "@") { - // When we encounter a namespace folder, augment all packages belonging to the scope - var pkgFiles = fs.readdirSync(filePath); - - pkgFiles.forEach(function(entry) { - if(stats.isDirectory()) { - var pkgFilePath = path.join(filePath, entry); - augmentPackageJSON(pkgFilePath, packages, dependencies); - } - }); - } else { - augmentPackageJSON(filePath, packages, dependencies); - } - } - }); - } - } - - processDependencies("node_modules", lockObj.packages, lockObj.dependencies); - - fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2)); - ''; - }; - - # Script that links bins defined in package.json to the node_modules bin directory - # NPM does not do this for top-level packages itself anymore as of v7 - linkBinsScript = writeTextFile { - name = "linkbins.js"; - text = '' - var fs = require('fs'); - var path = require('path'); - - var packageObj = JSON.parse(fs.readFileSync("package.json")); - - var nodeModules = Array(packageObj.name.split("/").length).fill("..").join(path.sep); - - if(packageObj.bin !== undefined) { - fs.mkdirSync(path.join(nodeModules, ".bin")) - - if(typeof packageObj.bin == "object") { - Object.keys(packageObj.bin).forEach(function(exe) { - if(fs.existsSync(packageObj.bin[exe])) { - console.log("linking bin '" + exe + "'"); - fs.symlinkSync( - path.join("..", packageObj.name, packageObj.bin[exe]), - path.join(nodeModules, ".bin", exe) - ); - } - else { - console.log("skipping non-existent bin '" + exe + "'"); - } - }) - } - else { - if(fs.existsSync(packageObj.bin)) { - console.log("linking bin '" + packageObj.bin + "'"); - fs.symlinkSync( - path.join("..", packageObj.name, packageObj.bin), - path.join(nodeModules, ".bin", packageObj.name.split("/").pop()) - ); - } - else { - console.log("skipping non-existent bin '" + packageObj.bin + "'"); - } - } - } - else if(packageObj.directories !== undefined && packageObj.directories.bin !== undefined) { - fs.mkdirSync(path.join(nodeModules, ".bin")) - - fs.readdirSync(packageObj.directories.bin).forEach(function(exe) { - if(fs.existsSync(path.join(packageObj.directories.bin, exe))) { - console.log("linking bin '" + exe + "'"); - fs.symlinkSync( - path.join("..", packageObj.name, packageObj.directories.bin, exe), - path.join(nodeModules, ".bin", exe) - ); - } - else { - console.log("skipping non-existent bin '" + exe + "'"); - } - }) - } - ''; - }; - - prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}: - let - forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com"; - in - '' - # Pinpoint the versions of all dependencies to the ones that are actually being used - echo "pinpointing versions of dependencies..." - source $pinpointDependenciesScriptPath - - # Patch the shebangs of the bundled modules to prevent them from - # calling executables outside the Nix store as much as possible - patchShebangs . - - # Deploy the Node.js package by running npm install. Since the - # dependencies have been provided already by ourselves, it should not - # attempt to install them again, which is good, because we want to make - # it Nix's responsibility. If it needs to install any dependencies - # anyway (e.g. because the dependency parameters are - # incomplete/incorrect), it fails. - # - # The other responsibilities of NPM are kept -- version checks, build - # steps, postprocessing etc. - - export HOME=$TMPDIR - cd "${packageName}" - runHook preRebuild - - ${lib.optionalString bypassCache '' - ${lib.optionalString reconstructLock '' - if [ -f package-lock.json ] - then - echo "WARNING: Reconstruct lock option enabled, but a lock file already exists!" - echo "This will most likely result in version mismatches! We will remove the lock file and regenerate it!" - rm package-lock.json - else - echo "No package-lock.json file found, reconstructing..." - fi - - node ${reconstructPackageLock} - ''} - - node ${addIntegrityFieldsScript} - ''} - - npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild - - runHook postRebuild - - if [ "''${dontNpmInstall-}" != "1" ] - then - # NPM tries to download packages even when they already exist if npm-shrinkwrap is used. - rm -f npm-shrinkwrap.json - - npm ${forceOfflineFlag} --nodedir=${nodeSources} --no-bin-links --ignore-scripts ${npmFlags} ${lib.optionalString production "--production"} install - fi - - # Link executables defined in package.json - node ${linkBinsScript} - ''; - - # Builds and composes an NPM package including all its dependencies - buildNodePackage = - { name - , packageName - , version ? null - , dependencies ? [] - , buildInputs ? [] - , production ? true - , npmFlags ? "" - , dontNpmInstall ? false - , bypassCache ? false - , reconstructLock ? false - , preRebuild ? "" - , dontStrip ? true - , unpackPhase ? "true" - , buildPhase ? "true" - , meta ? {} - , ... }@args: - - let - extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" "meta" ]; - in - stdenv.mkDerivation ({ - name = "${name}${if version == null then "" else "-${version}"}"; - buildInputs = [ tarWrapper python nodejs ] - ++ lib.optional (stdenv.isLinux) utillinux - ++ lib.optional (stdenv.isDarwin) libtool - ++ buildInputs; - - inherit nodejs; - - inherit dontStrip; # Stripping may fail a build for some package deployments - inherit dontNpmInstall preRebuild unpackPhase buildPhase; - - compositionScript = composePackage args; - pinpointDependenciesScript = pinpointDependenciesOfPackage args; - - passAsFile = [ "compositionScript" "pinpointDependenciesScript" ]; - - installPhase = '' - source ${installPackage} - - # Create and enter a root node_modules/ folder - mkdir -p $out/lib/node_modules - cd $out/lib/node_modules - - # Compose the package and all its dependencies - source $compositionScriptPath - - ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} - - # Create symlink to the deployed executable folder, if applicable - if [ -d "$out/lib/node_modules/.bin" ] - then - ln -s $out/lib/node_modules/.bin $out/bin - - # Fixup all executables - ls $out/bin/* | while read i - do - file="$(readlink -f "$i")" - chmod u+rwx "$file" - if isScript "$file" - then - sed -i 's/\r$//' "$file" # convert crlf to lf - fi - done - fi - - # Create symlinks to the deployed manual page folders, if applicable - if [ -d "$out/lib/node_modules/${packageName}/man" ] - then - mkdir -p $out/share - for dir in "$out/lib/node_modules/${packageName}/man/"* - do - mkdir -p $out/share/man/$(basename "$dir") - for page in "$dir"/* - do - ln -s $page $out/share/man/$(basename "$dir") - done - done - fi - - # Run post install hook, if provided - runHook postInstall - ''; - - meta = { - # default to Node.js' platforms - platforms = nodejs.meta.platforms; - } // meta; - } // extraArgs); - - # Builds a node environment (a node_modules folder and a set of binaries) - buildNodeDependencies = - { name - , packageName - , version ? null - , src - , dependencies ? [] - , buildInputs ? [] - , production ? true - , npmFlags ? "" - , dontNpmInstall ? false - , bypassCache ? false - , reconstructLock ? false - , dontStrip ? true - , unpackPhase ? "true" - , buildPhase ? "true" - , ... }@args: - - let - extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" ]; - in - stdenv.mkDerivation ({ - name = "node-dependencies-${name}${if version == null then "" else "-${version}"}"; - - buildInputs = [ tarWrapper python nodejs ] - ++ lib.optional (stdenv.isLinux) utillinux - ++ lib.optional (stdenv.isDarwin) libtool - ++ buildInputs; - - inherit dontStrip; # Stripping may fail a build for some package deployments - inherit dontNpmInstall unpackPhase buildPhase; - - includeScript = includeDependencies { inherit dependencies; }; - pinpointDependenciesScript = pinpointDependenciesOfPackage args; - - passAsFile = [ "includeScript" "pinpointDependenciesScript" ]; - - installPhase = '' - source ${installPackage} - - mkdir -p $out/${packageName} - cd $out/${packageName} - - source $includeScriptPath - - # Create fake package.json to make the npm commands work properly - cp ${src}/package.json . - chmod 644 package.json - ${lib.optionalString bypassCache '' - if [ -f ${src}/package-lock.json ] - then - cp ${src}/package-lock.json . - chmod 644 package-lock.json - fi - ''} - - # Go to the parent folder to make sure that all packages are pinpointed - cd .. - ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} - - ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} - - # Expose the executables that were installed - cd .. - ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} - - mv ${packageName} lib - ln -s $out/lib/node_modules/.bin $out/bin - ''; - } // extraArgs); - - # Builds a development shell - buildNodeShell = - { name - , packageName - , version ? null - , src - , dependencies ? [] - , buildInputs ? [] - , production ? true - , npmFlags ? "" - , dontNpmInstall ? false - , bypassCache ? false - , reconstructLock ? false - , dontStrip ? true - , unpackPhase ? "true" - , buildPhase ? "true" - , ... }@args: - - let - nodeDependencies = buildNodeDependencies args; - extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "unpackPhase" "buildPhase" ]; - in - stdenv.mkDerivation ({ - name = "node-shell-${name}${if version == null then "" else "-${version}"}"; - - buildInputs = [ python nodejs ] ++ lib.optional (stdenv.isLinux) utillinux ++ buildInputs; - buildCommand = '' - mkdir -p $out/bin - cat > $out/bin/shell <