curl-impersonate: init at 0.5.4 and replace curl-impersonate-bin

nixos/tests/all-tests.nix

@@ -156,6 +156,7 @@ in {
   couchdb = handleTest ./couchdb.nix {};
   cri-o = handleTestOn ["aarch64-linux" "x86_64-linux"] ./cri-o.nix {};
   cups-pdf = handleTest ./cups-pdf.nix {};
+  curl-impersonate = handleTest ./curl-impersonate.nix {};
   custom-ca = handleTest ./custom-ca.nix {};
   croc = handleTest ./croc.nix {};
   deluge = handleTest ./deluge.nix {};

nixos/tests/curl-impersonate.nix

@@ -0,0 +1,157 @@
+/*
+  Test suite for curl-impersonate
+
+  Abstract:
+    Uses the test suite from the curl-impersonate source repo which:
+
+    1. Performs requests with libcurl and captures the TLS client-hello
+       packets with tcpdump to compare against known-good signatures
+    2. Spins up an nghttpd2 server to test client HTTP/2 headers against
+       known-good headers
+
+    See https://github.com/lwthiker/curl-impersonate/tree/main/tests/signatures
+    for details.
+
+  Notes:
+    - We need to have our own web server running because the tests expect to be able
+      to hit domains like wikipedia.org and the sandbox has no internet
+    - We need to be able to do (verifying) TLS handshakes without internet access.
+      We do that by creating a trusted CA and issuing a cert that includes
+      all of the test domains as subject-alternative names and then spoofs the
+      hostnames in /etc/hosts.
+*/
+
+import ./make-test-python.nix ({ pkgs, lib, ... }: let
+  # Update with domains in TestImpersonate.TEST_URLS if needed from:
+  # https://github.com/lwthiker/curl-impersonate/blob/main/tests/test_impersonate.py
+  domains = [
+    "www.wikimedia.org"
+    "www.wikipedia.org"
+    "www.mozilla.org"
+    "www.apache.org"
+    "www.kernel.org"
+    "git-scm.com"
+  ];
+
+  tls-certs = let
+    # Configure CA with X.509 v3 extensions that would be trusted by curl
+    ca-cert-conf = pkgs.writeText "curl-impersonate-ca.cnf" ''
+      basicConstraints = critical, CA:TRUE
+      subjectKeyIdentifier = hash
+      authorityKeyIdentifier = keyid:always, issuer:always
+      keyUsage = critical, cRLSign, digitalSignature, keyCertSign
+    '';
+
+    # Configure leaf certificate with X.509 v3 extensions that would be trusted
+    # by curl and set subject-alternative names for test domains
+    tls-cert-conf = pkgs.writeText "curl-impersonate-tls.cnf" ''
+      basicConstraints = critical, CA:FALSE
+      subjectKeyIdentifier = hash
+      authorityKeyIdentifier = keyid:always, issuer:always
+      keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment, keyAgreement
+      extendedKeyUsage = critical, serverAuth
+      subjectAltName = @alt_names
+
+      [alt_names]
+      ${lib.concatStringsSep "\n" (lib.imap0 (idx: domain: "DNS.${toString idx} = ${domain}") domains)}
+    '';
+  in pkgs.runCommand "curl-impersonate-test-certs" {
+    nativeBuildInputs = [ pkgs.openssl ];
+  } ''
+    # create CA certificate and key
+    openssl req -newkey rsa:4096 -keyout ca-key.pem -out ca-csr.pem -nodes -subj '/CN=curl-impersonate-ca.nixos.test'
+    openssl x509 -req -sha512 -in ca-csr.pem -key ca-key.pem -out ca.pem -extfile ${ca-cert-conf} -days 36500
+    openssl x509 -in ca.pem -text
+
+    # create server certificate and key
+    openssl req -newkey rsa:4096 -keyout key.pem -out csr.pem -nodes -subj '/CN=curl-impersonate.nixos.test'
+    openssl x509 -req -sha512 -in csr.pem -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -extfile ${tls-cert-conf} -days 36500
+    openssl x509 -in cert.pem -text
+
+    # output CA cert and server cert and key
+    mkdir -p $out
+    cp key.pem cert.pem ca.pem $out
+  '';
+
+  # Test script
+  curl-impersonate-test = let
+    # Build miniature libcurl client used by test driver
+    minicurl = pkgs.runCommandCC "minicurl" {
+      buildInputs = [ pkgs.curl ];
+    } ''
+      mkdir -p $out/bin
+      $CC -Wall -Werror -o $out/bin/minicurl ${pkgs.curl-impersonate.src}/tests/minicurl.c `curl-config --libs`
+    '';
+  in pkgs.writeShellScript "curl-impersonate-test" ''
+    set -euxo pipefail
+
+    # Test driver requirements
+    export PATH="${with pkgs; lib.makeBinPath [
+      bash
+      coreutils
+      python3Packages.pytest
+      nghttp2
+      tcpdump
+    ]}"
+    export PYTHONPATH="${with pkgs.python3Packages; makePythonPath [
+      pyyaml
+      pytest-asyncio
+      dpkt
+    ]}"
+
+    # Prepare test root prefix
+    mkdir -p usr/{bin,lib}
+    cp -rs ${pkgs.curl-impersonate}/* ${minicurl}/* usr/
+
+    cp -r ${pkgs.curl-impersonate.src}/tests ./
+
+    # Run tests
+    cd tests
+    pytest . --install-dir ../usr --capture-interface eth1
+  '';
+in {
+  name = "curl-impersonate";
+
+  meta = with lib.maintainers; {
+    maintainers = [ lilyinstarlight ];
+  };
+
+  nodes = {
+    web = { nodes, pkgs, lib, config, ... }: {
+      networking.firewall.allowedTCPPorts = [ 80 443 ];
+
+      services = {
+        nginx = {
+          enable = true;
+          virtualHosts."curl-impersonate.nixos.test" = {
+            default = true;
+            addSSL = true;
+            sslCertificate = "${tls-certs}/cert.pem";
+            sslCertificateKey = "${tls-certs}/key.pem";
+          };
+        };
+      };
+    };
+
+    curl = { nodes, pkgs, lib, config, ... }: {
+      networking.extraHosts = lib.concatStringsSep "\n" (map (domain: "${nodes.web.networking.primaryIPAddress}  ${domain}") domains);
+
+      security.pki.certificateFiles = [ "${tls-certs}/ca.pem" ];
+    };
+  };
+
+  testScript = { nodes, ... }: ''
+    start_all()
+
+    with subtest("Wait for network"):
+        web.wait_for_unit("network-online.target")
+        curl.wait_for_unit("network-online.target")
+
+    with subtest("Wait for web server"):
+        web.wait_for_unit("nginx.service")
+        web.wait_for_open_port(443)
+
+    with subtest("Run curl-impersonate tests"):
+        curl.succeed("${curl-impersonate-test}")
+  '';
+})

pkgs/tools/networking/curl-impersonate/curl-impersonate-0.5.2-fix-shebangs.patch

@@ -0,0 +1,13 @@
+diff --git a/Makefile.in b/Makefile.in
+index 877c54f..3e39ed1 100644
+--- a/Makefile.in
++++ b/Makefile.in
+@@ -209,6 +209,8 @@ $(NSS_VERSION).tar.gz:
+
+ $(nss_static_libs): $(NSS_VERSION).tar.gz
+ 	tar xf $(NSS_VERSION).tar.gz
++	sed -i -e "1s@#!/usr/bin/env bash@#!$$(type -p bash)@" $(NSS_VERSION)/nss/build.sh
++	sed -i -e "s@/usr/bin/env grep@$$(type -p grep)@" $(NSS_VERSION)/nss/coreconf/config.gypi
+
+ ifeq ($(host),$(build))
+ 	# Native build, use NSS' build script.

pkgs/tools/networking/curl-impersonate/default.nix

@@ -1,27 +1,186 @@
-#TODO: It should be possible to build this from source, but it's currently a lot faster to just package the binaries.
-{ lib, stdenv, fetchzip, zlib, autoPatchelfHook }:
-stdenv.mkDerivation rec {
-  pname = "curl-impersonate-bin";
-  version = "v0.5.3";
-
-  src = fetchzip {
-    url = "https://github.com/lwthiker/curl-impersonate/releases/download/${version}/curl-impersonate-${version}.x86_64-linux-gnu.tar.gz";
-    sha256 = "sha256-+cH1swAIadIrWG9anzf0dcW6qyBjcKsUHFWdv75F49g=";
-    stripRoot = false;
+{ lib
+, stdenv
+, fetchFromGitHub
+, fetchpatch
+, callPackage
+, buildGoModule
+, installShellFiles
+, symlinkJoin
+, zlib
+, sqlite
+, cmake
+, python3
+, ninja
+, perl
+, autoconf
+, automake
+, libtool
+, darwin
+, cacert
+, unzip
+, go
+, p11-kit
+, nixosTests
+}:
+
+let
+  makeCurlImpersonate = { name, target }: stdenv.mkDerivation rec {
+    pname = "curl-impersonate-${name}";
+    version = "0.5.4";
+
+    src = fetchFromGitHub {
+      owner = "lwthiker";
+      repo = "curl-impersonate";
+      rev = "v${version}";
+      hash = "sha256-LBGWFal2szqgURIBCLB84kHWpdpt5quvBBZu6buGj2A=";
+    };
+
+    patches = [
+      # Fix shebangs in the NSS build script
+      # (can't just patchShebangs since makefile unpacks it)
+      ./curl-impersonate-0.5.2-fix-shebangs.patch
+    ];
+
+    strictDeps = true;
+
+    nativeBuildInputs = lib.optionals stdenv.isDarwin [
+      # Must come first so that it shadows the 'libtool' command but leaves 'libtoolize'
+      darwin.cctools
+    ] ++ [
+      installShellFiles
+      cmake
+      python3
+      python3.pkgs.gyp
+      ninja
+      perl
+      autoconf
+      automake
+      libtool
+      unzip
+      go
+    ];
+
+    buildInputs = [
+      zlib
+      sqlite
+    ];
+
+    configureFlags = [
+      "--with-ca-bundle=${if stdenv.isDarwin then "/etc/ssl/cert.pem" else "/etc/ssl/certs/ca-certificates.crt"}"
+      "--with-ca-path=${cacert}/etc/ssl/certs"
+    ];
+
+    buildFlags = [ "${target}-build" ];
+    checkTarget = "${target}-checkbuild";
+    installTargets = [ "${target}-install" ];
+
+    doCheck = true;
+
+    dontUseCmakeConfigure = true;
+    dontUseNinjaBuild = true;
+    dontUseNinjaInstall = true;
+    dontUseNinjaCheck = true;
+
+    postUnpack = lib.concatStringsSep "\n" (lib.mapAttrsToList (name: dep: "ln -sT ${dep.outPath} source/${name}") (lib.filterAttrs (n: v: v ? outPath) passthru.deps));
+
+    preConfigure = ''
+      export GOCACHE=$TMPDIR/go-cache
+      export GOPATH=$TMPDIR/go
+      export GOPROXY=file://${passthru.boringssl-go-modules}
+      export GOSUMDB=off
+
+      # Need to get value of $out for this flag
+      configureFlagsArray+=("--with-libnssckbi=$out/lib")
+    '';
+
+    postInstall = ''
+      # Remove vestigial *-config script
+      rm $out/bin/curl-impersonate-${name}-config
+
+      # Patch all shebangs of installed scripts
+      patchShebangs $out/bin
+
+      # Build and install completions for each curl binary
+
+      # Patch in correct binary name and alias it to all scripts
+      perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell zsh >$TMPDIR/curl-impersonate-${name}.zsh
+      substituteInPlace $TMPDIR/curl-impersonate-${name}.zsh \
+        --replace \
+          '#compdef curl' \
+          "#compdef curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' %f=curl-impersonate-${name}')"
+
+      perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell fish >$TMPDIR/curl-impersonate-${name}.fish
+      substituteInPlace $TMPDIR/curl-impersonate-${name}.fish \
+        --replace \
+          '--command curl' \
+          "--command curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' --command %f')"
+
+      # Install zsh and fish completions
+      installShellCompletion $TMPDIR/curl-impersonate-${name}.{zsh,fish}
+    '';
+
+    preFixup = let
+      libext = stdenv.hostPlatform.extensions.sharedLibrary;
+    in ''
+      # If libnssckbi.so is needed, link libnssckbi.so without needing nss in closure
+      if grep -F nssckbi $out/lib/libcurl-impersonate-*${libext} &>/dev/null; then
+        # NOTE: "p11-kit-trust" always ends in ".so" even when on darwin
+        ln -s ${p11-kit}/lib/pkcs11/p11-kit-trust.so $out/lib/libnssckbi${libext}
+        ${lib.optionalString stdenv.isLinux "patchelf --add-needed libnssckbi${libext} $out/lib/libcurl-impersonate-*${libext}"}
+      fi
+    '';
+
+    disallowedReferences = [ go ];
+
+    passthru = {
+      deps = callPackage ./deps.nix {};
+
+      boringssl-go-modules = (buildGoModule {
+        inherit (passthru.deps."boringssl.zip") name;
+
+        src = passthru.deps."boringssl.zip";
+        vendorHash = "sha256-ISmRdumckvSu7hBXrjvs5ZApShDiGLdD3T5B0fJ1x2Q=";
+
+        nativeBuildInputs = [ unzip ];
+
+        proxyVendor = true;
+      }).go-modules;
+    };
+
+    meta = with lib; {
+      description = "A special build of curl that can impersonate Chrome & Firefox";
+      homepage = "https://github.com/lwthiker/curl-impersonate";
+      license = with licenses; [ curl mit ];
+      maintainers = with maintainers; [ deliciouslytyped lilyinstarlight ];
+      platforms = platforms.unix;
+      knownVulnerabilities = [
+        "CVE-2023-32001"  # fopen TOCTOU race condition - https://curl.se/docs/CVE-2023-32001.html
+        "CVE-2022-43551"  # HSTS bypass - https://curl.se/docs/CVE-2022-43551.html
+        "CVE-2022-42916"  # HSTS bypass - https://curl.se/docs/CVE-2022-42916.html
+      ];
+    };
   };
+in
+
+symlinkJoin rec {
+  pname = "curl-impersonate";
+  inherit (passthru.curl-impersonate-ff) version meta;
+
+  name = "${pname}-${version}";
+
+  paths = [
+    passthru.curl-impersonate-ff
+    passthru.curl-impersonate-chrome
+  ];
+
+  passthru = {
+    curl-impersonate-ff = makeCurlImpersonate { name = "ff"; target = "firefox"; };
+    curl-impersonate-chrome = makeCurlImpersonate { name = "chrome"; target = "chrome"; };
 
-  nativeBuildInputs = [ autoPatchelfHook zlib ];
+    updateScript = ./update.sh;
 
-  installPhase = ''
-    mkdir -p $out/bin
-    cp * $out/bin
-  '';
+    inherit (passthru.curl-impersonate-ff) src;
 
-  meta = with lib; {
-    description = "curl-impersonate: A special build of curl that can impersonate Chrome & Firefox ";
-    homepage = "https://github.com/lwthiker/curl-impersonate";
-    license = with licenses; [ curl mit ];
-    maintainers = with maintainers; [ deliciouslytyped ];
-    platforms = platforms.linux; #TODO I'm unsure about the restrictions here, feel free to expand the platforms it if it works elsewhere.
+    tests = { inherit (nixosTests) curl-impersonate; };
   };
 }