llvmPackages_{12,13,14,15,16,17,git}.{libcxx,libcxxabi}: merge libcxxabi into libcxx

[ghost]

Description of changes

• merge libcxxabi into libcxx for LLVM 12, 13, 14, 15, 16, 17, and git.
• remove the link time workaround -lc++ -lc++abi from 58 packages as it is no longer required.
• fixes Some libc++abi symbols are missing with LLVM12+'s stdenv on Darwin #166205
• provides alternative fixes for. libcxx-{15,16,etc} segfault when multiple libcxx loaded #269548 segfault during substitution on x86-64_darwin nix#9640
• pkgsCross.x86_64-freebsd builds work again
• merge should be squashed

This change can be represented in 3 stages

1. merge libcxxabi into libcxx -- files: pkgs/development/compilers/llvm/[12, git]/{libcxx, libcxxabi}
2. update stdenv to account for merge -- files: stdenv.{adapters, cc.wrapper, darwin}
3. remove all references to libcxxabi outside of llvm (about 58 packages modified)

merging libcxxabi into libcxx

• take the union of the libcxxabi and libcxx cmake flags
• eliminate the libcxx-headers-only package - it was only needed to break libcxx <-> libcxxabi circular dependency
• libcxx.cxxabi is removed. external cxxabi (freebsd) will symlink headers / libs into libcxx.
• darwin will re-export the libcxxabi symbols into libcxx so linking -lc++ is sufficient.
• linux/freebsd libc++.so is a linker script LINK(libc++.so.1, -lc++abi) making -lc++ sufficient.
• libcxx/default.nix [12, 17] are identical except for patches and LIBCXX_ADDITIONAL_LIBRARIES (only used in 16+)
• git/libcxx/defaul.nix does not link with -nostdlib when useLLVM is true so flag is removed. this is not much different than before as libcxxabi used -nostdlib where libcxx did not, so libc was linked in anyway.

stdenv changes

• darwin bootstrap, remove references to libcxxabi and cxxabi
• cc-wrapper: remove c++ link workaround when libcxx.cxxabi doesn't exist (still exists for LLVM pre 12)
• adapter: update overrideLibcxx to account for a pkgs.stdenv that only has libcxx

58 package updates

• remove NIX_LDFLAGS = "-l${stdenv.cc.libcxx.cxxabi.libName} as no longer needed
• swift, nodejs_v8 remove libcxxabi references in the clang override

Supersedes #282624

tests bellow all pass:

linux x64 builds

{pkgs ? import ./. {}}:
pkgs.mkShell {
  packages = let
    inherit (pkgs) lib;
    versions = map toString (lib.range 12 17) ++  [ "git" ];
  in with pkgs; (
    (map (ver: pkgsCross.wasi32.${"llvmPackages_${ver}"}.libcxx) versions)
    ++ (map (ver: pkgsLLVM.${"llvmPackages_${ver}"}.libcxx) versions)
    ++ (map (ver: pkgs.${"llvmPackages_${ver}"}.libcxx) versions)
    ++ (map (ver: pkgsMusl.${"llvmPackages_${ver}"}.libcxx) versions)
    ++ (map (ver: pkgsCross.x86_64-freebsd.${"llvmPackages_${ver}"}.libcxx) versions)
  );

linux / darwin x64 builds

{pkgs ? import ./. {}}:
let
  inherit (pkgs) lib stdenv;
  libcxxabi-removed = (with pkgs; ([
    ast-grep
    bazel_7
    beancount-language-server
    bfc
    blackmagic-desktop-video
    boolector
    bpf-linker
    cbfmt
    cdrtools
    citrix_workspace
    crystal
    difftastic
    dolphin-emu
    done
    dump_syms
    espanso
    firefox
    flow
    frawk
    gcs
    geos
    harmonia
    hashcat
    haskellPackages.hercules-ci-cli
    jemalloc
    kanata
    kotatogram-desktop
    ladybird
    lean4
    lib2geom
    libserdes
    lua-language-server
    mold
    nickel
    nix-du
    nodejs_18
    nova-filters
    opam
    open-watcom-v2
    osl
    ouch
    p4
    pjsip
    postgresql15Packages.postgis
    proj
    prusa-slicer
    python3Packages.uamqp
    qdrant
    rust-code-analysis
    snapdragon-profiler
    sonic-server
    swift
    symbolicator
    teamspeak_client
    tectonic
    topiary
    unar
    usql
  ]));
  libcxxabi-removed-env = pkgs.buildEnv {
    name = "libcxxabi-removed-env";
    paths = lib.filter (p: lib.meta.availableOn stdenv p) libcxxabi-removed;
  };
  channel-blockers-env = pkgs.buildEnv {
    name = "channel-blockers-env";
    paths = lib.filter (p: lib.meta.availableOn stdenv p) (with pkgs; [
      cabal2nix cachix emacs ghc gimp git go inkscape mariadb nix-info-tested
      nix-info nix nixpkgs-review openssh openssl pandoc postgresql python3
      qt5.qtmultimedia ruby rustc stdenv # self.checks.${system}.tarball
      transmission-gtk vim
    ]);
  };
in
pkgs.mkShell {
  packages = [ libcxxabi-removed-env channel-blockers-env ]
  ++ lib.filter (p: lib.meta.availableOn stdenv p) (
    with pkgs; [ crystal_1_2  crystal_1_7  crystal_1_8  crystal_1_9 ]
  ) ;
}

dlopen and link tests

# shell.nix
{pkgs ? import ./. {}}:
let
  dltest = pkgs.callPackage (
    { lib
    , runCommandCC
    , writeText
    , libcxx
    }:
    let
      name = "dltest";
      code = writeText "${name}.c" ''
        #include <dlfcn.h>
        #include <stdio.h>

        int
        main(int argc, const char **argv)
        {
            int numFailed = 0;
            for (int i = 1; i < argc; i++) {
              int failed = dlopen(argv[i], RTLD_LOCAL | RTLD_LAZY) == 0;
              numFailed += failed;
              fprintf(stderr, "dlopen %s: %s\n", argv[i], failed ? "failed" : "succeeded");
            }
            return numFailed;
        }
      '';
    in
      runCommandCC name {src = code;} ''
        mkdir -p "$out"/bin
        cc -Wall -Werror -O2 -o "$out/bin/${name}" "$src" -ldl -L${lib.getLib libcxx}/lib -lc++
      ''
  ) {
    libcxx = pkgs.llvmPackages_git.libcxx;
  };

  linkTest = pkgs.callPackage (
    { runCommandWith
    , writeText
    , name
    , stdenv
    }:
    let
      runCommandCC = name: env: runCommandWith {
        stdenv = stdenv;
        runLocal = false;
        inherit name;
        derivationArgs = env;
      };

      code = writeText "${name}.cxx" ''
        #include <iostream>

        int
        main(void)
        {
            std::cout << "${name} succeeded" << std::endl;
            return 0;
        }
      '';
    in
      runCommandCC name {src = code;} ''
        mkdir -p "$out"/bin
        cd "$(mktemp -d)"
        $CXX $src -c -o main.o
        $CC main.o -o "$out/bin/${name}" -lc++
      ''
  );

  buildTest = pkgs.callPackage (
    { runCommandWith
    , writeText
    , name
    , stdenv
    }:
    let
      runCommandCC = name: env: runCommandWith {
        stdenv = stdenv;
        runLocal = false;
        inherit name;
        derivationArgs = env;
      };

      code = writeText "${name}.cxx" ''
        #include <iostream>

        int
        main(void)
        {
            std::cout << "${name} succeeded" << std::endl;
            return 0;
        }
      '';
    in
      runCommandCC name {src = code;} ''
        mkdir -p "$out"/bin
        $CXX $src -o "$out/bin/${name}"
      ''
  );

  shell =
  let
    inherit (pkgs) lib stdenv mkShell writeShellScriptBin;
    ext = stdenv.hostPlatform.extensions.sharedLibrary;
    #versions = [ "12" "13" "14" "15" "16" "17" "git" ];
    versions = map toString (lib.range 12 17) ++  [  "git" ];
    getLibcxx = libcxx: "${lib.getLib libcxx}/lib/libc++${ext}" + lib.optionalString stdenv.isLinux ".1";
    getStdenv = pfx: if stdenv.isLinux then pfx.libcxxStdenv else pfx.stdenv;
    getLinkTest = ver:
          linkTest {
            name = "main${ver}";
            stdenv = getStdenv pkgs.${"llvmPackages_${ver}"};
          };
    getBuildTest = ver:
          buildTest {
            name = "mainxx${ver}";
            stdenv = getStdenv pkgs.${"llvmPackages_${ver}"};
          };
  in
    mkShell {
      packages = (map getLinkTest versions) ++
        (map getBuildTest versions) ++ [
        dltest
        (writeShellScriptBin "runtests" ''
          set -e
          ${lib.getBin dltest}/bin/dltest \
            ${toString (map (ver: getLibcxx pkgs.${"llvmPackages_${ver}"}.libcxx) versions)}
          for ver in ${toString versions}; do
            mainxx$ver
            main$ver
          done
        '')
      ];
      shellHook = "runtests";
    };
in
  shell

Things done

• Built on platform(s)
  • x86_64-linux
  • aarch64-linux
  • x86_64-darwin
  • aarch64-darwin
• For non-Linux: Is sandboxing enabled in nix.conf? (See Nix manual)
  • sandbox = relaxed
  • sandbox = true
• Tested, as applicable:
  • NixOS test(s) (look inside nixos/tests)
  • and/or package tests
  • or, for functions and "core" functionality, tests in lib/tests or pkgs/test
  • made sure NixOS tests are linked to the relevant packages
• Tested compilation of all packages that depend on this change using nix-shell -p nixpkgs-review --run "nixpkgs-review rev HEAD". Note: all changes have to be committed, also see nixpkgs-review usage
• Tested basic functionality of all binary files (usually in ./result/bin/)
• 24.05 Release Notes (or backporting 23.05 and 23.11 Release notes)
  • (Package updates) Added a release notes entry if the change is major or breaking
  • (Module updates) Added a release notes entry if the change is significant
  • (Module addition) Added a release notes entry if adding a new NixOS module
• Fits CONTRIBUTING.md.

---

Add a 👍 reaction to pull requests you find important.

[nixos-discourse]

This pull request has been mentioned on NixOS Discourse. There might be relevant details there:

https://discourse.nixos.org/t/aarch64-darwin-rust-rocksdb-bindgen-linking-error/39931/3

[nixos-discourse]

This pull request has been mentioned on NixOS Discourse. There might be relevant details there:

https://discourse.nixos.org/t/darwin-updates-news/42249/4

[will]

I haven't fully confirmed it yet, but I suspect this commit broke statically building boehm-gc on aarch64-darwin http://discourse.nixos.org/t/tracking-down-what-broke-boehm-gc-static-aarch64-apple-darwin/42234/2

[ghost]

I haven't fully confirmed it yet, but I suspect this commit broke statically building boehm-gc on aarch64-darwin http://discourse.nixos.org/t/tracking-down-what-broke-boehm-gc-static-aarch64-apple-darwin/42234/2

yes, as noted https://discourse.nixos.org/t/tracking-down-what-broke-boehm-gc-static-aarch64-apple-darwin/42234/3 LLVM doesn't provide a mechanism for static builds that avoid having to specify -lc++abi. It's possible to just merge the libc++abi.a archive into libc++.a though that would be somewhat divergent behaviour as with vanilla LLVM static builds specifying both -lc++ -lc++abi is required.

[edit] PR #305876 merges libc++abi.a into libc++.a