Skip to content

glibc: add patch to use utf-8 when LANG is unset #61202

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
matthewbauer wants to merge 6 commits into from
Closed

glibc: add patch to use utf-8 when LANG is unset #61202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
265e077
glibc: add patch to use utf-8 when LANG is unset
matthewbauer May 9, 2019
e05d59a
treewide: don’t assume we’re all americans
matthewbauer May 9, 2019
0ebb6d2
darwin-stdenv: set LC_CTYPE to UTF-8
matthewbauer May 9, 2019
a5999f6
gnulib: skip localename tests
matthewbauer Jul 11, 2019
9b218d1
nixos: add release notes for glibc default locale
matthewbauer Jul 17, 2019
6b828a2
Merge branch 'staging' into glibc-default-to-utf8
matthewbauer Jul 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions nixos/doc/manual/release-notes/rl-1909.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,18 @@
installer after creating <literal>/var/lib/nextcloud</literal>.
</para>
</listitem>
<listitem>
<para>
The default locale used by Glibc is now
<literal>C.UTF-8</literal> instead of <literal>C</literal>.
This means that unicode works in Nix builders out of the box.
The previous behavior can be restored by setting
<literal>LANG="C"</literal>. More information on these settings
is available in the Glibc manual at <link
xlink:href="https://www.gnu.org/software/libc/manual/html_node/Locale-Categories.html">
7.3 Locale Categories</link>.
</para>
</lisitem>
</itemizedlist>
</section>
</section>
6 changes: 0 additions & 6 deletions pkgs/build-support/agda/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,6 @@ let
self.buildDepends;
buildDependsAgdaShareAgda = map (x: x + "/share/agda") self.buildDependsAgda;

# Not much choice here ;)
LANG = "en_US.UTF-8";
LOCALE_ARCHIVE = stdenv.lib.optionalString
stdenv.isLinux
"${glibcLocales}/lib/locale/locale-archive";

everythingFile = "Everything.agda";

propagatedBuildInputs = self.buildDependsAgda;
Expand Down
2 changes: 0 additions & 2 deletions pkgs/build-support/release/maven-build.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@ stdenv.mkDerivation ( rec {
runHook preSetupPhase

mkdir -p $out/nix-support
export LANG="en_US.UTF-8"
export LOCALE_ARCHIVE=$glibcLocales/lib/locale/locale-archive
export M2_REPO=$TMPDIR/repository

runHook postSetupPhase
Expand Down
5 changes: 1 addition & 4 deletions pkgs/development/haskell-modules/generic-builder.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ let
isCross = stdenv.buildPlatform != stdenv.hostPlatform;
inherit (buildPackages)
fetchurl removeReferencesTo
pkgconfig coreutils gnugrep gnused glibcLocales;
pkgconfig coreutils gnugrep gnused;
in

{ pname
Expand Down Expand Up @@ -262,8 +262,6 @@ stdenv.mkDerivation ({
buildInputs = otherBuildInputs ++ optionals (!isLibrary) propagatedBuildInputs;
propagatedBuildInputs = optionals isLibrary propagatedBuildInputs;

LANG = "en_US.UTF-8"; # GHC needs the locale configured during the Haddock phase.

prePatch = optionalString (editedCabalFile != null) ''
echo "Replace Cabal file with edited version from ${newCabalFileUrl}."
cp ${newCabalFile} ${pname}.cabal
Expand Down Expand Up @@ -513,6 +511,5 @@ stdenv.mkDerivation ({
// optionalAttrs (postFixup != "") { inherit postFixup; }
// optionalAttrs (dontStrip) { inherit dontStrip; }
// optionalAttrs (hardeningDisable != []) { inherit hardeningDisable; }
// optionalAttrs (stdenv.buildPlatform.libc == "glibc"){ LOCALE_ARCHIVE = "${glibcLocales}/lib/locale/locale-archive"; }
)
)
4 changes: 0 additions & 4 deletions pkgs/development/haskell-modules/make-package-set.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,6 @@ let
preferLocalBuild = true;
allowSubstitutes = false;
phases = ["installPhase"];
LANG = "en_US.UTF-8";
LOCALE_ARCHIVE = pkgs.lib.optionalString (buildPlatform.libc == "glibc") "${buildPackages.glibcLocales}/lib/locale/locale-archive";
installPhase = ''
export HOME="$TMP"
mkdir -p "$out"
Expand Down Expand Up @@ -311,8 +309,6 @@ in package-set { inherit pkgs stdenv callPackage; } self // {
nativeBuildInputs = [ ghcEnv ] ++ nativeBuildInputs ++ mkDrvArgs.nativeBuildInputs or [];
phases = ["installPhase"];
installPhase = "echo $nativeBuildInputs $buildInputs > $out";
LANG = "en_US.UTF-8";
LOCALE_ARCHIVE = pkgs.lib.optionalString (stdenv.hostPlatform.libc == "glibc") "${buildPackages.glibcLocales}/lib/locale/locale-archive";
"NIX_${ghcCommandCaps}" = "${ghcEnv}/bin/${ghcCommand}";
"NIX_${ghcCommandCaps}PKG" = "${ghcEnv}/bin/${ghcCommand}-pkg";
# TODO: is this still valid?
Expand Down
3 changes: 0 additions & 3 deletions pkgs/development/interpreters/elixir/generic-builder.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,6 @@ in

buildInputs = [ erlang rebar makeWrapper ];

LANG = "C.UTF-8";
LC_TYPE = "C.UTF-8";

setupHook = ./setup-hook.sh;

inherit debugInfo;
Expand Down
5 changes: 5 additions & 0 deletions pkgs/development/libraries/glibc/common.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,11 @@ stdenv.mkDerivation ({
url = "https://salsa.debian.org/glibc-team/glibc/raw/49767c9f7de4828220b691b29de0baf60d8a54ec/debian/patches/localedata/locale-C.diff";
sha256 = "0irj60hs2i91ilwg5w7sqrxb695c93xg0ik7yhhq9irprd7fidn4";
})
(fetchurl {
url = "https://bugs.debian.org/cgi-bin/bugreport.cgi?att=1;bug=874160;filename=0001-Default-to-C.UTF-8-on-setlocale-.-if-no-env-vars-are.patch;msg=5";
name = "0001-Default-to-C.UTF-8-on-setlocale-.-if-no-env-vars-are.patch";

@Mic92 Mic92 Jul 26, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So debian switched to C.UTF-8 now?

@matthewbauer matthewbauer Jul 26, 2019

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No this is just a proposed patch

@matthewbauer matthewbauer Jul 26, 2019

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=874160

sha256 = "05wnpib83ggqnr1c85ajrak00478hwalrb3q7pgnxlzs0axw2iyk";
})
]
++ lib.optional stdenv.isx86_64 ./fix-x64-abi.patch
++ lib.optional stdenv.hostPlatform.isMusl ./fix-rpc-types-musl-conflicts.patch
Expand Down
1 change: 1 addition & 0 deletions pkgs/stdenv/darwin/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ in rec {
export CMAKE_OSX_ARCHITECTURES=x86_64
# Workaround for https://openradar.appspot.com/22671534 on 10.11.
export gl_cv_func_getcwd_abort_bug=no
export LC_CTYPE=UTF-8

@Mic92 Mic92 May 9, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be still great if we could set LANG on macOS as well. Reason is, that $LC_CTYPE has a different semantics/priority compared to $LANG. LANG has a lower priority then all LC_* variables, so if an existing package already sets LC_ALL or similar to some value, it would be preferred.

@matthewbauer matthewbauer May 10, 2019 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It doesn't seem to work correctly:

$ env -i LANG=UTF-8 locale
LANG="UTF-8"
LC_COLLATE="C"
LC_CTYPE="C"
LC_MESSAGES="C"
LC_MONETARY="C"
LC_NUMERIC="C"
LC_TIME="C"
LC_ALL=

vs.

$ env -i LANG=en_US.UTF-8 locale
LANG="en_US.UTF-8"
LC_COLLATE="en_US.UTF-8"
LC_CTYPE="en_US.UTF-8"
LC_MESSAGES="en_US.UTF-8"
LC_MONETARY="en_US.UTF-8"
LC_NUMERIC="en_US.UTF-8"
LC_TIME="en_US.UTF-8"
LC_ALL=

vs.

$ env -i LANG=en_US.UTF-8 locale
LANG=
LC_COLLATE="C"
LC_CTYPE="UTF-8"
LC_MESSAGES="C"
LC_MONETARY="C"
LC_NUMERIC="C"
LC_TIME="C"
LC_ALL=

This is the same thing described in the FreeBSD proposal:

Introduce C.UTF-8 locale, using the same common CTYPE map as other UTF-8 locales ... and having all other components use C locale

UTF-8 is only valid for LC_CTYPE, which makes sense. This is a BSDism it looks like though, so documentation on it would be helpful. My thinking on this is that if a package really needs a non-UTF-8 encoding in 2019, they should specifically request it through setting LC_CTYPE or LC_ALL. Also just to show that LC_ALL does take precedence:

$ env -i LC_ALL=en_US.UTF-8 LC_CTYPE=UTF-8 locale
LANG=
LC_COLLATE="en_US.UTF-8"
LC_CTYPE="en_US.UTF-8"
LC_MESSAGES="en_US.UTF-8"
LC_MONETARY="en_US.UTF-8"
LC_NUMERIC="en_US.UTF-8"
LC_TIME="en_US.UTF-8"
LC_ALL="en_US.UTF-8"

@Mic92 Mic92 May 10, 2019 •
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I meant by having a C.UTF-8 locale generated on macOS instead of using LC_CTYPE=UTF-8.

'';

bootstrapTools = derivation rec {
Expand Down
5 changes: 5 additions & 0 deletions pkgs/tools/misc/coreutils/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ stdenv.mkDerivation rec {

sed '2i print "Skipping env -S test"; exit 77;' -i ./tests/misc/env-S.pl

# Skip default locale tests. These appear to assume that
# setlocale(LC_ALL, "") = "C" when the default in Nixpkgs is
# "C.UTF-8"
echo "int main() { return 77; }" > gnulib-tests/test-localename.c

# these tests fail in the unprivileged nix sandbox (without nix-daemon) as we break posix assumptions
for f in ./tests/chgrp/{basic.sh,recurse.sh,default-no-deref.sh,no-x.sh,posix-H.sh}; do
sed '2i echo Skipping chgrp && exit 77' -i "$f"
Expand Down
7 changes: 7 additions & 0 deletions pkgs/tools/misc/findutils/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ stdenv.mkDerivation rec {
./disable-getdtablesize-test.patch
];

postPatch = ''
# Skip default locale tests. These appear to assume that
# setlocale(LC_ALL, "") = "C" when the default in Nixpkgs is
# "C.UTF-8"
echo "int main() { return 77; }" > tests/test-localename.c
'';

buildInputs = [ coreutils ]; # bin/updatedb script needs to call sort

# Since glibc-2.25 the i686 tests hang reliably right after test-sleep.
Expand Down