Make cross compilation elegant

doc/cross-compilation.xml

@@ -187,7 +187,7 @@
       How does this work in practice? Nixpkgs is now structured so that build-time dependencies are taken from <varname>buildPackages</varname>, whereas run-time dependencies are taken from the top level attribute set.
       For example, <varname>buildPackages.gcc</varname> should be used at build time, while <varname>gcc</varname> should be used at run time.
       Now, for most of Nixpkgs's history, there was no <varname>buildPackages</varname>, and most packages have not been refactored to use it explicitly.
-      Instead, one can use the four attributes used for specifying dependencies as documented in <xref linkend="ssec-stdenv-attributes"/>.
+      Instead, one can use the six (<emphasis>gasp</emphasis>) attributes used for specifying dependencies as documented in <xref linkend="ssec-stdenv-dependencies"/>.
       We "splice" together the run-time and build-time package sets with <varname>callPackage</varname>, and then <varname>mkDerivation</varname> for each of four attributes pulls the right derivation out.
       This splicing can be skipped when not cross compiling as the package sets are the same, but is a bit slow for cross compiling.
       Because of this, a best-of-both-worlds solution is in the works with no splicing or explicit access of <varname>buildPackages</varname> needed.
@@ -200,6 +200,45 @@
     </para></note>
   </section>
 
+  <section>
+    <title>Cross packagaing cookbook</title>
+    <para>
+      Some frequently problems when packaging for cross compilation are good to just spell and answer.
+      Ideally the information above is exhaustive, so this section cannot provide any new information,
+      but its ludicrous and cruel to expect everyone to spend effort working through the interaction of many features just to figure out the same answer to the same common problem.
+      Feel free to add to this list!
+    </para>
+    <qandaset>
+      <qandaentry>
+        <question><para>
+          What if my package's build system needs to build a C program to be run under the build environment?
+        </para></question>
+        <answer><para>
+          <programlisting>depsBuildBuild = [ buildPackages.stdenv.cc ];</programlisting>
+          Add it to your <function>mkDerivation</function> invocation.
+        </para></answer>
+      </qandaentry>
+      <qandaentry>
+        <question><para>
+          My package fails to find <command>ar</command>.
+        </para></question>
+        <answer><para>
+          Many packages assume that an unprefixed <command>ar</command> is available, but Nix doesn't provide one.
+          It only provides a prefixed one, just as it only does for all the other binutils programs.
+          It may be necessary to patch the package to fix the build system to use a prefixed `ar`.
+        </para></answer>
+      </qandaentry>
+      <qandaentry>
+        <question><para>
+          My package's testsuite needs to run host platform code.
+        </para></question>
+        <answer><para>
+          <programlisting>doCheck = stdenv.hostPlatform != stdenv.buildPlatfrom;</programlisting>
+          Add it to your <function>mkDerivation</function> invocation.
+        </para></answer>
+      </qandaentry>
+    </qandaset>
+  </section>
 </section>
 
 <!--============================================================-->

nixos/doc/manual/release-notes/rl-1803.xml

@@ -103,6 +103,18 @@ following incompatible changes:</para>
       Other more obscure ones are just moved.
     </para>
   </listitem>
+  <listitem>
+    <para>
+      The propagation logic has been changed.
+      The new logic, along with new types of dependencies that go with, is thoroughly documented in the "Specifying dependencies" section of the "Standard Environment" chapter of the nixpkgs manual.
+      <!-- That's <xref linkend="ssec-stdenv-attributes"> were we to merge the manuals. -->
+      The old logic isn't but is easy to describe: dependencies were propagated as the same type of dependency no matter what.
+      In practice, that means that many <function>propagatedNativeBuildInputs</function> should instead be  <function>propagatedBuildInputs</function>.
+      Thankfully, that was and is the least used type of dependency.
+      Also, it means that some <function>propagatedBuildInputs</function> should instead be <function>depsTargetTargetPropagated</function>.
+      Other types dependencies should be unaffected.
+    </para>
+  </listitem>
 </itemizedlist>
 
 </section>

pkgs/applications/kde/libkcddb.nix

@@ -8,8 +8,8 @@ mkDerivation {
     license = with licenses; [ gpl2 lgpl21 bsd3 ];
     maintainers = with maintainers; [ peterhoeg ];
   };
-  nativeBuildInputs = [ extra-cmake-modules ];
-  buildInputs = [ qtbase kdoctools ];
+  nativeBuildInputs = [ extra-cmake-modules kdoctools ];
+  buildInputs = [ qtbase ];
   propagatedBuildInputs = [
     kcodecs ki18n kio kwidgetsaddons
     libmusicbrainz5

pkgs/applications/networking/browsers/lynx/default.nix

@@ -22,9 +22,9 @@ stdenv.mkDerivation rec {
 
   configureFlags = [ "--enable-widec" ] ++ stdenv.lib.optional sslSupport "--with-ssl";
 
-  nativeBuildInputs = stdenv.lib.optional sslSupport pkgconfig
-    ++ stdenv.lib.optional (hostPlatform != buildPlatform) buildPackages.stdenv.cc
-    ++ [ nukeReferences ];
+  depsBuildBuild = [ buildPackages.stdenv.cc ];
+  nativeBuildInputs = [ nukeReferences ]
+    ++ stdenv.lib.optional sslSupport pkgconfig;
 
   buildInputs = [ ncurses gzip ] ++ stdenv.lib.optional sslSupport openssl.dev;
 

pkgs/applications/office/skrooge/default.nix

@@ -14,9 +14,12 @@ mkDerivation rec {
     sha256 = "1dbvdrkdpgv39v8h7k3mri0nzlslfyd5kk410czj0jdn4qq400md";
   };
 
-  nativeBuildInputs = [ cmake extra-cmake-modules shared_mime_info ];
+  nativeBuildInputs = [
+    cmake extra-cmake-modules kdoctools shared_mime_info
+  ];
 
-  buildInputs = [ qtwebkit qtscript grantlee kxmlgui kwallet kparts kdoctools
+  buildInputs = [
+    qtwebkit qtscript grantlee kxmlgui kwallet kparts
     kjobwidgets kdesignerplugin kiconthemes knewstuff sqlcipher qca-qt5
     kactivities karchive kguiaddons knotifyconfig krunner kwindowsystem libofx
   ];

pkgs/applications/science/logic/coq/8.4.nix

@@ -60,7 +60,7 @@ stdenv.mkDerivation {
       fi
     }
 
-    envHooks=(''${envHooks[@]} addCoqPath)
+    addEnvHooks "$targetOffset" addCoqPath
   '';
 
   passthru = {

pkgs/applications/science/logic/coq/default.nix

@@ -109,7 +109,7 @@ self = stdenv.mkDerivation {
       fi
     }
 
-    envHooks=(''${envHooks[@]} addCoqPath)
+    addEnvHooks "$targetOffset" addCoqPath
   '';
 
   preConfigure = ''

pkgs/applications/science/math/R/setup-hook.sh

@@ -1,5 +1,7 @@
 addRLibPath () {
-    addToSearchPath R_LIBS_SITE $1/library
+    if [[ -d "$1/library" ]]; then
+        addToSearchPath R_LIBS_SITE "$1/library"
+    fi
 }
 
-envHooks+=(addRLibPath)
+addEnvHooks "$targetOffset" addRLibPath

pkgs/applications/video/mplayer/default.nix

@@ -102,7 +102,8 @@ stdenv.mkDerivation rec {
     rm -rf ffmpeg
   '';
 
-  nativeBuildInputs = [ buildPackages.stdenv.cc pkgconfig yasm ];
+  depsBuildBuild = [ buildPackages.stdenv.cc ];
+  nativeBuildInputs = [ pkgconfig yasm ];
   buildInputs = with stdenv.lib;
     [ freetype ffmpeg ]
     ++ optional aalibSupport aalib

pkgs/build-support/bintools-wrapper/default.nix

@@ -178,7 +178,7 @@ stdenv.mkDerivation {
       else throw "unknown emulation for platform: " + targetPlatform.config;
     in targetPlatform.platform.bfdEmulation or (fmt + "-" + arch);
 
-  propagatedBuildInputs = extraPackages;
+  depsTargetTargetPropagated = extraPackages;
 
   setupHook = ./setup-hook.sh;
 

pkgs/build-support/bintools-wrapper/setup-hook.sh

@@ -2,12 +2,20 @@
 #
 # See comments in cc-wrapper's setup hook. This works exactly the same way.
 
+set -u
+
+# Skip setup hook if we're neither a build-time dep, nor, temporarily, doing a
+# native compile.
+#
+# TODO(@Ericson2314): No native exception
+[[ -z ${crossConfig-} ]] || (( "$hostOffset" < 0 )) || return 0
+
 bintoolsWrapper_addLDVars () {
-    case $depOffset in
+    case $depHostOffset in
         -1) local role='BUILD_' ;;
         0)  local role='' ;;
         1)  local role='TARGET_' ;;
-        *)  echo "bintools-wrapper: Error: Cannot be used with $depOffset-offset deps, " >2;
+        *)  echo "bintools-wrapper: Error: Cannot be used with $depHostOffset-offset deps" >2;
             return 1 ;;
     esac
 
@@ -20,17 +28,29 @@ bintoolsWrapper_addLDVars () {
     fi
 }
 
-if [ -n "${crossConfig:-}" ]; then
-    export NIX_BINTOOLS_WRAPPER_@infixSalt@_TARGET_BUILD=1
-    role_pre='BUILD_'
-    role_post='_FOR_BUILD'
-else
-    export NIX_BINTOOLS_WRAPPER_@infixSalt@_TARGET_HOST=1
-    role_pre=""
-    role_post=''
-fi
+case $targetOffset in
+    -1)
+        export NIX_BINTOOLS_WRAPPER_@infixSalt@_TARGET_BUILD=1
+        role_pre='BUILD_'
+        role_post='_FOR_BUILD'
+        ;;
+    0)
+        export NIX_BINTOOLS_WRAPPER_@infixSalt@_TARGET_HOST=1
+        role_pre=''
+        role_post=''
+        ;;
+    1)
+        export NIX_BINTOOLS_WRAPPER_@infixSalt@_TARGET_TARGET=1
+        role_pre='TARGET_'
+        role_post='_FOR_TARGET'
+        ;;
+    *)
+        echo "cc-wrapper: used as improper sort of dependency" >2;
+        return 1
+        ;;
+esac
 
-envHooks+=(bintoolsWrapper_addLDVars)
+addEnvHooks "$targetOffset" bintoolsWrapper_addLDVars
 
 # shellcheck disable=SC2157
 if [ -n "@bintools_bin@" ]; then
@@ -65,3 +85,4 @@ done
 
 # No local scope in sourced file
 unset -v role_pre role_post cmd upper_case
+set +u

pkgs/build-support/cc-wrapper/default.nix

@@ -201,7 +201,8 @@ stdenv.mkDerivation {
       ln -s $ccPath/${targetPrefix}ghdl $out/bin/${targetPrefix}ghdl
     '';
 
-  propagatedBuildInputs = [ bintools ] ++ extraPackages;
+  propagatedBuildInputs = [ bintools ];
+  depsTargetTargetPropagated = extraPackages;
 
   setupHook = ./setup-hook.sh;
 

pkgs/build-support/cc-wrapper/setup-hook.sh

@@ -54,19 +54,26 @@
 # For more details, read the individual files where the mechanisms used to
 # accomplish this will be individually documented.
 
+set -u
+
+# Skip setup hook if we're neither a build-time dep, nor, temporarily, doing a
+# native compile.
+#
+# TODO(@Ericson2314): No native exception
+[[ -z ${crossConfig-} ]] || (( "$hostOffset" < 0 )) || return 0
 
 # It's fine that any other cc-wrapper will redefine this. Bash functions close
 # over no state, and there's no @-substitutions within, so any redefined
 # function is guaranteed to be exactly the same.
 ccWrapper_addCVars () {
-    # The `depOffset` describes how the platforms of the dependencies are slid
-    # relative to the depending package. It is brought into scope of the
-    # environment hook defined as the role of the dependency being applied.
-    case $depOffset in
+    # The `depHostOffset` describes how the host platform of the dependencies
+    # are slid relative to the depending package. It is brought into scope of
+    # the environment hook defined as the role of the dependency being applied.
+    case $depHostOffset in
         -1) local role='BUILD_' ;;
         0)  local role='' ;;
         1)  local role='TARGET_' ;;
-        *)  echo "cc-wrapper: Error: Cannot be used with $depOffset-offset deps, " >2;
+        *)  echo "cc-wrapper: Error: Cannot be used with $depHostOffset-offset deps" >2;
             return 1 ;;
     esac
 
@@ -87,20 +94,31 @@ ccWrapper_addCVars () {
 #
 # We also need to worry about what role is being added on *this* invocation of
 # setup-hook, which `role` tracks.
-if [ -n "${crossConfig:-}" ]; then
-    export NIX_CC_WRAPPER_@infixSalt@_TARGET_BUILD=1
-    role_pre='BUILD_'
-    role_post='_FOR_BUILD'
-else
-    export NIX_CC_WRAPPER_@infixSalt@_TARGET_HOST=1
-    role_pre=''
-    role_post=''
-fi
+case $targetOffset in
+    -1)
+        export NIX_CC_WRAPPER_@infixSalt@_TARGET_BUILD=1
+        role_pre='BUILD_'
+        role_post='_FOR_BUILD'
+        ;;
+    0)
+        export NIX_CC_WRAPPER_@infixSalt@_TARGET_HOST=1
+        role_pre=''
+        role_post=''
+        ;;
+    1)
+        export NIX_CC_WRAPPER_@infixSalt@_TARGET_TARGET=1
+        role_pre='TARGET_'
+        role_post='_FOR_TARGET'
+        ;;
+    *)
+        echo "cc-wrapper: used as improper sort of dependency" >2;
+        return 1
+        ;;
+esac
 
-# Eventually the exact sort of env-hook we create will depend on the role. This
-# is because based on what relative platform we are targeting, we use different
-# dependencies.
-envHooks+=(ccWrapper_addCVars)
+# We use the `targetOffset` to choose the right env hook to accumulate the right
+# sort of deps (those with that offset).
+addEnvHooks "$targetOffset" ccWrapper_addCVars
 
 # Note 1: these come *after* $out in the PATH (see setup.sh).
 # Note 2: phase separation makes this look useless to shellcheck.
@@ -131,3 +149,4 @@ export CXX${role_post}=@named_cxx@
 
 # No local scope in sourced file
 unset -v role_pre role_post
+set +u

pkgs/build-support/emacs/setup-hook.sh

@@ -4,4 +4,8 @@ addEmacsVars () {
   fi
 }
 
-envHooks+=(addEmacsVars)
+# If this is for a wrapper derivation, emacs and the dependencies are all
+# run-time dependencies. If this is for precompiling packages into bytecode,
+# emacs is a compile-time dependency of the package.
+addEnvHooks "$targetOffset" addEmacsVars
+addEnvHooks "$targetOffset" addEmacsVars

pkgs/build-support/gcc-wrapper-old/setup-hook.sh

@@ -1,4 +1,4 @@
-addCVars () {
+gccWrapperOld_addCVars () {
     if test -d $1/include; then
         export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -isystem $1/include"
     fi
@@ -12,7 +12,7 @@ addCVars () {
     fi
 }
 
-envHooks=(${envHooks[@]} addCVars)
+envBuildBuildHooks+=(gccWrapperOld_addCVars)
 
 # Note: these come *after* $out in the PATH (see setup.sh).
 

pkgs/build-support/setup-hooks/find-xml-catalogs.sh

@@ -18,5 +18,5 @@ if [ -z "$libxmlHookDone" ]; then
     # xmllint and xsltproc from looking in /etc/xml/catalog.
     export XML_CATALOG_FILES
     if [ -z "$XML_CATALOG_FILES" ]; then XML_CATALOG_FILES=" "; fi
-    envHooks+=(addXMLCatalogs)
+    addEnvHooks "$hostOffset" addXMLCatalogs
 fi

pkgs/build-support/setup-hooks/set-java-classpath.sh

@@ -10,4 +10,4 @@ addPkgToClassPath () {
     done
 }
 
-envHooks+=(addPkgToClassPath)
+addEnvHooks "$targetOffset" addPkgToClassPath

pkgs/build-support/setup-hooks/setup-debug-info-dirs.sh

@@ -2,4 +2,4 @@ setupDebugInfoDirs () {
     addToSearchPath NIX_DEBUG_INFO_DIRS $1/lib/debug
 }
 
-envHooks+=(setupDebugInfoDirs)
+addEnvHooks "$targetOffset" setupDebugInfoDirs