Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ci/OWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
/pkgs/build-support/cc-wrapper @Ericson2314
/pkgs/build-support/bintools-wrapper @Ericson2314
/pkgs/build-support/setup-hooks @Ericson2314
/pkgs/build-support/setup-hooks/arrayUtilities @ConnorBaker
/pkgs/build-support/setup-hooks/auto-patchelf.sh @layus
/pkgs/by-name/au/auto-patchelf @layus

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# shellcheck shell=bash

# getSortedMapKeys
# Stores the sorted keys of the input associative array referenced by inputMapRef in the indexed arrray referenced by
# outputArrRef.
#
# Note from the Bash manual on arrays:
# There is no maximum limit on the size of an array, nor any requirement that members be indexed or assigned contiguously.
# - https://www.gnu.org/software/bash/manual/html_node/Arrays.html
#
# Since no guarantees are made about the order in which associative maps are traversed, this function is primarly
# useful for getting rid of yet another source of non-determinism. As an added benefit, it checks that the arguments
# provided are of correct type, unlike native parameter expansion which will accept expansions of strings.
#
# Arguments:
# - inputMapRef: a reference to an associative array (not mutated)
# - outputArrRef: a reference to an indexed array (contents are replaced entirely)
#
# Returns 0.
getSortedMapKeys() {
if (($# != 2)); then
nixErrorLog "expected two arguments!"
nixErrorLog "usage: getSortedMapKeys inputMapRef outputArrRef"
exit 1
fi

local -rn inputMapRef="$1"
# shellcheck disable=SC2178
# Don't warn about outputArrRef being used as an array because it is an array.
local -rn outputArrRef="$2"

if ! isDeclaredMap "${!inputMapRef}"; then
nixErrorLog "first argument inputMapRef must be a reference to an associative array"
exit 1
elif ! isDeclaredArray "${!outputArrRef}"; then
nixErrorLog "second argument outputArrRef must be a reference to an indexed array"
exit 1
fi

# shellcheck disable=SC2034
local -a keys=("${!inputMapRef[@]}")
sortArray keys "${!outputArrRef}"

return 0
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
callPackages,
isDeclaredArray,
isDeclaredMap,
makeSetupHook,
sortArray,
}:
makeSetupHook {
name = "getSortedMapKeys";
propagatedBuildInputs = [
isDeclaredArray
isDeclaredMap
sortArray
];
passthru.tests = callPackages ./tests.nix { };
meta.description = "Gets the sorted indices of an associative array";
} ./getSortedMapKeys.bash
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# NOTE: Tests related to getSortedMapKeys go here.
{
getSortedMapKeys,
lib,
testers,
}:
let
inherit (lib.attrsets) recurseIntoAttrs;
inherit (testers) shellcheck shfmt testEqualArrayOrMap;

check =
{
name,
valuesMap,
expectedArray,
}:
(testEqualArrayOrMap {
inherit name valuesMap expectedArray;
script = ''
set -eu
nixLog "running getSortedMapKeys with valuesMap to populate actualArray"
getSortedMapKeys valuesMap actualArray
'';
}).overrideAttrs
(prevAttrs: {
nativeBuildInputs = prevAttrs.nativeBuildInputs or [ ] ++ [ getSortedMapKeys ];
});
in
recurseIntoAttrs {
shellcheck = shellcheck {
name = "getSortedMapKeys";
src = ./getSortedMapKeys.bash;
};

shfmt = shfmt {
name = "getSortedMapKeys";
src = ./getSortedMapKeys.bash;
};

empty = check {
name = "empty";
valuesMap = { };
expectedArray = [ ];
};

singleton = check {
name = "singleton";
valuesMap = {
"apple" = "fruit";
};
expectedArray = [ "apple" ];
};

keysAreSorted = check {
name = "keysAreSorted";
valuesMap = {
"apple" = "fruit";
"bee" = "insect";
"carrot" = "vegetable";
};
expectedArray = [
"apple"
"bee"
"carrot"
];
};

# NOTE: While keys can be whitespace, they cannot be null (empty).
keysCanBeWhitespace = check {
name = "keysCanBeWhitespace";
valuesMap = {
" " = 1;
" " = 2;
};
expectedArray = [
" "
" "
];
};
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# shellcheck shell=bash

# isDeclaredArray
# Tests if inputArrayRef refers to a declared, indexed array.
#
# Arguments:
# - inputArrayRef: a reference to an indexed array (not mutated)
#
# Returns 0 if the indexed array is declared, 1 otherwise.
isDeclaredArray() {
# NOTE: We must dereference the name ref to get the type of the underlying variable.
# shellcheck disable=SC2034
local -nr inputArrayRef="$1" && [[ ${!inputArrayRef@a} =~ a ]]
}
Comment on lines 10 to 14
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We do a bunch of checks for "is array" in stdenv/generic/setup.sh. Even though we use case statements there, I think abstracting away this into two calls to isDeclaredArray / isDeclaredMap makes the code much more readable.

If we wanted to use it there, we'd need to make this part of stdenv, though - not another setup hook. Thus.. I suggest to do just that, for those two functions.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These have more utility than just for stdenv, so I want them to remain separate.

That said, I'm not opposed to this implementation find its way into stdenv -- I just don't want them to be available solely through stdenv.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm. I thought that whatever was available in stdenv was available everywhere? What exactly is the benefit of having them available "separately"? How would that look like? I might be missing something, but I only see a negative effect here - aka that they are not available in stdenv right now.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One example is creating scripts which run outside stdenv -- for example, the magma.passthru.testers.all script introduced here: #414612.

Is there an easy way to ensure setup.sh and all the goodies it has are sourced in such scripts, without polluting the shell with a bunch of other variables?

Copy link
Contributor

@wolfgangwalther wolfgangwalther Jun 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This sounds a bit like we'd maybe want a separation of something like stdenv and stdlib? (where stdenv can make use of stdlib!)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd love to see something like that and am interested in helping make that happen --
in the short term, I'd like to have these merged so I can bring in fixes for CUDA setup hooks which require more complicated Bash functionality (and I hate writing common functionality in multiple places).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is a use-case for sortArray in stdenv now: getAllOutputNames returns a different order of outputs depending on whether __structuredAttrs is turned on or not. This causes things like #425323 (comment). If we were to sort these outputs by name, they would match between both cases.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I really like that these functions are separate so I can use them outside of things providing setup.sh and have an easy way to test them. I'm not opposed to the ol' copy/paste or including the files in mkDerivation's nativeBuildInputs... what are your thoughts? Although maybe copy-pasting would be better so any changes to these functions don't cause a rebuild of stdenv.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not opposed to the ol' copy/paste or including the files in mkDerivation's nativeBuildInputs... what are your thoughts?

I don't think we can (or should) use something from mkDerivation.nativeBuildInputs in setup.sh, aka in core stdenv.

I don't like copy&paste either.

I think this was still a good idea:

This sounds a bit like we'd maybe want a separation of something like stdenv and stdlib? (where stdenv can make use of stdlib!)

Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
callPackages,
makeSetupHook,
}:
makeSetupHook {
name = "isDeclaredArray";
passthru.tests = callPackages ./tests.nix { };
meta.description = "Tests if an array is declared";
} ./isDeclaredArray.bash
Loading