diff --git a/MODULE.bazel b/MODULE.bazel index e1566611e..68009fec9 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -79,6 +79,8 @@ rust.toolchain( rust_analyzer_version = "nightly/2025-05-21", rustfmt_version = "nightly/2025-05-21", sha256s = { + # Update the shas with update-module-hashes + # BEGIN SHAS "2025-05-21/cargo-nightly-x86_64-unknown-linux-gnu.tar.xz": "e866f249dfbdf10a68b7191c025257591e8a5aa2fede1663b34c88a4f4bb8a74", "2025-05-21/clippy-nightly-x86_64-unknown-linux-gnu.tar.xz": "0a312d722a94e3b9e1f7871d9a9af01d410917c2406dbf91d014c06fe79540fb", "2025-05-21/llvm-tools-nightly-x86_64-unknown-linux-gnu.tar.xz": "eee28e99ac24c27f3de969915e808c0645ee099b136e5547681110607d09d050", @@ -97,6 +99,7 @@ rust.toolchain( "rust-std-1.87.0-x86_64-unknown-linux-gnu.tar.xz": "1b57253bd32b8b292c965b3a2d992a266763158494cab8555584c09360b90f77", "rustc-1.87.0-aarch64-apple-darwin.tar.xz": "175800bc89cccd8f8ee2f3a4d07bdf98c163030fd5d3dc6d5b23cf4dd0a2a4c3", "rustc-1.87.0-x86_64-unknown-linux-gnu.tar.xz": "e8395c5c5756253b76107055e093ffbc4431af7b30aeebe72ce2684b9cb53973", + # END SHAS }, versions = [ "1.87.0", diff --git a/flake.nix b/flake.nix index 3e384a19a..6bd851790 100644 --- a/flake.nix +++ b/flake.nix @@ -150,12 +150,13 @@ } // (pkgs.lib.optionalAttrs isLinuxTarget { CARGO_BUILD_RUSTFLAGS = "-C target-feature=+crt-static"; + TARGET_CC = "${pkgs.lre.clang}/bin/customClang"; ${linkerEnvVar} = linkerPath; }); # Additional target for external dependencies to simplify caching. cargoArtifactsFor = p: (craneLibFor p).buildDepsOnly (commonArgsFor p); - nightlyCargoArtifactsFor = p: (craneLibFor p).buildDepsOnly (commonArgsFor p); + nightlyCargoArtifactsFor = p: (nightlyCraneLibFor p).buildDepsOnly (commonArgsFor p); nativelinkFor = p: (craneLibFor p).buildPackage ((commonArgsFor p) @@ -291,16 +292,7 @@ nativelinkCoverageFor = p: let coverageArgs = - (commonArgsFor p) - // { - # TODO(palfrey): For some reason we're triggering an edgecase where - # mimalloc builds against glibc headers in coverage - # builds. This leads to nonexistend __memcpy_chk and - # __memset_chk symbols if fortification is enabled. - # Our regular builds also have this issue, but we - # should investigate further. - hardeningDisable = ["fortify"]; - }; + commonArgsFor p; in (nightlyCraneLibFor p).cargoLlvmCov (coverageArgs // { @@ -381,7 +373,12 @@ buck2-with-nativelink-test = pkgs.callPackage integration_tests/buck2/buck2-with-nativelink-test.nix { inherit nativelink buck2; }; - + update-module-hashes = pkgs.callPackage tools/updaters/rewrite-module.nix { + python-with-requests = pkgs.python3.withPackages (ps: + with ps; [ + ps.requests + ]); + }; generate-bazel-rc = pkgs.callPackage tools/generate-bazel-rc/build.nix {craneLib = craneLibFor pkgs;}; generate-stores-config = pkgs.callPackage nativelink-config/generate-stores-config/build.nix {craneLib = craneLibFor pkgs;}; } @@ -466,6 +463,7 @@ pkgs.pre-commit pkgs.git-cliff pkgs.buck2 + packages.update-module-hashes # Rust bazel diff --git a/tools/nativelink-is-executable-test.nix b/tools/nativelink-is-executable-test.nix index 97cf52904..7de06473e 100644 --- a/tools/nativelink-is-executable-test.nix +++ b/tools/nativelink-is-executable-test.nix @@ -3,7 +3,7 @@ writeShellScriptBin, }: writeShellScriptBin "is-executable-test" '' - set -xuo pipefail + set -uo pipefail nativelink_output="$(${nativelink}/bin/nativelink 2>&1)" @@ -14,7 +14,8 @@ writeShellScriptBin "is-executable-test" '' Usage: nativelink For more information, try '--help'. - EOF) + EOF + ) if [ "$nativelink_output" = "$print_error_output" ]; then echo "The output of nativelink matches the print_error output." diff --git a/tools/updaters/cache/.gitignore b/tools/updaters/cache/.gitignore new file mode 100644 index 000000000..b5624b74f --- /dev/null +++ b/tools/updaters/cache/.gitignore @@ -0,0 +1 @@ +*.tar.xz diff --git a/tools/updaters/rewrite-module.nix b/tools/updaters/rewrite-module.nix new file mode 100644 index 000000000..4ea58591b --- /dev/null +++ b/tools/updaters/rewrite-module.nix @@ -0,0 +1,9 @@ +{ + python-with-requests, + writeShellScriptBin, +}: +writeShellScriptBin "update-module-hashes" '' + set -uo pipefail + + ${python-with-requests}/bin/python tools/updaters/rewrite-module.py MODULE.bazel +'' diff --git a/tools/updaters/rewrite-module.py b/tools/updaters/rewrite-module.py new file mode 100644 index 000000000..bc4cf7770 --- /dev/null +++ b/tools/updaters/rewrite-module.py @@ -0,0 +1,36 @@ +import re +import subprocess +import requests +import sys +import pathlib + +module_bazel_path = sys.argv[1] +cache_dir = pathlib.Path(__file__).parent.joinpath("cache") +cache_dir.mkdir(exist_ok=True) + +original = open(module_bazel_path).read() +begin_shas = re.search("# BEGIN SHAS\n", original).end() # pyright: ignore[reportOptionalMemberAccess] +end_shas = re.search("\n # END SHAS", original).start() # pyright: ignore[reportOptionalMemberAccess] +print(begin_shas, end_shas) +sha_pattern = re.compile(r"\"(.+\.tar\.xz)\": \"([0-9a-f]+)\"") + +results = "" + +for entry in sha_pattern.finditer(original, begin_shas, end_shas): + short_url, hash = entry.groups() + cache_path = cache_dir.joinpath(short_url.replace("/", "_")) + if not cache_path.exists(): + full_url = f"https://static.rust-lang.org/dist/{short_url}" + print("getting", full_url, cache_path) + req = requests.get(full_url) + with cache_path.open("wb") as f: + f.write(req.content) + sha256_cmd = subprocess.check_output(["sha256sum", cache_path.as_posix()], encoding="utf-8") + sha256 = sha256_cmd.split(" ")[0] + if results != "": + results += "\n" + results += f" \"{short_url}\": \"{sha256}\"," + +revised = original[:begin_shas] + results + original[end_shas:] +with open(module_bazel_path, "w") as f: + f.write(revised)