From 84335098d48cd8f4049153369b0902300116dc1a Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 11 May 2020 14:35:37 +0200 Subject: [PATCH 01/14] update rules_nixpkgs --- bazel_tools/nixpkgs-disable-http2.patch | 12 ++++-------- deps.bzl | 8 ++++---- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/bazel_tools/nixpkgs-disable-http2.patch b/bazel_tools/nixpkgs-disable-http2.patch index 3a8c9248c5ef..713c7283a3e5 100644 --- a/bazel_tools/nixpkgs-disable-http2.patch +++ b/bazel_tools/nixpkgs-disable-http2.patch @@ -1,8 +1,8 @@ diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl -index 263bb37..d363dfb 100644 +index fb87da4..8f30e51 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl -@@ -88,7 +88,7 @@ def _nixpkgs_package_impl(repository_ctx): +@@ -104,7 +104,7 @@ def _nixpkgs_package_impl(repository_ctx): "The NIX_PATH environment variable is not inherited." ) @@ -11,15 +11,11 @@ index 263bb37..d363dfb 100644 if repository_ctx.attr.nix_file and repository_ctx.attr.nix_file_content: fail("Specify one of 'nix_file' or 'nix_file_content', but not both.") elif repository_ctx.attr.nix_file: -diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl -index 263bb37..3360b9b 100644 ---- a/nixpkgs/nixpkgs.bzl -+++ b/nixpkgs/nixpkgs.bzl -@@ -117,6 +117,7 @@ def _nixpkgs_package_impl(repository_ctx): +@@ -133,6 +133,7 @@ def _nixpkgs_package_impl(repository_ctx): "bazel-support/nix-out-link", ]) + expr_args.extend(["--option", "http2", "false"]) expr_args.extend(repository_ctx.attr.nixopts) - # If repositories is not set, leave empty so nix will fail + for repo in repositories.keys(): diff --git a/deps.bzl b/deps.bzl index d358b2e2f88a..90c0388472ce 100644 --- a/deps.bzl +++ b/deps.bzl @@ -31,10 +31,10 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") rules_scala_version = "6c16cff213b76a4126bdc850956046da5db1daaa" -rules_haskell_version = "76f7b1c7328f59bf2f38b56bce0036153c65a40d" -rules_haskell_sha256 = "5553f76d40db672b499a76fd56fab75df6fe465531f2eedd7da39a3deaed54a6" -rules_nixpkgs_version = "c966bb8bd335f1e244c03efe6e7a1afa9784038e" -rules_nixpkgs_sha256 = "ccafea4fc4d5fa2ddba2882f76728558bfe2c12657f7f56078ece43a31761148" +rules_haskell_version = "ac87721a4dbc0f7dbe731df928d322f02ed93330" +rules_haskell_sha256 = "684f91defad36e9d6ce3ac4213864b89e8f6fe813508ae93bfe80996447a1516" +rules_nixpkgs_version = "d3c7bc94fed4001d5375632a936d743dc085c9a1" +rules_nixpkgs_sha256 = "903c6b98aa6a298bf45a6b931e77a3313c40a0cb1b44fa00d9792f9e8aedbb35" buildifier_version = "0.26.0" buildifier_sha256 = "86592d703ecbe0c5cbb5139333a63268cf58d7efd2c459c8be8e69e77d135e29" zlib_version = "cacf7f1d4e3d44d871b605da3b647f07d718623f" From 9ecb86e53ad9d4d7bd4de74a29fad85eb627e0e9 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 11 May 2020 15:07:31 +0200 Subject: [PATCH 02/14] Use hermetic nixpkgs cc toolchain CHANGELOG_BEGIN CHANGELOG_END --- .bazelrc | 4 + WORKSPACE | 5 +- bazel_tools/nixpkgs-disable-http2.patch | 13 +- .../nixpkgs-hermetic-cc-toolchain.patch | 1144 +++++++++++++++++ deps.bzl | 3 + nix/bazel-cc-toolchain.nix | 3 +- 6 files changed, 1161 insertions(+), 11 deletions(-) create mode 100644 bazel_tools/nixpkgs-hermetic-cc-toolchain.patch diff --git a/.bazelrc b/.bazelrc index fb85b7c52d7b..d844a1b49b6e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -77,6 +77,10 @@ build:darwin --host_platform=@rules_haskell//haskell/platforms:darwin_x86_64_nix # and GHC's gcc on Windows build:windows --crosstool_top=@rules_haskell_ghc_windows_amd64//:cc_toolchain +# Required for hermetic nixpkgs cc toolchain. +build:linux --incompatible_enable_cc_toolchain_resolution +build:darwin --incompatible_enable_cc_toolchain_resolution + # Bazel 1.0 disabled the bash test-runner on Windows. However, some of our # test-cases are implemented as bash scripts and rely on the bash test-runner. build:windows --noincompatible_windows_native_test_wrapper diff --git a/WORKSPACE b/WORKSPACE index ac39e57a2d4e..3a59d6fece2a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -33,7 +33,7 @@ register_toolchains( load("//bazel_tools/dev_env_tool:dev_env_tool.bzl", "dadew", "dev_env_tool") load( "@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", - "nixpkgs_cc_configure", + "nixpkgs_cc_configure_hermetic", "nixpkgs_local_repository", "nixpkgs_package", "nixpkgs_python_configure", @@ -98,10 +98,9 @@ common_nix_file_deps = [ ] # Use Nix provisioned cc toolchain -nixpkgs_cc_configure( +nixpkgs_cc_configure_hermetic( nix_file = "//nix:bazel-cc-toolchain.nix", nix_file_deps = common_nix_file_deps + [ - "//nix:bazel-cc-toolchain.nix", "//nix:tools/bazel-cc-toolchain/default.nix", ], repositories = dev_env_nix_repos, diff --git a/bazel_tools/nixpkgs-disable-http2.patch b/bazel_tools/nixpkgs-disable-http2.patch index 713c7283a3e5..a6deff1dbb11 100644 --- a/bazel_tools/nixpkgs-disable-http2.patch +++ b/bazel_tools/nixpkgs-disable-http2.patch @@ -1,8 +1,8 @@ diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl -index fb87da4..8f30e51 100644 +index 2f2c83d..61123c6 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl -@@ -104,7 +104,7 @@ def _nixpkgs_package_impl(repository_ctx): +@@ -156,7 +156,7 @@ def _nixpkgs_package_impl(repository_ctx): "The NIX_PATH environment variable is not inherited." ) @@ -11,11 +11,12 @@ index fb87da4..8f30e51 100644 if repository_ctx.attr.nix_file and repository_ctx.attr.nix_file_content: fail("Specify one of 'nix_file' or 'nix_file_content', but not both.") elif repository_ctx.attr.nix_file: -@@ -133,6 +133,7 @@ def _nixpkgs_package_impl(repository_ctx): +@@ -186,6 +186,8 @@ def _nixpkgs_package_impl(repository_ctx): "bazel-support/nix-out-link", ]) + expr_args.extend(["--option", "http2", "false"]) - expr_args.extend(repository_ctx.attr.nixopts) - - for repo in repositories.keys(): ++ + if repository_ctx.attr.expand_location: + expr_args.extend([ + _expand_location(repository_ctx, opt, nix_file_deps, "nixopts") diff --git a/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch b/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch new file mode 100644 index 000000000000..52093a596624 --- /dev/null +++ b/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch @@ -0,0 +1,1144 @@ +diff --git a/.bazelrc b/.bazelrc +index d226e65..cf344ef 100644 +--- a/.bazelrc ++++ b/.bazelrc +@@ -1 +1,2 @@ + build --host_platform=@io_tweag_rules_nixpkgs//nixpkgs/platforms:host ++build --incompatible_enable_cc_toolchain_resolution +diff --git a/BUILD.bazel b/BUILD.bazel +index e69de29..efcf274 100644 +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -0,0 +1,4 @@ ++exports_files([ ++ "nixpkgs.json", ++ "nixpkgs.nix", ++]) +diff --git a/CHANGELOG.md b/CHANGELOG.md +index 8596633..234c41b 100644 +--- a/CHANGELOG.md ++++ b/CHANGELOG.md +@@ -4,6 +4,27 @@ All notable changes to this project will be documented in this file. + + The format is based on [Keep a Changelog](https://keepachangelog.com/). + ++## [Unreleased] ++ ++[Unreleased]: https://github.com/tweag/rules_nixpkgs/compare/v0.7.0...HEAD ++ ++### Added ++ ++- Add `expand_location` attribute to `nixpkgs_package`. When enabled instances ++ of `$(location LABEL)` in the `nixopts` attribute will be expanded to the ++ file path of the file referenced by `LABEL`. ++ See [#128][#128]. ++- Define `nixpkgs_cc_configure_hermetic` in `//nixpkgs:repositories.bzl`. ++ See [#128][#128]. ++ ++### Deprecated ++ ++- `nixpkgs_cc_configure` has been deprecated in favor of ++ `nixpkgs_cc_configure_hermetic` and will be replaced by it in future. ++ See [#128][#128]. ++ ++[#128]: https://github.com/tweag/rules_nixpkgs/pull/128 ++ + ## [0.7.0] - 2020-04-20 + + [0.7.0]: https://github.com/tweag/rules_nixpkgs/compare/v0.6.0...v0.7.0 +diff --git a/README.md b/README.md +index 721f64d..ddd23d2 100644 +--- a/README.md ++++ b/README.md +@@ -176,7 +176,7 @@ Make the content of a Nixpkgs package available in the Bazel workspace. + nixpkgs_package( + name, attribute_path, nix_file, nix_file_deps, nix_file_content, + repository, repositories, build_file, build_file_content, nixopts, +- fail_not_supported, ++ expand_location, fail_not_supported, + ) + ``` + +@@ -306,6 +306,18 @@ filegroup( +

Extra flags to pass when calling Nix.

+ + ++ ++ nixopts ++ ++

Bool; optional

++

++ If set to True any instance of ++ $(location LABEL) in nixopts ++ will be replaced by the path to the file referenced by ++ LABEL relative to the workspace root. ++

++ ++ + + fail_not_supported + +@@ -321,6 +333,97 @@ filegroup( + + + ++### nixpkgs_cc_configure_hermetic ++ ++Use a CC toolchain from Nixpkgs. No-op if not a nix-based platform. ++ ++By default, Bazel auto-configures a CC toolchain from commands (e.g. ++`gcc`) available in the environment. To make builds more hermetic, use ++this rule to specify explicitly which commands the toolchain should use. ++ ++Specifically, it builds a Nix derivation that provides the CC toolchain tools ++in the `bin/` path and constructs a CC toolchain that uses those tools. ++ ++Note: ++ ++This requires `--incompatible_enable_cc_toolchain_resolution` to be ++enabled. See https://github.com/bazelbuild/bazel/issues/7260. ++Alternatively, you can set `--crosstool_top=@_config//:toolchain`. ++ ++Example: ++ ++```bzl ++nixpkgs_cc_configure_hermetic(repository = "@nixpkgs//:default.nix") ++``` ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++
Attributes
attribute_path ++

String; optional

++

Obtain the toolchain from the Nix expression under this attribute path. Requires `nix_file` or `nix_file_content`.

++ 		nix_file ++

String; optional

++

Obtain the toolchain from the Nix expression defined in this file. Specify only one of `nix_file` or `nix_file_content`.

++
nix_file_content ++

String; optional

++

Obtain the toolchain from the given Nix expression. Specify only one of `nix_file` or `nix_file_content`.

++
nix_file_deps ++

List of labels; optional

++

Additional files that the Nix expression depends on.

++
repository ++

Label; optional

++

Provides ``. Specify one of `repositories` or `repository`.

++
repositories ++

String-keyed label dict; optional

++

Provides `` and other repositories. Specify one of `repositories` or `repository`.

++
quiet ++

Bool; optional

++

Whether to hide `nix-build` output.

++
fail_not_supported ++

Bool; optional

++

Whether to fail if `nix-build` is not available.

++
++ + ### nixpkgs_cc_configure + + Tells Bazel to use compilers and linkers from Nixpkgs for the CC +@@ -328,6 +431,15 @@ toolchain. By default, Bazel autodetects a toolchain on the current + `PATH`. Overriding this autodetection makes builds more hermetic and + is considered a best practice. + ++Deprecated: ++ ++Use `nixpkgs_cc_configure_hermetic` instead. ++ ++This uses Bazel's autoconfigure toolchain under the hood, which is ++inhermetic. In particular, system include directories specified in the ++environment can leak in and affect the cache keys of target depending on ++the cc toolchain leading to cache misses. ++ + Example: + + ```bzl +diff --git a/WORKSPACE b/WORKSPACE +index f37ab6c..f8d40ad 100644 +--- a/WORKSPACE ++++ b/WORKSPACE +@@ -6,7 +6,7 @@ rules_nixpkgs_dependencies() + + load( + "//nixpkgs:nixpkgs.bzl", +- "nixpkgs_cc_configure", ++ "nixpkgs_cc_configure_hermetic", + "nixpkgs_git_repository", + "nixpkgs_local_repository", + "nixpkgs_package", +@@ -134,7 +134,7 @@ nixpkgs_package( + repository = "@nixpkgs", + ) + +-nixpkgs_cc_configure(repository = "@remote_nixpkgs") ++nixpkgs_cc_configure_hermetic(repository = "@remote_nixpkgs") + + nixpkgs_python_configure( + python2_attribute_path = "python2", +@@ -147,6 +147,27 @@ nixpkgs_package( + repository = "@remote_nixpkgs", + ) + ++nixpkgs_package( ++ name = "nixpkgs_location_expansion_test", ++ build_file_content = "exports_files(glob(['out/**']))", ++ expand_location = True, ++ nix_file = "//tests:location_expansion.nix", ++ nix_file_deps = [ ++ "//:nixpkgs.json", ++ "//:nixpkgs.nix", ++ "@io_tweag_rules_nixpkgs//tests:relative_imports/nixpkgs.nix", ++ ], ++ nixopts = [ ++ "--arg", ++ "attrs", ++ "{ nixpkgs_json = $(location //:nixpkgs.json); nixpkgs_nix = $(location //:nixpkgs.nix); }", ++ "--arg", ++ "relative_imports", ++ "$(location @io_tweag_rules_nixpkgs//tests:relative_imports/nixpkgs.nix)", ++ ], ++ repository = "@remote_nixpkgs", ++) ++ + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + + http_archive( +@@ -179,12 +200,11 @@ http_archive( + + load( + "//nixpkgs:toolchains/go.bzl", +- "nixpkgs_go_configure" ++ "nixpkgs_go_configure", + ) + + nixpkgs_go_configure(repository = "@nixpkgs") + +-load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains") ++load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") + + go_rules_dependencies() +- +diff --git a/nixpkgs/constraints/BUILD.bazel b/nixpkgs/constraints/BUILD.bazel +index 6662530..94678bd 100644 +--- a/nixpkgs/constraints/BUILD.bazel ++++ b/nixpkgs/constraints/BUILD.bazel +@@ -3,7 +3,7 @@ package(default_visibility = ["//visibility:public"]) + constraint_setting(name = "nix") + + constraint_value( +- name = "support_nix", ++ name = "support_nix", + constraint_setting = ":nix", + ) + +diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl +index fb87da4..52bbfcf 100644 +--- a/nixpkgs/nixpkgs.bzl ++++ b/nixpkgs/nixpkgs.bzl +@@ -1,7 +1,14 @@ + """Rules for importing Nixpkgs packages.""" + ++load("@bazel_skylib//lib:paths.bzl", "paths") ++load("@bazel_skylib//lib:sets.bzl", "sets") + load("@bazel_tools//tools/cpp:cc_configure.bzl", "cc_autoconf_impl") +-load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value") ++load( ++ "@bazel_tools//tools/cpp:lib_cc_configure.bzl", ++ "get_cpu_value", ++ "get_starlark_list", ++ "write_builtin_include_directory_paths", ++) + + def _nixpkgs_git_repository_impl(repository_ctx): + repository_ctx.file("BUILD") +@@ -70,6 +77,54 @@ nixpkgs_local_repository = repository_rule( + def _is_supported_platform(repository_ctx): + return repository_ctx.which("nix-build") != None + ++def _expand_location(repository_ctx, string, labels, attr = None): ++ """Expand `$(location label)` to a path. ++ ++ Attrs: ++ repository_ctx: The repository rule context. ++ string: string, Replace instances of `$(location )` in this string. ++ labels: dict from label to path: Known label to path mappings. ++ attr: string, The rule attribute to use for error reporting. ++ ++ Returns: ++ The string with all instances of `$(location )` replaced by paths. ++ """ ++ num = string.count("$(location ") ++ result = "" ++ offset = 0 ++ for i in range(num): ++ start = string.find("$(location ", offset) ++ label_start = start + len("$(location ") ++ label_end = string.find(")", label_start) ++ if label_end == -1: ++ fail("Unbalanced parentheses in location expansion for '{}'.".format(string[start:]), attr) ++ end = label_end + 1 ++ label_str = string[label_start:label_end] ++ label_candidates = [ ++ (lbl, path) ++ for (lbl, path) in labels.items() ++ if lbl.relative(label_str) == lbl ++ ] ++ if len(label_candidates) == 0: ++ fail("Unknown label '{}' in location expansion for '{}'.".format(label_str, string), attr) ++ elif len(label_candidates) > 1: ++ fail( ++ "Ambiguous label '{}' in location expansion for '{}'. Candidates: {}".format( ++ label_str, ++ string, ++ ", ".join([str(lbl) for lbl in label_candidates]), ++ ), ++ attr, ++ ) ++ location = paths.join(".", paths.relativize( ++ str(repository_ctx.path(label_candidates[0][1])), ++ str(repository_ctx.path(".")), ++ )) ++ result += string[offset:start] + location ++ offset = end ++ result += string[offset:] ++ return result ++ + def _nixpkgs_package_impl(repository_ctx): + repository = repository_ctx.attr.repository + repositories = repository_ctx.attr.repositories +@@ -117,8 +172,9 @@ def _nixpkgs_package_impl(repository_ctx): + else: + expr_args = ["-E", "import { config = {}; overlays = []; }"] + ++ nix_file_deps = {} + for dep in repository_ctx.attr.nix_file_deps: +- _cp(repository_ctx, dep) ++ nix_file_deps[dep] = _cp(repository_ctx, dep) + + expr_args.extend([ + "-A", +@@ -133,7 +189,13 @@ def _nixpkgs_package_impl(repository_ctx): + "bazel-support/nix-out-link", + ]) + +- expr_args.extend(repository_ctx.attr.nixopts) ++ if repository_ctx.attr.expand_location: ++ expr_args.extend([ ++ _expand_location(repository_ctx, opt, nix_file_deps, "nixopts") ++ for opt in repository_ctx.attr.nixopts ++ ]) ++ else: ++ expr_args.extend(repository_ctx.attr.nixopts) + + for repo in repositories.keys(): + path = str(repository_ctx.path(repo).dirname) + "/nix-file-deps" +@@ -206,7 +268,7 @@ def _nixpkgs_package_impl(repository_ctx): + if create_build_file_if_needed: + p = repository_ctx.path("BUILD") + if not p.exists: +- repository_ctx.template("BUILD", Label("@io_tweag_rules_nixpkgs//nixpkgs:BUILD.pkg")) ++ repository_ctx.template("BUILD", Label("@io_tweag_rules_nixpkgs//nixpkgs:BUILD.pkg")) + + _nixpkgs_package = repository_rule( + implementation = _nixpkgs_package_impl, +@@ -220,6 +282,7 @@ _nixpkgs_package = repository_rule( + "build_file": attr.label(), + "build_file_content": attr.string(), + "nixopts": attr.string_list(), ++ "expand_location": attr.bool(default = False), + "quiet": attr.bool(), + "fail_not_supported": attr.bool(default = True, doc = """ + If set to True (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to False calling this rule will succeed but no output will be generated. +@@ -244,6 +307,298 @@ def nixpkgs_package(*args, **kwargs): + else: + _nixpkgs_package(*args, **kwargs) + ++def _parse_cc_toolchain_info(content, filename): ++ """Parses the `CC_TOOLCHAIN_INFO` file generated by Nix. ++ ++ Attrs: ++ content: string, The content of the `CC_TOOLCHAIN_INFO` file. ++ filename: string, The path to the `CC_TOOLCHAIN_INFO` file, used for error reporting. ++ ++ Returns: ++ struct, The substitutions for `@bazel_tools//tools/cpp:BUILD.tpl`. ++ """ ++ ++ # Parse the content of CC_TOOLCHAIN_INFO. ++ # ++ # Each line has the form ++ # ++ # ... ++ info = {} ++ for line in content.splitlines(): ++ fields = line.split(" ") ++ if len(fields) == 0: ++ fail( ++ "Malformed CC_TOOLCHAIN_INFO '{}': Empty line encountered.".format(filename), ++ "cc_toolchain_info", ++ ) ++ info[fields[0]] = fields[1:] ++ ++ # Validate the keys in CC_TOOLCHAIN_INFO. ++ expected_keys = sets.make([ ++ "TOOL_NAMES", ++ "TOOL_PATHS", ++ "CXX_BUILTIN_INCLUDE_DIRECTORIES", ++ "COMPILER_FLAGS", ++ "CXX_FLAGS", ++ "LINK_FLAGS", ++ "LINK_LIBS", ++ "OPT_COMPILE_FLAGS", ++ "OPT_LINK_FLAGS", ++ "UNFILTERED_COMPILE_FLAGS", ++ "DBG_COMPILE_FLAGS", ++ "COVERAGE_COMPILE_FLAGS", ++ "COVERAGE_LINK_FLAGS", ++ "SUPPORTS_START_END_LIB", ++ ]) ++ actual_keys = sets.make(info.keys()) ++ missing_keys = sets.difference(expected_keys, actual_keys) ++ unexpected_keys = sets.difference(actual_keys, expected_keys) ++ if sets.length(missing_keys) > 0: ++ fail( ++ "Malformed CC_TOOLCHAIN_INFO '{}': Missing entries '{}'.".format( ++ filename, ++ "', '".join(sets.to_list(missing_keys)), ++ ), ++ "cc_toolchain_info", ++ ) ++ if sets.length(unexpected_keys) > 0: ++ fail( ++ "Malformed CC_TOOLCHAIN_INFO '{}': Unexpected entries '{}'.".format( ++ filename, ++ "', '".join(sets.to_list(unexpected_keys)), ++ ), ++ "cc_toolchain_info", ++ ) ++ ++ return struct( ++ tool_paths = { ++ tool: path ++ for (tool, path) in zip(info["TOOL_NAMES"], info["TOOL_PATHS"]) ++ }, ++ cxx_builtin_include_directories = info["CXX_BUILTIN_INCLUDE_DIRECTORIES"], ++ compiler_flags = info["COMPILER_FLAGS"], ++ cxx_flags = info["CXX_FLAGS"], ++ link_flags = info["LINK_FLAGS"], ++ link_libs = info["LINK_LIBS"], ++ opt_compile_flags = info["OPT_COMPILE_FLAGS"], ++ opt_link_flags = info["OPT_LINK_FLAGS"], ++ unfiltered_compile_flags = info["UNFILTERED_COMPILE_FLAGS"], ++ dbg_compile_flags = info["DBG_COMPILE_FLAGS"], ++ coverage_compile_flags = info["COVERAGE_COMPILE_FLAGS"], ++ coverage_link_flags = info["COVERAGE_LINK_FLAGS"], ++ supports_start_end_lib = info["SUPPORTS_START_END_LIB"] == "True", ++ ) ++ ++def _nixpkgs_cc_toolchain_config_impl(repository_ctx): ++ cpu_value = get_cpu_value(repository_ctx) ++ darwin = cpu_value == "darwin" ++ ++ cc_toolchain_info_file = repository_ctx.path(repository_ctx.attr.cc_toolchain_info) ++ if not cc_toolchain_info_file.exists and not repository_ctx.attr.fail_not_supported: ++ return ++ info = _parse_cc_toolchain_info( ++ repository_ctx.read(cc_toolchain_info_file), ++ cc_toolchain_info_file, ++ ) ++ ++ # Generate the cc_toolchain workspace following the example from ++ # `@bazel_tools//tools/cpp:unix_cc_configure.bzl`. ++ repository_ctx.symlink( ++ repository_ctx.path(repository_ctx.attr._unix_cc_toolchain_config), ++ "cc_toolchain_config.bzl", ++ ) ++ repository_ctx.symlink( ++ repository_ctx.path(repository_ctx.attr._armeabi_cc_toolchain_config), ++ "armeabi_cc_toolchain_config.bzl", ++ ) ++ cc_wrapper_src = ( ++ repository_ctx.attr._osx_cc_wrapper if darwin else repository_ctx.attr._linux_cc_wrapper ++ ) ++ repository_ctx.template( ++ "cc_wrapper.sh", ++ repository_ctx.path(cc_wrapper_src), ++ { ++ "%{cc}": info.tool_paths["gcc"], ++ "%{env}": "", ++ }, ++ ) ++ if darwin: ++ info.tool_paths["gcc"] = "cc_wrapper.sh" ++ info.tool_paths["ar"] = "/usr/bin/libtool" ++ write_builtin_include_directory_paths( ++ repository_ctx, ++ info.tool_paths["gcc"], ++ info.cxx_builtin_include_directories, ++ ) ++ repository_ctx.template( ++ "BUILD.bazel", ++ repository_ctx.path(repository_ctx.attr._build), ++ { ++ "%{cc_toolchain_identifier}": "local", ++ "%{name}": cpu_value, ++ "%{supports_param_files}": "0" if darwin else "1", ++ "%{cc_compiler_deps}": get_starlark_list( ++ [":builtin_include_directory_paths"] + ( ++ [":cc_wrapper"] if darwin else [] ++ ), ++ ), ++ "%{compiler}": "compiler", ++ "%{abi_version}": "local", ++ "%{abi_libc_version}": "local", ++ "%{host_system_name}": "local", ++ "%{target_libc}": "macosx" if darwin else "local", ++ "%{target_cpu}": cpu_value, ++ "%{target_system_name}": "local", ++ "%{tool_paths}": ",\n ".join( ++ ['"%s": "%s"' % (k, v) for (k, v) in info.tool_paths.items()], ++ ), ++ "%{cxx_builtin_include_directories}": get_starlark_list(info.cxx_builtin_include_directories), ++ "%{compile_flags}": get_starlark_list(info.compiler_flags), ++ "%{cxx_flags}": get_starlark_list(info.cxx_flags), ++ "%{link_flags}": get_starlark_list(info.link_flags), ++ "%{link_libs}": get_starlark_list(info.link_libs), ++ "%{opt_compile_flags}": get_starlark_list(info.opt_compile_flags), ++ "%{opt_link_flags}": get_starlark_list(info.opt_link_flags), ++ "%{unfiltered_compile_flags}": get_starlark_list(info.unfiltered_compile_flags), ++ "%{dbg_compile_flags}": get_starlark_list(info.dbg_compile_flags), ++ "%{coverage_compile_flags}": get_starlark_list(info.coverage_compile_flags), ++ "%{coverage_link_flags}": get_starlark_list(info.coverage_link_flags), ++ "%{supports_start_end_lib}": repr(info.supports_start_end_lib), ++ }, ++ ) ++ ++_nixpkgs_cc_toolchain_config = repository_rule( ++ _nixpkgs_cc_toolchain_config_impl, ++ attrs = { ++ "cc_toolchain_info": attr.label(), ++ "fail_not_supported": attr.bool(), ++ "_unix_cc_toolchain_config": attr.label( ++ default = Label("@bazel_tools//tools/cpp:unix_cc_toolchain_config.bzl"), ++ ), ++ "_armeabi_cc_toolchain_config": attr.label( ++ default = Label("@bazel_tools//tools/cpp:armeabi_cc_toolchain_config.bzl"), ++ ), ++ "_osx_cc_wrapper": attr.label( ++ default = Label("@bazel_tools//tools/cpp:osx_cc_wrapper.sh.tpl"), ++ ), ++ "_linux_cc_wrapper": attr.label( ++ default = Label("@bazel_tools//tools/cpp:linux_cc_wrapper.sh.tpl"), ++ ), ++ "_build": attr.label( ++ default = Label("@bazel_tools//tools/cpp:BUILD.tpl"), ++ ), ++ }, ++) ++ ++def _nixpkgs_cc_toolchain_impl(repository_ctx): ++ cpu = get_cpu_value(repository_ctx) ++ repository_ctx.file( ++ "BUILD.bazel", ++ executable = False, ++ content = """\ ++package(default_visibility = ["//visibility:public"]) ++ ++toolchain( ++ name = "toolchain", ++ toolchain = "@{name}_config//:cc-compiler-{cpu}", ++ toolchain_type = "@rules_cc//cc:toolchain_type", ++ exec_compatible_with = [ ++ "@platforms//cpu:x86_64", ++ "@platforms//os:{os}", ++ "@io_tweag_rules_nixpkgs//nixpkgs/constraints:support_nix", ++ ], ++ target_compatible_with = [ ++ "@platforms//cpu:x86_64", ++ "@platforms//os:{os}", ++ ], ++) ++""".format( ++ name = repository_ctx.name, ++ cpu = cpu, ++ os = {"darwin": "osx"}.get(cpu, "linux"), ++ ), ++ ) ++ ++_nixpkgs_cc_toolchain = repository_rule( ++ _nixpkgs_cc_toolchain_impl, ++) ++ ++def nixpkgs_cc_configure_hermetic( ++ name = "nixpkgs_cc_toolchain", ++ attribute_path = "", ++ nix_file = None, ++ nix_file_content = "", ++ nix_file_deps = [], ++ repositories = {}, ++ repository = None, ++ nixopts = [], ++ quiet = False, ++ fail_not_supported = True): ++ """Use a CC toolchain from Nixpkgs. No-op if not a nix-based platform. ++ ++ By default, Bazel auto-configures a CC toolchain from commands (e.g. ++ `gcc`) available in the environment. To make builds more hermetic, use ++ this rule to specify explicitly which commands the toolchain should use. ++ ++ NOTE: ++ This requires `--incompatible_enable_cc_toolchain_resolution` to be ++ enabled. See https://github.com/bazelbuild/bazel/issues/7260. ++ Alternatively, you can set `--crosstool_top=@_config//:toolchain`. ++ ++ Attrs: ++ attribute_path: optional, string, Obtain the toolchain from the Nix expression under this attribute path. Requires `nix_file` or `nix_file_content`. ++ nix_file: optional, Label, Obtain the toolchain from the Nix expression defined in this file. Specify only one of `nix_file` or `nix_file_content`. ++ nix_file_content: optional, string, Obtain the toolchain from the given Nix expression. Specify only one of `nix_file` or `nix_file_content`. ++ nix_file_deps: optional, list of Label, Additional files that the Nix expression depends on. ++ repositories: dict of Label to string, Provides `` and other repositories. Specify one of `repositories` or `repository`. ++ repository: Label, Provides ``. Specify one of `repositories` or `repository`. ++ quiet: bool, Whether to hide `nix-build` output. ++ fail_not_supported: bool, Whether to fail if `nix-build` is not available. ++ """ ++ ++ if attribute_path and not (nix_file or nix_file_content): ++ fail("'attribute_path' requires one of 'nix_file' or 'nix_file_content'", "attribute_path") ++ if nix_file and nix_file_content: ++ fail("Cannot specify both 'nix_file' and 'nix_file_content'.") ++ ++ nixopts = list(nixopts) ++ nix_file_deps = list(nix_file_deps) ++ if attribute_path: ++ # The `attribute_path` is forwarded to `cc.nix` as an argument. ++ nixopts.extend(["--argstr", "attribute_path", attribute_path]) ++ if nix_file: ++ nixopts.extend(["--arg", "nix_expr", "import $(location {})".format(nix_file)]) ++ nix_file_deps.append(nix_file) ++ if nix_file_content: ++ # The `nix_file_content` is forwarded to `cc.nix` as an argument. ++ nixopts.extend(["--arg", "nix_expr", nix_file_content]) ++ ++ # Invoke `toolchains/cc.nix` which generates `CC_TOOLCHAIN_INFO`. ++ nixpkgs_package( ++ name = "{}_cc".format(name), ++ nix_file = "@io_tweag_rules_nixpkgs//nixpkgs:toolchains/cc.nix", ++ nix_file_deps = nix_file_deps, ++ build_file_content = "exports_files(['CC_TOOLCHAIN_INFO'])", ++ repositories = repositories, ++ repository = repository, ++ nixopts = nixopts, ++ expand_location = True, ++ quiet = quiet, ++ fail_not_supported = fail_not_supported, ++ ) ++ ++ # Generate the `cc_toolchain_config` workspace. ++ _nixpkgs_cc_toolchain_config( ++ name = "{}_config".format(name), ++ cc_toolchain_info = "@{}_cc//:CC_TOOLCHAIN_INFO".format(name), ++ fail_not_supported = fail_not_supported, ++ ) ++ ++ # Generate the `cc_toolchain` workspace. ++ _nixpkgs_cc_toolchain(name = name) ++ ++ native.register_toolchains("@{}//:toolchain".format(name)) ++ + def _readlink(repository_ctx, path): + return repository_ctx.path(path).realpath + +@@ -323,6 +678,14 @@ def nixpkgs_cc_configure( + nixopts = []): + """Use a CC toolchain from Nixpkgs. No-op if not a nix-based platform. + ++ Deprecated: ++ Use `nixpkgs_cc_configure_hermetic` instead. ++ ++ This uses Bazel's autoconfigure toolchain under the hood, which is ++ inhermetic. In particular, system include directories specified in the ++ environment can leak in and affect the cache keys of target depending on ++ the cc toolchain leading to cache misses. ++ + By default, Bazel auto-configures a CC toolchain from commands (e.g. + `gcc`) available in the environment. To make builds more hermetic, use + this rule to specific explicitly which commands the toolchain should +diff --git a/nixpkgs/toolchains/cc.nix b/nixpkgs/toolchains/cc.nix +new file mode 100644 +index 0000000..4809408 +--- /dev/null ++++ b/nixpkgs/toolchains/cc.nix +@@ -0,0 +1,242 @@ ++with import { config = {}; overlays = []; }; ++ ++{ attribute_path ? null ++, nix_expr ? null ++}: ++ ++let ++ cc = ++ if isNull nix_expr then ++ buildEnv { ++ name = "bazel-nixpkgs-cc"; ++ # XXX: `gcov` is missing in `/bin`. ++ # It exists in `stdenv.cc.cc` but that collides with `stdenv.cc`. ++ paths = [ stdenv.cc binutils ]; ++ pathsToLink = [ "/bin" ]; ++ } ++ else if isNull attribute_path then ++ nix_expr ++ else ++ lib.attrByPath (lib.splitString "." attribute_path) null nix_expr ++ ; ++in ++ runCommand "bazel-nixpkgs-cc-toolchain" ++ { executable = false; ++ # Pointless to do this on a remote machine. ++ preferLocalBuild = true; ++ allowSubstitutes = false; ++ } ++ '' ++ # This constructs the substitutions for ++ # `@bazel_tools//tools/cpp:BUILD.tpl` following the example of ++ # `@bazel_tools//tools/cpp:unix_cc_configure.bzl` as of Bazel v2.1.0 git ++ # revision 0f4c498a270f05b3896d57055b6489e824821eda. ++ ++ # Determine toolchain tool paths. ++ # ++ # If a tool is not available then we use `bin/false` as a stand-in. ++ declare -A TOOLS=( [ar]=ar [cpp]=cpp [dwp]=dwp [gcc]=cc [gcov]=gcov [ld]=ld [nm]=nm [objcopy]=objcopy [objdump]=objdump [strip]=strip ) ++ TOOL_NAMES=(''${!TOOLS[@]}) ++ declare -A TOOL_PATHS=() ++ for tool_name in ''${!TOOLS[@]}; do ++ tool_path=${cc}/bin/''${TOOLS[$tool_name]} ++ if [[ -x $tool_path ]]; then ++ TOOL_PATHS[$tool_name]=$tool_path ++ else ++ TOOL_PATHS[$tool_name]=${coreutils}/bin/false ++ fi ++ done ++ cc=''${TOOL_PATHS[gcc]} ++ ++ # Check whether a flag is supported by the compiler. ++ # ++ # The logic checks whether the flag causes an error message that contains ++ # the flag (or a pattern) verbatim. The assumption is that this will be a ++ # message of the kind `unknown argument: XYZ`. This logic is copied and ++ # adapted to bash from `@bazel_tools//tools/cpp:unix_cc_configure.bzl`. ++ is_compiler_option_supported() { ++ local option="$1" ++ local pattern="''${2-$1}" ++ { $cc "$option" -o /dev/null -c -x c++ - <<<"int main {}" 2>&1 || true; } | grep -qv -- "$pattern" ++ } ++ is_linker_option_supported() { ++ local option="$1" ++ local pattern="''${2-$1}" ++ { $cc "$option" -o /dev/null -x c++ - <<<"int main {}" 2>&1 || true; } | grep -qv -- "$pattern" ++ } ++ add_compiler_option_if_supported() { ++ if is_compiler_option_supported "$@"; then ++ echo "$1" ++ fi ++ } ++ add_linker_option_if_supported() { ++ if is_linker_option_supported "$@"; then ++ echo "$1" ++ fi ++ } ++ ++ # Determine default include directories. ++ # ++ # This is copied and adapted to bash from ++ # `@bazel_tools//tools/cpp:unix_cc_configure.bzl`. ++ include_dirs_for() { ++ $cc -E -x "$1" - -v "''${@:2}" 2>&1 \ ++ | sed '1,/^#include <...>/d;/^[^ ]/,$d' \ ++ | xargs realpath -m ++ } ++ CXX_BUILTIN_INCLUDE_DIRECTORIES=($({ ++ include_dirs_for c ++ include_dirs_for c++ ++ if is_compiler_option_supported -fno-canonical-system-headers; then ++ include_dirs_for c -fno-canonical-system-headers ++ include_dirs_for c++ -std=c++0x -fno-canonical-system-headers ++ elif is_compiler_option_supported -no-canonical-prefixes; then ++ include_dirs_for c -no-canonical-prefixes ++ include_dirs_for c++ -std=c++0x -no-canonical-prefixes ++ fi ++ } 2>&1 | sort -u)) ++ ++ # Determine list of supported compiler and linker flags. ++ # ++ # This is copied and adapted to bash from ++ # `@bazel_tools//tools/cpp:unix_cc_configure.bzl`. ++ COMPILER_FLAGS=( ++ # Security hardening requires optimization. ++ # We need to undef it as some distributions now have it enabled by default. ++ -U_FORTIFY_SOURCE ++ -fstack-protector ++ # All warnings are enabled. Maybe enable -Werror as well? ++ -Wall ++ $( ++ # Enable a few more warnings that aren't part of -Wall. ++ add_compiler_option_if_supported -Wthread-safety ++ add_compiler_option_if_supported -Wself-assign ++ # Disable problematic warnings. ++ # XXX: Causes clang warning ++ add_compiler_option_if_supported -Wunused-but-set-parameter ++ # has false positives ++ # XXX: Causes clang warning ++ add_compiler_option_if_supported -Wno-free-nonheap-object ++ # Enable coloring even if there's no attached terminal. Bazel removes the ++ # escape sequences if --nocolor is specified. ++ add_compiler_option_if_supported -fcolor-diagnostics ++ ) ++ # Keep stack frames for debugging, even in opt mode. ++ -fno-omit-frame-pointer ++ ) ++ CXX_FLAGS=(-std=c++0x) ++ LINK_FLAGS=( ++ $( ++ # XXX: absolute path is only supported by clang. ++ if [[ -x ${cc}/bin/ld.gold ]]; then echo -fuse-ld=gold; fi ++ add_linker_option_if_supported -Wl,-no-as-needed -no-as-needed ++ add_linker_option_if_supported -Wl,-z,relro,-z,now -z ++ ) ++ ${ ++ if stdenv.isDarwin ++ then "-undefined dynamic_lookup -headerpad_max_install_names" ++ else "-B${cc}/bin" ++ } ++ $( ++ # Have gcc return the exit code from ld. ++ add_compiler_option_if_supported -pass-exit-codes ++ ) ++ -lstdc++ ++ -lm ++ ) ++ LINK_LIBS=() ++ OPT_COMPILE_FLAGS=( ++ # No debug symbols. ++ # Maybe we should enable https://gcc.gnu.org/wiki/DebugFission for opt or ++ # even generally? However, that can't happen here, as it requires special ++ # handling in Bazel. ++ -g0 ++ ++ # Conservative choice for -O ++ # -O3 can increase binary size and even slow down the resulting binaries. ++ # Profile first and / or use FDO if you need better performance than this. ++ -O2 ++ ++ # Security hardening on by default. ++ # Conservative choice; -D_FORTIFY_SOURCE=2 may be unsafe in some cases. ++ -D_FORTIFY_SOURCE=1 ++ ++ # Disable assertions ++ -DNDEBUG ++ ++ # Removal of unused code and data at link time (can this increase binary ++ # size in some cases?). ++ -ffunction-sections ++ -fdata-sections ++ ) ++ OPT_LINK_FLAGS=( ++ ${ ++ if stdenv.isDarwin ++ then "" ++ else "$(add_linker_option_if_supported -Wl,--gc-sections -gc-sections)" ++ } ++ ) ++ UNFILTERED_COMPILE_FLAGS=( ++ $( ++ if is_compiler_option_supported -fno-canonical-system-headers; then ++ echo -fno-canonical-system-headers ++ elif is_compiler_option_supported -no-canonical-prefixes; then ++ echo -no-canonical-prefixes ++ fi ++ ) ++ # Make C++ compilation deterministic. Use linkstamping instead of these ++ # compiler symbols. ++ -Wno-builtin-macro-redefined ++ -D__DATE__=\\\"redacted\\\" ++ -D__TIMESTAMP__=\\\"redacted\\\" ++ -D__TIME__=\\\"redacted\\\" ++ ) ++ DBG_COMPILE_FLAGS=(-g) ++ COVERAGE_COMPILE_FLAGS=( ++ ${ ++ if stdenv.isDarwin then ++ "-fprofile-instr-generate -fcoverage-mapping" ++ else ++ "--coverage" ++ } ++ ) ++ COVERAGE_LINK_FLAGS=( ++ ${ ++ if stdenv.isDarwin then ++ "-fprofile-instr-generate" ++ else ++ "--coverage" ++ } ++ ) ++ SUPPORTS_START_END_LIB=( ++ $( ++ if [[ -x ${cc}/bin/ld.gold ]]; then echo True; else echo False; fi ++ ) ++ ) ++ ++ # Write CC_TOOLCHAIN_INFO ++ # ++ # Each line has the following shape: ++ # ... ++ # I.e. each line contains one space-separated key-value pair, where the ++ # value is a space-separated list. ++ mkdir -p $out ++ write_info() { ++ local -n flags=$1 ++ echo $1 "''${flags[@]}" >>$out/CC_TOOLCHAIN_INFO ++ } ++ write_info TOOL_NAMES ++ write_info TOOL_PATHS ++ write_info CXX_BUILTIN_INCLUDE_DIRECTORIES ++ write_info COMPILER_FLAGS ++ write_info CXX_FLAGS ++ write_info LINK_FLAGS ++ write_info LINK_LIBS ++ write_info OPT_COMPILE_FLAGS ++ write_info OPT_LINK_FLAGS ++ write_info UNFILTERED_COMPILE_FLAGS ++ write_info DBG_COMPILE_FLAGS ++ write_info COVERAGE_COMPILE_FLAGS ++ write_info COVERAGE_LINK_FLAGS ++ write_info SUPPORTS_START_END_LIB ++ '' +diff --git a/nixpkgs/toolchains/go.bzl b/nixpkgs/toolchains/go.bzl +index 79cbb99..d8c6366 100644 +--- a/nixpkgs/toolchains/go.bzl ++++ b/nixpkgs/toolchains/go.bzl +@@ -2,20 +2,19 @@ load( + "@io_bazel_rules_go//go:deps.bzl", + "go_wrap_sdk", + ) +- + load( + "//nixpkgs:nixpkgs.bzl", +- "nixpkgs_package" ++ "nixpkgs_package", + ) + + def nixpkgs_go_configure( +- sdk_name = "go_sdk", +- repository = None, +- repositories = {}, +- nix_file = None, +- nix_file_deps = None, +- nix_file_content = None, +- nixopts = []): ++ sdk_name = "go_sdk", ++ repository = None, ++ repositories = {}, ++ nix_file = None, ++ nix_file_deps = None, ++ nix_file_content = None, ++ nixopts = []): + """ + Use go toolchain from Nixpkgs. Will fail if not a nix-based platform. + +@@ -41,7 +40,6 @@ def nixpkgs_go_configure( + } + """ + +- + nixpkgs_package( + name = "nixpkgs_go_toolchain", + repository = repository, +diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel +index 450810b..d86d6a5 100644 +--- a/tests/BUILD.bazel ++++ b/tests/BUILD.bazel +@@ -1,6 +1,7 @@ + package(default_testonly = 1) + + load("@io_bazel_rules_go//go:def.bzl", "go_binary") ++load("cc-test.bzl", "cc_toolchain_test") + + [ + # All of these tests use the "hello" binary to see +@@ -54,12 +55,41 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary") + ), + ] + +-# Test nixpkgs_cc_configure() by building some CC code. ++# Test nixopts location expansion ++sh_test( ++ name = "location-expansion-test", ++ srcs = ["location_expansion.sh"], ++ args = [ ++ "$(POSIX_DIFF)", ++ "$(rootpath //:nixpkgs.json)", ++ "$(rootpath //:nixpkgs.nix)", ++ "$(rootpath //tests:relative_imports/nixpkgs.nix)", ++ "$(rootpath @nixpkgs_location_expansion_test//:out/nixpkgs.json)", ++ "$(rootpath @nixpkgs_location_expansion_test//:out/nixpkgs.nix)", ++ "$(rootpath @nixpkgs_location_expansion_test//:out/relative_imports.nix)", ++ ], ++ data = [ ++ "//:nixpkgs.json", ++ "//:nixpkgs.nix", ++ "//tests:relative_imports/nixpkgs.nix", ++ "@nixpkgs_location_expansion_test//:out/nixpkgs.json", ++ "@nixpkgs_location_expansion_test//:out/nixpkgs.nix", ++ "@nixpkgs_location_expansion_test//:out/relative_imports.nix", ++ ], ++ toolchains = ["@rules_sh//sh/posix:make_variables"], ++) ++ ++# Test nixpkgs_cc_configure_hermetic() by building some CC code. + cc_binary( + name = "cc-test", + srcs = ["cc-test.cc"], + ) + ++# Test that nixpkgs_cc_configure_hermetic is selected. ++cc_toolchain_test( ++ name = "cc-toolchain", ++) ++ + # Test nixpkgs_python_configure() by running some Python code. + test_suite( + name = "python-test", +@@ -86,5 +116,5 @@ sh_test( + # Test nixpkgs_go_configure() + go_binary( + name = "go-test", +- srcs = ["go-test.go"] ++ srcs = ["go-test.go"], + ) +diff --git a/tests/cc-test.bzl b/tests/cc-test.bzl +new file mode 100644 +index 0000000..a5db3e9 +--- /dev/null ++++ b/tests/cc-test.bzl +@@ -0,0 +1,49 @@ ++load("@bazel_skylib//lib:paths.bzl", "paths") ++load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") ++ ++def _cc_toolchain_test_impl(ctx): ++ cc = find_cpp_toolchain(ctx) ++ executable = ctx.actions.declare_file(ctx.attr.name + ".sh") ++ cc_toolchain_info = ctx.file._cc_toolchain_info ++ cc_toolchain_info_path = ctx.expand_location( ++ "$(rootpath {})".format(str(ctx.attr._cc_toolchain_info.label)), ++ [ctx.attr._cc_toolchain_info], ++ ) ++ ctx.actions.write(executable, content = """\ ++# Find cc in CC_TOOLCHAIN_INFO ++while read -a line; do ++ if [[ ${{line[0]}} = TOOL_PATHS ]]; then ++ for item in ${{line[@]:1}}; do ++ if [[ $item = */bin/cc ]]; then ++ CC=$item ++ fi ++ done ++ fi ++done <{cc_toolchain_info_path} ++if [[ {cc} != $CC ]]; then ++ echo "Expected C compiler '$CC', but found '{cc}'." >&2 ++ exit 1 ++fi ++""".format( ++ cc = cc.compiler_executable, ++ cc_toolchain_info_path = cc_toolchain_info_path, ++ )) ++ return [DefaultInfo( ++ executable = executable, ++ runfiles = ctx.runfiles(files = [ctx.file._cc_toolchain_info]), ++ )] ++ ++cc_toolchain_test = rule( ++ _cc_toolchain_test_impl, ++ attrs = { ++ "_cc_toolchain": attr.label( ++ default = Label("@rules_cc//cc:current_cc_toolchain"), ++ ), ++ "_cc_toolchain_info": attr.label( ++ allow_single_file = True, ++ default = Label("@nixpkgs_cc_toolchain_cc//:CC_TOOLCHAIN_INFO"), ++ ), ++ }, ++ test = True, ++ toolchains = ["@rules_cc//cc:toolchain_type"], ++) +diff --git a/tests/location_expansion.nix b/tests/location_expansion.nix +new file mode 100644 +index 0000000..8023925 +--- /dev/null ++++ b/tests/location_expansion.nix +@@ -0,0 +1,17 @@ ++with import { config = {}; overlays = []; }; ++ ++{ attrs, relative_imports }: ++let ++ inherit (attrs) nixpkgs_json nixpkgs_nix; ++in ++ runCommand "location-expansion" ++ { ++ preferLocalBuild = true; ++ allowSubstitutes = false; ++ } ++ '' ++ mkdir -p $out/out ++ cp ${nixpkgs_json} $out/out/nixpkgs.json ++ cp ${nixpkgs_nix} $out/out/nixpkgs.nix ++ cp ${relative_imports} $out/out/relative_imports.nix ++ '' +diff --git a/tests/location_expansion.sh b/tests/location_expansion.sh +new file mode 100755 +index 0000000..9f4df14 +--- /dev/null ++++ b/tests/location_expansion.sh +@@ -0,0 +1,5 @@ ++DIFF="$1" ++ ++diff "$2" "$5" ++diff "$3" "$6" ++diff "$4" "$7" diff --git a/deps.bzl b/deps.bzl index 90c0388472ce..ed3583bdf010 100644 --- a/deps.bzl +++ b/deps.bzl @@ -89,6 +89,9 @@ def daml_deps(): urls = ["https://github.com/tweag/rules_nixpkgs/archive/%s.tar.gz" % rules_nixpkgs_version], sha256 = rules_nixpkgs_sha256, patches = [ + # Remove once https://github.com/tweag/rules_nixpkgs/pull/128 + # has been merged + "@com_github_digital_asset_daml//bazel_tools:nixpkgs-hermetic-cc-toolchain.patch", # On CI and locally we observe occasional segmantation faults # of nix. A known issue since Nix 2.2.2 is that HTTP2 support # can cause such segmentation faults. Since Nix 2.3.2 it is diff --git a/nix/bazel-cc-toolchain.nix b/nix/bazel-cc-toolchain.nix index a5e7a8421de9..b2c39d555be9 100644 --- a/nix/bazel-cc-toolchain.nix +++ b/nix/bazel-cc-toolchain.nix @@ -1,2 +1 @@ -{ system ? builtins.currentSystem }: -(import ./bazel.nix { inherit system; }).bazel-cc-toolchain +(import ./bazel.nix {}).bazel-cc-toolchain From 435fdd5653cdfa46e9066ab3c4d66ed297701015 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 14 May 2020 11:36:26 +0200 Subject: [PATCH 03/14] Work around Bazel's cc toolchain autodetection --- nix/default.nix | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/nix/default.nix b/nix/default.nix index 5397b48c05dd..ede68442f6d7 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -176,7 +176,22 @@ in rec { >&2 echo "Please run bazel inside of the dev-env" exit 1 fi - export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 + + # Workaround for Bazel insisting on autodecting cc toolchain. + # + # We enable `--incompatible_enable_cc_toolchain_resolution` and define a + # custom CC toolchain using `nixpkgs_cc_configure_hermetic`. Despite this + # Bazel insists on configuring the autodeting cc toolchain which fails if + # it cannot find a C compiler in `$PATH` or `$CC`. See + # https://github.com/bazelbuild/bazel/issues/5133. We can disable the + # autodetection, by setting `BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1`, but + # then toolchain resolution fails with + # + # cc_toolchain_suite '@local_config_cc//:toolchain' does not contain a toolchain for cpu 'k8' + # + # Even though the toolchain in `@local_config_cc` would be unused. + export CC=${cc}/bin/cc + # Set the JAVA_HOME to our JDK export JAVA_HOME=${jdk.home} export GIT_SSL_CAINFO="${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" From 8b1abc7f7b743003a9b8bbe1660fe565e9d194bb Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 14 May 2020 12:05:55 +0200 Subject: [PATCH 04/14] Use --crosstool_top for hermetic cc toolchain When using --incompatible_enable_cc_toolchain_resolution instead cc actions still depend on `external/local_config_cc/builtin_include_directory_paths` as well as `external/nixpkgs_cc_toolchain_config/builtin_include_directory_paths`. --- .bazelrc | 6 ++--- .../nixpkgs-hermetic-cc-toolchain.patch | 24 ++++++++++++------- nix/default.nix | 16 ------------- 3 files changed, 17 insertions(+), 29 deletions(-) diff --git a/.bazelrc b/.bazelrc index d844a1b49b6e..5f387963e289 100644 --- a/.bazelrc +++ b/.bazelrc @@ -73,14 +73,12 @@ build:windows --action_env=JAVA_HOME # Tell bazel to use the nixpkgs Haskell toolchain on Linux and Darwin build:linux --host_platform=@rules_haskell//haskell/platforms:linux_x86_64_nixpkgs +build:linux --crosstool_top=@nixpkgs_cc_toolchain_config//:toolchain build:darwin --host_platform=@rules_haskell//haskell/platforms:darwin_x86_64_nixpkgs +build:darwin --crosstool_top=@nixpkgs_cc_toolchain_config//:toolchain # and GHC's gcc on Windows build:windows --crosstool_top=@rules_haskell_ghc_windows_amd64//:cc_toolchain -# Required for hermetic nixpkgs cc toolchain. -build:linux --incompatible_enable_cc_toolchain_resolution -build:darwin --incompatible_enable_cc_toolchain_resolution - # Bazel 1.0 disabled the bash test-runner on Windows. However, some of our # test-cases are implemented as bash scripts and rely on the bash test-runner. build:windows --noincompatible_windows_native_test_wrapper diff --git a/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch b/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch index 52093a596624..47449279d232 100644 --- a/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch +++ b/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch @@ -271,10 +271,10 @@ index 6662530..94678bd 100644 ) diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl -index fb87da4..52bbfcf 100644 +index fb87da4..7334de0 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl -@@ -1,7 +1,14 @@ +@@ -1,7 +1,15 @@ """Rules for importing Nixpkgs packages.""" +load("@bazel_skylib//lib:paths.bzl", "paths") @@ -287,10 +287,11 @@ index fb87da4..52bbfcf 100644 + "get_starlark_list", + "write_builtin_include_directory_paths", +) ++load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") def _nixpkgs_git_repository_impl(repository_ctx): repository_ctx.file("BUILD") -@@ -70,6 +77,54 @@ nixpkgs_local_repository = repository_rule( +@@ -70,6 +78,54 @@ nixpkgs_local_repository = repository_rule( def _is_supported_platform(repository_ctx): return repository_ctx.which("nix-build") != None @@ -345,7 +346,7 @@ index fb87da4..52bbfcf 100644 def _nixpkgs_package_impl(repository_ctx): repository = repository_ctx.attr.repository repositories = repository_ctx.attr.repositories -@@ -117,8 +172,9 @@ def _nixpkgs_package_impl(repository_ctx): +@@ -117,8 +173,9 @@ def _nixpkgs_package_impl(repository_ctx): else: expr_args = ["-E", "import { config = {}; overlays = []; }"] @@ -356,7 +357,7 @@ index fb87da4..52bbfcf 100644 expr_args.extend([ "-A", -@@ -133,7 +189,13 @@ def _nixpkgs_package_impl(repository_ctx): +@@ -133,7 +190,13 @@ def _nixpkgs_package_impl(repository_ctx): "bazel-support/nix-out-link", ]) @@ -371,7 +372,7 @@ index fb87da4..52bbfcf 100644 for repo in repositories.keys(): path = str(repository_ctx.path(repo).dirname) + "/nix-file-deps" -@@ -206,7 +268,7 @@ def _nixpkgs_package_impl(repository_ctx): +@@ -206,7 +269,7 @@ def _nixpkgs_package_impl(repository_ctx): if create_build_file_if_needed: p = repository_ctx.path("BUILD") if not p.exists: @@ -380,7 +381,7 @@ index fb87da4..52bbfcf 100644 _nixpkgs_package = repository_rule( implementation = _nixpkgs_package_impl, -@@ -220,6 +282,7 @@ _nixpkgs_package = repository_rule( +@@ -220,6 +283,7 @@ _nixpkgs_package = repository_rule( "build_file": attr.label(), "build_file_content": attr.string(), "nixopts": attr.string_list(), @@ -388,7 +389,7 @@ index fb87da4..52bbfcf 100644 "quiet": attr.bool(), "fail_not_supported": attr.bool(default = True, doc = """ If set to True (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to False calling this rule will succeed but no output will be generated. -@@ -244,6 +307,298 @@ def nixpkgs_package(*args, **kwargs): +@@ -244,6 +308,303 @@ def nixpkgs_package(*args, **kwargs): else: _nixpkgs_package(*args, **kwargs) @@ -682,12 +683,17 @@ index fb87da4..52bbfcf 100644 + # Generate the `cc_toolchain` workspace. + _nixpkgs_cc_toolchain(name = name) + ++ maybe( ++ native.bind, ++ name = "cc_toolchain", ++ actual = "@{}_config//:toolchain".format(name), ++ ) + native.register_toolchains("@{}//:toolchain".format(name)) + def _readlink(repository_ctx, path): return repository_ctx.path(path).realpath -@@ -323,6 +678,14 @@ def nixpkgs_cc_configure( +@@ -323,6 +684,14 @@ def nixpkgs_cc_configure( nixopts = []): """Use a CC toolchain from Nixpkgs. No-op if not a nix-based platform. diff --git a/nix/default.nix b/nix/default.nix index ede68442f6d7..99c87b6ec9bb 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -176,22 +176,6 @@ in rec { >&2 echo "Please run bazel inside of the dev-env" exit 1 fi - - # Workaround for Bazel insisting on autodecting cc toolchain. - # - # We enable `--incompatible_enable_cc_toolchain_resolution` and define a - # custom CC toolchain using `nixpkgs_cc_configure_hermetic`. Despite this - # Bazel insists on configuring the autodeting cc toolchain which fails if - # it cannot find a C compiler in `$PATH` or `$CC`. See - # https://github.com/bazelbuild/bazel/issues/5133. We can disable the - # autodetection, by setting `BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1`, but - # then toolchain resolution fails with - # - # cc_toolchain_suite '@local_config_cc//:toolchain' does not contain a toolchain for cpu 'k8' - # - # Even though the toolchain in `@local_config_cc` would be unused. - export CC=${cc}/bin/cc - # Set the JAVA_HOME to our JDK export JAVA_HOME=${jdk.home} export GIT_SSL_CAINFO="${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" From 28ed88f44eb7e4f1d1f419e8bc21ab8095fa6714 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 14 May 2020 13:45:43 +0200 Subject: [PATCH 05/14] override local_config_cc --- .bazelrc | 2 - WORKSPACE | 4 ++ .../nixpkgs-hermetic-cc-toolchain.patch | 57 +++++++++++++------ 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/.bazelrc b/.bazelrc index 5f387963e289..fb85b7c52d7b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -73,9 +73,7 @@ build:windows --action_env=JAVA_HOME # Tell bazel to use the nixpkgs Haskell toolchain on Linux and Darwin build:linux --host_platform=@rules_haskell//haskell/platforms:linux_x86_64_nixpkgs -build:linux --crosstool_top=@nixpkgs_cc_toolchain_config//:toolchain build:darwin --host_platform=@rules_haskell//haskell/platforms:darwin_x86_64_nixpkgs -build:darwin --crosstool_top=@nixpkgs_cc_toolchain_config//:toolchain # and GHC's gcc on Windows build:windows --crosstool_top=@rules_haskell_ghc_windows_amd64//:cc_toolchain diff --git a/WORKSPACE b/WORKSPACE index 3a59d6fece2a..b38a22c213b3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -99,6 +99,10 @@ common_nix_file_deps = [ # Use Nix provisioned cc toolchain nixpkgs_cc_configure_hermetic( + # We override the Bazel's autodetect toolchain to avoid accidentaly + # dependencies on the inhermetic autodetected builtin include paths or + # builds failing due to Bazel not finding `cc` in `$PATH` or `$CC`. + name = "local_config_cc", nix_file = "//nix:bazel-cc-toolchain.nix", nix_file_deps = common_nix_file_deps + [ "//nix:tools/bazel-cc-toolchain/default.nix", diff --git a/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch b/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch index 47449279d232..c685396d4c35 100644 --- a/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch +++ b/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch @@ -47,7 +47,7 @@ index 8596633..234c41b 100644 [0.7.0]: https://github.com/tweag/rules_nixpkgs/compare/v0.6.0...v0.7.0 diff --git a/README.md b/README.md -index 721f64d..ddd23d2 100644 +index 721f64d..71180c4 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ Make the content of a Nixpkgs package available in the Bazel workspace. @@ -97,7 +97,7 @@ index 721f64d..ddd23d2 100644 + +This requires `--incompatible_enable_cc_toolchain_resolution` to be +enabled. See https://github.com/bazelbuild/bazel/issues/7260. -+Alternatively, you can set `--crosstool_top=@_config//:toolchain`. ++Alternatively, you can set `--crosstool_top=@//:toolchain`. + +Example: + @@ -271,7 +271,7 @@ index 6662530..94678bd 100644 ) diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl -index fb87da4..7334de0 100644 +index fb87da4..57ff187 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -1,7 +1,15 @@ @@ -389,7 +389,7 @@ index fb87da4..7334de0 100644 "quiet": attr.bool(), "fail_not_supported": attr.bool(default = True, doc = """ If set to True (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to False calling this rule will succeed but no output will be generated. -@@ -244,6 +308,303 @@ def nixpkgs_package(*args, **kwargs): +@@ -244,6 +308,324 @@ def nixpkgs_package(*args, **kwargs): else: _nixpkgs_package(*args, **kwargs) @@ -585,8 +585,8 @@ index fb87da4..7334de0 100644 +package(default_visibility = ["//visibility:public"]) + +toolchain( -+ name = "toolchain", -+ toolchain = "@{name}_config//:cc-compiler-{cpu}", ++ name = "cc-toolchain-{cpu}", ++ toolchain = "@{cc_toolchain_config}//:cc-compiler-{cpu}", + toolchain_type = "@rules_cc//cc:toolchain_type", + exec_compatible_with = [ + "@platforms//cpu:x86_64", @@ -598,8 +598,23 @@ index fb87da4..7334de0 100644 + "@platforms//os:{os}", + ], +) ++ ++toolchain( ++ name = "cc-toolchain-armeabi-v7a", ++ toolchain = "@{cc_toolchain_config}//:cc-compiler-armeabi-v7a", ++ toolchain_type = "@rules_cc//cc:toolchain_type", ++ exec_compatible_with = [ ++ "@platforms//cpu:x86_64", ++ "@platforms//os:{os}", ++ "@io_tweag_rules_nixpkgs//nixpkgs/constraints:support_nix", ++ ], ++ target_compatible_with = [ ++ "@platforms//cpu:arm", ++ "@platforms//os:android", ++ ], ++) +""".format( -+ name = repository_ctx.name, ++ cc_toolchain_config = repository_ctx.attr.cc_toolchain_config, + cpu = cpu, + os = {"darwin": "osx"}.get(cpu, "linux"), + ), @@ -607,10 +622,13 @@ index fb87da4..7334de0 100644 + +_nixpkgs_cc_toolchain = repository_rule( + _nixpkgs_cc_toolchain_impl, ++ attrs = { ++ "cc_toolchain_config": attr.string(), ++ }, +) + +def nixpkgs_cc_configure_hermetic( -+ name = "nixpkgs_cc_toolchain", ++ name = "nixpkgs_config_cc", + attribute_path = "", + nix_file = None, + nix_file_content = "", @@ -629,7 +647,7 @@ index fb87da4..7334de0 100644 + NOTE: + This requires `--incompatible_enable_cc_toolchain_resolution` to be + enabled. See https://github.com/bazelbuild/bazel/issues/7260. -+ Alternatively, you can set `--crosstool_top=@_config//:toolchain`. ++ Alternatively, you can set `--crosstool_top=@//:toolchain`. + + Attrs: + attribute_path: optional, string, Obtain the toolchain from the Nix expression under this attribute path. Requires `nix_file` or `nix_file_content`. @@ -661,7 +679,7 @@ index fb87da4..7334de0 100644 + + # Invoke `toolchains/cc.nix` which generates `CC_TOOLCHAIN_INFO`. + nixpkgs_package( -+ name = "{}_cc".format(name), ++ name = "{}_info".format(name), + nix_file = "@io_tweag_rules_nixpkgs//nixpkgs:toolchains/cc.nix", + nix_file_deps = nix_file_deps, + build_file_content = "exports_files(['CC_TOOLCHAIN_INFO'])", @@ -675,25 +693,28 @@ index fb87da4..7334de0 100644 + + # Generate the `cc_toolchain_config` workspace. + _nixpkgs_cc_toolchain_config( -+ name = "{}_config".format(name), -+ cc_toolchain_info = "@{}_cc//:CC_TOOLCHAIN_INFO".format(name), ++ name = "{}".format(name), ++ cc_toolchain_info = "@{}_info//:CC_TOOLCHAIN_INFO".format(name), + fail_not_supported = fail_not_supported, + ) + + # Generate the `cc_toolchain` workspace. -+ _nixpkgs_cc_toolchain(name = name) ++ _nixpkgs_cc_toolchain( ++ name = "{}_toolchains".format(name), ++ cc_toolchain_config = name, ++ ) + + maybe( + native.bind, + name = "cc_toolchain", -+ actual = "@{}_config//:toolchain".format(name), ++ actual = "@{}//:toolchain".format(name), + ) -+ native.register_toolchains("@{}//:toolchain".format(name)) ++ native.register_toolchains("@{}_toolchains//:all".format(name)) + def _readlink(repository_ctx, path): return repository_ctx.path(path).realpath -@@ -323,6 +684,14 @@ def nixpkgs_cc_configure( +@@ -323,6 +705,14 @@ def nixpkgs_cc_configure( nixopts = []): """Use a CC toolchain from Nixpkgs. No-op if not a nix-based platform. @@ -1061,7 +1082,7 @@ index 450810b..d86d6a5 100644 ) diff --git a/tests/cc-test.bzl b/tests/cc-test.bzl new file mode 100644 -index 0000000..a5db3e9 +index 0000000..c162e25 --- /dev/null +++ b/tests/cc-test.bzl @@ -0,0 +1,49 @@ @@ -1108,7 +1129,7 @@ index 0000000..a5db3e9 + ), + "_cc_toolchain_info": attr.label( + allow_single_file = True, -+ default = Label("@nixpkgs_cc_toolchain_cc//:CC_TOOLCHAIN_INFO"), ++ default = Label("@nixpkgs_config_cc_info//:CC_TOOLCHAIN_INFO"), + ), + }, + test = True, From 0143c56a5677c771dfd586553d4028fcd26c1cc0 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 14 May 2020 14:08:12 +0200 Subject: [PATCH 06/14] remove unused attribute --- bazel_tools/fat_cc_library.bzl | 8 -------- 1 file changed, 8 deletions(-) diff --git a/bazel_tools/fat_cc_library.bzl b/bazel_tools/fat_cc_library.bzl index 7e5a2568b085..23e902f7ea6a 100644 --- a/bazel_tools/fat_cc_library.bzl +++ b/bazel_tools/fat_cc_library.bzl @@ -113,14 +113,6 @@ fat_cc_library = rule( "_cc_toolchain": attr.label( default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), ), - "_cc_compiler": attr.label( - allow_files = True, - executable = True, - cfg = "host", - default = - # bin/cc is gcc on Darwin which fails to find libc++ - Label("@nixpkgs_cc_toolchain//:bin/clang") if is_darwin else None, - ), "whole_archive_flag": attr.string_list( # ld on MacOS doesn’t understand --whole-archive default = ["-Wl,-all_load"] if is_darwin else ["-Wl,--whole-archive"], From 39e3906eb5a89e4790d8c69b41882232b6194657 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 14 May 2020 14:14:06 +0200 Subject: [PATCH 07/14] Fix posix toolchain on Windows --- bazel_tools/dev_env_tool/dev_env_tool.bzl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bazel_tools/dev_env_tool/dev_env_tool.bzl b/bazel_tools/dev_env_tool/dev_env_tool.bzl index f2ca6db2d8bd..0a151333e6ed 100644 --- a/bazel_tools/dev_env_tool/dev_env_tool.bzl +++ b/bazel_tools/dev_env_tool/dev_env_tool.bzl @@ -197,7 +197,9 @@ def _dadew_sh_posix_config_impl(repository_ctx): load("@rules_sh//sh:posix.bzl", "sh_posix_toolchain") sh_posix_toolchain( name = "dadew_posix", - {commands} + cmds = {{ + {commands}, + }}, ) toolchain( name = "dadew_posix_toolchain", @@ -213,8 +215,8 @@ toolchain( ], ) """.format( - commands = ",\n ".join([ - '{cmd} = r"{path}"'.format(cmd = cmd, path = cmd_path).replace("\\", "/") + commands = ",\n ".join([ + 'r"{cmd}": r"{path}"'.format(cmd = cmd, path = cmd_path).replace("\\", "/") for (cmd, cmd_path) in commands.items() if cmd_path ]), From 5cf75822a9ab81129a5a517618023f3b24e7102c Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 14 May 2020 14:56:56 +0200 Subject: [PATCH 08/14] nixpkgs cc toolchain not on Windows --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index b38a22c213b3..922e8d0fc246 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -108,7 +108,7 @@ nixpkgs_cc_configure_hermetic( "//nix:tools/bazel-cc-toolchain/default.nix", ], repositories = dev_env_nix_repos, -) +) if not is_windows else None nixpkgs_python_configure(repository = "@nixpkgs") if not is_windows else None From 0336c896b08655643026dacc870ee05b2034d178 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Fri, 15 May 2020 13:57:04 +0200 Subject: [PATCH 09/14] Fix nixpkgs cc toolchain on MacOS --- .../nixpkgs-hermetic-cc-toolchain.patch | 132 ++++++++++++------ 1 file changed, 90 insertions(+), 42 deletions(-) diff --git a/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch b/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch index c685396d4c35..e5f747048a84 100644 --- a/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch +++ b/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch @@ -1,10 +1,15 @@ diff --git a/.bazelrc b/.bazelrc -index d226e65..cf344ef 100644 +index d226e65..687c136 100644 --- a/.bazelrc +++ b/.bazelrc -@@ -1 +1,2 @@ +@@ -1 +1,7 @@ build --host_platform=@io_tweag_rules_nixpkgs//nixpkgs/platforms:host -+build --incompatible_enable_cc_toolchain_resolution ++build --crosstool_top=@nixpkgs_config_cc//:toolchain ++# Using toolchain resolution can lead to spurious dependencies on ++# `@local_config_cc//:builtin_include_directory_paths`. This needs to be ++# resolved before `--incompatible_enable_cc_toolchain_resolution` can be ++# recommended for `nixpkgs_cc_configure_hermetic`. ++# build --incompatible_enable_cc_toolchain_resolution diff --git a/BUILD.bazel b/BUILD.bazel index e69de29..efcf274 100644 --- a/BUILD.bazel @@ -47,7 +52,7 @@ index 8596633..234c41b 100644 [0.7.0]: https://github.com/tweag/rules_nixpkgs/compare/v0.6.0...v0.7.0 diff --git a/README.md b/README.md -index 721f64d..71180c4 100644 +index 721f64d..07dbd8c 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ Make the content of a Nixpkgs package available in the Bazel workspace. @@ -78,7 +83,7 @@ index 721f64d..71180c4 100644 fail_not_supported -@@ -321,6 +333,97 @@ filegroup( +@@ -321,6 +333,96 @@ filegroup( @@ -95,9 +100,8 @@ index 721f64d..71180c4 100644 + +Note: + -+This requires `--incompatible_enable_cc_toolchain_resolution` to be -+enabled. See https://github.com/bazelbuild/bazel/issues/7260. -+Alternatively, you can set `--crosstool_top=@//:toolchain`. ++You need to configure `--crosstool_top=@//:toolchain` to activate this ++toolchain. + +Example: + @@ -176,7 +180,7 @@ index 721f64d..71180c4 100644 ### nixpkgs_cc_configure Tells Bazel to use compilers and linkers from Nixpkgs for the CC -@@ -328,6 +431,15 @@ toolchain. By default, Bazel autodetects a toolchain on the current +@@ -328,6 +430,15 @@ toolchain. By default, Bazel autodetects a toolchain on the current `PATH`. Overriding this autodetection makes builds more hermetic and is considered a best practice. @@ -271,7 +275,7 @@ index 6662530..94678bd 100644 ) diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl -index fb87da4..57ff187 100644 +index fb87da4..6bf2d90 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -1,7 +1,15 @@ @@ -389,7 +393,7 @@ index fb87da4..57ff187 100644 "quiet": attr.bool(), "fail_not_supported": attr.bool(default = True, doc = """ If set to True (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to False calling this rule will succeed but no output will be generated. -@@ -244,6 +308,324 @@ def nixpkgs_package(*args, **kwargs): +@@ -244,6 +308,323 @@ def nixpkgs_package(*args, **kwargs): else: _nixpkgs_package(*args, **kwargs) @@ -408,10 +412,10 @@ index fb87da4..57ff187 100644 + # + # Each line has the form + # -+ # ... ++ # :::... + info = {} + for line in content.splitlines(): -+ fields = line.split(" ") ++ fields = line.split(":") + if len(fields) == 0: + fail( + "Malformed CC_TOOLCHAIN_INFO '{}': Empty line encountered.".format(filename), @@ -644,10 +648,9 @@ index fb87da4..57ff187 100644 + `gcc`) available in the environment. To make builds more hermetic, use + this rule to specify explicitly which commands the toolchain should use. + -+ NOTE: -+ This requires `--incompatible_enable_cc_toolchain_resolution` to be -+ enabled. See https://github.com/bazelbuild/bazel/issues/7260. -+ Alternatively, you can set `--crosstool_top=@//:toolchain`. ++ Note: ++ You need to configure `--crosstool_top=@//:toolchain` to activate this ++ toolchain. + + Attrs: + attribute_path: optional, string, Obtain the toolchain from the Nix expression under this attribute path. Requires `nix_file` or `nix_file_content`. @@ -714,7 +717,7 @@ index fb87da4..57ff187 100644 def _readlink(repository_ctx, path): return repository_ctx.path(path).realpath -@@ -323,6 +705,14 @@ def nixpkgs_cc_configure( +@@ -323,6 +704,14 @@ def nixpkgs_cc_configure( nixopts = []): """Use a CC toolchain from Nixpkgs. No-op if not a nix-based platform. @@ -731,10 +734,10 @@ index fb87da4..57ff187 100644 this rule to specific explicitly which commands the toolchain should diff --git a/nixpkgs/toolchains/cc.nix b/nixpkgs/toolchains/cc.nix new file mode 100644 -index 0000000..4809408 +index 0000000..56ddae7 --- /dev/null +++ b/nixpkgs/toolchains/cc.nix -@@ -0,0 +1,242 @@ +@@ -0,0 +1,277 @@ +with import { config = {}; overlays = []; }; + +{ attribute_path ? null @@ -742,15 +745,43 @@ index 0000000..4809408 +}: + +let ++ defaultCcLinux = ++ buildEnv { ++ name = "bazel-nixpkgs-cc"; ++ # XXX: `gcov` is missing in `/bin`. ++ # It exists in `stdenv.cc.cc` but that collides with `stdenv.cc`. ++ paths = [ stdenv.cc binutils ]; ++ pathsToLink = [ "/bin" ]; ++ }; ++ defaultCcDarwin = ++ # Work around https://github.com/NixOS/nixpkgs/issues/42059. ++ # See also https://github.com/NixOS/nixpkgs/pull/41589. ++ with darwin.apple_sdk.frameworks; ++ runCommand "bazel-nixpkgs-cc-wrapper" ++ { ++ buildInputs = [ stdenv.cc makeWrapper ]; ++ } ++ '' ++ mkdir -p $out/bin ++ ++ for i in ${stdenv.cc}/bin/*; do ++ ln -sf $i $out/bin ++ done ++ ++ # Override cc ++ rm $out/bin/cc ++ makeWrapper ${stdenv.cc}/bin/cc $out/bin/cc --add-flags \ ++ "-isystem ${llvmPackages.libcxx}/include/c++/v1 \ ++ -F${CoreFoundation}/Library/Frameworks \ ++ -F${CoreServices}/Library/Frameworks \ ++ -F${Security}/Library/Frameworks \ ++ -F${Foundation}/Library/Frameworks \ ++ -L${libcxx}/lib \ ++ -L${darwin.libobjc}/lib" ++ ''; + cc = + if isNull nix_expr then -+ buildEnv { -+ name = "bazel-nixpkgs-cc"; -+ # XXX: `gcov` is missing in `/bin`. -+ # It exists in `stdenv.cc.cc` but that collides with `stdenv.cc`. -+ paths = [ stdenv.cc binutils ]; -+ pathsToLink = [ "/bin" ]; -+ } ++ if stdenv.isDarwin then defaultCcDarwin else defaultCcLinux + else if isNull attribute_path then + nix_expr + else @@ -794,12 +825,14 @@ index 0000000..4809408 + is_compiler_option_supported() { + local option="$1" + local pattern="''${2-$1}" -+ { $cc "$option" -o /dev/null -c -x c++ - <<<"int main {}" 2>&1 || true; } | grep -qv -- "$pattern" ++ { $cc "$option" -o /dev/null -c -x c++ - <<<"int main() {}" 2>&1 1>/dev/null || true; } \ ++ | grep -qe "$pattern" && return 1 || return 0 + } + is_linker_option_supported() { + local option="$1" + local pattern="''${2-$1}" -+ { $cc "$option" -o /dev/null -x c++ - <<<"int main {}" 2>&1 || true; } | grep -qv -- "$pattern" ++ { $cc "$option" -o /dev/null -x c++ - <<<"int main() {}" 2>&1 1>/dev/null || true; } \ ++ | grep -qe "$pattern" && return 1 || return 0 + } + add_compiler_option_if_supported() { + if is_compiler_option_supported "$@"; then @@ -816,10 +849,12 @@ index 0000000..4809408 + # + # This is copied and adapted to bash from + # `@bazel_tools//tools/cpp:unix_cc_configure.bzl`. ++ IFS=$'\n' + include_dirs_for() { + $cc -E -x "$1" - -v "''${@:2}" 2>&1 \ -+ | sed '1,/^#include <...>/d;/^[^ ]/,$d' \ -+ | xargs realpath -m ++ | sed '1,/^#include <...>/d;/^[^ ]/,$d;s/^ *//' \ ++ | tr '\n' '\0' \ ++ | xargs -0 realpath -ms + } + CXX_BUILTIN_INCLUDE_DIRECTORIES=($({ + include_dirs_for c @@ -832,6 +867,7 @@ index 0000000..4809408 + include_dirs_for c++ -std=c++0x -no-canonical-prefixes + fi + } 2>&1 | sort -u)) ++ unset IFS + + # Determine list of supported compiler and linker flags. + # @@ -954,13 +990,15 @@ index 0000000..4809408 + # Write CC_TOOLCHAIN_INFO + # + # Each line has the following shape: -+ # ... -+ # I.e. each line contains one space-separated key-value pair, where the -+ # value is a space-separated list. ++ # :::... ++ # I.e. each line contains one colon-separated key-value pair, where the ++ # value is a colon-separated list. + mkdir -p $out + write_info() { + local -n flags=$1 -+ echo $1 "''${flags[@]}" >>$out/CC_TOOLCHAIN_INFO ++ IFS=: ++ echo "$1:''${flags[*]}" >>$out/CC_TOOLCHAIN_INFO ++ unset IFS + } + write_info TOOL_NAMES + write_info TOOL_PATHS @@ -1082,10 +1120,10 @@ index 450810b..d86d6a5 100644 ) diff --git a/tests/cc-test.bzl b/tests/cc-test.bzl new file mode 100644 -index 0000000..c162e25 +index 0000000..8494cc8 --- /dev/null +++ b/tests/cc-test.bzl -@@ -0,0 +1,49 @@ +@@ -0,0 +1,59 @@ +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") + @@ -1099,7 +1137,7 @@ index 0000000..c162e25 + ) + ctx.actions.write(executable, content = """\ +# Find cc in CC_TOOLCHAIN_INFO -+while read -a line; do ++while IFS=: read -a line; do + if [[ ${{line[0]}} = TOOL_PATHS ]]; then + for item in ${{line[@]:1}}; do + if [[ $item = */bin/cc ]]; then @@ -1108,9 +1146,16 @@ index 0000000..c162e25 + done + fi +done <{cc_toolchain_info_path} -+if [[ {cc} != $CC ]]; then -+ echo "Expected C compiler '$CC', but found '{cc}'." >&2 -+ exit 1 ++if [[ {cc} = */cc_wrapper.sh ]]; then ++ grep -q "$CC" "{cc}" || {{ ++ echo "Expected C compiler '$CC' in wrapper script '{cc}'." >&2 ++ exit 1 ++ }} ++else ++ if [[ {cc} != $CC ]]; then ++ echo "Expected C compiler '$CC', but found '{cc}'." >&2 ++ exit 1 ++ fi +fi +""".format( + cc = cc.compiler_executable, @@ -1118,7 +1163,10 @@ index 0000000..c162e25 + )) + return [DefaultInfo( + executable = executable, -+ runfiles = ctx.runfiles(files = [ctx.file._cc_toolchain_info]), ++ runfiles = ctx.runfiles( ++ files = [ctx.file._cc_toolchain_info], ++ transitive_files = cc.all_files, ++ ), + )] + +cc_toolchain_test = rule( From 7c637fa3e633260acb564126ce182e82a9f53c75 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Fri, 15 May 2020 14:00:29 +0200 Subject: [PATCH 10/14] nixpkgs cc toolchain uses bin/cc --- nix/tools/bazel-cc-toolchain/default.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nix/tools/bazel-cc-toolchain/default.nix b/nix/tools/bazel-cc-toolchain/default.nix index a11f740485ba..e8e5072fc63f 100644 --- a/nix/tools/bazel-cc-toolchain/default.nix +++ b/nix/tools/bazel-cc-toolchain/default.nix @@ -27,10 +27,10 @@ let cc-darwin = ln -sf $i $out/bin done - # Override clang - rm $out/bin/clang + # Override cc + rm $out/bin/cc - makeWrapper ${stdenv.cc}/bin/clang $out/bin/clang \ + makeWrapper ${stdenv.cc}/bin/cc $out/bin/cc \ --add-flags "-Wno-unused-command-line-argument \ -mmacosx-version-min=10.14 \ -isystem ${llvmPackages.libcxx}/include/c++/v1 \ @@ -55,11 +55,11 @@ let cc-darwin = done # Override gcc - rm $out/bin/gcc + rm $out/bin/cc # We disable the fortify hardening as it causes issues with some # packages built with bazel that set these flags themselves. - makeWrapper ${stdenv.cc}/bin/gcc $out/bin/gcc \ + makeWrapper ${stdenv.cc}/bin/cc $out/bin/cc \ --set hardeningDisable fortify ''; From 727f585cda9fe6370b5635d7ba341a50f0fe720a Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Fri, 15 May 2020 17:30:47 +0200 Subject: [PATCH 11/14] Use darwin.binutils on MacOS --- .../nixpkgs-hermetic-cc-toolchain.patch | 26 ++++++++++--------- nix/tools/bazel-cc-toolchain/default.nix | 2 +- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch b/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch index e5f747048a84..82ef7e17f878 100644 --- a/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch +++ b/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch @@ -734,10 +734,10 @@ index fb87da4..6bf2d90 100644 this rule to specific explicitly which commands the toolchain should diff --git a/nixpkgs/toolchains/cc.nix b/nixpkgs/toolchains/cc.nix new file mode 100644 -index 0000000..56ddae7 +index 0000000..88cf785 --- /dev/null +++ b/nixpkgs/toolchains/cc.nix -@@ -0,0 +1,277 @@ +@@ -0,0 +1,279 @@ +with import { config = {}; overlays = []; }; + +{ attribute_path ? null @@ -745,15 +745,7 @@ index 0000000..56ddae7 +}: + +let -+ defaultCcLinux = -+ buildEnv { -+ name = "bazel-nixpkgs-cc"; -+ # XXX: `gcov` is missing in `/bin`. -+ # It exists in `stdenv.cc.cc` but that collides with `stdenv.cc`. -+ paths = [ stdenv.cc binutils ]; -+ pathsToLink = [ "/bin" ]; -+ }; -+ defaultCcDarwin = ++ darwinCC = + # Work around https://github.com/NixOS/nixpkgs/issues/42059. + # See also https://github.com/NixOS/nixpkgs/pull/41589. + with darwin.apple_sdk.frameworks; @@ -781,7 +773,17 @@ index 0000000..56ddae7 + ''; + cc = + if isNull nix_expr then -+ if stdenv.isDarwin then defaultCcDarwin else defaultCcLinux ++ buildEnv { ++ name = "bazel-nixpkgs-cc"; ++ # XXX: `gcov` is missing in `/bin`. ++ # It exists in `stdenv.cc.cc` but that collides with `stdenv.cc`. ++ paths = ++ if stdenv.isDarwin then ++ [ (overrideCC stdenv darwinCC).cc darwin.binutils ] ++ else ++ [ stdenv.cc binutils ]; ++ pathsToLink = [ "/bin" ]; ++ } + else if isNull attribute_path then + nix_expr + else diff --git a/nix/tools/bazel-cc-toolchain/default.nix b/nix/tools/bazel-cc-toolchain/default.nix index e8e5072fc63f..d37961fa07bc 100644 --- a/nix/tools/bazel-cc-toolchain/default.nix +++ b/nix/tools/bazel-cc-toolchain/default.nix @@ -70,5 +70,5 @@ let cc-darwin = in buildEnv { name = "bazel-cc-toolchain"; - paths = [ customStdenv.cc ] ++ (if stdenv.isDarwin then [ ] else [ binutils ]); + paths = [ customStdenv.cc ] ++ (if stdenv.isDarwin then [ darwin.binutils ] else [ binutils ]); } From 4408eca9c095d1e0b3099a3f2400a34f7b14d284 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 18 May 2020 10:45:32 +0200 Subject: [PATCH 12/14] Remove clang(++) and gcc (g++) symlinks The toolchain only considers `bin/cc` and having the other symlinks around could lead to confusion --- nix/tools/bazel-cc-toolchain/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/tools/bazel-cc-toolchain/default.nix b/nix/tools/bazel-cc-toolchain/default.nix index d37961fa07bc..1be07c61a4e3 100644 --- a/nix/tools/bazel-cc-toolchain/default.nix +++ b/nix/tools/bazel-cc-toolchain/default.nix @@ -28,7 +28,7 @@ let cc-darwin = done # Override cc - rm $out/bin/cc + rm $out/bin/cc $out/bin/clang $out/bin/clang++ makeWrapper ${stdenv.cc}/bin/cc $out/bin/cc \ --add-flags "-Wno-unused-command-line-argument \ @@ -55,7 +55,7 @@ let cc-darwin = done # Override gcc - rm $out/bin/cc + rm $out/bin/cc $out/bin/gcc $out/bin/g++ # We disable the fortify hardening as it causes issues with some # packages built with bazel that set these flags themselves. From f0fb522133cf3038ced936bbbd00dc514510ce29 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 18 May 2020 12:14:40 +0200 Subject: [PATCH 13/14] Use hermetic toolchain in compatibility workspace --- compatibility/WORKSPACE | 11 +++++++---- compatibility/deps.bzl | 6 ++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/compatibility/WORKSPACE b/compatibility/WORKSPACE index 5c9f787e53f8..0862f2a7e405 100644 --- a/compatibility/WORKSPACE +++ b/compatibility/WORKSPACE @@ -44,7 +44,7 @@ load( ) load( "@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", - "nixpkgs_cc_configure", + "nixpkgs_cc_configure_hermetic", "nixpkgs_local_repository", "nixpkgs_package", "nixpkgs_python_configure", @@ -78,14 +78,17 @@ nixpkgs_local_repository( ], ) -nixpkgs_cc_configure( +nixpkgs_cc_configure_hermetic( + # We override the Bazel's autodetect toolchain to avoid accidentaly + # dependencies on the inhermetic autodetected builtin include paths or + # builds failing due to Bazel not finding `cc` in `$PATH` or `$CC`. + name = "local_config_cc", nix_file = "@daml//nix:bazel-cc-toolchain.nix", nix_file_deps = common_nix_file_deps + [ - "@daml//nix:bazel-cc-toolchain.nix", "@daml//nix:tools/bazel-cc-toolchain/default.nix", ], repositories = dev_env_nix_repos, -) +) if not is_windows else None nixpkgs_package( name = "glibc_locales", diff --git a/compatibility/deps.bzl b/compatibility/deps.bzl index fa0757e0e25f..109c6a6c4c9b 100644 --- a/compatibility/deps.bzl +++ b/compatibility/deps.bzl @@ -64,6 +64,12 @@ def daml_deps(): strip_prefix = "rules_nixpkgs-%s" % rules_nixpkgs_version, urls = ["https://github.com/tweag/rules_nixpkgs/archive/%s.tar.gz" % rules_nixpkgs_version], sha256 = rules_nixpkgs_sha256, + patches = [ + # Remove once https://github.com/tweag/rules_nixpkgs/pull/128 + # has been merged + "@daml//bazel_tools:nixpkgs-hermetic-cc-toolchain.patch", + ], + patch_args = ["-p1"], ) if "com_github_bazelbuild_buildtools" not in native.existing_rules(): From bcdb916c0d94aead04759fa80443d95e751d3c95 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 18 May 2020 17:01:08 +0200 Subject: [PATCH 14/14] Avoid empty linker flags --- .../nixpkgs-hermetic-cc-toolchain.patch | 59 +++++++++++-------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch b/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch index 82ef7e17f878..33320e072f2c 100644 --- a/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch +++ b/bazel_tools/nixpkgs-hermetic-cc-toolchain.patch @@ -52,7 +52,7 @@ index 8596633..234c41b 100644 [0.7.0]: https://github.com/tweag/rules_nixpkgs/compare/v0.6.0...v0.7.0 diff --git a/README.md b/README.md -index 721f64d..07dbd8c 100644 +index 721f64d..3ee682b 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ Make the content of a Nixpkgs package available in the Bazel workspace. @@ -83,7 +83,7 @@ index 721f64d..07dbd8c 100644 fail_not_supported -@@ -321,6 +333,96 @@ filegroup( +@@ -321,6 +333,99 @@ filegroup( @@ -96,7 +96,10 @@ index 721f64d..07dbd8c 100644 +this rule to specify explicitly which commands the toolchain should use. + +Specifically, it builds a Nix derivation that provides the CC toolchain tools -+in the `bin/` path and constructs a CC toolchain that uses those tools. ++in the `bin/` path and constructs a CC toolchain that uses those tools. The ++following tools are expected `ar`, `cpp`, `dwp`, `cc`, `gcov`, `ld`, `nm`, ++`objcopy`, `objdump`, `strip`. Tools that aren't found are replaced by ++`${coreutils}/bin/false`. + +Note: + @@ -180,7 +183,7 @@ index 721f64d..07dbd8c 100644 ### nixpkgs_cc_configure Tells Bazel to use compilers and linkers from Nixpkgs for the CC -@@ -328,6 +430,15 @@ toolchain. By default, Bazel autodetects a toolchain on the current +@@ -328,6 +433,16 @@ toolchain. By default, Bazel autodetects a toolchain on the current `PATH`. Overriding this autodetection makes builds more hermetic and is considered a best practice. @@ -188,10 +191,11 @@ index 721f64d..07dbd8c 100644 + +Use `nixpkgs_cc_configure_hermetic` instead. + -+This uses Bazel's autoconfigure toolchain under the hood, which is -+inhermetic. In particular, system include directories specified in the -+environment can leak in and affect the cache keys of target depending on -+the cc toolchain leading to cache misses. ++While this improves upon Bazel's autoconfigure toolchain by picking tools from ++a Nix derivation rather than the environment, it is still not fully hermetic as ++it is affected by the environment. In particular, system include directories ++specified in the environment can leak in and affect the cache keys of targets ++depending on the cc toolchain leading to cache misses. + Example: @@ -275,7 +279,7 @@ index 6662530..94678bd 100644 ) diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl -index fb87da4..6bf2d90 100644 +index fb87da4..9a9ad75 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -1,7 +1,15 @@ @@ -393,7 +397,7 @@ index fb87da4..6bf2d90 100644 "quiet": attr.bool(), "fail_not_supported": attr.bool(default = True, doc = """ If set to True (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to False calling this rule will succeed but no output will be generated. -@@ -244,6 +308,323 @@ def nixpkgs_package(*args, **kwargs): +@@ -244,6 +308,329 @@ def nixpkgs_package(*args, **kwargs): else: _nixpkgs_package(*args, **kwargs) @@ -648,6 +652,12 @@ index fb87da4..6bf2d90 100644 + `gcc`) available in the environment. To make builds more hermetic, use + this rule to specify explicitly which commands the toolchain should use. + ++ Specifically, it builds a Nix derivation that provides the CC toolchain ++ tools in the `bin/` path and constructs a CC toolchain that uses those ++ tools. The following tools are expected `ar`, `cpp`, `dwp`, `cc`, `gcov`, ++ `ld`, `nm`, `objcopy`, `objdump`, `strip`. Tools that aren't found are ++ replaced by `${coreutils}/bin/false`. ++ + Note: + You need to configure `--crosstool_top=@//:toolchain` to activate this + toolchain. @@ -717,27 +727,29 @@ index fb87da4..6bf2d90 100644 def _readlink(repository_ctx, path): return repository_ctx.path(path).realpath -@@ -323,6 +704,14 @@ def nixpkgs_cc_configure( +@@ -323,6 +710,16 @@ def nixpkgs_cc_configure( nixopts = []): """Use a CC toolchain from Nixpkgs. No-op if not a nix-based platform. + Deprecated: + Use `nixpkgs_cc_configure_hermetic` instead. + -+ This uses Bazel's autoconfigure toolchain under the hood, which is -+ inhermetic. In particular, system include directories specified in the -+ environment can leak in and affect the cache keys of target depending on -+ the cc toolchain leading to cache misses. ++ While this improves upon Bazel's autoconfigure toolchain by picking tools ++ from a Nix derivation rather than the environment, it is still not fully ++ hermetic as it is affected by the environment. In particular, system ++ include directories specified in the environment can leak in and affect ++ the cache keys of targets depending on the cc toolchain leading to cache ++ misses. + By default, Bazel auto-configures a CC toolchain from commands (e.g. `gcc`) available in the environment. To make builds more hermetic, use this rule to specific explicitly which commands the toolchain should diff --git a/nixpkgs/toolchains/cc.nix b/nixpkgs/toolchains/cc.nix new file mode 100644 -index 0000000..88cf785 +index 0000000..6b1bff4 --- /dev/null +++ b/nixpkgs/toolchains/cc.nix -@@ -0,0 +1,279 @@ +@@ -0,0 +1,278 @@ +with import { config = {}; overlays = []; }; + +{ attribute_path ? null @@ -761,7 +773,7 @@ index 0000000..88cf785 + done + + # Override cc -+ rm $out/bin/cc ++ rm -f $out/bin/cc $out/bin/clang $out/bin/clang++ + makeWrapper ${stdenv.cc}/bin/cc $out/bin/cc --add-flags \ + "-isystem ${llvmPackages.libcxx}/include/c++/v1 \ + -F${CoreFoundation}/Library/Frameworks \ @@ -887,10 +899,8 @@ index 0000000..88cf785 + add_compiler_option_if_supported -Wthread-safety + add_compiler_option_if_supported -Wself-assign + # Disable problematic warnings. -+ # XXX: Causes clang warning + add_compiler_option_if_supported -Wunused-but-set-parameter + # has false positives -+ # XXX: Causes clang warning + add_compiler_option_if_supported -Wno-free-nonheap-object + # Enable coloring even if there's no attached terminal. Bazel removes the + # escape sequences if --nocolor is specified. @@ -902,7 +912,6 @@ index 0000000..88cf785 + CXX_FLAGS=(-std=c++0x) + LINK_FLAGS=( + $( -+ # XXX: absolute path is only supported by clang. + if [[ -x ${cc}/bin/ld.gold ]]; then echo -fuse-ld=gold; fi + add_linker_option_if_supported -Wl,-no-as-needed -no-as-needed + add_linker_option_if_supported -Wl,-z,relro,-z,now -z @@ -993,13 +1002,15 @@ index 0000000..88cf785 + # + # Each line has the following shape: + # :::... -+ # I.e. each line contains one colon-separated key-value pair, where the -+ # value is a colon-separated list. ++ # or ++ # ++ # I.e. each line is a colon-separated list of the key and the values. + mkdir -p $out + write_info() { + local -n flags=$1 ++ local output=( "$1" "''${flags[@]}" ) + IFS=: -+ echo "$1:''${flags[*]}" >>$out/CC_TOOLCHAIN_INFO ++ echo "''${output[*]}" >>$out/CC_TOOLCHAIN_INFO + unset IFS + } + write_info TOOL_NAMES