diff --git a/BUILD.bazel b/BUILD.bazel index e69de29bb..efcf274c9 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -0,0 +1,4 @@ +exports_files([ + "nixpkgs.json", + "nixpkgs.nix", +]) diff --git a/CHANGELOG.md b/CHANGELOG.md index 859663396..a1c6ddab2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/). +## [Unreleased] + +[Unreleased]: https://github.com/tweag/rules_nixpkgs/compare/v0.7.0...HEAD + +### Changed + +- The values in the `nixopts` attribute to `nixpkgs_package` are now subject to + location expansion. Any instance of `$(location LABEL)` in the `nixopts` + attribute will be expanded to the file path of the file referenced by + `LABEL`. To pass a plain `$` to Nix it must be escaped as `$$`. + See [#132][#132]. + +[#132]: https://github.com/tweag/rules_nixpkgs/pull/132 + ## [0.7.0] - 2020-04-20 [0.7.0]: https://github.com/tweag/rules_nixpkgs/compare/v0.6.0...v0.7.0 diff --git a/README.md b/README.md index 721f64dbb..de7f269e1 100644 --- a/README.md +++ b/README.md @@ -303,7 +303,12 @@ filegroup( nixopts

String list; optional

-

Extra flags to pass when calling Nix.

+

+ Extra flags to pass when calling Nix. Subject to location + expansion, any instance of $(location LABEL) will be + replaced by the path to the file ferenced by LABEL + relative to the workspace root. +

diff --git a/WORKSPACE b/WORKSPACE index 1f25ff149..85143ecc4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -16,6 +16,10 @@ load( # For tests +load("@bazel_skylib//lib:unittest.bzl", "register_unittest_toolchains") + +register_unittest_toolchains() + nixpkgs_git_repository( name = "remote_nixpkgs", remote = "https://github.com/NixOS/nixpkgs", @@ -170,6 +174,30 @@ nixpkgs_package( repository = "@remote_nixpkgs", ) +local_repository( + name = "nixpkgs_location_expansion_test_file", + path = "tests/location_expansion/test_repo", +) + +nixpkgs_package( + name = "nixpkgs_location_expansion_test", + build_file_content = "exports_files(glob(['out/**']))", + nix_file = "//tests:location_expansion.nix", + nix_file_deps = [ + "//tests:location_expansion/test_file", + "@nixpkgs_location_expansion_test_file//:test_file", + ], + nixopts = [ + "--arg", + "local_file", + "$(location //tests:location_expansion/test_file)", + "--arg", + "external_file", + "$(location @nixpkgs_location_expansion_test_file//:test_file)", + ], + repository = "@remote_nixpkgs", +) + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( @@ -202,12 +230,11 @@ http_archive( load( "//nixpkgs:toolchains/go.bzl", - "nixpkgs_go_configure" + "nixpkgs_go_configure", ) nixpkgs_go_configure(repository = "@nixpkgs") -load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains") +load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") go_rules_dependencies() - diff --git a/nixpkgs/BUILD.bazel b/nixpkgs/BUILD.bazel index 09fa6aa52..18f32ded3 100644 --- a/nixpkgs/BUILD.bazel +++ b/nixpkgs/BUILD.bazel @@ -6,7 +6,11 @@ exports_files([ "nixpkgs.bzl", ]) -filegroup(name = "srcs", srcs = glob(["**"]), visibility = ["//visibility:public"]) +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) # @bazel_tools//tools does not define a bzl_library itself, instead we are # supposed to define our own using the @bazel_tools//tools:bzl_srcs filegroup. @@ -33,9 +37,11 @@ bzl_library( name = "nixpkgs", srcs = [ "nixpkgs.bzl", + "private/location_expansion.bzl", ], visibility = ["//visibility:public"], deps = [ ":bazel_tools", + "@bazel_skylib//lib:paths", ], ) diff --git a/nixpkgs/constraints/BUILD.bazel b/nixpkgs/constraints/BUILD.bazel index 66625309b..94678bda9 100644 --- a/nixpkgs/constraints/BUILD.bazel +++ b/nixpkgs/constraints/BUILD.bazel @@ -3,7 +3,7 @@ package(default_visibility = ["//visibility:public"]) constraint_setting(name = "nix") constraint_value( - name = "support_nix", + name = "support_nix", constraint_setting = ":nix", ) diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index 3e6b0b5e6..40308ed98 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -2,11 +2,13 @@ load("@bazel_tools//tools/cpp:cc_configure.bzl", "cc_autoconf_impl") load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value") +load(":private/location_expansion.bzl", "expand_location") def _nixpkgs_git_repository_impl(repository_ctx): repository_ctx.file( "BUILD", - content = 'filegroup(name = "srcs", srcs = glob(["**"]), visibility = ["//visibility:public"])') + content = 'filegroup(name = "srcs", srcs = glob(["**"]), visibility = ["//visibility:public"])', + ) # Make "@nixpkgs" (syntactic sugar for "@nixpkgs//:nixpkgs") a valid # label for default.nix. @@ -119,8 +121,9 @@ def _nixpkgs_package_impl(repository_ctx): else: expr_args = ["-E", "import { config = {}; overlays = []; }"] + nix_file_deps = {} for dep in repository_ctx.attr.nix_file_deps: - _cp(repository_ctx, dep) + nix_file_deps[dep] = _cp(repository_ctx, dep) expr_args.extend([ "-A", @@ -135,7 +138,15 @@ def _nixpkgs_package_impl(repository_ctx): "bazel-support/nix-out-link", ]) - expr_args.extend(repository_ctx.attr.nixopts) + expr_args.extend([ + expand_location( + repository_ctx = repository_ctx, + string = opt, + labels = nix_file_deps, + attr = "nixopts", + ) + for opt in repository_ctx.attr.nixopts + ]) for repo in repositories.keys(): path = str(repository_ctx.path(repo).dirname) + "/nix-file-deps" @@ -208,7 +219,7 @@ def _nixpkgs_package_impl(repository_ctx): if create_build_file_if_needed: p = repository_ctx.path("BUILD") if not p.exists: - repository_ctx.template("BUILD", Label("@io_tweag_rules_nixpkgs//nixpkgs:BUILD.pkg")) + repository_ctx.template("BUILD", Label("@io_tweag_rules_nixpkgs//nixpkgs:BUILD.pkg")) _nixpkgs_package = repository_rule( implementation = _nixpkgs_package_impl, diff --git a/nixpkgs/private/location_expansion.bzl b/nixpkgs/private/location_expansion.bzl new file mode 100644 index 000000000..069058643 --- /dev/null +++ b/nixpkgs/private/location_expansion.bzl @@ -0,0 +1,128 @@ +load("@bazel_skylib//lib:paths.bzl", "paths") + +def parse_expand_location(string): + """Parse a string that might contain location expansion commands. + + Generates a list of pairs of command and argument. + The command can have the following values: + - `string`: argument is a string, append it to the result. + - `location`: argument is a label, append its location to the result. + + Attrs: + string: string, The string to parse. + + Returns: + (result, error): + result: The generated list of pairs of command and argument. + error: string or None, This is set if an error occurred. + """ + result = [] + offset = 0 + len_string = len(string) + + # Step through occurrences of `$`. This is bounded by the length of the string. + for _ in range(len_string): + # Find the position of the next `$`. + position = string.find("$", offset) + if position == -1: + position = len_string + + # Append the in-between literal string. + if offset < position: + result.append(("string", string[offset:position])) + + # Terminate at the end of the string. + if position == len_string: + break + + # Parse the `$` command. + if string[position:].startswith("$$"): + # Insert verbatim '$'. + result.append(("string", "$")) + offset = position + 2 + elif string[position:].startswith("$("): + # Expand a location command. + group_start = position + 2 + group_end = string.find(")", group_start) + if group_end == -1: + return (None, "Unbalanced parentheses in location expansion for '{}'.".format(string[position:])) + + group = string[group_start:group_end] + command = None + if group.startswith("location "): + label_str = group[len("location "):] + command = ("location", label_str) + else: + return (None, "Unrecognized location expansion '$({})'.".format(group)) + + result.append(command) + offset = group_end + 1 + else: + return (None, "Unescaped '$' in location expansion at position {} of input.".format(position)) + + return (result, None) + +def resolve_label(label_str, labels): + """Find the label that corresponds to the given string. + + Attr: + label_str: string, String representation of a label. + labels: dict from Label to path: Known label to path mappings. + + Returns: + (path, error): + path: path, The path to the resolved label + error: string or None, This is set if an error occurred. + """ + label_candidates = [ + (lbl, path) + for (lbl, path) in labels.items() + if lbl.relative(label_str) == lbl + ] + + if len(label_candidates) == 0: + return (None, "Unknown label '{}' in location expansion.".format(label_str)) + elif len(label_candidates) > 1: + return (None, "Ambiguous label '{}' in location expansion. Candidates: {}".format( + label_str, + ", ".join([str(lbl) for (lbl, _) in label_candidates]), + )) + + return (label_candidates[0][1], None) + +def expand_location(repository_ctx, string, labels, attr = None): + """Expand `$(location label)` to a path. + + Raises an error on unexpected occurrences of `$`. + Use `$$` to insert a verbatim `$`. + + Attrs: + repository_ctx: The repository rule context. + string: string, Replace instances of `$(location )` in this string. + labels: dict from label to path: Known label to path mappings. + attr: string, The rule attribute to use for error reporting. + + Returns: + The string with all instances of `$(location )` replaced by paths. + """ + (parsed, error) = parse_expand_location(string) + if error != None: + fail(error, attr) + + result = "" + for (command, argument) in parsed: + if command == "string": + result += argument + elif command == "location": + (label, error) = resolve_label(argument, labels) + if error != None: + fail(error, attr) + + result += paths.join(".", paths.relativize( + str(repository_ctx.path(label)), + str(repository_ctx.path(".")), + )) + else: + fail("Internal error: Unknown location expansion command '{}'.".format(command), attr) + + return result diff --git a/nixpkgs/toolchains/go.bzl b/nixpkgs/toolchains/go.bzl index 79cbb998c..d8c6366e2 100644 --- a/nixpkgs/toolchains/go.bzl +++ b/nixpkgs/toolchains/go.bzl @@ -2,20 +2,19 @@ load( "@io_bazel_rules_go//go:deps.bzl", "go_wrap_sdk", ) - load( "//nixpkgs:nixpkgs.bzl", - "nixpkgs_package" + "nixpkgs_package", ) def nixpkgs_go_configure( - sdk_name = "go_sdk", - repository = None, - repositories = {}, - nix_file = None, - nix_file_deps = None, - nix_file_content = None, - nixopts = []): + sdk_name = "go_sdk", + repository = None, + repositories = {}, + nix_file = None, + nix_file_deps = None, + nix_file_content = None, + nixopts = []): """ Use go toolchain from Nixpkgs. Will fail if not a nix-based platform. @@ -41,7 +40,6 @@ def nixpkgs_go_configure( } """ - nixpkgs_package( name = "nixpkgs_go_toolchain", repository = repository, diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index 16866a370..e782f747e 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -1,6 +1,9 @@ package(default_testonly = 1) load("@io_bazel_rules_go//go:def.bzl", "go_binary") +load(":location_expansion_unit_test.bzl", "expand_location_unit_test_suite") + +expand_location_unit_test_suite() [ # All of these tests use the "hello" binary to see @@ -54,6 +57,24 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary") ), ] +# Test nixopts location expansion +sh_test( + name = "location-expansion-test", + srcs = ["location_expansion.sh"], + args = [ + "$(POSIX_DIFF)", + "$(rootpath //tests:location_expansion/test_file)", + "$(rootpath @nixpkgs_location_expansion_test//:out/local_file)", + "$(rootpath @nixpkgs_location_expansion_test//:out/external_file)", + ], + data = [ + "//tests:location_expansion/test_file", + "@nixpkgs_location_expansion_test//:out/external_file", + "@nixpkgs_location_expansion_test//:out/local_file", + ], + toolchains = ["@rules_sh//sh/posix:make_variables"], +) + # Test nixpkgs_cc_configure() by building some CC code. cc_binary( name = "cc-test", @@ -86,7 +107,7 @@ sh_test( # Test nixpkgs_go_configure() go_binary( name = "go-test", - srcs = ["go-test.go"] + srcs = ["go-test.go"], ) sh_test( @@ -97,6 +118,6 @@ sh_test( "//nixpkgs:srcs", "//tests/invalid_nixpkgs_package:srcs", "@busybox_static//:bin", - "@nix-unstable//:bin" + "@nix-unstable//:bin", ], ) diff --git a/tests/invalid_nixpkgs_package/BUILD.bazel b/tests/invalid_nixpkgs_package/BUILD.bazel index 00cbfeaa5..70ff958ed 100644 --- a/tests/invalid_nixpkgs_package/BUILD.bazel +++ b/tests/invalid_nixpkgs_package/BUILD.bazel @@ -1 +1,5 @@ -filegroup(name = "srcs", srcs = glob(["**"]), visibility = ["//visibility:public"]) +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) diff --git a/tests/invalid_nixpkgs_package/workspace.bazel b/tests/invalid_nixpkgs_package/workspace.bazel index c43464952..2113a7733 100644 --- a/tests/invalid_nixpkgs_package/workspace.bazel +++ b/tests/invalid_nixpkgs_package/workspace.bazel @@ -1,5 +1,9 @@ workspace(name = "io_tweag_rules_nixpkgs") +load("//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") + +rules_nixpkgs_dependencies() + load( "//nixpkgs:nixpkgs.bzl", "nixpkgs_local_repository", diff --git a/tests/location_expansion.nix b/tests/location_expansion.nix new file mode 100644 index 000000000..d7639ba7f --- /dev/null +++ b/tests/location_expansion.nix @@ -0,0 +1,16 @@ +with import { config = {}; overlays = []; }; + +{ local_file, external_file }: +let + inherit (attrs) nixpkgs_json nixpkgs_nix; +in + runCommand "location-expansion" + { + preferLocalBuild = true; + allowSubstitutes = false; + } + '' + mkdir -p $out/out + cp ${local_file} $out/out/local_file + cp ${external_file} $out/out/external_file + '' diff --git a/tests/location_expansion.sh b/tests/location_expansion.sh new file mode 100755 index 000000000..5f60e2ae8 --- /dev/null +++ b/tests/location_expansion.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +# USAGE: +# location_expansion.sh DIFF REFERENCE FILE... +# +# Compares the given files to the reference file and fails if there is a difference. +DIFF="$1" +REFERENCE="$2" + +for file in "${@:3}"; do + "$DIFF" "$file" "$REFERENCE" +done diff --git a/tests/location_expansion/test_file b/tests/location_expansion/test_file new file mode 100644 index 000000000..b954d82d1 --- /dev/null +++ b/tests/location_expansion/test_file @@ -0,0 +1,11 @@ +# A few random numbers +3030205 +68380721 +93196393 +70110283 +92132420 +98602836 +37655207 +12502743 +73421895 +75878115 diff --git a/tests/location_expansion/test_repo/BUILD.bazel b/tests/location_expansion/test_repo/BUILD.bazel new file mode 100644 index 000000000..af49d1ebb --- /dev/null +++ b/tests/location_expansion/test_repo/BUILD.bazel @@ -0,0 +1 @@ +exports_files(glob(["*"])) diff --git a/tests/location_expansion/test_repo/WORKSPACE b/tests/location_expansion/test_repo/WORKSPACE new file mode 100644 index 000000000..e69de29bb diff --git a/tests/location_expansion/test_repo/test_file b/tests/location_expansion/test_repo/test_file new file mode 120000 index 000000000..a525bb2c1 --- /dev/null +++ b/tests/location_expansion/test_repo/test_file @@ -0,0 +1 @@ +../test_file \ No newline at end of file diff --git a/tests/location_expansion_unit_test.bzl b/tests/location_expansion_unit_test.bzl new file mode 100644 index 000000000..a924fe57c --- /dev/null +++ b/tests/location_expansion_unit_test.bzl @@ -0,0 +1,145 @@ +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") +load( + "//nixpkgs:private/location_expansion.bzl", + "parse_expand_location", + "resolve_label", +) + +def _parse_expand_location_test(ctx): + env = unittest.begin(ctx) + + asserts.equals( + env, + expected = ([], None), + actual = parse_expand_location(""), + msg = "Parses the empty string", + ) + + asserts.equals( + env, + expected = ([("string", "plain string")], None), + actual = parse_expand_location("plain string"), + msg = "Parses a plain string", + ) + + asserts.equals( + env, + expected = ([("string", "$")], None), + actual = parse_expand_location("$$"), + msg = "Parses an escaped dollar sign", + ) + + asserts.equals( + env, + expected = ([("location", "@workspace//package:target")], None), + actual = parse_expand_location("$(location @workspace//package:target)"), + msg = "Parses a location command", + ) + + asserts.equals( + env, + expected = ([ + ("string", "before "), + ("location", "//label:1"), + ("string", " "), + ("string", "$"), + ("string", " "), + ("location", "//label:2"), + ("string", " after"), + ], None), + actual = parse_expand_location( + "before $(location //label:1) $$ $(location //label:2) after", + ), + msg = "Parses a complex location expansion string", + ) + + asserts.equals( + env, + expected = (None, "Unescaped '$' in location expansion at position 0 of input."), + actual = parse_expand_location("$"), + msg = "Fails on unescaped dollar sign", + ) + + asserts.equals( + env, + expected = (None, "Unbalanced parentheses in location expansion for '$(location //label:1'."), + actual = parse_expand_location("$(location //label:1"), + msg = "Fails on unbalanced parentheses", + ) + + asserts.equals( + env, + expected = (None, "Unrecognized location expansion '$(misspelled)'."), + actual = parse_expand_location("$(misspelled)"), + msg = "Fails on unknown location expansion command", + ) + + return unittest.end(env) + +parse_expand_location_test = unittest.make(_parse_expand_location_test) + +def _resolve_label_test(ctx): + env = unittest.begin(ctx) + + asserts.equals( + env, + expected = ("correct/path", None), + actual = resolve_label( + "@workspace//package:target", + { + Label("@workspace//package:target"): "correct/path", + Label("@another//package:target"): "wrong/path", + }, + ), + msg = "Finds an absolute label", + ) + + asserts.equals( + env, + expected = ("correct/path", None), + actual = resolve_label( + "//package:target", + { + Label("@workspace//package:target"): "correct/path", + Label("@another//different:target"): "wrong/path", + }, + ), + msg = "Finds an unambiguous relative label", + ) + + asserts.equals( + env, + expected = (None, "Unknown label '@unknown//package:target' in location expansion."), + actual = resolve_label( + "@unknown//package:target", + { + Label("@workspace//package:target"): "wrong/path", + Label("@another//package:target"): "another/wrong/path", + }, + ), + msg = "Fails on an unknown label", + ) + + asserts.equals( + env, + expected = (None, "Ambiguous label '//package:target' in location expansion. Candidates: @workspace//package:target, @another//package:target"), + actual = resolve_label( + "//package:target", + { + Label("@workspace//package:target"): "wrong/path", + Label("@another//package:target"): "another/wrong/path", + }, + ), + msg = "Fails on an ambiguous relative label", + ) + + return unittest.end(env) + +resolve_label_test = unittest.make(_resolve_label_test) + +def expand_location_unit_test_suite(): + unittest.suite( + "expand_location_unit_test_suite", + parse_expand_location_test, + resolve_label_test, + )