diff --git a/.bazelrc.common b/.bazelrc.common index 640704b50..5d8b7e109 100644 --- a/.bazelrc.common +++ b/.bazelrc.common @@ -1,7 +1,8 @@ build --host_platform=@rules_nixpkgs_core//platforms:host -build:bzlmod --enable_bzlmod -build:bzlmod --registry=https://bcr.bazel.build +common:bzlmod --enable_bzlmod +common:bzlmod --registry=https://bcr.bazel.build +common:bzlmod --experimental_isolated_extension_usages test --test_output=errors diff --git a/.github/build-and-test b/.github/build-and-test index 476b24d62..ff4feacf0 100755 --- a/.github/build-and-test +++ b/.github/build-and-test @@ -36,6 +36,7 @@ if [[ ${BZLMOD_ENABLED-} = true ]]; then ) declare -ra test_dirs=( testing/core + testing/core/tests/intermediate_module testing/go-bzlmod testing/java testing/nodejs diff --git a/WORKSPACE b/WORKSPACE index f6d3838e1..08a14a7c8 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -24,19 +24,12 @@ docs_dependencies_2() load( "//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository", - "nixpkgs_local_repository", "nixpkgs_package", ) -nixpkgs_local_repository( - name = "nixpkgs", - nix_file = "@rules_nixpkgs_core//:nixpkgs.nix", - nix_file_deps = ["@rules_nixpkgs_core//:nixpkgs.json"], -) - nixpkgs_package( - name = "nix_2_7", - attribute_path = "nixVersions.nix_2_7", + name = "nix_2_10", + attribute_path = "nixVersions.nix_2_10", repositories = {"nixpkgs": "@nixpkgs"}, ) @@ -44,7 +37,10 @@ nixpkgs_package( # `run-test-invalid-nixpkgs-package`. nixpkgs_package( name = "coreutils_static", - attribute_path = "pkgsStatic.coreutils", + # Work around https://github.com/tweag/rules_nixpkgs/issues/424. + # `pkgsStatic.coreutils` stopped working on MacOS 11 with x86_64 as used on GitHub actions CI. + # Fall back to `pkgs.coreutils` on MacOS. + nix_file_content = "let pkgs = import { config = {}; overlays = []; }; in if pkgs.stdenv.isDarwin then pkgs.coreutils else pkgs.pkgsStatic.coreutils", repository = "@nixpkgs", ) diff --git a/core/MODULE.bazel b/core/MODULE.bazel index 320da3b2d..feec49cc5 100644 --- a/core/MODULE.bazel +++ b/core/MODULE.bazel @@ -7,10 +7,9 @@ bazel_dep(name = "platforms", version = "0.0.4") bazel_dep(name = "bazel_skylib", version = "1.0.3") nix_repo = use_extension("//extensions:repository.bzl", "nix_repo") -use_repo(nix_repo, "nixpkgs_repositories") -nix_repo.override(name = "nixpkgs") nix_repo.github( name = "nixpkgs", tag = "22.11", sha256 = "ddc3428d9e1a381b7476750ac4dbea7a42885cbbe6e1af44b21d6447c9609a6f", ) +use_repo(nix_repo, "nixpkgs") diff --git a/core/extensions/package.bzl b/core/extensions/package.bzl index 101f8de5a..1cdbbf4d6 100644 --- a/core/extensions/package.bzl +++ b/core/extensions/package.bzl @@ -1,68 +1,35 @@ """Defines the nix_pkg module extension. """ -load("//:nixpkgs.bzl", "nixpkgs_package") -load("//:util.bzl", "fail_on_err") -load("//private:module_registry.bzl", "registry") load("@bazel_skylib//lib:dicts.bzl", "dicts") -load("@bazel_skylib//lib:partial.bzl", "partial") -load("@nixpkgs_repositories//:defs.bzl", "nix_repo") - -_ACCESSOR = '''\ -def nix_pkg(module_name, name, label): - """Access a Nix package imported with `nix_pkg`. - - Args: - module_name: `String`; Name of the calling Bazel module. - This is needed until Bazel offers unique module identifiers, - see [#17652][bazel-17652]. - name: `String`; Name of the package. - label: `String`; Target within the package. - A string representation of a label under the package's external - workspace. - - Returns: - `Label`; The resolved label to the target within the package's external workspace. - - [bazel-17652]: https://github.com/bazelbuild/bazel/issues/17652 - """ - resolved = _fail_on_err( - _get_repository(module_name, name), - prefix = "Invalid Nix repository, you must use the nix_repo extension and request a global repository or register a local repository: ", - ) - return resolved.relative(label) -''' - -def _name_from_attr(attr): - """Generate a global workspace name from an attribute path. - """ - return attr - -def _handle_common_attrs(attrs): - kwargs = {} - - if bool(attrs.attr): - kwargs["attribute_path"] = attrs.attr +load("@bazel_skylib//lib:sets.bzl", "sets") +load("//:nixpkgs.bzl", "nixpkgs_package") - return kwargs +_DEFAULT_NIXKGS = "@nixpkgs" -def _handle_file_attrs(attrs): - kwargs = {"nix_file": attrs.file} +_ISOLATED_OR_ROOT_ONLY_ERROR = "Illegal use of the {tag_name} tag. The {tag_name} tag may only be used on an isolated module extension or in the root module or rules_nixpkgs_core." +_DUPLICATE_PACKAGE_NAME_ERROR = "Duplicate nix_pkg import due to {tag_name} tag. The package name '{package_name}' is already used." +_ISOLATED_NOT_ALLOWED_ERROR = "Illegal use of the {tag_name} tag. The {tag_name} tag may not be used on an isolated module extension." - if bool(attrs.file_deps): - kwargs["nix_file_deps"] = attrs.file_deps +# TODO[AH]: Add support to configure global default Nix options. - return kwargs +def _get_pkg_name(attrs): + if bool(attrs.name): + return attrs.name + elif bool(attrs.attr): + return attrs.attr + else: + fail('The `name` attribute must be set if `attr` is the empty string `""`.') -def _handle_expr_attrs(attrs): - kwargs = {"nix_file_content": attrs.expr} +def _handle_common_attrs(attrs): + kwargs = {} - if bool(attrs.file_deps): - kwargs["nix_file_deps"] = attrs.file_deps + kwargs["name"] = _get_pkg_name(attrs) + kwargs["attribute_path"] = attrs.attr return kwargs -def _handle_repo_attrs(key, attrs): +def _handle_repo_attrs(attrs): kwargs = {} repo_set = bool(attrs.repo) @@ -71,14 +38,15 @@ def _handle_repo_attrs(key, attrs): if repo_set and repos_set: fail("Duplicate Nix repositories. Specify at most one of `repo` and `repos`.") elif repo_set: - kwargs["repository"] = nix_repo(key, attrs.repo) + kwargs["repository"] = attrs.repo elif repos_set: kwargs["repositories"] = { - name: nix_repo(key, repo) - for name, repo in attrs.repos.items() + name: repo + for repo, names in attrs.repos.items() + for name in names.split(":") } else: - kwargs["repository"] = nix_repo(key, "nixpkgs") + kwargs["repository"] = _DEFAULT_NIXKGS return kwargs @@ -97,177 +65,174 @@ def _handle_build_attrs(attrs): return kwargs -def _handle_opts_attrs(attrs): - return {"nixopts": attrs.nixopts or []} +def _handle_file_attrs(attrs): + kwargs = {"nix_file": attrs.file} -def _attr_pkg(attr): - return partial.make( - nixpkgs_package, - attribute_path = attr.attr, - repository = nix_repo("rules_nixpkgs_core", "nixpkgs"), - ) + if bool(attrs.file_deps): + kwargs["nix_file_deps"] = attrs.file_deps -def _local_attr_pkg(key, local_attr): - kwargs = {} + return kwargs - if bool(local_attr.attr): - kwargs["attribute_path"] = local_attr.attr - else: - kwargs["attribute_path"] = local_attr.name +def _handle_expr_attrs(attrs): + kwargs = {"nix_file_content": attrs.expr} + + if bool(attrs.file_deps): + kwargs["nix_file_deps"] = attrs.file_deps - kwargs.update(_handle_repo_attrs(key, local_attr)) - kwargs.update(_handle_build_attrs(local_attr)) - kwargs.update(_handle_opts_attrs(local_attr)) + return kwargs - return partial.make( - nixpkgs_package, - **kwargs +def _handle_opts_attrs(attrs): + return {"nixopts": attrs.nixopts or []} + +def _default_pkg(default): + nixpkgs_package( + name = default.attr, + attribute_path = default.attr, + repository = _DEFAULT_NIXKGS, ) -def _local_file_pkg(key, local_file): - kwargs = _handle_common_attrs(local_file) - kwargs.update(_handle_repo_attrs(key, local_file)) - kwargs.update(_handle_build_attrs(local_file)) - kwargs.update(_handle_file_attrs(local_file)) - kwargs.update(_handle_opts_attrs(local_file)) +def _attr_pkg(attr): + kwargs = _handle_common_attrs(attr) + kwargs.update(_handle_repo_attrs(attr)) + kwargs.update(_handle_build_attrs(attr)) + kwargs.update(_handle_opts_attrs(attr)) + + nixpkgs_package(**kwargs) + +def _file_pkg(file): + kwargs = _handle_common_attrs(file) + kwargs.update(_handle_repo_attrs(file)) + kwargs.update(_handle_build_attrs(file)) + kwargs.update(_handle_file_attrs(file)) + kwargs.update(_handle_opts_attrs(file)) # Indicate that nixpkgs_package is called from a module extension to # enable required workarounds. # TODO[AH] Remove this once the workarounds are no longer required. kwargs["_bzlmod"] = True - return partial.make( - nixpkgs_package, - **kwargs - ) + nixpkgs_package(**kwargs) -def _local_expr_pkg(key, local_expr): - kwargs = _handle_common_attrs(local_expr) - kwargs.update(_handle_repo_attrs(key, local_expr)) - kwargs.update(_handle_build_attrs(local_expr)) - kwargs.update(_handle_expr_attrs(local_expr)) - kwargs.update(_handle_opts_attrs(local_expr)) +def _expr_pkg(expr): + kwargs = _handle_common_attrs(expr) + kwargs.update(_handle_repo_attrs(expr)) + kwargs.update(_handle_build_attrs(expr)) + kwargs.update(_handle_expr_attrs(expr)) + kwargs.update(_handle_opts_attrs(expr)) - return partial.make( - nixpkgs_package, - **kwargs - ) + nixpkgs_package(**kwargs) + +_OVERRIDE_TAGS = { + "attr": _attr_pkg, + "file": _file_pkg, + "expr": _expr_pkg, +} def _nix_pkg_impl(module_ctx): - r = registry.make() + all_pkgs = sets.make() + root_deps = sets.make() + root_dev_deps = sets.make() + + is_isolated = getattr(module_ctx, "is_isolated", False) + # This loop handles all tags that can create global package overrides, or + # generate isolated package instances. References to global packages are + # handled later. for mod in module_ctx.modules: - key = fail_on_err(registry.add_module(r, name = mod.name, version = mod.version)) - - for attr in mod.tags.attr: - name = _name_from_attr(attr.attr) - fail_on_err( - registry.use_global_repo(r, key = key, name = name), - prefix = "Cannot use unified Nix package: ", - ) - if not registry.has_global_repo(r, name = name): - fail_on_err( - registry.add_global_repo( - r, - name = name, - repo = _attr_pkg(attr), - ), - prefix = "Cannot define unified Nix package: ", - ) - - for local_attr in mod.tags.local_attr: - fail_on_err( - registry.add_local_repo( - r, - key = key, - name = local_attr.name, - repo = _local_attr_pkg(key, local_attr), - ), - prefix = "Cannot use Nix package: ", - ) - - for local_file in mod.tags.local_file: - fail_on_err( - registry.add_local_repo( - r, - key = key, - name = local_file.name, - repo = _local_file_pkg(key, local_file), - ), - prefix = "Cannot use Nix package: ", - ) - - for local_expr in mod.tags.local_expr: - fail_on_err( - registry.add_local_repo( - r, - key = key, - name = local_expr.name, - repo = _local_expr_pkg(key, local_expr), - ), - prefix = "Cannot use Nix package: ", - ) - - for repo_name, repo in registry.get_all_repositories(r).items(): - partial.call(repo, name = repo_name) - - fail_on_err( - registry.hub_repo(r, name = "nixpkgs_packages", accessor = _ACCESSOR), - prefix = "Failed to generate `nixpkgs_packages`: ", + module_pkgs = sets.make() + + is_root = mod.is_root + is_core = mod.name == "rules_nixpkgs_core" + may_override = is_isolated or is_root or is_core + + for tag_name, tag_fun in _OVERRIDE_TAGS.items(): + for tag in getattr(mod.tags, tag_name): + is_dev_dep = module_ctx.is_dev_dependency(tag) + + if not may_override: + fail(_ISOLATED_OR_ROOT_ONLY_ERROR.format(tag_name = tag_name)) + + pkg_name = _get_pkg_name(tag) + + if sets.contains(module_pkgs, pkg_name): + fail(_DUPLICATE_PACKAGE_NAME_ERROR.format(package_name = pkg_name, tag_name = tag_name)) + else: + sets.insert(module_pkgs, pkg_name) + + if is_root: + if is_dev_dep: + sets.insert(root_dev_deps, pkg_name) + else: + sets.insert(root_deps, pkg_name) + + if not sets.contains(all_pkgs, pkg_name): + sets.insert(all_pkgs, pkg_name) + tag_fun(tag) + + # Here we loop through the default tags only to check for duplicates. + # The imports or instantiations are performed later. + for default in mod.tags.default: + is_dev_dep = module_ctx.is_dev_dependency(default) + + if sets.contains(module_pkgs, default.attr): + if is_root and not is_dev_dep and sets.contains(root_dev_deps, default.attr): + # Collisions between default and overrides are allowed in + # the root module if the override is a dev-dependency and + # the default is not. + sets.remove(root_dev_deps, default.attr) + sets.insert(root_deps, default.attr) + else: + fail(_DUPLICATE_PACKAGE_NAME_ERROR.format(package_name = default.attr, tag_name = "default")) + else: + sets.insert(module_pkgs, default.attr) + + # This loop handles references to global packages. Any instance of a global + # override was already instantiated at this point, so we can resolve + # references from all modules. + for mod in module_ctx.modules: + is_root = mod.is_root + + for default in mod.tags.default: + is_dev_dep = module_ctx.is_dev_dependency(default) + + if is_isolated: + fail(_ISOLATED_NOT_ALLOWED_ERROR.format(tag_name = "default")) + + if not sets.contains(all_pkgs, default.attr): + sets.insert(all_pkgs, default.attr) + _default_pkg(default) + + if is_root: + if is_dev_dep: + sets.insert(root_dev_deps, default.attr) + else: + sets.insert(root_deps, default.attr) + + return module_ctx.extension_metadata( + root_module_direct_deps = sets.to_list(root_deps), + root_module_direct_dev_deps = sets.to_list(root_dev_deps), ) -_ATTR_ATTRS = { +_DEFAULT_ATTRS = { "attr": attr.string( doc = "The attribute path of the package to import. The attribute path is a sequence of attribute names separated by dots.", mandatory = True, ), } -_LOCAL_ATTR_ATTRS = { - "name": attr.string( - doc = "A unique name for this package. The name must be unique within the requesting module.", - mandatory = True, - ), - "attr": attr.string( - doc = "The attribute path of the package to import. The attribute path is a sequence of attribute names separated by dots. Defaults to `name`.", - mandatory = False, - ), -} - _COMMON_ATTRS = { - "name": attr.string( - doc = "A unique name for this package. The name must be unique within the requesting module.", - mandatory = True, - ), "attr": attr.string( - doc = "The attribute path of the package to import. The attribute path is a sequence of attribute names separated by dots. Import the top-level Nix expression if empty.", - mandatory = False, - ), -} - -_FILE_DEPS_ATTRS = { - "file_deps": attr.label_list( - doc = "Files required by the Nix expression.", - mandatory = False, - ), -} - -_FILE_ATTRS = { - "file": attr.label( - doc = "The file containing the Nix expression.", + doc = "The attribute path of the package to configure and import. The attribute path is a sequence of attribute names separated by dots.", mandatory = True, ), -} - -_EXPR_ATTRS = { - "expr": attr.string( - doc = "The Nix expression.", - mandatory = True, + "name": attr.string( + doc = "Configure and import the package under this name instead of the attribute path. Other modules must pass this name to the `default` tag to refer to this package.", + mandatory = False, ), } _REPO_ATTRS = { - "repo": attr.string( + "repo": attr.label( doc = """\ The Nix repository to use. Equivalent to `repos = {"nixpkgs": repo}`. @@ -275,13 +240,13 @@ Specify at most one of `repo` or `repos`. """, mandatory = False, ), - "repos": attr.string_dict( + "repos": attr.label_keyed_string_dict( doc = """\ -The Nix repositories to use. The dictionary keys represent the names of the -`NIX_PATH` entries. For example, `repositories = { "myrepo" : "somerepo" }` +The Nix repositories to use. The dictionary values represent the names of the +`NIX_PATH` entries. For example, `repositories = { "@somerepo" : "myrepo" }` would replace all instances of `` in the Nix code by the path to the -Nix repository `somerepo`. See the [relevant section in the nix -manual](https://nixos.org/nix/manual/#env-NIX_PATH) for more information. +Nix repository `@somerepo`. You can provide multiple `NIX_PATH` entry names for a single repository as a colon (`:`) separated string. See the [relevant section in the nix +manual](https://nixos.org/manual/nix/stable/command-ref/env-common.html#env-NIX_PATH) for more information. Specify at most one of `repo` or `repos`. """, mandatory = False, @@ -328,6 +293,27 @@ Specify at most one of `build_file` or `build_file_content`. ), } +_FILE_ATTRS = { + "file": attr.label( + doc = "The file containing the Nix expression.", + mandatory = True, + ), +} + +_EXPR_ATTRS = { + "expr": attr.string( + doc = "The Nix expression.", + mandatory = True, + ), +} + +_FILE_DEPS_ATTRS = { + "file_deps": attr.label_list( + doc = "Files required by the Nix expression.", + mandatory = False, + ), +} + _OPTS_ATTRS = { "nixopts": attr.string_list( doc = "Extra flags to pass when calling Nix. Note, this does not currently support location expansion.", @@ -336,32 +322,32 @@ _OPTS_ATTRS = { ), } -_attr_tag = tag_class( - attrs = _ATTR_ATTRS, - doc = "Import a globally unified Nix package. If multiple Bazel modules import the same nixpkgs attribute, then they will all use the same external Bazel repository that imports the Nix package.", +_default_tag = tag_class( + attrs = _DEFAULT_ATTRS, + doc = "Import a globally unified Nix package from the default nixpkgs repository. May not be used on an isolated module extension.", ) -_local_attr_tag = tag_class( - attrs = dicts.add(_LOCAL_ATTR_ATTRS, _REPO_ATTRS, _BUILD_ATTRS, _OPTS_ATTRS), - doc = "Import a Nix package by attribute path.", +_attr_tag = tag_class( + attrs = dicts.add(_COMMON_ATTRS, _REPO_ATTRS, _BUILD_ATTRS, _OPTS_ATTRS), + doc = "Configure and import a Nix package by attribute path. Overrides default imports of this package. May only be used on an isolated module extension or in the root module or rules_nixpkgs_core.", ) -_local_file_tag = tag_class( +_file_tag = tag_class( attrs = dicts.add(_COMMON_ATTRS, _REPO_ATTRS, _BUILD_ATTRS, _FILE_ATTRS, _FILE_DEPS_ATTRS, _OPTS_ATTRS), - doc = "Import a Nix package from a local file.", + doc = "Configure and import a Nix package from a file. Overrides default imports of this package. May only be used on an isolated module extension or in the root module or rules_nixpkgs_core.", ) -_local_expr_tag = tag_class( +_expr_tag = tag_class( attrs = dicts.add(_COMMON_ATTRS, _REPO_ATTRS, _BUILD_ATTRS, _EXPR_ATTRS, _FILE_DEPS_ATTRS, _OPTS_ATTRS), - doc = "Import a Nix package from a local expression.", + doc = "Configure and import a Nix package from a Nix expression. Overrides default imports of this package. May only be used on an isolated module extension or in the root module or rules_nixpkgs_core.", ) nix_pkg = module_extension( _nix_pkg_impl, tag_classes = { + "default": _default_tag, "attr": _attr_tag, - "local_attr": _local_attr_tag, - "local_file": _local_file_tag, - "local_expr": _local_expr_tag, + "file": _file_tag, + "expr": _expr_tag, }, ) diff --git a/core/extensions/repository.bzl b/core/extensions/repository.bzl index bb4f5f20f..22404f1e2 100644 --- a/core/extensions/repository.bzl +++ b/core/extensions/repository.bzl @@ -1,33 +1,14 @@ """Defines the nix_repo module extension. """ -load("//:nixpkgs.bzl", "nixpkgs_http_repository", "nixpkgs_local_repository") -load("//:util.bzl", "fail_on_err") -load("//private:module_registry.bzl", "registry") load("@bazel_skylib//lib:dicts.bzl", "dicts") -load("@bazel_skylib//lib:partial.bzl", "partial") - -_ACCESSOR = '''\ -def nix_repo(module_name, name): - """Access a Nix repository imported with `nix_repo`. - - Args: - module_name: `String`; Name of the calling Bazel module. - This is needed until Bazel offers unique module identifiers, - see [#17652][bazel-17652]. - name: `String`; Name of the repository. - - Returns: - The resolved label to the repository's entry point. - - [bazel-17652]: https://github.com/bazelbuild/bazel/issues/17652 - """ - resolved = _fail_on_err( - _get_repository(module_name, name), - prefix = "Invalid Nix repository, you must use the nix_repo extension and request a global repository or register a local repository: ", - ) - return resolved -''' +load("@bazel_skylib//lib:sets.bzl", "sets") +load("//:nixpkgs.bzl", "nixpkgs_http_repository", "nixpkgs_local_repository") + +_ISOLATED_OR_ROOT_ONLY_ERROR = "Illegal use of the {tag_name} tag. The {tag_name} tag may only be used on an isolated module extension or in the root module or rules_nixpkgs_core." +_ISOLATED_NOT_ALLOWED_ERROR = "Illegal use of the {tag_name} tag. The {tag_name} tag may not be used on an isolated module extension." +_DUPLICATE_REPOSITORY_NAME_ERROR = "Duplicate nix_repo import due to {tag_name} tag. The repository name '{repo_name}' is already used." +_UNKNOWN_REPOSITORY_REFERENCE_ERROR = "Reference to unknown repository '{repo_name}' encountered on {tag_name} tag." def _github_repo(github): tag_set = bool(github.tag) @@ -48,8 +29,8 @@ def _github_repo(github): url = "https://github.com/%s/%s/archive/%s" % (github.org, github.repo, archive) - return partial.make( - nixpkgs_http_repository, + nixpkgs_http_repository( + name = github.name, url = url, integrity = github.integrity, sha256 = github.sha256, @@ -66,8 +47,8 @@ def _http_repo(http): if not url_set and not urls_set: fail("Missing URL. Specify one of `url` or `urls`.") - return partial.make( - nixpkgs_http_repository, + nixpkgs_http_repository( + name = http.name, url = http.url if url_set else None, urls = http.urls if urls_set else None, integrity = http.integrity, @@ -76,93 +57,106 @@ def _http_repo(http): ) def _file_repo(file): - return partial.make( - nixpkgs_local_repository, + nixpkgs_local_repository( + name = file.name, nix_file = file.file, nix_file_deps = file.file_deps, ) def _expr_repo(expr): - return partial.make( - nixpkgs_local_repository, + nixpkgs_local_repository( + name = expr.name, nix_file_content = expr.expr, nix_file_deps = expr.file_deps, ) +_OVERRIDE_TAGS = { + "github": _github_repo, + "http": _http_repo, + "file": _file_repo, + "expr": _expr_repo, +} + def _nix_repo_impl(module_ctx): - r = registry.make() + all_repos = sets.make() + root_deps = sets.make() + root_dev_deps = sets.make() + + is_isolated = getattr(module_ctx, "is_isolated", False) + # This loop handles all tags that can create global repository overrides, + # or generate isolated repository instances. References to global + # repositories are handled later. for mod in module_ctx.modules: - key = fail_on_err(registry.add_module(r, name = mod.name, version = mod.version)) + module_repos = sets.make() + + is_root = mod.is_root + is_core = mod.name == "rules_nixpkgs_core" + may_override = is_isolated or is_root or is_core + + for tag_name, tag_fun in _OVERRIDE_TAGS.items(): + for tag in getattr(mod.tags, tag_name): + is_dev_dep = module_ctx.is_dev_dependency(tag) + + if not may_override: + fail(_ISOLATED_OR_ROOT_ONLY_ERROR.format(tag_name = tag_name)) + + if sets.contains(module_repos, tag.name): + fail(_DUPLICATE_REPOSITORY_NAME_ERROR.format(repo_name = tag.name, tag_name = tag_name)) + else: + sets.insert(module_repos, tag.name) + + if is_root: + if is_dev_dep: + sets.insert(root_dev_deps, tag.name) + else: + sets.insert(root_deps, tag.name) + + if not sets.contains(all_repos, tag.name): + sets.insert(all_repos, tag.name) + tag_fun(tag) + # Here we loop through the default tags only to check for duplicates. + # The imports are performed later. for default in mod.tags.default: - fail_on_err( - registry.use_global_repo(r, key = key, name = default.name), - prefix = "Cannot use global default repository: ", - ) - - for github in mod.tags.github: - fail_on_err( - registry.add_local_repo( - r, - key = key, - name = github.name, - repo = _github_repo(github), - ), - prefix = "Cannot import GitHub repository: ", - ) - - for http in mod.tags.http: - fail_on_err( - registry.add_local_repo( - r, - key = key, - name = http.name, - repo = _http_repo(http), - ), - prefix = "Cannot import HTTP repository: ", - ) - - for file in mod.tags.file: - fail_on_err( - registry.add_local_repo( - r, - key = key, - name = file.name, - repo = _file_repo(file), - ), - prefix = "Cannot import file repository: ", - ) - - for expr in mod.tags.expr: - fail_on_err( - registry.add_local_repo( - r, - key = key, - name = expr.name, - repo = _expr_repo(expr), - ), - prefix = "Cannot import expression repository: ", - ) - - for override in mod.tags.override: - prefix = "Cannot override global default repository: " - repo = fail_on_err( - registry.pop_local_repo(r, key = key, name = override.name), - prefix = prefix, - ) - registry.set_default_global_repo(r, name = override.name, repo = repo) - fail_on_err( - registry.use_global_repo(r, key = key, name = default.name), - prefix = prefix, - ) - - for repo_name, repo in registry.get_all_repositories(r).items(): - partial.call(repo, name = repo_name) - - fail_on_err( - registry.hub_repo(r, name = "nixpkgs_repositories", accessor = _ACCESSOR), - prefix = "Failed to generate `nixpkgs_repositories`: ", + is_dev_dep = module_ctx.is_dev_dependency(default) + + if sets.contains(module_repos, default.name): + if is_root and not is_dev_dep and sets.contains(root_dev_deps, default.name): + # Collisions between default and overrides are allowed in + # the root module if the override is a dev-dependency and + # the default is not. + sets.remove(root_dev_deps, default.name) + sets.insert(root_deps, default.name) + else: + fail(_DUPLICATE_REPOSITORY_NAME_ERROR.format(repo_name = default.name, tag_name = "default")) + else: + sets.insert(module_repos, default.name) + + # This loop handles references to global repositories. Any instance of a + # global override was already instantiated at this point, so we can resolve + # references from all modules. + for mod in module_ctx.modules: + is_root = mod.is_root + + for default in mod.tags.default: + is_dev_dep = module_ctx.is_dev_dependency(default) + + if is_isolated: + fail(_ISOLATED_NOT_ALLOWED_ERROR.format(tag_name = "default")) + + if not sets.contains(all_repos, default.name): + fail(_UNKNOWN_REPOSITORY_REFERENCE_ERROR.format(repo_name = default.name, tag_name = "default")) + + if is_root: + if is_dev_dep: + sets.insert(root_dev_deps, default.name) + else: + sets.insert(root_deps, default.name) + + return module_ctx.extension_metadata( + root_module_direct_deps = sets.to_list(root_deps), + root_module_direct_dev_deps = sets.to_list(root_dev_deps), ) _DEFAULT_ATTRS = { @@ -179,17 +173,6 @@ _NAME_ATTRS = { ), } -_INTEGRITY_ATTRS = { - "integrity": attr.string( - doc = "Expected checksum in Subresource Integrity format of the file downloaded. This must match the checksum of the file downloaded. _It is a security risk to omit the checksum as remote files can change._ At best omitting this field will make your build non-hermetic. It is optional to make development easier but either this attribute or `sha256` should be set before shipping.", - mandatory = False, - ), - "sha256": attr.string( - doc = "The expected SHA-256 of the file downloaded. This must match the SHA-256 of the file downloaded. _It is a security risk to omit the SHA-256 as remote files can change._ At best omitting this field will make your build non-hermetic. It is optional to make development easier but either this attribute or `integrity` should be set before shipping.", - mandatory = False, - ), -} - _GITHUB_ATTRS = { "org": attr.string( default = "NixOS", @@ -226,9 +209,13 @@ _HTTP_ATTRS = { ), } -_FILE_DEPS_ATTRS = { - "file_deps": attr.label_list( - doc = "List of files required by the Nix expression.", +_INTEGRITY_ATTRS = { + "integrity": attr.string( + doc = "Expected checksum in Subresource Integrity format of the file downloaded. This must match the checksum of the file downloaded. _It is a security risk to omit the checksum as remote files can change._ At best omitting this field will make your build non-hermetic. It is optional to make development easier but either this attribute or `sha256` should be set before shipping.", + mandatory = False, + ), + "sha256": attr.string( + doc = "The expected SHA-256 of the file downloaded. This must match the SHA-256 of the file downloaded. _It is a security risk to omit the SHA-256 as remote files can change._ At best omitting this field will make your build non-hermetic. It is optional to make development easier but either this attribute or `integrity` should be set before shipping.", mandatory = False, ), } @@ -248,41 +235,36 @@ _EXPR_ATTRS = { ), } -_OVERRIDE_ATTRS = { - "name": attr.string( - doc = "The name of the global default repository to set.", - mandatory = True, +_FILE_DEPS_ATTRS = { + "file_deps": attr.label_list( + doc = "List of files required by the Nix expression.", + mandatory = False, ), } _default_tag = tag_class( attrs = _DEFAULT_ATTRS, - doc = "Depend on this global default repository.", + doc = "Depend on this global default repository. May not be used on an isolated module extension.", ) _github_tag = tag_class( attrs = dicts.add(_NAME_ATTRS, _GITHUB_ATTRS, _INTEGRITY_ATTRS), - doc = "Import a Nix repository from Github.", + doc = "Import a Nix repository from Github. May only be used on an isolated module extension or in the root module or rules_nixpkgs_core.", ) _http_tag = tag_class( attrs = dicts.add(_NAME_ATTRS, _HTTP_ATTRS, _INTEGRITY_ATTRS), - doc = "Import a Nix repository from an HTTP URL.", + doc = "Import a Nix repository from an HTTP URL. May only be used on an isolated module extension or in the root module or rules_nixpkgs_core.", ) _file_tag = tag_class( attrs = dicts.add(_NAME_ATTRS, _FILE_ATTRS, _FILE_DEPS_ATTRS), - doc = "Import a Nix repository from a local file.", + doc = "Import a Nix repository from a local file. May only be used on an isolated module extension or in the root module or rules_nixpkgs_core.", ) _expr_tag = tag_class( attrs = dicts.add(_NAME_ATTRS, _EXPR_ATTRS, _FILE_DEPS_ATTRS), - doc = "Import a Nix repository from a Nix expression.", -) - -_override_tag = tag_class( - attrs = _OVERRIDE_ATTRS, - doc = "Define the global default Nix repository. May only be used in the root module or rules_nixpkgs_core.", + doc = "Import a Nix repository from a Nix expression. May only be used on an isolated module extension or in the root module or rules_nixpkgs_core.", ) nix_repo = module_extension( @@ -293,6 +275,5 @@ nix_repo = module_extension( "http": _http_tag, "file": _file_tag, "expr": _expr_tag, - "override": _override_tag, }, ) diff --git a/core/private/BUILD.bazel b/core/private/BUILD.bazel deleted file mode 100644 index e69de29bb..000000000 diff --git a/core/private/module_registry.bzl b/core/private/module_registry.bzl deleted file mode 100644 index eeacd9717..000000000 --- a/core/private/module_registry.bzl +++ /dev/null @@ -1,418 +0,0 @@ -"""This module implements a generic registry for hub repositories. -""" - -load("@bazel_skylib//lib:partial.bzl", "partial") - -_DUPLICATE_MODULE_ERROR = "Duplicate module '{desc}', previous version '{prev}'." -_DUPLICATE_LOCAL_REPO_ERROR = "Duplicate local repository '{name}', requested by module '{desc}'." -_DUPLICATE_GLOBAL_REPO_ERROR = "Duplicate global repository '{name}'." -_MODULE_NOT_FOUND_ERROR = "Module not found: '{key}'." -_LOCAL_REPO_NOT_FOUND_ERROR = "Local repository '{name}' not found, requested by module '{desc}'." -_GLOBAL_REPO_NOT_REGISTERED_ERROR = "Global repository '{name}' not registered, requested by module '{desc}'." - -def _make_registry(): - """Create a new registry object. - - Returns: - The registry object. - """ - return struct( - _modules = {}, - _global_repositories = {}, - ) - -def _module_key(name, version): - """Generate a key to identify a module. - - Args: - name: String, The name of the module. - version: String, The version of the module. - - Returns: - String, The generated key for the module. - """ - - # TODO[AH] Take the module version into account - # to support multi version overrides. - return name - -def _module_description(name, version): - """Generate a human readable description for a module. - - Args: - name: String, The name of the module. - version: String, The version of the module. - - Returns: - String, The generated description for the module. - """ - if version: - return "{} {}".format(name, version) - else: - return name - -def _make_module(*, name, version, description): - """Create a new module object. - - Args: - name: String, The name of the module. - version: String, The version of the module. - description: String, The description of the module. - - Returns: - The module object. - """ - return struct( - _name = name, - _version = version, - _description = description, - _local_repositories = {}, - _global_repositories = {}, - ) - -def _add_module(r, *, name, version): - """Register a new module. - - Args: - r: The registry object. - name: String, The name of the module to register. - version: String, The version of the module to register. - - Returns: - Tuple, the first element is the key to access the module in the registry, - the second element is an error message in case of failure - or None if successful. - """ - key = _module_key(name, version) - desc = _module_description(name, version) - - if key in r._modules: - return None, _DUPLICATE_MODULE_ERROR.format( - desc = desc, - prev = r._modules[key]._version, - ) - - r._modules[key] = _make_module( - name = name, - version = version, - description = desc, - ) - - return key, None - -def _use_global_repo(r, *, key, name): - """Register the dependency of a module on a global repository. - - Args: - r: The registry object. - key: String, The key of the module in the registry. - name: String, The name of the global repository. - - Returns: - Tuple, the first element is None, - the second element is an error message in case of failure - or None if successful. - """ - module = r._modules.get(key) - if module == None: - return None, _MODULE_NOT_FOUND_ERROR.format(key = key) - - module._global_repositories[name] = True - - return None, None - -def _add_local_repo(r, *, key, name, repo): - """Register a new local repository for the module. - - Args: - r: The registry object. - key: String, The key of the module in the registry. - name: String, The name of the local repository to register. - repo: The repository object to store in the registry. - - Returns: - Tuple, the first element is None, - the second element is an error message in case of failure - or None if successful. - """ - module = r._modules.get(key) - if module == None: - return None, _MODULE_NOT_FOUND_ERROR.format(key = key) - - if name in module._local_repositories: - return None, _DUPLICATE_LOCAL_REPO_ERROR.format(name = name, desc = module._description) - - module._local_repositories[name] = repo - - return None, None - -def _pop_local_repo(r, *, key, name): - """Remove the local repository from the registry and return its value. - - Fail if the repository is missing. - - Args: - r: The registry object. - key: String, The key of the module in the registry. - name: String, The name of the local repository to remove. - - Returns: - Tuple, the first element is the repository object if successful, - the second element is an error message in case of failure - or None if successful. - """ - module = r._modules.get(key) - if module == None: - return None, _MODULE_NOT_FOUND_ERROR.format(key = key) - - if not name in module._local_repositories: - return None, _LOCAL_REPO_NOT_FOUND_ERROR.format(name = name, desc = module._description) - - return module._local_repositories.pop(name), None - -def _set_default_global_repo(r, *, name, repo): - """Register a global repository if it has not been previously registered. - - Args: - r: The registry object. - name: String, The name of the global repository to register. - repo: Any, The repository object to store in the registry. - """ - if name not in r._global_repositories: - r._global_repositories[name] = repo - -def _has_global_repo(r, *, name): - """Check if the global repository exists in the registry. - - Args: - r: The registry object. - name: String, The name of the global repository to check. - - Returns: - Bool, True if the global repository exists, False otherwise. - """ - return name in r._global_repositories - -def _get_global_repo(r, *, name): - """Retrieve a global repository by its name. - - Args: - r: The registry object. - name: String, The name of the global repository to retrieve. - - Returns: - The repository object if found, None otherwise. - """ - return r._global_repositories.get(name) - -def _add_global_repo(r, *, name, repo): - """Register a new global repository. - - Args: - r: The registry object. - name: String, The name of the global repository to register. - repo: Any, The repository object to store in the registry. - - Returns: - Tuple, the first element is None, - the second element is an error message in case of failure - or None if successful. - """ - if name in r._global_repositories: - return None, _DUPLICATE_GLOBAL_REPO_ERROR.format(name = name) - - r._global_repositories[name] = repo - - return None, None - -def _canonical_repo_name(module_name, module_version, repo_name): - """Generate the canonical name for a repository. - - Args: - module_name: String, The name of the module. - module_version: String, The version of the module. - repo_name: String, The name of the repository. - - Returns: - String, The canonical name of the repository. - """ - return "{}_{}_{}".format(module_name, module_version, repo_name) - -def _get_all_repositories(r): - """Retrieve a dictionary containing all repositories. - - Args: - r: The registry object. - - Returns: - Dict, A dictionary containing all repositories, mapping canonical - repository names to stored repository objects. - """ - all_repositories = {} - - for name, repo in r._global_repositories.items(): - all_repositories[name] = repo - - for module_key, module in r._modules.items(): - for repo_name, repo in module._local_repositories.items(): - canonical_name = _canonical_repo_name(module._name, module._version, repo_name) - all_repositories[canonical_name] = repo - - return all_repositories - -def _get_all_module_scopes(r): - """Get mapping from module keys to all repositories used by that module. - - Args: - r: The registry object. - - Returns: - Tuple, the first element is a mapping of module keys - to all repositories used by that module, - the second element is an error message in case of failure - or None if successful. - """ - module_scopes = {} - - for module_key, module in r._modules.items(): - repos = {} - - for repo_name in module._global_repositories: - if not repo_name in r._global_repositories: - return None, _GLOBAL_REPO_NOT_REGISTERED_ERROR.format(name = repo_name, desc = module._description) - - repos[repo_name] = repo_name - - for repo_name, repo in module._local_repositories.items(): - canonical_name = _canonical_repo_name(module._name, module._version, repo_name) - repos[repo_name] = canonical_name - - module_scopes[module_key] = repos - - return module_scopes, None - -_NIXPKGS_REPOSITORIES_DEFS = '''\ -load("@rules_nixpkgs_core//:util.bzl", _fail_on_err = "fail_on_err") - -{accessor} - -def _get_repository(module_name, name): - """Access a registered repository from the module registry. - - Args: - module_name: `String`; Name of the calling Bazel module. - This is needed until Bazel offers unique module identifiers, - see [#17652][bazel-17652]. - name: `String`; Name of the repository. - - Returns: - Pair of (`Label`, optional `String`) - - `Label`; The resolved label to the repository's entry point. - - optional `String`; `None` indicates success, otherwise an error. - - [bazel-17652]: https://github.com/bazelbuild/bazel/issues/17652 - """ - key = module_name - - if key in _MODULES and key in _MODULE_SCOPES: - description = _MODULES[key] - repos = _MODULE_SCOPES[key] - else: - return None, "Unknown module - no module found under the key '{{}}'.".format(key) - - if name in repos: - repo_name = repos[name] - else: - return None, "Unknown repository - no repository named '{{}}' available for module '{{}}'.".format(name, description) - - return Label("@" + repo_name), None - -_MODULES = {{ -{modules} -}} -_MODULE_SCOPES = {{ -{module_scopes} -}} -''' - -def _format_module_scopes(module_scopes): - return "\n".join([ - " {}: {{\n{}\n }},".format(repr(key), "\n".join([ - " {}: {},".format(repr(repo_name), repr(workspace_name)) - for repo_name, workspace_name in sorted(repos.items()) - ])) - for key, repos in sorted(module_scopes.items()) - ]) - -def _format_modules(modules): - return "\n".join([ - " {}: {},".format(repr(key), repr(desc)) - for key, desc in sorted(modules.items()) - ]) - -def _hub_repo_rule_impl(repository_ctx): - module_scopes = {} - - for module_key, encoded_repos in repository_ctx.attr.module_scopes.items(): - repos = {} - - for encoded in encoded_repos: - repo_name, workspace_name = encoded.split("~", 1) - repos[repo_name] = workspace_name - - module_scopes[module_key] = repos - - defs = _NIXPKGS_REPOSITORIES_DEFS.format( - accessor = repository_ctx.attr.accessor, - modules = _format_modules(repository_ctx.attr.modules), - module_scopes = _format_module_scopes(module_scopes), - ) - repository_ctx.file("defs.bzl", content = defs, executable = False) - repository_ctx.file("BUILD.bazel", content = "", executable = False) - -_hub_repo_rule = repository_rule( - _hub_repo_rule_impl, - attrs = { - "accessor": attr.string(), - "modules": attr.string_dict(), - "module_scopes": attr.string_list_dict(), - }, -) - -def _hub_repo(r, *, name, accessor): - module_scopes, err = registry.get_all_module_scopes(r) - if err: - return None, err - - _hub_repo_rule( - name = name, - accessor = accessor, - modules = { - key: module._description - for key, module in r._modules.items() - }, - module_scopes = { - module_key: [ - "{}~{}".format(repo_name, canonical_name) - for repo_name, canonical_name in repos.items() - ] - for module_key, repos in module_scopes.items() - }, - ) - - return None, None - -registry = struct( - make = _make_registry, - add_module = _add_module, - has_global_repo = _has_global_repo, - get_global_repo = _get_global_repo, - add_global_repo = _add_global_repo, - set_default_global_repo = _set_default_global_repo, - use_global_repo = _use_global_repo, - add_local_repo = _add_local_repo, - pop_local_repo = _pop_local_repo, - get_all_repositories = _get_all_repositories, - get_all_module_scopes = _get_all_module_scopes, - hub_repo = _hub_repo, -) diff --git a/design/bzlmod/README.md b/design/bzlmod/README.md index 6adcaf05e..a20cbeb09 100644 --- a/design/bzlmod/README.md +++ b/design/bzlmod/README.md @@ -193,7 +193,10 @@ Support the use of Nix built packages as Bazel toolchains. into the package and toolchain repositories through that intermediary. A known repository that provides a macro to convert tag names into resolved labels to the repository could be such an intermediary, i.e. a hub - repository, see below. + repository, see below. Alternatively, the user could pass the required + repositories into the tag by label. The latter approach is simpler, uses + Bazel's core features, and introduces lower migration cost as WORKSPACE + users already used labels to refer to imports. ### Module Extensions Have Global Scope @@ -203,6 +206,8 @@ Support the use of Nix built packages as Bazel toolchains. extension. * Repository rules are assigned names in a global scope for the current module extension. + * Recent Bazel versions (starting from 6.3) introduced ["isolated" module + extensions][isolted-ext]. * Impact * Module extensions must either reconcile or avoid name clashes due to external workspaces defined based on tags requested by different Bazel @@ -210,6 +215,9 @@ Support the use of Nix built packages as Bazel toolchains. named `"nixpkgs"`, then the module extension must either unify that tag into a single external workspace under that name, or avoid collision by generating separate external workspaces with unique names. + * If the ability to define new names is limited to isolated extensions, the + root module, and rules_nixpkgs_core itself, then name can be easily + controlled by the user and unintended collisions can be avoided. ### External Workspaces Have Restricted Visibility @@ -279,6 +287,21 @@ Support the use of Nix built packages as Bazel toolchains. [auto-use-repo]: https://docs.google.com/document/d/1dj8SN5L6nwhNOufNqjBhYkk5f-BJI_FPYWKxlB3GAmA/edit?disco=AAAArdGBwhc +### Isolated Extensions and Automatic Imports + +* Features + * As mentioned above, module extensions can be declared as isolated to limit + their scope to one module. + * As mentioned above, Bazel can automatically manage the required `use_repo` + stanzas. +* Impact + * A hub-repository is no longer necessary to separate globally unified and + per-module scoped repositories. Instead, isolated extensions can be used to + generate per-module scoped repositories, while regular extensions manage + globally unified repositories. + * If repositories are explicitly imported using `use_repo` stanzas, then they + can be forwarded to other tags simply by label. + ### Nixpkgs Repositories or Packages Have No Convenient Canonical Name * Constraint @@ -306,6 +329,10 @@ Support the use of Nix built packages as Bazel toolchains. repositories and packages as defined in each Bazel module, taking into account nixpkgs repositories defined within the scope of that module, or explicitly imported from another module (if that's feasible). + * In light of isolated extensions and automatic imports we can restrict the + ability to assign non-canonical names to the root module and + rules_nixpkgs_core or isolated extensions and use attribute paths into the + default nixpkgs repository as canonical names for package imports. ### The Diamond Dependency Problem @@ -398,21 +425,28 @@ rules\_nixpkgs\_core itself will define a global default `nixpkgs` repository, any module can reference this global default repository like so. ```python -use_extension("//extensions:repository.bzl", "nix_repo") +nix_repo = use_extension("//extensions:repository.bzl", "nix_repo") nix_repo.default(name = "nixpkgs") + +use_repo(nix_repo, "nixpkgs") ``` ### Local Repository -Any Bazel module can define custom Nix repositories for local use. +Any Bazel module can use an isolated extension to define custom Nix +repositories for local use. ```python -nix_repo.github( +nix_repo_isolated = use_extension("//extensions:repository.bzl", "nix_repo", isolate = True) + +nix_repo_isolated.github( name = "nixpkgs-unstable", commit = "1eeea1f1922fb79a36008ba744310ccbf96130e2", sha256 = "d6759a60a91dfd03bdd4bf9c834e1acd348cf5ca80c6a6795af5838165bc7ea6", ) + +use_repo(nix_repo, "nixpkgs-unstable") ``` ### Repository Override @@ -420,63 +454,96 @@ nix_repo.github( The root module can override the default set by rules\_nixpkgs\_core. ```python -nix_repo.override(name = "nixpkgs") +nix_repo = use_extension("//extensions:repository.bzl", "nix_repo") + nix_repo.http( name = "nixpkgs", url = "https://github.com/NixOS/nixpkgs/archive/1eeea1f1922fb79a36008ba744310ccbf96130e2.tar.gz", sha256 = "d6759a60a91dfd03bdd4bf9c834e1acd348cf5ca80c6a6795af5838165bc7ea6", strip_prefix = "nixpkgs-1eeea1f1922fb79a36008ba744310ccbf96130e2", ) + +use_repo(nix_repo, "nixpkgs") ``` rules\_nixpkgs\_core uses the same mechanism to define the global default. -``` -nix_repo.override(name = "nixpkgs") -nix_repo.github( +A module can override the default as a dev-dependency, i.e. conditionally if it +is the root module. + +```python +nix_repo = use_extension("//extensions:repository.bzl", "nix_repo") +nix_repo.default(name = "nixpkgs") +use_repo(nix_repo, "nixpkgs") + +nix_repo_dev = use_extension("//extensions:repository.bzl", "nix_repo", dev_dependency = True) +nix_repo_dev.http( name = "nixpkgs", - tag = "22.11", - sha256 = "ddc3428d9e1a381b7476750ac4dbea7a42885cbbe6e1af44b21d6447c9609a6f", + url = "https://github.com/NixOS/nixpkgs/archive/1eeea1f1922fb79a36008ba744310ccbf96130e2.tar.gz", + sha256 = "d6759a60a91dfd03bdd4bf9c834e1acd348cf5ca80c6a6795af5838165bc7ea6", + strip_prefix = "nixpkgs-1eeea1f1922fb79a36008ba744310ccbf96130e2", ) ``` ### Unified Nix Package Bazel modules can depend on Nix packages by attribute path into a global Nix -repository (by default `nixpkgs`). These package references are unified -globally based on the attribute path, such that every Bazel module requesting -this package will be given the same instance of the package. This is meant to -avoid diamond dependency issues, see above. +repository (`nixpkgs`). These package references are unified globally based on +the attribute path, such that every Bazel module requesting this package will +be given the same instance of the package. This is meant to avoid diamond +dependency issues, see above. ```python -use_extension("@rules_nixpkgs_core//extensions:package.bzl", "nix_pkg") +nix_pkg = use_extension("@rules_nixpkgs_core//extensions:package.bzl", "nix_pkg") nix_pkg.attr(attr = "jq") nix_pkg.attr(attr = "gawk") + +use_repo(nix_pkg, "jq", "gawk") ``` ### Local Nix Package A Bazel module can import a custom Nix package from an expression or file and -provide a custom BUILD file template if required. The possibilities for -customization are too great to attempt global unification of such packages. If -two different Bazel modules effectively request the same such Nix package, then -rules\_nixkgs will still generate two separate external repositories to import -the package for each module. +provide a custom BUILD file template if required using an isoloated extension. +The possibilities for customization are too great to attempt global unification +of such packages. If two different Bazel modules effectively request the same +such Nix package, then rules\_nixkgs will still generate two separate external +repositories to import the package for each module. ```python -nix_pkgs.local_attr(name = "jq", repo = "nixpkgs-unstable") +nix_pkg_isolated = use_extension("@rules_nixpkgs_core//extensions:package.bzl", "nix_pkg", isolate = True) + +nix_pkg_isolated.attr(attr = "jq", repo = "@nixpkgs-unstable") -nix_pkg.local_expr( +nix_pkg_isolated.expr( name = "awk", + attr = "", expr = """\ with import { config = {}; overlays = []; }; gawk-with-extensions.override { extensions = with gawkextlib; [ csv json ]; } """, - repo = "nixpkgs-unstable", + repo = "@nixpkgs-unstable", ) + +use_repo(nix_pkg_isolated, "jq", "awk") +``` + +### Package Override + +The root module (and rules_nixpkgs_core) can override a globally unified +package, providing a custom repository, Nix expression, or BUILD file template, +such that other modules referring to the same attribute path will import the +same overriden instance. + +```python +use_extension("@rules_nixpkgs_core//extensions:package.bzl", "nix_pkg") + +nix_pkg.attr(attr = "jq", repo = "@nixpkgs-unstable") + +use_repo(nix_pkg, "jq") ``` ## Interface @@ -519,8 +586,6 @@ offers tags to define Nix repositories: * `name`: `String`; unique name. * `expr`: `String`; the Nix expression. * `file_deps`: optional, List of `Label`, files required by `expr`. -* `override(repo)` (only allowed in rules\_nixpkgs\_core and root)\ - * `repo`: `String`; The name of the repository to override. All `name` attributes define a unique name for the given Nix repository within the scope of the requesting module. @@ -528,40 +593,24 @@ the scope of the requesting module. The extension is defined in its own Starlark module under `@rules_nixpkgs_core//extensions:repository.bzl`. -The extension generates a hub repository called `nixpkgs_repositories` that -exposes a macro from `//:defs.bzl` to access the imported repositories from the -scope of the calling module: - -* `nix_repo(module, name)`\ - Attrs: - * `module`: `String`; name of the calling Bazel module.\ - Needed until Bazel offers an API to infer the calling module. - See, [#17652][bazel-17652].\ - Note, this is ambiguous for multi-version overrides.\ - TODO: Handle multi-version overrides. - * `name`: `String`; name of the repository.\ - This is the name used on the `nix_repo` tag. - - Returns:\ - The resolved `Label` object to the repository. - -Users are not expected to invoke `nix_repo` directly. Instead, it will be -invoked by the package and toolchain module extensions to access the relevant -repositories. +The extension returns `extension_metadata` (see [Automatic `use_repo` +fixups][auto-use-repo]) to declare which repositories are dependencies or +dev-dependencies of the root module, such that Bazel can check the imports and +generated buildozer commands to update the `use_repo` stanzas if needed. ### Nix Packages The `rules_nixpkgs_core` module exposes the module extension `nix_pkg` which offers tags to define Nix packages: -* `attr(attr)` (globally unified)\ +* `default(attr)` (globally unified)\ * `attr`: `String`; the attribute path.\ -* `local_attr(name, attr, repo, build_file, build_file_content)`\ - * `name`: `String`; unique name. - * `attr`: optional, `String`; the attribute path.\ - Default: `name`. - * `repo`: optional, `String`; the `nixpkgs` repository to import from.\ - Default: `nixpkgs`. +* `attr(name, attr, repo, build_file, build_file_content)`\ + * `name`: optional, `String`; unique name. + Default: `attr`. + * `attr`: `String`; the attribute path.\ + * `repo`: optional, `Label`; the `nixpkgs` repository to import from.\ + Default: `@nixpkgs`. * `build_file`: optional, `Label`; `BUILD` file to write into the external workspace.\ Specify at most one of `build_file` or `build_file_content`. @@ -569,16 +618,17 @@ offers tags to define Nix packages: the external workspace.\ Specify at most one of `build_file` or `build_file_content`. * `nixopts`: optional, List of `String`; Extra flags to pass to Nix. -* `local_file(name, attr, file, file_deps, repo, repos)`\ - * `name`: `String`; unique name. - * `attr`: optional, `String`; the attribute path.\ +* `file(name, attr, file, file_deps, repo, repos)`\ + * `name`: optional, `String`; unique name. + Default: `attr`. + * `attr`: `String`; the attribute path.\ * `file`: `Label`; the file containing the Nix expression. * `file_deps`: optional, List of `Label`, files required by `file`. - * `repo`: optional, `String`; use this `nixpkgs` repository. - Equivalent to `repos = {"nixpkgs": repo}`. + * `repo`: optional, `Label`; use this `nixpkgs` repository. + Equivalent to `repos = {repo: "nixpkgs"}`. Specify only one of `repo` or `repos`. - Default: `nixpkgs`. - * `repos`: optional, Dict of `String`; use these Nix repositories. + Default: `@nixpkgs`. + * `repos`: optional, Dict of `Label` to `String`; use these Nix repositories. The dictionary key represents the name of the `NIX_PATH` entry. Specify only one of `repo` or `repos`. * `build_file`: optional, `Label`; `BUILD` file to write into the external @@ -587,15 +637,16 @@ offers tags to define Nix packages: * `build_file_content`: optional, `Label`; `BUILD` file content to write into the external workspace.\ * `nixopts`: optional, List of `String`; Extra flags to pass to Nix. -* `local_expr(name, attr, expr, repo, repos)`\ - * `name`: `String`; unique name. - * `attr`: optional, `String`; the attribute path.\ +* `expr(name, attr, expr, repo, repos)`\ + * `name`: optional, `String`; unique name. + Default: `attr`. + * `attr`: `String`; the attribute path.\ * `expr`: `String`; the Nix expression. - * `repo`: optional, `String`; use this `nixpkgs` repository. - Equivalent to `repos = {"nixpkgs": repo}`. + * `repo`: optional, `Label`; use this `nixpkgs` repository. + Equivalent to `repos = {repo: "nixpkgs"}`. Specify only one of `repo` or `repos`. - Default: `nixpkgs`. - * `repos`: optional, Dict of `String`; use these Nix repositories. + Default: `@nixpkgs`. + * `repos`: optional, Dict of `Label` to `String`; use these Nix repositories. The dictionary key represents the name of the `NIX_PATH` entry. Specify only one of `repo` or `repos`. * `build_file`: optional, `Label`; `BUILD` file to write into the external @@ -605,29 +656,17 @@ offers tags to define Nix packages: the external workspace.\ * `nixopts`: optional, List of `String`; Extra flags to pass to Nix. -All `name` attributes define a unique name for the given Nix repository within -the scope of the requesting module. +All `name` attributes define the name for the generated external repository, +either globally, in case of override, or in the scope of the current module, in +case of an isolated extension. If `name` is not set, it defaults to `attr`. The extension is defined in its own Starlark module under `@rules_nixpkgs_core//extensions:package.bzl`. -The extension generates a hub repository called `nixpkgs_packages` that exposes -a macro from `//:defs.bzl` to access the imported packages from the scope of -the calling module: - -* `nix_pkg(module, name, label)`\ - Attrs: - * `module`: `String`; name of the calling Bazel module.\ - Needed until Bazel offers an API to infer the calling module. - See, [#17652][bazel-17652].\ - Note, this is ambiguous for multi-version overrides.\ - TODO: Handle multi-version overrides. - * `name`: `String`; name of the package.\ - This is the name used on the `nix_pkg` tag. - * `label`: `String`; the label to resolve within the package. - - Returns:\ - The resolved `Label` object. +The extension returns `extension_metadata` (see [Automatic `use_repo` +fixups][auto-use-repo]) to declare which packages are dependencies or +dev-dependencies of the root module, such that Bazel can check the imports and +generated buildozer commands to update the `use_repo` stanzas if needed. ### Nix Toolchains @@ -655,3 +694,4 @@ contains toolchain targets for all imported toolchains. The [bzlmod-undetected-cycles]: https://github.com/bazelbuild/bazel/issues/17564 [bzlmod-import]: https://bazelbuild.slack.com/archives/C014RARENH0/p1677600643532639?thread_ts=1677077009.456189&cid=C014RARENH0 [bazel-17652]: https://github.com/bazelbuild/bazel/issues/17652 +[isolated-ext]: https://github.com/bazelbuild/bazel/pull/18529 diff --git a/flake.lock b/flake.lock index dbe99c478..d1ab815af 100644 --- a/flake.lock +++ b/flake.lock @@ -21,11 +21,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1692799911, - "narHash": "sha256-3eihraek4qL744EvQXsK1Ha6C3CR7nnT8X2qWap4RNk=", + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", "owner": "numtide", "repo": "flake-utils", - "rev": "f9e7cf818399d17d347f847525c5a5a8032e4e44", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", "type": "github" }, "original": { @@ -36,16 +36,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1688392541, - "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=", + "lastModified": 1695644571, + "narHash": "sha256-asS9dCCdlt1lPq0DLwkVBbVoEKuEuz+Zi3DG7pR/RxA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b", + "rev": "6500b4580c2a1f3d0f980d32d285739d8e156d92", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-22.11", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 63abf7a17..f6fd9eb49 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,6 @@ { inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; flake-compat = { url = "github:edolstra/flake-compat"; flake = false; diff --git a/testing/core/.bazelignore b/testing/core/.bazelignore new file mode 100644 index 000000000..5377eb49f --- /dev/null +++ b/testing/core/.bazelignore @@ -0,0 +1 @@ +tests/intermediate_module diff --git a/testing/core/MODULE.bazel b/testing/core/MODULE.bazel index ae934bac1..25c3e657a 100644 --- a/testing/core/MODULE.bazel +++ b/testing/core/MODULE.bazel @@ -6,8 +6,13 @@ local_path_override( path = "../../core", ) -bazel_dep(name = "bazel_skylib", version = "1.0.3") +bazel_dep(name = "intermediate_module") +local_path_override( + module_name = "intermediate_module", + path = "tests/intermediate_module", +) +bazel_dep(name = "bazel_skylib", version = "1.0.3") bazel_dep(name = "nixpkgs_location_expansion_test_file") local_path_override( module_name = "nixpkgs_location_expansion_test_file", @@ -15,18 +20,17 @@ local_path_override( ) nix_repo = use_extension("@rules_nixpkgs_core//extensions:repository.bzl", "nix_repo") -use_repo(nix_repo, "nixpkgs_repositories") nix_repo.default(name = "nixpkgs") nix_repo.github( name = "remote_nixpkgs", - tag = "22.05", sha256 = "0f8c25433a6611fa5664797cd049c80faefec91575718794c701f3b033f2db01", + tag = "22.05", ) nix_repo.http( name = "http_nixpkgs", - url = "https://github.com/NixOS/nixpkgs/archive/refs/tags/22.05.tar.gz", sha256 = "0f8c25433a6611fa5664797cd049c80faefec91575718794c701f3b033f2db01", strip_prefix = "nixpkgs-22.05", + url = "https://github.com/NixOS/nixpkgs/archive/refs/tags/22.05.tar.gz", ) nix_repo.file( name = "file_nixpkgs", @@ -43,24 +47,43 @@ nix_repo.expr( "//:flake.lock", ], ) +use_repo( + nix_repo, + "file_nixpkgs", + "http_nixpkgs", + "nixpkgs", + "nixpkgs_content", + "remote_nixpkgs", +) nix_pkg = use_extension("@rules_nixpkgs_core//extensions:package.bzl", "nix_pkg") -use_repo(nix_pkg, "nixpkgs_packages") -nix_pkg.attr(attr = "hello") -nix_pkg.local_attr(name = "attribute-test", attr = "hello") -nix_pkg.local_attr(name = "nixpkgs-git-repository-test", attr = "hello", repo = "remote_nixpkgs") -nix_pkg.local_attr(name = "nixpkgs-http-repository-test", attr = "hello", repo = "http_nixpkgs") -nix_pkg.local_file( +nix_pkg.default(attr = "hello") +nix_pkg.attr( + name = "attribute-test", + attr = "hello", +) +nix_pkg.attr( + name = "nixpkgs-git-repository-test", + attr = "hello", + repo = "@remote_nixpkgs", +) +nix_pkg.attr( + name = "nixpkgs-http-repository-test", + attr = "hello", + repo = "@http_nixpkgs", +) +nix_pkg.file( name = "nix-file-test", attr = "hello", file = "//tests:nixpkgs.nix", ) -nix_pkg.local_file( +nix_pkg.file( name = "nix-file-deps-test", + attr = "", file = "//tests:hello.nix", file_deps = ["//tests:pkgname.nix"], ) -nix_pkg.local_file( +nix_pkg.file( name = "relative-imports", attr = "hello", file = "//tests:relative_imports.nix", @@ -70,13 +93,14 @@ nix_pkg.local_file( "//tests:relative_imports/nixpkgs.nix", ], ) -nix_pkg.local_file( +nix_pkg.file( name = "output-filegroup-test", + attr = "", file = "//tests:output.nix", ) -nix_pkg.local_file( +nix_pkg.file( name = "output-filegroup-manual-test", - file = "//tests:output.nix", + attr = "", build_file_content = """ package(default_visibility = [ "//visibility:public" ]) filegroup( @@ -84,32 +108,58 @@ filegroup( srcs = glob(["hi-i-exist", "hi-i-exist-too", "bin/*"]), ) """, + file = "//tests:output.nix", ) -nix_pkg.local_expr( - name = "expr-test", - expr = "let pkgs = import { config = {}; overlays = []; }; in pkgs.hello", - # Deliberately not nixpkgs, to test whether explict repository works. - repos = {"nixpkgs": "file_nixpkgs"}, -) -nix_pkg.local_expr( +nix_pkg.expr( name = "expr-attribute-test", attr = "hello", expr = "import { config = {}; overlays = []; }", ) -nix_pkg.local_expr( +nix_pkg.expr( + name = "extra-args-test", + attr = "", + expr = "{ packagePath }: (import { config = {}; overlays = []; }).${packagePath}", + nixopts = [ + "--argstr", + "packagePath", + "hello", + ], +) +nix_pkg.expr( + name = "expr-test", + attr = "", + expr = "let pkgs = import { config = {}; overlays = []; }; in pkgs.hello", + # Deliberately not @nixpkgs, to test whether explict repository works. + repos = {"@file_nixpkgs": "nixpkgs"}, +) +nix_pkg.expr( name = "nixpkgs-file-repository-test", + attr = "", expr = "with import {}; hello", - repo = "file_nixpkgs", + repo = "@file_nixpkgs", ) -nix_pkg.local_expr( +nix_pkg.expr( name = "nixpkgs-local-repository-test", + attr = "", expr = "with import {}; hello", - repo = "nixpkgs_content", -) -nix_pkg.local_expr( - name = "extra-args-test", - expr = "{ packagePath }: (import { config = {}; overlays = []; }).${packagePath}", - nixopts = ["--argstr", "packagePath", "hello"], + repo = "@nixpkgs_content", +) +use_repo( + nix_pkg, + "attribute-test", + "expr-attribute-test", + "expr-test", + "extra-args-test", + "hello", + "nix-file-deps-test", + "nix-file-test", + "nixpkgs-file-repository-test", + "nixpkgs-git-repository-test", + "nixpkgs-http-repository-test", + "nixpkgs-local-repository-test", + "output-filegroup-manual-test", + "output-filegroup-test", + "relative-imports", ) non_module_deps = use_extension("//:non_module_deps.bzl", "non_module_deps") diff --git a/testing/core/WORKSPACE b/testing/core/WORKSPACE index a88bf0211..d91fccaaa 100644 --- a/testing/core/WORKSPACE +++ b/testing/core/WORKSPACE @@ -20,16 +20,6 @@ load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") bazel_skylib_workspace() -local_repository( - name = "nixpkgs_repositories", - path = "workspace_nixpkgs_repositories", -) - -local_repository( - name = "nixpkgs_packages", - path = "workspace_nixpkgs_packages", -) - local_repository( name = "nixpkgs_location_expansion_test_file", path = "tests/location_expansion/test_repo", diff --git a/testing/core/tests/BUILD.bazel b/testing/core/tests/BUILD.bazel index eac05ed1c..62557e1f3 100644 --- a/testing/core/tests/BUILD.bazel +++ b/testing/core/tests/BUILD.bazel @@ -1,7 +1,6 @@ load("@bazel_skylib//rules:diff_test.bzl", "diff_test") -load("@nixpkgs_packages//:defs.bzl", "nix_pkg") load(":location_expansion_unit_test.bzl", "expand_location_unit_test_suite") -load(":module_registry_tests.bzl", "module_registry_test_suite") +load(":util.bzl", "is_bzlmod_enabled") package(default_testonly = 1) @@ -15,22 +14,22 @@ expand_location_unit_test_suite() name = "run-{0}".format(test), timeout = "short", srcs = ["test_bin.sh"], - args = ["$(location {0})".format(nix_pkg("rules_nixpkgs_core_testing", test, "//:bin"))], - data = [nix_pkg("rules_nixpkgs_core_testing", test, "//:bin")], + args = ["$(location @{0}//:bin)".format(test)], + data = ["@{}//:bin".format(test)], ) for test in [ "hello", "attribute-test", "nixpkgs-git-repository-test", "nixpkgs-http-repository-test", - "nixpkgs-file-repository-test", - "nixpkgs-local-repository-test", "nix-file-test", "nix-file-deps-test", - "expr-test", - "expr-attribute-test", "relative-imports", + "expr-attribute-test", "extra-args-test", + "expr-test", + "nixpkgs-file-repository-test", + "nixpkgs-local-repository-test", ] ] + [ # These tests use the nix package generated by ./output.nix @@ -44,9 +43,9 @@ expand_location_unit_test_suite() srcs = ["test_output.sh"], args = [ "2", - "$(locations {})".format(nix_pkg("rules_nixpkgs_core_testing", "output-filegroup-test", "//:include")), + "$(locations {})".format("@output-filegroup-test//:include"), ], - data = [nix_pkg("rules_nixpkgs_core_testing", "output-filegroup-test", "//:include")], + data = ["@output-filegroup-test//:include"], ), # Checks whether specifying a manual filegroup in the @@ -57,9 +56,9 @@ expand_location_unit_test_suite() srcs = ["test_output.sh"], args = [ "3", - "$(locations {})".format(nix_pkg("rules_nixpkgs_core_testing", "output-filegroup-manual-test", "//:manual-filegroup")), + "$(locations {})".format("@output-filegroup-manual-test//:manual-filegroup"), ], - data = [nix_pkg("rules_nixpkgs_core_testing", "output-filegroup-manual-test", "//:manual-filegroup")], + data = ["@output-filegroup-manual-test//:manual-filegroup"], ), ] + [ # All of these tests use the "hello" binary to see @@ -78,6 +77,13 @@ expand_location_unit_test_suite() ] ] +test_suite( + name = "intermediate_module_tests", + tests = [ + "@intermediate_module//:tests", + ], +) if is_bzlmod_enabled() else None + # Test nixopts location expansion test_suite( name = "location-expansion-test", @@ -112,5 +118,3 @@ diff_test( file1 = "//tests:location_expansion/test_file", file2 = "@nixpkgs_location_expansion_test//:out/argstr_external_file", ) - -module_registry_test_suite(name = "module_registry_test_suite") diff --git a/testing/core/tests/intermediate_module/.bazelrc b/testing/core/tests/intermediate_module/.bazelrc new file mode 100644 index 000000000..cb373f8fa --- /dev/null +++ b/testing/core/tests/intermediate_module/.bazelrc @@ -0,0 +1,3 @@ +import %workspace%/../../../../.bazelrc.common +import %workspace%/../../../../.bazelrc.java +common --override_module=rules_nixpkgs_core=../../../../core diff --git a/testing/core/tests/intermediate_module/BUILD.bazel b/testing/core/tests/intermediate_module/BUILD.bazel new file mode 100644 index 000000000..57eedba6d --- /dev/null +++ b/testing/core/tests/intermediate_module/BUILD.bazel @@ -0,0 +1,19 @@ +[ + sh_test( + name = "run-{0}".format(test), + timeout = "short", + srcs = ["test_bin.sh"], + args = ["$(location @{0}//:bin)".format(test)], + data = ["@{}//:bin".format(test)], + ) + for test in [ + "hello", + "nixpkgs-git-repository-test", + "isolated-test", + ] +] + +test_suite( + name = "tests", + visibility = ["//visibility:public"], +) diff --git a/testing/core/tests/intermediate_module/MODULE.bazel b/testing/core/tests/intermediate_module/MODULE.bazel new file mode 100644 index 000000000..6c0ccd335 --- /dev/null +++ b/testing/core/tests/intermediate_module/MODULE.bazel @@ -0,0 +1,59 @@ +module(name = "intermediate_module") + +bazel_dep(name = "rules_nixpkgs_core") + +nix_repo = use_extension("@rules_nixpkgs_core//extensions:repository.bzl", "nix_repo") +nix_repo.default(name = "nixpkgs") +nix_repo.default(name = "remote_nixpkgs") +use_repo(nix_repo, "nixpkgs", "remote_nixpkgs") + +nix_repo_dev = use_extension( + "@rules_nixpkgs_core//extensions:repository.bzl", + "nix_repo", + dev_dependency = True, +) +nix_repo_dev.github( + name = "remote_nixpkgs", + sha256 = "0f8c25433a6611fa5664797cd049c80faefec91575718794c701f3b033f2db01", + tag = "22.05", +) + +nix_repo_isolated = use_extension( + "@rules_nixpkgs_core//extensions:repository.bzl", + "nix_repo", + isolate = True, +) +nix_repo_isolated.github( + name = "isolated_nixpkgs", + sha256 = "0f8c25433a6611fa5664797cd049c80faefec91575718794c701f3b033f2db01", + tag = "22.05", +) +use_repo(nix_repo_isolated, "isolated_nixpkgs") + +nix_pkg = use_extension("@rules_nixpkgs_core//extensions:package.bzl", "nix_pkg") +nix_pkg.default(attr = "hello") +nix_pkg.default(attr = "nixpkgs-git-repository-test") +use_repo(nix_pkg, "hello", "nixpkgs-git-repository-test") + +nix_pkg_dev = use_extension( + "@rules_nixpkgs_core//extensions:package.bzl", + "nix_pkg", + dev_dependency = True, +) +nix_pkg_dev.attr( + name = "nixpkgs-git-repository-test", + attr = "hello", + repo = "@remote_nixpkgs", +) + +nix_pkg_isolated = use_extension( + "@rules_nixpkgs_core//extensions:package.bzl", + "nix_pkg", + isolate = True, +) +nix_pkg_isolated.attr( + name = "isolated-test", + attr = "hello", + repo = "@isolated_nixpkgs", +) +use_repo(nix_pkg_isolated, "isolated-test") diff --git a/testing/core/workspace_nixpkgs_packages/WORKSPACE b/testing/core/tests/intermediate_module/WORKSPACE similarity index 100% rename from testing/core/workspace_nixpkgs_packages/WORKSPACE rename to testing/core/tests/intermediate_module/WORKSPACE diff --git a/testing/core/tests/intermediate_module/test_bin.sh b/testing/core/tests/intermediate_module/test_bin.sh new file mode 100755 index 000000000..f4df0931b --- /dev/null +++ b/testing/core/tests/intermediate_module/test_bin.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +echo "Executing: " $@ +$@ diff --git a/testing/core/tests/module_registry_tests.bzl b/testing/core/tests/module_registry_tests.bzl deleted file mode 100644 index 926ce79d1..000000000 --- a/testing/core/tests/module_registry_tests.bzl +++ /dev/null @@ -1,193 +0,0 @@ -"""Unit tests for the hub repository registry. -""" - -load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") -load("@rules_nixpkgs_core//private:module_registry.bzl", "registry") - -def _add_module_test_impl(ctx): - env = unittest.begin(ctx) - - r = registry.make() - key, err = registry.add_module(r, name = "test_module", version = "1.0.0") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - _, err = registry.add_module(r, name = "test_module", version = "1.2.0") - asserts.equals(env, expected = "Duplicate module 'test_module 1.2.0', previous version '1.0.0'.", actual = err, msg = "Duplicate module error expected.") - - return unittest.end(env) - -_add_module_test = unittest.make(_add_module_test_impl) - -def _global_repo_test_impl(ctx): - env = unittest.begin(ctx) - - r = registry.make() - key, err = registry.add_module(r, name = "test_module", version = "1.0.0") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - _, err = registry.add_global_repo(r, name = "global_repo", repo = "repo") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - has_global_repo = registry.has_global_repo(r, name = "global_repo") - asserts.true(env, has_global_repo, msg = "Global repo should exist.") - - global_repo = registry.get_global_repo(r, name = "global_repo") - asserts.equals(env, expected = "repo", actual = global_repo, msg = "Unexpected repo value returned.") - - _, err = registry.add_global_repo(r, name = "global_repo", repo = "another_repo") - asserts.equals(env, expected = "Duplicate global repository 'global_repo'.", actual = err, msg = "Duplicate global repo error expected.") - - global_repo = registry.get_global_repo(r, name = "global_repo") - asserts.equals(env, expected = "repo", actual = global_repo, msg = "Unexpected repo value returned.") - - return unittest.end(env) - -_global_repo_test = unittest.make(_global_repo_test_impl) - -def _local_repo_test_impl(ctx): - env = unittest.begin(ctx) - - r = registry.make() - key, err = registry.add_module(r, name = "test_module", version = "1.0.0") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - _, err = registry.add_local_repo(r, key = key, name = "local_repo", repo = "repo") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - _, err = registry.add_local_repo(r, key = key, name = "local_repo", repo = "repo") - asserts.equals(env, expected = "Duplicate local repository 'local_repo', requested by module 'test_module 1.0.0'.", actual = err, msg = "Duplicate local repo error expected.") - - repo, err = registry.pop_local_repo(r, key = key, name = "local_repo") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - asserts.equals(env, expected = "repo", actual = repo, msg = "Unexpected repo value returned.") - - _, err = registry.pop_local_repo(r, key = key, name = "local_repo") - asserts.equals(env, expected = "Local repository 'local_repo' not found, requested by module 'test_module 1.0.0'.", actual = err, msg = "Local repo not found error expected.") - - return unittest.end(env) - -_local_repo_test = unittest.make(_local_repo_test_impl) - -def _use_global_repo_test_impl(ctx): - env = unittest.begin(ctx) - - r = registry.make() - key, err = registry.add_module(r, name = "test_module", version = "1.0.0") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - _, err = registry.use_global_repo(r, key = key, name = "global_repo") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - _, err = registry.use_global_repo(r, key = "nonexistent_key", name = "global_repo") - asserts.equals(env, expected = "Module not found: 'nonexistent_key'.", actual = err, msg = "Module not found error expected.") - - return unittest.end(env) - -_use_global_repo_test = unittest.make(_use_global_repo_test_impl) - -def _set_default_global_repo_test_impl(ctx): - env = unittest.begin(ctx) - - r = registry.make() - key, err = registry.add_module(r, name = "test_module", version = "1.0.0") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - registry.set_default_global_repo(r, name = "default_global_repo", repo = "repo") - - registry.set_default_global_repo(r, name = "default_global_repo", repo = "another_repo") - - global_repo = registry.get_global_repo(r, name = "default_global_repo") - asserts.equals(env, expected = "repo", actual = global_repo, msg = "Unexpected global default repo value returned.") - - return unittest.end(env) - -_set_default_global_repo_test = unittest.make(_set_default_global_repo_test_impl) - -def _get_all_repositories_test_impl(ctx): - env = unittest.begin(ctx) - - r = registry.make() - - key, err = registry.add_module(r, name = "test_module", version = "1.0.0") - asserts.equals(env, expected = None, actual = err, msg = "No error expected when adding module.") - - _, err = registry.add_local_repo(r, key = key, name = "local_repo", repo = "local_repo_obj") - asserts.equals(env, expected = None, actual = err, msg = "No error expected when adding local_repo.") - - _, err = registry.add_global_repo(r, name = "global_repo", repo = "global_repo_obj") - asserts.equals(env, expected = None, actual = err, msg = "No error expected when adding global_repo.") - - all_repositories = registry.get_all_repositories(r) - expected_repositories = { - "global_repo": "global_repo_obj", - "test_module_1.0.0_local_repo": "local_repo_obj", - } - asserts.equals(env, expected = expected_repositories, actual = all_repositories, msg = "Unexpected repositories returned.") - - return unittest.end(env) - -_get_all_repositories_test = unittest.make(_get_all_repositories_test_impl) - -def _get_all_module_scopes_test_impl(ctx): - env = unittest.begin(ctx) - - r = registry.make() - key1, err = registry.add_module(r, name = "test_module1", version = "1.0.0") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - key2, err = registry.add_module(r, name = "test_module2", version = "2.0.0") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - # Use global repositories before they are added - _, err = registry.use_global_repo(r, key = key1, name = "global_repo1") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - _, err = registry.use_global_repo(r, key = key2, name = "global_repo2") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - # Add one global repository - _, err = registry.add_global_repo(r, name = "global_repo1", repo = "global_repo1_obj") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - # Test with unregistered global repository - scopes, err = registry.get_all_module_scopes(r) - asserts.equals(env, expected = "Global repository 'global_repo2' not registered, requested by module 'test_module2 2.0.0'.", actual = err, msg = "Global repository not registered error expected.") - - # Add the other global repository - _, err = registry.add_global_repo(r, name = "global_repo2", repo = "global_repo2_obj") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - # Add local repositories - _, err = registry.add_local_repo(r, key = key1, name = "local_repo1", repo = "local_repo1_obj") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - _, err = registry.add_local_repo(r, key = key2, name = "local_repo2", repo = "local_repo2_obj") - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - - scopes, err = registry.get_all_module_scopes(r) - asserts.equals(env, expected = None, actual = err, msg = "No error expected.") - expected_scopes = { - key1: { - "global_repo1": "global_repo1", - "local_repo1": "test_module1_1.0.0_local_repo1", - }, - key2: { - "global_repo2": "global_repo2", - "local_repo2": "test_module2_2.0.0_local_repo2", - }, - } - asserts.equals(env, expected = expected_scopes, actual = scopes, msg = "Scopes do not match expected value.") - - return unittest.end(env) - -_get_all_module_scopes_test = unittest.make(_get_all_module_scopes_test_impl) - -def module_registry_test_suite(name): - unittest.suite( - name, - _add_module_test, - _global_repo_test, - _local_repo_test, - _use_global_repo_test, - _set_default_global_repo_test, - _get_all_repositories_test, - _get_all_module_scopes_test, - ) diff --git a/testing/core/tests/nixpkgs_repositories.bzl b/testing/core/tests/nixpkgs_repositories.bzl index 039af4365..306c2791d 100644 --- a/testing/core/tests/nixpkgs_repositories.bzl +++ b/testing/core/tests/nixpkgs_repositories.bzl @@ -1,5 +1,3 @@ -load("@nixpkgs_repositories//:defs.bzl", "nix_repo") -load("@nixpkgs_packages//:defs.bzl", "nix_pkg") load( "@rules_nixpkgs_core//:nixpkgs.bzl", "nixpkgs_flake_package", @@ -10,21 +8,18 @@ load( ) def nixpkgs_repositories(*, bzlmod): - if bzlmod: - nixpkgs = nix_repo("rules_nixpkgs_core_testing", "nixpkgs") - remote_nixpkgs = nix_repo("rules_nixpkgs_core_testing", "remote_nixpkgs") - http_nixpkgs = nix_repo("rules_nixpkgs_core_testing", "http_nixpkgs") - file_nixpkgs = nix_repo("rules_nixpkgs_core_testing", "file_nixpkgs") - nixpkgs_content = nix_repo("rules_nixpkgs_core_testing", "nixpkgs_content") - else: - nixpkgs = "@nixpkgs" + nixpkgs = "@nixpkgs" + remote_nixpkgs = "@remote_nixpkgs" + http_nixpkgs = "@http_nixpkgs" + file_nixpkgs = "@file_nixpkgs" + nixpkgs_content = "@nixpkgs_content" + if not bzlmod: nixpkgs_local_repository( name = "nixpkgs", nix_file = "//:nixpkgs.nix", nix_file_deps = ["//:flake.lock"], ) - remote_nixpkgs = "@remote_nixpkgs" nixpkgs_git_repository( name = "remote_nixpkgs", remote = "https://github.com/NixOS/nixpkgs", @@ -32,7 +27,6 @@ def nixpkgs_repositories(*, bzlmod): sha256 = "0f8c25433a6611fa5664797cd049c80faefec91575718794c701f3b033f2db01", ) - http_nixpkgs = "@http_nixpkgs" nixpkgs_http_repository( name = "http_nixpkgs", url = "https://github.com/NixOS/nixpkgs/archive/refs/tags/22.05.tar.gz", @@ -41,7 +35,6 @@ def nixpkgs_repositories(*, bzlmod): ) # same as @nixpkgs, only needed for bzlmod tests to distinguish `default` and `file`. - file_nixpkgs = "@file_nixpkgs" nixpkgs_local_repository( name = "file_nixpkgs", nix_file = "//:nixpkgs.nix", @@ -49,7 +42,6 @@ def nixpkgs_repositories(*, bzlmod): ) # same as @nixpkgs but using the `nix_file_content` parameter - nixpkgs_content = "@nixpkgs_content" nixpkgs_local_repository( name = "nixpkgs_content", nix_file_content = "import ./nixpkgs.nix", diff --git a/testing/core/tests/util.bzl b/testing/core/tests/util.bzl new file mode 100644 index 000000000..e03c599d9 --- /dev/null +++ b/testing/core/tests/util.bzl @@ -0,0 +1,4 @@ +def is_bzlmod_enabled(): + """Returns True if bzlmod is enabled.""" + # Labels are only canonicalized (@@ prefix) if bzlmod is enabled. + return str(Label("@//:BUILD.bazel")).startswith("@@") diff --git a/testing/core/workspace_nixpkgs_packages/BUILD.bazel b/testing/core/workspace_nixpkgs_packages/BUILD.bazel deleted file mode 100644 index e69de29bb..000000000 diff --git a/testing/core/workspace_nixpkgs_packages/defs.bzl b/testing/core/workspace_nixpkgs_packages/defs.bzl deleted file mode 100644 index c6565f93d..000000000 --- a/testing/core/workspace_nixpkgs_packages/defs.bzl +++ /dev/null @@ -1,3 +0,0 @@ -def nix_pkg(module_name, name, label): - # This is a WORKSPACE mode replacement for @nixpkgs_packages. - return Label("@" + name).relative(label) diff --git a/testing/core/workspace_nixpkgs_repositories/BUILD.bazel b/testing/core/workspace_nixpkgs_repositories/BUILD.bazel deleted file mode 100644 index e69de29bb..000000000 diff --git a/testing/core/workspace_nixpkgs_repositories/WORKSPACE b/testing/core/workspace_nixpkgs_repositories/WORKSPACE deleted file mode 100644 index e69de29bb..000000000 diff --git a/testing/core/workspace_nixpkgs_repositories/defs.bzl b/testing/core/workspace_nixpkgs_repositories/defs.bzl deleted file mode 100644 index afe33659c..000000000 --- a/testing/core/workspace_nixpkgs_repositories/defs.bzl +++ /dev/null @@ -1,2 +0,0 @@ -def nix_repo(): - fail("This is a WORKSPACE mode place-holder for @nixpkgs_repositories. Do not call it in WORKSPACE mode.") diff --git a/testing/go-bzlmod/tests/cc-patched.nix b/testing/go-bzlmod/tests/cc-patched.nix new file mode 100644 index 000000000..2dade31ec --- /dev/null +++ b/testing/go-bzlmod/tests/cc-patched.nix @@ -0,0 +1,61 @@ +let + ccPkgs = import { config = { }; overlays = [ ]; }; + pkgs = ccPkgs.buildPackages; + stdenv = ccPkgs.stdenv; + # The original `postLinkSignHook` from nixpkgs assumes `codesign_allocate` is + # in the PATH which is not the case when using our cc_wrapper. Set + # `CODESIGN_ALLOCATE` to an absolute path here and override the hook for + # `darwinCC` below. + postLinkSignHook = + with pkgs; writeTextFile { + name = "post-link-sign-hook"; + executable = true; + + text = '' + CODESIGN_ALLOCATE=${darwin.cctools}/bin/codesign_allocate \ + ${darwin.sigtool}/bin/codesign -f -s - "$linkerOutput" + ''; + }; + darwinCC = + # Work around https://github.com/NixOS/nixpkgs/issues/42059. + # See also https://github.com/NixOS/nixpkgs/pull/41589. + # + # Work around https://github.com/NixOS/nixpkgs/issues/258607 + # by patching cc-wrapper, see https://github.com/NixOS/nixpkgs/pull/258608 + pkgs.wrapCCWith rec { + cc = stdenv.cc.cc; + bintools = stdenv.cc.bintools.override { inherit postLinkSignHook; }; + extraBuildCommands = with pkgs.darwin.apple_sdk.frameworks; '' + sed -i.bak 's#--tmpdir cc-params.XXXXXX#"''${TMPDIR:-/tmp}/cc-params.XXXXXX"#' $out/bin/cc + echo "-Wno-unused-command-line-argument" >> $out/nix-support/cc-cflags + echo "-Wno-elaborated-enum-base" >> $out/nix-support/cc-cflags + echo "-isystem ${pkgs.llvmPackages.libcxx.dev}/include/c++/v1" >> $out/nix-support/cc-cflags + echo "-isystem ${pkgs.llvmPackages.clang-unwrapped.lib}/lib/clang/${cc.version}/include" >> $out/nix-support/cc-cflags + echo "-F${CoreFoundation}/Library/Frameworks" >> $out/nix-support/cc-cflags + echo "-F${CoreServices}/Library/Frameworks" >> $out/nix-support/cc-cflags + echo "-F${Security}/Library/Frameworks" >> $out/nix-support/cc-cflags + echo "-F${Foundation}/Library/Frameworks" >> $out/nix-support/cc-cflags + echo "-L${pkgs.llvmPackages.libcxx}/lib" >> $out/nix-support/cc-cflags + echo "-L${pkgs.llvmPackages.libcxxabi}/lib" >> $out/nix-support/cc-cflags + echo "-L${pkgs.libiconv}/lib" >> $out/nix-support/cc-cflags + echo "-L${pkgs.darwin.libobjc}/lib" >> $out/nix-support/cc-cflags + echo "-resource-dir=${pkgs.stdenv.cc}/resource-root" >> $out/nix-support/cc-cflags + ''; + }; +in +pkgs.buildEnv ( + let + cc = if stdenv.isDarwin then darwinCC else stdenv.cc; + in + { + name = "bazel-${cc.name}-wrapper"; + # XXX: `gcov` is missing in `/bin`. + # It exists in `stdenv.cc.cc` but that collides with `stdenv.cc`. + paths = [ cc cc.bintools ] ++ pkgs.lib.optional pkgs.stdenv.isDarwin pkgs.darwin.cctools; + pathsToLink = [ "/bin" ]; + passthru = { + inherit (cc) isClang targetPrefix; + orignalName = cc.name; + }; + } +) diff --git a/testing/go-bzlmod/tests/nixpkgs_repositories.bzl b/testing/go-bzlmod/tests/nixpkgs_repositories.bzl index 61f16f023..a69baf8a6 100644 --- a/testing/go-bzlmod/tests/nixpkgs_repositories.bzl +++ b/testing/go-bzlmod/tests/nixpkgs_repositories.bzl @@ -14,6 +14,7 @@ def nixpkgs_repositories(*, bzlmod): name = "nixpkgs_config_cc", repository = "@nixpkgs", register = not bzlmod, + nix_file = "//tests:cc-patched.nix", ) nixpkgs_java_configure( diff --git a/testing/go-workspace/tests/cc-patched.nix b/testing/go-workspace/tests/cc-patched.nix new file mode 120000 index 000000000..9246df538 --- /dev/null +++ b/testing/go-workspace/tests/cc-patched.nix @@ -0,0 +1 @@ +../../go-bzlmod/tests/cc-patched.nix \ No newline at end of file diff --git a/testing/go-workspace/tests/nixpkgs_repositories.bzl b/testing/go-workspace/tests/nixpkgs_repositories.bzl index 61f16f023..a69baf8a6 100644 --- a/testing/go-workspace/tests/nixpkgs_repositories.bzl +++ b/testing/go-workspace/tests/nixpkgs_repositories.bzl @@ -14,6 +14,7 @@ def nixpkgs_repositories(*, bzlmod): name = "nixpkgs_config_cc", repository = "@nixpkgs", register = not bzlmod, + nix_file = "//tests:cc-patched.nix", ) nixpkgs_java_configure( diff --git a/testing/python/.bazelrc b/testing/python/.bazelrc index ae8f32bd8..7799ea203 100644 --- a/testing/python/.bazelrc +++ b/testing/python/.bazelrc @@ -1,3 +1,7 @@ import %workspace%/../../.bazelrc.common import %workspace%/../../.bazelrc.cc import %workspace%/../../.bazelrc.java + +# The python-2.7.18.1 package is classified as insecure. +# It is used to test rules_nixpkgs_python support for PY2. +build --repo_env=NIXPKGS_ALLOW_INSECURE=1 diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index 28558d533..e4493ebed 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -8,7 +8,7 @@ sh_test( "//nixpkgs:srcs", "//tests/invalid_nixpkgs_package:srcs", "@coreutils_static//:bin", - "@nix_2_7//:bin", + "@nix_2_10//:bin", "@rules_nixpkgs_cc//:srcs", "@rules_nixpkgs_core//:srcs", "@rules_nixpkgs_java//:srcs", diff --git a/tests/test_invalid_nixpkgs_package.sh b/tests/test_invalid_nixpkgs_package.sh index a27137343..bd8f8ceaa 100755 --- a/tests/test_invalid_nixpkgs_package.sh +++ b/tests/test_invalid_nixpkgs_package.sh @@ -17,7 +17,7 @@ sed "s;COREUTILS-ABS-PATH;${PWD}/external/coreutils_static/bin/;g" -i default.ni # Bring a specific version of Nix which can be executed in the Bazel # linux sandbox. -export PATH=$PWD/external/nix_2_7/bin:$PATH +export PATH=$PWD/external/nix_2_10/bin:$PATH # This is a create all directories required to run Nix locally. mkdir -p ${TEST_TMPDIR}/nix/{store,var/nix,etc/nix}