diff --git a/android/android_toolchain.bzl b/android/android_toolchain.bzl index 9113911fc..e982c78e5 100644 --- a/android/android_toolchain.bzl +++ b/android/android_toolchain.bzl @@ -27,6 +27,7 @@ AndroidToolchainInfo = provider( "bundle_builder": provider_field(typing.Any, default = None), "combine_native_library_dirs": provider_field(typing.Any, default = None), "d8_command": provider_field(typing.Any, default = None), + "enabled_voltron_non_asset_libs": provider_field(typing.Any, default = None), "exo_resources_rewriter": provider_field(typing.Any, default = None), "exopackage_agent_apk": provider_field(typing.Any, default = None), "filter_dex_class_names": provider_field(typing.Any, default = None), diff --git a/apple/tools/linker_wrapper.py b/apple/tools/linker_wrapper.py new file mode 100644 index 000000000..77ede38b5 --- /dev/null +++ b/apple/tools/linker_wrapper.py @@ -0,0 +1,138 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +import argparse +import enum +import re +import subprocess +import sys + +from typing import List, Optional, Tuple + + +def _eprintln(msg: str) -> None: + print(msg, flush=True, file=sys.stderr) + + +def _expand_arg_files(args: List[str]) -> List[str]: + expanded_args = [] + for arg in args: + if arg.startswith("@"): + with open(arg[1:]) as argfile: + expanded_args.extend( + [line.strip('"') for line in argfile.read().splitlines()] + ) + else: + expanded_args.append(arg) + return expanded_args + + +def _seperate_wrapper_args_from_linker_args( + args: List[str], +) -> Tuple[List[str], List[str]]: + wrapper_args = [] + linker_args = [] + expanded_args = _expand_arg_files(args) + + i = 0 + while i < len(expanded_args): + if expanded_args[i] == "-Xwrapper": + wrapper_args.append(expanded_args[i + 1]) + i += 1 + else: + linker_args.append(expanded_args[i]) + i += 1 + + return wrapper_args, linker_args + + +def _diagnose_potential_unexported_symbol_issue( + unexported_symbol_lists: List[str], stderr: str +) -> Optional[str]: + stderr_lines = stderr.splitlines() + undefined_symbol_re = re.compile(r"undefined symbol:.*\(mangled: (\S+)\)") + undefined_symbols = set() + for stderr_line in stderr_lines: + match = re.search(undefined_symbol_re, stderr_line) + if match: + undefined_symbols.add(match.group(1)) + + if not undefined_symbols: + return None + + unexported_symbols = set() + incorrectly_unexported_symbols = set() + incorrect_unexported_symbol_lists = [] + for unexported_symbol_list in unexported_symbol_lists: + target_name, file_path = unexported_symbol_list.split(",") + with open(file_path, "r") as unexported_symbol_list_file: + unexported_symbols = set(unexported_symbol_list_file.read().splitlines()) + intersection = undefined_symbols & unexported_symbols + if intersection: + incorrectly_unexported_symbols.update(intersection) + incorrect_unexported_symbol_lists.append(target_name) + + if not incorrect_unexported_symbol_lists: + return None + + return f""" +UNEXPORTED SYMBOLS ERROR: + +At least one symbol is included in an unexported symbol list, but referenced across dylib boundaries. Please +run the following command to fix the unexported symbol lists: + +arc fix-unexported-symbol-lists {"".join(["--target " + target for target in incorrect_unexported_symbol_lists])} {" ".join(["--symbol " + symbol for symbol in sorted(incorrectly_unexported_symbols)])} + +Here is the linker failure message: +""" + + +class Linker(enum.Enum): + LLD = "lld" + LD64 = "ld64" + + +def _discover_linker(args: List[str]) -> Optional[Linker]: + for arg in args: + if arg.startswith("-fuse-ld="): + linker_name = arg.split("=")[-1] + if linker_name == Linker.LLD.value: + return Linker.LLD + elif linker_name == Linker.LD64.value: + return Linker.LD64 + else: + raise Exception(f"Unknown linker: {linker_name}") + + +def main(argv: List[str]) -> int: + wrapper_args, linker_args = _seperate_wrapper_args_from_linker_args(argv[1:]) + + parser = argparse.ArgumentParser() + parser.add_argument("-linker") + parser.add_argument("-unexported_symbol_list", action="append") + args = parser.parse_args(wrapper_args) + + linker = _discover_linker(linker_args) + if linker == Linker.LLD: + linker_args.extend( + ["-Xlinker", "-pika_include_mangled_names_in_undefined_symbol_errors"] + ) + + result = subprocess.run([args.linker] + linker_args, capture_output=True, text=True) + if result.returncode != 0: + if args.unexported_symbol_list and linker == Linker.LLD: + diagnosis = _diagnose_potential_unexported_symbol_issue( + args.unexported_symbol_list, result.stderr + ) + if diagnosis: + _eprintln(diagnosis) + _eprintln(result.stderr) + return result.returncode + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/cxx/anon_link.bzl b/cxx/anon_link.bzl index 456dc02c6..143f892ea 100644 --- a/cxx/anon_link.bzl +++ b/cxx/anon_link.bzl @@ -10,7 +10,11 @@ load( "ArtifactInfo", "make_artifact_tset", ) -load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxToolchainInfo") +load( + "@prelude//cxx:cxx_toolchain_types.bzl", + "CxxToolchainInfo", + "LinkerType", +) load("@prelude//cxx:cxx_utility.bzl", "cxx_attrs_get_allow_cache_upload") load("@prelude//linking:execution_preference.bzl", "LinkExecutionPreference") load( @@ -34,7 +38,7 @@ def _serialize_linkable(linkable): return ("archive", ( (linkable.archive.artifact, linkable.archive.external_objects), linkable.link_whole, - linkable.linker_type, + linkable.linker_type.value, linkable.supports_lto, )) @@ -42,7 +46,7 @@ def _serialize_linkable(linkable): return ("objects", ( linkable.objects, linkable.link_whole, - linkable.linker_type, + linkable.linker_type.value, )) if isinstance(linkable, SharedLibLinkable): @@ -107,7 +111,7 @@ def _deserialize_linkable(linkable: (str, typing.Any)) -> typing.Any: external_objects = external_objects, ), link_whole = link_whole, - linker_type = linker_type, + linker_type = LinkerType(linker_type), supports_lto = supports_lto, ) @@ -116,7 +120,7 @@ def _deserialize_linkable(linkable: (str, typing.Any)) -> typing.Any: return ObjectsLinkable( objects = objects, link_whole = link_whole, - linker_type = linker_type, + linker_type = LinkerType(linker_type), ) if typ == "shared": @@ -207,7 +211,7 @@ ANON_ATTRS = { # ObjectsLinkable attrs.list(attrs.source()), # objects attrs.bool(), # link_whole - attrs.string(), # linker_type + attrs.enum(LinkerType.values()), # linker_type ), attrs.tuple( # ArchiveLinkable @@ -217,7 +221,7 @@ ANON_ATTRS = { attrs.list(attrs.source()), # external_objects ), attrs.bool(), # link_whole - attrs.string(), # linker_type + attrs.enum(LinkerType.values()), # linker_type attrs.bool(), # supports_lto ), attrs.tuple( diff --git a/cxx/archive.bzl b/cxx/archive.bzl index b54b99ea3..f3149df7c 100644 --- a/cxx/archive.bzl +++ b/cxx/archive.bzl @@ -5,7 +5,7 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load("@prelude//cxx:cxx_toolchain_types.bzl", "LinkerInfo") +load("@prelude//cxx:cxx_toolchain_types.bzl", "LinkerInfo", "LinkerType") load("@prelude//linking:link_info.bzl", "Archive") load("@prelude//utils:argfile.bzl", "at_argfile") load("@prelude//utils:utils.bzl", "value_or") @@ -13,7 +13,7 @@ load(":cxx_context.bzl", "get_cxx_toolchain_info") def _archive_flags( archiver_type: str, - linker_type: str, + linker_type: LinkerType, use_archiver_flags: bool, symbol_table: bool, thin: bool) -> list[str]: @@ -43,7 +43,7 @@ def _archive_flags( flags += "T" # GNU archivers support generating deterministic archives. - if linker_type == "gnu": + if linker_type == LinkerType("gnu"): flags += "D" return [flags] diff --git a/cxx/comp_db.bzl b/cxx/comp_db.bzl index 884e9debb..f57111282 100644 --- a/cxx/comp_db.bzl +++ b/cxx/comp_db.bzl @@ -16,6 +16,7 @@ load(":cxx_context.bzl", "get_cxx_toolchain_info") # Provider that exposes the compilation database information CxxCompilationDbInfo = provider(fields = { + "compdb": provider_field(typing.Any, default = None), # path customly built compile_commands.json (used by Zephyr projects) "info": provider_field(typing.Any, default = None), # A map of the file (an `Artifact`) to its corresponding `CxxSrcCompileCommand` "platform": provider_field(typing.Any, default = None), # platform for this compilation database "toolchain": provider_field(typing.Any, default = None), # toolchain for this compilation database diff --git a/cxx/cxx_library.bzl b/cxx/cxx_library.bzl index 3f03fbb39..56796d9cf 100644 --- a/cxx/cxx_library.bzl +++ b/cxx/cxx_library.bzl @@ -150,7 +150,12 @@ load( "cxx_use_shlib_intfs", "cxx_use_shlib_intfs_mode", ) -load(":cxx_toolchain_types.bzl", "ShlibInterfacesMode", "is_bitcode_format") +load( + ":cxx_toolchain_types.bzl", + "LinkerType", + "ShlibInterfacesMode", + "is_bitcode_format", +) load( ":cxx_types.bzl", "CxxRuleConstructorParams", # @unused Used as a type @@ -1173,7 +1178,7 @@ def _strip_objects(ctx: AnalysisContext, objects: list[Artifact]) -> list[Artifa # Stripping is not supported on Windows linker_type = cxx_toolchain_info.linker_info.type - if linker_type == "windows": + if linker_type == LinkerType("windows"): return objects # Disable stripping if no `strip` binary was provided by the toolchain. @@ -1373,7 +1378,7 @@ def _static_library( # On darwin, the linked output references the archive that contains the # object files instead of the originating objects. object_external_debug_info = [] - if linker_type == "darwin": + if linker_type == LinkerType("darwin"): object_external_debug_info.append(archive.artifact) object_external_debug_info.extend(archive.external_objects) elif objects_have_external_debug_info: diff --git a/cxx/cxx_library_utility.bzl b/cxx/cxx_library_utility.bzl index a84a5615a..5b1e255e0 100644 --- a/cxx/cxx_library_utility.bzl +++ b/cxx/cxx_library_utility.bzl @@ -25,7 +25,11 @@ load( "from_named_set", ) load(":cxx_context.bzl", "get_cxx_platform_info", "get_cxx_toolchain_info") -load(":cxx_toolchain_types.bzl", "ShlibInterfacesMode") +load( + ":cxx_toolchain_types.bzl", + "LinkerType", + "ShlibInterfacesMode", +) load( ":headers.bzl", "cxx_attr_header_namespace", @@ -143,7 +147,7 @@ def cxx_attr_resources(ctx: AnalysisContext) -> dict[str, ArtifactOutputs]: return resources def cxx_is_gnu(ctx: AnalysisContext) -> bool: - return get_cxx_toolchain_info(ctx).linker_info.type == "gnu" + return get_cxx_toolchain_info(ctx).linker_info.type == LinkerType("gnu") def cxx_use_shlib_intfs(ctx: AnalysisContext) -> bool: """ diff --git a/cxx/cxx_link_utility.bzl b/cxx/cxx_link_utility.bzl index 3d2f89a2a..bf5c8ae31 100644 --- a/cxx/cxx_link_utility.bzl +++ b/cxx/cxx_link_utility.bzl @@ -7,7 +7,11 @@ load("@prelude//:artifact_tset.bzl", "project_artifacts") load("@prelude//:paths.bzl", "paths") -load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxToolchainInfo") +load( + "@prelude//cxx:cxx_toolchain_types.bzl", + "CxxToolchainInfo", + "LinkerType", +) load("@prelude//cxx:debug.bzl", "SplitDebugMode") load("@prelude//cxx:linker.bzl", "get_rpath_origin") load("@prelude//cxx:target_sdk_version.bzl", "get_target_sdk_version_linker_flags") @@ -42,14 +46,14 @@ def generates_split_debug(toolchain: CxxToolchainInfo): def linker_map_args(toolchain: CxxToolchainInfo, linker_map) -> LinkArgs: linker_type = toolchain.linker_info.type - if linker_type == "darwin": + if linker_type == LinkerType("darwin"): flags = [ "-Xlinker", "-map", "-Xlinker", linker_map, ] - elif linker_type == "gnu": + elif linker_type == LinkerType("gnu"): flags = [ "-Xlinker", "-Map", @@ -98,7 +102,7 @@ def make_link_args( linker_info = cxx_toolchain_info.linker_info linker_type = linker_info.type - if linker_type == "darwin": + if linker_type == LinkerType("darwin"): # Darwin requires a target triple specified to # control the deployment target being linked for. args.add(get_target_sdk_version_linker_flags(ctx)) @@ -132,7 +136,7 @@ def make_link_args( hidden.append(pdb_artifact.as_output()) filelists = None - if linker_type == "darwin": + if linker_type == LinkerType("darwin"): filelists = filter(None, [unpack_link_args_filelist(link) for link in links]) hidden.extend(filelists) @@ -196,7 +200,7 @@ def cxx_sanitizer_runtime_arguments( if not linker_info.sanitizer_runtime_files: fail("C++ sanitizer runtime enabled but there are no runtime files") - if linker_info.type == "darwin": + if linker_info.type == LinkerType("darwin"): # ignore_artifacts as the runtime directory is not required at _link_ time runtime_rpath = cmd_args(ignore_artifacts = True) runtime_files = linker_info.sanitizer_runtime_files @@ -247,7 +251,7 @@ def executable_shared_lib_arguments( linker_type = cxx_toolchain.linker_info.type if len(shared_libs) > 0: - if linker_type == "windows": + if linker_type == LinkerType("windows"): shared_libs_symlink_tree = [ctx.actions.symlink_file( shlib.lib.output.basename, shlib.lib.output, diff --git a/cxx/cxx_toolchain.bzl b/cxx/cxx_toolchain.bzl index d69de97a9..002049dcf 100644 --- a/cxx/cxx_toolchain.bzl +++ b/cxx/cxx_toolchain.bzl @@ -6,7 +6,27 @@ # of this source tree. load("@prelude//:is_full_meta_repo.bzl", "is_full_meta_repo") -load("@prelude//cxx:cxx_toolchain_types.bzl", "AsCompilerInfo", "AsmCompilerInfo", "BinaryUtilitiesInfo", "CCompilerInfo", "CudaCompilerInfo", "CvtresCompilerInfo", "CxxCompilerInfo", "CxxObjectFormat", "DepTrackingMode", "DistLtoToolsInfo", "HipCompilerInfo", "LinkerInfo", "PicBehavior", "RcCompilerInfo", "ShlibInterfacesMode", "StripFlagsInfo", "cxx_toolchain_infos") +load( + "@prelude//cxx:cxx_toolchain_types.bzl", + "AsCompilerInfo", + "AsmCompilerInfo", + "BinaryUtilitiesInfo", + "CCompilerInfo", + "CudaCompilerInfo", + "CvtresCompilerInfo", + "CxxCompilerInfo", + "CxxObjectFormat", + "DepTrackingMode", + "DistLtoToolsInfo", + "HipCompilerInfo", + "LinkerInfo", + "LinkerType", + "PicBehavior", + "RcCompilerInfo", + "ShlibInterfacesMode", + "StripFlagsInfo", + "cxx_toolchain_infos", +) load("@prelude//cxx:cxx_utility.bzl", "cxx_toolchain_allow_cache_upload_args") load("@prelude//cxx:debug.bzl", "SplitDebugMode") load("@prelude//cxx:headers.bzl", "HeaderMode", "HeadersAsRawHeadersMode") @@ -87,6 +107,7 @@ def cxx_toolchain_impl(ctx): preprocessor_flags = cmd_args(ctx.attrs.rc_preprocessor_flags), ) if ctx.attrs.rc_compiler else None + linker_type = LinkerType(ctx.attrs.linker_type) linker_info = LinkerInfo( archiver = ctx.attrs.archiver[RunInfo], archiver_flags = cmd_args(ctx.attrs.archiver_flags), @@ -98,7 +119,7 @@ def cxx_toolchain_impl(ctx): archive_symbol_table = ctx.attrs.archive_symbol_table, binary_extension = value_or(ctx.attrs.binary_extension, ""), generate_linker_maps = ctx.attrs.generate_linker_maps, - is_pdb_generated = is_pdb_generated(ctx.attrs.linker_type, ctx.attrs.linker_flags), + is_pdb_generated = is_pdb_generated(linker_type, ctx.attrs.linker_flags), link_binaries_locally = not value_or(ctx.attrs.cache_links, True), link_libraries_locally = False, link_style = LinkStyle(ctx.attrs.link_style), @@ -125,7 +146,7 @@ def cxx_toolchain_impl(ctx): static_dep_runtime_ld_flags = ctx.attrs.static_dep_runtime_ld_flags, static_library_extension = ctx.attrs.static_library_extension or "a", static_pic_dep_runtime_ld_flags = ctx.attrs.static_pic_dep_runtime_ld_flags, - type = ctx.attrs.linker_type, + type = linker_type, use_archiver_flags = ctx.attrs.use_archiver_flags, ) @@ -307,14 +328,14 @@ def _get_shared_library_name_default_prefix(ctx: AnalysisContext) -> str: return "" if extension == "dll" else "lib" def _get_shared_library_name_format(ctx: AnalysisContext) -> str: - linker_type = ctx.attrs.linker_type + linker_type = LinkerType(ctx.attrs.linker_type) extension = ctx.attrs.shared_library_extension if extension == "": extension = LINKERS[linker_type].default_shared_library_extension return "{}." + extension def _get_shared_library_versioned_name_format(ctx: AnalysisContext) -> str: - linker_type = ctx.attrs.linker_type + linker_type = LinkerType(ctx.attrs.linker_type) extension_format = ctx.attrs.shared_library_versioned_extension_format.replace("%s", "{}") if extension_format == "": extension_format = LINKERS[linker_type].default_shared_library_versioned_extension_format diff --git a/cxx/cxx_toolchain_types.bzl b/cxx/cxx_toolchain_types.bzl index 464c5743b..b73e6c723 100644 --- a/cxx/cxx_toolchain_types.bzl +++ b/cxx/cxx_toolchain_types.bzl @@ -7,7 +7,7 @@ load("@prelude//cxx:debug.bzl", "SplitDebugMode") -LinkerType = ["gnu", "darwin", "windows", "wasm"] +LinkerType = enum("gnu", "darwin", "windows", "wasm") ShlibInterfacesMode = enum("disabled", "enabled", "defined_only", "stub_from_library", "stub_from_headers") @@ -64,7 +64,7 @@ LinkerInfo = provider( "requires_objects": provider_field(typing.Any, default = None), "supports_distributed_thinlto": provider_field(typing.Any, default = None), "independent_shlib_interface_linker_flags": provider_field(typing.Any, default = None), - "type": provider_field(typing.Any, default = None), # of "LinkerType" type + "type": LinkerType, "use_archiver_flags": provider_field(typing.Any, default = None), "force_full_hybrid_if_capable": provider_field(typing.Any, default = None), "is_pdb_generated": provider_field(typing.Any, default = None), # bool diff --git a/cxx/headers.bzl b/cxx/headers.bzl index daf60e342..0e4f81917 100644 --- a/cxx/headers.bzl +++ b/cxx/headers.bzl @@ -6,6 +6,7 @@ # of this source tree. load("@prelude//:paths.bzl", "paths") +load("@prelude//cxx:cxx_toolchain_types.bzl", "LinkerType") load("@prelude//cxx:cxx_utility.bzl", "cxx_attrs_get_allow_cache_upload") load("@prelude//utils:expect.bzl", "expect") load("@prelude//utils:lazy.bzl", "lazy") @@ -334,7 +335,7 @@ def _get_dict_header_namespace(namespace: str, naming: CxxHeadersNaming) -> str: def _get_debug_prefix_args(ctx: AnalysisContext, header_dir: Artifact) -> [cmd_args, None]: # NOTE(@christylee): Do we need to enable debug-prefix-map for darwin and windows? - if get_cxx_toolchain_info(ctx).linker_info.type != "gnu": + if get_cxx_toolchain_info(ctx).linker_info.type != LinkerType("gnu"): return None fmt = "-fdebug-prefix-map={}=" + value_or(header_dir.owner.cell, ".") diff --git a/cxx/link.bzl b/cxx/link.bzl index d08d697a7..8657aec74 100644 --- a/cxx/link.bzl +++ b/cxx/link.bzl @@ -16,7 +16,11 @@ load( "bolt", "cxx_use_bolt", ) -load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxToolchainInfo") +load( + "@prelude//cxx:cxx_toolchain_types.bzl", + "CxxToolchainInfo", + "LinkerType", +) load( "@prelude//cxx/dist_lto:darwin_dist_lto.bzl", "cxx_darwin_dist_link", @@ -146,7 +150,7 @@ def cxx_link_into( fail("Cannot use distributed thinlto with sanitizer runtime") linker_type = linker_info.type - if linker_type == "darwin": + if linker_type == LinkerType("darwin"): exe = cxx_darwin_dist_link( ctx, output, @@ -155,7 +159,7 @@ def cxx_link_into( should_generate_dwp, is_result_executable, ) - elif linker_type == "gnu": + elif linker_type == LinkerType("gnu"): exe = cxx_gnu_dist_link( ctx, output, @@ -246,7 +250,7 @@ def cxx_link_into( all_link_args.add(link_cmd_parts.post_linker_flags) - if linker_info.type == "windows": + if linker_info.type == LinkerType("windows"): shell_quoted_args = cmd_args(all_link_args) else: shell_quoted_args = cmd_args(all_link_args, quote = "shell") diff --git a/cxx/linker.bzl b/cxx/linker.bzl index cee29d070..09ec0eea0 100644 --- a/cxx/linker.bzl +++ b/cxx/linker.bzl @@ -5,7 +5,7 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load("@prelude//cxx:cxx_toolchain_types.bzl", "LinkerInfo") +load("@prelude//cxx:cxx_toolchain_types.bzl", "LinkerInfo", "LinkerType") load("@prelude//utils:arglike.bzl", "ArgLike") load("@prelude//utils:expect.bzl", "expect") @@ -34,19 +34,19 @@ SharedLibraryFlagOverrides = record( ) LINKERS = { - "darwin": Linker( + LinkerType("darwin"): Linker( default_shared_library_extension = "dylib", default_shared_library_versioned_extension_format = "{}.dylib", shared_library_name_linker_flags_format = ["-install_name", "@rpath/{}"], shared_library_flags = ["-shared"], ), - "gnu": Linker( + LinkerType("gnu"): Linker( default_shared_library_extension = "so", default_shared_library_versioned_extension_format = "so.{}", shared_library_name_linker_flags_format = ["-Wl,-soname,{}"], shared_library_flags = ["-shared"], ), - "wasm": Linker( + LinkerType("wasm"): Linker( default_shared_library_extension = "wasm", default_shared_library_versioned_extension_format = "{}.wasm", shared_library_name_linker_flags_format = [], @@ -54,7 +54,7 @@ LINKERS = { # See https://github.com/WebAssembly/tool-conventions/blob/main/DynamicLinking.md#llvm-implementation shared_library_flags = ["-shared"], ), - "windows": Linker( + LinkerType("windows"): Linker( default_shared_library_extension = "dll", default_shared_library_versioned_extension_format = "dll", # NOTE(agallagher): I *think* windows doesn't support a flag to set the @@ -138,7 +138,7 @@ def get_default_shared_library_name(linker_info: LinkerInfo, label: Label): short_name = "{}_{}".format(_sanitize(label.package), _sanitize(label.name)) return get_shared_library_name(linker_info, short_name, apply_default_prefix = True) -def get_shared_library_name_linker_flags(linker_type: str, soname: str, flag_overrides: [SharedLibraryFlagOverrides, None] = None) -> list[str]: +def get_shared_library_name_linker_flags(linker_type: LinkerType, soname: str, flag_overrides: [SharedLibraryFlagOverrides, None] = None) -> list[str]: """ Arguments to pass to the linker to set the given soname. """ @@ -152,7 +152,7 @@ def get_shared_library_name_linker_flags(linker_type: str, soname: str, flag_ove for f in shared_library_name_linker_flags_format ] -def get_shared_library_flags(linker_type: str, flag_overrides: [SharedLibraryFlagOverrides, None] = None) -> list[ArgLike]: +def get_shared_library_flags(linker_type: LinkerType, flag_overrides: [SharedLibraryFlagOverrides, None] = None) -> list[ArgLike]: """ Arguments to pass to the linker to link a shared library. """ @@ -161,24 +161,24 @@ def get_shared_library_flags(linker_type: str, flag_overrides: [SharedLibraryFla return LINKERS[linker_type].shared_library_flags -def get_link_whole_args(linker_type: str, inputs: list[Artifact]) -> list[typing.Any]: +def get_link_whole_args(linker_type: LinkerType, inputs: list[Artifact]) -> list[typing.Any]: """ Return linker args used to always link all the given inputs. """ args = [] - if linker_type == "gnu": + if linker_type == LinkerType("gnu"): args.append("-Wl,--whole-archive") args.extend(inputs) args.append("-Wl,--no-whole-archive") - elif linker_type == "darwin": + elif linker_type == LinkerType("darwin"): for inp in inputs: args.append("-Xlinker") args.append("-force_load") args.append("-Xlinker") args.append(inp) - elif linker_type == "windows": + elif linker_type == LinkerType("windows"): for inp in inputs: args.append(inp) args.append("/WHOLEARCHIVE:" + inp.basename) @@ -187,42 +187,42 @@ def get_link_whole_args(linker_type: str, inputs: list[Artifact]) -> list[typing return args -def get_objects_as_library_args(linker_type: str, objects: list[Artifact]) -> list[typing.Any]: +def get_objects_as_library_args(linker_type: LinkerType, objects: list[Artifact]) -> list[typing.Any]: """ Return linker args used to link the given objects as a library. """ args = [] - if linker_type == "gnu": + if linker_type == LinkerType("gnu"): args.append("-Wl,--start-lib") args.extend(objects) args.append("-Wl,--end-lib") - elif linker_type == "darwin" or linker_type == "windows": + elif linker_type == LinkerType("darwin") or linker_type == LinkerType("windows"): args.extend(objects) else: fail("Linker type {} not supported".format(linker_type)) return args -def get_ignore_undefined_symbols_flags(linker_type: str) -> list[str]: +def get_ignore_undefined_symbols_flags(linker_type: LinkerType) -> list[str]: """ Return linker args used to suppress undefined symbol errors. """ args = [] - if linker_type == "gnu": + if linker_type == LinkerType("gnu"): args.append("-Wl,--allow-shlib-undefined") args.append("-Wl,--unresolved-symbols=ignore-all") - elif linker_type == "darwin": + elif linker_type == LinkerType("darwin"): args.append("-Wl,-undefined,dynamic_lookup") else: fail("Linker type {} not supported".format(linker_type)) return args -def get_no_as_needed_shared_libs_flags(linker_type: str) -> list[str]: +def get_no_as_needed_shared_libs_flags(linker_type: LinkerType) -> list[str]: """ Return linker args used to prevent linkers from dropping unused shared library dependencies from the e.g. DT_NEEDED tags of the link. @@ -230,26 +230,26 @@ def get_no_as_needed_shared_libs_flags(linker_type: str) -> list[str]: args = [] - if linker_type == "gnu": + if linker_type == LinkerType("gnu"): args.append("-Wl,--no-as-needed") - elif linker_type == "darwin": + elif linker_type == LinkerType("darwin"): pass else: fail("Linker type {} not supported".format(linker_type)) return args -def get_output_flags(linker_type: str, output: Artifact) -> list[ArgLike]: - if linker_type == "windows": +def get_output_flags(linker_type: LinkerType, output: Artifact) -> list[ArgLike]: + if linker_type == LinkerType("windows"): return ["/Brepro", cmd_args(output.as_output(), format = "/OUT:{}")] else: return ["-o", output.as_output()] def get_import_library( ctx: AnalysisContext, - linker_type: str, + linker_type: LinkerType, output_short_path: str) -> (Artifact | None, list[ArgLike]): - if linker_type == "windows": + if linker_type == LinkerType("windows"): import_library = ctx.actions.declare_output(output_short_path + ".imp.lib") return import_library, [cmd_args(import_library.as_output(), format = "/IMPLIB:{}")] else: @@ -257,8 +257,8 @@ def get_import_library( def get_deffile_flags( ctx: AnalysisContext, - linker_type: str) -> list[ArgLike]: - if linker_type == "windows" and ctx.attrs.deffile != None: + linker_type: LinkerType) -> list[ArgLike]: + if linker_type == LinkerType("windows") and ctx.attrs.deffile != None: return [ cmd_args(ctx.attrs.deffile, format = "/DEF:{}"), ] @@ -266,23 +266,23 @@ def get_deffile_flags( return [] def get_rpath_origin( - linker_type: str) -> str: + linker_type: LinkerType) -> str: """ Return the macro that runtime loaders resolve to the main executable at runtime. """ - if linker_type == "gnu": + if linker_type == LinkerType("gnu"): return "$ORIGIN" - if linker_type == "darwin": + if linker_type == LinkerType("darwin"): return "@loader_path" fail("Linker type {} not supported".format(linker_type)) def is_pdb_generated( - linker_type: str, + linker_type: LinkerType, linker_flags: list[[str, ResolvedStringWithMacros]]) -> bool: - if linker_type != "windows": + if linker_type != LinkerType("windows"): return False for flag in reversed(linker_flags): flag = str(flag).upper() diff --git a/cxx/omnibus.bzl b/cxx/omnibus.bzl index 12f4f696a..ad5a0f089 100644 --- a/cxx/omnibus.bzl +++ b/cxx/omnibus.bzl @@ -6,7 +6,11 @@ # of this source tree. load("@prelude//:local_only.bzl", "get_resolved_cxx_binary_link_execution_preference") -load("@prelude//cxx:cxx_toolchain_types.bzl", "PicBehavior") +load( + "@prelude//cxx:cxx_toolchain_types.bzl", + "LinkerType", + "PicBehavior", +) load( "@prelude//cxx:link.bzl", "CxxLinkResult", # @unused Used as a type @@ -513,7 +517,7 @@ def _create_omnibus( # Add global symbols version script. # FIXME(agallagher): Support global symbols for darwin. - if linker_info.type != "darwin": + if linker_info.type != LinkerType("darwin"): global_sym_vers = _create_global_symbols_version_script( ctx, # Extract symbols from roots... diff --git a/cxx/prebuilt_cxx_library_group.bzl b/cxx/prebuilt_cxx_library_group.bzl index eb3ad453f..6370818ec 100644 --- a/cxx/prebuilt_cxx_library_group.bzl +++ b/cxx/prebuilt_cxx_library_group.bzl @@ -5,7 +5,11 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load("@prelude//cxx:cxx_toolchain_types.bzl", "PicBehavior") +load( + "@prelude//cxx:cxx_toolchain_types.bzl", + "LinkerType", + "PicBehavior", +) load( "@prelude//cxx:preprocessor.bzl", "CPreprocessor", @@ -113,7 +117,7 @@ def _parse_macro(arg: str) -> [(str, str), None]: def _get_static_link_infos( ctx: AnalysisContext, - linker_type: str, + linker_type: LinkerType, libs: list[Artifact], args: list[str]) -> LinkInfos: """ diff --git a/cxx/preprocessor.bzl b/cxx/preprocessor.bzl index eb292337b..b530ed110 100644 --- a/cxx/preprocessor.bzl +++ b/cxx/preprocessor.bzl @@ -148,11 +148,14 @@ def cxx_inherited_preprocessor_infos(first_order_deps: list[Dependency]) -> list return filter(None, [x.get(CPreprocessorInfo) for x in first_order_deps]) def cxx_merge_cpreprocessors(ctx: AnalysisContext, own: list[CPreprocessor], xs: list[CPreprocessorInfo]) -> CPreprocessorInfo: + return cxx_merge_cpreprocessors_actions(ctx.actions, own, xs) + +def cxx_merge_cpreprocessors_actions(actions: AnalysisActions, own: list[CPreprocessor], xs: list[CPreprocessorInfo]) -> CPreprocessorInfo: kwargs = {"children": [x.set for x in xs]} if own: kwargs["value"] = own return CPreprocessorInfo( - set = ctx.actions.tset(CPreprocessorTSet, **kwargs), + set = actions.tset(CPreprocessorTSet, **kwargs), ) def _format_include_arg(flag: str, path: cmd_args, compiler_type: str) -> list[cmd_args]: diff --git a/cxx/symbols.bzl b/cxx/symbols.bzl index 4bebd9ff9..dbd8ca84c 100644 --- a/cxx/symbols.bzl +++ b/cxx/symbols.bzl @@ -6,7 +6,11 @@ # of this source tree. load("@prelude//:paths.bzl", "paths") -load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxToolchainInfo") +load( + "@prelude//cxx:cxx_toolchain_types.bzl", + "CxxToolchainInfo", + "LinkerType", +) load("@prelude//cxx:cxx_utility.bzl", "cxx_attrs_get_allow_cache_upload") load("@prelude//os_lookup:defs.bzl", "OsLookup") @@ -48,7 +52,7 @@ def _extract_symbol_names( nm_flags += "u" # darwin objects don't have dynamic symbol tables. - if dynamic and cxx_toolchain.linker_info.type != "darwin": + if dynamic and cxx_toolchain.linker_info.type != LinkerType("darwin"): nm_flags += "D" # llvm-nm supports -U for this but gnu nm doesn't. @@ -314,7 +318,7 @@ def get_undefined_symbols_args( category: [str, None] = None, identifier: [str, None] = None, prefer_local: bool = False) -> cmd_args: - if cxx_toolchain.linker_info.type == "gnu": + if cxx_toolchain.linker_info.type == LinkerType("gnu"): # linker script is only supported in gnu linkers linker_script = create_undefined_symbols_linker_script( ctx.actions, diff --git a/cxx/user/cxx_toolchain_override.bzl b/cxx/user/cxx_toolchain_override.bzl index a9041253f..320264dbe 100644 --- a/cxx/user/cxx_toolchain_override.bzl +++ b/cxx/user/cxx_toolchain_override.bzl @@ -5,7 +5,23 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load("@prelude//cxx:cxx_toolchain_types.bzl", "AsCompilerInfo", "AsmCompilerInfo", "BinaryUtilitiesInfo", "CCompilerInfo", "CxxCompilerInfo", "CxxObjectFormat", "CxxPlatformInfo", "CxxToolchainInfo", "LinkerInfo", "LinkerType", "PicBehavior", "ShlibInterfacesMode", "StripFlagsInfo", "cxx_toolchain_infos") +load( + "@prelude//cxx:cxx_toolchain_types.bzl", + "AsCompilerInfo", + "AsmCompilerInfo", + "BinaryUtilitiesInfo", + "CCompilerInfo", + "CxxCompilerInfo", + "CxxObjectFormat", + "CxxPlatformInfo", + "CxxToolchainInfo", + "LinkerInfo", + "LinkerType", + "PicBehavior", + "ShlibInterfacesMode", + "StripFlagsInfo", + "cxx_toolchain_infos", +) load("@prelude//cxx:cxx_utility.bzl", "cxx_toolchain_allow_cache_upload_args") load("@prelude//cxx:debug.bzl", "SplitDebugMode") load("@prelude//cxx:headers.bzl", "HeaderMode") @@ -68,7 +84,7 @@ def _cxx_toolchain_override(ctx): allow_cache_upload = _pick_raw(ctx.attrs.cxx_compiler_allow_cache_upload, base_cxx_info.allow_cache_upload), ) base_linker_info = base_toolchain.linker_info - linker_type = ctx.attrs.linker_type if ctx.attrs.linker_type != None else base_linker_info.type + linker_type = LinkerType(ctx.attrs.linker_type) if ctx.attrs.linker_type != None else base_linker_info.type pdb_expected = is_pdb_generated(linker_type, ctx.attrs.linker_flags) if ctx.attrs.linker_flags != None else base_linker_info.is_pdb_generated # This handles case when linker type is overridden to non-windows from @@ -77,7 +93,7 @@ def _cxx_toolchain_override(ctx): # we can't inspect base linker flags and disable PDB subtargets. # This shouldn't be a problem because to use windows linker after non-windows # linker flags should be changed as well. - pdb_expected = linker_type == "windows" and pdb_expected + pdb_expected = linker_type == LinkerType("windows") and pdb_expected shlib_interfaces = ShlibInterfacesMode(ctx.attrs.shared_library_interface_mode) if ctx.attrs.shared_library_interface_mode else None sanitizer_runtime_files = flatten([runtime_file[DefaultInfo].default_outputs for runtime_file in ctx.attrs.sanitizer_runtime_files]) if ctx.attrs.sanitizer_runtime_files != None else None linker_info = LinkerInfo( @@ -206,7 +222,7 @@ cxx_toolchain_override_registration_spec = RuleRegistrationSpec( "link_weight": attrs.option(attrs.int(), default = None), "linker": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), "linker_flags": attrs.option(attrs.list(attrs.arg()), default = None), - "linker_type": attrs.option(attrs.enum(LinkerType), default = None), + "linker_type": attrs.option(attrs.enum(LinkerType.values()), default = None), "llvm_link": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), "lto_mode": attrs.option(attrs.enum(LtoMode.values()), default = None), "min_sdk_version": attrs.option(attrs.string(), default = None), diff --git a/decls/common.bzl b/decls/common.bzl index 2f153480d..f662459f9 100644 --- a/decls/common.bzl +++ b/decls/common.bzl @@ -20,7 +20,7 @@ prelude_rule = record( further = field([str, None], None), attrs = field(dict[str, Attr]), impl = field([typing.Callable, None], None), - uses_plugins = field([list["PluginKind"], None], None), + uses_plugins = field([list[plugins.PluginKind], None], None), ) AbiGenerationMode = ["unknown", "class", "source", "migrating_to_source_only", "source_only", "unrecognized"] diff --git a/decls/haskell_common.bzl b/decls/haskell_common.bzl index 8a8ee90ec..0a9eaa729 100644 --- a/decls/haskell_common.bzl +++ b/decls/haskell_common.bzl @@ -24,11 +24,14 @@ def _deps_arg(): from which this rules sources import modules or native linkable rules exporting symbols this rules sources call into. """), + "srcs_deps": attrs.dict(attrs.source(), attrs.list(attrs.source()), default = {}, doc = """ + Allows to declare dependencies for sources manually, additionally to the dependencies automatically detected. + """), } def _compiler_flags_arg(): return { - "compiler_flags": attrs.list(attrs.string(), default = [], doc = """ + "compiler_flags": attrs.list(attrs.arg(), default = [], doc = """ Flags to pass to the Haskell compiler when compiling this rule's sources. """), } @@ -40,9 +43,62 @@ def _exported_linker_flags_arg(): """), } +def _scripts_arg(): + return { + "_generate_target_metadata": attrs.dep( + providers = [RunInfo], + default = "prelude//haskell/tools:generate_target_metadata", + ), + "_ghc_wrapper": attrs.dep( + providers = [RunInfo], + default = "prelude//haskell/tools:ghc_wrapper", + ), + } + +def _external_tools_arg(): + return { + "external_tools": attrs.list(attrs.dep(providers = [RunInfo]), default = [], doc = """ + External executables called from Haskell compiler during preprocessing or compilation. +"""), + } + +def _srcs_envs_arg(): + return { + "srcs_envs": attrs.dict(attrs.source(), attrs.dict(attrs.string(), attrs.arg()), default = {}, doc = """ + Individual run-time env for each source compilation. +"""), + } + +def _use_argsfile_at_link_arg(): + return { + "use_argsfile_at_link": attrs.bool(default = False, doc = """ + Use response file at linking. +"""), + } + +def _extra_libraries_arg(): + return { + "extra_libraries": attrs.list(attrs.dep(), default = [], doc = """ + Non-Haskell deps (C/C++ libraries) +"""), + } + +def _module_prefix_arg(): + return { + "module_prefix": attrs.option(attrs.string(), default = None, doc = """ + Module prefix if needed +"""), + } + haskell_common = struct( srcs_arg = _srcs_arg, deps_arg = _deps_arg, compiler_flags_arg = _compiler_flags_arg, exported_linker_flags_arg = _exported_linker_flags_arg, + scripts_arg = _scripts_arg, + external_tools_arg = _external_tools_arg, + srcs_envs_arg = _srcs_envs_arg, + use_argsfile_at_link_arg = _use_argsfile_at_link_arg, + extra_libraries_arg = _extra_libraries_arg, + module_prefix_arg = _module_prefix_arg, ) diff --git a/decls/haskell_rules.bzl b/decls/haskell_rules.bzl index 0233dcdd4..f20cc98c1 100644 --- a/decls/haskell_rules.bzl +++ b/decls/haskell_rules.bzl @@ -46,8 +46,14 @@ haskell_binary = prelude_rule( native_common.link_group_public_deps_label() | native_common.link_style() | haskell_common.srcs_arg() | + haskell_common.external_tools_arg() | + haskell_common.srcs_envs_arg () | + haskell_common.use_argsfile_at_link_arg () | + haskell_common.extra_libraries_arg () | haskell_common.compiler_flags_arg() | haskell_common.deps_arg() | + haskell_common.scripts_arg() | + haskell_common.module_prefix_arg() | buck.platform_deps_arg() | { "contacts": attrs.list(attrs.string(), default = []), @@ -163,8 +169,14 @@ haskell_library = prelude_rule( attrs = ( # @unsorted-dict-items haskell_common.srcs_arg() | + haskell_common.external_tools_arg() | + haskell_common.srcs_envs_arg() | + haskell_common.use_argsfile_at_link_arg() | + haskell_common.extra_libraries_arg() | haskell_common.compiler_flags_arg() | haskell_common.deps_arg() | + haskell_common.scripts_arg() | + haskell_common.module_prefix_arg() | buck.platform_deps_arg() | native_common.link_whole(link_whole_type = attrs.bool(default = False)) | native_common.preferred_linkage(preferred_linkage_type = attrs.enum(Linkage.values())) | @@ -184,6 +196,16 @@ haskell_library = prelude_rule( ), ) +haskell_toolchain_library = prelude_rule( + name = "haskell_toolchain_library", + docs = """ + Declare a library available as part of the GHC toolchain. + """, + attrs = { + }, +) + + haskell_prebuilt_library = prelude_rule( name = "haskell_prebuilt_library", docs = """ @@ -257,4 +279,5 @@ haskell_rules = struct( haskell_ide = haskell_ide, haskell_library = haskell_library, haskell_prebuilt_library = haskell_prebuilt_library, + haskell_toolchain_library = haskell_toolchain_library, ) diff --git a/genrule_toolchain.bzl b/genrule_toolchain.bzl index 38eafc371..fc5b69e18 100644 --- a/genrule_toolchain.bzl +++ b/genrule_toolchain.bzl @@ -9,5 +9,6 @@ GenruleToolchainInfo = provider( doc = "Genrule toolchain info", fields = { "zip_scrubber": provider_field(typing.Any, default = None), + "bash": provider_field(typing.Any, default = None), }, ) diff --git a/go/tools/go_list_wrapper.py b/go/tools/go_list_wrapper.py new file mode 100644 index 000000000..895f3185b --- /dev/null +++ b/go/tools/go_list_wrapper.py @@ -0,0 +1,40 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + + +import argparse +import os +import subprocess +import sys +from pathlib import Path + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument("--go", default="go", type=Path) + parser.add_argument("--workdir", type=Path) + parser.add_argument("--output", type=argparse.FileType("w"), default=sys.stdout) + parsed, unknown = parser.parse_known_args(argv[1:]) + + env = os.environ.copy() + # Make paths absolute, otherwise go build will fail. + if "GOROOT" in env: + env["GOROOT"] = os.path.realpath(env["GOROOT"]) + + env["GOCACHE"] = os.path.realpath(env["BUCK_SCRATCH_PATH"]) + + retcode = subprocess.call( + [parsed.go.resolve(), "list"] + unknown, + env=env, + stdout=parsed.output, + cwd=parsed.workdir, + ) + parsed.output.close() + return retcode + + +sys.exit(main(sys.argv)) diff --git a/haskell/compile.bzl b/haskell/compile.bzl index 222838398..87a33b5ba 100644 --- a/haskell/compile.bzl +++ b/haskell/compile.bzl @@ -5,55 +5,303 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. +load("@prelude//utils:arglike.bzl", "ArgLike") + load( "@prelude//cxx:preprocessor.bzl", "cxx_inherited_preprocessor_infos", - "cxx_merge_cpreprocessors", + "cxx_merge_cpreprocessors_actions", ) load( "@prelude//haskell:library_info.bzl", + "HaskellLibraryProvider", "HaskellLibraryInfoTSet", ) +load( + "@prelude//haskell:library_info.bzl", + "HaskellLibraryInfo", +) +load( + "@prelude//haskell:link_info.bzl", + "HaskellLinkInfo", +) load( "@prelude//haskell:toolchain.bzl", "HaskellToolchainInfo", + "HaskellToolchainLibrary", + "DynamicHaskellPackageDbInfo", + "HaskellPackageDbTSet", + "NativeToolchainLibrary", ) load( "@prelude//haskell:util.bzl", + "attr_deps", "attr_deps_haskell_lib_infos", "attr_deps_haskell_link_infos", + "attr_deps_haskell_toolchain_libraries", "get_artifact_suffix", + "get_source_prefixes", + "is_haskell_boot", "is_haskell_src", "output_extensions", + "src_to_module_name", "srcs_to_pairs", ) load( "@prelude//linking:link_info.bzl", "LinkStyle", ) -load("@prelude//utils:argfile.bzl", "at_argfile") +load("@prelude//utils:argfile.bzl", "argfile", "at_argfile") +load("@prelude//:paths.bzl", "paths") +load("@prelude//utils:graph_utils.bzl", "post_order_traversal") +load("@prelude//utils:strings.bzl", "strip_prefix") + +CompiledModuleInfo = provider(fields = { + "abi": provider_field(Artifact), + "interfaces": provider_field(list[Artifact]), + # TODO[AH] track this module's package-name/id & package-db instead. + "db_deps": provider_field(list[Artifact]), +}) + +def _compiled_module_project_as_abi(mod: CompiledModuleInfo) -> cmd_args: + return cmd_args(mod.abi) + +def _compiled_module_project_as_interfaces(mod: CompiledModuleInfo) -> cmd_args: + return cmd_args(mod.interfaces) + +def _compiled_module_reduce_as_packagedb_deps(children: list[dict[Artifact, None]], mod: CompiledModuleInfo | None) -> dict[Artifact, None]: + # TODO[AH] is there a better way to avoid duplicate package-dbs? + # Using a projection instead would produce duplicates. + result = {db: None for db in mod.db_deps} if mod else {} + for child in children: + result.update(child) + return result + +CompiledModuleTSet = transitive_set( + args_projections = { + "abi": _compiled_module_project_as_abi, + "interfaces": _compiled_module_project_as_interfaces, + }, + reductions = { + "packagedb_deps": _compiled_module_reduce_as_packagedb_deps, + }, +) + +DynamicCompileResultInfo = provider(fields = { + "modules": dict[str, CompiledModuleTSet], +}) # The type of the return value of the `_compile()` function. CompileResultInfo = record( - objects = field(Artifact), - hi = field(Artifact), + objects = field(list[Artifact]), + hi = field(list[Artifact]), stubs = field(Artifact), + hashes = field(list[Artifact]), producing_indices = field(bool), -) - -CompileArgsInfo = record( - result = field(CompileResultInfo), - srcs = field(cmd_args), - args_for_cmd = field(cmd_args), - args_for_file = field(cmd_args), + module_tsets = field(DynamicValue), ) PackagesInfo = record( exposed_package_args = cmd_args, packagedb_args = cmd_args, transitive_deps = field(HaskellLibraryInfoTSet), + bin_paths = cmd_args, ) +_Module = record( + source = field(Artifact), + interfaces = field(list[Artifact]), + hash = field(Artifact), + objects = field(list[Artifact]), + stub_dir = field(Artifact | None), + prefix_dir = field(str), +) + + +def _strip_prefix(prefix, s): + stripped = strip_prefix(prefix, s) + + return stripped if stripped != None else s + + +def _modules_by_name(ctx: AnalysisContext, *, sources: list[Artifact], link_style: LinkStyle, enable_profiling: bool, suffix: str, module_prefix: str | None) -> dict[str, _Module]: + modules = {} + + osuf, hisuf = output_extensions(link_style, enable_profiling) + + for src in sources: + bootsuf = "" + if is_haskell_boot(src.short_path): + bootsuf = "-boot" + elif not is_haskell_src(src.short_path): + continue + + module_name = src_to_module_name(src.short_path) + bootsuf + if module_prefix: + interface_path = paths.replace_extension(module_prefix.replace(".", "/") + "/" + src.short_path, "." + hisuf + bootsuf) + else: + interface_path = paths.replace_extension(src.short_path, "." + hisuf + bootsuf) + interface = ctx.actions.declare_output("mod-" + suffix, interface_path) + interfaces = [interface] + object_path = paths.replace_extension(src.short_path, "." + osuf + bootsuf) + object = ctx.actions.declare_output("mod-" + suffix, object_path) + objects = [object] + hash = ctx.actions.declare_output("mod-" + suffix, interface_path + ".hash") + + if link_style in [LinkStyle("static"), LinkStyle("static_pic")]: + dyn_osuf, dyn_hisuf = output_extensions(LinkStyle("shared"), enable_profiling) + interface_path = paths.replace_extension(src.short_path, "." + dyn_hisuf + bootsuf) + interface = ctx.actions.declare_output("mod-" + suffix, interface_path) + interfaces.append(interface) + object_path = paths.replace_extension(src.short_path, "." + dyn_osuf + bootsuf) + object = ctx.actions.declare_output("mod-" + suffix, object_path) + objects.append(object) + + if bootsuf == "": + stub_dir = ctx.actions.declare_output("stub-" + suffix + "-" + module_name, dir=True) + else: + stub_dir = None + + prefix_dir = "mod-" + suffix + + modules[module_name] = _Module( + source = src, + interfaces = interfaces, + hash = hash, + objects = objects, + stub_dir = stub_dir, + prefix_dir = prefix_dir) + + return modules + +def _dynamic_target_metadata_impl(actions, output, arg, pkg_deps) -> list[Provider]: + # Add -package-db and -package/-expose-package flags for each Haskell + # library dependency. + + packages_info = get_packages_info2( + actions, + arg.deps, + arg.direct_deps_link_info, + arg.haskell_toolchain, + arg.haskell_direct_deps_lib_infos, + LinkStyle("shared"), + specify_pkg_version = False, + enable_profiling = False, + use_empty_lib = True, + for_deps = True, + pkg_deps = pkg_deps, + ) + package_flag = _package_flag(arg.haskell_toolchain) + ghc_args = cmd_args() + ghc_args.add("-j") + ghc_args.add("-hide-all-packages") + + ghc_args.add(cmd_args(arg.toolchain_libs, prepend=package_flag)) + ghc_args.add(cmd_args(packages_info.exposed_package_args)) + ghc_args.add(cmd_args(packages_info.packagedb_args, prepend = "-package-db")) + ghc_args.add(arg.compiler_flags) + + md_args = cmd_args(arg.md_gen) + md_args.add(packages_info.bin_paths) + md_args.add("--ghc", arg.haskell_toolchain.compiler) + if arg.haskell_toolchain.use_persistent_workers: + md_args.add("--worker-target-id", "haskell_metadata") + md_args.add(cmd_args(ghc_args, format="--ghc-arg={}")) + md_args.add( + "--source-prefix", + arg.strip_prefix, + ) + md_args.add(cmd_args(arg.sources, format="--source={}")) + + md_args.add( + arg.lib_package_name_and_prefix, + ) + md_args.add("--output", output) + + actions.run( + md_args, + category = "haskell_metadata", + identifier = arg.suffix if arg.suffix else None, + weight = 8, + ) + + return [] + +_dynamic_target_metadata = dynamic_actions( + impl = _dynamic_target_metadata_impl, + attrs = { + "output": dynattrs.output(), + "arg": dynattrs.value(typing.Any), + "pkg_deps": dynattrs.option(dynattrs.dynamic_value()), + }, +) + +def target_metadata( + ctx: AnalysisContext, + *, + sources: list[Artifact], + suffix: str = "", + ) -> Artifact: + md_file = ctx.actions.declare_output(ctx.attrs.name + suffix + ".md.json") + md_gen = ctx.attrs._generate_target_metadata[RunInfo] + + libname = repr(ctx.label.path).replace("//", "_").replace("/", "_") + "_" + ctx.label.name + pkgname = libname.replace("_", "-") + + haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + toolchain_libs = [dep.name for dep in attr_deps_haskell_toolchain_libraries(ctx)] + + haskell_direct_deps_lib_infos = attr_deps_haskell_lib_infos( + ctx, + LinkStyle("shared"), + enable_profiling = False, + ) + + # The object and interface file paths are depending on the real module name + # as inferred by GHC, not the source file path; currently this requires the + # module name to correspond to the source file path as otherwise GHC will + # not be able to find the created object or interface files in the search + # path. + # + # (module X.Y.Z must be defined in a file at X/Y/Z.hs) + + ctx.actions.dynamic_output_new(_dynamic_target_metadata( + pkg_deps = haskell_toolchain.packages.dynamic if haskell_toolchain.packages else None, + output = md_file.as_output(), + arg = struct( + compiler_flags = ctx.attrs.compiler_flags, + deps = ctx.attrs.deps, + direct_deps_link_info = attr_deps_haskell_link_infos(ctx), + haskell_direct_deps_lib_infos = haskell_direct_deps_lib_infos, + haskell_toolchain = haskell_toolchain, + lib_package_name_and_prefix =_attr_deps_haskell_lib_package_name_and_prefix(ctx), + md_gen = md_gen, + sources = sources, + strip_prefix = _strip_prefix(str(ctx.label.cell_root), str(ctx.label.path)), + suffix = suffix, + toolchain_libs = toolchain_libs, + ), + )) + + return md_file + +def _attr_deps_haskell_lib_package_name_and_prefix(ctx: AnalysisContext) -> cmd_args: + args = cmd_args(prepend = "--package") + + for dep in attr_deps(ctx) + ctx.attrs.template_deps: + lib = dep.get(HaskellLibraryProvider) + if lib == None: + continue + + lib_info = lib.lib.values()[0] + args.add(cmd_args( + lib_info.name, + cmd_args(lib_info.db, parent = 1), + delimiter = ":", + )) + + return args + def _package_flag(toolchain: HaskellToolchainInfo) -> str: if toolchain.support_expose_package: return "-expose-package" @@ -64,13 +312,45 @@ def get_packages_info( ctx: AnalysisContext, link_style: LinkStyle, specify_pkg_version: bool, - enable_profiling: bool) -> PackagesInfo: + enable_profiling: bool, + use_empty_lib: bool) -> PackagesInfo: haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + haskell_direct_deps_lib_infos = attr_deps_haskell_lib_infos( + ctx, + link_style, + enable_profiling, + ) + + return get_packages_info2( + actions = ctx.actions, + deps = [], + direct_deps_link_info = attr_deps_haskell_link_infos(ctx), + haskell_toolchain = haskell_toolchain, + haskell_direct_deps_lib_infos = haskell_direct_deps_lib_infos, + link_style = link_style, + specify_pkg_version = specify_pkg_version, + enable_profiling = enable_profiling, + use_empty_lib = use_empty_lib, + pkg_deps = None, + ) + +def get_packages_info2( + actions: AnalysisActions, + deps: list[Dependency], + direct_deps_link_info: list[HaskellLinkInfo], + haskell_toolchain: HaskellToolchainInfo, + haskell_direct_deps_lib_infos: list[HaskellLibraryInfo], + link_style: LinkStyle, + specify_pkg_version: bool, + enable_profiling: bool, + use_empty_lib: bool, + pkg_deps: ResolvedDynamicValue | None, + for_deps: bool = False) -> PackagesInfo: + # Collect library dependencies. Note that these don't need to be in a # particular order. - direct_deps_link_info = attr_deps_haskell_link_infos(ctx) - libs = ctx.actions.tset( + libs = actions.tset( HaskellLibraryInfoTSet, children = [ lib.prof_info[link_style] if enable_profiling else lib.info[link_style] @@ -78,36 +358,56 @@ def get_packages_info( ], ) - # base is special and gets exposed by default package_flag = _package_flag(haskell_toolchain) - exposed_package_args = cmd_args([package_flag, "base"]) + hidden_args = [l for lib in libs.traverse() for l in lib.libs] + exposed_package_libs = cmd_args() + exposed_package_args = cmd_args() + + if for_deps: + get_db = lambda l: l.deps_db + elif use_empty_lib: + get_db = lambda l: l.empty_db + else: + get_db = lambda l: l.db packagedb_args = cmd_args() packagedb_set = {} for lib in libs.traverse(): - packagedb_set[lib.db] = None + packagedb_set[get_db(lib)] = None hidden_args = cmd_args(hidden = [ lib.import_dirs.values(), lib.stub_dirs, - # libs of dependencies might be needed at compile time if - # we're using Template Haskell: lib.libs, ]) - exposed_package_args.add(hidden_args) - packagedb_args.add(hidden_args) + if pkg_deps: + package_db = pkg_deps.providers[DynamicHaskellPackageDbInfo].packages + else: + package_db = {} + + direct_toolchain_libs = [ + dep[HaskellToolchainLibrary].name + for dep in deps + if HaskellToolchainLibrary in dep + ] + + toolchain_libs = direct_toolchain_libs + libs.reduce("packages") + + package_db_tset = actions.tset( + HaskellPackageDbTSet, + children = [package_db[name] for name in toolchain_libs if name in package_db] + ) # These we need to add for all the packages/dependencies, i.e. # direct and transitive (e.g. `fbcode-common-hs-util-hs-array`) - packagedb_args.add([cmd_args("-package-db", x) for x in packagedb_set]) + packagedb_args.add(packagedb_set.keys()) - haskell_direct_deps_lib_infos = attr_deps_haskell_lib_infos( - ctx, - link_style, - enable_profiling, - ) + packagedb_args.add(package_db_tset.project_as_args("package_db")) + + direct_package_paths = [package_db[name].value.path for name in direct_toolchain_libs if name in package_db] + bin_paths = cmd_args(direct_package_paths, format="--bin-path={}/bin") # Expose only the packages we depend on directly for lib in haskell_direct_deps_lib_infos: @@ -121,141 +421,522 @@ def get_packages_info( exposed_package_args = exposed_package_args, packagedb_args = packagedb_args, transitive_deps = libs, + bin_paths = bin_paths, ) -def compile_args( - ctx: AnalysisContext, - link_style: LinkStyle, - enable_profiling: bool, - pkgname = None, - suffix: str = "") -> CompileArgsInfo: - haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] - - compile_cmd = cmd_args() - compile_cmd.add(haskell_toolchain.compiler_flags) +CommonCompileModuleArgs = record( + command = field(cmd_args), + args_for_file = field(cmd_args), + package_env_args = field(cmd_args), +) +def _common_compile_module_args( + actions: AnalysisActions, + *, + compiler_flags: list[ArgLike], + ghc_wrapper: RunInfo, + haskell_toolchain: HaskellToolchainInfo, + pkg_deps: ResolvedDynamicValue | None, + enable_haddock: bool, + enable_profiling: bool, + link_style: LinkStyle, + main: None | str, + label: Label, + deps: list[Dependency], + external_tool_paths: list[RunInfo], + extra_libraries: list[Dependency], + sources: list[Artifact], + direct_deps_info: list[HaskellLibraryInfoTSet], + pkgname: str | None = None, +) -> CommonCompileModuleArgs: + + command = cmd_args(ghc_wrapper) + command.add("--ghc", haskell_toolchain.compiler) + if haskell_toolchain.use_persistent_workers and pkgname: + worker_target_id = pkgname + command.add("--worker-target-id", worker_target_id) # Some rules pass in RTS (e.g. `+RTS ... -RTS`) options for GHC, which can't # be parsed when inside an argsfile. - compile_cmd.add(ctx.attrs.compiler_flags) + command.add(haskell_toolchain.compiler_flags) + command.add(compiler_flags) - compile_args = cmd_args() - compile_args.add("-no-link", "-i") + command.add("-c") + + if main != None: + command.add(["-main-is", main]) + + if enable_haddock: + command.add("-haddock") + + non_haskell_sources = [ + src + for (path, src) in srcs_to_pairs(sources) + if not is_haskell_src(path) and not is_haskell_boot(path) + ] + + if non_haskell_sources: + warning("{} specifies non-haskell file in `srcs`, consider using `srcs_deps` instead".format(label)) + + args_for_file = cmd_args(hidden = non_haskell_sources) + + args_for_file.add("-no-link", "-i") + args_for_file.add("-hide-all-packages") if enable_profiling: - compile_args.add("-prof") + args_for_file.add("-prof") if link_style == LinkStyle("shared"): - compile_args.add("-dynamic", "-fPIC") + args_for_file.add("-dynamic", "-fPIC") elif link_style == LinkStyle("static_pic"): - compile_args.add("-fPIC", "-fexternal-dynamic-refs") + args_for_file.add("-fPIC", "-fexternal-dynamic-refs") osuf, hisuf = output_extensions(link_style, enable_profiling) - compile_args.add("-osuf", osuf, "-hisuf", hisuf) - - if getattr(ctx.attrs, "main", None) != None: - compile_args.add(["-main-is", ctx.attrs.main]) + args_for_file.add("-osuf", osuf, "-hisuf", hisuf) - artifact_suffix = get_artifact_suffix(link_style, enable_profiling, suffix) + # Add args from preprocess-able inputs. + inherited_pre = cxx_inherited_preprocessor_infos(deps) + pre = cxx_merge_cpreprocessors_actions(actions, [], inherited_pre) + pre_args = pre.set.project_as_args("args") + args_for_file.add(cmd_args(pre_args, format = "-optP={}")) - objects = ctx.actions.declare_output( - "objects-" + artifact_suffix, - dir = True, - ) - hi = ctx.actions.declare_output("hi-" + artifact_suffix, dir = True) - stubs = ctx.actions.declare_output("stubs-" + artifact_suffix, dir = True) - - compile_args.add( - "-odir", - objects.as_output(), - "-hidir", - hi.as_output(), - "-hiedir", - hi.as_output(), - "-stubdir", - stubs.as_output(), - ) + if pkgname: + args_for_file.add(["-this-unit-id", pkgname]) # Add -package-db and -package/-expose-package flags for each Haskell # library dependency. - packages_info = get_packages_info( - ctx, - link_style, - specify_pkg_version = False, - enable_profiling = enable_profiling, + + libs = actions.tset(HaskellLibraryInfoTSet, children = direct_deps_info) + + direct_toolchain_libs = [ + dep[HaskellToolchainLibrary].name + for dep in deps + if HaskellToolchainLibrary in dep + ] + toolchain_libs = direct_toolchain_libs + libs.reduce("packages") + + if haskell_toolchain.packages: + package_db = pkg_deps.providers[DynamicHaskellPackageDbInfo].packages + else: + package_db = [] + + package_db_tset = actions.tset( + HaskellPackageDbTSet, + children = [package_db[name] for name in toolchain_libs if name in package_db] ) - compile_args.add(packages_info.exposed_package_args) - compile_args.add(packages_info.packagedb_args) + direct_package_paths = [package_db[name].value.path for name in direct_toolchain_libs if name in package_db] + args_for_file.add(cmd_args( + direct_package_paths, + format="--bin-path={}/bin", + )) + + args_for_file.add(cmd_args( + external_tool_paths, + format="--bin-exe={}", + )) + + packagedb_args = cmd_args(libs.project_as_args("empty_package_db")) + packagedb_args.add(package_db_tset.project_as_args("package_db")) + + # TODO[AH] Avoid duplicates and share identical env files. + # The set of package-dbs can be known at the package level, not just the + # module level. So, we could generate this file outside of the + # dynamic_output action. + package_env_file = actions.declare_output(".".join([ + label.name, + "package-db", + output_extensions(link_style, enable_profiling)[1], + "env", + ])) + package_env = cmd_args(delimiter = "\n") + package_env.add(cmd_args( + packagedb_args, + format = "package-db {}", + ).relative_to(package_env_file, parent = 1)) + actions.write( + package_env_file, + package_env, + ) + package_env_args = cmd_args( + package_env_file, + prepend = "-package-env", + hidden = packagedb_args, + ) - # Add args from preprocess-able inputs. - inherited_pre = cxx_inherited_preprocessor_infos(ctx.attrs.deps) - pre = cxx_merge_cpreprocessors(ctx, [], inherited_pre) - pre_args = pre.set.project_as_args("args") - compile_args.add(cmd_args(pre_args, format = "-optP={}")) + return CommonCompileModuleArgs( + command = command, + args_for_file = args_for_file, + package_env_args = package_env_args, + ) - if pkgname: - compile_args.add(["-this-unit-id", pkgname]) - - arg_srcs = [] - hidden_srcs = [] - for (path, src) in srcs_to_pairs(ctx.attrs.srcs): - # hs-boot files aren't expected to be an argument to compiler but does need - # to be included in the directory of the associated src file - if is_haskell_src(path): - arg_srcs.append(src) +def _compile_module( + actions: AnalysisActions, + *, + common_args: CommonCompileModuleArgs, + link_style: LinkStyle, + enable_profiling: bool, + enable_th: bool, + haskell_toolchain: HaskellToolchainInfo, + label: Label, + module_name: str, + module: _Module, + module_tsets: dict[str, CompiledModuleTSet], + md_file: Artifact, + graph: dict[str, list[str]], + package_deps: dict[str, list[str]], + outputs: dict[Artifact, OutputArtifact], + artifact_suffix: str, + direct_deps_by_name: dict[str, typing.Any], + toolchain_deps_by_name: dict[str, None], + aux_deps: None | list[Artifact], + src_envs: None | dict[str, ArgLike], + source_prefixes: list[str], + extra_libraries: list[Dependency], +) -> CompiledModuleTSet: + # These compiler arguments can be passed in a response file. + compile_args_for_file = cmd_args(common_args.args_for_file, hidden = aux_deps or []) + + packagedb_tag = actions.artifact_tag() + compile_args_for_file.add(packagedb_tag.tag_artifacts(common_args.package_env_args)) + + dep_file = actions.declare_output(".".join([ + label.name, + module_name or "pkg", + "package-db", + output_extensions(link_style, enable_profiling)[1], + "dep", + ])).as_output() + tagged_dep_file = packagedb_tag.tag_artifacts(dep_file) + compile_args_for_file.add("--buck2-packagedb-dep", tagged_dep_file) + + objects = [outputs[obj] for obj in module.objects] + his = [outputs[hi] for hi in module.interfaces] + + compile_args_for_file.add("-o", objects[0]) + compile_args_for_file.add("-ohi", his[0]) + + # Set the output directories. We do not use the -outputdir flag, but set the directories individually. + # Note, the -outputdir option is shorthand for the combination of -odir, -hidir, -hiedir, -stubdir and -dumpdir. + # But setting -hidir effectively disables the use of the search path to look up interface files, + # as ghc exclusively looks in that directory when it is set. + for dir in ["o", "hie", "dump"]: + compile_args_for_file.add( + "-{}dir".format(dir), cmd_args([cmd_args(md_file, ignore_artifacts=True, parent=1), module.prefix_dir], delimiter="/"), + ) + if module.stub_dir != None: + stubs = outputs[module.stub_dir] + compile_args_for_file.add("-stubdir", stubs) + + if link_style in [LinkStyle("static_pic"), LinkStyle("static")]: + compile_args_for_file.add("-dynamic-too") + compile_args_for_file.add("-dyno", objects[1]) + compile_args_for_file.add("-dynohi", his[1]) + + compile_args_for_file.add(module.source) + + abi_tag = actions.artifact_tag() + + toolchain_deps = [] + library_deps = [] + exposed_package_modules = [] + exposed_package_dbs = [] + for dep_pkgname, dep_modules in package_deps.items(): + if dep_pkgname in toolchain_deps_by_name: + toolchain_deps.append(dep_pkgname) + elif dep_pkgname in direct_deps_by_name: + library_deps.append(dep_pkgname) + exposed_package_dbs.append(direct_deps_by_name[dep_pkgname][0]) + for dep_modname in dep_modules: + exposed_package_modules.append(direct_deps_by_name[dep_pkgname][1].providers[DynamicCompileResultInfo].modules[dep_modname]) else: - hidden_srcs.append(src) - srcs = cmd_args( - arg_srcs, - hidden = hidden_srcs, + fail("Unknown library dependency '{}'. Add the library to the `deps` attribute".format(dep_pkgname)) + + # Transitive module dependencies from other packages. + cross_package_modules = actions.tset( + CompiledModuleTSet, + children = exposed_package_modules, + ) + # Transitive module dependencies from the same package. + this_package_modules = [ + module_tsets[dep_name] + for dep_name in graph[module_name] + ] + + dependency_modules = actions.tset( + CompiledModuleTSet, + children = [cross_package_modules] + this_package_modules, ) - producing_indices = "-fwrite-ide-info" in ctx.attrs.compiler_flags + compile_cmd_args = [common_args.command] + compile_cmd_hidden = [ + abi_tag.tag_artifacts(dependency_modules.project_as_args("interfaces")), + dependency_modules.project_as_args("abi"), + ] + if src_envs: + for k, v in src_envs.items(): + compile_args_for_file.add(cmd_args( + k, + format="--extra-env-key={}", + )) + compile_args_for_file.add(cmd_args( + v, + format="--extra-env-value={}", + )) + if haskell_toolchain.use_argsfile: + compile_cmd_args.append(at_argfile( + actions = actions, + name = "haskell_compile_" + artifact_suffix + ".argsfile", + args = compile_args_for_file, + allow_args = True, + )) + else: + compile_cmd_args.append(compile_args_for_file) + + compile_cmd = cmd_args(compile_cmd_args, hidden = compile_cmd_hidden) + + # add each module dir prefix to search path + for prefix in source_prefixes: + compile_cmd.add( + cmd_args( + cmd_args(md_file, format = "-i{}", ignore_artifacts=True, parent=1), + "/", + paths.join(module.prefix_dir, prefix), + delimiter="" + ) + ) + + + compile_cmd.add(cmd_args(library_deps, prepend = "-package")) + compile_cmd.add(cmd_args(toolchain_deps, prepend = "-package")) + + # extra-libraries + extra_libs = [ + lib[NativeToolchainLibrary] + for lib in extra_libraries + if NativeToolchainLibrary in lib + ] + for l in extra_libs: + compile_cmd.add(l.lib_path) + compile_cmd.add("-l{}".format(l.name)) + + compile_cmd.add("-fwrite-if-simplified-core") + compile_cmd.add("-fpackage-db-byte-code") + if enable_th: + compile_cmd.add("-fprefer-byte-code") + + compile_cmd.add(cmd_args(dependency_modules.reduce("packagedb_deps").keys(), prepend = "--buck2-package-db")) + + dep_file = actions.declare_output("dep-{}_{}".format(module_name, artifact_suffix)).as_output() + + tagged_dep_file = abi_tag.tag_artifacts(dep_file) + + compile_cmd.add("--buck2-dep", tagged_dep_file) + compile_cmd.add("--abi-out", outputs[module.hash]) + + actions.run( + compile_cmd, category = "haskell_compile_" + artifact_suffix.replace("-", "_"), identifier = module_name, + dep_files = { + "abi": abi_tag, + "packagedb": packagedb_tag, + }, + # explicit turn this on for local_only actions to upload their results. + allow_cache_upload = True, + ) - return CompileArgsInfo( - result = CompileResultInfo( - objects = objects, - hi = hi, - stubs = stubs, - producing_indices = producing_indices, + module_tset = actions.tset( + CompiledModuleTSet, + value = CompiledModuleInfo( + abi = module.hash, + interfaces = module.interfaces, + db_deps = exposed_package_dbs, ), - srcs = srcs, - args_for_cmd = compile_cmd, - args_for_file = compile_args, + children = [cross_package_modules] + this_package_modules, ) + return module_tset + +def _dynamic_do_compile_impl(actions, md_file, pkg_deps, arg, direct_deps_by_name, outputs): + common_args = _common_compile_module_args( + actions, + compiler_flags = arg.compiler_flags, + deps = arg.deps, + external_tool_paths = arg.external_tool_paths, + extra_libraries = arg.extra_libraries, + ghc_wrapper = arg.ghc_wrapper, + haskell_toolchain = arg.haskell_toolchain, + label = arg.label, + main = arg.main, + pkg_deps = pkg_deps, + sources = arg.sources, + enable_haddock = arg.enable_haddock, + enable_profiling = arg.enable_profiling, + link_style = arg.link_style, + direct_deps_info = arg.direct_deps_info, + pkgname = arg.pkgname, + ) + + md = md_file.read_json() + th_modules = md["th_modules"] + module_map = md["module_mapping"] + graph = md["module_graph"] + package_deps = md["package_deps"] + + mapped_modules = { module_map.get(k, k): v for k, v in arg.modules.items() } + module_tsets = {} + source_prefixes = get_source_prefixes(arg.sources, module_map) + + for module_name in post_order_traversal(graph): + module = mapped_modules[module_name] + module_tsets[module_name] = _compile_module( + actions, + aux_deps = arg.sources_deps.get(module.source), + src_envs = arg.srcs_envs.get(module.source), + common_args = common_args, + link_style = arg.link_style, + enable_profiling = arg.enable_profiling, + enable_th = module_name in th_modules, + haskell_toolchain = arg.haskell_toolchain, + label = arg.label, + module_name = module_name, + module = module, + module_tsets = module_tsets, + graph = graph, + package_deps = package_deps.get(module_name, {}), + outputs = outputs, + md_file = arg.md_file, + artifact_suffix = arg.artifact_suffix, + direct_deps_by_name = direct_deps_by_name, + toolchain_deps_by_name = arg.toolchain_deps_by_name, + source_prefixes = source_prefixes, + extra_libraries = arg.extra_libraries, + ) + + return [DynamicCompileResultInfo(modules = module_tsets)] + + + +_dynamic_do_compile = dynamic_actions( + impl = _dynamic_do_compile_impl, + attrs = { + "md_file" : dynattrs.artifact_value(), + "arg" : dynattrs.value(typing.Any), + "pkg_deps": dynattrs.option(dynattrs.dynamic_value()), + "outputs": dynattrs.dict(Artifact, dynattrs.output()), + "direct_deps_by_name": dynattrs.dict(str, dynattrs.tuple(dynattrs.value(Artifact), dynattrs.dynamic_value())), + }, +) + # Compile all the context's sources. def compile( ctx: AnalysisContext, link_style: LinkStyle, enable_profiling: bool, + enable_haddock: bool, + md_file: Artifact, pkgname: str | None = None) -> CompileResultInfo: - haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] - compile_cmd = cmd_args(haskell_toolchain.compiler) + artifact_suffix = get_artifact_suffix(link_style, enable_profiling) - args = compile_args(ctx, link_style, enable_profiling, pkgname) + modules = _modules_by_name(ctx, sources = ctx.attrs.srcs, link_style = link_style, enable_profiling = enable_profiling, suffix = artifact_suffix, module_prefix = ctx.attrs.module_prefix) - compile_cmd.add(args.args_for_cmd) + haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] - artifact_suffix = get_artifact_suffix(link_style, enable_profiling) + interfaces = [interface for module in modules.values() for interface in module.interfaces] + objects = [object for module in modules.values() for object in module.objects] + stub_dirs = [ + module.stub_dir + for module in modules.values() + if module.stub_dir != None + ] + abi_hashes = [module.hash for module in modules.values()] - if args.args_for_file: - if haskell_toolchain.use_argsfile: - compile_cmd.add(at_argfile( - actions = ctx.actions, - name = artifact_suffix + ".haskell_compile_argsfile", - args = [args.args_for_file, args.srcs], - allow_args = True, - )) - else: - compile_cmd.add(args.args_for_file) - compile_cmd.add(args.srcs) + # Collect library dependencies. Note that these don't need to be in a + # particular order. + toolchain_deps_by_name = { + lib.name: None + for lib in attr_deps_haskell_toolchain_libraries(ctx) + } + direct_deps_info = [ + lib.prof_info[link_style] if enable_profiling else lib.info[link_style] + for lib in attr_deps_haskell_link_infos(ctx) + ] + + dyn_module_tsets = ctx.actions.dynamic_output_new(_dynamic_do_compile( + md_file = md_file, + pkg_deps = haskell_toolchain.packages.dynamic if haskell_toolchain.packages else None, + outputs = {o: o.as_output() for o in interfaces + objects + stub_dirs + abi_hashes}, + direct_deps_by_name = { + info.value.name: (info.value.empty_db, info.value.dynamic[enable_profiling]) + for info in direct_deps_info + }, + arg = struct( + artifact_suffix = artifact_suffix, + compiler_flags = ctx.attrs.compiler_flags, + deps = ctx.attrs.deps, + direct_deps_info = direct_deps_info, + enable_haddock = enable_haddock, + enable_profiling = enable_profiling, + external_tool_paths = [tool[RunInfo] for tool in ctx.attrs.external_tools], + ghc_wrapper = ctx.attrs._ghc_wrapper[RunInfo], + haskell_toolchain = haskell_toolchain, + label = ctx.label, + link_style = link_style, + main = getattr(ctx.attrs, "main", None), + md_file = md_file, + modules = modules, + pkgname = pkgname, + sources = ctx.attrs.srcs, + sources_deps = ctx.attrs.srcs_deps, + srcs_envs = ctx.attrs.srcs_envs, + toolchain_deps_by_name = toolchain_deps_by_name, + extra_libraries = ctx.attrs.extra_libraries, + ), + )) + + stubs_dir = ctx.actions.declare_output("stubs-" + artifact_suffix, dir=True) + + # collect the stubs from all modules into the stubs_dir + if ctx.attrs.use_argsfile_at_link: + stub_copy_cmd = cmd_args([ + "bash", "-euc", + """\ + mkdir -p \"$0\" + cat $1 | while read stub; do + find \"$stub\" -mindepth 1 -maxdepth 1 -exec cp -r -t \"$0\" '{}' ';' + done + """, + ]) + stub_copy_cmd.add(stubs_dir.as_output()) + stub_copy_cmd.add(argfile( + actions = ctx.actions, + name = "haskell_stubs_" + artifact_suffix + ".argsfile", + args = stub_dirs, + allow_args = True, + )) + else: + stub_copy_cmd = cmd_args([ + "bash", "-euc", + """\ + mkdir -p \"$0\" + for stub; do + find \"$stub\" -mindepth 1 -maxdepth 1 -exec cp -r -t \"$0\" '{}' ';' + done + """, + ]) + stub_copy_cmd.add(stubs_dir.as_output()) + stub_copy_cmd.add(stub_dirs) - artifact_suffix = get_artifact_suffix(link_style, enable_profiling) ctx.actions.run( - compile_cmd, - category = "haskell_compile_" + artifact_suffix.replace("-", "_"), - no_outputs_cleanup = True, + stub_copy_cmd, + category = "haskell_stubs", + identifier = artifact_suffix, + local_only = True, ) - return args.result + return CompileResultInfo( + objects = objects, + hi = interfaces, + hashes = abi_hashes, + stubs = stubs_dir, + producing_indices = False, + module_tsets = dyn_module_tsets, + ) diff --git a/haskell/haskell.bzl b/haskell/haskell.bzl index 497afe97a..5e3de9344 100644 --- a/haskell/haskell.bzl +++ b/haskell/haskell.bzl @@ -7,6 +7,7 @@ # Implementation of the Haskell build rules. +load("@prelude//utils:arglike.bzl", "ArgLike") load("@prelude//:paths.bzl", "paths") load("@prelude//cxx:archive.bzl", "make_archive") load( @@ -20,6 +21,7 @@ load( load( "@prelude//cxx:cxx_toolchain_types.bzl", "CxxToolchainInfo", + "LinkerType", "PicBehavior", ) load("@prelude//cxx:groups.bzl", "get_dedupped_roots_from_groups") @@ -53,6 +55,8 @@ load( "@prelude//haskell:compile.bzl", "CompileResultInfo", "compile", + "get_packages_info2", + "target_metadata", ) load( "@prelude//haskell:haskell_haddock.bzl", @@ -74,19 +78,24 @@ load( load( "@prelude//haskell:toolchain.bzl", "HaskellToolchainInfo", + "HaskellToolchainLibrary", + "HaskellPackageDbTSet", + "DynamicHaskellPackageDbInfo", ) load( "@prelude//haskell:util.bzl", "attr_deps", "attr_deps_haskell_link_infos_sans_template_deps", + "attr_deps_haskell_lib_infos", + "attr_deps_haskell_link_infos", + "attr_deps_haskell_toolchain_libraries", "attr_deps_merged_link_infos", "attr_deps_profiling_link_infos", "attr_deps_shared_library_infos", "get_artifact_suffix", - "is_haskell_src", "output_extensions", "src_to_module_name", - "srcs_to_pairs", + "get_source_prefixes", ) load( "@prelude//linking:link_groups.bzl", @@ -167,6 +176,11 @@ def _attr_preferred_linkage(ctx: AnalysisContext) -> Linkage: # -- +def haskell_toolchain_library_impl(ctx: AnalysisContext): + return [DefaultInfo(), HaskellToolchainLibrary(name = ctx.attrs.name)] + +# -- + def _get_haskell_prebuilt_libs( ctx, link_style: LinkStyle, @@ -229,9 +243,14 @@ def haskell_prebuilt_library_impl(ctx: AnalysisContext) -> list[Provider]: hlibinfo = HaskellLibraryInfo( name = ctx.attrs.name, db = ctx.attrs.db, + empty_db = None, + deps_db = None, + objects = {}, + dependencies = [], import_dirs = {}, stub_dirs = [], id = ctx.attrs.id, + dynamic = None, libs = libs, version = ctx.attrs.version, is_prebuilt = True, @@ -240,9 +259,14 @@ def haskell_prebuilt_library_impl(ctx: AnalysisContext) -> list[Provider]: prof_hlibinfo = HaskellLibraryInfo( name = ctx.attrs.name, db = ctx.attrs.db, + empty_db = None, + deps_db = None, + objects = {}, + dependencies = [], import_dirs = {}, stub_dirs = [], id = ctx.attrs.id, + dynamic = None, libs = prof_libs, version = ctx.attrs.version, is_prebuilt = True, @@ -252,7 +276,7 @@ def haskell_prebuilt_library_impl(ctx: AnalysisContext) -> list[Provider]: def archive_linkable(lib): return ArchiveLinkable( archive = Archive(artifact = lib), - linker_type = "gnu", + linker_type = LinkerType("gnu"), ) def shared_linkable(lib): @@ -371,24 +395,18 @@ def haskell_prebuilt_library_impl(ctx: AnalysisContext) -> list[Provider]: linkable_graph, ] -def _srcs_to_objfiles( - ctx: AnalysisContext, - odir: Artifact, - osuf: str) -> list[Artifact]: - objfiles = [] - for src, _ in srcs_to_pairs(ctx.attrs.srcs): - # Don't link boot sources, as they're only meant to be used for compiling. - if is_haskell_src(src): - objfiles.append(odir.project(paths.replace_extension(src, "." + osuf))) - return objfiles - +# Script to generate a GHC package-db entry for a new package. +# +# Sets --force so that ghc-pkg does not check for .hi, .so, ... files. +# This way package actions can be scheduled before actual build actions, +# don't lie on the critical path for a build, and don't form a bottleneck. _REGISTER_PACKAGE = """\ set -eu GHC_PKG=$1 DB=$2 PKGCONF=$3 "$GHC_PKG" init "$DB" -"$GHC_PKG" register --package-conf "$DB" --no-expand-pkgroot "$PKGCONF" +"$GHC_PKG" register --package-conf "$DB" --no-expand-pkgroot "$PKGCONF" --force -v0 """ # Create a package @@ -406,85 +424,114 @@ PKGCONF=$3 # - controlling module visibility: only dependencies that are # directly declared as dependencies may be used # -# - Template Haskell: the compiler needs to load libraries itself -# at compile time, so it uses the package specs to find out -# which libraries and where. +# - by GHCi when loading packages into the repl +# +# - when linking binaries statically, in order to pass libraries +# to the linker in the correct order def _make_package( ctx: AnalysisContext, link_style: LinkStyle, pkgname: str, - libname: str, + libname: str | None, hlis: list[HaskellLibraryInfo], - hi: dict[bool, Artifact], - lib: dict[bool, Artifact], - enable_profiling: bool) -> Artifact: + profiling: list[bool], + enable_profiling: bool, + use_empty_lib: bool, + md_file: Artifact, + for_deps: bool = False) -> Artifact: artifact_suffix = get_artifact_suffix(link_style, enable_profiling) - # Don't expose boot sources, as they're only meant to be used for compiling. - modules = [src_to_module_name(x) for x, _ in srcs_to_pairs(ctx.attrs.srcs) if is_haskell_src(x)] + def mk_artifact_dir(dir_prefix: str, profiled: bool, subdir: str = "") -> str: + suffix = get_artifact_suffix(link_style, profiled) + if subdir: + suffix = paths.join(suffix, subdir) + return "\"${pkgroot}/" + dir_prefix + "-" + suffix + "\"" + + if for_deps: + pkg_conf = ctx.actions.declare_output("pkg-" + artifact_suffix + "_deps.conf") + db = ctx.actions.declare_output("db-" + artifact_suffix + "_deps", dir = True) + elif use_empty_lib: + pkg_conf = ctx.actions.declare_output("pkg-" + artifact_suffix + "_empty.conf") + db = ctx.actions.declare_output("db-" + artifact_suffix + "_empty", dir = True) + else: + pkg_conf = ctx.actions.declare_output("pkg-" + artifact_suffix + ".conf") + db = ctx.actions.declare_output("db-" + artifact_suffix, dir = True) - if enable_profiling: - # Add the `-p` suffix otherwise ghc will look for objects - # following this logic (https://fburl.com/code/3gmobm5x) and will fail. - libname += "_p" + def write_package_conf(ctx, artifacts, outputs, md_file=md_file, libname=libname): + md = artifacts[md_file].read_json() + module_map = md["module_mapping"] - def mk_artifact_dir(dir_prefix: str, profiled: bool) -> str: - art_suff = get_artifact_suffix(link_style, profiled) - return "\"${pkgroot}/" + dir_prefix + "-" + art_suff + "\"" + source_prefixes = get_source_prefixes(ctx.attrs.srcs, module_map) - import_dirs = [ - mk_artifact_dir("hi", profiled) - for profiled in hi.keys() - ] - library_dirs = [ - mk_artifact_dir("lib", profiled) - for profiled in hi.keys() - ] + modules = [ + module + for module in md["module_graph"].keys() + if not module.endswith("-boot") + ] - conf = [ - "name: " + pkgname, - "version: 1.0.0", - "id: " + pkgname, - "key: " + pkgname, - "exposed: False", - "exposed-modules: " + ", ".join(modules), - "import-dirs:" + ", ".join(import_dirs), - "library-dirs:" + ", ".join(library_dirs), - "extra-libraries: " + libname, - "depends: " + ", ".join([lib.id for lib in hlis]), - ] - pkg_conf = ctx.actions.write("pkg-" + artifact_suffix + ".conf", conf) + # XXX use a single import dir when this package db is used for resolving dependencies with ghc -M, + # which works around an issue with multiple import dirs resulting in GHC trying to locate interface files + # for each exposed module + import_dirs = ["."] if for_deps else [ + mk_artifact_dir("mod", profiled, src_prefix) for profiled in profiling for src_prefix in source_prefixes + ] + + conf = [ + "name: " + pkgname, + "version: 1.0.0", + "id: " + pkgname, + "key: " + pkgname, + "exposed: False", + "exposed-modules: " + ", ".join(modules), + "import-dirs:" + ", ".join(import_dirs), + "depends: " + ", ".join([lib.id for lib in hlis]), + ] - db = ctx.actions.declare_output("db-" + artifact_suffix) + if not use_empty_lib: + if not libname: + fail("argument `libname` cannot be empty, when use_empty_lib == False") - # While the list of hlis is unique, there may be multiple packages in the same db. - # Cutting down the GHC_PACKAGE_PATH significantly speeds up GHC. - db_deps = {x.db: None for x in hlis}.keys() + if enable_profiling: + # Add the `-p` suffix otherwise ghc will look for objects + # following this logic (https://fburl.com/code/3gmobm5x) and will fail. + libname += "_p" - # So that ghc-pkg can find the DBs for the dependencies. We might - # be able to use flags for this instead, but this works. - ghc_package_path = cmd_args( - db_deps, - delimiter = ":", - ) + library_dirs = [mk_artifact_dir("lib", profiled) for profiled in profiling] + conf.append("library-dirs:" + ", ".join(library_dirs)) + conf.append("extra-libraries: " + libname) - haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] - ctx.actions.run( - cmd_args( - [ + ctx.actions.write(outputs[pkg_conf].as_output(), conf) + + db_deps = [x.db for x in hlis] + + # So that ghc-pkg can find the DBs for the dependencies. We might + # be able to use flags for this instead, but this works. + ghc_package_path = cmd_args( + db_deps, + delimiter = ":", + ) + + haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + ctx.actions.run( + cmd_args([ "sh", "-c", _REGISTER_PACKAGE, "", haskell_toolchain.packager, - db.as_output(), + outputs[db].as_output(), pkg_conf, - ], - # needs hi, because ghc-pkg checks that the .hi files exist - hidden = hi.values() + lib.values(), - ), - category = "haskell_package_" + artifact_suffix.replace("-", "_"), - env = {"GHC_PACKAGE_PATH": ghc_package_path} if db_deps else {}, + ]), + category = "haskell_package_" + artifact_suffix.replace("-", "_"), + identifier = "empty" if use_empty_lib else "final", + env = {"GHC_PACKAGE_PATH": ghc_package_path} if db_deps else {}, + ) + + ctx.actions.dynamic_output( + dynamic = [md_file], + inputs = [], + outputs = [pkg_conf.as_output(), db.as_output()], + f = write_package_conf ) return db @@ -497,10 +544,12 @@ HaskellLibBuildOutput = record( libs = list[Artifact], ) -def _get_haskell_shared_library_name_linker_flags(linker_type: str, soname: str) -> list[str]: - if linker_type == "gnu": +def _get_haskell_shared_library_name_linker_flags( + linker_type: LinkerType, + soname: str) -> list[str]: + if linker_type == LinkerType("gnu"): return ["-Wl,-soname,{}".format(soname)] - elif linker_type == "darwin": + elif linker_type == LinkerType("darwin"): # Passing `-install_name @rpath/...` or # `-Xlinker -install_name -Xlinker @rpath/...` instead causes # ghc-9.6.3: panic! (the 'impossible' happened) @@ -508,6 +557,71 @@ def _get_haskell_shared_library_name_linker_flags(linker_type: str, soname: str) else: fail("Unknown linker type '{}'.".format(linker_type)) +def _dynamic_link_shared_impl(actions, pkg_deps, lib, arg): + package_db = pkg_deps.providers[DynamicHaskellPackageDbInfo].packages + + package_db_tset = actions.tset( + HaskellPackageDbTSet, + children = [package_db[name] for name in arg.toolchain_libs if name in package_db] + ) + + link_args = cmd_args() + link_cmd_args = [cmd_args(arg.haskell_toolchain.linker)] + link_cmd_hidden = [] + + link_args.add(arg.haskell_toolchain.linker_flags) + link_args.add(arg.linker_flags) + link_args.add("-hide-all-packages") + link_args.add(cmd_args(arg.toolchain_libs, prepend = "-package")) + link_args.add(cmd_args(package_db_tset.project_as_args("package_db"), prepend="-package-db")) + link_args.add( + get_shared_library_flags(arg.linker_info.type), + "-dynamic", + cmd_args( + _get_haskell_shared_library_name_linker_flags(arg.linker_info.type, arg.libfile), + prepend = "-optl", + ), + ) + + link_args.add(arg.objects) + + link_args.add(cmd_args(unpack_link_args(arg.infos), prepend = "-optl")) + + if arg.use_argsfile_at_link: + link_cmd_args.append(at_argfile( + actions = actions, + name = "haskell_link_" + arg.artifact_suffix.replace("-", "_") + ".argsfile", + args = link_args, + allow_args = True, + )) + else: + link_cmd_args.append(link_args) + + link_cmd = cmd_args(link_cmd_args, hidden = link_cmd_hidden) + link_cmd.add("-o", lib) + + if arg.haskell_toolchain.use_persistent_workers: + link_cmd.add("--worker-target-id={}".format(arg.worker_target_id)) + link_cmd.add("--worker-close") + + actions.run( + link_cmd, + category = "haskell_link" + arg.artifact_suffix.replace("-", "_"), + # explicit turn this on for local_only actions to upload their results. + allow_cache_upload = True, + ) + + return [] + +_dynamic_link_shared = dynamic_actions( + impl = _dynamic_link_shared_impl, + attrs = { + "arg": dynattrs.value(typing.Any), + "lib": dynattrs.output(), + "pkg_deps": dynattrs.dynamic_value(), + }, +) + def _build_haskell_lib( ctx, libname: str, @@ -516,6 +630,8 @@ def _build_haskell_lib( nlis: list[MergedLinkInfo], # native link infos from all deps link_style: LinkStyle, enable_profiling: bool, + enable_haddock: bool, + md_file: Artifact, # The non-profiling artifacts are also needed to build the package for # profiling, so it should be passed when `enable_profiling` is True. non_profiling_hlib: [HaskellLibBuildOutput, None] = None) -> HaskellLibBuildOutput: @@ -524,13 +640,13 @@ def _build_haskell_lib( # Link the objects into a library haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] - osuf, _hisuf = output_extensions(link_style, enable_profiling) - # Compile the sources compiled = compile( ctx, link_style, enable_profiling = enable_profiling, + enable_haddock = enable_haddock, + md_file = md_file, pkgname = pkgname, ) solibs = {} @@ -551,37 +667,39 @@ def _build_haskell_lib( # only gather direct dependencies uniq_infos = [x[link_style].value for x in linfos] - objfiles = _srcs_to_objfiles(ctx, compiled.objects, osuf) + toolchain_libs = [dep.name for dep in attr_deps_haskell_toolchain_libraries(ctx)] if link_style == LinkStyle("shared"): lib = ctx.actions.declare_output(lib_short_path) - link = cmd_args( - [haskell_toolchain.linker] + - [haskell_toolchain.linker_flags] + - [ctx.attrs.linker_flags] + - ["-o", lib.as_output()] + - [ - get_shared_library_flags(linker_info.type), - "-dynamic", - cmd_args( - _get_haskell_shared_library_name_linker_flags(linker_info.type, libfile), - prepend = "-optl", - ), - ] + - [objfiles], - hidden = compiled.stubs, - ) + objects = [ + object + for object in compiled.objects + if not object.extension.endswith("-boot") + ] infos = get_link_args_for_strategy( ctx, nlis, to_link_strategy(link_style), ) - link.add(cmd_args(unpack_link_args(infos), prepend = "-optl")) - ctx.actions.run( - link, - category = "haskell_link" + artifact_suffix.replace("-", "_"), - ) + + ctx.actions.dynamic_output_new(_dynamic_link_shared( + pkg_deps = haskell_toolchain.packages.dynamic, + lib = lib.as_output(), + arg = struct( + artifact_suffix = artifact_suffix, + haskell_toolchain = haskell_toolchain, + infos = infos, + lib = lib, + libfile = libfile, + linker_flags = ctx.attrs.linker_flags, + linker_info = linker_info, + objects = objects, + toolchain_libs = toolchain_libs, + use_argsfile_at_link = ctx.attrs.use_argsfile_at_link, + worker_target_id = pkgname, + ), + )) solibs[libfile] = LinkedObject(output = lib, unstripped_output = lib) libs = [lib] @@ -592,7 +710,7 @@ def _build_haskell_lib( else: # static flavours # TODO: avoid making an archive for a single object, like cxx does # (but would that work with Template Haskell?) - archive = make_archive(ctx, lib_short_path, objfiles) + archive = make_archive(ctx, lib_short_path, compiled.objects) lib = archive.artifact libs = [lib] + archive.external_objects link_infos = LinkInfos( @@ -611,22 +729,29 @@ def _build_haskell_lib( if not non_profiling_hlib: fail("Non-profiling HaskellLibBuildOutput wasn't provided when building profiling lib") + dynamic = { + True: compiled.module_tsets, + False: non_profiling_hlib.compiled.module_tsets, + } import_artifacts = { True: compiled.hi, False: non_profiling_hlib.compiled.hi, } - library_artifacts = { - True: lib, - False: non_profiling_hlib.libs[0], + object_artifacts = { + True: compiled.objects, + False: non_profiling_hlib.compiled.objects, } all_libs = libs + non_profiling_hlib.libs stub_dirs = [compiled.stubs] + [non_profiling_hlib.compiled.stubs] else: + dynamic = { + False: compiled.module_tsets, + } import_artifacts = { False: compiled.hi, } - library_artifacts = { - False: lib, + object_artifacts = { + False: compiled.objects, } all_libs = libs stub_dirs = [compiled.stubs] @@ -637,21 +762,51 @@ def _build_haskell_lib( pkgname, libstem, uniq_infos, - import_artifacts, - library_artifacts, + import_artifacts.keys(), + enable_profiling = enable_profiling, + use_empty_lib = False, + md_file = md_file, + ) + empty_db = _make_package( + ctx, + link_style, + pkgname, + None, + uniq_infos, + import_artifacts.keys(), + enable_profiling = enable_profiling, + use_empty_lib = True, + md_file = md_file, + ) + deps_db = _make_package( + ctx, + link_style, + pkgname, + None, + uniq_infos, + import_artifacts.keys(), enable_profiling = enable_profiling, + use_empty_lib = True, + md_file = md_file, + for_deps = True, ) + hlib = HaskellLibraryInfo( name = pkgname, db = db, + empty_db = empty_db, + deps_db = deps_db, id = pkgname, + dynamic = dynamic, # TODO(ah) refine with dynamic projections import_dirs = import_artifacts, + objects = object_artifacts, stub_dirs = stub_dirs, libs = all_libs, version = "1.0.0", is_prebuilt = False, profiling_enabled = enable_profiling, + dependencies = toolchain_libs, ) return HaskellLibBuildOutput( @@ -683,9 +838,20 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: indexing_tsets = {} sub_targets = {} - libname = repr(ctx.label.path).replace("//", "_").replace("/", "_") + "_" + ctx.label.name + libprefix = repr(ctx.label.path).replace("//", "_").replace("/", "_") + + # avoid consecutive "--" in package name, which is not allowed by ghc-pkg. + if libprefix[-1] == '_': + libname = libprefix + ctx.label.name + else: + libname = libprefix + "_" + ctx.label.name pkgname = libname.replace("_", "-") + md_file = target_metadata( + ctx, + sources = ctx.attrs.srcs, + ) + # The non-profiling library is also needed to build the package with # profiling enabled, so we need to keep track of it for each link style. non_profiling_hlib = {} @@ -704,6 +870,9 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: nlis = nlis, link_style = link_style, enable_profiling = enable_profiling, + # enable haddock only for the first non-profiling hlib + enable_haddock = not enable_profiling and not non_profiling_hlib, + md_file = md_file, non_profiling_hlib = non_profiling_hlib.get(link_style), ) if not enable_profiling: @@ -716,19 +885,11 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: if enable_profiling: prof_hlib_infos[link_style] = hlib - prof_hlink_infos[link_style] = ctx.actions.tset( - HaskellLibraryInfoTSet, - value = hlib, - children = [li.prof_info[link_style] for li in hlis], - ) + prof_hlink_infos[link_style] = ctx.actions.tset(HaskellLibraryInfoTSet, value = hlib, children = [li.prof_info[link_style] for li in hlis]) prof_link_infos[link_style] = hlib_build_out.link_infos else: hlib_infos[link_style] = hlib - hlink_infos[link_style] = ctx.actions.tset( - HaskellLibraryInfoTSet, - value = hlib, - children = [li.info[link_style] for li in hlis], - ) + hlink_infos[link_style] = ctx.actions.tset(HaskellLibraryInfoTSet, value = hlib, children = [li.info[link_style] for li in hlis]) link_infos[link_style] = hlib_build_out.link_infos # Build the indices and create subtargets only once, with profiling @@ -746,6 +907,11 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: sub_targets[link_style.value.replace("_", "-")] = [DefaultInfo( default_outputs = libs, + sub_targets = _haskell_module_sub_targets( + compiled = compiled, + link_style = link_style, + enable_profiling = enable_profiling, + ), )] pic_behavior = ctx.attrs._cxx_toolchain[CxxToolchainInfo].pic_behavior @@ -822,6 +988,39 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: # )] pp = [] + haddock, = haskell_haddock_lib( + ctx, + pkgname, + non_profiling_hlib[LinkStyle("shared")].compiled, + md_file, + ), + + haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + + styles = [ + ctx.actions.declare_output("haddock-html", file) + for file in "synopsis.png linuwial.css quick-jump.css haddock-bundle.min.js".split() + ] + ctx.actions.run( + cmd_args( + haskell_toolchain.haddock, + "--gen-index", + "--optghc=-package-env=-", + "-o", cmd_args(styles[0].as_output(), parent=1), + hidden=[file.as_output() for file in styles] + ), + category = "haddock_styles", + ) + sub_targets.update({ + "haddock": [DefaultInfo( + default_outputs = haddock.html.values(), + sub_targets = { + module: [DefaultInfo(default_output = html, other_outputs=styles)] + for module, html in haddock.html.items() + } + )] + }) + providers = [ DefaultInfo( default_outputs = default_output, @@ -846,7 +1045,7 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: shared_libs, shared_library_infos, ), - haskell_haddock_lib(ctx, pkgname), + haddock, ] if indexing_tsets: @@ -888,7 +1087,7 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: def derive_indexing_tset( actions: AnalysisActions, link_style: LinkStyle, - value: Artifact | None, + value: list[Artifact] | None, children: list[Dependency]) -> HaskellIndexingTSet: index_children = [] for dep in children: @@ -903,6 +1102,100 @@ def derive_indexing_tset( children = index_children, ) +def _make_link_package( + ctx: AnalysisContext, + link_style: LinkStyle, + pkgname: str, + hlis: list[HaskellLibraryInfo], + static_libs: ArgLike) -> Artifact: + artifact_suffix = get_artifact_suffix(link_style, False) + + conf = cmd_args( + "name: " + pkgname, + "version: 1.0.0", + "id: " + pkgname, + "key: " + pkgname, + "exposed: False", + cmd_args(cmd_args(static_libs, delimiter = ", "), format = "ld-options: {}"), + "depends: " + ", ".join([lib.id for lib in hlis]), + ) + + pkg_conf = ctx.actions.write("pkg-" + artifact_suffix + "_link.conf", conf) + db = ctx.actions.declare_output("db-" + artifact_suffix + "_link", dir = True) + + # While the list of hlis is unique, there may be multiple packages in the same db. + # Cutting down the GHC_PACKAGE_PATH significantly speeds up GHC. + db_deps = {x.db: None for x in hlis}.keys() + + # So that ghc-pkg can find the DBs for the dependencies. We might + # be able to use flags for this instead, but this works. + ghc_package_path = cmd_args( + db_deps, + delimiter = ":", + ) + + haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + ctx.actions.run( + cmd_args([ + "sh", + "-c", + _REGISTER_PACKAGE, + "", + haskell_toolchain.packager, + db.as_output(), + pkg_conf, + ]), + category = "haskell_package_link" + artifact_suffix.replace("-", "_"), + env = {"GHC_PACKAGE_PATH": ghc_package_path}, + ) + + return db + +def _dynamic_link_binary_impl(actions, pkg_deps, output, arg): + link_cmd = arg.link.copy() # link is already frozen, make a copy + + # Add -package-db and -package/-expose-package flags for each Haskell + # library dependency. + packages_info = get_packages_info2( + actions, + deps = arg.deps, + direct_deps_link_info = arg.direct_deps_link_info, + haskell_toolchain = arg.haskell_toolchain, + haskell_direct_deps_lib_infos = arg.haskell_direct_deps_lib_infos, + link_style = arg.link_style, + pkg_deps = pkg_deps, + specify_pkg_version = False, + enable_profiling = arg.enable_profiling, + use_empty_lib = False, + ) + + link_cmd.add("-hide-all-packages") + link_cmd.add(cmd_args(arg.toolchain_libs, prepend = "-package")) + link_cmd.add(cmd_args(packages_info.exposed_package_args)) + link_cmd.add(cmd_args(packages_info.packagedb_args, prepend = "-package-db")) + link_cmd.add(arg.haskell_toolchain.linker_flags) + link_cmd.add(arg.linker_flags) + + link_cmd.add("-o", output) + + actions.run( + link_cmd, + category = "haskell_link", + # explicit turn this on for local_only actions to upload their results. + allow_cache_upload = True, + ) + + return [] + +_dynamic_link_binary = dynamic_actions( + impl = _dynamic_link_binary_impl, + attrs = { + "arg": dynattrs.value(typing.Any), + "pkg_deps": dynattrs.option(dynattrs.dynamic_value()), + "output": dynattrs.output(), + }, +) + def haskell_binary_impl(ctx: AnalysisContext) -> list[Provider]: enable_profiling = ctx.attrs.enable_profiling @@ -917,29 +1210,33 @@ def haskell_binary_impl(ctx: AnalysisContext) -> list[Provider]: if enable_profiling and link_style == LinkStyle("shared"): link_style = LinkStyle("static") + md_file = target_metadata(ctx, sources = ctx.attrs.srcs) + compiled = compile( ctx, link_style, enable_profiling = enable_profiling, + enable_haddock = False, + md_file = md_file, ) haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + toolchain_libs = [dep[HaskellToolchainLibrary].name for dep in ctx.attrs.deps if HaskellToolchainLibrary in dep] + output = ctx.actions.declare_output(ctx.attrs.name) - link = cmd_args( - [haskell_toolchain.compiler] + - ["-o", output.as_output()] + - [haskell_toolchain.linker_flags] + - [ctx.attrs.linker_flags], - hidden = compiled.stubs, - ) + link = cmd_args(haskell_toolchain.compiler) - link_args = cmd_args() + objects = {} - osuf, _hisuf = output_extensions(link_style, enable_profiling) + # only add the first object per module + # TODO[CB] restructure this to use a record / dict for compiled.objects + for obj in compiled.objects: + key = paths.replace_extension(obj.short_path, "") + if not key in objects: + objects[key] = obj - objfiles = _srcs_to_objfiles(ctx, compiled.objects, osuf) - link_args.add(objfiles) + link.add(objects.values()) indexing_tsets = {} if compiled.producing_indices: @@ -1087,15 +1384,52 @@ def haskell_binary_impl(ctx: AnalysisContext) -> list[Provider]: sos.extend(traverse_shared_library_info(shlib_info)) infos = get_link_args_for_strategy(ctx, nlis, to_link_strategy(link_style)) - link_args.add(cmd_args(unpack_link_args(infos), prepend = "-optl")) + if link_style in [LinkStyle("static"), LinkStyle("static_pic")]: + hlis = attr_deps_haskell_link_infos_sans_template_deps(ctx) + linfos = [x.prof_info if enable_profiling else x.info for x in hlis] + uniq_infos = [x[link_style].value for x in linfos] - link.add(at_argfile( - actions = ctx.actions, - name = "args.haskell_link_argsfile", - args = link_args, - allow_args = True, + pkgname = ctx.label.name + "-link" + linkable_artifacts = [ + f.archive.artifact + for link in infos.tset.infos.traverse(ordering = "topological") + for f in link.default.linkables + ] + db = _make_link_package( + ctx, + link_style, + pkgname, + uniq_infos, + linkable_artifacts, + ) + + link.add(cmd_args(db, prepend = "-package-db")) + link.add("-package", pkgname) + link.add(cmd_args(hidden = linkable_artifacts)) + else: + link.add(cmd_args(unpack_link_args(infos), prepend = "-optl")) + + haskell_direct_deps_lib_infos = attr_deps_haskell_lib_infos( + ctx, + link_style, + enable_profiling = enable_profiling, + ) + + ctx.actions.dynamic_output_new(_dynamic_link_binary( + pkg_deps = haskell_toolchain.packages.dynamic if haskell_toolchain.packages else None, + output = output.as_output(), + arg = struct( + deps = ctx.attrs.deps, + direct_deps_link_info = attr_deps_haskell_link_infos(ctx), + enable_profiling = enable_profiling, + haskell_direct_deps_lib_infos = haskell_direct_deps_lib_infos, + haskell_toolchain = haskell_toolchain, + link = link, + link_style = link_style, + linker_flags = ctx.attrs.linker_flags, + toolchain_libs = toolchain_libs, + ), )) - ctx.actions.run(link, category = "haskell_link") if link_style == LinkStyle("shared") or link_group_info != None: sos_dir = "__{}__shared_libs_symlink_tree".format(ctx.attrs.name) @@ -1111,8 +1445,18 @@ def haskell_binary_impl(ctx: AnalysisContext) -> list[Provider]: else: run = cmd_args(output) + sub_targets = {} + sub_targets.update(_haskell_module_sub_targets( + compiled = compiled, + link_style = link_style, + enable_profiling = enable_profiling, + )) + providers = [ - DefaultInfo(default_output = output), + DefaultInfo( + default_output = output, + sub_targets = sub_targets, + ), RunInfo(args = run), ] @@ -1120,3 +1464,18 @@ def haskell_binary_impl(ctx: AnalysisContext) -> list[Provider]: providers.append(HaskellIndexInfo(info = indexing_tsets)) return providers + +def _haskell_module_sub_targets(*, compiled, link_style, enable_profiling): + (osuf, hisuf) = output_extensions(link_style, enable_profiling) + return { + "interfaces": [DefaultInfo(sub_targets = { + src_to_module_name(hi.short_path): [DefaultInfo(default_output = hi)] + for hi in compiled.hi + if hi.extension[1:] == hisuf + })], + "objects": [DefaultInfo(sub_targets = { + src_to_module_name(o.short_path): [DefaultInfo(default_output = o)] + for o in compiled.objects + if o.extension[1:] == osuf + })], + } diff --git a/haskell/haskell_ghci.bzl b/haskell/haskell_ghci.bzl index 118e11f7e..817dc8163 100644 --- a/haskell/haskell_ghci.bzl +++ b/haskell/haskell_ghci.bzl @@ -43,6 +43,7 @@ load( ) load( "@prelude//linking:linkable_graph.bzl", + "LinkableGraph", "LinkableRootInfo", "create_linkable_graph", "get_deps_for_link", @@ -56,6 +57,10 @@ load( "with_unique_str_sonames", ) load("@prelude//linking:types.bzl", "Linkage") +load( + "@prelude//cxx:linker.bzl", + "get_rpath_origin", +) load( "@prelude//utils:graph_utils.bzl", "depth_first_traversal", @@ -179,7 +184,11 @@ def _build_haskell_omnibus_so(ctx: AnalysisContext) -> HaskellOmnibusData: for nlabel, n in graph_nodes.items() } - all_direct_deps = [dep.label for dep in all_deps] + all_direct_deps = [] + for dep in all_deps: + graph = dep.get(LinkableGraph) + if graph: + all_direct_deps.append(graph.label) dep_graph[ctx.label] = all_direct_deps # Need to exclude all transitive deps of excluded deps @@ -299,14 +308,14 @@ def _build_haskell_omnibus_so(ctx: AnalysisContext) -> HaskellOmnibusData: soname = "libghci_dependencies.so" extra_ldflags = [ "-rpath", - "$ORIGIN/{}".format(so_symlinks_root_path), + "{}/{}".format(get_rpath_origin(linker_info.type), so_symlinks_root_path) ] link_result = cxx_link_shared_library( ctx, soname, opts = link_options( links = [ - LinkArgs(flags = extra_ldflags), + LinkArgs(flags = cmd_args(cmd_args(extra_ldflags, delimiter=","), format="-Wl,{}")), LinkArgs(infos = body_link_infos.values()), LinkArgs(infos = tp_deps_shared_link_infos.values()), ], @@ -323,6 +332,11 @@ def _build_haskell_omnibus_so(ctx: AnalysisContext) -> HaskellOmnibusData: so_symlinks_root = so_symlinks_root, ) +def _get_default_output(dependency: Dependency | None) -> Artifact | None: + if dependency == None: + return None + return dependency.get(DefaultInfo).default_outputs[0] + # Use the script_template_processor.py script to generate a script from a # script template. def _replace_macros_in_script_template( @@ -342,16 +356,16 @@ def _replace_macros_in_script_template( # Optional string args srcs: [str, None] = None, output_name: [str, None] = None, - ghci_iserv_path: [str, None] = None, + ghci_iserv_path: [Artifact, None] = None, preload_libs: [str, None] = None) -> Artifact: toolchain_paths = { BINUTILS_PATH: haskell_toolchain.ghci_binutils_path, - GHCI_LIB_PATH: haskell_toolchain.ghci_lib_path, + GHCI_LIB_PATH: _get_default_output(haskell_toolchain.ghci_lib_path), CC_PATH: haskell_toolchain.ghci_cc_path, CPP_PATH: haskell_toolchain.ghci_cpp_path, CXX_PATH: haskell_toolchain.ghci_cxx_path, - GHCI_PACKAGER: haskell_toolchain.ghci_packager, - GHCI_GHC_PATH: haskell_toolchain.ghci_ghc_path, + GHCI_PACKAGER: _get_default_output(haskell_toolchain.ghci_packager), + GHCI_GHC_PATH: _get_default_output(haskell_toolchain.ghci_ghc_path), } if ghci_bin != None: @@ -365,7 +379,8 @@ def _replace_macros_in_script_template( replace_cmd = cmd_args(script_template_processor) replace_cmd.add(cmd_args(script_template, format = "--script_template={}")) for name, path in toolchain_paths.items(): - replace_cmd.add(cmd_args("--{}={}".format(name, path))) + if path: + replace_cmd.add(cmd_args(path, format = "--{}={{}}".format(name))) replace_cmd.add(cmd_args( final_script.as_output(), @@ -462,7 +477,7 @@ def _write_iserv_script( script_template = ghci_iserv_template, output_name = iserv_script_name, haskell_toolchain = haskell_toolchain, - ghci_iserv_path = ghci_iserv_path, + ghci_iserv_path = _get_default_output(ghci_iserv_path), preload_libs = preload_libs, ) return iserv_script @@ -620,13 +635,15 @@ def haskell_ghci_impl(ctx: AnalysisContext) -> list[Provider]: enable_profiling, ) - link_style = LinkStyle("static_pic") + link_style = LinkStyle("shared") + #link_style = LinkStyle("static_pic") packages_info = get_packages_info( ctx, link_style, specify_pkg_version = True, enable_profiling = enable_profiling, + use_empty_lib = False, ) # Create package db symlinks @@ -651,7 +668,8 @@ def haskell_ghci_impl(ctx: AnalysisContext) -> list[Provider]: for prof, import_dir in lib.import_dirs.items(): artifact_suffix = get_artifact_suffix(link_style, prof) - lib_symlinks["hi-" + artifact_suffix] = import_dir + for imp in import_dir: + lib_symlinks["mod-" + artifact_suffix + "/" + imp.short_path] = imp for o in lib.libs: lib_symlinks[o.short_path] = o @@ -720,7 +738,9 @@ def haskell_ghci_impl(ctx: AnalysisContext) -> list[Provider]: "__{}__".format(ctx.label.name), output_artifacts, ) - run = cmd_args(final_ghci_script, hidden = outputs) + ghci_bin_dep = ctx.attrs.ghci_bin_dep.get(RunInfo) + hidden_dep = [ghci_bin_dep] if ghci_bin_dep else [] + run = cmd_args(final_ghci_script, hidden=hidden_dep + outputs) return [ DefaultInfo(default_outputs = [root_output_dir]), diff --git a/haskell/haskell_haddock.bzl b/haskell/haskell_haddock.bzl index 4154e3aba..78190ab7c 100644 --- a/haskell/haskell_haddock.bzl +++ b/haskell/haskell_haddock.bzl @@ -5,7 +5,7 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load("@prelude//haskell:compile.bzl", "compile_args") +load("@prelude//haskell:compile.bzl", "CompileResultInfo", "CompiledModuleTSet", "DynamicCompileResultInfo") load("@prelude//haskell:link_info.bzl", "cxx_toolchain_link_style") load( "@prelude//haskell:toolchain.bzl", @@ -14,99 +14,202 @@ load( load( "@prelude//haskell:util.bzl", "attr_deps", + "attr_deps_haskell_link_infos", + "src_to_module_name", ) -load("@prelude//utils:argfile.bzl", "at_argfile") +load("@prelude//utils:graph_utils.bzl", "post_order_traversal") +load("@prelude//:paths.bzl", "paths") HaskellHaddockInfo = provider( fields = { - "html": provider_field(typing.Any, default = None), - "interface": provider_field(typing.Any, default = None), + "html": provider_field(dict[str, typing.Any], default = {}), + "interfaces": provider_field(list[typing.Any], default = []), }, ) -def haskell_haddock_lib(ctx: AnalysisContext, pkgname: str) -> Provider: - haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] - iface = ctx.actions.declare_output("haddock-interface") - odir = ctx.actions.declare_output("haddock-html", dir = True) +_HaddockInfo = record( + interface = Artifact, + haddock = Artifact, + html = Artifact, +) + +def _haskell_interfaces_args(info: _HaddockInfo): + return cmd_args(info.interface, format="--one-shot-dep-hi={}") - link_style = cxx_toolchain_link_style(ctx) - args = compile_args( - ctx, - link_style, - enable_profiling = False, - suffix = "-haddock", - pkgname = pkgname, +_HaddockInfoTSet = transitive_set( + args_projections = { + "interfaces": _haskell_interfaces_args + } +) + +def _haddock_module_to_html(module_name: str) -> str: + return module_name.replace(".", "-") + ".html" + +def _haddock_dump_interface( + actions: AnalysisActions, + cmd: cmd_args, + module_name: str, + module_tsets: dict[str, _HaddockInfoTSet], + haddock_info: _HaddockInfo, + module_deps: list[CompiledModuleTSet], + graph: dict[str, list[str]], + outputs: dict[Artifact, OutputArtifact]) -> _HaddockInfoTSet: + + # Transitive module dependencies from other packages. + cross_package_modules = actions.tset( + CompiledModuleTSet, + children = module_deps, ) + cross_interfaces = cross_package_modules.project_as_args("interfaces") + + # Transitive module dependencies from the same package. + this_package_modules = [ + module_tsets[dep_name] + for dep_name in graph[module_name] + ] + + expected_html = haddock_info.html + module_html = _haddock_module_to_html(module_name) + + if paths.basename(expected_html.short_path) != module_html: + html_output = actions.declare_output("haddock-html", module_html) + make_copy = True + else: + html_output = expected_html + make_copy = False + + actions.run( + cmd.copy().add( + "--odir", cmd_args(html_output.as_output(), parent = 1), + "--dump-interface", outputs[haddock_info.haddock], + "--html", + "--hoogle", + cmd_args( + haddock_info.interface, + format="--one-shot-hi={}"), + cmd_args( + [haddock_info.project_as_args("interfaces") for haddock_info in this_package_modules], + ), + cmd_args( + cross_interfaces, format="--one-shot-dep-hi={}" + ) + ), + category = "haskell_haddock", + identifier = module_name, + no_outputs_cleanup = True, + ) + if make_copy: + # XXX might as well use `symlink_file`` but that does not work with buck2 RE + # (see https://github.com/facebook/buck2/issues/222) + actions.copy_file(outputs[expected_html], html_output) + + return actions.tset( + _HaddockInfoTSet, + value = _HaddockInfo(interface = haddock_info.interface, haddock = haddock_info.haddock, html = haddock_info.html), + children = this_package_modules, + ) + +def _dynamic_haddock_dump_interfaces_impl(actions, md_file, dynamic_info_lib, outputs, arg): + md = md_file.read_json() + module_map = md["module_mapping"] + graph = md["module_graph"] + package_deps = md["package_deps"] + + haddock_infos = { module_map.get(k, k): v for k, v in arg.haddock_infos.items() } + module_tsets = {} + + for module_name in post_order_traversal(graph): + module_deps = [ + info.providers[DynamicCompileResultInfo].modules[mod] + for lib, info in dynamic_info_lib.items() + for mod in package_deps.get(module_name, {}).get(lib, []) + ] + + module_tsets[module_name] = _haddock_dump_interface( + actions, + arg.dyn_cmd.copy(), + module_name = module_name, + module_tsets = module_tsets, + haddock_info = haddock_infos[module_name], + module_deps = module_deps, + graph = graph, + outputs = outputs, + ) + + return [] + +_dynamic_haddock_dump_interfaces = dynamic_actions( + impl = _dynamic_haddock_dump_interfaces_impl, + attrs = { + "md_file": dynattrs.artifact_value(), + "arg": dynattrs.value(typing.Any), + "dynamic_info_lib": dynattrs.dict(str, dynattrs.dynamic_value()), + "outputs": dynattrs.dict(Artifact, dynattrs.output()), + }, +) + +def haskell_haddock_lib(ctx: AnalysisContext, pkgname: str, compiled: CompileResultInfo, md_file: Artifact) -> HaskellHaddockInfo: + haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + + link_style = cxx_toolchain_link_style(ctx) cmd = cmd_args(haskell_toolchain.haddock) - cmd.add(cmd_args(args.args_for_cmd, format = "--optghc={}")) + cmd.add( "--use-index", "doc-index.html", "--use-contents", "index.html", - "--html", - "--hoogle", "--no-tmp-comp-dir", "--no-warnings", - "--dump-interface", - iface.as_output(), - "--odir", - odir.as_output(), + "--optghc=-package-env=-", "--package-name", pkgname, ) - for lib in attr_deps(ctx): - hi = lib.get(HaskellHaddockInfo) - if hi != None: - cmd.add("--read-interface", hi.interface) - cmd.add(ctx.attrs.haddock_flags) source_entity = read_root_config("haskell", "haddock_source_entity", None) if source_entity: cmd.add("--source-entity", source_entity) - if args.args_for_file: - if haskell_toolchain.use_argsfile: - ghcargs = cmd_args(args.args_for_file, format = "--optghc={}") - cmd.add(at_argfile( - actions = ctx.actions, - name = "args.haskell_haddock_argsfile", - args = [ghcargs, args.srcs], - allow_args = True, - )) - else: - cmd.add(args.args_for_file) - - # Buck2 requires that the output artifacts are always produced, but Haddock only - # creates them if it needs to, so we need a wrapper script to mkdir the outputs. - script = ctx.actions.declare_output("haddock-script") - script_args = cmd_args([ - "mkdir", - "-p", - args.result.objects.as_output(), - args.result.hi.as_output(), - args.result.stubs.as_output(), - "&&", - cmd_args(cmd, quote = "shell"), - ], delimiter = " ") - ctx.actions.write( - script, - cmd_args("#!/bin/sh", script_args), - is_executable = True, - allow_args = True, - ) + haddock_infos = { + src_to_module_name(hi.short_path): _HaddockInfo( + interface = hi, + haddock = ctx.actions.declare_output("haddock-interface/{}.haddock".format(src_to_module_name(hi.short_path))), + html = ctx.actions.declare_output("haddock-html", _haddock_module_to_html(src_to_module_name(hi.short_path))), + ) + for hi in compiled.hi + if not hi.extension.endswith("-boot") + } - ctx.actions.run( - cmd_args(script, hidden = cmd), - category = "haskell_haddock", - no_outputs_cleanup = True, - ) + direct_deps_link_info = attr_deps_haskell_link_infos(ctx) - return HaskellHaddockInfo(interface = iface, html = odir) + ctx.actions.dynamic_output_new(_dynamic_haddock_dump_interfaces( + md_file = md_file, + dynamic_info_lib = { + info.value.name: info.value.dynamic[False] + for lib in direct_deps_link_info + for info in [ + #lib.prof_info[link_style] + #if enable_profiling else + lib.info[link_style], + ] + }, + outputs = {output: output.as_output() for info in haddock_infos.values() for output in [info.haddock, info.html]}, + arg = struct( + dyn_cmd = cmd.copy(), + haddock_infos = haddock_infos, + link_style = link_style, + md_file = md_file, + ), + )) + + return HaskellHaddockInfo( + interfaces = [i.haddock for i in haddock_infos.values()], + html = {module: i.html for module, i in haddock_infos.items()}, + ) def haskell_haddock_impl(ctx: AnalysisContext) -> list[Provider]: haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] @@ -116,6 +219,7 @@ def haskell_haddock_impl(ctx: AnalysisContext) -> list[Provider]: cmd = cmd_args(haskell_toolchain.haddock) cmd.add( + "--optghc=-package-env=-", "--gen-index", "--gen-contents", "-o", @@ -126,33 +230,35 @@ def haskell_haddock_impl(ctx: AnalysisContext) -> list[Provider]: for lib in attr_deps(ctx): hi = lib.get(HaskellHaddockInfo) if hi != None: - cmd.add("--read-interface", hi.interface) - dep_htmls.append(hi.html) + cmd.add(cmd_args(hi.interfaces, format="--read-interface={}")) + if hi.html: + dep_htmls.extend(hi.html.values()) cmd.add(ctx.attrs.haddock_flags) - script = ctx.actions.declare_output("haddock-script") script_args = cmd_args([ "#!/bin/sh", - "set -ueo pipefail", - cmd_args(cmd, delimiter = " ", quote = "shell"), - ]) - for dir in dep_htmls: - script_args.add( - cmd_args( - ["cp", "-Rf", "--reflink=auto", cmd_args(dir, format = "{}/*"), out.as_output()], - delimiter = " ", - ), + cmd_args( + cmd_args(cmd, delimiter = " ", quote = "shell"), + [ + cmd_args( + ["cp", "-f", html, cmd_args(out, ignore_artifacts = True)], + delimiter = " ", + ) for html in dep_htmls + ], + delimiter = " && \\\n " ) - ctx.actions.write( - script, + ]) + + script = ctx.actions.write( + "haddock-script", script_args, is_executable = True, - allow_args = True, + with_inputs = True, ) ctx.actions.run( - cmd_args(script, hidden = script_args), + cmd_args(script, hidden=out.as_output()), category = "haskell_haddock", no_outputs_cleanup = True, ) diff --git a/haskell/ide/ide.bxl b/haskell/ide/ide.bxl index 57fcd4c05..1607d45d3 100644 --- a/haskell/ide/ide.bxl +++ b/haskell/ide/ide.bxl @@ -7,7 +7,8 @@ load("@prelude//haskell:library_info.bzl", "HaskellLibraryProvider") load("@prelude//haskell:link_info.bzl", "HaskellLinkInfo") -load("@prelude//haskell:toolchain.bzl", "HaskellToolchainInfo") +load("@prelude//haskell:toolchain.bzl", "HaskellToolchainInfo", "HaskellToolchainLibrary") +load("@prelude//haskell:util.bzl", "is_haskell_src", "srcs_to_pairs") load("@prelude//linking:link_info.bzl", "LinkStyle") load("@prelude//paths.bzl", "paths") @@ -33,7 +34,7 @@ _HASKELL_BIN = "prelude//rules.bzl:haskell_binary" _HASKELL_IDE = "prelude//rules.bzl:haskell_ide" _HASKELL_LIB = "prelude//rules.bzl:haskell_library" -linkStyle = LinkStyle("static") +linkStyle = LinkStyle("shared") def _impl_target(ctx): target = ctx.cli_args.target @@ -142,20 +143,33 @@ def _solution_for_haskell_lib(ctx, target, exclude): hli = ctx.analysis(target).providers().get(HaskellLibraryProvider) haskellLibs = {} + toolchain_libs = [] + for dep in resolved_attrs.deps + resolved_attrs.template_deps: if exclude.get(dep.label) == None: providers = ctx.analysis(dep.label).providers() lb = providers.get(HaskellLinkInfo) if lb != None: haskellLibs[dep.label] = lb + continue + + lb = providers.get(HaskellToolchainLibrary) + if lb != None: + toolchain_libs.append(lb.name) - sources = [] - for item in ctx.output.ensure_multiple(resolved_attrs.srcs.values()): - sources.append(item.abs_path()) + target_srcs = { + k: v + for k, v in srcs_to_pairs(resolved_attrs.srcs) + if is_haskell_src(k) + } + sources = [ + item.abs_path() + for item in ctx.output.ensure_multiple(target_srcs.values()) + ] import_dirs = {} root = ctx.root() - for key, item in resolved_attrs.srcs.items(): + for key, item in target_srcs.items(): # because BXL won't give you the path of an ensured artifact sp = get_path_without_materialization(item, ctx) (_, ext) = paths.split_extension(sp) @@ -165,27 +179,40 @@ def _solution_for_haskell_lib(ctx, target, exclude): haskell_toolchain = ctx.analysis(resolved_attrs._haskell_toolchain.label) toolchain = haskell_toolchain.providers().get(HaskellToolchainInfo) - binutils_path = paths.join(root, toolchain.ghci_binutils_path) - cc_path = paths.join(root, toolchain.ghci_cc_path) - cxx_path = paths.join(root, toolchain.ghci_cxx_path) - cpp_path = paths.join(root, toolchain.ghci_cpp_path) - flags = [ "-this-unit-id", - "fbcode_fake_unit_id", + "{}-fake_unit_id".format(target.label.name), + "-dynamic", "-optP-undef", "-optP-traditional-cpp", "-I.", - "-no-global-package-db", "-no-user-package-db", "-hide-all-packages", - "-pgma%s" % cc_path, - "-pgml%s" % cxx_path, - "-pgmc%s" % cc_path, - "-pgmP%s" % cpp_path, - "-opta-B%s" % binutils_path, - "-optc-B%s" % binutils_path, + "-i", ] + + if toolchain.ghci_binutils_path: + binutils_path = paths.join(root, toolchain.ghci_binutils_path) + flags.extend([ + "-opta-B%s" % binutils_path, + "-optc-B%s" % binutils_path, + ]) + + if toolchain.ghci_cc_path: + cc_path = paths.join(root, toolchain.ghci_cc_path) + flags.extend([ + "-pgma%s" % cc_path, + "-pgmc%s" % cc_path, + ]) + + if toolchain.ghci_cxx_path: + cxx_path = paths.join(root, toolchain.ghci_cxx_path) + flags.append("-pgml%s" % cxx_path) + + if toolchain.ghci_cpp_path: + cpp_path = paths.join(root, toolchain.ghci_cpp_path) + flags.append("-pgmP%s" % cpp_path) + flags.extend(resolved_attrs.compiler_flags) return { @@ -193,6 +220,7 @@ def _solution_for_haskell_lib(ctx, target, exclude): "flags": flags, "generated_dependencies": externalSourcesForTarget(ctx, target), "haskell_deps": haskellLibs, + "toolchain_libs": toolchain_libs, "import_dirs": import_dirs.keys(), "sources": sources, "targets": targetsForTarget(ctx, target), @@ -267,8 +295,12 @@ def _assembleSolution(ctx, linkStyle, result): flags.append(hli.name) ctx.output.ensure_multiple(hli.stub_dirs) ctx.output.ensure_multiple(hli.libs) - ctx.output.ensure_multiple(hli.import_dirs.values()) + import_dirs = [d for ds in hli.import_dirs.values() for d in ds] + ctx.output.ensure_multiple(import_dirs) package_dbs[hli.db] = () + for lib in result["toolchain_libs"]: + flags.append("-package") + flags.append(lib) for pkgdb in ctx.output.ensure_multiple(package_dbs.keys()): flags.append("-package-db") flags.append(pkgdb.abs_path()) diff --git a/haskell/library_info.bzl b/haskell/library_info.bzl index 3b048f137..dbd80cbea 100644 --- a/haskell/library_info.bzl +++ b/haskell/library_info.bzl @@ -5,6 +5,8 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. +load("@prelude//utils:utils.bzl", "flatten", "dedupe_by_value") + # If the target is a haskell library, the HaskellLibraryProvider # contains its HaskellLibraryInfo. (in contrast to a HaskellLinkInfo, # which contains the HaskellLibraryInfo for all the transitive @@ -23,10 +25,18 @@ HaskellLibraryInfo = record( name = str, # package config database: e.g. platform009/build/ghc/lib/package.conf.d db = Artifact, + # package config database, referring to the empty lib which is only used for compilation + empty_db = Artifact | None, + # package config database, used for ghc -M + deps_db = Artifact | None, # e.g. "base-4.13.0.0" id = str, + # dynamic dependency information + dynamic = None | dict[bool, DynamicValue], # Import dirs indexed by profiling enabled/disabled - import_dirs = dict[bool, Artifact], + import_dirs = dict[bool, list[Artifact]], + # Object files indexed by profiling enabled/disabled + objects = dict[bool, list[Artifact]], stub_dirs = list[Artifact], # This field is only used as hidden inputs to compilation, to @@ -40,6 +50,32 @@ HaskellLibraryInfo = record( version = str, is_prebuilt = bool, profiling_enabled = bool, + # Package dependencies + dependencies = list[str], ) -HaskellLibraryInfoTSet = transitive_set() +def _project_as_package_db(lib: HaskellLibraryInfo): + return cmd_args(lib.db) + +def _project_as_empty_package_db(lib: HaskellLibraryInfo): + return cmd_args(lib.empty_db) + +def _project_as_deps_package_db(lib: HaskellLibraryInfo): + return cmd_args(lib.deps_db) + +def _get_package_deps(children: list[list[str]], lib: HaskellLibraryInfo | None): + flatted = flatten(children) + if lib: + flatted.extend(lib.dependencies) + return dedupe_by_value(flatted) + +HaskellLibraryInfoTSet = transitive_set( + args_projections = { + "package_db": _project_as_package_db, + "empty_package_db": _project_as_empty_package_db, + "deps_package_db": _project_as_deps_package_db, + }, + reductions = { + "packages": _get_package_deps, + }, +) diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl index f6c072fbf..b685df9ea 100644 --- a/haskell/toolchain.bzl +++ b/haskell/toolchain.bzl @@ -5,6 +5,8 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. +load("@prelude//utils:arglike.bzl", "ArgLike") + HaskellPlatformInfo = provider(fields = { "name": provider_field(typing.Any, default = None), }) @@ -37,5 +39,42 @@ HaskellToolchainInfo = provider( "ghci_packager": provider_field(typing.Any, default = None), "cache_links": provider_field(typing.Any, default = None), "script_template_processor": provider_field(typing.Any, default = None), + "packages": provider_field(typing.Any, default = None), + "use_persistent_workers": provider_field(typing.Any, default = None), + }, +) + +HaskellToolchainLibrary = provider( + fields = { + "name": provider_field(str), + }, +) + +HaskellPackagesInfo = record( + dynamic = DynamicValue, +) + +HaskellPackage = record( + db = ArgLike, + path = Artifact, +) + +def _haskell_package_info_as_package_db(p: HaskellPackage): + return cmd_args(p.db) + +HaskellPackageDbTSet = transitive_set( + args_projections = { + "package_db": _haskell_package_info_as_package_db, + } +) + +DynamicHaskellPackageDbInfo = provider(fields = { + "packages": dict[str, HaskellPackageDbTSet], +}) + +NativeToolchainLibrary = provider( + fields = { + "name": provider_field(str), + "lib_path": provider_field(typing.Any, default = None), }, ) diff --git a/haskell/tools/BUCK.v2 b/haskell/tools/BUCK.v2 index 3029719fc..e57116335 100644 --- a/haskell/tools/BUCK.v2 +++ b/haskell/tools/BUCK.v2 @@ -11,3 +11,15 @@ prelude.python_bootstrap_binary( main = "script_template_processor.py", visibility = ["PUBLIC"], ) + +prelude.python_bootstrap_binary( + name = "generate_target_metadata", + main = "generate_target_metadata.py", + visibility = ["PUBLIC"], +) + +prelude.python_bootstrap_binary( + name = "ghc_wrapper", + main = "ghc_wrapper.py", + visibility = ["PUBLIC"], +) diff --git a/haskell/tools/generate_target_metadata.py b/haskell/tools/generate_target_metadata.py new file mode 100755 index 000000000..749e6d351 --- /dev/null +++ b/haskell/tools/generate_target_metadata.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 + +"""Helper script to generate relevant metadata about Haskell targets. + +* The mapping from module source file to actual module name. +* The intra-package module dependency graph. +* The cross-package module dependencies. +* Which modules require Template Haskell. + +Note, boot files will be represented by a `-boot` suffix in the module name. + +The result is a JSON object with the following fields: +* `th_modules`: List of modules that require Template Haskell. +* `module_mapping`: Mapping from source inferred module name to actual module name, if different. +* `module_graph`: Intra-package module dependencies, `dict[modname, list[modname]]`. +* `package_deps`": Cross-package module dependencies, `dict[modname, dict[pkgname, list[modname]]`. +""" + +import argparse +import sys +import json +import os +from pathlib import Path +import shlex +import subprocess +import tempfile + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + fromfile_prefix_chars="@") + parser.add_argument( + "--output", + required=True, + type=argparse.FileType("w"), + help="Write package metadata to this file in JSON format.") + parser.add_argument( + "--worker-target-id", + required=False, + type=str, + help="Worker id") + parser.add_argument( + "--ghc", + required=True, + type=str, + help="Path to the Haskell compiler GHC.") + parser.add_argument( + "--ghc-arg", + required=False, + type=str, + action="append", + help="GHC compiler argument to forward to `ghc -M`, including package flags.") + parser.add_argument( + "--source-prefix", + required=True, + type=str, + help="The path prefix to strip of module sources to extract module names.") + parser.add_argument( + "--source", + required=True, + type=str, + action="append", + help="Haskell module source files of the current package.") + parser.add_argument( + "--package", + required=False, + type=str, + action="append", + default=[], + help="Package dependencies formated as `NAME:PREFIX_PATH`.") + parser.add_argument( + "--bin-path", + type=Path, + action="append", + default=[], + help="Add given path to PATH.", + ) + args = parser.parse_args() + + result = obtain_target_metadata(args) + + json.dump(result, args.output, indent=4, default=json_default_handler) + + +def json_default_handler(o): + if isinstance(o, set): + return sorted(o) + raise TypeError(f'Object of type {o.__class__.__name__} is not JSON serializable') + + +def obtain_target_metadata(args): + paths = [str(binpath) for binpath in args.bin_path if binpath.is_dir()] + ghc_depends = run_ghc_depends(args.ghc, args.ghc_arg, args.source, paths, args.worker_target_id) + th_modules = determine_th_modules(ghc_depends) + module_mapping = determine_module_mapping(ghc_depends, args.source_prefix) + module_graph = determine_module_graph(ghc_depends) + package_deps = determine_package_deps(ghc_depends) + return { + "th_modules": th_modules, + "module_mapping": module_mapping, + "module_graph": module_graph, + "package_deps": package_deps, + } + + +def load_toolchain_packages(filepath): + with open(filepath, "r") as f: + return json.load(f) + + +def determine_th_modules(ghc_depends): + return [ + modname + for modname, properties in ghc_depends.items() + if uses_th(properties.get("options", [])) + ] + + +__TH_EXTENSIONS = ["TemplateHaskell", "TemplateHaskellQuotes", "QuasiQuotes"] + + +def uses_th(opts): + """Determine if a Template Haskell extension is enabled.""" + return any([f"-X{ext}" in opts for ext in __TH_EXTENSIONS]) + + +def determine_module_mapping(ghc_depends, source_prefix): + result = {} + + for modname, properties in ghc_depends.items(): + sources = list(filter(is_haskell_src, properties.get("sources", []))) + + if len(sources) != 1: + raise RuntimeError(f"Expected exactly one Haskell source for module '{modname}' but got '{sources}'.") + + apparent_name = src_to_module_name(strip_prefix_(source_prefix, sources[0]).lstrip("/")) + + if apparent_name != modname: + result[apparent_name] = modname + + boot_properties = properties.get("boot", None) + if boot_properties != None: + boot_modname = modname + "-boot" + boot_sources = list(filter(is_haskell_boot, boot_properties.get("sources", []))) + + if len(boot_sources) != 1: + raise RuntimeError(f"Expected at most one Haskell boot file for module '{modname}' but got '{boot_sources}'.") + + boot_apparent_name = src_to_module_name(strip_prefix_(source_prefix, boot_sources[0]).lstrip("/")) + "-boot" + + if boot_apparent_name != boot_modname: + result[boot_apparent_name] = boot_modname + + return result + + +def determine_module_graph(ghc_depends): + module_deps = {} + for modname, description in ghc_depends.items(): + module_deps[modname] = description.get("modules", []) + [ + dep + "-boot" + for dep in description.get("modules-boot", []) + ] + + boot_description = description.get("boot", None) + if boot_description != None: + module_deps[modname + "-boot"] = boot_description.get("modules", []) + [ + dep + "-boot" + for dep in boot_description.get("modules-boot", []) + ] + + return module_deps + + +def determine_package_deps(ghc_depends): + package_deps = {} + + for modname, description in ghc_depends.items(): + for pkgdep in description.get("packages", {}): + pkgname = pkgdep.get("name") + package_deps.setdefault(modname, {})[pkgname] = pkgdep.get("modules", []) + + boot_description = description.get("boot", None) + if boot_description != None: + for pkgdep in boot_description.get("packages", {}): + pkgname = pkgdep.get("name") + package_deps.setdefault(modname + "-boot", {})[pkgname] = pkgdep.get("modules", []) + + return package_deps + + +def run_ghc_depends(ghc, ghc_args, sources, aux_paths, worker_target_id): + with tempfile.TemporaryDirectory() as dname: + json_fname = os.path.join(dname, "depends.json") + make_fname = os.path.join(dname, "depends.make") + haskell_sources = list(filter(is_haskell_src, sources)) + haskell_boot_sources = list(filter (is_haskell_boot, sources)) + if worker_target_id: + worker_args = ["--worker-target-id={}".format(worker_target_id)] + else: + worker_args = [] + args = [ + ghc, "-M", "-include-pkg-deps", + # Note: `-outputdir '.'` removes the prefix of all targets: + # backend/src/Foo/Util. => Foo/Util. + "-outputdir", ".", + "-dep-json", json_fname, + "-dep-makefile", make_fname, + ] + worker_args + ghc_args + haskell_sources + haskell_boot_sources + + env = os.environ.copy() + path = env.get("PATH", "") + env["PATH"] = os.pathsep.join([path] + aux_paths) + + res = subprocess.run(args, env=env, capture_output=True) + if res.returncode != 0: + # Write the GHC command on failure. + print(shlex.join(args), file=sys.stderr) + + # Always forward stdout/stderr. + # Note, Buck2 swallows stdout on successful builds. + # Redirect to stderr to avoid this. + sys.stderr.buffer.write(res.stdout) + sys.stderr.buffer.write(res.stderr) + + if res.returncode != 0: + # Fail if GHC failed. + sys.exit(res.returncode) + + with open(json_fname) as f: + return json.load(f) + + +def src_to_module_name(x): + base, _ = os.path.splitext(x) + return base.replace("/", ".") + + +def is_haskell_src(x): + _, ext = os.path.splitext(x) + return ext in HASKELL_EXTENSIONS + + +def is_haskell_boot(x): + _, ext = os.path.splitext(x) + return ext in HASKELL_BOOT_EXTENSIONS + + +HASKELL_EXTENSIONS = [ + ".hs", + ".lhs", + ".hsc", + ".chs", + ".x", + ".y", +] + + +HASKELL_BOOT_EXTENSIONS = [ + ".hs-boot", + ".lhs-boot", +] + + +def strip_prefix_(prefix, s): + stripped = strip_prefix(prefix, s) + + if stripped == None: + return s + + return stripped + + +def strip_prefix(prefix, s): + if s.startswith(prefix): + return s[len(prefix):] + + return None + + +if __name__ == "__main__": + main() diff --git a/haskell/tools/ghc_wrapper.py b/haskell/tools/ghc_wrapper.py new file mode 100755 index 000000000..ccd4bee13 --- /dev/null +++ b/haskell/tools/ghc_wrapper.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 + +"""Wrapper script to call ghc. + +It accepts a dep file where all used inputs are written to. For any passed ABI +hash file, the corresponding interface is marked as unused, so these can change +without triggering compilation actions. + +""" + +import argparse +import os +from pathlib import Path +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, add_help=False, fromfile_prefix_chars="@" + ) + parser.add_argument( + "--buck2-dep", + required=True, + help="Path to the dep file.", + ) + parser.add_argument( + "--buck2-packagedb-dep", + required=True, + help="Path to the dep file.", + ) + parser.add_argument( + "--buck2-package-db", + required=False, + nargs="*", + default=[], + help="Path to a package db that is used during the module compilation", + ) + parser.add_argument( + "--worker-target-id", required=False, type=str, help="worker target id", + ) + parser.add_argument( + "--worker-close", required=False, type=bool, default=False, help="worker close", + ) + parser.add_argument( + "--ghc", required=True, type=str, help="Path to the Haskell compiler GHC." + ) + parser.add_argument( + "--abi-out", + required=True, + type=Path, + help="Output path of the abi file to create.", + ) + parser.add_argument( + "--bin-path", + type=Path, + action="append", + default=[], + help="Add given path to PATH.", + ) + parser.add_argument( + "--bin-exe", + type=Path, + action="append", + default=[], + help="Add given exe (more specific than bin-path)", + ) + parser.add_argument( + "--extra-env-key", + type=str, + action="append", + default=[], + help="Extra environment variable name", + ) + parser.add_argument( + "--extra-env-value", + type=str, + action="append", + default=[], + help="Extra environment variable value", + ) + + args, ghc_args = parser.parse_known_args() + if args.worker_target_id: + worker_args = ["--worker-target-id={}".format(args.worker_target_id)] + (["--worker-close"] if args.worker_close else []) + use_persistent_workers = True + else: + worker_args = [] + use_persistent_workers = False + cmd = [args.ghc] + worker_args + ghc_args + + aux_paths = [str(binpath) for binpath in args.bin_path if binpath.is_dir()] + [str(os.path.dirname(binexepath)) for binexepath in args.bin_exe] + env = os.environ.copy() + path = env.get("PATH", "") + env["PATH"] = os.pathsep.join([path] + aux_paths) + + extra_env_keys = [str(k) for k in args.extra_env_key] + extra_env_values = [str(v) for v in args.extra_env_value] + assert len(extra_env_keys) == len(extra_env_values), "number of --extra-env-key and --extra-env-value flags must match" + n_extra_env = len(extra_env_keys) + if n_extra_env > 0: + for i in range(0, n_extra_env): + k = extra_env_keys[i] + v = extra_env_values[i] + env[k] = v + + # Note, Buck2 swallows stdout on successful builds. + # Redirect to stderr to avoid this. + returncode = subprocess.call(cmd, env=env, stdout=sys.stderr.buffer) + if returncode != 0: + return returncode + + recompute_abi_hash(args.ghc, args.abi_out, use_persistent_workers) + + # write an empty dep file, to signal that all tagged files are unused + try: + with open(args.buck2_dep, "w") as f: + f.write("\n") + + except Exception as e: + # remove incomplete dep file + os.remove(args.buck2_dep) + raise e + + # write an empty dep file, to signal that all tagged files are unused + try: + with open(args.buck2_packagedb_dep, "w") as f: + for db in args.buck2_package_db: + f.write(db + "\n") + if not args.buck2_package_db: + f.write("\n") + + except Exception as e: + # remove incomplete dep file + os.remove(args.buck2_packagedb_dep) + raise e + + return 0 + + +def recompute_abi_hash(ghc, abi_out, use_persistent_workers): + """Call ghc on the hi file and write the ABI hash to abi_out.""" + hi_file = abi_out.with_suffix("") + if use_persistent_workers: + worker_args = ["--worker-target-id=show-iface-abi-hash"] + else: + worker_args = [] + + cmd = [ghc, "-v0", "-package-env=-", "--show-iface-abi-hash", hi_file] + worker_args + + hash = subprocess.check_output(cmd, text=True).split(maxsplit=1)[0] + + abi_out.write_text(hash) + + +if __name__ == "__main__": + main() diff --git a/haskell/tools/script_template_processor.py b/haskell/tools/script_template_processor.py index a105c4dbf..b67e0e2ee 100644 --- a/haskell/tools/script_template_processor.py +++ b/haskell/tools/script_template_processor.py @@ -66,18 +66,26 @@ def _replace_template_values( string=script_template, ) - # user_ghci_path has to be handled separately because it needs to be passed - # with the ghci_lib_path as the `-B` argument. - ghci_lib_canonical_path = os.path.realpath( - rel_toolchain_paths["ghci_lib_path"], - ) if user_ghci_path is not None: - script_template = re.sub( - pattern="", - repl="${{DIR}}/{user_ghci_path} -B{ghci_lib_path}".format( + # user_ghci_path has to be handled separately because it needs to be passed + # with the ghci_lib_path as the `-B` argument. + ghci_lib_path = rel_toolchain_paths["ghci_lib_path"] + + if ghci_lib_path: + ghci_lib_canonical_path = os.path.realpath(ghci_lib_path) + + replacement="${{DIR}}/{user_ghci_path} -B{ghci_lib_path}".format( user_ghci_path=user_ghci_path, ghci_lib_path=ghci_lib_canonical_path, - ), + ) + else: + replacement="${{DIR}}/{user_ghci_path}".format( + user_ghci_path=user_ghci_path, + ) + + script_template = re.sub( + pattern="", + repl=replacement, string=script_template, ) diff --git a/haskell/util.bzl b/haskell/util.bzl index 80584cd3b..37c261cea 100644 --- a/haskell/util.bzl +++ b/haskell/util.bzl @@ -10,6 +10,10 @@ load( "@prelude//cxx:cxx_toolchain_types.bzl", "CxxPlatformInfo", ) +load( + "@prelude//haskell:toolchain.bzl", + "HaskellToolchainLibrary", +) load( "@prelude//haskell:library_info.bzl", "HaskellLibraryInfo", @@ -41,6 +45,11 @@ HASKELL_EXTENSIONS = [ ".y", ] +HASKELL_BOOT_EXTENSIONS = [ + ".hs-boot", + ".lhs-boot", +] + # We take a named_set for srcs, which is sometimes a list, sometimes a dict. # In future we should only accept a list, but for now, cope with both. def srcs_to_pairs(srcs) -> list[(str, Artifact)]: @@ -53,6 +62,10 @@ def is_haskell_src(x: str) -> bool: _, ext = paths.split_extension(x) return ext in HASKELL_EXTENSIONS +def is_haskell_boot(x: str) -> bool: + _, ext = paths.split_extension(x) + return ext in HASKELL_BOOT_EXTENSIONS + def src_to_module_name(x: str) -> str: base, _ext = paths.split_extension(x) return base.replace("/", ".") @@ -73,6 +86,15 @@ def attr_deps_haskell_link_infos(ctx: AnalysisContext) -> list[HaskellLinkInfo]: ], )) +def attr_deps_haskell_toolchain_libraries(ctx: AnalysisContext) -> list[HaskellToolchainLibrary]: + return filter( + None, + [ + d.get(HaskellToolchainLibrary) + for d in attr_deps(ctx) + ctx.attrs.template_deps + ], + ) + # DONT CALL THIS FUNCTION, you want attr_deps_haskell_link_infos instead def attr_deps_haskell_link_infos_sans_template_deps(ctx: AnalysisContext) -> list[HaskellLinkInfo]: return dedupe(filter( @@ -149,3 +171,34 @@ def get_artifact_suffix(link_style: LinkStyle, enable_profiling: bool, suffix: s if enable_profiling: artifact_suffix += "-prof" return artifact_suffix + suffix + +def _source_prefix(source: Artifact, module_name: str) -> str: + """Determine the directory prefix of the given artifact, considering that ghc has determined `module_name` for that file.""" + source_path = paths.replace_extension(source.short_path, "") + + module_name_for_file = src_to_module_name(source_path) + + # assert that source_path (without extension) and its module name have the same length + if len(source_path) != len(module_name_for_file): + fail("{} should have the same length as {}".format(source_path, module_name_for_file)) + + if module_name != module_name_for_file and module_name_for_file.endswith("." + module_name): + # N.B. the prefix could have some '.' characters in it, use the source_path to determine the prefix + return source_path[0:-len(module_name) - 1] + + return "" + + +def get_source_prefixes(srcs: list[Artifact], module_map: dict[str, str]) -> list[str]: + """Determine source prefixes for the given haskell files and a mapping from source file module name to module name.""" + source_prefixes = {} + for path, src in srcs_to_pairs(srcs): + if not is_haskell_src(path): + continue + + name = src_to_module_name(path) + real_name = module_map.get(name) + prefix = _source_prefix(src, real_name) if real_name else "" + source_prefixes[prefix] = None + + return source_prefixes.keys() diff --git a/java/tools/gen_class_to_source_map.py b/java/tools/gen_class_to_source_map.py index f86d83cd1..248cf9157 100644 --- a/java/tools/gen_class_to_source_map.py +++ b/java/tools/gen_class_to_source_map.py @@ -109,6 +109,19 @@ def main(argv): src_path, class_name.replace(".", "/") + src_path_ext ) + if not found: + # If the class is not present in the sources, we still want to + # include it if it has a prefix that we are interested in. + # certain classes in "androidx.databinding.*" are generated and it's useful to know their presence in jars + for prefix in args.include_classes_prefixes: + if classname.startswith(prefix): + classes.append( + { + "className": classname, + } + ) + break + json.dump( { "jarPath": args.jar, diff --git a/linking/link_info.bzl b/linking/link_info.bzl index e1e09ae9a..3b4725f53 100644 --- a/linking/link_info.bzl +++ b/linking/link_info.bzl @@ -10,7 +10,11 @@ load( "ArtifactTSet", "make_artifact_tset", ) -load("@prelude//cxx:cxx_toolchain_types.bzl", "PicBehavior") +load( + "@prelude//cxx:cxx_toolchain_types.bzl", + "LinkerType", + "PicBehavior", +) load( "@prelude//cxx:linker.bzl", "get_link_whole_args", @@ -77,7 +81,7 @@ ArchiveLinkable = record( archive = field(Archive), # If a bitcode bundle was created for this artifact it will be present here bitcode_bundle = field(Artifact | None, None), - linker_type = field(str), + linker_type = field(LinkerType), link_whole = field(bool, False), # Indicates if this archive may contain LTO bit code. Can be set to `False` # to e.g. tell dist LTO handling that a potentially expensive archive doesn't @@ -96,7 +100,7 @@ ObjectsLinkable = record( objects = field([list[Artifact], None], None), # Any of the objects that are in bitcode format bitcode_bundle = field(Artifact | None, None), - linker_type = field(str), + linker_type = field(LinkerType), link_whole = field(bool, False), ) @@ -900,13 +904,13 @@ def merge_swiftmodule_linkables(ctx: AnalysisContext, linkables: list[[Swiftmodu ], )) -def wrap_with_no_as_needed_shared_libs_flags(linker_type: str, link_info: LinkInfo) -> LinkInfo: +def wrap_with_no_as_needed_shared_libs_flags(linker_type: LinkerType, link_info: LinkInfo) -> LinkInfo: """ Wrap link info in args used to prevent linkers from dropping unused shared library dependencies from the e.g. DT_NEEDED tags of the link. """ - if linker_type == "gnu": + if linker_type == LinkerType("gnu"): return wrap_link_info( inner = link_info, pre_flags = ( @@ -916,7 +920,7 @@ def wrap_with_no_as_needed_shared_libs_flags(linker_type: str, link_info: LinkIn post_flags = ["-Wl,--pop-state"], ) - if linker_type == "darwin": + if linker_type == LinkerType("darwin"): return link_info fail("Linker type {} not supported".format(linker_type)) diff --git a/linking/lto.bzl b/linking/lto.bzl index f275d0059..fab91ec6d 100644 --- a/linking/lto.bzl +++ b/linking/lto.bzl @@ -5,7 +5,11 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. -load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxToolchainInfo") +load( + "@prelude//cxx:cxx_toolchain_types.bzl", + "CxxToolchainInfo", + "LinkerType", +) load("@prelude//cxx:debug.bzl", "SplitDebugMode") # Styles of LTO. @@ -50,7 +54,7 @@ def get_split_debug_lto_info(actions: AnalysisActions, cxx_toolchain: CxxToolcha # TODO: It might be nice to generalize a but more and move the darwin v. gnu # differences into toolchain settings (e.g. `split_debug_lto_flags_fmt`). - if linker_info.type == "darwin": + if linker_info.type == LinkerType("darwin"): # https://releases.llvm.org/14.0.0/tools/clang/docs/CommandGuide/clang.html#cmdoption-flto # We need to pass -object_path_lto to keep the temporary LTO object files around to use # for dSYM generation. @@ -74,7 +78,7 @@ def get_split_debug_lto_info(actions: AnalysisActions, cxx_toolchain: CxxToolcha linker_flags = linker_args, ) - if linker_info.type == "gnu": + if linker_info.type == LinkerType("gnu"): dwo_dir = actions.declare_output(name + ".dwo.d", dir = True) linker_flags = cmd_args([ diff --git a/linking/strip.bzl b/linking/strip.bzl index baf413680..9bcd22207 100644 --- a/linking/strip.bzl +++ b/linking/strip.bzl @@ -6,7 +6,11 @@ # of this source tree. load("@prelude//cxx:cxx_context.bzl", "get_cxx_toolchain_info") -load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxToolchainInfo") +load( + "@prelude//cxx:cxx_toolchain_types.bzl", + "CxxToolchainInfo", + "LinkerType", +) def _strip_debug_info(ctx: AnalysisContext, out: str, obj: Artifact) -> Artifact: """ @@ -15,7 +19,7 @@ def _strip_debug_info(ctx: AnalysisContext, out: str, obj: Artifact) -> Artifact cxx_toolchain = get_cxx_toolchain_info(ctx) strip = cxx_toolchain.binary_utilities_info.strip output = ctx.actions.declare_output("__stripped__", out) - if cxx_toolchain.linker_info.type == "gnu": + if cxx_toolchain.linker_info.type == LinkerType("gnu"): cmd = cmd_args([strip, "--strip-debug", "--strip-unneeded", "-o", output.as_output(), obj]) else: cmd = cmd_args([strip, "-S", "-o", output.as_output(), obj]) diff --git a/platforms/apple/BUCK b/platforms/apple/BUCK new file mode 100644 index 000000000..83f7fa856 --- /dev/null +++ b/platforms/apple/BUCK @@ -0,0 +1,3 @@ +load("@prelude//platforms/apple:build_mode.bzl", "config_settings") + +config_settings(config_setting_rule = config_setting) diff --git a/platforms/apple/constraints/BUCK b/platforms/apple/constraints/BUCK new file mode 100644 index 000000000..9c2001679 --- /dev/null +++ b/platforms/apple/constraints/BUCK @@ -0,0 +1,6 @@ +load("@prelude//platforms/apple:build_mode.bzl", "constraints") + +constraints( + constraint_setting_rule = constraint_setting, + constraint_value_rule = constraint_value, +) diff --git a/python/tools/run_inplace.py.in b/python/tools/run_inplace.py.in index 8ea96bae1..2795a82bc 100644 --- a/python/tools/run_inplace.py.in +++ b/python/tools/run_inplace.py.in @@ -185,6 +185,11 @@ if "PAR_APPEND_PYTHONPATH" in os.environ: # the originating PAR (and can be read via e.g. `/proc//environ`). os.environ["PAR_INVOKED_NAME_TAG"] = sys.argv[0] +# This environment variable is immediately unset on startup but will also appear +# in e.g. `multiprocessing` workers, and so serves as an audit trail back to +# the originating PAR (and can be read via e.g. `/proc//environ`). +os.environ["PAR_INVOKED_NAME_TAG"] = sys.argv[0] + if platform.system() == "Windows": # exec on Windows is not true exec - there is only 'spawn' ('CreateProcess'). # However, creating processes unnecessarily is painful, so we only do the spawn diff --git a/rules_impl.bzl b/rules_impl.bzl index 9a6f02656..0518124cf 100644 --- a/rules_impl.bzl +++ b/rules_impl.bzl @@ -29,7 +29,7 @@ load("@prelude//go:go_library.bzl", "go_library_impl") load("@prelude//go:go_stdlib.bzl", "go_stdlib_impl") load("@prelude//go:go_test.bzl", "go_test_impl") load("@prelude//go/transitions:defs.bzl", "asan_attr", "cgo_enabled_attr", "coverage_mode_attr", "go_binary_transition", "go_exported_library_transition", "go_test_transition", "race_attr", "tags_attr") -load("@prelude//haskell:haskell.bzl", "haskell_binary_impl", "haskell_library_impl", "haskell_prebuilt_library_impl") +load("@prelude//haskell:haskell.bzl", "haskell_binary_impl", "haskell_library_impl", "haskell_prebuilt_library_impl", "haskell_toolchain_library_impl") load("@prelude//haskell:haskell_ghci.bzl", "haskell_ghci_impl") load("@prelude//haskell:haskell_haddock.bzl", "haskell_haddock_impl") load("@prelude//haskell:haskell_ide.bzl", "haskell_ide_impl") @@ -188,6 +188,7 @@ extra_implemented_rules = struct( haskell_haddock = haskell_haddock_impl, haskell_ide = haskell_ide_impl, haskell_prebuilt_library = haskell_prebuilt_library_impl, + haskell_toolchain_library = haskell_toolchain_library_impl, #lua cxx_lua_extension = cxx_lua_extension_impl, diff --git a/rust/build_params.bzl b/rust/build_params.bzl index ec0b1df3d..f21c280c6 100644 --- a/rust/build_params.bzl +++ b/rust/build_params.bzl @@ -7,6 +7,7 @@ # Rules for mapping requirements to options +load("@prelude//cxx:cxx_toolchain_types.bzl", "LinkerType") load( "@prelude//linking:link_info.bzl", "LibOutputStyle", @@ -186,20 +187,20 @@ _RUST_STATIC_NON_PIC_LIBRARY = 7 _NATIVE_LINKABLE_STATIC_PIC = 8 _NATIVE_LINKABLE_STATIC_NON_PIC = 9 -def _executable_prefix_suffix(linker_type: str, target_os_type: OsLookup) -> (str, str): +def _executable_prefix_suffix(linker_type: LinkerType, target_os_type: OsLookup) -> (str, str): return { - "darwin": ("", ""), - "gnu": ("", ".exe") if target_os_type.platform == "windows" else ("", ""), - "wasm": ("", ".wasm"), - "windows": ("", ".exe"), + LinkerType("darwin"): ("", ""), + LinkerType("gnu"): ("", ".exe") if target_os_type.platform == "windows" else ("", ""), + LinkerType("wasm"): ("", ".wasm"), + LinkerType("windows"): ("", ".exe"), }[linker_type] -def _library_prefix_suffix(linker_type: str, target_os_type: OsLookup) -> (str, str): +def _library_prefix_suffix(linker_type: LinkerType, target_os_type: OsLookup) -> (str, str): return { - "darwin": ("lib", ".dylib"), - "gnu": ("", ".dll") if target_os_type.platform == "windows" else ("lib", ".so"), - "wasm": ("", ".wasm"), - "windows": ("", ".dll"), + LinkerType("darwin"): ("lib", ".dylib"), + LinkerType("gnu"): ("", ".dll") if target_os_type.platform == "windows" else ("lib", ".so"), + LinkerType("wasm"): ("", ".wasm"), + LinkerType("windows"): ("", ".dll"), }[linker_type] _BUILD_PARAMS = { @@ -338,7 +339,7 @@ def build_params( link_strategy: LinkStrategy | None, lib_output_style: LibOutputStyle | None, lang: LinkageLang, - linker_type: str, + linker_type: LinkerType, target_os_type: OsLookup) -> BuildParams: if rule == RuleType("binary"): expect(link_strategy != None) diff --git a/toolchains/cxx.bzl b/toolchains/cxx.bzl index 57da1f97d..3212e4696 100644 --- a/toolchains/cxx.bzl +++ b/toolchains/cxx.bzl @@ -14,6 +14,7 @@ load( "CxxPlatformInfo", "CxxToolchainInfo", "LinkerInfo", + "LinkerType", "PicBehavior", "RcCompilerInfo", "ShlibInterfacesMode", @@ -36,7 +37,7 @@ CxxToolsInfo = provider( "cvtres_compiler": provider_field(typing.Any, default = None), "cxx_compiler": provider_field(typing.Any, default = None), "linker": provider_field(typing.Any, default = None), - "linker_type": provider_field(typing.Any, default = None), + "linker_type": LinkerType, "rc_compiler": provider_field(typing.Any, default = None), }, ) @@ -90,7 +91,7 @@ def _cxx_toolchain_from_cxx_tools_info(ctx: AnalysisContext, cxx_tools_info: Cxx additional_linker_flags = ["-fuse-ld=lld"] if os == "linux" and cxx_tools_info.linker != "g++" and cxx_tools_info.cxx_compiler != "g++" else [] if os == "windows": - linker_type = "windows" + linker_type = LinkerType("windows") binary_extension = "exe" object_file_extension = "obj" static_library_extension = "lib" @@ -107,10 +108,10 @@ def _cxx_toolchain_from_cxx_tools_info(ctx: AnalysisContext, cxx_tools_info: Cxx shared_library_versioned_name_format = "{}.so.{}" if os == "macos": - linker_type = "darwin" + linker_type = LinkerType("darwin") pic_behavior = PicBehavior("always_enabled") else: - linker_type = "gnu" + linker_type = LinkerType("gnu") pic_behavior = PicBehavior("supported") if cxx_tools_info.compiler_type == "clang": diff --git a/toolchains/cxx/clang/tools.bzl b/toolchains/cxx/clang/tools.bzl index ac85a344b..05f4a6a1b 100644 --- a/toolchains/cxx/clang/tools.bzl +++ b/toolchains/cxx/clang/tools.bzl @@ -5,6 +5,7 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. +load("@prelude//cxx:cxx_toolchain_types.bzl", "LinkerType") load("@prelude//toolchains:cxx.bzl", "CxxToolsInfo") def _path_clang_tools_impl(_ctx) -> list[Provider]: @@ -21,7 +22,7 @@ def _path_clang_tools_impl(_ctx) -> list[Provider]: archiver = "ar", archiver_type = "gnu", linker = "clang++", - linker_type = "gnu", + linker_type = LinkerType("gnu"), ), ] diff --git a/toolchains/cxx/zig/BUCK b/toolchains/cxx/zig/BUCK new file mode 100644 index 000000000..e69de29bb diff --git a/toolchains/msvc/tools.bzl b/toolchains/msvc/tools.bzl index e5b058194..93ffa771d 100644 --- a/toolchains/msvc/tools.bzl +++ b/toolchains/msvc/tools.bzl @@ -5,6 +5,7 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. +load("@prelude//cxx:cxx_toolchain_types.bzl", "LinkerType") load("@prelude//toolchains:cxx.bzl", "CxxToolsInfo") load("@prelude//utils:cmd_script.bzl", "ScriptOs", "cmd_script") @@ -133,7 +134,7 @@ def _find_msvc_tools_impl(ctx: AnalysisContext) -> list[Provider]: archiver = lib_exe_script, archiver_type = "windows", linker = _windows_linker_wrapper(ctx, link_exe_script), - linker_type = "windows", + linker_type = LinkerType("windows"), ), ]