diff --git a/WORKSPACE b/WORKSPACE index 58d4e49d1..3065cd3bd 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -33,6 +33,32 @@ load("@os_info//:os_info.bzl", "is_linux", "is_windows") # bazel dependencies haskell_repositories() +load("@io_tweag_rules_haskell//haskell:cabal.bzl", "stack_install") + +stack_install( + name = "stackage", + packages = [ + # Core libraries + "array", + "base", + "directory", + "filepath", + "process", + # For tests + "streaming", + "void", + "hspec", + "hspec-core", + "lens-family-core", + "data-default-class", + "lens-labels", + "proto-lens", + "lens-family", + ], + snapshot = "lts-13.15", + deps = ["@zlib.dev//:zlib"], +) + rules_nixpkgs_version = "0.5.2" rules_nixpkgs_version_is_hash = False @@ -52,29 +78,10 @@ load( "nixpkgs_local_repository", "nixpkgs_package", ) -load( - "@io_tweag_rules_haskell//haskell:nixpkgs.bzl", - "haskell_nixpkgs_package", - "haskell_nixpkgs_packageset", -) -load( - "@io_tweag_rules_haskell//tests/external-haskell-repository:workspace_dummy.bzl", - "haskell_package_repository_dummy", -) -load( - "@io_tweag_rules_haskell//:constants.bzl", - "test_ghc_version", -) -haskell_nixpkgs_package( +nixpkgs_package( name = "ghc", - attribute_path = "haskellPackages.ghc", - nix_file = "//tests:ghc.nix", - nix_file_deps = ["//nixpkgs:default.nix"], - # rules_nixpkgs assumes we want to read from `` implicitly - # if `repository` is not set, but our nix_file uses `./nixpkgs/`. - # TODO(Profpatsch) - repositories = {"nixpkgs": "//nixpkgs:NOTUSED"}, + repository = "@nixpkgs", ) http_archive( @@ -109,18 +116,22 @@ test_repl_ghci_args = [ "-XOverloadedStrings", ] +load( + "@io_tweag_rules_haskell//:constants.bzl", + "test_ghc_version", +) load( "@io_tweag_rules_haskell//haskell:nixpkgs.bzl", "haskell_register_ghc_nixpkgs", ) haskell_register_ghc_nixpkgs( + attribute_path = "ghc", compiler_flags = test_compiler_flags, haddock_flags = test_haddock_flags, locale_archive = "@glibc_locales//:locale-archive", - nix_file = "//tests:ghc.nix", - nix_file_deps = ["//nixpkgs:default.nix"], repl_ghci_args = test_repl_ghci_args, + repositories = {"nixpkgs": "@nixpkgs"}, version = test_ghc_version, ) @@ -159,12 +170,29 @@ cc_library( name = "zlib", linkstatic = 1, srcs = [":lib"], - testonly = 1, ) """, repository = "@nixpkgs", ) +nixpkgs_package( + name = "c2hs", + attribute_path = "haskellPackages.c2hs", + repository = "@nixpkgs", +) + +nixpkgs_package( + name = "doctest", + attribute_path = "haskellPackages.doctest", + repository = "@nixpkgs", +) + +nixpkgs_package( + name = "proto-lens-protoc", + attribute_path = "haskellPackages.proto-lens-protoc", + repository = "@nixpkgs", +) + nixpkgs_package( name = "sphinx", attribute_path = "python36Packages.sphinx", @@ -191,14 +219,12 @@ package(default_visibility = ["//visibility:public"]) filegroup ( name = "include", srcs = glob(["include/*.h"]), - testonly = 1, ) cc_library( name = "zlib", deps = ["@zlib//:zlib"], hdrs = [":include"], - testonly = 1, strip_include_prefix = "include", ) """, @@ -219,22 +245,6 @@ filegroup( repository = "@nixpkgs", ) -haskell_nixpkgs_packageset( - name = "hackage-packages", - base_attribute_path = "haskellPackages", - nix_file = "//tests:ghc.nix", - nix_file_deps = ["//tests/haddock:libC.nix"], - nixopts = [ - "-j", - "1", - ], - repositories = {"nixpkgs": "@nixpkgs"}, -) - -load("@hackage-packages//:packages.bzl", "import_packages") - -import_packages(name = "hackage") - load("@bazel_tools//tools/build_defs/repo:jvm.bzl", "jvm_maven_import_external") jvm_maven_import_external( @@ -251,6 +261,11 @@ local_repository( path = "tests/c2hs/repo", ) +load( + "@io_tweag_rules_haskell//tests/external-haskell-repository:workspace_dummy.bzl", + "haskell_package_repository_dummy", +) + # dummy repo for the external haskell repo test (hazel) haskell_package_repository_dummy( name = "haskell_package_repository_dummy", diff --git a/constants.bzl b/constants.bzl index 930fc8e38..655358fad 100644 --- a/constants.bzl +++ b/constants.bzl @@ -1 +1 @@ -test_ghc_version = "8.6.4" +test_ghc_version = "8.6.5" diff --git a/haskell/assets/ghc_8_6_5_win_base.patch b/haskell/assets/ghc_8_6_5_win_base.patch new file mode 100644 index 000000000..d870deeff --- /dev/null +++ b/haskell/assets/ghc_8_6_5_win_base.patch @@ -0,0 +1,11 @@ +--- lib/package.conf.d/base-4.12.0.0.conf 2019-03-20 12:24:30.857292020 +0100 ++++ lib/package.conf.d/base-4.12.0.0.conf 2019-03-20 12:24:44.637400564 +0100 +@@ -79,7 +79,7 @@ + data-dir: $topdir\x86_64-windows-ghc-8.6.5\base-4.12.0.0 + hs-libraries: HSbase-4.12.0.0 + extra-libraries: +- wsock32 user32 shell32 msvcrt mingw32 mingwex ++ wsock32 user32 shell32 msvcrt mingw32 mingwex shlwapi + include-dirs: $topdir\base-4.12.0.0\include + includes: + HsBase.h diff --git a/haskell/cabal.bzl b/haskell/cabal.bzl index 9db613c93..648867803 100644 --- a/haskell/cabal.bzl +++ b/haskell/cabal.bzl @@ -243,3 +243,225 @@ However, using a plain `haskell_library` sometimes leads to better build times, and does not require drafting a `.cabal` file. """ + +def _chop_version(name): + """Remove any version component from the given package name.""" + return name.rpartition("-")[0] + +# Temporary hardcoded list of core libraries. This will no longer be +# necessary once Stack 2.0 is released. +# +# TODO remove this list and replace it with Stack's --global-hints +# mechanism. +_CORE_PACKAGES = [ + "Cabal", + "array", + "base", + "binary", + "bytestring", + "containers", + "deepseq", + "directory", + "filepath", + "ghc", + "ghc-boot", + "ghc-boot-th", + "ghc-compact", + "ghc-heap", + "ghc-prim", + "ghci", + "haskeline", + "hpc", + "integer-gmp", + "libiserv", + "mtl", + "parsec", + "pretty", + "process", + "rts", + "stm", + "template-haskell", + "terminfo", + "text", + "time", + "transformers", + "unix", + "xhtml", +] + +def _compute_dependency_graph(repository_ctx, versioned_packages, unversioned_packages): + """Given a list of root packages, compute a dependency graph. + + Returns: + dependencies: adjacency list of packages, represented as a dictionary. + transitive_unpacked_sdists: directory names of unpacked source distributions. + + """ + + if not versioned_packages and not unversioned_packages: + return ({}, []) + + # Unpack all given packages, then compute the transitive closure + # and unpack anything in the transitive closure as well. + stack_cmd = repository_ctx.which("stack") + if not stack_cmd: + fail("Cannot find stack command in your PATH.") + stack = [stack_cmd, "--no-nix", "--skip-ghc-check", "--system-ghc"] + if versioned_packages: + _execute_or_fail_loudly(repository_ctx, stack + ["unpack"] + versioned_packages) + stack = [stack_cmd, "--no-nix", "--skip-ghc-check", "--system-ghc", "--resolver", repository_ctx.attr.snapshot] + if unversioned_packages: + _execute_or_fail_loudly(repository_ctx, stack + ["unpack"] + unversioned_packages) + exec_result = _execute_or_fail_loudly(repository_ctx, ["ls"]) + unpacked_sdists = exec_result.stdout.splitlines() + stack_yaml_content = struct(resolver = "none", packages = unpacked_sdists).to_json() + repository_ctx.file("stack.yaml", content = stack_yaml_content, executable = False) + exec_result = _execute_or_fail_loudly( + repository_ctx, + stack + ["ls", "dependencies", "--separator=-"], + ) + transitive_unpacked_sdists = [ + unpacked_sdist + for unpacked_sdist in exec_result.stdout.splitlines() + if _chop_version(unpacked_sdist) not in _CORE_PACKAGES + ] + _execute_or_fail_loudly( + repository_ctx, + stack + ["unpack"] + [ + unpacked_sdist + for unpacked_sdist in transitive_unpacked_sdists + if unpacked_sdist not in unpacked_sdists + ], + ) + stack_yaml_content = struct(resolver = "none", packages = transitive_unpacked_sdists).to_json() + repository_ctx.file("stack.yaml", stack_yaml_content, executable = False) + + # Compute dependency graph. + all_packages = [_chop_version(dir) for dir in transitive_unpacked_sdists + _CORE_PACKAGES] + exec_result = _execute_or_fail_loudly( + repository_ctx, + stack + ["dot", "--external"], + ) + dependencies = {k: [] for k in all_packages} + for line in exec_result.stdout.splitlines(): + tokens = [w.strip('";') for w in line.split(" ")] + + # All lines of the form `"foo" -> "bar";` declare edges of the + # dependency graph in the Graphviz format. + if len(tokens) == 3 and tokens[1] == "->": + [src, _, dest] = tokens + if src in all_packages and dest in all_packages: + dependencies[src].append(dest) + return (dependencies, transitive_unpacked_sdists) + +def _stack_install_impl(repository_ctx): + packages = repository_ctx.attr.packages + non_core_packages = [ + package + for package in packages + if package not in _CORE_PACKAGES + ] + versioned_packages = [] + unversioned_packages = [] + for package in non_core_packages: + if package.rpartition("-")[2].replace(".", "").isdigit(): + versioned_packages.append(package) + else: + unversioned_packages.append(package) + (dependencies, transitive_unpacked_sdists) = _compute_dependency_graph( + repository_ctx, + versioned_packages, + unversioned_packages, + ) + + # Write out the dependency graph as a BUILD file. + build_file_builder = [] + build_file_builder.append(""" +load("@io_tweag_rules_haskell//haskell:cabal.bzl", "haskell_cabal_library") +load("@io_tweag_rules_haskell//haskell:haskell.bzl", "haskell_toolchain_library") +""") + extra_deps = [ + "@{}//{}:{}".format(label.workspace_name, label.package, label.name) + for label in repository_ctx.attr.deps + ] + for package in _CORE_PACKAGES: + if package in packages: + visibility = ["//visibility:public"] + else: + visibility = ["//visibility:private"] + build_file_builder.append( + """ +haskell_toolchain_library(name = "{name}", visibility = {visibility}) +""".format(name = package, visibility = visibility), + ) + for package in transitive_unpacked_sdists: + unversioned_package = _chop_version(package) + if unversioned_package in _CORE_PACKAGES: + continue + if unversioned_package in unversioned_packages or package in versioned_packages: + visibility = ["//visibility:public"] + else: + visibility = ["//visibility:private"] + build_file_builder.append( + """ +haskell_cabal_library( + name = "{name}", + srcs = glob(["{dir}/**"]), + deps = {deps}, + visibility = {visibility}, +) +""".format( + name = package, + dir = package, + deps = dependencies[unversioned_package] + extra_deps, + testonly = repository_ctx.attr.testonly, + visibility = visibility, + ), + ) + build_file_builder.append( + """alias(name = "{name}", actual = ":{actual}", visibility = {visibility})""".format( + name = unversioned_package, + actual = package, + visibility = visibility, + ), + ) + build_file_content = "\n".join(build_file_builder) + repository_ctx.file("BUILD.bazel", build_file_content, executable = False) + +stack_install = repository_rule( + _stack_install_impl, + attrs = { + "snapshot": attr.string( + doc = "The name of a Stackage snapshot.", + ), + "packages": attr.string_list( + doc = "A set of package identifiers. For packages in the snapshot, version numbers can be omitted.", + ), + "deps": attr.label_list( + doc = "Dependencies of the package set, e.g. system libraries or C/C++ libraries.", + ), + }, +) +"""Use Stack to download and extract Cabal source distributions. + +Example: + ```bzl + stack_install( + name = "stackage", + packages = ["conduit", "lens", "zlib"], + snapshot = "lts-13.15", + deps = ["@zlib.dev//:zlib"], + ) + ``` + +This rule will use Stack to compute the transitive closure of the +subset of the given snapshot listed in the `packages` attribute, and +generate a dependency graph. If a package in the closure depends on +system libraries or other external libraries, use the `deps` attribute +to list them. This attribute works like the +`--extra-{include,lib}-dirs` flags for Stack and cabal-install do. + +In the external repository defined by the rule, all given packages are +available as top-level targets named after each package. + +""" diff --git a/haskell/ghc_bindist.bzl b/haskell/ghc_bindist.bzl index 7acc332d7..d55ee1b3e 100644 --- a/haskell/ghc_bindist.bzl +++ b/haskell/ghc_bindist.bzl @@ -376,6 +376,7 @@ def ghc_bindist( patches = { "8.6.2": ["@io_tweag_rules_haskell//haskell:assets/ghc_8_6_2_win_base.patch"], "8.6.4": ["@io_tweag_rules_haskell//haskell:assets/ghc_8_6_4_win_base.patch"], + "8.6.5": ["@io_tweag_rules_haskell//haskell:assets/ghc_8_6_5_win_base.patch"], }.get(version) if target == "windows_amd64" else None extra_attrs = {"patches": patches, "patch_args": ["-p0"]} if patches else {} diff --git a/haskell/nixpkgs.bzl b/haskell/nixpkgs.bzl index 381edc2e2..bb505de39 100644 --- a/haskell/nixpkgs.bzl +++ b/haskell/nixpkgs.bzl @@ -394,7 +394,7 @@ def haskell_register_ghc_nixpkgs( toolchain_repo_name = "io_tweag_rules_haskell_ghc_nixpkgs_toolchain" # The package from the system. - haskell_nixpkgs_package( + nixpkgs_package( name = nixpkgs_ghc_repo_name, attribute_path = attribute_path, build_file = build_file, diff --git a/shell.nix b/shell.nix index 64287ca76..a2481c26b 100644 --- a/shell.nix +++ b/shell.nix @@ -16,6 +16,9 @@ mkShell { perl python3 bazel + # For stack_install. TODO Remove ghc when move to Stack 2. + stack + ghc # Needed for @com_github_golang_protobuf, itself needed by buildifier. git # Needed to get correct locale for tests with encoding diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index 276db5546..cc9799817 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -22,7 +22,7 @@ package(default_testonly = 1) haskell_doctest_toolchain( name = "doctest-toolchain", - doctest = "@hackage-doctest//:bin", + doctest = "@doctest//:bin", tags = ["requires_doctest"], ) @@ -34,7 +34,7 @@ haskell_doctest_toolchain( haskell_proto_toolchain( name = "protobuf-toolchain", testonly = 0, - plugin = "@hackage-proto-lens-protoc//:bin/proto-lens-protoc", + plugin = "@proto-lens-protoc//:bin/proto-lens-protoc", protoc = "@com_google_protobuf//:protoc", tags = ["requires_hackage"], deps = [ @@ -44,17 +44,17 @@ haskell_proto_toolchain( "//tests/hackage:deepseq", "//tests/hackage:mtl", "//tests/hackage:text", - "@hackage//:data-default-class", - "@hackage//:lens-family", - "@hackage//:lens-family-core", - "@hackage//:lens-labels", - "@hackage//:proto-lens", + "@stackage//:data-default-class", + "@stackage//:lens-family", + "@stackage//:lens-family-core", + "@stackage//:lens-labels", + "@stackage//:proto-lens", ], ) c2hs_toolchain( name = "c2hs-toolchain", - c2hs = "@hackage-c2hs//:bin", + c2hs = "@c2hs//:bin", tags = ["requires_c2hs"], ) @@ -316,7 +316,7 @@ haskell_binary( deps = [ "//tests/hackage:base", "//tests/hackage:process", - "@hackage//:hspec", - "@hackage//:hspec-core", + "@stackage//:hspec", + "@stackage//:hspec-core", ], ) diff --git a/tests/binary-with-prebuilt/BUILD.bazel b/tests/binary-with-prebuilt/BUILD.bazel index 99a22ae99..7b3a9e955 100644 --- a/tests/binary-with-prebuilt/BUILD.bazel +++ b/tests/binary-with-prebuilt/BUILD.bazel @@ -13,7 +13,7 @@ haskell_test( deps = [ "//tests/hackage:base", "//tests/hackage:template-haskell", - "@hackage//:streaming", - "@hackage//:void", + "@stackage//:streaming", + "@stackage//:void", ], ) diff --git a/tests/ghc.nix b/tests/ghc.nix deleted file mode 100644 index cde169991..000000000 --- a/tests/ghc.nix +++ /dev/null @@ -1,41 +0,0 @@ -{ pkgs ? import ../nixpkgs {} -# Whether we want to wrap the packages using . -# When this is called from inside bazel, we need to wrap the haskell package -# set using . Otherwise we don't need (and don't want) -# to. -, wrapPackages ? (builtins.tryEval ).success -}: - -with pkgs; - -let haskellPackages = pkgs.haskell.packages.ghc864.override { - overrides = with pkgs.haskell.lib; self: super: rec { - libc = import ./haddock/libC.nix self pkgs; - }; - }; - genBazelBuild = - if wrapPackages - then callPackage {} - else (x: x); - -in - { - ghc = haskellPackages.ghcWithPackages (p: with p; [ - - # haskell_proto_library inputs - bytestring - containers - data-default-class - lens-family - lens-labels - proto-lens - text - - # test inputs - libc - - # for test runner - hspec - ]); - haskellPackages = genBazelBuild haskellPackages; -} diff --git a/tests/haddock/BUILD.bazel b/tests/haddock/BUILD.bazel index d6035132a..bd637c354 100644 --- a/tests/haddock/BUILD.bazel +++ b/tests/haddock/BUILD.bazel @@ -48,7 +48,6 @@ haskell_library( ":haddock-lib-a", "//tests/hackage:base", "//tests/hackage:template-haskell", - "@hackage//:libc", "@zlib", ], ) diff --git a/tests/haddock/LibB.hs b/tests/haddock/LibB.hs index b4df1ffcb..3648ed2bb 100644 --- a/tests/haddock/LibB.hs +++ b/tests/haddock/LibB.hs @@ -5,17 +5,12 @@ module LibB where import LibA.A (a) import LibA (f) -import LibC (mytype, LibCType) import TH (foo) -- | Doc for 'x' using 'f' and 'a' and 'Int'. x :: Int x = const f a --- | This uses a type from an undocumented package -y :: LibCType -y = mytype - -- | A thing generated with TH. z :: String z = $foo diff --git a/tests/haddock/libC.nix b/tests/haddock/libC.nix deleted file mode 100644 index d34517883..000000000 --- a/tests/haddock/libC.nix +++ /dev/null @@ -1,47 +0,0 @@ -# A trivial `haskellPackages` library that has haddock generation disabled -self: pkgs: -let - # pkgs = import ../../nixpkgs.nix {}; - - libC = pkgs.writeText "LibC.hs" '' - {-# language NoImplicitPrelude #-} - module LibC where - - data LibCType = LibCType - - -- | myfunction - mytype :: LibCType - mytype = LibCType - ''; - - cabal = pkgs.writeText "libc.cabal" '' - name: libc - version: 0.1.0.0 - build-type: Simple - cabal-version: >=1.10 - - library - default-language: Haskell2010 - exposed-modules: LibC - ''; - - src = pkgs.runCommand "libc-src" {} '' - mkdir $out - cp ${libC} $out/LibC.hs - cp ${cabal} $out/libc.cabal - ''; - -in - # This call means the `.haddock` file is not generated, - # even though the ghc package still references the location - # where it would ordinarily be. - pkgs.haskell.lib.dontHaddock - - (self.callPackage - ({ mkDerivation }: mkDerivation { - pname = "libc"; - version = "0.1.0.0"; - src = src; - license = pkgs.lib.licenses.mit; - isExecutable = false; - }) {}) diff --git a/tests/repl-targets/BUILD.bazel b/tests/repl-targets/BUILD.bazel index 2fd78bfd6..f734412f8 100644 --- a/tests/repl-targets/BUILD.bazel +++ b/tests/repl-targets/BUILD.bazel @@ -59,7 +59,7 @@ haskell_library( deps = [ "//tests/data:ourclibrary", "//tests/hackage:base", - "@hackage//:array", + "@stackage//:array", "@zlib", ], ) diff --git a/tools/runfiles/BUILD.bazel b/tools/runfiles/BUILD.bazel index bc99b96c0..b49a80582 100644 --- a/tools/runfiles/BUILD.bazel +++ b/tools/runfiles/BUILD.bazel @@ -10,9 +10,9 @@ haskell_library( src_strip_prefix = "src", visibility = ["//visibility:public"], deps = [ - "@hackage//:base", - "@hackage//:directory", - "@hackage//:filepath", + "@stackage//:base", + "@stackage//:directory", + "@stackage//:filepath", ], ) @@ -24,8 +24,8 @@ haskell_test( src_strip_prefix = "bin", deps = [ ":runfiles", - "@hackage//:base", - "@hackage//:filepath", + "@stackage//:base", + "@stackage//:filepath", ], ) @@ -39,8 +39,8 @@ haskell_test( src_strip_prefix = "test", deps = [ ":runfiles", - "@hackage//:base", - "@hackage//:filepath", - "@hackage//:process", + "@stackage//:base", + "@stackage//:filepath", + "@stackage//:process", ], )