Refactor package configuration parsing in ls_modules and pkgdb_to_bzl

haskell/BUILD.bazel

@@ -1,5 +1,5 @@
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load("@rules_python//python:defs.bzl", "py_binary")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library")
 load(
     "@rules_haskell//haskell:private/haskell_impl.bzl",
     "haskell_toolchain_libraries",
@@ -92,10 +92,18 @@ py_binary(
     visibility = ["//visibility:public"],
 )
 
+py_library(
+    name = "package_configuration",
+    srcs = ["private/package_configuration.py"],
+    imports = ["private"],
+    visibility = ["//tests/package_configuration:__subpackages__"],
+)
+
 py_binary(
     name = "ls_modules",
     srcs = ["private/ls_modules.py"],
     visibility = ["//visibility:public"],
+    deps = [":package_configuration"],
 )
 
 sh_binary(

haskell/private/ls_modules.py

@@ -16,6 +16,8 @@
 import sys
 import io
 
+import package_configuration
+
 if len(sys.argv) != 7:
     sys.exit("Usage: %s <WITH_PROFILING> <DIRECTORY> <GLOBAL_PKG_DB> <HIDDEN_MODS_FILE> <REEXPORTED_MODS_FILE> <RESULT_FILE>" % sys.argv[0])
 
@@ -27,23 +29,22 @@
 results_file = sys.argv[6]
 
 with io.open(global_pkg_db_dump, "r", encoding='utf8') as f:
-    names = [line.split()[1] for line in f if line.startswith("name:")]
-    f.seek(0)
-    ids = [line.split()[1] for line in f if line.startswith("id:")]
-
-    # A few sanity checks.
-    assert len(names) == len(ids)
-
-    # compute duplicate, i.e. package name associated with multiples ids
-    duplicates = set()
-    if len(names) != len(set(names)):
-        duplicates = set([
-            name for name, count in collections.Counter(names).items()
-            if count > 1
-        ])
-
-    # This associate pkg name to pkg id
-    pkg_ids_map = dict(zip(names, ids))
+    name_id_pairs = [
+        (pkg.name, pkg.id)
+        for pkg in package_configuration.parse_package_database_dump(f)
+    ]
+
+# compute duplicate, i.e. package name associated with multiples ids
+names = [name for (name, _) in name_id_pairs]
+duplicates = set()
+if len(names) != len(set(names)):
+    duplicates = set([
+        name for name, count in collections.Counter(names).items()
+        if count > 1
+    ])
+
+# This associate pkg name to pkg id
+pkg_ids_map = dict(name_id_pairs)
 
 with io.open(hidden_modules_file, "r", encoding='utf8') as f:
     hidden_modules = [mod.strip() for mod in f.read().split(",")]

haskell/private/package_configuration.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+
+import collections
+import types
+
+def unfold_fields(lines):
+    """Unfold fields that were split over multiple lines.
+
+    Returns:
+        A list of strings. Each string represents one field (a name/value pair
+        separated by a colon).
+
+    >>> unfold_fields("foo  \n   bar  \n   baz  \nbiz   \nboz   ")
+    ['foo     bar     baz  ', 'biz   ', 'boz   ']
+    """
+    fields = []
+    for line in lines:
+        if line.startswith(" "):
+            fields[-1] += line
+        elif line.strip():
+            fields.append(line)
+    return fields
+
+PackageConfiguration = collections.namedtuple("PackageConfiguration", [
+    "name",
+    "version",
+    "id",
+    "include_dirs",
+    "library_dirs",
+    "dynamic_library_dirs",
+    "hs_libraries",
+    "depends",
+    "ld_options",
+    "extra_libraries",
+    "haddock_interfaces",
+    "haddock_html",
+])
+
+def parse_package_configuration(lines):
+    """Parses a single package configuration.
+
+    Returns:
+      An instance of `PackageConfiguration`.
+    """
+    pkg = types.SimpleNamespace(
+        include_dirs = [],
+        library_dirs = [],
+        dynamic_library_dirs = [],
+        depends = [],
+        hs_libraries = [],
+        ld_options = [],
+        extra_libraries = [],
+        haddock_interfaces = [],
+        haddock_html = None,
+    )
+    for field in unfold_fields(lines):
+        key, value = field.split(":", 1)
+        value = value.strip()
+        if key == "name":
+            pkg.name = value
+        elif key == "version":
+            pkg.version = value
+        elif key == "id":
+            pkg.id = value
+        elif key == "include-dirs":
+            pkg.include_dirs += value.split()
+        elif key == "library-dirs":
+            pkg.library_dirs += value.split()
+        elif key == "dynamic-library-dirs":
+            pkg.dynamic_library_dirs += value.split()
+        elif key == "hs-libraries":
+            pkg.hs_libraries += value.split()
+        elif key == "depends":
+            pkg.depends += value.split()
+        elif key == "ld-options":
+            pkg.ld_options += [opt.strip('"') for opt in value.split()]
+        elif key == "extra-libraries":
+            pkg.extra_libraries += value.split()
+        elif key == "haddock-interfaces":
+            pkg.haddock_interfaces += value.split()
+        elif key == "haddock-html":
+            pkg.haddock_html = value
+
+    return PackageConfiguration(**pkg.__dict__)
+
+def split_records(lines):
+    """Iterator over lists of lines separated by `---`.
+
+    Skips empty records.
+    """
+    separator = "---"
+    record = []
+    for line in lines:
+        if line.rstrip() == separator:
+            yield record
+            record = []
+        else:
+            record.append(line)
+    if record:
+        yield record
+
+def parse_package_database_dump(lines):
+    """Parse the output of ghc-pkg dump.
+
+    Assumes that records are separated by `---`.
+
+    Returns:
+      Iterable of `PackageConfiguration`.
+    """
+    return (
+        parse_package_configuration(record)
+        for record in split_records(lines)
+    )

haskell/private/pkgdb_to_bzl.py

@@ -15,6 +15,8 @@
 import textwrap
 import types
 
+import package_configuration
+
 if len(sys.argv) == 3:
     repo_dir = "external/" + sys.argv[1]
     topdir = sys.argv[2]
@@ -93,51 +95,12 @@ def hs_library_pattern(name, mode = "static", profiling = False):
 pkg_id_map = []
 for conf in glob.glob(os.path.join(topdir, "package.conf.d", "*.conf")):
     with open(conf, 'r') as f:
-        content = f.read()
-    fields = unfold_fields(content)
-    pkg = types.SimpleNamespace(
-        include_dirs = [],
-        library_dirs = [],
-        dynamic_library_dirs = [],
-        depends = [],
-        hs_libraries = [],
-        ld_options = [],
-        extra_libraries = [],
-        haddock_interfaces = [],
-        haddock_html = None,
-    )
-    for field in fields:
-        key, value = field.split(":", 1)
-        value = value.strip()
-        if key == "name":
-            pkg.name = value
-        elif key == "version":
-            pkg.version = value
-        elif key == "id":
-            pkg.id = value
-        elif key == "include-dirs":
-            pkg.include_dirs += value.split()
-        elif key == "library-dirs":
-            pkg.library_dirs += value.split()
-        elif key == "dynamic-library-dirs":
-            pkg.dynamic_library_dirs += value.split()
-        elif key == "hs-libraries":
-            pkg.hs_libraries += value.split()
-        elif key == "depends":
-            pkg.depends += value.split()
-        elif key == "ld-options":
-            pkg.ld_options += [opt.strip('"') for opt in value.split()]
-        elif key == "extra-libraries":
-            pkg.extra_libraries += value.split()
-        elif key == "haddock-interfaces":
-            pkg.haddock_interfaces += value.split()
-        elif key == "haddock-html":
-            pkg.haddock_html = value
+        pkg = package_configuration.parse_package_configuration(f)
 
     # pkgroot is not part of .conf files. It's a computed value. It is
     # defined to be the directory enclosing the package database
     # directory.
-    pkg.pkgroot = os.path.dirname(os.path.dirname(os.path.realpath(conf)))
+    pkgroot = os.path.dirname(os.path.dirname(os.path.realpath(conf)))
 
     pkg_id_map.append((pkg.name, pkg.id))
 
@@ -157,7 +120,7 @@ def hs_library_pattern(name, mode = "static", profiling = False):
     # generate the database entry even if no haddock was generated.
     haddock_html = None
     if pkg.haddock_html:
-        haddock_html = path_to_label(pkg.haddock_html, pkg.pkgroot)
+        haddock_html = path_to_label(pkg.haddock_html, pkgroot)
         if not haddock_html:
             haddock_html = os.path.join("haddock", "html", pkg.name)
             output.append("#SYMLINK: {} {}".format(pkg.haddock_html, haddock_html))
@@ -166,7 +129,7 @@ def hs_library_pattern(name, mode = "static", profiling = False):
     interface_id = 0
     haddock_interfaces = []
     for interface_path in pkg.haddock_interfaces:
-        interface = path_to_label(interface_path, pkg.pkgroot)
+        interface = path_to_label(interface_path, pkgroot)
         if not interface:
             interface = os.path.join(
                 "haddock",
@@ -202,43 +165,43 @@ def hs_library_pattern(name, mode = "static", profiling = False):
                 id = pkg.id,
                 version = pkg.version,
                 hdrs = "glob({}, allow_empty = True)".format([
-                    path_to_label("{}/**/*.h".format(include_dir), pkg.pkgroot)
+                    path_to_label("{}/**/*.h".format(include_dir), pkgroot)
                     for include_dir in pkg.include_dirs
-                    if path_to_label(include_dir, pkg.pkgroot)
+                    if path_to_label(include_dir, pkgroot)
                 ]),
                 includes = [
-                    "/".join([repo_dir, path_to_label(include_dir, pkg.pkgroot)])
+                    "/".join([repo_dir, path_to_label(include_dir, pkgroot)])
                     for include_dir in pkg.include_dirs
-                    if path_to_label(include_dir, pkg.pkgroot)
+                    if path_to_label(include_dir, pkgroot)
                 ],
                 static_libraries = "glob({}, allow_empty = True)".format([
-                    path_to_label("{}/{}".format(library_dir, pattern), pkg.pkgroot)
+                    path_to_label("{}/{}".format(library_dir, pattern), pkgroot)
                     for hs_library in pkg.hs_libraries
                     for pattern in hs_library_pattern(hs_library, mode = "static", profiling = False)
                     for library_dir in pkg.library_dirs
-                    if path_to_label(library_dir, pkg.pkgroot)
+                    if path_to_label(library_dir, pkgroot)
                 ]),
                 static_profiling_libraries = "glob({}, allow_empty = True)".format([
-                    path_to_label("{}/{}".format(library_dir, pattern), pkg.pkgroot)
+                    path_to_label("{}/{}".format(library_dir, pattern), pkgroot)
                     for hs_library in pkg.hs_libraries
                     for pattern in hs_library_pattern(hs_library, mode = "static", profiling = True)
                     for library_dir in pkg.library_dirs
-                    if path_to_label(library_dir, pkg.pkgroot)
+                    if path_to_label(library_dir, pkgroot)
                 ]),
                 shared_libraries = "glob({}, allow_empty = True)".format([
-                    path_to_label("{}/{}".format(dynamic_library_dir, pattern), pkg.pkgroot)
+                    path_to_label("{}/{}".format(dynamic_library_dir, pattern), pkgroot)
                     for hs_library in pkg.hs_libraries
                     for pattern in hs_library_pattern(hs_library, mode = "dynamic", profiling = False)
                     for dynamic_library_dir in pkg.dynamic_library_dirs + pkg.library_dirs
-                    if path_to_label(dynamic_library_dir, pkg.pkgroot)
+                    if path_to_label(dynamic_library_dir, pkgroot)
                 ]),
                 haddock_html = repr(haddock_html),
                 haddock_interfaces = repr(haddock_interfaces),
                 deps = pkg.depends,
                 linkopts = pkg.ld_options + [
                     "-L{}".format(library_dir)
                     for library_dir in pkg.dynamic_library_dirs + pkg.library_dirs
-                    if not path_to_label(library_dir, pkg.pkgroot)
+                    if not path_to_label(library_dir, pkgroot)
                 ] + [
                     "-l{}".format(extra_library)
                     for extra_library in pkg.extra_libraries

tests/package_configuration/BUILD.bazel

@@ -0,0 +1,7 @@
+load("@rules_python//python:defs.bzl", "py_test")
+
+py_test(
+    name = "package_configuration_test",
+    srcs = ["package_configuration_test.py"],
+    deps = ["//haskell:package_configuration"],
+)