Move pyre BXL script to buck2-prelude

python/sourcedb/build.bxl

@@ -0,0 +1,32 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+def _get_artifact(result: "bxl_build_result") -> "artifact":
+    # NOTE: the first artifact is always the source db json
+    # T124989384 will make this nicer
+    for artifact in result.artifacts():
+        return artifact
+    fail("Sourcedb rule must have at least one artifact")
+
+# Build sourcedb for the given targets, and return a mapping from target names
+# to the corresponding sourcedb JSON file location.
+def do_build(
+        ctx: "bxl_ctx",
+        targets: ["configured_target_label"]) -> {"target_label": "artifact"}:
+    # Build sourcedbs of all targets
+    configured_sub_targets = [
+        configured_sub_target(target, ["source-db-no-deps"])
+        for target in targets
+    ]
+    build_results = ctx.build(configured_sub_targets)
+
+    # Compute result dict
+    output = {}
+    for key, result in build_results.items():
+        path = _get_artifact(result)
+        output[key.raw_target()] = path
+    return output

python/sourcedb/classic.bxl

@@ -0,0 +1,41 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load(":build.bxl", "do_build")
+load(":merge.bxl", "do_merge")
+load(":query.bxl", "do_query")
+
+def _build_entry_point(ctx: "bxl_ctx") -> None:
+    query = ctx.cquery()
+    actions = ctx.bxl_actions.action_factory()
+    targets = do_query(ctx, query, actions, [query.eval(target) for target in ctx.cli_args.target])
+    built_sourcedbs = do_build(ctx, targets)
+
+    merged_sourcedb = do_merge(
+        ctx,
+        actions,
+        built_sourcedbs,
+        merger_target = "prelude//python/tools/sourcedb_merger:legacy_merge",
+        command_category = "pyre_legacy_merge_sourcedb",
+    )
+    ctx.output.print_json({"db": merged_sourcedb.abs_path()})
+
+build = bxl(
+    doc = """Build Python sourcedb for Pyre classic type checking server.
+
+    It takes a list of target patterns (usually obtained from Pyre local configuration
+    file), and will build source-db for those targets.
+    """,
+    impl = _build_entry_point,
+    cli_args = {
+        "target": cli_args.list(
+            cli_args.string(
+                doc = "Target pattern to build a source db for",
+            ),
+        ),
+    },
+)

python/sourcedb/code_navigation.bxl

@@ -0,0 +1,44 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load(":build.bxl", "do_build")
+load(":merge.bxl", "do_merge")
+load(":query.bxl", "do_query")
+
+def _build_entry_point(ctx: "bxl_ctx") -> None:
+    query = ctx.cquery()
+    actions = ctx.bxl_actions.action_factory()
+    root = ctx.root()
+
+    sources = ["{}/{}".format(root, source) for source in ctx.cli_args.source]
+    targets = do_query(ctx, query, actions, query.owner(sources))
+    built_sourcedbs = do_build(ctx, targets)
+
+    merged_sourcedb = do_merge(
+        ctx,
+        actions,
+        built_sourcedbs,
+        merger_target = "prelude//python/tools/sourcedb_merger:merge",
+        command_category = "pyre_merge_sourcedb",
+    )
+    ctx.output.print_json({"db": merged_sourcedb.abs_path()})
+
+build = bxl(
+    doc = """Build Python sourcedb for Pyre code navigation server.
+
+    It takes a list of file paths, and will find the owner targets for all
+    those files and build source-db for those owning targets.
+    """,
+    impl = _build_entry_point,
+    cli_args = {
+        "source": cli_args.list(
+            cli_args.string(
+                doc = "File to build a source db for (relative to source root)",
+            ),
+        ),
+    },
+)

python/sourcedb/merge.bxl

@@ -0,0 +1,35 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+def do_merge(
+        ctx: "bxl_ctx",
+        actions: "actions",
+        built_sourcedbs: {"target_label": "artifact"},
+        merger_target: str.type,
+        command_category: str.type) -> "ensured_artifact":
+    merger_input = actions.write_json("merge_input.json", built_sourcedbs)
+    merger_output = actions.declare_output("merged_db.json")
+    merger = ctx.analysis(
+        merger_target,
+        target_platform = "prelude//platforms:default",
+    ).providers()[RunInfo]
+
+    # Ensure all artifacts so merger is guaranteed to see them
+    ctx.output.ensure_multiple(built_sourcedbs.values())
+
+    command = cmd_args(merger)
+    command.add(merger_input)
+    command.add("--root")
+    command.add(ctx.root())
+    command.add("--output")
+    command.add(merger_output.as_output())
+
+    # Declare that the merger result depends on all sourcedbs
+    command.hidden(built_sourcedbs.values())
+
+    actions.run(command, category = command_category)
+    return ctx.output.ensure(merger_output)

python/sourcedb/query.bxl

@@ -0,0 +1,100 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load("@prelude//python:python.bzl", "PythonLibraryInfo", "PythonLibraryManifestsTSet")
+
+BUCK_PYTHON_RULE_KINDS = [
+    "python_binary",
+    "python_library",
+    "python_test",
+]
+BUCK_PYTHON_RULE_KIND_QUERY = "|".join(BUCK_PYTHON_RULE_KINDS)
+
+def _filter_root_targets(
+        query: "cqueryctx",
+        target_patterns: "_iterable_of_target_pattern") -> "target_set":
+    # Find all Pure-Python targets
+    candidate_targets = target_set()
+    for pattern in target_patterns:
+        candidate_targets += query.kind(
+            BUCK_PYTHON_RULE_KIND_QUERY,
+            pattern,
+        )
+
+    # Don't check generated rules
+    filtered_targets = candidate_targets - query.attrfilter(
+        "labels",
+        "generated",
+        candidate_targets,
+    )
+
+    # Do include unittest sources, which are marked as generated
+    filtered_targets = filtered_targets + query.attrfilter(
+        "labels",
+        "unittest-library",
+        candidate_targets,
+    )
+
+    # Provide an opt-out label
+    filtered_targets = filtered_targets - query.attrfilter(
+        "labels",
+        "no_pyre",
+        candidate_targets,
+    )
+    return filtered_targets
+
+def _get_python_library_manifest(
+        ctx: "bxl_ctx",
+        target: "target_node") -> [PythonLibraryManifestsTSet.type, None]:
+    providers = ctx.analysis(target).providers()
+    sub_target = providers[DefaultInfo].sub_targets.get("source-db-no-deps")
+    if sub_target == None:
+        return None
+    python_library_info = sub_target.get(PythonLibraryInfo)
+    if python_library_info == None:
+        return None
+    return python_library_info.manifests
+
+def _expand_and_filter_dependencies(
+        ctx: "bxl_ctx",
+        actions: "actions",
+        root_targets: "target_set") -> ["configured_target_label"]:
+    manifests_of_transitive_dependencies = actions.tset(
+        PythonLibraryManifestsTSet,
+        children = filter(None, [
+            _get_python_library_manifest(ctx, target)
+            for target in root_targets
+        ]),
+    )
+    return [
+        manifest.label.configured_target()
+        for manifest in manifests_of_transitive_dependencies.traverse()
+        if manifest.src_types != None
+    ]
+
+def do_query(
+        ctx: "bxl_ctx",
+        query: "cqueryctx",
+        actions: "actions",
+        target_patterns: "_iterable_of_target_pattern") -> ["configured_target_label"]:
+    root_targets = _filter_root_targets(query, target_patterns)
+    return _expand_and_filter_dependencies(ctx, actions, root_targets)
+
+def _do_query_entry_point(ctx: "bxl_ctx") -> None:
+    targets = do_query(ctx, ctx.cquery(), ctx.bxl_actions.action_factory(), ctx.cli_args.target)
+    ctx.output.print_json([target.raw_target() for target in targets])
+
+query = bxl(
+    doc = (
+        "Expand target patterns and look for all targets in their transitive" +
+        "dependencies that will be built by Pyre."
+    ),
+    impl = _do_query_entry_point,
+    cli_args = {
+        "target": cli_args.list(cli_args.target_expr()),
+    },
+)

python/tools/sourcedb_merger/TARGETS.v2

@@ -0,0 +1,38 @@
+prelude = native
+
+prelude.python_bootstrap_library(
+    name = "library-internal",
+    srcs = [
+        "inputs.py",
+        "legacy_outputs.py",
+        "outputs.py",
+    ],
+)
+
+prelude.python_library(
+    name = "library",
+    srcs = [
+        "inputs.py",
+        "legacy_outputs.py",
+        "outputs.py",
+    ],
+    visibility = ["PUBLIC"],
+)
+
+prelude.python_bootstrap_binary(
+    name = "merge",
+    main = "merge.py",
+    deps = [
+        ":library-internal",
+    ],
+    visibility = ["PUBLIC"],
+)
+
+prelude.python_bootstrap_binary(
+    name = "legacy_merge",
+    main = "legacy_merge.py",
+    deps = [
+        ":library-internal",
+    ],
+    visibility = ["PUBLIC"],
+)

python/tools/sourcedb_merger/inputs.py

@@ -0,0 +1,87 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+import dataclasses
+import json
+import pathlib
+from typing import Dict, Iterable, Mapping
+
+
+class BuildMapLoadError(Exception):
+    pass
+
+
+@dataclasses.dataclass(frozen=True)
+class Target:
+    name: str
+
+
+@dataclasses.dataclass(frozen=True)
+class PartialBuildMap:
+    content: Mapping[str, str] = dataclasses.field(default_factory=dict)
+
+    @staticmethod
+    def load_from_json(input_json: object) -> "PartialBuildMap":
+        if not isinstance(input_json, dict):
+            raise BuildMapLoadError(
+                "Input JSON for build map should be a dict."
+                f"Got {type(input_json)} instead"
+            )
+        result: Dict[str, str] = {}
+        for key, value in input_json.items():
+            if not isinstance(key, str):
+                raise BuildMapLoadError(
+                    f"Build map keys are expected to be strings. Got `{key}`."
+                )
+            if not isinstance(value, str):
+                raise BuildMapLoadError(
+                    f"Build map values are expected to be strings. Got `{value}`."
+                )
+            if pathlib.Path(key).suffix not in (".py", ".pyi"):
+                continue
+            result[key] = value
+        return PartialBuildMap(result)
+
+    @staticmethod
+    def load_from_path(input_path: pathlib.Path) -> "PartialBuildMap":
+        with open(input_path, "r") as input_file:
+            return PartialBuildMap.load_from_json(json.load(input_file))
+
+
+@dataclasses.dataclass(frozen=True)
+class TargetEntry:
+    target: Target
+    build_map: PartialBuildMap
+
+
+def load_targets_and_build_maps_from_json(
+    buck_root: pathlib.Path, input_json: object
+) -> Iterable[TargetEntry]:
+    if not isinstance(input_json, dict):
+        raise BuildMapLoadError(
+            f"Input JSON should be a dict. Got {type(input_json)} instead"
+        )
+    for key, value in input_json.items():
+        if not isinstance(key, str):
+            raise BuildMapLoadError(
+                f"Target keys are expected to be strings. Got `{key}`."
+            )
+        if not isinstance(value, str):
+            raise BuildMapLoadError(
+                f"Sourcedb file paths are expected to be strings. Got `{value}`."
+            )
+        yield TargetEntry(
+            target=Target(key),
+            build_map=PartialBuildMap.load_from_path(buck_root / value),
+        )
+
+
+def load_targets_and_build_maps_from_path(
+    buck_root: pathlib.Path, input_path: str
+) -> Iterable[TargetEntry]:
+    with open(buck_root / input_path, "r") as input_file:
+        return load_targets_and_build_maps_from_json(buck_root, json.load(input_file))