Add rule class definitions to query proto output

As rules are starlarkified and migrated out of Bazel, tools cannot rely any
longer on the (undocumented and deprecated) `info build-language` output to find
the definitions of rule classes. Moreover, the name of the rule class is no
longer unique: two different .bzl files might define different rule classes with
the same name - and both might be used in the same package.

For starlarkified rules, tools could use `starlark_doc_extract` targets to get
the rule class definition in proto form. However, this requires
(1) knowing in advance which .bzl file to introspect
(2) adding a `starlark_doc_extract` target to perform this introspection.

This is obviously not ideal when a tool needs to introspect targets without
changing the state of a repo.

The solution is to add `starlark_doc_extract`-like rule class definitions to
`query` command output. Since a RuleInfo proto (the rule class definition as
output by starlark_doc_extract and Stardoc) is large and expensive to generate,
we want to
* only output it optionally, when --proto:rule_classes flag is set
* only output each rule class definition once per output stream - in the first
  rule target having a given rule class key.
* add a rule class key, unique per rule class definition (not just rule class
  name!), so that later rule targets in the stream with the same rule class
  definition can be connected with the rule class definition provided in the first
  target.

RELNOTES: If --proto:rule_classes flag is enabled, query proto output will contain rule class definitions in Stardoc proto format.
PiperOrigin-RevId: 680642590
Change-Id: Ibdeb9c8acb7368b510f62d945c361a86cdb6f447

src/main/java/com/google/devtools/build/lib/query2/common/CommonQueryOptions.java

@@ -306,6 +306,18 @@ public LabelPrinter getLabelPrinter(
               + "that the attribute came from (empty string if it did not).")
   public boolean protoIncludeAttributeSourceAspects;
 
+  @Option(
+      name = "proto:rule_classes",
+      defaultValue = "false",
+      documentationCategory = OptionDocumentationCategory.QUERY,
+      effectTags = {OptionEffectTag.TERMINAL_OUTPUT},
+      help =
+          "Populate the rule_class_key field of each rule; and for the first rule with a given"
+              + " rule_class_key, also populate its rule_class_info proto field. The rule_class_key"
+              + " field uniquely identifies a rule class, and the rule_class_info field is a"
+              + " Stardoc-format rule class API definition.")
+  public boolean protoRuleClasses;
+
   /** An enum converter for {@code AspectResolver.Mode} . Should be used internally only. */
   public static class AspectResolutionModeConverter extends EnumConverter<Mode> {
     public AspectResolutionModeConverter() {

src/main/java/com/google/devtools/build/lib/query2/query/output/BUILD

@@ -28,6 +28,8 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/query2/compat:fake-load-target",
         "//src/main/java/com/google/devtools/build/lib/query2/engine",
         "//src/main/java/com/google/devtools/build/lib/query2/query/aspectresolvers",
+        "//src/main/java/com/google/devtools/build/lib/starlarkdocextract:labelrenderer",
+        "//src/main/java/com/google/devtools/build/lib/starlarkdocextract:ruleinfoextractor",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/com/google/devtools/common/options",

src/main/java/com/google/devtools/build/lib/query2/query/output/ProtoOutputFormatter.java

@@ -61,13 +61,17 @@
 import com.google.devtools.build.lib.query2.proto.proto2api.Build.QueryResult;
 import com.google.devtools.build.lib.query2.proto.proto2api.Build.SourceFile;
 import com.google.devtools.build.lib.query2.query.aspectresolvers.AspectResolver;
+import com.google.devtools.build.lib.starlarkdocextract.ExtractorContext;
+import com.google.devtools.build.lib.starlarkdocextract.LabelRenderer;
+import com.google.devtools.build.lib.starlarkdocextract.RuleInfoExtractor;
 import com.google.protobuf.CodedOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -114,6 +118,9 @@ public class ProtoOutputFormatter extends AbstractUnorderedFormatter {
   protected boolean includeAttributeSourceAspects = false;
   private HashFunction hashFunction = null;
 
+  /** Non-null if and only if --proto:rule_classes option is set. */
+  @Nullable private RuleClassInfoFormatter ruleClassInfoFormatter;
+
   @Nullable private EventHandler eventHandler;
 
   @Override
@@ -139,6 +146,7 @@ public void setOptions(
     this.includeDefinitionStack = options.protoIncludeDefinitionStack;
     this.includeAttributeSourceAspects = options.protoIncludeAttributeSourceAspects;
     this.hashFunction = hashFunction;
+    this.ruleClassInfoFormatter = options.protoRuleClasses ? new RuleClassInfoFormatter() : null;
   }
 
   @Override
@@ -272,6 +280,10 @@ public Build.Target toTargetProtoBuffer(
               FormatUtils.getRootRelativeLocation(fr.location, rule.getPackage()) + ": " + fr.name);
         }
       }
+
+      if (ruleClassInfoFormatter != null) {
+        ruleClassInfoFormatter.addRuleClassKeyAndInfoIfNeeded(rulePb, rule);
+      }
       targetPb.setType(RULE);
       targetPb.setRule(rulePb);
     } else if (target instanceof OutputFile outputFile) {
@@ -541,6 +553,33 @@ private static Object getFlattenedAttributeValues(Type<?> attrType, Rule rule, A
     throw new AssertionError("Unknown type: " + attrType);
   }
 
+  private static class RuleClassInfoFormatter {
+    private final HashSet<String> ruleClassKeys = new HashSet<>();
+    private final ExtractorContext extractorContext =
+        ExtractorContext.builder()
+            .labelRenderer(LabelRenderer.DEFAULT)
+            .extractNonStarlarkAttrs(true)
+            .build();
+
+    /**
+     * Sets the rule_class_key field, and if the rule class key has not been seen before, also sets
+     * the rule_class_info field.
+     */
+    public void addRuleClassKeyAndInfoIfNeeded(Build.Rule.Builder rulePb, Rule rule) {
+      String ruleClassKey = rule.getRuleClassObject().getKey();
+      rulePb.setRuleClassKey(ruleClassKey);
+      if (ruleClassKeys.add(ruleClassKey)) {
+        // TODO(b/368091415): instead of rule.getRuleClass(), we should be using the rule's public
+        // name. But to find the public name, we would need access to the globals dictionary of
+        // the compiled Starlark module in which the rule class was defined, which would have to
+        // be retrieved from skyframe.
+        rulePb.setRuleClassInfo(
+            RuleInfoExtractor.buildRuleInfo(
+                extractorContext, rule.getRuleClass(), rule.getRuleClassObject()));
+      }
+    }
+  }
+
   /**
    * Specialized {@link OutputFormatterCallback} implementation which produces a valid {@link
    * QueryResult} in streaming fashion. Internally this class makes some reasonably sound and stable

src/main/protobuf/BUILD

@@ -10,7 +10,6 @@ FILES = [
     "action_cache",
     "android_deploy_info",
     "bazel_flags",
-    "build",
     "builtin",
     "crash_debugging",
     "crosstool_config",
@@ -41,6 +40,22 @@ FILES = [
     deps = [":" + s + "_java_proto"],
 ) for s in FILES]
 
+proto_library(
+    name = "build_proto",
+    srcs = ["build.proto"],
+    deps = [":stardoc_output_proto"],
+)
+
+java_proto_library(
+    name = "build_java_proto",
+    deps = [":build_proto"],
+)
+
+java_library_srcs(
+    name = "build_java_proto_srcs",
+    deps = [":build_java_proto"],
+)
+
 proto_library(
     name = "analysis_v2_proto",
     srcs = ["analysis_v2.proto"],
@@ -411,6 +426,7 @@ filegroup(
         ":bazel_output_service_java_grpc_srcs",
         ":bazel_output_service_java_proto_srcs",
         ":bazel_output_service_rev2_java_proto_srcs",
+        ":build_java_proto_srcs",
         ":cache_salt_java_proto_srcs",
         ":command_line_java_proto_srcs",
         ":command_server_java_grpc_srcs",

src/main/protobuf/build.proto

@@ -19,6 +19,8 @@ syntax = "proto2";
 
 package blaze_query;
 
+import "src/main/protobuf/stardoc_output.proto";
+
 // option cc_api_version = 2;
 // option java_api_version = 1;
 option java_package = "com.google.devtools.build.lib.query2.proto.proto2api";
@@ -286,7 +288,11 @@ message Rule {
   // The name of the rule (formatted as an absolute label, e.g. //foo/bar:baz).
   required string name = 1;
 
-  // The rule class (e.g., java_library)
+  // The rule class name (e.g., java_library).
+  //
+  // Note that the rule class name may not uniquely identify a rule class, since
+  // two different .bzl files may define different rule classes with the same
+  // name. To uniquely identify the rule class, see rule_class_key field below.
   required string rule_class = 2;
 
   // The BUILD file and line number of the location (formatted as
@@ -336,6 +342,23 @@ message Rule {
   // enabled on the command line with the --proto:definition_stack flag or the
   // rule is a native one.
   repeated string definition_stack = 14;
+
+  // A key uniquely identifying the rule's rule class. Stable between repeated
+  // blaze query invocations (assuming that there are no changes to Starlark
+  // files and the same blaze binary is invoked with the same options).
+  //
+  // Requires --proto:rule_classes=true
+  optional string rule_class_key = 16;
+
+  // Stardoc-format rule class API definition for this rule. Includes both
+  // Starlark-defined and native (including inherited) attributes; does not
+  // include hidden or explicitly undocumented attributes.
+  //
+  // Populated only for the first rule in the stream with a given
+  // rule_class_key.
+  //
+  // Requires --proto:rule_classes=true
+  optional stardoc_output.RuleInfo rule_class_info = 17;
 }
 
 // Direct dependencies of a rule in <depLabel, depConfiguration> form.

src/main/protobuf/stardoc_output.proto

@@ -75,8 +75,12 @@ enum AttributeType {
 
 // Representation of a Starlark rule definition.
 message RuleInfo {
-  // The name under which the rule is made accessible to a user of this module,
-  // including any structs it is nested in, for example "foo.foo_library".
+  // In Stardoc and starlark_doc_extract output, this is the name under which
+  // the rule is made accessible to a user of this module, including any structs
+  // it is nested in, for example "foo.foo_library".
+  //
+  // In query output, this is the name under which the rule was defined (which
+  // might be a private symbol prefixed with "_").
   string rule_name = 1;
 
   // The documentation string of the rule.