Skip to content

Commit

Permalink
Add --output=files mode to cquery (bazelbuild#15979)
Browse files Browse the repository at this point in the history
With the new output mode `--output=files`, cquery lists all files advertised by the matched targets in the currently requested output groups.

This new mode has the following advantages over `--output=starlark` combined with an appropriate handcrafted `--starlark:expr`:
* provides a canonical answer to the very common "Where are my build outputs?" question
* is more friendly to new users as it doesn't require knowing about providers and non-BUILD dialect Starlark
* takes the value of `--output_groups` into account
* stays as close to the logic for build summaries printed by `bazel build` as possible

Fixes bazelbuild#8739

RELNOTES: `cquery`'s new output mode [`--output=files`](https://bazel.build/docs/cquery#files-output) lists the output files of the targets matching the query. It takes the current value of `--output_groups` into account.

Closes bazelbuild#15552.

PiperOrigin-RevId: 462630629
Change-Id: Ic648f22aa160ee57b476180561b444f08799ebb6
  • Loading branch information
fmeum authored Jul 26, 2022
1 parent 14c944a commit 04c373b
Show file tree
Hide file tree
Showing 17 changed files with 344 additions and 32 deletions.
20 changes: 20 additions & 0 deletions site/docs/cquery.html
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,26 @@ <h3>Graph output</h3>

</p>

<h3>Files output</h3>

<pre>
--output=files
</pre>

<p>
This option prints a list of the output files produced by each target matched
by the query similar to the list printed at the end of a <code>bazel build</code>
invocation. The output contains only the files advertised in the requested
output groups as determined by the
<a href="https://docs.bazel.build/version/main/reference/command-line-reference#flag--output_groups"><code>--output_groups</code></a>
flag and never contains source files.

Note: The output of <code>bazel cquery --output=files //pkg:foo</code> contains the output
files of <code>//pkg:foo</code> in <i>all</i> configurations that occur in the build (also
see the <a href="#target-pattern-evaluation">section on target pattern evaluation</a>. If that
is not desired, wrap you query in <a href="#config"><code>config(..., target)</code></a>.
</p>

<h3>Defining the output format using Starlark</h3>

<pre>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.analysis.configuredtargets.InputFileConfiguredTarget;
import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
import com.google.devtools.build.lib.analysis.test.TestProvider;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSet.Node;
Expand Down Expand Up @@ -236,6 +238,36 @@ static ArtifactsToBuild getAllArtifactsToBuild(
allOutputGroups.build(), /*allOutputGroupsImportant=*/ allOutputGroupsImportant);
}

/**
* Returns false if the build outputs provided by the target should never be shown to users.
*
* <p>Always returns false for hidden rules and source file targets.
*/
public static boolean shouldConsiderForDisplay(ConfiguredTarget configuredTarget) {
// TODO(bazel-team): this is quite ugly. Add a marker provider for this check.
if (configuredTarget instanceof InputFileConfiguredTarget) {
// Suppress display of source files (because we do no work to build them).
return false;
}
if (configuredTarget instanceof RuleConfiguredTarget) {
RuleConfiguredTarget ruleCt = (RuleConfiguredTarget) configuredTarget;
if (ruleCt.getRuleClassString().contains("$")) {
// Suppress display of hidden rules
return false;
}
}
return true;
}

/**
* Returns true if the given artifact should be shown to users as a build output.
*
* <p>Always returns false for middleman and source artifacts.
*/
public static boolean shouldDisplay(Artifact artifact) {
return !artifact.isSourceArtifact() && !artifact.isMiddlemanArtifact();
}

/**
* Recursive procedure filtering a target/aspect's declared {@code
* NestedSet<ArtifactsInOutputGroup>} and {@code NestedSet<Artifact>} to only include {@link
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,7 @@
import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper;
import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
import com.google.devtools.build.lib.analysis.configuredtargets.InputFileConfiguredTarget;
import com.google.devtools.build.lib.analysis.configuredtargets.OutputFileConfiguredTarget;
import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
Expand Down Expand Up @@ -140,7 +138,7 @@ void showBuildResult(
TopLevelArtifactHelper.getAllArtifactsToBuild(target, context)
.getImportantArtifacts()
.toList()) {
if (shouldPrint(artifact)) {
if (TopLevelArtifactHelper.shouldDisplay(artifact)) {
if (headerFlag) {
outErr.printErr("Target " + label + " up-to-date:\n");
headerFlag = false;
Expand Down Expand Up @@ -195,7 +193,7 @@ void showBuildResult(
outErr.printErr("Aspect " + aspectName + " of " + label + " up-to-date:\n");
headerFlag = false;
}
if (shouldPrint(importantArtifact)) {
if (TopLevelArtifactHelper.shouldDisplay(importantArtifact)) {
outErr.printErrLn(formatArtifactForShowResults(prettyPrinter, importantArtifact));
}
}
Expand All @@ -216,10 +214,6 @@ void showBuildResult(
}
}

private boolean shouldPrint(Artifact artifact) {
return !artifact.isSourceArtifact() && !artifact.isMiddlemanArtifact();
}

private String formatArtifactForShowResults(PathPrettyPrinter prettyPrinter, Artifact artifact) {
return " " + prettyPrinter.getPrettyPath(artifact.getPath().asFragment());
}
Expand Down Expand Up @@ -268,18 +262,9 @@ private Collection<ConfiguredTarget> filterTargetsToPrint(
Collection<ConfiguredTarget> configuredTargets) {
ImmutableList.Builder<ConfiguredTarget> result = ImmutableList.builder();
for (ConfiguredTarget configuredTarget : configuredTargets) {
// TODO(bazel-team): this is quite ugly. Add a marker provider for this check.
if (configuredTarget instanceof InputFileConfiguredTarget) {
// Suppress display of source files (because we do no work to build them).
if (!TopLevelArtifactHelper.shouldConsiderForDisplay(configuredTarget)) {
continue;
}
if (configuredTarget instanceof RuleConfiguredTarget) {
RuleConfiguredTarget ruleCt = (RuleConfiguredTarget) configuredTarget;
if (ruleCt.getRuleClassString().contains("$")) {
// Suppress display of hidden rules
continue;
}
}
if (configuredTarget instanceof OutputFileConfiguredTarget) {
// Suppress display of generated files (because they appear underneath
// their generating rule), EXCEPT those ones which are not part of the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ protected ConfiguredTargetQueryEnvironment getQueryEnvironment(
env.getRelativeWorkingDirectory(),
env.getPackageManager().getPackagePath(),
() -> walkableGraph,
cqueryOptions);
cqueryOptions,
request.getTopLevelArtifactContext());
}
}
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/query2/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/analysis:target_and_configuration",
"//src/main/java/com/google/devtools/build/lib/analysis:toolchain_collection",
"//src/main/java/com/google/devtools/build/lib/analysis:toolchain_context",
"//src/main/java/com/google/devtools/build/lib/analysis:top_level_artifact_context",
"//src/main/java/com/google/devtools/build/lib/buildeventstream",
"//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
"//src/main/java/com/google/devtools/build/lib/causes",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ class BuildOutputFormatterCallback extends CqueryThreadsafeCallback {
OutputStream out,
SkyframeExecutor skyframeExecutor,
TargetAccessor<KeyedConfiguredTarget> accessor) {
super(eventHandler, options, out, skyframeExecutor, accessor);
super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ false);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.google.common.util.concurrent.MoreExecutors;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.ConfiguredTargetValue;
import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.transitions.TransitionFactory;
import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
Expand Down Expand Up @@ -81,6 +82,8 @@ public class ConfiguredTargetQueryEnvironment

private CqueryOptions cqueryOptions;

private final TopLevelArtifactContext topLevelArtifactContext;

private final KeyExtractor<KeyedConfiguredTarget, ConfiguredTargetKey>
configuredTargetKeyExtractor;

Expand Down Expand Up @@ -119,7 +122,8 @@ public ConfiguredTargetQueryEnvironment(
PathFragment parserPrefix,
PathPackageLocator pkgPath,
Supplier<WalkableGraph> walkableGraphSupplier,
Set<Setting> settings)
Set<Setting> settings,
TopLevelArtifactContext topLevelArtifactContext)
throws InterruptedException {
super(
keepGoing,
Expand All @@ -135,6 +139,7 @@ public ConfiguredTargetQueryEnvironment(
this.configuredTargetKeyExtractor = KeyedConfiguredTarget::getConfiguredTargetKey;
this.transitiveConfigurations =
getTransitiveConfigurations(transitiveConfigurationKeys, walkableGraphSupplier.get());
this.topLevelArtifactContext = topLevelArtifactContext;
}

public ConfiguredTargetQueryEnvironment(
Expand All @@ -147,7 +152,8 @@ public ConfiguredTargetQueryEnvironment(
PathFragment parserPrefix,
PathPackageLocator pkgPath,
Supplier<WalkableGraph> walkableGraphSupplier,
CqueryOptions cqueryOptions)
CqueryOptions cqueryOptions,
TopLevelArtifactContext topLevelArtifactContext)
throws InterruptedException {
this(
keepGoing,
Expand All @@ -159,7 +165,8 @@ public ConfiguredTargetQueryEnvironment(
parserPrefix,
pkgPath,
walkableGraphSupplier,
cqueryOptions.toSettings());
cqueryOptions.toSettings(),
topLevelArtifactContext);
this.cqueryOptions = cqueryOptions;
}

Expand Down Expand Up @@ -270,7 +277,9 @@ private static ImmutableMap<String, BuildConfiguration> getTransitiveConfigurati
accessor,
kct -> getFwdDeps(ImmutableList.of(kct))),
new StarlarkOutputFormatterCallback(
eventHandler, cqueryOptions, out, skyframeExecutor, accessor));
eventHandler, cqueryOptions, out, skyframeExecutor, accessor),
new FilesOutputFormatterCallback(
eventHandler, cqueryOptions, out, skyframeExecutor, accessor, topLevelArtifactContext));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@


import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.query2.NamedThreadSafeOutputFormatterCallback;
Expand Down Expand Up @@ -55,14 +56,16 @@ public abstract class CqueryThreadsafeCallback
protected final ConfiguredTargetAccessor accessor;

private final List<String> result = new ArrayList<>();
private final boolean uniquifyResults;

@SuppressWarnings("DefaultCharset")
CqueryThreadsafeCallback(
ExtendedEventHandler eventHandler,
CqueryOptions options,
OutputStream out,
SkyframeExecutor skyframeExecutor,
TargetAccessor<KeyedConfiguredTarget> accessor) {
TargetAccessor<KeyedConfiguredTarget> accessor,
boolean uniquifyResults) {
this.eventHandler = eventHandler;
this.options = options;
if (out != null) {
Expand All @@ -72,6 +75,7 @@ public abstract class CqueryThreadsafeCallback
}
this.skyframeExecutor = skyframeExecutor;
this.accessor = (ConfiguredTargetAccessor) accessor;
this.uniquifyResults = uniquifyResults;
}

public void addResult(String string) {
Expand All @@ -86,7 +90,8 @@ public List<String> getResult() {
@Override
public void close(boolean failFast) throws InterruptedException, IOException {
if (!failFast && printStream != null) {
for (String s : result) {
List<String> resultsToPrint = uniquifyResults ? ImmutableSet.copyOf(result).asList() : result;
for (String s : resultsToPrint) {
// TODO(ulfjack): We should use queryOptions.getLineTerminator() instead.
printStream.append(s).append("\n");
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// Copyright 2022 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.query2.cquery;

import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.query2.engine.QueryEnvironment.TargetAccessor;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
import java.io.IOException;
import java.io.OutputStream;

/**
* Cquery output formatter that prints the set of output files advertised by the matched targets.
*/
public class FilesOutputFormatterCallback extends CqueryThreadsafeCallback {

private final TopLevelArtifactContext topLevelArtifactContext;

FilesOutputFormatterCallback(
ExtendedEventHandler eventHandler,
CqueryOptions options,
OutputStream out,
SkyframeExecutor skyframeExecutor,
TargetAccessor<KeyedConfiguredTarget> accessor,
TopLevelArtifactContext topLevelArtifactContext) {
// Different targets may provide the same artifact, so we deduplicate the collection of all
// results at the end.
super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ true);
this.topLevelArtifactContext = topLevelArtifactContext;
}

@Override
public String getName() {
return "files";
}

@Override
public void processOutput(Iterable<KeyedConfiguredTarget> partialResult)
throws IOException, InterruptedException {
for (KeyedConfiguredTarget keyedTarget : partialResult) {
ConfiguredTarget target = keyedTarget.getConfiguredTarget();
if (!TopLevelArtifactHelper.shouldConsiderForDisplay(target)) {
continue;
}
TopLevelArtifactHelper.getAllArtifactsToBuild(target, topLevelArtifactContext)
.getImportantArtifacts()
.toList()
.stream()
.filter(TopLevelArtifactHelper::shouldDisplay)
.map(Artifact::getExecPathString)
.forEach(this::addResult);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public Comparator<KeyedConfiguredTarget> comparator() {
SkyframeExecutor skyframeExecutor,
TargetAccessor<KeyedConfiguredTarget> accessor,
DepsRetriever depsRetriever) {
super(eventHandler, options, out, skyframeExecutor, accessor);
super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ false);
this.depsRetriever = depsRetriever;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public class LabelAndConfigurationOutputFormatterCallback extends CqueryThreadsa
SkyframeExecutor skyframeExecutor,
TargetAccessor<KeyedConfiguredTarget> accessor,
boolean showKind) {
super(eventHandler, options, out, skyframeExecutor, accessor);
super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ false);
this.showKind = showKind;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public String formatName() {
TargetAccessor<KeyedConfiguredTarget> accessor,
AspectResolver resolver,
OutputType outputType) {
super(eventHandler, options, out, skyframeExecutor, accessor);
super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ false);
this.outputType = outputType;
this.skyframeExecutor = skyframeExecutor;
this.resolver = resolver;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ public Object providers(ConfiguredTarget target) {
SkyframeExecutor skyframeExecutor,
TargetAccessor<KeyedConfiguredTarget> accessor)
throws QueryException, InterruptedException {
super(eventHandler, options, out, skyframeExecutor, accessor);
super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ false);

ParserInput input = null;
String exceptionMessagePrefix;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ public String getName() {
TargetAccessor<KeyedConfiguredTarget> accessor,
BuildConfiguration hostConfiguration,
@Nullable TransitionFactory<RuleTransitionData> trimmingTransitionFactory) {
super(eventHandler, options, out, skyframeExecutor, accessor);
super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ false);
this.hostConfiguration = hostConfiguration;
this.trimmingTransitionFactory = trimmingTransitionFactory;
this.partialResultMap = Maps.newHashMap();
Expand Down
Loading

0 comments on commit 04c373b

Please sign in to comment.