diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 4a272e29ecdd89..008242a1ae063f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -54,6 +54,7 @@ import com.google.devtools.build.lib.bazel.commands.FetchCommand; import com.google.devtools.build.lib.bazel.commands.ModCommand; import com.google.devtools.build.lib.bazel.commands.SyncCommand; +import com.google.devtools.build.lib.bazel.commands.VendorCommand; import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformFunction; import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformRule; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; @@ -154,6 +155,8 @@ public class BazelRepositoryModule extends BlazeModule { private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING; private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR; private LockfileMode bazelLockfileMode = LockfileMode.UPDATE; + + private Optional vendorDirectory; private List allowedYankedVersions = ImmutableList.of(); private SingleExtensionEvalFunction singleExtensionEvalFunction; private final ExecutorService repoFetchingWorkerThreadPool = @@ -214,6 +217,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde builder.addCommands(new FetchCommand()); builder.addCommands(new ModCommand()); builder.addCommands(new SyncCommand()); + builder.addCommands(new VendorCommand()); builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache)); } @@ -502,6 +506,16 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { bazelLockfileMode = repoOptions.lockfileMode; allowedYankedVersions = repoOptions.allowedYankedVersions; + if (repoOptions.vendorDirectory != null) { + vendorDirectory = + Optional.of( + repoOptions.vendorDirectory.isAbsolute() + ? filesystem.getPath(repoOptions.vendorDirectory) + : env.getWorkspace().getRelative(repoOptions.vendorDirectory)); + } else { + vendorDirectory = Optional.empty(); + } + if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) { registries = repoOptions.registries; } else { @@ -573,6 +587,8 @@ public ImmutableList getPrecomputedValues() { PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, bazelCompatibilityMode), PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, bazelLockfileMode), + PrecomputedValue.injected(RepositoryDelegatorFunction.IS_VENDOR_COMMAND, false), + PrecomputedValue.injected(RepositoryDelegatorFunction.VENDOR_DIRECTORY, vendorDirectory), PrecomputedValue.injected( YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, allowedYankedVersions)); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllFunction.java index 7f7f77a79020f8..40dfddf9e5ab50 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllFunction.java @@ -18,6 +18,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryFunction; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -31,8 +32,8 @@ import javax.annotation.Nullable; /** - * Void function designed to gather and fetch all the repositories without returning any specific - * result (empty value is returned). + * Gather and fetch all the repositories from MODULE.bazel resolution and extensions evaluation. If + * this is fetch configure, only configure repos will be fetched and returned */ public class BazelFetchAllFunction implements SkyFunction { @@ -53,14 +54,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept .filter(repo -> !repo.isMain()) .collect(toImmutableList())); - // 2. Run every extension found in the modules + // 2. Run every extension found in the modules & collect its generated repos ImmutableSet extensionIds = depGraphValue.getExtensionUsagesTable().rowKeySet(); ImmutableSet singleEvalKeys = extensionIds.stream().map(SingleExtensionEvalValue::key).collect(toImmutableSet()); SkyframeLookupResult singleEvalValues = env.getValuesAndExceptions(singleEvalKeys); - - // 3. For each extension, collect its generated repos for (SkyKey singleEvalKey : singleEvalKeys) { SingleExtensionEvalValue singleEvalValue = (SingleExtensionEvalValue) singleEvalValues.get(singleEvalKey); @@ -70,7 +69,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept reposToFetch.addAll(singleEvalValue.getCanonicalRepoNameToInternalNames().keySet()); } - // 4. If this is fetch configure, get repo rules and only collect repos marked as configure + // 3. If this is fetch configure, get repo rules and only collect repos marked as configure Boolean fetchConfigure = (Boolean) skyKey.argument(); if (fetchConfigure) { ImmutableSet repoRuleKeys = @@ -83,12 +82,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept return null; } if (StarlarkRepositoryFunction.isConfigureRule(repoRuleValue.getRule())) { - reposToFetch.add(RepositoryName.createUnvalidated(repoRuleValue.getRule().getName())); + reposToFetch.add((RepositoryName) repoRuleKey.argument()); } } } - // 5. Fetch all the collected repos + // 4. Fetch all the collected repos + List shouldVendor = new ArrayList<>(); ImmutableSet repoDelegatorKeys = reposToFetch.stream().map(RepositoryDirectoryValue::key).collect(toImmutableSet()); SkyframeLookupResult repoDirValues = env.getValuesAndExceptions(repoDelegatorKeys); @@ -98,9 +98,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept if (repoDirValue == null) { return null; } + if (!repoDirValue.excludeFromVendoring()) { + shouldVendor.add((RepositoryName) repoDelegatorKey.argument()); + } } - return BazelFetchAllValue.create(); + return BazelFetchAllValue.create( + ImmutableList.copyOf(reposToFetch), ImmutableList.copyOf(shouldVendor)); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllValue.java index 527e78eafa93ba..7e2831598c2bba 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllValue.java @@ -16,6 +16,8 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.skyframe.AbstractSkyKey; @@ -35,8 +37,13 @@ public static BazelFetchAllValue.Key key(Boolean configureEnabled) { return BazelFetchAllValue.Key.create(configureEnabled); } - public static BazelFetchAllValue create() { - return new AutoValue_BazelFetchAllValue(); + public abstract ImmutableList getFetchedRepos(); + + public abstract ImmutableList getReposToVendor(); + + public static BazelFetchAllValue create( + ImmutableList fetchedRepos, ImmutableList reposToVendor) { + return new AutoValue_BazelFetchAllValue(fetchedRepos, reposToVendor); } /** Key type for BazelFetchAllValue. */ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/InvalidArgumentException.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/InvalidArgumentException.java index ad7ea185396109..889b801de357e4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/InvalidArgumentException.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/InvalidArgumentException.java @@ -34,6 +34,10 @@ public InvalidArgumentException(String message, Code code) { this.code = code; } + public InvalidArgumentException(String message) { + this(message, Code.INVALID_ARGUMENTS); + } + public Code getCode() { return code; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD index 28cd553a3019c5..0ace636128ad20 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD @@ -20,6 +20,7 @@ java_library( "fetch.txt", "mod.txt", "sync.txt", + "vendor.txt", ], deps = [ "//src/main/java/com/google/devtools/build/lib:keep-going-option", @@ -66,6 +67,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/util:interrupted_failure_details", "//src/main/java/com/google/devtools/build/lib/util:maybe_complete_set", "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//src/main/java/com/google/devtools/common/options", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java index d3a79e668c5453..d745e80fb3ded8 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java @@ -18,11 +18,12 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.analysis.NoBuildEvent; import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent; import com.google.devtools.build.lib.bazel.bzlmod.BazelFetchAllValue; -import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.bazel.commands.RepositoryFetcher.RepositoryFetcherException; import com.google.devtools.build.lib.cmdline.RepositoryMapping; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.cmdline.TargetPattern; @@ -62,13 +63,11 @@ import com.google.devtools.build.lib.util.InterruptedFailureDetails; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; -import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsParsingResult; import java.io.IOException; import java.util.EnumSet; import java.util.List; -import net.starlark.java.eval.EvalException; /** Fetches external repositories. Which is so fetch. */ @Command( @@ -94,7 +93,7 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti PackageOptions pkgOptions = options.getOptions(PackageOptions.class); if (!pkgOptions.fetch) { return createFailedBlazeCommandResult( - env.getReporter(), Code.OPTIONS_INVALID, "You cannot run fetch with --fetch=false"); + env.getReporter(), Code.OPTIONS_INVALID, "You cannot run fetch with --nofetch"); } FetchOptions fetchOptions = options.getOptions(FetchOptions.class); int optionsCount = @@ -129,12 +128,21 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti PrecomputedValue.injected( RepositoryDelegatorFunction.FORCE_FETCH, env.getCommandId().toString()))); } - if (fetchOptions.all || fetchOptions.configure) { + try { + env.syncPackageLoading(options); + if (fetchOptions.all || fetchOptions.configure) { result = fetchAll(env, options, threadsOption, fetchOptions.configure); - } else if (!fetchOptions.repos.isEmpty()) { - result = fetchRepo(env, options, threadsOption, fetchOptions.repos); - } else { - result = fetchTarget(env, options, threadsOption); + } else if (!fetchOptions.repos.isEmpty()) { + result = fetchRepos(env, threadsOption, fetchOptions.repos); + } else { + result = fetchTarget(env, options, threadsOption); + } + } catch (AbruptExitException e) { + return createFailedBlazeCommandResult( + env.getReporter(), e.getMessage(), e.getDetailedExitCode()); + } catch (InterruptedException e) { + return createFailedBlazeCommandResult( + env.getReporter(), "Fetch interrupted: " + e.getMessage()); } env.getEventBus() .post( @@ -147,7 +155,8 @@ private BlazeCommandResult fetchAll( CommandEnvironment env, OptionsParsingResult options, LoadingPhaseThreadsOption threadsOption, - boolean configureEnabled) { + boolean configureEnabled) + throws InterruptedException { if (!options.getOptions(BuildLanguageOptions.class).enableBzlmod) { return createFailedBlazeCommandResult( env.getReporter(), @@ -161,86 +170,45 @@ private BlazeCommandResult fetchAll( .setEventHandler(env.getReporter()) .build(); - try { - env.syncPackageLoading(options); - EvaluationResult evaluationResult = - skyframeExecutor.prepareAndGet( - ImmutableSet.of(BazelFetchAllValue.key(configureEnabled)), evaluationContext); - if (evaluationResult.hasError()) { - Exception e = evaluationResult.getError().getException(); - return createFailedBlazeCommandResult( - env.getReporter(), - e != null ? e.getMessage() : "Unexpected error during fetching all external deps."); - } - // Everything is fetched successfully! - return BlazeCommandResult.success(); - } catch (AbruptExitException e) { - return createFailedBlazeCommandResult( - env.getReporter(), "Unknown error:: " + e.getMessage(), e.getDetailedExitCode()); - } catch (InterruptedException e) { + EvaluationResult evaluationResult = + skyframeExecutor.prepareAndGet( + ImmutableSet.of(BazelFetchAllValue.key(configureEnabled)), evaluationContext); + if (evaluationResult.hasError()) { + Exception e = evaluationResult.getError().getException(); return createFailedBlazeCommandResult( - env.getReporter(), "Fetch interrupted: " + e.getMessage()); + env.getReporter(), + e != null ? e.getMessage() : "Unexpected error during fetching all external deps."); } + return BlazeCommandResult.success(); } - private BlazeCommandResult fetchRepo( - CommandEnvironment env, - OptionsParsingResult options, - LoadingPhaseThreadsOption threadsOption, - List repos) { - SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor(); - EvaluationContext evaluationContext = - EvaluationContext.newBuilder() - .setParallelism(threadsOption.threads) - .setEventHandler(env.getReporter()) - .build(); + private BlazeCommandResult fetchRepos( + CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, List repos) + throws InterruptedException { try { - env.syncPackageLoading(options); - ImmutableSet.Builder repoDelegatorKeys = ImmutableSet.builder(); - for (String repo : repos) { - RepositoryName repoName = getRepositoryName(env, threadsOption, repo); - repoDelegatorKeys.add(RepositoryDirectoryValue.key(repoName)); - } - EvaluationResult evaluationResult = - skyframeExecutor.prepareAndGet(repoDelegatorKeys.build(), evaluationContext); - if (evaluationResult.hasError()) { - Exception e = evaluationResult.getError().getException(); - return createFailedBlazeCommandResult( - env.getReporter(), - e != null ? e.getMessage() : "Unexpected error during repository fetching."); - } + ImmutableMap repositoryNamesAndValues = + RepositoryFetcher.fetchRepos(repos, env, threadsOption); String notFoundRepos = - repoDelegatorKeys.build().stream() - .filter( - key -> !((RepositoryDirectoryValue) evaluationResult.get(key)).repositoryExists()) - .map(key -> ((RepositoryDirectoryValue) evaluationResult.get(key)).getErrorMsg()) + repositoryNamesAndValues.values().stream() + .filter(value -> !value.repositoryExists()) + .map(value -> value.getErrorMsg()) .collect(joining("; ")); if (!notFoundRepos.isEmpty()) { return createFailedBlazeCommandResult( env.getReporter(), "Fetching repos failed with errors: " + notFoundRepos); } - - // Everything has been fetched successfully! return BlazeCommandResult.success(); - } catch (AbruptExitException e) { - return createFailedBlazeCommandResult( - env.getReporter(), "Unknown error: " + e.getMessage(), e.getDetailedExitCode()); - } catch (InterruptedException e) { - return createFailedBlazeCommandResult( - env.getReporter(), "Fetch interrupted: " + e.getMessage()); - } catch (LabelSyntaxException | EvalException | IllegalArgumentException e) { - return createFailedBlazeCommandResult( - env.getReporter(), "Invalid repo name: " + e.getMessage()); } catch (RepositoryMappingResolutionException e) { return createFailedBlazeCommandResult( env.getReporter(), "Invalid repo name: " + e.getMessage(), e.getDetailedExitCode()); + } catch (RepositoryFetcherException e) { + return createFailedBlazeCommandResult(env.getReporter(), e.getMessage()); } } private BlazeCommandResult fetchTarget( - CommandEnvironment env, - OptionsParsingResult options, - LoadingPhaseThreadsOption threadsOption) { + CommandEnvironment env, OptionsParsingResult options, LoadingPhaseThreadsOption threadsOption) + throws InterruptedException { if (options.getResidue().isEmpty()) { return createFailedBlazeCommandResult( env.getReporter(), @@ -253,7 +221,6 @@ private BlazeCommandResult fetchTarget( boolean keepGoing = options.getOptions(KeepGoingOption.class).keepGoing; TargetPattern.Parser mainRepoTargetParser; try { - env.syncPackageLoading(options); RepositoryMapping repoMapping = env.getSkyframeExecutor() .getMainRepoMapping(keepGoing, threadsOption.threads, env.getReporter()); @@ -263,12 +230,6 @@ private BlazeCommandResult fetchTarget( } catch (RepositoryMappingResolutionException e) { return createFailedBlazeCommandResult( env.getReporter(), e.getMessage(), e.getDetailedExitCode()); - } catch (InterruptedException e) { - return createFailedBlazeCommandResult( - env.getReporter(), "Fetch interrupted: " + e.getMessage()); - } catch (AbruptExitException e) { - return createFailedBlazeCommandResult( - env.getReporter(), "Unknown error: " + e.getMessage(), e.getDetailedExitCode()); } // Querying for all of the dependencies of the targets has the side-effect of populating the @@ -363,29 +324,6 @@ public void processOutput(Iterable partialResult) { expr)); } - private RepositoryName getRepositoryName( - CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, String repoName) - throws EvalException, - LabelSyntaxException, - RepositoryMappingResolutionException, - InterruptedException { - if (repoName.startsWith("@@")) { // canonical RepoName - return RepositoryName.create(repoName.substring(2)); - } else if (repoName.startsWith("@")) { // apparent RepoName - RepositoryName.validateUserProvidedRepoName(repoName.substring(1)); - RepositoryMapping repoMapping = - env.getSkyframeExecutor() - .getMainRepoMapping( - env.getOptions().getOptions(KeepGoingOption.class).keepGoing, - threadsOption.threads, - env.getReporter()); - return repoMapping.get(repoName.substring(1)); - } else { - throw new IllegalArgumentException( - "The repo value has to be either apparent '@repo' or canonical '@@repo' repo name"); - } - } - private static BlazeCommandResult createFailedBlazeCommandResult( Reporter reporter, Code fetchCommandCode, String message) { return createFailedBlazeCommandResult( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/RepositoryFetcher.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/RepositoryFetcher.java new file mode 100644 index 00000000000000..498f27ab07811e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/RepositoryFetcher.java @@ -0,0 +1,138 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.commands; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.InvalidArgumentException; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.cmdline.RepositoryMapping; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue; +import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.KeepGoingOption; +import com.google.devtools.build.lib.runtime.LoadingPhaseThreadsOption; +import com.google.devtools.build.lib.skyframe.RepositoryMappingValue.RepositoryMappingResolutionException; +import com.google.devtools.build.skyframe.EvaluationContext; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import java.util.List; +import net.starlark.java.eval.EvalException; + +/** Fetches repositories for commands. */ +final class RepositoryFetcher { + + private final CommandEnvironment env; + private final LoadingPhaseThreadsOption threadsOption; + + private RepositoryFetcher( + CommandEnvironment env, + LoadingPhaseThreadsOption threadsOption) { + this.env = env; + this.threadsOption = threadsOption; + } + + static ImmutableMap fetchRepos( + List repos, + CommandEnvironment env, + LoadingPhaseThreadsOption threadsOption) + throws RepositoryMappingResolutionException, + InterruptedException, + RepositoryFetcherException { + return new RepositoryFetcher(env, threadsOption).fetchRepos(repos); + } + + private ImmutableMap fetchRepos(List repos) + throws InterruptedException, + RepositoryFetcherException, + RepositoryMappingResolutionException { + ImmutableSet reposnames = collectRepositoryNames(repos); + EvaluationResult evaluationResult = evaluateFetch(reposnames); + return reposnames.stream() + .collect( + toImmutableMap( + repoName -> repoName, + repoName -> + (RepositoryDirectoryValue) + evaluationResult.get(RepositoryDirectoryValue.key(repoName)))); + } + + private EvaluationResult evaluateFetch(ImmutableSet reposnames) + throws InterruptedException, RepositoryFetcherException { + EvaluationContext evaluationContext = + EvaluationContext.newBuilder() + .setParallelism(threadsOption.threads) + .setEventHandler(env.getReporter()) + .build(); + ImmutableSet repoDelegatorKeys = + reposnames.stream().map(RepositoryDirectoryValue::key).collect(toImmutableSet()); + EvaluationResult evaluationResult = + env.getSkyframeExecutor().prepareAndGet(repoDelegatorKeys, evaluationContext); + if (evaluationResult.hasError()) { + Exception e = evaluationResult.getError().getException(); + throw new RepositoryFetcherException( + e != null ? e.getMessage() : "Unexpected error during repository fetching."); + } + return evaluationResult; + } + + private ImmutableSet collectRepositoryNames(List repos) + throws InterruptedException, + RepositoryFetcherException, + RepositoryMappingResolutionException { + ImmutableSet.Builder reposnames = ImmutableSet.builder(); + for (String repo : repos) { + try { + reposnames.add(getRepositoryName(repo)); + } catch (LabelSyntaxException | EvalException | InvalidArgumentException e) { + throw new RepositoryFetcherException("Invalid repo name: " + e.getMessage()); + } + } + return reposnames.build(); + } + + private RepositoryName getRepositoryName(String repoName) + throws EvalException, + InterruptedException, + LabelSyntaxException, + InvalidArgumentException, + RepositoryMappingResolutionException { + if (repoName.startsWith("@@")) { // canonical RepoName + return RepositoryName.create(repoName.substring(2)); + } else if (repoName.startsWith("@")) { // apparent RepoName + RepositoryName.validateUserProvidedRepoName(repoName.substring(1)); + RepositoryMapping repoMapping = + env.getSkyframeExecutor() + .getMainRepoMapping( + env.getOptions().getOptions(KeepGoingOption.class).keepGoing, + threadsOption.threads, + env.getReporter()); + return repoMapping.get(repoName.substring(1)); + } else { + throw new InvalidArgumentException( + "The repo value has to be either apparent '@repo' or canonical '@@repo' repo name"); + } + } + + static class RepositoryFetcherException extends Exception { + public RepositoryFetcherException(String message) { + super(message); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java new file mode 100644 index 00000000000000..34835fcc90131a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java @@ -0,0 +1,311 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.commands; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.NoBuildEvent; +import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent; +import com.google.devtools.build.lib.bazel.bzlmod.BazelFetchAllValue; +import com.google.devtools.build.lib.bazel.commands.RepositoryFetcher.RepositoryFetcherException; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; +import com.google.devtools.build.lib.cmdline.LabelConstants; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; +import com.google.devtools.build.lib.pkgcache.PackageOptions; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; +import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandResult; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.KeepGoingOption; +import com.google.devtools.build.lib.runtime.LoadingPhaseThreadsOption; +import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; +import com.google.devtools.build.lib.server.FailureDetails.FetchCommand.Code; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.RepositoryMappingValue.RepositoryMappingResolutionException; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.DetailedExitCode; +import com.google.devtools.build.lib.util.InterruptedFailureDetails; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.skyframe.EvaluationContext; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.common.options.OptionsParsingResult; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Objects; +import javax.annotation.Nullable; + +/** Fetches external repositories into a specified directory. */ +@Command( + name = VendorCommand.NAME, + options = { + VendorOptions.class, + PackageOptions.class, + KeepGoingOption.class, + LoadingPhaseThreadsOption.class + }, + help = "resource:vendor.txt", + shortDescription = + "Fetches external repositories into a specific folder specified by the flag " + + "--vendor_dir.") +public final class VendorCommand implements BlazeCommand { + public static final String NAME = "vendor"; + + // TODO(salmasamy) decide on name and format + private static final String VENDOR_IGNORE = ".vendorignore"; + + @Override + public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult options) { + BlazeCommandResult invalidResult = validateOptions(env, options); + if (invalidResult != null) { + return invalidResult; + } + + env.getEventBus() + .post( + new NoBuildEvent( + env.getCommandName(), + env.getCommandStartTime(), + /* separateFinishedEvent= */ true, + /* showProgress= */ true, + env.getCommandId().toString())); + + // IS_VENDOR_COMMAND & VENDOR_DIR is already injected in "BazelRepositoryModule", we just need + // to update this value for the delegator function to recognize this call is from VendorCommand + env.getSkyframeExecutor() + .injectExtraPrecomputedValues( + ImmutableList.of( + PrecomputedValue.injected(RepositoryDelegatorFunction.IS_VENDOR_COMMAND, true))); + + BlazeCommandResult result; + VendorOptions vendorOptions = options.getOptions(VendorOptions.class); + PathFragment vendorDirectory = options.getOptions(RepositoryOptions.class).vendorDirectory; + LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); + try { + env.syncPackageLoading(options); + if (!vendorOptions.repos.isEmpty()) { + result = vendorRepos(env, threadsOption, vendorOptions.repos, vendorDirectory); + } else { + result = vendorAll(env, threadsOption, vendorDirectory); + } + } catch (AbruptExitException e) { + return createFailedBlazeCommandResult( + env.getReporter(), e.getMessage(), e.getDetailedExitCode()); + } catch (InterruptedException e) { + return createFailedBlazeCommandResult( + env.getReporter(), "Vendor interrupted: " + e.getMessage()); + } catch (IOException e) { + return createFailedBlazeCommandResult( + env.getReporter(), "Error while vendoring repos: " + e.getMessage()); + } + + env.getEventBus() + .post( + new NoBuildRequestFinishedEvent( + result.getExitCode(), env.getRuntime().getClock().currentTimeMillis())); + return result; + } + + @Nullable + private BlazeCommandResult validateOptions(CommandEnvironment env, OptionsParsingResult options) { + if (!options.getOptions(BuildLanguageOptions.class).enableBzlmod) { + return createFailedBlazeCommandResult( + env.getReporter(), + "Bzlmod has to be enabled for vendoring to work, run with --enable_bzlmod"); + } + if (options.getOptions(RepositoryOptions.class).vendorDirectory == null) { + return createFailedBlazeCommandResult( + env.getReporter(), + Code.OPTIONS_INVALID, + "You cannot run vendor without specifying --vendor_dir"); + } + if (!options.getOptions(PackageOptions.class).fetch) { + return createFailedBlazeCommandResult( + env.getReporter(), Code.OPTIONS_INVALID, "You cannot run vendor with --nofetch"); + } + return null; + } + + private BlazeCommandResult vendorAll( + CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, PathFragment vendorDirectory) + throws InterruptedException, IOException { + EvaluationContext evaluationContext = + EvaluationContext.newBuilder() + .setParallelism(threadsOption.threads) + .setEventHandler(env.getReporter()) + .build(); + + SkyKey fetchKey = BazelFetchAllValue.key(/* configureEnabled= */ false); + EvaluationResult evaluationResult = + env.getSkyframeExecutor().prepareAndGet(ImmutableSet.of(fetchKey), evaluationContext); + if (evaluationResult.hasError()) { + Exception e = evaluationResult.getError().getException(); + return createFailedBlazeCommandResult( + env.getReporter(), + e != null ? e.getMessage() : "Unexpected error during fetching all external deps."); + } + + BazelFetchAllValue fetchAllValue = (BazelFetchAllValue) evaluationResult.get(fetchKey); + vendor(env, vendorDirectory, fetchAllValue.getReposToVendor()); + return BlazeCommandResult.success(); + } + + private BlazeCommandResult vendorRepos( + CommandEnvironment env, + LoadingPhaseThreadsOption threadsOption, + List repos, + PathFragment vendorDirectory) + throws InterruptedException, IOException { + ImmutableMap repositoryNamesAndValues; + try { + repositoryNamesAndValues = RepositoryFetcher.fetchRepos(repos, env, threadsOption); + } catch (RepositoryMappingResolutionException e) { + return createFailedBlazeCommandResult( + env.getReporter(), "Invalid repo name: " + e.getMessage(), e.getDetailedExitCode()); + } catch (RepositoryFetcherException e) { + return createFailedBlazeCommandResult(env.getReporter(), e.getMessage()); + } + + // Split repos to found and not found, vendor found ones and report others + ImmutableList.Builder reposToVendor = ImmutableList.builder(); + List notFoundRepoErrors = new ArrayList<>(); + for (Entry entry : + repositoryNamesAndValues.entrySet()) { + if (entry.getValue().repositoryExists()) { + reposToVendor.add(entry.getKey()); + } else { + notFoundRepoErrors.add(entry.getValue().getErrorMsg()); + } + } + + vendor(env, vendorDirectory, reposToVendor.build()); + if (!notFoundRepoErrors.isEmpty()) { + return createFailedBlazeCommandResult( + env.getReporter(), "Vendoring some repos failed with errors: " + notFoundRepoErrors); + } + return BlazeCommandResult.success(); + } + + /** + * Copies the fetched repos from the external cache into the vendor directory, unless the repo is + * ignored or was already vendored and up-to-date + */ + private void vendor( + CommandEnvironment env, + PathFragment vendorDirectory, + ImmutableList reposToVendor) + throws IOException { + Path vendorPath = + vendorDirectory.isAbsolute() + ? env.getRuntime().getFileSystem().getPath(vendorDirectory) + : env.getWorkspace().getRelative(vendorDirectory); + Path externalPath = + env.getDirectories() + .getOutputBase() + .getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION); + Path vendorIgnore = vendorPath.getRelative(VENDOR_IGNORE); + + if (!vendorPath.exists()) { + vendorPath.createDirectory(); + } + + // exclude any ignored repo under .vendorignore + if (vendorIgnore.exists()) { + ImmutableSet ignoredRepos = + ImmutableSet.copyOf(FileSystemUtils.readLines(vendorIgnore, UTF_8)); + reposToVendor = + reposToVendor.stream() + .filter(repo -> !ignoredRepos.contains(repo.getName())) + .collect(toImmutableList()); + } else { + FileSystemUtils.createEmptyFile(vendorIgnore); + } + + // Update "out-of-date" repos under the vendor directory + for (RepositoryName repo : reposToVendor) { + if (!isRepoUpToDate(repo.getName(), vendorPath, externalPath)) { + Path repoUnderVendor = vendorPath.getRelative(repo.getName()); + if (!repoUnderVendor.exists()) { + repoUnderVendor.createDirectory(); + } + FileSystemUtils.copyTreesBelow( + externalPath.getRelative(repo.getName()), repoUnderVendor, Symlinks.NOFOLLOW); + FileSystemUtils.copyFile( + externalPath.getChild("@" + repo.getName() + ".marker"), + vendorPath.getChild("@" + repo.getName() + ".marker")); + } + } + } + + /** + * Returns whether the repo under vendor needs to be updated by comparing its marker file with the + * one under /external + */ + private boolean isRepoUpToDate(String repoName, Path vendorPath, Path externalPath) + throws IOException { + Path vendorMarkerFile = vendorPath.getChild("@" + repoName + ".marker"); + if (!vendorMarkerFile.exists()) { + return false; + } + + // Since this runs after BazelFetchAllFunction, its guaranteed that the marker files + // under $OUTPUT_BASE/external are up-to-date. We just need to compare it against the marker + // under vendor. + Path externalMarkerFile = externalPath.getChild("@" + repoName + ".marker"); + String vendorMarkerContent = FileSystemUtils.readContent(vendorMarkerFile, UTF_8); + String externalMarkerContent = FileSystemUtils.readContent(externalMarkerFile, UTF_8); + return Objects.equals(vendorMarkerContent, externalMarkerContent); + } + + private static BlazeCommandResult createFailedBlazeCommandResult( + Reporter reporter, Code fetchCommandCode, String message) { + return createFailedBlazeCommandResult( + reporter, + message, + DetailedExitCode.of( + FailureDetail.newBuilder() + .setMessage(message) + .setFetchCommand( + FailureDetails.FetchCommand.newBuilder().setCode(fetchCommandCode).build()) + .build())); + } + + private static BlazeCommandResult createFailedBlazeCommandResult( + Reporter reporter, String errorMessage) { + return createFailedBlazeCommandResult( + reporter, errorMessage, InterruptedFailureDetails.detailedExitCode(errorMessage)); + } + + private static BlazeCommandResult createFailedBlazeCommandResult( + Reporter reporter, String message, DetailedExitCode exitCode) { + reporter.handle(Event.error(message)); + return BlazeCommandResult.detailedExitCode(exitCode); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorOptions.java new file mode 100644 index 00000000000000..f1204632a2cd01 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorOptions.java @@ -0,0 +1,34 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.commands; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionDocumentationCategory; +import com.google.devtools.common.options.OptionEffectTag; +import com.google.devtools.common.options.OptionsBase; +import java.util.List; + +/** Defines the options specific to Bazel's vendor command. */ +public class VendorOptions extends OptionsBase { + @Option( + name = "repo", + defaultValue = "null", + allowMultiple = true, + documentationCategory = OptionDocumentationCategory.BZLMOD, + effectTags = {OptionEffectTag.CHANGES_INPUTS}, + help = + "Only vendors the specified repository, which can be either `@apparent_repo_name` or" + + " `@@canonical_repo_name`. This option can be set multiple times") + public List repos; +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/vendor.txt b/src/main/java/com/google/devtools/build/lib/bazel/commands/vendor.txt new file mode 100644 index 00000000000000..dfffde43ca8158 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/vendor.txt @@ -0,0 +1,7 @@ + +Usage: %{product} %{command} [