From 2b6733b0b074c50525cf243c0cfdfc7765dc2c84 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Wed, 24 Jan 2024 06:14:41 -0800 Subject: [PATCH 1/7] Vendoring (Related #19563) Vendor command fetches all external repositories into a folder specified by the flag --vendor_dir. When building using the same flag & value the vendored repos are used as long as they are up to date. For the initial vendoring or updating existing vendor repositories the new command vendor command is: `bazel vendor --vendor_dir={vendordir}` The {vendordir} parameter can be either a full path or just a folder name. Vendored repositories will be located under {workspace}/vendordir. To perform a build using vendored repositories: `bazel build {target} --vendor_dir={vendordir}` In this scenario, the vendor directory will be prioritized for repository retrieval. If the repository is not found or is outdated, the standard process will be followed, which involves checking cached repositories and fetching if not found. To exclude repositories from the vendoring process: A file named ".vendorignore" is generated under the vendor directory. By adding the repository name to this file, the vendoring process will ignore it. Also, any local or configure repos are ignored. PiperOrigin-RevId: 601099450 Change-Id: I08196c7ad924c1d22918ba09dba2ae774802a691 # Conflicts: # src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java # src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java --- .../lib/bazel/BazelRepositoryModule.java | 16 + .../bazel/bzlmod/BazelFetchAllFunction.java | 22 +- .../lib/bazel/bzlmod/BazelFetchAllValue.java | 11 +- .../devtools/build/lib/bazel/commands/BUILD | 2 + .../lib/bazel/commands/VendorCommand.java | 237 +++++++++++++ .../build/lib/bazel/commands/vendor.txt | 7 + .../bazel/repository/RepositoryOptions.java | 13 + .../RepositoryDelegatorFunction.java | 92 ++++- .../repository/RepositoryDirectoryValue.java | 32 +- .../skyframe/packages/BazelPackageLoader.java | 1 + .../build/lib/analysis/util/AnalysisMock.java | 1 + .../lib/analysis/util/AnalysisTestCase.java | 2 + .../analysis/util/ConfigurationTestCase.java | 2 + .../bzlmod/BazelLockFileFunctionTest.java | 2 + .../build/lib/bazel/bzlmod/DiscoveryTest.java | 2 + .../bzlmod/ModuleExtensionResolutionTest.java | 2 + .../bazel/bzlmod/ModuleFileFunctionTest.java | 2 + .../util/BuildIntegrationTestCase.java | 4 +- .../query2/testutil/SkyframeQueryHelper.java | 2 + .../repository/RepositoryDelegatorTest.java | 2 + ...ractCollectPackagesUnderDirectoryTest.java | 4 +- .../ContainingPackageLookupFunctionTest.java | 2 + .../build/lib/skyframe/FileFunctionTest.java | 1 + .../skyframe/PackageLookupFunctionTest.java | 2 + src/test/py/bazel/BUILD | 15 + src/test/py/bazel/bzlmod/bazel_vendor_test.py | 330 ++++++++++++++++++ src/test/py/bazel/test_base.py | 14 + 27 files changed, 797 insertions(+), 25 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/commands/vendor.txt create mode 100644 src/test/py/bazel/bzlmod/bazel_vendor_test.py diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 4a272e29ecdd89..008242a1ae063f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -54,6 +54,7 @@ import com.google.devtools.build.lib.bazel.commands.FetchCommand; import com.google.devtools.build.lib.bazel.commands.ModCommand; import com.google.devtools.build.lib.bazel.commands.SyncCommand; +import com.google.devtools.build.lib.bazel.commands.VendorCommand; import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformFunction; import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformRule; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; @@ -154,6 +155,8 @@ public class BazelRepositoryModule extends BlazeModule { private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING; private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR; private LockfileMode bazelLockfileMode = LockfileMode.UPDATE; + + private Optional vendorDirectory; private List allowedYankedVersions = ImmutableList.of(); private SingleExtensionEvalFunction singleExtensionEvalFunction; private final ExecutorService repoFetchingWorkerThreadPool = @@ -214,6 +217,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde builder.addCommands(new FetchCommand()); builder.addCommands(new ModCommand()); builder.addCommands(new SyncCommand()); + builder.addCommands(new VendorCommand()); builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache)); } @@ -502,6 +506,16 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { bazelLockfileMode = repoOptions.lockfileMode; allowedYankedVersions = repoOptions.allowedYankedVersions; + if (repoOptions.vendorDirectory != null) { + vendorDirectory = + Optional.of( + repoOptions.vendorDirectory.isAbsolute() + ? filesystem.getPath(repoOptions.vendorDirectory) + : env.getWorkspace().getRelative(repoOptions.vendorDirectory)); + } else { + vendorDirectory = Optional.empty(); + } + if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) { registries = repoOptions.registries; } else { @@ -573,6 +587,8 @@ public ImmutableList getPrecomputedValues() { PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, bazelCompatibilityMode), PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, bazelLockfileMode), + PrecomputedValue.injected(RepositoryDelegatorFunction.IS_VENDOR_COMMAND, false), + PrecomputedValue.injected(RepositoryDelegatorFunction.VENDOR_DIRECTORY, vendorDirectory), PrecomputedValue.injected( YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, allowedYankedVersions)); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllFunction.java index 7f7f77a79020f8..40dfddf9e5ab50 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllFunction.java @@ -18,6 +18,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryFunction; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -31,8 +32,8 @@ import javax.annotation.Nullable; /** - * Void function designed to gather and fetch all the repositories without returning any specific - * result (empty value is returned). + * Gather and fetch all the repositories from MODULE.bazel resolution and extensions evaluation. If + * this is fetch configure, only configure repos will be fetched and returned */ public class BazelFetchAllFunction implements SkyFunction { @@ -53,14 +54,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept .filter(repo -> !repo.isMain()) .collect(toImmutableList())); - // 2. Run every extension found in the modules + // 2. Run every extension found in the modules & collect its generated repos ImmutableSet extensionIds = depGraphValue.getExtensionUsagesTable().rowKeySet(); ImmutableSet singleEvalKeys = extensionIds.stream().map(SingleExtensionEvalValue::key).collect(toImmutableSet()); SkyframeLookupResult singleEvalValues = env.getValuesAndExceptions(singleEvalKeys); - - // 3. For each extension, collect its generated repos for (SkyKey singleEvalKey : singleEvalKeys) { SingleExtensionEvalValue singleEvalValue = (SingleExtensionEvalValue) singleEvalValues.get(singleEvalKey); @@ -70,7 +69,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept reposToFetch.addAll(singleEvalValue.getCanonicalRepoNameToInternalNames().keySet()); } - // 4. If this is fetch configure, get repo rules and only collect repos marked as configure + // 3. If this is fetch configure, get repo rules and only collect repos marked as configure Boolean fetchConfigure = (Boolean) skyKey.argument(); if (fetchConfigure) { ImmutableSet repoRuleKeys = @@ -83,12 +82,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept return null; } if (StarlarkRepositoryFunction.isConfigureRule(repoRuleValue.getRule())) { - reposToFetch.add(RepositoryName.createUnvalidated(repoRuleValue.getRule().getName())); + reposToFetch.add((RepositoryName) repoRuleKey.argument()); } } } - // 5. Fetch all the collected repos + // 4. Fetch all the collected repos + List shouldVendor = new ArrayList<>(); ImmutableSet repoDelegatorKeys = reposToFetch.stream().map(RepositoryDirectoryValue::key).collect(toImmutableSet()); SkyframeLookupResult repoDirValues = env.getValuesAndExceptions(repoDelegatorKeys); @@ -98,9 +98,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept if (repoDirValue == null) { return null; } + if (!repoDirValue.excludeFromVendoring()) { + shouldVendor.add((RepositoryName) repoDelegatorKey.argument()); + } } - return BazelFetchAllValue.create(); + return BazelFetchAllValue.create( + ImmutableList.copyOf(reposToFetch), ImmutableList.copyOf(shouldVendor)); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllValue.java index 527e78eafa93ba..7e2831598c2bba 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllValue.java @@ -16,6 +16,8 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.skyframe.AbstractSkyKey; @@ -35,8 +37,13 @@ public static BazelFetchAllValue.Key key(Boolean configureEnabled) { return BazelFetchAllValue.Key.create(configureEnabled); } - public static BazelFetchAllValue create() { - return new AutoValue_BazelFetchAllValue(); + public abstract ImmutableList getFetchedRepos(); + + public abstract ImmutableList getReposToVendor(); + + public static BazelFetchAllValue create( + ImmutableList fetchedRepos, ImmutableList reposToVendor) { + return new AutoValue_BazelFetchAllValue(fetchedRepos, reposToVendor); } /** Key type for BazelFetchAllValue. */ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD index 28cd553a3019c5..0ace636128ad20 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD @@ -20,6 +20,7 @@ java_library( "fetch.txt", "mod.txt", "sync.txt", + "vendor.txt", ], deps = [ "//src/main/java/com/google/devtools/build/lib:keep-going-option", @@ -66,6 +67,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/util:interrupted_failure_details", "//src/main/java/com/google/devtools/build/lib/util:maybe_complete_set", "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//src/main/java/com/google/devtools/common/options", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java new file mode 100644 index 00000000000000..5f4d72329edee4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java @@ -0,0 +1,237 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.commands; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.NoBuildEvent; +import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent; +import com.google.devtools.build.lib.bazel.bzlmod.BazelFetchAllValue; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; +import com.google.devtools.build.lib.cmdline.LabelConstants; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; +import com.google.devtools.build.lib.pkgcache.PackageOptions; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandResult; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.KeepGoingOption; +import com.google.devtools.build.lib.runtime.LoadingPhaseThreadsOption; +import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; +import com.google.devtools.build.lib.server.FailureDetails.FetchCommand.Code; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.DetailedExitCode; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.InterruptedFailureDetails; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.skyframe.EvaluationContext; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.common.options.OptionsParsingResult; +import java.io.IOException; +import java.util.Objects; + +/** Fetches external repositories into a specified directory. */ +@Command( + name = VendorCommand.NAME, + options = {PackageOptions.class, KeepGoingOption.class, LoadingPhaseThreadsOption.class}, + help = "resource:vendor.txt", + shortDescription = + "Fetches external repositories into a specific folder specified by the flag " + + "--vendor_dir.") +public final class VendorCommand implements BlazeCommand { + public static final String NAME = "vendor"; + + // TODO(salmasamy) decide on name and format + private static final String VENDOR_IGNORE = ".vendorignore"; + + @Override + public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult options) { + RepositoryOptions repoOptions = options.getOptions(RepositoryOptions.class); + if (!options.getOptions(BuildLanguageOptions.class).enableBzlmod) { + return createFailedBlazeCommandResult( + env.getReporter(), + "Bzlmod has to be enabled for vendoring to work, run with --enable_bzlmod"); + } + if (repoOptions.vendorDirectory == null) { + return createFailedBlazeCommandResult( + env.getReporter(), + Code.OPTIONS_INVALID, + "You cannot run vendor without specifying --vendor_dir"); + } + + LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); + SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor(); + EvaluationContext evaluationContext = + EvaluationContext.newBuilder() + .setParallelism(threadsOption.threads) + .setEventHandler(env.getReporter()) + .build(); + // IS_VENDOR_COMMAND & VENDOR_DIR is already injected in "BazelRepositoryModule", we just need + // to update this value for the delegator function to recognize this call is from VendorCommand + skyframeExecutor.injectExtraPrecomputedValues( + ImmutableList.of( + PrecomputedValue.injected(RepositoryDelegatorFunction.IS_VENDOR_COMMAND, true))); + + env.getEventBus() + .post( + new NoBuildEvent( + env.getCommandName(), + env.getCommandStartTime(), + /* separateFinishedEvent= */ true, + /* showProgress= */ true, + /* id= */ null)); + + // 1. Fetch all repos + SkyKey fetchKey = BazelFetchAllValue.key(/* configureEnabled= */ false); + try { + env.syncPackageLoading(options); + EvaluationResult evaluationResult = + skyframeExecutor.prepareAndGet(ImmutableSet.of(fetchKey), evaluationContext); + if (evaluationResult.hasError()) { + Exception e = evaluationResult.getError().getException(); + return createFailedBlazeCommandResult( + env.getReporter(), + e != null ? e.getMessage() : "Unexpected error during fetching all external deps."); + } + + // 2. Vendor repos + BazelFetchAllValue fetchAllValue = (BazelFetchAllValue) evaluationResult.get(fetchKey); + vendorRepos(env, repoOptions.vendorDirectory, fetchAllValue.getReposToVendor()); + + env.getEventBus() + .post( + new NoBuildRequestFinishedEvent( + ExitCode.SUCCESS, env.getRuntime().getClock().currentTimeMillis())); + return BlazeCommandResult.success(); + } catch (AbruptExitException e) { + return createFailedBlazeCommandResult( + env.getReporter(), + "Unknown error during vendoring: " + e.getMessage(), + e.getDetailedExitCode()); + } catch (InterruptedException e) { + return createFailedBlazeCommandResult( + env.getReporter(), "Fetch interrupted: " + e.getMessage()); + } catch (IOException e) { + return createFailedBlazeCommandResult( + env.getReporter(), "Error while vendoring repos: " + e.getMessage()); + } + } + + private void vendorRepos( + CommandEnvironment env, + PathFragment vendorDirectory, + ImmutableList reposToVendor) + throws IOException { + Path vendorPath = + vendorDirectory.isAbsolute() + ? env.getRuntime().getFileSystem().getPath(vendorDirectory) + : env.getWorkspace().getRelative(vendorDirectory); + Path externalPath = + env.getDirectories() + .getOutputBase() + .getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION); + Path vendorIgnore = vendorPath.getRelative(VENDOR_IGNORE); + + if (!vendorPath.exists()) { + vendorPath.createDirectory(); + } + + // exclude any ignored repo under .vendorignore + if (vendorIgnore.exists()) { + ImmutableSet ignoredRepos = + ImmutableSet.copyOf(FileSystemUtils.readLines(vendorIgnore, UTF_8)); + reposToVendor = + reposToVendor.stream() + .filter(repo -> !ignoredRepos.contains(repo.getName())) + .collect(toImmutableList()); + } else { + FileSystemUtils.createEmptyFile(vendorIgnore); + } + + // Update "out-of-date" repos under the vendor directory + for (RepositoryName repo : reposToVendor) { + if (!isRepoUpToDate(repo.getName(), vendorPath, externalPath)) { + Path repoUnderVendor = vendorPath.getRelative(repo.getName()); + if (!repoUnderVendor.exists()) { + repoUnderVendor.createDirectory(); + } + FileSystemUtils.copyTreesBelow( + externalPath.getRelative(repo.getName()), repoUnderVendor, Symlinks.NOFOLLOW); + FileSystemUtils.copyFile( + externalPath.getChild("@" + repo.getName() + ".marker"), + vendorPath.getChild("@" + repo.getName() + ".marker")); + } + } + } + + /** + * Returns whether the repo under vendor needs to be updated by comparing its marker file with the + * one under /external + */ + private boolean isRepoUpToDate(String repoName, Path vendorPath, Path externalPath) + throws IOException { + Path vendorMarkerFile = vendorPath.getChild("@" + repoName + ".marker"); + if (!vendorMarkerFile.exists()) { + return false; + } + + // Since this runs after BazelFetchAllFunction, its guaranteed that the marker files + // under $OUTPUT_BASE/external are up-to-date. We just need to compare it against the marker + // under vendor. + Path externalMarkerFile = externalPath.getChild("@" + repoName + ".marker"); + String vendorMarkerContent = FileSystemUtils.readContent(vendorMarkerFile, UTF_8); + String externalMarkerContent = FileSystemUtils.readContent(externalMarkerFile, UTF_8); + return Objects.equals(vendorMarkerContent, externalMarkerContent); + } + + private static BlazeCommandResult createFailedBlazeCommandResult( + Reporter reporter, Code fetchCommandCode, String message) { + return createFailedBlazeCommandResult( + reporter, + message, + DetailedExitCode.of( + FailureDetail.newBuilder() + .setMessage(message) + .setFetchCommand( + FailureDetails.FetchCommand.newBuilder().setCode(fetchCommandCode).build()) + .build())); + } + + private static BlazeCommandResult createFailedBlazeCommandResult( + Reporter reporter, String errorMessage) { + return createFailedBlazeCommandResult( + reporter, errorMessage, InterruptedFailureDetails.detailedExitCode(errorMessage)); + } + + private static BlazeCommandResult createFailedBlazeCommandResult( + Reporter reporter, String message, DetailedExitCode exitCode) { + reporter.handle(Event.error(message)); + return BlazeCommandResult.detailedExitCode(exitCode); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/vendor.txt b/src/main/java/com/google/devtools/build/lib/bazel/commands/vendor.txt new file mode 100644 index 00000000000000..dfffde43ca8158 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/vendor.txt @@ -0,0 +1,7 @@ + +Usage: %{product} %{command} [