[7.1.0] Add vendor mode

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -54,6 +54,7 @@
 import com.google.devtools.build.lib.bazel.commands.FetchCommand;
 import com.google.devtools.build.lib.bazel.commands.ModCommand;
 import com.google.devtools.build.lib.bazel.commands.SyncCommand;
+import com.google.devtools.build.lib.bazel.commands.VendorCommand;
 import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformFunction;
 import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformRule;
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
@@ -154,6 +155,8 @@ public class BazelRepositoryModule extends BlazeModule {
   private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING;
   private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR;
   private LockfileMode bazelLockfileMode = LockfileMode.UPDATE;
+
+  private Optional<Path> vendorDirectory;
   private List<String> allowedYankedVersions = ImmutableList.of();
   private SingleExtensionEvalFunction singleExtensionEvalFunction;
   private final ExecutorService repoFetchingWorkerThreadPool =
@@ -214,6 +217,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde
     builder.addCommands(new FetchCommand());
     builder.addCommands(new ModCommand());
     builder.addCommands(new SyncCommand());
+    builder.addCommands(new VendorCommand());
     builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache));
   }
 
@@ -502,6 +506,16 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
       bazelLockfileMode = repoOptions.lockfileMode;
       allowedYankedVersions = repoOptions.allowedYankedVersions;
 
+      if (repoOptions.vendorDirectory != null) {
+        vendorDirectory =
+            Optional.of(
+                repoOptions.vendorDirectory.isAbsolute()
+                    ? filesystem.getPath(repoOptions.vendorDirectory)
+                    : env.getWorkspace().getRelative(repoOptions.vendorDirectory));
+      } else {
+        vendorDirectory = Optional.empty();
+      }
+
       if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) {
         registries = repoOptions.registries;
       } else {
@@ -573,6 +587,8 @@ public ImmutableList<Injected> getPrecomputedValues() {
         PrecomputedValue.injected(
             BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, bazelCompatibilityMode),
         PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, bazelLockfileMode),
+        PrecomputedValue.injected(RepositoryDelegatorFunction.IS_VENDOR_COMMAND, false),
+        PrecomputedValue.injected(RepositoryDelegatorFunction.VENDOR_DIRECTORY, vendorDirectory),
         PrecomputedValue.injected(
             YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, allowedYankedVersions));
   }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllFunction.java

@@ -18,6 +18,7 @@
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryFunction;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
@@ -31,8 +32,8 @@
 import javax.annotation.Nullable;
 
 /**
- * Void function designed to gather and fetch all the repositories without returning any specific
- * result (empty value is returned).
+ * Gather and fetch all the repositories from MODULE.bazel resolution and extensions evaluation. If
+ * this is fetch configure, only configure repos will be fetched and returned
  */
 public class BazelFetchAllFunction implements SkyFunction {
 
@@ -53,14 +54,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept
             .filter(repo -> !repo.isMain())
             .collect(toImmutableList()));
 
-    // 2. Run every extension found in the modules
+    // 2. Run every extension found in the modules & collect its generated repos
     ImmutableSet<ModuleExtensionId> extensionIds =
         depGraphValue.getExtensionUsagesTable().rowKeySet();
     ImmutableSet<SkyKey> singleEvalKeys =
         extensionIds.stream().map(SingleExtensionEvalValue::key).collect(toImmutableSet());
     SkyframeLookupResult singleEvalValues = env.getValuesAndExceptions(singleEvalKeys);
-
-    // 3. For each extension, collect its generated repos
     for (SkyKey singleEvalKey : singleEvalKeys) {
       SingleExtensionEvalValue singleEvalValue =
           (SingleExtensionEvalValue) singleEvalValues.get(singleEvalKey);
@@ -70,7 +69,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept
       reposToFetch.addAll(singleEvalValue.getCanonicalRepoNameToInternalNames().keySet());
     }
 
-    // 4. If this is fetch configure, get repo rules and only collect repos marked as configure
+    // 3. If this is fetch configure, get repo rules and only collect repos marked as configure
     Boolean fetchConfigure = (Boolean) skyKey.argument();
     if (fetchConfigure) {
       ImmutableSet<SkyKey> repoRuleKeys =
@@ -83,12 +82,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept
           return null;
         }
         if (StarlarkRepositoryFunction.isConfigureRule(repoRuleValue.getRule())) {
-          reposToFetch.add(RepositoryName.createUnvalidated(repoRuleValue.getRule().getName()));
+          reposToFetch.add((RepositoryName) repoRuleKey.argument());
         }
       }
     }
 
-    // 5. Fetch all the collected repos
+    // 4. Fetch all the collected repos
+    List<RepositoryName> shouldVendor = new ArrayList<>();
     ImmutableSet<SkyKey> repoDelegatorKeys =
         reposToFetch.stream().map(RepositoryDirectoryValue::key).collect(toImmutableSet());
     SkyframeLookupResult repoDirValues = env.getValuesAndExceptions(repoDelegatorKeys);
@@ -98,9 +98,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept
       if (repoDirValue == null) {
         return null;
       }
+      if (!repoDirValue.excludeFromVendoring()) {
+        shouldVendor.add((RepositoryName) repoDelegatorKey.argument());
+      }
     }
 
-    return BazelFetchAllValue.create();
+    return BazelFetchAllValue.create(
+        ImmutableList.copyOf(reposToFetch), ImmutableList.copyOf(shouldVendor));
   }
 
 }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllValue.java

@@ -16,6 +16,8 @@
 package com.google.devtools.build.lib.bazel.bzlmod;
 
 import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.skyframe.SkyFunctions;
 import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
 import com.google.devtools.build.skyframe.AbstractSkyKey;
@@ -35,8 +37,13 @@ public static BazelFetchAllValue.Key key(Boolean configureEnabled) {
     return BazelFetchAllValue.Key.create(configureEnabled);
   }
 
-  public static BazelFetchAllValue create() {
-    return new AutoValue_BazelFetchAllValue();
+  public abstract ImmutableList<RepositoryName> getFetchedRepos();
+
+  public abstract ImmutableList<RepositoryName> getReposToVendor();
+
+  public static BazelFetchAllValue create(
+      ImmutableList<RepositoryName> fetchedRepos, ImmutableList<RepositoryName> reposToVendor) {
+    return new AutoValue_BazelFetchAllValue(fetchedRepos, reposToVendor);
   }
 
   /** Key type for BazelFetchAllValue. */

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/InvalidArgumentException.java

@@ -34,6 +34,10 @@ public InvalidArgumentException(String message, Code code) {
     this.code = code;
   }
 
+  public InvalidArgumentException(String message) {
+    this(message, Code.INVALID_ARGUMENTS);
+  }
+
   public Code getCode() {
     return code;
   }

src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD

@@ -20,6 +20,7 @@ java_library(
         "fetch.txt",
         "mod.txt",
         "sync.txt",
+        "vendor.txt",
     ],
     deps = [
         "//src/main/java/com/google/devtools/build/lib:keep-going-option",
@@ -66,6 +67,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/util:interrupted_failure_details",
         "//src/main/java/com/google/devtools/build/lib/util:maybe_complete_set",
         "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/com/google/devtools/build/skyframe",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
         "//src/main/java/com/google/devtools/common/options",