Implement RemoteDownloader w/ --experimental_remote_downloader

This is the Bazel client implementation of bazelbuild/proposals#160. It allows downloading of external dependencies to be delegated to a remote service.

TODOs:
- [x] Once bazelbuild/remote-apis#112 is merged, the vendored copy of `bazelbuild/remote-apis` should be updated. I've used a [WIP] placeholder for now.
- [x] If the general approach looks reasonable then I'll add tests. Currently I've been testing with an in-house implementation of the downloader server.

R: @buchgr @dslomov
CC: @EricBurnett @sstriker @ulfjack

Closes #10622.

PiperOrigin-RevId: 300116716

src/main/java/com/google/devtools/build/lib/BUILD

@@ -1252,6 +1252,7 @@ java_library(
         ":util",
         "//src/main/java/com/google/devtools/build/lib/actions",
         "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/transports",

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -33,6 +33,7 @@
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.RepositoryOverride;
 import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
+import com.google.devtools.build.lib.bazel.repository.downloader.DelegatingDownloader;
 import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
 import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
 import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryFunction;
@@ -97,8 +98,10 @@ public class BazelRepositoryModule extends BlazeModule {
   private final SkylarkRepositoryFunction skylarkRepositoryFunction;
   private final RepositoryCache repositoryCache = new RepositoryCache();
   private final HttpDownloader httpDownloader = new HttpDownloader();
+  private final DelegatingDownloader delegatingDownloader =
+      new DelegatingDownloader(httpDownloader);
   private final DownloadManager downloadManager =
-      new DownloadManager(repositoryCache, httpDownloader);
+      new DownloadManager(repositoryCache, delegatingDownloader);
   private final MutableSupplier<Map<String, String>> clientEnvironmentSupplier =
       new MutableSupplier<>();
   private ImmutableMap<RepositoryName, PathFragment> overrides = ImmutableMap.of();
@@ -334,6 +337,7 @@ public void beforeCommand(CommandEnvironment env) {
         remoteExecutor = remoteExecutorFactory.create();
       }
       skylarkRepositoryFunction.setRepositoryRemoteExecutor(remoteExecutor);
+      delegatingDownloader.setDelegate(env.getRuntime().getDownloaderSupplier().get());
     }
   }
 

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DelegatingDownloader.java

@@ -0,0 +1,64 @@
+// Copyright 2020 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository.downloader;
+
+import com.google.common.base.Optional;
+import com.google.devtools.build.lib.events.ExtendedEventHandler;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * A {@link Downloader} that delegates to another Downloader. Primarily useful for mutable
+ * dependency injection.
+ */
+public class DelegatingDownloader implements Downloader {
+  private final Downloader defaultDelegate;
+  @Nullable private Downloader delegate;
+
+  public DelegatingDownloader(Downloader defaultDelegate) {
+    this.defaultDelegate = defaultDelegate;
+  }
+
+  /**
+   * Sets the {@link Downloader} to delegate to. If setDelegate(null) is called, the default
+   * delegate passed to the constructor will be used.
+   */
+  public void setDelegate(@Nullable Downloader delegate) {
+    this.delegate = delegate;
+  }
+
+  @Override
+  public void download(
+      List<URL> urls,
+      Map<URI, Map<String, String>> authHeaders,
+      Optional<Checksum> checksum,
+      String canonicalId,
+      Path destination,
+      ExtendedEventHandler eventHandler,
+      Map<String, String> clientEnv)
+      throws IOException, InterruptedException {
+    Downloader downloader = defaultDelegate;
+    if (delegate != null) {
+      downloader = delegate;
+    }
+    downloader.download(
+        urls, authHeaders, checksum, canonicalId, destination, eventHandler, clientEnv);
+  }
+}

src/main/java/com/google/devtools/build/lib/remote/BUILD

@@ -51,12 +51,14 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/actions",
         "//src/main/java/com/google/devtools/build/lib/analysis/platform:platform_utils",
         "//src/main/java/com/google/devtools/build/lib/authandtls",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream",
         "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
         "//src/main/java/com/google/devtools/build/lib/concurrent",
         "//src/main/java/com/google/devtools/build/lib/profiler",
         "//src/main/java/com/google/devtools/build/lib/remote/common",
         "//src/main/java/com/google/devtools/build/lib/remote/disk",
+        "//src/main/java/com/google/devtools/build/lib/remote/downloader",
         "//src/main/java/com/google/devtools/build/lib/remote/http",
         "//src/main/java/com/google/devtools/build/lib/remote/logging",
         "//src/main/java/com/google/devtools/build/lib/remote/merkletree",

src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java

@@ -34,6 +34,7 @@
 import com.google.devtools.build.lib.analysis.test.TestProvider;
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
 import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
+import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
 import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader;
 import com.google.devtools.build.lib.buildeventstream.LocalFilesArtifactUploader;
 import com.google.devtools.build.lib.buildtool.BuildRequest;
@@ -43,6 +44,7 @@
 import com.google.devtools.build.lib.exec.ExecutorBuilder;
 import com.google.devtools.build.lib.packages.TargetUtils;
 import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
+import com.google.devtools.build.lib.remote.downloader.GrpcRemoteDownloader;
 import com.google.devtools.build.lib.remote.logging.LoggingInterceptor;
 import com.google.devtools.build.lib.remote.options.RemoteOptions;
 import com.google.devtools.build.lib.remote.options.RemoteOutputsMode;
@@ -57,6 +59,7 @@
 import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutorFactory;
 import com.google.devtools.build.lib.runtime.ServerBuilder;
 import com.google.devtools.build.lib.skyframe.AspectValue;
+import com.google.devtools.build.lib.skyframe.MutableSupplier;
 import com.google.devtools.build.lib.util.AbruptExitException;
 import com.google.devtools.build.lib.util.ExitCode;
 import com.google.devtools.build.lib.util.io.AsynchronousFileOutputStream;
@@ -70,6 +73,7 @@
 import io.grpc.Context;
 import java.io.IOException;
 import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.Executors;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -95,18 +99,26 @@ public final class RemoteModule extends BlazeModule {
   private final RepositoryRemoteExecutorFactoryDelegate repositoryRemoteExecutorFactoryDelegate =
       new RepositoryRemoteExecutorFactoryDelegate();
 
+  private final MutableSupplier<Downloader> remoteDownloaderSupplier = new MutableSupplier<>();
+
   @Override
   public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builder) {
     builder.addBuildEventArtifactUploaderFactory(
         buildEventArtifactUploaderFactoryDelegate, "remote");
     builder.setRepositoryRemoteExecutorFactory(repositoryRemoteExecutorFactoryDelegate);
+    builder.setDownloaderSupplier(remoteDownloaderSupplier);
   }
 
   /** Returns whether remote execution should be available. */
   public static boolean shouldEnableRemoteExecution(RemoteOptions options) {
     return !Strings.isNullOrEmpty(options.remoteExecutor);
   }
 
+  /** Returns whether remote downloading should be available. */
+  private static boolean shouldEnableRemoteDownloader(RemoteOptions options) {
+    return !Strings.isNullOrEmpty(options.remoteDownloader);
+  }
+
   private void verifyServerCapabilities(
       RemoteOptions remoteOptions,
       ReferenceCountedChannel channel,
@@ -160,6 +172,13 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
     boolean enableHttpCache = RemoteCacheClientFactory.isHttpCache(remoteOptions);
     boolean enableGrpcCache = GrpcCacheClient.isRemoteCacheOptions(remoteOptions);
     boolean enableRemoteExecution = shouldEnableRemoteExecution(remoteOptions);
+    boolean enableRemoteDownloader = shouldEnableRemoteDownloader(remoteOptions);
+
+    if (enableRemoteDownloader && !enableGrpcCache) {
+      throw new AbruptExitException(
+          "The remote downloader can only be used in combination with gRPC caching",
+          ExitCode.COMMAND_LINE_ERROR);
+    }
 
     if (!enableDiskCache && !enableHttpCache && !enableGrpcCache && !enableRemoteExecution) {
       // Quit if no remote caching or execution was enabled.
@@ -208,6 +227,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
 
       ReferenceCountedChannel execChannel = null;
       ReferenceCountedChannel cacheChannel = null;
+      ReferenceCountedChannel downloaderChannel = null;
       if (enableRemoteExecution) {
         ImmutableList.Builder<ClientInterceptor> interceptors = ImmutableList.builder();
         interceptors.add(TracingMetadataUtils.newExecHeadersInterceptor(remoteOptions));
@@ -242,6 +262,25 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
                 interceptors.build());
       }
 
+      if (enableRemoteDownloader) {
+        // Create a separate channel if --remote_downloader and --remote_cache point to different
+        // endpoints.
+        if (remoteOptions.remoteDownloader.equals(remoteOptions.remoteCache)) {
+          downloaderChannel = cacheChannel.retain();
+        } else {
+          ImmutableList.Builder<ClientInterceptor> interceptors = ImmutableList.builder();
+          if (loggingInterceptor != null) {
+            interceptors.add(loggingInterceptor);
+          }
+          downloaderChannel =
+              RemoteCacheClientFactory.createGrpcChannel(
+                  remoteOptions.remoteDownloader,
+                  remoteOptions.remoteProxy,
+                  authAndTlsOptions,
+                  interceptors.build());
+        }
+      }
+
       CallCredentials credentials = GoogleAuthUtils.newCallCredentials(authAndTlsOptions);
       RemoteRetrier retrier =
           new RemoteRetrier(
@@ -289,6 +328,9 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
               requestContext,
               remoteOptions.remoteInstanceName));
 
+      Context repoContext =
+          TracingMetadataUtils.contextWithMetadata(buildRequestId, invocationId, "repository_rule");
+
       if (enableRemoteExecution) {
         RemoteRetrier execRetrier =
             new RemoteRetrier(
@@ -308,9 +350,6 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
         actionContextProvider =
             RemoteActionContextProvider.createForRemoteExecution(
                 env, remoteCache, remoteExecutor, retryScheduler, digestUtil, logDir);
-        Context repoContext =
-            TracingMetadataUtils.contextWithMetadata(
-                buildRequestId, invocationId, "repository_rule");
         repositoryRemoteExecutorFactoryDelegate.init(
             new RemoteRepositoryRemoteExecutorFactory(
                 remoteCache,
@@ -336,6 +375,18 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
             RemoteActionContextProvider.createForRemoteCaching(
                 env, remoteCache, retryScheduler, digestUtil);
       }
+
+      if (enableRemoteDownloader) {
+        remoteDownloaderSupplier.set(
+            new GrpcRemoteDownloader(
+                downloaderChannel.retain(),
+                Optional.ofNullable(credentials),
+                retrier,
+                repoContext,
+                cacheClient,
+                remoteOptions));
+        downloaderChannel.release();
+      }
     } catch (IOException e) {
       env.getReporter().handle(Event.error(e.getMessage()));
       env.getBlazeModuleEnvironment()
@@ -468,6 +519,7 @@ public void afterCommand() throws AbruptExitException {
 
     buildEventArtifactUploaderFactoryDelegate.reset();
     repositoryRemoteExecutorFactoryDelegate.reset();
+    remoteDownloaderSupplier.set(null);
     actionContextProvider = null;
     actionInputFetcher = null;
     remoteOutputsMode = null;

src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD

@@ -15,7 +15,6 @@ java_library(
     deps = [
         "//src/main/java/com/google/devtools/build/lib:events",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
-        "//src/main/java/com/google/devtools/build/lib/remote",
         "//src/main/java/com/google/devtools/build/lib/remote:ReferenceCountedChannel",
         "//src/main/java/com/google/devtools/build/lib/remote:Retrier",
         "//src/main/java/com/google/devtools/build/lib/remote/common",

src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java

@@ -85,7 +85,16 @@ public final class RemoteOptions extends OptionsBase {
               + " https://docs.bazel.build/versions/master/remote-caching.html")
   public String remoteCache;
 
-  public final String remoteDownloader = "";
+  @Option(
+      name = "experimental_remote_downloader",
+      defaultValue = "null",
+      documentationCategory = OptionDocumentationCategory.REMOTE,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      help =
+          "A URI of a remote downloader endpoint. The supported schemas are grpc and grpcs"
+              + " (grpc with TLS enabled). If no schema is provided bazel will default to grpcs."
+              + " Specify grpc:// schema to disable TLS.")
+  public String remoteDownloader;
 
   @Option(
       name = "remote_header",

src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java

@@ -34,6 +34,7 @@
 import com.google.devtools.build.lib.analysis.ServerDirectories;
 import com.google.devtools.build.lib.analysis.config.BuildOptions;
 import com.google.devtools.build.lib.analysis.test.CoverageReportActionFactory;
+import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
 import com.google.devtools.build.lib.bugreport.BugReport;
 import com.google.devtools.build.lib.bugreport.BugReporter;
 import com.google.devtools.build.lib.buildeventstream.BuildEvent.LocalFile.LocalFileType;
@@ -123,6 +124,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
+import java.util.function.Supplier;
 import java.util.logging.Handler;
 import java.util.logging.Level;
 import java.util.logging.LogRecord;
@@ -175,6 +177,7 @@ public final class BlazeRuntime implements BugReport.BlazeRuntimeInterface {
   private final ImmutableMap<String, AuthHeadersProvider> authHeadersProviderMap;
   private final RetainedHeapLimiter retainedHeapLimiter = new RetainedHeapLimiter();
   @Nullable private final RepositoryRemoteExecutorFactory repositoryRemoteExecutorFactory;
+  private final Supplier<Downloader> downloaderSupplier;
 
   // Workspace state (currently exactly one workspace per server)
   private BlazeWorkspace workspace;
@@ -201,7 +204,8 @@ private BlazeRuntime(
       String productName,
       BuildEventArtifactUploaderFactoryMap buildEventArtifactUploaderFactoryMap,
       ImmutableMap<String, AuthHeadersProvider> authHeadersProviderMap,
-      RepositoryRemoteExecutorFactory repositoryRemoteExecutorFactory) {
+      RepositoryRemoteExecutorFactory repositoryRemoteExecutorFactory,
+      Supplier<Downloader> downloaderSupplier) {
     // Server state
     this.fileSystem = fileSystem;
     this.blazeModules = blazeModules;
@@ -231,6 +235,7 @@ private BlazeRuntime(
     this.authHeadersProviderMap =
         Preconditions.checkNotNull(authHeadersProviderMap, "authHeadersProviderMap");
     this.repositoryRemoteExecutorFactory = repositoryRemoteExecutorFactory;
+    this.downloaderSupplier = downloaderSupplier;
   }
 
   public BlazeWorkspace initWorkspace(BlazeDirectories directories, BinTools binTools)
@@ -1448,6 +1453,10 @@ public RepositoryRemoteExecutorFactory getRepositoryRemoteExecutorFactory() {
     return repositoryRemoteExecutorFactory;
   }
 
+  public Supplier<Downloader> getDownloaderSupplier() {
+    return downloaderSupplier;
+  }
+
   /**
    * A builder for {@link BlazeRuntime} objects. The only required fields are the {@link
    * BlazeDirectories}, and the {@link RuleClassProvider} (except for testing). All other fields
@@ -1589,7 +1598,8 @@ public BlazeRuntime build() throws AbruptExitException {
           productName,
           serverBuilder.getBuildEventArtifactUploaderMap(),
           serverBuilder.getAuthHeadersProvidersMap(),
-          serverBuilder.getRepositoryRemoteExecutorFactory());
+          serverBuilder.getRepositoryRemoteExecutorFactory(),
+          serverBuilder.getDownloaderSupplier());
     }
 
     public Builder setProductName(String productName) {