Skip to content

Commit 80a2d7c

Browse files
jmillikin-stripekatre
authored andcommitted
Implementation (but not plumbing) of the gRPC remote downloader
Extracted from bazelbuild#10622 Per discussion on that PR, there's still some unanswered questions about how exactly we plumb the new `Downloader` type into `RemoteModule`. And per bazelbuild#10742 (comment), it is unlikely that even heroic effort from me will get the full end-to-end functionality into v3.0. Given this, to simplify the review, I'm taking some of the bits the reviewer is happy with and moving them to a separate PR. After merger, `GrpcRemoteDownloader` and its tests will exist in the source tree, but will not yet be available as CLI options. R: @michajlo CC: @adunham-stripe @dslomov @EricBurnett @philwo @sstriker Closes bazelbuild#10914. PiperOrigin-RevId: 299908615
1 parent 63b01f7 commit 80a2d7c

File tree

16 files changed

+817
-10
lines changed

16 files changed

+817
-10
lines changed

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ public Path download(
187187

188188
try {
189189
downloader.download(
190-
urls, authHeaders, checksum, destination, eventHandler, clientEnv);
190+
urls, authHeaders, checksum, canonicalId, destination, eventHandler, clientEnv);
191191
} catch (InterruptedIOException e) {
192192
throw new InterruptedException(e.getMessage());
193193
}

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Downloader.java

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ void download(
4343
List<URL> urls,
4444
Map<URI, Map<String, String>> authHeaders,
4545
Optional<Checksum> checksum,
46+
String canonicalId,
4647
Path output,
4748
ExtendedEventHandler eventHandler,
4849
Map<String, String> clientEnv)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
// Copyright 2020 The Bazel Authors. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package com.google.devtools.build.lib.bazel.repository.downloader;
16+
17+
import com.google.common.hash.HashCode;
18+
import com.google.common.hash.Hasher;
19+
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
20+
import java.io.IOException;
21+
import java.io.OutputStream;
22+
import javax.annotation.Nullable;
23+
import javax.annotation.WillCloseWhenClosed;
24+
25+
/**
26+
* Output stream that guarantees its contents matches a hash code.
27+
*
28+
* <p>The actual checksum is computed gradually as the output is written. If it doesn't match, then
29+
* an {@link IOException} will be thrown when {@link #close()} is called. This error will be thrown
30+
* multiple times if these methods are called again for some reason.
31+
*
32+
* <p>Note that as the checksum can only be computed once the stream is closed, data will be written
33+
* to the underlying stream regardless of whether it matches the expected checksum.
34+
*
35+
* <p>This class is not thread safe, but it is safe to message pass this object between threads.
36+
*/
37+
@ThreadCompatible
38+
public final class HashOutputStream extends OutputStream {
39+
40+
private final OutputStream delegate;
41+
private final Hasher hasher;
42+
private final HashCode code;
43+
@Nullable private volatile HashCode actual;
44+
45+
public HashOutputStream(@WillCloseWhenClosed OutputStream delegate, Checksum checksum) {
46+
this.delegate = delegate;
47+
this.hasher = checksum.getKeyType().newHasher();
48+
this.code = checksum.getHashCode();
49+
}
50+
51+
@Override
52+
public void write(int buffer) throws IOException {
53+
hasher.putByte((byte) buffer);
54+
delegate.write(buffer);
55+
}
56+
57+
@Override
58+
public void write(byte[] buffer) throws IOException {
59+
hasher.putBytes(buffer);
60+
delegate.write(buffer);
61+
}
62+
63+
@Override
64+
public void write(byte[] buffer, int offset, int length) throws IOException {
65+
hasher.putBytes(buffer, offset, length);
66+
delegate.write(buffer, offset, length);
67+
}
68+
69+
@Override
70+
public void flush() throws IOException {
71+
delegate.flush();
72+
}
73+
74+
@Override
75+
public void close() throws IOException {
76+
delegate.close();
77+
check();
78+
}
79+
80+
private void check() throws IOException {
81+
if (actual == null) {
82+
actual = hasher.hash();
83+
}
84+
if (!code.equals(actual)) {
85+
throw new UnrecoverableHttpException(
86+
String.format("Checksum was %s but wanted %s", actual, code));
87+
}
88+
}
89+
}

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java

+1
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ public void download(
6262
List<URL> urls,
6363
Map<URI, Map<String, String>> authHeaders,
6464
Optional<Checksum> checksum,
65+
String canonicalId,
6566
Path destination,
6667
ExtendedEventHandler eventHandler,
6768
Map<String, String> clientEnv)

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/UnrecoverableHttpException.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616

1717
import java.io.IOException;
1818

19-
final class UnrecoverableHttpException extends IOException {
19+
/** Indicates an HTTP error that cannot be recovered from. */
20+
public final class UnrecoverableHttpException extends IOException {
2021
UnrecoverableHttpException(String message) {
2122
super(message);
2223
}

src/main/java/com/google/devtools/build/lib/remote/BUILD

+55-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ filegroup(
66
name = "srcs",
77
srcs = glob(["**"]) + [
88
"//src/main/java/com/google/devtools/build/lib/remote/common:srcs",
9+
"//src/main/java/com/google/devtools/build/lib/remote/downloader:srcs",
910
"//src/main/java/com/google/devtools/build/lib/remote/disk:srcs",
1011
"//src/main/java/com/google/devtools/build/lib/remote/http:srcs",
1112
"//src/main/java/com/google/devtools/build/lib/remote/logging:srcs",
@@ -18,13 +19,30 @@ filegroup(
1819

1920
java_library(
2021
name = "remote",
21-
srcs = glob(["*.java"]),
22+
srcs = glob(
23+
["*.java"],
24+
exclude = [
25+
"ExecutionStatusException.java",
26+
"ReferenceCountedChannel.java",
27+
"RemoteRetrier.java",
28+
"RemoteRetrierUtils.java",
29+
"Retrier.java",
30+
],
31+
),
2232
tags = ["bazel"],
33+
exports = [
34+
":ExecutionStatusException",
35+
":ReferenceCountedChannel",
36+
":Retrier",
37+
],
2338
runtime_deps = [
2439
# This is required for client TLS.
2540
"//third_party:netty_tcnative",
2641
],
2742
deps = [
43+
":ExecutionStatusException",
44+
":ReferenceCountedChannel",
45+
":Retrier",
2846
"//src/main/java/com/google/devtools/build/lib:build-base",
2947
"//src/main/java/com/google/devtools/build/lib:events",
3048
"//src/main/java/com/google/devtools/build/lib:packages-internal",
@@ -65,3 +83,39 @@ java_library(
6583
"@remoteapis//:build_bazel_semver_semver_java_proto",
6684
],
6785
)
86+
87+
java_library(
88+
name = "ExecutionStatusException",
89+
srcs = ["ExecutionStatusException.java"],
90+
deps = [
91+
"//third_party:jsr305",
92+
"//third_party/grpc:grpc-jar",
93+
"@googleapis//:google_rpc_status_java_proto",
94+
"@remoteapis//:build_bazel_remote_execution_v2_remote_execution_java_proto",
95+
],
96+
)
97+
98+
java_library(
99+
name = "ReferenceCountedChannel",
100+
srcs = ["ReferenceCountedChannel.java"],
101+
deps = [
102+
"//third_party:netty",
103+
"//third_party/grpc:grpc-jar",
104+
],
105+
)
106+
107+
java_library(
108+
name = "Retrier",
109+
srcs = [
110+
"RemoteRetrier.java",
111+
"RemoteRetrierUtils.java",
112+
"Retrier.java",
113+
],
114+
deps = [
115+
":ExecutionStatusException",
116+
"//src/main/java/com/google/devtools/build/lib/remote/options",
117+
"//third_party:guava",
118+
"//third_party:jsr305",
119+
"//third_party/grpc:grpc-jar",
120+
],
121+
)

src/main/java/com/google/devtools/build/lib/remote/ReferenceCountedChannel.java

+6-5
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,14 @@
2121
import io.netty.util.ReferenceCounted;
2222
import java.util.concurrent.TimeUnit;
2323

24-
/** A wrapper around a {@link io.grpc.ManagedChannel} exposing a reference count.
25-
* When instantiated the reference count is 1. {@link ManagedChannel#shutdown()} will be called
26-
* on the wrapped channel when the reference count reaches 0.
24+
/**
25+
* A wrapper around a {@link io.grpc.ManagedChannel} exposing a reference count. When instantiated
26+
* the reference count is 1. {@link ManagedChannel#shutdown()} will be called on the wrapped channel
27+
* when the reference count reaches 0.
2728
*
28-
* See {@link ReferenceCounted} for more information about reference counting.
29+
* <p>See {@link ReferenceCounted} for more information about reference counting.
2930
*/
30-
class ReferenceCountedChannel extends ManagedChannel implements ReferenceCounted {
31+
public class ReferenceCountedChannel extends ManagedChannel implements ReferenceCounted {
3132

3233
private final ManagedChannel channel;
3334
private final AbstractReferenceCounted referenceCounted = new AbstractReferenceCounted() {

src/main/java/com/google/devtools/build/lib/remote/RemoteRetrier.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,8 @@ public <T> T execute(Callable<T> call) throws IOException, InterruptedException
120120
}
121121
}
122122

123-
static class ExponentialBackoff implements Backoff {
123+
/** Backoff strategy that backs off exponentially. */
124+
public static class ExponentialBackoff implements Backoff {
124125

125126
private final long maxMillis;
126127
private long nextDelayMillis;
@@ -152,7 +153,7 @@ static class ExponentialBackoff implements Backoff {
152153
this.maxAttempts = maxAttempts;
153154
}
154155

155-
ExponentialBackoff(RemoteOptions options) {
156+
public ExponentialBackoff(RemoteOptions options) {
156157
this(
157158
/* initial = */ Duration.ofMillis(100),
158159
/* max = */ Duration.ofSeconds(5),
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
load("@rules_java//java:defs.bzl", "java_library")
2+
3+
package(
4+
default_visibility = ["//src:__subpackages__"],
5+
)
6+
7+
filegroup(
8+
name = "srcs",
9+
srcs = glob(["*"]),
10+
)
11+
12+
java_library(
13+
name = "downloader",
14+
srcs = glob(["*.java"]),
15+
deps = [
16+
"//src/main/java/com/google/devtools/build/lib:events",
17+
"//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
18+
"//src/main/java/com/google/devtools/build/lib/remote",
19+
"//src/main/java/com/google/devtools/build/lib/remote:ReferenceCountedChannel",
20+
"//src/main/java/com/google/devtools/build/lib/remote:Retrier",
21+
"//src/main/java/com/google/devtools/build/lib/remote/common",
22+
"//src/main/java/com/google/devtools/build/lib/remote/options",
23+
"//src/main/java/com/google/devtools/build/lib/remote/util",
24+
"//src/main/java/com/google/devtools/build/lib/vfs",
25+
"//third_party:gson",
26+
"//third_party:guava",
27+
"//third_party/grpc:grpc-jar",
28+
"@remoteapis//:build_bazel_remote_asset_v1_remote_asset_java_grpc",
29+
"@remoteapis//:build_bazel_remote_asset_v1_remote_asset_java_proto",
30+
"@remoteapis//:build_bazel_remote_execution_v2_remote_execution_java_proto",
31+
],
32+
)

0 commit comments

Comments
 (0)