diff --git a/src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java b/src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java index 742c74f3074404..fab5cf97938a7a 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java +++ b/src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java @@ -31,6 +31,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.devtools.build.lib.actions.ActionExecutionMetadata; import com.google.devtools.build.lib.actions.ActionUploadFinishedEvent; import com.google.devtools.build.lib.actions.ActionUploadStartedEvent; @@ -54,10 +55,12 @@ import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.Symlinks; import com.google.protobuf.ByteString; +import com.google.protobuf.CodedOutputStream; import com.google.protobuf.Timestamp; import io.reactivex.rxjava3.core.Completable; import io.reactivex.rxjava3.core.Flowable; import io.reactivex.rxjava3.core.Single; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.time.Duration; import java.time.Instant; @@ -65,9 +68,11 @@ import java.util.Collection; import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -303,25 +308,41 @@ private void addFile(Digest digest, Path file) throws IOException { digestToFile.put(digest, file); } + // Field numbers of the 'root' and 'directory' fields in the Tree message. + private static final int TREE_ROOT_FIELD_NUMBER = 1; + private static final int TREE_CHILDREN_FIELD_NUMBER = 2; + private void addDirectory(Path dir) throws ExecException, IOException { - Tree.Builder tree = Tree.newBuilder(); - Directory root = computeDirectory(dir, tree); - tree.setRoot(root); + Set directories = new LinkedHashSet<>(); + computeDirectory(dir, directories); + + // Convert individual Directory messages to a Tree message. As we want the + // records to be topologically sorted (parents before children), we iterate + // over the directories in reverse insertion order. + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + CodedOutputStream codedOutputStream = CodedOutputStream.newInstance(byteArrayOutputStream); + int fieldNumber = TREE_ROOT_FIELD_NUMBER; + for (ByteString directory : Lists.reverse(new ArrayList(directories))) { + codedOutputStream.writeBytes(fieldNumber, directory); + fieldNumber = TREE_CHILDREN_FIELD_NUMBER; + } + codedOutputStream.flush(); - ByteString data = tree.build().toByteString(); + ByteString data = ByteString.copyFrom(byteArrayOutputStream.toByteArray()); Digest digest = digestUtil.compute(data.toByteArray()); if (result != null) { result .addOutputDirectoriesBuilder() .setPath(remotePathResolver.localPathToOutputPath(dir)) - .setTreeDigest(digest); + .setTreeDigest(digest) + .setIsTopologicallySorted(true); } digestToBlobs.put(digest, data); } - private Directory computeDirectory(Path path, Tree.Builder tree) + private ByteString computeDirectory(Path path, Set directories) throws ExecException, IOException { Directory.Builder b = Directory.newBuilder(); @@ -332,9 +353,8 @@ private Directory computeDirectory(Path path, Tree.Builder tree) String name = dirent.getName(); Path child = path.getRelative(name); if (dirent.getType() == Dirent.Type.DIRECTORY) { - Directory dir = computeDirectory(child, tree); - b.addDirectoriesBuilder().setName(name).setDigest(digestUtil.compute(dir)); - tree.addChildren(dir); + ByteString dir = computeDirectory(child, directories); + b.addDirectoriesBuilder().setName(name).setDigest(digestUtil.compute(dir.toByteArray())); } else if (dirent.getType() == Dirent.Type.SYMLINK) { PathFragment target = child.readSymbolicLink(); if (!followSymlinks && !target.isAbsolute()) { @@ -353,9 +373,8 @@ private Directory computeDirectory(Path path, Tree.Builder tree) b.addFilesBuilder().setName(name).setDigest(digest).setIsExecutable(child.isExecutable()); digestToFile.put(digest, child); } else if (statFollow.isDirectory()) { - Directory dir = computeDirectory(child, tree); - b.addDirectoriesBuilder().setName(name).setDigest(digestUtil.compute(dir)); - tree.addChildren(dir); + ByteString dir = computeDirectory(child, directories); + b.addDirectoriesBuilder().setName(name).setDigest(digestUtil.compute(dir.toByteArray())); } else { illegalOutput(child); } @@ -368,7 +387,9 @@ private Directory computeDirectory(Path path, Tree.Builder tree) } } - return b.build(); + ByteString directory = b.build().toByteString(); + directories.add(directory); + return directory; } private void illegalOutput(Path path) throws ExecException { diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java index 4df12d4cb1e107..1bc5be0235c44e 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java @@ -369,7 +369,10 @@ public void updateActionResult( .setPath("a/foo") .setDigest(fooDigest) .setIsExecutable(true); - expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("bar") + .setTreeDigest(barDigest) + .setIsTopologicallySorted(true); assertThat(result).isEqualTo(expectedResult.build()); } @@ -409,7 +412,10 @@ public void updateActionResult( ActionResult result = uploadDirectory(remoteCache, ImmutableList.of(barDir)); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("bar") + .setTreeDigest(barDigest) + .setIsTopologicallySorted(true); assertThat(result).isEqualTo(expectedResult.build()); } @@ -472,7 +478,10 @@ public void updateActionResult( ActionResult result = uploadDirectory(remoteCache, ImmutableList.of(barDir)); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("bar") + .setTreeDigest(barDigest) + .setIsTopologicallySorted(true); assertThat(result).isEqualTo(expectedResult.build()); } diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java index dc69fbe03f8042..2d1a8fe87602c4 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java @@ -1345,7 +1345,10 @@ public void uploadOutputs_uploadDirectory_works() throws Exception { .setPath("outputs/a/foo") .setDigest(fooDigest) .setIsExecutable(true); - expectedResult.addOutputDirectoriesBuilder().setPath("outputs/bar").setTreeDigest(barDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("outputs/bar") + .setTreeDigest(barDigest) + .setIsTopologicallySorted(true); assertThat(manifest.getActionResult()).isEqualTo(expectedResult.build()); ImmutableList toQuery = ImmutableList.of(fooDigest, quxDigest, barDigest); @@ -1383,7 +1386,10 @@ public void uploadOutputs_uploadEmptyDirectory_works() throws Exception { // assert ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("outputs/bar").setTreeDigest(barDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("outputs/bar") + .setTreeDigest(barDigest) + .setIsTopologicallySorted(true); assertThat(manifest.getActionResult()).isEqualTo(expectedResult.build()); assertThat( getFromFuture( @@ -1448,7 +1454,10 @@ public void uploadOutputs_uploadNestedDirectory_works() throws Exception { // assert ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("outputs/bar").setTreeDigest(barDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("outputs/bar") + .setTreeDigest(barDigest) + .setIsTopologicallySorted(true); assertThat(manifest.getActionResult()).isEqualTo(expectedResult.build()); ImmutableList toQuery = ImmutableList.of(wobbleDigest, quxDigest, barDigest); diff --git a/src/test/java/com/google/devtools/build/lib/remote/UploadManifestTest.java b/src/test/java/com/google/devtools/build/lib/remote/UploadManifestTest.java index bc758073d7a9c9..b1a3a2f7594174 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/UploadManifestTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/UploadManifestTest.java @@ -41,6 +41,8 @@ import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -249,7 +251,10 @@ public void actionResult_followSymlinks_absoluteDirectorySymlinkAsDirectory() th Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("link").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("link") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -309,7 +314,10 @@ public void actionResult_noFollowSymlinks_absoluteDirectorySymlinkAsDirectory() Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("link").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("link") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -369,7 +377,10 @@ public void actionResult_followSymlinks_relativeDirectorySymlinkAsDirectory() th Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("link").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("link") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -570,7 +581,10 @@ public void actionResult_followSymlinks_absoluteFileSymlinkInDirectoryAsFile() t Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("dir") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -615,7 +629,10 @@ public void actionResult_followSymlinks_absoluteDirectorySymlinkInDirectoryAsDir Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("dir") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -651,7 +668,10 @@ public void actionResult_noFollowSymlinks_absoluteFileSymlinkInDirectoryAsFile() Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("dir") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -696,7 +716,10 @@ public void actionResult_noFollowSymlinks_absoluteDirectorySymlinkInDirectoryAsD Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("dir") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -731,7 +754,10 @@ public void actionResult_followSymlinks_relativeFileSymlinkInDirectoryAsFile() t Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("dir") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -776,7 +802,10 @@ public void actionResult_followSymlinks_relativeDirectorySymlinkInDirectoryAsDir Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("dir") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -811,7 +840,10 @@ public void actionResult_noFollowSymlinks_relativeFileSymlinkInDirectoryAsSymlin Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("dir") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -848,7 +880,10 @@ public void actionResult_noFollowSymlinks_relativeDirectorySymlinkInDirectoryAsS Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("dir") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -933,7 +968,10 @@ public void actionResult_noFollowSymlinks_relativeDirectorySymlinkInDirectoryAsS Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("dir") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -968,7 +1006,10 @@ public void actionResult_noFollowSymlinks_relativeDirectorySymlinkInDirectoryAsS Digest treeDigest = digestUtil.compute(tree); ActionResult.Builder expectedResult = ActionResult.newBuilder(); - expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); + expectedResult.addOutputDirectoriesBuilder() + .setPath("dir") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); assertThat(result.build()).isEqualTo(expectedResult.build()); } @@ -1053,6 +1094,76 @@ public void actionResult_followSymlinks_specialFileSymlinkInDirectoryError() thr assertThat(e).hasMessageThat().contains("dir/link"); } + @Test + public void actionResult_topologicallySortedAndDeduplicatedTree() + throws Exception { + // Create 5^3 identical files named "dir/%d/%d/%d/file". + Path dir = execRoot.getRelative("dir"); + dir.createDirectory(); + byte fileContents[] = new byte[] {1, 2, 3, 4, 5}; + final int childrenPerDirectory = 5; + for (int a = 0; a < childrenPerDirectory; a++) { + Path pathA = dir.getRelative(Integer.toString(a)); + pathA.createDirectory(); + for (int b = 0; b < childrenPerDirectory; b++) { + Path pathB = pathA.getRelative(Integer.toString(b)); + pathB.createDirectory(); + for (int c = 0; c < childrenPerDirectory; c++) { + Path pathC = pathB.getRelative(Integer.toString(c)); + pathC.createDirectory(); + Path file = pathC.getRelative("file"); + FileSystemUtils.writeContent(file, fileContents); + } + } + } + + ActionResult.Builder result = ActionResult.newBuilder(); + UploadManifest um = + new UploadManifest( + digestUtil, + remotePathResolver, + result, + /*followSymlinks=*/ false, + /*allowDanglingSymlinks=*/ false, + /*allowAbsoluteSymlinks=*/ false); + um.addFiles(ImmutableList.of(dir)); + + // Even though we constructed 1 + 5 + 5^2 + 5^3 directories, the resulting + // Tree message should only contain four unique instances. The directories + // should also be topologically sorted. + List children = new ArrayList<>(); + Directory root = Directory.newBuilder() + .addFiles( + FileNode.newBuilder(). + setName("file").setDigest(digestUtil.compute(fileContents))) + .build(); + for (int depth = 0; depth < 3; depth++) { + Directory.Builder b = Directory.newBuilder(); + Digest parentDigest = digestUtil.compute(root.toByteArray()); + for (int i = 0; i < childrenPerDirectory; i++) { + b.addDirectories( + DirectoryNode.newBuilder() + .setName(Integer.toString(i)) + .setDigest(parentDigest)); + } + children.add(0, root); + root = b.build(); + } + Tree tree = + Tree.newBuilder() + .setRoot(root) + .addAllChildren(children) + .build(); + Digest treeDigest = digestUtil.compute(tree); + + ActionResult.Builder expectedResult = ActionResult.newBuilder(); + expectedResult.addOutputDirectoriesBuilder() + .setPath("dir") + .setTreeDigest(treeDigest) + .setIsTopologicallySorted(true); + assertThat(result.build()).isEqualTo(expectedResult.build()); + } + private Path createSpecialFile(String execPath) throws IOException { Path special = mock(Path.class); when(special.statIfFound(Symlinks.NOFOLLOW)).thenReturn(SPECIAL_FILE_STATUS); diff --git a/third_party/remoteapis/build/bazel/remote/asset/v1/remote_asset.proto b/third_party/remoteapis/build/bazel/remote/asset/v1/remote_asset.proto index e11fc7b174a7ad..4d9be8175a9885 100644 --- a/third_party/remoteapis/build/bazel/remote/asset/v1/remote_asset.proto +++ b/third_party/remoteapis/build/bazel/remote/asset/v1/remote_asset.proto @@ -23,7 +23,7 @@ import "google/protobuf/timestamp.proto"; import "google/rpc/status.proto"; option csharp_namespace = "Build.Bazel.Remote.Asset.v1"; -option go_package = "remoteasset"; +option go_package = "github.com/bazelbuild/remote-apis/build/bazel/remote/asset/v1;remoteasset"; option java_multiple_files = true; option java_outer_classname = "RemoteAssetProto"; option java_package = "build.bazel.remote.asset.v1"; diff --git a/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto b/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto index 9b5806449a5fcf..60753f78ab59a5 100644 --- a/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto +++ b/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto @@ -26,7 +26,7 @@ import "google/protobuf/wrappers.proto"; import "google/rpc/status.proto"; option csharp_namespace = "Build.Bazel.Remote.Execution.V2"; -option go_package = "remoteexecution"; +option go_package = "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2;remoteexecution"; option java_multiple_files = true; option java_outer_classname = "RemoteExecutionProto"; option java_package = "build.bazel.remote.execution.v2"; @@ -255,10 +255,11 @@ service ActionCache { // // When attempting an upload, if another client has already completed the upload // (which may occur in the middle of a single upload if another client uploads -// the same blob concurrently), the request will terminate immediately with -// a response whose `committed_size` is the full size of the uploaded file -// (regardless of how much data was transmitted by the client). If the client -// completes the upload but the +// the same blob concurrently), the request will terminate immediately without +// error, and with a response whose `committed_size` is the value `-1` if this +// is a compressed upload, or with the full size of the uploaded file if this is +// an uncompressed upload (regardless of how much data was transmitted by the +// client). If the client completes the upload but the // [Digest][build.bazel.remote.execution.v2.Digest] does not match, an // `INVALID_ARGUMENT` error will be returned. In either case, the client should // not attempt to retry the upload. @@ -423,6 +424,8 @@ service Capabilities { // CacheCapabilities and ExecutionCapabilities. // * Execution only endpoints should return ExecutionCapabilities. // * CAS + Action Cache only endpoints should return CacheCapabilities. + // + // There are no method-specific errors. rpc GetCapabilities(GetCapabilitiesRequest) returns (ServerCapabilities) { option (google.api.http) = { get: "/v2/{instance_name=**}/capabilities" @@ -475,6 +478,14 @@ message Action { // timeout that is longer than the server's maximum timeout, the server MUST // reject the request. // + // The timeout is only intended to cover the "execution" of the specified + // action and not time in queue nor any overheads before or after execution + // such as marshalling inputs/outputs. The server SHOULD avoid including time + // spent the client doesn't have control over, and MAY extend or reduce the + // timeout to account for delays or speedups that occur during execution + // itself (e.g., lazily loading data from the Content Addressable Storage, + // live migration of virtual machines, emulation overhead). + // // The timeout is a part of the // [Action][build.bazel.remote.execution.v2.Action] message, and // therefore two `Actions` with different timeouts are different, even if they @@ -529,9 +540,21 @@ message Command { string value = 2; } - // The arguments to the command. The first argument must be the path to the - // executable, which must be either a relative path, in which case it is - // evaluated with respect to the input root, or an absolute path. + // The arguments to the command. + // + // The first argument specifies the command to run, which may be either an + // absolute path, a path relative to the working directory, or an unqualified + // path (without path separators) which will be resolved using the operating + // system's equivalent of the PATH environment variable. Path separators + // native to the operating system running on the worker SHOULD be used. If the + // `environment_variables` list contains an entry for the PATH environment + // variable, it SHOULD be respected. If not, the resolution process is + // implementation-defined. + // + // Changed in v2.3. v2.2 and older require that no PATH lookups are performed, + // and that relative paths are resolved relative to the input root. This + // behavior can, however, not be relied upon, as most implementations already + // followed the rules described above. repeated string arguments = 1; // The environment variables to set when running the program. The worker may @@ -605,10 +628,10 @@ message Command { // The type of the output (file or directory) is not specified, and will be // determined by the server after action execution. If the resulting path is // a file, it will be returned in an - // [OutputFile][build.bazel.remote.execution.v2.OutputFile]) typed field. + // [OutputFile][build.bazel.remote.execution.v2.OutputFile] typed field. // If the path is a directory, the entire directory structure will be returned // as a [Tree][build.bazel.remote.execution.v2.Tree] message digest, see - // [OutputDirectory][build.bazel.remote.execution.v2.OutputDirectory]) + // [OutputDirectory][build.bazel.remote.execution.v2.OutputDirectory] // Other files or directories that may be created during command execution // are discarded. // @@ -942,6 +965,25 @@ message ExecutedActionMetadata { // When the worker completed executing the action command. google.protobuf.Timestamp execution_completed_timestamp = 8; + // New in v2.3: the amount of time the worker spent executing the action + // command, potentially computed using a worker-specific virtual clock. + // + // The virtual execution duration is only intended to cover the "execution" of + // the specified action and not time in queue nor any overheads before or + // after execution such as marshalling inputs/outputs. The server SHOULD avoid + // including time spent the client doesn't have control over, and MAY extend + // or reduce the execution duration to account for delays or speedups that + // occur during execution itself (e.g., lazily loading data from the Content + // Addressable Storage, live migration of virtual machines, emulation + // overhead). + // + // The method of timekeeping used to compute the virtual execution duration + // MUST be consistent with what is used to enforce the + // [Action][[build.bazel.remote.execution.v2.Action]'s `timeout`. There is no + // relationship between the virtual execution duration and the values of + // `execution_start_timestamp` and `execution_completed_timestamp`. + google.protobuf.Duration virtual_execution_duration = 12; + // When the worker started uploading action outputs. google.protobuf.Timestamp output_upload_start_timestamp = 9; @@ -1105,6 +1147,7 @@ message ActionResult { // [GetActionResultRequest][build.bazel.remote.execution.v2.GetActionResultRequest] // message. The server MAY omit inlining, even if requested, and MUST do so if inlining // would cause the response to exceed message size limits. + // Clients SHOULD NOT populate this field when uploading to the cache. bytes stdout_raw = 5; // The digest for a blob containing the standard output of the action, which @@ -1117,6 +1160,7 @@ message ActionResult { // [GetActionResultRequest][build.bazel.remote.execution.v2.GetActionResultRequest] // message. The server MAY omit inlining, even if requested, and MUST do so if inlining // would cause the response to exceed message size limits. + // Clients SHOULD NOT populate this field when uploading to the cache. bytes stderr_raw = 7; // The digest for a blob containing the standard error of the action, which @@ -1151,6 +1195,7 @@ message OutputFile { // [GetActionResultRequest][build.bazel.remote.execution.v2.GetActionResultRequest] // message. The server MAY omit inlining, even if requested, and MUST do so if inlining // would cause the response to exceed message size limits. + // Clients SHOULD NOT populate this field when uploading to the cache. bytes contents = 5; // The supported node properties of the OutputFile, if requested by the Action. @@ -1169,6 +1214,9 @@ message Tree { // recursively, all its children. In order to reconstruct the directory tree, // the client must take the digests of each of the child directories and then // build up a tree starting from the `root`. + // Servers SHOULD ensure that these are ordered consistently such that two + // actions producing equivalent output directories on the same server + // implementation also produce Tree messages with matching digests. repeated Directory children = 2; } @@ -1187,6 +1235,43 @@ message OutputDirectory { // [Tree][build.bazel.remote.execution.v2.Tree] proto containing the // directory's contents. Digest tree_digest = 3; + + // If set, consumers MAY make the following assumptions about the + // directories contained in the the Tree, so that it may be + // instantiated on a local file system by scanning through it + // sequentially: + // + // - All directories with the same binary representation are stored + // exactly once. + // - All directories, apart from the root directory, are referenced by + // at least one parent directory. + // - Directories are stored in topological order, with parents being + // stored before the child. The root directory is thus the first to + // be stored. + // + // Additionally, the Tree MUST be encoded as a stream of records, + // where each record has the following format: + // + // - A tag byte, having one of the following two values: + // - (1 << 3) | 2 == 0x0a: First record (the root directory). + // - (2 << 3) | 2 == 0x12: Any subsequent records (child directories). + // - The size of the directory, encoded as a base 128 varint. + // - The contents of the directory, encoded as a binary serialized + // Protobuf message. + // + // This encoding is a subset of the Protobuf wire format of the Tree + // message. As it is only permitted to store data associated with + // field numbers 1 and 2, the tag MUST be encoded as a single byte. + // More details on the Protobuf wire format can be found here: + // https://developers.google.com/protocol-buffers/docs/encoding + // + // It is recommended that implementations using this feature construct + // Tree objects manually using the specification given above, as + // opposed to using a Protobuf library to marshal a full Tree message. + // As individual Directory messages already need to be marshaled to + // compute their digests, constructing the Tree object manually avoids + // redundant marshaling. + bool is_topologically_sorted = 4; } // An `OutputSymlink` is similar to a @@ -1334,6 +1419,17 @@ message ExecuteResponse { } // The current stage of action execution. +// +// Even though these stages are numbered according to the order in which +// they generally occur, there is no requirement that the remote +// execution system reports events along this order. For example, an +// operation MAY transition from the EXECUTING stage back to QUEUED +// in case the hardware on which the operation executes fails. +// +// If and only if the remote execution system reports that an operation +// has reached the COMPLETED stage, it MUST set the [done +// field][google.longrunning.Operation.done] of the +// [Operation][google.longrunning.Operation] and terminate the stream. message ExecutionStage { enum Value { // Invalid value. @@ -1469,6 +1565,12 @@ message BatchUpdateBlobsRequest { // The raw binary data. bytes data = 2; + + // The format of `data`. Must be `IDENTITY`/unspecified, or one of the + // compressors advertised by the + // [CacheCapabilities.supported_batch_compressors][build.bazel.remote.execution.v2.CacheCapabilities.supported_batch_compressors] + // field. + Compressor.Value compressor = 3; } // The instance of the execution system to operate against. A server may @@ -1510,6 +1612,10 @@ message BatchReadBlobsRequest { // The individual blob digests. repeated Digest digests = 2; + + // A list of acceptable encodings for the returned inlined data, in no + // particular order. `IDENTITY` is always allowed even if not specified here. + repeated Compressor.Value acceptable_compressors = 3; } // A response message for @@ -1523,6 +1629,10 @@ message BatchReadBlobsResponse { // The raw binary data. bytes data = 2; + // The format the data is encoded in. MUST be `IDENTITY`/unspecified, + // or one of the acceptable compressors specified in the `BatchReadBlobsRequest`. + Compressor.Value compressor = 4; + // The result of attempting to download that blob. google.rpc.Status status = 3; } @@ -1724,6 +1834,11 @@ message CacheCapabilities { // Note that this does not imply which if any compressors are supported by // the server at the gRPC level. repeated Compressor.Value supported_compressors = 6; + + // Compressors supported for inlined data in + // [BatchUpdateBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchUpdateBlobs] + // requests. + repeated Compressor.Value supported_batch_update_compressors = 7; } // Capabilities of the remote execution system. diff --git a/third_party/remoteapis/build/bazel/semver/semver.proto b/third_party/remoteapis/build/bazel/semver/semver.proto index 3b626b7e47c050..44f83f857648af 100644 --- a/third_party/remoteapis/build/bazel/semver/semver.proto +++ b/third_party/remoteapis/build/bazel/semver/semver.proto @@ -17,7 +17,7 @@ syntax = "proto3"; package build.bazel.semver; option csharp_namespace = "Build.Bazel.Semver"; -option go_package = "semver"; +option go_package = "github.com/bazelbuild/remote-apis/build/bazel/semver"; option java_multiple_files = true; option java_outer_classname = "SemverProto"; option java_package = "build.bazel.semver";