Skip to content

Commit

Permalink
Add tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@coeuvre
coeuvre committed Feb 13, 2023
1 parent 09c0b5a commit da9e3bf
Show file tree
Hide file tree
Showing 6 changed files with 141 additions and 32 deletions.
13 changes: 13 additions & 0 deletions src/main/java/com/google/devtools/build/lib/actions/ActionCacheUtils.java
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,16 @@
// Copyright 2023 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.actions;

import com.google.devtools.build.lib.actions.cache.ActionCache;
Expand Down
8 changes: 5 additions & 3 deletions src/test/java/com/google/devtools/build/lib/remote/ActionInputPrefetcherTestBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ protected Artifact createRemoteArtifact(
String contents,
@Nullable PathFragment materializationExecPath,
Map<ActionInput, FileArtifactValue> metadata,
Map<HashCode, byte[]> cas) {
@Nullable Map<HashCode, byte[]> cas) {
Path p = artifactRoot.getRoot().getRelative(pathFragment);
Artifact a = ActionsTestUtil.createArtifact(artifactRoot, p);
byte[] contentsBytes = contents.getBytes(UTF_8);
Expand All @@ -103,15 +103,17 @@ protected Artifact createRemoteArtifact(
"action-id",
materializationExecPath);
metadata.put(a, f);
cas.put(hashCode, contentsBytes);
if (cas != null) {
cas.put(hashCode, contentsBytes);
}
return a;
}

protected Artifact createRemoteArtifact(
String pathFragment,
String contents,
Map<ActionInput, FileArtifactValue> metadata,
Map<HashCode, byte[]> cas) {
@Nullable Map<HashCode, byte[]> cas) {
return createRemoteArtifact(
pathFragment, contents, /* materializationExecPath= */ null, metadata, cas);
}
Expand Down
1 change: 1 addition & 0 deletions src/test/java/com/google/devtools/build/lib/remote/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ java_test(
"//src/main/java/com/google/devtools/build/lib/remote",
"//src/main/java/com/google/devtools/build/lib/standalone",
"//src/main/java/com/google/devtools/build/lib/util:os",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/test/java/com/google/devtools/build/lib/remote/util:integration_test_utils",
"//third_party:guava",
"//third_party:junit4",
Expand Down
52 changes: 51 additions & 1 deletion src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import com.google.devtools.build.lib.runtime.BuildSummaryStatsModule;
import com.google.devtools.build.lib.standalone.StandaloneModule;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import java.io.IOException;
import org.junit.After;
import org.junit.Test;
Expand Down Expand Up @@ -69,6 +70,7 @@ protected void setDownloadAll() {
@Override
protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception {
return super.getRuntimeBuilder()
.addBlazeModule(new RemoteModule())
.addBlazeModule(new BuildSummaryStatsModule())
.addBlazeModule(new BlockWaitingModule());
}
Expand All @@ -79,7 +81,6 @@ protected ImmutableList<BlazeModule> getSpawnModules() {
.addAll(super.getSpawnModules())
.add(new StandaloneModule())
.add(new CredentialModule())
.add(new RemoteModule())
.add(new DynamicExecutionModule())
.build();
}
Expand Down Expand Up @@ -415,4 +416,53 @@ public void symlinkToNestedDirectory() throws Exception {

buildTarget("//a:one_local", "//a:two_local", "//a:one_remote", "//a:two_remote");
}

@Test
public void remoteCacheEvictBlobs_incrementalBuildCanContinue() throws Exception {
// Arrange: Prepare workspace and populate remote cache
write(
"a/BUILD",
"genrule(",
" name = 'foo',",
" srcs = ['foo.in'],",
" outs = ['foo.out'],",
" cmd = 'cat $(SRCS) > $@',",
")",
"genrule(",
" name = 'bar',",
" srcs = ['foo.out', 'bar.in'],",
" outs = ['bar.out'],",
" cmd = 'cat $(SRCS) > $@',",
" tags = ['no-remote-exec'],",
")");
write("a/foo.in", "foo");
write("a/bar.in", "bar");

// Populate remote cache
buildTarget("//a:bar");
var bytes = FileSystemUtils.readContent(getOutputPath("a/foo.out"));
var hashCode = getDigestHashFunction().getHashFunction().hashBytes(bytes);
getOutputPath("a/foo.out").delete();
getOutputPath("a/bar.out").delete();
getOutputBase().getRelative("action_cache").deleteTreesBelow();
restartServer();

// Clean build, foo.out isn't downloaded
buildTarget("//a:bar");
assertOutputDoesNotExist("a/foo.out");

// Evict blobs from remote cache
worker.restart();

// trigger build error
write("a/bar.in", "updated bar");
// Build failed because of remote cache eviction
assertThrows(BuildFailedException.class, () -> buildTarget("//a:bar"));

// Act: Do an incremental build without "clean" or "shutdown"
buildTarget("//a:bar");

// Assert: target was successfully built
assertValidOutputFile("a/bar.out", "foo\nupdated bar\n");
}
}
20 changes: 20 additions & 0 deletions src/test/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcherTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,22 @@
package com.google.devtools.build.lib.remote;

import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;

import build.bazel.remote.execution.v2.CacheCapabilities;
import build.bazel.remote.execution.v2.Digest;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.eventbus.EventBus;
import com.google.common.hash.HashCode;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.MetadataProvider;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.remote.common.BulkTransferException;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.remote.util.InMemoryCacheClient;
Expand Down Expand Up @@ -127,6 +132,21 @@ public void testStagingEmptyVirtualActionInput() throws Exception {
assertThat(actionInputFetcher.downloadsInProgress()).isEmpty();
}

@Test
public void missingInputs_addedToList() {
Map<ActionInput, FileArtifactValue> metadata = new HashMap<>();
Map<HashCode, byte[]> cas = new HashMap<>();
Artifact a = createRemoteArtifact("file", "hello world", metadata, /* cas= */ null);
MetadataProvider metadataProvider = new StaticMetadataProvider(metadata);
AbstractActionInputPrefetcher prefetcher = createPrefetcher(cas);

assertThrows(
BulkTransferException.class,
() -> wait(prefetcher.prefetchFiles(metadata.keySet(), metadataProvider)));

assertThat(prefetcher.getMissingActionInputs()).contains(a);
}

private RemoteCache newCache(
RemoteOptions options, DigestUtil digestUtil, Map<HashCode, byte[]> cas) {
Map<Digest, byte[]> cacheEntries = Maps.newHashMapWithExpectedSize(cas.size());
Expand Down
79 changes: 51 additions & 28 deletions src/test/java/com/google/devtools/build/lib/remote/util/IntegrationTestUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@
import java.net.DatagramSocket;
import java.net.ServerSocket;
import java.net.SocketException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Comparator;
import java.util.Random;

/** Integration test utilities. */
Expand Down Expand Up @@ -84,30 +88,9 @@ public static WorkerInstance startWorker(boolean useHttp)
PathFragment casPath = testTmpDir.getRelative("remote.cas_path");
PathFragment pidPath = testTmpDir.getRelative("remote.pid_file");
int workerPort = pickUnusedRandomPort();
ensureMkdir(workPath);
ensureMkdir(casPath);
String workerPath = Runfiles.create().rlocation(WORKER_PATH.getSafePathString());
Subprocess workerProcess =
new SubprocessBuilder()
.setArgv(
ImmutableList.of(
workerPath,
"--work_path=" + workPath.getSafePathString(),
"--cas_path=" + casPath.getSafePathString(),
(useHttp ? "--http_listen_port=" : "--listen_port=") + workerPort,
"--pid_file=" + pidPath))
.start();

File pidFile = new File(pidPath.getSafePathString());
while (!pidFile.exists()) {
if (!workerProcess.isAlive()) {
String message = new String(workerProcess.getErrorStream().readAllBytes(), UTF_8);
throw new IOException("Failed to start worker: " + message);
}
Thread.sleep(1);
}

return new WorkerInstance(workerProcess, workerPort, workPath, casPath, pidPath);
var worker = new WorkerInstance(useHttp, workerPort, workPath, casPath, pidPath);
worker.start();
return worker;
}

private static void ensureMkdir(PathFragment path) throws IOException {
Expand All @@ -121,27 +104,67 @@ private static void ensureMkdir(PathFragment path) throws IOException {
}

public static class WorkerInstance {
private final Subprocess process;
private Subprocess process;
private final boolean useHttp;
private final int port;
private final PathFragment workPath;
private final PathFragment casPath;
private final PathFragment pidPath;

private WorkerInstance(
Subprocess process,
boolean useHttp,
int port,
PathFragment workPath,
PathFragment casPath,
PathFragment pidPath) {
this.process = process;
this.useHttp = useHttp;
this.port = port;
this.workPath = workPath;
this.casPath = casPath;
this.pidPath = pidPath;
}

public void stop() {
private void start() throws IOException, InterruptedException {
ensureMkdir(workPath);
ensureMkdir(casPath);

var workerPath = Runfiles.create().rlocation(WORKER_PATH.getSafePathString());
process =
new SubprocessBuilder()
.setArgv(
ImmutableList.of(
workerPath,
"--work_path=" + workPath.getSafePathString(),
"--cas_path=" + casPath.getSafePathString(),
(useHttp ? "--http_listen_port=" : "--listen_port=") + port,
"--pid_file=" + pidPath))
.start();

File pidFile = new File(pidPath.getSafePathString());
while (!pidFile.exists()) {
if (!process.isAlive()) {
String message = new String(process.getErrorStream().readAllBytes(), UTF_8);
throw new IOException("Failed to start worker: " + message);
}
Thread.sleep(1);
}
}

public void restart() throws IOException, InterruptedException {
stop();
start();
}

public void stop() throws IOException {
process.destroyAndWait();
deleteDir(workPath);
deleteDir(casPath);
}

private static void deleteDir(PathFragment path) throws IOException {
try (var stream = Files.walk(Paths.get(path.getSafePathString()))) {
stream.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
}
}

public int getPort() {
Expand Down

0 comments on commit da9e3bf

Please sign in to comment.