diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java index 92303949c79a25..ce283b1accbc75 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java @@ -381,11 +381,7 @@ public DiscoveredModulesPruner getDiscoveredModulesPruner() { return discoveredModulesPruner; } - /** - * This only exists for loose header checking (and shouldn't exist at all). - * - *

Do NOT use from any other place. - */ + /** This only exists for loose header checking and as a helper for digest computations. */ public SyscallCache getSyscallCache() { return syscallCache; } diff --git a/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java b/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java index d74ed514535e7d..4166cc0b00628c 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java +++ b/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java @@ -29,6 +29,7 @@ import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.SkyValue; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -189,8 +190,8 @@ interface Singleton {} @SerializationConstant public static final FileArtifactValue OMITTED_FILE_MARKER = new OmittedFileValue(); - public static FileArtifactValue createForSourceArtifact(Artifact artifact, FileValue fileValue) - throws IOException { + public static FileArtifactValue createForSourceArtifact( + Artifact artifact, FileValue fileValue, SyscallCache syscallCache) throws IOException { // Artifacts with known generating actions should obtain the derived artifact's SkyValue // from the generating action, instead. Preconditions.checkState(!artifact.hasKnownGeneratingAction()); @@ -201,7 +202,8 @@ public static FileArtifactValue createForSourceArtifact(Artifact artifact, FileV isFile, isFile ? fileValue.getSize() : 0, isFile ? fileValue.realFileStateValue().getContentsProxy() : null, - isFile ? fileValue.getDigest() : null); + isFile ? fileValue.getDigest() : null, + syscallCache); } public static FileArtifactValue createFromInjectedDigest( @@ -219,16 +221,27 @@ public static FileArtifactValue createForTesting(Path path) throws IOException { // Caution: there's a race condition between stating the file and computing the digest. We need // to stat first, since we're using the stat to detect changes. We follow symlinks here to be // consistent with getDigest. - return createFromStat(path, path.stat(Symlinks.FOLLOW)); + return createFromStat(path, path.stat(Symlinks.FOLLOW), SyscallCache.NO_CACHE); } - public static FileArtifactValue createFromStat(Path path, FileStatus stat) throws IOException { + public static FileArtifactValue createFromStat( + Path path, FileStatus stat, SyscallCache syscallCache) throws IOException { return create( - path, stat.isFile(), stat.getSize(), FileContentsProxy.create(stat), /*digest=*/ null); + path, + stat.isFile(), + stat.getSize(), + FileContentsProxy.create(stat), + /*digest=*/ null, + syscallCache); } private static FileArtifactValue create( - Path path, boolean isFile, long size, FileContentsProxy proxy, @Nullable byte[] digest) + Path path, + boolean isFile, + long size, + FileContentsProxy proxy, + @Nullable byte[] digest, + SyscallCache syscallCache) throws IOException { if (!isFile) { // In this case, we need to store the mtime because the action cache uses mtime for @@ -237,7 +250,7 @@ private static FileArtifactValue create( return new DirectoryArtifactValue(path.getLastModifiedTime()); } if (digest == null) { - digest = DigestUtils.getDigestWithManualFallback(path, size); + digest = DigestUtils.getDigestWithManualFallback(path, size, syscallCache); } Preconditions.checkState(digest != null, path); return createForNormalFile(digest, proxy, size); @@ -275,9 +288,9 @@ public static FileArtifactValue createForNormalFile( * Create a FileArtifactValue using the {@link Path} and size. FileArtifactValue#create will * handle getting the digest using the Path and size values. */ - public static FileArtifactValue createForNormalFileUsingPath(Path path, long size) - throws IOException { - return create(path, /*isFile=*/ true, size, /*proxy=*/ null, /*digest=*/ null); + public static FileArtifactValue createForNormalFileUsingPath( + Path path, long size, SyscallCache syscallCache) throws IOException { + return create(path, /*isFile=*/ true, size, /*proxy=*/ null, /*digest=*/ null, syscallCache); } public static FileArtifactValue createForDirectoryWithHash(byte[] digest) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD index b6f8718b65d7bd..47e7e4e3326f04 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD @@ -166,6 +166,7 @@ java_library( srcs = ["ResolvedEvent.java"], deps = [ "//src/main/java/com/google/devtools/build/lib/events", + "//src/main/java/com/google/devtools/build/lib/vfs", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index ec7fd5a0d044d2..12ab3a33148ecc 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -270,6 +270,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { ProcessWrapper processWrapper = ProcessWrapper.fromCommandEnvironment(env); starlarkRepositoryFunction.setProcessWrapper(processWrapper); + starlarkRepositoryFunction.setSyscallCache(env.getSyscallCache()); singleExtensionEvalFunction.setProcessWrapper(processWrapper); RepositoryOptions repoOptions = env.getOptions().getOptions(RepositoryOptions.class); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/ResolvedEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/ResolvedEvent.java index 89c9cc14c4ce49..af12b47619de80 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/ResolvedEvent.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/ResolvedEvent.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.bazel; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.vfs.SyscallCache; /** Interface for events reporting information to be added to a resolved file. */ public interface ResolvedEvent extends ExtendedEventHandler.ProgressLike { @@ -22,5 +23,5 @@ public interface ResolvedEvent extends ExtendedEventHandler.ProgressLike { String getName(); /** The entry for the list of resolved Information. */ - Object getResolvedInformation(); + Object getResolvedInformation(SyscallCache syscallCache); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java b/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java index 051db540190b63..030ea2dbc5e7fd 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java @@ -117,7 +117,10 @@ private void initOutputs(CommandEnvironment env) throws IOException { spawnLogContext = new SpawnLogContext( - env.getExecRoot(), outStream, env.getOptions().getOptions(RemoteOptions.class)); + env.getExecRoot(), + outStream, + env.getOptions().getOptions(RemoteOptions.class), + env.getSyscallCache()); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java index bd29405836091d..8c84cb0b6c333d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java @@ -52,6 +52,7 @@ import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; import com.google.devtools.build.skyframe.SkyKey; @@ -263,7 +264,7 @@ public String getName() { } @Override - public Object getResolvedInformation() { + public Object getResolvedInformation(SyscallCache syscallCache) { return ImmutableMap.builder() .put(ResolvedHashesFunction.ORIGINAL_RULE_CLASS, "bind") .put( @@ -295,7 +296,7 @@ public String getName() { } @Override - public Object getResolvedInformation() { + public Object getResolvedInformation(SyscallCache syscallCache) { return ImmutableMap.builder() .put(ResolvedHashesFunction.ORIGINAL_RULE_CLASS, ruleName) .put( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalConfigPlatformFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalConfigPlatformFunction.java index f57f63c1304f86..2672784b3f8557 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalConfigPlatformFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalConfigPlatformFunction.java @@ -26,6 +26,7 @@ import com.google.devtools.build.lib.util.CPU; import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; @@ -85,7 +86,7 @@ public String getName() { } @Override - public Object getResolvedInformation() { + public Object getResolvedInformation(SyscallCache syscallCache) { String repr = String.format("local_config_platform(name = '%s')", name); return ImmutableMap.builder() .put(ResolvedHashesFunction.ORIGINAL_RULE_CLASS, LocalConfigPlatformRule.NAME) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java index 0d9d01fba68641..97f8dca2cd9273 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java @@ -30,6 +30,7 @@ import com.google.devtools.build.lib.packages.StructImpl; import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.SyscallCache; import java.io.IOException; import java.util.List; import java.util.Map; @@ -164,13 +165,13 @@ public RepositoryResolvedEvent(Rule rule, StructImpl attrs, Path outputDirectory * Ensure that the {@code resolvedInformation} and the {@code directoryDigest} fields are * initialized properly. Does nothing, if the values are computed already. */ - private synchronized void finalizeResolvedInformation() { + private synchronized void finalizeResolvedInformation(SyscallCache syscallCache) { if (resolvedInformation != null) { return; } String digest = "[unavailable]"; try { - digest = outputDirectory.getDirectoryDigest(); + digest = outputDirectory.getDirectoryDigest(syscallCache); repositoryBuilder.put(OUTPUT_TREE_HASH, digest); } catch (IOException e) { // Digest not available, but we still have to report that a repository rule @@ -190,8 +191,8 @@ private synchronized void finalizeResolvedInformation() { * Returns the entry for the given rule invocation in a format suitable for WORKSPACE.resolved. */ @Override - public Object getResolvedInformation() { - finalizeResolvedInformation(); + public Object getResolvedInformation(SyscallCache syscallCache) { + finalizeResolvedInformation(syscallCache); return resolvedInformation; } @@ -201,8 +202,8 @@ public String getName() { return name; } - public String getDirectoryDigest() { - finalizeResolvedInformation(); + public String getDirectoryDigest(SyscallCache syscallCache) { + finalizeResolvedInformation(syscallCache); return directoryDigest; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java index ccbc982c7073ee..5f17822eff410c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java @@ -24,6 +24,7 @@ import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.common.options.OptionsBase; import java.io.File; import java.io.IOException; @@ -41,6 +42,7 @@ public final class RepositoryResolvedModule extends BlazeModule { private Map resolvedValues; private String resolvedFile; private ImmutableList orderedNames; + private SyscallCache syscallCache; @Override public Iterable> getCommandOptions(Command command) { @@ -61,6 +63,7 @@ public void beforeCommand(CommandEnvironment env) { } else { this.resolvedFile = null; } + this.syscallCache = env.getSyscallCache(); } @Override @@ -97,7 +100,7 @@ public void repositoryOrderEvent(RepositoryOrderEvent event) { @Subscribe public void resolved(ResolvedEvent event) { if (resolvedValues != null) { - resolvedValues.put(event.getName(), event.getResolvedInformation()); + resolvedValues.put(event.getName(), event.getResolvedInformation(syscallCache)); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java index 90414eb424297e..138c756f1584d6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java @@ -43,6 +43,7 @@ import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; @@ -66,6 +67,7 @@ public class StarlarkRepositoryFunction extends RepositoryFunction { private double timeoutScaling = 1.0; @Nullable private ProcessWrapper processWrapper = null; @Nullable private RepositoryRemoteExecutor repositoryRemoteExecutor; + @Nullable private SyscallCache syscallCache; public StarlarkRepositoryFunction(DownloadManager downloadManager) { this.downloadManager = downloadManager; @@ -79,6 +81,10 @@ public void setProcessWrapper(@Nullable ProcessWrapper processWrapper) { this.processWrapper = processWrapper; } + public void setSyscallCache(SyscallCache syscallCache) { + this.syscallCache = syscallCache; + } + static String describeSemantics(StarlarkSemantics semantics) { // Here we use the hash code provided by AutoValue. This is unique, as long // as the number of bits in the StarlarkSemantics is small enough. We will have to @@ -236,7 +242,7 @@ public RepositoryDirectoryValue.Builder fetch( if (verificationRules.contains(ruleClass)) { String expectedHash = resolvedHashes.get(rule.getName()); if (expectedHash != null) { - String actualHash = resolved.getDirectoryDigest(); + String actualHash = resolved.getDirectoryDigest(syscallCache); if (!expectedHash.equals(actualHash)) { throw new RepositoryFunctionException( new IOException( diff --git a/src/main/java/com/google/devtools/build/lib/exec/RunfilesTreeUpdater.java b/src/main/java/com/google/devtools/build/lib/exec/RunfilesTreeUpdater.java index 3316af9f9352c7..57bdd64e751491 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/RunfilesTreeUpdater.java +++ b/src/main/java/com/google/devtools/build/lib/exec/RunfilesTreeUpdater.java @@ -24,6 +24,7 @@ import com.google.devtools.build.lib.vfs.DigestUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import java.io.IOException; import java.util.Arrays; import java.util.HashMap; @@ -66,7 +67,8 @@ private static void updateRunfilesTree( BinTools binTools, ImmutableMap env, OutErr outErr, - boolean enableRunfiles) + boolean enableRunfiles, + SyscallCache syscallCache) throws IOException, ExecException, InterruptedException { Path runfilesDirPath = execRoot.getRelative(runfilesDir); Path inputManifest = RunfilesSupport.inputManifestPath(runfilesDirPath); @@ -82,8 +84,9 @@ private static void updateRunfilesTree( // an up-to-date check. if (!outputManifest.isSymbolicLink() && Arrays.equals( - DigestUtils.getDigestWithManualFallbackWhenSizeUnknown(outputManifest), - DigestUtils.getDigestWithManualFallbackWhenSizeUnknown(inputManifest))) { + DigestUtils.getDigestWithManualFallbackWhenSizeUnknown(outputManifest, syscallCache), + DigestUtils.getDigestWithManualFallbackWhenSizeUnknown( + inputManifest, syscallCache))) { return; } } catch (IOException e) { @@ -131,7 +134,8 @@ public void updateRunfilesDirectory( RunfilesSupplier runfilesSupplier, BinTools binTools, ImmutableMap env, - OutErr outErr) + OutErr outErr, + SyscallCache syscallCache) throws ExecException, IOException, InterruptedException { for (Map.Entry> runfiles : runfilesSupplier.getMappings().entrySet()) { @@ -154,7 +158,8 @@ public void updateRunfilesDirectory( binTools, env, outErr, - runfilesSupplier.isRunfileLinksEnabled(runfilesDir)); + runfilesSupplier.isRunfileLinksEnabled(runfilesDir), + syscallCache); } } finally { decrementRefcnt(runfilesDir); diff --git a/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java b/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java index 1a0ff6284ca715..cc4fda2a5b81d1 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java +++ b/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java @@ -23,6 +23,7 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.lib.vfs.SyscallCache; import java.io.IOException; import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; @@ -46,8 +47,10 @@ public class SingleBuildFileCache implements MetadataProvider { // unlikely that this default will adversely affect memory in most cases. .initialCapacity(10000) .build(); + private final SyscallCache syscallCache; - public SingleBuildFileCache(String cwd, FileSystem fs) { + public SingleBuildFileCache(String cwd, FileSystem fs, SyscallCache syscallCache) { + this.syscallCache = syscallCache; this.execRoot = fs.getPath(cwd); } @@ -60,7 +63,12 @@ public FileArtifactValue getMetadata(ActionInput input) throws IOException { Path path = ActionInputHelper.toInputPath(input, execRoot); FileArtifactValue metadata; try { - metadata = FileArtifactValue.createFromStat(path, path.stat(Symlinks.FOLLOW)); + metadata = + FileArtifactValue.createFromStat( + path, + // TODO(b/199940216): should we use syscallCache here since caching anyway? + path.stat(Symlinks.FOLLOW), + syscallCache); } catch (IOException e) { return new ActionInputMetadata(input, e); } diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java index b1eab176103c58..1ca1e83eb33f70 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java +++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java @@ -38,6 +38,7 @@ import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.protobuf.util.Durations; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -57,17 +58,22 @@ * A logging utility for spawns. */ public class SpawnLogContext implements ActionContext { - private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + private final Path execRoot; private final MessageOutputStream executionLog; @Nullable private final RemoteOptions remoteOptions; + private final SyscallCache syscallCache; public SpawnLogContext( - Path execRoot, MessageOutputStream executionLog, @Nullable RemoteOptions remoteOptions) { + Path execRoot, + MessageOutputStream executionLog, + @Nullable RemoteOptions remoteOptions, + SyscallCache syscallCache) { this.execRoot = execRoot; this.executionLog = executionLog; this.remoteOptions = remoteOptions; + this.syscallCache = syscallCache; } /** Log the executed spawn to the output stream. */ @@ -99,7 +105,7 @@ public void logSpawn( if (inputPath.isDirectory()) { listDirectoryContents(inputPath, builder::addInputs, metadataProvider); } else { - Digest digest = computeDigest(input, null, metadataProvider); + Digest digest = computeDigest(input, null, metadataProvider, syscallCache); builder.addInputsBuilder().setPath(input.getExecPathString()).setDigest(digest); } } @@ -120,7 +126,8 @@ public void logSpawn( File.Builder outputBuilder = builder.addActualOutputsBuilder(); outputBuilder.setPath(path.relativeTo(execRoot).toString()); try { - outputBuilder.setDigest(computeDigest(e.getValue(), path, metadataProvider)); + outputBuilder.setDigest( + computeDigest(e.getValue(), path, metadataProvider, syscallCache)); } catch (IOException ex) { logger.atWarning().withCause(ex).log("Error computing spawn event output properties"); } @@ -191,7 +198,7 @@ private void listDirectoryContents( addFile.accept( File.newBuilder() .setPath(child.relativeTo(execRoot).toString()) - .setDigest(computeDigest(null, child, metadataProvider)) + .setDigest(computeDigest(null, child, metadataProvider, syscallCache)) .build()); } } @@ -205,7 +212,10 @@ private void listDirectoryContents( * Metadata cache first, if it is available, and fall back to digesting the contents manually. */ private Digest computeDigest( - @Nullable ActionInput input, @Nullable Path path, MetadataProvider metadataProvider) + @Nullable ActionInput input, + @Nullable Path path, + MetadataProvider metadataProvider, + SyscallCache syscallCache) throws IOException { Preconditions.checkArgument(input != null || path != null); DigestHashFunction hashFunction = execRoot.getFileSystem().getDigestFunction(); @@ -243,7 +253,9 @@ private Digest computeDigest( long fileSize = path.getFileSize(); return digest .setHash( - HashCode.fromBytes(DigestUtils.getDigestWithManualFallback(path, fileSize)).toString()) + HashCode.fromBytes( + DigestUtils.getDigestWithManualFallback(path, fileSize, syscallCache)) + .toString()) .setSizeBytes(fileSize) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java index 259b7e0319dd69..15e1b5db9017af 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java @@ -59,6 +59,7 @@ import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.errorprone.annotations.FormatMethod; import com.google.errorprone.annotations.FormatString; import java.io.File; @@ -91,6 +92,7 @@ public class LocalSpawnRunner implements SpawnRunner { private final String hostName; private final LocalExecutionOptions localExecutionOptions; + private final SyscallCache syscallCache; @Nullable private final ProcessWrapper processWrapper; @@ -106,10 +108,12 @@ public LocalSpawnRunner( LocalEnvProvider localEnvProvider, BinTools binTools, ProcessWrapper processWrapper, + SyscallCache syscallCache, RunfilesTreeUpdater runfilesTreeUpdater) { this.execRoot = execRoot; this.processWrapper = processWrapper; this.localExecutionOptions = Preconditions.checkNotNull(localExecutionOptions); + this.syscallCache = syscallCache; this.hostName = NetUtil.getCachedShortHostName(); this.resourceManager = resourceManager; this.localEnvProvider = localEnvProvider; @@ -134,7 +138,8 @@ public SpawnResult exec(Spawn spawn, SpawnExecutionContext context) spawn.getRunfilesSupplier(), binTools, spawn.getEnvironment(), - context.getFileOutErr()); + context.getFileOutErr(), + syscallCache); spawnMetrics.addSetupTime(setupTimeStopwatch.elapsed()); try (SilentCloseable c = diff --git a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java index 7c16c2973542f0..c1cfc69d4bc36e 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java +++ b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java @@ -33,6 +33,7 @@ import com.google.devtools.build.lib.remote.util.DigestUtil; import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.SyscallCache; import io.netty.util.AbstractReferenceCounted; import io.netty.util.ReferenceCounted; import io.reactivex.rxjava3.core.Flowable; @@ -68,6 +69,7 @@ class ByteStreamBuildEventArtifactUploader extends AbstractReferenceCounted private final Set omittedFiles = Sets.newConcurrentHashSet(); private final Set omittedTreeRoots = Sets.newConcurrentHashSet(); + private final SyscallCache syscallCache; ByteStreamBuildEventArtifactUploader( Executor executor, @@ -76,7 +78,8 @@ class ByteStreamBuildEventArtifactUploader extends AbstractReferenceCounted RemoteCache remoteCache, String remoteServerInstanceName, String buildRequestId, - String commandId) { + String commandId, + SyscallCache syscallCache) { this.executor = executor; this.reporter = reporter; this.verboseFailures = verboseFailures; @@ -85,6 +88,7 @@ class ByteStreamBuildEventArtifactUploader extends AbstractReferenceCounted this.commandId = commandId; this.remoteServerInstanceName = remoteServerInstanceName; this.scheduler = Schedulers.from(executor); + this.syscallCache = syscallCache; } public void omitFile(Path file) { @@ -151,7 +155,7 @@ private PathMetadata readPathMetadata(Path file) throws IOException { } } - DigestUtil digestUtil = new DigestUtil(file.getFileSystem().getDigestFunction()); + DigestUtil digestUtil = new DigestUtil(syscallCache, file.getFileSystem().getDigestFunction()); Digest digest = digestUtil.compute(file); return new PathMetadata(file, digest, /* directory= */ false, isRemoteFile(file)); } diff --git a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java index 23fa5d513d1137..5ff3cfa89c1946 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java +++ b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java @@ -63,7 +63,8 @@ public BuildEventArtifactUploader create(CommandEnvironment env) { remoteCache.retain(), remoteServerInstanceName, buildRequestId, - commandId); + commandId, + env.getSyscallCache()); return uploader; } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index 30560a6235ba8a..47bc77e9d63ea1 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java @@ -260,7 +260,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { AuthAndTLSOptions authAndTlsOptions = env.getOptions().getOptions(AuthAndTLSOptions.class); DigestHashFunction hashFn = env.getRuntime().getFileSystem().getDigestFunction(); - DigestUtil digestUtil = new DigestUtil(hashFn); + DigestUtil digestUtil = new DigestUtil(env.getSyscallCache(), hashFn); boolean verboseFailures = false; ExecutionOptions executionOptions = env.getOptions().getOptions(ExecutionOptions.class); diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java b/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java index 1f949e4ea8f2a2..59d5faf13b1571 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java +++ b/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.vfs.DigestHashFunction; import com.google.devtools.build.lib.vfs.DigestUtils; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.protobuf.Message; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -32,10 +33,11 @@ /** Utility methods to work with {@link Digest}. */ public class DigestUtil { - + private final SyscallCache syscallCache; private final DigestHashFunction hashFn; - public DigestUtil(DigestHashFunction hashFn) { + public DigestUtil(SyscallCache syscallCache, DigestHashFunction hashFn) { + this.syscallCache = syscallCache; this.hashFn = hashFn; } @@ -60,7 +62,8 @@ public Digest compute(Path file) throws IOException { } public Digest compute(Path file, long fileSize) throws IOException { - return buildDigest(DigestUtils.getDigestWithManualFallback(file, fileSize), fileSize); + return buildDigest( + DigestUtils.getDigestWithManualFallback(file, fileSize, syscallCache), fileSize); } public Digest compute(VirtualActionInput input) throws IOException { diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java index 53b619a151d0bc..79e623f589b2bc 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java @@ -21,6 +21,7 @@ import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyKey; import java.util.Map; @@ -90,7 +91,7 @@ public String getName() { } @Override - public Object getResolvedInformation() { + public Object getResolvedInformation(SyscallCache syscallCache) { return ImmutableMap.builder() .put(ResolvedHashesFunction.ORIGINAL_RULE_CLASS, "local_repository") .put( diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java index c2e0de349c9fe8..c11922568ddf97 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java @@ -28,6 +28,7 @@ import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.Root; import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; @@ -204,7 +205,7 @@ public String getName() { } @Override - public Object getResolvedInformation() { + public Object getResolvedInformation(SyscallCache syscallCache) { return ImmutableMap.builder() .put(ResolvedHashesFunction.ORIGINAL_RULE_CLASS, "new_local_repository") .put(ResolvedHashesFunction.ORIGINAL_ATTRIBUTES, orig) diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java index 7d2fce955225d5..de0b296ea2edd0 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java @@ -791,7 +791,8 @@ public MetadataProvider getFileCache() { synchronized (fileCacheLock) { if (fileCache == null) { fileCache = - new SingleBuildFileCache(getExecRoot().getPathString(), getRuntime().getFileSystem()); + new SingleBuildFileCache( + getExecRoot().getPathString(), getRuntime().getFileSystem(), syscallCache); } return fileCache; } diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java index 1f2b1c104a9dcc..f5ecba2fc71312 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java @@ -33,7 +33,6 @@ import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.exec.RunfilesTreeUpdater; import com.google.devtools.build.lib.exec.SpawnRunner; -import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionContext; import com.google.devtools.build.lib.exec.SpawnStrategyRegistry; import com.google.devtools.build.lib.exec.TreeDeleter; import com.google.devtools.build.lib.exec.local.LocalEnvProvider; @@ -453,6 +452,7 @@ private static SpawnRunner createFallbackRunner(CommandEnvironment env) { env.getBlazeWorkspace().getBinTools(), ProcessWrapper.fromCommandEnvironment(env), // TODO(buchgr): Replace singleton by a command-scoped RunfilesTreeUpdater + env.getSyscallCache(), RunfilesTreeUpdater.INSTANCE); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java index e8cd2c9551622b..f170fee5e24458 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java @@ -47,6 +47,7 @@ import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; @@ -68,6 +69,7 @@ class ArtifactFunction implements SkyFunction { private final Supplier mkdirForTreeArtifacts; private final MetadataConsumerForMetrics sourceArtifactsSeen; + private final Supplier syscallCache; static final class MissingArtifactValue implements SkyValue { private final DetailedExitCode detailedExitCode; @@ -92,9 +94,12 @@ public String toString() { } public ArtifactFunction( - Supplier mkdirForTreeArtifacts, MetadataConsumerForMetrics sourceArtifactsSeen) { + Supplier mkdirForTreeArtifacts, + MetadataConsumerForMetrics sourceArtifactsSeen, + Supplier syscallCache) { this.mkdirForTreeArtifacts = mkdirForTreeArtifacts; this.sourceArtifactsSeen = sourceArtifactsSeen; + this.syscallCache = syscallCache; } @Override @@ -274,7 +279,8 @@ private SkyValue createSourceValue(Artifact artifact, Environment env) if (!fileValue.isDirectory() || !TrackSourceDirectoriesFlag.trackSourceDirectories()) { FileArtifactValue metadata; try { - metadata = FileArtifactValue.createForSourceArtifact(artifact, fileValue); + metadata = + FileArtifactValue.createForSourceArtifact(artifact, fileValue, syscallCache.get()); } catch (IOException e) { throw new ArtifactFunctionException( SourceArtifactException.create(artifact, e), Transience.TRANSIENT); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java index 26835121de1c3a..12f56ec6ff6b39 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java @@ -66,6 +66,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import javax.annotation.Nullable; /** A {@link SkyFunction} to build {@link RecursiveFilesystemTraversalValue}s. */ @@ -144,6 +145,12 @@ private static final class RecursiveFilesystemTraversalFunctionException extends } } + private final Supplier syscallCache; + + RecursiveFilesystemTraversalFunction(Supplier syscallCache) { + this.syscallCache = syscallCache; + } + @Nullable @Override public SkyValue compute(SkyKey skyKey, Environment env) @@ -153,7 +160,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) Profiler.instance() .profile(ProfilerTask.FILESYSTEM_TRAVERSAL, traversal.getRoot().toString())) { // Stat the traversal root. - FileInfo rootInfo = lookUpFileInfo(env, traversal); + FileInfo rootInfo = lookUpFileInfo(env, traversal, syscallCache.get()); if (rootInfo == null) { return null; } @@ -188,7 +195,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) } // Otherwise the root is a directory or a symlink to one. - PkgLookupResult pkgLookupResult = checkIfPackage(env, traversal, rootInfo); + PkgLookupResult pkgLookupResult = + checkIfPackage(env, traversal, rootInfo, syscallCache.get()); if (pkgLookupResult == null) { return null; } @@ -306,7 +314,8 @@ public String toString() { } @Nullable - private static FileInfo lookUpFileInfo(Environment env, TraversalRequest traversal) + private static FileInfo lookUpFileInfo( + Environment env, TraversalRequest traversal, SyscallCache syscallCache) throws IOException, InterruptedException { if (traversal.isRootGenerated) { HasDigest fsVal = null; @@ -366,7 +375,8 @@ private static FileInfo lookUpFileInfo(Environment env, TraversalRequest travers if (fsVal == null) { fsVal = fileState; } - return new FileInfo(type, withDigest(fsVal, path), realPath, unresolvedLinkTarget); + return new FileInfo( + type, withDigest(fsVal, path, syscallCache), realPath, unresolvedLinkTarget); } } else { // Stat the file. @@ -403,14 +413,14 @@ private static FileInfo lookUpFileInfo(Environment env, TraversalRequest travers Path path = traversal.root.asRootedPath().asPath(); return new FileInfo( type, - withDigest(fileValue.realFileStateValue(), path), + withDigest(fileValue.realFileStateValue(), path, syscallCache), fileValue.realRootedPath(), unresolvedLinkTarget); } else { // If it doesn't exist, or it's a dangling symlink, we still want to handle that gracefully. return new FileInfo( fileValue.isSymlink() ? FileType.DANGLING_SYMLINK : FileType.NONEXISTENT, - withDigest(fileValue.realFileStateValue(), null), + withDigest(fileValue.realFileStateValue(), null, syscallCache), null, fileValue.isSymlink() ? fileValue.getUnresolvedLinkTarget() : null); } @@ -430,7 +440,8 @@ private static FileInfo lookUpFileInfo(Environment env, TraversalRequest travers * @return transformed HasDigest value based on the digest field and object type. */ @VisibleForTesting - static HasDigest withDigest(HasDigest fsVal, Path path) throws IOException { + static HasDigest withDigest(HasDigest fsVal, Path path, SyscallCache syscallCache) + throws IOException { if (fsVal instanceof FileStateValue) { FileStateValue fsv = (FileStateValue) fsVal; if (fsv instanceof RegularFileStateValue) { @@ -440,7 +451,7 @@ static HasDigest withDigest(HasDigest fsVal, Path path) throws IOException { ? FileArtifactValue.createForVirtualActionInput(rfsv.getDigest(), rfsv.getSize()) // Otherwise, create a file FileArtifactValue (RegularFileArtifactValue) based on the // path and size. - : FileArtifactValue.createForNormalFileUsingPath(path, rfsv.getSize()); + : FileArtifactValue.createForNormalFileUsingPath(path, rfsv.getSize(), syscallCache); } return new HasDigest.ByteStringDigest(fsv.getValueFingerprint()); } else if (fsVal instanceof FileArtifactValue) { @@ -452,7 +463,7 @@ static HasDigest withDigest(HasDigest fsVal, Path path) throws IOException { // In the case there is a directory, the HasDigest value should not be converted. Otherwise, // if the HasDigest value is a file, convert it using the Path and size values. return fav.getType().isFile() - ? FileArtifactValue.createForNormalFileUsingPath(path, fav.getSize()) + ? FileArtifactValue.createForNormalFileUsingPath(path, fav.getSize(), syscallCache) : new HasDigest.ByteStringDigest(fav.getValueFingerprint()); } return fsVal; @@ -510,7 +521,7 @@ public String toString() { * a package is found, but under a different root than expected) */ private static PkgLookupResult checkIfPackage( - Environment env, TraversalRequest traversal, FileInfo rootInfo) + Environment env, TraversalRequest traversal, FileInfo rootInfo, SyscallCache syscallCache) throws IOException, InterruptedException, BuildFileNotFoundException { Preconditions.checkArgument(rootInfo.type.exists() && !rootInfo.type.isFile(), "{%s} {%s}", traversal, rootInfo); @@ -540,7 +551,7 @@ private static PkgLookupResult checkIfPackage( // However the root of this package is different from what we expected. stat() the real // BUILD file of that package. traversal = traversal.forChangedRootPath(pkgRoot); - rootInfo = lookUpFileInfo(env, traversal); + rootInfo = lookUpFileInfo(env, traversal, syscallCache); Verify.verify(rootInfo.type.exists(), "{%s} {%s}", traversal, rootInfo); } return PkgLookupResult.pkg(traversal, rootInfo); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index ec1a6f8d6b7131..5b6f252e5d8dec 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -599,7 +599,8 @@ private ImmutableMap skyFunctions() { Artifact.ARTIFACT, new ArtifactFunction( () -> !skyframeActionExecutor.actionFileSystemType().inMemoryFileSystem(), - sourceArtifactsSeen)); + sourceArtifactsSeen, + syscallCacheRef::get)); map.put( SkyFunctions.BUILD_INFO_COLLECTION, new BuildInfoCollectionFunction(actionKeyContext, artifactFactory)); @@ -610,7 +611,8 @@ private ImmutableMap skyFunctions() { map.put(SkyFunctions.ACTION_EXECUTION, actionExecutionFunction); this.actionExecutionFunction = actionExecutionFunction; map.put( - SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, new RecursiveFilesystemTraversalFunction()); + SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, + new RecursiveFilesystemTraversalFunction(syscallCacheRef::get)); map.put(SkyFunctions.FILESET_ENTRY, new FilesetEntryFunction(directories::getExecRoot)); map.put( SkyFunctions.ACTION_TEMPLATE_EXPANSION, diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java index 79b92a2c530d99..1fde3429a28306 100644 --- a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java +++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java @@ -82,6 +82,7 @@ public void registerSpawnStrategies( LocalEnvProvider.forCurrentOs(env.getClientEnv()), env.getBlazeWorkspace().getBinTools(), ProcessWrapper.fromCommandEnvironment(env), + env.getSyscallCache(), // TODO(buchgr): Replace singleton by a command-scoped RunfilesTreeUpdater RunfilesTreeUpdater.INSTANCE); diff --git a/src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java b/src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java index d3841ea8ad0306..afb473a1766311 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java @@ -156,8 +156,9 @@ public static CacheStats getCacheStats() { * serially or in parallel. Files larger than a certain threshold will be read serially, in * order to avoid excessive disk seeks. */ - public static byte[] getDigestWithManualFallback(Path path, long fileSize) throws IOException { - byte[] digest = path.getFastDigest(); + public static byte[] getDigestWithManualFallback(Path path, long fileSize, SyscallCache syscalls) + throws IOException { + byte[] digest = syscalls.getFastDigest(path); return digest != null ? digest : manuallyComputeDigest(path, fileSize); } @@ -171,8 +172,9 @@ public static byte[] getDigestWithManualFallback(Path path, long fileSize) throw * * @param path Path of the file. */ - public static byte[] getDigestWithManualFallbackWhenSizeUnknown(Path path) throws IOException { - return getDigestWithManualFallback(path, -1); + public static byte[] getDigestWithManualFallbackWhenSizeUnknown(Path path, SyscallCache syscalls) + throws IOException { + return getDigestWithManualFallback(path, -1, syscalls); } /** diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Path.java b/src/main/java/com/google/devtools/build/lib/vfs/Path.java index efcdf584f0bc2c..e7b921336bbc81 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/Path.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/Path.java @@ -703,7 +703,7 @@ public byte[] getDigest() throws IOException { * @return a string representation of the bash of the directory * @throws IOException if the digest could not be computed for any reason */ - public String getDirectoryDigest() throws IOException { + public String getDirectoryDigest(SyscallCache syscallCache) throws IOException { ImmutableList entries = ImmutableList.sortedCopyOf(fileSystem.getDirectoryEntries(asFragment())); Hasher hasher = fileSystem.getDigestFunction().getHashFunction().newHasher(); @@ -717,9 +717,10 @@ public String getDirectoryDigest() throws IOException { } else { hasher.putChar('-'); } - hasher.putBytes(DigestUtils.getDigestWithManualFallback(path, stat.getSize())); + hasher.putBytes( + DigestUtils.getDigestWithManualFallback(path, stat.getSize(), syscallCache)); } else if (stat.isDirectory()) { - hasher.putChar('d').putUnencodedChars(path.getDirectoryDigest()); + hasher.putChar('d').putUnencodedChars(path.getDirectoryDigest(syscallCache)); } else if (stat.isSymbolicLink()) { PathFragment link = path.readSymbolicLink(); if (link.isAbsolute()) { @@ -731,7 +732,8 @@ public String getDirectoryDigest() throws IOException { } else { hasher.putChar('-'); } - hasher.putBytes(DigestUtils.getDigestWithManualFallbackWhenSizeUnknown(resolved)); + hasher.putBytes( + DigestUtils.getDigestWithManualFallbackWhenSizeUnknown(resolved, syscallCache)); } else { // link to a non-file: include the link itself in the hash hasher.putChar('l').putUnencodedChars(link.toString()); diff --git a/src/main/java/com/google/devtools/build/lib/vfs/SyscallCache.java b/src/main/java/com/google/devtools/build/lib/vfs/SyscallCache.java index 752197a18d6924..a9d02641586c71 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/SyscallCache.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/SyscallCache.java @@ -53,6 +53,14 @@ public Dirent.Type getType(Path path, Symlinks symlinks) throws IOException { */ Dirent.Type getType(Path path, Symlinks symlinks) throws IOException; + default byte[] getFastDigest(Path path) throws IOException { + return path.getFastDigest(); + } + + default byte[] getDigest(Path path) throws IOException { + return path.getDigest(); + } + static Dirent.Type statusToDirentType(FileStatus status) { if (status == null) { return null; diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java index 0670f97a3a40de..cf40bb3e2a5259 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java @@ -160,7 +160,8 @@ public void registerSpawnStrategies( RunfilesTreeUpdater.INSTANCE, env.getOptions().getOptions(WorkerOptions.class), env.getEventBus(), - Runtime.getRuntime()); + Runtime.getRuntime(), + env.getSyscallCache()); ExecutionOptions executionOptions = checkNotNull(env.getOptions().getOptions(ExecutionOptions.class)); registryBuilder.registerStrategy( diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java index dc30adc6996342..8914e3deb2e983 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java @@ -62,6 +62,7 @@ import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest; import com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse; import com.google.protobuf.ByteString; @@ -108,6 +109,7 @@ final class WorkerSpawnRunner implements SpawnRunner { private final WorkerParser workerParser; private final AtomicInteger requestIdCounter = new AtomicInteger(1); private final Runtime runtime; + private final SyscallCache syscallCache; /** Mapping of worker ids to their metrics. */ private Map workerIdToWorkerMetric = new ConcurrentHashMap<>(); @@ -123,7 +125,8 @@ public WorkerSpawnRunner( RunfilesTreeUpdater runfilesTreeUpdater, WorkerOptions workerOptions, EventBus eventBus, - Runtime runtime) { + Runtime runtime, + SyscallCache syscallCache) { this.helpers = helpers; this.execRoot = execRoot; this.workers = Preconditions.checkNotNull(workers); @@ -131,6 +134,7 @@ public WorkerSpawnRunner( this.binTools = binTools; this.resourceManager = resourceManager; this.runfilesTreeUpdater = runfilesTreeUpdater; + this.syscallCache = syscallCache; this.workerParser = new WorkerParser(execRoot, workerOptions, localEnvProvider, binTools); this.workerOptions = workerOptions; this.runtime = runtime; @@ -186,7 +190,8 @@ public SpawnResult exec(Spawn spawn, SpawnExecutionContext context) spawn.getRunfilesSupplier(), binTools, spawn.getEnvironment(), - context.getFileOutErr()); + context.getFileOutErr(), + syscallCache); MetadataProvider inputFileCache = context.getMetadataProvider(); diff --git a/src/test/java/com/google/devtools/build/lib/actions/ExecutableSymlinkActionTest.java b/src/test/java/com/google/devtools/build/lib/actions/ExecutableSymlinkActionTest.java index f98e0668f807a0..a71afcb6297f96 100644 --- a/src/test/java/com/google/devtools/build/lib/actions/ExecutableSymlinkActionTest.java +++ b/src/test/java/com/google/devtools/build/lib/actions/ExecutableSymlinkActionTest.java @@ -68,7 +68,8 @@ private ActionExecutionContext createContext() { Path execRoot = executor.getExecRoot(); return new ActionExecutionContext( executor, - new SingleBuildFileCache(execRoot.getPathString(), execRoot.getFileSystem()), + new SingleBuildFileCache( + execRoot.getPathString(), execRoot.getFileSystem(), SyscallCache.NO_CACHE), ActionInputPrefetcher.NONE, actionKeyContext, /*metadataHandler=*/ null, diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java index 24683c2ead2b65..3cf19c689e9141 100644 --- a/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java +++ b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java @@ -163,7 +163,8 @@ public static ActionExecutionContext createContext( Map clientEnv) { return new ActionExecutionContext( executor, - new SingleBuildFileCache(execRoot.getPathString(), execRoot.getFileSystem()), + new SingleBuildFileCache( + execRoot.getPathString(), execRoot.getFileSystem(), SyscallCache.NO_CACHE), ActionInputPrefetcher.NONE, actionKeyContext, metadataHandler, @@ -218,7 +219,8 @@ public static ActionExecutionContext createContextForInputDiscovery( DiscoveredModulesPruner discoveredModulesPruner) { return ActionExecutionContext.forInputDiscovery( executor, - new SingleBuildFileCache(execRoot.getPathString(), execRoot.getFileSystem()), + new SingleBuildFileCache( + execRoot.getPathString(), execRoot.getFileSystem(), SyscallCache.NO_CACHE), ActionInputPrefetcher.NONE, actionKeyContext, metadataHandler, diff --git a/src/test/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategyTest.java b/src/test/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategyTest.java index 3906420294f33d..1731188552ad5d 100644 --- a/src/test/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategyTest.java +++ b/src/test/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategyTest.java @@ -53,6 +53,7 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import com.google.protobuf.Duration; @@ -466,7 +467,8 @@ private void setUpExecutionContext(RemoteOptions remoteOptions) throws Exception when(actionExecutionContext.getContext(eq(SpawnCache.class))).thenReturn(SpawnCache.NO_CACHE); when(actionExecutionContext.getExecRoot()).thenReturn(execRoot); when(actionExecutionContext.getContext(eq(SpawnLogContext.class))) - .thenReturn(new SpawnLogContext(execRoot, messageOutput, remoteOptions)); + .thenReturn( + new SpawnLogContext(execRoot, messageOutput, remoteOptions, SyscallCache.NO_CACHE)); when(spawnRunner.execAsync(any(Spawn.class), any(SpawnExecutionContext.class))) .thenReturn( FutureSpawn.immediate( diff --git a/src/test/java/com/google/devtools/build/lib/exec/SingleBuildFileCacheTest.java b/src/test/java/com/google/devtools/build/lib/exec/SingleBuildFileCacheTest.java index fa23eecad17aa7..5b990c8c0d2b6b 100644 --- a/src/test/java/com/google/devtools/build/lib/exec/SingleBuildFileCacheTest.java +++ b/src/test/java/com/google/devtools/build/lib/exec/SingleBuildFileCacheTest.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import java.io.IOException; import java.io.InputStream; @@ -70,7 +71,7 @@ protected byte[] getFastDigest(PathFragment path) throws IOException { return null; } }; - underTest = new SingleBuildFileCache("/", fs); + underTest = new SingleBuildFileCache("/", fs, SyscallCache.NO_CACHE); FileSystemUtils.createEmptyFile(fs.getPath("/empty")); } diff --git a/src/test/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunnerTest.java index 69d2a7bb5f7dbd..ca1a8a2a234a42 100644 --- a/src/test/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunnerTest.java +++ b/src/test/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunnerTest.java @@ -74,6 +74,7 @@ import com.google.devtools.build.lib.vfs.JavaIoFileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Converters.RegexPatternConverter; import com.google.devtools.common.options.Options; @@ -122,6 +123,7 @@ public TestedLocalSpawnRunner( localEnvProvider, /*binTools=*/ null, processWrapper, + SyscallCache.NO_CACHE, Mockito.mock(RunfilesTreeUpdater.class)); } @@ -703,6 +705,7 @@ public void interruptWaitsForProcessExit() throws Exception { LocalEnvProvider.forCurrentOs(ImmutableMap.of()), /*binTools=*/ null, /*processWrapper=*/ null, + SyscallCache.NO_CACHE, Mockito.mock(RunfilesTreeUpdater.class)); FileOutErr fileOutErr = new FileOutErr(tempDir.getRelative("stdout"), tempDir.getRelative("stderr")); @@ -963,6 +966,7 @@ public void hasExecutionStatistics_whenOptionIsEnabled() throws Exception { binTools, new ProcessWrapper( processWrapperPath, /*killDelay=*/ Duration.ZERO, /*gracefulSigterm=*/ false), + SyscallCache.NO_CACHE, Mockito.mock(RunfilesTreeUpdater.class)); Spawn spawn = @@ -1028,6 +1032,7 @@ public void hasNoExecutionStatistics_whenOptionIsDisabled() throws Exception { binTools, new ProcessWrapper( processWrapperPath, /*killDelay=*/ Duration.ZERO, /*gracefulSigterm=*/ false), + SyscallCache.NO_CACHE, Mockito.mock(RunfilesTreeUpdater.class)); Spawn spawn = diff --git a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java index fdf575e16347e1..41076d4514396e 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java @@ -62,6 +62,7 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import io.grpc.Server; @@ -90,9 +91,10 @@ /** Test for {@link ByteStreamBuildEventArtifactUploader}. */ @RunWith(JUnit4.class) public class ByteStreamBuildEventArtifactUploaderTest { - @Rule public final RxNoGlobalErrorsRule rxNoGlobalErrorsRule = new RxNoGlobalErrorsRule(); + private static final DigestUtil DIGEST_UTIL = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); - private static final DigestUtil DIGEST_UTIL = new DigestUtil(DigestHashFunction.SHA256); + @Rule public final RxNoGlobalErrorsRule rxNoGlobalErrorsRule = new RxNoGlobalErrorsRule(); private final Reporter reporter = new Reporter(new EventBus()); private final StoredEventHandler eventHandler = new StoredEventHandler(); @@ -491,7 +493,8 @@ private ByteStreamBuildEventArtifactUploader newArtifactUploader( remoteCache, /*remoteServerInstanceName=*/ "localhost/instance", /*buildRequestId=*/ "none", - /*commandId=*/ "none"); + /*commandId=*/ "none", + SyscallCache.NO_CACHE); } private static class StaticMissingDigestsFinder implements MissingDigestsFinder { diff --git a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java index 2623921f4a9e15..adcc14ab9fab7a 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java @@ -45,6 +45,7 @@ import com.google.devtools.build.lib.remote.util.TestUtils; import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.protobuf.ByteString; import io.grpc.BindableService; import io.grpc.CallCredentials; @@ -97,7 +98,8 @@ @RunWith(JUnit4.class) public class ByteStreamUploaderTest { - private static final DigestUtil DIGEST_UTIL = new DigestUtil(DigestHashFunction.SHA256); + private static final DigestUtil DIGEST_UTIL = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); private static final int CHUNK_SIZE = 10; private static final String INSTANCE_NAME = "foo"; diff --git a/src/test/java/com/google/devtools/build/lib/remote/FakeActionInputFileCache.java b/src/test/java/com/google/devtools/build/lib/remote/FakeActionInputFileCache.java index d29645ca650b44..a876bc4bacb959 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/FakeActionInputFileCache.java +++ b/src/test/java/com/google/devtools/build/lib/remote/FakeActionInputFileCache.java @@ -28,6 +28,7 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.lib.vfs.SyscallCache; import java.io.IOException; /** A fake implementation of the {@link MetadataProvider} interface. */ @@ -38,7 +39,8 @@ final class FakeActionInputFileCache implements MetadataProvider { FakeActionInputFileCache(Path execRoot) { this.execRoot = execRoot; - this.digestUtil = new DigestUtil(execRoot.getFileSystem().getDigestFunction()); + this.digestUtil = + new DigestUtil(SyscallCache.NO_CACHE, execRoot.getFileSystem().getDigestFunction()); } @Override diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java index 2f7187c57dee75..aacbfb74d97f15 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java @@ -78,6 +78,7 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import com.google.protobuf.ByteString; @@ -124,7 +125,8 @@ @RunWith(JUnit4.class) public class GrpcCacheClientTest { - protected static final DigestUtil DIGEST_UTIL = new DigestUtil(DigestHashFunction.SHA256); + protected static final DigestUtil DIGEST_UTIL = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); private FileSystem fs; private Path execRoot; diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTest.java index a34bb4806f7387..061ab11524b7f7 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTest.java @@ -34,6 +34,7 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -183,7 +184,9 @@ private Artifact createLocalArtifact(String pathFragment, String contents, Actio // Caution: there's a race condition between stating the file and computing the // digest. We need to stat first, since we're using the stat to detect changes. // We follow symlinks here to be consistent with getDigest. - inputs.putWithNoDepOwner(a, FileArtifactValue.createFromStat(path, path.stat(Symlinks.FOLLOW))); + inputs.putWithNoDepOwner( + a, + FileArtifactValue.createFromStat(path, path.stat(Symlinks.FOLLOW), SyscallCache.NO_CACHE)); return a; } } diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcherTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcherTest.java index 9223028ba887ed..5327189d37ac66 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcherTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcherTest.java @@ -45,6 +45,7 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import com.google.protobuf.ByteString; @@ -81,7 +82,7 @@ public void setUp() throws IOException { artifactRoot = ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, "root"); artifactRoot.getRoot().asPath().createDirectoryAndParents(); options = Options.getDefaults(RemoteOptions.class); - digestUtil = new DigestUtil(HASH_FUNCTION); + digestUtil = new DigestUtil(SyscallCache.NO_CACHE, HASH_FUNCTION); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactoryTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactoryTest.java index 5a03070df9fcfb..4a44a5acc7e688 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactoryTest.java @@ -28,6 +28,7 @@ import com.google.devtools.build.lib.vfs.DigestHashFunction; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import java.io.IOException; @@ -39,8 +40,8 @@ /** Tests for {@link RemoteCacheClientFactory}. */ @RunWith(JUnit4.class) public class RemoteCacheClientFactoryTest { - - private final DigestUtil digestUtil = new DigestUtil(DigestHashFunction.SHA256); + private final DigestUtil digestUtil = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); private RemoteOptions remoteOptions; private final AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class); diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTest.java index 3b3772cc7004e2..ec4e7038a933b9 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTest.java @@ -50,6 +50,7 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import com.google.protobuf.ByteString; @@ -73,12 +74,12 @@ /** Tests for {@link RemoteCache}. */ @RunWith(JUnit4.class) public class RemoteCacheTest { - private RemoteActionExecutionContext context; private FileSystem fs; private Path execRoot; ArtifactRoot artifactRoot; - private final DigestUtil digestUtil = new DigestUtil(DigestHashFunction.SHA256); + private final DigestUtil digestUtil = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); private FakeActionInputFileCache fakeFileCache; private ListeningScheduledExecutorService retryService; diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java index b90cc763060c69..1f9f6bd8065d35 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java @@ -102,6 +102,7 @@ import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import com.google.protobuf.ByteString; @@ -124,7 +125,8 @@ public class RemoteExecutionServiceTest { @Rule public final RxNoGlobalErrorsRule rxNoGlobalErrorsRule = new RxNoGlobalErrorsRule(); - private final DigestUtil digestUtil = new DigestUtil(DigestHashFunction.SHA256); + private final DigestUtil digestUtil = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); private final Reporter reporter = new Reporter(new EventBus()); private final StoredEventHandler eventHandler = new StoredEventHandler(); diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutorTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutorTest.java index 4d59b78031cc33..2a95c1349db099 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutorTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutorTest.java @@ -31,6 +31,7 @@ import com.google.devtools.build.lib.remote.util.DigestUtil; import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor.ExecutionResult; import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.protobuf.ByteString; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -45,8 +46,8 @@ /** Tests for {@link com.google.devtools.build.lib.remote.RemoteRepositoryRemoteExecutor}. */ @RunWith(JUnit4.class) public class RemoteRepositoryRemoteExecutorTest { - - public static final DigestUtil DIGEST_UTIL = new DigestUtil(DigestHashFunction.SHA256); + public static final DigestUtil DIGEST_UTIL = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); @Mock public RemoteExecutionCache remoteCache; diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java index 4996893f56e73e..53a9eee43e10ac 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java @@ -79,6 +79,7 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import java.io.IOException; @@ -233,7 +234,7 @@ private RemoteSpawnCache remoteSpawnCacheWithOptions(RemoteOptions options) { public final void setUp() throws Exception { MockitoAnnotations.initMocks(this); fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256); - digestUtil = new DigestUtil(DigestHashFunction.SHA256); + digestUtil = new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); execRoot = fs.getPath("/exec/root"); execRoot.createDirectoryAndParents(); fakeFileCache = new FakeActionInputFileCache(execRoot); diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java index 2bc4abc8ff40d2..d7ca873267073e 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java @@ -99,6 +99,7 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import com.google.longrunning.Operation; @@ -156,7 +157,7 @@ public class RemoteSpawnRunnerTest { @Before public final void setUp() throws Exception { MockitoAnnotations.initMocks(this); - digestUtil = new DigestUtil(DigestHashFunction.SHA256); + digestUtil = new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); FileSystem fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256); execRoot = fs.getPath("/exec/root"); logDir = fs.getPath("/server-logs"); diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java index 2aa8d0e0fed0e5..8b2d0b815ea9b1 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java @@ -85,6 +85,7 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import com.google.longrunning.Operation; @@ -124,10 +125,10 @@ /** Tests for {@link RemoteSpawnRunner} in combination with {@link GrpcRemoteExecutor}. */ @RunWith(JUnit4.class) public class RemoteSpawnRunnerWithGrpcRemoteExecutorTest { + private static final DigestUtil DIGEST_UTIL = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); private final Reporter reporter = new Reporter(new EventBus()); - private static final DigestUtil DIGEST_UTIL = new DigestUtil(DigestHashFunction.SHA256); - private final MutableHandlerRegistry serviceRegistry = new MutableHandlerRegistry(); private FileSystem fs; private Path execRoot; diff --git a/src/test/java/com/google/devtools/build/lib/remote/UploadManifestTest.java b/src/test/java/com/google/devtools/build/lib/remote/UploadManifestTest.java index 6afd2b818039ed..7aac9775b43528 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/UploadManifestTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/UploadManifestTest.java @@ -33,6 +33,7 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import java.io.IOException; import org.junit.Before; @@ -43,7 +44,8 @@ /** Tests for {@link UploadManifest}. */ @RunWith(JUnit4.class) public class UploadManifestTest { - private final DigestUtil digestUtil = new DigestUtil(DigestHashFunction.SHA256); + private final DigestUtil digestUtil = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); private Path execRoot; private RemotePathResolver remotePathResolver; diff --git a/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java index 699076407647c2..0a64f932ffcb1c 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java @@ -49,6 +49,7 @@ import com.google.devtools.build.lib.testutil.Scratch; import com.google.devtools.build.lib.vfs.DigestHashFunction; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.common.options.Options; import com.google.protobuf.ByteString; import io.grpc.CallCredentials; @@ -78,7 +79,8 @@ @RunWith(JUnit4.class) public class GrpcRemoteDownloaderTest { - private static final DigestUtil DIGEST_UTIL = new DigestUtil(DigestHashFunction.SHA256); + private static final DigestUtil DIGEST_UTIL = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); private final MutableHandlerRegistry serviceRegistry = new MutableHandlerRegistry(); private final String fakeServerName = "fake server for " + getClass(); diff --git a/src/test/java/com/google/devtools/build/lib/remote/http/AbstractHttpHandlerTest.java b/src/test/java/com/google/devtools/build/lib/remote/http/AbstractHttpHandlerTest.java index 61c0f33a39b117..d6e9d6e901e5cb 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/http/AbstractHttpHandlerTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/http/AbstractHttpHandlerTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.Maps; import com.google.devtools.build.lib.remote.util.DigestUtil; import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.SyscallCache; import io.netty.channel.ChannelPromise; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http.HttpHeaderNames; @@ -37,8 +38,8 @@ @RunWith(JUnit4.class) @SuppressWarnings("FutureReturnValueIgnored") public abstract class AbstractHttpHandlerTest { - - private static final DigestUtil DIGEST_UTIL = new DigestUtil(DigestHashFunction.SHA256); + private static final DigestUtil DIGEST_UTIL = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); private static final Digest DIGEST = DIGEST_UTIL.computeAsUtf8("foo"); @Test diff --git a/src/test/java/com/google/devtools/build/lib/remote/http/HttpCacheClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/http/HttpCacheClientTest.java index 3dbd1c6b8727de..d80996e00c26e6 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/http/HttpCacheClientTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/http/HttpCacheClientTest.java @@ -37,6 +37,7 @@ import com.google.devtools.build.lib.remote.util.DigestUtil; import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.remote.worker.http.HttpCacheServerHandler; import com.google.devtools.common.options.Options; import com.google.protobuf.ByteString; @@ -103,8 +104,8 @@ @RunWith(Parameterized.class) @SuppressWarnings("FutureReturnValueIgnored") public class HttpCacheClientTest { - - private static final DigestUtil DIGEST_UTIL = new DigestUtil(DigestHashFunction.SHA256); + private static final DigestUtil DIGEST_UTIL = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); private static final Digest DIGEST = DIGEST_UTIL.computeAsUtf8("File Contents"); private RemoteActionExecutionContext remoteActionExecutionContext; diff --git a/src/test/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandlerTest.java b/src/test/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandlerTest.java index c77c575e30424e..35eecfbd90bcd6 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandlerTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandlerTest.java @@ -22,6 +22,7 @@ import com.google.common.net.HttpHeaders; import com.google.devtools.build.lib.remote.util.DigestUtil; import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.SyscallCache; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; @@ -50,9 +51,9 @@ @RunWith(JUnit4.class) @SuppressWarnings("FutureReturnValueIgnored") public class HttpDownloadHandlerTest extends AbstractHttpHandlerTest { - private static final URI CACHE_URI = URI.create("http://storage.googleapis.com:80/cache-bucket"); - private static final DigestUtil DIGEST_UTIL = new DigestUtil(DigestHashFunction.SHA256); + private static final DigestUtil DIGEST_UTIL = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); private static final Digest DIGEST = DIGEST_UTIL.computeAsUtf8("foo"); /** diff --git a/src/test/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeTest.java b/src/test/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeTest.java index c7abbb726232d7..54965dbd1b04f9 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeTest.java @@ -27,6 +27,7 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import java.io.IOException; import java.util.ArrayList; @@ -50,7 +51,7 @@ public void setup() { FileSystem fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256); execRoot = fs.getPath("/exec"); artifactRoot = ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, "srcs"); - digestUtil = new DigestUtil(fs.getDigestFunction()); + digestUtil = new DigestUtil(SyscallCache.NO_CACHE, fs.getDigestFunction()); } protected abstract DirectoryTree build(Path... paths) throws IOException; diff --git a/src/test/java/com/google/devtools/build/lib/remote/merkletree/MerkleTreeTest.java b/src/test/java/com/google/devtools/build/lib/remote/merkletree/MerkleTreeTest.java index 221fdc1638a125..af96bb2d46f106 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/merkletree/MerkleTreeTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/merkletree/MerkleTreeTest.java @@ -35,6 +35,7 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import java.io.IOException; import java.util.Arrays; @@ -61,7 +62,7 @@ public void setup() { FileSystem fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256); execRoot = fs.getPath("/exec"); artifactRoot = ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, "srcs"); - digestUtil = new DigestUtil(fs.getDigestFunction()); + digestUtil = new DigestUtil(SyscallCache.NO_CACHE, fs.getDigestFunction()); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java index 98b07072c18adb..743a1f6c566613 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java @@ -110,7 +110,8 @@ public void baseSetUp() throws Exception { .put(FileValue.FILE, new FileFunction(pkgLocator)) .put( Artifact.ARTIFACT, - new ArtifactFunction(() -> true, MetadataConsumerForMetrics.NO_OP)) + new ArtifactFunction( + () -> true, MetadataConsumerForMetrics.NO_OP, () -> SyscallCache.NO_CACHE)) .put(SkyFunctions.ACTION_EXECUTION, new SimpleActionExecutionFunction()) .put( SkyFunctions.PACKAGE, diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunctionTest.java index 1e15d11aae70b3..7f9b9783a9c208 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunctionTest.java @@ -114,7 +114,8 @@ public void setUp() throws Exception { SkyFunctions.DIRECTORY_LISTING_STATE, new DirectoryListingStateFunction(externalFilesHelper, () -> SyscallCache.NO_CACHE)); skyFunctions.put( - SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, new RecursiveFilesystemTraversalFunction()); + SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, + new RecursiveFilesystemTraversalFunction(() -> SyscallCache.NO_CACHE)); skyFunctions.put( SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction( diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunctionTest.java index 3917326c86cb3a..b568858c390c06 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunctionTest.java @@ -154,7 +154,8 @@ public void setUp() { SkyFunctions.DIRECTORY_LISTING_STATE, new DirectoryListingStateFunction(externalFilesHelper, () -> SyscallCache.NO_CACHE)); skyFunctions.put( - SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, new RecursiveFilesystemTraversalFunction()); + SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, + new RecursiveFilesystemTraversalFunction(() -> SyscallCache.NO_CACHE)); skyFunctions.put( SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction( @@ -1090,13 +1091,15 @@ public void testWithDigestFileArtifactValue() throws Exception { // file artifacts will return the same bytes as it was initialized with byte[] expectedBytes = new byte[] {1, 2, 3}; FileArtifactValue fav = FileArtifactValue.createForVirtualActionInput(expectedBytes, 10L); - HasDigest result = RecursiveFilesystemTraversalFunction.withDigest(fav, null); + HasDigest result = + RecursiveFilesystemTraversalFunction.withDigest(fav, null, SyscallCache.NO_CACHE); assertThat(result).isInstanceOf(FileArtifactValue.class); assertThat(result.getDigest()).isEqualTo(expectedBytes); // Directories do not have digest but the result will have a fingerprinted digest FileArtifactValue directoryFav = FileArtifactValue.createForDirectoryWithMtime(10L); - HasDigest directoryResult = RecursiveFilesystemTraversalFunction.withDigest(directoryFav, null); + HasDigest directoryResult = + RecursiveFilesystemTraversalFunction.withDigest(directoryFav, null, SyscallCache.NO_CACHE); assertThat(directoryResult).isInstanceOf(HasDigest.ByteStringDigest.class); assertThat(directoryResult.getDigest()).isNotNull(); } @@ -1107,14 +1110,16 @@ public void testWithDigestFileStateValue() throws Exception { byte[] expectedBytes = new byte[] {1, 2, 3}; RegularFileStateValue withDigest = new RegularFileStateValue(10L, expectedBytes, /* contentsProxy */ null); - HasDigest result = RecursiveFilesystemTraversalFunction.withDigest(withDigest, null); + HasDigest result = + RecursiveFilesystemTraversalFunction.withDigest(withDigest, null, SyscallCache.NO_CACHE); assertThat(result).isInstanceOf(FileArtifactValue.class); assertThat(result.getDigest()).isEqualTo(expectedBytes); // FileStateValue will be transformed with fingerprinted digest RootedPath rootedPath = rootedPath("bar", "foo"); - FileStateValue fsv = FileStateValue.create(rootedPath, null); - HasDigest fsvResult = RecursiveFilesystemTraversalFunction.withDigest(fsv, null); + FileStateValue fsv = FileStateValue.create(rootedPath, SyscallCache.NO_CACHE, /*tsgm=*/ null); + HasDigest fsvResult = + RecursiveFilesystemTraversalFunction.withDigest(fsv, null, SyscallCache.NO_CACHE); assertThat(fsvResult).isInstanceOf(HasDigest.ByteStringDigest.class); assertThat(fsvResult.getDigest()).isNotNull(); } @@ -1132,7 +1137,8 @@ public void testRegularFileStateValueWithoutDigest() throws Exception { null, /* contentsProxy */ FileContentsProxy.create(status)); HasDigest withoutDigestResult = - RecursiveFilesystemTraversalFunction.withDigest(withoutDigest, rootedPath.asPath()); + RecursiveFilesystemTraversalFunction.withDigest( + withoutDigest, rootedPath.asPath(), SyscallCache.NO_CACHE); // withDigest will construct a FileArtifactValue using the Path assertThat(withoutDigestResult).isInstanceOf(FileArtifactValue.class); assertThat(withoutDigestResult.getDigest()).isNotNull(); @@ -1142,7 +1148,9 @@ public void testRegularFileStateValueWithoutDigest() throws Exception { public void testWithDigestByteStringDigest() throws Exception { byte[] expectedBytes = new byte[] {1, 2, 3}; HasDigest.ByteStringDigest byteStringDigest = new HasDigest.ByteStringDigest(expectedBytes); - HasDigest result = RecursiveFilesystemTraversalFunction.withDigest(byteStringDigest, null); + HasDigest result = + RecursiveFilesystemTraversalFunction.withDigest( + byteStringDigest, null, SyscallCache.NO_CACHE); assertThat(result).isInstanceOf(HasDigest.ByteStringDigest.class); assertThat(result.getDigest()).isEqualTo(expectedBytes); } diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java b/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java index 507a2c0a688f54..2efd420bfc8a0b 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java @@ -245,7 +245,8 @@ protected BuilderWithResult createBuilder( new ActionLogBufferPathGenerator(actionOutputBase, actionOutputBase)); MetadataProvider cache = - new SingleBuildFileCache(rootDirectory.getPathString(), scratch.getFileSystem()); + new SingleBuildFileCache( + rootDirectory.getPathString(), scratch.getFileSystem(), SyscallCache.NO_CACHE); skyframeActionExecutor.configure( cache, ActionInputPrefetcher.NONE, DiscoveredModulesPruner.DEFAULT); @@ -259,7 +260,8 @@ protected BuilderWithResult createBuilder( .put(FileValue.FILE, new FileFunction(pkgLocator)) .put( Artifact.ARTIFACT, - new ArtifactFunction(() -> true, MetadataConsumerForMetrics.NO_OP)) + new ArtifactFunction( + () -> true, MetadataConsumerForMetrics.NO_OP, () -> SyscallCache.NO_CACHE)) .put( SkyFunctions.ACTION_EXECUTION, new ActionExecutionFunction( diff --git a/src/test/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategyTest.java b/src/test/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategyTest.java index b10759fecd53ac..c9e568134e5fa6 100644 --- a/src/test/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategyTest.java +++ b/src/test/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategyTest.java @@ -155,6 +155,7 @@ public final void setUp() throws Exception { (env, binTools1, fallbackTmpDir) -> ImmutableMap.copyOf(env), binTools, /*processWrapper=*/ null, + SyscallCache.NO_CACHE, Mockito.mock(RunfilesTreeUpdater.class)), /*verboseFailures=*/ false); this.executor = @@ -203,7 +204,8 @@ private ActionExecutionContext createContext() { Path execRoot = executor.getExecRoot(); return new ActionExecutionContext( executor, - new SingleBuildFileCache(execRoot.getPathString(), execRoot.getFileSystem()), + new SingleBuildFileCache( + execRoot.getPathString(), execRoot.getFileSystem(), SyscallCache.NO_CACHE), ActionInputPrefetcher.NONE, new ActionKeyContext(), /*metadataHandler=*/ null, diff --git a/src/test/java/com/google/devtools/build/lib/unix/UnixDigestHashAttributeNameTest.java b/src/test/java/com/google/devtools/build/lib/unix/UnixDigestHashAttributeNameTest.java index fda9a009f41d9f..97001c1ed4bb36 100644 --- a/src/test/java/com/google/devtools/build/lib/unix/UnixDigestHashAttributeNameTest.java +++ b/src/test/java/com/google/devtools/build/lib/unix/UnixDigestHashAttributeNameTest.java @@ -20,6 +20,7 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.FileSystemTest; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import org.junit.Test; /** Test for {@link com.google.devtools.build.lib.unix.UnixFileSystem#getFastDigest}. */ @@ -41,7 +42,9 @@ public void testFoo() throws Exception { // Instead of actually trying to access this file, a call to getxattr() should be made. We // intercept this call and return a fake extended attribute value, thereby causing the checksum // computation to be skipped entirely. - assertThat(DigestUtils.getDigestWithManualFallback(absolutize("myfile"), 123)) + assertThat( + DigestUtils.getDigestWithManualFallback( + absolutize("myfile"), 123, SyscallCache.NO_CACHE)) .isEqualTo(FAKE_DIGEST); } diff --git a/src/test/java/com/google/devtools/build/lib/vfs/DigestUtilsTest.java b/src/test/java/com/google/devtools/build/lib/vfs/DigestUtilsTest.java index 910276c7b57d81..3dfbf0eb968d22 100644 --- a/src/test/java/com/google/devtools/build/lib/vfs/DigestUtilsTest.java +++ b/src/test/java/com/google/devtools/build/lib/vfs/DigestUtilsTest.java @@ -74,9 +74,13 @@ protected byte[] getFastDigest(PathFragment path) throws IOException { FileSystemUtils.writeContentAsLatin1(myFile2, "b".repeat(fileSize2)); TestThread thread1 = - new TestThread(() -> DigestUtils.getDigestWithManualFallback(myFile1, fileSize1)); + new TestThread( + () -> + DigestUtils.getDigestWithManualFallback(myFile1, fileSize1, SyscallCache.NO_CACHE)); TestThread thread2 = - new TestThread(() -> DigestUtils.getDigestWithManualFallback(myFile2, fileSize2)); + new TestThread( + () -> + DigestUtils.getDigestWithManualFallback(myFile2, fileSize2, SyscallCache.NO_CACHE)); thread1.start(); thread2.start(); if (!expectConcurrent) { // Synchronized case. @@ -116,11 +120,14 @@ protected byte[] getDigest(PathFragment path) throws IOException { Path file = tracingFileSystem.getPath("/file.txt"); FileSystemUtils.writeContentAsLatin1(file, "some contents"); - byte[] digest1 = DigestUtils.getDigestWithManualFallback(file, file.getFileSize()); + byte[] digest1 = + DigestUtils.getDigestWithManualFallback(file, file.getFileSize(), SyscallCache.NO_CACHE); assertThat(getFastDigestCounter.get()).isEqualTo(1); assertThat(getDigestCounter.get()).isEqualTo(1); - assertThat(DigestUtils.getDigestWithManualFallback(file, file.getFileSize())) + assertThat( + DigestUtils.getDigestWithManualFallback( + file, file.getFileSize(), SyscallCache.NO_CACHE)) .isEqualTo(digest1); assertThat(getFastDigestCounter.get()).isEqualTo(2); assertThat(getDigestCounter.get()).isEqualTo(1); // Cached. diff --git a/src/test/java/com/google/devtools/build/lib/worker/WorkerSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/worker/WorkerSpawnRunnerTest.java index 2e6f134d3b68fb..877a9f22394e7b 100644 --- a/src/test/java/com/google/devtools/build/lib/worker/WorkerSpawnRunnerTest.java +++ b/src/test/java/com/google/devtools/build/lib/worker/WorkerSpawnRunnerTest.java @@ -52,6 +52,7 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.build.lib.worker.WorkerPool.WorkerPoolConfig; import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest; @@ -131,7 +132,8 @@ public void testExecInWorker_happyPath() throws ExecException, InterruptedExcept /* runfilestTreeUpdater */ null, new WorkerOptions(), eventBus, - runtime); + runtime, + SyscallCache.NO_CACHE); WorkerKey key = createWorkerKey(fs, "mnem", false); Path logFile = fs.getPath("/worker.log"); when(worker.getResponse(0)) @@ -169,7 +171,8 @@ public void testExecInWorker_finishesAsyncOnInterrupt() throws InterruptedExcept /* runfilesTreeUpdater=*/ null, new WorkerOptions(), eventBus, - runtime); + runtime, + SyscallCache.NO_CACHE); WorkerKey key = createWorkerKey(fs, "mnem", false); Path logFile = fs.getPath("/worker.log"); when(worker.getResponse(anyInt())) @@ -213,7 +216,8 @@ public void testExecInWorker_sendsCancelMessageOnInterrupt() /* runfilesTreeUpdater=*/ null, workerOptions, eventBus, - runtime); + runtime, + SyscallCache.NO_CACHE); WorkerKey key = createWorkerKey(fs, "mnem", false); Path logFile = fs.getPath("/worker.log"); Semaphore secondResponseRequested = new Semaphore(0); @@ -271,7 +275,8 @@ public void testExecInWorker_unsandboxedDiesOnInterrupt() /* runfilesTreeUpdater=*/ null, workerOptions, eventBus, - runtime); + runtime, + SyscallCache.NO_CACHE); WorkerKey key = createWorkerKey(fs, "mnem", false); Path logFile = fs.getPath("/worker.log"); when(worker.getResponse(anyInt())).thenThrow(new InterruptedException()); @@ -316,7 +321,8 @@ public void testExecInWorker_noMultiplexWithDynamic() /* runfilestTreeUpdater */ null, workerOptions, eventBus, - runtime); + runtime, + SyscallCache.NO_CACHE); // This worker key just so happens to be multiplex and require sandboxing. WorkerKey key = createWorkerKey(WorkerProtocolFormat.JSON, fs, true); Path logFile = fs.getPath("/worker.log"); @@ -356,7 +362,8 @@ private void assertRecordedResponsethrowsException(String recordedResponse, Stri /* runfilestTreeUpdater */ null, new WorkerOptions(), eventBus, - runtime); + runtime, + SyscallCache.NO_CACHE); WorkerKey key = createWorkerKey(fs, "mnem", false); Path logFile = fs.getPath("/worker.log"); when(worker.getLogFile()).thenReturn(logFile); @@ -408,7 +415,8 @@ public void testCollectStats_ignoreSpaces() throws Exception { /* runfilestTreeUpdater */ null, new WorkerOptions(), eventBus, - runtime); + runtime, + SyscallCache.NO_CACHE); String psOutput = " PID \t RSS\n 1 3216 \t\n \t 2 \t 4096 \t"; InputStream psStream = new ByteArrayInputStream(psOutput.getBytes(UTF_8)); @@ -439,7 +447,8 @@ public void testCollectStats_filterInvalidPids() throws Exception { /* runfilestTreeUpdater */ null, new WorkerOptions(), eventBus, - runtime); + runtime, + SyscallCache.NO_CACHE); String psOutput = "PID RSS \n 1 3216"; InputStream psStream = new ByteArrayInputStream(psOutput.getBytes(UTF_8)); diff --git a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java index 41ef41f7e81813..73f6b0a4b543bc 100644 --- a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java +++ b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java @@ -45,6 +45,7 @@ import com.google.devtools.build.lib.vfs.JavaIoFileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.remote.worker.http.HttpCacheServerInitializer; import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.OptionsParsingException; @@ -255,7 +256,7 @@ public static void main(String[] args) throws Exception { Path casPath = remoteWorkerOptions.casPath != null ? fs.getPath(remoteWorkerOptions.casPath) : null; - DigestUtil digestUtil = new DigestUtil(fs.getDigestFunction()); + DigestUtil digestUtil = new DigestUtil(SyscallCache.NO_CACHE, fs.getDigestFunction()); OnDiskBlobStoreCache cache = new OnDiskBlobStoreCache(remoteOptions, casPath, digestUtil); ListeningScheduledExecutorService retryService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1));