New flag --experimental_worker_sandbox_hardening to use the linux s…

…andbox for worker sandboxing.

When enabled, the existing `linux-sandbox` code will be used for worker sandboxing, running the workers in a locked-down state. Some flags that affect sandbox behaviour also affect this worker sandbox.

This is still very experimental and will change over time. Workers that expect the contents of `/tmp` to stay the same between build requests will not work with hardened sandbox.

PiperOrigin-RevId: 483951030
Change-Id: I343cdcebb0f3d5215c0f98cf5e7f067d860335ba

src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxUtil.java

@@ -14,7 +14,7 @@
 
 package com.google.devtools.build.lib.sandbox;
 
-import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.build.lib.runtime.BlazeWorkspace;
 import com.google.devtools.build.lib.util.OsUtils;
 import com.google.devtools.build.lib.vfs.Path;
 
@@ -23,14 +23,14 @@ public final class LinuxSandboxUtil {
   private static final String LINUX_SANDBOX = "linux-sandbox" + OsUtils.executableExtension();
 
   /** Returns whether using the {@code linux-sandbox} is supported in the command environment. */
-  public static boolean isSupported(CommandEnvironment cmdEnv) {
+  public static boolean isSupported(BlazeWorkspace blazeWorkspace) {
     // We can only use the linux-sandbox if the linux-sandbox exists in the embedded tools.
     // This might not always be the case, e.g. while bootstrapping.
-    return getLinuxSandbox(cmdEnv) != null;
+    return getLinuxSandbox(blazeWorkspace) != null;
   }
 
   /** Returns the path of the {@code linux-sandbox} binary, or null if it doesn't exist. */
-  public static Path getLinuxSandbox(CommandEnvironment cmdEnv) {
-    return cmdEnv.getBlazeWorkspace().getBinTools().getEmbeddedPath(LINUX_SANDBOX);
+  public static Path getLinuxSandbox(BlazeWorkspace blazeWorkspace) {
+    return blazeWorkspace.getBinTools().getEmbeddedPath(LINUX_SANDBOX);
   }
 }

src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java

@@ -74,10 +74,10 @@ public static boolean isSupported(final CommandEnvironment cmdEnv) throws Interr
     if (OS.getCurrent() != OS.LINUX) {
       return false;
     }
-    if (!LinuxSandboxUtil.isSupported(cmdEnv)) {
+    if (!LinuxSandboxUtil.isSupported(cmdEnv.getBlazeWorkspace())) {
       return false;
     }
-    Path linuxSandbox = LinuxSandboxUtil.getLinuxSandbox(cmdEnv);
+    Path linuxSandbox = LinuxSandboxUtil.getLinuxSandbox(cmdEnv.getBlazeWorkspace());
     Boolean isSupported;
     synchronized (isSupportedMap) {
       isSupported = isSupportedMap.get(linuxSandbox);
@@ -156,7 +156,7 @@ private static boolean computeIsSupported(CommandEnvironment cmdEnv, Path linuxS
     this.blazeDirs = cmdEnv.getDirectories();
     this.execRoot = cmdEnv.getExecRoot();
     this.allowNetwork = helpers.shouldAllowNetwork(cmdEnv.getOptions());
-    this.linuxSandbox = LinuxSandboxUtil.getLinuxSandbox(cmdEnv);
+    this.linuxSandbox = LinuxSandboxUtil.getLinuxSandbox(cmdEnv.getBlazeWorkspace());
     this.sandboxBase = sandboxBase;
     this.inaccessibleHelperFile = inaccessibleHelperFile;
     this.inaccessibleHelperDir = inaccessibleHelperDir;

src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java

@@ -177,7 +177,6 @@ public static void cleanExisting(
         parent = parent.getParentDirectory();
       }
     }
-
     cleanRecursively(root, inputs, inputsToCreate, dirsToCreate, workDir, prefixDirs);
   }
 

src/main/java/com/google/devtools/build/lib/worker/BUILD

@@ -37,6 +37,7 @@ java_library(
         ":worker_protocol",
         "//src/main/java/com/google/devtools/build/lib/actions",
         "//src/main/java/com/google/devtools/build/lib/events",
+        "//src/main/java/com/google/devtools/build/lib/sandbox:linux_sandbox_command_line_builder",
         "//src/main/java/com/google/devtools/build/lib/sandbox:sandbox_helpers",
         "//src/main/java/com/google/devtools/build/lib/shell",
         "//src/main/java/com/google/devtools/build/lib/vfs",
@@ -144,6 +145,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/exec:spawn_strategy_registry",
         "//src/main/java/com/google/devtools/build/lib/exec/local",
         "//src/main/java/com/google/devtools/build/lib/runtime/commands/events",
+        "//src/main/java/com/google/devtools/build/lib/sandbox",
         "//src/main/java/com/google/devtools/build/lib/sandbox:sandbox_helpers",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/common/options",

src/main/java/com/google/devtools/build/lib/worker/SandboxedWorker.java

@@ -14,29 +14,187 @@
 
 package com.google.devtools.build.lib.worker;
 
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
 import com.google.common.flogger.GoogleLogger;
+import com.google.devtools.build.lib.sandbox.LinuxSandboxCommandLineBuilder;
 import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxInputs;
 import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxOutputs;
+import com.google.devtools.build.lib.shell.Subprocess;
+import com.google.devtools.build.lib.shell.SubprocessBuilder;
+import com.google.devtools.build.lib.vfs.FileSystem;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
+import java.io.File;
 import java.io.IOException;
 import java.util.Set;
+import java.util.SortedMap;
+import javax.annotation.Nullable;
 
 /** A {@link SingleplexWorker} that runs inside a sandboxed execution root. */
 final class SandboxedWorker extends SingleplexWorker {
+
+  public static final String TMP_DIR_MOUNT_NAME = "_tmp";
+
+  @AutoValue
+  public abstract static class WorkerSandboxOptions {
+    // Need to have this data class because we can't depend on SandboxOptions in here.
+    abstract boolean fakeHostname();
+
+    abstract boolean fakeUsername();
+
+    abstract boolean debugMode();
+
+    abstract ImmutableList<PathFragment> tmpfsPath();
+
+    abstract ImmutableList<String> writablePaths();
+
+    abstract Path sandboxBinary();
+
+    public static WorkerSandboxOptions create(
+        Path sandboxBinary,
+        boolean fakeHostname,
+        boolean fakeUsername,
+        boolean debugMode,
+        ImmutableList<PathFragment> tmpfsPath,
+        ImmutableList<String> writablePaths) {
+      return new AutoValue_SandboxedWorker_WorkerSandboxOptions(
+          fakeHostname, fakeUsername, debugMode, tmpfsPath, writablePaths, sandboxBinary);
+    }
+  }
+
   private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
   private final WorkerExecRoot workerExecRoot;
+  /** Options specific to hardened sandbox, null if not using that. */
+  @Nullable private final WorkerSandboxOptions hardenedSandboxOptions;
 
-  SandboxedWorker(WorkerKey workerKey, int workerId, Path workDir, Path logFile) {
+  SandboxedWorker(
+      WorkerKey workerKey,
+      int workerId,
+      Path workDir,
+      Path logFile,
+      @Nullable WorkerSandboxOptions hardenedSandboxOptions) {
     super(workerKey, workerId, workDir, logFile);
-    this.workerExecRoot = new WorkerExecRoot(workDir);
+    this.workerExecRoot =
+        new WorkerExecRoot(
+            workDir,
+            hardenedSandboxOptions != null
+                ? ImmutableList.of(PathFragment.create("../" + TMP_DIR_MOUNT_NAME))
+                : ImmutableList.of());
+    this.hardenedSandboxOptions = hardenedSandboxOptions;
   }
 
   @Override
   public boolean isSandboxed() {
     return true;
   }
 
+  private ImmutableSet<Path> getWritableDirs(Path sandboxExecRoot) throws IOException {
+    // We have to make the TEST_TMPDIR directory writable if it is specified.
+    ImmutableSet.Builder<Path> writableDirs =
+        ImmutableSet.<Path>builder().add(sandboxExecRoot).add(sandboxExecRoot.getRelative("/tmp"));
+
+    FileSystem fileSystem = sandboxExecRoot.getFileSystem();
+    for (String writablePath : hardenedSandboxOptions.writablePaths()) {
+      Path path = fileSystem.getPath(writablePath);
+      writableDirs.add(path);
+      if (path.isSymbolicLink()) {
+        writableDirs.add(path.resolveSymbolicLinks());
+      }
+    }
+
+    FileSystem fs = sandboxExecRoot.getFileSystem();
+    writableDirs.add(fs.getPath("/dev/shm").resolveSymbolicLinks());
+    writableDirs.add(fs.getPath("/tmp"));
+
+    return writableDirs.build();
+  }
+
+  private SortedMap<Path, Path> getBindMounts(Path sandboxExecRoot, @Nullable Path sandboxTmp) {
+    Path tmpPath = sandboxExecRoot.getFileSystem().getPath("/tmp");
+    final SortedMap<Path, Path> bindMounts = Maps.newTreeMap();
+    // Mount a fresh, empty temporary directory as /tmp for each sandbox rather than reusing the
+    // host filesystem's /tmp. Since we're in a worker, we clean this dir between requests.
+    bindMounts.put(tmpPath, sandboxTmp);
+    // TODO(larsrc): Apply InaccessiblePaths
+    // for (Path inaccessiblePath : getInaccessiblePaths()) {
+    //   if (inaccessiblePath.isDirectory(Symlinks.NOFOLLOW)) {
+    //     bindMounts.put(inaccessiblePath, inaccessibleHelperDir);
+    //   } else {
+    //     bindMounts.put(inaccessiblePath, inaccessibleHelperFile);
+    //   }
+    // }
+    // validateBindMounts(bindMounts);
+    return bindMounts;
+  }
+
+  @Override
+  protected Subprocess createProcess() throws IOException {
+    // TODO(larsrc): Check that execRoot and outputBase are not under /tmp
+    // TODO(larsrc): Maybe deduplicate this code copied from super.createProcess()
+    if (hardenedSandboxOptions != null) {
+      this.shutdownHook =
+          new Thread(
+              () -> {
+                this.shutdownHook = null;
+                this.destroy();
+              });
+      Runtime.getRuntime().addShutdownHook(shutdownHook);
+
+      // TODO(larsrc): Figure out what of the environment rewrite needs doing.
+      // ImmutableMap<String, String> environment =
+      //     localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), binTools, "/tmp");
+
+      // TODO(larsrc): Figure out which things can change and make sure workers get restarted
+      // ImmutableSet<Path> writableDirs = getWritableDirs(workerExecRoot, environment);
+
+      ImmutableList<String> args = workerKey.getArgs();
+      File executable = new File(args.get(0));
+      if (!executable.isAbsolute() && executable.getParent() != null) {
+        args =
+            ImmutableList.<String>builderWithExpectedSize(args.size())
+                .add(new File(workDir.getPathFile(), args.get(0)).getAbsolutePath())
+                .addAll(args.subList(1, args.size()))
+                .build();
+      }
+
+      // In hardened mode, we bindmount a temp dir. We put the mount dir in the parent directory to
+      // avoid clashes with workspace files.
+      Path sandboxTmp = workDir.getParentDirectory().getRelative(TMP_DIR_MOUNT_NAME);
+      sandboxTmp.createDirectoryAndParents();
+
+      // TODO(larsrc): Need to make error messages go to stderr.
+      LinuxSandboxCommandLineBuilder commandLineBuilder =
+          LinuxSandboxCommandLineBuilder.commandLineBuilder(
+                  this.hardenedSandboxOptions.sandboxBinary(), args)
+              .setWritableFilesAndDirectories(getWritableDirs(workDir))
+              // Need all the sandbox options passed in here?
+              .setTmpfsDirectories(ImmutableSet.copyOf(this.hardenedSandboxOptions.tmpfsPath()))
+              .setBindMounts(getBindMounts(workDir, sandboxTmp))
+              .setUseFakeHostname(this.hardenedSandboxOptions.fakeHostname())
+              // Mostly tests require network, and some blaze run commands
+              .setCreateNetworkNamespace(true)
+              .setUseDebugMode(hardenedSandboxOptions.debugMode());
+
+      if (this.hardenedSandboxOptions.fakeUsername()) {
+        commandLineBuilder.setUseFakeUsername(true);
+      }
+
+      SubprocessBuilder processBuilder = new SubprocessBuilder();
+      ImmutableList<String> argv = commandLineBuilder.build();
+      processBuilder.setArgv(argv);
+      processBuilder.setWorkingDirectory(workDir.getPathFile());
+      processBuilder.setStderr(logFile.getPathFile());
+      processBuilder.setEnv(workerKey.getEnv());
+
+      return processBuilder.start();
+    } else {
+      return super.createProcess();
+    }
+  }
+
   @Override
   public void prepareExecution(
       SandboxInputs inputFiles, SandboxOutputs outputs, Set<PathFragment> workerFiles)

src/main/java/com/google/devtools/build/lib/worker/SingleplexWorker.java

@@ -64,14 +64,14 @@ class SingleplexWorker extends Worker {
    * zombie processes. Unfortunately, shutdown hooks are not guaranteed to be called, but this is
    * the best we can do. This must be set when a process is created.
    */
-  private Thread shutdownHook;
+  protected Thread shutdownHook;
 
   SingleplexWorker(WorkerKey workerKey, int workerId, final Path workDir, Path logFile) {
     super(workerKey, workerId, logFile);
     this.workDir = workDir;
   }
 
-  Subprocess createProcess() throws IOException {
+  protected Subprocess createProcess() throws IOException {
     this.shutdownHook =
         new Thread(
             () -> {

src/main/java/com/google/devtools/build/lib/worker/WorkerExecRoot.java

@@ -23,14 +23,24 @@
 import com.google.devtools.build.lib.vfs.PathFragment;
 import java.io.IOException;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Set;
 
 /** Creates and manages the contents of a working directory of a persistent worker. */
 final class WorkerExecRoot {
   private final Path workDir;
+  private final List<PathFragment> extraDirs;
 
-  public WorkerExecRoot(Path workDir) {
+  /**
+   * Creates a new WorkerExecRoot.
+   *
+   * @param workDir The directory (workspace dir) that the worker will be executing in.
+   * @param extraDirs Directories that must survive sandbox cleanup, e.g. for things that are
+   *     bind-mounted.
+   */
+  public WorkerExecRoot(Path workDir, List<PathFragment> extraDirs) {
     this.workDir = workDir;
+    this.extraDirs = extraDirs;
   }
 
   public void createFileSystem(
@@ -41,7 +51,7 @@ public void createFileSystem(
     // First compute all the inputs and directories that we need. This is based only on
     // `workerFiles`, `inputs` and `outputs` and won't do any I/O.
     Set<PathFragment> inputsToCreate = new LinkedHashSet<>();
-    LinkedHashSet<PathFragment> dirsToCreate = new LinkedHashSet<>();
+    LinkedHashSet<PathFragment> dirsToCreate = new LinkedHashSet<>(extraDirs);
     SandboxHelpers.populateInputsAndDirsToCreate(
         ImmutableSet.of(),
         inputsToCreate,

src/main/java/com/google/devtools/build/lib/worker/WorkerFactory.java

@@ -19,6 +19,7 @@
 import com.google.devtools.build.lib.events.Reporter;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.worker.SandboxedWorker.WorkerSandboxOptions;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Locale;
@@ -45,9 +46,19 @@ public class WorkerFactory extends BaseKeyedPooledObjectFactory<WorkerKey, Worke
 
   private final Path workerBaseDir;
   private Reporter reporter;
+  /**
+   * Options specific to hardened sandbox. Null if {@code --experimental_worker_sandbox_hardening}
+   * is not set.
+   */
+  @Nullable private final WorkerSandboxOptions hardenedSandboxOptions;
 
   public WorkerFactory(Path workerBaseDir) {
+    this(workerBaseDir, null);
+  }
+
+  public WorkerFactory(Path workerBaseDir, @Nullable WorkerSandboxOptions hardenedSandboxOptions) {
     this.workerBaseDir = workerBaseDir;
+    this.hardenedSandboxOptions = hardenedSandboxOptions;
   }
 
   public void setReporter(Reporter reporter) {
@@ -72,7 +83,7 @@ public Worker create(WorkerKey key) throws IOException {
         worker = new SandboxedWorkerProxy(key, workerId, logFile, workerMultiplexer, workDir);
       } else {
         Path workDir = getSandboxedWorkerPath(key, workerId);
-        worker = new SandboxedWorker(key, workerId, workDir, logFile);
+        worker = new SandboxedWorker(key, workerId, workDir, logFile, hardenedSandboxOptions);
       }
     } else if (key.isMultiplex()) {
       WorkerMultiplexer workerMultiplexer = WorkerMultiplexerManager.getInstance(key, logFile);
@@ -196,12 +207,13 @@ public boolean equals(Object o) {
       return false;
     }
     WorkerFactory that = (WorkerFactory) o;
-    return workerBaseDir.equals(that.workerBaseDir);
+    return workerBaseDir.equals(that.workerBaseDir)
+        && Objects.equals(this.hardenedSandboxOptions, that.hardenedSandboxOptions);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hashCode(workerBaseDir);
+    return Objects.hash(workerBaseDir, hardenedSandboxOptions);
   }
 
   /** This class simultaneously sends messages to a logger and an event reporter. */