refactor: split bounded query runners

containers/bounded-query/Dockerfile

@@ -53,6 +53,10 @@ RUN chmod -R a-w /opt/awf \
     && node --check /opt/awf/broker/framing.js \
     && node --check /opt/awf/broker/workspace.js \
     && node --check /opt/awf/broker/query-runner.js \
+    && node --check /opt/awf/broker/query-runner-spec.js \
+    && node --check /opt/awf/broker/docker-client.js \
+    && node --check /opt/awf/broker/docker-query-runner.js \
+    && node --check /opt/awf/broker/gvisor-query-runner.js \
     && node --check /opt/awf/broker/healthcheck.js
 
 # Fixed broker-only mount points.

containers/bounded-query/broker/broker.js

@@ -11,7 +11,6 @@ const {
 const { createLedger } = require('./ledger');
 const { createRealClock, waitForBucket } = require('./scheduler');
 const defaultWorkspace = require('./workspace');
-const defaultRunner = require('./query-runner');
 
 /**
  * The trusted bounded-query broker (protocol v2).
@@ -51,7 +50,10 @@ const defaultRunner = require('./query-runner');
 function createBroker(params) {
   const { config, seedMap, runId, audit } = params;
   const workspace = params.workspace || defaultWorkspace;
-  const runner = params.runner || defaultRunner;
+  if (!params.runner) {
+    throw new Error('createBroker requires a trusted QueryRunner');
+  }
+  const runner = params.runner;
   const clock = params.clock || createRealClock();
   const ledger = params.ledger || createLedger(seedMap);
 
@@ -241,6 +243,11 @@ function createBroker(params) {
       return queued;
     },
 
+    /** Resolves when every admitted invocation has finished broker-side work. */
+    drain() {
+      return tail;
+    },
+
     /** @internal Exposed for tests. */
     get invocationsUsed() {
       return invocationsUsed;

containers/bounded-query/broker/config.js

@@ -74,9 +74,9 @@ function loadConfig() {
     throw new Error('AWF_BOUNDED_QUERY_MEMORY must be a Docker memory limit (e.g. "512m")');
   }
 
-  const dockerRuntime = process.env.AWF_BOUNDED_QUERY_RUNTIME || '';
-  if (dockerRuntime && !/^[A-Za-z0-9_.-]+$/.test(dockerRuntime)) {
-    throw new Error('AWF_BOUNDED_QUERY_RUNTIME contains unexpected characters');
+  const queryBackend = requireEnv('AWF_BOUNDED_QUERY_BACKEND');
+  if (queryBackend !== 'docker' && queryBackend !== 'gvisor') {
+    throw new Error(`Unsupported AWF_BOUNDED_QUERY_BACKEND: ${queryBackend}`);
   }
 
   return {
@@ -97,7 +97,7 @@ function loadConfig() {
     // The daemon resolves query bind-mount sources in *its* filesystem view,
     // which is not necessarily the broker's (ARC/DinD split filesystems).
     hostWorkDir: requireEnv('AWF_BOUNDED_QUERY_HOST_WORK_DIR'),
-    dockerRuntime,
+    queryBackend,
     timeoutSeconds: parseTimeoutSeconds(),
     maxInvocations: parsePositiveInt('AWF_BOUNDED_QUERY_MAX_INVOCATIONS', 32),
     memoryLimit,

containers/bounded-query/broker/docker-client.js

@@ -0,0 +1,31 @@
+'use strict';
+
+const { execFile } = require('child_process');
+
+/**
+ * Executes the Docker CLI with bounded output and no inherited credentials.
+ */
+function runDocker(args, timeoutMs) {
+  return new Promise((resolve) => {
+    execFile(
+      'docker',
+      args,
+      {
+        timeout: timeoutMs,
+        killSignal: 'SIGKILL',
+        maxBuffer: 64 * 1024,
+        env: { PATH: process.env.PATH || '/usr/local/bin:/usr/bin:/bin' },
+      },
+      (error, stdout, stderr) => {
+        resolve({
+          exitCode: error && typeof error.code === 'number' ? error.code : error ? 1 : 0,
+          timedOut: Boolean(error && error.killed),
+          stderr: typeof stderr === 'string' ? stderr.slice(0, 2000) : '',
+          stdout: typeof stdout === 'string' ? stdout.slice(0, 2000) : '',
+        });
+      },
+    );
+  });
+}
+
+module.exports = { runDocker };

containers/bounded-query/broker/docker-query-runner.js

@@ -0,0 +1,111 @@
+'use strict';
+
+const defaultDockerClient = require('./docker-client');
+const {
+  CLI_GRACE_MS,
+  buildRemoveArgs,
+  deriveQueryContainerSpec,
+  normalizeTimeoutMs,
+} = require('./query-runner-spec');
+
+/**
+ * QueryRunner using the Docker daemon's default OCI runtime.
+ *
+ * The optional runtimeName is constructor-controlled so subclasses can select
+ * a fixed trusted runtime without accepting runtime data per invocation.
+ */
+class DockerQueryRunner {
+  constructor(config, deps = {}, runtimeName = undefined) {
+    this.config = config;
+    this.runtimeName = runtimeName;
+    this.docker = deps.docker || defaultDockerClient;
+    this.cleanupTail = Promise.resolve();
+  }
+
+  async assertAvailable() {
+    const image = await this.docker.runDocker(['image', 'inspect', this.config.queryImage], 60_000);
+    if (image.exitCode !== 0) {
+      throw new Error(`Query image is not available locally: ${this.config.queryImage}`);
+    }
+  }
+
+  spec(runId, invocationId) {
+    return deriveQueryContainerSpec({
+      config: this.config,
+      runId,
+      invocationId,
+      runtimeName: this.runtimeName,
+    });
+  }
+
+  async listContainerIds(args) {
+    const listed = await this.docker.runDocker(args, 30_000);
+    if (listed.exitCode !== 0) {
+      throw new Error('Failed to reconcile bounded-query containers');
+    }
+    const ids = listed.stdout.split('\n').map((id) => id.trim()).filter(Boolean);
+    if (ids.some((id) => !/^[0-9a-f]{12,64}$/.test(id))) {
+      throw new Error('Docker returned an invalid bounded-query container id');
+    }
+    return ids;
+  }
+
+  async removeListed(args) {
+    const ids = await this.listContainerIds(args);
+    if (ids.length === 0) return;
+    const removed = await this.docker.runDocker(buildRemoveArgs(ids), 30_000);
+    if (removed.exitCode !== 0) {
+      throw new Error('Failed to remove bounded-query containers');
+    }
+  }
+
+  serializeCleanup(operation) {
+    const queued = this.cleanupTail.then(operation, operation);
+    this.cleanupTail = queued.then(
+      () => undefined,
+      () => undefined,
+    );
+    return queued;
+  }
+
+  async reconcileRun(runId) {
+    const spec = this.spec(runId, 'reconcile');
+    await this.serializeCleanup(() => this.removeListed(spec.runListArgs));
+  }
+
+  async cleanupInvocation(runId, invocationId) {
+    const spec = this.spec(runId, invocationId);
+    await this.serializeCleanup(() => this.removeListed(spec.invocationListArgs));
+  }
+
+  async runQueryContainer(params) {
+    const spec = this.spec(params.runId, params.invocationId);
+    const timeoutMs = normalizeTimeoutMs(
+      (params.timeoutMs ?? this.config.timeoutSeconds * 1000) + CLI_GRACE_MS,
+    );
+
+    let result;
+    let runError;
+    try {
+      result = await this.docker.runDocker(spec.launchArgs, timeoutMs);
+    } catch (error) {
+      runError = error;
+    }
+
+    try {
+      await this.cleanupInvocation(params.runId, params.invocationId);
+    } catch (cleanupError) {
+      // A successful `docker run` means the container has already stopped, so
+      // preserve its result as before this refactor. Timeout/error paths may
+      // still have a live sandbox and therefore fail closed when cleanup fails.
+      if (!result || result.timedOut || result.exitCode !== 0) {
+        throw cleanupError;
+      }
+    }
+
+    if (runError) throw runError;
+    return result;
+  }
+}
+
+module.exports = { DockerQueryRunner };

containers/bounded-query/broker/gvisor-query-runner.js

@@ -0,0 +1,35 @@
+'use strict';
+
+const { DockerQueryRunner } = require('./docker-query-runner');
+
+const RUNSC_RUNTIME = 'runsc';
+
+/** QueryRunner using Docker with the fixed runsc OCI runtime. */
+class GvisorQueryRunner extends DockerQueryRunner {
+  constructor(config, deps = {}) {
+    super(config, deps, RUNSC_RUNTIME);
+  }
+
+  async assertAvailable() {
+    await super.assertAvailable();
+    const result = await this.docker.runDocker(
+      ['info', '--format', '{{json .Runtimes}}'],
+      30_000,
+    );
+    if (result.exitCode !== 0) {
+      throw new Error('Unable to inspect Docker OCI runtimes for gVisor');
+    }
+
+    let runtimes;
+    try {
+      runtimes = JSON.parse(result.stdout);
+    } catch {
+      throw new Error('Docker returned malformed OCI runtime information');
+    }
+    if (!runtimes || !Object.prototype.hasOwnProperty.call(runtimes, RUNSC_RUNTIME)) {
+      throw new Error('gVisor query backend requires the runsc OCI runtime; no fallback is permitted');
+    }
+  }
+}
+
+module.exports = { GvisorQueryRunner, RUNSC_RUNTIME };

containers/bounded-query/broker/query-runner-spec.js

@@ -0,0 +1,116 @@
+'use strict';
+
+/** Extra grace beyond the query's wall-clock budget for docker CLI overhead. */
+const CLI_GRACE_MS = 5_000;
+
+/** Maximum file size a query may create, in bytes (per-file RLIMIT_FSIZE). */
+const QUERY_MAX_FILE_BYTES = 512 * 1024 * 1024;
+
+/** Aggregate size limit for the query's writable tmpfs workspace in bytes. */
+const QUERY_WORKSPACE_TMPFS_BYTES = 1024 * 1024 * 1024;
+
+const RUN_LABEL = 'awf.bounded-query.run';
+const INVOCATION_LABEL = 'awf.bounded-query.invocation';
+const TRUSTED_ID_PATTERN = /^[a-z0-9][a-z0-9-]{0,63}$/;
+
+/** Converts a monotonic-clock duration to the integer milliseconds Node requires. */
+function normalizeTimeoutMs(timeoutMs) {
+  return Math.max(1, Math.ceil(timeoutMs));
+}
+
+function assertTrustedId(name, value) {
+  if (typeof value !== 'string' || !TRUSTED_ID_PATTERN.test(value)) {
+    throw new Error(`${name} is not a broker-generated identifier`);
+  }
+}
+
+function freezeArray(values) {
+  return Object.freeze(values);
+}
+
+/**
+ * Derives every daemon-facing query setting from trusted config and
+ * broker-generated identifiers. No request object is accepted by this API.
+ */
+function deriveQueryContainerSpec({ config, runId, invocationId, runtimeName }) {
+  assertTrustedId('runId', runId);
+  assertTrustedId('invocationId', invocationId);
+  if (runtimeName !== undefined && runtimeName !== 'runsc') {
+    throw new Error(`Unsupported OCI runtime in query runner: ${runtimeName}`);
+  }
+
+  const containerName = `awf-query-${runId.slice(0, 12)}-${invocationId}`;
+  const hostInvocationDir = `${config.hostWorkDir}/${invocationId}`;
+  const runLabel = `${RUN_LABEL}=${runId}`;
+  const invocationLabel = `${INVOCATION_LABEL}=${invocationId}`;
+  const launchArgs = [
+    'run',
+    '--pull', 'never',
+    '--name', containerName,
+    '--label', runLabel,
+    '--label', invocationLabel,
+    '--network', 'none',
+    '--read-only',
+    '--user', `${config.queryUid}:${config.queryGid}`,
+    '--cap-drop', 'ALL',
+    '--security-opt', 'no-new-privileges:true',
+    '--security-opt', `seccomp=${config.querySeccompPath}`,
+    '--memory', config.memoryLimit,
+    '--memory-swap', config.memoryLimit,
+    '--cpus', '1',
+    '--pids-limit', '128',
+    '--ulimit', `fsize=${QUERY_MAX_FILE_BYTES}`,
+    '--ulimit', 'nofile=1024:1024',
+    '--tmpfs', '/tmp:rw,noexec,nosuid,nodev,size=16m',
+    '--tmpfs', `/query:rw,nosuid,nodev,size=${QUERY_WORKSPACE_TMPFS_BYTES},uid=${config.queryUid},gid=${config.queryGid},mode=0700`,
+    '--hostname', 'query',
+    '--workdir', config.queryMountDir,
+    '--env', 'HOME=/tmp',
+    '--env', 'PYTHONDONTWRITEBYTECODE=1',
+    '--env', 'PYTHONUNBUFFERED=1',
+    '-v', `${hostInvocationDir}/repo:/awf/seed:ro`,
+    '-v', `${hostInvocationDir}/out:${config.queryMountDir}/out:rw`,
+    '-v', `${hostInvocationDir}/script.py:${config.queryScriptPath}:ro`,
+  ];
+
+  if (runtimeName !== undefined) {
+    launchArgs.push('--runtime', runtimeName);
+  }
+  launchArgs.push('--entrypoint', '/usr/local/bin/run-query', config.queryImage);
+
+  return Object.freeze({
+    containerName,
+    runtimeName,
+    launchArgs: freezeArray(launchArgs),
+    invocationListArgs: freezeArray([
+      'ps', '-aq',
+      '--filter', `label=${runLabel}`,
+      '--filter', `label=${invocationLabel}`,
+    ]),
+    runListArgs: freezeArray(['ps', '-aq', '--filter', `label=${runLabel}`]),
+  });
+}
+
+/** Compatibility helper for focused argument tests. */
+function buildQueryArgs(params) {
+  return deriveQueryContainerSpec({
+    ...params,
+    runtimeName: params.runtimeName,
+  }).launchArgs;
+}
+
+function buildRemoveArgs(containerIds) {
+  return freezeArray(['rm', '-f', ...containerIds]);
+}
+
+module.exports = {
+  CLI_GRACE_MS,
+  INVOCATION_LABEL,
+  QUERY_MAX_FILE_BYTES,
+  QUERY_WORKSPACE_TMPFS_BYTES,
+  RUN_LABEL,
+  buildQueryArgs,
+  buildRemoveArgs,
+  deriveQueryContainerSpec,
+  normalizeTimeoutMs,
+};