fabric storage

.github/workflows/witnesses.yml

@@ -858,6 +858,31 @@ jobs:
           if (! '[' '-f' "$GUNBC_FLOOR_RECEIPT" ']') || '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']' || ('[' "$GUNBC_FLOOR_CLASS" '=' 'infra' ']' && (! ('[' '-f' "$GUNBC_FLOOR_RECEIPT" ']' && 'grep' '-q' 'class=structural' "$GUNBC_FLOOR_RECEIPT"))) || ('[' "$GUNBC_FLOOR_CLASS" '=' 'none' ']' && (! ('[' '-f' "$GUNBC_FLOOR_RECEIPT" ']' && 'grep' '-q' 'class=structural' "$GUNBC_FLOOR_RECEIPT")) && (! ('[' '-f' "$GUNBC_FLOOR_RECEIPT" ']' && 'grep' '-q' 'class=infra' "$GUNBC_FLOOR_RECEIPT"))); then 'printf' 'class=%s\nsignature=%s\nexit=%s\n' "$GUNBC_FLOOR_CLASS" "$GUNBC_FLOOR_SIGNATURE" "$GUNBC_FLOOR_EXIT" > "$GUNBC_FLOOR_RECEIPT"; if '[' "$GUNBC_FLOOR_CLASS" '=' 'infra' ']'; then 'echo' '::error title=environment::floor_class='"$GUNBC_FLOOR_CLASS"' signature='"$GUNBC_FLOOR_SIGNATURE"' exit='"$GUNBC_FLOOR_EXIT"'; this is not a verdict about the diff. Attempt receipt: '"$GUNBC_FLOOR_RECEIPT"; fi; if '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']'; then 'echo' '::error title=subject::floor_class='"$GUNBC_FLOOR_CLASS"' exit='"$GUNBC_FLOOR_EXIT"'; read the step log for the subject defect'; fi; fi
           'exit' "$GUNBC_FLOOR_EXIT"
 
+        if: "!cancelled()"
+      - name: Run the slice cost controls
+        id: slice_cost_controls
+        run: |+
+          GUNBC_FLOOR_LOG='gunbc-floor-cmd.log'
+          'set' '+e'
+          'set' '-o' 'pipefail'
+          ('sh' '-e' '-c' 'cargo test --release -p v1-stage0-runtime --lib -- list_slice_tests') 2>&1 | 'tee' "$GUNBC_FLOOR_LOG"
+          GUNBC_FLOOR_EXIT="$?"
+          GUNBC_FLOOR_RECEIPT='gunbc-floor-outcome.txt'
+          GUNBC_FLOOR_CLASS='structural'
+          GUNBC_FLOOR_SIGNATURE=''
+          if '[' "$GUNBC_FLOOR_EXIT" '-eq' '126' ']'; then GUNBC_FLOOR_CLASS='infra'; GUNBC_FLOOR_SIGNATURE='CommandInvokedCannotExecute'; fi
+          if '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']' && '[' "$GUNBC_FLOOR_EXIT" '-eq' '127' ']'; then GUNBC_FLOOR_CLASS='infra'; GUNBC_FLOOR_SIGNATURE='CommandNotFound'; fi
+          if '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']' && '[' "$GUNBC_FLOOR_EXIT" '-eq' '0' ']'; then GUNBC_FLOOR_CLASS='none'; GUNBC_FLOOR_SIGNATURE=''; fi
+          if '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']' && '[' '-f' "$GUNBC_FLOOR_LOG" ']' && 'grep' '-q' 'MemoryStallRefusedPageThrash' "$GUNBC_FLOOR_LOG"; then GUNBC_FLOOR_CLASS='infra'; GUNBC_FLOOR_SIGNATURE='MemoryStallRefusedPageThrash'; fi
+          if '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']' && '[' '-f' "$GUNBC_FLOOR_LOG" ']' && 'grep' '-q' 'The runner has received a shutdown signal' "$GUNBC_FLOOR_LOG"; then GUNBC_FLOOR_CLASS='infra'; GUNBC_FLOOR_SIGNATURE='RunnerLost'; fi
+          if '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']' && '[' '-f' "$GUNBC_FLOOR_LOG" ']' && 'grep' '-q' 'sccache: error: failed to execute compile' "$GUNBC_FLOOR_LOG"; then GUNBC_FLOOR_CLASS='infra'; GUNBC_FLOOR_SIGNATURE='SccacheFailedToExecuteCompile'; fi
+          if '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']' && '[' '-f' "$GUNBC_FLOOR_LOG" ']' && 'grep' '-q' '(exit status: 254)' "$GUNBC_FLOOR_LOG"; then GUNBC_FLOOR_CLASS='infra'; GUNBC_FLOOR_SIGNATURE='SccacheRustcWrapperExit254'; fi
+          if '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']' && '[' '-f' "$GUNBC_FLOOR_LOG" ']' && 'grep' '-q' 'sccache: encountered fatal error' "$GUNBC_FLOOR_LOG"; then GUNBC_FLOOR_CLASS='infra'; GUNBC_FLOOR_SIGNATURE='SccacheFatalError'; fi
+          if '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']' && '[' '-f' "$GUNBC_FLOOR_LOG" ']' && 'grep' '-q' 'failed to spawn' "$GUNBC_FLOOR_LOG"; then GUNBC_FLOOR_CLASS='infra'; GUNBC_FLOOR_SIGNATURE='ProcessSpawnFailure'; fi
+          if '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']' && '[' '-f' "$GUNBC_FLOOR_LOG" ']' && 'grep' '-q' 'Resource temporarily unavailable' "$GUNBC_FLOOR_LOG"; then GUNBC_FLOOR_CLASS='infra'; GUNBC_FLOOR_SIGNATURE='ResourceTemporarilyUnavailable'; fi
+          if (! '[' '-f' "$GUNBC_FLOOR_RECEIPT" ']') || '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']' || ('[' "$GUNBC_FLOOR_CLASS" '=' 'infra' ']' && (! ('[' '-f' "$GUNBC_FLOOR_RECEIPT" ']' && 'grep' '-q' 'class=structural' "$GUNBC_FLOOR_RECEIPT"))) || ('[' "$GUNBC_FLOOR_CLASS" '=' 'none' ']' && (! ('[' '-f' "$GUNBC_FLOOR_RECEIPT" ']' && 'grep' '-q' 'class=structural' "$GUNBC_FLOOR_RECEIPT")) && (! ('[' '-f' "$GUNBC_FLOOR_RECEIPT" ']' && 'grep' '-q' 'class=infra' "$GUNBC_FLOOR_RECEIPT"))); then 'printf' 'class=%s\nsignature=%s\nexit=%s\n' "$GUNBC_FLOOR_CLASS" "$GUNBC_FLOOR_SIGNATURE" "$GUNBC_FLOOR_EXIT" > "$GUNBC_FLOOR_RECEIPT"; if '[' "$GUNBC_FLOOR_CLASS" '=' 'infra' ']'; then 'echo' '::error title=environment::floor_class='"$GUNBC_FLOOR_CLASS"' signature='"$GUNBC_FLOOR_SIGNATURE"' exit='"$GUNBC_FLOOR_EXIT"'; this is not a verdict about the diff. Attempt receipt: '"$GUNBC_FLOOR_RECEIPT"; fi; if '[' "$GUNBC_FLOOR_CLASS" '=' 'structural' ']'; then 'echo' '::error title=subject::floor_class='"$GUNBC_FLOOR_CLASS"' exit='"$GUNBC_FLOOR_EXIT"'; read the step log for the subject defect'; fi; fi
+          'exit' "$GUNBC_FLOOR_EXIT"
+
         if: "!cancelled()"
       - name: "Stage0 mirrors match one emission by this seed: required CI build lane"
         id: stage0_regen

dag/extdeps/crypto/seeded_filler.dag

@@ -0,0 +1,94 @@
+module extdeps.crypto.seeded_filler
+
+import std.types { String, NonEmptyStr, Int, FilePath }
+import std.measure { ByteSize, byte_size, measure_count }
+import std.optional { Present, Absent }
+import std.decl_ref { DeclarationRef, WholeDeclaration }
+import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef }
+import extdeps.uri { Uri, Https }
+import extdeps.shell
+import extdeps.tools.coreutils_truncate
+import extdeps.tools.coreutils_stat
+import extdeps.tools.openssl
+
+// A DETERMINISTIC, NON-SECRET, INCOMPRESSIBLE FILE OF EXACTLY N BYTES, KEYED BY A SEED.
+//
+// The construction is the GNU coreutils manual's own seeded random source ("Sources of random data"):
+// the AES-256-CTR keystream that `openssl enc -aes-256-ctr -pass pass:<seed> -nosalt` produces over
+// zeros. Here the zeros are a file `truncate -s N` sized exactly, so the keystream is exactly N bytes
+// long without a pipe: CTR is a stream mode, so output length equals input length. It is
+// reproducible (one seed, one key, one stream), two seeds give unrelated streams, and an AES
+// keystream is indistinguishable from random, so neither gzip nor a chunk store's dedup finds
+// structure in it -- which is why it, and not zeros or a repeated pattern, stands in for content
+// whose SIZE must be carried while its bytes must not be: a compressible filler would understate
+// what storing the real content costs. The seed is in argv and is NOT a secret; the filler carries
+// no information beyond its seed and its length.
+data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority {
+  uri: Uri { scheme: Https, locator: "www.gnu.org/software/coreutils/manual/html_node/Random-sources.html" }
+}
+
+data seeded_filler_openssl_enc_authority: ExternalAuthority = ExternalAuthority {
+  uri: Uri { scheme: Https, locator: "docs.openssl.org/3.0/man1/openssl-enc/" }
+}
+
+data extdeps_model_scope: ExternalModelScope = ExternalModelScope {
+  subject: ExternalSubjectRef {
+    declaration: DeclarationRef { module_path: "extdeps.crypto.seeded_filler", decl_name: "seeded_filler_write", field: WholeDeclaration }
+  },
+  first_citation: extdeps.crypto.seeded_filler.extdeps_external_authority_anchor,
+  further_citations: [extdeps.crypto.seeded_filler.seeded_filler_openssl_enc_authority]
+}
+
+// WHICH STEP REFUSED. The size readback is a step: a filler whose stat size is not the requested
+// size is refused, never returned.
+type SeededFillerStep
+  = FillerZeroSized
+  | FillerKeystreamWritten
+  | FillerZeroRemoved
+  | FillerSizeReadBack
+
+type SeededFiller
+  = SeededFillerWritten { path: NonEmptyStr, size: ByteSize }
+  | SeededFillerRefused { path: NonEmptyStr, step: SeededFillerStep, detail: String }
+
+fn seeded_filler_step_words(step: SeededFillerStep) -> String {
+  match step {
+    FillerZeroSized => "truncate"
+    FillerKeystreamWritten => "openssl-enc"
+    FillerZeroRemoved => "zero-input-removal"
+    FillerSizeReadBack => "size-readback"
+  }
+}
+
+// WRITE THE FILLER AT `path`, whose parent directory must exist. The zero input is a sibling
+// `<path>.zero`, sparse on every filesystem truncate extends, and removed before returning.
+fn seeded_filler_write(seed: NonEmptyStr, size: ByteSize, path: NonEmptyStr) -> SeededFiller {
+  let zero = concat(path as String, ".zero") as NonEmptyStr
+  let sized = coreutils.Truncate.ToSize(size_bytes: to_string(measure_count(m: size)) as NonEmptyStr, path: zero)
+  if sized.success == false {
+    SeededFillerRefused { path: path, step: FillerZeroSized, detail: "truncate could not size the zero input" }
+  } else {
+    let enc = openssl.Enc.Aes256CtrPassphrase(passphrase: seed, input_path: zero, output_path: path)
+    let removed = shell.Remove.FileForce(path: zero as FilePath)
+    if enc.success == false {
+      SeededFillerRefused { path: path, step: FillerKeystreamWritten, detail: enc.stderr }
+    } else if removed.success == false {
+      SeededFillerRefused { path: path, step: FillerZeroRemoved, detail: concat("the zero input could not be removed: ", zero as String) }
+    } else {
+      seeded_filler_readback(path: path, size: size)
+    }
+  }
+}
+
+fn seeded_filler_readback(path: NonEmptyStr, size: ByteSize) -> SeededFiller {
+  let st = coreutils.Stat.PathSize(path: path)
+  match parse_int(s: replace(st.size_text, "\n", "")) {
+    Absent => SeededFillerRefused { path: path, step: FillerSizeReadBack, detail: concat("stat printed no size: ", st.size_text) }
+    Present { value: n } =>
+      if st.success && n == measure_count(m: size) {
+        SeededFillerWritten { path: path, size: size }
+      } else {
+        SeededFillerRefused { path: path, step: FillerSizeReadBack, detail: join(["the filler is ", to_string(n), " bytes, not ", to_string(measure_count(m: size))], "") }
+      }
+  }
+}

dag/extdeps/exec/command.dag

@@ -107,6 +107,9 @@ fn argv_command(program: ProgramIdentity, arguments: List<String>) -> ArgvComman
     decl_ref(module_path: "extdeps.tools.tar", decl_name: "tar_extract_command"),
     decl_ref(module_path: "extdeps.tools.tar", decl_name: "tar_create_from_list_command"),
     decl_ref(module_path: "extdeps.tools.tar", decl_name: "tar_list_command"),
+    decl_ref(module_path: "extdeps.tools.tar", decl_name: "tar_create_from_two_lists_command"),
+    decl_ref(module_path: "extdeps.tools.tar", decl_name: "tar_list_sizes_command"),
+    decl_ref(module_path: "extdeps.tools.tar", decl_name: "tar_extract_listed_command"),
     decl_ref(module_path: "extdeps.tools.btrfs_progs", decl_name: "btrfs_subvolume_create_command"),
     decl_ref(module_path: "extdeps.tools.btrfs_progs", decl_name: "btrfs_subvolume_snapshot_read_only_command"),
     decl_ref(module_path: "extdeps.tools.btrfs_progs", decl_name: "btrfs_subvolume_delete_command"),
@@ -115,6 +118,7 @@ fn argv_command(program: ProgramIdentity, arguments: List<String>) -> ArgvComman
     decl_ref(module_path: "extdeps.tools.findutils", decl_name: "find_regular_files_sha256_command"),
     decl_ref(module_path: "extdeps.tools.findutils", decl_name: "find_owner_executable_files_command"),
     decl_ref(module_path: "extdeps.tools.findutils", decl_name: "find_symlink_targets_command"),
+    decl_ref(module_path: "extdeps.tools.findutils", decl_name: "find_regular_file_size_mtime_command"),
     decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "mv_exact_destination_command"),
     decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "sort_in_place_command"),
     decl_ref(module_path: "extdeps.tools.unzip", decl_name: "unzip_extract_command"),

dag/extdeps/languages/rust/emit.dag

@@ -76,8 +76,8 @@ data rust_simple_method_specs: List<SimpleMethodSpec> = [
   { method_name: "first",     template: "\{recv\}.first().cloned()",                                                                   wraps_result: false },
   { method_name: "enumerate", template: "\{recv\}.iter().cloned().enumerate().map(|(i, v)| (i as i64, v)).collect::<Vec<_>>()",         wraps_result: true },
   { method_name: "chars",     template: "\{recv\}.chars().map(|c| c as i64).collect::<Vec<_>>()",                                      wraps_result: true },
-  { method_name: "skip",      template: "\{recv\}.iter().cloned().skip(\{arg\} as usize).collect::<Vec<_>>()",                         wraps_result: true },
-  { method_name: "take",      template: "\{recv\}.iter().cloned().take(\{arg\} as usize).collect::<Vec<_>>()",                         wraps_result: true },
+  { method_name: "skip",      template: "v1_rt::list_skip(&\{recv\}, \{arg\})",                                                       wraps_result: true },
+  { method_name: "take",      template: "v1_rt::list_take(&\{recv\}, \{arg\})",                                                       wraps_result: true },
   { method_name: "is_empty",  template: "\{recv\}.is_empty()",                                                                       wraps_result: false }
 ]
 

dag/extdeps/tools/coreutils_truncate.dag

@@ -0,0 +1,29 @@
+module extdeps.tools.coreutils_truncate
+
+import extdeps.external_authority { ExternalAuthority }
+import extdeps.uri { Uri, Https }
+import std.types { Bool, NonEmptyStr, Unit }
+
+data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority {
+  uri: Uri {
+    scheme: Https
+    locator: "www.gnu.org/software/coreutils/manual/html_node/truncate-invocation.html"
+  }
+}
+
+// `truncate -s N FILE` creates FILE if it is absent and sets its size to EXACTLY N bytes; bytes it
+// adds read as zero (coreutils "truncate invocation"). -s takes a bare decimal byte count here, never
+// a suffixed one, so the size is the caller's integer and nothing the tool reinterprets. -- ends the
+// options, so a path beginning with '-' is a path.
+service coreutils.Truncate {
+  operation ToSize {
+    requires none
+    input { size_bytes: NonEmptyStr, path: NonEmptyStr }
+    output { success: Bool from "exit_success" }
+    transport shell { argv: ["truncate", "-s", "{size_bytes}", "--", "{path}"] }
+    exit {
+      0 => Unit
+      nonzero => String "truncate could not size the path"
+    }
+  }
+}

dag/extdeps/tools/findutils.dag

@@ -101,3 +101,27 @@ fn find_owner_executable_files_command(root: String) -> ArgvCommand {
 fn find_symlink_targets_command(root: String) -> ArgvCommand {
   argv_command(program: find_program(), arguments: [root, "-type", "l", "-printf", "l %P -> %l\\n"])
 }
+
+// THE REGULAR FILES UNDER A ROOT WITH THEIR SIZE AND MODIFICATION TIME, ONE
+// `<size>\t<mtime>\t<relative path>` LINE EACH, AND NOTHING READ BUT METADATA. Every -printf
+// directive here is answered from the inode find already holds from its own stat(2) of the entry
+// (find manual, "Format Directives"): %s is st_size in bytes, %T@ is st_mtime as seconds since the
+// epoch with a fractional part, %P is the path with the starting point removed. No directive opens
+// the file, and the expression carries no action that runs another program, so the command reads
+// no file content -- find_command_runs_another_program below is how a consumer checks the second
+// half on the value rather than on this note. The path is LAST so a name holding a tab survives as
+// the rest of the line; the escapes are find's own, so the argv word carries a backslash and a
+// letter, never a control character.
+fn find_regular_file_size_mtime_command(root: String) -> ArgvCommand {
+  argv_command(program: find_program(), arguments: [root, "-type", "f", "-printf", "%s\\t%T@\\t%P\\n"])
+}
+
+// THE ACTIONS THAT MAKE find RUN ANOTHER PROGRAM ON WHAT IT MATCHED (find manual, "Run Commands"):
+// -exec and -execdir, and their prompting forms -ok and -okdir. find itself reads directory entries
+// and inodes; every content reader it can reach -- cat, sha256sum, grep -- arrives through one of
+// these four. find_regular_files_sha256_command above is a real builder that uses one.
+data find_program_running_actions: List<String> = ["-exec", "-execdir", "-ok", "-okdir"]
+
+fn find_command_runs_another_program(command: ArgvCommand) -> Bool {
+  any(command.arguments, a => any(find_program_running_actions, x => a == x))
+}

dag/extdeps/tools/gzip.dag

@@ -58,3 +58,19 @@ service gzip.Gzip {
     }
   }
 }
+
+// COMPRESS ONE FILE BESIDE ITSELF: `--keep --best --force`, which writes <path>.gz at level 9, keeps
+// the input, and replaces a stale <path>.gz (gzip manual, "Invoking gzip"). The compressed size is a
+// stat of <path>.gz; it is how a caller asks whether bytes are compressible without carrying them.
+service gzip.Compress {
+  operation BesideKeepBest {
+    requires none
+    input { path: NonEmptyStr }
+    output { success: Bool from "exit_success" }
+    transport shell { argv: ["gzip", "--keep", "--best", "--force", "--", "{path}"] }
+    exit {
+      0 => Unit
+      nonzero => String "gzip could not compress the path"
+    }
+  }
+}

dag/extdeps/tools/openssl.dag

@@ -113,3 +113,28 @@ data extdeps_model_scope: ExternalModelScope = ExternalModelScope {
   first_citation: extdeps.tools.openssl.extdeps_external_authority_anchor,
   further_citations: []
 }
+
+// `openssl enc -aes-256-ctr` ENCRYPTS ONE FILE INTO ANOTHER UNDER AES-256 IN COUNTER MODE (openssl-enc,
+// docs.openssl.org/3.0/man1/openssl-enc/). CTR is a stream mode, so the output is exactly as long as
+// the input -- no padding, and -nosalt drops the "Salted__" header -- and is the input XOR the AES
+// keystream. -pass pass:<p> derives the key and IV from the passphrase; -md sha256 pins that
+// derivation's digest, which is the default from OpenSSL 1.1.0 on, so 1.1 and 3.x hosts agree. The
+// passphrase is in argv and is therefore NOT a secret: this operation is for keyed, reproducible
+// streams whose key anyone may know (extdeps.crypto.seeded_filler), never for confidentiality.
+service openssl.Enc {
+  operation Aes256CtrPassphrase {
+    requires none
+    input { passphrase: NonEmptyStr, input_path: NonEmptyStr, output_path: NonEmptyStr }
+    output {
+      success: Bool from "exit_success"
+      stderr: String from "stderr"
+    }
+    transport shell {
+      argv: [openssl_program().invocation, "enc", "-aes-256-ctr", "-nosalt", "-md", "sha256", "-pass", "pass:{passphrase}", "-in", "{input_path}", "-out", "{output_path}"]
+    }
+    exit {
+      0 => Unit
+      nonzero => String "openssl enc refused"
+    }
+  }
+}