Promote develop to main: the backup pull, capture/, the media gate, and mtime-restored hard links

.editorconfig

@@ -63,7 +63,7 @@ end_of_line = unset
 insert_final_newline = false
 trim_trailing_whitespace = false
 
-# Caddy map files are tab-separated redirect tables generated by `checks/build-redirects.py`.
+# Caddy map files are space-separated redirect tables generated by `capture/build-redirects.py`.
 # Trailing whitespace is significant to the parse, and the generator owns the formatting.
 [deploy/maps/*.map]
 trim_trailing_whitespace = false

.gitattributes

@@ -15,17 +15,35 @@
 # A CRLF shebang breaks execution, so scripts stay LF regardless of the default.
 *.sh text eol=lf
 
-# The URL-parity and redirect-map generators are shebang-executable and run by path in CI.
-# They are pinned individually rather than by a blanket `*.py` rule.
-checks/build-redirects.py text eol=lf
+# Shebang-executable files carrying no extension, which every rule above matches by
+# extension and therefore misses. `ops/vps-backup-pull` is run by systemd on the backup
+# host. A named line per file, as with the Python rules below, and `check-eol-pins.py`
+# fails if a tracked shebang file ever lands without one.
+ops/vps-backup-pull text eol=lf
+
+# Shebang-executable Python, where a CRLF on line one is a broken interpreter line rather
+# than a cosmetic difference. `capture/` is pinned whole, because everything there is a
+# script; under `checks/` the two executables are named, because that directory also holds
+# lists and fixtures. Neither is a blanket `*.py` rule, so a future non-executable module
+# elsewhere is not swept in by accident, and a new executable needs a line here.
+capture/*.py text eol=lf
 checks/check-url-parity.py text eol=lf
+checks/check-env-docs.py text eol=lf
+checks/check-eol-pins.py text eol=lf
 
-# These formats are parsed line by line by a daemon rather than by a shell.
-# Caddy and OpenSSH both reject or silently mis-parse a CRLF file.
-# The deploy shell is an extensionless shebang script that matches no rule above.
+# Caddy config is parsed line by line by a daemon rather than by a shell, and a CRLF file
+# is rejected or silently mis-parsed. Both files are named: the bundle's `Caddyfile`, and
+# the bootstrap that is installed into the container's config directory and is the only
+# Caddy file outside the release.
+#
+# The restricted `authorized_keys` and the forced-command deploy shell were pinned here
+# too, and this repository has never carried either: they live on the server, described in
+# OPERATIONS.md "Server Hardening". A pin binds nothing for a file that does not exist,
+# and the comment claiming to cover "the extensionless shebang script" is what let the
+# real one, `ops/vps-backup-pull`, sit unpinned above. `check-eol-pins.py` now fails on a
+# pattern matching no tracked file, so neither can come back silently.
 deploy/Caddyfile text eol=lf
-deploy/blog-deploy-shell text eol=lf
-deploy/authorized_keys text eol=lf
+deploy/bootstrap.Caddyfile text eol=lf
 
 # Caddy map files are tabular data read by `map` directives.
 # They stay LF for the same reason as the Caddyfile.

.github/workflows/deploy-site-task.yml

@@ -69,21 +69,47 @@ jobs:
     steps:
 
       # Full history, because a shallow clone silently changes page metadata if git info is on.
+      # The mtime restore below needs it too: a shallow clone has no commit to date a file from.
       - name: Checkout code step
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           fetch-depth: 0
 
-      # The pin lives in the action, so the deploy and validation cannot install different generators.
-      - name: Install Hugo step
-        uses: ./.github/actions/install-hugo
-
-      # REQUIRE_BROTLI below makes a missing binary fatal, so this keeps the build from failing.
-      - name: Install brotli step
+      # One update for the job, because each one is a network round trip that can fail on its
+      # own. REQUIRE_BROTLI later makes a missing brotli fatal, so this keeps the build from
+      # failing, and git-restore-mtime is what the next step runs.
+      - name: Install build tools step
         run: |
           set -Eeuo pipefail
           sudo apt-get update
-          sudo apt-get install --yes --no-install-recommends brotli
+          sudo apt-get install --yes --no-install-recommends brotli git-restore-mtime
+
+      # Git stores no mtimes, so a checkout stamps every file with the moment it was written.
+      # The deploy uploads with --link-dest against the previous release, and a file only links
+      # when size and mtime both match, so today nothing links and every release is a full copy.
+      # Restoring the last-commit time makes static/ match between releases: measured across two
+      # independent clones, all 1052 files land on identical mtimes, which is the same 1052 Hugo
+      # reports as static files and the same 1052 that link on a locally built release.
+      #
+      # static/ only. The generated pages are written fresh by every build and can never match,
+      # and walking the whole tree to prove that costs history reads for nothing.
+      #
+      # ORDERING: this is deliberately behind the live media check that #64 added. While every
+      # file arrives as a fresh inode, the upload re-asserts the mode contract on every deploy.
+      # Once a third of the tree arrives as hard links, a link carries the mode its inode chain
+      # began with, so a media file that acquires a bad one stays present, correctly named and
+      # unreadable, through every later release. The live check is what notices that, by
+      # requesting images and failing on the 403.
+      # `git restore-mtime`, the subcommand form, because the package installs into git's
+      # exec-path at /usr/lib/git-core rather than onto PATH, so the bare name does not resolve.
+      - name: Restore file mtimes step
+        run: |
+          set -Eeuo pipefail
+          git restore-mtime static
+
+      # The pin lives in the action, so the deploy and validation cannot install different generators.
+      - name: Install Hugo step
+        uses: ./.github/actions/install-hugo
 
       # Derived once and used three times, as the directory name, the stamp, and EXPECT_RELEASE.
       # Deriving it twice yields ids seconds apart, and the gate then asserts a phantom version.

.github/workflows/validate-task.yml

@@ -53,11 +53,13 @@ jobs:
         run: docker run --rm --pull=always -v "$PWD":/check --workdir /check mstruebing/editorconfig-checker:latest
 
       # The shell clean-compile is shellcheck at default severity plus `shfmt -d`, both reporting nothing.
-      # The formatter reads .editorconfig, which pins these scripts to tabs.
+      # The formatter reads .editorconfig, which pins the .sh files to tabs.
+      # ops/vps-backup-pull carries no extension, so it takes the [*] default of four spaces instead.
+      # Use `-d` rather than `-w` here and locally: the container writes as root and would take ownership of the tree.
       - name: Lint shell scripts step
         run: |
           set -Eeuo pipefail
-          scripts=(checks/check-live-urls.sh deploy/make-release.sh)
+          scripts=(checks/check-live-urls.sh deploy/make-release.sh ops/vps-backup-pull ops/install.sh capture/run-wp2hugo.sh)
           docker run --rm --pull=always -v "$PWD":/mnt --workdir /mnt \
             koalaman/shellcheck:stable "${scripts[@]}"
           docker run --rm --pull=always -v "$PWD":/mnt --workdir /mnt \
@@ -71,6 +73,18 @@ jobs:
           done
           python3 -c 'import yaml,sys; yaml.safe_load(open("hugo.yaml"))'
 
+      # Every configuration value is described once, in ENVIRONMENT.md.
+      # A new value gets added wherever its author is working, and nothing else notices a
+      # missing row. Runs both directions: undocumented values, and rows describing nothing.
+      - name: Check environment docs step
+        run: python3 checks/check-env-docs.py
+
+      # The .gitattributes line-ending pins are hand-maintained, and nothing read them back.
+      # This is what reads them back: it fails on a tracked shebang file with no LF pin,
+      # and on a pin naming no tracked file.
+      - name: Check line-ending pins step
+        run: python3 checks/check-eol-pins.py
+
       # The pin lives in the action, so validation and the deploy cannot install different generators.
       - name: Install Hugo step
         uses: ./.github/actions/install-hugo

.gitignore

@@ -19,7 +19,7 @@ hugo_stats.json
 *.user
 .claude
 
-# Python byproducts from the check and redirect-map generators under `checks/`.
+# Python byproducts from the gates under `checks/` and the provenance tools under `capture/`.
 __pycache__/
 *.py[cod]
 .venv/
@@ -30,13 +30,16 @@ __pycache__/
 # Host-specific values: deploy roots, base URLs, container names, and uids.
 # Each names one particular machine rather than the project.
 # The whole directory is ignored so a value added later lands ignored by default.
-# `deploy/env.example` is the committed template and sits outside the directory.
-# The last pattern is the backstop for one written outside the directory, matching the
-# `<server>.<environment>.env` shape those files are named for rather than a single literal name.
-# `deploy/env.example` does not end in `.env`, so it is unaffected.
+# `example.env` is the committed template and sits at the repository root.
+# The `*.env` pattern is the backstop for a real environment file written outside the
+# directory, matching the `<server>.<environment>.env` shape those files are named for
+# rather than a single literal name. It also matches the template, so the template is
+# negated on the line after it, anchored so it only exempts the one at the root. Order
+# matters: a negation placed before its pattern does nothing.
 secrets/
 **/secrets
 *.env
+!/example.env
 
 # The working copies of the host channel described in OPERATIONS.md.
 # They carry server internals, and the host's own backup is what makes them durable.