feat(cliproxy): back up manager analytics to S3

home-manager/services/cliproxyapi/README.md

@@ -7,7 +7,7 @@ This directory contains the Nix-based configuration for the cliproxyapi service
 ### Services
 
 1. **cliproxyapi** - Main proxy server on port 8317
-2. **cliproxyapi-backup** - File watcher that syncs auth files to S3
+2. **cliproxyapi-backup** - File watcher and wall-clock hourly job that syncs auth files and CPA Manager Plus analytics to S3
 
 ### Scripts
 
@@ -30,7 +30,11 @@ This directory contains the Nix-based configuration for the cliproxyapi service
 ~/.ccs/cliproxy/auth/              # CCS auth directory (synced from local)
 
 S3 Storage:
-└── s3://cliproxyapi/auths/        # Auth storage
+├── s3://cliproxyapi/auths/        # Auth storage
+├── s3://cliproxyapi/cpa-manager-plus/analytics-backup-HH.tar.gz
+│                                     # 24 hourly rollback slots (UTC)
+└── s3://cliproxyapi/cpa-manager-plus/analytics-backup.tar.gz
+                                      # Latest SQLite snapshot + matching data.key
 ```
 
 ## Data Flow
@@ -56,10 +60,17 @@ key error when S3 already has auths.
 1. Pull from S3 `auths/` → local
 2. Copy local → CCS auth dir
 
-### Backup (on file change)
+### Backup (on auth-file change and each wall-clock hour)
 
 1. Push local → S3 `auths/`
 2. Copy local → CCS auth dir
+3. Create and integrity-check an online CPA Manager Plus SQLite snapshot
+4. Archive the snapshot with its matching `data.key` and upload it to S3
+
+The SQLite online backup command is required because the live analytics database
+uses WAL mode. Copying only `usage.sqlite` while CPA Manager Plus is running can
+produce an incomplete backup. Each run updates the `latest` archive and its UTC
+hour slot, retaining up to 24 hourly rollback points without unbounded growth.
 
 ### WatchPaths (file watchers)
 

home-manager/services/cliproxyapi/default.nix

@@ -11,6 +11,8 @@ let
 
   commonScript = pkgs.replaceVars ./scripts/common.sh {
     aws = "${pkgs.awscli2}/bin/aws";
+    sqlite3 = "${pkgs.sqlite}/bin/sqlite3";
+    tar = "${pkgs.gnutar}/bin/tar";
   };
 
   hydrateScript = pkgs.replaceVars ./scripts/hydrate.sh {
@@ -98,13 +100,17 @@ in
             pkgs.bash
             pkgs.coreutils
             pkgs.awscli2
+            pkgs.gnutar
+            pkgs.gzip
+            pkgs.sqlite
           ]
         }:/opt/homebrew/bin:/usr/local/bin:/usr/bin";
       };
       WatchPaths = [
         "${homeDir}/.cli-proxy-api/objectstore/auths"
         "${homeDir}/.ccs/cliproxy/auth"
       ];
+      StartCalendarInterval = [ { Minute = 0; } ];
       RunAtLoad = true;
       StandardOutPath = "/tmp/cliproxyapi-backup.log";
       StandardErrorPath = "/tmp/cliproxyapi-backup.error.log";
@@ -215,11 +221,26 @@ in
           pkgs.bash
           pkgs.awscli2
           pkgs.coreutils
+          pkgs.gnutar
+          pkgs.gzip
+          pkgs.sqlite
         ]
       }";
+      UMask = "0077";
     };
   };
 
+  systemd.user.timers.cliproxyapi-backup = lib.mkIf pkgs.stdenv.isLinux {
+    Unit.Description = "Periodically back up CLIProxyAPI and CPA Manager Plus data";
+    Timer = {
+      OnBootSec = "5min";
+      OnCalendar = "hourly";
+      Persistent = true;
+      Unit = "cliproxyapi-backup.service";
+    };
+    Install.WantedBy = [ "timers.target" ];
+  };
+
   # Periodic sync - pull auth files from S3 every 5 minutes
   systemd.user.timers.cliproxyapi-sync = lib.mkIf pkgs.stdenv.isLinux {
     Unit.Description = "Periodically sync auth files from S3";

home-manager/services/cliproxyapi/scripts/backup.sh

@@ -7,21 +7,26 @@ set -euo pipefail
 AUTH_DIR="${HOME}/.cli-proxy-api/objectstore/auths"
 CCS_AUTH_DIR="${HOME}/.ccs/cliproxy/auth"
 cliproxy_init_objectstore_env
+CPA_MANAGER_PLUS_DATA_DIR="${CPA_MANAGER_PLUS_DATA_DIR:-${HOME}/.cpa-manager-plus}"
 
 if ! cliproxy_has_objectstore_credentials; then
   echo "⚠️  Missing S3 credentials, skipping backup" >&2
   exit 0
 fi
 
-if [ ! -d "$AUTH_DIR" ] || [ -z "$(ls -A "$AUTH_DIR" 2>/dev/null)" ]; then
-  echo "⚠️  No auth files to backup" >&2
-  exit 0
-fi
+if [ -d "$AUTH_DIR" ] && [ -n "$(ls -A "$AUTH_DIR" 2>/dev/null)" ]; then
+  echo "[$(date)] Backing up auth files..." >&2
 
-echo "[$(date)] Backing up auth files..." >&2
+  cliproxy_sync_auth_to_s3 "$AUTH_DIR"
 
-cliproxy_sync_auth_to_s3 "$AUTH_DIR"
+  # Also sync back to CCS auth dir so ccs can find the tokens
+  mkdir -p "$CCS_AUTH_DIR"
+  cp -fu "$AUTH_DIR"/*.json "$CCS_AUTH_DIR/" 2>/dev/null || true
+else
+  echo "⚠️  No auth files to backup" >&2
+fi
 
-# Also sync back to CCS auth dir so ccs can find the tokens
-mkdir -p "$CCS_AUTH_DIR"
-cp -u "$AUTH_DIR"/*.json "$CCS_AUTH_DIR/" 2>/dev/null || true
+if [ -d "$CPA_MANAGER_PLUS_DATA_DIR" ]; then
+  echo "[$(date)] Backing up CPA Manager Plus analytics..." >&2
+  cliproxy_backup_manager_data "$CPA_MANAGER_PLUS_DATA_DIR"
+fi

home-manager/services/cliproxyapi/scripts/common.sh

@@ -88,3 +88,63 @@ cliproxy_download_usage_from_s3() {
     "$(cliproxy_usage_s3_uri)" \
     "$dst" || true
 }
+
+cliproxy_manager_backup_s3_uri() {
+  local object_name="${1:-analytics-backup.tar.gz}"
+  printf 's3://%s/cpa-manager-plus/%s' "${OBJECTSTORE_BUCKET:?OBJECTSTORE_BUCKET is required}" "$object_name"
+}
+
+cliproxy_backup_manager_data() (
+  set -euo pipefail
+  umask 077
+
+  local data_dir="$1"
+  local database_path="${data_dir}/usage.sqlite"
+  local data_key_path="${data_dir}/data.key"
+  local backup_root snapshot_dir snapshot_path archive_path integrity hourly_object
+
+  if [ ! -f "$database_path" ] || [ ! -f "$data_key_path" ]; then
+    echo "⚠️  CPA Manager Plus database or data key is missing; skipping analytics backup" >&2
+    return 0
+  fi
+
+  backup_root="$(mktemp -d "${TMPDIR:-/tmp}/cpa-manager-plus-backup.XXXXXX")"
+  trap 'rm -rf -- "$backup_root"' EXIT
+  snapshot_dir="${backup_root}/cpa-manager-plus"
+  snapshot_path="${snapshot_dir}/usage.sqlite"
+  archive_path="${backup_root}/analytics-backup.tar.gz"
+  mkdir -p "$snapshot_dir"
+
+  # The live database uses WAL mode. SQLite's online backup command produces a
+  # consistent standalone database without copying transient -wal/-shm files.
+  @sqlite3@ "$database_path" ".timeout 5000" ".backup '${snapshot_path}'"
+
+  integrity="$(@sqlite3@ "$snapshot_path" "PRAGMA integrity_check;")"
+  if [ "$integrity" != "ok" ]; then
+    echo "CPA Manager Plus SQLite snapshot failed its integrity check" >&2
+    return 1
+  fi
+
+  install -m 600 "$data_key_path" "${snapshot_dir}/data.key"
+  @tar@ -czf "$archive_path" -C "$snapshot_dir" usage.sqlite data.key
+  chmod 600 "$archive_path"
+
+  # Keep one rollback point per UTC hour in addition to the convenient latest
+  # object. Auth-file triggers within the same hour replace only that hour's slot.
+  hourly_object="analytics-backup-$(date -u +%H).tar.gz"
+  AWS_ACCESS_KEY_ID="$OBJECTSTORE_ACCESS_KEY" \
+    AWS_SECRET_ACCESS_KEY="$OBJECTSTORE_SECRET_KEY" \
+    @aws@ s3 cp \
+    --endpoint-url="$OBJECTSTORE_ENDPOINT" \
+    --only-show-errors \
+    "$archive_path" \
+    "$(cliproxy_manager_backup_s3_uri "$hourly_object")"
+
+  AWS_ACCESS_KEY_ID="$OBJECTSTORE_ACCESS_KEY" \
+    AWS_SECRET_ACCESS_KEY="$OBJECTSTORE_SECRET_KEY" \
+    @aws@ s3 cp \
+    --endpoint-url="$OBJECTSTORE_ENDPOINT" \
+    --only-show-errors \
+    "$archive_path" \
+    "$(cliproxy_manager_backup_s3_uri)"
+)

home-manager/services/cpa-manager-plus/start.sh

@@ -51,10 +51,23 @@ if ! ensure_container_removed; then
   exit 1
 fi
 
+host_uid="$(id -u)"
+host_gid="$(id -g)"
+
+# Older launches ran as root inside the container. Migrate the bind-mounted data
+# before switching to the service user's numeric identity.
+@docker@ run --rm \
+  --user 0:0 \
+  -v "$DATA_DIR:/data" \
+  --entrypoint chown \
+  "$IMAGE" \
+  -R "${host_uid}:${host_gid}" /data
+
 docker_args=(
   run
   --rm
   --name "$CONTAINER_NAME"
+  --user "${host_uid}:${host_gid}"
   --network host
   --ulimit nofile=65536:65536
   -v "$DATA_DIR:/data"

spec/cliproxyapi_backup_spec.sh

@@ -3,6 +3,7 @@
 
 Describe 'cliproxyapi backup scripts'
 SCRIPTS_DIR="$PWD/home-manager/services/cliproxyapi/scripts"
+NIX_MODULE="$PWD/home-manager/services/cliproxyapi/default.nix"
 
 # Preprocess scripts once at describe-time
 __PREPROCESSED_DIR=$(mktemp -d)
@@ -13,6 +14,8 @@ __BACKUP_SCRIPT="$__PREPROCESSED_DIR/backup.sh"
 # Preprocess common.sh
 sed \
   -e 's|@aws@|aws|g' \
+  -e 's|@sqlite3@|sqlite3|g' \
+  -e 's|@tar@|tar|g' \
   "$SCRIPTS_DIR/common.sh" >"$__COMMON_SCRIPT"
 chmod +x "$__COMMON_SCRIPT"
 
@@ -58,7 +61,7 @@ Before 'setup'
 After 'cleanup'
 
 It 'pulls from the configured S3 auth path'
-When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__HYDRATE_SCRIPT"'" 2>&1; cat "$MOCK_LOG" 2>/dev/null || true'
+When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__HYDRATE_SCRIPT"'" 2>&1; status=$?; cat "$MOCK_LOG" 2>/dev/null || true; exit "$status"'
 The status should be success
 The output should include 's3://cliproxyapi/auths/'
 End
@@ -70,7 +73,7 @@ OBJECTSTORE_SECRET_KEY=test_secret
 OBJECTSTORE_ENDPOINT=https://test.endpoint.com
 OBJECTSTORE_BUCKET=custom-bucket
 ENV
-When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__HYDRATE_SCRIPT"'" 2>&1; cat "$MOCK_LOG" 2>/dev/null || true'
+When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__HYDRATE_SCRIPT"'" 2>&1; status=$?; cat "$MOCK_LOG" 2>/dev/null || true; exit "$status"'
 The status should be success
 The output should include 's3://custom-bucket/auths/'
 End
@@ -87,6 +90,25 @@ Describe 'backup.sh'
 
 setup() {
   mock_bin_setup aws
+  cat >"$MOCK_BIN/sqlite3" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+: "${MOCK_LOG:?MOCK_LOG must be set}"
+printf '%s\n' "$0 $*" >>"$MOCK_LOG"
+
+command="${*: -1}"
+case "$command" in
+  ".backup '"*)
+    snapshot_path="${command#.backup \'}"
+    snapshot_path="${snapshot_path%\'}"
+    : >"$snapshot_path"
+    ;;
+  'PRAGMA integrity_check;')
+    printf '%s\n' "${SQLITE_INTEGRITY_RESULT:-ok}"
+    ;;
+esac
+EOF
+  chmod +x "$MOCK_BIN/sqlite3"
   TEMP_HOME=$(mktemp -d)
   mkdir -p "$TEMP_HOME/.cli-proxy-api/objectstore/auths"
   mkdir -p "$TEMP_HOME/.ccs/cliproxy/auth"
@@ -120,7 +142,7 @@ End
 
 It 'pushes to S3 when auth files exist'
 touch "$TEMP_HOME/.cli-proxy-api/objectstore/auths/test-auth.json"
-When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__BACKUP_SCRIPT"'" 2>&1; cat "$MOCK_LOG" 2>/dev/null || true'
+When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__BACKUP_SCRIPT"'" 2>&1; status=$?; cat "$MOCK_LOG" 2>/dev/null || true; exit "$status"'
 The status should be success
 The output should include 's3://cliproxyapi/auths/'
 End
@@ -133,11 +155,33 @@ OBJECTSTORE_ENDPOINT=https://test.endpoint.com
 OBJECTSTORE_BUCKET=custom-bucket
 ENV
 touch "$TEMP_HOME/.cli-proxy-api/objectstore/auths/test-auth.json"
-When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__BACKUP_SCRIPT"'" 2>&1; cat "$MOCK_LOG" 2>/dev/null || true'
+When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__BACKUP_SCRIPT"'" 2>&1; status=$?; cat "$MOCK_LOG" 2>/dev/null || true; exit "$status"'
 The status should be success
 The output should include 's3://custom-bucket/auths/'
 End
 
+It 'uploads a consistent CPA Manager Plus archive through the existing backup service'
+mkdir -p "$TEMP_HOME/.cpa-manager-plus"
+touch "$TEMP_HOME/.cpa-manager-plus/usage.sqlite"
+touch "$TEMP_HOME/.cpa-manager-plus/data.key"
+When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__BACKUP_SCRIPT"'" 2>&1; status=$?; cat "$MOCK_LOG" 2>/dev/null || true; exit "$status"'
+The status should be success
+The output should include ".backup '"
+The output should include 'PRAGMA integrity_check;'
+The output should match pattern '*s3://cliproxyapi/cpa-manager-plus/analytics-backup-??.tar.gz*'
+The output should include 's3://cliproxyapi/cpa-manager-plus/analytics-backup.tar.gz'
+End
+
+It 'fails closed and skips analytics uploads when the snapshot is corrupt'
+mkdir -p "$TEMP_HOME/.cpa-manager-plus"
+touch "$TEMP_HOME/.cpa-manager-plus/usage.sqlite"
+touch "$TEMP_HOME/.cpa-manager-plus/data.key"
+When run bash -c 'HOME="'"$TEMP_HOME"'" SQLITE_INTEGRITY_RESULT=corrupt bash "'"$__BACKUP_SCRIPT"'" 2>&1; status=$?; cat "$MOCK_LOG" 2>/dev/null || true; exit "$status"'
+The status should be failure
+The output should include 'SQLite snapshot failed its integrity check'
+The output should not include 's3://cliproxyapi/cpa-manager-plus/'
+End
+
 It 'skips when credentials are missing'
 rm -f "$TEMP_HOME/dotfiles/.env"
 touch "$TEMP_HOME/.cli-proxy-api/objectstore/auths/test-auth.json"
@@ -147,6 +191,15 @@ The output should include 'Missing S3 credentials'
 End
 End
 
+Describe 'backup scheduling'
+It 'uses wall-clock schedules independent of auth-file path triggers'
+When run grep -E 'StartCalendarInterval|OnCalendar = "hourly"' "$NIX_MODULE"
+The status should be success
+The output should include 'StartCalendarInterval'
+The output should include 'OnCalendar = "hourly"'
+End
+End
+
 # Cleanup preprocessed scripts at end
 cleanup_preprocessed() {
   rm -rf "$__PREPROCESSED_DIR"

spec/cpa_manager_plus_spec.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# shellcheck disable=SC2016
 
 Describe 'CPA Manager Plus service'
 NIX_MODULE="$PWD/home-manager/services/cpa-manager-plus/default.nix"
@@ -34,6 +35,12 @@ The output should include 'seakee/cpa-manager-plus:latest'
 The output should include '--network host'
 End
 
+It 'migrates persistent data ownership and runs as the service user'
+When run grep -E -- '--entrypoint chown|--user "\$\{host_uid\}:\$\{host_gid\}"' "$START_SCRIPT"
+The output should include '--entrypoint chown'
+The output should include '--user "${host_uid}:${host_gid}"'
+End
+
 It 'persists SQLite and the encryption key under the data mount'
 When run grep -E 'USAGE_DB_PATH=/data/usage.sqlite|CPA_MANAGER_DATA_KEY_PATH=/data/data.key' "$START_SCRIPT"
 The output should include 'USAGE_DB_PATH=/data/usage.sqlite'