diff --git a/home-manager/services/cliproxyapi/README.md b/home-manager/services/cliproxyapi/README.md index 68843ce3e..2fab6bcaf 100644 --- a/home-manager/services/cliproxyapi/README.md +++ b/home-manager/services/cliproxyapi/README.md @@ -7,7 +7,7 @@ This directory contains the Nix-based configuration for the cliproxyapi service ### Services 1. **cliproxyapi** - Main proxy server on port 8317 -2. **cliproxyapi-backup** - File watcher that syncs auth files to S3 +2. **cliproxyapi-backup** - File watcher and wall-clock hourly job that syncs auth files and CPA Manager Plus analytics to S3 ### Scripts @@ -30,7 +30,11 @@ This directory contains the Nix-based configuration for the cliproxyapi service ~/.ccs/cliproxy/auth/ # CCS auth directory (synced from local) S3 Storage: -└── s3://cliproxyapi/auths/ # Auth storage +├── s3://cliproxyapi/auths/ # Auth storage +├── s3://cliproxyapi/cpa-manager-plus/analytics-backup-HH.tar.gz +│ # 24 hourly rollback slots (UTC) +└── s3://cliproxyapi/cpa-manager-plus/analytics-backup.tar.gz + # Latest SQLite snapshot + matching data.key ``` ## Data Flow @@ -56,10 +60,17 @@ key error when S3 already has auths. 1. Pull from S3 `auths/` → local 2. Copy local → CCS auth dir -### Backup (on file change) +### Backup (on auth-file change and each wall-clock hour) 1. Push local → S3 `auths/` 2. Copy local → CCS auth dir +3. Create and integrity-check an online CPA Manager Plus SQLite snapshot +4. Archive the snapshot with its matching `data.key` and upload it to S3 + +The SQLite online backup command is required because the live analytics database +uses WAL mode. Copying only `usage.sqlite` while CPA Manager Plus is running can +produce an incomplete backup. Each run updates the `latest` archive and its UTC +hour slot, retaining up to 24 hourly rollback points without unbounded growth. ### WatchPaths (file watchers) diff --git a/home-manager/services/cliproxyapi/default.nix b/home-manager/services/cliproxyapi/default.nix index 03a525502..abba50cb6 100644 --- a/home-manager/services/cliproxyapi/default.nix +++ b/home-manager/services/cliproxyapi/default.nix @@ -11,6 +11,8 @@ let commonScript = pkgs.replaceVars ./scripts/common.sh { aws = "${pkgs.awscli2}/bin/aws"; + sqlite3 = "${pkgs.sqlite}/bin/sqlite3"; + tar = "${pkgs.gnutar}/bin/tar"; }; hydrateScript = pkgs.replaceVars ./scripts/hydrate.sh { @@ -98,6 +100,9 @@ in pkgs.bash pkgs.coreutils pkgs.awscli2 + pkgs.gnutar + pkgs.gzip + pkgs.sqlite ] }:/opt/homebrew/bin:/usr/local/bin:/usr/bin"; }; @@ -105,6 +110,7 @@ in "${homeDir}/.cli-proxy-api/objectstore/auths" "${homeDir}/.ccs/cliproxy/auth" ]; + StartCalendarInterval = [ { Minute = 0; } ]; RunAtLoad = true; StandardOutPath = "/tmp/cliproxyapi-backup.log"; StandardErrorPath = "/tmp/cliproxyapi-backup.error.log"; @@ -215,11 +221,26 @@ in pkgs.bash pkgs.awscli2 pkgs.coreutils + pkgs.gnutar + pkgs.gzip + pkgs.sqlite ] }"; + UMask = "0077"; }; }; + systemd.user.timers.cliproxyapi-backup = lib.mkIf pkgs.stdenv.isLinux { + Unit.Description = "Periodically back up CLIProxyAPI and CPA Manager Plus data"; + Timer = { + OnBootSec = "5min"; + OnCalendar = "hourly"; + Persistent = true; + Unit = "cliproxyapi-backup.service"; + }; + Install.WantedBy = [ "timers.target" ]; + }; + # Periodic sync - pull auth files from S3 every 5 minutes systemd.user.timers.cliproxyapi-sync = lib.mkIf pkgs.stdenv.isLinux { Unit.Description = "Periodically sync auth files from S3"; diff --git a/home-manager/services/cliproxyapi/scripts/backup.sh b/home-manager/services/cliproxyapi/scripts/backup.sh index 8892bf0eb..cfce9c3be 100644 --- a/home-manager/services/cliproxyapi/scripts/backup.sh +++ b/home-manager/services/cliproxyapi/scripts/backup.sh @@ -7,21 +7,26 @@ set -euo pipefail AUTH_DIR="${HOME}/.cli-proxy-api/objectstore/auths" CCS_AUTH_DIR="${HOME}/.ccs/cliproxy/auth" cliproxy_init_objectstore_env +CPA_MANAGER_PLUS_DATA_DIR="${CPA_MANAGER_PLUS_DATA_DIR:-${HOME}/.cpa-manager-plus}" if ! cliproxy_has_objectstore_credentials; then echo "⚠️ Missing S3 credentials, skipping backup" >&2 exit 0 fi -if [ ! -d "$AUTH_DIR" ] || [ -z "$(ls -A "$AUTH_DIR" 2>/dev/null)" ]; then - echo "⚠️ No auth files to backup" >&2 - exit 0 -fi +if [ -d "$AUTH_DIR" ] && [ -n "$(ls -A "$AUTH_DIR" 2>/dev/null)" ]; then + echo "[$(date)] Backing up auth files..." >&2 -echo "[$(date)] Backing up auth files..." >&2 + cliproxy_sync_auth_to_s3 "$AUTH_DIR" -cliproxy_sync_auth_to_s3 "$AUTH_DIR" + # Also sync back to CCS auth dir so ccs can find the tokens + mkdir -p "$CCS_AUTH_DIR" + cp -fu "$AUTH_DIR"/*.json "$CCS_AUTH_DIR/" 2>/dev/null || true +else + echo "⚠️ No auth files to backup" >&2 +fi -# Also sync back to CCS auth dir so ccs can find the tokens -mkdir -p "$CCS_AUTH_DIR" -cp -u "$AUTH_DIR"/*.json "$CCS_AUTH_DIR/" 2>/dev/null || true +if [ -d "$CPA_MANAGER_PLUS_DATA_DIR" ]; then + echo "[$(date)] Backing up CPA Manager Plus analytics..." >&2 + cliproxy_backup_manager_data "$CPA_MANAGER_PLUS_DATA_DIR" +fi diff --git a/home-manager/services/cliproxyapi/scripts/common.sh b/home-manager/services/cliproxyapi/scripts/common.sh index a4c89657c..33ff17e89 100644 --- a/home-manager/services/cliproxyapi/scripts/common.sh +++ b/home-manager/services/cliproxyapi/scripts/common.sh @@ -88,3 +88,63 @@ cliproxy_download_usage_from_s3() { "$(cliproxy_usage_s3_uri)" \ "$dst" || true } + +cliproxy_manager_backup_s3_uri() { + local object_name="${1:-analytics-backup.tar.gz}" + printf 's3://%s/cpa-manager-plus/%s' "${OBJECTSTORE_BUCKET:?OBJECTSTORE_BUCKET is required}" "$object_name" +} + +cliproxy_backup_manager_data() ( + set -euo pipefail + umask 077 + + local data_dir="$1" + local database_path="${data_dir}/usage.sqlite" + local data_key_path="${data_dir}/data.key" + local backup_root snapshot_dir snapshot_path archive_path integrity hourly_object + + if [ ! -f "$database_path" ] || [ ! -f "$data_key_path" ]; then + echo "⚠️ CPA Manager Plus database or data key is missing; skipping analytics backup" >&2 + return 0 + fi + + backup_root="$(mktemp -d "${TMPDIR:-/tmp}/cpa-manager-plus-backup.XXXXXX")" + trap 'rm -rf -- "$backup_root"' EXIT + snapshot_dir="${backup_root}/cpa-manager-plus" + snapshot_path="${snapshot_dir}/usage.sqlite" + archive_path="${backup_root}/analytics-backup.tar.gz" + mkdir -p "$snapshot_dir" + + # The live database uses WAL mode. SQLite's online backup command produces a + # consistent standalone database without copying transient -wal/-shm files. + @sqlite3@ "$database_path" ".timeout 5000" ".backup '${snapshot_path}'" + + integrity="$(@sqlite3@ "$snapshot_path" "PRAGMA integrity_check;")" + if [ "$integrity" != "ok" ]; then + echo "CPA Manager Plus SQLite snapshot failed its integrity check" >&2 + return 1 + fi + + install -m 600 "$data_key_path" "${snapshot_dir}/data.key" + @tar@ -czf "$archive_path" -C "$snapshot_dir" usage.sqlite data.key + chmod 600 "$archive_path" + + # Keep one rollback point per UTC hour in addition to the convenient latest + # object. Auth-file triggers within the same hour replace only that hour's slot. + hourly_object="analytics-backup-$(date -u +%H).tar.gz" + AWS_ACCESS_KEY_ID="$OBJECTSTORE_ACCESS_KEY" \ + AWS_SECRET_ACCESS_KEY="$OBJECTSTORE_SECRET_KEY" \ + @aws@ s3 cp \ + --endpoint-url="$OBJECTSTORE_ENDPOINT" \ + --only-show-errors \ + "$archive_path" \ + "$(cliproxy_manager_backup_s3_uri "$hourly_object")" + + AWS_ACCESS_KEY_ID="$OBJECTSTORE_ACCESS_KEY" \ + AWS_SECRET_ACCESS_KEY="$OBJECTSTORE_SECRET_KEY" \ + @aws@ s3 cp \ + --endpoint-url="$OBJECTSTORE_ENDPOINT" \ + --only-show-errors \ + "$archive_path" \ + "$(cliproxy_manager_backup_s3_uri)" +) diff --git a/home-manager/services/cpa-manager-plus/start.sh b/home-manager/services/cpa-manager-plus/start.sh index 90523d800..f6e9c4a8d 100644 --- a/home-manager/services/cpa-manager-plus/start.sh +++ b/home-manager/services/cpa-manager-plus/start.sh @@ -51,10 +51,23 @@ if ! ensure_container_removed; then exit 1 fi +host_uid="$(id -u)" +host_gid="$(id -g)" + +# Older launches ran as root inside the container. Migrate the bind-mounted data +# before switching to the service user's numeric identity. +@docker@ run --rm \ + --user 0:0 \ + -v "$DATA_DIR:/data" \ + --entrypoint chown \ + "$IMAGE" \ + -R "${host_uid}:${host_gid}" /data + docker_args=( run --rm --name "$CONTAINER_NAME" + --user "${host_uid}:${host_gid}" --network host --ulimit nofile=65536:65536 -v "$DATA_DIR:/data" diff --git a/spec/cliproxyapi_backup_spec.sh b/spec/cliproxyapi_backup_spec.sh index 0f0df417b..5a58309d5 100644 --- a/spec/cliproxyapi_backup_spec.sh +++ b/spec/cliproxyapi_backup_spec.sh @@ -3,6 +3,7 @@ Describe 'cliproxyapi backup scripts' SCRIPTS_DIR="$PWD/home-manager/services/cliproxyapi/scripts" +NIX_MODULE="$PWD/home-manager/services/cliproxyapi/default.nix" # Preprocess scripts once at describe-time __PREPROCESSED_DIR=$(mktemp -d) @@ -13,6 +14,8 @@ __BACKUP_SCRIPT="$__PREPROCESSED_DIR/backup.sh" # Preprocess common.sh sed \ -e 's|@aws@|aws|g' \ + -e 's|@sqlite3@|sqlite3|g' \ + -e 's|@tar@|tar|g' \ "$SCRIPTS_DIR/common.sh" >"$__COMMON_SCRIPT" chmod +x "$__COMMON_SCRIPT" @@ -58,7 +61,7 @@ Before 'setup' After 'cleanup' It 'pulls from the configured S3 auth path' -When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__HYDRATE_SCRIPT"'" 2>&1; cat "$MOCK_LOG" 2>/dev/null || true' +When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__HYDRATE_SCRIPT"'" 2>&1; status=$?; cat "$MOCK_LOG" 2>/dev/null || true; exit "$status"' The status should be success The output should include 's3://cliproxyapi/auths/' End @@ -70,7 +73,7 @@ OBJECTSTORE_SECRET_KEY=test_secret OBJECTSTORE_ENDPOINT=https://test.endpoint.com OBJECTSTORE_BUCKET=custom-bucket ENV -When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__HYDRATE_SCRIPT"'" 2>&1; cat "$MOCK_LOG" 2>/dev/null || true' +When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__HYDRATE_SCRIPT"'" 2>&1; status=$?; cat "$MOCK_LOG" 2>/dev/null || true; exit "$status"' The status should be success The output should include 's3://custom-bucket/auths/' End @@ -87,6 +90,25 @@ Describe 'backup.sh' setup() { mock_bin_setup aws + cat >"$MOCK_BIN/sqlite3" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +: "${MOCK_LOG:?MOCK_LOG must be set}" +printf '%s\n' "$0 $*" >>"$MOCK_LOG" + +command="${*: -1}" +case "$command" in + ".backup '"*) + snapshot_path="${command#.backup \'}" + snapshot_path="${snapshot_path%\'}" + : >"$snapshot_path" + ;; + 'PRAGMA integrity_check;') + printf '%s\n' "${SQLITE_INTEGRITY_RESULT:-ok}" + ;; +esac +EOF + chmod +x "$MOCK_BIN/sqlite3" TEMP_HOME=$(mktemp -d) mkdir -p "$TEMP_HOME/.cli-proxy-api/objectstore/auths" mkdir -p "$TEMP_HOME/.ccs/cliproxy/auth" @@ -120,7 +142,7 @@ End It 'pushes to S3 when auth files exist' touch "$TEMP_HOME/.cli-proxy-api/objectstore/auths/test-auth.json" -When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__BACKUP_SCRIPT"'" 2>&1; cat "$MOCK_LOG" 2>/dev/null || true' +When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__BACKUP_SCRIPT"'" 2>&1; status=$?; cat "$MOCK_LOG" 2>/dev/null || true; exit "$status"' The status should be success The output should include 's3://cliproxyapi/auths/' End @@ -133,11 +155,33 @@ OBJECTSTORE_ENDPOINT=https://test.endpoint.com OBJECTSTORE_BUCKET=custom-bucket ENV touch "$TEMP_HOME/.cli-proxy-api/objectstore/auths/test-auth.json" -When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__BACKUP_SCRIPT"'" 2>&1; cat "$MOCK_LOG" 2>/dev/null || true' +When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__BACKUP_SCRIPT"'" 2>&1; status=$?; cat "$MOCK_LOG" 2>/dev/null || true; exit "$status"' The status should be success The output should include 's3://custom-bucket/auths/' End +It 'uploads a consistent CPA Manager Plus archive through the existing backup service' +mkdir -p "$TEMP_HOME/.cpa-manager-plus" +touch "$TEMP_HOME/.cpa-manager-plus/usage.sqlite" +touch "$TEMP_HOME/.cpa-manager-plus/data.key" +When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__BACKUP_SCRIPT"'" 2>&1; status=$?; cat "$MOCK_LOG" 2>/dev/null || true; exit "$status"' +The status should be success +The output should include ".backup '" +The output should include 'PRAGMA integrity_check;' +The output should match pattern '*s3://cliproxyapi/cpa-manager-plus/analytics-backup-??.tar.gz*' +The output should include 's3://cliproxyapi/cpa-manager-plus/analytics-backup.tar.gz' +End + +It 'fails closed and skips analytics uploads when the snapshot is corrupt' +mkdir -p "$TEMP_HOME/.cpa-manager-plus" +touch "$TEMP_HOME/.cpa-manager-plus/usage.sqlite" +touch "$TEMP_HOME/.cpa-manager-plus/data.key" +When run bash -c 'HOME="'"$TEMP_HOME"'" SQLITE_INTEGRITY_RESULT=corrupt bash "'"$__BACKUP_SCRIPT"'" 2>&1; status=$?; cat "$MOCK_LOG" 2>/dev/null || true; exit "$status"' +The status should be failure +The output should include 'SQLite snapshot failed its integrity check' +The output should not include 's3://cliproxyapi/cpa-manager-plus/' +End + It 'skips when credentials are missing' rm -f "$TEMP_HOME/dotfiles/.env" touch "$TEMP_HOME/.cli-proxy-api/objectstore/auths/test-auth.json" @@ -147,6 +191,15 @@ The output should include 'Missing S3 credentials' End End +Describe 'backup scheduling' +It 'uses wall-clock schedules independent of auth-file path triggers' +When run grep -E 'StartCalendarInterval|OnCalendar = "hourly"' "$NIX_MODULE" +The status should be success +The output should include 'StartCalendarInterval' +The output should include 'OnCalendar = "hourly"' +End +End + # Cleanup preprocessed scripts at end cleanup_preprocessed() { rm -rf "$__PREPROCESSED_DIR" diff --git a/spec/cpa_manager_plus_spec.sh b/spec/cpa_manager_plus_spec.sh index 77d35173b..a8c381601 100644 --- a/spec/cpa_manager_plus_spec.sh +++ b/spec/cpa_manager_plus_spec.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2016 Describe 'CPA Manager Plus service' NIX_MODULE="$PWD/home-manager/services/cpa-manager-plus/default.nix" @@ -34,6 +35,12 @@ The output should include 'seakee/cpa-manager-plus:latest' The output should include '--network host' End +It 'migrates persistent data ownership and runs as the service user' +When run grep -E -- '--entrypoint chown|--user "\$\{host_uid\}:\$\{host_gid\}"' "$START_SCRIPT" +The output should include '--entrypoint chown' +The output should include '--user "${host_uid}:${host_gid}"' +End + It 'persists SQLite and the encryption key under the data mount' When run grep -E 'USAGE_DB_PATH=/data/usage.sqlite|CPA_MANAGER_DATA_KEY_PATH=/data/data.key' "$START_SCRIPT" The output should include 'USAGE_DB_PATH=/data/usage.sqlite'