diff --git a/Makefile b/Makefile index 3e04a5583..3402b607d 100644 --- a/Makefile +++ b/Makefile @@ -778,11 +778,10 @@ launchctl-ollama: ## Restart ollama launchd agent. systemctl: systemctl-cliproxyapi systemctl-code-syncer systemctl-docker-postgres systemctl-dotfiles-updater systemctl-ollama systemctl-openclaw ## Restart all systemd user services. .PHONY: systemctl-cliproxyapi -systemctl-cliproxyapi: ## Pull latest image and restart cliproxyapi systemd user service. - @echo "🔄 Restarting cliproxyapi..." +systemctl-cliproxyapi: ## Reload systemd units for cliproxyapi (home-manager handles restart). + @echo "🔄 Reloading cliproxyapi..." @systemctl --user daemon-reload - @systemctl --user restart cliproxyapi.service || true - @echo "✅ cliproxyapi restarted" + @echo "✅ cliproxyapi reloaded" .PHONY: systemctl-code-syncer systemctl-code-syncer: ## Restart code-syncer systemd user service. diff --git a/config/claude/settings.json b/config/claude/settings.json index efbf3a7ed..9005f3a4d 100644 --- a/config/claude/settings.json +++ b/config/claude/settings.json @@ -330,6 +330,8 @@ "Bash(rm -rf /*:*)", "Bash(rm -rf ~/*:*)", "Bash(chmod -R 777:*)", + "Bash(docker system prune -a:*)", + "Bash(docker system prune -f:*)", "Bash(mkfs:*)", "Bash(dd if=:*)", "Bash(git push --force origin main:*)", diff --git a/config/openclaw/hydrate.sh b/config/openclaw/hydrate.sh index bdd388494..782ffb2d5 100644 --- a/config/openclaw/hydrate.sh +++ b/config/openclaw/hydrate.sh @@ -9,6 +9,7 @@ STATE_DIR="${OPENCLAW_STATE_DIR:-${HOME}/.openclaw}" CONFIG="${OPENCLAW_CONFIG_PATH:-${STATE_DIR}/openclaw.json}" TEMPLATE="@template@" SECRETS_DIR="${HOME}/.config/openclaw" +CLIPROXY_CONFIG="${OPENCLAW_CLIPROXY_CONFIG_PATH:-${HOME}/.cli-proxy-api/config.yaml}" ENV_FILE="${HOME}/dotfiles/.env" # Source .env if it exists @@ -30,6 +31,23 @@ read_secret() { echo "" } +read_cliproxy_api_key_from_config() { + local config_file="$1" + [ -f "$config_file" ] || return 0 + + awk ' + /^api-keys:/ { in_api_keys = 1; next } + in_api_keys && /^ - / { + value = $0 + sub(/^ - "/, "", value) + sub(/"$/, "", value) + print value + exit + } + in_api_keys && /^[^[:space:]]/ { exit } + ' "$config_file" +} + # Load gateway token (required for both modes) GATEWAY_TOKEN="${OPENCLAW_GATEWAY_TOKEN:-${GATEWAY_TOKEN:-$(read_secret "${SECRETS_DIR}/gateway-token")}}" @@ -42,7 +60,17 @@ mkdir -p "$STATE_DIR" if [ "$MODE" = "gateway" ]; then # Gateway mode (Kyber): hydrate full template and start gateway - CLIPROXY_API_KEY="${OPENCLAW_CLIPROXY_API_KEY:-${CLIPROXY_API_KEY:-$(read_secret "${SECRETS_DIR}/cliproxy-key")}}" + cliproxy_api_key_from_env="${CLIPROXY_API_KEY:-}" + CLIPROXY_API_KEY="${OPENCLAW_CLIPROXY_API_KEY:-}" + if [ -z "$CLIPROXY_API_KEY" ]; then + CLIPROXY_API_KEY="$(read_cliproxy_api_key_from_config "$CLIPROXY_CONFIG")" + fi + if [ -z "$CLIPROXY_API_KEY" ]; then + CLIPROXY_API_KEY="$cliproxy_api_key_from_env" + fi + if [ -z "$CLIPROXY_API_KEY" ]; then + CLIPROXY_API_KEY="$(read_secret "${SECRETS_DIR}/cliproxy-key")" + fi TELEGRAM_TOKEN="${OPENCLAW_TELEGRAM_TOKEN:-${TELEGRAM_TOKEN:-$(read_secret "${SECRETS_DIR}/telegram-token")}}" WHATSAPP_ALLOW_FROM="${OPENCLAW_WHATSAPP_ALLOW_FROM:-${WHATSAPP_ALLOW_FROM:-$(read_secret "${SECRETS_DIR}/whatsapp-allow-from")}}" ANTHROPIC_API_KEY="${OPENCLAW_ANTHROPIC_API_KEY:-${ANTHROPIC_API_KEY:-$(read_secret "${SECRETS_DIR}/anthropic-key")}}" diff --git a/config/pi/settings.json b/config/pi/settings.json index f9388376c..c6114ca3b 100644 --- a/config/pi/settings.json +++ b/config/pi/settings.json @@ -24,7 +24,8 @@ "keepRecentTokens": 20000 }, "packages": [ - "https://github.com/davebcn87/pi-autoresearch" + "https://github.com/davebcn87/pi-autoresearch", + "https://github.com/cagdotin/agents" ], "skills": [], "retry": { diff --git a/home-manager/services/cliproxyapi/README.md b/home-manager/services/cliproxyapi/README.md index c791ebd0e..5820cad47 100644 --- a/home-manager/services/cliproxyapi/README.md +++ b/home-manager/services/cliproxyapi/README.md @@ -30,31 +30,13 @@ This directory contains the Nix-based configuration for the cliproxyapi service ~/.ccs/cliproxy/auth/ # CCS auth directory (synced from local) S3 Storage: -├── s3://cliproxyapi/auths/ # Primary storage -└── s3://cliproxyapi/backup/auths/ # Redundant backup +└── s3://cliproxyapi/auths/ # Auth storage ``` ## Data Flow -``` -┌─────────────────────────────────────────────────────────┐ -│ S3 (Source of Truth) │ -│ ┌──────────────┐ ┌───────────────────────┐ │ -│ │ auths/ │ │ backup/auths/ │ │ -│ └──────┬───────┘ └───────────┬───────────┘ │ -└─────────┼─────────────────────────────┼─────────────────┘ - │ │ - ▼ hydrate.sh ▼ -┌─────────────────────────────────────────────────────────┐ -│ ~/.cli-proxy-api/objectstore/auths/ │ -│ (local cache) │ -└─────────────────────────┬───────────────────────────────┘ - │ - ▼ backup.sh (on file change) -┌─────────────────────────────────────────────────────────┐ -│ ~/.ccs/cliproxy/auth/ │ -│ (CCS compatibility) │ -└─────────────────────────────────────────────────────────┘ +``` +S3 auths/ -> ~/.cli-proxy-api/objectstore/auths -> ~/.ccs/cliproxy/auth ``` ### Pre-start guard (service) @@ -72,14 +54,12 @@ key error when S3 already has auths. ### Hydrate (on activation/switch) 1. Pull from S3 `auths/` → local -2. Pull from S3 `backup/auths/` → local (takes precedence, overwrites conflicts) -3. Copy local → CCS auth dir +2. Copy local → CCS auth dir ### Backup (on file change) 1. Push local → S3 `auths/` -2. Push local → S3 `backup/auths/` -3. Copy local → CCS auth dir +2. Copy local → CCS auth dir ### WatchPaths (file watchers) diff --git a/home-manager/services/cliproxyapi/default.nix b/home-manager/services/cliproxyapi/default.nix index edcf438f3..dd8974421 100644 --- a/home-manager/services/cliproxyapi/default.nix +++ b/home-manager/services/cliproxyapi/default.nix @@ -9,17 +9,22 @@ let config.home.homeDirectory or (if pkgs.stdenv.isDarwin then builtins.getEnv "HOME" else "/home/${config.home.username}"); - hydrateScript = pkgs.replaceVars ./scripts/hydrate.sh { + commonScript = pkgs.replaceVars ./scripts/common.sh { aws = "${pkgs.awscli2}/bin/aws"; }; + hydrateScript = pkgs.replaceVars ./scripts/hydrate.sh { + common = commonScript; + }; + backupScript = pkgs.replaceVars ./scripts/backup.sh { - aws = "${pkgs.awscli2}/bin/aws"; + common = commonScript; }; startScript = pkgs.replaceVars ./scripts/start.sh { sed = "${pkgs.gnused}/bin/sed"; aws = "${pkgs.awscli2}/bin/aws"; + common = commonScript; }; # Smart wrapper that handles both NixOS and non-NixOS Linux @@ -45,7 +50,7 @@ let ''; wrapperScript = pkgs.replaceVars ./scripts/wrapper.sh { - aws = "${pkgs.awscli2}/bin/aws"; + common = commonScript; }; cliWrapper = pkgs.writeShellScriptBin "cliproxyapi" (builtins.readFile wrapperScript); diff --git a/home-manager/services/cliproxyapi/scripts/backup.sh b/home-manager/services/cliproxyapi/scripts/backup.sh index 325313837..8892bf0eb 100644 --- a/home-manager/services/cliproxyapi/scripts/backup.sh +++ b/home-manager/services/cliproxyapi/scripts/backup.sh @@ -2,28 +2,13 @@ # Push auth files from local cache to S3 # shellcheck source=/dev/null set -euo pipefail +. "@common@" AUTH_DIR="${HOME}/.cli-proxy-api/objectstore/auths" CCS_AUTH_DIR="${HOME}/.ccs/cliproxy/auth" -ENV_FILE="${HOME}/dotfiles/.env" +cliproxy_init_objectstore_env -if [ -f "$ENV_FILE" ]; then - set -a - . "$ENV_FILE" - set +a -fi - -strip_quotes() { - local v="$1" - v="${v%\"}" - v="${v#\"}" - printf '%s' "$v" -} -ENDPOINT="$(strip_quotes "${OBJECTSTORE_ENDPOINT:-}")" -ACCESS_KEY="$(strip_quotes "${OBJECTSTORE_ACCESS_KEY:-}")" -SECRET_KEY="$(strip_quotes "${OBJECTSTORE_SECRET_KEY:-}")" - -if [ -z "$ENDPOINT" ] || [ -z "$ACCESS_KEY" ] || [ -z "$SECRET_KEY" ]; then +if ! cliproxy_has_objectstore_credentials; then echo "⚠️ Missing S3 credentials, skipping backup" >&2 exit 0 fi @@ -35,21 +20,7 @@ fi echo "[$(date)] Backing up auth files..." >&2 -AWS_ACCESS_KEY_ID="$ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$SECRET_KEY" \ - @aws@ s3 sync \ - --endpoint-url="$ENDPOINT" \ - --no-progress \ - "$AUTH_DIR/" \ - "s3://cliproxyapi/auths/" && echo "✅ Backed up to S3 auths/" >&2 - -AWS_ACCESS_KEY_ID="$ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$SECRET_KEY" \ - @aws@ s3 sync \ - --endpoint-url="$ENDPOINT" \ - --no-progress \ - "$AUTH_DIR/" \ - "s3://cliproxyapi/backup/auths/" && echo "✅ Backed up to S3 backup/auths/" >&2 +cliproxy_sync_auth_to_s3 "$AUTH_DIR" # Also sync back to CCS auth dir so ccs can find the tokens mkdir -p "$CCS_AUTH_DIR" diff --git a/home-manager/services/cliproxyapi/scripts/common.sh b/home-manager/services/cliproxyapi/scripts/common.sh new file mode 100644 index 000000000..84f7641c4 --- /dev/null +++ b/home-manager/services/cliproxyapi/scripts/common.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +cliproxy_load_env() { + local env_file="${HOME}/dotfiles/.env" + if [ -f "$env_file" ]; then + set -a + # shellcheck source=/dev/null + . "$env_file" + set +a + fi +} + +cliproxy_strip_quotes() { + local value="$1" + value="${value%\"}" + value="${value#\"}" + printf '%s' "$value" +} + +cliproxy_init_objectstore_env() { + cliproxy_load_env + OBJECTSTORE_ENDPOINT="$(cliproxy_strip_quotes "${OBJECTSTORE_ENDPOINT:-}")" + OBJECTSTORE_BUCKET="$(cliproxy_strip_quotes "${OBJECTSTORE_BUCKET:-cliproxyapi}")" + OBJECTSTORE_ACCESS_KEY="$(cliproxy_strip_quotes "${OBJECTSTORE_ACCESS_KEY:-}")" + OBJECTSTORE_SECRET_KEY="$(cliproxy_strip_quotes "${OBJECTSTORE_SECRET_KEY:-}")" + export OBJECTSTORE_ENDPOINT OBJECTSTORE_BUCKET OBJECTSTORE_ACCESS_KEY OBJECTSTORE_SECRET_KEY +} + +cliproxy_has_objectstore_credentials() { + [ -n "${OBJECTSTORE_ENDPOINT:-}" ] && + [ -n "${OBJECTSTORE_ACCESS_KEY:-}" ] && + [ -n "${OBJECTSTORE_SECRET_KEY:-}" ] +} + +cliproxy_auth_s3_uri() { + printf 's3://%s/auths/' "${OBJECTSTORE_BUCKET:?OBJECTSTORE_BUCKET is required}" +} + +cliproxy_s3_sync() { + local source_path="$1" + local destination_path="$2" + AWS_ACCESS_KEY_ID="${OBJECTSTORE_ACCESS_KEY:?OBJECTSTORE_ACCESS_KEY is required}" \ + AWS_SECRET_ACCESS_KEY="${OBJECTSTORE_SECRET_KEY:?OBJECTSTORE_SECRET_KEY is required}" \ + @aws@ s3 sync \ + --endpoint-url="${OBJECTSTORE_ENDPOINT:?OBJECTSTORE_ENDPOINT is required}" \ + --no-progress \ + "$source_path" \ + "$destination_path" || true +} + +cliproxy_sync_auth_from_s3() { + local auth_dir="$1" + mkdir -p "$auth_dir" + cliproxy_s3_sync "$(cliproxy_auth_s3_uri)" "$auth_dir/" +} + +cliproxy_sync_auth_to_s3() { + local auth_dir="$1" + cliproxy_s3_sync "$auth_dir/" "$(cliproxy_auth_s3_uri)" +} diff --git a/home-manager/services/cliproxyapi/scripts/hydrate.sh b/home-manager/services/cliproxyapi/scripts/hydrate.sh index 4b0188683..f847bf295 100644 --- a/home-manager/services/cliproxyapi/scripts/hydrate.sh +++ b/home-manager/services/cliproxyapi/scripts/hydrate.sh @@ -2,49 +2,18 @@ # Pull auth files from S3 to local cache # shellcheck source=/dev/null set -euo pipefail +. "@common@" AUTH_DIR="${HOME}/.cli-proxy-api/objectstore/auths" CCS_AUTH_DIR="${HOME}/.ccs/cliproxy/auth" -ENV_FILE="${HOME}/dotfiles/.env" +cliproxy_init_objectstore_env -if [ -f "$ENV_FILE" ]; then - set -a - . "$ENV_FILE" - set +a -fi - -strip_quotes() { - local v="$1" - v="${v%\"}" - v="${v#\"}" - printf '%s' "$v" -} -ENDPOINT="$(strip_quotes "${OBJECTSTORE_ENDPOINT:-}")" -ACCESS_KEY="$(strip_quotes "${OBJECTSTORE_ACCESS_KEY:-}")" -SECRET_KEY="$(strip_quotes "${OBJECTSTORE_SECRET_KEY:-}")" - -if [ -z "$ENDPOINT" ] || [ -z "$ACCESS_KEY" ] || [ -z "$SECRET_KEY" ]; then +if ! cliproxy_has_objectstore_credentials; then echo "⚠️ Missing S3 credentials, skipping hydrate" >&2 exit 0 fi -mkdir -p "$AUTH_DIR" - -AWS_ACCESS_KEY_ID="$ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$SECRET_KEY" \ - @aws@ s3 sync \ - --endpoint-url="$ENDPOINT" \ - --no-progress \ - "s3://cliproxyapi/auths/" \ - "$AUTH_DIR/" && echo "✅ Hydrated from S3 auths/" >&2 - -AWS_ACCESS_KEY_ID="$ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$SECRET_KEY" \ - @aws@ s3 sync \ - --endpoint-url="$ENDPOINT" \ - --no-progress \ - "s3://cliproxyapi/backup/auths/" \ - "$AUTH_DIR/" && echo "✅ Hydrated from S3 backup/auths/" >&2 +cliproxy_sync_auth_from_s3 "$AUTH_DIR" # Also sync to CCS auth dir so ccs can find the tokens mkdir -p "$CCS_AUTH_DIR" diff --git a/home-manager/services/cliproxyapi/scripts/start.sh b/home-manager/services/cliproxyapi/scripts/start.sh index 93fbd7c45..de28f10d6 100755 --- a/home-manager/services/cliproxyapi/scripts/start.sh +++ b/home-manager/services/cliproxyapi/scripts/start.sh @@ -1,74 +1,31 @@ #!/usr/bin/env bash # shellcheck source=/dev/null set -euo pipefail +. "@common@" CONFIG_DIR="${HOME}/.cli-proxy-api" TEMPLATE="$CONFIG_DIR/config.template.yaml" CONFIG="$CONFIG_DIR/config.yaml" -ENV_FILE="${HOME}/dotfiles/.env" AUTH_DIR="${CONFIG_DIR}/objectstore/auths" USAGE_EXPORT_FILE="${CONFIG_DIR}/usage-export.json" MANAGEMENT_URL="${CLIPROXY_MANAGEMENT_URL:-http://127.0.0.1:8317/v0/management}" -if [ -f "$ENV_FILE" ]; then - set -a - . "$ENV_FILE" - set +a -fi - -strip_quotes() { - local v="$1" - v="${v%\"}" - v="${v#\"}" - printf '%s' "$v" -} -OBJECTSTORE_ENDPOINT="$(strip_quotes "${OBJECTSTORE_ENDPOINT:-}")" -OBJECTSTORE_BUCKET="$(strip_quotes "${OBJECTSTORE_BUCKET:-cliproxyapi}")" -OBJECTSTORE_ACCESS_KEY="$(strip_quotes "${OBJECTSTORE_ACCESS_KEY:-}")" -OBJECTSTORE_SECRET_KEY="$(strip_quotes "${OBJECTSTORE_SECRET_KEY:-}")" +cliproxy_init_objectstore_env OBJECTSTORE_LOCAL_PATH="$CONFIG_DIR" MANAGEMENT_PASSWORD="${CLIPROXY_MANAGEMENT_PASSWORD:-}" MANAGEMENT_KEY="${CLIPROXY_MANAGEMENT_PASSWORD:-${CLIPROXY_MANAGEMENT_KEY:-}}" export OBJECTSTORE_ENDPOINT OBJECTSTORE_BUCKET OBJECTSTORE_ACCESS_KEY OBJECTSTORE_SECRET_KEY OBJECTSTORE_LOCAL_PATH MANAGEMENT_PASSWORD -if [ -n "$OBJECTSTORE_ENDPOINT" ] && [ -n "$OBJECTSTORE_ACCESS_KEY" ] && [ -n "$OBJECTSTORE_SECRET_KEY" ]; then +if cliproxy_has_objectstore_credentials; then mkdir -p "$AUTH_DIR" if [ -z "$(ls -A "$AUTH_DIR" 2>/dev/null)" ]; then echo "⚠️ Local auth cache empty; hydrating from S3" >&2 - AWS_ACCESS_KEY_ID="$OBJECTSTORE_ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$OBJECTSTORE_SECRET_KEY" \ - @aws@ s3 sync \ - --endpoint-url="$OBJECTSTORE_ENDPOINT" \ - --no-progress \ - "s3://${OBJECTSTORE_BUCKET}/auths/" \ - "$AUTH_DIR/" || true - - AWS_ACCESS_KEY_ID="$OBJECTSTORE_ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$OBJECTSTORE_SECRET_KEY" \ - @aws@ s3 sync \ - --endpoint-url="$OBJECTSTORE_ENDPOINT" \ - --no-progress \ - "s3://${OBJECTSTORE_BUCKET}/backup/auths/" \ - "$AUTH_DIR/" || true + cliproxy_sync_auth_from_s3 "$AUTH_DIR" fi if [ -n "$(ls -A "$AUTH_DIR" 2>/dev/null)" ]; then - AWS_ACCESS_KEY_ID="$OBJECTSTORE_ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$OBJECTSTORE_SECRET_KEY" \ - @aws@ s3 sync \ - --endpoint-url="$OBJECTSTORE_ENDPOINT" \ - --no-progress \ - "$AUTH_DIR/" \ - "s3://${OBJECTSTORE_BUCKET}/auths/" || true - - AWS_ACCESS_KEY_ID="$OBJECTSTORE_ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$OBJECTSTORE_SECRET_KEY" \ - @aws@ s3 sync \ - --endpoint-url="$OBJECTSTORE_ENDPOINT" \ - --no-progress \ - "$AUTH_DIR/" \ - "s3://${OBJECTSTORE_BUCKET}/backup/auths/" || true + cliproxy_sync_auth_to_s3 "$AUTH_DIR" fi fi @@ -151,7 +108,7 @@ usage_import() { # shellcheck disable=SC2329 # Invoked via trap usage_export() { - if [ -z "$MANAGEMENT_KEY" ]; then + if [ -z "${MANAGEMENT_KEY:-}" ] || [ -z "${USAGE_EXPORT_FILE:-}" ]; then return 0 fi mkdir -p "$(dirname "$USAGE_EXPORT_FILE")" @@ -190,6 +147,7 @@ if [ "$(uname)" = "Linux" ] && command -v docker >/dev/null 2>&1; then echo "⏭️ Skipping docker pull (docker not accessible)" fi + docker stop cliproxyapi 2>/dev/null || true docker rm -f cliproxyapi 2>/dev/null || true mkdir -p "$CONFIG_DIR/logs" docker run --rm \ diff --git a/home-manager/services/cliproxyapi/scripts/wrapper.sh b/home-manager/services/cliproxyapi/scripts/wrapper.sh index 638692d49..4fce932f6 100644 --- a/home-manager/services/cliproxyapi/scripts/wrapper.sh +++ b/home-manager/services/cliproxyapi/scripts/wrapper.sh @@ -1,65 +1,21 @@ #!/usr/bin/env bash # shellcheck source=/dev/null set -euo pipefail +. "@common@" CONFIG_DIR="${HOME}/.cli-proxy-api" AUTH_DIR="${CONFIG_DIR}/objectstore/auths" -ENV_FILE="${HOME}/dotfiles/.env" -if [ -f "$ENV_FILE" ]; then - set -a - . "$ENV_FILE" - set +a -fi - -strip_quotes() { - local v="$1" - v="${v%\"}" - v="${v#\"}" - printf '%s' "$v" -} - -OBJECTSTORE_ENDPOINT="$(strip_quotes "${OBJECTSTORE_ENDPOINT:-}")" -OBJECTSTORE_BUCKET="$(strip_quotes "${OBJECTSTORE_BUCKET:-cliproxyapi}")" -OBJECTSTORE_ACCESS_KEY="$(strip_quotes "${OBJECTSTORE_ACCESS_KEY:-}")" -OBJECTSTORE_SECRET_KEY="$(strip_quotes "${OBJECTSTORE_SECRET_KEY:-}")" +cliproxy_init_objectstore_env OBJECTSTORE_LOCAL_PATH="$CONFIG_DIR" export OBJECTSTORE_ENDPOINT OBJECTSTORE_BUCKET OBJECTSTORE_ACCESS_KEY OBJECTSTORE_SECRET_KEY OBJECTSTORE_LOCAL_PATH -if [ -n "$OBJECTSTORE_ENDPOINT" ] && [ -n "$OBJECTSTORE_ACCESS_KEY" ] && [ -n "$OBJECTSTORE_SECRET_KEY" ]; then +if cliproxy_has_objectstore_credentials; then mkdir -p "$AUTH_DIR" if [ -n "$(ls -A "$AUTH_DIR" 2>/dev/null)" ]; then - AWS_ACCESS_KEY_ID="$OBJECTSTORE_ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$OBJECTSTORE_SECRET_KEY" \ - @aws@ s3 sync \ - --endpoint-url="$OBJECTSTORE_ENDPOINT" \ - --no-progress \ - "$AUTH_DIR/" \ - "s3://${OBJECTSTORE_BUCKET}/auths/" || true - - AWS_ACCESS_KEY_ID="$OBJECTSTORE_ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$OBJECTSTORE_SECRET_KEY" \ - @aws@ s3 sync \ - --endpoint-url="$OBJECTSTORE_ENDPOINT" \ - --no-progress \ - "$AUTH_DIR/" \ - "s3://${OBJECTSTORE_BUCKET}/backup/auths/" || true + cliproxy_sync_auth_to_s3 "$AUTH_DIR" else - AWS_ACCESS_KEY_ID="$OBJECTSTORE_ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$OBJECTSTORE_SECRET_KEY" \ - @aws@ s3 sync \ - --endpoint-url="$OBJECTSTORE_ENDPOINT" \ - --no-progress \ - "s3://${OBJECTSTORE_BUCKET}/auths/" \ - "$AUTH_DIR/" || true - - AWS_ACCESS_KEY_ID="$OBJECTSTORE_ACCESS_KEY" \ - AWS_SECRET_ACCESS_KEY="$OBJECTSTORE_SECRET_KEY" \ - @aws@ s3 sync \ - --endpoint-url="$OBJECTSTORE_ENDPOINT" \ - --no-progress \ - "s3://${OBJECTSTORE_BUCKET}/backup/auths/" \ - "$AUTH_DIR/" || true + cliproxy_sync_auth_from_s3 "$AUTH_DIR" fi fi diff --git a/spec/cliproxyapi_backup_spec.sh b/spec/cliproxyapi_backup_spec.sh index 94c269a54..0f0df417b 100644 --- a/spec/cliproxyapi_backup_spec.sh +++ b/spec/cliproxyapi_backup_spec.sh @@ -6,18 +6,25 @@ SCRIPTS_DIR="$PWD/home-manager/services/cliproxyapi/scripts" # Preprocess scripts once at describe-time __PREPROCESSED_DIR=$(mktemp -d) +__COMMON_SCRIPT="$__PREPROCESSED_DIR/common.sh" __HYDRATE_SCRIPT="$__PREPROCESSED_DIR/hydrate.sh" __BACKUP_SCRIPT="$__PREPROCESSED_DIR/backup.sh" -# Preprocess hydrate.sh +# Preprocess common.sh sed \ -e 's|@aws@|aws|g' \ + "$SCRIPTS_DIR/common.sh" >"$__COMMON_SCRIPT" +chmod +x "$__COMMON_SCRIPT" + +# Preprocess hydrate.sh +sed \ + -e 's|@common@|'"$__COMMON_SCRIPT"'|g' \ "$SCRIPTS_DIR/hydrate.sh" >"$__HYDRATE_SCRIPT" chmod +x "$__HYDRATE_SCRIPT" # Preprocess backup.sh sed \ - -e 's|@aws@|aws|g' \ + -e 's|@common@|'"$__COMMON_SCRIPT"'|g' \ "$SCRIPTS_DIR/backup.sh" >"$__BACKUP_SCRIPT" chmod +x "$__BACKUP_SCRIPT" @@ -50,11 +57,22 @@ cleanup() { Before 'setup' After 'cleanup' -It 'pulls from S3 auths and backup/auths' +It 'pulls from the configured S3 auth path' When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__HYDRATE_SCRIPT"'" 2>&1; cat "$MOCK_LOG" 2>/dev/null || true' The status should be success The output should include 's3://cliproxyapi/auths/' -The output should include 's3://cliproxyapi/backup/auths/' +End + +It 'uses OBJECTSTORE_BUCKET for hydrate paths' +cat >"$TEMP_HOME/dotfiles/.env" <<'ENV' +OBJECTSTORE_ACCESS_KEY=test_key +OBJECTSTORE_SECRET_KEY=test_secret +OBJECTSTORE_ENDPOINT=https://test.endpoint.com +OBJECTSTORE_BUCKET=custom-bucket +ENV +When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__HYDRATE_SCRIPT"'" 2>&1; cat "$MOCK_LOG" 2>/dev/null || true' +The status should be success +The output should include 's3://custom-bucket/auths/' End It 'skips when credentials are missing' @@ -105,7 +123,19 @@ touch "$TEMP_HOME/.cli-proxy-api/objectstore/auths/test-auth.json" When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__BACKUP_SCRIPT"'" 2>&1; cat "$MOCK_LOG" 2>/dev/null || true' The status should be success The output should include 's3://cliproxyapi/auths/' -The output should include 's3://cliproxyapi/backup/auths/' +End + +It 'uses OBJECTSTORE_BUCKET for backup paths' +cat >"$TEMP_HOME/dotfiles/.env" <<'ENV' +OBJECTSTORE_ACCESS_KEY=test_key +OBJECTSTORE_SECRET_KEY=test_secret +OBJECTSTORE_ENDPOINT=https://test.endpoint.com +OBJECTSTORE_BUCKET=custom-bucket +ENV +touch "$TEMP_HOME/.cli-proxy-api/objectstore/auths/test-auth.json" +When run bash -c 'HOME="'"$TEMP_HOME"'" bash "'"$__BACKUP_SCRIPT"'" 2>&1; cat "$MOCK_LOG" 2>/dev/null || true' +The status should be success +The output should include 's3://custom-bucket/auths/' End It 'skips when credentials are missing' diff --git a/spec/coverage_spec.sh b/spec/coverage_spec.sh index 0ac9b9da2..453765b05 100644 --- a/spec/coverage_spec.sh +++ b/spec/coverage_spec.sh @@ -53,6 +53,10 @@ It 'has spec file for home-manager/services/cliproxyapi/scripts/backup-auth.sh' The path "spec/cliproxyapi_backup_spec.sh" should be exist End +It 'has spec file for home-manager/services/cliproxyapi/scripts/common.sh' +The path "spec/cliproxyapi_backup_spec.sh" should be exist +End + It 'has spec file for home-manager/services/code-syncer/sync.sh' The path "spec/code_syncer_spec.sh" should be exist End @@ -152,6 +156,7 @@ home-manager/programs/neovim/run_tests.sh home-manager/programs/tmux/session-logger.sh home-manager/services/brew-upgrader/upgrade.sh home-manager/services/cliproxyapi/scripts/backup.sh +home-manager/services/cliproxyapi/scripts/common.sh home-manager/services/cliproxyapi/scripts/hydrate.sh home-manager/services/cliproxyapi/scripts/start.sh home-manager/services/cliproxyapi/scripts/wrapper.sh diff --git a/spec/openclaw_hydrate_spec.sh b/spec/openclaw_hydrate_spec.sh index f2cf96c13..187126980 100644 --- a/spec/openclaw_hydrate_spec.sh +++ b/spec/openclaw_hydrate_spec.sh @@ -31,10 +31,20 @@ It 'reads from dotfiles .env file' When run bash -c "grep 'ENV_FILE=' '$SCRIPT'" The output should include 'dotfiles/.env' End + +It 'uses cliproxyapi config as the Kyber source of truth' +When run bash -c "grep 'CLIPROXY_CONFIG=' '$SCRIPT'" +The output should include '.cli-proxy-api/config.yaml' +End End Describe 'secret loading' -It 'loads CLIPROXY_API_KEY from file' +It 'loads CLIPROXY_API_KEY from cliproxyapi config root' +When run bash -c "grep 'read_cliproxy_api_key_from_config' '$SCRIPT'" +The output should include 'read_cliproxy_api_key_from_config' +End + +It 'falls back to cliproxy key file' When run bash -c "grep 'CLIPROXY_API_KEY' '$SCRIPT'" The output should include 'cliproxy-key' End @@ -60,6 +70,70 @@ The output should include 'whatsapp-allow-from' End End +Describe 'gateway api key resolution' +setup_gateway() { + TEMP_HOME=$(mktemp -d) + mkdir -p "$TEMP_HOME/.cli-proxy-api" + mkdir -p "$TEMP_HOME/.config/openclaw" + mkdir -p "$TEMP_HOME/openclaw/bin" + mkdir -p "$TEMP_HOME/chromium/bin" + mkdir -p "$TEMP_HOME/templates" + + cat >"$TEMP_HOME/.cli-proxy-api/config.yaml" <<'YAML' +api-keys: + - "from-cliproxy-config" +YAML + + cat >"$TEMP_HOME/.config/openclaw/gateway-token" <<'EOF' +gateway-token +EOF + + cat >"$TEMP_HOME/.config/openclaw/cliproxy-key" <<'EOF' +from-secret-file +EOF + + cat >"$TEMP_HOME/templates/openclaw.json" <<'EOF' +{"apiKey":"__CLIPROXY_API_KEY__","token":"__GATEWAY_TOKEN__","workspace":"__WORKSPACE__","home":"__HOME__","chromium":"__CHROMIUM_PATH__"} +EOF + + cat >"$TEMP_HOME/openclaw/bin/openclaw" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + chmod +x "$TEMP_HOME/openclaw/bin/openclaw" + + cat >"$TEMP_HOME/chromium/bin/chromium" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + chmod +x "$TEMP_HOME/chromium/bin/chromium" + + PREPROCESSED_SCRIPT="$TEMP_HOME/hydrate.sh" + sed \ + -e 's|@mode@|gateway|g' \ + -e 's|@sed@|sed|g' \ + -e 's|@template@|'"$TEMP_HOME"'/templates/openclaw.json|g' \ + -e 's|@chromium@|'"$TEMP_HOME"'/chromium|g' \ + -e 's|@openclaw@|'"$TEMP_HOME"'/openclaw|g' \ + "$SCRIPT" >"$PREPROCESSED_SCRIPT" + chmod +x "$PREPROCESSED_SCRIPT" +} + +cleanup_gateway() { + rm -rf "$TEMP_HOME" +} + +Before 'setup_gateway' +After 'cleanup_gateway' + +It 'prefers the root api-keys entry from cliproxyapi config over the secret file' +When run bash -c 'HOME="'"$TEMP_HOME"'" OPENCLAW_CONFIG_PATH="'"$TEMP_HOME"'/generated-openclaw.json" bash "'"$PREPROCESSED_SCRIPT"'" >/dev/null 2>&1; cat "'"$TEMP_HOME"'/generated-openclaw.json"' +The status should be success +The output should include 'from-cliproxy-config' +The output should not include 'from-secret-file' +End +End + Describe 'config generation' It 'uses sed to substitute values in template' When run bash -c "grep '@sed@' '$SCRIPT'"