diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 2dfc4d888..78ec2771c 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -213,7 +213,8 @@ PMOVES.AI is a **production-ready multi-agent orchestration platform** featuring - Services: DB (Postgres 17.6.1), GoTrue, PostgREST v14.3, Kong 3.7.1, Realtime v2.72.0, Storage v1.37.1, Studio, imgproxy, pg-meta, Edge Functions, Analytics (Logflare), Vector, Supavisor - Canonical consumer URL: `http://supabase-kong:8000/rest/v1` (via Kong gateway) - Standard variable names: `JWT_SECRET`, `ANON_KEY`, `SERVICE_ROLE_KEY` (SUPABASE_* aliases for compat) -n**Neo4j** [Port 7474 HTTP, 7687 Bolt, 2004 Metrics] + +**Neo4j** [Port 7474 HTTP, 7687 Bolt] - Graph database for knowledge management, CHIT consciousness taxonomy, agent memory - Profile-based integration: make -C pmoves neo4j-local-up - API: POST http://localhost:7474/db/neo4j/tx/commit (Cypher transactions) @@ -221,22 +222,11 @@ n**Neo4j** [Port 7474 HTTP, 7687 Bolt, 2004 Metrics] - **Use for:** Graph queries, relationship traversal, CHIT consciousness taxonomy, agent memory - **Submodule:** PMOVES-Neo4j (root-level, follows PMOVES-supabase pattern) - **See:** `PMOVES-Neo4j/CLAUDE.md` for submodule context -- Schema: `pmoves_core`, Archon prompts -- **Use for:** Metadata storage, content records, agent state, auth, connection pooling **Qdrant** [Port 6333] - Vector embeddings for semantic search - Collection: `pmoves_chunks` -**Neo4j** [Port 7474 HTTP, 7687 Bolt, 2004 Metrics] -- Graph database for knowledge management, CHIT consciousness taxonomy, agent memory -- Profile-based integration: make -C pmoves neo4j-local-up -- API: POST http://localhost:7474/db/neo4j/tx/commit (Cypher transactions) -- Health: GET http://localhost:7474/db/neo4j/health -- **Use for:** Graph queries, relationship traversal, CHIT consciousness taxonomy, agent memory -- **Submodule:** PMOVES-Neo4j (root-level, follows PMOVES-supabase pattern) -- **See:** `PMOVES-Neo4j/CLAUDE.md` for submodule context - **Meilisearch** [Port 7700] - Full-text keyword search - Typo-tolerant, substring search diff --git a/pmoves/Makefile b/pmoves/Makefile index 5505336dc..bcdd987fb 100644 --- a/pmoves/Makefile +++ b/pmoves/Makefile @@ -640,60 +640,6 @@ supabase-clean: ## Stop + remove Supabase containers and volumes @$(DC) --profile supabase-local down -v --remove-orphans >/dev/null 2>&1 || true @echo "✅ Supabase stack cleaned" -.PHONY: neo4j-up neo4j-down neo4j-restart neo4j-logs neo4j-migrate neo4j-seed neo4j-bootstrap neo4j-status - -neo4j-up: ## Start Neo4j submodule stack - @echo "→ Starting Neo4j from submodule..." - @$(MAKE) --no-print-directory -C pmoves/integrations/neo4j up - -neo4j-down: ## Stop Neo4j submodule stack - @echo "→ Stopping Neo4j from submodule..." - @$(MAKE) --no-print-directory -C pmoves/integrations/neo4j down - -neo4j-restart: ## Restart Neo4j submodule stack - @echo "→ Restarting Neo4j from submodule..." - @$(MAKE) --no-print-directory -C pmoves/integrations/neo4j restart - -neo4j-logs: ## View Neo4j logs - @$(MAKE) --no-print-directory -C pmoves/integrations/neo4j logs - -neo4j-migrate: ## Run Neo4j migration (usage: make neo4j-migrate VERSION=003) - @if [ -z "$(VERSION)" ]; then \ - echo "Usage: make neo4j-migrate VERSION=003"; \ - echo ""; \ - echo "Available migrations:"; \ - ls -1 pmoves/integrations/neo4j/db/migrations/*.cypher 2>/dev/null | while read -r f; do \ - echo " • $$(basename "$f")"; \ - done; \ - exit 1; \ - fi - @$(MAKE) --no-print-directory -C pmoves/integrations/neo4j migrate VERSION=$(VERSION) - -neo4j-seed: ## Load Neo4j seed data (usage: make neo4j-seed SEED=001_person_aliases.csv) - @if [ -z "$(SEED)" ]; then \ - echo "Usage: make neo4j-seed SEED=001_person_aliases.csv"; \ - echo ""; \ - echo "Available seeds:"; \ - ls -1 pmoves/integrations/neo4j/db/seeds/*.csv 2>/dev/null | while read -r f; do \ - echo " • $$(basename "$f")"; \ - done; \ - exit 1; \ - fi - @$(MAKE) --no-print-directory -C pmoves/integrations/neo4j seed SEED=$(SEED) - -neo4j-bootstrap: ## Initialize Neo4j (run migrations and seeds) - @echo "→ Bootstrapping Neo4j from submodule..." - @$(MAKE) --no-print-directory -C pmoves/integrations/neo4j bootstrap - -neo4j-status: ## Check Neo4j status from submodule - @$(MAKE) --no-print-directory -C pmoves/integrations/neo4j status - -# Legacy compatibility (delegates to submodule) -.PHONY: neo4j-bootstrap-legacy -neo4j-bootstrap-legacy: ## Legacy bootstrap script (deprecated: use neo4j-bootstrap) - @echo "⚠️ This target is deprecated. Use: make neo4j-bootstrap" - @$(MAKE) --no-print-directory neo4j-bootstrap - .PHONY: seed-data seed-data: ## Seed Qdrant/Meilisearch with demo corpus via Hi-RAG v2 @echo "📚 Seeding Qdrant/Meili demo corpus..." @@ -1917,10 +1863,6 @@ neo4j-reset: ## DANGEROUS: wipe Neo4j volume and recreate with current NEO4J_AUT "/var/lib/neo4j/bin/cypher-shell -u \"$$user\" -p \"$$pass\" \"RETURN 1\""; \ ' -neo4j-status: ## Show Neo4j logs and health line - @$(DC) ps neo4j || true - @$(DC) logs --tail 60 neo4j || true - brand-verify: ## Verify key branded endpoints respond @echo "Presign:" && curl -fsS http://localhost:8088/healthz && echo @echo "Supabase REST:" && curl -fsS -o /dev/null -w '%{http_code}\n' http://host.docker.internal:65421/rest/v1 || true @@ -2820,49 +2762,49 @@ github-app-test: ## Test GitHub App token minting (requires services running) neo4j-up: ## Start Neo4j submodule stack @echo "→ Starting Neo4j from submodule..." - @$(MAKE) --no-print-directory -C PMOVES-Neo4j up + @$(MAKE) --no-print-directory -C ../PMOVES-Neo4j up neo4j-down: ## Stop Neo4j submodule stack @echo "→ Stopping Neo4j from submodule..." - @$(MAKE) --no-print-directory -C PMOVES-Neo4j down + @$(MAKE) --no-print-directory -C ../PMOVES-Neo4j down neo4j-restart: ## Restart Neo4j submodule stack @echo "→ Restarting Neo4j from submodule..." - @$(MAKE) --no-print-directory -C PMOVES-Neo4j restart + @$(MAKE) --no-print-directory -C ../PMOVES-Neo4j restart neo4j-logs: ## View Neo4j logs - @$(MAKE) --no-print-directory -C PMOVES-Neo4j logs + @$(MAKE) --no-print-directory -C ../PMOVES-Neo4j logs neo4j-migrate: ## Run Neo4j migration (usage: make neo4j-migrate VERSION=003) @if [ -z "$(VERSION)" ]; then \ echo "Usage: make neo4j-migrate VERSION=003"; \ echo ""; \ echo "Available migrations:"; \ - ls -1 PMOVES-Neo4j/db/migrations/*.cypher 2>/dev/null | while read -r f; do \ + ls -1 ../PMOVES-Neo4j/db/migrations/*.cypher 2>/dev/null | while read -r f; do \ echo " • $$(basename "$$f")"; \ done; \ exit 1; \ fi - @$(MAKE) --no-print-directory -C PMOVES-Neo4j migrate VERSION=$(VERSION) + @$(MAKE) --no-print-directory -C ../PMOVES-Neo4j migrate VERSION=$(VERSION) neo4j-seed: ## Load Neo4j seed data (usage: make neo4j-seed SEED=001_person_aliases.csv) @if [ -z "$(SEED)" ]; then \ echo "Usage: make neo4j-seed SEED=001_person_aliases.csv"; \ echo ""; \ echo "Available seeds:"; \ - ls -1 PMOVES-Neo4j/db/seeds/*.csv 2>/dev/null | while read -r f; do \ + ls -1 ../PMOVES-Neo4j/db/seeds/*.csv 2>/dev/null | while read -r f; do \ echo " • $$(basename "$$f")"; \ done; \ exit 1; \ fi - @$(MAKE) --no-print-directory -C PMOVES-Neo4j seed SEED=$(SEED) + @$(MAKE) --no-print-directory -C ../PMOVES-Neo4j seed SEED=$(SEED) neo4j-bootstrap: ## Initialize Neo4j (run migrations and seeds) @echo "→ Bootstrapping Neo4j from submodule..." - @$(MAKE) --no-print-directory -C PMOVES-Neo4j bootstrap + @$(MAKE) --no-print-directory -C ../PMOVES-Neo4j bootstrap neo4j-status: ## Check Neo4j status from submodule - @$(MAKE) --no-print-directory -C PMOVES-Neo4j status + @$(MAKE) --no-print-directory -C ../PMOVES-Neo4j status # Legacy compatibility .PHONY: neo4j-bootstrap-legacy @@ -2881,7 +2823,8 @@ neo4j-local-up: ## Start Neo4j with profile (make neo4j-local-up) neo4j-local-down: ## Stop Neo4j with profile (make neo4j-local-down) @echo "→ Stopping Neo4j with neo4j-local profile..." - @$(DC) --profile neo4j-local down -v --remove-orphans + @$(DC) --profile neo4j-local stop neo4j >/dev/null 2>&1 || true + @$(DC) --profile neo4j-local rm -f neo4j >/dev/null 2>&1 || true @echo "✅ Neo4j stopped" neo4j-local-status: ## Check Neo4j status (make neo4j-local-status) @@ -2890,10 +2833,10 @@ neo4j-local-status: ## Check Neo4j status (make neo4j-local-status) .PHONY: neo4j-backup neo4j-restore -neo4j-backup: ## Backup Neo4j database to pmoves/backups/ with timestamp +neo4j-backup: ## Backup Neo4j database to backups/ with timestamp @echo "→ Creating Neo4j backup..." @timestamp=$$(date +%Y%m%d_%H%M%S); \ - backup_dir="pmoves/backups"; \ + backup_dir="backups"; \ mkdir -p "$$backup_dir"; \ container_name=$$($(DC) ps -q neo4j 2>/dev/null | head -1); \ if [ -z "$$container_name" ]; then \ @@ -2903,20 +2846,21 @@ neo4j-backup: ## Backup Neo4j database to pmoves/backups/ with timestamp container_name=$$($(DC) ps -q neo4j 2>/dev/null | head -1); \ fi; \ if [ -n "$$container_name" ]; then \ - docker exec $$container_name neo4j-admin database dump \ + if docker exec $$container_name neo4j-admin database dump \ --to-path=/backups \ --overwrite-destination=true \ --username=neo4j \ - --password="$${NEO4J_PASSWORD:-changeme}" 2>/dev/null || \ - docker exec $$container_name cypher-shell -u neo4j -p "$${NEO4J_PASSWORD:-changeme}" \ - "CALL apoc.export.cypher.all(null, 'file:///backups/neo4j_$${timestamp}.cypher', {})"; \ - docker cp $$container_name:/backups/neo4j.dump "$$backup_dir/neo4j_$${timestamp}.dump" 2>/dev/null || \ - echo "⚠️ Backup file not found at /backups/neo4j.dump"; \ - if [ -f "$$backup_dir/neo4j_$${timestamp}.dump" ]; then \ - echo "✅ Backup created: $$backup_dir/neo4j_$${timestamp}.dump"; \ - ls -lh "$$backup_dir/neo4j_$${timestamp}.dump"; \ + --password="$${NEO4J_PASSWORD:-changeme}" 2>/dev/null; then \ + docker cp $$container_name:/backups/neo4j.dump "$$backup_dir/neo4j_$${timestamp}.dump" 2>/dev/null && \ + echo "✅ Backup created: $$backup_dir/neo4j_$${timestamp}.dump" && \ + ls -lh "$$backup_dir/neo4j_$${timestamp}.dump"; \ + elif docker exec $$container_name cypher-shell -u neo4j -p "$${NEO4J_PASSWORD:-changeme}" \ + "CALL apoc.export.cypher.all(null, 'file:///backups/neo4j_$${timestamp}.cypher', {})"; then \ + docker cp $$container_name:/backups/neo4j_$${timestamp}.cypher "$$backup_dir/neo4j_$${timestamp}.cypher" 2>/dev/null && \ + echo "✅ Backup created: $$backup_dir/neo4j_$${timestamp}.cypher" && \ + ls -lh "$$backup_dir/neo4j_$${timestamp}.cypher"; \ else \ - echo "❌ Backup failed - no file created"; \ + echo "❌ Backup failed - neither dump nor cypher export succeeded"; \ exit 1; \ fi; \ else \ @@ -2924,14 +2868,14 @@ neo4j-backup: ## Backup Neo4j database to pmoves/backups/ with timestamp exit 1; \ fi -neo4j-restore: ## Restore Neo4j from backup (usage: make neo4j-restore BACKUP=pmoves/backups/neo4j_YYYYMMDD_HHMMSS.dump) +neo4j-restore: ## Restore Neo4j from backup (usage: make neo4j-restore BACKUP=backups/neo4j_YYYYMMDD_HHMMSS.dump) @if [ -z "$(BACKUP)" ]; then \ echo "❌ Error: BACKUP parameter required"; \ echo ""; \ - echo "Usage: make neo4j-restore BACKUP=pmoves/backups/neo4j_20260313_120000.dump"; \ + echo "Usage: make neo4j-restore BACKUP=backups/neo4j_20260313_120000.dump"; \ echo ""; \ echo "Available backups:"; \ - ls -lh pmoves/backups/neo4j_*.dump 2>/dev/null || echo " No backups found in pmoves/backups/"; \ + ls -lh backups/neo4j_*.dump 2>/dev/null || echo " No backups found in backups/"; \ exit 1; \ fi @if [ ! -f "$(BACKUP)" ]; then \ @@ -2943,7 +2887,7 @@ neo4j-restore: ## Restore Neo4j from backup (usage: make neo4j-restore BACKUP=pm @echo "Press Ctrl+C to cancel, or wait 5 seconds to continue..." @sleep 5 @$(MAKE) --no-print-directory neo4j-local-down - @docker volume rm pmoves_neo4jdata 2>/dev/null || true + @docker volume rm pmoves_neo4j-data 2>/dev/null || true @echo "→ Starting fresh Neo4j..." @$(MAKE) --no-print-directory neo4j-local-up @sleep 15 diff --git a/pmoves/docs/NEO4J_INTEGRATION_GUIDE.md b/pmoves/docs/NEO4J_INTEGRATION_GUIDE.md index fbff08558..afe5b0d15 100644 --- a/pmoves/docs/NEO4J_INTEGRATION_GUIDE.md +++ b/pmoves/docs/NEO4J_INTEGRATION_GUIDE.md @@ -56,7 +56,7 @@ PMOVES-Neo4j/ at root level (follows PMOVES-supabase pattern) make -C pmoves neo4j-backup # Restore -make -C pmoves neo4j-restore BACKUP=pmoves/backups/neo4j_YYYYMMDD_HHMMSS.dump +make -C pmoves neo4j-restore BACKUP=backups/neo4j_YYYYMMDD_HHMMSS.dump ``` ## Monitoring @@ -68,7 +68,7 @@ curl http://localhost:7474/db/neo4j/health ``` ### Grafana Dashboard -- URL: http://localhost:3000 +- URL: http://localhost:3002 - Dashboard: Neo4j Overview - Panels: Database size, heap usage, transaction rate, cache performance diff --git a/pmoves/monitoring/prometheus/prometheus.yml b/pmoves/monitoring/prometheus/prometheus.yml index 2fec0a1c9..404ef2204 100644 --- a/pmoves/monitoring/prometheus/prometheus.yml +++ b/pmoves/monitoring/prometheus/prometheus.yml @@ -43,6 +43,12 @@ scrape_configs: static_configs: - targets: ["supaserch:8099"] + - job_name: neo4j + static_configs: + - targets: ["neo4j:2004"] + metrics_path: '/metrics' + scrape_interval: 15s + - job_name: gpu-orchestrator static_configs: - targets: ["gpu-orchestrator:8200"] diff --git a/pmoves/scripts/backup-neo4j.sh b/pmoves/scripts/backup-neo4j.sh index 2141765ae..db91aa6d6 100644 --- a/pmoves/scripts/backup-neo4j.sh +++ b/pmoves/scripts/backup-neo4j.sh @@ -2,7 +2,7 @@ # Neo4j Database Backup Script # ============================================================================= # Automated backup script for Neo4j graph database. -# Creates timestamped dumps to pmoves/backups/ with rotation. +# Creates timestamped dumps to backups/ with rotation. # # Usage: ./pmoves/scripts/backup-neo4j.sh [retention_days] # Default retention: 7 days @@ -13,12 +13,22 @@ set -euo pipefail +# Script directory (for loading env.shared) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +# Load credentials if env.shared exists +if [ -f "$PROJECT_ROOT/env.shared" ]; then + source "$PROJECT_ROOT/env.shared" +fi + # Configuration -BACKUP_DIR="pmoves/backups" +BACKUP_DIR="backups" CONTAINER_NAME="pmoves-neo4j-1" RETENTION_DAYS=${1:-7} TIMESTAMP=$(date +%Y%m%d_%H%M%S) BACKUP_FILE="neo4j_${TIMESTAMP}.dump" +CONTAINER_BACKUP_DIR="/data/backups" # Colors for output RED='\033[0;31m' @@ -55,21 +65,25 @@ if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then exit 1 fi -# Get Neo4j password from environment +# Get Neo4j password from environment (with fallback) NEO4J_PASSWORD=${NEO4J_PASSWORD:-changeme} +# Create backup directory inside container +log_info "Creating backup directory inside container..." +docker exec "$CONTAINER_NAME" mkdir -p "$CONTAINER_BACKUP_DIR" + # Create backup using neo4j-admin log_info "Creating Neo4j backup: ${BACKUP_FILE}" if docker exec "$CONTAINER_NAME" neo4j-admin database dump \ - --to-path=/backups \ + --to-path="$CONTAINER_BACKUP_DIR" \ --overwrite-destination=true \ --username=neo4j \ --password="$NEO4J_PASSWORD" 2>/dev/null; then - + # Copy backup from container - docker cp "$CONTAINER_NAME:/backups/neo4j.dump" "$BACKUP_DIR/$BACKUP_FILE" - + docker cp "$CONTAINER_NAME:$CONTAINER_BACKUP_DIR/neo4j.dump" "$BACKUP_DIR/$BACKUP_FILE" + # Verify backup file exists and is not empty if [ -f "$BACKUP_DIR/$BACKUP_FILE" ] && [ -s "$BACKUP_DIR/$BACKUP_FILE" ]; then BACKUP_SIZE=$(du -h "$BACKUP_DIR/$BACKUP_FILE" | cut -f1)