diff --git a/docker/postgres/postgresql.conf.custom b/docker/postgres/postgresql.conf.custom index 12dbb557a..9d60aceb6 100644 --- a/docker/postgres/postgresql.conf.custom +++ b/docker/postgres/postgresql.conf.custom @@ -1,183 +1,175 @@ # ============================================================================= -# Custom PostgreSQL Configuration for Production Monitoring +# Optional PostgreSQL monitoring configuration # ============================================================================= -# This file extends the base PostgreSQL configuration with enhanced monitoring -# and performance tracking capabilities across ALL databases in the cluster. +# This file is shipped as an operator-applied example. The current container +# image copies it into /etc/postgresql but does NOT automatically include it in +# the running cluster configuration. Applying it is a deployment decision. # -# Apply these settings via: -# 1. Include in main postgresql.conf: include = 'postgresql.conf.custom' -# 2. Or copy contents to main postgresql.conf -# 3. Or apply via ALTER SYSTEM commands +# Privacy boundary: pg-llm-batch can process prompts, responses, credentials, +# and identifiers. The package therefore disables persistent SQL statement/bind +# logging and pg_stat_statements query-text retention in this example. Live +# pg_stat_activity can still expose bounded current/recent query text while +# track_activities is enabled; that volatile privileged surface is documented +# separately and must be access-controlled by the deployment. +# Enabling broad SQL logging is not by itself evidence of SOC 2, ISO 27001, +# PCI-DSS, CSAP, or any other certification/compliance outcome. +# +# PostgreSQL 16 explicitly warns that logged statements can reveal sensitive +# data and plaintext passwords. See: +# https://www.postgresql.org/docs/16/runtime-config-logging.html # ============================================================================= # ============================================================================= # SHARED LIBRARIES (requires restart) # ============================================================================= -shared_preload_libraries = 'pg_stat_statements,pg_cron' +# Query statistics are a true opt-in. PostgreSQL documents that loading +# pg_stat_statements reserves shared memory even when pg_stat_statements.track +# is none, so this general profile does not preload the module. A deployment +# that needs query-level statistics must deliberately add pg_stat_statements to +# shared_preload_libraries, restart PostgreSQL, create the extension in the +# intended database, and enable only the reviewed tracking surface. +shared_preload_libraries = 'pg_cron' -# ============================================================================= -# QUERY TEXT SIZE LIMITS -# ============================================================================= -# Increase from default 1024 to 32KB to capture full query text -track_activity_query_size = 32768 +# pg_stat_activity remains a volatile live-query surface while track_activities +# is enabled. Bound its retained text and restrict pg_read_all_stats/superuser +# access; see docs/doctoring/postgresql-logging-privacy.md. +track_activity_query_size = 1024 # ============================================================================= # PG_STAT_STATEMENTS CONFIGURATION # ============================================================================= -# Track query statistics across ALL databases in the cluster -pg_stat_statements.max = 10000 # Track up to 10000 unique statements -pg_stat_statements.track = 'all' # Track all statements (not just top-level) -pg_stat_statements.track_utility = on # Include DDL, VACUUM, ANALYZE, etc. -pg_stat_statements.track_planning = on # Track planning time (PG 13+) -pg_stat_statements.save = on # Persist stats across restarts - -# ============================================================================= -# LOGGING CONFIGURATION -# ============================================================================= -# Log all slow queries to PostgreSQL log files -log_min_duration_statement = 1000 # Log queries taking > 1 second -log_min_duration_sample = 500 # Sample queries 500ms-1s for performance analysis -log_statement_sample_rate = 0.01 # Additional 1% sampling for performance metrics (NOT for audit) -log_transaction_sample_rate = 0.01 # Transaction-level sampling for workload analysis - -# IMPORTANT: These sample rates DO NOT affect audit logging! -# With log_statement = 'all', 100% of queries are logged regardless of sample rates -# Sample rates only control ADDITIONAL performance statistics collection - -# Statement logging for comprehensive audit trail -log_statement = 'all' # Log EVERYTHING for complete audit trail - # 'none' = no logging (security risk!) - # 'ddl' = CREATE, ALTER, DROP only (insufficient!) - # 'mod' = DDL + INSERT, UPDATE, DELETE (minimum for compliance) - # 'all' = DDL + DML + SELECT (required for data security) - -# Enhanced log format with full context -log_line_prefix = '%t [%p]: user=%u,db=%d,app=%a,client=%h,query_id=%Q ' -log_checkpoints = on # Log checkpoint activity -log_connections = on # Log new connections -log_disconnections = on # Log disconnections -log_lock_waits = on # Log lock wait events -log_temp_files = 0 # Log all temp file usage -log_autovacuum_min_duration = 0 # Log all autovacuum runs - -# ============================================================================= -# PERFORMANCE TRACKING -# ============================================================================= -track_io_timing = on # Track I/O timing for queries -track_wal_io_timing = on # Track WAL I/O timing (PG 14+) -track_functions = 'all' # Track function call counts and time -track_commit_timestamp = on # Track transaction commit times - -# ============================================================================= -# STATISTICS COLLECTION -# ============================================================================= -# Query Monitoring Statistics (NOT Cost-Based Optimizer) -stats_fetch_consistency = 'cache' # Use cached stats for consistency -compute_query_id = 'on' # Generate query IDs for tracking - -# ============================================================================= -# COST-BASED OPTIMIZER CONFIGURATION (Separate from logging) -# ============================================================================= -# These settings affect query planning, NOT audit logging -default_statistics_target = 100 # Default histogram buckets (10-10000) -# random_page_cost = 1.1 # SSD: 1.1, HDD: 4.0 -# effective_cache_size = '4GB' # Total memory available for caching -# work_mem = '4MB' # Memory per sort/hash operation - -# Auto-ANALYZE for CBO statistics updates -# autovacuum_analyze_threshold = 50 # Min rows before analyze -# autovacuum_analyze_scale_factor = 0.1 # 10% of table size change triggers analyze - -# ============================================================================= -# AUDIT AND COMPLIANCE CONFIGURATION (Enterprise-grade) -# ============================================================================= -# Comprehensive audit logging for compliance (SOC2, ISO27001, PCI-DSS) -log_replication_commands = on # Log replication commands -log_rotation_age = 1h # Rotate logs hourly for high-volume systems -log_rotation_size = 1GB # Stay within PostgreSQL 16's per-file limit -log_truncate_on_rotation = off # Never truncate (audit trail preservation) -log_file_mode = 0600 # Secure file permissions -log_destination = 'csvlog' # CSV format for structured analysis - -# Archive and compression strategy (requires external tools) -# archive_mode = on # Enable archiving -# archive_command = 'gzip < %p > /archive/%f.gz && aws s3 cp /archive/%f.gz s3://audit-logs/%f.gz' - -# Row-level security auditing (when applicable) -row_security = on # Enable row security policies - -# Additional audit trail for sensitive operations -log_error_verbosity = 'verbose' # Include SQLSTATE in logs -log_hostname = on # Log hostname for multi-server tracking -log_timezone = 'UTC' # Use UTC for consistency - -# ============================================================================= -# MEMORY AND RESOURCE TRACKING -# ============================================================================= -log_executor_stats = off # Don't log per-query (too verbose) -log_parser_stats = off # Don't log parser stats -log_planner_stats = off # Don't log planner stats -log_statement_stats = off # Don't log statement stats - -# But DO track these at system level: -track_counts = on # Track table/index access counts -track_activities = on # Track current query activity - -# ============================================================================= -# PG_CRON CONFIGURATION (for scheduled monitoring tasks) -# ============================================================================= -cron.database_name = 'postgres' # Database where pg_cron metadata is stored -cron.use_background_workers = on # Use background workers for jobs -cron.max_running_jobs = 10 # Max concurrent cron jobs - -# ============================================================================= -# LOG RETENTION AND COMPLIANCE STRATEGY -# ============================================================================= -# Enterprise audit log retention requirements: -# - Financial: 7 years (SOX, SEC Rule 17a-4) -# - Healthcare: 6 years (HIPAA) -# - Payment Card: 3 years (PCI-DSS) -# - EU Data: 6 years (GDPR for financial data) -# - General Business: 3-7 years depending on jurisdiction - -# Recommended retention strategy: -# 1. HOT storage (local SSD): Last 7 days - immediate access -# 2. WARM storage (NAS/SAN): 7-90 days - quick retrieval -# 3. COLD storage (S3 Glacier): 90 days - 1 year - compliance retrieval -# 4. ARCHIVE (S3 Deep Archive): 1-7 years - legal hold - -# Estimated storage requirements (with log_statement = 'all'): -# - 1,000 queries/second = ~86M queries/day -# - Average log entry: ~500 bytes -# - Daily raw logs: ~43GB -# - Daily compressed (gzip -9): ~4.3GB -# - Yearly compressed: ~1.5TB -# - 7-year archive: ~10.5TB compressed - -# Log shipping and archival commands: -# log_directory = '/var/log/postgresql/audit' # Separate audit directory -# log_filename = 'audit_%Y%m%d_%H%M%S.csv' # Timestamp in filename - -# External archival script example (run via cron): -# #!/bin/bash -# find /var/log/postgresql/audit -name "*.csv" -mmin +60 | while read file; do -# gzip -9 "$file" -# aws s3 cp "$file.gz" s3://audit-logs/$(date +%Y/%m/%d)/ -# # Or use rsync for network storage: -# # rsync -av "$file.gz" backup-server:/audit-archive/$(date +%Y/%m/%d)/ -# done - -# ============================================================================= -# NOTES FOR CLUSTER-WIDE MONITORING -# ============================================================================= -# 1. pg_stat_statements is automatically available in ALL databases once loaded -# 2. Query the pg_stat_statements view from any database to see cluster-wide stats -# 3. Use dbid::regclass to identify which database each query belongs to -# 4. For true multi-database monitoring, consider: -# - Creating monitoring objects in 'postgres' database as central location -# - Using foreign data wrappers (postgres_fdw) to aggregate from all DBs -# - Setting up a dedicated monitoring database with cross-DB views -# -# CRITICAL: With log_statement = 'all', ensure adequate storage provisioning -# Monitor disk usage with: SELECT pg_size_pretty(pg_database_size('postgres')); -# Set up alerts for disk usage > 80% to prevent audit log loss +# Query-level statement collection is opt-in because pg_stat_statements stores +# representative query text. These placeholder settings remain fail-safe if an +# operator later preloads the module; collection stays disabled until explicitly +# changed under a reviewed purpose/access/retention policy. +pg_stat_statements.max = 10000 +pg_stat_statements.track = 'none' +pg_stat_statements.track_utility = off +pg_stat_statements.track_planning = off +pg_stat_statements.save = off + +# ============================================================================= +# CONTENT-SAFE SERVER LOGGING BASELINE +# ============================================================================= +# Do not persist SQL statement text or per-statement duration events merely to +# obtain operational evidence. Higher-level database/application metrics remain +# available below without forcing persistent query-content retention. +log_statement = 'none' +log_min_duration_statement = -1 +log_min_duration_sample = -1 +log_statement_sample_rate = 0 +log_transaction_sample_rate = 0 +log_duration = off + +# Error messages remain available, but failing SQL text, bind values, QUERY, and +# CONTEXT payloads are excluded from this baseline. PANIC is used because +# log_min_error_statement controls statement-text inclusion, not whether the +# error itself is logged through log_min_messages. +log_min_error_statement = PANIC +log_parameter_max_length = 0 +log_parameter_max_length_on_error = 0 +log_error_verbosity = 'terse' + +# The regular prefix intentionally omits remote-host escapes and SQL text. +# PostgreSQL CSV records nevertheless have a fixed client host:port field for +# emitted backend log entries. Treat that client network metadata, together with +# user/database/application identifiers, as access-controlled operational data. +# Connection/disconnection event logging is therefore opt-in rather than an +# unnecessary default source of additional persistent network-identity records. +log_line_prefix = '%m [%p]: user=%u,db=%d,app=%a,query_id=%Q ' +log_checkpoints = on +log_connections = off +log_disconnections = off +log_lock_waits = on + +# PostgreSQL documents that zero logs every temporary file name/size and every +# autovacuum action respectively. Avoid those unconditional high-volume event +# streams in the generic profile: temporary-file logging is off by default, and +# autovacuum logging stays at PostgreSQL's documented 10-minute threshold. +# Deployments may opt in to a lower threshold after defining a concrete purpose, +# storage/retention budget, and operator response path. +log_temp_files = -1 +log_autovacuum_min_duration = 10min +log_replication_commands = off +log_hostname = off +log_timezone = 'UTC' + +# ============================================================================= +# PERFORMANCE / STATISTICS TRACKING +# ============================================================================= +# PostgreSQL documents that I/O timing repeatedly reads the operating-system +# clock and can impose significant platform-dependent overhead. Keep both timing +# collectors off in the general example; deployments that need them should +# benchmark the target host with pg_test_timing and opt in deliberately. +track_io_timing = off +track_wal_io_timing = off + +# Function-call timing/count collection is optional statistics work. PostgreSQL +# defaults this setting to none; keep the generic profile at that boundary so a +# deployment opts in only for a concrete diagnostic need after measuring a +# representative workload and accepting the added collection overhead. +track_functions = 'none' + +# PostgreSQL records extra per-transaction commit metadata in pg_commit_ts when +# this server-start option is enabled, and the documented default is off. Keep +# it opt-in unless a deployment has a concrete commit-timestamp/replication need. +track_commit_timestamp = off +stats_fetch_consistency = 'cache' +# Keep the PostgreSQL default auto mode instead of forcing query-ID calculation +# for every statement. Adding pg_stat_statements later can request query IDs as +# documented, without imposing that work on deployments that leave it disabled. +compute_query_id = 'auto' +default_statistics_target = 100 +track_counts = on +track_activities = on + +# ============================================================================= +# LOG FILE SAFETY / ROTATION +# ============================================================================= +# PostgreSQL requires logging_collector for csvlog output. Enable it explicitly +# so an operator applying this example gets the declared structured destination +# instead of an internally inconsistent configuration. The collector changes +# log routing, not the content-retention policy above, and requires server start. +logging_collector = on + +# Rotation bounds file growth; it does not define retention. Retention, export, +# deletion, legal hold, backup expiry, residency, and external log-shipping are +# host-owned data-governance decisions and must not be inferred from this file. +log_rotation_age = 1h +log_rotation_size = 1GB +log_truncate_on_rotation = off +log_file_mode = 0600 +log_destination = 'csvlog' + +# ============================================================================= +# ROW SECURITY / PG_CRON +# ============================================================================= +row_security = on +cron.database_name = 'postgres' +cron.use_background_workers = on +cron.max_running_jobs = 10 + +# ============================================================================= +# OPERATOR NOTES +# ============================================================================= +# - This example is not automatically loaded by the bundled image. +# - Keep SQL/bind-value logging and pg_stat_statements collection/preloading +# disabled unless a separately reviewed deployment has a concrete purpose, +# authorization model, minimal retention, access audit, encryption boundary, +# storage/performance budget, and incident/deletion procedure. +# - Keep all-temp-file logging disabled and do not lower the 10-minute autovacuum +# threshold without a defined diagnostic purpose and bounded log-storage plan. +# - Keep track_io_timing/track_wal_io_timing off unless the deployment has +# measured clock-read cost with pg_test_timing and accepts the runtime overhead. +# - Keep track_functions off unless function-call statistics have a reviewed +# diagnostic purpose and measured representative-workload overhead. +# - Keep track_commit_timestamp off unless commit-time metadata has a concrete +# purpose; enabling it requires server start and writes extra pg_commit_ts data. +# - CSV output has a fixed client host:port field on emitted records. Keep +# log_connections/log_disconnections off unless connection-audit events have a +# reviewed purpose, access boundary, retention policy, and storage budget. +# - track_activities remains enabled for operational diagnosis; treat the +# bounded volatile pg_stat_activity query field as sensitive live data and +# restrict privileged statistics access accordingly. +# - Prefer low-cardinality application telemetry and database operational metrics +# when query content is not needed to answer the operational question. diff --git a/docs/doctoring/postgresql-logging-privacy.md b/docs/doctoring/postgresql-logging-privacy.md new file mode 100644 index 000000000..bc89a8cd7 --- /dev/null +++ b/docs/doctoring/postgresql-logging-privacy.md @@ -0,0 +1,167 @@ +# PostgreSQL logging privacy boundary + +## Scope + +`docker/postgres/postgresql.conf.custom` is an **optional operator-applied configuration surface**. The current container copies the file but does not automatically include it in the running PostgreSQL cluster. This doctoring therefore describes the safety contract for deployments that deliberately apply the file; it does not claim protected `main` currently enables these settings. + +pg-llm-batch can process prompts, provider configuration, identifiers, lifecycle evidence, and secret-management operations. SQL statement text and bind values can therefore contain personal, confidential, or credential-bearing content. The package must preserve the business data itself for authorized batch work; the risk treatment is to avoid unnecessary secondary copies in logs/statistics rather than destructively masking production data. + +## Root cause + +The former example enabled `log_statement = 'all'`, slow/sample statement logging, transaction statement sampling, verbose error output, and `pg_stat_statements` collection/persistence, and described blanket SQL logging plus jurisdiction-specific multi-year retention as if they were generally required for compliance. PostgreSQL 16 explicitly warns that logged statements can reveal sensitive data and plaintext passwords, and extended-query protocol statement logging can include bind parameter values. PostgreSQL also documents that `pg_stat_statements` stores representative query text, with normalization caveats. The old example therefore made disclosure and retention a side effect of generic monitoring guidance rather than an explicit data-governance decision. + +A separate operability inconsistency remained after the privacy repair: the example selected `csvlog` but did not enable `logging_collector`. PostgreSQL 16 explicitly requires the collector to generate CSV-format log output. An operator could therefore apply a configuration that advertised structured CSV logging and rotation without satisfying the server-start prerequisite that makes that destination effective. + +A further privacy review found that the example still enabled `log_connections` and `log_disconnections` while claiming its operational log context excluded client addresses. PostgreSQL documents that connection log messages expose the client IP address when hostname lookup is disabled, and CSV log records have a fixed **client host:port** field. `log_hostname = off` prevents hostname resolution; it is not a client-network-metadata suppression control. The prior wording therefore understated persistent **client network metadata** and caused avoidable connection/disconnection event copies. + +A reliability/performance review then found that the generic optional profile enabled both `track_io_timing` and `track_wal_io_timing` unconditionally. PostgreSQL documents that each setting repeatedly queries the operating system for the current time and can impose significant platform-dependent **timing overhead**, and specifically recommends `pg_test_timing` to measure that cost. A generic monitoring example should therefore not silently opt every deployment into timing instrumentation whose cost depends on the target host. + +The same statistics review found `track_functions = all`. PostgreSQL's cumulative **statistics collection** has execution cost, and `track_functions` specifically collects call counts and elapsed execution time for procedural-language and SQL functions when enabled. The PostgreSQL default is `none`. Enabling **function statistics** across every deployment therefore creates avoidable instrumentation **overhead** without proving that the resulting data is needed for a concrete operational question. + +A further transaction-metadata review found `track_commit_timestamp = on`. PostgreSQL documents this as a server-start option whose **default is off**, and its transaction-processing documentation states that enabling it records additional information in the `pg_commit_ts` directory for committed transactions. Commit-time metadata can be useful for specific replication/conflict or forensic questions, but collecting an additional persistent transaction record for every deployment without a defined consumer is not a neutral monitoring default. + +A log-volume review found two unconditional event streams that contradicted the same purpose-bound/storage-bounded design. PostgreSQL documents that `log_temp_files = 0` logs **every temporary file name and size** when each file is deleted, and that `log_autovacuum_min_duration = 0` **logs all autovacuum actions**. Those events can be frequent on sort/hash-heavy or maintenance-active workloads. Rotation limits individual file growth, not aggregate retention or event generation, so logging every event by default is not a bounded storage policy. + +A final query-statistics resource review found that setting `pg_stat_statements.track = none` did not make the feature a true opt-in. The optional profile still placed `pg_stat_statements` in `shared_preload_libraries` and forced `compute_query_id = on`. PostgreSQL 16 documents that the module consumes shared memory whenever it is loaded **even when tracking is `none`**, and that the module requires query identifiers when active. Reserving module memory and forcing query-ID calculation in a profile whose stated default is “query statistics disabled” is avoidable package-default work. The root-cause remedy is to remove the module from the default preload list and restore `compute_query_id = auto`; a deployment that needs query statistics must explicitly preload the module, restart, create the extension in the intended database, and enable a reviewed tracking mode. + +## Decision + +The reviewed baseline keeps ordinary SQL statement text and bind values out of server logs, makes query-text statistics collection and its preload cost opt-in, avoids connection-event logging unless a deployment has an explicit need, and keeps optional high-volume or material-cost monitoring bounded: + +- `log_statement = none`; +- `log_min_duration_statement = -1` and `log_min_duration_sample = -1`; +- statement and transaction sample rates are `0`; +- `log_duration = off` to avoid high-volume per-statement events by default; +- `log_parameter_max_length = 0` and `log_parameter_max_length_on_error = 0`; +- `log_min_error_statement = PANIC` so ordinary errors do not add failing statement text; +- `log_error_verbosity = terse` so PostgreSQL omits `DETAIL`, `HINT`, `QUERY`, and `CONTEXT` error fields; +- `log_connections = off` and `log_disconnections = off` so connection lifecycle events are **opt-in** rather than an unconditional source of client-network records; +- `log_temp_files = -1` so temporary-file name/size events are **opt-in** rather than emitted for every temporary file; +- `log_autovacuum_min_duration = 10min`, PostgreSQL 16's documented default, so the generic profile does not log every autovacuum action while retaining a conservative long-running-maintenance signal; +- `pg_stat_statements` is **not** in `shared_preload_libraries`; its tracking/planning/utility/save settings remain fail-safe placeholders at disabled values for a later deliberate preload; +- `compute_query_id = auto` instead of forcing query-ID calculation for deployments that do not enable a module requiring it; +- `track_io_timing = off` and `track_wal_io_timing = off` so platform-dependent timing overhead is not imposed until an operator has measured and accepted it; +- `track_functions = none` so function-call timing/count collection remains an **opt-in** diagnostic instead of package-default work; and +- `track_commit_timestamp = off` so additional per-transaction commit metadata is not written without a concrete purpose. + +Checkpoint and lock-wait events, conservative long-running autovacuum evidence, table/index counters, and activity-state telemetry remain available. Query IDs and query-level statement statistics remain available as explicit opt-ins rather than unconditional work. Temporary-file logging and lower autovacuum thresholds remain explicit opt-ins. This is not a claim that every remaining monitoring surface is content-free or cost-free: CSV client metadata, live activity tracking, and cumulative statistics have explicit residual boundaries. + +This is **selective disclosure**, not blanket masking. The source data remains available to the authorized application/database path. If an embedding organization has a genuine requirement for content-bearing database audit logs, connection audit events, query-level `pg_stat_statements`, temporary-file diagnostics, more aggressive autovacuum logging, high-resolution I/O timing, function-call statistics, or commit timestamps, it must enable only the necessary surface under a purpose-specific authorization, least-privilege access model, retention/deletion schedule where data is persisted, encryption/storage boundary, access audit, legal basis where applicable, performance/storage budget, and incident procedure. + +## Query-statistics preload and shared-memory boundary + +PostgreSQL 16 requires `pg_stat_statements` to be loaded through `shared_preload_libraries` because it allocates shared memory. The same primary documentation explicitly states that this memory is consumed whenever the module is loaded, **even if `pg_stat_statements.track = none`**. Keeping the module preloaded while calling query statistics “disabled” therefore avoided query-text collection but still imposed a server-start resource decision on every deployment that applied the optional profile. + +The generic profile now leaves `pg_stat_statements` out of `shared_preload_libraries` and uses PostgreSQL's `compute_query_id = auto` mode rather than forcing query-ID computation. The `pg_stat_statements.*` settings remain at fail-safe disabled values so a later operator preload does not silently enable collection merely because the configuration file is present. + +A deployment that deliberately needs query-level statistics must treat the capability as a coordinated opt-in: add `pg_stat_statements` to `shared_preload_libraries`, restart PostgreSQL, create the extension only in the intended database, select a reviewed `track` mode, define privileged access to representative query text, and accept the module's shared-memory/query-ID cost under a measured capacity budget. Disabling the feature again requires removing the preload entry and restarting; changing only `track` to `none` stops statement collection but does **not** reclaim the module's preload memory. + +## Temporary-file and autovacuum log-volume boundary + +PostgreSQL 16 defines `log_temp_files` as logging temporary file names and sizes when files are deleted. A value of `0` logs all temporary files; `-1` disables the event class. Temporary files can be produced by sorts, hashes, and temporary query results, so unconditional logging can create a workload-dependent event stream even when SQL statement logging itself is disabled. The generic profile therefore uses `log_temp_files = -1`. A deployment may opt in with a positive size threshold when it has a concrete spill-diagnosis question and a reviewed storage/retention budget; using `0` should be a deliberate short-lived diagnostic decision rather than the package baseline. + +PostgreSQL 16 likewise documents that `log_autovacuum_min_duration = 0` logs all autovacuum actions, while its default is `10min`. The generic profile restores `10min` rather than emitting every maintenance action. Operators may lower the threshold when they need finer autovacuum evidence, but the decision should define the diagnostic purpose, expected event volume, log destination capacity, retention/deletion behavior, and response procedure. Setting `-1` is also available to a deployment that must suppress the event class entirely. + +These settings bound event generation; they do not make PostgreSQL-managed files self-retaining or self-deleting. Issue #120 separately owns the container-native/storage-lifecycle integration question. Rollback for this slice is configuration-only: a deployment can restore its prior thresholds if its own approved monitoring requirement needs them, but the package default must not silently return to all-temp-file/all-autovacuum logging. + +## CSV log routing and retention boundary + +The optional example keeps `log_destination = 'csvlog'` for structured operational records and sets `logging_collector = on`, because PostgreSQL requires the collector to generate CSV-format output. `logging_collector` is a **server start** parameter: applying the file therefore requires a restart/start boundary before the declared CSV destination becomes effective. + +This change repairs **log routing** only. It does not widen the event/content classes permitted by the privacy settings above, and enabling the collector **does not define retention**. PostgreSQL's rotation knobs bound individual file age/size; they do not establish business retention, deletion, legal hold, backup expiry, residency, or external log-shipping policy. Those remain deployment-owned governance decisions. Operators should size and protect the collector's destination storage and keep file permissions/access aligned with the data classifications that remain in operational metadata. + +If a deployment intentionally routes server logs to a platform-owned stderr/journald/logging pipeline instead of PostgreSQL-managed CSV files, it should use a deployment overlay that changes both destination and related collector/rotation settings coherently rather than leaving an ineffective `csvlog` declaration. + +## Timing instrumentation overhead boundary + +PostgreSQL documents `track_io_timing` and `track_wal_io_timing` as disabled by default because they repeatedly query the operating system for timing information and can cause significant overhead on some platforms. The safe generic profile therefore keeps both settings `off`. + +A deployment that needs block/WAL timing must treat the feature as an **opt-in** performance decision. Measure the target host with PostgreSQL's `pg_test_timing`, evaluate the result under representative workload and concurrency, and enable only the timing classes whose diagnostic value justifies the measured runtime cost. The package does not assume that cloud, VM, bare-metal, or container clock-read costs are interchangeable, and a green functional test is not evidence that the timing overhead is acceptable for a production workload. + +Disabling these two timing collectors does not disable ordinary database activity counters, activity-state visibility, checkpoint/lock/autovacuum events, application-level telemetry, or the deployment's ability to opt into query identifiers later. It only avoids imposing optional high-frequency clock reads before a deployment has established a performance budget. + +## Function-statistics overhead boundary + +PostgreSQL documents that cumulative **statistics collection** adds some execution overhead and that `track_functions` defaults to `none`. Setting it to `all` collects **function statistics** for procedural-language functions and SQL-language functions that PostgreSQL considers trackable, including call counts and execution time. That can be useful for targeted diagnosis, but it is not necessary for every pg-llm-batch deployment. + +The generic profile therefore keeps `track_functions = none`. A deployment may **opt-in** to `pl` or `all` only when function-level attribution answers a concrete operational question. Before enabling it, measure representative workload and concurrency with the deployment's normal observability stack active, compare throughput/latency/CPU effects against the same workload with function tracking disabled, and record the accepted performance budget. Functional correctness alone is not evidence that instrumentation overhead is commercially acceptable. + +This change does not disable `track_counts`, `track_activities`, checkpoint/lock/autovacuum logging, application-level OpenTelemetry, or deliberate query-statistics enablement. If function statistics prove too expensive or are no longer needed, rollback is simply to restore `track_functions = none`; no business data migration is required. + +## Commit-timestamp metadata boundary + +PostgreSQL documents `track_commit_timestamp` as a boolean **server start** parameter whose **default is off**. When it is enabled, PostgreSQL records commit times and stores additional committed-transaction information in the `pg_commit_ts` directory. Those records support APIs such as `pg_xact_commit_timestamp()` and may be useful for specific replication-conflict or forensic workflows, but they are not required by pg-llm-batch's ordinary queue, lifecycle, readiness, or provider operations. + +The generic profile therefore keeps `track_commit_timestamp = off`. A deployment may **opt-in** only when it has a concrete consumer for commit-time evidence and has accepted the extra transaction-metadata storage/write path plus the server-start change boundary. The decision should state which operator or replication procedure consumes the data, who may access it, and how the deployment handles restart/rollback. Enabling the setting is not a substitute for application audit events, durable checkpoint evidence, or release provenance. + +Turning the option off again is a server-start configuration change; it does not erase or rewrite pg-llm-batch business tables. PostgreSQL also documents that commit timestamp information is eventually removed during vacuum, so this facility must not be treated as a package-owned durable audit-retention mechanism. + +## Client-network metadata residual boundary + +PostgreSQL's CSV schema contains a fixed **client host:port** column for emitted records from client backends. Consequently, even though the regular `log_line_prefix` does not include `%h` or `%r`, choosing `csvlog` can still persist client network metadata whenever another enabled event produces a backend log row. The package does not represent CSV output as network-identifier-free. + +`log_connections` logs connection attempts plus successful authentication/authorization, and `log_disconnections` logs session termination with similar connection information. The optional baseline therefore leaves both `log_connections` and `log_disconnections` off. A deployment that needs connection-audit events may opt in, but must classify client IP/port as operational data and define a purpose, access boundary, retention/deletion policy, storage budget, and incident handling before doing so. Turning `log_hostname` off avoids reverse hostname lookup and additional hostname disclosure; it does **not** remove the client IP/port already carried by connection messages or the CSV field. + +For environments where retaining the CSV client-address field on other operational events is unacceptable, use a deployment-owned logging destination/collector overlay whose emitted fields satisfy that deployment's data-minimization policy. That is an explicit operability/privacy tradeoff rather than a package-wide masking transformation. + +## Live `pg_stat_activity` residual boundary + +The baseline deliberately leaves `track_activities = on` because current-session state is useful for operational diagnosis. PostgreSQL documents that `pg_stat_activity.query` exposes the current or most recent **query text** for a backend and that the text is truncated according to `track_activity_query_size`; this example keeps that bound at 1024 bytes. This activity record is **volatile** server state rather than the persistent server-log or `pg_stat_statements` store removed above, but it can still contain prompt, identifier, configuration, or credential-management content while a session is observable. + +Access is therefore part of the trust boundary. PostgreSQL restricts visibility of security-sensitive activity fields for other users; superusers and roles with `pg_read_all_stats` can see information for all sessions. Deployments must grant those roles only to purpose-authorized operators and audit privileged access. If a deployment cannot accept live query-text visibility even under that access model, it may set `track_activities = off`, but that deliberately sacrifices current-command/activity diagnostics and must be evaluated as an operability tradeoff rather than presented as a free privacy switch. + +This residual is why the configuration and doctoring say **persistent SQL/bind-value logging and query-stat retention are disabled by default**, not that PostgreSQL has no in-memory query text anywhere. + +## `pg_stat_statements` residual boundary + +The optional file no longer preloads `pg_stat_statements`; its fail-safe GUC placeholders keep collection disabled if an operator later chooses to preload the module. PostgreSQL documents that an enabled extension retains representative query text; literal constants are commonly normalized but can still appear in some circumstances, and cross-user text is restricted to privileged roles. Those controls reduce exposure but do not remove the need for data classification, purpose, access, retention, and shared-memory/performance governance. `pg_stat_statements.track = none` is a collection switch, not a way to reclaim preload memory after the module has been loaded. + +## Compliance / certification boundary + +No PostgreSQL logging knob proves SOC 2, ISO/IEC 27001, PCI DSS, CSAP, privacy-law compliance, or any certification. NIST SP 800-53 is a risk-managed control catalog whose controls are selected and tailored to mission/business needs; it does not prescribe that applications persist all SQL text or fixed universal retention periods. Retention and audit scope are therefore deployment-governance decisions, not package constants. + +## Test-first evidence + +RED source `72f3e1c245c4a26d1778802bab0661973143fe04` added `tests/test_postgres_logging_privacy_contract.py`. CI run `31429883906` reproduced the exact initial defect: Python 3.12 reported two intended failures because `log_statement` was still `all` and the blanket audit/compliance prose was still present. + +A second test-first refinement on `9014c1337486c29208757178aade3d70f2d132a8` added explicit opt-in requirements for `pg_stat_statements` and disabled per-statement duration logging before the corresponding configuration change. Subsequent source `52f7879d875eb86a58ab9f93142d324b2ecd31be` implements that narrower content-retention boundary. + +RED source `0984c66e8d7a6ba713446860b575bf582bc74c41` then made the remaining live-activity assurance explicit. CI `31432404368` failed the intended contract because the doctoring did not yet name `pg_stat_activity` or its live query-text visibility. This document closes that documentation boundary without disabling the operationally useful activity collector. + +RED source `4111a9fba56920046a0a9eb83ccce4ca87d8f418` added the CSV routing regression. CI `31434551317` failed on Python 3.14 with `KeyError: 'logging_collector'`, proving the optional file selected `csvlog` without its required collector. Production source `9e40e38ed071288341d8854fe678a181d7c3dc51` enables the collector. Documentation RED `dc168c154a23371d94739b84241c787e677cc94d` then failed CI `31434729307` because this doctoring had not yet explained the routing, restart, and retention boundary. + +RED source `29bd3c5ff153ae75a503106104b522147b200ce2` added a connection-metadata minimization contract. CI `31435768146` failed on the intended first boundary because `log_connections` was still `on` (`1 failed, 355 passed, 3 deselected` on Python 3.14). Production source `ce09c5074d7d88f34ec58078a45743e890043f0e` disables both connection and disconnection event logging and corrects the configuration's client-address claim. This document records the remaining CSV `client host:port` field and the purpose-bound opt-in policy rather than falsely claiming the structured destination is network-metadata-free. + +RED source `923f1ec87e5296efe96e9e4f1f5438ae99fabe2a` added the timing-overhead contract. CI `31436821943` failed exactly because `track_io_timing` remained `on` (`1 failed, 356 passed, 3 deselected` on Python 3.10); the same test also requires `track_wal_io_timing` to remain opt-in and this doctoring to bind the decision to `pg_test_timing`. Production source `88133a4755fcd7599331958e8ea269cce6dd83a6` turns both timing collectors off in the generic profile. This document closes the documented feasibility/measurement boundary without removing the metrics from deployments that explicitly accept their measured cost. + +RED source `64fed077663877d939a86b9641bbb5960ddb3823` added the function-statistics contract. CI `31437980674` failed exactly because `track_functions` remained `all` (`1 failed, 357 passed, 3 deselected` on Python 3.10). Production source `ee555f69ab08a1eb000ed546945e29a7e34312ab` restores PostgreSQL's generic `none` boundary. This document adds the purpose/measurement/rollback contract so an operator can still opt in to function-call statistics after accepting representative-workload overhead rather than receiving it silently. + +RED source `755487e1830fb7defc64626b3f5d6b301c1aa2f1` adds the commit-timestamp contract before the configuration repair. At that source, the new regression is deterministically RED because the optional profile still sets `track_commit_timestamp = on`. Production source `54476a262b354a88065b0c30d9188e0002b2846c` restores PostgreSQL's documented default-off boundary. This doctoring closes the explicit purpose, `pg_commit_ts`, server-start, rollback, and non-audit-retention semantics without removing commit timestamps from deployments that deliberately require them. + +RED source `5b14c0bb7e0256c8c33b9f2fc176e390fcc873c7` added the high-volume event regression. CI `31448131801` failed exactly on `log_temp_files = 0` (`1 failed, 359 passed, 3 deselected` on Python 3.12), proving the optional profile still emitted every temporary-file event. Production source `798ce22de75da91b792b54996b6f75bfbc5da7df` disables temporary-file logging by default and restores PostgreSQL's 10-minute autovacuum threshold. This document binds both settings to purpose, event-volume, storage, retention, and rollback decisions. + +RED source `e831063b422c51c80bd58cdc737b664e48647e03` added the query-statistics preload regression before the configuration repair. Its superseded PR workflows were cancelled after the production commit, so they are **not** passing or failing CI evidence. The test is nevertheless deterministic against that source: `shared_preload_libraries` still contained `pg_stat_statements` and `compute_query_id` was still `on`. Production source `bee5ba0ccaea0859fb54cfdc9067b672f585b712` removes the module from the default preload list and restores `compute_query_id = auto`. Final GREEN evidence must come from the later unchanged head; cancelled predecessor runs do not transfer. + +No predecessor or synthetic-merge result transfers to later heads; final acceptance requires fresh validation of the unchanged final source under current repository governance. + +## Rollback and recovery + +If the safer baseline prevents an operator from satisfying a documented, purpose-specific audit requirement, do **not** restore blanket logging in the package default. Instead, maintain a deployment-owned overlay that enables only the necessary event/content classes for the authorized scope, defines access/retention/deletion, and can be disabled independently. If accidental content or unnecessary client-network logging is discovered, stop the relevant logging path, preserve only evidence required by the incident/legal process, rotate or revoke exposed credentials where relevant, and follow the deployment's deletion/backup-expiry procedure for unnecessary copies. + +If query statistics are no longer needed, set tracking to `none`, remove `pg_stat_statements` from `shared_preload_libraries`, restore `compute_query_id = auto` unless another reviewed consumer requires query IDs, and restart PostgreSQL to release the module's shared-memory allocation. If PostgreSQL-managed CSV collection is operationally unsuitable, use a deployment-owned overlay to select the intended logging destination and disable/adjust `logging_collector` coherently. If temporary-file or autovacuum logging produces excessive volume, restore `log_temp_files = -1` and `log_autovacuum_min_duration = 10min` (or a deployment-approved higher/off threshold) before re-establishing a measurement baseline. If timing telemetry imposes unacceptable overhead, disable `track_io_timing` and `track_wal_io_timing` and re-establish a measurement baseline before any narrower re-enable. If function statistics impose unacceptable overhead or no longer have a reviewed diagnostic purpose, restore `track_functions = none`. If commit timestamps are no longer needed, restore `track_commit_timestamp = off` and restart under the deployment's change procedure. Rollback must not silently restore broad SQL/bind logging, unconditional connection event logging, all-event temp/autovacuum logging, fixed retention claims, or unmeasured/unneeded instrumentation. + +## APA 7 references + +Joint Task Force. (2025). *Security and privacy controls for information systems and organizations* (NIST Special Publication 800-53, Revision 5, Release 5.2.0). National Institute of Standards and Technology. https://csrc.nist.gov/projects/cprt/catalog + +PostgreSQL Global Development Group. (2026). *PostgreSQL 16 documentation: Error reporting and logging*. https://www.postgresql.org/docs/16/runtime-config-logging.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 16 documentation: Run-time statistics*. https://www.postgresql.org/docs/16/runtime-config-statistics.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 16 documentation: The cumulative statistics system*. https://www.postgresql.org/docs/16/monitoring-stats.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 16 documentation: Replication*. https://www.postgresql.org/docs/16/runtime-config-replication.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 16 documentation: Transactions and identifiers*. https://www.postgresql.org/docs/16/transaction-id.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 16 documentation: pg_stat_statements—Track statistics of SQL planning and execution*. https://www.postgresql.org/docs/16/pgstatstatements.html diff --git a/tests/test_postgres_logging_privacy_contract.py b/tests/test_postgres_logging_privacy_contract.py new file mode 100644 index 000000000..ec18c9dd2 --- /dev/null +++ b/tests/test_postgres_logging_privacy_contract.py @@ -0,0 +1,207 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Regression contracts for the optional PostgreSQL logging configuration.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = ROOT / "docker/postgres/postgresql.conf.custom" +DOCTORING = ROOT / "docs/doctoring/postgresql-logging-privacy.md" + + +def _settings() -> dict[str, str]: + """Return active ``key = value`` settings with trailing comments removed.""" + settings: dict[str, str] = {} + for raw_line in CONFIG.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + settings[key.strip()] = value.split("#", 1)[0].strip().strip("'\"") + return settings + + +def test_optional_postgres_logging_does_not_capture_sql_or_bind_values_by_default() -> None: + """The reviewed baseline must not persist prompt/secret-bearing SQL text.""" + settings = _settings() + + assert settings["log_statement"] == "none" + assert settings["log_min_duration_statement"] == "-1" + assert settings["log_min_duration_sample"] == "-1" + assert settings["log_statement_sample_rate"] == "0" + assert settings["log_transaction_sample_rate"] == "0" + assert settings["log_duration"].lower() == "off" + assert settings["log_min_error_statement"].lower() == "panic" + assert settings["log_parameter_max_length"] == "0" + assert settings["log_parameter_max_length_on_error"] == "0" + assert settings["log_error_verbosity"].lower() == "terse" + + +def test_optional_timing_metrics_are_opt_in_for_predictable_overhead() -> None: + """Timing metrics with platform-dependent cost must remain explicit opt-ins.""" + settings = _settings() + doctoring = " ".join(DOCTORING.read_text(encoding="utf-8").lower().split()) + + assert settings["track_io_timing"].lower() == "off" + assert settings["track_wal_io_timing"].lower() == "off" + for phrase in ( + "track_io_timing", + "track_wal_io_timing", + "pg_test_timing", + "timing overhead", + "opt-in", + ): + assert phrase in doctoring, phrase + + +def test_function_statistics_are_opt_in_for_bounded_monitoring_overhead() -> None: + """Function-call timing must not be enabled for every deployment by default.""" + settings = _settings() + doctoring = " ".join(DOCTORING.read_text(encoding="utf-8").lower().split()) + + assert settings["track_functions"].lower() == "none" + for phrase in ( + "track_functions", + "function statistics", + "statistics collection", + "overhead", + "opt-in", + ): + assert phrase in doctoring, phrase + + +def test_commit_timestamp_tracking_is_opt_in_for_bounded_transaction_metadata() -> None: + """Commit timestamps must not create extra transaction metadata without purpose.""" + settings = _settings() + doctoring = " ".join(DOCTORING.read_text(encoding="utf-8").lower().split()) + + assert settings["track_commit_timestamp"].lower() == "off" + for phrase in ( + "track_commit_timestamp", + "pg_commit_ts", + "server start", + "default is off", + "opt-in", + ): + assert phrase in doctoring, phrase + + +def test_high_volume_temp_and_autovacuum_logging_is_not_unconditionally_enabled() -> None: + """Generic monitoring must not emit every temp-file and autovacuum event.""" + settings = _settings() + doctoring = " ".join(DOCTORING.read_text(encoding="utf-8").lower().split()) + + assert settings["log_temp_files"] == "-1" + assert settings["log_autovacuum_min_duration"] == "10min" + for phrase in ( + "log_temp_files", + "temporary file names and sizes", + "log_autovacuum_min_duration", + "logs all autovacuum actions", + "10min", + "opt-in", + ): + assert phrase in doctoring, phrase + + +def test_csv_logging_enables_the_required_logging_collector() -> None: + """A configured CSV destination must enable PostgreSQL's logging collector.""" + settings = _settings() + destinations = {item.strip() for item in settings["log_destination"].split(",")} + + assert "csvlog" in destinations + assert settings["logging_collector"].lower() == "on" + + +def test_csv_logging_collector_contract_is_documented() -> None: + """Doctoring must explain routing, restart, and retention semantics.""" + doctoring = " ".join(DOCTORING.read_text(encoding="utf-8").lower().split()) + + for phrase in ( + "logging_collector", + "csvlog", + "server start", + "log routing", + "does not define retention", + ): + assert phrase in doctoring, phrase + + +def test_connection_event_logging_is_opt_in_and_network_metadata_is_documented() -> None: + """Connection events must not create avoidable client-network log copies.""" + settings = _settings() + doctoring = " ".join(DOCTORING.read_text(encoding="utf-8").lower().split()) + + assert settings["log_connections"].lower() == "off" + assert settings["log_disconnections"].lower() == "off" + for phrase in ( + "client host:port", + "log_connections", + "log_disconnections", + "client network metadata", + "opt-in", + ): + assert phrase in doctoring, phrase + + +def test_optional_query_statistics_do_not_retain_query_text_without_opt_in() -> None: + """Representative query-text collection must be disabled in the package baseline.""" + settings = _settings() + + assert settings["pg_stat_statements.track"].lower() == "none" + assert settings["pg_stat_statements.track_utility"].lower() == "off" + assert settings["pg_stat_statements.track_planning"].lower() == "off" + assert settings["pg_stat_statements.save"].lower() == "off" + + +def test_query_statistics_preload_is_opt_in_for_bounded_shared_memory() -> None: + """Disabled query statistics must not still reserve preload/query-id resources.""" + settings = _settings() + doctoring = " ".join(DOCTORING.read_text(encoding="utf-8").lower().split()) + preloaded = { + item.strip() for item in settings["shared_preload_libraries"].split(",") + } + + assert "pg_stat_statements" not in preloaded + assert settings["compute_query_id"].lower() == "auto" + for phrase in ( + "shared_preload_libraries", + "shared memory", + "even if `pg_stat_statements.track = none`", + "compute_query_id = auto", + "restart postgresql", + "opt-in", + ): + assert phrase in doctoring, phrase + + +def test_activity_tracking_query_text_residual_is_explicit() -> None: + """Live pg_stat_activity query text must remain an explicit residual boundary.""" + settings = _settings() + doctoring = " ".join(DOCTORING.read_text(encoding="utf-8").lower().split()) + + assert settings["track_activities"].lower() == "on" + assert settings["track_activity_query_size"] == "1024" + for phrase in ( + "pg_stat_activity", + "query text", + "track_activities", + "volatile", + "pg_read_all_stats", + ): + assert phrase in doctoring, phrase + + +def test_optional_postgres_logging_does_not_claim_blanket_sql_logging_is_compliance() -> None: + """Operator guidance must not equate plaintext SQL retention with compliance.""" + normalized = " ".join(CONFIG.read_text(encoding="utf-8").lower().split()) + + for prohibited in ( + "log everything for complete audit trail", + "required for data security", + "minimum for compliance", + "financial: 7 years", + "healthcare: 6 years", + "eu data: 6 years", + ): + assert prohibited not in normalized, prohibited