Repository navigation
feat(recovery): restore bounded PostgreSQL logical archives #209
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
d029efd
ad6f598
8f6970d
eec38ad
ad9df6b
fcf7518
1cc06ae
22b0b3d
453f424
99c9f24
c622481
92f0e60
e7957d1
afbe449
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,268 @@ | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| """Restore bounded PostgreSQL logical archives into a caller-selected target.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import os | ||
| import re | ||
| import stat | ||
| import subprocess | ||
| from dataclasses import dataclass | ||
|
|
||
|
|
||
| _SERVICE_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z") | ||
| _MAX_TIMEOUT_SECONDS = 86_400 | ||
| _MAX_CONNECT_TIMEOUT_SECONDS = 60 | ||
| _MAX_SIGNED_BIGINT = (1 << 63) - 1 | ||
| _DEFAULT_MAXIMUM_ARCHIVE_SIZE_BYTES = 64 * 1024 * 1024 * 1024 | ||
| _MISSING_ARCHIVE_METADATA = object() | ||
| _INHERITED_LIBPQ_VARIABLES = frozenset( | ||
| { | ||
| "PGPASSWORD", | ||
| "PGPASSFILE", | ||
| "PGSERVICEFILE", | ||
| } | ||
| ) | ||
|
|
||
|
|
||
| class PostgresLogicalRestoreError(RuntimeError): | ||
| """Report a fail-closed PostgreSQL logical-restore execution violation.""" | ||
|
|
||
|
|
||
| @dataclass(frozen=True, slots=True) | ||
| class PostgresLogicalRestoreResult: | ||
| """Describe the bounded archive consumed by one successful logical restore.""" | ||
|
|
||
| size_bytes: int | ||
|
|
||
|
|
||
| def _parameters_are_valid( | ||
| service_name: object, | ||
| input_descriptor: object, | ||
| source_superusers_trusted: object, | ||
| pg_restore_executable: object, | ||
| timeout_seconds: object, | ||
| connect_timeout_seconds: object, | ||
| maximum_archive_size_bytes: object, | ||
| ) -> bool: | ||
| """Return whether execution-authority parameters satisfy the narrow contract.""" | ||
| return ( | ||
| type(service_name) is str | ||
| and _SERVICE_NAME_RE.fullmatch(service_name) is not None | ||
| and type(input_descriptor) is int | ||
| and input_descriptor >= 0 | ||
| and type(source_superusers_trusted) is bool | ||
| and type(pg_restore_executable) is str | ||
| and os.path.isabs(pg_restore_executable) | ||
| and os.path.basename(pg_restore_executable) == "pg_restore" | ||
| and type(timeout_seconds) is int | ||
| and 1 <= timeout_seconds <= _MAX_TIMEOUT_SECONDS | ||
| and type(connect_timeout_seconds) is int | ||
| and 1 <= connect_timeout_seconds <= _MAX_CONNECT_TIMEOUT_SECONDS | ||
| and type(maximum_archive_size_bytes) is int | ||
| and 1 <= maximum_archive_size_bytes <= _MAX_SIGNED_BIGINT | ||
| ) | ||
|
|
||
|
|
||
| def _archive_is_owner_only(mode: int) -> bool: | ||
| """Return whether an archive mode grants no group or other permissions.""" | ||
| return mode & (stat.S_IRWXG | stat.S_IRWXO) == 0 | ||
|
|
||
|
|
||
| def _archive_metadata(status: object) -> tuple[object, ...]: | ||
| """Return observable file identity metadata used to detect archive mutation.""" | ||
| return ( | ||
| getattr(status, "st_mode", _MISSING_ARCHIVE_METADATA), | ||
| getattr(status, "st_size", _MISSING_ARCHIVE_METADATA), | ||
| getattr(status, "st_nlink", _MISSING_ARCHIVE_METADATA), | ||
| getattr(status, "st_dev", _MISSING_ARCHIVE_METADATA), | ||
| getattr(status, "st_ino", _MISSING_ARCHIVE_METADATA), | ||
| getattr(status, "st_mtime_ns", _MISSING_ARCHIVE_METADATA), | ||
| getattr(status, "st_ctime_ns", _MISSING_ARCHIVE_METADATA), | ||
| ) | ||
|
|
||
|
|
||
| def _inspect_initial_archive( | ||
| input_descriptor: int, | ||
| maximum_archive_size_bytes: int, | ||
| ) -> os.stat_result: | ||
| """Require a private, bounded, non-empty regular archive at offset zero.""" | ||
| try: | ||
| status = os.fstat(input_descriptor) | ||
| except (OSError, ValueError): | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore archive could not be inspected" | ||
| ) from None | ||
|
|
||
| if not stat.S_ISREG(status.st_mode): | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore archive must be a private regular file" | ||
| ) | ||
| if status.st_size <= 0 or status.st_size > maximum_archive_size_bytes: | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore archive must be non-empty and bounded" | ||
| ) | ||
| try: | ||
| offset = os.lseek(input_descriptor, 0, os.SEEK_CUR) | ||
| except (OSError, ValueError): | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore archive could not be inspected" | ||
| ) from None | ||
| if offset != 0: | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore archive must start at offset zero" | ||
| ) | ||
| if not _archive_is_owner_only(status.st_mode): | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore archive must be owner-only" | ||
| ) | ||
| if status.st_nlink != 1: | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore archive must have one link" | ||
| ) | ||
| return status | ||
|
|
||
|
|
||
| def _libpq_environment(connect_timeout_seconds: int) -> dict[str, str]: | ||
| """Return allowlisted libpq credentials plus the bounded connect timeout.""" | ||
| environment = { | ||
| key: os.environ[key] | ||
| for key in _INHERITED_LIBPQ_VARIABLES | ||
| if key in os.environ | ||
| } | ||
| environment["PGCONNECT_TIMEOUT"] = str(connect_timeout_seconds) | ||
| return environment | ||
|
|
||
|
|
||
| def _run_pg_restore( | ||
| *, | ||
| service_name: str, | ||
| input_descriptor: int, | ||
| pg_restore_executable: str, | ||
| timeout_seconds: int, | ||
| connect_timeout_seconds: int, | ||
| ) -> None: | ||
| """Run one shell-free, single-transaction pg_restore with bounded diagnostics.""" | ||
| arguments = [ | ||
| pg_restore_executable, | ||
| "--single-transaction", | ||
| "--exit-on-error", | ||
| f"--dbname=service={service_name}", | ||
| ] | ||
| environment = _libpq_environment(connect_timeout_seconds) | ||
| try: | ||
| completed = subprocess.run( | ||
| arguments, | ||
| stdin=input_descriptor, | ||
| stdout=subprocess.DEVNULL, | ||
| stderr=subprocess.DEVNULL, | ||
| timeout=timeout_seconds, | ||
| check=False, | ||
| close_fds=True, | ||
| env=environment, | ||
| ) | ||
| except FileNotFoundError: | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore executable unavailable" | ||
| ) from None | ||
| except subprocess.TimeoutExpired: | ||
| raise PostgresLogicalRestoreError("PostgreSQL logical restore timed out") from None | ||
| except Exception: | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore execution failed" | ||
| ) from None | ||
|
|
||
| if type(completed) is not subprocess.CompletedProcess: | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore execution failed" | ||
| ) | ||
| if completed.returncode != 0: | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore command failed" | ||
| ) | ||
|
|
||
|
|
||
| def _verify_archive_unchanged( | ||
| input_descriptor: int, | ||
| initial_status: os.stat_result, | ||
| ) -> int: | ||
| """Require the exact bounded archive to remain private and fully consumed.""" | ||
| try: | ||
| final_status = os.fstat(input_descriptor) | ||
| offset = os.lseek(input_descriptor, 0, os.SEEK_CUR) | ||
| except (OSError, ValueError): | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore archive could not be verified" | ||
| ) from None | ||
|
|
||
| if _archive_metadata(final_status) != _archive_metadata(initial_status): | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore archive changed during execution" | ||
| ) | ||
| if offset != initial_status.st_size: | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore archive was not consumed completely" | ||
| ) | ||
| return initial_status.st_size | ||
|
|
||
|
|
||
| def restore_postgres_logical_backup( | ||
| service_name: str, | ||
| input_descriptor: int, | ||
| *, | ||
| source_superusers_trusted: bool = False, | ||
| pg_restore_executable: str, | ||
| timeout_seconds: int = 1800, | ||
| connect_timeout_seconds: int = 15, | ||
| maximum_archive_size_bytes: int = _DEFAULT_MAXIMUM_ARCHIVE_SIZE_BYTES, | ||
| ) -> PostgresLogicalRestoreResult: | ||
| """Restore one bounded custom archive through a caller-owned file descriptor. | ||
|
|
||
| The caller must explicitly assert that the archive originates from trusted source | ||
| superusers. This assertion is a caller-owned precondition, not package proof that | ||
| archive definitions, ownership, or privileges are safe. The caller also selects | ||
| the target libpq service and is responsible for making that service an isolated | ||
| recovery target; the service name is not an authorization or proof-of-isolation | ||
| boundary. The package does not receive an archive path, place credentials in | ||
| process arguments, or reflect archive/database content in diagnostics. Only | ||
| ``PGPASSWORD``, ``PGPASSFILE``, and ``PGSERVICEFILE`` may be inherited, so ambient | ||
| host/database/options/SSL-mode variables cannot silently redirect or weaken the | ||
| target session. The validated non-secret service selector is supplied through | ||
| ``--dbname=service=...`` so ``pg_restore`` performs a direct database restore | ||
| rather than merely rendering SQL. The command runs with one transaction and exits | ||
| on the first SQL error, so timeout or execution failure does not intentionally | ||
| commit a partial package restore. Descriptor identity and observable archive | ||
| metadata are revalidated after the restore to reject in-place mutation detected | ||
| during execution. | ||
| """ | ||
|
Comment on lines
+209
to
+237
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 📐 Maintainability & Code Quality | 🟠 Major | ⚡ Quick win 직접 SQL 복원·롤백 계약 변경에 대한 문서를 갱신하십시오. 이 모듈은 README, 운영자 가이드, 아키텍처 문서, ADR, doctoring, CHANGELOG를 갱신하십시오. 특히 다음 항목을 기록하십시오.
원하시면 CHANGELOG 항목과 운영자 가이드 초안을 생성하겠습니다. As per coding guidelines: "Update the README, operator guide, architecture, ADR, doctoring, and CHANGELOG whenever tenant identity, role, migration, direct-SQL, or rollback contracts change." 🤖 Prompt for AI AgentsSource: Coding guidelines |
||
| if not _parameters_are_valid( | ||
| service_name, | ||
| input_descriptor, | ||
| source_superusers_trusted, | ||
| pg_restore_executable, | ||
| timeout_seconds, | ||
| connect_timeout_seconds, | ||
| maximum_archive_size_bytes, | ||
| ): | ||
| raise PostgresLogicalRestoreError( | ||
| "invalid PostgreSQL logical restore parameters" | ||
| ) | ||
| if not source_superusers_trusted: | ||
| raise PostgresLogicalRestoreError( | ||
| "PostgreSQL logical restore requires trusted source superusers" | ||
| ) | ||
|
|
||
| initial_status = _inspect_initial_archive( | ||
| input_descriptor, | ||
| maximum_archive_size_bytes, | ||
| ) | ||
| _run_pg_restore( | ||
| service_name=service_name, | ||
| input_descriptor=input_descriptor, | ||
| pg_restore_executable=pg_restore_executable, | ||
| timeout_seconds=timeout_seconds, | ||
| connect_timeout_seconds=connect_timeout_seconds, | ||
| ) | ||
| return PostgresLogicalRestoreResult( | ||
| size_bytes=_verify_archive_unchanged(input_descriptor, initial_status) | ||
| ) | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do not merge this EOF consumption check. Custom-format
pg_restoreseeks to the TOC and data blocks on a regular file, so a successful isolated restore can leave this shared descriptor mid-archive after--single-transactionhas already committed. The API would then report failure on a target that already changed.Keep the metadata fingerprint. Drop the
offset == st_sizerequirement. The repair is oncursor/bc-b38b4506-5199-4c13-903f-d073b7923acb-a453(2f7f0b0/3627dc3) with a seek-based regression and the operator contract indocs/doctoring/postgres-logical-restore.md.