Skip to content

PG-1441 Do not replicate relation keys #234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jeltz merged 4 commits into from
Apr 23, 2025
Merged

PG-1441 Do not replicate relation keys #234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
885b567
PG-1441 Refactor code for generating keys in SMGR code
jeltz Apr 17, 2025
88f586f
PG-1441 Write simple TAP test for replication of pg_tde
jeltz Apr 17, 2025
dbe2d8f
PG-1441 Do not generate relation keys in the SMGR on redo
jeltz Apr 16, 2025
1e92263
PG-1441 Do not replicate relation keys
jeltz Apr 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion contrib/pg_tde/meson.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ tap_tests = [
't/009_wal_encrypt.pl',
't/010_change_key_provider.pl',
't/011_unlogged_tables.pl',
't/012_replication.pl',
]

tests += {
Expand All @@ -125,7 +126,8 @@ tests += {
'runningcheck': false,
},
'tap': {
'tests': tap_tests },
'tests': tap_tests
},
}

pg_tde_inc = incdir
Expand Down
138 changes: 57 additions & 81 deletions contrib/pg_tde/src/access/pg_tde_tdemap.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,13 +121,13 @@ static WALKeyCacheRec *pg_tde_add_wal_key_to_cache(InternalKey *cached_key, XLog
static InternalKey *pg_tde_put_key_into_cache(const RelFileLocator *locator, InternalKey *key);

#ifndef FRONTEND
static InternalKey *pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type);
static InternalKey *pg_tde_create_local_key(const RelFileLocator *newrlocator, uint32 entry_type);
static InternalKey *pg_tde_create_smgr_key_temp(const RelFileLocator *newrlocator);
static InternalKey *pg_tde_create_smgr_key_perm(const RelFileLocator *newrlocator);
static void pg_tde_generate_internal_key(InternalKey *int_key, uint32 entry_type);
static int pg_tde_file_header_write(const char *tde_filename, int fd, const TDESignedPrincipalKeyInfo *signed_key_info, off_t *bytes_written);
static void pg_tde_sign_principal_key_info(TDESignedPrincipalKeyInfo *signed_key_info, const TDEPrincipalKey *principal_key);
static void pg_tde_write_one_map_entry(int fd, const TDEMapEntry *map_entry, off_t *offset, const char *db_map_path);
static void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *rel_key_data, TDEPrincipalKey *principal_key, bool write_xlog);
static void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *rel_key_data, TDEPrincipalKey *principal_key);
static int keyrotation_init_file(const TDESignedPrincipalKeyInfo *signed_key_info, char *rotated_filename, const char *filename, off_t *curr_pos);
static void finalize_key_rotation(const char *path_old, const char *path_new);
static int pg_tde_open_file_write(const char *tde_filename, const TDESignedPrincipalKeyInfo *signed_key_info, bool truncate, off_t *curr_pos);
Expand All @@ -137,22 +137,32 @@ InternalKey *
pg_tde_create_smgr_key(const RelFileLocatorBackend *newrlocator)
{
if (RelFileLocatorBackendIsTemp(*newrlocator))
return pg_tde_create_local_key(&newrlocator->locator, TDE_KEY_TYPE_SMGR);
return pg_tde_create_smgr_key_temp(&newrlocator->locator);
else
return pg_tde_create_key_map_entry(&newrlocator->locator, TDE_KEY_TYPE_SMGR);
return pg_tde_create_smgr_key_perm(&newrlocator->locator);
}

static InternalKey *
pg_tde_create_smgr_key_temp(const RelFileLocator *newrlocator)
{
InternalKey int_key;

pg_tde_generate_internal_key(&int_key, TDE_KEY_TYPE_SMGR);

return pg_tde_put_key_into_cache(newrlocator, &int_key);
}

/*
* Generate an encrypted key for the relation and store it in the keymap file.
*/
static InternalKey *
pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type)
pg_tde_create_smgr_key_perm(const RelFileLocator *newrlocator)
{
InternalKey rel_key_data;
TDEPrincipalKey *principal_key;
LWLock *lock_pk = tde_lwlock_enc_keys();
XLogRelKey xlrec = {
.rlocator = *newrlocator,
};

pg_tde_generate_internal_key(&rel_key_data, entry_type);
pg_tde_generate_internal_key(&rel_key_data, TDE_KEY_TYPE_SMGR);

LWLockAcquire(lock_pk, LW_EXCLUSIVE);
principal_key = GetPrincipalKey(newrlocator->dbOid, LW_EXCLUSIVE);
Expand All @@ -164,20 +174,49 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type
}

/* Add the encrypted key to the key map data file structure. */
pg_tde_write_key_map_entry(newrlocator, &rel_key_data, principal_key, true);
pg_tde_write_key_map_entry(newrlocator, &rel_key_data, principal_key);
LWLockRelease(lock_pk);

/*
* It is fine to write the to WAL after writing to the file since we have
* not WAL logged the SMGR CREATE event either.
*/
XLogBeginInsert();
XLogRegisterData((char *) &xlrec, sizeof(xlrec));
XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_RELATION_KEY);

return pg_tde_put_key_into_cache(newrlocator, &rel_key_data);
}

static InternalKey *
pg_tde_create_local_key(const RelFileLocator *newrlocator, uint32 entry_type)
void
pg_tde_create_smgr_key_perm_redo(const RelFileLocator *newrlocator)
{
InternalKey int_key;
InternalKey rel_key_data;
InternalKey *old_key;
TDEPrincipalKey *principal_key;
LWLock *lock_pk = tde_lwlock_enc_keys();

pg_tde_generate_internal_key(&int_key, entry_type);
if ((old_key = pg_tde_get_key_from_file(newrlocator, TDE_KEY_TYPE_SMGR)))
{
pfree(old_key);
LWLockRelease(lock_pk);
return;
}

return pg_tde_put_key_into_cache(newrlocator, &int_key);
pg_tde_generate_internal_key(&rel_key_data, TDE_KEY_TYPE_SMGR);

LWLockAcquire(lock_pk, LW_EXCLUSIVE);
principal_key = GetPrincipalKey(newrlocator->dbOid, LW_EXCLUSIVE);
if (principal_key == NULL)
{
ereport(ERROR,
errmsg("principal key not configured"),
errhint("create one using pg_tde_set_key before using encrypted tables"));
}

/* Add the encrypted key to the key map data file structure. */
pg_tde_write_key_map_entry(newrlocator, &rel_key_data, principal_key);
LWLockRelease(lock_pk);
}

static void
Expand Down Expand Up @@ -240,7 +279,7 @@ pg_tde_create_wal_key(InternalKey *rel_key_data, const RelFileLocator *newrlocat
/*
* Add the encrypted key to the key map data file structure.
*/
pg_tde_write_key_map_entry(newrlocator, rel_key_data, principal_key, false);
pg_tde_write_key_map_entry(newrlocator, rel_key_data, principal_key);

LWLockRelease(tde_lwlock_enc_keys());
}
Expand Down Expand Up @@ -415,7 +454,7 @@ pg_tde_write_one_map_entry(int fd, const TDEMapEntry *map_entry, off_t *offset,
* concurrent in place updates leading to data conflicts.
*/
void
pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *rel_key_data, TDEPrincipalKey *principal_key, bool write_xlog)
pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *rel_key_data, TDEPrincipalKey *principal_key)
{
char db_map_path[MAXPGPATH];
int map_fd;
Expand Down Expand Up @@ -458,75 +497,12 @@ pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *rel_key_
/* Initialize map entry and encrypt key */
pg_tde_initialize_map_entry(&write_map_entry, principal_key, rlocator, rel_key_data);

if (write_xlog)
{
XLogRelKey xlrec;

xlrec.mapEntry = write_map_entry;
xlrec.pkInfo = signed_key_Info;

XLogBeginInsert();
XLogRegisterData((char *) &xlrec, sizeof(xlrec));
XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_RELATION_KEY);
}

/* Write the given entry at curr_pos; i.e. the free entry. */
pg_tde_write_one_map_entry(map_fd, &write_map_entry, &curr_pos, db_map_path);

close(map_fd);
}

/*
* Write and already encrypted entry to the key map.
*
* The caller must hold an exclusive lock on the map file to avoid
* concurrent in place updates leading to data conflicts.
*/
void
pg_tde_write_key_map_entry_redo(const TDEMapEntry *write_map_entry, TDESignedPrincipalKeyInfo *signed_key_info)
{
char db_map_path[MAXPGPATH];
int map_fd;
off_t curr_pos = 0;

pg_tde_set_db_file_path(signed_key_info->data.databaseId, db_map_path);

LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE);

/* Open and validate file for basic correctness. */
map_fd = pg_tde_open_file_write(db_map_path, signed_key_info, false, &curr_pos);

/*
* Read until we find an empty slot. Otherwise, read until end. This seems
* to be less frequent than vacuum. So let's keep this function here
* rather than overloading the vacuum process.
*/
while (1)
{
TDEMapEntry read_map_entry;
off_t prev_pos = curr_pos;

if (!pg_tde_read_one_map_entry(map_fd, &read_map_entry, &curr_pos))
{
curr_pos = prev_pos;
break;
}

if (read_map_entry.flags == MAP_ENTRY_EMPTY)
{
curr_pos = prev_pos;
break;
}
}

/* Write the given entry at curr_pos; i.e. the free entry. */
pg_tde_write_one_map_entry(map_fd, write_map_entry, &curr_pos, db_map_path);

close(map_fd);

LWLockRelease(tde_lwlock_enc_keys());
}

/*
* Mark relation map entry as free and overwrite the key
*
Expand Down
4 changes: 2 additions & 2 deletions contrib/pg_tde/src/access/pg_tde_xlog.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ tdeheap_rmgr_redo(XLogReaderState *record)
{
XLogRelKey *xlrec = (XLogRelKey *) XLogRecGetData(record);

pg_tde_write_key_map_entry_redo(&xlrec->mapEntry, &xlrec->pkInfo);
pg_tde_create_smgr_key_perm_redo(&xlrec->rlocator);
}
else if (info == XLOG_TDE_ADD_PRINCIPAL_KEY)
{
Expand Down Expand Up @@ -93,7 +93,7 @@ tdeheap_rmgr_desc(StringInfo buf, XLogReaderState *record)
{
XLogRelKey *xlrec = (XLogRelKey *) XLogRecGetData(record);

appendStringInfo(buf, "rel: %u/%u/%u", xlrec->mapEntry.spcOid, xlrec->pkInfo.data.databaseId, xlrec->mapEntry.relNumber);
appendStringInfo(buf, "rel: %u/%u/%u", xlrec->rlocator.spcOid, xlrec->rlocator.dbOid, xlrec->rlocator.relNumber);
}
else if (info == XLOG_TDE_ADD_PRINCIPAL_KEY)
{
Expand Down
5 changes: 2 additions & 3 deletions contrib/pg_tde/src/include/access/pg_tde_tdemap.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,7 @@ typedef struct TDEMapEntry

typedef struct XLogRelKey
{
TDEMapEntry mapEntry;
TDESignedPrincipalKeyInfo pkInfo;
RelFileLocator rlocator;
} XLogRelKey;

/*
Expand Down Expand Up @@ -99,9 +98,9 @@ extern WALKeyCacheRec *pg_tde_get_wal_cache_keys(void);
extern void pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path);

extern InternalKey *pg_tde_create_smgr_key(const RelFileLocatorBackend *newrlocator);
extern void pg_tde_create_smgr_key_perm_redo(const RelFileLocator *newrlocator);
extern void pg_tde_create_wal_key(InternalKey *rel_key_data, const RelFileLocator *newrlocator, uint32 flags);
extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator);
extern void pg_tde_write_key_map_entry_redo(const TDEMapEntry *write_map_entry, TDESignedPrincipalKeyInfo *signed_key_info);

#define PG_TDE_MAP_FILENAME "pg_tde_%d_map"

Expand Down
Loading