Skip to content

Commit

Permalink
Merge pull request #14897 from craftcms/feature/backup-restore-closure
Browse files Browse the repository at this point in the history 
Accept closure for backup/restore commands
  • Loading branch information
@brandonkelly
brandonkelly authored Apr 29, 2024
2 parents 0fffc42 + e67a80d commit 7f8c1fb
Show file tree
Hide file tree
Showing 5 changed files with 186 additions and 109 deletions.
60 changes: 52 additions & 8 deletions src/config/GeneralConfig.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

namespace craft\config;

use Closure;
use Craft;
use craft\helpers\ConfigHelper;
use craft\helpers\DateTimeHelper;
Expand Down Expand Up @@ -419,7 +420,7 @@ class GeneralConfig extends BaseConfig
public bool $backupOnUpdate = true;

/**
* @var string|null|false The shell command that Craft should execute to create a database backup.
* @var string|null|false|Closure The shell command that Craft should execute to create a database backup.
*
* When set to `null` (default), Craft will run `mysqldump` or `pg_dump`, provided that those libraries are in the `$PATH` variable
* for the system user running the web server.
Expand Down Expand Up @@ -447,7 +448,29 @@ class GeneralConfig extends BaseConfig
*
* @group Environment
*/
public string|null|false $backupCommand = null;
public string|null|false|Closure $backupCommand = null;

/**
* @var string|null The output format to pass to `pg_dump` when backing up the database.
*
* This setting has no effect with MySQL databases.
*
* Valid options are `custom`, `directory`, `tar`, or `plain`.
* When set to `null` (default), `pg_restore` will default to `plain`
* @see https://www.postgresql.org/docs/current/app-pgdump.html
*
* ::: code
* ```php Static Config
* ->backupCommandFormat('custom')
* ```
* ```shell Environment Override
* CRAFT_BACKUP_COMMAND_FORMAT=custom
* ```
* :::
*
* @group Environment
*/
public ?string $backupCommandFormat = null;

/**
* @var string|null The base URL Craft should use when generating control panel URLs.
Expand Down Expand Up @@ -2430,7 +2453,7 @@ class GeneralConfig extends BaseConfig
public string $resourceBaseUrl = '@web/cpresources';

/**
* @var string|null|false The shell command Craft should execute to restore a database backup.
* @var string|null|false|Closure The shell command Craft should execute to restore a database backup.
*
* By default Craft will run `mysql` or `psql`, provided those libraries are in the `$PATH` variable for the user the web server is running as.
*
Expand All @@ -2456,7 +2479,7 @@ class GeneralConfig extends BaseConfig
*
* @group Environment
*/
public string|null|false $restoreCommand = null;
public string|null|false|Closure $restoreCommand = null;

/**
* @var bool Whether asset URLs should be revved so browsers don’t load cached versions when they’re modified.
Expand Down Expand Up @@ -3568,17 +3591,38 @@ public function backupOnUpdate(bool $value = true): self
* ```
*
* @group Environment
* @param string|null|false $value
* @param string|null|false|Closure $value
* @return self
* @see $backupCommand
* @since 4.2.0
*/
public function backupCommand(string|null|false $value): self
public function backupCommand(string|null|false|Closure $value): self
{
$this->backupCommand = $value;
return $this;
}

/**
* The output format to pass to `pg_dump` when backing up the database.
*
* This setting has no effect with MySQL databases.
*
* Valid options are `custom`, `directory`, `tar`, or `plain`.
* When set to `null` (default), `pg_restore` will default to `plain`
* @see https://www.postgresql.org/docs/current/app-pgdump.html
*
* @group Environment
* @param string $value
* @return self
* @see $backupCommandFormat
* @since 4.9.0
*/
public function backupCommandFormat(string $value): self
{
$this->backupCommandFormat = $value;
return $this;
}

/**
* The base URL Craft should use when generating control panel URLs.
*
Expand Down Expand Up @@ -5875,12 +5919,12 @@ public function resourceBaseUrl(string $value): self
* ```
*
* @group Environment
* @param string|null|false $value
* @param string|null|false|Closure $value
* @return self
* @see $restoreCommand
* @since 4.2.0
*/
public function restoreCommand(string|null|false $value): self
public function restoreCommand(string|null|false|Closure $value): self
{
$this->restoreCommand = $value;
return $this;
Expand Down
2 changes: 1 addition & 1 deletion src/console/controllers/DbController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ public function actionBackup(?string $path = null): int
$db->backupTo($path);
if ($this->zip) {
$zipPath = FileHelper::zip($path);
unlink($path);
FileHelper::unlink($path);
$path = $zipPath;
}
} catch (Throwable $e) {
Expand Down
28 changes: 15 additions & 13 deletions src/db/Connection.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,10 +209,11 @@ public function getBackupFilePath(): string
$version = Craft::$app->getInfo()->version ?? Craft::$app->getVersion();
$filename = ($systemName ? "$systemName--" : '') . gmdate('Y-m-d-His') . "--v$version";
$backupPath = Craft::$app->getPath()->getDbBackupPath();
$path = $backupPath . DIRECTORY_SEPARATOR . $filename . '.sql';
$path = $backupPath . DIRECTORY_SEPARATOR . $filename . $this->_getDumpExtension();

$i = 0;
while (file_exists($path)) {
$path = $backupPath . DIRECTORY_SEPARATOR . $filename . '--' . ++$i . '.sql';
$path = $backupPath . DIRECTORY_SEPARATOR . $filename . '--' . ++$i . $this->_getDumpExtension();
}
return $path;
}
Expand Down Expand Up @@ -271,12 +272,10 @@ public function backupTo(string $filePath): void
// Determine the command that should be executed
$backupCommand = Craft::$app->getConfig()->getGeneral()->backupCommand;

if ($backupCommand === null) {
$backupCommand = $this->getSchema()->getDefaultBackupCommand($event->ignoreTables);
}

if ($backupCommand === false) {
throw new Exception('Database not backed up because the backup command is false.');
} elseif ($backupCommand === null || $backupCommand instanceof \Closure) {
$backupCommand = $this->getSchema()->getDefaultBackupCommand($event->ignoreTables);
}

// Create the shell command
Expand All @@ -297,10 +296,10 @@ public function backupTo(string $filePath): void
if ($generalConfig->maxBackups) {
$backupPath = Craft::$app->getPath()->getDbBackupPath();

// Grab all .sql files in the backup folder.
// Grab all .sql/.dump files in the backup folder.
$files = array_merge(
glob($backupPath . DIRECTORY_SEPARATOR . '*.sql'),
glob($backupPath . DIRECTORY_SEPARATOR . '*.sql.zip'),
glob($backupPath . DIRECTORY_SEPARATOR . "*.{$this->_getDumpExtension()}"),
glob($backupPath . DIRECTORY_SEPARATOR . "*.{$this->_getDumpExtension()}.zip"),
);

// Sort them by file modified time descending (newest first).
Expand Down Expand Up @@ -337,12 +336,10 @@ public function restore(string $filePath): void
// Determine the command that should be executed
$restoreCommand = Craft::$app->getConfig()->getGeneral()->restoreCommand;

if ($restoreCommand === null) {
$restoreCommand = $this->getSchema()->getDefaultRestoreCommand();
}

if ($restoreCommand === false) {
throw new Exception('Database not restored because the restore command is false.');
} elseif ($restoreCommand === null || $restoreCommand instanceof \Closure) {
$restoreCommand = $this->getSchema()->getDefaultRestoreCommand();
}

// Create the shell command
Expand Down Expand Up @@ -470,6 +467,11 @@ public function trigger($name, Event $event = null)
parent::trigger($name, $event);
}

private function _getDumpExtension(): string
{
return $this->getIsPgsql() && $this->getSchema()->usePgRestore() ? '.dump' : '.sql';
}

/**
* Generates a FK, index, or PK name.
*
Expand Down
126 changes: 66 additions & 60 deletions src/db/mysql/Schema.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,79 +152,57 @@ public function createColumnSchemaBuilder($type, $length = null): ColumnSchemaBu
*/
public function getDefaultBackupCommand(?array $ignoreTables = null): string
{
$useSingleTransaction = true;
$serverVersion = App::normalizeVersion($this->getServerVersion());

$baseCommand = (new ShellCommand('mysqldump'))
->addArg('--defaults-file=', $this->_createDumpConfigFile())
->addArg('--add-drop-table')
->addArg('--comments')
->addArg('--create-options')
->addArg('--dump-date')
->addArg('--no-autocommit')
->addArg('--routines')
->addArg('--default-character-set=', Craft::$app->getConfig()->getDb()->charset)
->addArg('--set-charset')
->addArg('--triggers')
->addArg('--no-tablespaces');

$serverVersion = App::normalizeVersion(Craft::$app->getDb()->getServerVersion());
$isMySQL5 = version_compare($serverVersion, '8', '<');
$isMySQL8 = version_compare($serverVersion, '8', '>=');
$ignoreTables = $ignoreTables ?? Craft::$app->getDb()->getIgnoredBackupTables();
$commandFromConfig = Craft::$app->getConfig()->getGeneral()->backupCommand;

// https://bugs.mysql.com/bug.php?id=109685
if (($isMySQL5 && version_compare($serverVersion, '5.7.41', '>=')) ||
($isMySQL8 && version_compare($serverVersion, '8.0.32', '>='))) {
$useSingleTransaction = false;
}

$defaultArgs =
' --defaults-file="' . $this->_createDumpConfigFile() . '"' .
' --add-drop-table' .
' --comments' .
' --create-options' .
' --dump-date' .
' --no-autocommit' .
' --routines' .
' --default-character-set=' . Craft::$app->getConfig()->getDb()->charset .
' --set-charset' .
' --triggers' .
' --no-tablespaces';
$useSingleTransaction =
($isMySQL5 && version_compare($serverVersion, '5.7.41', '>=')) ||
($isMySQL8 && version_compare($serverVersion, '8.0.32', '>='));

if ($useSingleTransaction) {
$defaultArgs .= ' --single-transaction';
$baseCommand->addArg('--single-transaction');
}

// Find out if the db/dump client supports column-statistics
$shellCommand = new ShellCommand();

if (Platform::isWindows()) {
$shellCommand->setCommand('mysqldump --help | findstr "column-statistics"');
} else {
$shellCommand->setCommand('mysqldump --help | grep "column-statistics"');
if ($this->supportsColumnStatistics()) {
$baseCommand->addArg('--column-statistics=', '0');
}

// If we don't have proc_open, maybe we've got exec
if (!function_exists('proc_open') && function_exists('exec')) {
$shellCommand->useExec = true;
}

$success = $shellCommand->execute();
$schemaDump = (clone $baseCommand)
->addArg('--no-data')
->addArg('--result-file=', '{file}')
->addArg('{database}');

// if there was output, then column-statistics is supported and we should disable it
if ($success && $shellCommand->getOutput()) {
$defaultArgs .= ' --column-statistics=0';
}
$dataDump = (clone $baseCommand)
->addArg('--no-create-info');

if ($ignoreTables === null) {
$ignoreTables = $this->db->getIgnoredBackupTables();
}
$ignoreTableArgs = [];
foreach ($ignoreTables as $table) {
$table = $this->getRawTableName($table);
$ignoreTableArgs[] = "--ignore-table={database}.$table";
$dataDump->addArg('--ignore-table', "{schema}.$table");
}

$schemaDump = 'mysqldump' .
$defaultArgs .
' --no-data' .
' --result-file="{file}"' .
' {database}';

$dataDump = 'mysqldump' .
$defaultArgs .
' --no-create-info' .
' ' . implode(' ', $ignoreTableArgs) .
' {database}' .
' >> "{file}"';
if ($commandFromConfig instanceof \Closure) {
$schemaDump = $commandFromConfig($schemaDump);
$dataDump = $commandFromConfig($dataDump);
}

return $schemaDump . ' && ' . $dataDump;
return "{$schemaDump->getExecCommand()} && {$dataDump->getExecCommand()} >> {file}";
}

/**
Expand All @@ -235,10 +213,16 @@ public function getDefaultBackupCommand(?array $ignoreTables = null): string
*/
public function getDefaultRestoreCommand(): string
{
return 'mysql' .
' --defaults-file="' . $this->_createDumpConfigFile() . '"' .
' {database}' .
' < "{file}"';
$commandFromConfig = Craft::$app->getConfig()->getGeneral()->restoreCommand;
$command = (new ShellCommand('mysql'))
->addArg('--defaults-file=', $this->_createDumpConfigFile())
->addArg('{database}');

if ($commandFromConfig instanceof \Closure) {
$command = $commandFromConfig($command);
}

return $command->getExecCommand() . ' < "{file}"';
}

/**
Expand Down Expand Up @@ -383,6 +367,28 @@ protected function findConstraints($table): void
}
}

protected function supportsColumnStatistics(): bool
{
// Find out if the db/dump client supports column-statistics
$shellCommand = new ShellCommand();

if (Platform::isWindows()) {
$shellCommand->setCommand('mysqldump --help | findstr "column-statistics"');
} else {
$shellCommand->setCommand('mysqldump --help | grep "column-statistics"');
}

// If we don't have proc_open, maybe we've got exec
if (!function_exists('proc_open') && function_exists('exec')) {
$shellCommand->useExec = true;
}

$success = $shellCommand->execute();

// if there was output, then column-statistics is supported
return $success && $shellCommand->getOutput();
}

/**
* Creates a temporary my.cnf file based on the DB config settings.
*
Expand Down
Loading

0 comments on commit 7f8c1fb

Please sign in to comment.