Merge pull request #64 from swiftotter/56-table-ref-issues

#56 & #61 - Dump refactored

.driver/environments.yaml.dist

@@ -10,5 +10,7 @@ environments:
           # For example, if your `TABLE_NAME` is `core_config_data`, Driver will search for a table in the database that
           # ends with `core_config_data`. Thus, `core_config_data` and `sample_core_config_data` would all match.
     ignored_tables: # OPTIONAL, tables listed here will be ignored in the final dump with:
-                    # mysqldump ... --ignored-tables=DATABASE.table_1
+                    # mysqldump ... --ignored-tables=TABLE_NAME
+      - TABLE_NAME
+    empty_tables: # OPTIONAL, tables listed here will be dumped without data
       - TABLE_NAME

.driver/environments.yaml.example

@@ -29,48 +29,7 @@ environments:
         - "UPDATE {{table_name}} SET value = 'store.local' WHERE path LIKE 'web/cookie/cookie_domain' AND scope_id = 0;"
         - "UPDATE {{table_name}} SET value = 'localhost' WHERE path LIKE 'catalog/search/elasticsearch%_server_hostname' AND scope_id = 0;"
     ignored_tables:
-      - setup_module
-      - customer_address_entity
-      - customer_address_entity_datetime
-      - customer_address_entity_decimal
-      - customer_address_entity_int
-      - customer_address_entity_text
-      - customer_address_entity_varchar
-      - customer_entity
-      - customer_entity_datetime
-      - customer_entity_decimal
-      - customer_entity_int
-      - customer_entity_text
-      - customer_entity_varchar
-      - sales_creditmemo
-      - sales_credimemo_comment
-      - sales_creditmemo_grid
-      - sales_creditmemo_item
-      - sales_invoice
-      - sales_invoice_comment
-      - sales_invoice_grid
-      - sales_invoice_item
-      - sales_order
-      - sales_order_address
-      - sales_order_grid
-      - sales_order_item
-      - sales_order_payment
-      - sales_order_status_history
-      - sales_shipment
-      - sales_shipment_comment
-      - sales_shipment_grid
-      - sales_shipment_item
-      - sales_shipment_track
-      - sales_invoiced_aggregated
-      - sales_invoiced_aggregated_order
-      - sales_payment_transaction
-      - sales_order_aggregated_created
-      - sales_order_tax
-      - sales_order_tax_item
-      - sales_quote
-      - sales_quote_address
-      - sales_quote_address_item
-      - sales_quote_item
-      - sales_quote_item_option
-      - sales_quote_payment
-      - sales_quote_shipping_rate
+      - some_non_magento_table
+    empty_tables:
+      - customer_log
+      - customer_visitor

src/Engines/MySql/Export/CommandAssembler.php

@@ -7,6 +7,7 @@
 use Driver\Engines\ConnectionInterface;
 use Driver\Pipeline\Environment\EnvironmentInterface;
 
+use function array_diff;
 use function array_unshift;
 use function implode;
 use function in_array;
@@ -26,54 +27,101 @@ public function __construct(TablesProvider $tablesProvider)
     public function execute(
         ConnectionInterface $connection,
         EnvironmentInterface $environment,
-        string $dumpFile
+        string $dumpFile,
+        string $triggersDumpFile
     ): array {
+        $allTables = $this->tablesProvider->getAllTables($connection);
         $ignoredTables = $this->tablesProvider->getIgnoredTables($environment);
+        if (array_diff($allTables, $ignoredTables) === []) {
+            return [];
+        }
         $emptyTables = $this->tablesProvider->getEmptyTables($environment);
-        foreach ($this->tablesProvider->getAllTables($connection) as $table) {
+        $commands = [$this->getStructureCommand($connection, $ignoredTables, $dumpFile)];
+        foreach ($allTables as $table) {
             if (in_array($table, $ignoredTables) || in_array($table, $emptyTables)) {
                 continue;
             }
-            $commands[] = $this->getSingleCommand($connection, [$table], $dumpFile);
-        }
-        if (!empty($emptyTables)) {
-            $commands[] = $this->getSingleCommand($connection, $emptyTables, $dumpFile, false);
-        }
-        if (empty($commands)) {
-            return [];
+            $commands[] = $this->getDataCommand($connection, [$table], $dumpFile);
         }
         array_unshift(
             $commands,
             "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'"
-            . ">> $dumpFile"
+            . " | gzip >> $dumpFile"
         );
-        $commands[] = "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> $dumpFile";
-        $commands[] = "cat $dumpFile | "
-            . "sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > $dumpFile.gz";
+        $commands[] = "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' | gzip >> $dumpFile";
+        $commands[] = $this->getTriggersCommand($connection, $ignoredTables, $triggersDumpFile);
         return $commands;
     }
 
     /**
-     * @param string[] $tables
+     * @param string[] $ignoredTables
      */
-    private function getSingleCommand(
+    private function getStructureCommand(
         ConnectionInterface $connection,
-        array $tables,
-        string $dumpFile,
-        bool $withData = true
+        array $ignoredTables,
+        string $dumpFile
     ): string {
         $parts = [
             "mysqldump --user=\"{$connection->getUser()}\"",
             "--password=\"{$connection->getPassword()}\"",
             "--single-transaction",
             "--no-tablespaces",
+            "--no-data",
+            "--skip-triggers",
+            "--host={$connection->getHost()}",
+            $connection->getDatabase()
+        ];
+        foreach ($ignoredTables as $table) {
+            $parts[] = "--ignore-table={$connection->getDatabase()}.{$table}";
+        }
+        $parts[] = "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g'";
+        $parts[] = "| gzip";
+        $parts[] = ">> $dumpFile";
+        return implode(' ', $parts);
+    }
+
+    /**
+     * @param string[] $tables
+     */
+    private function getDataCommand(ConnectionInterface $connection, array $tables, string $dumpFile): string
+    {
+        $parts = [
+            "mysqldump --user=\"{$connection->getUser()}\"",
+            "--password=\"{$connection->getPassword()}\"",
+            "--single-transaction",
+            "--no-tablespaces",
+            "--no-create-info",
+            "--skip-triggers",
             "--host={$connection->getHost()}",
             $connection->getDatabase(),
             implode(' ', $tables)
         ];
-        if (!$withData) {
-            $parts[] = '--no-data';
+        $parts[] = "| gzip";
+        $parts[] = ">> $dumpFile";
+        return implode(' ', $parts);
+    }
+
+    /**
+     * @param string[] $ignoredTables
+     */
+    private function getTriggersCommand(ConnectionInterface $connection, array $ignoredTables, string $dumpFile): string
+    {
+        $parts = [
+            "mysqldump --user=\"{$connection->getUser()}\"",
+            "--password=\"{$connection->getPassword()}\"",
+            "--single-transaction",
+            "--no-tablespaces",
+            "--no-data",
+            "--no-create-info",
+            "--triggers",
+            "--host={$connection->getHost()}",
+            $connection->getDatabase()
+        ];
+        foreach ($ignoredTables as $table) {
+            $parts[] = "--ignore-table={$connection->getDatabase()}.{$table}";
         }
+        $parts[] = "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g'";
+        $parts[] = "| gzip";
         $parts[] = ">> $dumpFile";
         return implode(' ', $parts);
     }

src/Engines/MySql/Export/Primary.php

@@ -27,7 +27,8 @@ class Primary extends Command implements CommandInterface, CleanupInterface
     private array $properties;
     private LoggerInterface $logger;
     private Random $random;
-    private ?string $path = null;
+    /** @var array<string, string> */
+    private array $filePaths = [];
     private Configuration $configuration;
     private ConsoleOutput $output;
     private CommandAssembler $commandAssembler;
@@ -58,7 +59,8 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm
         $this->output->writeln("<comment>Exporting database from local MySql</comment>");
 
         try {
-            $commands = $this->commandAssembler->execute($this->localConnection, $environment, $this->getDumpFile());
+            $commands = $this->commandAssembler
+                ->execute($this->localConnection, $environment, $this->getDumpFile(), $this->getDumpFile('triggers'));
             if (empty($commands)) {
                 throw new RuntimeException('Nothing to import');
             }
@@ -70,7 +72,7 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm
                 $result = system($command, $resultCode);
                 if ($result === false || $resultCode !== 0) {
                     $message = sprintf('Error (%s) when executing command: %s', $resultCode, $strippedCommand);
-                    $this->output->writeln("<error>${$message}</error>");
+                    $this->output->writeln("<error>$message</error>");
                     throw new RuntimeException($message);
                 }
             }
@@ -82,14 +84,17 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm
         $this->logger->notice("Database dump has completed.");
         $this->output->writeln("<info>Database dump has completed.</info>");
         return $transport->withStatus(new Status('sandbox_init', 'success'))
-            ->withNewData('dump-file', $this->getDumpFile());
+            ->withNewData('dump-file', $this->getDumpFile())
+            ->withNewData('triggers-dump-file', $this->getDumpFile('triggers'));
     }
 
     public function cleanup(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface
     {
-        if ($this->getDumpFile() && file_exists($this->getDumpFile())) {
-            @unlink($this->getDumpFile());
-        }
+        array_walk($this->filePaths, function (string $filePath): void {
+            if ($filePath && file_exists($filePath)) {
+                @unlink($filePath);
+            }
+        });
         return $transport;
     }
 
@@ -99,18 +104,18 @@ public function getProperties(): array
         return $this->properties;
     }
 
-    private function getDumpFile(): string
+    private function getDumpFile(string $code = 'default'): string
     {
-        if (!$this->path) {
+        if (!\array_key_exists($code, $this->filePaths)) {
             $path = $this->configuration->getNode('connections/mysql/dump-path');
             if (!$path) {
                 $path = self::DEFAULT_DUMP_PATH;
             }
-            $filename = 'driver-' . $this->random->getRandomString(6) . '.sql';
+            $filename = 'driver-' . $this->random->getRandomString(6) . '.gz';
 
-            $this->path = $path . '/' . $filename;
+            $this->filePaths[$code] = $path . '/' . $filename;
         }
 
-        return $this->path;
+        return $this->filePaths[$code];
     }
 }

src/Engines/MySql/Sandbox/Export.php

@@ -71,7 +71,11 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm
         );
 
         $environmentName = $environment->getName();
-        $command = $this->assembleCommand($environmentName, $environment->getIgnoredTables());
+        $command = $this->assembleCommand(
+            $environmentName,
+            $environment->getIgnoredTables(),
+            $transport->getData('triggers-dump-file')
+        );
 
         $this->files[] = $this->getFilename($environmentName);
 
@@ -101,30 +105,23 @@ public function cleanup(TransportInterface $transport, EnvironmentInterface $env
     /**
      * @param string[] $ignoredTables
      */
-    private function assembleCommand(string $environmentName, array $ignoredTables): string
+    private function assembleCommand(string $environmentName, array $ignoredTables, string $triggersDumpFile): string
     {
+        $filename = $this->getFilename($environmentName);
         $command = implode(' ', array_merge([
             "mysqldump --user={$this->connection->getUser()}",
             "--password={$this->connection->getPassword()}",
             "--host={$this->connection->getHost()}",
             "--port={$this->connection->getPort()}",
             "--no-tablespaces"
         ], $this->getIgnoredTables($ignoredTables)));
-
         $command .= " {$this->connection->getDatabase()} ";
         $command .= "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' ";
-
         if ($this->compressOutput()) {
-            $command .= ' ' . implode(' ', [
-                '|',
-                'gzip --best'
-            ]);
+            $command .= "| gzip --best ";
         }
-
-        $command .= ' ' . implode(' ', [
-            '>',
-            $this->getFilename($environmentName)
-        ]);
+        $command .= "> $filename;";
+        $command .= ($this->compressOutput() ? "cat" : "gunzip < ") . " $triggersDumpFile >> $filename;";
 
         return $command;
     }

src/Engines/MySql/Sandbox/Import.php

@@ -74,14 +74,13 @@ public function getProperties(): array
     public function assembleCommand(string $path): string
     {
         $command = implode(' ', [
+            "gunzip < $path |",
             "mysql --user={$this->remoteConnection->getUser()}",
-                "--password={$this->remoteConnection->getPassword()}",
-                "--host={$this->remoteConnection->getHost()}",
-                "--port={$this->remoteConnection->getPort()}",
-                $this->remoteConnection->useSsl() ? "--ssl-ca={$this->ssl->getPath()}" : "",
-                "{$this->remoteConnection->getDatabase()}",
-            "<",
-            $path
+            "--password={$this->remoteConnection->getPassword()}",
+            "--host={$this->remoteConnection->getHost()}",
+            "--port={$this->remoteConnection->getPort()}",
+            $this->remoteConnection->useSsl() ? "--ssl-ca={$this->ssl->getPath()}" : "",
+            "{$this->remoteConnection->getDatabase()}"
         ]);
 
         if (

src/Engines/MySql/Transformation.php

@@ -78,10 +78,7 @@ private function applyTransformationsTo(ReconnectingPDO $connection, array $tran
                     $ex->getMessage(),
                     $ex->getTraceAsString()
                 ]);
-                $this->output->writeln("<error>Query transformation failed: " . $query, [
-                    $ex->getMessage(),
-                    $ex->getTraceAsString()
-                ] . '</error>');
+                $this->output->writeln("<error>Query transformation failed: " . $query . '</error>');
             }
         });
     }