Allow EAV and other extended table types

README.md

@@ -27,9 +27,63 @@ For example, to override the `admin.yaml` for Magento 2, you place a file in `co
 admin:
 ```
 
+### Partial anonymisation
+
+You can affect only certain records by including a 'where' clause - for example to avoid anonymising certain admin accounts, or to preserve data used in unit tests, like this:
+
+```yaml
+customers:
+  customer_entity:
+    provider: # this sets options specific to the type of table
+      where: "`email` not like '%@mycompany.com'" # leave mycompany.com emails alone
+```
+
+### Delete Data
+
+You might want to fully or partially delete data - eg. if your developers don't need sales orders, or you want to keep the database size a lot smaller than the production database.  Specify the 'delete' option.
+
+When deleting some Magento data, eg. sales orders, add the command line option `--with-integrity` which enforces foreign key checks, so for example sales\_invoice records will be deleted automatically if their parent sales\_order is deleted:
+
+```yaml
+orders:
+  sales_order:
+    provider:
+      delete: true
+      where: "customer_id != 3" # delete all except customer 3's orders because we use that for testing
+    # no need to specify columns if you're using 'delete'      
+```
+
+If you use 'delete' without a 'where', and without '--with-integrity', it will use 'truncate' to delete the entire table.  It will not use truncate if --with-integrity is specified since that bypasses key checks.
+
+### Magento EAV Attributes
+
+You can use the Magento2Eav table type to treat EAV attributes just like normal columns, eg.
+
+```yaml
+products:
+  catalog_product_entity: # specify the base table of the entity
+    provider:
+      name: \Elgentos\Masquerade\Provider\Table\Magento2Eav
+      where: "sku != 'TESTPRODUCT'" # you can still use 'where' and 'delete'
+    columns:
+      my_custom_attribute:
+        formatter: sentence
+      my_other_attribute:
+        formatter: email
+
+  catalog_category_entity:
+    provider: \Elgentos\Masquerade\Provider\Table\Magento2Eav # shortcut if not using 'where' or 'delete'
+    columns:
+      description: # refer to EAV attributes like normal columns
+        formatter: paragraph
+
+```
+
+### Formatter Options
+
 For formatters, you can use all default [Faker formatters](https://github.com/fzaninotto/Faker#formatters).
 
-#### Custom Providers / Formatters
+#### Custom Data Providers / Formatters
 
 You can also create your own custom providers with formatters. They need to extend `Faker\Provider\Base` and they need to live in either `~/.masquerade` or `.masquerade` relative from where you run masquerade.
 
@@ -63,6 +117,48 @@ customer:
           name: woopwoop
 ```
 
+### Custom Table Type Providers
+
+Some systems have linked tables containing related data - eg. Magento's EAV system, Drupal's entity fields and Wordpress's post metadata tables.  You can provide custom table types like this:
+
+An example file `.masquerade/Custom/WoopTable.php`;
+
+```php
+<?php
+
+namespace Custom;
+
+use Elgentos\Masquerade\Provider\Table\Base;
+
+class WoopTable extends Base {
+
+    public function setup() // do any one-off work - eg. delete/truncate, find EAV attributes, create temporary tables
+
+    public function columns() // return a list of the column names that will be faked - these don't have to be real database fields
+
+    public function getPrimaryKey() // if you inherit \Elgentos\Masquerade\Provider\Table\Simple, this will be guessed
+
+    public function update($primaryKey, [field=>value, ...]) // update a record - handle the update of any special field types here
+
+    public function query() // return an Illuminate database query object giving all the records you want to affect, and selecting all the columns
+}
+```
+
+And then use it in your YAML file. A provider needs to be set on the table level, and can be a simple class name, or a set of options which are available to your class.  See the documentation in the 'Base' class, and the code for the 'Simple' table type for more details.
+
+```yaml
+customer:
+  customer_entity:
+    provider:
+      class: \Custom\WoopTable
+      option1: "test"
+      option2: false
+    columns:
+      firstname:
+        formatter:
+          name: firstName
+```
+
 ### Installation
 
 Download the phar file:
@@ -93,6 +189,7 @@ Options:
       --prefix[=PREFIX]      Database prefix [empty]
       --locale[=LOCALE]      Locale for Faker data [en_US]
       --group[=GROUP]        Comma-separated groups to run masquerade on [all]
+      --with-integrity       Run with foreign key checks enabled
 ```
 
 You can also set these variables in a `config.yaml` file in the same location as where you run masquerade from, for example:

src/Elgentos/Masquerade/Console/RunCommand.php

@@ -11,6 +11,7 @@
 use Faker\Factory as FakerFactory;
 use Symfony\Component\Console\Helper\ProgressBar;
 use Illuminate\Database\Capsule\Manager as Capsule;
+use Illuminate\Support\Arr;
 
 class RunCommand extends Command
 {
@@ -22,6 +23,8 @@ class RunCommand extends Command
 
     const VERSION = '0.1.13';
 
+    const DEFAULT_QUERY_PROVIDER = \Elgentos\Masquerade\Provider\Table\Simple::class;
+
     protected $config;
 
     /**
@@ -87,7 +90,8 @@ protected function configure()
             ->addOption('prefix', null, InputOption::VALUE_OPTIONAL, 'Database prefix [empty]')
             ->addOption('locale', null, InputOption::VALUE_OPTIONAL, 'Locale for Faker data [en_US]')
             ->addOption('group', null, InputOption::VALUE_OPTIONAL, 'Which groups to run masquerade on [all]')
-            ->addOption('charset', null, InputOption::VALUE_OPTIONAL, 'Database charset [utf8]');
+            ->addOption('charset', null, InputOption::VALUE_OPTIONAL, 'Database charset [utf8]')
+            ->addOption('with-integrity', null, InputOption::VALUE_NONE, 'Run with foreign key checks enabled');
     }
 
     /**
@@ -123,39 +127,29 @@ protected function execute(InputInterface $input, OutputInterface $output)
      */
     private function fakeData(array $table) : void
     {
-        if (!$this->db->getSchemaBuilder()->hasTable($table['name'])) {
-            $this->output->writeln('Table ' . $table['name'] . ' does not exist.');
-            return;
-        }
-
-        foreach ($table['columns'] as $columnName => $columnData) {
-            if (!$this->db->getSchemaBuilder()->hasColumn($table['name'], $columnName)) {
-                unset($table['columns'][$columnName]);
-                $this->output->writeln('Column ' . $columnName . ' in table ' . $table['name'] . ' does not exist; skip it.');
-            }
+        $tableProviderData = Arr::get($table, 'provider', []);
+        if (is_string($tableProviderData)) {
+            $tableProviderData = ['class' => $tableProviderData]; // just a class rather than array of options
         }
+        $tableProviderClass = Arr::get($tableProviderData, 'class', self::DEFAULT_QUERY_PROVIDER);
+        $tableProvider = new $tableProviderClass($this->input, $this->output, $this->db, $table, $tableProviderData);
 
         $this->output->writeln('');
-        $this->output->writeln('Updating ' . $table['name']);
+        $this->output->writeln('Updating ' . $table['name'] . ' using '. $tableProviderClass);
+
+        $tableProvider->setup();
 
-        $totalRows = $this->db->table($table['name'])->count();
+        $totalRows = $tableProvider->count();
         $progressBar = new ProgressBar($this->output, $totalRows);
         $progressBar->setRedrawFrequency($this->calculateRedrawFrequency($totalRows));
         $progressBar->start();
 
-        $primaryKey = array_get($table, 'pk', 'entity_id');
+        $primaryKey = $tableProvider->getPrimaryKey();
 
-        // Null columns before run to avoid integrity constrains errors
-        foreach ($table['columns'] as $columnName => $columnData) {
-            if (array_get($columnData, 'nullColumnBeforeRun', false)) {
-                $this->db->table($table['name'])->update([$columnName => null]);
-            }
-        }
-
-        $this->db->table($table['name'])->orderBy($primaryKey)->chunk(100, function ($rows) use ($table, $progressBar, $primaryKey) {
+        $tableProvider->query()->chunk(100, function ($rows) use ($table, $progressBar, $primaryKey, $tableProvider) {
             foreach ($rows as $row) {
                 $updates = [];
-                foreach ($table['columns'] as $columnName => $columnData) {
+                foreach ($tableProvider->columns() as $columnName => $columnData) {
                     $formatter = array_get($columnData, 'formatter.name');
                     $formatterData = array_get($columnData, 'formatter');
                     $providerClassName = array_get($columnData, 'provider', false);
@@ -190,7 +184,7 @@ private function fakeData(array $table) : void
                         $updates[$columnName] = null;
                     }
                 }
-                $this->db->table($table['name'])->where($primaryKey, $row->{$primaryKey})->update($updates);
+                $tableProvider->update($row->{$primaryKey}, $updates);
                 $progressBar->advance();
             }
         });
@@ -253,7 +247,10 @@ private function setup()
         ]);
 
         $this->db = $capsule->getConnection();
-        $this->db->statement('SET FOREIGN_KEY_CHECKS=0');
+        if (!$this->input->getOption('with-integrity')) {
+            $this->output->writeln('[Foreign key constraint checking is off - deletions will not affect linked tables]');
+            $this->db->statement('SET FOREIGN_KEY_CHECKS=0');
+        }
 
         $this->locale = $this->input->getOption('locale') ?? $databaseConfig['locale'] ?? 'en_US';
 

src/Elgentos/Masquerade/Provider/Table/Base.php

@@ -0,0 +1,98 @@
+<?php
+
+namespace Elgentos\Masquerade\Provider\Table;
+
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Input\InputInterface;
+
+/**
+ * All table providers must inherit this
+ *
+ * example config:
+ *
+ * group1:   # the group
+ *   table1:  # the entity name
+ *     provider: \Elgentos\Masquerade\Provider\Table\YourProviderClass
+ *     columns:
+ *       cost_price:
+ *         ...your usual column formatting goes here...
+ *
+ * OR, if your provider takes options:
+ *
+ * group1:
+ *   table1:
+ *     provider:
+ *       class: \My\Custom\Class\Name
+ *       option1: "some value"
+ *       option2: "some value"
+ *
+ * In your class, the provider options will be accessible as $this->options['option1'] etc, and table data as $this->table[...]
+ *
+ * The setup() method will be called before any processing.
+ *
+ *
+ */
+
+abstract class Base
+{
+    protected $output;
+    protected $input;
+    protected $db;
+    protected $table;
+    protected $options = [];
+
+    public function __construct(InputInterface $input, OutputInterface $output, \Illuminate\Database\Connection $db, array $tableData, array $providerData = [])
+    {
+        $this->output = $output;
+        $this->input = $input;
+        $this->db = $db;
+        $this->table = $tableData;
+        $this->options = $providerData;
+    }
+
+    /**
+     * Do any setup or validation work here, eg. extracting details from the database for later use,
+     * removing unused columns, etc
+     *
+     * @return void
+     */
+    public function setup()
+    {
+    }
+
+    /**
+     * Return the name of the primary key column in the query returned by ->query()
+     * @return string
+     */
+    abstract public function getPrimaryKey();
+
+    /**
+     * Return the columns with their config
+     * @return array
+     */
+    public function columns()
+    {
+        return $this->table['columns'];
+    }
+
+    /**
+     * @return int The number of rows which will be affected
+     */
+    abstract public function count();
+
+    /**
+     * Update a set of columns for a specific primary key
+     * @param string|int Primary Key
+     * @param array in the form [column_name => value, ...]
+     * @return void
+     */
+    abstract public function update($primaryKey, array $updates);
+
+    /**
+     * Return a query builder which will return the column names returned by $this->columns()
+     * It should be ordered by primary key
+     *
+     * @return \Illuminate\Database\Query\Builder
+     */
+    abstract public function query() : \Illuminate\Database\Query\Builder;
+}