Search Kit - For "Entity" displays, provide TABLE/VIEW options

CRM/Core/BAO/SchemaHandler.php

@@ -1000,4 +1000,15 @@ public static function getDBCharset() {
     return CRM_Core_DAO::singleValueQuery('SELECT @@character_set_database');
   }
 
+  /**
+   * @param string $table
+   * @return string|null
+   *   Ex: 'BASE TABLE' or 'VIEW'
+   */
+  public static function getTableType(string $table): ?string {
+    return \CRM_Core_DAO::singleValueQuery(
+      'SELECT TABLE_TYPE  FROM information_schema.tables  WHERE TABLE_SCHEMA=database() AND TABLE_NAME LIKE %1',
+      [1 => [$table, 'String']]);
+  }
+
 }

ext/search_kit/Civi/Api4/Action/SKEntity/Refresh.php

@@ -2,10 +2,9 @@
 
 namespace Civi\Api4\Action\SKEntity;
 
-use Civi\API\Request;
 use Civi\Api4\Generic\AbstractAction;
 use Civi\Api4\Generic\Result;
-use Civi\Api4\Query\Api4SelectQuery;
+use Civi\Search\SKEntityGenerator;
 
 /**
  * Store the results of a SearchDisplay as a SQL table.
@@ -29,25 +28,11 @@ public function _run(Result $result) {
       ->addWhere('name', '=', $displayName)
       ->execute()->single();
 
-    $apiParams = $display['saved_search_id.api_params'];
-    // Add orderBy to api params
-    foreach ($display['settings']['sort'] ?? [] as $item) {
-      $apiParams['orderBy'][$item[0]] = $item[1];
+    if (($display['settings']['data_mode'] ?? 'table') !== 'table') {
+      return;
     }
-    // Set select clause to match display columns
-    $select = [];
-    foreach ($display['settings']['columns'] as $column) {
-      foreach ($apiParams['select'] as $selectExpr) {
-        if ($selectExpr === $column['key'] || str_ends_with($selectExpr, " AS {$column['key']}")) {
-          $select[] = $selectExpr;
-          continue 2;
-        }
-      }
-    }
-    $apiParams['select'] = $select;
-    $api = Request::create($display['saved_search_id.api_entity'], 'get', $apiParams);
-    $query = new Api4SelectQuery($api);
-    $query->forceSelectId = FALSE;
+
+    $query = (new SKEntityGenerator())->createQuery($display['saved_search_id.api_entity'], $display['saved_search_id.api_params'], $display['settings']);
     $sql = $query->getSql();
     $tableName = _getSearchKitDisplayTableName($displayName);
     $columnSpecs = array_column($display['settings']['columns'], 'spec');

ext/search_kit/Civi/Api4/Event/Subscriber/SKEntitySubscriber.php

@@ -19,6 +19,7 @@
 use Civi\Core\Event\PostEvent;
 use Civi\Core\Event\PreEvent;
 use Civi\Core\Service\AutoService;
+use Civi\Search\SKEntityGenerator;
 use Symfony\Component\EventDispatcher\EventSubscriberInterface;
 
 /**
@@ -36,8 +37,8 @@ class SKEntitySubscriber extends AutoService implements EventSubscriberInterface
   public static function getSubscribedEvents(): array {
     return [
       'civi.api4.entityTypes' => 'on_civi_api4_entityTypes',
-      'hook_civicrm_pre' => 'onPreSaveDisplay',
-      'hook_civicrm_post' => 'onPostSaveDisplay',
+      'hook_civicrm_pre::SearchDisplay' => 'onPreSaveDisplay',
+      'hook_civicrm_post::SearchDisplay' => 'onPostSaveDisplay',
     ];
   }
 
@@ -83,6 +84,7 @@ public function onPreSaveDisplay(PreEvent $event): void {
     // Drop the old table if it exists
     if ($oldName) {
       \CRM_Core_BAO_SchemaHandler::dropTable(_getSearchKitDisplayTableName($oldName));
+      \CRM_Core_DAO::executeQuery(sprintf('DROP VIEW IF EXISTS `%s`', _getSearchKitDisplayTableName($oldName)));
     }
     if ($event->action === 'delete') {
       // Delete scheduled jobs when deleting entity
@@ -119,9 +121,34 @@ public function onPreSaveDisplay(PreEvent $event): void {
     }
     // Store new settings with added column spec
     $event->params['settings'] = $newSettings;
-    $sql = \CRM_Core_BAO_SchemaHandler::buildTableSQL($table);
-    // do not i18n-rewrite
-    \CRM_Core_DAO::executeQuery($sql, [], TRUE, NULL, FALSE, FALSE);
+
+    $mode = $event->params['settings']['data_mode'] ?? 'table';
+    switch ($mode) {
+      case 'table':
+      case '':
+        $sql = \CRM_Core_BAO_SchemaHandler::buildTableSQL($table);
+        // do not i18n-rewrite
+        \CRM_Core_DAO::executeQuery($sql, [], TRUE, NULL, FALSE, FALSE);
+        break;
+
+      case 'view':
+        $tableName = _getSearchKitDisplayTableName($newName);
+        $tempSettings = $event->params['settings'];
+        $query = (new SKEntityGenerator())->createQuery($this->savedSearch['api_entity'], $this->savedSearch['api_params'], $tempSettings);
+        $columnSpecs = array_column($tempSettings['columns'], 'spec');
+        $columns = implode(', ', array_column($columnSpecs, 'name'));
+
+        $sql = "CREATE VIEW `$tableName` (_row, $columns) AS " . $query->getSql();
+        $sql = preg_replace('/ SELECT /', ' SELECT row_number() over () AS _row, ', $sql, 1);
+        // Q: Do we really need _row? What are the performance implications?
+
+        // do not i18n-rewrite
+        \CRM_Core_DAO::executeQuery($sql, [], TRUE, NULL, FALSE, FALSE);
+        break;
+
+      default:
+        throw new \LogicException("Search display $event->id has invalid mode ($mode)");
+    }
   }
 
   /**

ext/search_kit/Civi/Search/SKEntityGenerator.php

@@ -0,0 +1,53 @@
+<?php
+/*
+ +--------------------------------------------------------------------+
+ | Copyright CiviCRM LLC. All rights reserved.                        |
+ |                                                                    |
+ | This work is published under the GNU AGPLv3 license with some      |
+ | permitted exceptions and without any warranty. For full license    |
+ | and copyright information, see https://civicrm.org/licensing       |
+ +--------------------------------------------------------------------+
+ */
+
+namespace Civi\Search;
+
+use Civi\API\Request;
+use Civi\Api4\Query\Api4SelectQuery;
+
+class SKEntityGenerator {
+
+  /**
+   * @param string $realEntity
+   *   The underlying API entity that we need to query
+   * @param array $realParams
+   *   Basic API request
+   * @param array $settings
+   *   The settings from the SearchDisplay record.
+   * @return \Civi\Api4\Query\Api4SelectQuery
+   *   A prepared query
+   * @throws \Civi\API\Exception\NotImplementedException
+   */
+  public function createQuery(string $realEntity, array $realParams, array $settings): Api4SelectQuery {
+    $apiParams = $realParams;
+    // Add orderBy to api params
+    foreach ($settings['sort'] ?? [] as $item) {
+      $apiParams['orderBy'][$item[0]] = $item[1];
+    }
+    // Set select clause to match display columns
+    $select = [];
+    foreach ($settings['columns'] as $column) {
+      foreach ($apiParams['select'] as $selectExpr) {
+        if ($selectExpr === $column['key'] || str_ends_with($selectExpr, " AS {$column['key']}")) {
+          $select[] = $selectExpr;
+          continue 2;
+        }
+      }
+    }
+    $apiParams['select'] = $select;
+    $api = Request::create($realEntity, 'get', $apiParams);
+    $query = new Api4SelectQuery($api);
+    $query->forceSelectId = FALSE;
+    return $query;
+  }
+
+}

ext/search_kit/ang/crmSearchAdmin/displays/searchAdminDisplayEntity.component.js

@@ -17,6 +17,12 @@
       $scope.hs = crmUiHelp({file: 'CRM/Search/Help/DisplayTypeEntity'});
 
       this.permissions = CRM.crmSearchAdmin.permissions;
+      this.dataModes = [
+        {id: 'table', text: ts('MySQL Table')},
+        {id: 'view', text: ts('MySQL View')}
+        // {id: 'cte', text: ts('MySQL Table Expression')}
+      ];
+      ctrl.isDataMode = (m) => (m == (ctrl.display.settings.data_mode || 'table'));
 
       this.$onInit = function () {
         ctrl.jobFrequency = CRM.crmSearchAdmin.jobFrequency;

ext/search_kit/ang/crmSearchAdmin/displays/searchAdminDisplayEntity.html

@@ -5,19 +5,24 @@
 <div>
   <textarea class="form-control" placeholder="{{:: ts('Description') }}" ng-model="$ctrl.display.settings.description"></textarea>
 </div>
+<div class="form-inline">
+  <label for="display_data_mode">{{:: ts('Data Mode') }}</label>
+  <a crm-ui-help="hs({id: 'data_mode', title: ts('Data Mode')})"></a>
+  <input id="display_data_mode" class="form-control" crm-ui-select="{data: $ctrl.dataModes, placeholder: ts('MySQL Table')}" ng-model="$ctrl.display.settings.data_mode">
+</div>
 <div class="form-inline">
   <label for="crm-search-admin-display-api">{{:: ts('API Name') }} <span class="crm-marker">*</span></label>
   <div class="input-group">
     <span class="input-group-addon">SK_</span>
     <input id="crm-search-admin-display-api" type="text" class="form-control" ng-model="$ctrl.display.name" required />
   </div>
 </div>
-<p ng-if="$ctrl.display.id">
+<p ng-if="$ctrl.isDataMode('table') && $ctrl.display.id">
   <i class="crm-i fa-clock-o"></i>
   <strong ng-if="$ctrl.display._refresh_date">{{:: ts('Last refreshed: %1. Click "Save" to refresh now.', {1: $ctrl.display._refresh_date}) }}</strong>
   <strong ng-if="!$ctrl.display._refresh_date">{{:: ts('Checking last refresh date...') }}</strong>
 </p>
-<div class="form-inline" ng-if="$ctrl.display._job">
+<div class="form-inline" ng-if="$ctrl.isDataMode('table') && $ctrl.display._job">
   <div class="checkbox-inline form-control">
     <label>
       <input type="checkbox" ng-model="$ctrl.display._job.is_active">

ext/search_kit/templates/CRM/Search/Help/DisplayTypeEntity.hlp

@@ -2,3 +2,31 @@
   <p>{ts}Set the permission level needed to view this entity.{/ts}</p>
   <p>{ts}Users without this permission will not be able to see this entity in SearchKit nor in any search queries or displays that use this entity.{/ts}</p>
 {/htxt}
+{htxt id="data_mode"}
+  <p>{ts}Determine which data-storage features to use when generating this entity.{/ts}</p>
+  <ul>
+    {* We should probably give some more hints about when to use each. Included draft notes about expected trade-offs. *}
+
+    {* TABLE: Expected trade-offs:
+      Strength: Stable data. Stable structure. Analytics with grouping/aggregation/function-evaluation.
+      Weakness: Volatile data. Volatile structure.
+    *}
+    <li>{ts}<strong>MySQL Table</strong>: Create a persistent MySQL table and re-fill it periodically.{/ts}</li>
+    {* VIEW: Expected trade-offs:
+      Strength: Volatile data. Stable structure. Simple variations on existing entities. Targeted access to subsets of data.
+      Weakness: Volatile structure. Analytics with grouping/aggregation/function-evaluation.
+    *}
+    <li>{ts}<strong>MySQL View</strong>: Create a persistent MySQL view. Better suited to volatile data with a stable data-structure. Better suited to targeted data-access but not to analytics.{/ts}</li>
+    {* CTE: Expected trade-offs:
+      Strength: Volatile data. Volatile structure.
+      Moderate: Analytics with grouping/aggregation/function-evaluation.
+      Weakness: Targeted access to subsets of data.
+
+      Compared to VIEWs, a key difference is CTEs are likely to build an on-demand temporary table to cache intermediate
+      results. This is good if you need to full-table operations like grouping/aggregating. But it may be bad if you're
+      only accessing a small portion of the data.
+    <li>{ts}<strong>MySQL CTE</strong>: Create dynamic MySQL query using Common Table Expression. Better suited to volatile {/ts}</li>
+    *}
+  </ul>
+
+{/htxt}

ext/search_kit/tests/phpunit/api/v4/SearchDisplay/EntityDisplayTest.php

@@ -22,7 +22,17 @@ public function setUpHeadless(): CiviEnvBuilder {
       ->apply();
   }
 
-  public function testEntityDisplay() {
+  public static function getDataModes(): array {
+    return [
+      ['table'],
+      ['view'],
+    ];
+  }
+
+  /**
+   * @dataProvider getDataModes
+   */
+  public function testEntityDisplay(string $dataMode) {
     $lastName = uniqid(__FUNCTION__);
 
     $this->saveTestRecords('Contact', [
@@ -51,6 +61,7 @@ public function testEntityDisplay() {
       ->addValue('label', 'MyNewEntity')
       ->addValue('name', 'MyNewEntity')
       ->addValue('settings', [
+        'data_mode' => $dataMode,
         'columns' => [
           [
             'key' => 'id',
@@ -90,22 +101,33 @@ public function testEntityDisplay() {
       ])
       ->execute()->first();
 
+    $expectTypes = ['table' => 'BASE TABLE', 'view' => 'VIEW'];
+    $this->assertEquals($expectTypes[$dataMode], \CRM_Core_BAO_SchemaHandler::getTableType('civicrm_sk_my_new_entity'));
+
     $schema = \CRM_Core_DAO::executeQuery('DESCRIBE civicrm_sk_my_new_entity')->fetchAll();
     $this->assertCount(7, $schema);
     $this->assertEquals('_row', $schema[0]['Field']);
-    $this->assertStringStartsWith('int', $schema[0]['Type']);
-    $this->assertEquals('PRI', $schema[0]['Key']);
+    // $this->assertStringStartsWith('int', $schema[0]['Type']);
+    $this->assertMatchesRegularExpression('/^(int|bigint)/', $schema[0]['Type']);
+    if ($dataMode === 'table') {
+      $this->assertEquals('PRI', $schema[0]['Key']);
+    }
 
-    // created_date and modified_date should be NULLable and have no default/extra
-    $this->assertEmpty($schema[5]['Default']);
-    $this->assertEmpty($schema[5]['Extra']);
-    $this->assertEquals('YES', $schema[5]['Null']);
-    $this->assertEmpty($schema[6]['Default']);
-    $this->assertEmpty($schema[6]['Extra']);
-    $this->assertEquals('YES', $schema[5]['Null']);
+    if ($dataMode === 'table') {
+      // created_date and modified_date should be NULLable and have no default/extra
+      $this->assertEmpty($schema[5]['Default']);
+      $this->assertEmpty($schema[5]['Extra']);
+      $this->assertEquals('YES', $schema[5]['Null']);
+      $this->assertEmpty($schema[6]['Default']);
+      $this->assertEmpty($schema[6]['Extra']);
+      $this->assertEquals('YES', $schema[5]['Null']);
+    }
 
-    $rows = \CRM_Core_DAO::executeQuery('SELECT * FROM civicrm_sk_my_new_entity')->fetchAll();
-    $this->assertCount(0, $rows);
+    if ($dataMode === 'table') {
+      // SQL table is not yet hydrated.
+      $rows = \CRM_Core_DAO::executeQuery('SELECT * FROM civicrm_sk_my_new_entity')->fetchAll();
+      $this->assertCount(0, $rows);
+    }
 
     $getFields = civicrm_api4('SK_MyNewEntity', 'getFields', ['loadOptions' => TRUE])->indexBy('name');
     $this->assertNotEmpty($getFields['prefix_id']['options'][1]);
@@ -160,7 +182,10 @@ public function testEntityDisplay() {
     $this->assertStringContainsString('SK_MyNewEntity', $e->getMessage());
   }
 
-  public function testEntityDisplayWithJoin() {
+  /**
+   * @dataProvider getDataModes
+   */
+  public function testEntityDisplayWithJoin(string $dataMode) {
 
     $lastName = uniqid(__FUNCTION__);
     $contacts = (array) $this->saveTestRecords('Individual', [
@@ -198,6 +223,7 @@ public function testEntityDisplayWithJoin() {
       ->addValue('label', 'MyNewEntityWithJoin')
       ->addValue('name', 'MyNewEntityWithJoin')
       ->addValue('settings', [
+        'data_mode' => $dataMode,
         // Additional column data will be filled in automatically
         // @see SKEntitySubscriber::formatFieldSpec
         'columns' => [
@@ -271,7 +297,10 @@ public function testEntityDisplayWithJoin() {
     $this->assertSame('Event_SK_MyNewEntityWithJoin_Contact_Participant_contact_id_01_event_id', $eventJoin[0]['alias']);
   }
 
-  public function testEntityWithSqlFunctions(): void {
+  /**
+   * @dataProvider getDataModes
+   */
+  public function testEntityWithSqlFunctions(string $dataMode): void {
     $cids = $this->saveTestRecords('Individual', [
       'records' => [
         ['first_name' => 'A', 'last_name' => 'A'],
@@ -324,6 +353,7 @@ public function testEntityWithSqlFunctions(): void {
       'saved_search_id.name' => 'Major_Donors_entity_test',
       'type' => 'entity',
       'settings' => [
+        'data_mode' => $dataMode,
         'sort' => [
           ['YEAR_receive_date', 'ASC'],
         ],
@@ -361,9 +391,9 @@ public function testEntityWithSqlFunctions(): void {
     $schema = \CRM_Core_DAO::executeQuery('DESCRIBE civicrm_sk_major_donors_entity_test_db_entity1')->fetchAll();
     $this->assertCount(6, $schema);
     $this->assertEquals('_row', $schema[0]['Field']);
-    $this->assertStringStartsWith('int', $schema[0]['Type']);
-    $this->assertStringStartsWith('int', $schema[1]['Type']);
-    $this->assertStringStartsWith('int', $schema[2]['Type']);
+    $this->assertMatchesRegularExpression('/^(int|bigint)/', $schema[0]['Type']);
+    $this->assertMatchesRegularExpression('/^(int|bigint)/', $schema[1]['Type']);
+    $this->assertMatchesRegularExpression('/^(int|bigint)/', $schema[2]['Type']);
     $this->assertStringStartsWith('text', $schema[3]['Type']);
     $this->assertStringStartsWith('decimal', $schema[4]['Type']);
     $this->assertStringStartsWith('text', $schema[3]['Type']);