Merge pull request #6590 from magento-performance/MCP-180

MCP-180: elastic index improvements

app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php

@@ -43,10 +43,10 @@ class BatchSizeCalculator
     private const DEPLOYMENT_CONFIG_INDEXER_BATCHES = 'indexer/batch_size/';
 
     /**
-     * BatchSizeCalculator constructor.
      * @param array $batchRowsCount
      * @param array $estimators
      * @param array $batchSizeAdjusters
+     * @param DeploymentConfig|null $deploymentConfig
      */
     public function __construct(
         array $batchRowsCount,

app/code/Magento/CatalogSearch/Model/Indexer/Fulltext.php

@@ -111,7 +111,7 @@ class Fulltext implements
      * @param array $data
      * @param ProcessManager|null $processManager
      * @param int|null $batchSize
-     * @param DeploymentConfig $deploymentConfig
+     * @param DeploymentConfig|null $deploymentConfig
      * @SuppressWarnings(PHPMD.UnusedFormalParameter)
      */
     public function __construct(
@@ -182,7 +182,7 @@ public function executeByDimensions(array $dimensions, \Traversable $entityIds =
             $i = 0;
 
             $this->batchSize = $this->deploymentConfig->get(
-                self::DEPLOYMENT_CONFIG_INDEXER_BATCHES . self::INDEXER_ID
+                self::DEPLOYMENT_CONFIG_INDEXER_BATCHES . self::INDEXER_ID . '/partial_reindex'
             ) ?? $this->batchSize;
 
             foreach ($entityIds as $entityId) {

app/code/Magento/CatalogSearch/Model/Indexer/Fulltext/Action/Full.php

@@ -7,6 +7,7 @@
 
 use Magento\Catalog\Api\Data\ProductInterface;
 use Magento\CatalogSearch\Model\Indexer\Fulltext;
+use Magento\Framework\App\DeploymentConfig;
 use Magento\Framework\App\ObjectManager;
 use Magento\Framework\App\ResourceConnection;
 
@@ -199,6 +200,19 @@ class Full
     private $batchSize;
 
     /**
+     * @var DeploymentConfig|null
+     */
+    private $deploymentConfig;
+
+    /**
+     * Deployment config path
+     *
+     * @var string
+     */
+    private const DEPLOYMENT_CONFIG_INDEXER_BATCHES = 'indexer/batch_size/';
+
+    /**
+     * Full constructor.
      * @param ResourceConnection $resource
      * @param \Magento\Catalog\Model\Product\Type $catalogProductType
      * @param \Magento\Eav\Model\Config $eavConfig
@@ -216,10 +230,12 @@ class Full
      * @param \Magento\CatalogSearch\Model\ResourceModel\Fulltext $fulltextResource
      * @param \Magento\Framework\Search\Request\DimensionFactory $dimensionFactory
      * @param \Magento\Framework\Indexer\ConfigInterface $indexerConfig
-     * @param mixed $indexIteratorFactory
+     * @param null $indexIteratorFactory
      * @param \Magento\Framework\EntityManager\MetadataPool|null $metadataPool
      * @param DataProvider|null $dataProvider
      * @param int $batchSize
+     * @param DeploymentConfig|null $deploymentConfig
+     *
      * @SuppressWarnings(PHPMD.ExcessiveParameterList)
      * @SuppressWarnings(PHPMD.UnusedFormalParameter)
      */
@@ -244,7 +260,8 @@ public function __construct(
         $indexIteratorFactory = null,
         \Magento\Framework\EntityManager\MetadataPool $metadataPool = null,
         DataProvider $dataProvider = null,
-        $batchSize = 500
+        $batchSize = 500,
+        ?DeploymentConfig $deploymentConfig = null
     ) {
         $this->resource = $resource;
         $this->connection = $resource->getConnection();
@@ -268,6 +285,7 @@ public function __construct(
             ->get(\Magento\Framework\EntityManager\MetadataPool::class);
         $this->dataProvider = $dataProvider ?: ObjectManager::getInstance()->get(DataProvider::class);
         $this->batchSize = $batchSize;
+        $this->deploymentConfig = $deploymentConfig ?: ObjectManager::getInstance()->get(DeploymentConfig::class);
     }
 
     /**
@@ -360,6 +378,9 @@ public function rebuildStoreIndex($storeId, $productIds = null)
         ];
 
         $lastProductId = 0;
+        $this->batchSize = $this->deploymentConfig->get(
+            self::DEPLOYMENT_CONFIG_INDEXER_BATCHES . Fulltext::INDEXER_ID . '/mysql_get'
+        ) ?? $this->batchSize;
         $products = $this->dataProvider
             ->getSearchableProducts($storeId, $staticFields, $productIds, $lastProductId, $this->batchSize);
         while (count($products) > 0) {

app/code/Magento/Elasticsearch/Model/Indexer/IndexerHandler.php

@@ -5,8 +5,11 @@
  */
 namespace Magento\Elasticsearch\Model\Indexer;
 
+use Magento\CatalogSearch\Model\Indexer\Fulltext;
 use Magento\Elasticsearch\Model\Adapter\Elasticsearch as ElasticsearchAdapter;
 use Magento\Elasticsearch\Model\Adapter\Index\IndexNameResolver;
+use Magento\Framework\App\DeploymentConfig;
+use Magento\Framework\App\ObjectManager;
 use Magento\Framework\App\ScopeResolverInterface;
 use Magento\Framework\Indexer\IndexStructureInterface;
 use Magento\Framework\Indexer\SaveHandler\Batch;
@@ -59,13 +62,27 @@ class IndexerHandler implements IndexerInterface
     private $scopeResolver;
 
     /**
+     * @var DeploymentConfig|null
+     */
+    private $deploymentConfig;
+
+    /**
+     * Deployment config path
+     *
+     * @var string
+     */
+    private const DEPLOYMENT_CONFIG_INDEXER_BATCHES = 'indexer/batch_size/';
+
+    /**
+     * IndexerHandler constructor.
      * @param IndexStructureInterface $indexStructure
      * @param ElasticsearchAdapter $adapter
      * @param IndexNameResolver $indexNameResolver
      * @param Batch $batch
      * @param ScopeResolverInterface $scopeResolver
      * @param array $data
      * @param int $batchSize
+     * @param DeploymentConfig|null $deploymentConfig
      */
     public function __construct(
         IndexStructureInterface $indexStructure,
@@ -74,7 +91,8 @@ public function __construct(
         Batch $batch,
         ScopeResolverInterface $scopeResolver,
         array $data = [],
-        $batchSize = self::DEFAULT_BATCH_SIZE
+        $batchSize = self::DEFAULT_BATCH_SIZE,
+        ?DeploymentConfig $deploymentConfig = null
     ) {
         $this->indexStructure = $indexStructure;
         $this->adapter = $adapter;
@@ -83,6 +101,7 @@ public function __construct(
         $this->data = $data;
         $this->batchSize = $batchSize;
         $this->scopeResolver = $scopeResolver;
+        $this->deploymentConfig = $deploymentConfig ?: ObjectManager::getInstance()->get(DeploymentConfig::class);
     }
 
     /**
@@ -92,6 +111,11 @@ public function saveIndex($dimensions, \Traversable $documents)
     {
         $dimension = current($dimensions);
         $scopeId = $this->scopeResolver->getScope($dimension->getValue())->getId();
+
+        $this->batchSize = $this->deploymentConfig->get(
+            self::DEPLOYMENT_CONFIG_INDEXER_BATCHES . Fulltext::INDEXER_ID . '/elastic_save'
+        ) ?? $this->batchSize;
+
         foreach ($this->batch->getItems($documents, $this->batchSize) as $documentsBatch) {
             $docs = $this->adapter->prepareDocsPerStore($documentsBatch, $scopeId);
             $this->adapter->addDocs($docs, $scopeId, $this->getIndexerId());

app/code/Magento/Indexer/Console/Command/IndexerReindexCommand.php

@@ -16,6 +16,7 @@
 use Magento\Framework\Indexer\IndexerRegistry;
 use Magento\Framework\Indexer\StateInterface;
 use Magento\Indexer\Model\Processor\MakeSharedIndexValid;
+use Psr\Log\LoggerInterface;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 
@@ -50,21 +51,29 @@ class IndexerReindexCommand extends AbstractIndexerManageCommand
      */
     private $makeSharedValid;
 
+    /**
+     * @var LoggerInterface
+     */
+    private $logger;
+
     /**
      * @param ObjectManagerFactory $objectManagerFactory
      * @param IndexerRegistry|null $indexerRegistry
      * @param DependencyInfoProvider|null $dependencyInfoProvider
      * @param MakeSharedIndexValid|null $makeSharedValid
+     * @param LoggerInterface|null $logger
      */
     public function __construct(
         ObjectManagerFactory $objectManagerFactory,
         IndexerRegistry $indexerRegistry = null,
         DependencyInfoProvider $dependencyInfoProvider = null,
-        MakeSharedIndexValid $makeSharedValid = null
+        MakeSharedIndexValid $makeSharedValid = null,
+        ?LoggerInterface $logger = null
     ) {
         $this->indexerRegistry = $indexerRegistry;
         $this->dependencyInfoProvider = $dependencyInfoProvider;
         $this->makeSharedValid = $makeSharedValid;
+        $this->logger = $logger ?: ObjectManager::getInstance()->get(LoggerInterface::class);
         parent::__construct($objectManagerFactory);
     }
 
@@ -108,15 +117,14 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 $output->writeln(
                     __('has been rebuilt successfully in %time', ['time' => gmdate('H:i:s', $resultTime)])
                 );
-            } catch (LocalizedException $e) {
-                $output->writeln(__('exception: %message', ['message' => $e->getMessage()]));
-                $returnValue = Cli::RETURN_FAILURE;
-            } catch (\Exception $e) {
-                $output->writeln('process unknown error:');
+            } catch (\Throwable $e) {
+                $output->writeln('process error during indexation process:');
                 $output->writeln($e->getMessage());
 
                 $output->writeln($e->getTraceAsString(), OutputInterface::VERBOSITY_DEBUG);
                 $returnValue = Cli::RETURN_FAILURE;
+
+                $this->logger->critical($e->getMessage());
             }
         }
 

app/code/Magento/Indexer/Test/Unit/Console/Command/IndexerReindexCommandTest.php

@@ -422,27 +422,6 @@ public function executeWithIndexDataProvider()
         ];
     }
 
-    public function testExecuteWithLocalizedException()
-    {
-        $this->configureAdminArea();
-        $indexerOne = $this->getIndexerMock(
-            ['reindexAll', 'getStatus'],
-            ['indexer_id' => 'indexer_1', 'title' => self::STUB_INDEXER_NAME]
-        );
-        $localizedException = new LocalizedException(new Phrase('Some Exception Message'));
-        $indexerOne->expects($this->once())->method('reindexAll')->willThrowException($localizedException);
-        $this->initIndexerCollectionByItems([$indexerOne]);
-        $this->command = new IndexerReindexCommand($this->objectManagerFactory);
-        $commandTester = new CommandTester($this->command);
-        $commandTester->execute(['index' => ['indexer_1']]);
-        $actualValue = $commandTester->getDisplay();
-        $this->assertSame(Cli::RETURN_FAILURE, $commandTester->getStatusCode());
-        $this->assertStringStartsWith(
-            self::STUB_INDEXER_NAME . ' index exception: Some Exception Message',
-            $actualValue
-        );
-    }
-
     public function testExecuteWithException()
     {
         $this->configureAdminArea();
@@ -459,7 +438,10 @@ public function testExecuteWithException()
         $commandTester->execute(['index' => ['indexer_1']]);
         $actualValue = $commandTester->getDisplay();
         $this->assertSame(Cli::RETURN_FAILURE, $commandTester->getStatusCode());
-        $this->assertStringStartsWith('Title_indexer_1' . ' index process unknown error:', $actualValue);
+        $this->assertStringStartsWith(
+            'Title_indexer_1' . ' index process error during indexation process:',
+            $actualValue
+        );
     }
 
     public function testExecuteWithExceptionInGetIndexers()