From fa8516eab682e4262a2363dd245a824979488c61 Mon Sep 17 00:00:00 2001 From: Tseho Date: Wed, 9 Nov 2022 00:23:24 +0100 Subject: [PATCH 1/2] OCT-169: fix migration of ES index --- ...d_id_to_events_api_debug_index_mapping.php | 78 +++++++++++++++++++ ...ts_api_debug_index_mapping_Integration.php | 41 +++++++++- 2 files changed, 118 insertions(+), 1 deletion(-) diff --git a/upgrades/schema/Version_7_0_20221026154157_add_id_to_events_api_debug_index_mapping.php b/upgrades/schema/Version_7_0_20221026154157_add_id_to_events_api_debug_index_mapping.php index eb1393ee8499..eb45a6837573 100644 --- a/upgrades/schema/Version_7_0_20221026154157_add_id_to_events_api_debug_index_mapping.php +++ b/upgrades/schema/Version_7_0_20221026154157_add_id_to_events_api_debug_index_mapping.php @@ -9,6 +9,7 @@ use Elasticsearch\ClientBuilder; use Symfony\Component\DependencyInjection\ContainerAwareInterface; use Symfony\Component\DependencyInjection\ContainerInterface; +use Symfony\Component\Yaml\Yaml; use Webmozart\Assert\Assert; /** @@ -33,6 +34,11 @@ public function up(Schema $schema): void throw new \RuntimeException('Unable to retrieve existing mapping.'); } + if ('text' === (current($existingMapping)['mappings']['properties']['id']['type'] ?? '')) { + $this->reindexWhenThereIsAlreadyAnId(); + return; + } + $client->putMapping([ 'index' => $eventsApiDebugIndexName, 'body' => [ @@ -41,7 +47,79 @@ public function up(Schema $schema): void ], ], ]); + } + + private function reindexWhenThereIsAlreadyAnId(): void + { + $builder = $this->container->get('akeneo_elasticsearch.client_builder'); + $builder->setHosts([$this->container->getParameter('index_hosts')]); + /** @var \Elasticsearch\Client $client */ + $client = $builder->build(); + $alias = $this->container->getParameter('events_api_debug_index_name'); + $copy = sprintf('%s_copy', $alias); + $indice = array_keys($client->indices()->getAlias(['name' => $alias]))[0]; + + $mapping = $this->getMappingConfiguration(); + + $client->indices()->create([ + 'index' => $copy, + 'body' => $mapping, + ]); + + $client->reindex([ + 'refresh' => true, + 'body' => [ + 'source' => [ + 'index' => $indice, + ], + 'dest' => [ + 'index' => $copy, + ], + ], + ]); + + $client->indices()->putAlias([ + 'name' => $alias, + 'index' => $copy, + ]); + + $client->indices()->delete([ + 'index' => $indice, + ]); + + $client->indices()->create([ + 'index' => $indice, + 'body' => $mapping, + ]); + + $client->reindex([ + 'refresh' => true, + 'body' => [ + 'source' => [ + 'index' => $copy, + ], + 'dest' => [ + 'index' => $indice, + ], + ], + ]); + + $client->indices()->putAlias([ + 'name' => $alias, + 'index' => $indice, + ]); + + $client->indices()->delete([ + 'index' => $copy, + ]); + } + + private function getMappingConfiguration(): array + { + $ceDir = $this->container->getParameter('pim_ce_dev_src_folder_location'); + $path = "{$ceDir}/src/Akeneo/Connectivity/Connection/back/Infrastructure/Symfony/Resources/elasticsearch/events_api_debug_mapping.yml"; + return Yaml::parseFile($path); } public function down(Schema $schema): void diff --git a/upgrades/test_schema/Version_7_0_20221026154157_add_id_to_events_api_debug_index_mapping_Integration.php b/upgrades/test_schema/Version_7_0_20221026154157_add_id_to_events_api_debug_index_mapping_Integration.php index 40bc29cefe37..cf93e0abed28 100644 --- a/upgrades/test_schema/Version_7_0_20221026154157_add_id_to_events_api_debug_index_mapping_Integration.php +++ b/upgrades/test_schema/Version_7_0_20221026154157_add_id_to_events_api_debug_index_mapping_Integration.php @@ -4,10 +4,12 @@ namespace Pim\Upgrade\Schema\Tests; +use Akeneo\Connectivity\Connection\Domain\Webhook\Model\EventsApiDebugLogLevels; use Akeneo\Test\Integration\TestCase; use Akeneo\Tool\Bundle\ElasticsearchBundle\Client; use Akeneo\Tool\Bundle\ElasticsearchBundle\IndexConfiguration\Loader; use Elasticsearch\Client as NativeClient; +use Ramsey\Uuid\Uuid; use Symfony\Component\DependencyInjection\ParameterBag\ParameterBagInterface; use Symfony\Component\Yaml\Yaml; @@ -37,7 +39,7 @@ protected function setUp(): void $this->eventsApiDebugClient = $this->get('akeneo_connectivity.client.events_api_debug'); } - public function test_it_adds_the_entity_updated_property_to_the_mapping(): void + public function test_it_adds_the_id_property_to_the_mapping(): void { $this->recreateIndexWithoutIdFieldInTheMapping(); $properties = $this->getIndexMappingProperties(); @@ -50,6 +52,43 @@ public function test_it_adds_the_entity_updated_property_to_the_mapping(): void self::assertSame(['type' => 'keyword'], $properties['id']); } + public function test_it_changes_the_id_property_from_text_to_keyword(): void + { + $this->recreateIndexWithoutIdFieldInTheMapping(); + self::assertArrayNotHasKey('id', $this->getIndexMappingProperties()); + + $this->nativeClient->index([ + 'index' => self::getContainer()->getParameter('events_api_debug_index_name'), + 'body' => [ + 'id' => 'aa63292c-a06c-4c50-afb9-c98c97dc8a13', + 'timestamp' => '1667946703', + 'level' => 'notice', + 'message' => 'foobar', + 'connection_code' => 'foo', + 'context' => [], + ], + ]); + $this->nativeClient->indices()->refresh(); + self::assertSame('text', $this->getIndexMappingProperties()['id']['type']); + + $this->reExecuteMigration(self::MIGRATION_LABEL); + + $properties = $this->getIndexMappingProperties(); + self::assertSame('keyword', $properties['id']['type']); + + $documents = $this->nativeClient->search([ + 'index' => self::getContainer()->getParameter('events_api_debug_index_name'), + ]); + self::assertSame([ + 'id' => 'aa63292c-a06c-4c50-afb9-c98c97dc8a13', + 'timestamp' => '1667946703', + 'level' => 'notice', + 'message' => 'foobar', + 'connection_code' => 'foo', + 'context' => [], + ], $documents['hits']['hits'][0][('_source')]); + } + private function recreateIndexWithoutIdFieldInTheMapping(): void { $configFiles = $this->getParameter('elasticsearch_index_configuration_files'); From 6592223e395c1930ab5904a137106220fc99710f Mon Sep 17 00:00:00 2001 From: Tseho Date: Wed, 9 Nov 2022 09:32:48 +0100 Subject: [PATCH 2/2] OCT-169: apply same fix to connection error index --- upgrades/.php_cd.php | 1 + ...ield_to_connection_error_index_mapping.php | 79 +++++++++++++++++++ ...ection_error_index_mapping_Integration.php | 37 ++++++++- 3 files changed, 114 insertions(+), 3 deletions(-) diff --git a/upgrades/.php_cd.php b/upgrades/.php_cd.php index 6bf50969f23a..1dc3b13fc8a1 100644 --- a/upgrades/.php_cd.php +++ b/upgrades/.php_cd.php @@ -41,6 +41,7 @@ 'Symfony\Component\Console\Input\ArrayInput', 'Symfony\Component\Console\Output\BufferedOutput', 'Symfony\Component\DependencyInjection\ParameterBag\ParameterBag', + 'Symfony\Component\Yaml\Yaml', 'Webmozart\Assert\Assert', ] )->in('Pim\Upgrade\Schema'), diff --git a/upgrades/schema/Version_7_0_20221027152057_add_id_field_to_connection_error_index_mapping.php b/upgrades/schema/Version_7_0_20221027152057_add_id_field_to_connection_error_index_mapping.php index e50d4d205aa7..d917512e4c84 100644 --- a/upgrades/schema/Version_7_0_20221027152057_add_id_field_to_connection_error_index_mapping.php +++ b/upgrades/schema/Version_7_0_20221027152057_add_id_field_to_connection_error_index_mapping.php @@ -9,6 +9,7 @@ use Elasticsearch\ClientBuilder; use Symfony\Component\DependencyInjection\ContainerAwareInterface; use Symfony\Component\DependencyInjection\ContainerInterface; +use Symfony\Component\Yaml\Yaml; use Webmozart\Assert\Assert; /** @@ -33,6 +34,11 @@ public function up(Schema $schema) : void throw new \RuntimeException('Unable to retrieve existing mapping.'); } + if ('text' === (current($existingMapping)['mappings']['properties']['id']['type'] ?? '')) { + $this->reindexWhenThereIsAlreadyAnId(); + return; + } + $client->putMapping([ 'index' => $connectionErrorIndexName, 'body' => [ @@ -43,6 +49,79 @@ public function up(Schema $schema) : void ]); } + private function reindexWhenThereIsAlreadyAnId(): void + { + $builder = $this->container->get('akeneo_elasticsearch.client_builder'); + $builder->setHosts([$this->container->getParameter('index_hosts')]); + /** @var \Elasticsearch\Client $client */ + $client = $builder->build(); + $alias = $this->container->getParameter('connection_error_index_name'); + $copy = sprintf('%s_copy', $alias); + $indice = array_keys($client->indices()->getAlias(['name' => $alias]))[0]; + + $mapping = $this->getMappingConfiguration(); + + $client->indices()->create([ + 'index' => $copy, + 'body' => $mapping, + ]); + + $client->reindex([ + 'refresh' => true, + 'body' => [ + 'source' => [ + 'index' => $indice, + ], + 'dest' => [ + 'index' => $copy, + ], + ], + ]); + + $client->indices()->putAlias([ + 'name' => $alias, + 'index' => $copy, + ]); + + $client->indices()->delete([ + 'index' => $indice, + ]); + + $client->indices()->create([ + 'index' => $indice, + 'body' => $mapping, + ]); + + $client->reindex([ + 'refresh' => true, + 'body' => [ + 'source' => [ + 'index' => $copy, + ], + 'dest' => [ + 'index' => $indice, + ], + ], + ]); + + $client->indices()->putAlias([ + 'name' => $alias, + 'index' => $indice, + ]); + + $client->indices()->delete([ + 'index' => $copy, + ]); + } + + private function getMappingConfiguration(): array + { + $ceDir = $this->container->getParameter('pim_ce_dev_src_folder_location'); + $path = "{$ceDir}/src/Akeneo/Connectivity/Connection/back/Infrastructure/Symfony/Resources/elasticsearch/connection_error_mapping.yml"; + + return Yaml::parseFile($path); + } + public function down(Schema $schema) : void { $this->throwIrreversibleMigrationException(); diff --git a/upgrades/test_schema/Version_7_0_20221027152057_add_id_field_to_connection_error_index_mapping_Integration.php b/upgrades/test_schema/Version_7_0_20221027152057_add_id_field_to_connection_error_index_mapping_Integration.php index 471e1923977b..e4e568c94d27 100644 --- a/upgrades/test_schema/Version_7_0_20221027152057_add_id_field_to_connection_error_index_mapping_Integration.php +++ b/upgrades/test_schema/Version_7_0_20221027152057_add_id_field_to_connection_error_index_mapping_Integration.php @@ -37,10 +37,8 @@ protected function setUp(): void $this->connectionErrorClient = $this->get('akeneo_connectivity.client.connection_error'); } - /** @test */ - public function it_adds_the_entity_updated_property_to_the_mapping(): void + public function test_it_adds_the_id_property_to_the_mapping(): void { - $this->recreateConnectionErrorIndexWithoutIdInTheMapping(); $properties = $this->getMappingProperties(); self::assertArrayNotHasKey('id', $properties); @@ -52,6 +50,39 @@ public function it_adds_the_entity_updated_property_to_the_mapping(): void self::assertSame(['type' => 'keyword'], $properties['id']); } + public function test_it_changes_the_id_property_from_text_to_keyword(): void + { + $this->recreateConnectionErrorIndexWithoutIdInTheMapping(); + self::assertArrayNotHasKey('id', $this->getMappingProperties()); + + $this->nativeClient->index([ + 'index' => self::getContainer()->getParameter('connection_error_index_name'), + 'body' => [ + 'id' => 'aa63292c-a06c-4c50-afb9-c98c97dc8a13', + 'connection_code' => 'foo', + 'content' => [], + 'error_datetime' => '2021-01-03T02:30:00+01:00', + ], + ]); + $this->nativeClient->indices()->refresh(); + self::assertSame('text', $this->getMappingProperties()['id']['type']); + + $this->reExecuteMigration(self::MIGRATION_LABEL); + + $properties = $this->getMappingProperties(); + self::assertSame('keyword', $properties['id']['type']); + + $documents = $this->nativeClient->search([ + 'index' => self::getContainer()->getParameter('connection_error_index_name'), + ]); + self::assertSame([ + 'id' => 'aa63292c-a06c-4c50-afb9-c98c97dc8a13', + 'connection_code' => 'foo', + 'content' => [], + 'error_datetime' => '2021-01-03T02:30:00+01:00', + ], $documents['hits']['hits'][0][('_source')]); + } + private function recreateConnectionErrorIndexWithoutIdInTheMapping(): void { $configFiles = $this->getParameter('elasticsearch_index_configuration_files');