Skip to content

Commit 8e2d474

Browse files
giovannialbero1992ruflin
giovannialbero1992
authored and
ruflin
committed
Added Reindex and deprecated CrossIndex (#1311) (#1315)
1 parent d9c59b5 commit 8e2d474

File tree

4 files changed

+324
-1
lines changed

4 files changed

+324
-1
lines changed

‎CHANGELOG.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,14 @@ All notable changes to this project will be documented in this file based on the
1616
- Add support for Health parameters for Cluster\Health endpoint (new prop : delayed_unassigned_shards, number_of_pending_tasks, number_of_in_flight_fetch, task_max_waiting_in_queue_millis, active_shards_percent_as_number)
1717
- Add support for querystring in Type. this allow to use `update_all_types` in type mapping in order to resolve conflicts between fields in different types. [Conflicts between fields in different types](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-put-mapping.html#merging-conflicts)
1818
- Added `\Elastica\Query\ParentId` to avoid join with parent documents [#1287](https://github.com/ruflin/Elastica/issues/1287)
19+
- Added `\Elastica\Reindex` for reindexing between indices [#1311](https://github.com/ruflin/Elastica/issues/1311)
1920

2021
### Improvements
2122

2223
- Added support for `other_bucket` and `other_bucket_key` paramters on `Elastica\Aggregation\Filters`
2324

2425
### Deprecated
25-
26+
- Deprecated `Tool\CrossIndex` use `\Elastica\Reindex` instead [#1311](https://github.com/ruflin/Elastica/issues/1311)
2627

2728
## [Unreleased](https://github.com/ruflin/Elastica/compare/5.1.0...5.2.0)
2829

‎lib/Elastica/Reindex.php

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
<?php
2+
namespace Elastica;
3+
4+
use Elastica\Query\AbstractQuery;
5+
6+
class Reindex
7+
{
8+
const VERSION_TYPE = 'version_type';
9+
const VERSION_TYPE_INTERNAL = 'internal';
10+
const VERSION_TYPE_EXTERNAL = 'external';
11+
const OPERATION_TYPE = 'op_type';
12+
const OPERATION_TYPE_CREATE = 'create';
13+
const CONFLICTS = 'conflicts';
14+
const CONFLICTS_PROCEED = 'proceed';
15+
const TYPE = 'type';
16+
const SIZE = 'size';
17+
const QUERY = 'query';
18+
19+
/**
20+
* @var Index
21+
*/
22+
protected $_oldIndex;
23+
24+
/**
25+
* @var Index
26+
*/
27+
protected $_newIndex;
28+
29+
/**
30+
* @var array
31+
*/
32+
protected $_options;
33+
34+
public function __construct(Index $oldIndex, Index $newIndex, array $options = [])
35+
{
36+
$this->_oldIndex = $oldIndex;
37+
$this->_newIndex = $newIndex;
38+
$this->_options = $options;
39+
}
40+
41+
public function run()
42+
{
43+
$body = $this->_getBody($this->_oldIndex, $this->_newIndex, $this->_options);
44+
45+
$reindexEndpoint = new \Elasticsearch\Endpoints\Reindex();
46+
$reindexEndpoint->setBody($body);
47+
48+
$this->_oldIndex->getClient()->requestEndpoint($reindexEndpoint);
49+
$this->_newIndex->refresh();
50+
51+
return $this->_newIndex;
52+
}
53+
54+
protected function _getBody($oldIndex, $newIndex, $options)
55+
{
56+
$body = array_merge([
57+
'source' => $this->_getSourcePartBody($oldIndex, $options),
58+
'dest' => $this->_getDestPartBody($newIndex, $options)
59+
], $this->_resolveBodyOptions($options));
60+
61+
return $body;
62+
}
63+
64+
protected function _getSourcePartBody(Index $index, array $options)
65+
{
66+
$sourceBody = array_merge([
67+
'index' => $index->getName(),
68+
], $this->_resolveSourceOptions($options));
69+
70+
$sourceBody = $this->_setSourceQuery($sourceBody);
71+
$sourceBody = $this->_setSourceType($sourceBody);
72+
73+
return $sourceBody;
74+
}
75+
76+
protected function _getDestPartBody(Index $index, array $options)
77+
{
78+
return array_merge([
79+
'index' => $index->getName(),
80+
], $this->_resolveDestOptions($options));
81+
}
82+
83+
private function _resolveSourceOptions(array $options)
84+
{
85+
return array_intersect_key($options, [
86+
self::TYPE => null,
87+
self::QUERY => null,
88+
]);
89+
}
90+
91+
private function _resolveDestOptions(array $options)
92+
{
93+
return array_intersect_key($options, [
94+
self::VERSION_TYPE => null,
95+
self::OPERATION_TYPE => null,
96+
]);
97+
}
98+
99+
private function _resolveBodyOptions(array $options)
100+
{
101+
return array_intersect_key($options, [
102+
self::SIZE => null,
103+
self::CONFLICTS => null,
104+
]);
105+
}
106+
107+
/**
108+
* @param array $sourceBody
109+
*
110+
* @return array
111+
*/
112+
private function _setSourceQuery(array $sourceBody)
113+
{
114+
if (isset($sourceBody[self::QUERY]) && $sourceBody[self::QUERY] instanceof AbstractQuery) {
115+
$sourceBody[self::QUERY] = $sourceBody[self::QUERY]->toArray();
116+
}
117+
return $sourceBody;
118+
}
119+
120+
/**
121+
* @param array $sourceBody
122+
*
123+
* @return array
124+
*/
125+
private function _setSourceType(array $sourceBody)
126+
{
127+
if (isset($sourceBody[self::TYPE]) && !is_array($sourceBody[self::TYPE])) {
128+
$sourceBody[self::TYPE] = [$sourceBody[self::TYPE]];
129+
}
130+
if (isset($sourceBody[self::TYPE])) {
131+
foreach ($sourceBody[self::TYPE] as $key => $type) {
132+
if ($type instanceof Type) {
133+
$sourceBody[self::TYPE][$key] = $type->getName();
134+
}
135+
}
136+
}
137+
return $sourceBody;
138+
}
139+
}

‎lib/Elastica/Tool/CrossIndex.php

+2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
* Functions to move documents and types between indices.
1313
*
1414
* @author Manuel Andreo Garcia <[email protected]>
15+
*
16+
* @deprecated use Reindex instead. This class will be removed in further Elastica releases.
1517
*/
1618
class CrossIndex
1719
{

‎test/Elastica/ReindexTest.php

+181
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
<?php
2+
namespace Elastica\Test;
3+
4+
use Elastica\Document;
5+
use Elastica\Index;
6+
use Elastica\Query\Match;
7+
use Elastica\Reindex;
8+
use Elastica\Type;
9+
10+
class ReindexTest extends Base
11+
{
12+
/**
13+
* Test default reindex.
14+
*
15+
* @group functional
16+
*/
17+
public function testReindex()
18+
{
19+
$oldIndex = $this->_createIndex('idx1', true, 2);
20+
$this->_addDocs($oldIndex->getType('resetTest'), 10);
21+
22+
$newIndex = $this->_createIndex('idx2', true, 2);
23+
24+
$reindex = new Reindex($oldIndex, $newIndex);
25+
$this->assertInstanceOf(
26+
Index::class,
27+
$newIndex
28+
);
29+
$newIndex = $reindex->run();
30+
31+
$this->assertEquals(10, $newIndex->count());
32+
33+
$oldResult = [];
34+
35+
foreach ($oldIndex->search()->getResults() as $result) {
36+
$oldResult[] = $result->getData();
37+
}
38+
39+
$newResult = [];
40+
41+
foreach ($newIndex->search()->getResults() as $result) {
42+
$newResult[] = $result->getData();
43+
}
44+
45+
$this->assertEquals($oldResult, $newResult);
46+
}
47+
48+
/**
49+
* Test reindex type option.
50+
*
51+
* @group functional
52+
*/
53+
public function testReindexTypeOption()
54+
{
55+
$oldIndex = $this->_createIndex('', true, 2);
56+
$type1 = $oldIndex->getType('crossIndexTest_1');
57+
$type2 = $oldIndex->getType('crossIndexTest_2');
58+
59+
$docs1 = $this->_addDocs($type1, 10);
60+
$docs2 = $this->_addDocs($type2, 10);
61+
62+
$newIndex = $this->_createIndex(null, true, 2);
63+
64+
$reindex = new Reindex($oldIndex, $newIndex, [
65+
Reindex::TYPE => 'crossIndexTest_1',
66+
]);
67+
$reindex->run();
68+
69+
$this->assertEquals(10, $newIndex->count());
70+
$newIndex->deleteDocuments($docs1);
71+
72+
// string
73+
$reindex = new Reindex($oldIndex, $newIndex, [
74+
Reindex::TYPE => 'crossIndexTest_2',
75+
]);
76+
$reindex->run();
77+
$this->assertEquals(10, $newIndex->count());
78+
$newIndex->deleteDocuments($docs2);
79+
80+
// array
81+
$reindex = new Reindex($oldIndex, $newIndex, [
82+
Reindex::TYPE => [
83+
$type1,
84+
'crossIndexTest_2',
85+
],
86+
]);
87+
$reindex->run();
88+
$this->assertEquals(20, $newIndex->count());
89+
}
90+
91+
/**
92+
* @group functional
93+
*/
94+
public function testReindexOpTypeOptionWithProceedSetOnConflicts()
95+
{
96+
$oldIndex = $this->_createIndex('idx1', true, 2);
97+
$type1 = $oldIndex->getType('crossIndexTest_1');
98+
99+
$docs1 = $this->_addDocs($type1, 10);
100+
101+
$subDocs1 = array_splice($docs1, 0, 5);
102+
103+
$newIndex = $this->_createIndex('idx2', true, 2);
104+
$newIndex->addDocuments($subDocs1);
105+
$newIndex->refresh();
106+
107+
$this->assertEquals(5, $newIndex->count());
108+
109+
$reindex = new Reindex($oldIndex, $newIndex, [
110+
Reindex::OPERATION_TYPE => Reindex::OPERATION_TYPE_CREATE,
111+
Reindex::CONFLICTS => Reindex::CONFLICTS_PROCEED,
112+
]);
113+
114+
$reindex->run();
115+
116+
$this->assertEquals(10, $newIndex->count());
117+
}
118+
119+
/**
120+
* @group functional
121+
*/
122+
public function testReindexWithQueryOption()
123+
{
124+
$oldIndex = $this->_createIndex('idx1', true, 2);
125+
$type1 = $oldIndex->getType('crossIndexTest_1');
126+
$docs1 = $this->_addDocs($type1, 10);
127+
128+
$newIndex = $this->_createIndex('idx2', true, 2);
129+
130+
$query = new Match('id', 8);
131+
132+
$reindex = new Reindex($oldIndex, $newIndex, [
133+
Reindex::QUERY => $query,
134+
]);
135+
$reindex->run();
136+
137+
$results = $newIndex->search()->getResults();
138+
$this->assertEquals(1, $newIndex->count());
139+
foreach ($results as $result) {
140+
$this->assertEquals($docs1[7]->getData(), $result->getData());
141+
}
142+
}
143+
144+
/**
145+
* @group functional
146+
*/
147+
public function testReindexWithSizeOption()
148+
{
149+
$oldIndex = $this->_createIndex('idx1', true, 2);
150+
$type1 = $oldIndex->getType('crossIndexTest_1');
151+
$this->_addDocs($type1, 10);
152+
153+
$newIndex = $this->_createIndex('idx2', true, 2);
154+
155+
$reindex = new Reindex($oldIndex, $newIndex, [
156+
Reindex::SIZE => 5,
157+
]);
158+
$reindex->run();
159+
160+
$this->assertEquals(5, $newIndex->count());
161+
}
162+
163+
/**
164+
* @param Type $type
165+
* @param int $docs
166+
*
167+
* @return array
168+
*/
169+
private function _addDocs(Type $type, $docs)
170+
{
171+
$insert = [];
172+
for ($i = 1; $i <= $docs; ++$i) {
173+
$insert[] = new Document($i, ['id' => $i, 'key' => 'value']);
174+
}
175+
176+
$type->addDocuments($insert);
177+
$type->getIndex()->refresh();
178+
179+
return $insert;
180+
}
181+
}

0 commit comments

Comments
 (0)