Skip to content

Commit 37a9893

Browse files
ruflinruflin
committed
Merge pull request #853 from webdevsHub/reindex
Added Tool\CrossIndex class
2 parents 1eee419 + aad5fdc commit 37a9893

File tree

4 files changed

+297
-2
lines changed

4 files changed

+297
-2
lines changed

‎lib/Elastica/Tool/CrossIndex.php

+161
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
<?php
2+
3+
namespace Elastica\Tool;
4+
5+
use Elastica\Bulk;
6+
use Elastica\Index;
7+
use Elastica\Query\MatchAll;
8+
use Elastica\ScanAndScroll;
9+
use Elastica\Search;
10+
use Elastica\Type;
11+
12+
/**
13+
* Functions to move documents and types between indices
14+
*
15+
* @author Manuel Andreo Garcia <[email protected]>
16+
*/
17+
class CrossIndex
18+
{
19+
/**
20+
* Type option
21+
*
22+
* type: string | string[] | \Elastica\Type | \Elastica\Type[] | null
23+
* default: null (means all types)
24+
*/
25+
const OPTION_TYPE = 'type';
26+
27+
/**
28+
* Query option
29+
*
30+
* type: see \Elastica\Query::create()
31+
* default: Elastica\Query\MatchAll
32+
*/
33+
const OPTION_QUERY = 'query';
34+
35+
/**
36+
* Expiry time option
37+
*
38+
* type: string (see Elastica\ScanAndScroll)
39+
* default: '1m'
40+
*/
41+
const OPTION_EXPIRY_TIME = 'expiryTime';
42+
43+
/**
44+
* Size per shard option
45+
*
46+
* type: int (see Elastica\ScanAndScroll)
47+
* default: 1000
48+
*/
49+
const OPTION_SIZE_PER_SHARD = 'sizePerShard';
50+
51+
/**
52+
* Reindex documents from an old index to a new index
53+
*
54+
* @link https://www.elastic.co/guide/en/elasticsearch/guide/master/reindex.html
55+
*
56+
* @param \Elastica\Index $oldIndex
57+
* @param \Elastica\Index $newIndex
58+
* @param array $options keys: CrossIndex::OPTION_* constants
59+
*
60+
* @return \Elastica\Index The new index object
61+
*/
62+
public static function reindex(
63+
Index $oldIndex,
64+
Index $newIndex,
65+
array $options = array()
66+
) {
67+
// prepare search
68+
$search = new Search($oldIndex->getClient());
69+
70+
$options = array_merge(
71+
array(
72+
self::OPTION_TYPE => null,
73+
self::OPTION_QUERY => new MatchAll(),
74+
self::OPTION_EXPIRY_TIME => '1m',
75+
self::OPTION_SIZE_PER_SHARD => 1000,
76+
),
77+
$options
78+
);
79+
80+
$search->addIndex($oldIndex);
81+
if (isset($options[self::OPTION_TYPE])) {
82+
$type = $options[self::OPTION_TYPE];
83+
$search->addTypes(is_array($type) ? $type : array($type));
84+
}
85+
$search->setQuery($options[self::OPTION_QUERY]);
86+
87+
// search on old index and bulk insert in new index
88+
$scanAndScroll = new ScanAndScroll(
89+
$search,
90+
$options[self::OPTION_EXPIRY_TIME],
91+
$options[self::OPTION_SIZE_PER_SHARD]
92+
);
93+
foreach ($scanAndScroll as $resultSet) {
94+
$bulk = new Bulk($newIndex->getClient());
95+
$bulk->setIndex($newIndex);
96+
97+
foreach ($resultSet as $result) {
98+
$action = new Bulk\Action();
99+
$action->setType($result->getType());
100+
$action->setId($result->getId());
101+
$action->setSource($result->getData());
102+
103+
$bulk->addAction($action);
104+
}
105+
106+
$bulk->send();
107+
}
108+
109+
$newIndex->refresh();
110+
111+
return $newIndex;
112+
}
113+
114+
/**
115+
* Copies type mappings and documents from an old index to a new index
116+
*
117+
* @see \Elastica\Tool\CrossIndex::reindex()
118+
*
119+
* @param \Elastica\Index $oldIndex
120+
* @param \Elastica\Index $newIndex
121+
* @param array $options keys: CrossIndex::OPTION_* constants
122+
*
123+
* @return \Elastica\Index The new index object
124+
*/
125+
public static function copy(
126+
Index $oldIndex,
127+
Index $newIndex,
128+
array $options = array()
129+
) {
130+
// normalize types to array of string
131+
$types = array();
132+
if (isset($options[self::OPTION_TYPE])) {
133+
$types = $options[self::OPTION_TYPE];
134+
$types = is_array($types) ? $types : array($types);
135+
136+
$types = array_map(
137+
function ($type) {
138+
if ($type instanceof Type) {
139+
$type = $type->getName();
140+
}
141+
142+
return (string) $type;
143+
},
144+
$types
145+
);
146+
}
147+
148+
// copy mapping
149+
foreach ($oldIndex->getMapping() as $type => $mapping) {
150+
if (!empty($types) && !in_array($type, $types, true)) {
151+
continue;
152+
}
153+
154+
$type = new Type($newIndex, $type);
155+
$type->setMapping($mapping['properties']);
156+
}
157+
158+
// copy documents
159+
return self::reindex($oldIndex, $newIndex, $options);
160+
}
161+
}

‎lib/Elastica/Util.php

-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
namespace Elastica;
44

5-
use Elastica\Bulk\Action;
65
/**
76
* Elastica tools
87
*

‎test/lib/Elastica/Test/Tool/CrossIndexTest.php

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
<?php
2+
3+
namespace Elastica\Test\Tool;
4+
5+
use Elastica\Document;
6+
use Elastica\Test\Base;
7+
use Elastica\Tool\CrossIndex;
8+
use Elastica\Type;
9+
10+
class CrossIndexTest extends Base
11+
{
12+
/**
13+
* Test default reindex
14+
*/
15+
public function testReindex()
16+
{
17+
$oldIndex = $this->_createIndex(null, true, 2);
18+
$this->_addDocs($oldIndex->getType('crossIndexTest'), 10);
19+
20+
$newIndex = $this->_createIndex(null, true, 2);
21+
22+
$this->assertInstanceOf(
23+
'Elastica\Index',
24+
CrossIndex::reindex($oldIndex, $newIndex)
25+
);
26+
27+
$this->assertEquals(10, $newIndex->count());
28+
}
29+
30+
/**
31+
* Test reindex type option
32+
*/
33+
public function testReindexTypeOption()
34+
{
35+
$oldIndex = $this->_createIndex('', true, 2);
36+
$type1 = $oldIndex->getType('crossIndexTest_1');
37+
$type2 = $oldIndex->getType('crossIndexTest_2');
38+
39+
$docs1 = $this->_addDocs($type1, 10);
40+
$docs2 = $this->_addDocs($type2, 10);
41+
42+
$newIndex = $this->_createIndex(null, true, 2);
43+
44+
// \Elastica\Type
45+
CrossIndex::reindex($oldIndex, $newIndex, array(
46+
CrossIndex::OPTION_TYPE => $type1,
47+
));
48+
$this->assertEquals(10, $newIndex->count());
49+
$newIndex->deleteDocuments($docs1);
50+
51+
// string
52+
CrossIndex::reindex($oldIndex, $newIndex, array(
53+
CrossIndex::OPTION_TYPE => 'crossIndexTest_2',
54+
));
55+
$this->assertEquals(10, $newIndex->count());
56+
$newIndex->deleteDocuments($docs2);
57+
58+
// array
59+
CrossIndex::reindex($oldIndex, $newIndex, array(
60+
CrossIndex::OPTION_TYPE => array(
61+
'crossIndexTest_1',
62+
$type2,
63+
),
64+
));
65+
$this->assertEquals(20, $newIndex->count());
66+
}
67+
68+
/**
69+
* Test default copy
70+
*/
71+
public function testCopy()
72+
{
73+
$oldIndex = $this->_createIndex(null, true, 2);
74+
$newIndex = $this->_createIndex(null, true, 2);
75+
76+
$oldType = $oldIndex->getType('copy_test');
77+
$oldMapping = array(
78+
'name' => array(
79+
'type' => 'string',
80+
'store' => true,
81+
),
82+
);
83+
$oldType->setMapping($oldMapping);
84+
$docs = $this->_addDocs($oldType, 10);
85+
86+
// mapping
87+
$this->assertInstanceOf(
88+
'Elastica\Index',
89+
CrossIndex::copy($oldIndex, $newIndex)
90+
);
91+
92+
$newMapping = $newIndex->getType('copy_test')->getMapping();
93+
if (!isset($newMapping['copy_test']['properties']['name'])) {
94+
$this->fail('could not request new mapping');
95+
}
96+
97+
$this->assertEquals(
98+
$oldMapping['name'],
99+
$newMapping['copy_test']['properties']['name']
100+
);
101+
102+
// document copy
103+
$this->assertEquals(10, $newIndex->count());
104+
$newIndex->deleteDocuments($docs);
105+
106+
// ignore mapping
107+
$ignoredType = $oldIndex->getType('copy_test_1');
108+
$this->_addDocs($ignoredType, 10);
109+
110+
CrossIndex::copy($oldIndex, $newIndex, array(
111+
CrossIndex::OPTION_TYPE => $oldType,
112+
));
113+
114+
$this->assertFalse($newIndex->getType($ignoredType->getName())->exists());
115+
$this->assertEquals(10, $newIndex->count());
116+
}
117+
118+
/**
119+
* @param Type $type
120+
* @param int $docs
121+
*
122+
* @return array
123+
*/
124+
private function _addDocs(Type $type, $docs)
125+
{
126+
$insert = array();
127+
for ($i = 1; $i <= $docs; $i++) {
128+
$insert[] = new Document($i, array('_id' => $i, 'key' => 'value'));
129+
}
130+
131+
$type->addDocuments($insert);
132+
$type->getIndex()->refresh();
133+
134+
return $insert;
135+
}
136+
}

‎test/lib/Elastica/Test/UtilTest.php

-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
use Elastica\Request;
77
use Elastica\Test\Base as BaseTest;
88
use Elastica\Util;
9-
use Elastica\Exception\ResponseException;
109

1110
class UtilTest extends BaseTest
1211
{

0 commit comments

Comments
 (0)