Skip to content

Commit 8c50ee1

Browse files
risdenkrisdenk
committed
[Solr] Add Solr 6 binding
1 parent a13fc48 commit 8c50ee1

File tree

20 files changed

+967
-14
lines changed

20 files changed

+967
-14
lines changed

‎bin/ycsb

+2-1
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,8 @@ DATABASES = {
8383
"redis" : "com.yahoo.ycsb.db.RedisClient",
8484
"riak" : "com.yahoo.ycsb.db.riak.RiakKVClient",
8585
"s3" : "com.yahoo.ycsb.db.S3Client",
86-
"solr" : "com.yahoo.ycsb.db.SolrClient",
86+
"solr" : "com.yahoo.ycsb.db.solr.SolrClient",
87+
"solr6" : "com.yahoo.ycsb.db.solr6.SolrClient",
8788
"tarantool" : "com.yahoo.ycsb.db.TarantoolClient",
8889
"voldemort" : "com.yahoo.ycsb.db.VoldemortClient"
8990
}

‎distribution/pom.xml

+5
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,11 @@ LICENSE file.
179179
<artifactId>solr-binding</artifactId>
180180
<version>${project.version}</version>
181181
</dependency>
182+
<dependency>
183+
<groupId>com.yahoo.ycsb</groupId>
184+
<artifactId>solr6-binding</artifactId>
185+
<version>${project.version}</version>
186+
</dependency>
182187
<dependency>
183188
<groupId>com.yahoo.ycsb</groupId>
184189
<artifactId>tarantool-binding</artifactId>

‎pom.xml

+2
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ LICENSE file.
9595
<riak.version>2.0.5</riak.version>
9696
<aerospike.version>3.1.2</aerospike.version>
9797
<solr.version>5.4.0</solr.version>
98+
<solr6.version>6.2.0</solr6.version>
9899
<arangodb.version>2.7.3</arangodb.version>
99100
</properties>
100101

@@ -133,6 +134,7 @@ LICENSE file.
133134
<module>riak</module>
134135
<module>s3</module>
135136
<module>solr</module>
137+
<module>solr6</module>
136138
<module>tarantool</module>
137139
<!--<module>voldemort</module>-->
138140
</modules>

‎solr/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ LICENSE file.
4646
<dependency>
4747
<groupId>org.slf4j</groupId>
4848
<artifactId>slf4j-log4j12</artifactId>
49-
<version>1.7.10</version>
49+
<version>1.7.21</version>
5050
</dependency>
5151

5252
<dependency>

‎solr/src/main/java/com/yahoo/ycsb/db/SolrClient.java renamed to ‎solr/src/main/java/com/yahoo/ycsb/db/solr/SolrClient.java

+2-6
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* LICENSE file.
1616
*/
1717

18-
package com.yahoo.ycsb.db;
18+
package com.yahoo.ycsb.db.solr;
1919

2020
import com.yahoo.ycsb.ByteIterator;
2121
import com.yahoo.ycsb.DB;
@@ -34,12 +34,8 @@
3434
import org.apache.solr.common.SolrInputDocument;
3535

3636
import java.io.IOException;
37-
import java.util.Collections;
38-
import java.util.HashMap;
37+
import java.util.*;
3938
import java.util.Map.Entry;
40-
import java.util.Properties;
41-
import java.util.Set;
42-
import java.util.Vector;
4339

4440
/**
4541
* Solr client for YCSB framework.

‎solr/src/main/java/com/yahoo/ycsb/db/package-info.java renamed to ‎solr/src/main/java/com/yahoo/ycsb/db/solr/package-info.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,5 @@
1919
* The YCSB binding for
2020
* <a href="http://lucene.apache.org/solr/">Solr</a>.
2121
*/
22-
package com.yahoo.ycsb.db;
22+
package com.yahoo.ycsb.db.solr;
2323

‎solr/src/test/java/com/yahoo/ycsb/db/SolrClientBaseTest.java renamed to ‎solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientBaseTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* LICENSE file.
1616
*/
1717

18-
package com.yahoo.ycsb.db;
18+
package com.yahoo.ycsb.db.solr;
1919

2020
import com.yahoo.ycsb.ByteIterator;
2121
import com.yahoo.ycsb.DB;

‎solr/src/test/java/com/yahoo/ycsb/db/SolrClientCloudTest.java renamed to ‎solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientCloudTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* permissions and limitations under the License. See accompanying
1515
* LICENSE file.
1616
*/
17-
package com.yahoo.ycsb.db;
17+
package com.yahoo.ycsb.db.solr;
1818

1919
import com.yahoo.ycsb.DB;
2020
import org.junit.After;

‎solr/src/test/java/com/yahoo/ycsb/db/SolrClientTest.java renamed to ‎solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientTest.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,13 @@
1414
* permissions and limitations under the License. See accompanying
1515
* LICENSE file.
1616
*/
17-
package com.yahoo.ycsb.db;
17+
package com.yahoo.ycsb.db.solr;
1818

1919
import com.yahoo.ycsb.DB;
2020
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
2121
import org.junit.After;
2222

23-
import java.net.URL;
2423
import java.util.Properties;
25-
import java.util.Set;
2624

2725
import static org.junit.Assume.assumeNoException;
2826

‎solr6/README.md

+110
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
<!--
2+
Copyright (c) 2016 YCSB contributors. All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License"); you
5+
may not use this file except in compliance with the License. You
6+
may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13+
implied. See the License for the specific language governing
14+
permissions and limitations under the License. See accompanying
15+
LICENSE file.
16+
-->
17+
18+
## Quick Start
19+
20+
This section describes how to run YCSB on Solr running locally.
21+
22+
### 1. Set Up YCSB
23+
24+
Clone the YCSB git repository and compile:
25+
26+
git clone git://github.com/brianfrankcooper/YCSB.git
27+
cd YCSB
28+
mvn -pl com.yahoo.ycsb:solr6-binding -am clean package
29+
30+
### 2. Set Up Solr
31+
32+
There must be a running Solr instance with a core/collection pre-defined and configured.
33+
- See this [API](https://cwiki.apache.org/confluence/display/solr/CoreAdmin+API#CoreAdminAPI-CREATE) reference on how to create a core.
34+
- See this [API](https://cwiki.apache.org/confluence/display/solr/Collections+API#CollectionsAPI-api1) reference on how to create a collection in SolrCloud mode.
35+
36+
The `conf/schema.xml` configuration file present in the core/collection just created must be configured to handle the expected field names during benchmarking.
37+
Below illustrates a sample from a schema config file that matches the default field names used by the ycsb client:
38+
39+
<field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false"/>
40+
<field name="field0" type="text_general" indexed="true" stored="true"/>
41+
<field name="field1" type="text_general" indexed="true" stored="true"/>
42+
<field name="field2" type="text_general" indexed="true" stored="true"/>
43+
<field name="field3" type="text_general" indexed="true" stored="true"/>
44+
<field name="field4" type="text_general" indexed="true" stored="true"/>
45+
<field name="field5" type="text_general" indexed="true" stored="true"/>
46+
<field name="field6" type="text_general" indexed="true" stored="true"/>
47+
<field name="field7" type="text_general" indexed="true" stored="true"/>
48+
<field name="field8" type="text_general" indexed="true" stored="true"/>
49+
<field name="field9" type="text_general" indexed="true" stored="true"/>
50+
51+
If running in SolrCloud mode ensure there is an external Zookeeper cluster running.
52+
- See [here](https://cwiki.apache.org/confluence/display/solr/Setting+Up+an+External+ZooKeeper+Ensemble) for details on how to set up an external Zookeeper cluster.
53+
- See [here](https://cwiki.apache.org/confluence/display/solr/Using+ZooKeeper+to+Manage+Configuration+Files) for instructions on how to use Zookeeper to manage your core/collection configuration files.
54+
55+
### 3. Run YCSB
56+
57+
Now you are ready to run! First, load the data:
58+
59+
./bin/ycsb load solr6 -s -P workloads/workloada -p table=<core/collection name>
60+
61+
Then, run the workload:
62+
63+
./bin/ycsb run solr6 -s -P workloads/workloada -p table=<core/collection name>
64+
65+
For further configuration see below:
66+
67+
### Default Configuration Parameters
68+
The default settings for the Solr node that is created is as follows:
69+
70+
- `solr.cloud`
71+
- A Boolean value indicating if Solr is running in SolrCloud mode. If so there must be an external Zookeeper cluster running also.
72+
- Default value is `false` and therefore expects solr to be running in stand-alone mode.
73+
74+
- `solr.base.url`
75+
- The base URL in which to interface with a running Solr instance in stand-alone mode
76+
- Default value is `http://localhost:8983/solr
77+
78+
- `solr.commit.within.time`
79+
- The max time in ms to wait for a commit when in batch mode, ignored otherwise
80+
- Default value is `1000ms`
81+
82+
- `solr.batch.mode`
83+
- Indicates if inserts/updates/deletes should be commited in batches (frequency controlled by the `solr.commit.within.time` parameter) or commit 1 document at a time.
84+
- Default value is `false`
85+
86+
- `solr.zookeeper.hosts`
87+
- A list of comma seperated host:port pairs of Zookeeper nodes used to manage SolrCloud configurations.
88+
- Must be passed when in [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) mode.
89+
- Default value is `localhost:2181`
90+
91+
### Custom Configuration
92+
If you wish to customize the settings used to create the Solr node
93+
you can created a new property file that contains your desired Solr
94+
node settings and pass it in via the parameter to 'bin/ycsb' script. Note that
95+
the default properties will be kept if you don't explicitly overwrite them.
96+
97+
Assuming that we have a properties file named "myproperties.data" that contains
98+
custom Solr node configuration you can execute the following to
99+
pass it into the Solr client:
100+
101+
./bin/ycsb run solr6 -P workloads/workloada -P myproperties.data -s
102+
103+
If you wish to use SolrCloud mode ensure a Solr cluster is running with an
104+
external zookeeper cluster and an appropriate collection has been created.
105+
Make sure to pass the following properties as parameters to 'bin/ycsb' script.
106+
107+
solr.cloud=true
108+
solr.zookeeper.hosts=<zkHost2>:<zkPort1>,...,<zkHostN>:<zkPortN>
109+
110+

‎solr6/pom.xml

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Copyright (c) 2012 - 2016 YCSB contributors.
4+
All rights reserved.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License"); you
7+
may not use this file except in compliance with the License. You
8+
may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15+
implied. See the License for the specific language governing
16+
permissions and limitations under the License. See accompanying
17+
LICENSE file.
18+
-->
19+
20+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
21+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
22+
<modelVersion>4.0.0</modelVersion>
23+
<parent>
24+
<groupId>com.yahoo.ycsb</groupId>
25+
<artifactId>binding-parent</artifactId>
26+
<version>0.12.0-SNAPSHOT</version>
27+
<relativePath>../binding-parent</relativePath>
28+
</parent>
29+
30+
<artifactId>solr6-binding</artifactId>
31+
<name>Solr 6 Binding</name>
32+
<packaging>jar</packaging>
33+
34+
<dependencies>
35+
<dependency>
36+
<groupId>com.yahoo.ycsb</groupId>
37+
<artifactId>core</artifactId>
38+
<version>${project.version}</version>
39+
<scope>provided</scope>
40+
</dependency>
41+
<dependency>
42+
<groupId>org.apache.solr</groupId>
43+
<artifactId>solr-solrj</artifactId>
44+
<version>${solr6.version}</version>
45+
</dependency>
46+
<dependency>
47+
<groupId>org.slf4j</groupId>
48+
<artifactId>slf4j-log4j12</artifactId>
49+
<version>1.7.21</version>
50+
</dependency>
51+
52+
<dependency>
53+
<groupId>org.apache.solr</groupId>
54+
<artifactId>solr-test-framework</artifactId>
55+
<version>${solr6.version}</version>
56+
<scope>test</scope>
57+
</dependency>
58+
</dependencies>
59+
</project>

0 commit comments

Comments
 (0)