diff --git a/bin/bindings.properties b/bin/bindings.properties index 9afdc436aa..d75730c12e 100644 --- a/bin/bindings.properties +++ b/bin/bindings.properties @@ -58,7 +58,8 @@ rados:com.yahoo.ycsb.db.RadosClient redis:com.yahoo.ycsb.db.RedisClient riak:com.yahoo.ycsb.db.riak.RiakKVClient s3:com.yahoo.ycsb.db.S3Client -solr:com.yahoo.ycsb.db.SolrClient +solr:com.yahoo.ycsb.db.solr.SolrClient +solr6:com.yahoo.ycsb.db.solr6.SolrClient tarantool:com.yahoo.ycsb.db.TarantoolClient voldemort:com.yahoo.ycsb.db.VoldemortClient diff --git a/bin/ycsb b/bin/ycsb index 470fdd8d2a..e0f7092e62 100755 --- a/bin/ycsb +++ b/bin/ycsb @@ -83,7 +83,8 @@ DATABASES = { "redis" : "com.yahoo.ycsb.db.RedisClient", "riak" : "com.yahoo.ycsb.db.riak.RiakKVClient", "s3" : "com.yahoo.ycsb.db.S3Client", - "solr" : "com.yahoo.ycsb.db.SolrClient", + "solr" : "com.yahoo.ycsb.db.solr.SolrClient", + "solr6" : "com.yahoo.ycsb.db.solr6.SolrClient", "tarantool" : "com.yahoo.ycsb.db.TarantoolClient", "voldemort" : "com.yahoo.ycsb.db.VoldemortClient" } diff --git a/distribution/pom.xml b/distribution/pom.xml index d8af2145d8..ff3b111602 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -179,6 +179,11 @@ LICENSE file. solr-binding ${project.version} + + com.yahoo.ycsb + solr6-binding + ${project.version} + com.yahoo.ycsb tarantool-binding diff --git a/pom.xml b/pom.xml index 58db4afd81..510dace400 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,8 @@ LICENSE file. 1.6.5 2.0.5 3.1.2 - 5.5.3 + 5.4.0 + 6.2.1 2.7.3 4.0.0 @@ -135,6 +136,7 @@ LICENSE file. riak s3 solr + solr6 tarantool diff --git a/solr/pom.xml b/solr/pom.xml index 09ada0d856..8dd3e668be 100644 --- a/solr/pom.xml +++ b/solr/pom.xml @@ -51,7 +51,7 @@ LICENSE file. org.slf4j slf4j-log4j12 - 1.7.10 + 1.7.21 diff --git a/solr/src/main/java/com/yahoo/ycsb/db/SolrClient.java b/solr/src/main/java/com/yahoo/ycsb/db/solr/SolrClient.java similarity index 98% rename from solr/src/main/java/com/yahoo/ycsb/db/SolrClient.java rename to solr/src/main/java/com/yahoo/ycsb/db/solr/SolrClient.java index bbc90775ae..f6a78e91e3 100644 --- a/solr/src/main/java/com/yahoo/ycsb/db/SolrClient.java +++ b/solr/src/main/java/com/yahoo/ycsb/db/solr/SolrClient.java @@ -15,7 +15,7 @@ * LICENSE file. */ -package com.yahoo.ycsb.db; +package com.yahoo.ycsb.db.solr; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; @@ -34,12 +34,8 @@ import org.apache.solr.common.SolrInputDocument; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; +import java.util.*; import java.util.Map.Entry; -import java.util.Properties; -import java.util.Set; -import java.util.Vector; /** * Solr client for YCSB framework. diff --git a/solr/src/main/java/com/yahoo/ycsb/db/package-info.java b/solr/src/main/java/com/yahoo/ycsb/db/solr/package-info.java similarity index 95% rename from solr/src/main/java/com/yahoo/ycsb/db/package-info.java rename to solr/src/main/java/com/yahoo/ycsb/db/solr/package-info.java index 98fd53e960..1c5e8f202d 100644 --- a/solr/src/main/java/com/yahoo/ycsb/db/package-info.java +++ b/solr/src/main/java/com/yahoo/ycsb/db/solr/package-info.java @@ -19,5 +19,5 @@ * The YCSB binding for * Solr. */ -package com.yahoo.ycsb.db; +package com.yahoo.ycsb.db.solr; diff --git a/solr/src/test/java/com/yahoo/ycsb/db/SolrClientBaseTest.java b/solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientBaseTest.java similarity index 99% rename from solr/src/test/java/com/yahoo/ycsb/db/SolrClientBaseTest.java rename to solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientBaseTest.java index c6bf53a16f..40959fec2e 100644 --- a/solr/src/test/java/com/yahoo/ycsb/db/SolrClientBaseTest.java +++ b/solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientBaseTest.java @@ -15,7 +15,7 @@ * LICENSE file. */ -package com.yahoo.ycsb.db; +package com.yahoo.ycsb.db.solr; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; diff --git a/solr/src/test/java/com/yahoo/ycsb/db/SolrClientCloudTest.java b/solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientCloudTest.java similarity index 97% rename from solr/src/test/java/com/yahoo/ycsb/db/SolrClientCloudTest.java rename to solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientCloudTest.java index 379d405a98..94449e05c7 100644 --- a/solr/src/test/java/com/yahoo/ycsb/db/SolrClientCloudTest.java +++ b/solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientCloudTest.java @@ -14,7 +14,7 @@ * permissions and limitations under the License. See accompanying * LICENSE file. */ -package com.yahoo.ycsb.db; +package com.yahoo.ycsb.db.solr; import com.yahoo.ycsb.DB; import org.junit.After; diff --git a/solr/src/test/java/com/yahoo/ycsb/db/SolrClientTest.java b/solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientTest.java similarity index 96% rename from solr/src/test/java/com/yahoo/ycsb/db/SolrClientTest.java rename to solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientTest.java index 82fb866c18..c476c80e93 100644 --- a/solr/src/test/java/com/yahoo/ycsb/db/SolrClientTest.java +++ b/solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientTest.java @@ -14,15 +14,13 @@ * permissions and limitations under the License. See accompanying * LICENSE file. */ -package com.yahoo.ycsb.db; +package com.yahoo.ycsb.db.solr; import com.yahoo.ycsb.DB; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.junit.After; -import java.net.URL; import java.util.Properties; -import java.util.Set; import static org.junit.Assume.assumeNoException; diff --git a/solr6/README.md b/solr6/README.md new file mode 100644 index 0000000000..3aa079ea57 --- /dev/null +++ b/solr6/README.md @@ -0,0 +1,110 @@ + + +## Quick Start + +This section describes how to run YCSB on Solr running locally. + +### 1. Set Up YCSB + +Clone the YCSB git repository and compile: + + git clone git://github.com/brianfrankcooper/YCSB.git + cd YCSB + mvn -pl com.yahoo.ycsb:solr6-binding -am clean package + +### 2. Set Up Solr + +There must be a running Solr instance with a core/collection pre-defined and configured. +- See this [API](https://cwiki.apache.org/confluence/display/solr/CoreAdmin+API#CoreAdminAPI-CREATE) reference on how to create a core. +- See this [API](https://cwiki.apache.org/confluence/display/solr/Collections+API#CollectionsAPI-api1) reference on how to create a collection in SolrCloud mode. + +The `conf/schema.xml` configuration file present in the core/collection just created must be configured to handle the expected field names during benchmarking. +Below illustrates a sample from a schema config file that matches the default field names used by the ycsb client: + + + + + + + + + + + + + +If running in SolrCloud mode ensure there is an external Zookeeper cluster running. +- See [here](https://cwiki.apache.org/confluence/display/solr/Setting+Up+an+External+ZooKeeper+Ensemble) for details on how to set up an external Zookeeper cluster. +- See [here](https://cwiki.apache.org/confluence/display/solr/Using+ZooKeeper+to+Manage+Configuration+Files) for instructions on how to use Zookeeper to manage your core/collection configuration files. + +### 3. Run YCSB + +Now you are ready to run! First, load the data: + + ./bin/ycsb load solr6 -s -P workloads/workloada -p table= + +Then, run the workload: + + ./bin/ycsb run solr6 -s -P workloads/workloada -p table= + +For further configuration see below: + +### Default Configuration Parameters +The default settings for the Solr node that is created is as follows: + +- `solr.cloud` + - A Boolean value indicating if Solr is running in SolrCloud mode. If so there must be an external Zookeeper cluster running also. + - Default value is `false` and therefore expects solr to be running in stand-alone mode. + +- `solr.base.url` + - The base URL in which to interface with a running Solr instance in stand-alone mode + - Default value is `http://localhost:8983/solr + +- `solr.commit.within.time` + - The max time in ms to wait for a commit when in batch mode, ignored otherwise + - Default value is `1000ms` + +- `solr.batch.mode` + - Indicates if inserts/updates/deletes should be commited in batches (frequency controlled by the `solr.commit.within.time` parameter) or commit 1 document at a time. + - Default value is `false` + +- `solr.zookeeper.hosts` + - A list of comma seperated host:port pairs of Zookeeper nodes used to manage SolrCloud configurations. + - Must be passed when in [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) mode. + - Default value is `localhost:2181` + +### Custom Configuration +If you wish to customize the settings used to create the Solr node +you can created a new property file that contains your desired Solr +node settings and pass it in via the parameter to 'bin/ycsb' script. Note that +the default properties will be kept if you don't explicitly overwrite them. + +Assuming that we have a properties file named "myproperties.data" that contains +custom Solr node configuration you can execute the following to +pass it into the Solr client: + + ./bin/ycsb run solr6 -P workloads/workloada -P myproperties.data -s + +If you wish to use SolrCloud mode ensure a Solr cluster is running with an +external zookeeper cluster and an appropriate collection has been created. +Make sure to pass the following properties as parameters to 'bin/ycsb' script. + + solr.cloud=true + solr.zookeeper.hosts=:,...,: + + diff --git a/solr6/pom.xml b/solr6/pom.xml new file mode 100644 index 0000000000..f23b3704b8 --- /dev/null +++ b/solr6/pom.xml @@ -0,0 +1,79 @@ + + + + + 4.0.0 + + com.yahoo.ycsb + binding-parent + 0.12.0-SNAPSHOT + ../binding-parent + + + solr6-binding + Solr 6 Binding + jar + + + + true + + + + + com.yahoo.ycsb + core + ${project.version} + provided + + + org.apache.solr + solr-solrj + ${solr6.version} + + + org.slf4j + slf4j-log4j12 + 1.7.21 + + + + org.apache.solr + solr-test-framework + ${solr6.version} + test + + + + + + + jdk8-tests + + 1.8 + + + false + + + + diff --git a/solr6/src/main/java/com/yahoo/ycsb/db/solr6/SolrClient.java b/solr6/src/main/java/com/yahoo/ycsb/db/solr6/SolrClient.java new file mode 100644 index 0000000000..0f18fa21e4 --- /dev/null +++ b/solr6/src/main/java/com/yahoo/ycsb/db/solr6/SolrClient.java @@ -0,0 +1,323 @@ +/** + * Copyright (c) 2016 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package com.yahoo.ycsb.db.solr6; + +import com.yahoo.ycsb.ByteIterator; +import com.yahoo.ycsb.DB; +import com.yahoo.ycsb.DBException; +import com.yahoo.ycsb.Status; +import com.yahoo.ycsb.StringByteIterator; + +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; + +import java.io.IOException; +import java.util.*; +import java.util.Map.Entry; + +/** + * Solr client for YCSB framework. + * + *

+ * Default properties to set: + *

+ *
    + * See README.md + *
+ * + */ +public class SolrClient extends DB { + + public static final String DEFAULT_CLOUD_MODE = "false"; + public static final String DEFAULT_BATCH_MODE = "false"; + public static final String DEFAULT_ZOOKEEPER_HOSTS = "localhost:2181"; + public static final String DEFAULT_SOLR_BASE_URL = "http://localhost:8983/solr"; + public static final String DEFAULT_COMMIT_WITHIN_TIME = "1000"; + + private org.apache.solr.client.solrj.SolrClient client; + private Integer commitTime; + private Boolean batchMode; + + /** + * Initialize any state for this DB. Called once per DB instance; there is one DB instance per + * client thread. + */ + @Override + public void init() throws DBException { + Properties props = getProperties(); + commitTime = Integer + .parseInt(props.getProperty("solr.commit.within.time", DEFAULT_COMMIT_WITHIN_TIME)); + batchMode = Boolean.parseBoolean(props.getProperty("solr.batch.mode", DEFAULT_BATCH_MODE)); + + // Check if Solr cluster is running in SolrCloud or Stand-alone mode + Boolean cloudMode = Boolean.parseBoolean(props.getProperty("solr.cloud", DEFAULT_CLOUD_MODE)); + System.err.println("Solr Cloud Mode = " + cloudMode); + if (cloudMode) { + System.err.println("Solr Zookeeper Remote Hosts = " + + props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS)); + client = new CloudSolrClient.Builder().withZkHost( + Arrays.asList(props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS).split(","))).build(); + } else { + client = new HttpSolrClient.Builder(props.getProperty("solr.base.url", DEFAULT_SOLR_BASE_URL)).build(); + } + } + + @Override + public void cleanup() throws DBException { + try { + client.close(); + } catch (IOException e) { + throw new DBException(e); + } + } + + /** + * Insert a record in the database. Any field/value pairs in the specified values HashMap will be + * written into the record with the specified record key. + * + * @param table + * The name of the table + * @param key + * The record key of the record to insert. + * @param values + * A HashMap of field/value pairs to insert in the record + * @return Zero on success, a non-zero error code on error. See this class's description for a + * discussion of error codes. + */ + @Override + public Status insert(String table, String key, HashMap values) { + try { + SolrInputDocument doc = new SolrInputDocument(); + + doc.addField("id", key); + for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) { + doc.addField(entry.getKey(), entry.getValue()); + } + UpdateResponse response; + if (batchMode) { + response = client.add(table, doc, commitTime); + } else { + response = client.add(table, doc); + client.commit(table); + } + return checkStatus(response.getStatus()); + + } catch (IOException | SolrServerException e) { + e.printStackTrace(); + } + return Status.ERROR; + } + + /** + * Delete a record from the database. + * + * @param table + * The name of the table + * @param key + * The record key of the record to delete. + * @return Zero on success, a non-zero error code on error. See this class's description for a + * discussion of error codes. + */ + @Override + public Status delete(String table, String key) { + try { + UpdateResponse response; + if (batchMode) { + response = client.deleteById(table, key, commitTime); + } else { + response = client.deleteById(table, key); + client.commit(table); + } + return checkStatus(response.getStatus()); + } catch (IOException | SolrServerException e) { + e.printStackTrace(); + } + return Status.ERROR; + } + + /** + * Read a record from the database. Each field/value pair from the result will be stored in a + * HashMap. + * + * @param table + * The name of the table + * @param key + * The record key of the record to read. + * @param fields + * The list of fields to read, or null for all of them + * @param result + * A HashMap of field/value pairs for the result + * @return Zero on success, a non-zero error code on error or "not found". + */ + @Override + public Status read(String table, String key, Set fields, + HashMap result) { + try { + Boolean returnFields = false; + String[] fieldList = null; + if (fields != null) { + returnFields = true; + fieldList = fields.toArray(new String[fields.size()]); + } + SolrQuery query = new SolrQuery(); + query.setQuery("id:" + key); + if (returnFields) { + query.setFields(fieldList); + } + final QueryResponse response = client.query(table, query); + SolrDocumentList results = response.getResults(); + if ((results != null) && (results.getNumFound() > 0)) { + for (String field : results.get(0).getFieldNames()) { + result.put(field, + new StringByteIterator(String.valueOf(results.get(0).getFirstValue(field)))); + } + } + return checkStatus(response.getStatus()); + } catch (IOException | SolrServerException e) { + e.printStackTrace(); + } + return Status.ERROR; + } + + /** + * Update a record in the database. Any field/value pairs in the specified values HashMap will be + * written into the record with the specified record key, overwriting any existing values with the + * same field name. + * + * @param table + * The name of the table + * @param key + * The record key of the record to write. + * @param values + * A HashMap of field/value pairs to update in the record + * @return Zero on success, a non-zero error code on error. See this class's description for a + * discussion of error codes. + */ + @Override + public Status update(String table, String key, HashMap values) { + try { + SolrInputDocument updatedDoc = new SolrInputDocument(); + updatedDoc.addField("id", key); + + for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) { + updatedDoc.addField(entry.getKey(), Collections.singletonMap("set", entry.getValue())); + } + + UpdateResponse writeResponse; + if (batchMode) { + writeResponse = client.add(table, updatedDoc, commitTime); + } else { + writeResponse = client.add(table, updatedDoc); + client.commit(table); + } + return checkStatus(writeResponse.getStatus()); + } catch (IOException | SolrServerException e) { + e.printStackTrace(); + } + return Status.ERROR; + } + + /** + * Perform a range scan for a set of records in the database. Each field/value pair from the + * result will be stored in a HashMap. + * + * @param table + * The name of the table + * @param startkey + * The record key of the first record to read. + * @param recordcount + * The number of records to read + * @param fields + * The list of fields to read, or null for all of them + * @param result + * A Vector of HashMaps, where each HashMap is a set field/value pairs for one record + * @return Zero on success, a non-zero error code on error. See this class's description for a + * discussion of error codes. + */ + @Override + public Status scan(String table, String startkey, int recordcount, Set fields, + Vector> result) { + try { + Boolean returnFields = false; + String[] fieldList = null; + if (fields != null) { + returnFields = true; + fieldList = fields.toArray(new String[fields.size()]); + } + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.setParam("fq", "id:[ " + startkey + " TO * ]"); + if (returnFields) { + query.setFields(fieldList); + } + query.setRows(recordcount); + final QueryResponse response = client.query(table, query); + SolrDocumentList results = response.getResults(); + + HashMap entry; + + for (SolrDocument hit : results) { + entry = new HashMap<>((int) results.getNumFound()); + for (String field : hit.getFieldNames()) { + entry.put(field, new StringByteIterator(String.valueOf(hit.getFirstValue(field)))); + } + result.add(entry); + } + return checkStatus(response.getStatus()); + } catch (IOException | SolrServerException e) { + e.printStackTrace(); + } + return Status.ERROR; + } + + private Status checkStatus(int status) { + Status responseStatus; + switch (status) { + case 0: + responseStatus = Status.OK; + break; + case 400: + responseStatus = Status.BAD_REQUEST; + break; + case 403: + responseStatus = Status.FORBIDDEN; + break; + case 404: + responseStatus = Status.NOT_FOUND; + break; + case 500: + responseStatus = Status.ERROR; + break; + case 503: + responseStatus = Status.SERVICE_UNAVAILABLE; + break; + default: + responseStatus = Status.UNEXPECTED_STATE; + break; + } + return responseStatus; + } + +} diff --git a/solr6/src/main/java/com/yahoo/ycsb/db/solr6/package-info.java b/solr6/src/main/java/com/yahoo/ycsb/db/solr6/package-info.java new file mode 100644 index 0000000000..94e05ee044 --- /dev/null +++ b/solr6/src/main/java/com/yahoo/ycsb/db/solr6/package-info.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2016 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +/** + * The YCSB binding for + * Solr. + */ +package com.yahoo.ycsb.db.solr6; + diff --git a/solr6/src/main/resources/log4j.properties b/solr6/src/main/resources/log4j.properties new file mode 100644 index 0000000000..0ede1d4c14 --- /dev/null +++ b/solr6/src/main/resources/log4j.properties @@ -0,0 +1,8 @@ +# Root logger option +log4j.rootLogger=INFO, stderr + +# Direct log messages to stderr +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientBaseTest.java b/solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientBaseTest.java new file mode 100644 index 0000000000..06fdb76e4e --- /dev/null +++ b/solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientBaseTest.java @@ -0,0 +1,160 @@ +/** + * Copyright (c) 2016 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package com.yahoo.ycsb.db.solr6; + +import com.yahoo.ycsb.ByteIterator; +import com.yahoo.ycsb.DB; +import com.yahoo.ycsb.Status; +import com.yahoo.ycsb.StringByteIterator; +import org.apache.solr.client.solrj.embedded.JettyConfig; +import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.common.util.NamedList; +import org.junit.*; + +import java.io.File; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Properties; +import java.util.Set; +import java.util.Vector; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public abstract class SolrClientBaseTest { + + protected static MiniSolrCloudCluster miniSolrCloudCluster; + private DB instance; + private final static HashMap MOCK_DATA; + protected final static String MOCK_TABLE = "ycsb"; + private final static String MOCK_KEY0 = "0"; + private final static String MOCK_KEY1 = "1"; + private final static int NUM_RECORDS = 10; + + static { + MOCK_DATA = new HashMap<>(NUM_RECORDS); + for (int i = 0; i < NUM_RECORDS; i++) { + MOCK_DATA.put("field" + i, new StringByteIterator("value" + i)); + } + } + + @BeforeClass + public static void onlyOnce() throws Exception { + Path miniSolrCloudClusterTempDirectory = Files.createTempDirectory("miniSolrCloudCluster"); + miniSolrCloudClusterTempDirectory.toFile().deleteOnExit(); + miniSolrCloudCluster = new MiniSolrCloudCluster(1, miniSolrCloudClusterTempDirectory, JettyConfig.builder().build()); + + // Upload Solr configuration + URL configDir = SolrClientBaseTest.class.getClassLoader().getResource("solr_config"); + assertNotNull(configDir); + miniSolrCloudCluster.uploadConfigDir(new File(configDir.toURI()), MOCK_TABLE); + } + + @AfterClass + public static void destroy() throws Exception { + if(miniSolrCloudCluster != null) { + miniSolrCloudCluster.shutdown(); + } + } + + @Before + public void setup() throws Exception { + NamedList namedList = miniSolrCloudCluster.createCollection(MOCK_TABLE, 1, 1, MOCK_TABLE, null); + assertEquals(namedList.indexOf("success", 0), 1); + Thread.sleep(1000); + + instance = getDB(); + } + + @After + public void tearDown() throws Exception { + if(miniSolrCloudCluster != null) { + NamedList namedList = miniSolrCloudCluster.deleteCollection(MOCK_TABLE); + assertEquals(namedList.indexOf("success", 0), 1); + Thread.sleep(1000); + } + } + + @Test + public void testInsert() throws Exception { + Status result = instance.insert(MOCK_TABLE, MOCK_KEY0, MOCK_DATA); + assertEquals(Status.OK, result); + } + + @Test + public void testDelete() throws Exception { + Status result = instance.delete(MOCK_TABLE, MOCK_KEY1); + assertEquals(Status.OK, result); + } + + @Test + public void testRead() throws Exception { + Set fields = MOCK_DATA.keySet(); + HashMap resultParam = new HashMap<>(NUM_RECORDS); + Status result = instance.read(MOCK_TABLE, MOCK_KEY1, fields, resultParam); + assertEquals(Status.OK, result); + } + + @Test + public void testUpdate() throws Exception { + HashMap newValues = new HashMap<>(NUM_RECORDS); + + for (int i = 0; i < NUM_RECORDS; i++) { + newValues.put("field" + i, new StringByteIterator("newvalue" + i)); + } + + Status result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues); + assertEquals(Status.OK, result); + + //validate that the values changed + HashMap resultParam = new HashMap<>(NUM_RECORDS); + instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam); + + for (int i = 0; i < NUM_RECORDS; i++) { + assertEquals("newvalue" + i, resultParam.get("field" + i).toString()); + } + } + + @Test + public void testScan() throws Exception { + Set fields = MOCK_DATA.keySet(); + Vector> resultParam = new Vector<>(NUM_RECORDS); + Status result = instance.scan(MOCK_TABLE, MOCK_KEY1, NUM_RECORDS, fields, resultParam); + assertEquals(Status.OK, result); + } + + /** + * Gets the test DB. + * + * @return The test DB. + */ + protected DB getDB() { + return getDB(new Properties()); + } + + /** + * Gets the test DB. + * + * @param props + * Properties to pass to the client. + * @return The test DB. + */ + protected abstract DB getDB(Properties props); +} diff --git a/solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientCloudTest.java b/solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientCloudTest.java new file mode 100644 index 0000000000..2cfefd17c2 --- /dev/null +++ b/solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientCloudTest.java @@ -0,0 +1,56 @@ +/** + * Copyright (c) 2016 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package com.yahoo.ycsb.db.solr6; + +import com.yahoo.ycsb.DB; +import org.junit.After; + +import java.util.Properties; + +import static org.junit.Assume.assumeNoException; + +public class SolrClientCloudTest extends SolrClientBaseTest { + + private SolrClient instance; + + @After + public void tearDown() throws Exception { + try { + if(instance != null) { + instance.cleanup(); + } + } finally { + super.tearDown(); + } + } + + @Override + protected DB getDB(Properties props) { + instance = new SolrClient(); + + props.setProperty("solr.cloud", "true"); + props.setProperty("solr.zookeeper.hosts", miniSolrCloudCluster.getSolrClient().getZkHost()); + + instance.setProperties(props); + try { + instance.init(); + } catch (Exception error) { + assumeNoException(error); + } + return instance; + } +} diff --git a/solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientTest.java b/solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientTest.java new file mode 100644 index 0000000000..17373dffbf --- /dev/null +++ b/solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientTest.java @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2016 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package com.yahoo.ycsb.db.solr6; + +import com.yahoo.ycsb.DB; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.junit.After; + +import java.util.Properties; + +import static org.junit.Assume.assumeNoException; + +public class SolrClientTest extends SolrClientBaseTest { + + private SolrClient instance; + + @After + public void tearDown() throws Exception { + try { + if(instance != null) { + instance.cleanup(); + } + } finally { + super.tearDown(); + } + } + + @Override + protected DB getDB(Properties props) { + instance = new SolrClient(); + + // Use the first Solr server in the cluster. + // Doesn't matter if there are more since requests will be forwarded properly by Solr. + JettySolrRunner jettySolrRunner = miniSolrCloudCluster.getJettySolrRunners().get(0); + String solrBaseUrl = String.format("http://localhost:%s%s", jettySolrRunner.getLocalPort(), + jettySolrRunner.getBaseUrl()); + + props.setProperty("solr.base.url", solrBaseUrl); + instance.setProperties(props); + + try { + instance.init(); + } catch (Exception error) { + assumeNoException(error); + } + return instance; + } +} diff --git a/solr6/src/test/resources/log4j.properties b/solr6/src/test/resources/log4j.properties new file mode 100644 index 0000000000..599b505128 --- /dev/null +++ b/solr6/src/test/resources/log4j.properties @@ -0,0 +1,29 @@ +# Copyright (c) 2015-2016 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. +# + +# Root logger option +log4j.rootLogger=INFO, stderr + +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %-5p %c %x - %m%n + +# Suppress messages from ZooKeeper +log4j.logger.org.apache.zookeeper=ERROR +# Solr classes are too chatty in test at INFO +log4j.logger.org.apache.solr=ERROR +log4j.logger.org.eclipse.jetty=ERROR diff --git a/solr6/src/test/resources/solr_config/schema.xml b/solr6/src/test/resources/solr_config/schema.xml new file mode 100644 index 0000000000..55323ee4c1 --- /dev/null +++ b/solr6/src/test/resources/solr_config/schema.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + diff --git a/solr6/src/test/resources/solr_config/solrconfig.xml b/solr6/src/test/resources/solr_config/solrconfig.xml new file mode 100644 index 0000000000..7c58457dcd --- /dev/null +++ b/solr6/src/test/resources/solr_config/solrconfig.xml @@ -0,0 +1,63 @@ + + + + + + + + ${tests.luceneMatchVersion:LATEST} + + ${useCompoundFile:false} + + ${solr.data.dir:} + + + + + ${solr.data.dir:} + + + + + + + + + + + + *:* + + + all + + server-enabled.txt + + + + + solr + + + +