diff --git a/.gitignore b/.gitignore index cf4397b..76e8aa5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ * !/**/ !**/*.java +!**/*.xml !**/*.jar !**/*.md !.gitignore diff --git a/pom.xml b/pom.xml index ec68f1f..d85ace9 100644 --- a/pom.xml +++ b/pom.xml @@ -45,6 +45,10 @@ 8 8 8 + 1.21 + 2.6 + 3.5.1 + 1.12 @@ -59,10 +63,16 @@ 4.12 test + + org.openjdk.jmh + jmh-core + ${jmh.version} + test + redis.clients jedis - 3.0.1 + 3.1.0 @@ -79,6 +89,27 @@ + + + org.codehaus.mojo + build-helper-maven-plugin + ${maven-build-helper-plugin.version} + + + add-test-source + generate-test-sources + + add-test-source + + + + src/test/perf + + + + + + org.codehaus.mojo cobertura-maven-plugin @@ -94,11 +125,28 @@ org.apache.maven.plugins maven-compiler-plugin - 3.1 + ${maven-compiler-plugin.version} 1.8 1.8 + + + + testCompile + + + + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + + org.sonatype.plugins @@ -118,6 +166,32 @@ false + + + maven-assembly-plugin + ${maven-assembly-plugin.version} + + src/main/assembly/perf-tests.xml + + + + make-assembly + package + + single + + + true + + + org.openjdk.jmh.Main + + + + + + + @@ -149,7 +223,7 @@ true -Xdoclint:none - -Xdoclint:none + diff --git a/src/main/assembly/perf-tests.xml b/src/main/assembly/perf-tests.xml new file mode 100644 index 0000000..57d9eba --- /dev/null +++ b/src/main/assembly/perf-tests.xml @@ -0,0 +1,28 @@ + + perf-tests + + jar + + false + + + / + true + true + test + + + + + ${project.build.directory}/test-classes + / + + **/* + + true + + + \ No newline at end of file diff --git a/src/main/java/io/redisearch/client/Client.java b/src/main/java/io/redisearch/client/Client.java index 17dac52..54161a6 100644 --- a/src/main/java/io/redisearch/client/Client.java +++ b/src/main/java/io/redisearch/client/Client.java @@ -372,6 +372,7 @@ public boolean addDocument(Document doc, AddOptions options) { public boolean[] addDocuments(Document... docs){ return addDocuments(new AddOptions(), docs); } + /** * Add a batch of documents to the index @@ -381,11 +382,13 @@ public boolean[] addDocuments(Document... docs){ */ @Override public boolean[] addDocuments(AddOptions options, Document... docs){ + try (Jedis conn = _conn()) { + Pipeline p = conn.pipelined(); for(Document doc : docs) { - addDocument(doc, options, conn); + addDocument(doc, options, p); } - List objects = conn.getClient().getMany(docs.length); + List objects = p.syncAndReturnAll(); boolean[] results = new boolean[docs.length]; int i=0; for(Object obj : objects) { @@ -395,13 +398,24 @@ public boolean[] addDocuments(AddOptions options, Document... docs){ return results; } } - + + private void addDocument(Document doc, AddOptions options, Pipeline p) { + byte[][] args = addDocumentArgs(doc, options); + p.sendCommand(commands.getAddCommand(), args); + + } + private BinaryClient addDocument(Document doc, AddOptions options, Jedis conn) { + byte[][] args = addDocumentArgs(doc, options); + return sendCommand(conn, commands.getAddCommand(), args); + } + + private byte[][] addDocumentArgs(Document doc, AddOptions options) { ArrayList args = new ArrayList<>(); args.add(endocdedIndexName); args.add(SafeEncoder.encode(doc.getId())); args.add(Protocol.toByteArray(doc.getScore())); - + if (options.getNosave()) { args.add(Keywords.NOSAVE.getRaw()); } @@ -426,8 +440,7 @@ private BinaryClient addDocument(Document doc, AddOptions options, Jedis conn) { Object value = ent.getValue(); args.add(value instanceof byte[] ? (byte[])value : SafeEncoder.encode(value.toString())); } - - return sendCommand(conn, commands.getAddCommand(), args.toArray(new byte[args.size()][])); + return args.toArray(new byte[args.size()][]); } /** diff --git a/src/test/perf/redisearch/client/DocumentIngestionBenchmark.java b/src/test/perf/redisearch/client/DocumentIngestionBenchmark.java new file mode 100644 index 0000000..216ca5c --- /dev/null +++ b/src/test/perf/redisearch/client/DocumentIngestionBenchmark.java @@ -0,0 +1,176 @@ +package redisearch.client; + +import io.redisearch.Document; +import io.redisearch.Schema; +import io.redisearch.client.Client; +import org.openjdk.jmh.Main; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.ThreadParams; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; + +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * Each thread will have a dedicated JRediSearch client. + */ +@BenchmarkMode({Mode.Throughput}) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Benchmark) +public class DocumentIngestionBenchmark { + + static final Logger LOG = LoggerFactory.getLogger(Main.class); + static final int RANDOM_SEED = 12345; + static final int JREDISCLIENT_TIMEOUT = 500; + static final int JREDISCLIENT_POOLSIZE = 100; + static final int MULTIDOCUMENT_SIZE = 10; + static final int DOCS_SIZE = 10000; + static final String INDEX_NAME = "indexName"; + static final String HOST_IP = "127.0.0.1"; + static final int HOST_PORT = 6379; + + static Options opt; + + private io.redisearch.client.Client client; + private Jedis jedis_client; + + /* + * ============================== HOW TO RUN THIS TEST: ==================================== + * + * You can run this test: + * + * a) Via the command line: + * $ mvn clean install + * $ java -jar target/jredisearch-1.1.0-SNAPSHOT-perf-tests.jar -wi 1 -i 5 -t 16 -f 5 + * (we requested 1 warmup iterations, 5 iterations, 8 threads, and 5 forks) + * + */ + public static void main(String[] args) throws RunnerException { + + opt = new OptionsBuilder() + .include(DocumentIngestionBenchmark.class.getSimpleName()) + .warmupIterations(1) + .measurementIterations(5) + .threads(4) + .forks(5) + .build(); + + new Runner(opt).run(); + + } + + @Setup + public void setup(BenchmarkParams params) { + + client = new Client(INDEX_NAME, HOST_IP, HOST_PORT, JREDISCLIENT_TIMEOUT, JREDISCLIENT_POOLSIZE); + jedis_client = new Jedis(HOST_IP, HOST_PORT); + jedis_client.flushAll(); + client.dropIndex(true); + + LOG.info("Started Creating Schema"); + Schema sc = new Schema(); + sc.addSortableTextField("name", 1.0); + sc.addSortableNumericField("count"); + sc.addSortableNumericField("height"); + sc.addSortableNumericField("height2"); + sc.addSortableNumericField("height3"); + sc.addSortableNumericField("height4"); + sc.addSortableNumericField("height5"); + sc.addSortableNumericField("height6"); + sc.addSortableNumericField("height7"); + sc.addSortableNumericField("height8"); + sc.addSortableNumericField("height9"); + sc.addSortableNumericField("height10"); + sc.addSortableNumericField("height11"); + sc.addSortableNumericField("height12"); + client.createIndex(sc, Client.IndexOptions.defaultOptions()); + LOG.info("Finished Creating Schema"); + client.close(); + + + } + + @TearDown(Level.Trial) + public void doTearDown() { + //clean the db after the benchmark + jedis_client = new Jedis(HOST_IP, HOST_PORT, 30000); + jedis_client.flushAll(); + } + + @Benchmark + @OperationsPerInvocation(1) + public void Client_addDocument(NumericValues numv) { + numv.client.addDocument(new Document(String.format("multidoc:thread%d:%d", numv.id, numv.docPos), numv.fieldsList.get(numv.random.nextInt(DOCS_SIZE))) + ); + numv.docPos++; + } + + @Benchmark + @OperationsPerInvocation(MULTIDOCUMENT_SIZE) + public void Client_addDocuments(NumericValues numv) { + List docs = new ArrayList(); + for (int i = 0; i < MULTIDOCUMENT_SIZE; i++) { + docs.add(new Document(String.format("multidoc:thread%d:%d", numv.id, numv.docPos), numv.fieldsList.get(numv.random.nextInt(DOCS_SIZE)))); + numv.docPos++; + } + numv.client.addDocuments(docs.toArray(new Document[docs.size()])); + + } + + @State(Scope.Thread) + public static class NumericValues { + Random random; + private int id; + private io.redisearch.client.Client client; + private int docPos; + private List> fieldsList; + + + @Setup + public void setup(ThreadParams threads) { + client = new Client(INDEX_NAME, HOST_IP, HOST_PORT, JREDISCLIENT_TIMEOUT, JREDISCLIENT_POOLSIZE); + id = threads.getThreadIndex(); + random = new Random(); + random.setSeed(RANDOM_SEED); + docPos = 0; + LOG.info("Started Creating FieldsList"); + + fieldsList = new ArrayList>(); + for (int i = 0; i < DOCS_SIZE; i++) { + Map fields = new HashMap<>(); + + fields.put("count", random.nextFloat()); + fields.put("height", random.nextFloat()); + fields.put("height2", random.nextFloat()); + fields.put("height3", random.nextFloat()); + fields.put("height4", random.nextFloat()); + fields.put("height5", random.nextFloat()); + fields.put("height6", random.nextFloat()); + fields.put("height7", random.nextFloat()); + fields.put("height8", random.nextFloat()); + fields.put("height9", random.nextFloat()); + fields.put("height10", random.nextFloat()); + fields.put("height11", random.nextFloat()); + fields.put("height12", random.nextFloat()); + fieldsList.add(fields); + } + LOG.info("Finished Creating FieldsList"); + + } + + @TearDown(Level.Trial) + public void doTearDown() { + } + + + } + + +}