From a6ea6147fa68bca2a8f869479dee28d620d93dbd Mon Sep 17 00:00:00 2001 From: Eric Zhang <31218485+16BitNarwhal@users.noreply.github.com> Date: Thu, 16 May 2024 09:35:36 -0400 Subject: [PATCH] Update API retrieval results format (#2499) * change format to match RunOutputWriter jsonl format * remove whitespace --- .../frontend/pages/components/Dropdown.tsx | 2 - .../frontend/pages/components/SearchBar.tsx | 17 +++--- src/main/frontend/types/QueryResult.tsx | 5 -- .../java/io/anserini/server/Controller.java | 16 +++++- .../java/io/anserini/server/QueryResult.java | 55 ------------------- .../io/anserini/server/SearchService.java | 50 ++++++++--------- .../io/anserini/server/ControllerTest.java | 15 +++-- 7 files changed, 57 insertions(+), 103 deletions(-) delete mode 100644 src/main/frontend/types/QueryResult.tsx delete mode 100644 src/main/java/io/anserini/server/QueryResult.java diff --git a/src/main/frontend/pages/components/Dropdown.tsx b/src/main/frontend/pages/components/Dropdown.tsx index 758da8b9a3..f0ea5a26ee 100644 --- a/src/main/frontend/pages/components/Dropdown.tsx +++ b/src/main/frontend/pages/components/Dropdown.tsx @@ -91,8 +91,6 @@ const Dropdown: React.FC = ({ onSelect }) => { ]; }); setCollections(generatedMap); - - console.log('collections', collections); }, []); return ( diff --git a/src/main/frontend/pages/components/SearchBar.tsx b/src/main/frontend/pages/components/SearchBar.tsx index aef3c85d81..d22b8d119e 100644 --- a/src/main/frontend/pages/components/SearchBar.tsx +++ b/src/main/frontend/pages/components/SearchBar.tsx @@ -15,12 +15,11 @@ */ import React, { useEffect, useState } from 'react'; -import { QueryResult } from '../../types/QueryResult'; import Dropdown from './Dropdown'; const SearchBar: React.FC = () => { const [loading, setLoading] = useState(false); - const [results, setResults] = useState([]); + const [results, setResults] = useState>([]); const [query, setQuery] = useState(''); const [index, setIndex] = useState(''); @@ -32,8 +31,8 @@ const SearchBar: React.FC = () => { endpoint += `/search?query=${query}`; const response = await fetch(endpoint); - const data: QueryResult[] = await response.json(); - setResults(data); + const data = await response.json(); + setResults(data.candidates); } catch (error) { console.error("Failed to fetch data: ", error); setResults([]); @@ -66,10 +65,14 @@ const SearchBar: React.FC = () => { {loading &&

Loading...

}
    - {results.map((result) => ( -
    + {results.map((result, index) => ( +

    Document ID: {result.docid} Score: {result.score}

    -

    {result.content}

    + {Object.entries(result.doc).map(([key, value]) => ( +

    + {key}: {value as React.ReactNode} +

    + ))}
    ))}
diff --git a/src/main/frontend/types/QueryResult.tsx b/src/main/frontend/types/QueryResult.tsx deleted file mode 100644 index 3ab9c13439..0000000000 --- a/src/main/frontend/types/QueryResult.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export interface QueryResult { - docid: string; - content: string; - score: number; -} \ No newline at end of file diff --git a/src/main/java/io/anserini/server/Controller.java b/src/main/java/io/anserini/server/Controller.java index 5150db9dd2..5eabbeb6e4 100644 --- a/src/main/java/io/anserini/server/Controller.java +++ b/src/main/java/io/anserini/server/Controller.java @@ -17,6 +17,8 @@ package io.anserini.server; import java.util.List; +import java.util.LinkedHashMap; +import java.util.Map; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestParam; @@ -31,15 +33,23 @@ public class Controller { private static final String DEFAULT_COLLECTION = "msmarco-v1-passage"; @RequestMapping(method = RequestMethod.GET, path = {"/collection/{collection}/search", "/search"}) - public List search(@PathVariable(value = "collection", required = false) String collection, - @RequestParam("query") String query) { + public Map search(@PathVariable(value = "collection", required = false) String collection, + @RequestParam("query") String query, + @RequestParam(value = "hits", defaultValue = "10") int hits, + @RequestParam(value = "qid", defaultValue = "") String qid) { if (collection == null) { collection = DEFAULT_COLLECTION; } SearchService searchService = new SearchService(collection); - return searchService.search(query, 10); + List> candidates = searchService.search(query, hits); + + Map queryMap = new LinkedHashMap<>(); + queryMap.put("query", new LinkedHashMap<>(Map.of("qid", qid, "text", query))); + queryMap.put("candidates", candidates); + + return queryMap; } } \ No newline at end of file diff --git a/src/main/java/io/anserini/server/QueryResult.java b/src/main/java/io/anserini/server/QueryResult.java deleted file mode 100644 index 3d530cf543..0000000000 --- a/src/main/java/io/anserini/server/QueryResult.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Anserini: A Lucene toolkit for reproducible information retrieval research - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.anserini.server; - -public class QueryResult { - private String docid; - private String content; - private float score; - - public QueryResult(String docid, String content, float score) { - this.docid = docid; - this.content = content; - this.score = score; - } - - // Getters - public String getDocid() { - return docid; - } - - public String getContent() { - return content; - } - - public float getScore() { - return score; - } - - // Setters - public void setDocid(String docid) { - this.docid = docid; - } - - public void setContent(String content) { - this.content = content; - } - - public void setRank(int rank) { - this.score = rank; - } -} \ No newline at end of file diff --git a/src/main/java/io/anserini/server/SearchService.java b/src/main/java/io/anserini/server/SearchService.java index 266326c47c..81281e60a7 100644 --- a/src/main/java/io/anserini/server/SearchService.java +++ b/src/main/java/io/anserini/server/SearchService.java @@ -16,6 +16,7 @@ package io.anserini.server; +import io.anserini.index.Constants; import io.anserini.search.ScoredDoc; import io.anserini.search.SimpleSearcher; import io.anserini.util.PrebuiltIndexHandler; @@ -24,15 +25,19 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; public class SearchService { - final private String indexDir; - final private float k1 = 0.9f; - final private float b = 0.4f; + private final String indexDir; + private final float k1 = 0.9f; + private final float b = 0.4f; + private final ObjectMapper mapper = new ObjectMapper(); public SearchService(String prebuiltIndex) { PrebuiltIndexHandler handler = new PrebuiltIndexHandler(prebuiltIndex); @@ -45,37 +50,28 @@ public SearchService(String prebuiltIndex) { } } - public List search(String query, int hits) { + public List> search(String query, int hits) { try { - // index, k1, b, hits SimpleSearcher searcher = new SimpleSearcher(indexDir); searcher.set_bm25(k1, b); ScoredDoc[] results = searcher.search(query, hits); - List resultStrings = List.of(results).stream() - .map(result -> { - try { - String jsonString = searcher.doc_raw(result.lucene_docid); - ObjectMapper mapper = new ObjectMapper(); - JsonNode jsonNode = mapper.readTree(jsonString); - String content; - if (jsonNode.get("contents") != null) content = jsonNode.get("contents").asText(); - else if (jsonNode.get("text") != null) content = jsonNode.get("text").asText(); - else if (jsonNode.get("passage") != null) content = jsonNode.get("passage").asText(); - else if (jsonNode.get("body") != null) content = jsonNode.get("body").asText(); - else if (jsonNode.get("segment") != null) content = jsonNode.get("segment").asText(); - else content = jsonNode.toString(); - return new QueryResult(result.docid, content, result.score); - } catch (Exception e) { - e.printStackTrace(); - return null; - } - }).filter(Objects::nonNull).collect(Collectors.toList()); - searcher.close(); - return resultStrings; + List> candidates = new ArrayList<>(); + for (ScoredDoc r : results) { + String raw = r.lucene_document.get(Constants.RAW); + JsonNode rootNode = mapper.readTree(raw); + Map content = mapper.convertValue(rootNode, Map.class); + content.remove("docid"); + Map candidate = new LinkedHashMap<>(); + candidate.put("docid", r.docid); + candidate.put("score", r.score); + candidate.put("doc", content); + candidates.add(candidate); + } + return candidates; } catch (Exception e) { - // Consume exception and return empty list e.printStackTrace(); return List.of(); } } + } \ No newline at end of file diff --git a/src/test/java/io/anserini/server/ControllerTest.java b/src/test/java/io/anserini/server/ControllerTest.java index a110652683..3ee0608630 100644 --- a/src/test/java/io/anserini/server/ControllerTest.java +++ b/src/test/java/io/anserini/server/ControllerTest.java @@ -18,10 +18,13 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; public class ControllerTest { @@ -29,10 +32,14 @@ public class ControllerTest { public void testSearch() throws Exception { Controller controller = new Controller(); - List results = controller.search(null, "Albert Einstein"); + Map results = controller.search(null, "Albert Einstein", 10, ""); + assertNotNull(results); + assertTrue(results.get("candidates") instanceof List); - assertEquals(results.size(), 10); - assertEquals(results.get(0).getDocid(), "3553430"); + @SuppressWarnings("unchecked") + List> candidates = (List>) results.get("candidates"); + assertEquals(10, candidates.size()); + assertEquals("3553430", candidates.get(0).get("docid")); } @Test @@ -40,7 +47,7 @@ public void testIndexNotFound() throws Exception { Controller controller = new Controller(); assertThrows(RuntimeException.class, () -> { - List results = controller.search("nonexistent-index", "Albert Einstein"); + Map results = controller.search("nonexistent-index", "Albert Einstein", 10, ""); }); }