From f77956e2966ee9b47401d3131ff4da68313ff74f Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 20 Sep 2024 04:43:11 +0300 Subject: [PATCH 001/104] Convert to record --- .../gui/shared/SharedDatabaseUIManager.java | 10 +++--- .../jabref/logic/shared/DBMSSynchronizer.java | 4 +-- .../shared/event/ConnectionLostEvent.java | 18 +++------- .../event/SharedEntriesNotPresentEvent.java | 18 +++------- .../shared/event/UpdateRefusedEvent.java | 34 ++++--------------- .../PostgresSQLNotificationListener.java | 2 +- .../shared/SynchronizationSimulatorTest.java | 2 +- 7 files changed, 24 insertions(+), 64 deletions(-) diff --git a/src/main/java/org/jabref/gui/shared/SharedDatabaseUIManager.java b/src/main/java/org/jabref/gui/shared/SharedDatabaseUIManager.java index d3f2989b36e..ce41cfb5ca7 100644 --- a/src/main/java/org/jabref/gui/shared/SharedDatabaseUIManager.java +++ b/src/main/java/org/jabref/gui/shared/SharedDatabaseUIManager.java @@ -97,7 +97,7 @@ public void listen(ConnectionLostEvent connectionLostEvent) { tabContainer.closeTab(tabContainer.getCurrentLibraryTab()); dialogService.showCustomDialogAndWait(new SharedDatabaseLoginDialogView(tabContainer)); } else if (answer.get().equals(workOffline)) { - connectionLostEvent.getBibDatabaseContext().convertToLocalDatabase(); + connectionLostEvent.bibDatabaseContext().convertToLocalDatabase(); tabContainer.getLibraryTabs().forEach(tab -> tab.updateTabTitle(tab.isModified())); dialogService.notify(Localization.lang("Working offline.")); } @@ -110,8 +110,8 @@ public void listen(ConnectionLostEvent connectionLostEvent) { public void listen(UpdateRefusedEvent updateRefusedEvent) { dialogService.notify(Localization.lang("Update refused.")); - BibEntry localBibEntry = updateRefusedEvent.getLocalBibEntry(); - BibEntry sharedBibEntry = updateRefusedEvent.getSharedBibEntry(); + BibEntry localBibEntry = updateRefusedEvent.localBibEntry(); + BibEntry sharedBibEntry = updateRefusedEvent.sharedBibEntry(); String message = Localization.lang("Update could not be performed due to existing change conflicts.") + "\r\n" + Localization.lang("You are not working on the newest version of BibEntry.") + "\r\n" + @@ -144,9 +144,9 @@ public void listen(SharedEntriesNotPresentEvent event) { LibraryTab libraryTab = tabContainer.getCurrentLibraryTab(); EntryEditor entryEditor = libraryTab.getEntryEditor(); - libraryTab.getUndoManager().addEdit(new UndoableRemoveEntries(libraryTab.getDatabase(), event.getBibEntries())); + libraryTab.getUndoManager().addEdit(new UndoableRemoveEntries(libraryTab.getDatabase(), event.bibEntries())); - if (entryEditor != null && (event.getBibEntries().contains(entryEditor.getCurrentlyEditedEntry()))) { + if (entryEditor != null && (event.bibEntries().contains(entryEditor.getCurrentlyEditedEntry()))) { dialogService.showInformationDialogAndWait(Localization.lang("Shared entry is no longer present"), Localization.lang("The entry you currently work on has been deleted on the shared side.") + "\n" diff --git a/src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java b/src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java index 020b4c8de9d..d388de0b884 100644 --- a/src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java +++ b/src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java @@ -327,7 +327,7 @@ public void pullChanges() { * Synchronizes local BibEntries only if last entry changes still remain */ public void pullLastEntryChanges() { - if (!lastEntryChanged.isEmpty()) { + if (lastEntryChanged.isPresent()) { if (!checkCurrentConnection()) { return; } @@ -342,7 +342,7 @@ public void pullLastEntryChanges() { * Synchronizes local BibEntries and pulls remaining last entry changes */ private void pullWithLastEntry() { - if (!lastEntryChanged.isEmpty() && isPresentLocalBibEntry(lastEntryChanged.get())) { + if (lastEntryChanged.isPresent() && isPresentLocalBibEntry(lastEntryChanged.get())) { synchronizeSharedEntry(lastEntryChanged.get()); } lastEntryChanged = Optional.empty(); diff --git a/src/main/java/org/jabref/logic/shared/event/ConnectionLostEvent.java b/src/main/java/org/jabref/logic/shared/event/ConnectionLostEvent.java index da38a78b4eb..94064145ba7 100644 --- a/src/main/java/org/jabref/logic/shared/event/ConnectionLostEvent.java +++ b/src/main/java/org/jabref/logic/shared/event/ConnectionLostEvent.java @@ -4,19 +4,9 @@ /** * A new {@link ConnectionLostEvent} is fired, when the connection to the shared database gets lost. + * + * @param bibDatabaseContext Affected {@link BibDatabaseContext} */ -public class ConnectionLostEvent { - - private final BibDatabaseContext bibDatabaseContext; - - /** - * @param bibDatabaseContext Affected {@link BibDatabaseContext} - */ - public ConnectionLostEvent(BibDatabaseContext bibDatabaseContext) { - this.bibDatabaseContext = bibDatabaseContext; - } - - public BibDatabaseContext getBibDatabaseContext() { - return this.bibDatabaseContext; - } +public record ConnectionLostEvent( + BibDatabaseContext bibDatabaseContext) { } diff --git a/src/main/java/org/jabref/logic/shared/event/SharedEntriesNotPresentEvent.java b/src/main/java/org/jabref/logic/shared/event/SharedEntriesNotPresentEvent.java index 6972e60e90d..cd1f4c2ff85 100644 --- a/src/main/java/org/jabref/logic/shared/event/SharedEntriesNotPresentEvent.java +++ b/src/main/java/org/jabref/logic/shared/event/SharedEntriesNotPresentEvent.java @@ -7,19 +7,9 @@ /** * This event is fired when the user tries to push changes of one or more obsolete * {@link BibEntry} to the server. + * + * @param bibEntries Affected {@link BibEntry} */ -public class SharedEntriesNotPresentEvent { - - private final List bibEntries; - - /** - * @param bibEntries Affected {@link BibEntry} - */ - public SharedEntriesNotPresentEvent(List bibEntries) { - this.bibEntries = bibEntries; - } - - public List getBibEntries() { - return this.bibEntries; - } +public record SharedEntriesNotPresentEvent( + List bibEntries) { } diff --git a/src/main/java/org/jabref/logic/shared/event/UpdateRefusedEvent.java b/src/main/java/org/jabref/logic/shared/event/UpdateRefusedEvent.java index ffced9d8af3..1dce167b7b2 100644 --- a/src/main/java/org/jabref/logic/shared/event/UpdateRefusedEvent.java +++ b/src/main/java/org/jabref/logic/shared/event/UpdateRefusedEvent.java @@ -5,32 +5,12 @@ /** * A new {@link UpdateRefusedEvent} is fired, when the user tries to push changes of an obsolete {@link BibEntry} to the server. + * + * @param bibDatabaseContext Affected {@link BibDatabaseContext} + * @param localBibEntry Affected {@link BibEntry} */ -public class UpdateRefusedEvent { - - private final BibDatabaseContext bibDatabaseContext; - private final BibEntry localBibEntry; - private final BibEntry sharedBibEntry; - - /** - * @param bibDatabaseContext Affected {@link BibDatabaseContext} - * @param localBibEntry Affected {@link BibEntry} - */ - public UpdateRefusedEvent(BibDatabaseContext bibDatabaseContext, BibEntry localBibEntry, BibEntry sharedBibEntry) { - this.bibDatabaseContext = bibDatabaseContext; - this.localBibEntry = localBibEntry; - this.sharedBibEntry = sharedBibEntry; - } - - public BibDatabaseContext getBibDatabaseContext() { - return this.bibDatabaseContext; - } - - public BibEntry getLocalBibEntry() { - return localBibEntry; - } - - public BibEntry getSharedBibEntry() { - return sharedBibEntry; - } +public record UpdateRefusedEvent( + BibDatabaseContext bibDatabaseContext, + BibEntry localBibEntry, + BibEntry sharedBibEntry) { } diff --git a/src/main/java/org/jabref/logic/shared/listener/PostgresSQLNotificationListener.java b/src/main/java/org/jabref/logic/shared/listener/PostgresSQLNotificationListener.java index 96b8b7ccaf6..9d9d8819445 100644 --- a/src/main/java/org/jabref/logic/shared/listener/PostgresSQLNotificationListener.java +++ b/src/main/java/org/jabref/logic/shared/listener/PostgresSQLNotificationListener.java @@ -32,7 +32,7 @@ public void run() { try { // noinspection InfiniteLoopStatement while (!stop) { - PGNotification notifications[] = pgConnection.getNotifications(); + PGNotification[] notifications = pgConnection.getNotifications(); if (notifications != null) { for (PGNotification notification : notifications) { diff --git a/src/test/java/org/jabref/logic/shared/SynchronizationSimulatorTest.java b/src/test/java/org/jabref/logic/shared/SynchronizationSimulatorTest.java index 2558cbdb85f..f4d09b06904 100644 --- a/src/test/java/org/jabref/logic/shared/SynchronizationSimulatorTest.java +++ b/src/test/java/org/jabref/logic/shared/SynchronizationSimulatorTest.java @@ -146,7 +146,7 @@ void simulateUpdateOnNoLongerExistingEntry() throws Exception { // here a new SharedEntryNotPresentEvent has been thrown. In this case the user B would get an pop-up window. assertNotNull(eventListenerB.getSharedEntriesNotPresentEvent()); - assertEquals(List.of(bibEntryOfClientB), eventListenerB.getSharedEntriesNotPresentEvent().getBibEntries()); + assertEquals(List.of(bibEntryOfClientB), eventListenerB.getSharedEntriesNotPresentEvent().bibEntries()); } @Test From 0219ddf34e07f9e0cf43e423eda99946a197383c Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 20 Sep 2024 09:31:35 +0300 Subject: [PATCH 002/104] Add embedded-postgres dependency --- build.gradle | 3 + external-libraries.md | 120 ++++++++++++++++++++++----------- src/main/java/module-info.java | 4 ++ 3 files changed, 86 insertions(+), 41 deletions(-) diff --git a/build.gradle b/build.gradle index cd344966b18..cc764f65bb4 100644 --- a/build.gradle +++ b/build.gradle @@ -352,6 +352,9 @@ dependencies { // Even if "compileOnly" is used, IntelliJ always adds to module-info.java. To avoid issues during committing, we use "implementation" instead of "compileOnly" implementation 'io.github.adr:e-adr:2.0.0-SNAPSHOT' + implementation 'io.zonky.test:embedded-postgres:2.0.7' + implementation enforcedPlatform('io.zonky.test.postgres:embedded-postgres-binaries-bom:16.4.0') + testImplementation 'io.github.classgraph:classgraph:4.8.175' testImplementation 'org.junit.jupiter:junit-jupiter:5.11.0' testImplementation 'org.junit.platform:junit-platform-launcher:1.10.3' diff --git a/external-libraries.md b/external-libraries.md index 73bbbdbc2c7..9b0d4feccf3 100644 --- a/external-libraries.md +++ b/external-libraries.md @@ -356,6 +356,13 @@ URL: https://github.com/java-diff-utils/java-diff-utils License: Apache-2.0 ``` +```yaml +Id: io.zonky.test:embedded-postgres +Project: embedded-postgres +URL: https://github.com/zonkyio/embedded-postgres +License: Apache-2.0 +``` + ```yaml Id: jakarta.annotation:jakarata.annotation-api Project: Jakarta Annotations @@ -699,8 +706,12 @@ License: BSD-3-Clause 3. (on WSL) `sed 's/[^a-z]*//' < build/reports/project/dependencies.txt | sed "s/\(.*\) .*/\1/" | grep -v "\->" | sort | uniq > build/dependencies-for-external-libraries.txt` ```text +ai.djl.huggingface:tokenizers:0.29.0 +ai.djl.pytorch:pytorch-engine:0.30.0 +ai.djl.pytorch:pytorch-model-zoo:0.30.0 +ai.djl:api:0.30.0 at.favre.lib:hkdf:1.1.0 -com.dlsc.gemsfx:gemsfx:2.32.0 +com.dlsc.gemsfx:gemsfx:2.48.0 com.dlsc.pickerfx:pickerfx:1.3.1 com.dlsc.unitfx:unitfx:1.0.10 com.fasterxml.jackson.core:jackson-annotations:2.17.2 @@ -717,29 +728,37 @@ com.github.sialcasa.mvvmFX:mvvmfx-validation:f195849ca9 com.github.tomtung:latex2unicode_2.13:0.3.2 com.github.vatbub:mslinks:1.0.6.2 com.github.weisj:jsvg:1.2.0 -com.google.code.gson:gson:2.10.1 -com.google.errorprone:error_prone_annotations:2.26.1 +com.google.code.gson:gson:2.11.0 +com.google.errorprone:error_prone_annotations:2.27.0 com.google.guava:failureaccess:1.0.2 com.google.guava:guava:33.1.0-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:3.0.0 com.googlecode.javaewah:JavaEWAH:1.2.3 com.googlecode.plist:dd-plist:1.28 -com.h2database:h2-mvstore:2.2.224 -com.kohlschutter.junixsocket:junixsocket-common:2.9.1 -com.kohlschutter.junixsocket:junixsocket-core:2.9.1 -com.kohlschutter.junixsocket:junixsocket-mysql:2.9.1 -com.kohlschutter.junixsocket:junixsocket-native-common:2.9.1 -com.konghq:unirest-java-core:4.4.0 -com.konghq:unirest-modules-gson:4.4.0 +com.h2database:h2-mvstore:2.3.232 +com.knuddels:jtokkit:1.1.0 +com.kohlschutter.junixsocket:junixsocket-common:2.10.0 +com.kohlschutter.junixsocket:junixsocket-core:2.10.0 +com.kohlschutter.junixsocket:junixsocket-mysql:2.10.0 +com.kohlschutter.junixsocket:junixsocket-native-common:2.10.0 +com.konghq:unirest-java-core:4.4.4 +com.konghq:unirest-modules-gson:4.4.4 com.oracle.ojdbc:ojdbc10:19.3.0.0 com.oracle.ojdbc:ons:19.3.0.0 com.oracle.ojdbc:osdt_cert:19.3.0.0 com.oracle.ojdbc:osdt_core:19.3.0.0 com.oracle.ojdbc:simplefan:19.3.0.0 com.oracle.ojdbc:ucp:19.3.0.0 +com.squareup.okhttp3:logging-interceptor:4.12.0 +com.squareup.okhttp3:okhttp-sse:4.12.0 +com.squareup.okhttp3:okhttp:4.12.0 +com.squareup.okio:okio-jvm:3.6.0 +com.squareup.okio:okio:3.6.0 +com.squareup.retrofit2:converter-gson:2.9.0 +com.squareup.retrofit2:converter-jackson:2.9.0 +com.squareup.retrofit2:retrofit:2.11.0 com.sun.istack:istack-commons-runtime:4.1.2 -com.tobiasdiez:easybind:2.2.1-SNAPSHOT com.vladsch.flexmark:flexmark-ext-emoji:0.64.8 com.vladsch.flexmark:flexmark-ext-gfm-strikethrough:0.64.8 com.vladsch.flexmark:flexmark-ext-ins:0.64.8 @@ -762,12 +781,12 @@ com.vladsch.flexmark:flexmark-util-visitor:0.64.8 com.vladsch.flexmark:flexmark-util:0.64.8 com.vladsch.flexmark:flexmark:0.64.8 commons-beanutils:commons-beanutils:1.9.4 -commons-cli:commons-cli:1.8.0 -commons-codec:commons-codec:1.17.0 +commons-cli:commons-cli:1.9.0 +commons-codec:commons-codec:1.17.1 commons-collections:commons-collections:3.2.2 commons-digester:commons-digester:2.1 commons-io:commons-io:2.16.1 -commons-logging:commons-logging:1.3.2 +commons-logging:commons-logging:1.3.4 commons-validator:commons-validator:1.8.0 de.rototor.jeuclid:jeuclid-core:3.1.11 de.rototor.snuggletex:snuggletex-core:1.3.0 @@ -776,9 +795,20 @@ de.rototor.snuggletex:snuggletex:1.3.0 de.saxsys:mvvmfx:1.8.0 de.swiesend:secret-service:1.8.1-jdk17 de.undercouch:citeproc-java:3.1.0 +dev.ai4j:openai4j:0.20.0 +dev.langchain4j:langchain4j-core:0.34.0 +dev.langchain4j:langchain4j-google-ai-gemini:0.34.0 +dev.langchain4j:langchain4j-hugging-face:0.34.0 +dev.langchain4j:langchain4j-mistral-ai:0.34.0 +dev.langchain4j:langchain4j-open-ai:0.34.0 +dev.langchain4j:langchain4j:0.34.0 eu.lestard:doc-annotations:0.2 info.debatty:java-string-similarity:2.0.0 +io.github.adr:e-adr:2.0.0-SNAPSHOT io.github.java-diff-utils:java-diff-utils:4.12 +io.github.stefanbratanov:jvm-openai:0.11.0 +io.github.thibaultmeyer:cuid:2.0.3 +io.zonky.test:embedded-postgres:2.0.7 jakarta.activation:jakarta.activation-api:2.1.3 jakarta.annotation:jakarta.annotation-api:2.1.1 jakarta.inject:jakarta.inject-api:2.0.1 @@ -788,30 +818,32 @@ jakarta.xml.bind:jakarta.xml.bind-api:4.0.2 javax.measure:unit-api:2.2 net.harawata:appdirs:1.2.2 net.java.dev.jna:jna-platform:5.13.0 -net.java.dev.jna:jna:5.13.0 +net.java.dev.jna:jna:5.14.0 net.jcip:jcip-annotations:1.0 net.jodah:typetools:0.6.1 net.synedra:validatorfx:0.5.0 one.jpro.jproutils:tree-showing:0.2.2 -org.antlr:antlr4-runtime:4.13.1 +org.antlr:antlr4-runtime:4.13.2 +org.apache.commons:commons-compress:1.27.1 org.apache.commons:commons-csv:1.11.0 -org.apache.commons:commons-lang3:3.14.0 +org.apache.commons:commons-lang3:3.17.0 org.apache.commons:commons-text:1.12.0 org.apache.httpcomponents.client5:httpclient5:5.3.1 org.apache.httpcomponents.core5:httpcore5-h2:5.2.4 org.apache.httpcomponents.core5:httpcore5:5.2.4 -org.apache.logging.log4j:log4j-api:2.23.1 -org.apache.logging.log4j:log4j-to-slf4j:2.23.1 +org.apache.logging.log4j:log4j-api:2.24.0 +org.apache.logging.log4j:log4j-to-slf4j:2.24.0 org.apache.lucene:lucene-analysis-common:9.11.1 org.apache.lucene:lucene-core:9.11.1 org.apache.lucene:lucene-highlighter:9.11.1 org.apache.lucene:lucene-queries:9.11.1 org.apache.lucene:lucene-queryparser:9.11.1 org.apache.lucene:lucene-sandbox:9.11.1 -org.apache.pdfbox:fontbox:3.0.2 -org.apache.pdfbox:pdfbox-io:3.0.2 -org.apache.pdfbox:pdfbox:3.0.2 -org.apache.pdfbox:xmpbox:3.0.2 +org.apache.opennlp:opennlp-tools:1.9.4 +org.apache.pdfbox:fontbox:3.0.3 +org.apache.pdfbox:pdfbox-io:3.0.3 +org.apache.pdfbox:pdfbox:3.0.3 +org.apache.pdfbox:xmpbox:3.0.3 org.apiguardian:apiguardian-api:1.1.2 org.bouncycastle:bcprov-jdk18on:1.78.1 org.checkerframework:checker-qual:3.42.0 @@ -833,18 +865,23 @@ org.glassfish.hk2:osgi-resource-locator:1.0.3 org.glassfish.jaxb:jaxb-core:4.0.3 org.glassfish.jaxb:jaxb-runtime:4.0.3 org.glassfish.jaxb:txw2:4.0.3 -org.glassfish.jersey.containers:jersey-container-grizzly2-http:3.1.7 -org.glassfish.jersey.core:jersey-client:3.1.7 -org.glassfish.jersey.core:jersey-common:3.1.7 -org.glassfish.jersey.core:jersey-server:3.1.7 -org.glassfish.jersey.inject:jersey-hk2:3.1.7 +org.glassfish.jersey.containers:jersey-container-grizzly2-http:3.1.8 +org.glassfish.jersey.core:jersey-client:3.1.8 +org.glassfish.jersey.core:jersey-common:3.1.8 +org.glassfish.jersey.core:jersey-server:3.1.8 +org.glassfish.jersey.inject:jersey-hk2:3.1.8 org.jabref:afterburner.fx:2.0.0 +org.jabref:easybind:2.2.1-SNAPSHOT org.javassist:javassist:3.30.2-GA org.jbibtex:jbibtex:1.0.20 +org.jetbrains.kotlin:kotlin-stdlib-common:2.0.20 +org.jetbrains.kotlin:kotlin-stdlib-jdk7:2.0.20 +org.jetbrains.kotlin:kotlin-stdlib-jdk8:2.0.20 +org.jetbrains.kotlin:kotlin-stdlib:2.0.20 org.jetbrains:annotations:24.0.1 org.jooq:jool:0.9.15 -org.jsoup:jsoup:1.17.2 -org.jspecify:jspecify:0.3.0 +org.jsoup:jsoup:1.18.1 +org.jspecify:jspecify:1.0.0 org.kordamp.ikonli:ikonli-bootstrapicons-pack:12.3.1 org.kordamp.ikonli:ikonli-core:12.3.1 org.kordamp.ikonli:ikonli-javafx:12.3.1 @@ -854,22 +891,23 @@ org.kordamp.ikonli:ikonli-materialdesign2-pack:12.3.1 org.libreoffice:libreoffice:24.2.3 org.libreoffice:unoloader:24.2.3 org.mariadb.jdbc:mariadb-java-client:2.7.9 -org.openjfx:javafx-base:22.0.1 -org.openjfx:javafx-controls:22.0.1 -org.openjfx:javafx-fxml:22.0.1 -org.openjfx:javafx-graphics:22.0.1 -org.openjfx:javafx-media:22.0.1 -org.openjfx:javafx-swing:22.0.1 -org.openjfx:javafx-web:22.0.1 -org.postgresql:postgresql:42.7.3 +org.openjfx:javafx-base:23 +org.openjfx:javafx-controls:23 +org.openjfx:javafx-fxml:23 +org.openjfx:javafx-graphics:23 +org.openjfx:javafx-media:23 +org.openjfx:javafx-swing:23 +org.openjfx:javafx-web:23 +org.postgresql:postgresql:42.7.4 org.reactfx:reactfx:2.0-M5 org.scala-lang:scala-library:2.13.8 -org.slf4j:jul-to-slf4j:2.0.13 -org.slf4j:slf4j-api:2.0.13 +org.slf4j:jul-to-slf4j:2.0.16 +org.slf4j:slf4j-api:2.0.16 org.tinylog:slf4j-tinylog:2.7.0 org.tinylog:tinylog-api:2.7.0 org.tinylog:tinylog-impl:2.7.0 -org.yaml:snakeyaml:2.2 +org.tukaani:xz:1.9 +org.yaml:snakeyaml:2.3 pt.davidafsilva.apple:jkeychain:1.1.0 tech.units:indriya:2.2 tech.uom.lib:uom-lib-common:2.2 diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index dc1fce79618..be39d323332 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -97,6 +97,8 @@ // endregion // region: SQL databases + requires embedded.postgres; + requires org.tukaani.xz; requires ojdbc10; requires org.postgresql.jdbc; requires org.mariadb.jdbc; @@ -108,6 +110,8 @@ requires io.github.javadiffutils; requires java.string.similarity; requires org.apache.commons.cli; + requires org.apache.commons.codec; + requires org.apache.commons.compress; requires org.apache.commons.csv; requires org.apache.commons.io; requires org.apache.commons.lang3; From 351d01cd516553fa80ad65595dc1a813ac043938 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 22 Sep 2024 09:05:35 +0300 Subject: [PATCH 003/104] Setup embedded postgre server --- src/main/java/org/jabref/Launcher.java | 4 ++ src/main/java/org/jabref/gui/JabRefGUI.java | 4 ++ .../jabref/logic/search/PostgreServer.java | 67 +++++++++++++++++++ .../org/jabref/logic/shared/DBMSType.java | 2 +- src/main/resources/tinylog.properties | 2 + 5 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/jabref/logic/search/PostgreServer.java diff --git a/src/main/java/org/jabref/Launcher.java b/src/main/java/org/jabref/Launcher.java index 1d3cb9cb3e1..da2f9cde7e8 100644 --- a/src/main/java/org/jabref/Launcher.java +++ b/src/main/java/org/jabref/Launcher.java @@ -30,6 +30,7 @@ import org.jabref.logic.protectedterms.ProtectedTermsLoader; import org.jabref.logic.remote.RemotePreferences; import org.jabref.logic.remote.client.RemoteClient; +import org.jabref.logic.search.PostgreServer; import org.jabref.logic.util.BuildInfo; import org.jabref.logic.util.Directories; import org.jabref.logic.util.HeadlessExecutorService; @@ -92,6 +93,9 @@ public static void main(String[] args) { DirectoryMonitor directoryMonitor = new DefaultDirectoryMonitor(); Injector.setModelOrService(DirectoryMonitor.class, directoryMonitor); + PostgreServer postgreServer = new PostgreServer(); + Injector.setModelOrService(PostgreServer.class, postgreServer); + // Process arguments ArgumentProcessor argumentProcessor = new ArgumentProcessor( args, diff --git a/src/main/java/org/jabref/gui/JabRefGUI.java b/src/main/java/org/jabref/gui/JabRefGUI.java index d6a80bf3947..dc464b1cfa1 100644 --- a/src/main/java/org/jabref/gui/JabRefGUI.java +++ b/src/main/java/org/jabref/gui/JabRefGUI.java @@ -32,6 +32,7 @@ import org.jabref.logic.net.ProxyRegisterer; import org.jabref.logic.remote.RemotePreferences; import org.jabref.logic.remote.server.RemoteListenerServerManager; +import org.jabref.logic.search.PostgreServer; import org.jabref.logic.util.BuildInfo; import org.jabref.logic.util.FallbackExceptionHandler; import org.jabref.logic.util.HeadlessExecutorService; @@ -398,6 +399,9 @@ public static void shutdownThreadPools() { LOGGER.trace("Shutting down directoryMonitor"); DirectoryMonitor directoryMonitor = Injector.instantiateModelOrService(DirectoryMonitor.class); directoryMonitor.shutdown(); + LOGGER.trace("Shutting down postgreServer"); + PostgreServer postgreServer = Injector.instantiateModelOrService(PostgreServer.class); + postgreServer.shutdown(); LOGGER.trace("Shutting down HeadlessExecutorService"); HeadlessExecutorService.INSTANCE.shutdownEverything(); LOGGER.trace("Finished shutdownThreadPools"); diff --git a/src/main/java/org/jabref/logic/search/PostgreServer.java b/src/main/java/org/jabref/logic/search/PostgreServer.java new file mode 100644 index 00000000000..8ecf2de4dbd --- /dev/null +++ b/src/main/java/org/jabref/logic/search/PostgreServer.java @@ -0,0 +1,67 @@ +package org.jabref.logic.search; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; + +import javax.sql.DataSource; + +import io.zonky.test.db.postgres.embedded.EmbeddedPostgres; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PostgreServer { + private static final Logger LOGGER = LoggerFactory.getLogger(PostgreServer.class); + private final EmbeddedPostgres embeddedPostgres; + private final DataSource dataSource; + + public PostgreServer() { + EmbeddedPostgres embeddedPostgres; + try { + embeddedPostgres = EmbeddedPostgres.builder().start(); + LOGGER.info("Postgres server started, connection port: {}", embeddedPostgres.getPort()); + } catch (IOException e) { + LOGGER.error("Could not start Postgres server", e); + this.embeddedPostgres = null; + this.dataSource = null; + return; + } + + this.embeddedPostgres = embeddedPostgres; + // TODO: Use the default database (postgres) and the default schema (public) or create a new one? + this.dataSource = embeddedPostgres.getPostgresDatabase(); + addTrigramExtension(); + } + + private void addTrigramExtension() { + try (Connection connection = getConnection()) { + if (connection != null) { + LOGGER.debug("Adding trigram extension to Postgres server"); + connection.createStatement().execute("CREATE EXTENSION IF NOT EXISTS pg_trgm"); + } + } catch (SQLException e) { + LOGGER.error("Could not add trigram extension to Postgres server", e); + } + } + + public Connection getConnection() { + if (dataSource != null) { + try { + return dataSource.getConnection(); + } catch (SQLException e) { + LOGGER.error("Could not get connection to Postgres server", e); + } + } + return null; + } + + public void shutdown() { + if (embeddedPostgres != null) { + try { + embeddedPostgres.close(); + } catch (IOException e) { + LOGGER.error("Could not shutdown Postgres server", e); + } + } + } +} diff --git a/src/main/java/org/jabref/logic/shared/DBMSType.java b/src/main/java/org/jabref/logic/shared/DBMSType.java index 8a3717abaf0..ebb49730b9e 100644 --- a/src/main/java/org/jabref/logic/shared/DBMSType.java +++ b/src/main/java/org/jabref/logic/shared/DBMSType.java @@ -13,7 +13,7 @@ public enum DBMSType { private final String type; private final String driverPath; - private String urlPattern; + private final String urlPattern; private final int defaultPort; DBMSType(String type, String driverPath, String urlPattern, int defaultPort) { diff --git a/src/main/resources/tinylog.properties b/src/main/resources/tinylog.properties index 0d5ea5c4a90..2b4bd0c3c28 100644 --- a/src/main/resources/tinylog.properties +++ b/src/main/resources/tinylog.properties @@ -30,3 +30,5 @@ level@org.jabref.http.server.Server = debug #level@org.jabref.logic.ai.FileEmbeddingsManager = trace #level@org.jabref.logic.ai.impl.embeddings.LowLevelIngestor = trace #level@org.jabref.logic.ai.chat.AiChatLogic = trace +level@org.jabref.logic.search = debug +level@org.jabref.model.search = debug From 9efee369df610c1f019e2b2e2400d810a5367baa Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 22 Sep 2024 14:00:00 +0300 Subject: [PATCH 004/104] Indexing entries with Postgres --- src/main/java/org/jabref/gui/LibraryTab.java | 3 +- .../jabref/logic/search/LuceneManager.java | 26 ++- .../jabref/logic/search/PostgreServer.java | 4 +- .../search/indexing/PostgreConstants.java | 22 ++ .../logic/search/indexing/PostgreIndexer.java | 200 ++++++++++++++++++ .../model/database/BibDatabaseContext.java | 8 + 6 files changed, 251 insertions(+), 12 deletions(-) create mode 100644 src/main/java/org/jabref/logic/search/indexing/PostgreConstants.java create mode 100644 src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 93901045848..d14184ab0b2 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -92,7 +92,6 @@ import org.jabref.model.entry.event.FieldChangedEvent; import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.FieldFactory; -import org.jabref.model.entry.field.StandardField; import org.jabref.model.groups.GroupTreeNode; import org.jabref.model.search.SearchQuery; import org.jabref.model.util.DirectoryMonitor; @@ -1150,7 +1149,7 @@ public void listen(EntriesRemovedEvent removedEntriesEvent) { @Subscribe public void listen(FieldChangedEvent fieldChangedEvent) { - luceneManager.updateEntry(fieldChangedEvent.getBibEntry(), fieldChangedEvent.getOldValue(), fieldChangedEvent.getNewValue(), fieldChangedEvent.getField().equals(StandardField.FILE)); + luceneManager.updateEntry(fieldChangedEvent); } } diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index 2d0c56f2fcd..2996a1c010f 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -10,12 +10,15 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.search.indexing.BibFieldsIndexer; import org.jabref.logic.search.indexing.DefaultLinkedFilesIndexer; +import org.jabref.logic.search.indexing.PostgreIndexer; import org.jabref.logic.search.indexing.ReadOnlyLinkedFilesIndexer; import org.jabref.logic.search.retrieval.LuceneSearcher; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.event.FieldChangedEvent; +import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; @@ -23,6 +26,7 @@ import org.jabref.model.search.event.IndexRemovedEvent; import org.jabref.model.search.event.IndexStartedEvent; +import com.airhacks.afterburner.injection.Injector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +41,7 @@ public class LuceneManager { private final LuceneIndexer bibFieldsIndexer; private final LuceneIndexer linkedFilesIndexer; private final LuceneSearcher luceneSearcher; + private final PostgreIndexer postgreIndexer; public LuceneManager(BibDatabaseContext databaseContext, TaskExecutor executor, FilePreferences preferences) { this.taskExecutor = executor; @@ -57,6 +62,8 @@ public LuceneManager(BibDatabaseContext databaseContext, TaskExecutor executor, linkedFilesIndexer = indexer; this.luceneSearcher = new LuceneSearcher(databaseContext, bibFieldsIndexer, linkedFilesIndexer, preferences); + PostgreServer postgreServer = Injector.instantiateModelOrService(PostgreServer.class); + postgreIndexer = new PostgreIndexer(databaseContext, postgreServer.getConnection()); updateOnStart(); } @@ -78,7 +85,7 @@ private void updateOnStart() { new BackgroundTask<>() { @Override public Object call() { - bibFieldsIndexer.updateOnStart(this); + postgreIndexer.updateOnStart(this); return null; } }.showToUser(true) @@ -101,7 +108,7 @@ public void addToIndex(List entries) { new BackgroundTask<>() { @Override public Object call() { - bibFieldsIndexer.addToIndex(entries, this); + postgreIndexer.addToIndex(entries, this); return null; } }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(entries))) @@ -122,7 +129,7 @@ public void removeFromIndex(List entries) { new BackgroundTask<>() { @Override public Object call() { - bibFieldsIndexer.removeFromIndex(entries, this); + postgreIndexer.removeFromIndex(entries, this); return null; } }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexRemovedEvent(entries))) @@ -139,21 +146,21 @@ public Object call() { } } - public void updateEntry(BibEntry entry, String oldValue, String newValue, boolean isLinkedFile) { + public void updateEntry(FieldChangedEvent event) { new BackgroundTask<>() { @Override public Object call() { - bibFieldsIndexer.updateEntry(entry, oldValue, newValue, this); + postgreIndexer.updateEntry(event.getBibEntry(), event.getField()); return null; } - }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(List.of(entry)))) + }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(List.of(event.getBibEntry())))) .executeWith(taskExecutor); - if (isLinkedFile && shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { + if (shouldIndexLinkedFiles.get() && event.getField().equals(StandardField.FILE) && !isLinkedFilesIndexerBlocked.get()) { new BackgroundTask<>() { @Override public Object call() { - linkedFilesIndexer.updateEntry(entry, oldValue, newValue, this); + linkedFilesIndexer.updateEntry(event.getBibEntry(), event.getOldValue(), event.getNewValue(), this); return null; } }.executeWith(taskExecutor); @@ -164,7 +171,7 @@ public void updateAfterDropFiles(BibEntry entry) { new BackgroundTask<>() { @Override public Object call() { - bibFieldsIndexer.updateEntry(entry, "", "", this); + postgreIndexer.updateEntry(entry, StandardField.FILE); return null; } }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(List.of(entry)))) @@ -206,6 +213,7 @@ public void close() { bibFieldsIndexer.close(); shouldIndexLinkedFiles.removeListener(preferencesListener); linkedFilesIndexer.close(); + postgreIndexer.close(); databaseContext.getDatabase().postEvent(new IndexClosedEvent()); } diff --git a/src/main/java/org/jabref/logic/search/PostgreServer.java b/src/main/java/org/jabref/logic/search/PostgreServer.java index 8ecf2de4dbd..699dad540da 100644 --- a/src/main/java/org/jabref/logic/search/PostgreServer.java +++ b/src/main/java/org/jabref/logic/search/PostgreServer.java @@ -18,7 +18,9 @@ public class PostgreServer { public PostgreServer() { EmbeddedPostgres embeddedPostgres; try { - embeddedPostgres = EmbeddedPostgres.builder().start(); + embeddedPostgres = EmbeddedPostgres.builder() + .setPort(5033) + .start(); LOGGER.info("Postgres server started, connection port: {}", embeddedPostgres.getPort()); } catch (IOException e) { LOGGER.error("Could not start Postgres server", e); diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreConstants.java b/src/main/java/org/jabref/logic/search/indexing/PostgreConstants.java new file mode 100644 index 00000000000..375a5216c7b --- /dev/null +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreConstants.java @@ -0,0 +1,22 @@ +package org.jabref.logic.search.indexing; + +public enum PostgreConstants { + ENTRY_ID("entry_id"), + FIELD_NAME("field_name"), + FIELD_VALUE("field_value"); + + private final String columnName; + + PostgreConstants(String columnName) { + this.columnName = columnName; + } + + public String getIndexName(String tableName) { + return String.format("%s_%s_index", tableName, this.columnName); + } + + @Override + public String toString() { + return columnName; + } +} diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java new file mode 100644 index 00000000000..4db760cef8d --- /dev/null +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -0,0 +1,200 @@ +package org.jabref.logic.search.indexing; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Collection; +import java.util.Map; + +import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.BackgroundTask; +import org.jabref.logic.util.HeadlessExecutorService; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.Field; +import org.jabref.model.search.SearchFieldConstants; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PostgreIndexer { + private static final Logger LOGGER = LoggerFactory.getLogger(PostgreIndexer.class); + private static int NUMBER_OF_UNSAVED_LIBRARIES = 1; + + private final BibDatabaseContext databaseContext; + private final Connection connection; + private final String libraryName; + private final String tableName; + + public PostgreIndexer(BibDatabaseContext databaseContext, Connection connection) { + this.databaseContext = databaseContext; + this.connection = connection; + this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElse("unsaved"); + if ("unsaved".equals(databaseContext.getPostgreTableName())) { + this.tableName = "unsaved" + NUMBER_OF_UNSAVED_LIBRARIES++; + } else { + this.tableName = databaseContext.getPostgreTableName(); + } + setup(); + } + + /** + * Creates a table for the library in the database, and sets up indexes on the columns. + */ + private void setup() { + try { + connection.createStatement().executeUpdate(""" + CREATE TABLE IF NOT EXISTS "%s" ( + %s TEXT NOT NULL, + %s TEXT NOT NULL, + %s TEXT, + PRIMARY KEY (%s, %s) + ) + """.formatted(tableName, + PostgreConstants.ENTRY_ID, + PostgreConstants.FIELD_NAME, + PostgreConstants.FIELD_VALUE, + PostgreConstants.ENTRY_ID, + PostgreConstants.FIELD_NAME)); + LOGGER.debug("Created table for library: {}", libraryName); + + // btree index on id column + connection.createStatement().executeUpdate(""" + CREATE INDEX "%s" ON "%s" ("%s") + """.formatted(PostgreConstants.ENTRY_ID.getIndexName(tableName), tableName, PostgreConstants.ENTRY_ID)); + + // btree index on field name column + connection.createStatement().executeUpdate(""" + CREATE INDEX "%s" ON "%s" ("%s") + """.formatted(PostgreConstants.FIELD_NAME.getIndexName(tableName), tableName, PostgreConstants.FIELD_NAME)); + + // trigram index on field value column + connection.createStatement().executeUpdate(""" + CREATE INDEX "%s" ON "%s" USING gin ("%s" gin_trgm_ops) + """.formatted(PostgreConstants.FIELD_VALUE.getIndexName(tableName), tableName, PostgreConstants.FIELD_VALUE)); + + LOGGER.debug("Created indexes for library: {}", libraryName); + } catch (SQLException e) { + LOGGER.error("Could not create table for library: {}", libraryName, e); + } + } + + public void updateOnStart(BackgroundTask task) { + addToIndex(databaseContext.getDatabase().getEntries(), task); + } + + public void addToIndex(Collection entries, BackgroundTask task) { + task.setTitle(Localization.lang("Indexing bib fields for %0", libraryName)); + int i = 1; + long startTime = System.currentTimeMillis(); + LOGGER.debug("Adding {} entries to index", entries.size()); + for (BibEntry entry : entries) { + if (task.isCancelled()) { + LOGGER.debug("Indexing canceled"); + return; + } + addToIndex(entry); + task.updateProgress(i, entries.size()); + task.updateMessage(Localization.lang("%0 of %1 entries added to the index.", i, entries.size())); + i++; + } + LOGGER.debug("Added {} entries to index in {} ms", entries.size(), System.currentTimeMillis() - startTime); + } + + private void addToIndex(BibEntry bibEntry) { + String insertFieldQuery = """ + INSERT INTO "%s" ("%s", "%s", "%s") + VALUES (?, ?, ?) + """.formatted(tableName, + PostgreConstants.ENTRY_ID, + PostgreConstants.FIELD_NAME, + PostgreConstants.FIELD_VALUE); + + try (PreparedStatement preparedStatement = connection.prepareStatement(insertFieldQuery)) { + String entryId = bibEntry.getId(); + for (Map.Entry field : bibEntry.getFieldMap().entrySet()) { + preparedStatement.setString(1, entryId); + preparedStatement.setString(2, field.getKey().getName()); + preparedStatement.setString(3, field.getValue()); + preparedStatement.addBatch(); + } + + // add entry type + preparedStatement.setString(1, entryId); + preparedStatement.setString(2, SearchFieldConstants.ENTRY_TYPE.toString()); + preparedStatement.setString(3, bibEntry.getType().getName()); + preparedStatement.addBatch(); + + preparedStatement.executeBatch(); + } catch (SQLException e) { + LOGGER.error("Could not add an entry to the index.", e); + } + } + + public void removeFromIndex(Collection entries, BackgroundTask task) { + task.setTitle(Localization.lang("Removing entries from index for %0", libraryName)); + int i = 1; + for (BibEntry entry : entries) { + if (task.isCancelled()) { + LOGGER.debug("Removing entries canceled"); + return; + } + removeFromIndex(entry); + task.updateProgress(i, entries.size()); + task.updateMessage(Localization.lang("%0 of %1 entries removed from the index.", i, entries.size())); + i++; + } + } + + private void removeFromIndex(BibEntry entry) { + try { + connection.createStatement().executeUpdate(""" + DELETE FROM "%s" + WHERE "%s" = '%s' + """.formatted(tableName, PostgreConstants.ENTRY_ID, entry.getId())); + LOGGER.debug("Entry {} removed from index", entry.getId()); + } catch (SQLException e) { + LOGGER.error("Error deleting entry from index", e); + } + } + + public void updateEntry(BibEntry entry, Field field) { + try { + // Use upsert to add the field to the index if it doesn't exist, or update it if it does + connection.createStatement().executeUpdate(""" + INSERT INTO "%s" ("%s", "%s", "%s") + VALUES ('%s', '%s', '%s') + ON CONFLICT ("%s", "%s") DO UPDATE + SET "%s" = EXCLUDED."%s" + """.formatted(tableName, + PostgreConstants.ENTRY_ID, + PostgreConstants.FIELD_NAME, + PostgreConstants.FIELD_VALUE, + entry.getId(), + field.getName(), + entry.getField(field).orElse(""), + PostgreConstants.ENTRY_ID, + PostgreConstants.FIELD_NAME, + PostgreConstants.FIELD_VALUE, + PostgreConstants.FIELD_VALUE)); + + LOGGER.debug("Updated entry {} in index", entry.getId()); + } catch (SQLException e) { + LOGGER.error("Error updating entry in index", e); + } + } + + public void close() { + HeadlessExecutorService.INSTANCE.execute(() -> { + try { + LOGGER.debug("Closing connection to Postgres server for library: {}", libraryName); + connection.createStatement().executeUpdate(""" + DROP TABLE IF EXISTS "%s" + """.formatted(tableName)); + connection.close(); + } catch (SQLException e) { + LOGGER.error("Could not drop table for library: {}", libraryName, e); + } + }); + } +} diff --git a/src/main/java/org/jabref/model/database/BibDatabaseContext.java b/src/main/java/org/jabref/model/database/BibDatabaseContext.java index 9d34e8398e8..bafbf49a2e5 100644 --- a/src/main/java/org/jabref/model/database/BibDatabaseContext.java +++ b/src/main/java/org/jabref/model/database/BibDatabaseContext.java @@ -274,6 +274,14 @@ public Path getFulltextIndexPath() { return indexPath; } + public String getPostgreTableName() { + if (getDatabasePath().isPresent()) { + Path databasePath = getDatabasePath().get(); + return BackupFileUtil.getUniqueFilePrefix(databasePath) + "--" + databasePath.getFileName(); + } + return "unsaved"; + } + @Override public String toString() { return "BibDatabaseContext{" + From 24e222ab913da7156508d69b5d4707d4c3dec569 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 22 Sep 2024 14:18:30 +0300 Subject: [PATCH 005/104] Remove Lucene bib fields indexer --- .../gui/maintable/MainTableDataModel.java | 2 - .../RebuildFulltextSearchIndexAction.java | 2 +- .../jabref/logic/search/LuceneManager.java | 30 +-- .../search/indexing/BibFieldsIndexer.java | 175 ---------------- .../logic/search/indexing/PostgreIndexer.java | 24 ++- .../search/retrieval/LinkedFilesSearcher.java | 124 +++++++++++ .../search/retrieval/LuceneSearcher.java | 195 ------------------ 7 files changed, 148 insertions(+), 404 deletions(-) delete mode 100644 src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java create mode 100644 src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java delete mode 100644 src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 5339d4220b7..d0a7aae33e6 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -51,7 +51,6 @@ public class MainTableDataModel { private final SearchPreferences searchPreferences; private final NameDisplayPreferences nameDisplayPreferences; private final BibDatabaseContext bibDatabaseContext; - private final StateManager stateManager; private final TaskExecutor taskExecutor; private final Subscription searchQuerySubscription; private final Subscription searchDisplayModeSubscription; @@ -75,7 +74,6 @@ public MainTableDataModel(BibDatabaseContext context, this.searchPreferences = preferences.getSearchPreferences(); this.nameDisplayPreferences = preferences.getNameDisplayPreferences(); this.taskExecutor = taskExecutor; - this.stateManager = stateManager; this.luceneManager = luceneManager; this.bibDatabaseContext = context; this.searchQueryProperty = searchQueryProperty; diff --git a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java index c623edb60e4..73ebef568b1 100644 --- a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java +++ b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java @@ -53,6 +53,6 @@ private void rebuildIndex() { if (!shouldContinue || stateManager.getActiveDatabase().isEmpty()) { return; } - tabSupplier.get().getLuceneManager().rebuildIndex(); + tabSupplier.get().getLuceneManager().rebuildFullTextIndex(); } } diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index 2996a1c010f..594b0e48c0c 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -8,11 +8,10 @@ import javafx.beans.value.ChangeListener; import org.jabref.logic.FilePreferences; -import org.jabref.logic.search.indexing.BibFieldsIndexer; import org.jabref.logic.search.indexing.DefaultLinkedFilesIndexer; import org.jabref.logic.search.indexing.PostgreIndexer; import org.jabref.logic.search.indexing.ReadOnlyLinkedFilesIndexer; -import org.jabref.logic.search.retrieval.LuceneSearcher; +import org.jabref.logic.search.retrieval.LinkedFilesSearcher; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; @@ -38,9 +37,8 @@ public class LuceneManager { private final BooleanProperty shouldIndexLinkedFiles; private final BooleanProperty isLinkedFilesIndexerBlocked = new SimpleBooleanProperty(false); private final ChangeListener preferencesListener; - private final LuceneIndexer bibFieldsIndexer; private final LuceneIndexer linkedFilesIndexer; - private final LuceneSearcher luceneSearcher; + private final LinkedFilesSearcher linkedFilesSearcher; private final PostgreIndexer postgreIndexer; public LuceneManager(BibDatabaseContext databaseContext, TaskExecutor executor, FilePreferences preferences) { @@ -50,8 +48,6 @@ public LuceneManager(BibDatabaseContext databaseContext, TaskExecutor executor, this.preferencesListener = (observable, oldValue, newValue) -> bindToPreferences(newValue); this.shouldIndexLinkedFiles.addListener(preferencesListener); - this.bibFieldsIndexer = new BibFieldsIndexer(databaseContext); - LuceneIndexer indexer; try { indexer = new DefaultLinkedFilesIndexer(databaseContext, preferences); @@ -61,7 +57,7 @@ public LuceneManager(BibDatabaseContext databaseContext, TaskExecutor executor, } linkedFilesIndexer = indexer; - this.luceneSearcher = new LuceneSearcher(databaseContext, bibFieldsIndexer, linkedFilesIndexer, preferences); + this.linkedFilesSearcher = new LinkedFilesSearcher(databaseContext, linkedFilesIndexer, preferences); PostgreServer postgreServer = Injector.instantiateModelOrService(PostgreServer.class); postgreIndexer = new PostgreIndexer(databaseContext, postgreServer.getConnection()); updateOnStart(); @@ -188,16 +184,7 @@ public Object call() { } } - public void rebuildIndex() { - new BackgroundTask<>() { - @Override - public Object call() { - bibFieldsIndexer.rebuildIndex(this); - return null; - } - }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexStartedEvent())) - .showToUser(true).executeWith(taskExecutor); - + public void rebuildFullTextIndex() { if (shouldIndexLinkedFiles.get()) { new BackgroundTask<>() { @Override @@ -210,15 +197,14 @@ public Object call() { } public void close() { - bibFieldsIndexer.close(); + postgreIndexer.close(); shouldIndexLinkedFiles.removeListener(preferencesListener); linkedFilesIndexer.close(); - postgreIndexer.close(); databaseContext.getDatabase().postEvent(new IndexClosedEvent()); } public void closeAndWait() { - bibFieldsIndexer.closeAndWait(); + postgreIndexer.closeAndWait(); shouldIndexLinkedFiles.removeListener(preferencesListener); linkedFilesIndexer.closeAndWait(); databaseContext.getDatabase().postEvent(new IndexClosedEvent()); @@ -232,7 +218,7 @@ public AutoCloseable blockLinkedFileIndexer() { public SearchResults search(SearchQuery query) { if (query.isValid()) { - query.setSearchResults(luceneSearcher.search(query.getParsedQuery(), query.getSearchFlags())); + query.setSearchResults(linkedFilesSearcher.search(query.getParsedQuery(), query.getSearchFlags())); } else { query.setSearchResults(new SearchResults()); } @@ -240,6 +226,6 @@ public SearchResults search(SearchQuery query) { } public boolean isEntryMatched(BibEntry entry, SearchQuery query) { - return luceneSearcher.isEntryMatched(entry, query); + return true; } } diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java deleted file mode 100644 index eb0141f3f92..00000000000 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ /dev/null @@ -1,175 +0,0 @@ -package org.jabref.logic.search.indexing; - -import java.io.IOException; -import java.util.Collection; -import java.util.Map; - -import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.LuceneIndexer; -import org.jabref.logic.util.BackgroundTask; -import org.jabref.logic.util.HeadlessExecutorService; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.Field; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.search.SearchFieldConstants; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.SearcherManager; -import org.apache.lucene.store.ByteBuffersDirectory; -import org.apache.lucene.store.Directory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BibFieldsIndexer implements LuceneIndexer { - private static final Logger LOGGER = LoggerFactory.getLogger(BibFieldsIndexer.class); - private final BibDatabaseContext databaseContext; - private final String libraryName; - private final Directory indexDirectory; - private IndexWriter indexWriter; - private SearcherManager searcherManager; - - public BibFieldsIndexer(BibDatabaseContext databaseContext) { - this.databaseContext = databaseContext; - this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElse("unsaved"); - - IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.LATEX_AWARE_NGRAM_ANALYZER); - - this.indexDirectory = new ByteBuffersDirectory(); - try { - this.indexWriter = new IndexWriter(indexDirectory, config); - this.searcherManager = new SearcherManager(indexWriter, null); - } catch (IOException e) { - LOGGER.error("Error initializing bib fields index", e); - } - } - - @Override - public void updateOnStart(BackgroundTask task) { - addToIndex(databaseContext.getDatabase().getEntries(), task); - } - - @Override - public void addToIndex(Collection entries, BackgroundTask task) { - task.setTitle(Localization.lang("Indexing bib fields for %0", libraryName)); - int i = 1; - long startTime = System.currentTimeMillis(); - LOGGER.debug("Adding {} entries to index", entries.size()); - for (BibEntry entry : entries) { - if (task.isCancelled()) { - LOGGER.debug("Indexing canceled"); - return; - } - addToIndex(entry); - task.updateProgress(i, entries.size()); - task.updateMessage(Localization.lang("%0 of %1 entries added to the index.", i, entries.size())); - i++; - } - LOGGER.debug("Added {} entries to index in {} ms", entries.size(), System.currentTimeMillis() - startTime); - } - - private void addToIndex(BibEntry bibEntry) { - try { - Document document = new Document(); - org.apache.lucene.document.Field.Store storeEnabled = org.apache.lucene.document.Field.Store.YES; - org.apache.lucene.document.Field.Store storeDisabled = org.apache.lucene.document.Field.Store.NO; - document.add(new StringField(SearchFieldConstants.ENTRY_ID.toString(), bibEntry.getId(), storeEnabled)); - document.add(new TextField(SearchFieldConstants.ENTRY_TYPE.toString(), bibEntry.getType().getName(), storeDisabled)); - - StringBuilder allFields = new StringBuilder(bibEntry.getType().getName()); - for (Map.Entry mapEntry : bibEntry.getFieldMap().entrySet()) { - document.add(new TextField(mapEntry.getKey().getName(), mapEntry.getValue(), storeDisabled)); - if (mapEntry.getKey().equals(StandardField.GROUPS)) { - // Do not add groups to the allFields field: https://github.com/JabRef/jabref/issues/7996 - continue; - } - allFields.append('\n').append(mapEntry.getValue()); - } - document.add(new TextField(SearchFieldConstants.DEFAULT_FIELD.toString(), allFields.toString(), storeDisabled)); - indexWriter.addDocument(document); - } catch (IOException e) { - LOGGER.warn("Could not add an entry to the index.", e); - } - } - - @Override - public void removeFromIndex(Collection entries, BackgroundTask task) { - task.setTitle(Localization.lang("Removing entries from index for %0", libraryName)); - int i = 1; - for (BibEntry entry : entries) { - if (task.isCancelled()) { - LOGGER.debug("Removing entries canceled"); - return; - } - removeFromIndex(entry); - task.updateProgress(i, entries.size()); - task.updateMessage(Localization.lang("%0 of %1 entries removed from the index.", i, entries.size())); - i++; - } - } - - private void removeFromIndex(BibEntry entry) { - try { - indexWriter.deleteDocuments((new Term(SearchFieldConstants.ENTRY_ID.toString(), entry.getId()))); - LOGGER.debug("Entry {} removed from index", entry.getId()); - } catch (IOException e) { - LOGGER.error("Error deleting entry from index", e); - } - } - - @Override - public void updateEntry(BibEntry entry, String oldValue, String newValue, BackgroundTask task) { - LOGGER.debug("Updating entry {} in index", entry.getId()); - removeFromIndex(entry); - addToIndex(entry); - } - - @Override - public void removeAllFromIndex() { - try { - LOGGER.debug("Removing all bib fields from index"); - indexWriter.deleteAll(); - LOGGER.debug("All bib fields removed from index"); - } catch (IOException e) { - LOGGER.error("Error deleting all linked files from index", e); - } - } - - @Override - public void rebuildIndex(BackgroundTask task) { - removeAllFromIndex(); - addToIndex(databaseContext.getDatabase().getEntries(), task); - } - - @Override - public SearcherManager getSearcherManager() { - return searcherManager; - } - - @Override - public void close() { - HeadlessExecutorService.INSTANCE.execute(this::closeIndex); - } - - @Override - public void closeAndWait() { - HeadlessExecutorService.INSTANCE.executeAndWait(this::closeIndex); - } - - private void closeIndex() { - try { - LOGGER.debug("Closing bib fields index"); - searcherManager.close(); - indexWriter.close(); - indexDirectory.close(); - LOGGER.debug("Bib fields index closed"); - } catch (IOException e) { - LOGGER.error("Error while closing bib fields index", e); - } - } -} diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java index 4db760cef8d..77542aed06a 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -185,16 +185,22 @@ ON CONFLICT ("%s", "%s") DO UPDATE } public void close() { - HeadlessExecutorService.INSTANCE.execute(() -> { - try { - LOGGER.debug("Closing connection to Postgres server for library: {}", libraryName); - connection.createStatement().executeUpdate(""" + HeadlessExecutorService.INSTANCE.execute(this::closeIndex); + } + + public void closeAndWait() { + HeadlessExecutorService.INSTANCE.executeAndWait(this::closeIndex); + } + + private void closeIndex() { + try { + LOGGER.debug("Closing connection to Postgres server for library: {}", libraryName); + connection.createStatement().executeUpdate(""" DROP TABLE IF EXISTS "%s" """.formatted(tableName)); - connection.close(); - } catch (SQLException e) { - LOGGER.error("Could not drop table for library: {}", libraryName, e); - } - }); + connection.close(); + } catch (SQLException e) { + LOGGER.error("Could not drop table for library: {}", libraryName, e); + } } } diff --git a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java new file mode 100644 index 00000000000..5fbbe714434 --- /dev/null +++ b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java @@ -0,0 +1,124 @@ +package org.jabref.logic.search.retrieval; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.jabref.logic.FilePreferences; +import org.jabref.logic.search.LuceneIndexer; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.LinkedFile; +import org.jabref.model.search.SearchFieldConstants; +import org.jabref.model.search.SearchFlags; +import org.jabref.model.search.SearchResult; +import org.jabref.model.search.SearchResults; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.StoredFields; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.SearcherManager; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.highlight.Highlighter; +import org.apache.lucene.search.highlight.QueryScorer; +import org.apache.lucene.search.highlight.SimpleHTMLFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class LinkedFilesSearcher { + private static final Logger LOGGER = LoggerFactory.getLogger(LinkedFilesSearcher.class); + + private final FilePreferences filePreferences; + private final BibDatabaseContext databaseContext; + private final SearcherManager searcherManager; + + public LinkedFilesSearcher(BibDatabaseContext databaseContext, LuceneIndexer linkedFilesIndexer, FilePreferences filePreferences) { + this.searcherManager = linkedFilesIndexer.getSearcherManager(); + this.databaseContext = databaseContext; + this.filePreferences = filePreferences; + } + + public SearchResults search(Query searchQuery, EnumSet searchFlags) { + boolean shouldSearchInLinkedFiles = searchFlags.contains(SearchFlags.FULLTEXT) && filePreferences.shouldFulltextIndexLinkedFiles(); + if (!shouldSearchInLinkedFiles) { + return new SearchResults(); + } + + LOGGER.debug("Searching in linked files with query: {}", searchQuery); + try { + IndexSearcher linkedFilesIndexSearcher = acquireIndexSearcher(searcherManager); + SearchResults searchResults = search(linkedFilesIndexSearcher, searchQuery); + releaseIndexSearcher(searcherManager, linkedFilesIndexSearcher); + return searchResults; + } catch (IOException | IndexSearcher.TooManyClauses e) { + LOGGER.error("Error during linked files search execution", e); + } + return new SearchResults(); + } + + private SearchResults search(IndexSearcher indexSearcher, Query searchQuery) throws IOException { + TopDocs topDocs = indexSearcher.search(searchQuery, Integer.MAX_VALUE); + StoredFields storedFields = indexSearcher.storedFields(); + LOGGER.debug("Found {} matching documents", topDocs.totalHits.value); + return getSearchResults(topDocs, storedFields, searchQuery); + } + + private SearchResults getSearchResults(TopDocs topDocs, StoredFields storedFields, Query searchQuery) throws IOException { + SearchResults searchResults = new SearchResults(); + long startTime = System.currentTimeMillis(); + + Map> linkedFilesMap = getLinkedFilesMap(); + Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter("", ""), new QueryScorer(searchQuery)); + + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { + Document document = storedFields.document(scoreDoc.doc); + String fileLink = getFieldContents(document, SearchFieldConstants.PATH); + + if (!fileLink.isEmpty()) { + List entriesWithFile = linkedFilesMap.get(fileLink); + if (entriesWithFile != null && !entriesWithFile.isEmpty()) { + SearchResult searchResult = new SearchResult( + scoreDoc.score, + fileLink, + getFieldContents(document, SearchFieldConstants.CONTENT), + getFieldContents(document, SearchFieldConstants.ANNOTATIONS), + Integer.parseInt(getFieldContents(document, SearchFieldConstants.PAGE_NUMBER)), + highlighter); + searchResults.addSearchResult(entriesWithFile, searchResult); + } + } + } + LOGGER.debug("Getting linked files results took {} ms", System.currentTimeMillis() - startTime); + return searchResults; + } + + private Map> getLinkedFilesMap() { + // fileLink to List of entry IDs + Map> linkedFilesMap = new HashMap<>(); + for (BibEntry bibEntry : databaseContext.getEntries()) { + for (LinkedFile linkedFile : bibEntry.getFiles()) { + linkedFilesMap.computeIfAbsent(linkedFile.getLink(), k -> new ArrayList<>()).add(bibEntry.getId()); + } + } + return linkedFilesMap; + } + + private static String getFieldContents(Document document, SearchFieldConstants field) { + return Optional.ofNullable(document.get(field.toString())).orElse(""); + } + + private static IndexSearcher acquireIndexSearcher(SearcherManager searcherManager) throws IOException { + searcherManager.maybeRefreshBlocking(); + return searcherManager.acquire(); + } + + private static void releaseIndexSearcher(SearcherManager searcherManager, IndexSearcher indexSearcher) throws IOException { + searcherManager.release(indexSearcher); + } +} diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java deleted file mode 100644 index 944cfad66ce..00000000000 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ /dev/null @@ -1,195 +0,0 @@ -package org.jabref.logic.search.retrieval; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import org.jabref.logic.FilePreferences; -import org.jabref.logic.search.LuceneIndexer; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.LinkedFile; -import org.jabref.model.search.SearchFieldConstants; -import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.SearchQuery; -import org.jabref.model.search.SearchResult; -import org.jabref.model.search.SearchResults; - -import org.apache.lucene.document.Document; -import org.apache.lucene.index.MultiReader; -import org.apache.lucene.index.StoredFields; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.SearcherManager; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.highlight.Highlighter; -import org.apache.lucene.search.highlight.QueryScorer; -import org.apache.lucene.search.highlight.SimpleHTMLFormatter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public final class LuceneSearcher { - private static final Logger LOGGER = LoggerFactory.getLogger(LuceneSearcher.class); - - private final FilePreferences filePreferences; - private final BibDatabaseContext databaseContext; - private final SearcherManager bibFieldsSearcherManager; - private final SearcherManager linkedFilesSearcherManager; - - public LuceneSearcher(BibDatabaseContext databaseContext, LuceneIndexer bibFieldsIndexer, LuceneIndexer linkedFilesIndexer, FilePreferences filePreferences) { - this.bibFieldsSearcherManager = bibFieldsIndexer.getSearcherManager(); - this.linkedFilesSearcherManager = linkedFilesIndexer.getSearcherManager(); - this.databaseContext = databaseContext; - this.filePreferences = filePreferences; - } - - public boolean isEntryMatched(BibEntry entry, SearchQuery searchQuery) { - BooleanQuery booleanQuery = buildBooleanQueryForEntry(entry, searchQuery); - SearchResults results = search(booleanQuery, searchQuery.getSearchFlags()); - return results.getSearchScoreForEntry(entry) > 0; - } - - private BooleanQuery buildBooleanQueryForEntry(BibEntry entry, SearchQuery searchQuery) { - Query parsedQuery = searchQuery.getParsedQuery(); - TermQuery entryIdQuery = new TermQuery(new Term(SearchFieldConstants.ENTRY_ID.toString(), entry.getId())); - return new BooleanQuery.Builder() - .add(parsedQuery, BooleanClause.Occur.MUST) - .add(entryIdQuery, BooleanClause.Occur.MUST) - .build(); - } - - public SearchResults search(Query searchQuery, EnumSet searchFlags) { - LOGGER.debug("Executing search with query: {}", searchQuery); - try { - boolean shouldSearchInLinkedFiles = searchFlags.contains(SearchFlags.FULLTEXT) && filePreferences.shouldFulltextIndexLinkedFiles(); - return performSearch(searchQuery, shouldSearchInLinkedFiles); - } catch (IOException | IndexSearcher.TooManyClauses e) { - LOGGER.error("Error during search execution", e); - } - return new SearchResults(); - } - - private SearchResults performSearch(Query searchQuery, boolean shouldSearchInLinkedFiles) throws IOException { - if (shouldSearchInLinkedFiles) { - return searchInBibFieldsAndLinkedFiles(searchQuery); - } else { - return searchInBibFields(searchQuery); - } - } - - private SearchResults searchInBibFieldsAndLinkedFiles(Query searchQuery) throws IOException { - IndexSearcher bibFieldsIndexSearcher = acquireIndexSearcher(bibFieldsSearcherManager); - IndexSearcher linkedFilesIndexSearcher = acquireIndexSearcher(linkedFilesSearcherManager); - try { - MultiReader multiReader = new MultiReader(bibFieldsIndexSearcher.getIndexReader(), linkedFilesIndexSearcher.getIndexReader()); - IndexSearcher indexSearcher = new IndexSearcher(multiReader); - return search(indexSearcher, searchQuery, true); - } finally { - releaseIndexSearcher(bibFieldsSearcherManager, bibFieldsIndexSearcher); - releaseIndexSearcher(linkedFilesSearcherManager, linkedFilesIndexSearcher); - } - } - - private SearchResults searchInBibFields(Query searchQuery) throws IOException { - IndexSearcher indexSearcher = acquireIndexSearcher(bibFieldsSearcherManager); - try { - return search(indexSearcher, searchQuery, false); - } finally { - releaseIndexSearcher(bibFieldsSearcherManager, indexSearcher); - } - } - - private SearchResults search(IndexSearcher indexSearcher, Query searchQuery, boolean shouldSearchInLinkedFiles) throws IOException { - TopDocs topDocs = indexSearcher.search(searchQuery, Integer.MAX_VALUE); - StoredFields storedFields = indexSearcher.storedFields(); - LOGGER.debug("Found {} matching documents", topDocs.totalHits.value); - return getSearchResults(topDocs, storedFields, searchQuery, shouldSearchInLinkedFiles); - } - - private SearchResults getSearchResults(TopDocs topDocs, StoredFields storedFields, Query searchQuery, boolean shouldSearchInLinkedFiles) throws IOException { - SearchResults searchResults = new SearchResults(); - long startTime = System.currentTimeMillis(); - - if (shouldSearchInLinkedFiles) { - getBibFieldsAndLinkedFilesResults(topDocs, storedFields, searchQuery, searchResults); - } else { - getBibFieldsResults(topDocs, storedFields, searchResults); - } - - LOGGER.debug("Getting search results took {} ms", System.currentTimeMillis() - startTime); - return searchResults; - } - - private void getBibFieldsAndLinkedFilesResults(TopDocs topDocs, StoredFields storedFields, Query searchQuery, SearchResults searchResults) throws IOException { - Map> linkedFilesMap = getLinkedFilesMap(); - Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter("", ""), new QueryScorer(searchQuery)); - - for (ScoreDoc scoreDoc : topDocs.scoreDocs) { - Document document = storedFields.document(scoreDoc.doc); - String fileLink = getFieldContents(document, SearchFieldConstants.PATH); - - if (!fileLink.isEmpty()) { - addLinkedFileToResults(document, fileLink, linkedFilesMap, highlighter, searchResults, scoreDoc.score); - } else { - addBibEntryToResults(document, searchResults, scoreDoc.score); - } - } - } - - private void getBibFieldsResults(TopDocs topDocs, StoredFields storedFields, SearchResults searchResults) throws IOException { - for (ScoreDoc scoreDoc : topDocs.scoreDocs) { - Document document = storedFields.document(scoreDoc.doc); - addBibEntryToResults(document, searchResults, scoreDoc.score); - } - } - - private void addLinkedFileToResults(Document document, String fileLink, Map> linkedFilesMap, Highlighter highlighter, SearchResults searchResults, float score) { - List entriesWithFile = linkedFilesMap.get(fileLink); - if (entriesWithFile != null && !entriesWithFile.isEmpty()) { - SearchResult searchResult = new SearchResult(score, fileLink, - getFieldContents(document, SearchFieldConstants.CONTENT), - getFieldContents(document, SearchFieldConstants.ANNOTATIONS), - Integer.parseInt(getFieldContents(document, SearchFieldConstants.PAGE_NUMBER)), - highlighter); - searchResults.addSearchResult(entriesWithFile, searchResult); - } - } - - private void addBibEntryToResults(Document document, SearchResults searchResults, float score) { - String entryId = getFieldContents(document, SearchFieldConstants.ENTRY_ID); - searchResults.addSearchResult(entryId, new SearchResult(score)); - } - - private Map> getLinkedFilesMap() { - // fileLink to List of entry IDs - Map> linkedFilesMap = new HashMap<>(); - for (BibEntry bibEntry : databaseContext.getEntries()) { - for (LinkedFile linkedFile : bibEntry.getFiles()) { - linkedFilesMap.computeIfAbsent(linkedFile.getLink(), k -> new ArrayList<>()).add(bibEntry.getId()); - } - } - return linkedFilesMap; - } - - private static String getFieldContents(Document document, SearchFieldConstants field) { - return Optional.ofNullable(document.get(field.toString())).orElse(""); - } - - private static IndexSearcher acquireIndexSearcher(SearcherManager searcherManager) throws IOException { - searcherManager.maybeRefreshBlocking(); - return searcherManager.acquire(); - } - - private static void releaseIndexSearcher(SearcherManager searcherManager, IndexSearcher indexSearcher) throws IOException { - searcherManager.release(indexSearcher); - } -} From bc93bf1e124b30735acedeb05de2ee0404270f9c Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 22 Sep 2024 15:09:01 +0300 Subject: [PATCH 006/104] Use PreparedStatement to fix escaping characters --- .../logic/search/indexing/PostgreIndexer.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java index 77542aed06a..44be8df4684 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -161,24 +161,27 @@ private void removeFromIndex(BibEntry entry) { public void updateEntry(BibEntry entry, Field field) { try { // Use upsert to add the field to the index if it doesn't exist, or update it if it does - connection.createStatement().executeUpdate(""" - INSERT INTO "%s" ("%s", "%s", "%s") - VALUES ('%s', '%s', '%s') - ON CONFLICT ("%s", "%s") DO UPDATE - SET "%s" = EXCLUDED."%s" - """.formatted(tableName, + String updateQuery = """ + INSERT INTO "%s" ("%s", "%s", "%s") + VALUES (?, ?, ?) + ON CONFLICT ("%s", "%s") DO UPDATE + SET "%s" = EXCLUDED."%s" + """.formatted(tableName, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, PostgreConstants.FIELD_VALUE, - entry.getId(), - field.getName(), - entry.getField(field).orElse(""), PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, PostgreConstants.FIELD_VALUE, - PostgreConstants.FIELD_VALUE)); - - LOGGER.debug("Updated entry {} in index", entry.getId()); + PostgreConstants.FIELD_VALUE); + + try (PreparedStatement preparedStatement = connection.prepareStatement(updateQuery)) { + preparedStatement.setString(1, entry.getId()); + preparedStatement.setString(2, field.getName()); + preparedStatement.setString(3, entry.getField(field).orElse("")); + preparedStatement.executeUpdate(); + LOGGER.debug("Updated entry {} in index", entry.getId()); + } } catch (SQLException e) { LOGGER.error("Error updating entry in index", e); } From fb648654070e0e5805ebb1621fa97f22e2f8e345 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 22 Sep 2024 14:40:08 +0300 Subject: [PATCH 007/104] Rename LuceneManager to IndexManager --- .../org/jabref/cli/ArgumentProcessor.java | 2 +- src/main/java/org/jabref/gui/LibraryTab.java | 34 +++++++++---------- .../java/org/jabref/gui/StateManager.java | 12 +++---- .../jabref/gui/entryeditor/CommentsTab.java | 6 ++-- .../gui/entryeditor/DeprecatedFieldsTab.java | 6 ++-- .../entryeditor/DetailOptionalFieldsTab.java | 6 ++-- .../jabref/gui/entryeditor/EntryEditor.java | 20 +++++------ .../gui/entryeditor/FieldsEditorTab.java | 10 +++--- .../ImportantOptionalFieldsTab.java | 6 ++-- .../entryeditor/OptionalFieldsTabBase.java | 6 ++-- .../gui/entryeditor/OtherFieldsTab.java | 6 ++-- .../jabref/gui/entryeditor/PreviewTab.java | 10 +++--- .../gui/entryeditor/RequiredFieldsTab.java | 6 ++-- .../gui/entryeditor/UserDefinedFieldsTab.java | 6 ++-- .../gui/exporter/SaveDatabaseAction.java | 8 ++--- .../ExternalFilesEntryLinker.java | 14 ++++---- .../gui/groups/GroupDialogViewModel.java | 8 ++--- .../jabref/gui/groups/GroupNodeViewModel.java | 18 +++++----- .../org/jabref/gui/maintable/MainTable.java | 4 +-- .../gui/maintable/MainTableDataModel.java | 20 +++++------ .../org/jabref/gui/preview/PreviewPanel.java | 8 ++--- .../RebuildFulltextSearchIndexAction.java | 2 +- .../search/SearchResultsTableDataModel.java | 4 +-- .../jabref/logic/search/DatabaseSearcher.java | 20 +++++------ .../{LuceneManager.java => IndexManager.java} | 27 ++++++++------- .../Analyzer/LatexAwareNGramAnalyzer.java | 32 ----------------- .../model/search/SearchFieldConstants.java | 2 -- .../gui/entryeditor/CommentsTabTest.java | 4 +-- 28 files changed, 136 insertions(+), 171 deletions(-) rename src/main/java/org/jabref/logic/search/{LuceneManager.java => IndexManager.java} (91%) delete mode 100644 src/main/java/org/jabref/model/search/Analyzer/LatexAwareNGramAnalyzer.java diff --git a/src/main/java/org/jabref/cli/ArgumentProcessor.java b/src/main/java/org/jabref/cli/ArgumentProcessor.java index 64edddac7a4..3416a6688e3 100644 --- a/src/main/java/org/jabref/cli/ArgumentProcessor.java +++ b/src/main/java/org/jabref/cli/ArgumentProcessor.java @@ -467,7 +467,7 @@ private boolean exportMatches(List loaded) { List matches; try { - // extract current thread task executor from luceneManager + // extract current thread task executor from indexManager matches = new DatabaseSearcher(query, databaseContext, new CurrentThreadTaskExecutor(), cliPreferences.getFilePreferences()).getMatches(); } catch (IOException e) { LOGGER.error("Error occurred when searching", e); diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index d14184ab0b2..61f14427ab4 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -72,7 +72,7 @@ import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; import org.jabref.logic.pdf.FileAnnotationCache; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.shared.DatabaseLocation; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.TaskExecutor; @@ -166,7 +166,7 @@ private enum PanelMode { MAIN_TABLE, MAIN_TABLE_AND_ENTRY_EDITOR } private final DirectoryMonitorManager directoryMonitorManager; private ImportHandler importHandler; - private LuceneManager luceneManager; + private IndexManager indexManager; /** * @param isDummyContext Indicates whether the database context is a dummy. A dummy context is used to display a progress indicator while parsing the database. @@ -209,7 +209,7 @@ private LibraryTab(BibDatabaseContext bibDatabaseContext, private void initializeComponentsAndListeners(boolean isDummyContext) { if (!isDummyContext) { - createLuceneManager(); + createIndexManager(); } if (tableModel != null) { @@ -220,7 +220,7 @@ private void initializeComponentsAndListeners(boolean isDummyContext) { bibDatabaseContext.getMetaData().registerListener(this); this.selectedGroupsProperty = new SimpleListProperty<>(stateManager.getSelectedGroups(bibDatabaseContext)); - this.tableModel = new MainTableDataModel(getBibDatabaseContext(), preferences, taskExecutor, stateManager, getLuceneManager(), selectedGroupsProperty(), searchQueryProperty(), resultSizeProperty()); + this.tableModel = new MainTableDataModel(getBibDatabaseContext(), preferences, taskExecutor, getIndexManager(), selectedGroupsProperty(), searchQueryProperty(), resultSizeProperty()); new CitationStyleCache(bibDatabaseContext); annotationCache = new FileAnnotationCache(bibDatabaseContext, preferences.getFilePreferences()); @@ -314,17 +314,17 @@ private void onDatabaseLoadingSucceed(ParserResult result) { dataLoadingTask = null; } - public void createLuceneManager() { - luceneManager = new LuceneManager(bibDatabaseContext, taskExecutor, preferences.getFilePreferences()); - stateManager.setLuceneManager(bibDatabaseContext, luceneManager); + public void createIndexManager() { + indexManager = new IndexManager(bibDatabaseContext, taskExecutor, preferences.getFilePreferences()); + stateManager.setIndexManager(bibDatabaseContext, indexManager); } - public LuceneManager getLuceneManager() { - return luceneManager; + public IndexManager getIndexManager() { + return indexManager; } - public void closeLuceneManger() { - luceneManager.close(); + public void closeIndexManger() { + indexManager.close(); } private void onDatabaseLoadingFailed(Exception ex) { @@ -786,11 +786,11 @@ private void onClosed(Event event) { LOGGER.error("Problem when closing directory monitor", e); } try { - if (luceneManager != null) { - luceneManager.close(); + if (indexManager != null) { + indexManager.close(); } } catch (RuntimeException e) { - LOGGER.error("Problem when closing lucene indexer", e); + LOGGER.error("Problem when closing index manager", e); } try { AutosaveManager.shutdown(bibDatabaseContext); @@ -1139,17 +1139,17 @@ private class IndexUpdateListener { @Subscribe public void listen(EntriesAddedEvent addedEntryEvent) { - luceneManager.addToIndex(addedEntryEvent.getBibEntries()); + indexManager.addToIndex(addedEntryEvent.getBibEntries()); } @Subscribe public void listen(EntriesRemovedEvent removedEntriesEvent) { - luceneManager.removeFromIndex(removedEntriesEvent.getBibEntries()); + indexManager.removeFromIndex(removedEntriesEvent.getBibEntries()); } @Subscribe public void listen(FieldChangedEvent fieldChangedEvent) { - luceneManager.updateEntry(fieldChangedEvent); + indexManager.updateEntry(fieldChangedEvent); } } diff --git a/src/main/java/org/jabref/gui/StateManager.java b/src/main/java/org/jabref/gui/StateManager.java index cf73b00da94..ff9a5717d37 100644 --- a/src/main/java/org/jabref/gui/StateManager.java +++ b/src/main/java/org/jabref/gui/StateManager.java @@ -25,7 +25,7 @@ import org.jabref.gui.util.CustomLocalDragboard; import org.jabref.gui.util.DialogWindowState; import org.jabref.gui.util.OptionalObjectProperty; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.BackgroundTask; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -59,7 +59,7 @@ public class StateManager { private final OptionalObjectProperty activeTab = OptionalObjectProperty.empty(); private final ObservableList selectedEntries = FXCollections.observableArrayList(); private final ObservableMap> selectedGroups = FXCollections.observableHashMap(); - private final ObservableMap luceneManagers = FXCollections.observableHashMap(); + private final ObservableMap indexManagers = FXCollections.observableHashMap(); private final OptionalObjectProperty activeSearchQuery = OptionalObjectProperty.empty(); private final OptionalObjectProperty activeGlobalSearchQuery = OptionalObjectProperty.empty(); private final IntegerProperty searchResultSize = new SimpleIntegerProperty(0); @@ -124,12 +124,12 @@ public void clearSelectedGroups(BibDatabaseContext context) { selectedGroups.computeIfAbsent(context.getUid(), k -> FXCollections.observableArrayList()).clear(); } - public void setLuceneManager(BibDatabaseContext database, LuceneManager luceneManager) { - luceneManagers.put(database.getUid(), luceneManager); + public void setIndexManager(BibDatabaseContext database, IndexManager indexManager) { + indexManagers.put(database.getUid(), indexManager); } - public Optional getLuceneManager(BibDatabaseContext database) { - return Optional.ofNullable(luceneManagers.get(database.getUid())); + public Optional getIndexManager(BibDatabaseContext database) { + return Optional.ofNullable(indexManagers.get(database.getUid())); } public Optional getActiveDatabase() { diff --git a/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java b/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java index 428052847bf..fe0f7691006 100644 --- a/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java @@ -28,7 +28,7 @@ import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -55,7 +55,7 @@ public CommentsTab(GuiPreferences preferences, ThemeManager themeManager, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository, - LuceneManager luceneManager, + IndexManager indexManager, OptionalObjectProperty searchQueryProperty) { super( false, @@ -69,7 +69,7 @@ public CommentsTab(GuiPreferences preferences, themeManager, taskExecutor, journalAbbreviationRepository, - luceneManager, + indexManager, searchQueryProperty ); this.defaultOwner = preferences.getOwnerPreferences().getDefaultOwner().toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]", "-"); diff --git a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java index 3c962be547b..cdc001bad58 100644 --- a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java @@ -19,7 +19,7 @@ import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; @@ -47,9 +47,9 @@ public DeprecatedFieldsTab(BibDatabaseContext databaseContext, BibEntryTypesManager entryTypesManager, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository, - LuceneManager luceneManager, + IndexManager indexManager, OptionalObjectProperty searchQueryProperty) { - super(false, databaseContext, suggestionProviders, undoManager, undoAction, redoAction, dialogService, preferences, themeManager, taskExecutor, journalAbbreviationRepository, luceneManager, searchQueryProperty); + super(false, databaseContext, suggestionProviders, undoManager, undoAction, redoAction, dialogService, preferences, themeManager, taskExecutor, journalAbbreviationRepository, indexManager, searchQueryProperty); this.entryTypesManager = entryTypesManager; setText(Localization.lang("Deprecated fields")); diff --git a/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java index 4458d80a405..f01e6fa9318 100644 --- a/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java @@ -11,7 +11,7 @@ import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntryTypesManager; @@ -32,7 +32,7 @@ public DetailOptionalFieldsTab(BibDatabaseContext databaseContext, BibEntryTypesManager entryTypesManager, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository, - LuceneManager luceneManager, + IndexManager indexManager, OptionalObjectProperty searchQueryProperty) { super( Localization.lang("Optional fields 2"), @@ -48,7 +48,7 @@ public DetailOptionalFieldsTab(BibDatabaseContext databaseContext, entryTypesManager, taskExecutor, journalAbbreviationRepository, - luceneManager, + indexManager, searchQueryProperty ); } diff --git a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java index 7e99897cadb..00398df8c84 100644 --- a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java +++ b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java @@ -177,11 +177,11 @@ private void setupDragAndDrop(LibraryTab libraryTab) { switch (event.getTransferMode()) { case COPY -> { LOGGER.debug("Mode COPY"); - fileLinker.copyFilesToFileDirAndAddToEntry(entry, draggedFiles, libraryTab.getLuceneManager()); + fileLinker.copyFilesToFileDirAndAddToEntry(entry, draggedFiles, libraryTab.getIndexManager()); } case MOVE -> { LOGGER.debug("Mode MOVE"); - fileLinker.moveFilesToFileDirRenameAndAddToEntry(entry, draggedFiles, libraryTab.getLuceneManager()); + fileLinker.moveFilesToFileDirRenameAndAddToEntry(entry, draggedFiles, libraryTab.getIndexManager()); } case LINK -> { LOGGER.debug("Mode LINK"); @@ -278,21 +278,21 @@ private void navigateToNextEntry() { private List createTabs() { List tabs = new LinkedList<>(); - tabs.add(new PreviewTab(databaseContext, dialogService, preferences, themeManager, taskExecutor, libraryTab.getLuceneManager(), libraryTab.searchQueryProperty())); + tabs.add(new PreviewTab(databaseContext, dialogService, preferences, themeManager, taskExecutor, libraryTab.getIndexManager(), libraryTab.searchQueryProperty())); // Required, optional (important+detail), deprecated, and "other" fields - tabs.add(new RequiredFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, preferences, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository, libraryTab.getLuceneManager(), libraryTab.searchQueryProperty())); - tabs.add(new ImportantOptionalFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, preferences, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository, libraryTab.getLuceneManager(), libraryTab.searchQueryProperty())); - tabs.add(new DetailOptionalFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, preferences, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository, libraryTab.getLuceneManager(), libraryTab.searchQueryProperty())); - tabs.add(new DeprecatedFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, preferences, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository, libraryTab.getLuceneManager(), libraryTab.searchQueryProperty())); - tabs.add(new OtherFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, preferences, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository, libraryTab.getLuceneManager(), libraryTab.searchQueryProperty())); + tabs.add(new RequiredFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, preferences, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository, libraryTab.getIndexManager(), libraryTab.searchQueryProperty())); + tabs.add(new ImportantOptionalFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, preferences, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository, libraryTab.getIndexManager(), libraryTab.searchQueryProperty())); + tabs.add(new DetailOptionalFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, preferences, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository, libraryTab.getIndexManager(), libraryTab.searchQueryProperty())); + tabs.add(new DeprecatedFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, preferences, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository, libraryTab.getIndexManager(), libraryTab.searchQueryProperty())); + tabs.add(new OtherFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, preferences, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository, libraryTab.getIndexManager(), libraryTab.searchQueryProperty())); // Comment Tab: Tab for general and user-specific comments - tabs.add(new CommentsTab(preferences, databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, themeManager, taskExecutor, journalAbbreviationRepository, libraryTab.getLuceneManager(), libraryTab.searchQueryProperty())); + tabs.add(new CommentsTab(preferences, databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, themeManager, taskExecutor, journalAbbreviationRepository, libraryTab.getIndexManager(), libraryTab.searchQueryProperty())); Map> entryEditorTabList = getAdditionalUserConfiguredTabs(); for (Map.Entry> tab : entryEditorTabList.entrySet()) { - tabs.add(new UserDefinedFieldsTab(tab.getKey(), tab.getValue(), databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, preferences, themeManager, taskExecutor, journalAbbreviationRepository, libraryTab.getLuceneManager(), libraryTab.searchQueryProperty())); + tabs.add(new UserDefinedFieldsTab(tab.getKey(), tab.getValue(), databaseContext, libraryTab.getSuggestionProviders(), undoManager, undoAction, redoAction, dialogService, preferences, themeManager, taskExecutor, journalAbbreviationRepository, libraryTab.getIndexManager(), libraryTab.searchQueryProperty())); } tabs.add(new MathSciNetTab()); diff --git a/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java b/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java index 41b29ee8f42..25a6036f551 100644 --- a/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java @@ -36,7 +36,7 @@ import org.jabref.gui.undo.UndoAction; import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.journals.JournalAbbreviationRepository; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -64,7 +64,7 @@ abstract class FieldsEditorTab extends EntryEditorTab implements OffersPreview { private final JournalAbbreviationRepository journalAbbreviationRepository; private PreviewPanel previewPanel; private final UndoManager undoManager; - private final LuceneManager luceneManager; + private final IndexManager indexManager; private final OptionalObjectProperty searchQueryProperty; private Collection fields = new ArrayList<>(); @SuppressWarnings("FieldCanBeLocal") @@ -81,7 +81,7 @@ public FieldsEditorTab(boolean compressed, ThemeManager themeManager, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository, - LuceneManager luceneManager, + IndexManager indexManager, OptionalObjectProperty searchQueryProperty) { this.isCompressed = compressed; this.databaseContext = Objects.requireNonNull(databaseContext); @@ -94,7 +94,7 @@ public FieldsEditorTab(boolean compressed, this.themeManager = themeManager; this.taskExecutor = Objects.requireNonNull(taskExecutor); this.journalAbbreviationRepository = Objects.requireNonNull(journalAbbreviationRepository); - this.luceneManager = luceneManager; + this.indexManager = indexManager; this.searchQueryProperty = searchQueryProperty; } @@ -265,7 +265,7 @@ private void initPanel() { preferences, themeManager, taskExecutor, - luceneManager, + indexManager, searchQueryProperty); EasyBind.subscribe(preferences.getPreviewPreferences().showPreviewAsExtraTabProperty(), show -> { if (show) { diff --git a/src/main/java/org/jabref/gui/entryeditor/ImportantOptionalFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/ImportantOptionalFieldsTab.java index 64004b70848..49534abf2de 100644 --- a/src/main/java/org/jabref/gui/entryeditor/ImportantOptionalFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/ImportantOptionalFieldsTab.java @@ -11,7 +11,7 @@ import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntryTypesManager; @@ -32,7 +32,7 @@ public ImportantOptionalFieldsTab(BibDatabaseContext databaseContext, BibEntryTypesManager entryTypesManager, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository, - LuceneManager luceneManager, + IndexManager indexManager, OptionalObjectProperty searchQueryProperty) { super( Localization.lang("Optional fields"), @@ -48,7 +48,7 @@ public ImportantOptionalFieldsTab(BibDatabaseContext databaseContext, entryTypesManager, taskExecutor, journalAbbreviationRepository, - luceneManager, + indexManager, searchQueryProperty ); } diff --git a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java index 0d59a31bbf9..6b0379d99b6 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java +++ b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java @@ -18,7 +18,7 @@ import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; @@ -45,7 +45,7 @@ public OptionalFieldsTabBase(String title, BibEntryTypesManager entryTypesManager, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository, - LuceneManager luceneManager, + IndexManager indexManager, OptionalObjectProperty searchQueryProperty) { super(true, databaseContext, @@ -58,7 +58,7 @@ public OptionalFieldsTabBase(String title, themeManager, taskExecutor, journalAbbreviationRepository, - luceneManager, + indexManager, searchQueryProperty); this.entryTypesManager = entryTypesManager; this.isImportantOptionalFields = isImportantOptionalFields; diff --git a/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java index 5fda9368885..8fb4f16d24f 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java @@ -22,7 +22,7 @@ import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; @@ -52,7 +52,7 @@ public OtherFieldsTab(BibDatabaseContext databaseContext, BibEntryTypesManager entryTypesManager, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository, - LuceneManager luceneManager, + IndexManager indexManager, OptionalObjectProperty searchQueryProperty) { super(false, databaseContext, @@ -65,7 +65,7 @@ public OtherFieldsTab(BibDatabaseContext databaseContext, themeManager, taskExecutor, journalAbbreviationRepository, - luceneManager, + indexManager, searchQueryProperty); this.entryTypesManager = entryTypesManager; diff --git a/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java b/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java index 52b51b67b7e..843485c3218 100644 --- a/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java @@ -7,7 +7,7 @@ import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -20,7 +20,7 @@ public class PreviewTab extends EntryEditorTab implements OffersPreview { private final GuiPreferences preferences; private final ThemeManager themeManager; private final TaskExecutor taskExecutor; - private final LuceneManager luceneManager; + private final IndexManager indexManager; private final OptionalObjectProperty searchQueryProperty; private PreviewPanel previewPanel; @@ -29,14 +29,14 @@ public PreviewTab(BibDatabaseContext databaseContext, GuiPreferences preferences, ThemeManager themeManager, TaskExecutor taskExecutor, - LuceneManager luceneManager, + IndexManager indexManager, OptionalObjectProperty searchQueryProperty) { this.databaseContext = databaseContext; this.dialogService = dialogService; this.preferences = preferences; this.themeManager = themeManager; this.taskExecutor = taskExecutor; - this.luceneManager = luceneManager; + this.indexManager = indexManager; this.searchQueryProperty = searchQueryProperty; setGraphic(IconTheme.JabRefIcons.TOGGLE_ENTRY_PREVIEW.getGraphicNode()); @@ -65,7 +65,7 @@ public boolean shouldShow(BibEntry entry) { @Override protected void bindToEntry(BibEntry entry) { if (previewPanel == null) { - previewPanel = new PreviewPanel(databaseContext, dialogService, preferences.getKeyBindingRepository(), preferences, themeManager, taskExecutor, luceneManager, searchQueryProperty); + previewPanel = new PreviewPanel(databaseContext, dialogService, preferences.getKeyBindingRepository(), preferences, themeManager, taskExecutor, indexManager, searchQueryProperty); setContent(previewPanel); } diff --git a/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java index 5966c329fda..33c2ec1a2a1 100644 --- a/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java @@ -18,7 +18,7 @@ import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -45,10 +45,10 @@ public RequiredFieldsTab(BibDatabaseContext databaseContext, BibEntryTypesManager entryTypesManager, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository, - LuceneManager luceneManager, + IndexManager indexManager, OptionalObjectProperty searchQueryProperty) { super(false, databaseContext, suggestionProviders, undoManager, undoAction, redoAction, dialogService, - preferences, themeManager, taskExecutor, journalAbbreviationRepository, luceneManager, searchQueryProperty); + preferences, themeManager, taskExecutor, journalAbbreviationRepository, indexManager, searchQueryProperty); this.entryTypesManager = entryTypesManager; setText(Localization.lang("Required fields")); setTooltip(new Tooltip(Localization.lang("Show required fields"))); diff --git a/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java index 3e09cdb59e8..17c3477827e 100644 --- a/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java @@ -15,7 +15,7 @@ import org.jabref.gui.undo.UndoAction; import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.journals.JournalAbbreviationRepository; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -37,9 +37,9 @@ public UserDefinedFieldsTab(String name, ThemeManager themeManager, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository, - LuceneManager luceneManager, + IndexManager indexManager, OptionalObjectProperty searchQueryProperty) { - super(false, databaseContext, suggestionProviders, undoManager, undoAction, redoAction, dialogService, preferences, themeManager, taskExecutor, journalAbbreviationRepository, luceneManager, searchQueryProperty); + super(false, databaseContext, suggestionProviders, undoManager, undoAction, redoAction, dialogService, preferences, themeManager, taskExecutor, journalAbbreviationRepository, indexManager, searchQueryProperty); this.fields = new LinkedHashSet<>(fields); diff --git a/src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java b/src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java index 6a2dd9cb42b..10487ead6ad 100644 --- a/src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java +++ b/src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java @@ -139,10 +139,10 @@ boolean saveAs(Path file, SaveDatabaseMode mode) { Optional databasePath = context.getDatabasePath(); if (databasePath.isPresent()) { - // Close AutosaveManager, BackupManager, and LuceneManager for original library + // Close AutosaveManager, BackupManager, and IndexManager for original library AutosaveManager.shutdown(context); BackupManager.shutdown(context, this.preferences.getFilePreferences().getBackupDirectory(), preferences.getFilePreferences().shouldCreateBackup()); - libraryTab.closeLuceneManger(); + libraryTab.closeIndexManger(); } // Set new location @@ -160,10 +160,10 @@ boolean saveAs(Path file, SaveDatabaseMode mode) { context.setDatabasePath(file); libraryTab.updateTabTitle(false); - // Reset (here: uninstall and install again) AutosaveManager, BackupManager and LuceneManager for the new file name + // Reset (here: uninstall and install again) AutosaveManager, BackupManager and IndexManager for the new file name libraryTab.resetChangeMonitor(); libraryTab.installAutosaveManagerAndBackupManager(); - libraryTab.createLuceneManager(); + libraryTab.createIndexManager(); preferences.getLastFilesOpenedPreferences().getFileHistory().newFile(file); } diff --git a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java index 85b1a83213e..fdfd00dfb75 100644 --- a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java +++ b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java @@ -17,7 +17,7 @@ import org.jabref.logic.cleanup.MoveFilesCleanup; import org.jabref.logic.cleanup.RenamePdfCleanup; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.io.FileNameCleaner; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.database.BibDatabaseContext; @@ -79,19 +79,19 @@ public void addFilesToEntry(BibEntry entry, List files) { } } - public void moveFilesToFileDirRenameAndAddToEntry(BibEntry entry, List files, LuceneManager luceneManager) { - try (AutoCloseable blocker = luceneManager.blockLinkedFileIndexer()) { + public void moveFilesToFileDirRenameAndAddToEntry(BibEntry entry, List files, IndexManager indexManager) { + try (AutoCloseable blocker = indexManager.blockLinkedFileIndexer()) { addFilesToEntry(entry, files); moveLinkedFilesToFileDir(entry); renameLinkedFilesToPattern(entry); } catch (Exception e) { LOGGER.error("Could not block LinkedFilesIndexer", e); } - luceneManager.updateAfterDropFiles(entry); + indexManager.updateAfterDropFiles(entry); } - public void copyFilesToFileDirAndAddToEntry(BibEntry entry, List files, LuceneManager luceneManager) { - try (AutoCloseable blocker = luceneManager.blockLinkedFileIndexer()) { + public void copyFilesToFileDirAndAddToEntry(BibEntry entry, List files, IndexManager indexManager) { + try (AutoCloseable blocker = indexManager.blockLinkedFileIndexer()) { for (Path file : files) { copyFileToFileDir(file) .ifPresent(copiedFile -> addFilesToEntry(entry, Collections.singletonList(copiedFile))); @@ -100,7 +100,7 @@ public void copyFilesToFileDirAndAddToEntry(BibEntry entry, List files, Lu } catch (Exception e) { LOGGER.error("Could not block LinkedFilesIndexer", e); } - luceneManager.updateAfterDropFiles(entry); + indexManager.updateAfterDropFiles(entry); } private List getValidFileNames(List filesToAdd) { diff --git a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java index 1c36289b9db..060fdf2e257 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java @@ -32,7 +32,7 @@ import org.jabref.logic.auxparser.DefaultAuxParser; import org.jabref.logic.groups.DefaultGroupsFactory; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.StandardFileType; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.database.BibDatabase; @@ -331,10 +331,10 @@ public AbstractGroup resultConverter(ButtonType button) { } } - Optional luceneManager = stateManager.getLuceneManager(currentDatabase); - if (luceneManager.isPresent()) { + Optional indexManager = stateManager.getIndexManager(currentDatabase); + if (indexManager.isPresent()) { SearchGroup searchGroup = (SearchGroup) resultingGroup; - searchGroup.setMatchedEntries(luceneManager.get().search(searchGroup.getQuery()).getMatchedEntries()); + searchGroup.setMatchedEntries(indexManager.get().search(searchGroup.getQuery()).getMatchedEntries()); } } else if (typeAutoProperty.getValue()) { if (autoGroupKeywordsOptionProperty.getValue()) { diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index cc64665235f..43c3050cb00 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -101,9 +101,9 @@ public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager state if (groupNode.getGroup() instanceof TexGroup) { databaseContext.getMetaData().groupsBinding().addListener(new WeakInvalidationListener(onInvalidatedGroup)); } else if (groupNode.getGroup() instanceof SearchGroup searchGroup) { - stateManager.getLuceneManager(databaseContext).ifPresent(luceneManager -> { + stateManager.getIndexManager(databaseContext).ifPresent(indexManager -> { BackgroundTask.wrap(() -> { - searchGroup.setMatchedEntries(luceneManager.search(searchGroup.getQuery()).getMatchedEntries()); + searchGroup.setMatchedEntries(indexManager.search(searchGroup.getQuery()).getMatchedEntries()); }).onSuccess(success -> { refreshGroup(); databaseContext.getMetaData().groupsBinding().invalidate(); @@ -127,7 +127,7 @@ public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager state // 'all' returns 'true' for empty streams, so this has to be checked explicitly allSelectedEntriesMatched = selectedEntriesMatchStatus.isEmptyBinding().not().and(selectedEntriesMatchStatus.allMatch(matched -> matched)); - this.databaseContext.getDatabase().registerListener(new LuceneIndexListener()); + this.databaseContext.getDatabase().registerListener(new SearchIndexListener()); } public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager stateManager, TaskExecutor taskExecutor, AbstractGroup group, CustomLocalDragboard localDragboard, GuiPreferences preferences) { @@ -251,7 +251,7 @@ public GroupTreeNode getGroupNode() { /** * Gets invoked if an entry in the current database changes. * - * @implNote Search groups are updated in {@link LuceneIndexListener}. + * @implNote Search groups are updated in {@link SearchIndexListener}. */ private void onDatabaseChanged(ListChangeListener.Change change) { if (groupNode.getGroup() instanceof SearchGroup) { @@ -536,13 +536,13 @@ public boolean isEditable() { } } - class LuceneIndexListener { + class SearchIndexListener { @Subscribe public void listen(IndexStartedEvent event) { if (groupNode.getGroup() instanceof SearchGroup searchGroup) { - stateManager.getLuceneManager(databaseContext).ifPresent(luceneManager -> { + stateManager.getIndexManager(databaseContext).ifPresent(indexManager -> { BackgroundTask.wrap(() -> { - searchGroup.setMatchedEntries(luceneManager.search(searchGroup.getQuery()).getMatchedEntries()); + searchGroup.setMatchedEntries(indexManager.search(searchGroup.getQuery()).getMatchedEntries()); }).onSuccess(success -> { refreshGroup(); databaseContext.getMetaData().groupsBinding().invalidate(); @@ -554,10 +554,10 @@ public void listen(IndexStartedEvent event) { @Subscribe public void listen(IndexAddedOrUpdatedEvent event) { if (groupNode.getGroup() instanceof SearchGroup searchGroup) { - stateManager.getLuceneManager(databaseContext).ifPresent(luceneManager -> { + stateManager.getIndexManager(databaseContext).ifPresent(indexManager -> { BackgroundTask.wrap(() -> { for (BibEntry entry : event.entries()) { - searchGroup.updateMatches(entry, luceneManager.isEntryMatched(entry, searchGroup.getQuery())); + searchGroup.updateMatches(entry, indexManager.isEntryMatched(entry, searchGroup.getQuery())); } }).onFinished(() -> { for (BibEntry entry : event.entries()) { diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.java b/src/main/java/org/jabref/gui/maintable/MainTable.java index 4719e2e8220..84d468156e5 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.java +++ b/src/main/java/org/jabref/gui/maintable/MainTable.java @@ -428,11 +428,11 @@ private void handleOnDragDropped(TableRow row, BibEntryT } case MOVE -> { LOGGER.debug("Mode MOVE"); // alt on win - importHandler.getLinker().moveFilesToFileDirRenameAndAddToEntry(entry, files, libraryTab.getLuceneManager()); + importHandler.getLinker().moveFilesToFileDirRenameAndAddToEntry(entry, files, libraryTab.getIndexManager()); } case COPY -> { LOGGER.debug("Mode Copy"); // ctrl on win - importHandler.getLinker().copyFilesToFileDirAndAddToEntry(entry, files, libraryTab.getLuceneManager()); + importHandler.getLinker().copyFilesToFileDirAndAddToEntry(entry, files, libraryTab.getIndexManager()); } } } diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index d0a7aae33e6..910db9f5196 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -12,7 +12,6 @@ import javafx.collections.transformation.FilteredList; import javafx.collections.transformation.SortedList; -import org.jabref.gui.StateManager; import org.jabref.gui.groups.GroupViewMode; import org.jabref.gui.groups.GroupsPreferences; import org.jabref.gui.preferences.GuiPreferences; @@ -20,7 +19,7 @@ import org.jabref.gui.util.BindingsHelper; import org.jabref.gui.util.FilteredListProxy; import org.jabref.gui.util.OptionalObjectProperty; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.search.SearchDisplayMode; import org.jabref.logic.search.SearchPreferences; import org.jabref.logic.util.BackgroundTask; @@ -56,17 +55,16 @@ public class MainTableDataModel { private final Subscription searchDisplayModeSubscription; private final Subscription selectedGroupsSubscription; private final Subscription groupViewModeSubscription; - private final LuceneIndexListener indexUpdatedListener; + private final SearchIndexListener indexUpdatedListener; private final OptionalObjectProperty searchQueryProperty; - @Nullable private final LuceneManager luceneManager; + @Nullable private final IndexManager indexManager; private Optional groupsMatcher; public MainTableDataModel(BibDatabaseContext context, GuiPreferences preferences, TaskExecutor taskExecutor, - StateManager stateManager, - @Nullable LuceneManager luceneManager, + @Nullable IndexManager indexManager, ListProperty selectedGroupsProperty, OptionalObjectProperty searchQueryProperty, IntegerProperty resultSizeProperty) { @@ -74,10 +72,10 @@ public MainTableDataModel(BibDatabaseContext context, this.searchPreferences = preferences.getSearchPreferences(); this.nameDisplayPreferences = preferences.getNameDisplayPreferences(); this.taskExecutor = taskExecutor; - this.luceneManager = luceneManager; + this.indexManager = indexManager; this.bibDatabaseContext = context; this.searchQueryProperty = searchQueryProperty; - this.indexUpdatedListener = new LuceneIndexListener(); + this.indexUpdatedListener = new SearchIndexListener(); this.groupsMatcher = createGroupMatcher(selectedGroupsProperty.get(), groupsPreferences); this.bibDatabaseContext.getDatabase().registerListener(indexUpdatedListener); @@ -100,7 +98,7 @@ public MainTableDataModel(BibDatabaseContext context, private void updateSearchMatches(Optional query) { BackgroundTask.wrap(() -> { if (query.isPresent()) { - SearchResults results = luceneManager.search(query.get()); + SearchResults results = indexManager.search(query.get()); setSearchMatches(results); } else { clearSearchMatches(); @@ -202,7 +200,7 @@ public void resetFieldFormatter() { this.fieldValueFormatter.setValue(new MainTableFieldValueFormatter(nameDisplayPreferences, bibDatabaseContext)); } - class LuceneIndexListener { + class SearchIndexListener { @Subscribe public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { indexAddedOrUpdatedEvent.entries().forEach(entry -> { @@ -216,7 +214,7 @@ public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { SearchQuery searchQuery = searchQueryProperty.get().get(); String newSearchExpression = "+" + SearchFieldConstants.ENTRY_ID + ":" + entry.getId() + " +" + searchQuery.getSearchExpression(); SearchQuery entryQuery = new SearchQuery(newSearchExpression, searchQuery.getSearchFlags()); - SearchResults results = luceneManager.search(entryQuery); + SearchResults results = indexManager.search(entryQuery); viewModel.searchScoreProperty().set(results.getSearchScoreForEntry(entry)); viewModel.hasFullTextResultsProperty().set(results.hasFulltextResults(entry)); diff --git a/src/main/java/org/jabref/gui/preview/PreviewPanel.java b/src/main/java/org/jabref/gui/preview/PreviewPanel.java index 1120ad4fc61..48ab602a55c 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewPanel.java +++ b/src/main/java/org/jabref/gui/preview/PreviewPanel.java @@ -26,7 +26,7 @@ import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.l10n.Localization; import org.jabref.logic.preview.PreviewLayout; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -52,7 +52,7 @@ public PreviewPanel(BibDatabaseContext database, GuiPreferences preferences, ThemeManager themeManager, TaskExecutor taskExecutor, - LuceneManager luceneManager, + IndexManager indexManager, OptionalObjectProperty searchQueryProperty) { this.keyBindingRepository = keyBindingRepository; this.dialogService = dialogService; @@ -88,7 +88,7 @@ public PreviewPanel(BibDatabaseContext database, if (event.getTransferMode() == TransferMode.MOVE) { LOGGER.debug("Mode MOVE"); // shift on win or no modifier - fileLinker.moveFilesToFileDirRenameAndAddToEntry(entry, files, luceneManager); + fileLinker.moveFilesToFileDirRenameAndAddToEntry(entry, files, indexManager); } if (event.getTransferMode() == TransferMode.LINK) { LOGGER.debug("Node LINK"); // alt on win @@ -96,7 +96,7 @@ public PreviewPanel(BibDatabaseContext database, } if (event.getTransferMode() == TransferMode.COPY) { LOGGER.debug("Mode Copy"); // ctrl on win, no modifier on Xubuntu - fileLinker.copyFilesToFileDirAndAddToEntry(entry, files, luceneManager); + fileLinker.copyFilesToFileDirAndAddToEntry(entry, files, indexManager); } success = true; } diff --git a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java index 73ebef568b1..1e1ce28a72c 100644 --- a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java +++ b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java @@ -53,6 +53,6 @@ private void rebuildIndex() { if (!shouldContinue || stateManager.getActiveDatabase().isEmpty()) { return; } - tabSupplier.get().getLuceneManager().rebuildFullTextIndex(); + tabSupplier.get().getIndexManager().rebuildFullTextIndex(); } } diff --git a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java index b6f7503ccbb..63246e1a8d3 100644 --- a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java +++ b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java @@ -66,8 +66,8 @@ private void updateSearchMatches(Optional query) { if (query.isPresent()) { SearchResults searchResults = new SearchResults(); for (BibDatabaseContext context : stateManager.getOpenDatabases()) { - stateManager.getLuceneManager(context).ifPresent(luceneManager -> { - searchResults.mergeSearchResults(luceneManager.search(query.get())); + stateManager.getIndexManager(context).ifPresent(indexManager -> { + searchResults.mergeSearchResults(indexManager.search(query.get())); }); } for (BibEntryTableViewModel entry : entriesViewModel) { diff --git a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java index 09f15d48207..76424e0e6ad 100644 --- a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java +++ b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java @@ -20,13 +20,13 @@ public class DatabaseSearcher { private final BibDatabaseContext databaseContext; private final SearchQuery query; - private final LuceneManager luceneManager; + private final IndexManager indexManager; - // get rid of task executor here or add a constuctor overload? + // get rid of task executor here or add a constructor overload? public DatabaseSearcher(SearchQuery query, BibDatabaseContext databaseContext, TaskExecutor taskExecutor, FilePreferences filePreferences) throws IOException { this.databaseContext = databaseContext; this.query = Objects.requireNonNull(query); - this.luceneManager = new LuceneManager(databaseContext, taskExecutor, filePreferences); + this.indexManager = new IndexManager(databaseContext, taskExecutor, filePreferences); } /** @@ -37,15 +37,15 @@ public List getMatches() { if (!query.isValid()) { LOGGER.warn("Search failed: invalid search expression"); - luceneManager.closeAndWait(); + indexManager.closeAndWait(); return Collections.emptyList(); } - List matchEntries = luceneManager.search(query) - .getMatchedEntries() - .stream() - .map(entryId -> databaseContext.getDatabase().getEntryById(entryId)) - .toList(); - luceneManager.closeAndWait(); + List matchEntries = indexManager.search(query) + .getMatchedEntries() + .stream() + .map(entryId -> databaseContext.getDatabase().getEntryById(entryId)) + .toList(); + indexManager.closeAndWait(); return BibDatabases.purgeEmptyEntries(matchEntries); } } diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java similarity index 91% rename from src/main/java/org/jabref/logic/search/LuceneManager.java rename to src/main/java/org/jabref/logic/search/IndexManager.java index 594b0e48c0c..33aa4084ceb 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -29,25 +29,28 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class LuceneManager { - private static final Logger LOGGER = LoggerFactory.getLogger(LuceneManager.class); +public class IndexManager { + private static final Logger LOGGER = LoggerFactory.getLogger(IndexManager.class); private final TaskExecutor taskExecutor; private final BibDatabaseContext databaseContext; private final BooleanProperty shouldIndexLinkedFiles; private final BooleanProperty isLinkedFilesIndexerBlocked = new SimpleBooleanProperty(false); private final ChangeListener preferencesListener; + private final PostgreIndexer bibFieldsIndexer; private final LuceneIndexer linkedFilesIndexer; private final LinkedFilesSearcher linkedFilesSearcher; - private final PostgreIndexer postgreIndexer; - public LuceneManager(BibDatabaseContext databaseContext, TaskExecutor executor, FilePreferences preferences) { + public IndexManager(BibDatabaseContext databaseContext, TaskExecutor executor, FilePreferences preferences) { this.taskExecutor = executor; this.databaseContext = databaseContext; this.shouldIndexLinkedFiles = preferences.fulltextIndexLinkedFilesProperty(); this.preferencesListener = (observable, oldValue, newValue) -> bindToPreferences(newValue); this.shouldIndexLinkedFiles.addListener(preferencesListener); + PostgreServer postgreServer = Injector.instantiateModelOrService(PostgreServer.class); + bibFieldsIndexer = new PostgreIndexer(databaseContext, postgreServer.getConnection()); + LuceneIndexer indexer; try { indexer = new DefaultLinkedFilesIndexer(databaseContext, preferences); @@ -58,8 +61,6 @@ public LuceneManager(BibDatabaseContext databaseContext, TaskExecutor executor, linkedFilesIndexer = indexer; this.linkedFilesSearcher = new LinkedFilesSearcher(databaseContext, linkedFilesIndexer, preferences); - PostgreServer postgreServer = Injector.instantiateModelOrService(PostgreServer.class); - postgreIndexer = new PostgreIndexer(databaseContext, postgreServer.getConnection()); updateOnStart(); } @@ -81,7 +82,7 @@ private void updateOnStart() { new BackgroundTask<>() { @Override public Object call() { - postgreIndexer.updateOnStart(this); + bibFieldsIndexer.updateOnStart(this); return null; } }.showToUser(true) @@ -104,7 +105,7 @@ public void addToIndex(List entries) { new BackgroundTask<>() { @Override public Object call() { - postgreIndexer.addToIndex(entries, this); + bibFieldsIndexer.addToIndex(entries, this); return null; } }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(entries))) @@ -125,7 +126,7 @@ public void removeFromIndex(List entries) { new BackgroundTask<>() { @Override public Object call() { - postgreIndexer.removeFromIndex(entries, this); + bibFieldsIndexer.removeFromIndex(entries, this); return null; } }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexRemovedEvent(entries))) @@ -146,7 +147,7 @@ public void updateEntry(FieldChangedEvent event) { new BackgroundTask<>() { @Override public Object call() { - postgreIndexer.updateEntry(event.getBibEntry(), event.getField()); + bibFieldsIndexer.updateEntry(event.getBibEntry(), event.getField()); return null; } }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(List.of(event.getBibEntry())))) @@ -167,7 +168,7 @@ public void updateAfterDropFiles(BibEntry entry) { new BackgroundTask<>() { @Override public Object call() { - postgreIndexer.updateEntry(entry, StandardField.FILE); + bibFieldsIndexer.updateEntry(entry, StandardField.FILE); return null; } }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(List.of(entry)))) @@ -197,14 +198,14 @@ public Object call() { } public void close() { - postgreIndexer.close(); + bibFieldsIndexer.close(); shouldIndexLinkedFiles.removeListener(preferencesListener); linkedFilesIndexer.close(); databaseContext.getDatabase().postEvent(new IndexClosedEvent()); } public void closeAndWait() { - postgreIndexer.closeAndWait(); + bibFieldsIndexer.closeAndWait(); shouldIndexLinkedFiles.removeListener(preferencesListener); linkedFilesIndexer.closeAndWait(); databaseContext.getDatabase().postEvent(new IndexClosedEvent()); diff --git a/src/main/java/org/jabref/model/search/Analyzer/LatexAwareNGramAnalyzer.java b/src/main/java/org/jabref/model/search/Analyzer/LatexAwareNGramAnalyzer.java deleted file mode 100644 index aafb993390d..00000000000 --- a/src/main/java/org/jabref/model/search/Analyzer/LatexAwareNGramAnalyzer.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.jabref.model.search.Analyzer; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.LowerCaseFilter; -import org.apache.lucene.analysis.StopFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.core.WhitespaceTokenizer; -import org.apache.lucene.analysis.en.EnglishAnalyzer; -import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; -import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter; - -public class LatexAwareNGramAnalyzer extends Analyzer { - private final int minGram; - private final int maxGram; - - public LatexAwareNGramAnalyzer(int minGram, int maxGram) { - this.minGram = minGram; - this.maxGram = maxGram; - } - - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new WhitespaceTokenizer(); - TokenStream result = new LatexToUnicodeFoldingFilter(source); - result = new StopFilter(result, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); - result = new ASCIIFoldingFilter(result); - result = new LowerCaseFilter(result); - result = new EdgeNGramTokenFilter(result, minGram, maxGram, true); - return new TokenStreamComponents(source, result); - } -} diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java index 12aa58576da..65a4436ddb6 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -3,7 +3,6 @@ import java.util.List; import org.jabref.model.search.Analyzer.LatexAwareAnalyzer; -import org.jabref.model.search.Analyzer.LatexAwareNGramAnalyzer; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.en.EnglishAnalyzer; @@ -28,7 +27,6 @@ public enum SearchFieldConstants { public static final Analyzer LINKED_FILES_ANALYZER = new EnglishAnalyzer(); public static final Analyzer LATEX_AWARE_ANALYZER = new LatexAwareAnalyzer(); - public static final Analyzer LATEX_AWARE_NGRAM_ANALYZER = new LatexAwareNGramAnalyzer(1, Integer.MAX_VALUE); public static final List PDF_FIELDS = List.of(CONTENT.toString(), ANNOTATIONS.toString()); private final String field; diff --git a/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java b/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java index 0ca82db33ce..f09dd84092b 100644 --- a/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java +++ b/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java @@ -15,7 +15,7 @@ import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.preferences.OwnerPreferences; -import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.search.IndexManager; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; @@ -98,7 +98,7 @@ void setUp() { themeManager, taskExecutor, journalAbbreviationRepository, - mock(LuceneManager.class), + mock(IndexManager.class), OptionalObjectProperty.empty() ); } From 7839c68a5fb143fb308ada3131213995f54b0baa Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Sun, 22 Sep 2024 23:22:56 +0200 Subject: [PATCH 008/104] Begin to implement "new" search syntax Co-authored-by: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> Co-authored-by: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> --- src/main/antlr4/org/jabref/search/Search.g4 | 4 +- .../logic/search/SearchToSqlConversion.java | 29 +++++ .../logic/search/SearchToSqlVisitor.java | 115 ++++++++++++++++++ .../migrations/SearchToLuceneMigration.java | 4 + .../search/SearchToSqlConversionTest.java | 16 +++ 5 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/jabref/logic/search/SearchToSqlConversion.java create mode 100644 src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java create mode 100644 src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java diff --git a/src/main/antlr4/org/jabref/search/Search.g4 b/src/main/antlr4/org/jabref/search/Search.g4 index e2ee78b6c9d..5124e82d9d9 100644 --- a/src/main/antlr4/org/jabref/search/Search.g4 +++ b/src/main/antlr4/org/jabref/search/Search.g4 @@ -13,6 +13,8 @@ RPAREN:')'; EQUAL:'='; // semantically the same as CONTAINS EEQUAL:'=='; // semantically the same as MATCHES NEQUAL:'!='; +CEQUAL:'=!'; // case sensitive contains +CEEQUAL:'==!'; // case sensitive exact match AND:[aA][nN][dD]; // 'and' case insensitive OR:[oO][rR]; // 'or' case insensitive @@ -42,7 +44,7 @@ expression: ; comparison: - left=name operator=(CONTAINS | MATCHES | EQUAL | EEQUAL | NEQUAL) right=name // example: author != miller + left=name operator=(CONTAINS | MATCHES | EQUAL | EEQUAL | NEQUAL | CEQUAL | CEEQUAL) right=name // example: author != miller | right=name // example: miller (search all fields) ; diff --git a/src/main/java/org/jabref/logic/search/SearchToSqlConversion.java b/src/main/java/org/jabref/logic/search/SearchToSqlConversion.java new file mode 100644 index 00000000000..20a7845d257 --- /dev/null +++ b/src/main/java/org/jabref/logic/search/SearchToSqlConversion.java @@ -0,0 +1,29 @@ +package org.jabref.logic.search; + +import org.jabref.model.search.ThrowingErrorListener; +import org.jabref.search.SearchLexer; +import org.jabref.search.SearchParser; + +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.CommonTokenStream; + +public class SearchToSqlConversion { + + public static String searchToSql(String table, String searchExpression) { + SearchParser.StartContext context = getStartContext(searchExpression); + SearchToSqlVisitor searchToSqlVisitor = new SearchToSqlVisitor(table); + return searchToSqlVisitor.visit(context); + } + + private static SearchParser.StartContext getStartContext(String searchExpression) { + SearchLexer lexer = new SearchLexer(new ANTLRInputStream(searchExpression)); + lexer.removeErrorListeners(); // no infos on file system + lexer.addErrorListener(ThrowingErrorListener.INSTANCE); + SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); + parser.removeErrorListeners(); // no infos on file system + parser.addErrorListener(ThrowingErrorListener.INSTANCE); + parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors + return parser.start(); + } +} diff --git a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java new file mode 100644 index 00000000000..e4cb03fed0a --- /dev/null +++ b/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java @@ -0,0 +1,115 @@ +package org.jabref.logic.search; + +import java.util.EnumSet; +import java.util.Optional; + +import org.jabref.logic.search.indexing.PostgreConstants; +import org.jabref.model.search.SearchFieldConstants; +import org.jabref.search.SearchBaseVisitor; +import org.jabref.search.SearchParser; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Similar class: {@link org.jabref.migrations.SearchToLuceneMigration} + */ +public class SearchToSqlVisitor extends SearchBaseVisitor { + + private static final Logger LOGGER = LoggerFactory.getLogger(SearchToSqlVisitor.class); + private final String tableName; + + public SearchToSqlVisitor(String tableName) { + this.tableName = tableName; + } + + private enum SearchTermFlag { + REGULAR_EXPRESSION, CASE_SENSITIVE, EXACT_MATCH + } + + @Override + public String visitStart(SearchParser.StartContext ctx) { + return "SELECT " + PostgreConstants.ENTRY_ID + " FROM " + tableName + " WHERE " + visit(ctx.expression()); + } + + @Override + public String visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { + return "NOT " + visit(ctx.expression()); + } + + @Override + public String visitParenExpression(SearchParser.ParenExpressionContext ctx) { + return "(" + visit(ctx.expression()) + ")"; + } + + @Override + public String visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { + if ("AND".equalsIgnoreCase(ctx.operator.getText())) { + return visit(ctx.left) + " AND " + visit(ctx.right); + } else { + return visit(ctx.left) + " OR " + visit(ctx.right); + } + } + + @Override + public String visitComparison(SearchParser.ComparisonContext context) { + // The comparison is a leaf node in the tree + + // remove possible enclosing " symbols + String right = context.right.getText(); + if (right.startsWith("\"") && right.endsWith("\"")) { + right = right.substring(1, right.length() - 1); + } + + Optional fieldDescriptor = Optional.ofNullable(context.left); + int startIndex = context.getStart().getStartIndex(); + int stopIndex = context.getStop().getStopIndex(); + if (fieldDescriptor.isPresent()) { + String field = fieldDescriptor.get().getText(); + + // Direct comparison does not work + // context.CONTAINS() and others are null if absent (thus, we cannot check for getText()) + EnumSet searchFlags = EnumSet.noneOf(SearchTermFlag.class); + if (context.MATCHES() != null || context.EEQUAL() != null) { + searchFlags.add(SearchTermFlag.EXACT_MATCH); + } + // TODO: Add check for regular expression + // TODO: Add check for case sensitivity + + // NEQUAL is treated at unaryExpression + assert (context.NEQUAL() == null); + + return getFieldQueryNode(field, right, searchFlags); + } else { + // Query without any field name + return getFieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), right, EnumSet.noneOf(SearchTermFlag.class)); + } + } + + private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { + /* + field = switch (field) { + case "anyfield" -> field(PostgreConstants.FIELD_VALUE.toString()).eq; + case "anykeyword" -> StandardField.KEYWORDS.getName(); + case "key" -> InternalField.KEY_FIELD.getName(); + default -> field; + }; + + if (isRegularExpression || forceRegex) { + // Lucene does a sanity check on the positions, thus we provide other fake positions + return new RegexpQueryNode(field, term, 0, term.length()); + } + return new FieldQueryNode(field, term, startIndex, stopIndex); + */ + // TODO: Handle search flags + String operator = "~*"; + if (searchFlags.equals(EnumSet.of(SearchTermFlag.REGULAR_EXPRESSION))) { + operator = "~*"; + } else if (searchFlags.equals(EnumSet.of(SearchTermFlag.CASE_SENSITIVE, SearchTermFlag.EXACT_MATCH))) { + return "(" + PostgreConstants.FIELD_NAME + " = '" + field + "' AND " + PostgreConstants.FIELD_VALUE + " ~ '\\y'" + term + "\\y')"; + } else if (searchFlags.equals(EnumSet.of(SearchTermFlag.CASE_SENSITIVE))) { + return "(" + PostgreConstants.FIELD_NAME + " = '" + field + "' AND " + PostgreConstants.FIELD_VALUE + " ~ '\\y'" + term + "\\y')"; + } + return "(" + PostgreConstants.FIELD_NAME + " = '" + field + "' AND " + PostgreConstants.FIELD_VALUE + " " + operator + " '" + term + "')"; + } +} diff --git a/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java b/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java index 4a0d2db6297..c56fd255dec 100644 --- a/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java +++ b/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java @@ -10,6 +10,10 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.standard.parser.EscapeQuerySyntaxImpl; +/** + * @deprecated This class is deprecated and will be removed in the future. We use Search.g4 as main search grammar + */ +@Deprecated public class SearchToLuceneMigration { public static String migrateToLuceneSyntax(String searchExpression, boolean isRegularExpression) { SearchParser.StartContext context = getStartContext(searchExpression); diff --git a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java new file mode 100644 index 00000000000..462a0592be4 --- /dev/null +++ b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java @@ -0,0 +1,16 @@ +package org.jabref.logic.search; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class SearchToSqlConversionTest { + @ParameterizedTest + @CsvSource({ + "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value ~* 'compute'), title=compute" + }) + void conversion(String expected, String input) { + assertEquals(expected, SearchToSqlConversion.searchToSql("tableName", input)); + } +} From 65fd4fe52606a9e7d52024bc3d1b8a6789876382 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 23 Sep 2024 00:39:50 +0200 Subject: [PATCH 009/104] Some new operators --- src/main/antlr4/org/jabref/search/Search.g4 | 12 ++++- .../logic/search/SearchToSqlVisitor.java | 45 +++++++++++++++---- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/main/antlr4/org/jabref/search/Search.g4 b/src/main/antlr4/org/jabref/search/Search.g4 index 5124e82d9d9..da473f2e626 100644 --- a/src/main/antlr4/org/jabref/search/Search.g4 +++ b/src/main/antlr4/org/jabref/search/Search.g4 @@ -11,10 +11,18 @@ LPAREN:'('; RPAREN:')'; EQUAL:'='; // semantically the same as CONTAINS -EEQUAL:'=='; // semantically the same as MATCHES NEQUAL:'!='; + +EEQUAL:'=='; // semantically the same as MATCHES +NEEQUAL:'!=='; // negated semantically the same as MATCHES + CEQUAL:'=!'; // case sensitive contains +NCEQUAL:'!=!'; // negated case sensitive contains + CEEQUAL:'==!'; // case sensitive exact match +NCEEQUAL:'!==!'; // negated case sensitive exact match + +REQUAL:'=~'; // regex check AND:[aA][nN][dD]; // 'and' case insensitive OR:[oO][rR]; // 'or' case insensitive @@ -44,7 +52,7 @@ expression: ; comparison: - left=name operator=(CONTAINS | MATCHES | EQUAL | EEQUAL | NEQUAL | CEQUAL | CEEQUAL) right=name // example: author != miller + left=name operator=(CONTAINS | MATCHES | EQUAL | EEQUAL | NEQUAL | CEQUAL | CEEQUAL | REQUAL) right=name // example: author != miller | right=name // example: miller (search all fields) ; diff --git a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java index e4cb03fed0a..9e6a816a68c 100644 --- a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java @@ -24,7 +24,10 @@ public SearchToSqlVisitor(String tableName) { } private enum SearchTermFlag { - REGULAR_EXPRESSION, CASE_SENSITIVE, EXACT_MATCH + REGULAR_EXPRESSION, // mutually exclusive to the others + NEGATION, + CASE_SENSITIVE, CASE_INSENSITIVE, // mutually exclusive + EXACT_MATCH, INEXACT_MATCH // mutually exclusive } @Override @@ -70,11 +73,33 @@ public String visitComparison(SearchParser.ComparisonContext context) { // Direct comparison does not work // context.CONTAINS() and others are null if absent (thus, we cannot check for getText()) EnumSet searchFlags = EnumSet.noneOf(SearchTermFlag.class); - if (context.MATCHES() != null || context.EEQUAL() != null) { - searchFlags.add(SearchTermFlag.EXACT_MATCH); + if (context.REQUAL() != null) { + searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); + } else { + if (context.CONTAINS() != null || context.EQUAL() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); + } else if (context.NEQUAL() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_SENSITIVE); + searchFlags.add(SearchTermFlag.NEGATION); + } else if (context.CEQUAL() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_SENSITIVE); +// } else if (context.NCEQUAL() != null) { +// searchFlags.add(SearchTermFlag.INEXACT_MATCH); +// searchFlags.add(SearchTermFlag.CASE_SENSITIVE); +// searchFlags.add(SearchTermFlag.NEGATION); + } else if (context.MATCHES() != null || context.EEQUAL() != null) { + searchFlags.add(SearchTermFlag.EXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); + } else if (context.CEEQUAL() != null) { + searchFlags.add(SearchTermFlag.EXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_SENSITIVE); + } else if (context.NEQUAL() != null) { + searchFlags.add(SearchTermFlag.NEGATION); + } } - // TODO: Add check for regular expression - // TODO: Add check for case sensitivity // NEQUAL is treated at unaryExpression assert (context.NEQUAL() == null); @@ -82,7 +107,7 @@ public String visitComparison(SearchParser.ComparisonContext context) { return getFieldQueryNode(field, right, searchFlags); } else { // Query without any field name - return getFieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), right, EnumSet.noneOf(SearchTermFlag.class)); + return getFieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), right, EnumSet.of(SearchTermFlag.INEXACT_MATCH, SearchTermFlag.CASE_INSENSITIVE)); } } @@ -103,13 +128,15 @@ private String getFieldQueryNode(String field, String term, EnumSet Date: Mon, 23 Sep 2024 05:16:04 +0300 Subject: [PATCH 010/104] Change log level of EmbeddedPostgres --- src/main/resources/tinylog.properties | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/resources/tinylog.properties b/src/main/resources/tinylog.properties index 2b4bd0c3c28..2b99c963cef 100644 --- a/src/main/resources/tinylog.properties +++ b/src/main/resources/tinylog.properties @@ -12,11 +12,7 @@ level@org.apache.pdfbox.pdmodel.PDSimpleFont = error level@org.apache.fontbox.util.autodetect.FontFileFinder = warn level@org.apache.fontbox.ttf = warn level@ai.djl = info - -#level@org.jabref.model.entry.BibEntry = debug -#level@org.jabref.gui.maintable.PersistenceVisualStateTable = debug - -level@org.jabref.http.server.Server = debug +level@io.zonky.test.db.postgres.embedded = warn #level@org.jabref.gui.JabRefGUI = debug From 1e3482d84d24cb8b218f6817cbc2bc1cd5cccdf2 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 23 Sep 2024 09:01:20 +0300 Subject: [PATCH 011/104] Handle search flags --- src/main/antlr4/org/jabref/search/Search.g4 | 14 ++--- .../logic/search/SearchToSqlVisitor.java | 51 ++++++++++--------- .../search/SearchToSqlConversionTest.java | 18 ++++++- 3 files changed, 52 insertions(+), 31 deletions(-) diff --git a/src/main/antlr4/org/jabref/search/Search.g4 b/src/main/antlr4/org/jabref/search/Search.g4 index da473f2e626..338ac91a454 100644 --- a/src/main/antlr4/org/jabref/search/Search.g4 +++ b/src/main/antlr4/org/jabref/search/Search.g4 @@ -10,16 +10,16 @@ WS: [ \t] -> skip; // whitespace is ignored/skipped LPAREN:'('; RPAREN:')'; -EQUAL:'='; // semantically the same as CONTAINS -NEQUAL:'!='; +EQUAL:'='; // case insensitive contains, semantically the same as CONTAINS +CEQUAL:'=!'; // case sensitive contains -EEQUAL:'=='; // semantically the same as MATCHES -NEEQUAL:'!=='; // negated semantically the same as MATCHES +EEQUAL:'=='; // exact match case insensitive, semantically the same as MATCHES +CEEQUAL:'==!'; // exact match case sensitive -CEQUAL:'=!'; // case sensitive contains +NEQUAL:'!='; // negated case insensitive contains NCEQUAL:'!=!'; // negated case sensitive contains -CEEQUAL:'==!'; // case sensitive exact match +NEEQUAL:'!=='; // negated case insensitive exact match NCEEQUAL:'!==!'; // negated case sensitive exact match REQUAL:'=~'; // regex check @@ -52,7 +52,7 @@ expression: ; comparison: - left=name operator=(CONTAINS | MATCHES | EQUAL | EEQUAL | NEQUAL | CEQUAL | CEEQUAL | REQUAL) right=name // example: author != miller + left=name operator=(CONTAINS | EQUAL | CEQUAL | MATCHES | EEQUAL | CEEQUAL | NEQUAL | NCEQUAL | NEEQUAL | NCEEQUAL | REQUAL) right=name // example: author != miller | right=name // example: miller (search all fields) ; diff --git a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java index 9e6a816a68c..1eabc397c16 100644 --- a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java @@ -65,8 +65,6 @@ public String visitComparison(SearchParser.ComparisonContext context) { } Optional fieldDescriptor = Optional.ofNullable(context.left); - int startIndex = context.getStart().getStartIndex(); - int stopIndex = context.getStop().getStopIndex(); if (fieldDescriptor.isPresent()) { String field = fieldDescriptor.get().getText(); @@ -79,17 +77,9 @@ public String visitComparison(SearchParser.ComparisonContext context) { if (context.CONTAINS() != null || context.EQUAL() != null) { searchFlags.add(SearchTermFlag.INEXACT_MATCH); searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); - } else if (context.NEQUAL() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_SENSITIVE); - searchFlags.add(SearchTermFlag.NEGATION); } else if (context.CEQUAL() != null) { searchFlags.add(SearchTermFlag.INEXACT_MATCH); searchFlags.add(SearchTermFlag.CASE_SENSITIVE); -// } else if (context.NCEQUAL() != null) { -// searchFlags.add(SearchTermFlag.INEXACT_MATCH); -// searchFlags.add(SearchTermFlag.CASE_SENSITIVE); -// searchFlags.add(SearchTermFlag.NEGATION); } else if (context.MATCHES() != null || context.EEQUAL() != null) { searchFlags.add(SearchTermFlag.EXACT_MATCH); searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); @@ -97,13 +87,22 @@ public String visitComparison(SearchParser.ComparisonContext context) { searchFlags.add(SearchTermFlag.EXACT_MATCH); searchFlags.add(SearchTermFlag.CASE_SENSITIVE); } else if (context.NEQUAL() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); + searchFlags.add(SearchTermFlag.NEGATION); + } else if (context.NCEQUAL() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_SENSITIVE); + searchFlags.add(SearchTermFlag.NEGATION); + } else if (context.NEEQUAL() != null) { + searchFlags.add(SearchTermFlag.EXACT_MATCH); + searchFlags.add(SearchTermFlag.NEGATION); + } else if (context.NCEEQUAL() != null) { + searchFlags.add(SearchTermFlag.EXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_SENSITIVE); searchFlags.add(SearchTermFlag.NEGATION); } } - - // NEQUAL is treated at unaryExpression - assert (context.NEQUAL() == null); - return getFieldQueryNode(field, right, searchFlags); } else { // Query without any field name @@ -126,17 +125,23 @@ private String getFieldQueryNode(String field, String term, EnumSet Date: Mon, 23 Sep 2024 09:10:31 +0300 Subject: [PATCH 012/104] Fix default field --- .../jabref/logic/search/SearchToSqlVisitor.java | 14 ++++++-------- .../logic/search/SearchToSqlConversionTest.java | 2 ++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java index 1eabc397c16..71d7c8e86ff 100644 --- a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java @@ -4,6 +4,8 @@ import java.util.Optional; import org.jabref.logic.search.indexing.PostgreConstants; +import org.jabref.model.entry.field.InternalField; +import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.SearchFieldConstants; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; @@ -111,20 +113,13 @@ public String visitComparison(SearchParser.ComparisonContext context) { } private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { - /* field = switch (field) { - case "anyfield" -> field(PostgreConstants.FIELD_VALUE.toString()).eq; + case "anyfield" -> SearchFieldConstants.DEFAULT_FIELD.toString(); case "anykeyword" -> StandardField.KEYWORDS.getName(); case "key" -> InternalField.KEY_FIELD.getName(); default -> field; }; - if (isRegularExpression || forceRegex) { - // Lucene does a sanity check on the positions, thus we provide other fake positions - return new RegexpQueryNode(field, term, 0, term.length()); - } - return new FieldQueryNode(field, term, startIndex, stopIndex); - */ String operator = ""; String prefixSuffix = ""; @@ -142,6 +137,9 @@ private String getFieldQueryNode(String field, String term, EnumSet Date: Mon, 23 Sep 2024 10:18:07 +0200 Subject: [PATCH 013/104] Fix handling of anyfield (and add "any" as alias) --- .../logic/search/SearchToSqlVisitor.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java index 71d7c8e86ff..865d9eacaee 100644 --- a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java @@ -5,7 +5,6 @@ import org.jabref.logic.search.indexing.PostgreConstants; import org.jabref.model.entry.field.InternalField; -import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.SearchFieldConstants; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; @@ -113,13 +112,6 @@ public String visitComparison(SearchParser.ComparisonContext context) { } private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { - field = switch (field) { - case "anyfield" -> SearchFieldConstants.DEFAULT_FIELD.toString(); - case "anykeyword" -> StandardField.KEYWORDS.getName(); - case "key" -> InternalField.KEY_FIELD.getName(); - default -> field; - }; - String operator = ""; String prefixSuffix = ""; @@ -137,6 +129,19 @@ private String getFieldQueryNode(String field, String term, EnumSet InternalField.KEY_FIELD.getName(); + default -> field; + }; + if (SearchFieldConstants.DEFAULT_FIELD.toString().equals(field)) { return "(" + PostgreConstants.FIELD_VALUE + " " + operator + " '" + prefixSuffix + term + prefixSuffix + "')"; } From 2749b866ad80801e3dd217b525b1d1b845dc0e77 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 23 Sep 2024 10:20:58 +0200 Subject: [PATCH 014/104] Openrewrite... --- .../java/org/jabref/logic/search/indexing/PostgreConstants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreConstants.java b/src/main/java/org/jabref/logic/search/indexing/PostgreConstants.java index 375a5216c7b..8889547b80a 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreConstants.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreConstants.java @@ -12,7 +12,7 @@ public enum PostgreConstants { } public String getIndexName(String tableName) { - return String.format("%s_%s_index", tableName, this.columnName); + return "%s_%s_index".formatted(tableName, this.columnName); } @Override From 58d7e250c3c2b6cc494184373bb94bf39a166286 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 23 Sep 2024 11:26:10 +0300 Subject: [PATCH 015/104] More test cases --- .../search/SearchToSqlConversionTest.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java index 716c4b3098d..a5be0e7367b 100644 --- a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java +++ b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java @@ -9,7 +9,25 @@ class SearchToSqlConversionTest { @ParameterizedTest @CsvSource({ // Default search, query without any field name (case insensitive contains) - "SELECT entry_id FROM tableName WHERE (field_value ~* 'compute'), compute", + "SELECT entry_id FROM tableName WHERE (field_value ~* 'computer'), computer", + "SELECT entry_id FROM tableName WHERE (field_value ~* 'computer science'), \"computer science\"", // Phrase search + "SELECT entry_id FROM tableName WHERE (field_value ~* 'computer') OR (field_value ~* 'science'), computer science", // Should be searched as a phrase or as two separate words (OR)? + "SELECT entry_id FROM tableName WHERE (field_value ~* '!computer'), !computer", // Is the explanation should be escaped? + "SELECT entry_id FROM tableName WHERE (field_value ~* '!computer'), \"!computer\"", + // search in all fields case sensitive contains + "SELECT entry_id FROM tableName WHERE (field_value ~ 'computer'), any=!computer", + "SELECT entry_id FROM tableName WHERE (field_value ~ '!computer'), any=! !computer", // Is the explanation should be escaped? + // Regex search + "SELECT entry_id FROM tableName WHERE (field_value ~* 'Jabref.*Search'), Jabref.*Search", + "SELECT entry_id FROM tableName WHERE (field_value ~* 'Jabref.*Search'), \"Jabref.*Search\"", // This is wrong, this query should be a string literal searching for "Jabref.*Search" as string, should use syntax LIKE %...% + // And + "SELECT entry_id FROM tableName WHERE (field_value ~* 'computer') AND (field_value ~* 'science'), computer AND science", + // Or + "SELECT entry_id FROM tableName WHERE (field_value ~* 'computer') OR (field_value ~* 'science'), computer OR science", + // Grouping + "SELECT entry_id FROM tableName WHERE ((field_value ~* 'computer') AND (field_value ~* 'science')) OR (field_value ~* 'math'), (computer AND science) OR math", + "SELECT entry_id FROM tableName WHERE (field_value ~* 'computer') AND ((field_value ~* 'science') OR (field_value ~* 'math')), computer AND (science OR math)", + "SELECT entry_id FROM tableName WHERE ((field_value ~* 'computer') OR (field_value ~* 'science')) AND ((field_value ~* 'math') OR (field_value ~* 'physics')), (computer OR science) AND (math OR physics)", // case insensitive contains "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value ~* 'compute'), title=compute", // case sensitive contains From 9028748654a79b4e637de55f0a14523a1de98fdd Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 23 Sep 2024 11:34:49 +0200 Subject: [PATCH 016/104] Remove non-covered libraries --- external-libraries.md | 112 ++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 71 deletions(-) diff --git a/external-libraries.md b/external-libraries.md index 9b0d4feccf3..a05e3757ad9 100644 --- a/external-libraries.md +++ b/external-libraries.md @@ -706,12 +706,8 @@ License: BSD-3-Clause 3. (on WSL) `sed 's/[^a-z]*//' < build/reports/project/dependencies.txt | sed "s/\(.*\) .*/\1/" | grep -v "\->" | sort | uniq > build/dependencies-for-external-libraries.txt` ```text -ai.djl.huggingface:tokenizers:0.29.0 -ai.djl.pytorch:pytorch-engine:0.30.0 -ai.djl.pytorch:pytorch-model-zoo:0.30.0 -ai.djl:api:0.30.0 at.favre.lib:hkdf:1.1.0 -com.dlsc.gemsfx:gemsfx:2.48.0 +com.dlsc.gemsfx:gemsfx:2.32.0 com.dlsc.pickerfx:pickerfx:1.3.1 com.dlsc.unitfx:unitfx:1.0.10 com.fasterxml.jackson.core:jackson-annotations:2.17.2 @@ -728,37 +724,29 @@ com.github.sialcasa.mvvmFX:mvvmfx-validation:f195849ca9 com.github.tomtung:latex2unicode_2.13:0.3.2 com.github.vatbub:mslinks:1.0.6.2 com.github.weisj:jsvg:1.2.0 -com.google.code.gson:gson:2.11.0 -com.google.errorprone:error_prone_annotations:2.27.0 +com.google.code.gson:gson:2.10.1 +com.google.errorprone:error_prone_annotations:2.26.1 com.google.guava:failureaccess:1.0.2 com.google.guava:guava:33.1.0-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:3.0.0 com.googlecode.javaewah:JavaEWAH:1.2.3 com.googlecode.plist:dd-plist:1.28 -com.h2database:h2-mvstore:2.3.232 -com.knuddels:jtokkit:1.1.0 -com.kohlschutter.junixsocket:junixsocket-common:2.10.0 -com.kohlschutter.junixsocket:junixsocket-core:2.10.0 -com.kohlschutter.junixsocket:junixsocket-mysql:2.10.0 -com.kohlschutter.junixsocket:junixsocket-native-common:2.10.0 -com.konghq:unirest-java-core:4.4.4 -com.konghq:unirest-modules-gson:4.4.4 +com.h2database:h2-mvstore:2.2.224 +com.kohlschutter.junixsocket:junixsocket-common:2.9.1 +com.kohlschutter.junixsocket:junixsocket-core:2.9.1 +com.kohlschutter.junixsocket:junixsocket-mysql:2.9.1 +com.kohlschutter.junixsocket:junixsocket-native-common:2.9.1 +com.konghq:unirest-java-core:4.4.0 +com.konghq:unirest-modules-gson:4.4.0 com.oracle.ojdbc:ojdbc10:19.3.0.0 com.oracle.ojdbc:ons:19.3.0.0 com.oracle.ojdbc:osdt_cert:19.3.0.0 com.oracle.ojdbc:osdt_core:19.3.0.0 com.oracle.ojdbc:simplefan:19.3.0.0 com.oracle.ojdbc:ucp:19.3.0.0 -com.squareup.okhttp3:logging-interceptor:4.12.0 -com.squareup.okhttp3:okhttp-sse:4.12.0 -com.squareup.okhttp3:okhttp:4.12.0 -com.squareup.okio:okio-jvm:3.6.0 -com.squareup.okio:okio:3.6.0 -com.squareup.retrofit2:converter-gson:2.9.0 -com.squareup.retrofit2:converter-jackson:2.9.0 -com.squareup.retrofit2:retrofit:2.11.0 com.sun.istack:istack-commons-runtime:4.1.2 +com.tobiasdiez:easybind:2.2.1-SNAPSHOT com.vladsch.flexmark:flexmark-ext-emoji:0.64.8 com.vladsch.flexmark:flexmark-ext-gfm-strikethrough:0.64.8 com.vladsch.flexmark:flexmark-ext-ins:0.64.8 @@ -781,12 +769,12 @@ com.vladsch.flexmark:flexmark-util-visitor:0.64.8 com.vladsch.flexmark:flexmark-util:0.64.8 com.vladsch.flexmark:flexmark:0.64.8 commons-beanutils:commons-beanutils:1.9.4 -commons-cli:commons-cli:1.9.0 -commons-codec:commons-codec:1.17.1 +commons-cli:commons-cli:1.8.0 +commons-codec:commons-codec:1.17.0 commons-collections:commons-collections:3.2.2 commons-digester:commons-digester:2.1 commons-io:commons-io:2.16.1 -commons-logging:commons-logging:1.3.4 +commons-logging:commons-logging:1.3.2 commons-validator:commons-validator:1.8.0 de.rototor.jeuclid:jeuclid-core:3.1.11 de.rototor.snuggletex:snuggletex-core:1.3.0 @@ -795,19 +783,9 @@ de.rototor.snuggletex:snuggletex:1.3.0 de.saxsys:mvvmfx:1.8.0 de.swiesend:secret-service:1.8.1-jdk17 de.undercouch:citeproc-java:3.1.0 -dev.ai4j:openai4j:0.20.0 -dev.langchain4j:langchain4j-core:0.34.0 -dev.langchain4j:langchain4j-google-ai-gemini:0.34.0 -dev.langchain4j:langchain4j-hugging-face:0.34.0 -dev.langchain4j:langchain4j-mistral-ai:0.34.0 -dev.langchain4j:langchain4j-open-ai:0.34.0 -dev.langchain4j:langchain4j:0.34.0 eu.lestard:doc-annotations:0.2 info.debatty:java-string-similarity:2.0.0 -io.github.adr:e-adr:2.0.0-SNAPSHOT io.github.java-diff-utils:java-diff-utils:4.12 -io.github.stefanbratanov:jvm-openai:0.11.0 -io.github.thibaultmeyer:cuid:2.0.3 io.zonky.test:embedded-postgres:2.0.7 jakarta.activation:jakarta.activation-api:2.1.3 jakarta.annotation:jakarta.annotation-api:2.1.1 @@ -818,32 +796,30 @@ jakarta.xml.bind:jakarta.xml.bind-api:4.0.2 javax.measure:unit-api:2.2 net.harawata:appdirs:1.2.2 net.java.dev.jna:jna-platform:5.13.0 -net.java.dev.jna:jna:5.14.0 +net.java.dev.jna:jna:5.13.0 net.jcip:jcip-annotations:1.0 net.jodah:typetools:0.6.1 net.synedra:validatorfx:0.5.0 one.jpro.jproutils:tree-showing:0.2.2 -org.antlr:antlr4-runtime:4.13.2 -org.apache.commons:commons-compress:1.27.1 +org.antlr:antlr4-runtime:4.13.1 org.apache.commons:commons-csv:1.11.0 -org.apache.commons:commons-lang3:3.17.0 +org.apache.commons:commons-lang3:3.14.0 org.apache.commons:commons-text:1.12.0 org.apache.httpcomponents.client5:httpclient5:5.3.1 org.apache.httpcomponents.core5:httpcore5-h2:5.2.4 org.apache.httpcomponents.core5:httpcore5:5.2.4 -org.apache.logging.log4j:log4j-api:2.24.0 -org.apache.logging.log4j:log4j-to-slf4j:2.24.0 +org.apache.logging.log4j:log4j-api:2.23.1 +org.apache.logging.log4j:log4j-to-slf4j:2.23.1 org.apache.lucene:lucene-analysis-common:9.11.1 org.apache.lucene:lucene-core:9.11.1 org.apache.lucene:lucene-highlighter:9.11.1 org.apache.lucene:lucene-queries:9.11.1 org.apache.lucene:lucene-queryparser:9.11.1 org.apache.lucene:lucene-sandbox:9.11.1 -org.apache.opennlp:opennlp-tools:1.9.4 -org.apache.pdfbox:fontbox:3.0.3 -org.apache.pdfbox:pdfbox-io:3.0.3 -org.apache.pdfbox:pdfbox:3.0.3 -org.apache.pdfbox:xmpbox:3.0.3 +org.apache.pdfbox:fontbox:3.0.2 +org.apache.pdfbox:pdfbox-io:3.0.2 +org.apache.pdfbox:pdfbox:3.0.2 +org.apache.pdfbox:xmpbox:3.0.2 org.apiguardian:apiguardian-api:1.1.2 org.bouncycastle:bcprov-jdk18on:1.78.1 org.checkerframework:checker-qual:3.42.0 @@ -865,23 +841,18 @@ org.glassfish.hk2:osgi-resource-locator:1.0.3 org.glassfish.jaxb:jaxb-core:4.0.3 org.glassfish.jaxb:jaxb-runtime:4.0.3 org.glassfish.jaxb:txw2:4.0.3 -org.glassfish.jersey.containers:jersey-container-grizzly2-http:3.1.8 -org.glassfish.jersey.core:jersey-client:3.1.8 -org.glassfish.jersey.core:jersey-common:3.1.8 -org.glassfish.jersey.core:jersey-server:3.1.8 -org.glassfish.jersey.inject:jersey-hk2:3.1.8 +org.glassfish.jersey.containers:jersey-container-grizzly2-http:3.1.7 +org.glassfish.jersey.core:jersey-client:3.1.7 +org.glassfish.jersey.core:jersey-common:3.1.7 +org.glassfish.jersey.core:jersey-server:3.1.7 +org.glassfish.jersey.inject:jersey-hk2:3.1.7 org.jabref:afterburner.fx:2.0.0 -org.jabref:easybind:2.2.1-SNAPSHOT org.javassist:javassist:3.30.2-GA org.jbibtex:jbibtex:1.0.20 -org.jetbrains.kotlin:kotlin-stdlib-common:2.0.20 -org.jetbrains.kotlin:kotlin-stdlib-jdk7:2.0.20 -org.jetbrains.kotlin:kotlin-stdlib-jdk8:2.0.20 -org.jetbrains.kotlin:kotlin-stdlib:2.0.20 org.jetbrains:annotations:24.0.1 org.jooq:jool:0.9.15 -org.jsoup:jsoup:1.18.1 -org.jspecify:jspecify:1.0.0 +org.jsoup:jsoup:1.17.2 +org.jspecify:jspecify:0.3.0 org.kordamp.ikonli:ikonli-bootstrapicons-pack:12.3.1 org.kordamp.ikonli:ikonli-core:12.3.1 org.kordamp.ikonli:ikonli-javafx:12.3.1 @@ -891,23 +862,22 @@ org.kordamp.ikonli:ikonli-materialdesign2-pack:12.3.1 org.libreoffice:libreoffice:24.2.3 org.libreoffice:unoloader:24.2.3 org.mariadb.jdbc:mariadb-java-client:2.7.9 -org.openjfx:javafx-base:23 -org.openjfx:javafx-controls:23 -org.openjfx:javafx-fxml:23 -org.openjfx:javafx-graphics:23 -org.openjfx:javafx-media:23 -org.openjfx:javafx-swing:23 -org.openjfx:javafx-web:23 -org.postgresql:postgresql:42.7.4 +org.openjfx:javafx-base:22.0.1 +org.openjfx:javafx-controls:22.0.1 +org.openjfx:javafx-fxml:22.0.1 +org.openjfx:javafx-graphics:22.0.1 +org.openjfx:javafx-media:22.0.1 +org.openjfx:javafx-swing:22.0.1 +org.openjfx:javafx-web:22.0.1 +org.postgresql:postgresql:42.7.3 org.reactfx:reactfx:2.0-M5 org.scala-lang:scala-library:2.13.8 -org.slf4j:jul-to-slf4j:2.0.16 -org.slf4j:slf4j-api:2.0.16 +org.slf4j:jul-to-slf4j:2.0.13 +org.slf4j:slf4j-api:2.0.13 org.tinylog:slf4j-tinylog:2.7.0 org.tinylog:tinylog-api:2.7.0 org.tinylog:tinylog-impl:2.7.0 -org.tukaani:xz:1.9 -org.yaml:snakeyaml:2.3 +org.yaml:snakeyaml:2.2 pt.davidafsilva.apple:jkeychain:1.1.0 tech.units:indriya:2.2 tech.uom.lib:uom-lib-common:2.2 From 64bc03c53dbc316a724331e2bcbb66f8c45886a1 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 23 Sep 2024 15:36:16 +0300 Subject: [PATCH 017/104] Use LIKE syntax as default instead of regex Disabled exact match --- src/main/antlr4/org/jabref/search/Search.g4 | 10 +- .../logic/search/SearchToSqlVisitor.java | 96 ++++++++++--------- .../search/SearchToSqlConversionTest.java | 51 +++++----- 3 files changed, 83 insertions(+), 74 deletions(-) diff --git a/src/main/antlr4/org/jabref/search/Search.g4 b/src/main/antlr4/org/jabref/search/Search.g4 index 338ac91a454..bfaeb1b57b1 100644 --- a/src/main/antlr4/org/jabref/search/Search.g4 +++ b/src/main/antlr4/org/jabref/search/Search.g4 @@ -14,15 +14,15 @@ EQUAL:'='; // case insensitive contains, semantically the same as CONTAINS CEQUAL:'=!'; // case sensitive contains EEQUAL:'=='; // exact match case insensitive, semantically the same as MATCHES -CEEQUAL:'==!'; // exact match case sensitive NEQUAL:'!='; // negated case insensitive contains NCEQUAL:'!=!'; // negated case sensitive contains -NEEQUAL:'!=='; // negated case insensitive exact match -NCEEQUAL:'!==!'; // negated case sensitive exact match +REQUAL:'=~'; // regex check case insensitive +CREEQUAL:'=~!'; // regex check case sensitive -REQUAL:'=~'; // regex check +NREQUAL:'!=~'; // negated regex check case insensitive +NCREEQUAL:'!=~!'; // negated regex check case sensitive AND:[aA][nN][dD]; // 'and' case insensitive OR:[oO][rR]; // 'or' case insensitive @@ -52,7 +52,7 @@ expression: ; comparison: - left=name operator=(CONTAINS | EQUAL | CEQUAL | MATCHES | EEQUAL | CEEQUAL | NEQUAL | NCEQUAL | NEEQUAL | NCEEQUAL | REQUAL) right=name // example: author != miller + left=name operator=(CONTAINS | EQUAL | CEQUAL | MATCHES | EEQUAL | NEQUAL | NCEQUAL | REQUAL | CREEQUAL | NREQUAL | NCREEQUAL) right=name // example: author != miller | right=name // example: miller (search all fields) ; diff --git a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java index 865d9eacaee..23d9564b53a 100644 --- a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java @@ -5,6 +5,7 @@ import org.jabref.logic.search.indexing.PostgreConstants; import org.jabref.model.entry.field.InternalField; +import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.SearchFieldConstants; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; @@ -28,7 +29,7 @@ private enum SearchTermFlag { REGULAR_EXPRESSION, // mutually exclusive to the others NEGATION, CASE_SENSITIVE, CASE_INSENSITIVE, // mutually exclusive - EXACT_MATCH, INEXACT_MATCH // mutually exclusive + INEXACT_MATCH // mutually exclusive } @Override @@ -74,35 +75,32 @@ public String visitComparison(SearchParser.ComparisonContext context) { EnumSet searchFlags = EnumSet.noneOf(SearchTermFlag.class); if (context.REQUAL() != null) { searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); - } else { - if (context.CONTAINS() != null || context.EQUAL() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); - } else if (context.CEQUAL() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_SENSITIVE); - } else if (context.MATCHES() != null || context.EEQUAL() != null) { - searchFlags.add(SearchTermFlag.EXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); - } else if (context.CEEQUAL() != null) { - searchFlags.add(SearchTermFlag.EXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_SENSITIVE); - } else if (context.NEQUAL() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); - searchFlags.add(SearchTermFlag.NEGATION); - } else if (context.NCEQUAL() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_SENSITIVE); - searchFlags.add(SearchTermFlag.NEGATION); - } else if (context.NEEQUAL() != null) { - searchFlags.add(SearchTermFlag.EXACT_MATCH); - searchFlags.add(SearchTermFlag.NEGATION); - } else if (context.NCEEQUAL() != null) { - searchFlags.add(SearchTermFlag.EXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_SENSITIVE); - searchFlags.add(SearchTermFlag.NEGATION); - } + searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); + } else if (context.CREEQUAL() != null) { + searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); + searchFlags.add(SearchTermFlag.CASE_SENSITIVE); + } else if (context.NREQUAL() != null) { + searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); + searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); + searchFlags.add(SearchTermFlag.NEGATION); + } else if (context.NCREEQUAL() != null) { + searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); + searchFlags.add(SearchTermFlag.CASE_SENSITIVE); + searchFlags.add(SearchTermFlag.NEGATION); + } else if (context.CONTAINS() != null || context.EQUAL() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); + } else if (context.CEQUAL() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_SENSITIVE); + } else if (context.NEQUAL() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); + searchFlags.add(SearchTermFlag.NEGATION); + } else if (context.NCEQUAL() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_SENSITIVE); + searchFlags.add(SearchTermFlag.NEGATION); } return getFieldQueryNode(field, right, searchFlags); } else { @@ -115,36 +113,40 @@ private String getFieldQueryNode(String field, String term, EnumSet InternalField.KEY_FIELD.getName(); + case "keywords" -> StandardField.KEYWORDS.getName(); default -> field; }; - if (SearchFieldConstants.DEFAULT_FIELD.toString().equals(field)) { - return "(" + PostgreConstants.FIELD_VALUE + " " + operator + " '" + prefixSuffix + term + prefixSuffix + "')"; - } return "(" + PostgreConstants.FIELD_NAME + " = '" + field + "' AND " + PostgreConstants.FIELD_VALUE + " " + operator + " '" + prefixSuffix + term + prefixSuffix + "')"; } } diff --git a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java index a5be0e7367b..f7cdd097f14 100644 --- a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java +++ b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java @@ -9,41 +9,48 @@ class SearchToSqlConversionTest { @ParameterizedTest @CsvSource({ // Default search, query without any field name (case insensitive contains) - "SELECT entry_id FROM tableName WHERE (field_value ~* 'computer'), computer", - "SELECT entry_id FROM tableName WHERE (field_value ~* 'computer science'), \"computer science\"", // Phrase search - "SELECT entry_id FROM tableName WHERE (field_value ~* 'computer') OR (field_value ~* 'science'), computer science", // Should be searched as a phrase or as two separate words (OR)? - "SELECT entry_id FROM tableName WHERE (field_value ~* '!computer'), !computer", // Is the explanation should be escaped? - "SELECT entry_id FROM tableName WHERE (field_value ~* '!computer'), \"!computer\"", + "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%computer%'), computer", + "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%computer science%'), \"computer science\"", // Phrase search + "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%computer%') OR (field_value ILIKE '%science%'), computer science", // Should be searched as a phrase or as two separate words (OR)? + "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%!computer%'), !computer", // Is the explanation should be escaped? + "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%!computer%'), \"!computer\"", // search in all fields case sensitive contains - "SELECT entry_id FROM tableName WHERE (field_value ~ 'computer'), any=!computer", - "SELECT entry_id FROM tableName WHERE (field_value ~ '!computer'), any=! !computer", // Is the explanation should be escaped? + "SELECT entry_id FROM tableName WHERE (field_value LIKE '%computer%'), any=!computer", + "SELECT entry_id FROM tableName WHERE (field_value LIKE '%!computer%'), any=! !computer", // Is the explanation should be escaped? // Regex search - "SELECT entry_id FROM tableName WHERE (field_value ~* 'Jabref.*Search'), Jabref.*Search", - "SELECT entry_id FROM tableName WHERE (field_value ~* 'Jabref.*Search'), \"Jabref.*Search\"", // This is wrong, this query should be a string literal searching for "Jabref.*Search" as string, should use syntax LIKE %...% + "SELECT entry_id FROM tableName WHERE (field_value ~* 'Jabref.*Search'), any=~Jabref.*Search", // And - "SELECT entry_id FROM tableName WHERE (field_value ~* 'computer') AND (field_value ~* 'science'), computer AND science", + "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%computer%') AND (field_value ILIKE '%science%'), computer AND science", // Or - "SELECT entry_id FROM tableName WHERE (field_value ~* 'computer') OR (field_value ~* 'science'), computer OR science", + "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%computer%') OR (field_value ILIKE '%science%'), computer OR science", // Grouping - "SELECT entry_id FROM tableName WHERE ((field_value ~* 'computer') AND (field_value ~* 'science')) OR (field_value ~* 'math'), (computer AND science) OR math", - "SELECT entry_id FROM tableName WHERE (field_value ~* 'computer') AND ((field_value ~* 'science') OR (field_value ~* 'math')), computer AND (science OR math)", - "SELECT entry_id FROM tableName WHERE ((field_value ~* 'computer') OR (field_value ~* 'science')) AND ((field_value ~* 'math') OR (field_value ~* 'physics')), (computer OR science) AND (math OR physics)", + "SELECT entry_id FROM tableName WHERE ((field_value ILIKE '%computer%') AND (field_value ILIKE '%science%')) OR (field_value ILIKE '%math%'), (computer AND science) OR math", + "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%computer%') AND ((field_value ILIKE '%science%') OR (field_value ILIKE '%math%')), computer AND (science OR math)", + "SELECT entry_id FROM tableName WHERE ((field_value ILIKE '%computer%') OR (field_value ILIKE '%science%')) AND ((field_value ILIKE '%math%') OR (field_value ILIKE '%physics%')), (computer OR science) AND (math OR physics)", // case insensitive contains - "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value ~* 'compute'), title=compute", + "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value ILIKE '%compute%'), title=compute", // case sensitive contains - "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value ~ 'compute'), title=!compute", + "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value LIKE '%compute%'), title=!compute", // exact match case insensitive - "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value ~* '\\ycompute\\y'), title==compute", + // "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value ~* '\\ycompute\\y'), title==compute", // exact match case sensitive - "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value ~ '\\ycompute\\y'), title==!compute", + // "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value ~ '\\ycompute\\y'), title==!compute", // negated case insensitive contains - "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value !~* 'compute'), title!=compute", + "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value NOT ILIKE '%compute%'), title!=compute", // negated case sensitive contains - "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value !~ 'compute'), title !=! compute", + "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value NOT LIKE '%compute%'), title !=! compute", // negated case insensitive exact match - "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value !~* '\\ycompute\\y'), title !== compute", + // "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value !~* '\\ycompute\\y'), title !== compute", // negated case sensitive exact match - "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value !~ '\\ycompute\\y'), title !==! compute", + // "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value !~ '\\ycompute\\y'), title !==! compute", + + // Special characters + "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%{IEEE}%'), {IEEE}", + "SELECT entry_id FROM tableName WHERE (field_name = 'author' AND field_value ILIKE '%{IEEE}%'), author={IEEE}", + // R\"ock + "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%R\\\"ock%'), R\\\"ock", + // Breitenb{\"{u}}cher + "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%Breitenb{\\\"{u}}cher%'), Breitenb{\\\"{u}}cher", }) void conversion(String expected, String input) { From f61314cf58d85446cdaa6d6016a4cc4f3dee1fa5 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 24 Sep 2024 05:01:57 +0300 Subject: [PATCH 018/104] Update module-info.java --- external-libraries.md | 10 ++++++---- src/main/java/module-info.java | 1 - 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/external-libraries.md b/external-libraries.md index ed58af72867..257aea187cb 100644 --- a/external-libraries.md +++ b/external-libraries.md @@ -350,10 +350,10 @@ License: MIT ``` ```yaml -Id:io.github.adr:e-adr -Project:EmbeddedArchitecturalDecisionRecords -URL:https://github.com/adr/e-adr/ -License:EPL-2.0 +Id: io.github.adr:e-adr +Project: EmbeddedArchitecturalDecisionRecords +URL: https://github.com/adr/e-adr/ +License: EPL-2.0 ``` ```yaml @@ -821,6 +821,7 @@ net.jodah:typetools:0.6.1 net.synedra:validatorfx:0.5.0 one.jpro.jproutils:tree-showing:0.2.2 org.antlr:antlr4-runtime:4.13.2 +org.apache.commons:commons-compress:1.27.1 org.apache.commons:commons-csv:1.11.0 org.apache.commons:commons-lang3:3.17.0 org.apache.commons:commons-text:1.12.0 @@ -897,6 +898,7 @@ org.slf4j:slf4j-api:2.0.16 org.tinylog:slf4j-tinylog:2.7.0 org.tinylog:tinylog-api:2.7.0 org.tinylog:tinylog-impl:2.7.0 +org.tukaani:xz:1.9 org.yaml:snakeyaml:2.3 pt.davidafsilva.apple:jkeychain:1.1.0 tech.units:indriya:2.2 diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 7471b765652..349b1f5da0e 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -110,7 +110,6 @@ requires io.github.javadiffutils; requires java.string.similarity; requires org.apache.commons.cli; - requires org.apache.commons.codec; requires org.apache.commons.compress; requires org.apache.commons.csv; requires org.apache.commons.io; From 250a1b0aa6a5578b8e3ec03c8a9fada9b203d31f Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 24 Sep 2024 12:13:24 +0300 Subject: [PATCH 019/104] Create "query" package --- src/main/java/org/jabref/cli/ArgumentProcessor.java | 2 +- src/main/java/org/jabref/gui/LibraryTab.java | 2 +- src/main/java/org/jabref/gui/StateManager.java | 2 +- src/main/java/org/jabref/gui/entryeditor/CommentsTab.java | 2 +- .../org/jabref/gui/entryeditor/DeprecatedFieldsTab.java | 2 +- .../org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java | 2 +- .../java/org/jabref/gui/entryeditor/FieldsEditorTab.java | 2 +- .../jabref/gui/entryeditor/ImportantOptionalFieldsTab.java | 2 +- .../org/jabref/gui/entryeditor/OptionalFieldsTabBase.java | 2 +- .../java/org/jabref/gui/entryeditor/OtherFieldsTab.java | 2 +- src/main/java/org/jabref/gui/entryeditor/PreviewTab.java | 2 +- .../java/org/jabref/gui/entryeditor/RequiredFieldsTab.java | 2 +- src/main/java/org/jabref/gui/entryeditor/SourceTab.java | 2 +- .../org/jabref/gui/entryeditor/UserDefinedFieldsTab.java | 2 +- .../fileannotationtab/FulltextSearchResultsTab.java | 6 +++--- .../java/org/jabref/gui/maintable/MainTableDataModel.java | 6 +++--- src/main/java/org/jabref/gui/preview/PreviewPanel.java | 2 +- src/main/java/org/jabref/gui/preview/PreviewViewer.java | 2 +- src/main/java/org/jabref/gui/search/GlobalSearchBar.java | 4 ++-- .../org/jabref/gui/search/SearchResultsTableDataModel.java | 4 ++-- .../org/jabref/logic/preferences/JabRefCliPreferences.java | 2 +- src/main/java/org/jabref/logic/search/DatabaseSearcher.java | 2 +- src/main/java/org/jabref/logic/search/IndexManager.java | 4 ++-- .../java/org/jabref/logic/search/SearchPreferences.java | 1 + .../org/jabref/logic/search/indexing/PostgreIndexer.java | 1 + .../logic/search/{ => query}/SearchToSqlConversion.java | 2 +- .../jabref/logic/search/{ => query}/SearchToSqlVisitor.java | 4 ++-- .../jabref/logic/search/retrieval/LinkedFilesSearcher.java | 4 ++-- src/main/java/org/jabref/model/groups/SearchGroup.java | 2 +- .../search/indexing => model/search}/PostgreConstants.java | 2 +- .../jabref/{logic => model}/search/SearchDisplayMode.java | 2 +- .../java/org/jabref/model/search/SearchFieldConstants.java | 2 +- .../search/{Analyzer => analyzer}/LatexAwareAnalyzer.java | 2 +- .../{Analyzer => analyzer}/LatexToUnicodeFoldingFilter.java | 2 +- .../org/jabref/model/search/{ => query}/SearchQuery.java | 5 ++++- .../org/jabref/model/search/{ => query}/SearchResult.java | 4 +++- .../org/jabref/model/search/{ => query}/SearchResults.java | 2 +- src/test/java/org/jabref/cli/ArgumentProcessorTest.java | 2 +- .../java/org/jabref/logic/search/DatabaseSearcherTest.java | 2 +- .../logic/search/DatabaseSearcherWithBibFilesTest.java | 2 +- .../org/jabref/logic/search/SearchToSqlConversionTest.java | 2 ++ 41 files changed, 56 insertions(+), 47 deletions(-) rename src/main/java/org/jabref/logic/search/{ => query}/SearchToSqlConversion.java (96%) rename src/main/java/org/jabref/logic/search/{ => query}/SearchToSqlVisitor.java (98%) rename src/main/java/org/jabref/{logic/search/indexing => model/search}/PostgreConstants.java (91%) rename src/main/java/org/jabref/{logic => model}/search/SearchDisplayMode.java (63%) rename src/main/java/org/jabref/model/search/{Analyzer => analyzer}/LatexAwareAnalyzer.java (96%) rename src/main/java/org/jabref/model/search/{Analyzer => analyzer}/LatexToUnicodeFoldingFilter.java (98%) rename src/main/java/org/jabref/model/search/{ => query}/SearchQuery.java (97%) rename src/main/java/org/jabref/model/search/{ => query}/SearchResult.java (97%) rename src/main/java/org/jabref/model/search/{ => query}/SearchResults.java (98%) diff --git a/src/main/java/org/jabref/cli/ArgumentProcessor.java b/src/main/java/org/jabref/cli/ArgumentProcessor.java index 3416a6688e3..86ca3bf8ac0 100644 --- a/src/main/java/org/jabref/cli/ArgumentProcessor.java +++ b/src/main/java/org/jabref/cli/ArgumentProcessor.java @@ -55,7 +55,7 @@ import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryTypesManager; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import org.jabref.model.strings.StringUtil; import org.jabref.model.util.DummyFileUpdateMonitor; import org.jabref.model.util.FileUpdateMonitor; diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 61f14427ab4..2f9566202a5 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -93,7 +93,7 @@ import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.FieldFactory; import org.jabref.model.groups.GroupTreeNode; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import org.jabref.model.util.DirectoryMonitor; import org.jabref.model.util.DirectoryMonitorManager; import org.jabref.model.util.FileUpdateMonitor; diff --git a/src/main/java/org/jabref/gui/StateManager.java b/src/main/java/org/jabref/gui/StateManager.java index 3c6ed12037e..f0585cbcf3f 100644 --- a/src/main/java/org/jabref/gui/StateManager.java +++ b/src/main/java/org/jabref/gui/StateManager.java @@ -29,7 +29,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.EasyBinding; diff --git a/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java b/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java index fe0f7691006..070b3594a28 100644 --- a/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java @@ -35,7 +35,7 @@ import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UserSpecificCommentField; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; public class CommentsTab extends FieldsEditorTab { public static final String NAME = "Comments"; diff --git a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java index cdc001bad58..8fd8b739a79 100644 --- a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java @@ -27,7 +27,7 @@ import org.jabref.model.entry.BibEntryType; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.model.entry.field.Field; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import com.tobiasdiez.easybind.EasyBind; diff --git a/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java index f01e6fa9318..fb03e121a52 100644 --- a/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java @@ -15,7 +15,7 @@ import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntryTypesManager; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; public class DetailOptionalFieldsTab extends OptionalFieldsTabBase { diff --git a/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java b/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java index 25a6036f551..b9506739495 100644 --- a/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java @@ -41,7 +41,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.Subscription; diff --git a/src/main/java/org/jabref/gui/entryeditor/ImportantOptionalFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/ImportantOptionalFieldsTab.java index 49534abf2de..70d831742c3 100644 --- a/src/main/java/org/jabref/gui/entryeditor/ImportantOptionalFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/ImportantOptionalFieldsTab.java @@ -15,7 +15,7 @@ import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntryTypesManager; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; public class ImportantOptionalFieldsTab extends OptionalFieldsTabBase { diff --git a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java index 6b0379d99b6..a1d62df4303 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java +++ b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java @@ -26,7 +26,7 @@ import org.jabref.model.entry.BibEntryType; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.model.entry.field.Field; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; public class OptionalFieldsTabBase extends FieldsEditorTab { private final BibEntryTypesManager entryTypesManager; diff --git a/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java index 8fb4f16d24f..2e9aed08484 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java @@ -33,7 +33,7 @@ import org.jabref.model.entry.field.InternalField; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UserSpecificCommentField; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; public class OtherFieldsTab extends FieldsEditorTab { diff --git a/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java b/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java index 843485c3218..b2ba9839749 100644 --- a/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java @@ -11,7 +11,7 @@ import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; public class PreviewTab extends EntryEditorTab implements OffersPreview { public static final String NAME = "Preview"; diff --git a/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java index 33c2ec1a2a1..78ed8ffbe8f 100644 --- a/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java @@ -27,7 +27,7 @@ import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.InternalField; import org.jabref.model.entry.field.OrFields; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; public class RequiredFieldsTab extends FieldsEditorTab { diff --git a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java index 6161e3a24ed..0a3fc57fd26 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java @@ -49,7 +49,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.model.entry.field.Field; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import org.jabref.model.util.FileUpdateMonitor; import de.saxsys.mvvmfx.utils.validation.ObservableRuleBasedValidator; diff --git a/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java index 17c3477827e..c1aedc08f3e 100644 --- a/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java @@ -20,7 +20,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; public class UserDefinedFieldsTab extends FieldsEditorTab { private final LinkedHashSet fields; diff --git a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java index 01441568e60..5e98f7d737d 100644 --- a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java @@ -34,9 +34,9 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.SearchQuery; -import org.jabref.model.search.SearchResult; -import org.jabref.model.search.SearchResults; +import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.search.query.SearchResult; +import org.jabref.model.search.query.SearchResults; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 910db9f5196..633145240c7 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -20,20 +20,20 @@ import org.jabref.gui.util.FilteredListProxy; import org.jabref.gui.util.OptionalObjectProperty; import org.jabref.logic.search.IndexManager; -import org.jabref.logic.search.SearchDisplayMode; import org.jabref.logic.search.SearchPreferences; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; +import org.jabref.model.search.SearchDisplayMode; import org.jabref.model.search.SearchFieldConstants; -import org.jabref.model.search.SearchQuery; -import org.jabref.model.search.SearchResults; import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; import org.jabref.model.search.event.IndexStartedEvent; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; +import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.search.query.SearchResults; import com.google.common.eventbus.Subscribe; import com.tobiasdiez.easybind.EasyBind; diff --git a/src/main/java/org/jabref/gui/preview/PreviewPanel.java b/src/main/java/org/jabref/gui/preview/PreviewPanel.java index 48ab602a55c..8d3842d2454 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewPanel.java +++ b/src/main/java/org/jabref/gui/preview/PreviewPanel.java @@ -30,7 +30,7 @@ import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/jabref/gui/preview/PreviewViewer.java b/src/main/java/org/jabref/gui/preview/PreviewViewer.java index 26322517004..cb7c084dc64 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewViewer.java +++ b/src/main/java/org/jabref/gui/preview/PreviewViewer.java @@ -31,7 +31,7 @@ import org.jabref.logic.util.WebViewStore; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import com.airhacks.afterburner.injection.Injector; import org.slf4j.Logger; diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index e57f2e47643..b74121f7a39 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -59,11 +59,11 @@ import org.jabref.gui.util.UiTaskExecutor; import org.jabref.logic.l10n.Localization; import org.jabref.logic.preferences.AutoCompleteFirstNameMode; -import org.jabref.logic.search.SearchDisplayMode; import org.jabref.logic.search.SearchPreferences; import org.jabref.model.entry.Author; +import org.jabref.model.search.SearchDisplayMode; import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import com.tobiasdiez.easybind.EasyBind; import impl.org.controlsfx.skin.AutoCompletePopup; diff --git a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java index 63246e1a8d3..b90b8160367 100644 --- a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java +++ b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java @@ -21,8 +21,8 @@ import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import org.jabref.model.search.SearchQuery; -import org.jabref.model.search.SearchResults; +import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.search.query.SearchResults; import com.tobiasdiez.easybind.EasyBind; diff --git a/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index 3423df38a8d..4ce72f23e99 100644 --- a/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -76,7 +76,7 @@ import org.jabref.logic.protectedterms.ProtectedTermsLoader; import org.jabref.logic.protectedterms.ProtectedTermsPreferences; import org.jabref.logic.remote.RemotePreferences; -import org.jabref.logic.search.SearchDisplayMode; +import org.jabref.model.search.SearchDisplayMode; import org.jabref.logic.search.SearchPreferences; import org.jabref.logic.shared.prefs.SharedDatabasePreferences; import org.jabref.logic.shared.security.Password; diff --git a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java index 76424e0e6ad..e031553d48e 100644 --- a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java +++ b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java @@ -10,7 +10,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabases; import org.jabref.model.entry.BibEntry; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index 33aa4084ceb..012c37498bc 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -18,8 +18,8 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.event.FieldChangedEvent; import org.jabref.model.entry.field.StandardField; -import org.jabref.model.search.SearchQuery; -import org.jabref.model.search.SearchResults; +import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.search.query.SearchResults; import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; import org.jabref.model.search.event.IndexClosedEvent; import org.jabref.model.search.event.IndexRemovedEvent; diff --git a/src/main/java/org/jabref/logic/search/SearchPreferences.java b/src/main/java/org/jabref/logic/search/SearchPreferences.java index 07a687c75ee..e6ff3d34dc6 100644 --- a/src/main/java/org/jabref/logic/search/SearchPreferences.java +++ b/src/main/java/org/jabref/logic/search/SearchPreferences.java @@ -11,6 +11,7 @@ import javafx.collections.FXCollections; import javafx.collections.ObservableSet; +import org.jabref.model.search.SearchDisplayMode; import org.jabref.model.search.SearchFlags; import com.google.common.annotations.VisibleForTesting; diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java index 44be8df4684..eb73493f4f0 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -12,6 +12,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; +import org.jabref.model.search.PostgreConstants; import org.jabref.model.search.SearchFieldConstants; import org.slf4j.Logger; diff --git a/src/main/java/org/jabref/logic/search/SearchToSqlConversion.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java similarity index 96% rename from src/main/java/org/jabref/logic/search/SearchToSqlConversion.java rename to src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java index 20a7845d257..5522dfef5ad 100644 --- a/src/main/java/org/jabref/logic/search/SearchToSqlConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java @@ -1,4 +1,4 @@ -package org.jabref.logic.search; +package org.jabref.logic.search.query; import org.jabref.model.search.ThrowingErrorListener; import org.jabref.search.SearchLexer; diff --git a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java similarity index 98% rename from src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java rename to src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 23d9564b53a..3b30d63f6d2 100644 --- a/src/main/java/org/jabref/logic/search/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -1,11 +1,11 @@ -package org.jabref.logic.search; +package org.jabref.logic.search.query; import java.util.EnumSet; import java.util.Optional; -import org.jabref.logic.search.indexing.PostgreConstants; import org.jabref.model.entry.field.InternalField; import org.jabref.model.entry.field.StandardField; +import org.jabref.model.search.PostgreConstants; import org.jabref.model.search.SearchFieldConstants; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; diff --git a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java index 5fbbe714434..fc18df702e6 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java @@ -15,8 +15,8 @@ import org.jabref.model.entry.LinkedFile; import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.SearchResult; -import org.jabref.model.search.SearchResults; +import org.jabref.model.search.query.SearchResult; +import org.jabref.model.search.query.SearchResults; import org.apache.lucene.document.Document; import org.apache.lucene.index.StoredFields; diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index e366f8c6d83..5684736a949 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -8,7 +8,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import io.github.adr.linked.ADR; import org.slf4j.Logger; diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreConstants.java b/src/main/java/org/jabref/model/search/PostgreConstants.java similarity index 91% rename from src/main/java/org/jabref/logic/search/indexing/PostgreConstants.java rename to src/main/java/org/jabref/model/search/PostgreConstants.java index 8889547b80a..1e86690ce9e 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreConstants.java +++ b/src/main/java/org/jabref/model/search/PostgreConstants.java @@ -1,4 +1,4 @@ -package org.jabref.logic.search.indexing; +package org.jabref.model.search; public enum PostgreConstants { ENTRY_ID("entry_id"), diff --git a/src/main/java/org/jabref/logic/search/SearchDisplayMode.java b/src/main/java/org/jabref/model/search/SearchDisplayMode.java similarity index 63% rename from src/main/java/org/jabref/logic/search/SearchDisplayMode.java rename to src/main/java/org/jabref/model/search/SearchDisplayMode.java index a05465fa7d4..a7440bbcaec 100644 --- a/src/main/java/org/jabref/logic/search/SearchDisplayMode.java +++ b/src/main/java/org/jabref/model/search/SearchDisplayMode.java @@ -1,4 +1,4 @@ -package org.jabref.logic.search; +package org.jabref.model.search; public enum SearchDisplayMode { FLOAT, diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java index 65a4436ddb6..c64f88d7884 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -2,7 +2,7 @@ import java.util.List; -import org.jabref.model.search.Analyzer.LatexAwareAnalyzer; +import org.jabref.model.search.analyzer.LatexAwareAnalyzer; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.en.EnglishAnalyzer; diff --git a/src/main/java/org/jabref/model/search/Analyzer/LatexAwareAnalyzer.java b/src/main/java/org/jabref/model/search/analyzer/LatexAwareAnalyzer.java similarity index 96% rename from src/main/java/org/jabref/model/search/Analyzer/LatexAwareAnalyzer.java rename to src/main/java/org/jabref/model/search/analyzer/LatexAwareAnalyzer.java index a67f1883b80..437575ff810 100644 --- a/src/main/java/org/jabref/model/search/Analyzer/LatexAwareAnalyzer.java +++ b/src/main/java/org/jabref/model/search/analyzer/LatexAwareAnalyzer.java @@ -1,4 +1,4 @@ -package org.jabref.model.search.Analyzer; +package org.jabref.model.search.analyzer; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.LowerCaseFilter; diff --git a/src/main/java/org/jabref/model/search/Analyzer/LatexToUnicodeFoldingFilter.java b/src/main/java/org/jabref/model/search/analyzer/LatexToUnicodeFoldingFilter.java similarity index 98% rename from src/main/java/org/jabref/model/search/Analyzer/LatexToUnicodeFoldingFilter.java rename to src/main/java/org/jabref/model/search/analyzer/LatexToUnicodeFoldingFilter.java index 5cd7fd64042..9bea4abb467 100644 --- a/src/main/java/org/jabref/model/search/Analyzer/LatexToUnicodeFoldingFilter.java +++ b/src/main/java/org/jabref/model/search/analyzer/LatexToUnicodeFoldingFilter.java @@ -1,4 +1,4 @@ -package org.jabref.model.search.Analyzer; +package org.jabref.model.search.analyzer; import java.io.IOException; import java.util.Arrays; diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/query/SearchQuery.java similarity index 97% rename from src/main/java/org/jabref/model/search/SearchQuery.java rename to src/main/java/org/jabref/model/search/query/SearchQuery.java index b76fc036fce..cdfb774c7a4 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/query/SearchQuery.java @@ -1,4 +1,4 @@ -package org.jabref.model.search; +package org.jabref.model.search.query; import java.util.Arrays; import java.util.Collections; @@ -12,6 +12,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.jabref.model.search.SearchFieldConstants; +import org.jabref.model.search.SearchFlags; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; diff --git a/src/main/java/org/jabref/model/search/SearchResult.java b/src/main/java/org/jabref/model/search/query/SearchResult.java similarity index 97% rename from src/main/java/org/jabref/model/search/SearchResult.java rename to src/main/java/org/jabref/model/search/query/SearchResult.java index a26a443d21a..e35b302834e 100644 --- a/src/main/java/org/jabref/model/search/SearchResult.java +++ b/src/main/java/org/jabref/model/search/query/SearchResult.java @@ -1,9 +1,11 @@ -package org.jabref.model.search; +package org.jabref.model.search.query; import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.jabref.model.search.SearchFieldConstants; + import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.search.highlight.Highlighter; import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; diff --git a/src/main/java/org/jabref/model/search/SearchResults.java b/src/main/java/org/jabref/model/search/query/SearchResults.java similarity index 98% rename from src/main/java/org/jabref/model/search/SearchResults.java rename to src/main/java/org/jabref/model/search/query/SearchResults.java index 3c0acb85ed7..1394b11fe81 100644 --- a/src/main/java/org/jabref/model/search/SearchResults.java +++ b/src/main/java/org/jabref/model/search/query/SearchResults.java @@ -1,4 +1,4 @@ -package org.jabref.model.search; +package org.jabref.model.search.query; import java.util.ArrayList; import java.util.Collection; diff --git a/src/test/java/org/jabref/cli/ArgumentProcessorTest.java b/src/test/java/org/jabref/cli/ArgumentProcessorTest.java index 9f662dfe8da..d83def7c47d 100644 --- a/src/test/java/org/jabref/cli/ArgumentProcessorTest.java +++ b/src/test/java/org/jabref/cli/ArgumentProcessorTest.java @@ -17,7 +17,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ImporterPreferences; import org.jabref.logic.importer.fileformat.BibtexImporter; -import org.jabref.logic.search.SearchDisplayMode; +import org.jabref.model.search.SearchDisplayMode; import org.jabref.logic.search.SearchPreferences; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryTypesManager; diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index 2b2d1eca991..74acacead18 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -16,7 +16,7 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.io.TempDir; diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java index 13e59c1b683..78afa46e006 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java @@ -21,7 +21,7 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.SearchQuery; +import org.jabref.model.search.query.SearchQuery; import org.jabref.model.util.DummyFileUpdateMonitor; import org.hamcrest.Matchers; diff --git a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java index f7cdd097f14..411e4807688 100644 --- a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java +++ b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java @@ -1,5 +1,7 @@ package org.jabref.logic.search; +import org.jabref.logic.search.query.SearchToSqlConversion; + import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; From 5314e6683e54041ac5c44d212d3703ccd8dcb13a Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 24 Sep 2024 13:46:30 +0300 Subject: [PATCH 020/104] Postgres searcher --- .../org/jabref/logic/search/IndexManager.java | 19 +++++++--- .../jabref/logic/search/PostgreServer.java | 1 - .../logic/search/indexing/PostgreIndexer.java | 4 ++ .../search/query/SearchToSqlVisitor.java | 2 +- .../search/retrieval/BibFieldsSearcher.java | 38 +++++++++++++++++++ .../model/search/query/SearchQuery.java | 5 +++ .../model/search/query/SqlSearchQuery.java | 25 ++++++++++++ .../search/SearchToSqlConversionTest.java | 2 +- 8 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java create mode 100644 src/main/java/org/jabref/model/search/query/SqlSearchQuery.java diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index 012c37498bc..88ef38bc450 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -11,6 +11,7 @@ import org.jabref.logic.search.indexing.DefaultLinkedFilesIndexer; import org.jabref.logic.search.indexing.PostgreIndexer; import org.jabref.logic.search.indexing.ReadOnlyLinkedFilesIndexer; +import org.jabref.logic.search.retrieval.BibFieldsSearcher; import org.jabref.logic.search.retrieval.LinkedFilesSearcher; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.TaskExecutor; @@ -18,12 +19,13 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.event.FieldChangedEvent; import org.jabref.model.entry.field.StandardField; -import org.jabref.model.search.query.SearchQuery; -import org.jabref.model.search.query.SearchResults; +import org.jabref.model.search.SearchFlags; import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; import org.jabref.model.search.event.IndexClosedEvent; import org.jabref.model.search.event.IndexRemovedEvent; import org.jabref.model.search.event.IndexStartedEvent; +import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.search.query.SearchResults; import com.airhacks.afterburner.injection.Injector; import org.slf4j.Logger; @@ -39,6 +41,7 @@ public class IndexManager { private final ChangeListener preferencesListener; private final PostgreIndexer bibFieldsIndexer; private final LuceneIndexer linkedFilesIndexer; + private final BibFieldsSearcher bibFieldsSearcher; private final LinkedFilesSearcher linkedFilesSearcher; public IndexManager(BibDatabaseContext databaseContext, TaskExecutor executor, FilePreferences preferences) { @@ -60,6 +63,7 @@ public IndexManager(BibDatabaseContext databaseContext, TaskExecutor executor, F } linkedFilesIndexer = indexer; + this.bibFieldsSearcher = new BibFieldsSearcher(postgreServer.getConnection()); this.linkedFilesSearcher = new LinkedFilesSearcher(databaseContext, linkedFilesIndexer, preferences); updateOnStart(); } @@ -218,10 +222,15 @@ public AutoCloseable blockLinkedFileIndexer() { } public SearchResults search(SearchQuery query) { - if (query.isValid()) { - query.setSearchResults(linkedFilesSearcher.search(query.getParsedQuery(), query.getSearchFlags())); +// if (query.isValid()) { +// query.setSearchResults(linkedFilesSearcher.search(query.getParsedQuery(), query.getSearchFlags())); +// } else { +// query.setSearchResults(new SearchResults()); +// } + if (query.getSearchFlags().contains(SearchFlags.FULLTEXT)) { + // TODO: merge results from lucene and postgres } else { - query.setSearchResults(new SearchResults()); + query.setSearchResults(bibFieldsSearcher.search(query.getSqlQuery(bibFieldsIndexer.getTableName()))); } return query.getSearchResults(); } diff --git a/src/main/java/org/jabref/logic/search/PostgreServer.java b/src/main/java/org/jabref/logic/search/PostgreServer.java index 699dad540da..afe2a414bb8 100644 --- a/src/main/java/org/jabref/logic/search/PostgreServer.java +++ b/src/main/java/org/jabref/logic/search/PostgreServer.java @@ -19,7 +19,6 @@ public PostgreServer() { EmbeddedPostgres embeddedPostgres; try { embeddedPostgres = EmbeddedPostgres.builder() - .setPort(5033) .start(); LOGGER.info("Postgres server started, connection port: {}", embeddedPostgres.getPort()); } catch (IOException e) { diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java index eb73493f4f0..aadb335584f 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -39,6 +39,10 @@ public PostgreIndexer(BibDatabaseContext databaseContext, Connection connection) setup(); } + public String getTableName() { + return tableName; + } + /** * Creates a table for the library in the database, and sets up indexes on the columns. */ diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 3b30d63f6d2..fd51c9daf2f 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -34,7 +34,7 @@ private enum SearchTermFlag { @Override public String visitStart(SearchParser.StartContext ctx) { - return "SELECT " + PostgreConstants.ENTRY_ID + " FROM " + tableName + " WHERE " + visit(ctx.expression()); + return "SELECT " + PostgreConstants.ENTRY_ID + " FROM \"" + tableName + "\" WHERE " + visit(ctx.expression()) + " GROUP BY " + PostgreConstants.ENTRY_ID; } @Override diff --git a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java new file mode 100644 index 00000000000..fcdc6ecf630 --- /dev/null +++ b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java @@ -0,0 +1,38 @@ +package org.jabref.logic.search.retrieval; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.jabref.model.search.query.SearchResult; +import org.jabref.model.search.query.SearchResults; +import org.jabref.model.search.query.SqlSearchQuery; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BibFieldsSearcher { + private static final Logger LOGGER = LoggerFactory.getLogger(BibFieldsSearcher.class); + + private final Connection connection; + + public BibFieldsSearcher(Connection connection) { + this.connection = connection; + } + + public SearchResults search(SqlSearchQuery searchQuery) { + LOGGER.debug("Searching in bib fields with query: {}", searchQuery.getQuery()); + SearchResults searchResults = new SearchResults(); + try (PreparedStatement preparedStatement = connection.prepareStatement(searchQuery.getQuery())) { + ResultSet resultSet = preparedStatement.executeQuery(); + while (resultSet.next()) { + String entryId = resultSet.getString(1); + searchResults.addSearchResult(entryId, new SearchResult(1)); + } + } catch (SQLException e) { + LOGGER.error("Error during bib fields search execution", e); + } + return searchResults; + } +} diff --git a/src/main/java/org/jabref/model/search/query/SearchQuery.java b/src/main/java/org/jabref/model/search/query/SearchQuery.java index cdfb774c7a4..8a0821b26c7 100644 --- a/src/main/java/org/jabref/model/search/query/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/query/SearchQuery.java @@ -12,6 +12,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.jabref.logic.search.query.SearchToSqlConversion; import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.SearchFlags; @@ -100,6 +101,10 @@ public SearchQuery(String query, EnumSet searchFlags) { } } + public SqlSearchQuery getSqlQuery(String tableName) { + return new SqlSearchQuery(SearchToSqlConversion.searchToSql(tableName, query)); + } + public String getSearchExpression() { return query; } diff --git a/src/main/java/org/jabref/model/search/query/SqlSearchQuery.java b/src/main/java/org/jabref/model/search/query/SqlSearchQuery.java new file mode 100644 index 00000000000..db96a93da81 --- /dev/null +++ b/src/main/java/org/jabref/model/search/query/SqlSearchQuery.java @@ -0,0 +1,25 @@ +package org.jabref.model.search.query; + +import java.util.Objects; + +public class SqlSearchQuery { + + private final String query; + private SearchResults searchResults; + + public SqlSearchQuery(String query) { + this.query = Objects.requireNonNull(query); + } + + public String getQuery() { + return query; + } + + public SearchResults getSearchResults() { + return searchResults; + } + + public void setSearchResults(SearchResults searchResults) { + this.searchResults = searchResults; + } +} diff --git a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java index 411e4807688..0041e0f36a1 100644 --- a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java +++ b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java @@ -56,6 +56,6 @@ class SearchToSqlConversionTest { }) void conversion(String expected, String input) { - assertEquals(expected, SearchToSqlConversion.searchToSql("tableName", input)); + assertEquals(expected + " GROUP BY entry_id", SearchToSqlConversion.searchToSql("tableName", input)); } } From fd2ace1868f270d8d94f9e8e28be3ae9a50a3cd1 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 24 Sep 2024 16:00:34 +0300 Subject: [PATCH 021/104] Return back the exact match operator --- src/main/antlr4/org/jabref/search/Search.g4 | 10 ++- .../search/query/SearchToSqlVisitor.java | 59 +++++++++---- .../search/SearchToSqlConversionTest.java | 88 ++++++++++++------- 3 files changed, 102 insertions(+), 55 deletions(-) diff --git a/src/main/antlr4/org/jabref/search/Search.g4 b/src/main/antlr4/org/jabref/search/Search.g4 index bfaeb1b57b1..e739c0d9a0e 100644 --- a/src/main/antlr4/org/jabref/search/Search.g4 +++ b/src/main/antlr4/org/jabref/search/Search.g4 @@ -14,12 +14,16 @@ EQUAL:'='; // case insensitive contains, semantically the same as CONTAINS CEQUAL:'=!'; // case sensitive contains EEQUAL:'=='; // exact match case insensitive, semantically the same as MATCHES +CEEQUAL:'==!'; // exact match case sensitive + +REQUAL:'=~'; // regex check case insensitive +CREEQUAL:'=~!'; // regex check case sensitive NEQUAL:'!='; // negated case insensitive contains NCEQUAL:'!=!'; // negated case sensitive contains -REQUAL:'=~'; // regex check case insensitive -CREEQUAL:'=~!'; // regex check case sensitive +NEEQUAL:'!=='; // negated case insensitive exact match +NCEEQUAL:'!==!'; // negated case sensitive exact match NREQUAL:'!=~'; // negated regex check case insensitive NCREEQUAL:'!=~!'; // negated regex check case sensitive @@ -52,7 +56,7 @@ expression: ; comparison: - left=name operator=(CONTAINS | EQUAL | CEQUAL | MATCHES | EEQUAL | NEQUAL | NCEQUAL | REQUAL | CREEQUAL | NREQUAL | NCREEQUAL) right=name // example: author != miller + left=name operator=(CONTAINS | EQUAL | CEQUAL | MATCHES | EEQUAL | CEEQUAL | REQUAL | CREEQUAL | NEQUAL | NCEQUAL | NEEQUAL | NCEEQUAL | NREQUAL | NCREEQUAL) right=name // example: author != miller | right=name // example: miller (search all fields) ; diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index fd51c9daf2f..83fec7ec6e0 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -26,10 +26,10 @@ public SearchToSqlVisitor(String tableName) { } private enum SearchTermFlag { - REGULAR_EXPRESSION, // mutually exclusive to the others + REGULAR_EXPRESSION, // mutually exclusive to exact/inexact match NEGATION, CASE_SENSITIVE, CASE_INSENSITIVE, // mutually exclusive - INEXACT_MATCH // mutually exclusive + EXACT_MATCH, INEXACT_MATCH // mutually exclusive } @Override @@ -73,35 +73,50 @@ public String visitComparison(SearchParser.ComparisonContext context) { // Direct comparison does not work // context.CONTAINS() and others are null if absent (thus, we cannot check for getText()) EnumSet searchFlags = EnumSet.noneOf(SearchTermFlag.class); - if (context.REQUAL() != null) { + if (context.EQUAL() != null || context.CONTAINS() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); + } else if (context.CEQUAL() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_SENSITIVE); + } else if (context.EEQUAL() != null || context.MATCHES() != null) { + searchFlags.add(SearchTermFlag.EXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); + } else if (context.CEEQUAL() != null) { + searchFlags.add(SearchTermFlag.EXACT_MATCH); + searchFlags.add(SearchTermFlag.CASE_SENSITIVE); + } else if (context.REQUAL() != null) { searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); } else if (context.CREEQUAL() != null) { searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); searchFlags.add(SearchTermFlag.CASE_SENSITIVE); - } else if (context.NREQUAL() != null) { - searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); + } else if (context.NEQUAL() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); searchFlags.add(SearchTermFlag.NEGATION); - } else if (context.NCREEQUAL() != null) { - searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); + } else if (context.NCEQUAL() != null) { + searchFlags.add(SearchTermFlag.INEXACT_MATCH); searchFlags.add(SearchTermFlag.CASE_SENSITIVE); searchFlags.add(SearchTermFlag.NEGATION); - } else if (context.CONTAINS() != null || context.EQUAL() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); + } else if (context.NEEQUAL() != null) { + searchFlags.add(SearchTermFlag.EXACT_MATCH); searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); - } else if (context.CEQUAL() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.NEGATION); + } else if (context.NCEEQUAL() != null) { + searchFlags.add(SearchTermFlag.EXACT_MATCH); searchFlags.add(SearchTermFlag.CASE_SENSITIVE); - } else if (context.NEQUAL() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); + searchFlags.add(SearchTermFlag.NEGATION); + } else if (context.NREQUAL() != null) { + searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); searchFlags.add(SearchTermFlag.NEGATION); - } else if (context.NCEQUAL() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); + } else if (context.NCREEQUAL() != null) { + searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); searchFlags.add(SearchTermFlag.CASE_SENSITIVE); searchFlags.add(SearchTermFlag.NEGATION); } + return getFieldQueryNode(field, right, searchFlags); } else { // Query without any field name @@ -124,7 +139,10 @@ private String getFieldQueryNode(String field, String term, EnumSet InternalField.KEY_FIELD.getName(); - case "keywords" -> StandardField.KEYWORDS.getName(); - default -> field; + case "key" -> + InternalField.KEY_FIELD.getName(); + case "anykeyword" -> + StandardField.KEYWORDS.getName(); + default -> + field; }; return "(" + PostgreConstants.FIELD_NAME + " = '" + field + "' AND " + PostgreConstants.FIELD_VALUE + " " + operator + " '" + prefixSuffix + term + prefixSuffix + "')"; diff --git a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java index 0041e0f36a1..f98e1178fd3 100644 --- a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java +++ b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java @@ -10,52 +10,74 @@ class SearchToSqlConversionTest { @ParameterizedTest @CsvSource({ - // Default search, query without any field name (case insensitive contains) - "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%computer%'), computer", - "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%computer science%'), \"computer science\"", // Phrase search - "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%computer%') OR (field_value ILIKE '%science%'), computer science", // Should be searched as a phrase or as two separate words (OR)? - "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%!computer%'), !computer", // Is the explanation should be escaped? - "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%!computer%'), \"!computer\"", - // search in all fields case sensitive contains - "SELECT entry_id FROM tableName WHERE (field_value LIKE '%computer%'), any=!computer", - "SELECT entry_id FROM tableName WHERE (field_value LIKE '%!computer%'), any=! !computer", // Is the explanation should be escaped? - // Regex search - "SELECT entry_id FROM tableName WHERE (field_value ~* 'Jabref.*Search'), any=~Jabref.*Search", - // And - "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%computer%') AND (field_value ILIKE '%science%'), computer AND science", - // Or - "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%computer%') OR (field_value ILIKE '%science%'), computer OR science", - // Grouping - "SELECT entry_id FROM tableName WHERE ((field_value ILIKE '%computer%') AND (field_value ILIKE '%science%')) OR (field_value ILIKE '%math%'), (computer AND science) OR math", - "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%computer%') AND ((field_value ILIKE '%science%') OR (field_value ILIKE '%math%')), computer AND (science OR math)", - "SELECT entry_id FROM tableName WHERE ((field_value ILIKE '%computer%') OR (field_value ILIKE '%science%')) AND ((field_value ILIKE '%math%') OR (field_value ILIKE '%physics%')), (computer OR science) AND (math OR physics)", // case insensitive contains - "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value ILIKE '%compute%'), title=compute", + "(field_name = 'title' AND field_value ILIKE '%compute%'), title=compute", + // case sensitive contains - "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value LIKE '%compute%'), title=!compute", + "(field_name = 'title' AND field_value LIKE '%compute%'), title=!compute", + // exact match case insensitive - // "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value ~* '\\ycompute\\y'), title==compute", + "(field_name = 'title' AND field_value ILIKE 'compute'), title==compute", + // exact match case sensitive - // "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value ~ '\\ycompute\\y'), title==!compute", + "(field_name = 'title' AND field_value LIKE 'compute'), title==!compute", + + // Regex search case insensitive + "(field_value ~* 'Jabref.*Search'), any=~Jabref.*Search", + + // Regex search case sensitive + "(field_value ~ 'Jabref.*Search'), any=~!Jabref.*Search", + // negated case insensitive contains - "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value NOT ILIKE '%compute%'), title!=compute", + "(field_name = 'title' AND field_value NOT ILIKE '%compute%'), title!=compute", + // negated case sensitive contains - "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value NOT LIKE '%compute%'), title !=! compute", + "(field_name = 'title' AND field_value NOT LIKE '%compute%'), title !=! compute", + // negated case insensitive exact match - // "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value !~* '\\ycompute\\y'), title !== compute", + "(field_name = 'title' AND field_value NOT ILIKE 'compute'), title !== compute", + // negated case sensitive exact match - // "SELECT entry_id FROM tableName WHERE (field_name = 'title' AND field_value !~ '\\ycompute\\y'), title !==! compute", + "(field_name = 'title' AND field_value NOT LIKE 'compute'), title !==! compute", + + // negated regex search case insensitive + "(field_value !~* 'Jabref.*Search'), any!=~Jabref.*Search", + + // negated regex search case sensitive + "(field_value !~ 'Jabref.*Search'), any!=~!Jabref.*Search", + + // Default search, query without any field name (case insensitive contains) + "(field_value ILIKE '%computer%'), computer", + "(field_value ILIKE '%computer science%'), \"computer science\"", // Phrase search + "(field_value ILIKE '%computer%') OR (field_value ILIKE '%science%'), computer science", // Should be searched as a phrase or as two separate words (OR)? + "(field_value ILIKE '%!computer%'), !computer", // Is the explanation should be escaped? + "(field_value ILIKE '%!computer%'), \"!computer\"", + + // search in all fields case sensitive contains + "(field_value LIKE '%computer%'), any=!computer", + "(field_value LIKE '%!computer%'), any=! !computer", // Is the explanation should be escaped? + + // And + "(field_value ILIKE '%computer%') AND (field_value ILIKE '%science%'), computer AND science", + + // Or + "(field_value ILIKE '%computer%') OR (field_value ILIKE '%science%'), computer OR science", + + // Grouping + "((field_value ILIKE '%computer%') AND (field_value ILIKE '%science%')) OR (field_value ILIKE '%math%'), (computer AND science) OR math", + "(field_value ILIKE '%computer%') AND ((field_value ILIKE '%science%') OR (field_value ILIKE '%math%')), computer AND (science OR math)", + "((field_value ILIKE '%computer%') OR (field_value ILIKE '%science%')) AND ((field_value ILIKE '%math%') OR (field_value ILIKE '%physics%')), (computer OR science) AND (math OR physics)", // Special characters - "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%{IEEE}%'), {IEEE}", - "SELECT entry_id FROM tableName WHERE (field_name = 'author' AND field_value ILIKE '%{IEEE}%'), author={IEEE}", + "(field_value ILIKE '%{IEEE}%'), {IEEE}", + "(field_name = 'author' AND field_value ILIKE '%{IEEE}%'), author={IEEE}", + // R\"ock - "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%R\\\"ock%'), R\\\"ock", + "(field_value ILIKE '%R\\\"ock%'), R\\\"ock", // Breitenb{\"{u}}cher - "SELECT entry_id FROM tableName WHERE (field_value ILIKE '%Breitenb{\\\"{u}}cher%'), Breitenb{\\\"{u}}cher", + "(field_value ILIKE '%Breitenb{\\\"{u}}cher%'), Breitenb{\\\"{u}}cher", }) - void conversion(String expected, String input) { - assertEquals(expected + " GROUP BY entry_id", SearchToSqlConversion.searchToSql("tableName", input)); + assertEquals("SELECT entry_id FROM \"tableName\" WHERE " + expected + " GROUP BY entry_id", SearchToSqlConversion.searchToSql("tableName", input)); } } From aa55eb9efb9f0c3d38ce28730ba0c429b71bdf70 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 24 Sep 2024 16:03:51 +0300 Subject: [PATCH 022/104] checkstyle --- .../java/org/jabref/logic/preferences/JabRefCliPreferences.java | 2 +- src/test/java/org/jabref/cli/ArgumentProcessorTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index 4ce72f23e99..f01a9d99537 100644 --- a/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -76,7 +76,6 @@ import org.jabref.logic.protectedterms.ProtectedTermsLoader; import org.jabref.logic.protectedterms.ProtectedTermsPreferences; import org.jabref.logic.remote.RemotePreferences; -import org.jabref.model.search.SearchDisplayMode; import org.jabref.logic.search.SearchPreferences; import org.jabref.logic.shared.prefs.SharedDatabasePreferences; import org.jabref.logic.shared.security.Password; @@ -99,6 +98,7 @@ import org.jabref.model.entry.types.EntryTypeFactory; import org.jabref.model.metadata.SaveOrder; import org.jabref.model.metadata.SelfContainedSaveOrder; +import org.jabref.model.search.SearchDisplayMode; import org.jabref.model.search.SearchFlags; import org.jabref.model.strings.StringUtil; diff --git a/src/test/java/org/jabref/cli/ArgumentProcessorTest.java b/src/test/java/org/jabref/cli/ArgumentProcessorTest.java index d83def7c47d..6baa662a9c8 100644 --- a/src/test/java/org/jabref/cli/ArgumentProcessorTest.java +++ b/src/test/java/org/jabref/cli/ArgumentProcessorTest.java @@ -17,12 +17,12 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ImporterPreferences; import org.jabref.logic.importer.fileformat.BibtexImporter; -import org.jabref.model.search.SearchDisplayMode; import org.jabref.logic.search.SearchPreferences; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.model.metadata.SaveOrder; import org.jabref.model.metadata.SelfContainedSaveOrder; +import org.jabref.model.search.SearchDisplayMode; import org.jabref.model.search.SearchFlags; import org.jabref.model.util.DummyFileUpdateMonitor; import org.jabref.model.util.FileUpdateMonitor; From 239fde0c372b7196c36ed58a87cadf418377561c Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 24 Sep 2024 15:34:51 +0200 Subject: [PATCH 023/104] Add link --- .../org/jabref/logic/search/query/SearchToSqlVisitor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 83fec7ec6e0..d98453b6c32 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -14,7 +14,9 @@ import org.slf4j.LoggerFactory; /** - * Similar class: {@link org.jabref.migrations.SearchToLuceneMigration} + * Converts to a query processable by the scheme created by {@link org.jabref.logic.search.indexing.PostgreIndexer}. + * + * @implNote Similar class: {@link org.jabref.migrations.SearchToLuceneMigration} */ public class SearchToSqlVisitor extends SearchBaseVisitor { From 08b1dfb447a05b9c14c12cff1b4c56740505361a Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 24 Sep 2024 16:42:03 +0200 Subject: [PATCH 024/104] WIP --- .../logic/search/indexing/PostgreIndexer.java | 54 ++++++++++++++++--- .../java/org/jabref/model/entry/BibEntry.java | 1 + .../model/entry/field/FieldFactory.java | 2 +- .../jabref/model/search/PostgreConstants.java | 4 +- 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java index aadb335584f..ecdc86d1749 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -20,6 +20,7 @@ public class PostgreIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(PostgreIndexer.class); + private static final String TABLE_STRING_NAME_SPLIT_VALUES_PREFIX = "_split_values"; private static int NUMBER_OF_UNSAVED_LIBRARIES = 1; private final BibDatabaseContext databaseContext; @@ -53,30 +54,61 @@ private void setup() { %s TEXT NOT NULL, %s TEXT NOT NULL, %s TEXT, + %s TEXT, PRIMARY KEY (%s, %s) ) """.formatted(tableName, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, - PostgreConstants.FIELD_VALUE, + PostgreConstants.FIELD_VALUE_LITERAL, + PostgreConstants.FIELD_VALUE_TRANSFORMED, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME)); - LOGGER.debug("Created table for library: {}", libraryName); + + String tableNameSplitValues = tableName + "_split_values"; + connection.createStatement().executeUpdate(""" + CREATE TABLE IF NOT EXISTS "%s" ( + %s TEXT NOT NULL, + %s TEXT NOT NULL, + %s TEXT + ) + """.formatted(tableNameSplitValues, + PostgreConstants.ENTRY_ID, + PostgreConstants.FIELD_NAME, + PostgreConstants.FIELD_SPLIT_VALUE)); + + LOGGER.debug("Created tables for library: {}", libraryName); // btree index on id column connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") """.formatted(PostgreConstants.ENTRY_ID.getIndexName(tableName), tableName, PostgreConstants.ENTRY_ID)); + connection.createStatement().executeUpdate(""" + CREATE INDEX "%s" ON "%s" ("%s") + """.formatted(PostgreConstants.ENTRY_ID.getIndexName(tableNameSplitValues), tableName, PostgreConstants.ENTRY_ID)); // btree index on field name column connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") """.formatted(PostgreConstants.FIELD_NAME.getIndexName(tableName), tableName, PostgreConstants.FIELD_NAME)); + connection.createStatement().executeUpdate(""" + CREATE INDEX "%s" ON "%s" ("%s") + """.formatted(PostgreConstants.FIELD_NAME.getIndexName(tableNameSplitValues), tableName, PostgreConstants.FIELD_NAME)); // trigram index on field value column connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" USING gin ("%s" gin_trgm_ops) - """.formatted(PostgreConstants.FIELD_VALUE.getIndexName(tableName), tableName, PostgreConstants.FIELD_VALUE)); + """.formatted(PostgreConstants.FIELD_VALUE_LITERAL.getIndexName(tableName), tableName, PostgreConstants.FIELD_VALUE_LITERAL)); + + // trigram index on field value column + connection.createStatement().executeUpdate(""" + CREATE INDEX "%s" ON "%s" USING gin ("%s" gin_trgm_ops) + """.formatted(PostgreConstants.FIELD_VALUE_TRANSFORMED.getIndexName(tableName), tableName, PostgreConstants.FIELD_VALUE_TRANSFORMED)); + + // btree index on spilt values column + connection.createStatement().executeUpdate(""" + CREATE INDEX "%s" ON "%s" ("%s") + """.formatted(PostgreConstants.FIELD_SPLIT_VALUE.getIndexName(tableName), tableName, PostgreConstants.FIELD_SPLIT_VALUE)); LOGGER.debug("Created indexes for library: {}", libraryName); } catch (SQLException e) { @@ -108,12 +140,13 @@ public void addToIndex(Collection entries, BackgroundTask task) { private void addToIndex(BibEntry bibEntry) { String insertFieldQuery = """ - INSERT INTO "%s" ("%s", "%s", "%s") - VALUES (?, ?, ?) + INSERT INTO "%s" ("%s", "%s", "%s", "%s") + VALUES (?, ?, ?, ?) """.formatted(tableName, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, - PostgreConstants.FIELD_VALUE); + PostgreConstants.FIELD_VALUE_LITERAL, + PostgreConstants.FIELD_VALUE_TRANSFORMED); try (PreparedStatement preparedStatement = connection.prepareStatement(insertFieldQuery)) { String entryId = bibEntry.getId(); @@ -121,6 +154,14 @@ private void addToIndex(BibEntry bibEntry) { preparedStatement.setString(1, entryId); preparedStatement.setString(2, field.getKey().getName()); preparedStatement.setString(3, field.getValue()); + + // If a field exists, there also exists a resolved field latex free. + // We add a `.orElse("")` only because there could be some flaw in the future in the code - and we want to have search working even if the flaws are present. + // To uncover these flaws, we add the "assert" statement. + // One potential future flaw is that the bibEntry is modified concurrently and the field being deleted. + assert bibEntry.getResolvedFieldOrAliasLatexFree(field.getKey(), this.databaseContext.getDatabase()).isPresent(); + preparedStatement.setString(4, bibEntry.getResolvedFieldOrAliasLatexFree(field.getKey(), this.databaseContext.getDatabase()).orElse("")); + preparedStatement.addBatch(); } @@ -128,6 +169,7 @@ private void addToIndex(BibEntry bibEntry) { preparedStatement.setString(1, entryId); preparedStatement.setString(2, SearchFieldConstants.ENTRY_TYPE.toString()); preparedStatement.setString(3, bibEntry.getType().getName()); + preparedStatement.setString(4, bibEntry.getType().getName()); preparedStatement.addBatch(); preparedStatement.executeBatch(); diff --git a/src/main/java/org/jabref/model/entry/BibEntry.java b/src/main/java/org/jabref/model/entry/BibEntry.java index f0d2d43689e..d42c7734cc9 100644 --- a/src/main/java/org/jabref/model/entry/BibEntry.java +++ b/src/main/java/org/jabref/model/entry/BibEntry.java @@ -483,6 +483,7 @@ public Optional getFieldLatexFree(Field field) { } else { Optional fieldValue = getField(field); if (fieldValue.isPresent()) { + // TODO: To we need FieldFactory.isLaTeXField(field) here to filter? String latexFreeValue = LatexToUnicodeAdapter.format(fieldValue.get()).intern(); latexFreeFields.put(field, latexFreeValue); return Optional.of(latexFreeValue); diff --git a/src/main/java/org/jabref/model/entry/field/FieldFactory.java b/src/main/java/org/jabref/model/entry/field/FieldFactory.java index ff660ad4ab6..fe76330a35f 100644 --- a/src/main/java/org/jabref/model/entry/field/FieldFactory.java +++ b/src/main/java/org/jabref/model/entry/field/FieldFactory.java @@ -55,7 +55,7 @@ public static String serializeOrFieldsList(Set fields) { * Checks whether the given field contains LaTeX code or something else */ public static boolean isLatexField(Field field) { - return Collections.disjoint(field.getProperties(), Set.of(FieldProperty.VERBATIM, FieldProperty.MARKDOWN)); + return Collections.disjoint(field.getProperties(), Set.of(FieldProperty.VERBATIM, FieldProperty.MARKDOWN, FieldProperty.NUMERIC, FieldProperty.DATE, FieldProperty.MULTIPLE_ENTRY_LINK)); } /** diff --git a/src/main/java/org/jabref/model/search/PostgreConstants.java b/src/main/java/org/jabref/model/search/PostgreConstants.java index 1e86690ce9e..0527c5d1d5e 100644 --- a/src/main/java/org/jabref/model/search/PostgreConstants.java +++ b/src/main/java/org/jabref/model/search/PostgreConstants.java @@ -3,7 +3,9 @@ public enum PostgreConstants { ENTRY_ID("entry_id"), FIELD_NAME("field_name"), - FIELD_VALUE("field_value"); + FIELD_VALUE_LITERAL("field_value_literal"), // contains the value as-is + FIELD_VALUE_TRANSFORMED("field_value_transforemd"), // contains the value transformed for better querying + FIELD_SPLIT_VALUE("field_split_value"); private final String columnName; From ab87dd591828e90d97a5a77e67dd0409ad6dbe62 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 24 Sep 2024 17:53:49 +0300 Subject: [PATCH 025/104] Fix compilation --- .../logic/search/indexing/PostgreIndexer.java | 35 +++++++++++-------- .../search/query/SearchToSqlVisitor.java | 4 +-- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java index ecdc86d1749..e2dbc515716 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -5,6 +5,7 @@ import java.sql.SQLException; import java.util.Collection; import java.util.Map; +import java.util.Optional; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; @@ -20,7 +21,7 @@ public class PostgreIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(PostgreIndexer.class); - private static final String TABLE_STRING_NAME_SPLIT_VALUES_PREFIX = "_split_values"; + private static final String SPLIT_VALUES_PREFIX = "_split_values"; private static int NUMBER_OF_UNSAVED_LIBRARIES = 1; private final BibDatabaseContext databaseContext; @@ -83,6 +84,7 @@ PRIMARY KEY (%s, %s) connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") """.formatted(PostgreConstants.ENTRY_ID.getIndexName(tableName), tableName, PostgreConstants.ENTRY_ID)); + connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") """.formatted(PostgreConstants.ENTRY_ID.getIndexName(tableNameSplitValues), tableName, PostgreConstants.ENTRY_ID)); @@ -91,10 +93,16 @@ PRIMARY KEY (%s, %s) connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") """.formatted(PostgreConstants.FIELD_NAME.getIndexName(tableName), tableName, PostgreConstants.FIELD_NAME)); + connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") """.formatted(PostgreConstants.FIELD_NAME.getIndexName(tableNameSplitValues), tableName, PostgreConstants.FIELD_NAME)); + // btree index on spilt values column + connection.createStatement().executeUpdate(""" + CREATE INDEX "%s" ON "%s" ("%s") + """.formatted(PostgreConstants.FIELD_SPLIT_VALUE.getIndexName(tableNameSplitValues), tableName, PostgreConstants.FIELD_SPLIT_VALUE)); + // trigram index on field value column connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" USING gin ("%s" gin_trgm_ops) @@ -105,11 +113,6 @@ PRIMARY KEY (%s, %s) CREATE INDEX "%s" ON "%s" USING gin ("%s" gin_trgm_ops) """.formatted(PostgreConstants.FIELD_VALUE_TRANSFORMED.getIndexName(tableName), tableName, PostgreConstants.FIELD_VALUE_TRANSFORMED)); - // btree index on spilt values column - connection.createStatement().executeUpdate(""" - CREATE INDEX "%s" ON "%s" ("%s") - """.formatted(PostgreConstants.FIELD_SPLIT_VALUE.getIndexName(tableName), tableName, PostgreConstants.FIELD_SPLIT_VALUE)); - LOGGER.debug("Created indexes for library: {}", libraryName); } catch (SQLException e) { LOGGER.error("Could not create table for library: {}", libraryName, e); @@ -140,8 +143,8 @@ public void addToIndex(Collection entries, BackgroundTask task) { private void addToIndex(BibEntry bibEntry) { String insertFieldQuery = """ - INSERT INTO "%s" ("%s", "%s", "%s", "%s") - VALUES (?, ?, ?, ?) + INSERT INTO "%s" ("%s", "%s", "%s", "%s") + VALUES (?, ?, ?, ?) """.formatted(tableName, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, @@ -159,8 +162,9 @@ private void addToIndex(BibEntry bibEntry) { // We add a `.orElse("")` only because there could be some flaw in the future in the code - and we want to have search working even if the flaws are present. // To uncover these flaws, we add the "assert" statement. // One potential future flaw is that the bibEntry is modified concurrently and the field being deleted. - assert bibEntry.getResolvedFieldOrAliasLatexFree(field.getKey(), this.databaseContext.getDatabase()).isPresent(); - preparedStatement.setString(4, bibEntry.getResolvedFieldOrAliasLatexFree(field.getKey(), this.databaseContext.getDatabase()).orElse("")); + Optional resolvedFieldLatexFree = bibEntry.getResolvedFieldOrAliasLatexFree(field.getKey(), this.databaseContext.getDatabase()); + assert resolvedFieldLatexFree.isPresent(); + preparedStatement.setString(4, resolvedFieldLatexFree.orElse("")); preparedStatement.addBatch(); } @@ -209,18 +213,19 @@ public void updateEntry(BibEntry entry, Field field) { try { // Use upsert to add the field to the index if it doesn't exist, or update it if it does String updateQuery = """ - INSERT INTO "%s" ("%s", "%s", "%s") - VALUES (?, ?, ?) + INSERT INTO "%s" ("%s", "%s", "%s", "%s") + VALUES (?, ?, ?, ?) ON CONFLICT ("%s", "%s") DO UPDATE SET "%s" = EXCLUDED."%s" """.formatted(tableName, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, - PostgreConstants.FIELD_VALUE, + PostgreConstants.FIELD_VALUE_LITERAL, + PostgreConstants.FIELD_VALUE_TRANSFORMED, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, - PostgreConstants.FIELD_VALUE, - PostgreConstants.FIELD_VALUE); + PostgreConstants.FIELD_VALUE_LITERAL, + PostgreConstants.FIELD_VALUE_LITERAL); try (PreparedStatement preparedStatement = connection.prepareStatement(updateQuery)) { preparedStatement.setString(1, entry.getId()); diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index d98453b6c32..4e630f2de1e 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -158,7 +158,7 @@ private String getFieldQueryNode(String field, String term, EnumSet Date: Thu, 26 Sep 2024 12:13:46 +0300 Subject: [PATCH 026/104] WIP --- .../org/jabref/cli/ArgumentProcessor.java | 2 +- src/main/java/org/jabref/gui/LibraryTab.java | 2 +- .../jabref/logic/search/DatabaseSearcher.java | 6 +-- .../org/jabref/logic/search/IndexManager.java | 16 +++--- .../logic/search/indexing/PostgreIndexer.java | 49 ++++++++++++++----- .../java/org/jabref/model/entry/Author.java | 18 +++---- .../jabref/model/search/PostgreConstants.java | 12 +++++ 7 files changed, 70 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/jabref/cli/ArgumentProcessor.java b/src/main/java/org/jabref/cli/ArgumentProcessor.java index 86ca3bf8ac0..ae1ea8cf350 100644 --- a/src/main/java/org/jabref/cli/ArgumentProcessor.java +++ b/src/main/java/org/jabref/cli/ArgumentProcessor.java @@ -468,7 +468,7 @@ private boolean exportMatches(List loaded) { List matches; try { // extract current thread task executor from indexManager - matches = new DatabaseSearcher(query, databaseContext, new CurrentThreadTaskExecutor(), cliPreferences.getFilePreferences()).getMatches(); + matches = new DatabaseSearcher(query, databaseContext, new CurrentThreadTaskExecutor(), cliPreferences).getMatches(); } catch (IOException e) { LOGGER.error("Error occurred when searching", e); return false; diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 2f9566202a5..8ca7f71b5cd 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -315,7 +315,7 @@ private void onDatabaseLoadingSucceed(ParserResult result) { } public void createIndexManager() { - indexManager = new IndexManager(bibDatabaseContext, taskExecutor, preferences.getFilePreferences()); + indexManager = new IndexManager(bibDatabaseContext, taskExecutor, preferences); stateManager.setIndexManager(bibDatabaseContext, indexManager); } diff --git a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java index e031553d48e..6b51f651539 100644 --- a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java +++ b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java @@ -5,7 +5,7 @@ import java.util.List; import java.util.Objects; -import org.jabref.logic.FilePreferences; +import org.jabref.logic.preferences.CliPreferences; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabases; @@ -23,10 +23,10 @@ public class DatabaseSearcher { private final IndexManager indexManager; // get rid of task executor here or add a constructor overload? - public DatabaseSearcher(SearchQuery query, BibDatabaseContext databaseContext, TaskExecutor taskExecutor, FilePreferences filePreferences) throws IOException { + public DatabaseSearcher(SearchQuery query, BibDatabaseContext databaseContext, TaskExecutor taskExecutor, CliPreferences preferences) throws IOException { this.databaseContext = databaseContext; this.query = Objects.requireNonNull(query); - this.indexManager = new IndexManager(databaseContext, taskExecutor, filePreferences); + this.indexManager = new IndexManager(databaseContext, taskExecutor, preferences); } /** diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index ee497ed7205..35b77980946 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -7,7 +7,7 @@ import javafx.beans.property.SimpleBooleanProperty; import javafx.beans.value.ChangeListener; -import org.jabref.logic.FilePreferences; +import org.jabref.logic.preferences.CliPreferences; import org.jabref.logic.search.indexing.DefaultLinkedFilesIndexer; import org.jabref.logic.search.indexing.PostgreIndexer; import org.jabref.logic.search.indexing.ReadOnlyLinkedFilesIndexer; @@ -44,19 +44,19 @@ public class IndexManager { private final BibFieldsSearcher bibFieldsSearcher; private final LinkedFilesSearcher linkedFilesSearcher; - public IndexManager(BibDatabaseContext databaseContext, TaskExecutor executor, FilePreferences preferences) { + public IndexManager(BibDatabaseContext databaseContext, TaskExecutor executor, CliPreferences preferences) { this.taskExecutor = executor; this.databaseContext = databaseContext; - this.shouldIndexLinkedFiles = preferences.fulltextIndexLinkedFilesProperty(); + this.shouldIndexLinkedFiles = preferences.getFilePreferences().fulltextIndexLinkedFilesProperty(); this.preferencesListener = (observable, oldValue, newValue) -> bindToPreferences(newValue); this.shouldIndexLinkedFiles.addListener(preferencesListener); PostgreServer postgreServer = Injector.instantiateModelOrService(PostgreServer.class); - bibFieldsIndexer = new PostgreIndexer(databaseContext, postgreServer.getConnection()); + bibFieldsIndexer = new PostgreIndexer(preferences.getBibEntryPreferences(), databaseContext, postgreServer.getConnection()); LuceneIndexer indexer; try { - indexer = new DefaultLinkedFilesIndexer(databaseContext, preferences); + indexer = new DefaultLinkedFilesIndexer(databaseContext, preferences.getFilePreferences()); } catch (IOException e) { LOGGER.debug("Error initializing linked files index - using read only index"); indexer = new ReadOnlyLinkedFilesIndexer(databaseContext); @@ -64,7 +64,7 @@ public IndexManager(BibDatabaseContext databaseContext, TaskExecutor executor, F linkedFilesIndexer = indexer; this.bibFieldsSearcher = new BibFieldsSearcher(postgreServer.getConnection()); - this.linkedFilesSearcher = new LinkedFilesSearcher(databaseContext, linkedFilesIndexer, preferences); + this.linkedFilesSearcher = new LinkedFilesSearcher(databaseContext, linkedFilesIndexer, preferences.getFilePreferences()); updateOnStart(); } @@ -150,7 +150,7 @@ public void updateEntry(FieldChangedEvent event) { new BackgroundTask<>() { @Override public Object call() { - bibFieldsIndexer.updateEntry(event.getBibEntry(), event.getField()); + bibFieldsIndexer.updateEntry(event.getBibEntry(), event.getField(), event.getOldValue(), event.getNewValue()); return null; } }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(List.of(event.getBibEntry())))) @@ -171,7 +171,7 @@ public void updateAfterDropFiles(BibEntry entry) { new BackgroundTask<>() { @Override public Object call() { - bibFieldsIndexer.updateEntry(entry, StandardField.FILE); +// bibFieldsIndexer.updateEntry(entry, StandardField.FILE); return null; } }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(List.of(entry)))) diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java index 661f595379d..6909dfad005 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -12,6 +12,8 @@ import org.jabref.logic.util.HeadlessExecutorService; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.BibEntryPreferences; +import org.jabref.model.entry.KeywordList; import org.jabref.model.entry.field.Field; import org.jabref.model.search.PostgreConstants; import org.jabref.model.search.SearchFieldConstants; @@ -19,17 +21,25 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.jabref.model.entry.field.StandardField.AUTHOR; +import static org.jabref.model.entry.field.StandardField.CROSSREF; +import static org.jabref.model.entry.field.StandardField.EDITOR; +import static org.jabref.model.entry.field.StandardField.FILE; +import static org.jabref.model.entry.field.StandardField.GROUPS; +import static org.jabref.model.entry.field.StandardField.KEYWORDS; + public class PostgreIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(PostgreIndexer.class); - private static final String SPLIT_VALUES_PREFIX = "_split_values"; private static int NUMBER_OF_UNSAVED_LIBRARIES = 1; + private final BibEntryPreferences bibEntryPreferences; private final BibDatabaseContext databaseContext; private final Connection connection; private final String libraryName; private final String tableName; - public PostgreIndexer(BibDatabaseContext databaseContext, Connection connection) { + public PostgreIndexer(BibEntryPreferences bibEntryPreferences, BibDatabaseContext databaseContext, Connection connection) { + this.bibEntryPreferences = bibEntryPreferences; this.databaseContext = databaseContext; this.connection = connection; this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElse("unsaved"); @@ -108,7 +118,7 @@ PRIMARY KEY (%s, %s) CREATE INDEX "%s" ON "%s" USING gin ("%s" gin_trgm_ops) """.formatted(PostgreConstants.FIELD_VALUE_LITERAL.getIndexName(tableName), tableName, PostgreConstants.FIELD_VALUE_LITERAL)); - // trigram index on field value column + // trigram index on field value transformed column connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" USING gin ("%s" gin_trgm_ops) """.formatted(PostgreConstants.FIELD_VALUE_TRANSFORMED.getIndexName(tableName), tableName, PostgreConstants.FIELD_VALUE_TRANSFORMED)); @@ -170,6 +180,28 @@ private void addToIndex(BibEntry bibEntry) { preparedStatement.setString(4, resolvedFieldLatexFree.orElse("")); preparedStatement.addBatch(); + + if () + + if (PostgreConstants.MULTI_VALUE_FIELDS.contains(field.getKey())) { + switch (field.getKey()) { + case AUTHOR -> { + } + case EDITOR -> { + } + case CROSSREF -> { + } + case KEYWORDS -> { + KeywordList keywordList = KeywordList.parse(field.getValue(), bibEntryPreferences.getKeywordSeparator()); + } + case GROUPS -> { + } + case FILE -> { + } + default -> { + } + } + } } // add entry type @@ -215,23 +247,16 @@ private void removeFromIndex(BibEntry entry) { } } - public void updateEntry(BibEntry entry, Field field) { + public void updateEntry(BibEntry entry, Field field, String oldValue, String newValue) { try { - // Use upsert to add the field to the index if it doesn't exist, or update it if it does String updateQuery = """ INSERT INTO "%s" ("%s", "%s", "%s", "%s") VALUES (?, ?, ?, ?) - ON CONFLICT ("%s", "%s") DO UPDATE - SET "%s" = EXCLUDED."%s" """.formatted(tableName, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, PostgreConstants.FIELD_VALUE_LITERAL, - PostgreConstants.FIELD_VALUE_TRANSFORMED, - PostgreConstants.ENTRY_ID, - PostgreConstants.FIELD_NAME, - PostgreConstants.FIELD_VALUE_LITERAL, - PostgreConstants.FIELD_VALUE_LITERAL); + PostgreConstants.FIELD_VALUE_TRANSFORMED); try (PreparedStatement preparedStatement = connection.prepareStatement(updateQuery)) { preparedStatement.setString(1, entry.getId()); diff --git a/src/main/java/org/jabref/model/entry/Author.java b/src/main/java/org/jabref/model/entry/Author.java index 5892ca0b457..a8a24c91a56 100644 --- a/src/main/java/org/jabref/model/entry/Author.java +++ b/src/main/java/org/jabref/model/entry/Author.java @@ -241,7 +241,7 @@ public Optional getNameSuffix() { * @return 'von Last' */ public String getNamePrefixAndFamilyName() { - if (namePrefix == null || "".equals(namePrefix)) { + if (namePrefix == null || namePrefix.isEmpty()) { return getFamilyName().orElse(""); } else { return familyName == null ? namePrefix : namePrefix + ' ' + familyName; @@ -285,14 +285,12 @@ public String getGivenFamily(boolean abbr) { @Override public String toString() { - final StringBuilder sb = new StringBuilder("Author{"); - sb.append("givenName='").append(givenName).append('\''); - sb.append(", givenNameAbbreviated='").append(givenNameAbbreviated).append('\''); - sb.append(", namePrefix='").append(namePrefix).append('\''); - sb.append(", familyName='").append(familyName).append('\''); - sb.append(", nameSuffix='").append(nameSuffix).append('\''); - sb.append('}'); - return sb.toString(); + return "Author{" + "givenName='" + givenName + '\'' + + ", givenNameAbbreviated='" + givenNameAbbreviated + '\'' + + ", namePrefix='" + namePrefix + '\'' + + ", familyName='" + familyName + '\'' + + ", nameSuffix='" + nameSuffix + '\'' + + '}'; } /** @@ -305,7 +303,7 @@ public String getNameForAlphabetization() { getFamilyName().ifPresent(res::append); getNameSuffix().ifPresent(jr -> res.append(", ").append(jr)); getGivenNameAbbreviated().ifPresent(firstA -> res.append(", ").append(firstA)); - while ((res.length() > 0) && (res.charAt(0) == '{')) { + while ((!res.isEmpty()) && (res.charAt(0) == '{')) { res.deleteCharAt(0); } return res.toString(); diff --git a/src/main/java/org/jabref/model/search/PostgreConstants.java b/src/main/java/org/jabref/model/search/PostgreConstants.java index 0527c5d1d5e..83c319685be 100644 --- a/src/main/java/org/jabref/model/search/PostgreConstants.java +++ b/src/main/java/org/jabref/model/search/PostgreConstants.java @@ -1,5 +1,10 @@ package org.jabref.model.search; +import java.util.Set; + +import org.jabref.model.entry.field.Field; +import org.jabref.model.entry.field.StandardField; + public enum PostgreConstants { ENTRY_ID("entry_id"), FIELD_NAME("field_name"), @@ -7,6 +12,13 @@ public enum PostgreConstants { FIELD_VALUE_TRANSFORMED("field_value_transforemd"), // contains the value transformed for better querying FIELD_SPLIT_VALUE("field_split_value"); + public static final Set MULTI_VALUE_FIELDS = Set.of( + StandardField.AUTHOR, + StandardField.EDITOR, + StandardField.CROSSREF, + StandardField.KEYWORDS, + StandardField.GROUPS, + StandardField.FILE); private final String columnName; PostgreConstants(String columnName) { From 98bfffa070d3bf0651b6a9fac7742c2341d407c4 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 26 Sep 2024 11:43:27 +0200 Subject: [PATCH 027/104] Intermediate result Co-authored-by: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> --- .../logic/search/indexing/PostgreIndexer.java | 31 +++++++++---- .../search/query/SearchToSqlVisitor.java | 44 ++++++++++++++++++- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java index 6909dfad005..c3330d124d0 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -37,6 +37,7 @@ public class PostgreIndexer { private final Connection connection; private final String libraryName; private final String tableName; + private final String tableNameSplitValues; public PostgreIndexer(BibEntryPreferences bibEntryPreferences, BibDatabaseContext databaseContext, Connection connection) { this.bibEntryPreferences = bibEntryPreferences; @@ -48,6 +49,7 @@ public PostgreIndexer(BibEntryPreferences bibEntryPreferences, BibDatabaseContex } else { this.tableName = databaseContext.getPostgreTableName(); } + tableNameSplitValues = tableName + "_split_values"; setup(); } @@ -76,7 +78,6 @@ PRIMARY KEY (%s, %s) PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME)); - String tableNameSplitValues = tableName + "_split_values"; connection.createStatement().executeUpdate(""" CREATE TABLE IF NOT EXISTS "%s" ( %s TEXT NOT NULL, @@ -166,25 +167,37 @@ private void addToIndex(BibEntry bibEntry) { try (PreparedStatement preparedStatement = connection.prepareStatement(insertFieldQuery)) { String entryId = bibEntry.getId(); - for (Map.Entry field : bibEntry.getFieldMap().entrySet()) { + for (Map.Entry fieldPair : bibEntry.getFieldMap().entrySet()) { + Field field = fieldPair.getKey(); + String value = fieldPair.getValue(); + preparedStatement.setString(1, entryId); - preparedStatement.setString(2, field.getKey().getName()); - preparedStatement.setString(3, field.getValue()); + preparedStatement.setString(2, field.getName()); + preparedStatement.setString(3, value); // If a field exists, there also exists a resolved field latex free. // We add a `.orElse("")` only because there could be some flaw in the future in the code - and we want to have search working even if the flaws are present. // To uncover these flaws, we add the "assert" statement. // One potential future flaw is that the bibEntry is modified concurrently and the field being deleted. - Optional resolvedFieldLatexFree = bibEntry.getResolvedFieldOrAliasLatexFree(field.getKey(), this.databaseContext.getDatabase()); + Optional resolvedFieldLatexFree = bibEntry.getResolvedFieldOrAliasLatexFree(field, this.databaseContext.getDatabase()); assert resolvedFieldLatexFree.isPresent(); preparedStatement.setString(4, resolvedFieldLatexFree.orElse("")); preparedStatement.addBatch(); - if () + // region Handling of known multi-value fields + // split and convert to unicode + /* + if (field.getProperties().contains(FieldProperty.PERSON_NAMES) { + authorList = AuthorListParser.parse(value); + } else { + + } + */ + // endregion - if (PostgreConstants.MULTI_VALUE_FIELDS.contains(field.getKey())) { - switch (field.getKey()) { + if (PostgreConstants.MULTI_VALUE_FIELDS.contains(field)) { + switch (field) { case AUTHOR -> { } case EDITOR -> { @@ -192,7 +205,7 @@ private void addToIndex(BibEntry bibEntry) { case CROSSREF -> { } case KEYWORDS -> { - KeywordList keywordList = KeywordList.parse(field.getValue(), bibEntryPreferences.getKeywordSeparator()); + KeywordList keywordList = KeywordList.parse(value, bibEntryPreferences.getKeywordSeparator()); } case GROUPS -> { } diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 4e630f2de1e..c57b2e76594 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -21,6 +21,7 @@ public class SearchToSqlVisitor extends SearchBaseVisitor { private static final Logger LOGGER = LoggerFactory.getLogger(SearchToSqlVisitor.class); + private final String tableName; public SearchToSqlVisitor(String tableName) { @@ -36,7 +37,24 @@ private enum SearchTermFlag { @Override public String visitStart(SearchParser.StartContext ctx) { - return "SELECT " + PostgreConstants.ENTRY_ID + " FROM \"" + tableName + "\" WHERE " + visit(ctx.expression()) + " GROUP BY " + PostgreConstants.ENTRY_ID; + String whereClause = visit(ctx.expression()); + return """ + SELECT %s + FROM "%s" AS main_table + LEFT JOIN "%s_split_values" AS split_table + ON main_table.%s = split_table.%s + AND main_table.%s = split_table.%s + WHERE %s + GROUP BY %s + """.formatted(PostgreConstants.ENTRY_ID, + tableName, + tableName, + PostgreConstants.ENTRY_ID, + PostgreConstants.ENTRY_ID, + PostgreConstants.FIELD_NAME, + PostgreConstants.FIELD_NAME, + whereClause, + PostgreConstants.ENTRY_ID); } @Override @@ -127,6 +145,23 @@ public String visitComparison(SearchParser.ComparisonContext context) { } private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { + String additionalCondition = null; + if (searchFlags.contains((SearchTermFlag.EXACT_MATCH))) { + // additionally search in second table + + String operator = ""; + if (searchFlags.contains(SearchTermFlag.NEGATION)) { + operator = "NOT "; + } + + if (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE)) { + operator += "LIKE"; + } else { + operator += "ILIKE"; + } + additionalCondition = "(split_table.field_name = '\" + field + \"' AND split_table.field_value " + operator + " '" + term + "')"; + } + String operator = ""; String prefixSuffix = ""; @@ -170,6 +205,11 @@ private String getFieldQueryNode(String field, String term, EnumSet Date: Thu, 26 Sep 2024 12:09:14 +0200 Subject: [PATCH 028/104] Query should be OK Co-authored-by: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> --- .../logic/search/indexing/PostgreIndexer.java | 35 ++++++++++++------- .../search/query/SearchToSqlVisitor.java | 20 +++++++---- .../jabref/model/search/PostgreConstants.java | 3 +- src/main/resources/tinylog.properties | 2 ++ 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java index c3330d124d0..010c2b3dbae 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -82,37 +82,37 @@ PRIMARY KEY (%s, %s) CREATE TABLE IF NOT EXISTS "%s" ( %s TEXT NOT NULL, %s TEXT NOT NULL, + %s TEXT, %s TEXT - ) + ) """.formatted(tableNameSplitValues, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, - PostgreConstants.FIELD_SPLIT_VALUE)); + PostgreConstants.FIELD_VALUE_LITERAL, + PostgreConstants.FIELD_VALUE_TRANSFORMED)); LOGGER.debug("Created tables for library: {}", libraryName); - - // btree index on id column + } catch (SQLException e) { + LOGGER.error("Could not create tables for library: {}", libraryName, e); + } + try { + // region btree index on id column connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") """.formatted(PostgreConstants.ENTRY_ID.getIndexName(tableName), tableName, PostgreConstants.ENTRY_ID)); - connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") """.formatted(PostgreConstants.ENTRY_ID.getIndexName(tableNameSplitValues), tableName, PostgreConstants.ENTRY_ID)); + // endregion - // btree index on field name column + // region btree index on field name column connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") """.formatted(PostgreConstants.FIELD_NAME.getIndexName(tableName), tableName, PostgreConstants.FIELD_NAME)); - connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") """.formatted(PostgreConstants.FIELD_NAME.getIndexName(tableNameSplitValues), tableName, PostgreConstants.FIELD_NAME)); - - // btree index on spilt values column - connection.createStatement().executeUpdate(""" - CREATE INDEX "%s" ON "%s" ("%s") - """.formatted(PostgreConstants.FIELD_SPLIT_VALUE.getIndexName(tableNameSplitValues), tableName, PostgreConstants.FIELD_SPLIT_VALUE)); + // endregion // trigram index on field value column connection.createStatement().executeUpdate(""" @@ -124,9 +124,18 @@ PRIMARY KEY (%s, %s) CREATE INDEX "%s" ON "%s" USING gin ("%s" gin_trgm_ops) """.formatted(PostgreConstants.FIELD_VALUE_TRANSFORMED.getIndexName(tableName), tableName, PostgreConstants.FIELD_VALUE_TRANSFORMED)); + // region btree index on spilt values column + connection.createStatement().executeUpdate(""" + CREATE INDEX "%s" ON "%s" ("%s") + """.formatted(PostgreConstants.FIELD_VALUE_LITERAL.getIndexName(tableNameSplitValues), tableName, PostgreConstants.FIELD_VALUE_LITERAL)); + connection.createStatement().executeUpdate(""" + CREATE INDEX "%s" ON "%s" ("%s") + """.formatted(PostgreConstants.FIELD_VALUE_TRANSFORMED.getIndexName(tableNameSplitValues), tableName, PostgreConstants.FIELD_VALUE_TRANSFORMED)); + // endregion + LOGGER.debug("Created indexes for library: {}", libraryName); } catch (SQLException e) { - LOGGER.error("Could not create table for library: {}", libraryName, e); + LOGGER.error("Could not create indexes for library: {}", libraryName, e); } } diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index c57b2e76594..a73cf0137cb 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -38,14 +38,14 @@ private enum SearchTermFlag { @Override public String visitStart(SearchParser.StartContext ctx) { String whereClause = visit(ctx.expression()); - return """ - SELECT %s + String result = """ + SELECT main_table.%s FROM "%s" AS main_table LEFT JOIN "%s_split_values" AS split_table ON main_table.%s = split_table.%s AND main_table.%s = split_table.%s WHERE %s - GROUP BY %s + GROUP BY main_table.%s """.formatted(PostgreConstants.ENTRY_ID, tableName, tableName, @@ -55,6 +55,8 @@ public String visitStart(SearchParser.StartContext ctx) { PostgreConstants.FIELD_NAME, whereClause, PostgreConstants.ENTRY_ID); + LOGGER.trace("Converted search query to SQL: {}", result); + return result; } @Override @@ -159,7 +161,11 @@ private String getFieldQueryNode(String field, String term, EnumSet MULTI_VALUE_FIELDS = Set.of( StandardField.AUTHOR, diff --git a/src/main/resources/tinylog.properties b/src/main/resources/tinylog.properties index 2b99c963cef..7e9f20f3165 100644 --- a/src/main/resources/tinylog.properties +++ b/src/main/resources/tinylog.properties @@ -16,6 +16,8 @@ level@io.zonky.test.db.postgres.embedded = warn #level@org.jabref.gui.JabRefGUI = debug +level@org.jabref.logic.search.query.SearchToSqlVisitor = trace + # AI debugging #level@ai.djl = debug #level@org.jabref.gui.entryeditor.aichattab.AiChat = trace From b473b6cfb9e50476bc1184432d12dc844a9e23f2 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 26 Sep 2024 13:20:36 +0200 Subject: [PATCH 029/104] Indexing of split values Co-authored-by: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> --- .../logic/search/indexing/PostgreIndexer.java | 128 ++++++++++++------ 1 file changed, 88 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java index 010c2b3dbae..1581878eed7 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -3,33 +3,34 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.Arrays; import java.util.Collection; import java.util.Map; import java.util.Optional; +import java.util.regex.Pattern; import org.jabref.logic.l10n.Localization; +import org.jabref.logic.layout.format.LatexToUnicodeFormatter; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.HeadlessExecutorService; import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.AuthorList; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryPreferences; import org.jabref.model.entry.KeywordList; import org.jabref.model.entry.field.Field; +import org.jabref.model.entry.field.FieldProperty; +import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.PostgreConstants; import org.jabref.model.search.SearchFieldConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.jabref.model.entry.field.StandardField.AUTHOR; -import static org.jabref.model.entry.field.StandardField.CROSSREF; -import static org.jabref.model.entry.field.StandardField.EDITOR; -import static org.jabref.model.entry.field.StandardField.FILE; -import static org.jabref.model.entry.field.StandardField.GROUPS; -import static org.jabref.model.entry.field.StandardField.KEYWORDS; - public class PostgreIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(PostgreIndexer.class); + private static final LatexToUnicodeFormatter LATEX_TO_UNICODE_FORMATTER = new LatexToUnicodeFormatter(); + private static final Pattern GROUPS_SEPARATOR_REGEX = Pattern.compile("\s*,\s*"); private static int NUMBER_OF_UNSAVED_LIBRARIES = 1; private final BibEntryPreferences bibEntryPreferences; @@ -174,59 +175,53 @@ private void addToIndex(BibEntry bibEntry) { PostgreConstants.FIELD_VALUE_LITERAL, PostgreConstants.FIELD_VALUE_TRANSFORMED); - try (PreparedStatement preparedStatement = connection.prepareStatement(insertFieldQuery)) { + String insertIntoSplitTable = """ + INSERT INTO "%s" ("%s", "%s", "%s", "%s") + VALUES (?, ?, ?, ?) + """.formatted(tableNameSplitValues, + PostgreConstants.ENTRY_ID, + PostgreConstants.FIELD_NAME, + PostgreConstants.FIELD_VALUE_LITERAL, + PostgreConstants.FIELD_VALUE_TRANSFORMED); + + try (PreparedStatement preparedStatement = connection.prepareStatement(insertFieldQuery); + PreparedStatement preparedStatementSplitValues = connection.prepareStatement(insertIntoSplitTable)) { String entryId = bibEntry.getId(); for (Map.Entry fieldPair : bibEntry.getFieldMap().entrySet()) { Field field = fieldPair.getKey(); String value = fieldPair.getValue(); - preparedStatement.setString(1, entryId); - preparedStatement.setString(2, field.getName()); - preparedStatement.setString(3, value); - // If a field exists, there also exists a resolved field latex free. // We add a `.orElse("")` only because there could be some flaw in the future in the code - and we want to have search working even if the flaws are present. // To uncover these flaws, we add the "assert" statement. // One potential future flaw is that the bibEntry is modified concurrently and the field being deleted. Optional resolvedFieldLatexFree = bibEntry.getResolvedFieldOrAliasLatexFree(field, this.databaseContext.getDatabase()); assert resolvedFieldLatexFree.isPresent(); - preparedStatement.setString(4, resolvedFieldLatexFree.orElse("")); - - preparedStatement.addBatch(); + prepareStatement(preparedStatement, entryId, field, value, resolvedFieldLatexFree.orElse("")); // region Handling of known multi-value fields // split and convert to unicode - /* - if (field.getProperties().contains(FieldProperty.PERSON_NAMES) { - authorList = AuthorListParser.parse(value); + if (field.getProperties().contains(FieldProperty.PERSON_NAMES)) { + addAuthors(value, preparedStatementSplitValues, entryId, field); + } else if (field == StandardField.KEYWORDS) { + addKeywords(value, preparedStatementSplitValues, entryId, field); + } else if (field == StandardField.GROUPS) { + addGroups(value, preparedStatementSplitValues, entryId, field); + } else if (field.getProperties().contains(FieldProperty.MULTIPLE_ENTRY_LINK)) { + addEntryLinks(bibEntry, field, preparedStatementSplitValues, entryId); + } else if (field == StandardField.FILE) { + // No handling of File, because due to relative paths, we think, there won't be any exact match operation + // We could add the filename itself (with and without extension). However, the user can also use regular expressions to achieve the same. + // The use case to search for file names seems pretty seldom, therefore we omit it. } else { - + // No other multi-value fields are known + // No action needed -> main table has the value } - */ // endregion - - if (PostgreConstants.MULTI_VALUE_FIELDS.contains(field)) { - switch (field) { - case AUTHOR -> { - } - case EDITOR -> { - } - case CROSSREF -> { - } - case KEYWORDS -> { - KeywordList keywordList = KeywordList.parse(value, bibEntryPreferences.getKeywordSeparator()); - } - case GROUPS -> { - } - case FILE -> { - } - default -> { - } - } - } } // add entry type + // Separate code, because ENTRY_TYPE is not a Field preparedStatement.setString(1, entryId); preparedStatement.setString(2, SearchFieldConstants.ENTRY_TYPE.toString()); preparedStatement.setString(3, bibEntry.getType().getName()); @@ -234,11 +229,64 @@ private void addToIndex(BibEntry bibEntry) { preparedStatement.addBatch(); preparedStatement.executeBatch(); + preparedStatementSplitValues.executeBatch(); } catch (SQLException e) { LOGGER.error("Could not add an entry to the index.", e); } } + private void addEntryLinks(BibEntry bibEntry, Field field, PreparedStatement preparedStatementSplitValues, String entryId) { + bibEntry.getEntryLinkList(field, databaseContext.getDatabase()).stream().distinct().forEach(link -> { + doInsert(preparedStatementSplitValues, entryId, field, link.getKey()); + }); + } + + private static void addGroups(String value, PreparedStatement preparedStatementSplitValues, String entryId, Field field) { + // We could use KeywordList, but we are afraid that group names could have ">" in their name and then they would not be handled correctly + Arrays.stream(GROUPS_SEPARATOR_REGEX.split(value)) + .distinct() + .forEach(group -> { + doInsert(preparedStatementSplitValues, entryId, field, group); + }); + } + + private void addKeywords(String keywordsString, PreparedStatement preparedStatementSplitValues, String entryId, Field field) { + KeywordList keywordList = KeywordList.parse(keywordsString, bibEntryPreferences.getKeywordSeparator()); + keywordList.stream().flatMap(keyword -> keyword.flatten().stream()).forEach(keyword -> { + String value = keyword.toString(); + doInsert(preparedStatementSplitValues, entryId, field, value); + }); + } + + private static void addAuthors(String value, PreparedStatement preparedStatementSplitValues, String entryId, Field field) { + AuthorList.parse(value).getAuthors().forEach(author -> { + // Author object does not support literal values + // We use the method giving us the most complete information for the literal value; + String literal = author.getFamilyGiven(false); + String transformed = author.latexFree().getFamilyGiven(false); + prepareStatement(preparedStatementSplitValues, entryId, field, literal, transformed); + }); + } + + private static void doInsert(PreparedStatement preparedStatement, String entryId, Field field, String value) { + prepareStatement(preparedStatement, entryId, field, value, LATEX_TO_UNICODE_FORMATTER.format(value)); + } + + /** + * The values are passed as they should be inserted into the database table + */ + private static void prepareStatement(PreparedStatement preparedStatement, String entryId, Field field, String value, String normalized) { + try { + preparedStatement.setString(1, entryId); + preparedStatement.setString(2, field.getName()); + preparedStatement.setString(3, value); + preparedStatement.setString(4, normalized); + preparedStatement.addBatch(); + } catch (SQLException e) { + LOGGER.error("Could not add field {} having value {} of entry {} to the index.", field.getName(), value, entryId, e); + } + } + public void removeFromIndex(Collection entries, BackgroundTask task) { if (entries.size() > 1) { task.showToUser(true); From ad8824d029c45c2947d0912c4f51ff7b57c06a42 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 26 Sep 2024 23:54:30 +0300 Subject: [PATCH 030/104] Fix tests compile --- .../java/org/jabref/logic/search/DatabaseSearcherTest.java | 4 +++- .../jabref/logic/search/DatabaseSearcherWithBibFilesTest.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index 74acacead18..9e2be7d7996 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -9,6 +9,7 @@ import javafx.beans.property.BooleanProperty; import org.jabref.logic.FilePreferences; +import org.jabref.logic.preferences.CliPreferences; import org.jabref.logic.util.CurrentThreadTaskExecutor; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; @@ -32,6 +33,7 @@ public class DatabaseSearcherTest { private static final TaskExecutor TASK_EXECUTOR = new CurrentThreadTaskExecutor(); private BibDatabaseContext databaseContext; + private final CliPreferences preferences = mock(CliPreferences.class); private final FilePreferences filePreferences = mock(FilePreferences.class); @TempDir private Path indexDir; @@ -50,7 +52,7 @@ void testDatabaseSearcher(List expectedMatches, SearchQuery query, Lis for (BibEntry entry : entries) { databaseContext.getDatabase().insertEntry(entry); } - List matches = new DatabaseSearcher(query, databaseContext, TASK_EXECUTOR, filePreferences).getMatches(); + List matches = new DatabaseSearcher(query, databaseContext, TASK_EXECUTOR, preferences).getMatches(); assertEquals(expectedMatches, matches); } diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java index 78afa46e006..5d2736796d3 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java @@ -12,6 +12,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.fileformat.BibtexImporter; +import org.jabref.logic.preferences.CliPreferences; import org.jabref.logic.util.CurrentThreadTaskExecutor; import org.jabref.logic.util.StandardFileType; import org.jabref.logic.util.TaskExecutor; @@ -67,6 +68,7 @@ class DatabaseSearcherWithBibFilesTest { .withCitationKey("minimal-note-mixed-case") .withFiles(List.of(new LinkedFile("", "minimal-note-mixed-case.pdf", StandardFileType.PDF.getName()))); + private final CliPreferences preferences = mock(CliPreferences.class); private final FilePreferences filePreferences = mock(FilePreferences.class); @TempDir @@ -134,7 +136,7 @@ private static Stream searchLibrary() { @MethodSource void searchLibrary(List expected, String testFile, String query, EnumSet searchFlags) throws Exception { BibDatabaseContext databaseContext = initializeDatabaseFromPath(testFile); - List matches = new DatabaseSearcher(new SearchQuery(query, searchFlags), databaseContext, TASK_EXECUTOR, filePreferences).getMatches(); + List matches = new DatabaseSearcher(new SearchQuery(query, searchFlags), databaseContext, TASK_EXECUTOR, preferences).getMatches(); assertThat(expected, Matchers.containsInAnyOrder(matches.toArray())); } } From 97dc941c3661b1fb06bbca68dbc7ff4ca219a987 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 26 Sep 2024 23:55:15 +0300 Subject: [PATCH 031/104] Use first name Last name for authors --- .../logic/search/indexing/PostgreIndexer.java | 6 +++--- .../org/jabref/model/search/PostgreConstants.java | 13 +------------ 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java index 1581878eed7..36463b9be14 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -50,7 +50,7 @@ public PostgreIndexer(BibEntryPreferences bibEntryPreferences, BibDatabaseContex } else { this.tableName = databaseContext.getPostgreTableName(); } - tableNameSplitValues = tableName + "_split_values"; + tableNameSplitValues = tableName + PostgreConstants.TABLE_NAME_SUFFIX; setup(); } @@ -262,8 +262,8 @@ private static void addAuthors(String value, PreparedStatement preparedStatement AuthorList.parse(value).getAuthors().forEach(author -> { // Author object does not support literal values // We use the method giving us the most complete information for the literal value; - String literal = author.getFamilyGiven(false); - String transformed = author.latexFree().getFamilyGiven(false); + String literal = author.getGivenFamily(false); + String transformed = author.latexFree().getGivenFamily(false); prepareStatement(preparedStatementSplitValues, entryId, field, literal, transformed); }); } diff --git a/src/main/java/org/jabref/model/search/PostgreConstants.java b/src/main/java/org/jabref/model/search/PostgreConstants.java index 491e829755b..f5092d8556f 100644 --- a/src/main/java/org/jabref/model/search/PostgreConstants.java +++ b/src/main/java/org/jabref/model/search/PostgreConstants.java @@ -1,23 +1,12 @@ package org.jabref.model.search; -import java.util.Set; - -import org.jabref.model.entry.field.Field; -import org.jabref.model.entry.field.StandardField; - public enum PostgreConstants { ENTRY_ID("entry_id"), FIELD_NAME("field_name"), FIELD_VALUE_LITERAL("field_value_literal"), // contains the value as-is FIELD_VALUE_TRANSFORMED("field_value_transformed"); // contains the value transformed for better querying - public static final Set MULTI_VALUE_FIELDS = Set.of( - StandardField.AUTHOR, - StandardField.EDITOR, - StandardField.CROSSREF, - StandardField.KEYWORDS, - StandardField.GROUPS, - StandardField.FILE); + public static final String TABLE_NAME_SUFFIX = "_split_values"; private final String columnName; PostgreConstants(String columnName) { From ff79dd3fffd738ab86c62dd64330c1b1f8e89f36 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 26 Sep 2024 23:55:48 +0300 Subject: [PATCH 032/104] Refactor SQL query visitor --- .../search/query/SearchToSqlVisitor.java | 190 ++++++++---------- 1 file changed, 81 insertions(+), 109 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index a73cf0137cb..0c632c9089c 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -6,7 +6,6 @@ import org.jabref.model.entry.field.InternalField; import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.PostgreConstants; -import org.jabref.model.search.SearchFieldConstants; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; @@ -21,11 +20,15 @@ public class SearchToSqlVisitor extends SearchBaseVisitor { private static final Logger LOGGER = LoggerFactory.getLogger(SearchToSqlVisitor.class); + private static final String MAIN_TABLE = "main_table"; + private static final String SPLIT_TABLE = "split_table"; - private final String tableName; + private final String mainTableName; + private final String splitTableName; - public SearchToSqlVisitor(String tableName) { - this.tableName = tableName; + public SearchToSqlVisitor(String mainTableName) { + this.mainTableName = mainTableName; + this.splitTableName = mainTableName + PostgreConstants.TABLE_NAME_SUFFIX; } private enum SearchTermFlag { @@ -39,22 +42,22 @@ private enum SearchTermFlag { public String visitStart(SearchParser.StartContext ctx) { String whereClause = visit(ctx.expression()); String result = """ - SELECT main_table.%s - FROM "%s" AS main_table - LEFT JOIN "%s_split_values" AS split_table - ON main_table.%s = split_table.%s - AND main_table.%s = split_table.%s - WHERE %s - GROUP BY main_table.%s - """.formatted(PostgreConstants.ENTRY_ID, - tableName, - tableName, - PostgreConstants.ENTRY_ID, - PostgreConstants.ENTRY_ID, - PostgreConstants.FIELD_NAME, - PostgreConstants.FIELD_NAME, + SELECT %s.%s + FROM "%s" AS %s + LEFT JOIN "%s" AS %s + ON (%s.%s = %s.%s AND %s.%s = %s.%s) + WHERE (%s) + GROUP BY %s.%s + """.formatted( + PostgreConstants.ENTRY_ID, MAIN_TABLE, + mainTableName, MAIN_TABLE, + splitTableName, SPLIT_TABLE, + MAIN_TABLE, PostgreConstants.ENTRY_ID, + SPLIT_TABLE, PostgreConstants.ENTRY_ID, + MAIN_TABLE, PostgreConstants.FIELD_NAME, + SPLIT_TABLE, PostgreConstants.FIELD_NAME, whereClause, - PostgreConstants.ENTRY_ID); + MAIN_TABLE, PostgreConstants.ENTRY_ID); LOGGER.trace("Converted search query to SQL: {}", result); return result; } @@ -96,128 +99,97 @@ public String visitComparison(SearchParser.ComparisonContext context) { // context.CONTAINS() and others are null if absent (thus, we cannot check for getText()) EnumSet searchFlags = EnumSet.noneOf(SearchTermFlag.class); if (context.EQUAL() != null || context.CONTAINS() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); + setFlags(searchFlags, SearchTermFlag.INEXACT_MATCH, false, false); } else if (context.CEQUAL() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_SENSITIVE); + setFlags(searchFlags, SearchTermFlag.INEXACT_MATCH, true, false); } else if (context.EEQUAL() != null || context.MATCHES() != null) { - searchFlags.add(SearchTermFlag.EXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); + setFlags(searchFlags, SearchTermFlag.EXACT_MATCH, false, false); } else if (context.CEEQUAL() != null) { - searchFlags.add(SearchTermFlag.EXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_SENSITIVE); + setFlags(searchFlags, SearchTermFlag.EXACT_MATCH, true, false); } else if (context.REQUAL() != null) { - searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); - searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); + setFlags(searchFlags, SearchTermFlag.REGULAR_EXPRESSION, false, false); } else if (context.CREEQUAL() != null) { - searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); - searchFlags.add(SearchTermFlag.CASE_SENSITIVE); + setFlags(searchFlags, SearchTermFlag.REGULAR_EXPRESSION, true, false); } else if (context.NEQUAL() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); - searchFlags.add(SearchTermFlag.NEGATION); + setFlags(searchFlags, SearchTermFlag.INEXACT_MATCH, false, true); } else if (context.NCEQUAL() != null) { - searchFlags.add(SearchTermFlag.INEXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_SENSITIVE); - searchFlags.add(SearchTermFlag.NEGATION); + setFlags(searchFlags, SearchTermFlag.INEXACT_MATCH, true, true); } else if (context.NEEQUAL() != null) { - searchFlags.add(SearchTermFlag.EXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); - searchFlags.add(SearchTermFlag.NEGATION); + setFlags(searchFlags, SearchTermFlag.EXACT_MATCH, false, true); } else if (context.NCEEQUAL() != null) { - searchFlags.add(SearchTermFlag.EXACT_MATCH); - searchFlags.add(SearchTermFlag.CASE_SENSITIVE); - searchFlags.add(SearchTermFlag.NEGATION); + setFlags(searchFlags, SearchTermFlag.EXACT_MATCH, true, true); } else if (context.NREQUAL() != null) { - searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); - searchFlags.add(SearchTermFlag.CASE_INSENSITIVE); - searchFlags.add(SearchTermFlag.NEGATION); + setFlags(searchFlags, SearchTermFlag.REGULAR_EXPRESSION, false, true); } else if (context.NCREEQUAL() != null) { - searchFlags.add(SearchTermFlag.REGULAR_EXPRESSION); - searchFlags.add(SearchTermFlag.CASE_SENSITIVE); - searchFlags.add(SearchTermFlag.NEGATION); + setFlags(searchFlags, SearchTermFlag.REGULAR_EXPRESSION, true, true); } return getFieldQueryNode(field, right, searchFlags); } else { // Query without any field name - return getFieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), right, EnumSet.of(SearchTermFlag.INEXACT_MATCH, SearchTermFlag.CASE_INSENSITIVE)); + return getFieldQueryNode("any", right, EnumSet.of(SearchTermFlag.INEXACT_MATCH, SearchTermFlag.CASE_INSENSITIVE)); } } private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { - String additionalCondition = null; - if (searchFlags.contains((SearchTermFlag.EXACT_MATCH))) { - // additionally search in second table + StringBuilder whereClause = new StringBuilder(); + String operator = getOperator(searchFlags); + String prefixSuffix = searchFlags.contains(SearchTermFlag.INEXACT_MATCH) ? "%" : ""; - String operator = ""; - if (searchFlags.contains(SearchTermFlag.NEGATION)) { - operator = "NOT "; - } + // Pseudo-fields + field = switch (field) { + case "key" -> InternalField.KEY_FIELD.getName(); + case "anykeyword" -> StandardField.KEYWORDS.getName(); + default -> field; + }; - if (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE)) { - operator += "LIKE"; - } else { - operator += "ILIKE"; + if ("anyfield".equals(field) || "any".equals(field)) { + whereClause.append(buildTableQuery(MAIN_TABLE, operator, prefixSuffix, term)); + if (searchFlags.contains(SearchTermFlag.EXACT_MATCH)) { + whereClause.append(" OR ").append(buildTableQuery(SPLIT_TABLE, operator, prefixSuffix, term)); + } + } else { + whereClause.append(buildFieldQuery(MAIN_TABLE, field, operator, prefixSuffix, term)); + if (searchFlags.contains(SearchTermFlag.EXACT_MATCH)) { + whereClause.append(" OR ").append(buildFieldQuery(SPLIT_TABLE, field, operator, prefixSuffix, term)); } - additionalCondition = "(split_table." + PostgreConstants.FIELD_NAME + "='\" + field + \"' AND (" + - "(split_table." + PostgreConstants.FIELD_VALUE_LITERAL + " " + operator + " '" + term + "')" + - " OR " + - "(split_table." + PostgreConstants.FIELD_VALUE_TRANSFORMED + " " + operator + " '" + term + "')" + - "))"; } - String operator = ""; - String prefixSuffix = ""; + return whereClause.toString(); + } + private String getOperator(EnumSet searchFlags) { if (searchFlags.contains(SearchTermFlag.REGULAR_EXPRESSION)) { - if (searchFlags.contains(SearchTermFlag.NEGATION)) { - operator = "!"; - } - - if (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE)) { - operator += "~"; - } else { - operator += "~*"; - } + return (searchFlags.contains(SearchTermFlag.NEGATION) ? "!" : "") + + (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "~" : "~*"); } else { - if (searchFlags.contains(SearchTermFlag.INEXACT_MATCH)) { - prefixSuffix = "%"; - } - - if (searchFlags.contains(SearchTermFlag.NEGATION)) { - operator = "NOT "; - } - - if (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE)) { - operator += "LIKE"; - } else { - operator += "ILIKE"; - } + return (searchFlags.contains(SearchTermFlag.NEGATION) ? "NOT " : "") + + (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "LIKE" : "ILIKE"); } + } - // Pseudo-fields - if ("anyfield".equals(field) || "any".equals(field)) { - // TODO: also query _TRANSFORMED - return "(main_table." + PostgreConstants.FIELD_VALUE_LITERAL + " " + operator + " '" + prefixSuffix + term + prefixSuffix + "')"; - } + private String buildTableQuery(String tableName, String operator, String prefixSuffix, String term) { + return """ + (%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')""".formatted( + tableName, PostgreConstants.FIELD_VALUE_LITERAL, + operator, + prefixSuffix, term, prefixSuffix, + tableName, PostgreConstants.FIELD_VALUE_TRANSFORMED, + operator, prefixSuffix, term, prefixSuffix); + } - field = switch (field) { - case "key" -> - InternalField.KEY_FIELD.getName(); - case "anykeyword" -> - StandardField.KEYWORDS.getName(); - default -> - field; - }; + private String buildFieldQuery(String tableName, String field, String operator, String prefixSuffix, String term) { + return """ + ((%s.%s = '%s') AND (%s))""".formatted( + tableName, PostgreConstants.FIELD_NAME, field, + buildTableQuery(tableName, operator, prefixSuffix, term)); + } - // TODO: also query _TRANSFORMED - String resultMainTable = "(main_table." + PostgreConstants.FIELD_NAME + " = '" + field + "' AND main_table." + PostgreConstants.FIELD_VALUE_LITERAL + " " + operator + " '" + prefixSuffix + term + prefixSuffix + "')"; - if (additionalCondition != null) { - return "(" + resultMainTable + " OR " + additionalCondition + ")"; - } else { - return resultMainTable; + private static void setFlags(EnumSet flags, SearchTermFlag matchType, boolean caseSensitive, boolean negation) { + flags.add(matchType); + flags.add(caseSensitive ? SearchTermFlag.CASE_SENSITIVE : SearchTermFlag.CASE_INSENSITIVE); + if (negation) { + flags.add(SearchTermFlag.NEGATION); } } } From baa73cbc6ce217e0172e7db73c1d24d872639f66 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 27 Sep 2024 00:26:58 +0300 Subject: [PATCH 033/104] Adapt tests --- .../search/query/SearchToSqlConversion.java | 8 +++ .../search/query/SearchToSqlVisitor.java | 12 +++- .../search/SearchToSqlConversionTest.java | 61 ++++++++++--------- 3 files changed, 48 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java index 5522dfef5ad..e34aa32b92f 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java @@ -4,6 +4,7 @@ import org.jabref.search.SearchLexer; import org.jabref.search.SearchParser; +import com.google.common.annotations.VisibleForTesting; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CommonTokenStream; @@ -26,4 +27,11 @@ private static SearchParser.StartContext getStartContext(String searchExpression parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors return parser.start(); } + + @VisibleForTesting + public static String getWhereClause(String table, String searchExpression) { + SearchParser.StartContext context = getStartContext(searchExpression); + SearchToSqlVisitor searchToSqlVisitor = new SearchToSqlVisitor(table); + return searchToSqlVisitor.getWhereClause(context); + } } diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 0c632c9089c..685bf054d48 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -9,6 +9,7 @@ import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; +import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,9 +20,9 @@ */ public class SearchToSqlVisitor extends SearchBaseVisitor { + public static final String MAIN_TABLE = "main_table"; + public static final String SPLIT_TABLE = "split_table"; private static final Logger LOGGER = LoggerFactory.getLogger(SearchToSqlVisitor.class); - private static final String MAIN_TABLE = "main_table"; - private static final String SPLIT_TABLE = "split_table"; private final String mainTableName; private final String splitTableName; @@ -40,7 +41,7 @@ private enum SearchTermFlag { @Override public String visitStart(SearchParser.StartContext ctx) { - String whereClause = visit(ctx.expression()); + String whereClause = getWhereClause(ctx); String result = """ SELECT %s.%s FROM "%s" AS %s @@ -62,6 +63,11 @@ public String visitStart(SearchParser.StartContext ctx) { return result; } + @VisibleForTesting + public String getWhereClause(SearchParser.StartContext ctx) { + return visit(ctx.expression()); + } + @Override public String visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { return "NOT " + visit(ctx.expression()); diff --git a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java index f98e1178fd3..8519d727d0b 100644 --- a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java +++ b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java @@ -11,73 +11,74 @@ class SearchToSqlConversionTest { @ParameterizedTest @CsvSource({ // case insensitive contains - "(field_name = 'title' AND field_value ILIKE '%compute%'), title=compute", + "((main_table.field_name = 'title') AND ((main_table.field_value_literal ILIKE '%compute%') OR (main_table.field_value_transformed ILIKE '%compute%'))), title=compute", // case sensitive contains - "(field_name = 'title' AND field_value LIKE '%compute%'), title=!compute", + "((main_table.field_name = 'title') AND ((main_table.field_value_literal LIKE '%compute%') OR (main_table.field_value_transformed LIKE '%compute%'))), title=!compute", // exact match case insensitive - "(field_name = 'title' AND field_value ILIKE 'compute'), title==compute", + "((main_table.field_name = 'title') AND ((main_table.field_value_literal ILIKE 'compute') OR (main_table.field_value_transformed ILIKE 'compute'))) OR ((split_table.field_name = 'title') AND ((split_table.field_value_literal ILIKE 'compute') OR (split_table.field_value_transformed ILIKE 'compute'))), title==compute", // exact match case sensitive - "(field_name = 'title' AND field_value LIKE 'compute'), title==!compute", + "((main_table.field_name = 'title') AND ((main_table.field_value_literal LIKE 'compute') OR (main_table.field_value_transformed LIKE 'compute'))) OR ((split_table.field_name = 'title') AND ((split_table.field_value_literal LIKE 'compute') OR (split_table.field_value_transformed LIKE 'compute'))), title==!compute", // Regex search case insensitive - "(field_value ~* 'Jabref.*Search'), any=~Jabref.*Search", + "(main_table.field_value_literal ~* 'Jabref.*Search') OR (main_table.field_value_transformed ~* 'Jabref.*Search'), any=~Jabref.*Search", // Regex search case sensitive - "(field_value ~ 'Jabref.*Search'), any=~!Jabref.*Search", + "(main_table.field_value_literal ~ 'Jabref.*Search') OR (main_table.field_value_transformed ~ 'Jabref.*Search'), any=~!Jabref.*Search", // negated case insensitive contains - "(field_name = 'title' AND field_value NOT ILIKE '%compute%'), title!=compute", + "((main_table.field_name = 'title') AND ((main_table.field_value_literal NOT ILIKE '%compute%') OR (main_table.field_value_transformed NOT ILIKE '%compute%'))), title!=compute", // negated case sensitive contains - "(field_name = 'title' AND field_value NOT LIKE '%compute%'), title !=! compute", + "((main_table.field_name = 'title') AND ((main_table.field_value_literal NOT LIKE '%compute%') OR (main_table.field_value_transformed NOT LIKE '%compute%'))), title !=! compute", // negated case insensitive exact match - "(field_name = 'title' AND field_value NOT ILIKE 'compute'), title !== compute", + "((main_table.field_name = 'title') AND ((main_table.field_value_literal NOT ILIKE 'compute') OR (main_table.field_value_transformed NOT ILIKE 'compute'))) OR ((split_table.field_name = 'title') AND ((split_table.field_value_literal NOT ILIKE 'compute') OR (split_table.field_value_transformed NOT ILIKE 'compute'))), title !== compute", // negated case sensitive exact match - "(field_name = 'title' AND field_value NOT LIKE 'compute'), title !==! compute", + "((main_table.field_name = 'title') AND ((main_table.field_value_literal NOT LIKE 'compute') OR (main_table.field_value_transformed NOT LIKE 'compute'))) OR ((split_table.field_name = 'title') AND ((split_table.field_value_literal NOT LIKE 'compute') OR (split_table.field_value_transformed NOT LIKE 'compute'))), title !==! compute", // negated regex search case insensitive - "(field_value !~* 'Jabref.*Search'), any!=~Jabref.*Search", + "(main_table.field_value_literal !~* 'Jabref.*Search') OR (main_table.field_value_transformed !~* 'Jabref.*Search'), any!=~Jabref.*Search", // negated regex search case sensitive - "(field_value !~ 'Jabref.*Search'), any!=~!Jabref.*Search", + "(main_table.field_value_literal !~ 'Jabref.*Search') OR (main_table.field_value_transformed !~ 'Jabref.*Search'), any!=~!Jabref.*Search", // Default search, query without any field name (case insensitive contains) - "(field_value ILIKE '%computer%'), computer", - "(field_value ILIKE '%computer science%'), \"computer science\"", // Phrase search - "(field_value ILIKE '%computer%') OR (field_value ILIKE '%science%'), computer science", // Should be searched as a phrase or as two separate words (OR)? - "(field_value ILIKE '%!computer%'), !computer", // Is the explanation should be escaped? - "(field_value ILIKE '%!computer%'), \"!computer\"", + "(main_table.field_value_literal ILIKE '%computer%') OR (main_table.field_value_transformed ILIKE '%computer%'), computer", + "(main_table.field_value_literal ILIKE '%computer science%') OR (main_table.field_value_transformed ILIKE '%computer science%'), \"computer science\"", // Phrase search + "(field_value ILIKE '%computer%') OR (field_value ILIKE '%science%'), computer science", // Should be searched as a phrase or as two separate words (OR)? (Throw exception) + "(field_value ILIKE '%!computer%'), !computer", // Is the explanation should be escaped? (Throw exception) + "(main_table.field_value_literal ILIKE '%!computer%') OR (main_table.field_value_transformed ILIKE '%!computer%'), \"!computer\"", // search in all fields case sensitive contains - "(field_value LIKE '%computer%'), any=!computer", - "(field_value LIKE '%!computer%'), any=! !computer", // Is the explanation should be escaped? + "(main_table.field_value_literal LIKE '%computer%') OR (main_table.field_value_transformed LIKE '%computer%'), any=!computer", + "(field_value LIKE '%!computer%'), any=! !computer", // Is the explanation should be escaped? (Throw exception) // And - "(field_value ILIKE '%computer%') AND (field_value ILIKE '%science%'), computer AND science", + "(main_table.field_value_literal ILIKE '%computer%') OR (main_table.field_value_transformed ILIKE '%computer%') AND (main_table.field_value_literal ILIKE '%science%') OR (main_table.field_value_transformed ILIKE '%science%'), computer AND science", // Or - "(field_value ILIKE '%computer%') OR (field_value ILIKE '%science%'), computer OR science", + "(main_table.field_value_literal ILIKE '%computer%') OR (main_table.field_value_transformed ILIKE '%computer%') OR (main_table.field_value_literal ILIKE '%science%') OR (main_table.field_value_transformed ILIKE '%science%'), computer OR science", // Grouping - "((field_value ILIKE '%computer%') AND (field_value ILIKE '%science%')) OR (field_value ILIKE '%math%'), (computer AND science) OR math", - "(field_value ILIKE '%computer%') AND ((field_value ILIKE '%science%') OR (field_value ILIKE '%math%')), computer AND (science OR math)", - "((field_value ILIKE '%computer%') OR (field_value ILIKE '%science%')) AND ((field_value ILIKE '%math%') OR (field_value ILIKE '%physics%')), (computer OR science) AND (math OR physics)", + "((main_table.field_value_literal ILIKE '%computer%') OR (main_table.field_value_transformed ILIKE '%computer%') AND (main_table.field_value_literal ILIKE '%science%') OR (main_table.field_value_transformed ILIKE '%science%')) OR (main_table.field_value_literal ILIKE '%math%') OR (main_table.field_value_transformed ILIKE '%math%'), (computer AND science) OR math", + "(main_table.field_value_literal ILIKE '%computer%') OR (main_table.field_value_transformed ILIKE '%computer%') AND ((main_table.field_value_literal ILIKE '%science%') OR (main_table.field_value_transformed ILIKE '%science%') OR (main_table.field_value_literal ILIKE '%math%') OR (main_table.field_value_transformed ILIKE '%math%')), computer AND (science OR math)", + "((main_table.field_value_literal ILIKE '%computer%') OR (main_table.field_value_transformed ILIKE '%computer%') OR (main_table.field_value_literal ILIKE '%science%') OR (main_table.field_value_transformed ILIKE '%science%')) AND ((main_table.field_value_literal ILIKE '%math%') OR (main_table.field_value_transformed ILIKE '%math%') OR (main_table.field_value_literal ILIKE '%physics%') OR (main_table.field_value_transformed ILIKE '%physics%')), (computer OR science) AND (math OR physics)", // Special characters - "(field_value ILIKE '%{IEEE}%'), {IEEE}", - "(field_name = 'author' AND field_value ILIKE '%{IEEE}%'), author={IEEE}", + "(main_table.field_value_literal ILIKE '%{IEEE}%') OR (main_table.field_value_transformed ILIKE '%{IEEE}%'), {IEEE}", + "((main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE '%{IEEE}%') OR (main_table.field_value_transformed ILIKE '%{IEEE}%'))), author={IEEE}", // R\"ock - "(field_value ILIKE '%R\\\"ock%'), R\\\"ock", + "(field_value ILIKE '%R\\\"ock%'), R\\\"ock", // (Throw exception) // Breitenb{\"{u}}cher - "(field_value ILIKE '%Breitenb{\\\"{u}}cher%'), Breitenb{\\\"{u}}cher", + "(field_value ILIKE '%Breitenb{\\\"{u}}cher%'), Breitenb{\\\"{u}}cher", // (Throw exception) }) - void conversion(String expected, String input) { - assertEquals("SELECT entry_id FROM \"tableName\" WHERE " + expected + " GROUP BY entry_id", SearchToSqlConversion.searchToSql("tableName", input)); + + void conversion(String expectedWhereClause, String input) { + assertEquals(expectedWhereClause, SearchToSqlConversion.getWhereClause("tableName", input)); } } From 066c7430bff48b23fa2ed46b427c3901b1436d68 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 27 Sep 2024 20:51:02 +0300 Subject: [PATCH 034/104] Use join with EXACT_MATCH only --- .../logic/search/indexing/PostgreIndexer.java | 97 ++++++++++++------- .../search/query/SearchToSqlVisitor.java | 91 +++++++++-------- .../jabref/model/search/PostgreConstants.java | 1 + 3 files changed, 117 insertions(+), 72 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java index 36463b9be14..19edb9e929d 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java @@ -33,17 +33,17 @@ public class PostgreIndexer { private static final Pattern GROUPS_SEPARATOR_REGEX = Pattern.compile("\s*,\s*"); private static int NUMBER_OF_UNSAVED_LIBRARIES = 1; - private final BibEntryPreferences bibEntryPreferences; private final BibDatabaseContext databaseContext; private final Connection connection; private final String libraryName; private final String tableName; private final String tableNameSplitValues; + private final Character keywordSeparator; public PostgreIndexer(BibEntryPreferences bibEntryPreferences, BibDatabaseContext databaseContext, Connection connection) { - this.bibEntryPreferences = bibEntryPreferences; this.databaseContext = databaseContext; this.connection = connection; + this.keywordSeparator = bibEntryPreferences.getKeywordSeparator(); this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElse("unsaved"); if ("unsaved".equals(databaseContext.getPostgreTableName())) { this.tableName = "unsaved" + NUMBER_OF_UNSAVED_LIBRARIES++; @@ -71,7 +71,8 @@ private void setup() { %s TEXT, PRIMARY KEY (%s, %s) ) - """.formatted(tableName, + """.formatted( + tableName, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, PostgreConstants.FIELD_VALUE_LITERAL, @@ -86,7 +87,8 @@ PRIMARY KEY (%s, %s) %s TEXT, %s TEXT ) - """.formatted(tableNameSplitValues, + """.formatted( + tableNameSplitValues, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, PostgreConstants.FIELD_VALUE_LITERAL, @@ -100,38 +102,65 @@ PRIMARY KEY (%s, %s) // region btree index on id column connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") - """.formatted(PostgreConstants.ENTRY_ID.getIndexName(tableName), tableName, PostgreConstants.ENTRY_ID)); + """.formatted( + PostgreConstants.ENTRY_ID.getIndexName(tableName), + tableName, + PostgreConstants.ENTRY_ID)); + connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") - """.formatted(PostgreConstants.ENTRY_ID.getIndexName(tableNameSplitValues), tableName, PostgreConstants.ENTRY_ID)); + """.formatted( + PostgreConstants.ENTRY_ID.getIndexName(tableNameSplitValues), + tableName, + PostgreConstants.ENTRY_ID)); // endregion // region btree index on field name column connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") - """.formatted(PostgreConstants.FIELD_NAME.getIndexName(tableName), tableName, PostgreConstants.FIELD_NAME)); + """.formatted( + PostgreConstants.FIELD_NAME.getIndexName(tableName), + tableName, + PostgreConstants.FIELD_NAME)); + connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" ("%s") - """.formatted(PostgreConstants.FIELD_NAME.getIndexName(tableNameSplitValues), tableName, PostgreConstants.FIELD_NAME)); + """.formatted( + PostgreConstants.FIELD_NAME.getIndexName(tableNameSplitValues), + tableName, + PostgreConstants.FIELD_NAME)); // endregion // trigram index on field value column connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" USING gin ("%s" gin_trgm_ops) - """.formatted(PostgreConstants.FIELD_VALUE_LITERAL.getIndexName(tableName), tableName, PostgreConstants.FIELD_VALUE_LITERAL)); + """.formatted( + PostgreConstants.FIELD_VALUE_LITERAL.getIndexName(tableName), + tableName, + PostgreConstants.FIELD_VALUE_LITERAL)); // trigram index on field value transformed column connection.createStatement().executeUpdate(""" CREATE INDEX "%s" ON "%s" USING gin ("%s" gin_trgm_ops) - """.formatted(PostgreConstants.FIELD_VALUE_TRANSFORMED.getIndexName(tableName), tableName, PostgreConstants.FIELD_VALUE_TRANSFORMED)); + """.formatted( + PostgreConstants.FIELD_VALUE_TRANSFORMED.getIndexName(tableName), + tableName, + PostgreConstants.FIELD_VALUE_TRANSFORMED)); // region btree index on spilt values column - connection.createStatement().executeUpdate(""" - CREATE INDEX "%s" ON "%s" ("%s") - """.formatted(PostgreConstants.FIELD_VALUE_LITERAL.getIndexName(tableNameSplitValues), tableName, PostgreConstants.FIELD_VALUE_LITERAL)); - connection.createStatement().executeUpdate(""" - CREATE INDEX "%s" ON "%s" ("%s") - """.formatted(PostgreConstants.FIELD_VALUE_TRANSFORMED.getIndexName(tableNameSplitValues), tableName, PostgreConstants.FIELD_VALUE_TRANSFORMED)); +// connection.createStatement().executeUpdate(""" +// CREATE INDEX "%s" ON "%s" ("%s") +// """.formatted( +// PostgreConstants.FIELD_VALUE_LITERAL.getIndexName(tableNameSplitValues), +// tableName, +// PostgreConstants.FIELD_VALUE_LITERAL)); +// +// connection.createStatement().executeUpdate(""" +// CREATE INDEX "%s" ON "%s" ("%s") +// """.formatted( +// PostgreConstants.FIELD_VALUE_TRANSFORMED.getIndexName(tableNameSplitValues), +// tableName, +// PostgreConstants.FIELD_VALUE_TRANSFORMED)); // endregion LOGGER.debug("Created indexes for library: {}", libraryName); @@ -167,9 +196,10 @@ public void addToIndex(Collection entries, BackgroundTask task) { private void addToIndex(BibEntry bibEntry) { String insertFieldQuery = """ - INSERT INTO "%s" ("%s", "%s", "%s", "%s") - VALUES (?, ?, ?, ?) - """.formatted(tableName, + INSERT INTO "%s" ("%s", "%s", "%s", "%s") + VALUES (?, ?, ?, ?) + """.formatted( + tableName, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, PostgreConstants.FIELD_VALUE_LITERAL, @@ -178,7 +208,8 @@ private void addToIndex(BibEntry bibEntry) { String insertIntoSplitTable = """ INSERT INTO "%s" ("%s", "%s", "%s", "%s") VALUES (?, ?, ?, ?) - """.formatted(tableNameSplitValues, + """.formatted( + tableNameSplitValues, PostgreConstants.ENTRY_ID, PostgreConstants.FIELD_NAME, PostgreConstants.FIELD_VALUE_LITERAL, @@ -197,14 +228,14 @@ private void addToIndex(BibEntry bibEntry) { // One potential future flaw is that the bibEntry is modified concurrently and the field being deleted. Optional resolvedFieldLatexFree = bibEntry.getResolvedFieldOrAliasLatexFree(field, this.databaseContext.getDatabase()); assert resolvedFieldLatexFree.isPresent(); - prepareStatement(preparedStatement, entryId, field, value, resolvedFieldLatexFree.orElse("")); + addBatch(preparedStatement, entryId, field, value, resolvedFieldLatexFree.orElse("")); // region Handling of known multi-value fields - // split and convert to unicode + // split and convert to Unicode if (field.getProperties().contains(FieldProperty.PERSON_NAMES)) { addAuthors(value, preparedStatementSplitValues, entryId, field); } else if (field == StandardField.KEYWORDS) { - addKeywords(value, preparedStatementSplitValues, entryId, field); + addKeywords(value, preparedStatementSplitValues, entryId, field, keywordSeparator); } else if (field == StandardField.GROUPS) { addGroups(value, preparedStatementSplitValues, entryId, field); } else if (field.getProperties().contains(FieldProperty.MULTIPLE_ENTRY_LINK)) { @@ -237,24 +268,24 @@ private void addToIndex(BibEntry bibEntry) { private void addEntryLinks(BibEntry bibEntry, Field field, PreparedStatement preparedStatementSplitValues, String entryId) { bibEntry.getEntryLinkList(field, databaseContext.getDatabase()).stream().distinct().forEach(link -> { - doInsert(preparedStatementSplitValues, entryId, field, link.getKey()); + addBatch(preparedStatementSplitValues, entryId, field, link.getKey()); }); } private static void addGroups(String value, PreparedStatement preparedStatementSplitValues, String entryId, Field field) { - // We could use KeywordList, but we are afraid that group names could have ">" in their name and then they would not be handled correctly + // We could use KeywordList, but we are afraid that group names could have ">" in their name, and then they would not be handled correctly Arrays.stream(GROUPS_SEPARATOR_REGEX.split(value)) .distinct() .forEach(group -> { - doInsert(preparedStatementSplitValues, entryId, field, group); + addBatch(preparedStatementSplitValues, entryId, field, group); }); } - private void addKeywords(String keywordsString, PreparedStatement preparedStatementSplitValues, String entryId, Field field) { - KeywordList keywordList = KeywordList.parse(keywordsString, bibEntryPreferences.getKeywordSeparator()); + private static void addKeywords(String keywordsString, PreparedStatement preparedStatementSplitValues, String entryId, Field field, Character keywordSeparator) { + KeywordList keywordList = KeywordList.parse(keywordsString, keywordSeparator); keywordList.stream().flatMap(keyword -> keyword.flatten().stream()).forEach(keyword -> { String value = keyword.toString(); - doInsert(preparedStatementSplitValues, entryId, field, value); + addBatch(preparedStatementSplitValues, entryId, field, value); }); } @@ -264,18 +295,18 @@ private static void addAuthors(String value, PreparedStatement preparedStatement // We use the method giving us the most complete information for the literal value; String literal = author.getGivenFamily(false); String transformed = author.latexFree().getGivenFamily(false); - prepareStatement(preparedStatementSplitValues, entryId, field, literal, transformed); + addBatch(preparedStatementSplitValues, entryId, field, literal, transformed); }); } - private static void doInsert(PreparedStatement preparedStatement, String entryId, Field field, String value) { - prepareStatement(preparedStatement, entryId, field, value, LATEX_TO_UNICODE_FORMATTER.format(value)); + private static void addBatch(PreparedStatement preparedStatement, String entryId, Field field, String value) { + addBatch(preparedStatement, entryId, field, value, LATEX_TO_UNICODE_FORMATTER.format(value)); } /** * The values are passed as they should be inserted into the database table */ - private static void prepareStatement(PreparedStatement preparedStatement, String entryId, Field field, String value, String normalized) { + private static void addBatch(PreparedStatement preparedStatement, String entryId, Field field, String value, String normalized) { try { preparedStatement.setString(1, entryId); preparedStatement.setString(2, field.getName()); diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 685bf054d48..970d5d98730 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -20,12 +20,13 @@ */ public class SearchToSqlVisitor extends SearchBaseVisitor { - public static final String MAIN_TABLE = "main_table"; - public static final String SPLIT_TABLE = "split_table"; private static final Logger LOGGER = LoggerFactory.getLogger(SearchToSqlVisitor.class); + private static final String MAIN_TABLE = "main_table"; + private static final String SPLIT_TABLE = "split_table"; private final String mainTableName; private final String splitTableName; + private boolean isExactMatch = false; public SearchToSqlVisitor(String mainTableName) { this.mainTableName = mainTableName; @@ -42,23 +43,36 @@ private enum SearchTermFlag { @Override public String visitStart(SearchParser.StartContext ctx) { String whereClause = getWhereClause(ctx); - String result = """ - SELECT %s.%s - FROM "%s" AS %s - LEFT JOIN "%s" AS %s - ON (%s.%s = %s.%s AND %s.%s = %s.%s) - WHERE (%s) - GROUP BY %s.%s - """.formatted( - PostgreConstants.ENTRY_ID, MAIN_TABLE, - mainTableName, MAIN_TABLE, - splitTableName, SPLIT_TABLE, - MAIN_TABLE, PostgreConstants.ENTRY_ID, - SPLIT_TABLE, PostgreConstants.ENTRY_ID, - MAIN_TABLE, PostgreConstants.FIELD_NAME, - SPLIT_TABLE, PostgreConstants.FIELD_NAME, - whereClause, - MAIN_TABLE, PostgreConstants.ENTRY_ID); + String result; + + if (isExactMatch) { + result = """ + SELECT %s.%s FROM "%s" AS %s + LEFT JOIN "%s" AS %s + ON (%s.%s = %s.%s AND %s.%s = %s.%s) + WHERE (%s) + GROUP BY %s.%s + """.formatted( + MAIN_TABLE, PostgreConstants.ENTRY_ID, + mainTableName, MAIN_TABLE, + splitTableName, SPLIT_TABLE, + MAIN_TABLE, PostgreConstants.ENTRY_ID, + SPLIT_TABLE, PostgreConstants.ENTRY_ID, + MAIN_TABLE, PostgreConstants.FIELD_NAME, + SPLIT_TABLE, PostgreConstants.FIELD_NAME, + whereClause, + MAIN_TABLE, PostgreConstants.ENTRY_ID); + } else { + result = """ + SELECT %s.%s FROM "%s" AS %s + WHERE (%s) + GROUP BY %s.%s + """.formatted( + MAIN_TABLE, PostgreConstants.ENTRY_ID, + mainTableName, MAIN_TABLE, + whereClause, + MAIN_TABLE, PostgreConstants.ENTRY_ID); + } LOGGER.trace("Converted search query to SQL: {}", result); return result; } @@ -137,7 +151,17 @@ public String visitComparison(SearchParser.ComparisonContext context) { } } - private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { + private void setFlags(EnumSet flags, SearchTermFlag matchType, boolean caseSensitive, boolean negation) { + isExactMatch |= matchType.equals(SearchTermFlag.EXACT_MATCH); + flags.add(matchType); + + flags.add(caseSensitive ? SearchTermFlag.CASE_SENSITIVE : SearchTermFlag.CASE_INSENSITIVE); + if (negation) { + flags.add(SearchTermFlag.NEGATION); + } + } + + private static String getFieldQueryNode(String field, String term, EnumSet searchFlags) { StringBuilder whereClause = new StringBuilder(); String operator = getOperator(searchFlags); String prefixSuffix = searchFlags.contains(SearchTermFlag.INEXACT_MATCH) ? "%" : ""; @@ -164,38 +188,27 @@ private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { - if (searchFlags.contains(SearchTermFlag.REGULAR_EXPRESSION)) { - return (searchFlags.contains(SearchTermFlag.NEGATION) ? "!" : "") + - (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "~" : "~*"); - } else { - return (searchFlags.contains(SearchTermFlag.NEGATION) ? "NOT " : "") + - (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "LIKE" : "ILIKE"); - } + private static String getOperator(EnumSet searchFlags) { + return searchFlags.contains(SearchTermFlag.REGULAR_EXPRESSION) + ? (searchFlags.contains(SearchTermFlag.NEGATION) ? "!" : "") + (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "~" : "~*") + : (searchFlags.contains(SearchTermFlag.NEGATION) ? "NOT " : "") + (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "LIKE" : "ILIKE"); } - private String buildTableQuery(String tableName, String operator, String prefixSuffix, String term) { + private static String buildTableQuery(String tableName, String operator, String prefixSuffix, String term) { return """ (%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')""".formatted( tableName, PostgreConstants.FIELD_VALUE_LITERAL, operator, prefixSuffix, term, prefixSuffix, tableName, PostgreConstants.FIELD_VALUE_TRANSFORMED, - operator, prefixSuffix, term, prefixSuffix); + operator, + prefixSuffix, term, prefixSuffix); } - private String buildFieldQuery(String tableName, String field, String operator, String prefixSuffix, String term) { + private static String buildFieldQuery(String tableName, String field, String operator, String prefixSuffix, String term) { return """ ((%s.%s = '%s') AND (%s))""".formatted( tableName, PostgreConstants.FIELD_NAME, field, buildTableQuery(tableName, operator, prefixSuffix, term)); } - - private static void setFlags(EnumSet flags, SearchTermFlag matchType, boolean caseSensitive, boolean negation) { - flags.add(matchType); - flags.add(caseSensitive ? SearchTermFlag.CASE_SENSITIVE : SearchTermFlag.CASE_INSENSITIVE); - if (negation) { - flags.add(SearchTermFlag.NEGATION); - } - } } diff --git a/src/main/java/org/jabref/model/search/PostgreConstants.java b/src/main/java/org/jabref/model/search/PostgreConstants.java index f5092d8556f..4b6dbe7584f 100644 --- a/src/main/java/org/jabref/model/search/PostgreConstants.java +++ b/src/main/java/org/jabref/model/search/PostgreConstants.java @@ -7,6 +7,7 @@ public enum PostgreConstants { FIELD_VALUE_TRANSFORMED("field_value_transformed"); // contains the value transformed for better querying public static final String TABLE_NAME_SUFFIX = "_split_values"; + private final String columnName; PostgreConstants(String columnName) { From d9f93c40e8590b2ee11bca773b1b05f5252e528c Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sat, 28 Sep 2024 18:19:36 +0300 Subject: [PATCH 035/104] Update to Postgres 17 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 0d2eb110367..1643de2a6ec 100644 --- a/build.gradle +++ b/build.gradle @@ -371,7 +371,7 @@ dependencies { implementation 'io.github.adr:e-adr:2.0.0-SNAPSHOT' implementation 'io.zonky.test:embedded-postgres:2.0.7' - implementation enforcedPlatform('io.zonky.test.postgres:embedded-postgres-binaries-bom:16.4.0') + implementation enforcedPlatform('io.zonky.test.postgres:embedded-postgres-binaries-bom:17.0.0') testImplementation 'io.github.classgraph:classgraph:4.8.175' testImplementation 'org.junit.jupiter:junit-jupiter:5.11.0' From 38c02655d05d71bf9ad48b76a073eec4893073f7 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sat, 28 Sep 2024 20:52:12 +0300 Subject: [PATCH 036/104] Attempt to use sub-queries with CTEs --- .../search/query/SearchToSqlConversion.java | 8 -- .../search/query/SearchToSqlVisitor.java | 117 +++++++++--------- .../search/SearchToSqlConversionTest.java | 9 +- 3 files changed, 69 insertions(+), 65 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java index e34aa32b92f..5522dfef5ad 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java @@ -4,7 +4,6 @@ import org.jabref.search.SearchLexer; import org.jabref.search.SearchParser; -import com.google.common.annotations.VisibleForTesting; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CommonTokenStream; @@ -27,11 +26,4 @@ private static SearchParser.StartContext getStartContext(String searchExpression parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors return parser.start(); } - - @VisibleForTesting - public static String getWhereClause(String table, String searchExpression) { - SearchParser.StartContext context = getStartContext(searchExpression); - SearchToSqlVisitor searchToSqlVisitor = new SearchToSqlVisitor(table); - return searchToSqlVisitor.getWhereClause(context); - } } diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 970d5d98730..c96e369e70a 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -22,15 +22,13 @@ public class SearchToSqlVisitor extends SearchBaseVisitor { private static final Logger LOGGER = LoggerFactory.getLogger(SearchToSqlVisitor.class); private static final String MAIN_TABLE = "main_table"; - private static final String SPLIT_TABLE = "split_table"; private final String mainTableName; - private final String splitTableName; - private boolean isExactMatch = false; + + private int cteCount = 0; public SearchToSqlVisitor(String mainTableName) { this.mainTableName = mainTableName; - this.splitTableName = mainTableName + PostgreConstants.TABLE_NAME_SUFFIX; } private enum SearchTermFlag { @@ -44,35 +42,11 @@ private enum SearchTermFlag { public String visitStart(SearchParser.StartContext ctx) { String whereClause = getWhereClause(ctx); String result; + result = """ + WITH %s + SELECT * FROM cte%s + """.formatted(whereClause, cteCount - 1); - if (isExactMatch) { - result = """ - SELECT %s.%s FROM "%s" AS %s - LEFT JOIN "%s" AS %s - ON (%s.%s = %s.%s AND %s.%s = %s.%s) - WHERE (%s) - GROUP BY %s.%s - """.formatted( - MAIN_TABLE, PostgreConstants.ENTRY_ID, - mainTableName, MAIN_TABLE, - splitTableName, SPLIT_TABLE, - MAIN_TABLE, PostgreConstants.ENTRY_ID, - SPLIT_TABLE, PostgreConstants.ENTRY_ID, - MAIN_TABLE, PostgreConstants.FIELD_NAME, - SPLIT_TABLE, PostgreConstants.FIELD_NAME, - whereClause, - MAIN_TABLE, PostgreConstants.ENTRY_ID); - } else { - result = """ - SELECT %s.%s FROM "%s" AS %s - WHERE (%s) - GROUP BY %s.%s - """.formatted( - MAIN_TABLE, PostgreConstants.ENTRY_ID, - mainTableName, MAIN_TABLE, - whereClause, - MAIN_TABLE, PostgreConstants.ENTRY_ID); - } LOGGER.trace("Converted search query to SQL: {}", result); return result; } @@ -89,16 +63,32 @@ public String visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { @Override public String visitParenExpression(SearchParser.ParenExpressionContext ctx) { - return "(" + visit(ctx.expression()) + ")"; + return visit(ctx.expression()); } @Override public String visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { - if ("AND".equalsIgnoreCase(ctx.operator.getText())) { - return visit(ctx.left) + " AND " + visit(ctx.right); - } else { - return visit(ctx.left) + " OR " + visit(ctx.right); - } + String left = visit(ctx.left); + String right = visit(ctx.right); + return """ + %s, + %s, + cte%s AS ( + SELECT %s + FROM cte%s + %s + SELECT %s + FROM cte%s + ) + """.formatted( + left, + right, + cteCount++, + PostgreConstants.ENTRY_ID, + cteCount - 3, + "AND".equalsIgnoreCase(ctx.operator.getText()) ? "INTERSECT" : "UNION", + PostgreConstants.ENTRY_ID, + cteCount - 2); } @Override @@ -152,7 +142,6 @@ public String visitComparison(SearchParser.ComparisonContext context) { } private void setFlags(EnumSet flags, SearchTermFlag matchType, boolean caseSensitive, boolean negation) { - isExactMatch |= matchType.equals(SearchTermFlag.EXACT_MATCH); flags.add(matchType); flags.add(caseSensitive ? SearchTermFlag.CASE_SENSITIVE : SearchTermFlag.CASE_INSENSITIVE); @@ -161,7 +150,7 @@ private void setFlags(EnumSet flags, SearchTermFlag matchType, b } } - private static String getFieldQueryNode(String field, String term, EnumSet searchFlags) { + private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { StringBuilder whereClause = new StringBuilder(); String operator = getOperator(searchFlags); String prefixSuffix = searchFlags.contains(SearchTermFlag.INEXACT_MATCH) ? "%" : ""; @@ -174,15 +163,9 @@ private static String getFieldQueryNode(String field, String term, EnumSet searchFlags) { : (searchFlags.contains(SearchTermFlag.NEGATION) ? "NOT " : "") + (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "LIKE" : "ILIKE"); } - private static String buildTableQuery(String tableName, String operator, String prefixSuffix, String term) { + private String buildFieldQuery(String field, String operator, String prefixSuffix, String term) { return """ - (%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')""".formatted( - tableName, PostgreConstants.FIELD_VALUE_LITERAL, + cte%s AS ( + SELECT %s.%s + FROM "%s" AS %s + WHERE (%s.%s = '%s') AND ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) + ) + """.formatted( + cteCount++, + MAIN_TABLE, PostgreConstants.ENTRY_ID, + mainTableName, MAIN_TABLE, + MAIN_TABLE, PostgreConstants.FIELD_NAME, + field, + MAIN_TABLE, PostgreConstants.FIELD_VALUE_LITERAL, operator, prefixSuffix, term, prefixSuffix, - tableName, PostgreConstants.FIELD_VALUE_TRANSFORMED, + MAIN_TABLE, PostgreConstants.FIELD_VALUE_TRANSFORMED, operator, prefixSuffix, term, prefixSuffix); } - private static String buildFieldQuery(String tableName, String field, String operator, String prefixSuffix, String term) { + private String buildAnyFieldQuery(String operator, String prefixSuffix, String term) { return """ - ((%s.%s = '%s') AND (%s))""".formatted( - tableName, PostgreConstants.FIELD_NAME, field, - buildTableQuery(tableName, operator, prefixSuffix, term)); + cte%s AS ( + SELECT %s.%s + FROM "%s" AS %s + WHERE ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) + ) + """.formatted( + cteCount++, + MAIN_TABLE, PostgreConstants.ENTRY_ID, + mainTableName, MAIN_TABLE, + MAIN_TABLE, PostgreConstants.FIELD_VALUE_LITERAL, + operator, + prefixSuffix, term, prefixSuffix, + MAIN_TABLE, PostgreConstants.FIELD_VALUE_TRANSFORMED, + operator, + prefixSuffix, term, prefixSuffix); } } diff --git a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java index 8519d727d0b..b44d10e4be8 100644 --- a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java +++ b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java @@ -10,6 +10,13 @@ class SearchToSqlConversionTest { @ParameterizedTest @CsvSource({ + "(), alex", + "(), author=alex", + "(), title=dino AND author=alex", + "(), author=computer AND editor=science OR title=math", + "(), (author=computer AND editor=science) OR title=math", + "(), author=computer AND (editor=science OR title=math)", + "(), (author=computer AND editor=science) OR (title=math AND year=2021)", // case insensitive contains "((main_table.field_name = 'title') AND ((main_table.field_value_literal ILIKE '%compute%') OR (main_table.field_value_transformed ILIKE '%compute%'))), title=compute", @@ -79,6 +86,6 @@ class SearchToSqlConversionTest { }) void conversion(String expectedWhereClause, String input) { - assertEquals(expectedWhereClause, SearchToSqlConversion.getWhereClause("tableName", input)); + assertEquals(expectedWhereClause, SearchToSqlConversion.searchToSql("tableName", input)); } } From e8a28366f3beef0e84fd393d9b4bdcbc433d3175 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 29 Sep 2024 01:06:22 +0300 Subject: [PATCH 037/104] Fix CTEs sub-queries and grouping TODO: EXCAT_MATCH to search in split table --- .../search/query/SearchToSqlConversion.java | 4 +- .../search/query/SearchToSqlVisitor.java | 115 +++++++++++------- 2 files changed, 75 insertions(+), 44 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java index 5522dfef5ad..265b3cf55c5 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java @@ -4,8 +4,8 @@ import org.jabref.search.SearchLexer; import org.jabref.search.SearchParser; -import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonTokenStream; public class SearchToSqlConversion { @@ -17,7 +17,7 @@ public static String searchToSql(String table, String searchExpression) { } private static SearchParser.StartContext getStartContext(String searchExpression) { - SearchLexer lexer = new SearchLexer(new ANTLRInputStream(searchExpression)); + SearchLexer lexer = new SearchLexer(CharStreams.fromString(searchExpression)); lexer.removeErrorListeners(); // no infos on file system lexer.addErrorListener(ThrowingErrorListener.INSTANCE); SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index c96e369e70a..522da052d82 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -1,6 +1,8 @@ package org.jabref.logic.search.query; +import java.util.ArrayList; import java.util.EnumSet; +import java.util.List; import java.util.Optional; import org.jabref.model.entry.field.InternalField; @@ -9,7 +11,6 @@ import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; -import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,8 +25,8 @@ public class SearchToSqlVisitor extends SearchBaseVisitor { private static final String MAIN_TABLE = "main_table"; private final String mainTableName; - - private int cteCount = 0; + private final List ctes = new ArrayList<>(); + private int cteCounter = 0; public SearchToSqlVisitor(String mainTableName) { this.mainTableName = mainTableName; @@ -40,55 +41,82 @@ private enum SearchTermFlag { @Override public String visitStart(SearchParser.StartContext ctx) { - String whereClause = getWhereClause(ctx); - String result; - result = """ - WITH %s - SELECT * FROM cte%s - """.formatted(whereClause, cteCount - 1); - - LOGGER.trace("Converted search query to SQL: {}", result); - return result; - } + String query = visit(ctx.expression()); - @VisibleForTesting - public String getWhereClause(SearchParser.StartContext ctx) { - return visit(ctx.expression()); - } + StringBuilder sql = new StringBuilder("WITH\n"); + for (String cte : ctes) { + sql.append(cte).append(",\n"); + } - @Override - public String visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { - return "NOT " + visit(ctx.expression()); + // Remove the last comma and newline + if (!ctes.isEmpty()) { + sql.setLength(sql.length() - 2); + } + + sql.append("SELECT * FROM ").append(query); + LOGGER.trace("Converted search query to SQL: {}", sql); + return sql.toString(); } @Override - public String visitParenExpression(SearchParser.ParenExpressionContext ctx) { - return visit(ctx.expression()); + public String visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { + String subQuery = visit(ctx.expression()); + String cte = """ + cte%d AS ( + SELECT %s + FROM %s + EXCEPT + SELECT %s + FROM "%s" + ) + """.formatted( + cteCounter, + PostgreConstants.ENTRY_ID, + subQuery, + PostgreConstants.ENTRY_ID, + mainTableName); + ctes.add(cte); + return "cte" + cteCounter++; } @Override public String visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { String left = visit(ctx.left); String right = visit(ctx.right); - return """ - %s, - %s, - cte%s AS ( + String operator = "AND".equalsIgnoreCase(ctx.operator.getText()) ? "INTERSECT" : "UNION"; + + String cte = """ + cte%d AS ( SELECT %s - FROM cte%s + FROM %s %s SELECT %s - FROM cte%s + FROM %s ) """.formatted( - left, - right, - cteCount++, + cteCounter, PostgreConstants.ENTRY_ID, - cteCount - 3, - "AND".equalsIgnoreCase(ctx.operator.getText()) ? "INTERSECT" : "UNION", + left, + operator, PostgreConstants.ENTRY_ID, - cteCount - 2); + right); + ctes.add(cte); + return "cte" + cteCounter++; + } + + @Override + public String visitParenExpression(SearchParser.ParenExpressionContext ctx) { + return visit(ctx.expression()); + } + + @Override + public String visitAtomExpression(SearchParser.AtomExpressionContext ctx) { + return visit(ctx.comparison()); + } + + @Override + public String visitName(SearchParser.NameContext ctx) { + return ctx.getText(); } @Override @@ -102,6 +130,7 @@ public String visitComparison(SearchParser.ComparisonContext context) { } Optional fieldDescriptor = Optional.ofNullable(context.left); + String cte; if (fieldDescriptor.isPresent()) { String field = fieldDescriptor.get().getText(); @@ -134,11 +163,13 @@ public String visitComparison(SearchParser.ComparisonContext context) { setFlags(searchFlags, SearchTermFlag.REGULAR_EXPRESSION, true, true); } - return getFieldQueryNode(field, right, searchFlags); + cte = getFieldQueryNode(field, right, searchFlags); } else { // Query without any field name - return getFieldQueryNode("any", right, EnumSet.of(SearchTermFlag.INEXACT_MATCH, SearchTermFlag.CASE_INSENSITIVE)); + cte = getFieldQueryNode("any", right, EnumSet.of(SearchTermFlag.INEXACT_MATCH, SearchTermFlag.CASE_INSENSITIVE)); } + ctes.add(cte); + return "cte" + cteCounter++; } private void setFlags(EnumSet flags, SearchTermFlag matchType, boolean caseSensitive, boolean negation) { @@ -163,7 +194,7 @@ private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { private String buildFieldQuery(String field, String operator, String prefixSuffix, String term) { return """ - cte%s AS ( + cte%d AS ( SELECT %s.%s FROM "%s" AS %s WHERE (%s.%s = '%s') AND ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) ) """.formatted( - cteCount++, + cteCounter, MAIN_TABLE, PostgreConstants.ENTRY_ID, mainTableName, MAIN_TABLE, MAIN_TABLE, PostgreConstants.FIELD_NAME, @@ -198,15 +229,15 @@ private String buildFieldQuery(String field, String operator, String prefixSuffi prefixSuffix, term, prefixSuffix); } - private String buildAnyFieldQuery(String operator, String prefixSuffix, String term) { + private String buildAllFieldsQuery(String operator, String prefixSuffix, String term) { return """ - cte%s AS ( + cte%d AS ( SELECT %s.%s FROM "%s" AS %s WHERE ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) ) """.formatted( - cteCount++, + cteCounter, MAIN_TABLE, PostgreConstants.ENTRY_ID, mainTableName, MAIN_TABLE, MAIN_TABLE, PostgreConstants.FIELD_VALUE_LITERAL, From 9073d0c227e22b54f50ce296f20407eb8fc872d5 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 29 Sep 2024 01:23:00 +0300 Subject: [PATCH 038/104] group matches by entry_id --- .../java/org/jabref/logic/search/query/SearchToSqlVisitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 522da052d82..28e689f7bb4 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -53,7 +53,7 @@ public String visitStart(SearchParser.StartContext ctx) { sql.setLength(sql.length() - 2); } - sql.append("SELECT * FROM ").append(query); + sql.append("SELECT * FROM ").append(query).append(" GROUP BY ").append(PostgreConstants.ENTRY_ID); LOGGER.trace("Converted search query to SQL: {}", sql); return sql.toString(); } From a34f8bdf9839e43eabaf98000aeca9753f516d75 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 30 Sep 2024 19:36:06 +0300 Subject: [PATCH 039/104] Use NOT IN for negation queries --- .../search/query/SearchToSqlVisitor.java | 106 ++++++++++++++---- .../search/SearchToSqlConversionTest.java | 1 + 2 files changed, 84 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 28e689f7bb4..852339fdbf2 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -23,6 +23,7 @@ public class SearchToSqlVisitor extends SearchBaseVisitor { private static final Logger LOGGER = LoggerFactory.getLogger(SearchToSqlVisitor.class); private static final String MAIN_TABLE = "main_table"; + private static final String INNER_TABLE = "inner_table"; private final String mainTableName; private final List ctes = new ArrayList<>(); @@ -44,9 +45,7 @@ public String visitStart(SearchParser.StartContext ctx) { String query = visit(ctx.expression()); StringBuilder sql = new StringBuilder("WITH\n"); - for (String cte : ctes) { - sql.append(cte).append(",\n"); - } + ctes.forEach(cte -> sql.append(cte).append(",\n")); // Remove the last comma and newline if (!ctes.isEmpty()) { @@ -172,17 +171,8 @@ public String visitComparison(SearchParser.ComparisonContext context) { return "cte" + cteCounter++; } - private void setFlags(EnumSet flags, SearchTermFlag matchType, boolean caseSensitive, boolean negation) { - flags.add(matchType); - - flags.add(caseSensitive ? SearchTermFlag.CASE_SENSITIVE : SearchTermFlag.CASE_INSENSITIVE); - if (negation) { - flags.add(SearchTermFlag.NEGATION); - } - } - private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { - StringBuilder whereClause = new StringBuilder(); + String cte; String operator = getOperator(searchFlags); String prefixSuffix = searchFlags.contains(SearchTermFlag.INEXACT_MATCH) ? "%" : ""; @@ -194,18 +184,19 @@ private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { - return searchFlags.contains(SearchTermFlag.REGULAR_EXPRESSION) - ? (searchFlags.contains(SearchTermFlag.NEGATION) ? "!" : "") + (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "~" : "~*") - : (searchFlags.contains(SearchTermFlag.NEGATION) ? "NOT " : "") + (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "LIKE" : "ILIKE"); + return cte; } private String buildFieldQuery(String field, String operator, String prefixSuffix, String term) { @@ -229,6 +220,34 @@ private String buildFieldQuery(String field, String operator, String prefixSuffi prefixSuffix, term, prefixSuffix); } + private String buildNegationFieldQuery(String field, String operator, String prefixSuffix, String term) { + return """ + cte%d AS ( + SELECT %s.%s + FROM "%s" AS %s + WHERE %s.%s NOT IN ( + SELECT %s.%s + FROM "%s" AS %s + WHERE (%s.%s = '%s') AND ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) + ) + ) + """.formatted( + cteCounter, + MAIN_TABLE, PostgreConstants.ENTRY_ID, + mainTableName, MAIN_TABLE, + MAIN_TABLE, PostgreConstants.ENTRY_ID, + INNER_TABLE, PostgreConstants.ENTRY_ID, + mainTableName, INNER_TABLE, + INNER_TABLE, PostgreConstants.FIELD_NAME, + field, + INNER_TABLE, PostgreConstants.FIELD_VALUE_LITERAL, + operator, + prefixSuffix, term, prefixSuffix, + INNER_TABLE, PostgreConstants.FIELD_VALUE_TRANSFORMED, + operator, + prefixSuffix, term, prefixSuffix); + } + private String buildAllFieldsQuery(String operator, String prefixSuffix, String term) { return """ cte%d AS ( @@ -247,4 +266,45 @@ private String buildAllFieldsQuery(String operator, String prefixSuffix, String operator, prefixSuffix, term, prefixSuffix); } + + private String buildNegationAllFieldsQuery(String operator, String prefixSuffix, String term) { + return """ + cte%d AS ( + SELECT %s.%s + FROM "%s" AS %s + WHERE %s.%s NOT IN ( + SELECT %s.%s + FROM "%s" AS %s + WHERE ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) + ) + ) + """.formatted( + cteCounter, + MAIN_TABLE, PostgreConstants.ENTRY_ID, + mainTableName, MAIN_TABLE, + MAIN_TABLE, PostgreConstants.ENTRY_ID, + INNER_TABLE, PostgreConstants.ENTRY_ID, + mainTableName, INNER_TABLE, + INNER_TABLE, PostgreConstants.FIELD_VALUE_LITERAL, + operator, + prefixSuffix, term, prefixSuffix, + INNER_TABLE, PostgreConstants.FIELD_VALUE_TRANSFORMED, + operator, + prefixSuffix, term, prefixSuffix); + } + + private static void setFlags(EnumSet flags, SearchTermFlag matchType, boolean caseSensitive, boolean negation) { + flags.add(matchType); + + flags.add(caseSensitive ? SearchTermFlag.CASE_SENSITIVE : SearchTermFlag.CASE_INSENSITIVE); + if (negation) { + flags.add(SearchTermFlag.NEGATION); + } + } + + private static String getOperator(EnumSet searchFlags) { + return searchFlags.contains(SearchTermFlag.REGULAR_EXPRESSION) + ? (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "~" : "~*") + : (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "LIKE" : "ILIKE"); + } } diff --git a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java index b44d10e4be8..5bd5453040c 100644 --- a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java +++ b/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java @@ -16,6 +16,7 @@ class SearchToSqlConversionTest { "(), author=computer AND editor=science OR title=math", "(), (author=computer AND editor=science) OR title=math", "(), author=computer AND (editor=science OR title=math)", + "(), (title = computer OR comment = science) AND (title = math OR comment = physics)", "(), (author=computer AND editor=science) OR (title=math AND year=2021)", // case insensitive contains "((main_table.field_name = 'title') AND ((main_table.field_value_literal ILIKE '%compute%') OR (main_table.field_value_transformed ILIKE '%compute%'))), title=compute", From 1fe2b06c2bfb05314742ac56afd50dc1d2ae42ea Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 30 Sep 2024 20:01:46 +0300 Subject: [PATCH 040/104] Fix unary NOT operator --- .../search/query/SearchToSqlVisitor.java | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 852339fdbf2..6135851106d 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -7,13 +7,17 @@ import org.jabref.model.entry.field.InternalField; import org.jabref.model.entry.field.StandardField; -import org.jabref.model.search.PostgreConstants; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.jabref.model.search.PostgreConstants.ENTRY_ID; +import static org.jabref.model.search.PostgreConstants.FIELD_NAME; +import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_LITERAL; +import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_TRANSFORMED; + /** * Converts to a query processable by the scheme created by {@link org.jabref.logic.search.indexing.PostgreIndexer}. * @@ -52,7 +56,7 @@ public String visitStart(SearchParser.StartContext ctx) { sql.setLength(sql.length() - 2); } - sql.append("SELECT * FROM ").append(query).append(" GROUP BY ").append(PostgreConstants.ENTRY_ID); + sql.append("SELECT * FROM ").append(query).append(" GROUP BY ").append(ENTRY_ID); LOGGER.trace("Converted search query to SQL: {}", sql); return sql.toString(); } @@ -62,18 +66,20 @@ public String visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { String subQuery = visit(ctx.expression()); String cte = """ cte%d AS ( - SELECT %s - FROM %s - EXCEPT - SELECT %s - FROM "%s" + SELECT %s.%s + FROM "%s" AS %s + WHERE %s.%s NOT IN ( + SELECT %s + FROM %s + ) ) """.formatted( cteCounter, - PostgreConstants.ENTRY_ID, - subQuery, - PostgreConstants.ENTRY_ID, - mainTableName); + MAIN_TABLE, ENTRY_ID, + mainTableName, MAIN_TABLE, + MAIN_TABLE, ENTRY_ID, + ENTRY_ID, + subQuery); ctes.add(cte); return "cte" + cteCounter++; } @@ -94,10 +100,10 @@ public String visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { ) """.formatted( cteCounter, - PostgreConstants.ENTRY_ID, + ENTRY_ID, left, operator, - PostgreConstants.ENTRY_ID, + ENTRY_ID, right); ctes.add(cte); return "cte" + cteCounter++; @@ -208,14 +214,14 @@ private String buildFieldQuery(String field, String operator, String prefixSuffi ) """.formatted( cteCounter, - MAIN_TABLE, PostgreConstants.ENTRY_ID, + MAIN_TABLE, ENTRY_ID, mainTableName, MAIN_TABLE, - MAIN_TABLE, PostgreConstants.FIELD_NAME, + MAIN_TABLE, FIELD_NAME, field, - MAIN_TABLE, PostgreConstants.FIELD_VALUE_LITERAL, + MAIN_TABLE, FIELD_VALUE_LITERAL, operator, prefixSuffix, term, prefixSuffix, - MAIN_TABLE, PostgreConstants.FIELD_VALUE_TRANSFORMED, + MAIN_TABLE, FIELD_VALUE_TRANSFORMED, operator, prefixSuffix, term, prefixSuffix); } @@ -233,17 +239,17 @@ private String buildNegationFieldQuery(String field, String operator, String pre ) """.formatted( cteCounter, - MAIN_TABLE, PostgreConstants.ENTRY_ID, + MAIN_TABLE, ENTRY_ID, mainTableName, MAIN_TABLE, - MAIN_TABLE, PostgreConstants.ENTRY_ID, - INNER_TABLE, PostgreConstants.ENTRY_ID, + MAIN_TABLE, ENTRY_ID, + INNER_TABLE, ENTRY_ID, mainTableName, INNER_TABLE, - INNER_TABLE, PostgreConstants.FIELD_NAME, + INNER_TABLE, FIELD_NAME, field, - INNER_TABLE, PostgreConstants.FIELD_VALUE_LITERAL, + INNER_TABLE, FIELD_VALUE_LITERAL, operator, prefixSuffix, term, prefixSuffix, - INNER_TABLE, PostgreConstants.FIELD_VALUE_TRANSFORMED, + INNER_TABLE, FIELD_VALUE_TRANSFORMED, operator, prefixSuffix, term, prefixSuffix); } @@ -257,12 +263,12 @@ private String buildAllFieldsQuery(String operator, String prefixSuffix, String ) """.formatted( cteCounter, - MAIN_TABLE, PostgreConstants.ENTRY_ID, + MAIN_TABLE, ENTRY_ID, mainTableName, MAIN_TABLE, - MAIN_TABLE, PostgreConstants.FIELD_VALUE_LITERAL, + MAIN_TABLE, FIELD_VALUE_LITERAL, operator, prefixSuffix, term, prefixSuffix, - MAIN_TABLE, PostgreConstants.FIELD_VALUE_TRANSFORMED, + MAIN_TABLE, FIELD_VALUE_TRANSFORMED, operator, prefixSuffix, term, prefixSuffix); } @@ -280,15 +286,15 @@ private String buildNegationAllFieldsQuery(String operator, String prefixSuffix, ) """.formatted( cteCounter, - MAIN_TABLE, PostgreConstants.ENTRY_ID, + MAIN_TABLE, ENTRY_ID, mainTableName, MAIN_TABLE, - MAIN_TABLE, PostgreConstants.ENTRY_ID, - INNER_TABLE, PostgreConstants.ENTRY_ID, + MAIN_TABLE, ENTRY_ID, + INNER_TABLE, ENTRY_ID, mainTableName, INNER_TABLE, - INNER_TABLE, PostgreConstants.FIELD_VALUE_LITERAL, + INNER_TABLE, FIELD_VALUE_LITERAL, operator, prefixSuffix, term, prefixSuffix, - INNER_TABLE, PostgreConstants.FIELD_VALUE_TRANSFORMED, + INNER_TABLE, FIELD_VALUE_TRANSFORMED, operator, prefixSuffix, term, prefixSuffix); } From c6869cc8e1c711065275042fcee2ef9c002bef19 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 30 Sep 2024 20:55:03 +0300 Subject: [PATCH 041/104] Use split values table for EXACT_MATCH queries --- .../search/query/SearchToSqlVisitor.java | 155 ++++++++++++++++-- 1 file changed, 145 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 6135851106d..caa102f6feb 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -17,6 +17,7 @@ import static org.jabref.model.search.PostgreConstants.FIELD_NAME; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_LITERAL; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_TRANSFORMED; +import static org.jabref.model.search.PostgreConstants.TABLE_NAME_SUFFIX; /** * Converts to a query processable by the scheme created by {@link org.jabref.logic.search.indexing.PostgreIndexer}. @@ -27,14 +28,18 @@ public class SearchToSqlVisitor extends SearchBaseVisitor { private static final Logger LOGGER = LoggerFactory.getLogger(SearchToSqlVisitor.class); private static final String MAIN_TABLE = "main_table"; + private static final String SPLIT_TABLE = "split_table"; private static final String INNER_TABLE = "inner_table"; private final String mainTableName; + private final String splitValuesTableName; + private final List ctes = new ArrayList<>(); private int cteCounter = 0; public SearchToSqlVisitor(String mainTableName) { this.mainTableName = mainTableName; + this.splitValuesTableName = mainTableName + TABLE_NAME_SUFFIX; } private enum SearchTermFlag { @@ -190,22 +195,30 @@ private String getFieldQueryNode(String field, String term, EnumSet Date: Wed, 2 Oct 2024 18:10:39 +0300 Subject: [PATCH 042/104] Prepare for linked files index (full-text) --- .../org/jabref/logic/search/IndexManager.java | 8 +- .../jabref/logic/search/PostgreServer.java | 23 ++- ...tgreIndexer.java => BibFieldsIndexer.java} | 170 ++++++++++-------- .../search/query/SearchToSqlVisitor.java | 3 +- .../org/jabref/logic/util/Directories.java | 7 + .../model/database/BibDatabaseContext.java | 5 +- .../jabref/model/search/PostgreConstants.java | 20 +-- 7 files changed, 138 insertions(+), 98 deletions(-) rename src/main/java/org/jabref/logic/search/indexing/{PostgreIndexer.java => BibFieldsIndexer.java} (76%) diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index 35b77980946..9e3504bb36e 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -9,7 +9,7 @@ import org.jabref.logic.preferences.CliPreferences; import org.jabref.logic.search.indexing.DefaultLinkedFilesIndexer; -import org.jabref.logic.search.indexing.PostgreIndexer; +import org.jabref.logic.search.indexing.BibFieldsIndexer; import org.jabref.logic.search.indexing.ReadOnlyLinkedFilesIndexer; import org.jabref.logic.search.retrieval.BibFieldsSearcher; import org.jabref.logic.search.retrieval.LinkedFilesSearcher; @@ -39,7 +39,7 @@ public class IndexManager { private final BooleanProperty shouldIndexLinkedFiles; private final BooleanProperty isLinkedFilesIndexerBlocked = new SimpleBooleanProperty(false); private final ChangeListener preferencesListener; - private final PostgreIndexer bibFieldsIndexer; + private final BibFieldsIndexer bibFieldsIndexer; private final LuceneIndexer linkedFilesIndexer; private final BibFieldsSearcher bibFieldsSearcher; private final LinkedFilesSearcher linkedFilesSearcher; @@ -52,7 +52,7 @@ public IndexManager(BibDatabaseContext databaseContext, TaskExecutor executor, C this.shouldIndexLinkedFiles.addListener(preferencesListener); PostgreServer postgreServer = Injector.instantiateModelOrService(PostgreServer.class); - bibFieldsIndexer = new PostgreIndexer(preferences.getBibEntryPreferences(), databaseContext, postgreServer.getConnection()); + bibFieldsIndexer = new BibFieldsIndexer(preferences.getBibEntryPreferences(), databaseContext, postgreServer.getConnection()); LuceneIndexer indexer; try { @@ -229,7 +229,7 @@ public SearchResults search(SearchQuery query) { if (query.getSearchFlags().contains(SearchFlags.FULLTEXT)) { // TODO: merge results from lucene and postgres } else { - query.setSearchResults(bibFieldsSearcher.search(query.getSqlQuery(bibFieldsIndexer.getTableName()))); + query.setSearchResults(bibFieldsSearcher.search(query.getSqlQuery(bibFieldsIndexer.getMainTable()))); } return query.getSearchResults(); } diff --git a/src/main/java/org/jabref/logic/search/PostgreServer.java b/src/main/java/org/jabref/logic/search/PostgreServer.java index afe2a414bb8..25488005fbe 100644 --- a/src/main/java/org/jabref/logic/search/PostgreServer.java +++ b/src/main/java/org/jabref/logic/search/PostgreServer.java @@ -6,10 +6,15 @@ import javax.sql.DataSource; +import org.jabref.logic.util.Directories; + import io.zonky.test.db.postgres.embedded.EmbeddedPostgres; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.jabref.model.search.PostgreConstants.BIB_FIELDS_SCHEME; +import static org.jabref.model.search.PostgreConstants.LINKED_FILES_SCHEME; + public class PostgreServer { private static final Logger LOGGER = LoggerFactory.getLogger(PostgreServer.class); private final EmbeddedPostgres embeddedPostgres; @@ -19,6 +24,9 @@ public PostgreServer() { EmbeddedPostgres embeddedPostgres; try { embeddedPostgres = EmbeddedPostgres.builder() + .setCleanDataDirectory(false) + .setDataDirectory(Directories.getPostgresDataDirectory()) + .setPort(10456) .start(); LOGGER.info("Postgres server started, connection port: {}", embeddedPostgres.getPort()); } catch (IOException e) { @@ -29,9 +37,22 @@ public PostgreServer() { } this.embeddedPostgres = embeddedPostgres; - // TODO: Use the default database (postgres) and the default schema (public) or create a new one? this.dataSource = embeddedPostgres.getPostgresDatabase(); addTrigramExtension(); + createScheme(); + } + + private void createScheme() { + try (Connection connection = getConnection()) { + if (connection != null) { + LOGGER.debug("Creating scheme for bib fields and linked files"); + connection.createStatement().execute("DROP SCHEMA IF EXISTS " + BIB_FIELDS_SCHEME); + connection.createStatement().execute("CREATE SCHEMA " + BIB_FIELDS_SCHEME); + connection.createStatement().execute("CREATE SCHEMA IF NOT EXISTS " + LINKED_FILES_SCHEME); + } + } catch (SQLException e) { + LOGGER.error("Could not create scheme for bib fields and linked files", e); + } } private void addTrigramExtension() { diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java similarity index 76% rename from src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java rename to src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index 19edb9e929d..9dab45aff5a 100644 --- a/src/main/java/org/jabref/logic/search/indexing/PostgreIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -27,35 +27,47 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PostgreIndexer { - private static final Logger LOGGER = LoggerFactory.getLogger(PostgreIndexer.class); +import static org.jabref.model.search.PostgreConstants.ENTRY_ID; +import static org.jabref.model.search.PostgreConstants.FIELD_NAME; +import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_LITERAL; +import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_TRANSFORMED; +import static org.jabref.model.search.PostgreConstants.TABLE_NAME_SUFFIX; + +public class BibFieldsIndexer { + private static final Logger LOGGER = LoggerFactory.getLogger(BibFieldsIndexer.class); private static final LatexToUnicodeFormatter LATEX_TO_UNICODE_FORMATTER = new LatexToUnicodeFormatter(); private static final Pattern GROUPS_SEPARATOR_REGEX = Pattern.compile("\s*,\s*"); - private static int NUMBER_OF_UNSAVED_LIBRARIES = 1; private final BibDatabaseContext databaseContext; private final Connection connection; private final String libraryName; - private final String tableName; - private final String tableNameSplitValues; + private final String mainTable; + private final String schemaMainTableReference; + private final String splitValuesTable; + private final String schemaSplitValuesTableReference; private final Character keywordSeparator; - public PostgreIndexer(BibEntryPreferences bibEntryPreferences, BibDatabaseContext databaseContext, Connection connection) { + public BibFieldsIndexer(BibEntryPreferences bibEntryPreferences, BibDatabaseContext databaseContext, Connection connection) { this.databaseContext = databaseContext; this.connection = connection; this.keywordSeparator = bibEntryPreferences.getKeywordSeparator(); this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElse("unsaved"); - if ("unsaved".equals(databaseContext.getPostgreTableName())) { - this.tableName = "unsaved" + NUMBER_OF_UNSAVED_LIBRARIES++; - } else { - this.tableName = databaseContext.getPostgreTableName(); - } - tableNameSplitValues = tableName + PostgreConstants.TABLE_NAME_SUFFIX; + + this.mainTable = databaseContext.getUniqueName(); + this.splitValuesTable = mainTable + TABLE_NAME_SUFFIX; + + this.schemaMainTableReference = """ + "%s"."%s" + """.formatted(PostgreConstants.BIB_FIELDS_SCHEME, mainTable); + this.schemaSplitValuesTableReference = """ + "%s"."%s" + """.formatted(PostgreConstants.BIB_FIELDS_SCHEME, splitValuesTable); + // TODO: Set-up should be in a background task setup(); } - public String getTableName() { - return tableName; + public String getMainTable() { + return mainTable; } /** @@ -64,7 +76,7 @@ public String getTableName() { private void setup() { try { connection.createStatement().executeUpdate(""" - CREATE TABLE IF NOT EXISTS "%s" ( + CREATE TABLE IF NOT EXISTS %s ( %s TEXT NOT NULL, %s TEXT NOT NULL, %s TEXT, @@ -72,27 +84,26 @@ private void setup() { PRIMARY KEY (%s, %s) ) """.formatted( - tableName, - PostgreConstants.ENTRY_ID, - PostgreConstants.FIELD_NAME, - PostgreConstants.FIELD_VALUE_LITERAL, - PostgreConstants.FIELD_VALUE_TRANSFORMED, - PostgreConstants.ENTRY_ID, - PostgreConstants.FIELD_NAME)); + schemaMainTableReference, + ENTRY_ID, + FIELD_NAME, + FIELD_VALUE_LITERAL, + FIELD_VALUE_TRANSFORMED, + ENTRY_ID, FIELD_NAME)); connection.createStatement().executeUpdate(""" - CREATE TABLE IF NOT EXISTS "%s" ( + CREATE TABLE IF NOT EXISTS %s ( %s TEXT NOT NULL, %s TEXT NOT NULL, %s TEXT, %s TEXT ) """.formatted( - tableNameSplitValues, - PostgreConstants.ENTRY_ID, - PostgreConstants.FIELD_NAME, - PostgreConstants.FIELD_VALUE_LITERAL, - PostgreConstants.FIELD_VALUE_TRANSFORMED)); + schemaSplitValuesTableReference, + ENTRY_ID, + FIELD_NAME, + FIELD_VALUE_LITERAL, + FIELD_VALUE_TRANSFORMED)); LOGGER.debug("Created tables for library: {}", libraryName); } catch (SQLException e) { @@ -101,66 +112,66 @@ PRIMARY KEY (%s, %s) try { // region btree index on id column connection.createStatement().executeUpdate(""" - CREATE INDEX "%s" ON "%s" ("%s") + CREATE INDEX "%s_%s_index" ON %s ("%s") """.formatted( - PostgreConstants.ENTRY_ID.getIndexName(tableName), - tableName, - PostgreConstants.ENTRY_ID)); + mainTable, ENTRY_ID, + schemaMainTableReference, + ENTRY_ID)); connection.createStatement().executeUpdate(""" - CREATE INDEX "%s" ON "%s" ("%s") + CREATE INDEX "%s_%s_index" ON %s ("%s") """.formatted( - PostgreConstants.ENTRY_ID.getIndexName(tableNameSplitValues), - tableName, - PostgreConstants.ENTRY_ID)); + splitValuesTable, ENTRY_ID, + schemaSplitValuesTableReference, + ENTRY_ID)); // endregion // region btree index on field name column connection.createStatement().executeUpdate(""" - CREATE INDEX "%s" ON "%s" ("%s") + CREATE INDEX "%s_%s_index" ON %s ("%s") """.formatted( - PostgreConstants.FIELD_NAME.getIndexName(tableName), - tableName, - PostgreConstants.FIELD_NAME)); + mainTable, FIELD_NAME, + schemaMainTableReference, + FIELD_NAME)); connection.createStatement().executeUpdate(""" - CREATE INDEX "%s" ON "%s" ("%s") + CREATE INDEX "%s_%s_index" ON %s ("%s") """.formatted( - PostgreConstants.FIELD_NAME.getIndexName(tableNameSplitValues), - tableName, - PostgreConstants.FIELD_NAME)); + splitValuesTable, FIELD_NAME, + schemaSplitValuesTableReference, + FIELD_NAME)); // endregion // trigram index on field value column connection.createStatement().executeUpdate(""" - CREATE INDEX "%s" ON "%s" USING gin ("%s" gin_trgm_ops) + CREATE INDEX "%s_%s_index" ON %s USING gin ("%s" gin_trgm_ops) """.formatted( - PostgreConstants.FIELD_VALUE_LITERAL.getIndexName(tableName), - tableName, - PostgreConstants.FIELD_VALUE_LITERAL)); + mainTable, FIELD_VALUE_LITERAL, + schemaMainTableReference, + FIELD_VALUE_LITERAL)); // trigram index on field value transformed column connection.createStatement().executeUpdate(""" - CREATE INDEX "%s" ON "%s" USING gin ("%s" gin_trgm_ops) + CREATE INDEX "%s_%s_index" ON %s USING gin ("%s" gin_trgm_ops) """.formatted( - PostgreConstants.FIELD_VALUE_TRANSFORMED.getIndexName(tableName), - tableName, - PostgreConstants.FIELD_VALUE_TRANSFORMED)); + mainTable, FIELD_VALUE_TRANSFORMED, + schemaMainTableReference, + FIELD_VALUE_TRANSFORMED)); // region btree index on spilt values column // connection.createStatement().executeUpdate(""" // CREATE INDEX "%s" ON "%s" ("%s") // """.formatted( -// PostgreConstants.FIELD_VALUE_LITERAL.getIndexName(tableNameSplitValues), +// FIELD_VALUE_LITERAL.getIndexName(tableNameSplitValues), // tableName, -// PostgreConstants.FIELD_VALUE_LITERAL)); +// FIELD_VALUE_LITERAL)); // // connection.createStatement().executeUpdate(""" // CREATE INDEX "%s" ON "%s" ("%s") // """.formatted( -// PostgreConstants.FIELD_VALUE_TRANSFORMED.getIndexName(tableNameSplitValues), +// FIELD_VALUE_TRANSFORMED.getIndexName(tableNameSplitValues), // tableName, -// PostgreConstants.FIELD_VALUE_TRANSFORMED)); +// FIELD_VALUE_TRANSFORMED)); // endregion LOGGER.debug("Created indexes for library: {}", libraryName); @@ -196,24 +207,24 @@ public void addToIndex(Collection entries, BackgroundTask task) { private void addToIndex(BibEntry bibEntry) { String insertFieldQuery = """ - INSERT INTO "%s" ("%s", "%s", "%s", "%s") + INSERT INTO %s ("%s", "%s", "%s", "%s") VALUES (?, ?, ?, ?) """.formatted( - tableName, - PostgreConstants.ENTRY_ID, - PostgreConstants.FIELD_NAME, - PostgreConstants.FIELD_VALUE_LITERAL, - PostgreConstants.FIELD_VALUE_TRANSFORMED); + schemaMainTableReference, + ENTRY_ID, + FIELD_NAME, + FIELD_VALUE_LITERAL, + FIELD_VALUE_TRANSFORMED); String insertIntoSplitTable = """ - INSERT INTO "%s" ("%s", "%s", "%s", "%s") + INSERT INTO %s ("%s", "%s", "%s", "%s") VALUES (?, ?, ?, ?) """.formatted( - tableNameSplitValues, - PostgreConstants.ENTRY_ID, - PostgreConstants.FIELD_NAME, - PostgreConstants.FIELD_VALUE_LITERAL, - PostgreConstants.FIELD_VALUE_TRANSFORMED); + schemaSplitValuesTableReference, + ENTRY_ID, + FIELD_NAME, + FIELD_VALUE_LITERAL, + FIELD_VALUE_TRANSFORMED); try (PreparedStatement preparedStatement = connection.prepareStatement(insertFieldQuery); PreparedStatement preparedStatementSplitValues = connection.prepareStatement(insertIntoSplitTable)) { @@ -339,9 +350,9 @@ public void removeFromIndex(Collection entries, BackgroundTask task private void removeFromIndex(BibEntry entry) { try { connection.createStatement().executeUpdate(""" - DELETE FROM "%s" + DELETE FROM %s WHERE "%s" = '%s' - """.formatted(tableName, PostgreConstants.ENTRY_ID, entry.getId())); + """.formatted(schemaMainTableReference, ENTRY_ID, entry.getId())); LOGGER.debug("Entry {} removed from index", entry.getId()); } catch (SQLException e) { LOGGER.error("Error deleting entry from index", e); @@ -351,13 +362,13 @@ private void removeFromIndex(BibEntry entry) { public void updateEntry(BibEntry entry, Field field, String oldValue, String newValue) { try { String updateQuery = """ - INSERT INTO "%s" ("%s", "%s", "%s", "%s") + INSERT INTO %s ("%s", "%s", "%s", "%s") VALUES (?, ?, ?, ?) - """.formatted(tableName, - PostgreConstants.ENTRY_ID, - PostgreConstants.FIELD_NAME, - PostgreConstants.FIELD_VALUE_LITERAL, - PostgreConstants.FIELD_VALUE_TRANSFORMED); + """.formatted(schemaSplitValuesTableReference, + ENTRY_ID, + FIELD_NAME, + FIELD_VALUE_LITERAL, + FIELD_VALUE_TRANSFORMED); try (PreparedStatement preparedStatement = connection.prepareStatement(updateQuery)) { preparedStatement.setString(1, entry.getId()); @@ -383,8 +394,11 @@ private void closeIndex() { try { LOGGER.debug("Closing connection to Postgres server for library: {}", libraryName); connection.createStatement().executeUpdate(""" - DROP TABLE IF EXISTS "%s" - """.formatted(tableName)); + DROP TABLE IF EXISTS %s + """.formatted(schemaMainTableReference)); + connection.createStatement().executeUpdate(""" + DROP TABLE IF EXISTS %s + """.formatted(schemaSplitValuesTableReference)); connection.close(); } catch (SQLException e) { LOGGER.error("Could not drop table for library: {}", libraryName, e); diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index caa102f6feb..d2b04c37fb3 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.Optional; +import org.jabref.logic.search.indexing.BibFieldsIndexer; import org.jabref.model.entry.field.InternalField; import org.jabref.model.entry.field.StandardField; import org.jabref.search.SearchBaseVisitor; @@ -20,7 +21,7 @@ import static org.jabref.model.search.PostgreConstants.TABLE_NAME_SUFFIX; /** - * Converts to a query processable by the scheme created by {@link org.jabref.logic.search.indexing.PostgreIndexer}. + * Converts to a query processable by the scheme created by {@link BibFieldsIndexer}. * * @implNote Similar class: {@link org.jabref.migrations.SearchToLuceneMigration} */ diff --git a/src/main/java/org/jabref/logic/util/Directories.java b/src/main/java/org/jabref/logic/util/Directories.java index 00396975da7..c7eccc4b384 100644 --- a/src/main/java/org/jabref/logic/util/Directories.java +++ b/src/main/java/org/jabref/logic/util/Directories.java @@ -49,6 +49,13 @@ public static Path getFulltextIndexBaseDirectory() { OS.APP_DIR_APP_AUTHOR)); } + public static Path getPostgresDataDirectory() { + return Path.of(AppDirsFactory.getInstance() + .getUserDataDir(OS.APP_DIR_APP_NAME, + "postgres" + File.separator, + OS.APP_DIR_APP_AUTHOR)); + } + public static Path getAiFilesDirectory() { return Path.of(AppDirsFactory.getInstance() .getUserDataDir(OS.APP_DIR_APP_NAME, diff --git a/src/main/java/org/jabref/model/database/BibDatabaseContext.java b/src/main/java/org/jabref/model/database/BibDatabaseContext.java index bafbf49a2e5..f50029d9073 100644 --- a/src/main/java/org/jabref/model/database/BibDatabaseContext.java +++ b/src/main/java/org/jabref/model/database/BibDatabaseContext.java @@ -40,6 +40,7 @@ public class BibDatabaseContext { private static final Logger LOGGER = LoggerFactory.getLogger(BibDatabaseContext.class); + private static int NUMBER_OF_UNSAVED_LIBRARIES = 0; private final BibDatabase database; private MetaData metaData; @@ -274,12 +275,12 @@ public Path getFulltextIndexPath() { return indexPath; } - public String getPostgreTableName() { + public String getUniqueName() { if (getDatabasePath().isPresent()) { Path databasePath = getDatabasePath().get(); return BackupFileUtil.getUniqueFilePrefix(databasePath) + "--" + databasePath.getFileName(); } - return "unsaved"; + return "unsaved" + NUMBER_OF_UNSAVED_LIBRARIES++; } @Override diff --git a/src/main/java/org/jabref/model/search/PostgreConstants.java b/src/main/java/org/jabref/model/search/PostgreConstants.java index 4b6dbe7584f..c5c5ac34fc9 100644 --- a/src/main/java/org/jabref/model/search/PostgreConstants.java +++ b/src/main/java/org/jabref/model/search/PostgreConstants.java @@ -1,25 +1,21 @@ package org.jabref.model.search; public enum PostgreConstants { + BIB_FIELDS_SCHEME("bib_fields"), + LINKED_FILES_SCHEME("linked_files"), ENTRY_ID("entry_id"), FIELD_NAME("field_name"), FIELD_VALUE_LITERAL("field_value_literal"), // contains the value as-is - FIELD_VALUE_TRANSFORMED("field_value_transformed"); // contains the value transformed for better querying + FIELD_VALUE_TRANSFORMED("field_value_transformed"), // contains the value transformed for better querying + TABLE_NAME_SUFFIX("_split_values"); + private final String value; - public static final String TABLE_NAME_SUFFIX = "_split_values"; - - private final String columnName; - - PostgreConstants(String columnName) { - this.columnName = columnName; - } - - public String getIndexName(String tableName) { - return "%s_%s_index".formatted(tableName, this.columnName); + PostgreConstants(String value) { + this.value = value; } @Override public String toString() { - return columnName; + return value; } } From a0cc53991dd566daeea54024d1a82d0cf8bb0d77 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 2 Oct 2024 19:10:12 +0300 Subject: [PATCH 043/104] Prepare linked files tables --- .../org/jabref/logic/search/IndexManager.java | 5 +- .../search/indexing/BibFieldsIndexer.java | 6 +- .../indexing/PostgresLinkedFilesIndexer.java | 97 +++++++++++++++++++ .../search/query/SearchToSqlVisitor.java | 4 +- .../jabref/model/search/PostgreConstants.java | 12 ++- 5 files changed, 116 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/jabref/logic/search/indexing/PostgresLinkedFilesIndexer.java diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index 9e3504bb36e..835325d79a4 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -8,8 +8,9 @@ import javafx.beans.value.ChangeListener; import org.jabref.logic.preferences.CliPreferences; -import org.jabref.logic.search.indexing.DefaultLinkedFilesIndexer; import org.jabref.logic.search.indexing.BibFieldsIndexer; +import org.jabref.logic.search.indexing.DefaultLinkedFilesIndexer; +import org.jabref.logic.search.indexing.PostgresLinkedFilesIndexer; import org.jabref.logic.search.indexing.ReadOnlyLinkedFilesIndexer; import org.jabref.logic.search.retrieval.BibFieldsSearcher; import org.jabref.logic.search.retrieval.LinkedFilesSearcher; @@ -42,6 +43,7 @@ public class IndexManager { private final BibFieldsIndexer bibFieldsIndexer; private final LuceneIndexer linkedFilesIndexer; private final BibFieldsSearcher bibFieldsSearcher; + private final PostgresLinkedFilesIndexer postgresLinkedFilesIndexer; private final LinkedFilesSearcher linkedFilesSearcher; public IndexManager(BibDatabaseContext databaseContext, TaskExecutor executor, CliPreferences preferences) { @@ -53,6 +55,7 @@ public IndexManager(BibDatabaseContext databaseContext, TaskExecutor executor, C PostgreServer postgreServer = Injector.instantiateModelOrService(PostgreServer.class); bibFieldsIndexer = new BibFieldsIndexer(preferences.getBibEntryPreferences(), databaseContext, postgreServer.getConnection()); + postgresLinkedFilesIndexer = new PostgresLinkedFilesIndexer(databaseContext, preferences.getFilePreferences(), postgreServer.getConnection()); LuceneIndexer indexer; try { diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index 9dab45aff5a..ddb51ecca70 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -31,7 +31,7 @@ import static org.jabref.model.search.PostgreConstants.FIELD_NAME; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_LITERAL; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_TRANSFORMED; -import static org.jabref.model.search.PostgreConstants.TABLE_NAME_SUFFIX; +import static org.jabref.model.search.PostgreConstants.SPLIT_TABLE_SUFFIX; public class BibFieldsIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(BibFieldsIndexer.class); @@ -54,7 +54,7 @@ public BibFieldsIndexer(BibEntryPreferences bibEntryPreferences, BibDatabaseCont this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElse("unsaved"); this.mainTable = databaseContext.getUniqueName(); - this.splitValuesTable = mainTable + TABLE_NAME_SUFFIX; + this.splitValuesTable = mainTable + SPLIT_TABLE_SUFFIX; this.schemaMainTableReference = """ "%s"."%s" @@ -97,7 +97,7 @@ PRIMARY KEY (%s, %s) %s TEXT NOT NULL, %s TEXT, %s TEXT - ) + ) """.formatted( schemaSplitValuesTableReference, ENTRY_ID, diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgresLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgresLinkedFilesIndexer.java new file mode 100644 index 00000000000..8c81c3a983c --- /dev/null +++ b/src/main/java/org/jabref/logic/search/indexing/PostgresLinkedFilesIndexer.java @@ -0,0 +1,97 @@ +package org.jabref.logic.search.indexing; + +import java.sql.Connection; +import java.sql.SQLException; + +import org.jabref.logic.FilePreferences; +import org.jabref.model.database.BibDatabaseContext; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.jabref.model.search.PostgreConstants.CONTENT_TABLE_SUFFIX; +import static org.jabref.model.search.PostgreConstants.FILE_MODIFIED; +import static org.jabref.model.search.PostgreConstants.FILE_PATH; +import static org.jabref.model.search.PostgreConstants.LINKED_ENTRIES; +import static org.jabref.model.search.PostgreConstants.LINKED_FILES_SCHEME; +import static org.jabref.model.search.PostgreConstants.PAGE_ANNOTATIONS; +import static org.jabref.model.search.PostgreConstants.PAGE_CONTENT; +import static org.jabref.model.search.PostgreConstants.PAGE_NUMBER; + +public class PostgresLinkedFilesIndexer { + private static final Logger LOGGER = LoggerFactory.getLogger(PostgresLinkedFilesIndexer.class); + + private final BibDatabaseContext databaseContext; + private final FilePreferences filePreferences; + private final Connection connection; + private final String libraryName; + private final String mainTable; + private final String contentTable; + private final String schemaMainTableReference; + private final String schemaContentTableReference; + + public PostgresLinkedFilesIndexer(BibDatabaseContext databaseContext, FilePreferences filePreferences, Connection connection) { + this.databaseContext = databaseContext; + this.filePreferences = filePreferences; + this.connection = connection; + this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElse("unsaved"); + this.mainTable = databaseContext.getUniqueName(); + this.contentTable = mainTable + CONTENT_TABLE_SUFFIX; + this.schemaMainTableReference = """ + "%s"."%s" + """.formatted(LINKED_FILES_SCHEME, mainTable); + this.schemaContentTableReference = """ + "%s"."%s" + """.formatted(LINKED_FILES_SCHEME, contentTable); + setup(); + } + + private void setup() { + try { + connection.createStatement().executeUpdate(""" + CREATE TABLE IF NOT EXISTS %s ( + %s TEXT NOT NULL, + %s TEXT NOT NULL, + %s TEXT NOT NULL, + PRIMARY KEY (%s) + ) + """.formatted( + schemaMainTableReference, + FILE_PATH, + FILE_MODIFIED, + LINKED_ENTRIES, + FILE_PATH)); + + connection.createStatement().executeUpdate(""" + CREATE TABLE IF NOT EXISTS %s ( + %s TEXT NOT NULL REFERENCES %s(%s), + %s INT NOT NULL, + %s TEXT, + %s TEXT, + PRIMARY KEY (%s, %s) + ) + """.formatted( + schemaContentTableReference, + FILE_PATH, + schemaMainTableReference, FILE_PATH, + PAGE_NUMBER, + PAGE_CONTENT, + PAGE_ANNOTATIONS, + FILE_PATH, PAGE_NUMBER)); + + // full text index on page content, annotations + connection.createStatement().executeUpdate(""" + CREATE INDEX IF NOT EXISTS "%s_content_idx" + ON %s + USING GIN (to_tsvector('english', %s || ' ' || %s)) + """.formatted( + contentTable, + schemaContentTableReference, + PAGE_CONTENT, PAGE_ANNOTATIONS)); + + LOGGER.debug("Created full-text tables for library: {}", libraryName); + } catch (SQLException e) { + LOGGER.error("Could not create full-text tables for library: {}", libraryName, e); + } + } +} diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index d2b04c37fb3..aae9fc9d3d5 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -18,7 +18,7 @@ import static org.jabref.model.search.PostgreConstants.FIELD_NAME; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_LITERAL; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_TRANSFORMED; -import static org.jabref.model.search.PostgreConstants.TABLE_NAME_SUFFIX; +import static org.jabref.model.search.PostgreConstants.SPLIT_TABLE_SUFFIX; /** * Converts to a query processable by the scheme created by {@link BibFieldsIndexer}. @@ -40,7 +40,7 @@ public class SearchToSqlVisitor extends SearchBaseVisitor { public SearchToSqlVisitor(String mainTableName) { this.mainTableName = mainTableName; - this.splitValuesTableName = mainTableName + TABLE_NAME_SUFFIX; + this.splitValuesTableName = mainTableName + SPLIT_TABLE_SUFFIX; } private enum SearchTermFlag { diff --git a/src/main/java/org/jabref/model/search/PostgreConstants.java b/src/main/java/org/jabref/model/search/PostgreConstants.java index c5c5ac34fc9..ce002e6c9d4 100644 --- a/src/main/java/org/jabref/model/search/PostgreConstants.java +++ b/src/main/java/org/jabref/model/search/PostgreConstants.java @@ -2,12 +2,20 @@ public enum PostgreConstants { BIB_FIELDS_SCHEME("bib_fields"), - LINKED_FILES_SCHEME("linked_files"), + SPLIT_TABLE_SUFFIX("_split_values"), ENTRY_ID("entry_id"), FIELD_NAME("field_name"), FIELD_VALUE_LITERAL("field_value_literal"), // contains the value as-is FIELD_VALUE_TRANSFORMED("field_value_transformed"), // contains the value transformed for better querying - TABLE_NAME_SUFFIX("_split_values"); + LINKED_FILES_SCHEME("linked_files"), + CONTENT_TABLE_SUFFIX("_file_content"), + FILE_PATH("path"), + FILE_MODIFIED("modified"), + LINKED_ENTRIES("entries"), + PAGE_NUMBER("pageNumber"), + PAGE_CONTENT("content"), + PAGE_ANNOTATIONS("annotations"); + private final String value; PostgreConstants(String value) { From 27a0143d1b6f5291817fed3518069d5f213d2e82 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 2 Oct 2024 20:57:18 +0300 Subject: [PATCH 044/104] Fix searching --- .../org/jabref/logic/search/IndexManager.java | 2 +- .../search/indexing/BibFieldsIndexer.java | 16 +++---- .../search/query/SearchToSqlVisitor.java | 42 +++++++++---------- .../jabref/model/search/PostgreConstants.java | 8 ++++ .../model/search/query/SearchQuery.java | 4 +- 5 files changed, 38 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index 835325d79a4..48f471452ad 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -232,7 +232,7 @@ public SearchResults search(SearchQuery query) { if (query.getSearchFlags().contains(SearchFlags.FULLTEXT)) { // TODO: merge results from lucene and postgres } else { - query.setSearchResults(bibFieldsSearcher.search(query.getSqlQuery(bibFieldsIndexer.getMainTable()))); + query.setSearchResults(bibFieldsSearcher.search(query.getSqlQuery(bibFieldsIndexer.getTable()))); } return query.getSearchResults(); } diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index ddb51ecca70..87eeb7f6236 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -56,20 +56,12 @@ public BibFieldsIndexer(BibEntryPreferences bibEntryPreferences, BibDatabaseCont this.mainTable = databaseContext.getUniqueName(); this.splitValuesTable = mainTable + SPLIT_TABLE_SUFFIX; - this.schemaMainTableReference = """ - "%s"."%s" - """.formatted(PostgreConstants.BIB_FIELDS_SCHEME, mainTable); - this.schemaSplitValuesTableReference = """ - "%s"."%s" - """.formatted(PostgreConstants.BIB_FIELDS_SCHEME, splitValuesTable); + this.schemaMainTableReference = PostgreConstants.getMainTableSchemaReference(mainTable); + this.schemaSplitValuesTableReference = PostgreConstants.getSplitTableSchemaReference(mainTable); // TODO: Set-up should be in a background task setup(); } - public String getMainTable() { - return mainTable; - } - /** * Creates a table for the library in the database, and sets up indexes on the columns. */ @@ -382,6 +374,10 @@ public void updateEntry(BibEntry entry, Field field, String oldValue, String new } } + public String getTable() { + return mainTable; + } + public void close() { HeadlessExecutorService.INSTANCE.execute(this::closeIndex); } diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index aae9fc9d3d5..963fe5cdc22 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -8,6 +8,7 @@ import org.jabref.logic.search.indexing.BibFieldsIndexer; import org.jabref.model.entry.field.InternalField; import org.jabref.model.entry.field.StandardField; +import org.jabref.model.search.PostgreConstants; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; @@ -18,7 +19,6 @@ import static org.jabref.model.search.PostgreConstants.FIELD_NAME; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_LITERAL; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_TRANSFORMED; -import static org.jabref.model.search.PostgreConstants.SPLIT_TABLE_SUFFIX; /** * Converts to a query processable by the scheme created by {@link BibFieldsIndexer}. @@ -38,9 +38,9 @@ public class SearchToSqlVisitor extends SearchBaseVisitor { private final List ctes = new ArrayList<>(); private int cteCounter = 0; - public SearchToSqlVisitor(String mainTableName) { - this.mainTableName = mainTableName; - this.splitValuesTableName = mainTableName + SPLIT_TABLE_SUFFIX; + public SearchToSqlVisitor(String table) { + this.mainTableName = PostgreConstants.getMainTableSchemaReference(table); + this.splitValuesTableName = PostgreConstants.getSplitTableSchemaReference(table); } private enum SearchTermFlag { @@ -73,7 +73,7 @@ public String visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { String cte = """ cte%d AS ( SELECT %s.%s - FROM "%s" AS %s + FROM %s AS %s WHERE %s.%s NOT IN ( SELECT %s FROM %s @@ -223,7 +223,7 @@ private String buildContainsFieldQuery(String field, String operator, String pre return """ cte%d AS ( SELECT %s.%s - FROM "%s" AS %s + FROM %s AS %s WHERE (%s.%s = '%s') AND ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) ) """.formatted( @@ -244,10 +244,10 @@ private String buildContainsNegationFieldQuery(String field, String operator, St return """ cte%d AS ( SELECT %s.%s - FROM "%s" AS %s + FROM %s AS %s WHERE %s.%s NOT IN ( SELECT %s.%s - FROM "%s" AS %s + FROM %s AS %s WHERE (%s.%s = '%s') AND ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) ) ) @@ -272,8 +272,8 @@ private String buildExactFieldQuery(String field, String operator, String term) return """ cte%d AS ( SELECT %s.%s - FROM "%s" AS %s - LEFT JOIN "%s" AS %s + FROM %s AS %s + LEFT JOIN %s AS %s ON (%s.%s = %s.%s AND %s.%s = %s.%s) WHERE ( (%s.%s = '%s') AND ((%s.%s %s '%s') OR (%s.%s %s '%s')) @@ -300,11 +300,11 @@ private String buildExactNegationFieldQuery(String field, String operator, Strin return """ cte%d AS ( SELECT %s.%s - FROM "%s" AS %s + FROM %s AS %s WHERE %s.%s NOT IN ( SELECT %s.%s - FROM "%s" AS %s - LEFT JOIN "%s" AS %s + FROM %s AS %s + LEFT JOIN %s AS %s ON (%s.%s = %s.%s AND %s.%s = %s.%s) WHERE ( (%s.%s = '%s') AND ((%s.%s %s '%s') OR (%s.%s %s '%s')) @@ -335,7 +335,7 @@ private String buildContainsAnyFieldQuery(String operator, String prefixSuffix, return """ cte%d AS ( SELECT %s.%s - FROM "%s" AS %s + FROM %s AS %s WHERE ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) ) """.formatted( @@ -354,8 +354,8 @@ private String buildExactAnyFieldQuery(String operator, String term) { return """ cte%d AS ( SELECT %s.%s - FROM "%s" AS %s - LEFT JOIN "%s" AS %s + FROM %s AS %s + LEFT JOIN %s AS %s ON (%s.%s = %s.%s AND %s.%s = %s.%s) WHERE ( (%s.%s %s '%s') OR (%s.%s %s '%s') @@ -380,11 +380,11 @@ private String buildExactNegationAnyFieldQuery(String operator, String term) { return """ cte%d AS ( SELECT %s.%s - FROM "%s" AS %s + FROM %s AS %s WHERE %s.%s NOT IN ( SELECT %s.%s - FROM "%s" AS %s - LEFT JOIN "%s" AS %s + FROM %s AS %s + LEFT JOIN %s AS %s ON (%s.%s = %s.%s AND %s.%s = %s.%s) WHERE ( (%s.%s %s '%s') OR (%s.%s %s '%s') @@ -413,10 +413,10 @@ private String buildContainsNegationAnyFieldQuery(String operator, String prefix return """ cte%d AS ( SELECT %s.%s - FROM "%s" AS %s + FROM %s AS %s WHERE %s.%s NOT IN ( SELECT %s.%s - FROM "%s" AS %s + FROM %s AS %s WHERE ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) ) ) diff --git a/src/main/java/org/jabref/model/search/PostgreConstants.java b/src/main/java/org/jabref/model/search/PostgreConstants.java index ce002e6c9d4..df6989c172e 100644 --- a/src/main/java/org/jabref/model/search/PostgreConstants.java +++ b/src/main/java/org/jabref/model/search/PostgreConstants.java @@ -22,6 +22,14 @@ public enum PostgreConstants { this.value = value; } + public static String getMainTableSchemaReference(String mainTable) { + return BIB_FIELDS_SCHEME + ".\"" + mainTable + "\""; + } + + public static String getSplitTableSchemaReference(String mainTable) { + return BIB_FIELDS_SCHEME + ".\"" + mainTable + SPLIT_TABLE_SUFFIX + "\""; + } + @Override public String toString() { return value; diff --git a/src/main/java/org/jabref/model/search/query/SearchQuery.java b/src/main/java/org/jabref/model/search/query/SearchQuery.java index 8a0821b26c7..aef9d2ab3e6 100644 --- a/src/main/java/org/jabref/model/search/query/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/query/SearchQuery.java @@ -101,8 +101,8 @@ public SearchQuery(String query, EnumSet searchFlags) { } } - public SqlSearchQuery getSqlQuery(String tableName) { - return new SqlSearchQuery(SearchToSqlConversion.searchToSql(tableName, query)); + public SqlSearchQuery getSqlQuery(String table) { + return new SqlSearchQuery(SearchToSqlConversion.searchToSql(table, query)); } public String getSearchExpression() { From 1c951fb03fa90b5cdc6c52cb2c80472ccb397996 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 2 Oct 2024 21:10:09 +0300 Subject: [PATCH 045/104] Use multi column index --- .../search/indexing/BibFieldsIndexer.java | 29 +++++-------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index 87eeb7f6236..970c53a5c8c 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -136,34 +136,19 @@ PRIMARY KEY (%s, %s) // trigram index on field value column connection.createStatement().executeUpdate(""" - CREATE INDEX "%s_%s_index" ON %s USING gin ("%s" gin_trgm_ops) + CREATE INDEX "%s_%s_index" ON %s USING gin ("%s" gin_trgm_ops, "%s" gin_trgm_ops) """.formatted( mainTable, FIELD_VALUE_LITERAL, schemaMainTableReference, - FIELD_VALUE_LITERAL)); + FIELD_VALUE_LITERAL, FIELD_VALUE_TRANSFORMED)); - // trigram index on field value transformed column + // region btree index on spilt table connection.createStatement().executeUpdate(""" - CREATE INDEX "%s_%s_index" ON %s USING gin ("%s" gin_trgm_ops) + CREATE INDEX "%s_%s_index" ON %s ("%s", "%s") """.formatted( - mainTable, FIELD_VALUE_TRANSFORMED, - schemaMainTableReference, - FIELD_VALUE_TRANSFORMED)); - - // region btree index on spilt values column -// connection.createStatement().executeUpdate(""" -// CREATE INDEX "%s" ON "%s" ("%s") -// """.formatted( -// FIELD_VALUE_LITERAL.getIndexName(tableNameSplitValues), -// tableName, -// FIELD_VALUE_LITERAL)); -// -// connection.createStatement().executeUpdate(""" -// CREATE INDEX "%s" ON "%s" ("%s") -// """.formatted( -// FIELD_VALUE_TRANSFORMED.getIndexName(tableNameSplitValues), -// tableName, -// FIELD_VALUE_TRANSFORMED)); + splitValuesTable, FIELD_VALUE_LITERAL, + schemaSplitValuesTableReference, + FIELD_VALUE_LITERAL, FIELD_VALUE_TRANSFORMED)); // endregion LOGGER.debug("Created indexes for library: {}", libraryName); From 981f158d8c7ca62356efbe4807c733795fe3c13f Mon Sep 17 00:00:00 2001 From: Siedlerchr Date: Wed, 2 Oct 2024 21:09:14 +0200 Subject: [PATCH 046/104] fix merged module issues --- build.gradle | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.gradle b/build.gradle index 99804fafbec..61aa7a91b89 100644 --- a/build.gradle +++ b/build.gradle @@ -796,9 +796,13 @@ jlink { requires 'org.apache.commons.lang3' requires 'org.apache.commons.logging' requires 'org.apache.commons.text' + requires 'org.apache.commons.codec' + requires 'org.apache.commons.io' + requires 'org.apache.commons.compress' requires 'org.freedesktop.dbus' requires 'org.jsoup' requires 'org.slf4j' + requires 'org.tukaani.xz'; uses 'ai.djl.engine.EngineProvider' uses 'ai.djl.repository.RepositoryFactory' uses 'ai.djl.repository.zoo.ZooProvider' @@ -810,6 +814,7 @@ jlink { uses 'org.mariadb.jdbc.authentication.AuthenticationPlugin' uses 'org.mariadb.jdbc.credential.CredentialPlugin' uses 'org.mariadb.jdbc.tls.TlsSocketPlugin' + uses 'org.postgresql.shaded.com.ongres.stringprep.Profile' provides 'org.mariadb.jdbc.tls.TlsSocketPlugin' with 'org.mariadb.jdbc.internal.protocol.tls.DefaultTlsSocketPlugin' provides 'java.sql.Driver' with 'org.postgresql.Driver' From 29c179e868526ef5fc446b5a8350c3adbf0d25c1 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 4 Oct 2024 12:45:07 +0300 Subject: [PATCH 047/104] Fix update event --- .../org/jabref/logic/search/IndexManager.java | 9 -- .../search/indexing/BibFieldsIndexer.java | 128 +++++++++++++++--- 2 files changed, 112 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index 48f471452ad..e4be44fe4be 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -171,15 +171,6 @@ public Object call() { } public void updateAfterDropFiles(BibEntry entry) { - new BackgroundTask<>() { - @Override - public Object call() { -// bibFieldsIndexer.updateEntry(entry, StandardField.FILE); - return null; - } - }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(List.of(entry)))) - .executeWith(taskExecutor); - if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { new BackgroundTask<>() { @Override diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index 970c53a5c8c..085d9be776b 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -330,6 +330,10 @@ private void removeFromIndex(BibEntry entry) { DELETE FROM %s WHERE "%s" = '%s' """.formatted(schemaMainTableReference, ENTRY_ID, entry.getId())); + connection.createStatement().executeUpdate(""" + DELETE FROM %s + WHERE "%s" = '%s' + """.formatted(schemaSplitValuesTableReference, ENTRY_ID, entry.getId())); LOGGER.debug("Entry {} removed from index", entry.getId()); } catch (SQLException e) { LOGGER.error("Error deleting entry from index", e); @@ -337,25 +341,117 @@ private void removeFromIndex(BibEntry entry) { } public void updateEntry(BibEntry entry, Field field, String oldValue, String newValue) { + if (oldValue == null || oldValue.isEmpty()) { + insertField(entry, field, true); + } else if (newValue == null || newValue.isEmpty()) { + removeField(entry, field, true); + } else { + updateField(entry, field); + } + } + + private void insertField(BibEntry entry, Field field, boolean insertIntoMainTable) { + String insertFieldQuery = """ + INSERT INTO %s ("%s", "%s", "%s", "%s") + VALUES (?, ?, ?, ?) + """.formatted( + schemaMainTableReference, + ENTRY_ID, + FIELD_NAME, + FIELD_VALUE_LITERAL, + FIELD_VALUE_TRANSFORMED); + + if (insertIntoMainTable) { + try (PreparedStatement preparedStatement = connection.prepareStatement(insertFieldQuery)) { + String entryId = entry.getId(); + String value = entry.getField(field).orElse(""); + + Optional resolvedFieldLatexFree = entry.getResolvedFieldOrAliasLatexFree(field, this.databaseContext.getDatabase()); + assert resolvedFieldLatexFree.isPresent(); + addBatch(preparedStatement, entryId, field, value, resolvedFieldLatexFree.orElse("")); + preparedStatement.executeBatch(); + } catch (SQLException e) { + LOGGER.error("Could not add an entry to the index.", e); + } + } + + String insertIntoSplitTable = """ + INSERT INTO %s ("%s", "%s", "%s", "%s") + VALUES (?, ?, ?, ?) + """.formatted( + schemaSplitValuesTableReference, + ENTRY_ID, + FIELD_NAME, + FIELD_VALUE_LITERAL, + FIELD_VALUE_TRANSFORMED); + + try (PreparedStatement preparedStatement = connection.prepareStatement(insertIntoSplitTable)) { + String entryId = entry.getId(); + String value = entry.getField(field).orElse(""); + + if (field.getProperties().contains(FieldProperty.PERSON_NAMES)) { + addAuthors(value, preparedStatement, entryId, field); + } else if (field == StandardField.KEYWORDS) { + addKeywords(value, preparedStatement, entryId, field, keywordSeparator); + } else if (field == StandardField.GROUPS) { + addGroups(value, preparedStatement, entryId, field); + } else if (field.getProperties().contains(FieldProperty.MULTIPLE_ENTRY_LINK)) { + addEntryLinks(entry, field, preparedStatement, entryId); + } else if (field == StandardField.FILE) { + // No handling of File, because due to relative paths, we think, there won't be any exact match operation + } + preparedStatement.executeBatch(); + } catch (SQLException e) { + LOGGER.error("Could not add an entry to the index.", e); + } + } + + private void removeField(BibEntry entry, Field field, boolean removeFromMainTable) { try { - String updateQuery = """ - INSERT INTO %s ("%s", "%s", "%s", "%s") - VALUES (?, ?, ?, ?) - """.formatted(schemaSplitValuesTableReference, - ENTRY_ID, - FIELD_NAME, - FIELD_VALUE_LITERAL, - FIELD_VALUE_TRANSFORMED); - - try (PreparedStatement preparedStatement = connection.prepareStatement(updateQuery)) { - preparedStatement.setString(1, entry.getId()); - preparedStatement.setString(2, field.getName()); - preparedStatement.setString(3, entry.getField(field).orElse("")); - preparedStatement.executeUpdate(); - LOGGER.debug("Updated entry {} in index", entry.getId()); + if (removeFromMainTable) { + connection.createStatement().executeUpdate(""" + DELETE FROM %s + WHERE "%s" = '%s' AND "%s" = '%s' + """.formatted(schemaMainTableReference, ENTRY_ID, entry.getId(), FIELD_NAME, field.getName())); } + connection.createStatement().executeUpdate(""" + DELETE FROM %s + WHERE "%s" = '%s' AND "%s" = '%s' + """.formatted(schemaSplitValuesTableReference, ENTRY_ID, entry.getId(), FIELD_NAME, field.getName())); + LOGGER.debug("Field {} removed from entry {} in index", field.getName(), entry.getId()); + } catch (SQLException e) { + LOGGER.error("Error deleting field from entry in index", e); + } + } + + private void updateField(BibEntry entry, Field field) { + // remove from split table, and reinsert to it, and update in the main table + removeField(entry, field, false); + insertField(entry, field, false); + String updateFieldQuery = """ + UPDATE %s + SET "%s" = ?, "%s" = ? + WHERE "%s" = ? AND "%s" = ? + """.formatted( + schemaMainTableReference, + FIELD_VALUE_LITERAL, + FIELD_VALUE_TRANSFORMED, + ENTRY_ID, + FIELD_NAME); + + try (PreparedStatement preparedStatement = connection.prepareStatement(updateFieldQuery)) { + String entryId = entry.getId(); + String value = entry.getField(field).orElse(""); + + Optional resolvedFieldLatexFree = entry.getResolvedFieldOrAliasLatexFree(field, this.databaseContext.getDatabase()); + assert resolvedFieldLatexFree.isPresent(); + preparedStatement.setString(1, value); + preparedStatement.setString(2, resolvedFieldLatexFree.orElse("")); + preparedStatement.setString(3, entryId); + preparedStatement.setString(4, field.getName()); + preparedStatement.executeUpdate(); } catch (SQLException e) { - LOGGER.error("Error updating entry in index", e); + LOGGER.error("Could not update an entry in the index.", e); } } From e3d0628abd07f4d241ff6271eb81a8b592a9e040 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 4 Oct 2024 13:44:44 +0300 Subject: [PATCH 048/104] Remove postgres linked files indexer --- .../org/jabref/logic/search/IndexManager.java | 3 - .../jabref/logic/search/PostgreServer.java | 11 +-- .../indexing/PostgresLinkedFilesIndexer.java | 97 ------------------- .../org/jabref/logic/util/Directories.java | 7 -- .../jabref/model/search/PostgreConstants.java | 10 +- 5 files changed, 3 insertions(+), 125 deletions(-) delete mode 100644 src/main/java/org/jabref/logic/search/indexing/PostgresLinkedFilesIndexer.java diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index e4be44fe4be..718a9ee7185 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -10,7 +10,6 @@ import org.jabref.logic.preferences.CliPreferences; import org.jabref.logic.search.indexing.BibFieldsIndexer; import org.jabref.logic.search.indexing.DefaultLinkedFilesIndexer; -import org.jabref.logic.search.indexing.PostgresLinkedFilesIndexer; import org.jabref.logic.search.indexing.ReadOnlyLinkedFilesIndexer; import org.jabref.logic.search.retrieval.BibFieldsSearcher; import org.jabref.logic.search.retrieval.LinkedFilesSearcher; @@ -43,7 +42,6 @@ public class IndexManager { private final BibFieldsIndexer bibFieldsIndexer; private final LuceneIndexer linkedFilesIndexer; private final BibFieldsSearcher bibFieldsSearcher; - private final PostgresLinkedFilesIndexer postgresLinkedFilesIndexer; private final LinkedFilesSearcher linkedFilesSearcher; public IndexManager(BibDatabaseContext databaseContext, TaskExecutor executor, CliPreferences preferences) { @@ -55,7 +53,6 @@ public IndexManager(BibDatabaseContext databaseContext, TaskExecutor executor, C PostgreServer postgreServer = Injector.instantiateModelOrService(PostgreServer.class); bibFieldsIndexer = new BibFieldsIndexer(preferences.getBibEntryPreferences(), databaseContext, postgreServer.getConnection()); - postgresLinkedFilesIndexer = new PostgresLinkedFilesIndexer(databaseContext, preferences.getFilePreferences(), postgreServer.getConnection()); LuceneIndexer indexer; try { diff --git a/src/main/java/org/jabref/logic/search/PostgreServer.java b/src/main/java/org/jabref/logic/search/PostgreServer.java index 25488005fbe..4392cf173f2 100644 --- a/src/main/java/org/jabref/logic/search/PostgreServer.java +++ b/src/main/java/org/jabref/logic/search/PostgreServer.java @@ -6,14 +6,11 @@ import javax.sql.DataSource; -import org.jabref.logic.util.Directories; - import io.zonky.test.db.postgres.embedded.EmbeddedPostgres; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.jabref.model.search.PostgreConstants.BIB_FIELDS_SCHEME; -import static org.jabref.model.search.PostgreConstants.LINKED_FILES_SCHEME; public class PostgreServer { private static final Logger LOGGER = LoggerFactory.getLogger(PostgreServer.class); @@ -24,9 +21,6 @@ public PostgreServer() { EmbeddedPostgres embeddedPostgres; try { embeddedPostgres = EmbeddedPostgres.builder() - .setCleanDataDirectory(false) - .setDataDirectory(Directories.getPostgresDataDirectory()) - .setPort(10456) .start(); LOGGER.info("Postgres server started, connection port: {}", embeddedPostgres.getPort()); } catch (IOException e) { @@ -45,13 +39,12 @@ public PostgreServer() { private void createScheme() { try (Connection connection = getConnection()) { if (connection != null) { - LOGGER.debug("Creating scheme for bib fields and linked files"); + LOGGER.debug("Creating scheme for bib fields"); connection.createStatement().execute("DROP SCHEMA IF EXISTS " + BIB_FIELDS_SCHEME); connection.createStatement().execute("CREATE SCHEMA " + BIB_FIELDS_SCHEME); - connection.createStatement().execute("CREATE SCHEMA IF NOT EXISTS " + LINKED_FILES_SCHEME); } } catch (SQLException e) { - LOGGER.error("Could not create scheme for bib fields and linked files", e); + LOGGER.error("Could not create scheme for bib fields", e); } } diff --git a/src/main/java/org/jabref/logic/search/indexing/PostgresLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/PostgresLinkedFilesIndexer.java deleted file mode 100644 index 8c81c3a983c..00000000000 --- a/src/main/java/org/jabref/logic/search/indexing/PostgresLinkedFilesIndexer.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.jabref.logic.search.indexing; - -import java.sql.Connection; -import java.sql.SQLException; - -import org.jabref.logic.FilePreferences; -import org.jabref.model.database.BibDatabaseContext; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.jabref.model.search.PostgreConstants.CONTENT_TABLE_SUFFIX; -import static org.jabref.model.search.PostgreConstants.FILE_MODIFIED; -import static org.jabref.model.search.PostgreConstants.FILE_PATH; -import static org.jabref.model.search.PostgreConstants.LINKED_ENTRIES; -import static org.jabref.model.search.PostgreConstants.LINKED_FILES_SCHEME; -import static org.jabref.model.search.PostgreConstants.PAGE_ANNOTATIONS; -import static org.jabref.model.search.PostgreConstants.PAGE_CONTENT; -import static org.jabref.model.search.PostgreConstants.PAGE_NUMBER; - -public class PostgresLinkedFilesIndexer { - private static final Logger LOGGER = LoggerFactory.getLogger(PostgresLinkedFilesIndexer.class); - - private final BibDatabaseContext databaseContext; - private final FilePreferences filePreferences; - private final Connection connection; - private final String libraryName; - private final String mainTable; - private final String contentTable; - private final String schemaMainTableReference; - private final String schemaContentTableReference; - - public PostgresLinkedFilesIndexer(BibDatabaseContext databaseContext, FilePreferences filePreferences, Connection connection) { - this.databaseContext = databaseContext; - this.filePreferences = filePreferences; - this.connection = connection; - this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElse("unsaved"); - this.mainTable = databaseContext.getUniqueName(); - this.contentTable = mainTable + CONTENT_TABLE_SUFFIX; - this.schemaMainTableReference = """ - "%s"."%s" - """.formatted(LINKED_FILES_SCHEME, mainTable); - this.schemaContentTableReference = """ - "%s"."%s" - """.formatted(LINKED_FILES_SCHEME, contentTable); - setup(); - } - - private void setup() { - try { - connection.createStatement().executeUpdate(""" - CREATE TABLE IF NOT EXISTS %s ( - %s TEXT NOT NULL, - %s TEXT NOT NULL, - %s TEXT NOT NULL, - PRIMARY KEY (%s) - ) - """.formatted( - schemaMainTableReference, - FILE_PATH, - FILE_MODIFIED, - LINKED_ENTRIES, - FILE_PATH)); - - connection.createStatement().executeUpdate(""" - CREATE TABLE IF NOT EXISTS %s ( - %s TEXT NOT NULL REFERENCES %s(%s), - %s INT NOT NULL, - %s TEXT, - %s TEXT, - PRIMARY KEY (%s, %s) - ) - """.formatted( - schemaContentTableReference, - FILE_PATH, - schemaMainTableReference, FILE_PATH, - PAGE_NUMBER, - PAGE_CONTENT, - PAGE_ANNOTATIONS, - FILE_PATH, PAGE_NUMBER)); - - // full text index on page content, annotations - connection.createStatement().executeUpdate(""" - CREATE INDEX IF NOT EXISTS "%s_content_idx" - ON %s - USING GIN (to_tsvector('english', %s || ' ' || %s)) - """.formatted( - contentTable, - schemaContentTableReference, - PAGE_CONTENT, PAGE_ANNOTATIONS)); - - LOGGER.debug("Created full-text tables for library: {}", libraryName); - } catch (SQLException e) { - LOGGER.error("Could not create full-text tables for library: {}", libraryName, e); - } - } -} diff --git a/src/main/java/org/jabref/logic/util/Directories.java b/src/main/java/org/jabref/logic/util/Directories.java index c7eccc4b384..00396975da7 100644 --- a/src/main/java/org/jabref/logic/util/Directories.java +++ b/src/main/java/org/jabref/logic/util/Directories.java @@ -49,13 +49,6 @@ public static Path getFulltextIndexBaseDirectory() { OS.APP_DIR_APP_AUTHOR)); } - public static Path getPostgresDataDirectory() { - return Path.of(AppDirsFactory.getInstance() - .getUserDataDir(OS.APP_DIR_APP_NAME, - "postgres" + File.separator, - OS.APP_DIR_APP_AUTHOR)); - } - public static Path getAiFilesDirectory() { return Path.of(AppDirsFactory.getInstance() .getUserDataDir(OS.APP_DIR_APP_NAME, diff --git a/src/main/java/org/jabref/model/search/PostgreConstants.java b/src/main/java/org/jabref/model/search/PostgreConstants.java index df6989c172e..b5653219b42 100644 --- a/src/main/java/org/jabref/model/search/PostgreConstants.java +++ b/src/main/java/org/jabref/model/search/PostgreConstants.java @@ -6,15 +6,7 @@ public enum PostgreConstants { ENTRY_ID("entry_id"), FIELD_NAME("field_name"), FIELD_VALUE_LITERAL("field_value_literal"), // contains the value as-is - FIELD_VALUE_TRANSFORMED("field_value_transformed"), // contains the value transformed for better querying - LINKED_FILES_SCHEME("linked_files"), - CONTENT_TABLE_SUFFIX("_file_content"), - FILE_PATH("path"), - FILE_MODIFIED("modified"), - LINKED_ENTRIES("entries"), - PAGE_NUMBER("pageNumber"), - PAGE_CONTENT("content"), - PAGE_ANNOTATIONS("annotations"); + FIELD_VALUE_TRANSFORMED("field_value_transformed"); // contains the value transformed for better querying private final String value; From fb6f0135cf91ab24d570d0144e0346f9375f04ac Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 4 Oct 2024 13:49:01 +0300 Subject: [PATCH 049/104] Remove and insert field on update event --- .../org/jabref/logic/search/IndexManager.java | 11 ++- .../search/indexing/BibFieldsIndexer.java | 76 +++++-------------- 2 files changed, 28 insertions(+), 59 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index 718a9ee7185..0108ed395cb 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -150,7 +150,7 @@ public void updateEntry(FieldChangedEvent event) { new BackgroundTask<>() { @Override public Object call() { - bibFieldsIndexer.updateEntry(event.getBibEntry(), event.getField(), event.getOldValue(), event.getNewValue()); + bibFieldsIndexer.updateEntry(event.getBibEntry(), event.getField()); return null; } }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(List.of(event.getBibEntry())))) @@ -168,6 +168,15 @@ public Object call() { } public void updateAfterDropFiles(BibEntry entry) { + new BackgroundTask<>() { + @Override + public Object call() { + bibFieldsIndexer.updateEntry(entry, StandardField.FILE); + return null; + } + }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(List.of(entry)))) + .executeWith(taskExecutor); + if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { new BackgroundTask<>() { @Override diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index 085d9be776b..9ce3e8d566a 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -340,17 +340,12 @@ private void removeFromIndex(BibEntry entry) { } } - public void updateEntry(BibEntry entry, Field field, String oldValue, String newValue) { - if (oldValue == null || oldValue.isEmpty()) { - insertField(entry, field, true); - } else if (newValue == null || newValue.isEmpty()) { - removeField(entry, field, true); - } else { - updateField(entry, field); - } + public void updateEntry(BibEntry entry, Field field) { + removeField(entry, field); + insertField(entry, field); } - private void insertField(BibEntry entry, Field field, boolean insertIntoMainTable) { + private void insertField(BibEntry entry, Field field) { String insertFieldQuery = """ INSERT INTO %s ("%s", "%s", "%s", "%s") VALUES (?, ?, ?, ?) @@ -361,18 +356,16 @@ private void insertField(BibEntry entry, Field field, boolean insertIntoMainTabl FIELD_VALUE_LITERAL, FIELD_VALUE_TRANSFORMED); - if (insertIntoMainTable) { - try (PreparedStatement preparedStatement = connection.prepareStatement(insertFieldQuery)) { - String entryId = entry.getId(); - String value = entry.getField(field).orElse(""); + try (PreparedStatement preparedStatement = connection.prepareStatement(insertFieldQuery)) { + String entryId = entry.getId(); + String value = entry.getField(field).orElse(""); - Optional resolvedFieldLatexFree = entry.getResolvedFieldOrAliasLatexFree(field, this.databaseContext.getDatabase()); - assert resolvedFieldLatexFree.isPresent(); - addBatch(preparedStatement, entryId, field, value, resolvedFieldLatexFree.orElse("")); - preparedStatement.executeBatch(); - } catch (SQLException e) { - LOGGER.error("Could not add an entry to the index.", e); - } + Optional resolvedFieldLatexFree = entry.getResolvedFieldOrAliasLatexFree(field, this.databaseContext.getDatabase()); + assert resolvedFieldLatexFree.isPresent(); + addBatch(preparedStatement, entryId, field, value, resolvedFieldLatexFree.orElse("")); + preparedStatement.executeBatch(); + } catch (SQLException e) { + LOGGER.error("Could not add an entry to the index.", e); } String insertIntoSplitTable = """ @@ -406,14 +399,12 @@ private void insertField(BibEntry entry, Field field, boolean insertIntoMainTabl } } - private void removeField(BibEntry entry, Field field, boolean removeFromMainTable) { + private void removeField(BibEntry entry, Field field) { try { - if (removeFromMainTable) { - connection.createStatement().executeUpdate(""" - DELETE FROM %s - WHERE "%s" = '%s' AND "%s" = '%s' - """.formatted(schemaMainTableReference, ENTRY_ID, entry.getId(), FIELD_NAME, field.getName())); - } + connection.createStatement().executeUpdate(""" + DELETE FROM %s + WHERE "%s" = '%s' AND "%s" = '%s' + """.formatted(schemaMainTableReference, ENTRY_ID, entry.getId(), FIELD_NAME, field.getName())); connection.createStatement().executeUpdate(""" DELETE FROM %s WHERE "%s" = '%s' AND "%s" = '%s' @@ -424,37 +415,6 @@ private void removeField(BibEntry entry, Field field, boolean removeFromMainTabl } } - private void updateField(BibEntry entry, Field field) { - // remove from split table, and reinsert to it, and update in the main table - removeField(entry, field, false); - insertField(entry, field, false); - String updateFieldQuery = """ - UPDATE %s - SET "%s" = ?, "%s" = ? - WHERE "%s" = ? AND "%s" = ? - """.formatted( - schemaMainTableReference, - FIELD_VALUE_LITERAL, - FIELD_VALUE_TRANSFORMED, - ENTRY_ID, - FIELD_NAME); - - try (PreparedStatement preparedStatement = connection.prepareStatement(updateFieldQuery)) { - String entryId = entry.getId(); - String value = entry.getField(field).orElse(""); - - Optional resolvedFieldLatexFree = entry.getResolvedFieldOrAliasLatexFree(field, this.databaseContext.getDatabase()); - assert resolvedFieldLatexFree.isPresent(); - preparedStatement.setString(1, value); - preparedStatement.setString(2, resolvedFieldLatexFree.orElse("")); - preparedStatement.setString(3, entryId); - preparedStatement.setString(4, field.getName()); - preparedStatement.executeUpdate(); - } catch (SQLException e) { - LOGGER.error("Could not update an entry in the index.", e); - } - } - public String getTable() { return mainTable; } From 65d76810f294567acc509f2a32cb58a7445f49ee Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 4 Oct 2024 14:07:54 +0300 Subject: [PATCH 050/104] Remove search score column --- .../gui/maintable/BibEntryTableViewModel.java | 7 ------- .../gui/maintable/MainTableColumnFactory.java | 19 ------------------- .../gui/maintable/MainTableColumnModel.java | 3 +-- .../gui/maintable/MainTableDataModel.java | 12 ++++-------- .../preferences/table/TableTabViewModel.java | 1 - .../search/SearchResultsTableDataModel.java | 4 +--- .../search/retrieval/BibFieldsSearcher.java | 2 +- .../search/retrieval/LinkedFilesSearcher.java | 1 - .../model/search/query/SearchResult.java | 17 +++++------------ .../model/search/query/SearchResults.java | 10 ++-------- 10 files changed, 14 insertions(+), 62 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java index aa8fdc38ec3..613a102d9df 100644 --- a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java +++ b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java @@ -14,11 +14,9 @@ import javafx.beans.binding.Binding; import javafx.beans.binding.Bindings; import javafx.beans.property.BooleanProperty; -import javafx.beans.property.FloatProperty; import javafx.beans.property.ObjectProperty; import javafx.beans.property.ReadOnlyStringWrapper; import javafx.beans.property.SimpleBooleanProperty; -import javafx.beans.property.SimpleFloatProperty; import javafx.beans.property.SimpleObjectProperty; import javafx.beans.property.StringProperty; import javafx.beans.value.ObservableValue; @@ -50,7 +48,6 @@ public class BibEntryTableViewModel { private final EasyBinding> linkedIdentifiers; private final Binding> matchedGroups; private final BibDatabaseContext bibDatabaseContext; - private final FloatProperty searchScore = new SimpleFloatProperty(0); private final BooleanProperty hasFullTextResults = new SimpleBooleanProperty(false); private final BooleanProperty isMatchedBySearch = new SimpleBooleanProperty(true); private final BooleanProperty isVisibleBySearch = new SimpleBooleanProperty(true); @@ -163,10 +160,6 @@ public BibDatabaseContext getBibDatabaseContext() { return bibDatabaseContext; } - public FloatProperty searchScoreProperty() { - return searchScore; - } - public BooleanProperty hasFullTextResultsProperty() { return hasFullTextResults; } diff --git a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java index bd592f6b22c..d4d9b8fb576 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java @@ -33,7 +33,6 @@ import org.jabref.gui.maintable.columns.SpecialFieldColumn; import org.jabref.gui.preferences.GuiPreferences; import org.jabref.gui.search.MatchCategory; -import org.jabref.gui.search.SearchType; import org.jabref.gui.specialfields.SpecialFieldValueViewModel; import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.ValueTableCellFactory; @@ -87,9 +86,6 @@ public MainTableColumnFactory(BibDatabaseContext database, public TableColumn createColumn(MainTableColumnModel column) { TableColumn returnColumn = null; switch (column.getType()) { - case MATCH_SCORE: - returnColumn = createScoreColumn(column); - break; case INDEX: returnColumn = createIndexColumn(column); break; @@ -163,21 +159,6 @@ private TableColumn createMatchCategoryCo return column; } - private TableColumn createScoreColumn(MainTableColumnModel columnModel) { - TableColumn column = new MainTableColumn<>(columnModel); - Node header = new Text(Localization.lang("Score")); - header.getStyleClass().add("mainTable-header"); - Tooltip.install(header, new Tooltip(MainTableColumnModel.Type.MATCH_SCORE.getDisplayName())); - column.setGraphic(header); - column.setStyle("-fx-alignment: CENTER-RIGHT;"); - column.setCellValueFactory(cellData -> cellData.getValue().searchScoreProperty()); - new ValueTableCellFactory().withText(String::valueOf).install(column); - column.setSortable(true); - column.setReorderable(false); - column.visibleProperty().bind(stateManager.activeSearchQuery(SearchType.NORMAL_SEARCH).isPresent()); - return column; - } - /** * Creates a column with a continuous number */ diff --git a/src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java b/src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java index d0d5f7366de..b8f89ac144e 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java @@ -36,7 +36,6 @@ public class MainTableColumnModel { private static final Logger LOGGER = LoggerFactory.getLogger(MainTableColumnModel.class); public enum Type { MATCH_CATEGORY("match_category"), // Not localized, because this column is always hidden - MATCH_SCORE("match_score", Localization.lang("Match score")), INDEX("index", Localization.lang("Index")), EXTRAFILE("extrafile", Localization.lang("File type")), FILES("files", Localization.lang("Linked files")), @@ -153,7 +152,7 @@ public String getName() { public String getDisplayName() { if ((Type.ICON_COLUMNS.contains(typeProperty.getValue()) && qualifierProperty.getValue().isBlank()) - || (typeProperty.getValue() == Type.INDEX) || typeProperty.getValue() == Type.MATCH_SCORE) { + || (typeProperty.getValue() == Type.INDEX)) { return typeProperty.getValue().getDisplayName(); } else { // In case an OrField is used, `FieldFactory.parseField` returns UnknownField, which leads to diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 633145240c7..7c59c105a49 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -98,8 +98,7 @@ public MainTableDataModel(BibDatabaseContext context, private void updateSearchMatches(Optional query) { BackgroundTask.wrap(() -> { if (query.isPresent()) { - SearchResults results = indexManager.search(query.get()); - setSearchMatches(results); + setSearchMatches(indexManager.search(query.get())); } else { clearSearchMatches(); } @@ -109,16 +108,15 @@ private void updateSearchMatches(Optional query) { private void setSearchMatches(SearchResults results) { boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; entriesViewModel.forEach(entry -> { - entry.searchScoreProperty().set(results.getSearchScoreForEntry(entry.getEntry())); entry.hasFullTextResultsProperty().set(results.hasFulltextResults(entry.getEntry())); - updateEntrySearchMatch(entry, entry.searchScoreProperty().get() > 0, isFloatingMode); + updateEntrySearchMatch(entry, results.isMatched(entry.getEntry()), isFloatingMode); }); } private void clearSearchMatches() { boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; entriesViewModel.forEach(entry -> { - entry.searchScoreProperty().set(0); + entry.isMatchedBySearch().set(true); entry.hasFullTextResultsProperty().set(false); updateEntrySearchMatch(entry, true, isFloatingMode); }); @@ -216,11 +214,9 @@ public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { SearchQuery entryQuery = new SearchQuery(newSearchExpression, searchQuery.getSearchFlags()); SearchResults results = indexManager.search(entryQuery); - viewModel.searchScoreProperty().set(results.getSearchScoreForEntry(entry)); viewModel.hasFullTextResultsProperty().set(results.hasFulltextResults(entry)); - isMatched = viewModel.searchScoreProperty().get() > 0; + isMatched = results.isMatched(entry); } else { - viewModel.searchScoreProperty().set(0); viewModel.hasFullTextResultsProperty().set(false); } diff --git a/src/main/java/org/jabref/gui/preferences/table/TableTabViewModel.java b/src/main/java/org/jabref/gui/preferences/table/TableTabViewModel.java index 1df137f28e4..12ae933265e 100644 --- a/src/main/java/org/jabref/gui/preferences/table/TableTabViewModel.java +++ b/src/main/java/org/jabref/gui/preferences/table/TableTabViewModel.java @@ -123,7 +123,6 @@ public void setValues() { availableColumnsProperty.clear(); availableColumnsProperty.addAll( - new MainTableColumnModel(MainTableColumnModel.Type.MATCH_SCORE), new MainTableColumnModel(MainTableColumnModel.Type.INDEX), new MainTableColumnModel(MainTableColumnModel.Type.LINKED_IDENTIFIER), new MainTableColumnModel(MainTableColumnModel.Type.GROUPS), diff --git a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java index b90b8160367..435372a2cb2 100644 --- a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java +++ b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java @@ -71,13 +71,11 @@ private void updateSearchMatches(Optional query) { }); } for (BibEntryTableViewModel entry : entriesViewModel) { - entry.searchScoreProperty().set(searchResults.getSearchScoreForEntry(entry.getEntry())); entry.hasFullTextResultsProperty().set(searchResults.hasFulltextResults(entry.getEntry())); - entry.isVisibleBySearch().set(entry.searchScoreProperty().get() > 0); + entry.isVisibleBySearch().set(searchResults.isMatched(entry.getEntry())); } } else { for (BibEntryTableViewModel entry : entriesViewModel) { - entry.searchScoreProperty().set(0); entry.hasFullTextResultsProperty().set(false); entry.isVisibleBySearch().set(true); } diff --git a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java index fcdc6ecf630..8efb3d5dfb0 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java @@ -28,7 +28,7 @@ public SearchResults search(SqlSearchQuery searchQuery) { ResultSet resultSet = preparedStatement.executeQuery(); while (resultSet.next()) { String entryId = resultSet.getString(1); - searchResults.addSearchResult(entryId, new SearchResult(1)); + searchResults.addSearchResult(entryId, new SearchResult()); } } catch (SQLException e) { LOGGER.error("Error during bib fields search execution", e); diff --git a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java index fc18df702e6..c38d70a89d4 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java @@ -84,7 +84,6 @@ private SearchResults getSearchResults(TopDocs topDocs, StoredFields storedField List entriesWithFile = linkedFilesMap.get(fileLink); if (entriesWithFile != null && !entriesWithFile.isEmpty()) { SearchResult searchResult = new SearchResult( - scoreDoc.score, fileLink, getFieldContents(document, SearchFieldConstants.CONTENT), getFieldContents(document, SearchFieldConstants.ANNOTATIONS), diff --git a/src/main/java/org/jabref/model/search/query/SearchResult.java b/src/main/java/org/jabref/model/search/query/SearchResult.java index e35b302834e..dde7c933e26 100644 --- a/src/main/java/org/jabref/model/search/query/SearchResult.java +++ b/src/main/java/org/jabref/model/search/query/SearchResult.java @@ -13,7 +13,6 @@ public final class SearchResult { - private final float searchScore; private final boolean hasFulltextResults; private final String path; private final String pageContent; @@ -23,14 +22,12 @@ public final class SearchResult { private List contentResultStringsHtml; private List annotationsResultStringsHtml; - private SearchResult(float searchScore, - boolean hasFulltextResults, + private SearchResult(boolean hasFulltextResults, String path, String pageContent, String annotation, int pageNumber, Highlighter highlighter) { - this.searchScore = searchScore; this.hasFulltextResults = hasFulltextResults; this.path = path; this.pageContent = pageContent; @@ -39,12 +36,12 @@ private SearchResult(float searchScore, this.highlighter = highlighter; } - public SearchResult(float searchScore) { - this(searchScore, false, "", "", "", -1, null); + public SearchResult() { + this(false, "", "", "", -1, null); } - public SearchResult(float searchScore, String path, String pageContent, String annotation, int pageNumber, Highlighter highlighter) { - this(searchScore, true, path, pageContent, annotation, pageNumber, highlighter); + public SearchResult(String path, String pageContent, String annotation, int pageNumber, Highlighter highlighter) { + this(true, path, pageContent, annotation, pageNumber, highlighter); } public List getContentResultStringsHtml() { @@ -61,10 +58,6 @@ public List getAnnotationsResultStringsHtml() { return annotationsResultStringsHtml; } - public float getSearchScore() { - return searchScore; - } - public boolean hasFulltextResults() { return hasFulltextResults; } diff --git a/src/main/java/org/jabref/model/search/query/SearchResults.java b/src/main/java/org/jabref/model/search/query/SearchResults.java index 1394b11fe81..e955a5031c3 100644 --- a/src/main/java/org/jabref/model/search/query/SearchResults.java +++ b/src/main/java/org/jabref/model/search/query/SearchResults.java @@ -25,14 +25,8 @@ public void addSearchResult(Collection entries, SearchResult result) { entries.forEach(entry -> addSearchResult(entry, result)); } - public float getSearchScoreForEntry(BibEntry entry) { - if (searchResults.containsKey(entry.getId())) { - return searchResults.get(entry.getId()) - .stream() - .map(SearchResult::getSearchScore) - .reduce(0f, Float::max); - } - return 0f; + public boolean isMatched(BibEntry entry) { + return searchResults.containsKey(entry.getId()); } public boolean hasFulltextResults(BibEntry entry) { From f54bf6e16b0b311564054237d932af13724ca1ad Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 4 Oct 2024 14:57:48 +0300 Subject: [PATCH 051/104] Update search groups matches --- .../gui/groups/GroupDialogViewModel.java | 2 +- .../jabref/gui/groups/GroupNodeViewModel.java | 6 +- .../gui/maintable/MainTableDataModel.java | 10 ++- .../org/jabref/logic/search/IndexManager.java | 11 ++- .../search/query/SearchToSqlConversion.java | 4 + .../search/query/SearchToSqlVisitor.java | 14 +++- .../search/retrieval/BibFieldsSearcher.java | 26 +++++-- .../org/jabref/model/groups/SearchGroup.java | 22 +++--- .../model/search/query/SearchQuery.java | 74 ++++++------------- .../model/search/query/SqlSearchQuery.java | 25 ------- 10 files changed, 88 insertions(+), 106 deletions(-) delete mode 100644 src/main/java/org/jabref/model/search/query/SqlSearchQuery.java diff --git a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java index 060fdf2e257..bc56ced4046 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java @@ -334,7 +334,7 @@ public AbstractGroup resultConverter(ButtonType button) { Optional indexManager = stateManager.getIndexManager(currentDatabase); if (indexManager.isPresent()) { SearchGroup searchGroup = (SearchGroup) resultingGroup; - searchGroup.setMatchedEntries(indexManager.get().search(searchGroup.getQuery()).getMatchedEntries()); + searchGroup.setMatchedEntries(indexManager.get().search(searchGroup.getSearchQuery()).getMatchedEntries()); } } else if (typeAutoProperty.getValue()) { if (autoGroupKeywordsOptionProperty.getValue()) { diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index 43c3050cb00..b662b1ae115 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -103,7 +103,7 @@ public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager state } else if (groupNode.getGroup() instanceof SearchGroup searchGroup) { stateManager.getIndexManager(databaseContext).ifPresent(indexManager -> { BackgroundTask.wrap(() -> { - searchGroup.setMatchedEntries(indexManager.search(searchGroup.getQuery()).getMatchedEntries()); + searchGroup.setMatchedEntries(indexManager.search(searchGroup.getSearchQuery()).getMatchedEntries()); }).onSuccess(success -> { refreshGroup(); databaseContext.getMetaData().groupsBinding().invalidate(); @@ -542,7 +542,7 @@ public void listen(IndexStartedEvent event) { if (groupNode.getGroup() instanceof SearchGroup searchGroup) { stateManager.getIndexManager(databaseContext).ifPresent(indexManager -> { BackgroundTask.wrap(() -> { - searchGroup.setMatchedEntries(indexManager.search(searchGroup.getQuery()).getMatchedEntries()); + searchGroup.setMatchedEntries(indexManager.search(searchGroup.getSearchQuery()).getMatchedEntries()); }).onSuccess(success -> { refreshGroup(); databaseContext.getMetaData().groupsBinding().invalidate(); @@ -557,7 +557,7 @@ public void listen(IndexAddedOrUpdatedEvent event) { stateManager.getIndexManager(databaseContext).ifPresent(indexManager -> { BackgroundTask.wrap(() -> { for (BibEntry entry : event.entries()) { - searchGroup.updateMatches(entry, indexManager.isEntryMatched(entry, searchGroup.getQuery())); + searchGroup.updateMatches(entry, indexManager.isEntryMatched(entry, searchGroup.getSearchQuery())); } }).onFinished(() -> { for (BibEntry entry : event.entries()) { diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 7c59c105a49..4c4d6a11c75 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -27,7 +27,6 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; import org.jabref.model.search.SearchDisplayMode; -import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; import org.jabref.model.search.event.IndexStartedEvent; import org.jabref.model.search.matchers.MatcherSet; @@ -40,6 +39,8 @@ import com.tobiasdiez.easybind.Subscription; import org.jspecify.annotations.Nullable; +import static org.jabref.model.search.PostgreConstants.ENTRY_ID; + public class MainTableDataModel { private final ObservableList entriesViewModel; @@ -207,16 +208,17 @@ public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { if (index >= 0) { BibEntryTableViewModel viewModel = entriesViewModel.get(index); boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; - boolean isMatched = true; + boolean isMatched; if (searchQueryProperty.get().isPresent()) { SearchQuery searchQuery = searchQueryProperty.get().get(); - String newSearchExpression = "+" + SearchFieldConstants.ENTRY_ID + ":" + entry.getId() + " +" + searchQuery.getSearchExpression(); + String newSearchExpression = "(" + ENTRY_ID + "= " + entry.getId() + ") AND (" + searchQuery.getSearchExpression() + ")"; SearchQuery entryQuery = new SearchQuery(newSearchExpression, searchQuery.getSearchFlags()); SearchResults results = indexManager.search(entryQuery); - viewModel.hasFullTextResultsProperty().set(results.hasFulltextResults(entry)); isMatched = results.isMatched(entry); + viewModel.hasFullTextResultsProperty().set(results.hasFulltextResults(entry)); } else { + isMatched = true; viewModel.hasFullTextResultsProperty().set(false); } diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index 0108ed395cb..454528620f7 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -63,7 +63,7 @@ public IndexManager(BibDatabaseContext databaseContext, TaskExecutor executor, C } linkedFilesIndexer = indexer; - this.bibFieldsSearcher = new BibFieldsSearcher(postgreServer.getConnection()); + this.bibFieldsSearcher = new BibFieldsSearcher(postgreServer.getConnection(), bibFieldsIndexer.getTable()); this.linkedFilesSearcher = new LinkedFilesSearcher(databaseContext, linkedFilesIndexer, preferences.getFilePreferences()); updateOnStart(); } @@ -221,6 +221,8 @@ public AutoCloseable blockLinkedFileIndexer() { } public SearchResults search(SearchQuery query) { + SearchResults searchResults = new SearchResults(); + // if (query.isValid()) { // query.setSearchResults(linkedFilesSearcher.search(query.getParsedQuery(), query.getSearchFlags())); // } else { @@ -229,12 +231,15 @@ public SearchResults search(SearchQuery query) { if (query.getSearchFlags().contains(SearchFlags.FULLTEXT)) { // TODO: merge results from lucene and postgres } else { - query.setSearchResults(bibFieldsSearcher.search(query.getSqlQuery(bibFieldsIndexer.getTable()))); + query.setSearchResults(bibFieldsSearcher.search(query)); } return query.getSearchResults(); } + /** + * @implNote No need to check for full-text searches as this method only used by the search groups + */ public boolean isEntryMatched(BibEntry entry, SearchQuery query) { - return true; + return bibFieldsSearcher.isMatched(entry, query); } } diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java index 265b3cf55c5..8cf9b7884d2 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java @@ -7,10 +7,14 @@ import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonTokenStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SearchToSqlConversion { + private static final Logger LOGGER = LoggerFactory.getLogger(SearchToSqlConversion.class); public static String searchToSql(String table, String searchExpression) { + LOGGER.debug("Converting search expression to SQL: {}", searchExpression); SearchParser.StartContext context = getStartContext(searchExpression); SearchToSqlVisitor searchToSqlVisitor = new SearchToSqlVisitor(table); return searchToSqlVisitor.visit(context); diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 963fe5cdc22..49ae5d3fe0a 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -195,7 +195,19 @@ private String getFieldQueryNode(String field, String term, EnumSet field; }; - if ("anyfield".equals(field) || "any".equals(field)) { + if (ENTRY_ID.toString().equals(field)) { + cte = """ + cte%d AS ( + SELECT %s + FROM %s + WHERE %s = '%s' + ) + """.formatted( + cteCounter, + ENTRY_ID, + mainTableName, + ENTRY_ID, term); + } else if ("anyfield".equals(field) || "any".equals(field)) { if (searchFlags.contains(SearchTermFlag.EXACT_MATCH)) { cte = searchFlags.contains(SearchTermFlag.NEGATION) ? buildExactNegationAnyFieldQuery(operator, term) diff --git a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java index 8efb3d5dfb0..f090b488e2b 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java @@ -5,26 +5,42 @@ import java.sql.ResultSet; import java.sql.SQLException; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.search.query.SearchQuery; import org.jabref.model.search.query.SearchResult; import org.jabref.model.search.query.SearchResults; -import org.jabref.model.search.query.SqlSearchQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.jabref.model.search.PostgreConstants.ENTRY_ID; + public class BibFieldsSearcher { private static final Logger LOGGER = LoggerFactory.getLogger(BibFieldsSearcher.class); private final Connection connection; + private final String tableName; - public BibFieldsSearcher(Connection connection) { + public BibFieldsSearcher(Connection connection, String tableName) { this.connection = connection; + this.tableName = tableName; + } + + public boolean isMatched(BibEntry entry, SearchQuery searchQuery) { + SearchQuery newSearchQuery = createBooleanQueryForEntry(entry, searchQuery); + return search(newSearchQuery).isMatched(entry); + } + + private static SearchQuery createBooleanQueryForEntry(BibEntry entry, SearchQuery oldSearchQuery) { + String newSearchExpression = "( " + ENTRY_ID + "= " + entry.getId() + ") AND (" + oldSearchQuery.getSearchExpression() + " )"; + return new SearchQuery(newSearchExpression, oldSearchQuery.getSearchFlags()); } - public SearchResults search(SqlSearchQuery searchQuery) { - LOGGER.debug("Searching in bib fields with query: {}", searchQuery.getQuery()); + public SearchResults search(SearchQuery searchQuery) { + String sqlQuery = searchQuery.getSqlQuery(tableName); + LOGGER.debug("Searching in bib fields with query: {}", sqlQuery); SearchResults searchResults = new SearchResults(); - try (PreparedStatement preparedStatement = connection.prepareStatement(searchQuery.getQuery())) { + try (PreparedStatement preparedStatement = connection.prepareStatement(sqlQuery)) { ResultSet resultSet = preparedStatement.executeQuery(); while (resultSet.next()) { String entryId = resultSet.getString(1); diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index 5684736a949..71deb84c521 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -24,15 +24,11 @@ public class SearchGroup extends AbstractGroup { @ADR(38) private final Set matchedEntries = new HashSet<>(); - private SearchQuery query; + private SearchQuery searchQuery; public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags) { super(name, context); - this.query = new SearchQuery(searchExpression, searchFlags); - } - - public String getSearchExpression() { - return query.getSearchExpression(); + this.searchQuery = new SearchQuery(searchExpression, searchFlags); } /** @@ -41,15 +37,19 @@ public String getSearchExpression() { */ public void setSearchExpression(String searchExpression) { LOGGER.debug("Setting search expression {}", searchExpression); - this.query = new SearchQuery(searchExpression, query.getSearchFlags()); + this.searchQuery = new SearchQuery(searchExpression, searchQuery.getSearchFlags()); + } + + public String getSearchExpression() { + return searchQuery.getSearchExpression(); } - public SearchQuery getQuery() { - return query; + public SearchQuery getSearchQuery() { + return searchQuery; } public EnumSet getSearchFlags() { - return query.getSearchFlags(); + return searchQuery.getSearchFlags(); } public void setMatchedEntries(Collection entriesId) { @@ -98,7 +98,7 @@ public AbstractGroup deepCopy() { @Override public String toString() { - return "SearchGroup [query=" + query + ", name=" + name + ", searchFlags=" + getSearchFlags() + ", context=" + context + ", color=" + color + ", isExpanded=" + isExpanded + ", description=" + description + ", iconName=" + iconName + "]"; + return "SearchGroup [query=" + searchQuery + ", name=" + name + ", searchFlags=" + getSearchFlags() + ", context=" + context + ", color=" + color + ", isExpanded=" + isExpanded + ", description=" + description + ", iconName=" + iconName + "]"; } @Override diff --git a/src/main/java/org/jabref/model/search/query/SearchQuery.java b/src/main/java/org/jabref/model/search/query/SearchQuery.java index aef9d2ab3e6..2acb1e816e8 100644 --- a/src/main/java/org/jabref/model/search/query/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/query/SearchQuery.java @@ -1,11 +1,8 @@ package org.jabref.model.search.query; -import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.regex.Pattern; @@ -13,15 +10,9 @@ import java.util.stream.Stream; import org.jabref.logic.search.query.SearchToSqlConversion; -import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.SearchFlags; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; -import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; import org.apache.lucene.search.Query; -import org.apache.lucene.search.highlight.QueryTermExtractor; -import org.apache.lucene.search.highlight.WeightedTerm; public class SearchQuery { /** @@ -61,52 +52,35 @@ String format(String regex) { abstract String format(String regex); } - private final String query; + private final String searchExpression; private final EnumSet searchFlags; - private Query parsedQuery; private String parseError; + private String sqlQuery; + private Query luceneQuery; private SearchResults searchResults; - public SearchQuery(String query, EnumSet searchFlags) { - this.query = Objects.requireNonNull(query); + public SearchQuery(String searchExpression, EnumSet searchFlags) { + this.searchExpression = Objects.requireNonNull(searchExpression); this.searchFlags = searchFlags; + } - Map boosts = new HashMap<>(); - Map fieldAnalyzers = new HashMap<>(); - - if (searchFlags.contains(SearchFlags.FULLTEXT)) { - boosts.put(SearchFieldConstants.DEFAULT_FIELD.toString(), 4F); - SearchFieldConstants.PDF_FIELDS.forEach(field -> { - boosts.put(field, 1F); - fieldAnalyzers.put(field, SearchFieldConstants.LINKED_FILES_ANALYZER); - }); - } else { - boosts.put(SearchFieldConstants.DEFAULT_FIELD.toString(), 1F); - } - - String[] fieldsToSearchArray = new String[boosts.size()]; - boosts.keySet().toArray(fieldsToSearchArray); - - PerFieldAnalyzerWrapper analyzerWrapper = new PerFieldAnalyzerWrapper(SearchFieldConstants.LATEX_AWARE_ANALYZER, fieldAnalyzers); - MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, analyzerWrapper, boosts); - queryParser.setAllowLeadingWildcard(true); - - try { - parsedQuery = queryParser.parse(query); - parseError = null; - } catch (Exception e) { - parsedQuery = null; - parseError = e.getMessage(); + public String getSqlQuery(String table) { + if (sqlQuery == null) { + sqlQuery = SearchToSqlConversion.searchToSql(table, searchExpression); } + return sqlQuery; } - public SqlSearchQuery getSqlQuery(String table) { - return new SqlSearchQuery(SearchToSqlConversion.searchToSql(table, query)); + public Query getLuceneQuery() { + if (luceneQuery == null) { + // TODO: convert to lucene query + } + return luceneQuery; } public String getSearchExpression() { - return query; + return searchExpression; } public SearchResults getSearchResults() { @@ -119,7 +93,7 @@ public void setSearchResults(SearchResults searchResults) { @Override public String toString() { - return query; + return searchExpression; } @Override @@ -130,23 +104,19 @@ public boolean equals(Object o) { if (!(o instanceof SearchQuery that)) { return false; } - return Objects.equals(query, that.query) + return Objects.equals(searchExpression, that.searchExpression) && Objects.equals(searchFlags, that.searchFlags); } @Override public int hashCode() { - return Objects.hash(query, searchFlags); + return Objects.hash(searchExpression, searchFlags); } public boolean isValid() { return parseError == null; } - public Query getParsedQuery() { - return parsedQuery; - } - public EnumSet getSearchFlags() { return searchFlags; } @@ -156,14 +126,12 @@ public EnumSet getSearchFlags() { */ public List getSearchWords() { if (searchFlags.contains(SearchFlags.REGULAR_EXPRESSION)) { - return Collections.singletonList(query); + return Collections.singletonList(searchExpression); } if (!isValid()) { return List.of(); } - return Arrays.stream(QueryTermExtractor.getTerms(parsedQuery)) - .map(WeightedTerm::getTerm) - .toList(); + return List.of(); } // Returns a regular expression pattern in the form (w1)|(w2)| ... wi are escaped if no regular expression search is enabled diff --git a/src/main/java/org/jabref/model/search/query/SqlSearchQuery.java b/src/main/java/org/jabref/model/search/query/SqlSearchQuery.java deleted file mode 100644 index db96a93da81..00000000000 --- a/src/main/java/org/jabref/model/search/query/SqlSearchQuery.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.jabref.model.search.query; - -import java.util.Objects; - -public class SqlSearchQuery { - - private final String query; - private SearchResults searchResults; - - public SqlSearchQuery(String query) { - this.query = Objects.requireNonNull(query); - } - - public String getQuery() { - return query; - } - - public SearchResults getSearchResults() { - return searchResults; - } - - public void setSearchResults(SearchResults searchResults) { - this.searchResults = searchResults; - } -} From 6ab8f5543c1ce46d9c858862bab60fd4e1e9f0c3 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 4 Oct 2024 16:05:55 +0300 Subject: [PATCH 052/104] Remove search_score from tale preferences --- .../java/org/jabref/gui/preferences/JabRefGuiPreferences.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/preferences/JabRefGuiPreferences.java b/src/main/java/org/jabref/gui/preferences/JabRefGuiPreferences.java index d3cfb4de424..1011f83dc81 100644 --- a/src/main/java/org/jabref/gui/preferences/JabRefGuiPreferences.java +++ b/src/main/java/org/jabref/gui/preferences/JabRefGuiPreferences.java @@ -373,8 +373,8 @@ private JabRefGuiPreferences() { // region: Main table, main table column, and search dialog column preferences defaults.put(EXTRA_FILE_COLUMNS, Boolean.FALSE); - defaults.put(COLUMN_NAMES, "search_score;groups;group_icons;files;linked_id;field:entrytype;field:author/editor;field:title;field:year;field:journal/booktitle;special:ranking;special:readstatus;special:priority"); - defaults.put(COLUMN_WIDTHS, "50;28;40;28;28;75;300;470;60;130;50;50;50"); + defaults.put(COLUMN_NAMES, "groups;group_icons;files;linked_id;field:entrytype;field:author/editor;field:title;field:year;field:journal/booktitle;special:ranking;special:readstatus;special:priority"); + defaults.put(COLUMN_WIDTHS, "28;40;28;28;75;300;470;60;130;50;50;50"); defaults.put(SIDE_PANE_COMPONENT_NAMES, ""); defaults.put(SIDE_PANE_COMPONENT_PREFERRED_POSITIONS, ""); From 4fc088fe415118292a2ca6bd5d8a049a9d2b5fcd Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 4 Oct 2024 19:04:17 +0300 Subject: [PATCH 053/104] Migrate search groups flags to new syntax --- .../gui/groups/GroupDialogViewModel.java | 3 +- .../actions/CheckForNewEntryTypesAction.java | 2 +- .../importer/actions/GUIPostOpenAction.java | 6 +- .../actions/MergeReviewIntoCommentAction.java | 2 +- .../importer/actions/OpenDatabaseAction.java | 2 +- .../actions/SearchGroupsMigrationAction.java | 28 +++-- .../org/jabref/logic/search/IndexManager.java | 7 +- .../query/SearchFlagsToExpressionVisitor.java | 115 ++++++++++++++++++ ...ersion.java => SearchQueryConversion.java} | 18 ++- .../search/query}/SearchToLuceneVisitor.java | 4 +- .../search/query/SearchToSqlVisitor.java | 66 +++++----- .../ConvertLegacyExplicitGroups.java | 3 +- .../migrations/SearchToLuceneMigration.java | 35 ------ .../model/search/query/SearchQuery.java | 12 +- .../model/search/query/SearchTermFlag.java | 7 ++ .../{ => query}/LuceneQueryParserTest.java | 2 +- .../query/SearchQueryFlagsConversionTest.java | 90 ++++++++++++++ .../SearchQueryLuceneConversionTest.java} | 8 +- .../SearchQuerySQLConversionTest.java} | 8 +- 19 files changed, 304 insertions(+), 114 deletions(-) create mode 100644 src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java rename src/main/java/org/jabref/logic/search/query/{SearchToSqlConversion.java => SearchQueryConversion.java} (67%) rename src/main/java/org/jabref/{migrations => logic/search/query}/SearchToLuceneVisitor.java (99%) delete mode 100644 src/main/java/org/jabref/migrations/SearchToLuceneMigration.java create mode 100644 src/main/java/org/jabref/model/search/query/SearchTermFlag.java rename src/test/java/org/jabref/logic/search/{ => query}/LuceneQueryParserTest.java (97%) create mode 100644 src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java rename src/test/java/org/jabref/{migrations/SearchToLuceneMigrationTest.java => logic/search/query/SearchQueryLuceneConversionTest.java} (94%) rename src/test/java/org/jabref/logic/search/{SearchToSqlConversionTest.java => query/SearchQuerySQLConversionTest.java} (97%) diff --git a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java index bc56ced4046..1e9c9c4eb49 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java @@ -327,13 +327,14 @@ public AbstractGroup resultConverter(ButtonType button) { // Otherwise, it means that the user did not accept the migration to the new version. Optional groups = currentDatabase.getMetaData().getGroups(); if (groups.filter(this::groupOrSubgroupIsSearchGroup).isEmpty()) { - currentDatabase.getMetaData().setGroupSearchSyntaxVersion(SearchGroupsMigrationAction.VERSION_6_0_ALPHA); + currentDatabase.getMetaData().setGroupSearchSyntaxVersion(SearchGroupsMigrationAction.VERSION_6_0_ALPHA_1); } } Optional indexManager = stateManager.getIndexManager(currentDatabase); if (indexManager.isPresent()) { SearchGroup searchGroup = (SearchGroup) resultingGroup; + // TODO: search should be done in a background thread searchGroup.setMatchedEntries(indexManager.get().search(searchGroup.getSearchQuery()).getMatchedEntries()); } } else if (typeAutoProperty.getValue()) { diff --git a/src/main/java/org/jabref/gui/importer/actions/CheckForNewEntryTypesAction.java b/src/main/java/org/jabref/gui/importer/actions/CheckForNewEntryTypesAction.java index 68b086f4ba8..e63af24ab95 100644 --- a/src/main/java/org/jabref/gui/importer/actions/CheckForNewEntryTypesAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/CheckForNewEntryTypesAction.java @@ -21,7 +21,7 @@ public class CheckForNewEntryTypesAction implements GUIPostOpenAction { @Override - public boolean isActionNecessary(ParserResult parserResult, CliPreferences preferences) { + public boolean isActionNecessary(ParserResult parserResult, DialogService dialogService, CliPreferences preferences) { return !getListOfUnknownAndUnequalCustomizations(parserResult, preferences.getLibraryPreferences()).isEmpty(); } diff --git a/src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java b/src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java index 2551845ea61..f4eec73e339 100644 --- a/src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java @@ -9,7 +9,7 @@ * opening a BIB file into JabRef. This can for instance be file upgrade actions * that should be offered due to new features in JabRef, and may depend on e.g. * which JabRef version the file was last written by. - * + *

* This interface is introduced in an attempt to add such functionality in a * flexible manner. */ @@ -22,12 +22,12 @@ public interface GUIPostOpenAction { * @param pr The result of the BIB parse operation. * @return true if the action should be called, false otherwise. */ - boolean isActionNecessary(ParserResult pr, CliPreferences preferences); + boolean isActionNecessary(ParserResult pr, DialogService dialogService, CliPreferences preferences); /** * This method is called after the new database has been added to the GUI, if * the isActionNecessary() method returned true. - * + *

* Note: if several such methods need to be called sequentially, it is * important that all implementations of this method do not return * until the operation is finished. diff --git a/src/main/java/org/jabref/gui/importer/actions/MergeReviewIntoCommentAction.java b/src/main/java/org/jabref/gui/importer/actions/MergeReviewIntoCommentAction.java index 9ccec061d23..3d69e620558 100644 --- a/src/main/java/org/jabref/gui/importer/actions/MergeReviewIntoCommentAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/MergeReviewIntoCommentAction.java @@ -11,7 +11,7 @@ public class MergeReviewIntoCommentAction implements GUIPostOpenAction { @Override - public boolean isActionNecessary(ParserResult parserResult, CliPreferences preferences) { + public boolean isActionNecessary(ParserResult parserResult, DialogService dialogService, CliPreferences preferences) { return MergeReviewIntoCommentMigration.needsMigration(parserResult); } diff --git a/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java b/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java index d30c7d79a3d..50b1ca8edd0 100644 --- a/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java @@ -94,7 +94,7 @@ public OpenDatabaseAction(LibraryTabContainer tabContainer, public static void performPostOpenActions(ParserResult result, DialogService dialogService, CliPreferences preferences) { for (GUIPostOpenAction action : OpenDatabaseAction.POST_OPEN_ACTIONS) { - if (action.isActionNecessary(result, preferences)) { + if (action.isActionNecessary(result, dialogService, preferences)) { action.performAction(result, dialogService, preferences); } } diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index 9e2ca086ab6..1b7376698b4 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -7,11 +7,10 @@ import org.jabref.logic.importer.ParserResult; import org.jabref.logic.l10n.Localization; import org.jabref.logic.preferences.CliPreferences; +import org.jabref.logic.search.query.SearchQueryConversion; import org.jabref.logic.util.Version; -import org.jabref.migrations.SearchToLuceneMigration; import org.jabref.model.groups.GroupTreeNode; import org.jabref.model.groups.SearchGroup; -import org.jabref.model.search.SearchFlags; import org.antlr.v4.runtime.misc.ParseCancellationException; @@ -24,11 +23,17 @@ public class SearchGroupsMigrationAction implements GUIPostOpenAction { // We cannot have this constant in `Version.java` because of recursion errors // Thus, we keep it here, because it is (currently) used only in the context of groups migration. public static final Version VERSION_6_0_ALPHA = Version.parse("6.0-alpha"); + public static final Version VERSION_6_0_ALPHA_1 = Version.parse("6.0-alpha_1"); @Override - public boolean isActionNecessary(ParserResult parserResult, CliPreferences preferences) { - if (parserResult.getMetaData().getGroupSearchSyntaxVersion().isPresent()) { - // Currently the presence of any version is enough to know that no migration is necessary + public boolean isActionNecessary(ParserResult parserResult, DialogService dialogService, CliPreferences preferences) { + Optional currentVersion = parserResult.getMetaData().getGroupSearchSyntaxVersion(); + if (currentVersion.isPresent()) { + if (currentVersion.get().equals(VERSION_6_0_ALPHA)) { + dialogService.showErrorDialogAndWait(Localization.lang("Search groups migration"), + Localization.lang("The search groups syntax has been reverted to the old one. " + + "Please use the backup you made before migrating to 6.0-alpha.")); + } return false; } @@ -57,21 +62,22 @@ public void performAction(ParserResult parserResult, DialogService dialogService } parserResult.getMetaData().getGroups().ifPresent(groupTreeNode -> migrateGroups(groupTreeNode, dialogService)); - parserResult.getMetaData().setGroupSearchSyntaxVersion(VERSION_6_0_ALPHA); + parserResult.getMetaData().setGroupSearchSyntaxVersion(VERSION_6_0_ALPHA_1); parserResult.setChangedOnMigration(true); } private void migrateGroups(GroupTreeNode node, DialogService dialogService) { if (node.getGroup() instanceof SearchGroup searchGroup) { try { - String luceneSearchExpression = SearchToLuceneMigration.migrateToLuceneSyntax(searchGroup.getSearchExpression(), searchGroup.getSearchFlags().contains(SearchFlags.REGULAR_EXPRESSION)); - searchGroup.setSearchExpression(luceneSearchExpression); + String newSearchExpression = SearchQueryConversion.flagsToSearchExpression(searchGroup.getSearchExpression(), searchGroup.getSearchFlags()); + searchGroup.setSearchExpression(newSearchExpression); } catch (ParseCancellationException e) { - Optional luceneSearchExpression = dialogService.showInputDialogWithDefaultAndWait( + Optional newSearchExpression = dialogService.showInputDialogWithDefaultAndWait( Localization.lang("Search group migration failed"), - Localization.lang("The search group '%0' could not be migrated. Please enter the new search expression.", searchGroup.getName()), + Localization.lang("The search group '%0' could not be migrated. Please enter the new search expression.", + searchGroup.getName()), searchGroup.getSearchExpression()); - luceneSearchExpression.ifPresent(searchGroup::setSearchExpression); + newSearchExpression.ifPresent(searchGroup::setSearchExpression); } } for (GroupTreeNode child : node.getChildren()) { diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index 454528620f7..7feae979af2 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -222,7 +222,7 @@ public AutoCloseable blockLinkedFileIndexer() { public SearchResults search(SearchQuery query) { SearchResults searchResults = new SearchResults(); - + searchResults.mergeSearchResults(bibFieldsSearcher.search(query)); // if (query.isValid()) { // query.setSearchResults(linkedFilesSearcher.search(query.getParsedQuery(), query.getSearchFlags())); // } else { @@ -230,10 +230,9 @@ public SearchResults search(SearchQuery query) { // } if (query.getSearchFlags().contains(SearchFlags.FULLTEXT)) { // TODO: merge results from lucene and postgres - } else { - query.setSearchResults(bibFieldsSearcher.search(query)); } - return query.getSearchResults(); + query.setSearchResults(searchResults); + return searchResults; } /** diff --git a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java new file mode 100644 index 00000000000..38b0f6ae4ad --- /dev/null +++ b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java @@ -0,0 +1,115 @@ +package org.jabref.logic.search.query; + +import java.util.EnumSet; +import java.util.Optional; + +import org.jabref.model.search.SearchFlags; +import org.jabref.model.search.query.SearchTermFlag; +import org.jabref.search.SearchBaseVisitor; +import org.jabref.search.SearchParser; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.jabref.model.search.query.SearchTermFlag.CASE_INSENSITIVE; +import static org.jabref.model.search.query.SearchTermFlag.CASE_SENSITIVE; +import static org.jabref.model.search.query.SearchTermFlag.EXACT_MATCH; +import static org.jabref.model.search.query.SearchTermFlag.INEXACT_MATCH; +import static org.jabref.model.search.query.SearchTermFlag.NEGATION; +import static org.jabref.model.search.query.SearchTermFlag.REGULAR_EXPRESSION; + +public class SearchFlagsToExpressionVisitor extends SearchBaseVisitor { + + private static final Logger LOGGER = LoggerFactory.getLogger(SearchFlagsToExpressionVisitor.class); + + private final boolean isCaseSensitive; + private final boolean isRegularExpression; + + public SearchFlagsToExpressionVisitor(EnumSet searchFlags) { + LOGGER.debug("Converting search flags to search expression: {}", searchFlags); + this.isCaseSensitive = searchFlags.contains(SearchFlags.CASE_SENSITIVE); + this.isRegularExpression = searchFlags.contains(SearchFlags.REGULAR_EXPRESSION); + } + + @Override + public String visitStart(SearchParser.StartContext context) { + return visit(context.expression()); + } + + @Override + public String visitParenExpression(SearchParser.ParenExpressionContext ctx) { + return "(" + visit(ctx.expression()) + ")"; + } + + @Override + public String visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { + return "NOT " + visit(ctx.expression()); + } + + @Override + public String visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { + return visit(ctx.left) + " " + ctx.operator.getText() + " " + visit(ctx.right); + } + + public String visitComparison(SearchParser.ComparisonContext context) { + String right = context.right.getText(); + + Optional fieldDescriptor = Optional.ofNullable(context.left); + EnumSet termFlags = EnumSet.noneOf(SearchTermFlag.class); + + if (fieldDescriptor.isPresent()) { + String field = fieldDescriptor.get().getText(); + + termFlags.add(isCaseSensitive ? CASE_SENSITIVE : CASE_INSENSITIVE); + if (context.NEQUAL() != null) { + termFlags.add(NEGATION); + } + + if (isRegularExpression) { + termFlags.add(REGULAR_EXPRESSION); + } else { + if (context.EQUAL() != null || context.CONTAINS() != null || context.NEQUAL() != null) { + termFlags.add(INEXACT_MATCH); + } else if (context.EEQUAL() != null || context.MATCHES() != null) { + termFlags.add(EXACT_MATCH); + } + } + return getFieldQueryNode(field, right, termFlags); + } else { + termFlags.add(isCaseSensitive ? CASE_SENSITIVE : CASE_INSENSITIVE); + if (isRegularExpression) { + termFlags.add(REGULAR_EXPRESSION); + } else { + termFlags.add(INEXACT_MATCH); + } + return getFieldQueryNode("any", right, termFlags); + } + } + + private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { + String operator = getOperator(searchFlags); + return field + " " + operator + " " + term; + } + + private static String getOperator(EnumSet searchFlags) { + StringBuilder operator = new StringBuilder(); + + if (searchFlags.contains(NEGATION)) { + operator.append("!"); + } + + if (searchFlags.contains(INEXACT_MATCH)) { + operator.append("="); + } else if (searchFlags.contains(EXACT_MATCH)) { + operator.append("=="); + } else if (searchFlags.contains(REGULAR_EXPRESSION)) { + operator.append("=~"); + } + + if (searchFlags.contains(CASE_SENSITIVE)) { + operator.append("!"); + } + + return operator.toString(); + } +} diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java similarity index 67% rename from src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java rename to src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java index 8cf9b7884d2..932fec56eaa 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java @@ -1,5 +1,8 @@ package org.jabref.logic.search.query; +import java.util.EnumSet; + +import org.jabref.model.search.SearchFlags; import org.jabref.model.search.ThrowingErrorListener; import org.jabref.search.SearchLexer; import org.jabref.search.SearchParser; @@ -10,8 +13,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class SearchToSqlConversion { - private static final Logger LOGGER = LoggerFactory.getLogger(SearchToSqlConversion.class); +public class SearchQueryConversion { + private static final Logger LOGGER = LoggerFactory.getLogger(SearchQueryConversion.class); public static String searchToSql(String table, String searchExpression) { LOGGER.debug("Converting search expression to SQL: {}", searchExpression); @@ -20,6 +23,17 @@ public static String searchToSql(String table, String searchExpression) { return searchToSqlVisitor.visit(context); } + public static String flagsToSearchExpression(String searchExpression, EnumSet searchFlags) { + LOGGER.debug("Converting search flags to search expression: {}, flags {}", searchExpression, searchFlags); + SearchParser.StartContext context = getStartContext(searchExpression); + return new SearchFlagsToExpressionVisitor(searchFlags).visit(context); + } + + public static String searchToLucene(String searchExpression) { + LOGGER.debug("Converting search expression to Lucene: {}", searchExpression); + return ""; + } + private static SearchParser.StartContext getStartContext(String searchExpression) { SearchLexer lexer = new SearchLexer(CharStreams.fromString(searchExpression)); lexer.removeErrorListeners(); // no infos on file system diff --git a/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java similarity index 99% rename from src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java rename to src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java index aabdf570185..5f9e82f9f16 100644 --- a/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java @@ -1,4 +1,4 @@ -package org.jabref.migrations; +package org.jabref.logic.search.query; import java.util.List; import java.util.Optional; @@ -21,7 +21,7 @@ /** * Converts to a Lucene index with the assumption that the ngram analyzer is used. - * + *

* Tests are located in {@link org.jabref.migrations.SearchToLuceneMigrationTest}. */ public class SearchToLuceneVisitor extends SearchBaseVisitor { diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 49ae5d3fe0a..5f54170ee94 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -9,6 +9,7 @@ import org.jabref.model.entry.field.InternalField; import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.PostgreConstants; +import org.jabref.model.search.query.SearchTermFlag; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; @@ -19,11 +20,15 @@ import static org.jabref.model.search.PostgreConstants.FIELD_NAME; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_LITERAL; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_TRANSFORMED; +import static org.jabref.model.search.query.SearchTermFlag.CASE_INSENSITIVE; +import static org.jabref.model.search.query.SearchTermFlag.CASE_SENSITIVE; +import static org.jabref.model.search.query.SearchTermFlag.EXACT_MATCH; +import static org.jabref.model.search.query.SearchTermFlag.INEXACT_MATCH; +import static org.jabref.model.search.query.SearchTermFlag.NEGATION; +import static org.jabref.model.search.query.SearchTermFlag.REGULAR_EXPRESSION; /** * Converts to a query processable by the scheme created by {@link BibFieldsIndexer}. - * - * @implNote Similar class: {@link org.jabref.migrations.SearchToLuceneMigration} */ public class SearchToSqlVisitor extends SearchBaseVisitor { @@ -43,13 +48,6 @@ public SearchToSqlVisitor(String table) { this.splitValuesTableName = PostgreConstants.getSplitTableSchemaReference(table); } - private enum SearchTermFlag { - REGULAR_EXPRESSION, // mutually exclusive to exact/inexact match - NEGATION, - CASE_SENSITIVE, CASE_INSENSITIVE, // mutually exclusive - EXACT_MATCH, INEXACT_MATCH // mutually exclusive - } - @Override public String visitStart(SearchParser.StartContext ctx) { String query = visit(ctx.expression()); @@ -149,35 +147,35 @@ public String visitComparison(SearchParser.ComparisonContext context) { // context.CONTAINS() and others are null if absent (thus, we cannot check for getText()) EnumSet searchFlags = EnumSet.noneOf(SearchTermFlag.class); if (context.EQUAL() != null || context.CONTAINS() != null) { - setFlags(searchFlags, SearchTermFlag.INEXACT_MATCH, false, false); + setFlags(searchFlags, INEXACT_MATCH, false, false); } else if (context.CEQUAL() != null) { - setFlags(searchFlags, SearchTermFlag.INEXACT_MATCH, true, false); + setFlags(searchFlags, INEXACT_MATCH, true, false); } else if (context.EEQUAL() != null || context.MATCHES() != null) { - setFlags(searchFlags, SearchTermFlag.EXACT_MATCH, false, false); + setFlags(searchFlags, EXACT_MATCH, false, false); } else if (context.CEEQUAL() != null) { - setFlags(searchFlags, SearchTermFlag.EXACT_MATCH, true, false); + setFlags(searchFlags, EXACT_MATCH, true, false); } else if (context.REQUAL() != null) { - setFlags(searchFlags, SearchTermFlag.REGULAR_EXPRESSION, false, false); + setFlags(searchFlags, REGULAR_EXPRESSION, false, false); } else if (context.CREEQUAL() != null) { - setFlags(searchFlags, SearchTermFlag.REGULAR_EXPRESSION, true, false); + setFlags(searchFlags, REGULAR_EXPRESSION, true, false); } else if (context.NEQUAL() != null) { - setFlags(searchFlags, SearchTermFlag.INEXACT_MATCH, false, true); + setFlags(searchFlags, INEXACT_MATCH, false, true); } else if (context.NCEQUAL() != null) { - setFlags(searchFlags, SearchTermFlag.INEXACT_MATCH, true, true); + setFlags(searchFlags, INEXACT_MATCH, true, true); } else if (context.NEEQUAL() != null) { - setFlags(searchFlags, SearchTermFlag.EXACT_MATCH, false, true); + setFlags(searchFlags, EXACT_MATCH, false, true); } else if (context.NCEEQUAL() != null) { - setFlags(searchFlags, SearchTermFlag.EXACT_MATCH, true, true); + setFlags(searchFlags, EXACT_MATCH, true, true); } else if (context.NREQUAL() != null) { - setFlags(searchFlags, SearchTermFlag.REGULAR_EXPRESSION, false, true); + setFlags(searchFlags, REGULAR_EXPRESSION, false, true); } else if (context.NCREEQUAL() != null) { - setFlags(searchFlags, SearchTermFlag.REGULAR_EXPRESSION, true, true); + setFlags(searchFlags, REGULAR_EXPRESSION, true, true); } cte = getFieldQueryNode(field, right, searchFlags); } else { // Query without any field name - cte = getFieldQueryNode("any", right, EnumSet.of(SearchTermFlag.INEXACT_MATCH, SearchTermFlag.CASE_INSENSITIVE)); + cte = getFieldQueryNode("any", right, EnumSet.of(INEXACT_MATCH, CASE_INSENSITIVE)); } ctes.add(cte); return "cte" + cteCounter++; @@ -186,7 +184,7 @@ public String visitComparison(SearchParser.ComparisonContext context) { private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { String cte; String operator = getOperator(searchFlags); - String prefixSuffix = searchFlags.contains(SearchTermFlag.INEXACT_MATCH) ? "%" : ""; + String prefixSuffix = searchFlags.contains(INEXACT_MATCH) ? "%" : ""; // Pseudo-fields field = switch (field) { @@ -208,22 +206,22 @@ private String getFieldQueryNode(String field, String term, EnumSet flags, SearchTermFlag matchType, boolean caseSensitive, boolean negation) { flags.add(matchType); - flags.add(caseSensitive ? SearchTermFlag.CASE_SENSITIVE : SearchTermFlag.CASE_INSENSITIVE); + flags.add(caseSensitive ? CASE_SENSITIVE : CASE_INSENSITIVE); if (negation) { - flags.add(SearchTermFlag.NEGATION); + flags.add(NEGATION); } } private static String getOperator(EnumSet searchFlags) { - return searchFlags.contains(SearchTermFlag.REGULAR_EXPRESSION) - ? (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "~" : "~*") - : (searchFlags.contains(SearchTermFlag.CASE_SENSITIVE) ? "LIKE" : "ILIKE"); + return searchFlags.contains(REGULAR_EXPRESSION) + ? (searchFlags.contains(CASE_SENSITIVE) ? "~" : "~*") + : (searchFlags.contains(CASE_SENSITIVE) ? "LIKE" : "ILIKE"); } } diff --git a/src/main/java/org/jabref/migrations/ConvertLegacyExplicitGroups.java b/src/main/java/org/jabref/migrations/ConvertLegacyExplicitGroups.java index 63f1ab809c4..1e1f6b992dc 100644 --- a/src/main/java/org/jabref/migrations/ConvertLegacyExplicitGroups.java +++ b/src/main/java/org/jabref/migrations/ConvertLegacyExplicitGroups.java @@ -36,8 +36,7 @@ private List getExplicitGroupsWithLegacyKeys(GroupTreeNode node) Objects.requireNonNull(node); List findings = new ArrayList<>(); - if (node.getGroup() instanceof ExplicitGroup) { - ExplicitGroup group = (ExplicitGroup) node.getGroup(); + if (node.getGroup() instanceof ExplicitGroup group) { if (!group.getLegacyEntryKeys().isEmpty()) { findings.add(group); } diff --git a/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java b/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java deleted file mode 100644 index c56fd255dec..00000000000 --- a/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.jabref.migrations; - -import org.jabref.model.search.ThrowingErrorListener; -import org.jabref.search.SearchLexer; -import org.jabref.search.SearchParser; - -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.BailErrorStrategy; -import org.antlr.v4.runtime.CommonTokenStream; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.EscapeQuerySyntaxImpl; - -/** - * @deprecated This class is deprecated and will be removed in the future. We use Search.g4 as main search grammar - */ -@Deprecated -public class SearchToLuceneMigration { - public static String migrateToLuceneSyntax(String searchExpression, boolean isRegularExpression) { - SearchParser.StartContext context = getStartContext(searchExpression); - SearchToLuceneVisitor searchToLuceneVisitor = new SearchToLuceneVisitor(isRegularExpression); - QueryNode luceneQueryNode = searchToLuceneVisitor.visit(context); - return luceneQueryNode.toQueryString(new EscapeQuerySyntaxImpl()).toString(); - } - - private static SearchParser.StartContext getStartContext(String searchExpression) { - SearchLexer lexer = new SearchLexer(new ANTLRInputStream(searchExpression)); - lexer.removeErrorListeners(); // no infos on file system - lexer.addErrorListener(ThrowingErrorListener.INSTANCE); - SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); - parser.removeErrorListeners(); // no infos on file system - parser.addErrorListener(ThrowingErrorListener.INSTANCE); - parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors - return parser.start(); - } -} diff --git a/src/main/java/org/jabref/model/search/query/SearchQuery.java b/src/main/java/org/jabref/model/search/query/SearchQuery.java index 2acb1e816e8..eb349c99c03 100644 --- a/src/main/java/org/jabref/model/search/query/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/query/SearchQuery.java @@ -9,11 +9,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import org.jabref.logic.search.query.SearchToSqlConversion; +import org.jabref.logic.search.query.SearchQueryConversion; import org.jabref.model.search.SearchFlags; -import org.apache.lucene.search.Query; - public class SearchQuery { /** * The mode of escaping special characters in regular expressions @@ -57,7 +55,7 @@ String format(String regex) { private String parseError; private String sqlQuery; - private Query luceneQuery; + private String luceneQuery; private SearchResults searchResults; public SearchQuery(String searchExpression, EnumSet searchFlags) { @@ -67,14 +65,14 @@ public SearchQuery(String searchExpression, EnumSet searchFlags) { public String getSqlQuery(String table) { if (sqlQuery == null) { - sqlQuery = SearchToSqlConversion.searchToSql(table, searchExpression); + sqlQuery = SearchQueryConversion.searchToSql(table, searchExpression); } return sqlQuery; } - public Query getLuceneQuery() { + public String getLuceneQuery() { if (luceneQuery == null) { - // TODO: convert to lucene query + luceneQuery = SearchQueryConversion.searchToLucene(searchExpression); } return luceneQuery; } diff --git a/src/main/java/org/jabref/model/search/query/SearchTermFlag.java b/src/main/java/org/jabref/model/search/query/SearchTermFlag.java new file mode 100644 index 00000000000..5707e80a4c2 --- /dev/null +++ b/src/main/java/org/jabref/model/search/query/SearchTermFlag.java @@ -0,0 +1,7 @@ +package org.jabref.model.search.query; + +public enum SearchTermFlag { + EXACT_MATCH, INEXACT_MATCH, REGULAR_EXPRESSION, // mutually exclusive + CASE_SENSITIVE, CASE_INSENSITIVE, // mutually exclusive + NEGATION, +} diff --git a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java b/src/test/java/org/jabref/logic/search/query/LuceneQueryParserTest.java similarity index 97% rename from src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java rename to src/test/java/org/jabref/logic/search/query/LuceneQueryParserTest.java index 8d495609da4..bf1429d1c11 100644 --- a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java +++ b/src/test/java/org/jabref/logic/search/query/LuceneQueryParserTest.java @@ -1,4 +1,4 @@ -package org.jabref.logic.search; +package org.jabref.logic.search.query; import java.util.stream.Stream; diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java new file mode 100644 index 00000000000..f7aba638a8e --- /dev/null +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java @@ -0,0 +1,90 @@ +package org.jabref.logic.search.query; + +import java.util.EnumSet; +import java.util.stream.Stream; + +import org.jabref.model.search.SearchFlags; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class SearchQueryFlagsConversionTest { + + private static Stream testSearchConversion() { + return Stream.of( + createTestCases( + "Term", + "any = Term", + "any =! Term", + "any =~ Term", + "any =~! Term" + ), + + createTestCases( + "title = Term", + "title = Term", + "title =! Term", + "title =~ Term", + "title =~! Term" + ), + createTestCases( + "title == Term", + "title == Term", + "title ==! Term", + "title =~ Term", + "title =~! Term" + ), + + createTestCases( + "title != Term", + "title != Term", + "title !=! Term", + "title !=~ Term", + "title !=~! Term" + ), + + createTestCases( + "title = Tem AND author = Alex", + "title = Tem AND author = Alex", + "title =! Tem AND author =! Alex", + "title =~ Tem AND author =~ Alex", + "title =~! Tem AND author =~! Alex" + ), + + createTestCases( + "(title = Tem) AND (author = Alex)", + "(title = Tem) AND (author = Alex)", + "(title =! Tem) AND (author =! Alex)", + "(title =~ Tem) AND (author =~ Alex)", + "(title =~! Tem) AND (author =~! Alex)" + ), + + createTestCases( + "(title = \"Tem\" AND author != Alex) OR term", + "(title = \"Tem\" AND author != Alex) OR any = term", + "(title =! \"Tem\" AND author !=! Alex) OR any =! term", + "(title =~ \"Tem\" AND author !=~ Alex) OR any =~ term", + "(title =~! \"Tem\" AND author !=~! Alex) OR any =~! term" + ) + ).flatMap(stream -> stream); + } + + private static Stream createTestCases(String query, String noneExpected, String caseSensitiveExpected, String regexExpected, String bothExpected) { + return Stream.of( + Arguments.of(noneExpected, query, EnumSet.noneOf(SearchFlags.class)), + Arguments.of(caseSensitiveExpected, query, EnumSet.of(SearchFlags.CASE_SENSITIVE)), + Arguments.of(regexExpected, query, EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), + Arguments.of(bothExpected, query, EnumSet.of(SearchFlags.CASE_SENSITIVE, SearchFlags.REGULAR_EXPRESSION)) + ); + } + + @ParameterizedTest + @MethodSource + void testSearchConversion(String expected, String query, EnumSet flags) { + String result = SearchQueryConversion.flagsToSearchExpression(query, flags); + assertEquals(expected, result); + } +} diff --git a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java similarity index 94% rename from src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java rename to src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java index 37ee05fb358..c84a196bfdd 100644 --- a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java @@ -1,4 +1,4 @@ -package org.jabref.migrations; +package org.jabref.logic.search.query; import java.util.stream.Stream; @@ -8,7 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -class SearchToLuceneMigrationTest { +class SearchQueryLuceneConversionTest { public static Stream transformationNormal() { return Stream.of( @@ -50,7 +50,7 @@ public static Stream transformationNormal() { @ParameterizedTest @MethodSource void transformationNormal(String expected, String query) { - String result = SearchToLuceneMigration.migrateToLuceneSyntax(query, false); + String result = SearchQueryConversion.searchToLucene(query); assertEquals(expected, result); } @@ -67,7 +67,7 @@ public static Stream transformationRegularExpression() { @ParameterizedTest @MethodSource void transformationRegularExpression(String expected, String query) { - String result = SearchToLuceneMigration.migrateToLuceneSyntax(query, true); + String result = SearchQueryConversion.searchToLucene(query); assertEquals(expected, result); } } diff --git a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java similarity index 97% rename from src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java rename to src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java index 5bd5453040c..8e3bf5f2a3d 100644 --- a/src/test/java/org/jabref/logic/search/SearchToSqlConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java @@ -1,13 +1,11 @@ -package org.jabref.logic.search; - -import org.jabref.logic.search.query.SearchToSqlConversion; +package org.jabref.logic.search.query; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import static org.junit.jupiter.api.Assertions.assertEquals; -class SearchToSqlConversionTest { +class SearchQuerySQLConversionTest { @ParameterizedTest @CsvSource({ "(), alex", @@ -87,6 +85,6 @@ class SearchToSqlConversionTest { }) void conversion(String expectedWhereClause, String input) { - assertEquals(expectedWhereClause, SearchToSqlConversion.searchToSql("tableName", input)); + assertEquals(expectedWhereClause, SearchQueryConversion.searchToSql("tableName", input)); } } From 1b1f0ff30554a62a90663c3df719b8fe1cf901ce Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 4 Oct 2024 19:08:14 +0300 Subject: [PATCH 054/104] Localization --- .../gui/importer/actions/SearchGroupsMigrationAction.java | 5 ++--- src/main/resources/l10n/JabRef_en.properties | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index 1b7376698b4..a2863a362d0 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -30,9 +30,8 @@ public boolean isActionNecessary(ParserResult parserResult, DialogService dialog Optional currentVersion = parserResult.getMetaData().getGroupSearchSyntaxVersion(); if (currentVersion.isPresent()) { if (currentVersion.get().equals(VERSION_6_0_ALPHA)) { - dialogService.showErrorDialogAndWait(Localization.lang("Search groups migration"), - Localization.lang("The search groups syntax has been reverted to the old one. " - + "Please use the backup you made before migrating to 6.0-alpha.")); + dialogService.showErrorDialogAndWait(Localization.lang("Search groups migration of %0", parserResult.getPath().map(Path::toString).orElse(""), + Localization.lang("The search groups syntax has been reverted to the old one. Please use the backup you made before migrating to 6.0-alpha."))); } return false; } diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 454a38cf4a6..5ca0112a42a 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -271,6 +271,7 @@ Each\ line\ must\ be\ of\ the\ following\ form\:\ \'tab\:field1;field2;...;field Search\ groups\ migration\ of\ %0=Search groups migration of %0 The\ search\ groups\ syntax\ is\ outdated.\ Do\ you\ want\ to\ migrate\ to\ the\ new\ syntax?=The search groups syntax is outdated. Do you want to migrate to the new syntax? +The\ search\ groups\ syntax\ has\ been\ reverted\ to\ the\ old\ one.\ Please\ use\ the\ backup\ you\ made\ before\ migrating\ to\ 6.0-alpha. = The search groups syntax has been reverted to the old one. Please use the backup you made before migrating to 6.0-alpha. Migrate=Migrate Keep\ as\ is=Keep as is Search\ group\ migration\ failed=Search group migration failed From 855cf3cb08ab6c2bac83bcb68aa943b587d0f28c Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 4 Oct 2024 19:15:31 +0300 Subject: [PATCH 055/104] Fix dialog message --- .../gui/importer/actions/SearchGroupsMigrationAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index a2863a362d0..e1a4cc3daee 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -30,8 +30,8 @@ public boolean isActionNecessary(ParserResult parserResult, DialogService dialog Optional currentVersion = parserResult.getMetaData().getGroupSearchSyntaxVersion(); if (currentVersion.isPresent()) { if (currentVersion.get().equals(VERSION_6_0_ALPHA)) { - dialogService.showErrorDialogAndWait(Localization.lang("Search groups migration of %0", parserResult.getPath().map(Path::toString).orElse(""), - Localization.lang("The search groups syntax has been reverted to the old one. Please use the backup you made before migrating to 6.0-alpha."))); + dialogService.showErrorDialogAndWait(Localization.lang("Search groups migration of %0", parserResult.getPath().map(Path::toString).orElse("")), + Localization.lang("The search groups syntax has been reverted to the old one. Please use the backup you made before migrating to 6.0-alpha.")); } return false; } From 9a35891a73d30964d0d8ccb62327c70a27f19416 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 4 Oct 2024 20:47:01 +0300 Subject: [PATCH 056/104] Ignores groups field from default searches Fixes https://github.com/JabRef/jabref/issues/7996 --- .../search/query/SearchToSqlVisitor.java | 160 ++++++++++-------- 1 file changed, 90 insertions(+), 70 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 5f54170ee94..63592d2361f 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -29,6 +29,7 @@ /** * Converts to a query processable by the scheme created by {@link BibFieldsIndexer}. + * Tests are located in {@link org.jabref.logic.search.query.SearchQuerySQLConversionTest}. */ public class SearchToSqlVisitor extends SearchBaseVisitor { @@ -36,6 +37,7 @@ public class SearchToSqlVisitor extends SearchBaseVisitor { private static final String MAIN_TABLE = "main_table"; private static final String SPLIT_TABLE = "split_table"; private static final String INNER_TABLE = "inner_table"; + private static final String GROUPS_FIELD = StandardField.GROUPS.getName(); private final String mainTableName; private final String splitValuesTableName; @@ -70,12 +72,12 @@ public String visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { String subQuery = visit(ctx.expression()); String cte = """ cte%d AS ( - SELECT %s.%s - FROM %s AS %s - WHERE %s.%s NOT IN ( - SELECT %s - FROM %s - ) + SELECT %s.%s + FROM %s AS %s + WHERE %s.%s NOT IN ( + SELECT %s + FROM %s + ) ) """.formatted( cteCounter, @@ -96,11 +98,11 @@ public String visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { String cte = """ cte%d AS ( - SELECT %s - FROM %s - %s - SELECT %s - FROM %s + SELECT %s + FROM %s + %s + SELECT %s + FROM %s ) """.formatted( cteCounter, @@ -196,9 +198,9 @@ private String getFieldQueryNode(String field, String term, EnumSet Date: Fri, 4 Oct 2024 21:24:49 +0300 Subject: [PATCH 057/104] Use TYPE_HEADER field for entrytype --- .../search/indexing/BibFieldsIndexer.java | 121 +++++++++--------- 1 file changed, 58 insertions(+), 63 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index 9ce3e8d566a..bc39accde09 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -22,11 +22,11 @@ import org.jabref.model.entry.field.FieldProperty; import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.PostgreConstants; -import org.jabref.model.search.SearchFieldConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.jabref.model.entry.field.InternalField.TYPE_HEADER; import static org.jabref.model.search.PostgreConstants.ENTRY_ID; import static org.jabref.model.search.PostgreConstants.FIELD_NAME; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_LITERAL; @@ -240,12 +240,7 @@ private void addToIndex(BibEntry bibEntry) { } // add entry type - // Separate code, because ENTRY_TYPE is not a Field - preparedStatement.setString(1, entryId); - preparedStatement.setString(2, SearchFieldConstants.ENTRY_TYPE.toString()); - preparedStatement.setString(3, bibEntry.getType().getName()); - preparedStatement.setString(4, bibEntry.getType().getName()); - preparedStatement.addBatch(); + addBatch(preparedStatement, entryId, TYPE_HEADER, bibEntry.getType().getName()); preparedStatement.executeBatch(); preparedStatementSplitValues.executeBatch(); @@ -254,58 +249,6 @@ private void addToIndex(BibEntry bibEntry) { } } - private void addEntryLinks(BibEntry bibEntry, Field field, PreparedStatement preparedStatementSplitValues, String entryId) { - bibEntry.getEntryLinkList(field, databaseContext.getDatabase()).stream().distinct().forEach(link -> { - addBatch(preparedStatementSplitValues, entryId, field, link.getKey()); - }); - } - - private static void addGroups(String value, PreparedStatement preparedStatementSplitValues, String entryId, Field field) { - // We could use KeywordList, but we are afraid that group names could have ">" in their name, and then they would not be handled correctly - Arrays.stream(GROUPS_SEPARATOR_REGEX.split(value)) - .distinct() - .forEach(group -> { - addBatch(preparedStatementSplitValues, entryId, field, group); - }); - } - - private static void addKeywords(String keywordsString, PreparedStatement preparedStatementSplitValues, String entryId, Field field, Character keywordSeparator) { - KeywordList keywordList = KeywordList.parse(keywordsString, keywordSeparator); - keywordList.stream().flatMap(keyword -> keyword.flatten().stream()).forEach(keyword -> { - String value = keyword.toString(); - addBatch(preparedStatementSplitValues, entryId, field, value); - }); - } - - private static void addAuthors(String value, PreparedStatement preparedStatementSplitValues, String entryId, Field field) { - AuthorList.parse(value).getAuthors().forEach(author -> { - // Author object does not support literal values - // We use the method giving us the most complete information for the literal value; - String literal = author.getGivenFamily(false); - String transformed = author.latexFree().getGivenFamily(false); - addBatch(preparedStatementSplitValues, entryId, field, literal, transformed); - }); - } - - private static void addBatch(PreparedStatement preparedStatement, String entryId, Field field, String value) { - addBatch(preparedStatement, entryId, field, value, LATEX_TO_UNICODE_FORMATTER.format(value)); - } - - /** - * The values are passed as they should be inserted into the database table - */ - private static void addBatch(PreparedStatement preparedStatement, String entryId, Field field, String value, String normalized) { - try { - preparedStatement.setString(1, entryId); - preparedStatement.setString(2, field.getName()); - preparedStatement.setString(3, value); - preparedStatement.setString(4, normalized); - preparedStatement.addBatch(); - } catch (SQLException e) { - LOGGER.error("Could not add field {} having value {} of entry {} to the index.", field.getName(), value, entryId, e); - } - } - public void removeFromIndex(Collection entries, BackgroundTask task) { if (entries.size() > 1) { task.showToUser(true); @@ -415,10 +358,6 @@ private void removeField(BibEntry entry, Field field) { } } - public String getTable() { - return mainTable; - } - public void close() { HeadlessExecutorService.INSTANCE.execute(this::closeIndex); } @@ -441,4 +380,60 @@ private void closeIndex() { LOGGER.error("Could not drop table for library: {}", libraryName, e); } } + + public String getTable() { + return mainTable; + } + + private void addEntryLinks(BibEntry bibEntry, Field field, PreparedStatement preparedStatementSplitValues, String entryId) { + bibEntry.getEntryLinkList(field, databaseContext.getDatabase()).stream().distinct().forEach(link -> { + addBatch(preparedStatementSplitValues, entryId, field, link.getKey()); + }); + } + + private static void addGroups(String value, PreparedStatement preparedStatementSplitValues, String entryId, Field field) { + // We could use KeywordList, but we are afraid that group names could have ">" in their name, and then they would not be handled correctly + Arrays.stream(GROUPS_SEPARATOR_REGEX.split(value)) + .distinct() + .forEach(group -> { + addBatch(preparedStatementSplitValues, entryId, field, group); + }); + } + + private static void addKeywords(String keywordsString, PreparedStatement preparedStatementSplitValues, String entryId, Field field, Character keywordSeparator) { + KeywordList keywordList = KeywordList.parse(keywordsString, keywordSeparator); + keywordList.stream().flatMap(keyword -> keyword.flatten().stream()).forEach(keyword -> { + String value = keyword.toString(); + addBatch(preparedStatementSplitValues, entryId, field, value); + }); + } + + private static void addAuthors(String value, PreparedStatement preparedStatementSplitValues, String entryId, Field field) { + AuthorList.parse(value).getAuthors().forEach(author -> { + // Author object does not support literal values + // We use the method giving us the most complete information for the literal value; + String literal = author.getGivenFamily(false); + String transformed = author.latexFree().getGivenFamily(false); + addBatch(preparedStatementSplitValues, entryId, field, literal, transformed); + }); + } + + private static void addBatch(PreparedStatement preparedStatement, String entryId, Field field, String value) { + addBatch(preparedStatement, entryId, field, value, LATEX_TO_UNICODE_FORMATTER.format(value)); + } + + /** + * The values are passed as they should be inserted into the database table + */ + private static void addBatch(PreparedStatement preparedStatement, String entryId, Field field, String value, String normalized) { + try { + preparedStatement.setString(1, entryId); + preparedStatement.setString(2, field.getName()); + preparedStatement.setString(3, value); + preparedStatement.setString(4, normalized); + preparedStatement.addBatch(); + } catch (SQLException e) { + LOGGER.error("Could not add field {} having value {} of entry {} to the index.", field.getName(), value, entryId, e); + } + } } From 8450f8918d5d681252613fb3540f7fad2afabe5e Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sat, 5 Oct 2024 03:31:23 +0300 Subject: [PATCH 058/104] Search to Lucene query for linked files searching --- src/main/java/module-info.java | 4 +- .../indexing/DefaultLinkedFilesIndexer.java | 12 +- .../logic/search/indexing/DocumentReader.java | 10 +- .../query/SearchFlagsToExpressionVisitor.java | 3 + .../search/query/SearchQueryConversion.java | 6 +- .../search/query/SearchToLuceneVisitor.java | 204 +++++++++--------- .../search/query/SearchToSqlVisitor.java | 3 +- .../search/retrieval/LinkedFilesSearcher.java | 12 +- .../org/jabref/logic/util/Directories.java | 4 +- ...nstants.java => LinkedFilesConstants.java} | 10 +- .../search/analyzer/LatexAwareAnalyzer.java | 26 --- .../analyzer/LatexToUnicodeFoldingFilter.java | 89 -------- .../model/search/query/SearchQuery.java | 6 +- .../model/search/query/SearchResult.java | 10 +- .../search/query/LuceneQueryParserTest.java | 40 ---- .../SearchQueryLuceneConversionTest.java | 83 ++++--- 16 files changed, 175 insertions(+), 347 deletions(-) rename src/main/java/org/jabref/model/search/{SearchFieldConstants.java => LinkedFilesConstants.java} (75%) delete mode 100644 src/main/java/org/jabref/model/search/analyzer/LatexAwareAnalyzer.java delete mode 100644 src/main/java/org/jabref/model/search/analyzer/LatexToUnicodeFoldingFilter.java delete mode 100644 src/test/java/org/jabref/logic/search/query/LuceneQueryParserTest.java diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 11c2fcf9608..a2f598be32c 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -163,8 +163,8 @@ // endregion // region: Lucene - /** - * In case the version is updated, please also increment {@link org.jabref.model.search.SearchFieldConstants#VERSION} to trigger reindexing. + /* + * In case the version is updated, please also increment {@link org.jabref.model.search.LinkedFilesConstants.VERSION} to trigger reindexing. */ uses org.apache.lucene.codecs.lucene99.Lucene99Codec; requires org.apache.lucene.analysis.common; diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index 78c9d736e72..50683c8d8f6 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -26,7 +26,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; -import org.jabref.model.search.SearchFieldConstants; +import org.jabref.model.search.LinkedFilesConstants; import org.apache.commons.io.FileUtils; import org.apache.lucene.document.Document; @@ -65,7 +65,7 @@ public DefaultLinkedFilesIndexer(BibDatabaseContext databaseContext, FilePrefere this.indexedFiles = new ConcurrentHashMap<>(); indexDirectoryPath = databaseContext.getFulltextIndexPath(); - IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.LINKED_FILES_ANALYZER); + IndexWriterConfig config = new IndexWriterConfig(LinkedFilesConstants.LINKED_FILES_ANALYZER); if ("unsaved".equals(indexDirectoryPath.getFileName().toString())) { config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); indexDirectoryPath = indexDirectoryPath.resolveSibling("unsaved" + NUMBER_OF_UNSAVED_LIBRARIES++); @@ -192,7 +192,7 @@ private void removeFromIndex(Set links) { for (String fileLink : links) { try { LOGGER.debug("Removing file {} from index.", fileLink); - indexWriter.deleteDocuments(new Term(SearchFieldConstants.PATH.toString(), fileLink)); + indexWriter.deleteDocuments(new Term(LinkedFilesConstants.PATH.toString(), fileLink)); indexedFiles.remove(fileLink); } catch (IOException e) { LOGGER.warn("Could not remove linked file {} from index.", fileLink, e); @@ -236,15 +236,15 @@ private Map getLinkedFilesFromIndex() { LOGGER.debug("Getting all linked files from index."); Map linkedFiles = new HashMap<>(); try { - TermQuery query = new TermQuery(new Term(SearchFieldConstants.PAGE_NUMBER.toString(), "1")); + TermQuery query = new TermQuery(new Term(LinkedFilesConstants.PAGE_NUMBER.toString(), "1")); searcherManager.maybeRefresh(); IndexSearcher searcher = searcherManager.acquire(); StoredFields storedFields = searcher.storedFields(); TopDocs allDocs = searcher.search(query, Integer.MAX_VALUE); for (ScoreDoc scoreDoc : allDocs.scoreDocs) { Document doc = storedFields.document(scoreDoc.doc); - var pathField = doc.getField(SearchFieldConstants.PATH.toString()); - var modifiedField = doc.getField(SearchFieldConstants.MODIFIED.toString()); + var pathField = doc.getField(LinkedFilesConstants.PATH.toString()); + var modifiedField = doc.getField(LinkedFilesConstants.MODIFIED.toString()); if (pathField != null && modifiedField != null) { linkedFiles.put(pathField.stringValue(), Long.valueOf(modifiedField.stringValue())); } diff --git a/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java b/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java index eca0723827a..b0437d45a12 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java +++ b/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java @@ -23,11 +23,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.jabref.model.search.SearchFieldConstants.ANNOTATIONS; -import static org.jabref.model.search.SearchFieldConstants.CONTENT; -import static org.jabref.model.search.SearchFieldConstants.MODIFIED; -import static org.jabref.model.search.SearchFieldConstants.PAGE_NUMBER; -import static org.jabref.model.search.SearchFieldConstants.PATH; +import static org.jabref.model.search.LinkedFilesConstants.ANNOTATIONS; +import static org.jabref.model.search.LinkedFilesConstants.CONTENT; +import static org.jabref.model.search.LinkedFilesConstants.MODIFIED; +import static org.jabref.model.search.LinkedFilesConstants.PAGE_NUMBER; +import static org.jabref.model.search.LinkedFilesConstants.PATH; /** * Utility class for reading the data from LinkedFiles of a BibEntry for Lucene. diff --git a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java index 38b0f6ae4ad..3f10249cc33 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java @@ -18,6 +18,9 @@ import static org.jabref.model.search.query.SearchTermFlag.NEGATION; import static org.jabref.model.search.query.SearchTermFlag.REGULAR_EXPRESSION; +/** + * Tests are located in {@link org.jabref.logic.search.query.SearchQueryFlagsConversionTest}. + */ public class SearchFlagsToExpressionVisitor extends SearchBaseVisitor { private static final Logger LOGGER = LoggerFactory.getLogger(SearchFlagsToExpressionVisitor.class); diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java index 932fec56eaa..5b699b68026 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java @@ -10,6 +10,7 @@ import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonTokenStream; +import org.apache.lucene.search.Query; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,9 +30,10 @@ public static String flagsToSearchExpression(String searchExpression, EnumSet - * Tests are located in {@link org.jabref.migrations.SearchToLuceneMigrationTest}. + * Tests are located in {@link org.jabref.logic.search.query.SearchToLuceneVisitor}. */ -public class SearchToLuceneVisitor extends SearchBaseVisitor { +public class SearchToLuceneVisitor extends SearchBaseVisitor { - private static final Logger LOGGER = LoggerFactory.getLogger(SearchToLuceneVisitor.class); + private static final List SEARCH_FIELDS = LinkedFilesConstants.PDF_FIELDS; - private final boolean isRegularExpression; + private final QueryBuilder queryBuilder; - public SearchToLuceneVisitor(boolean isRegularExpression) { - this.isRegularExpression = isRegularExpression; + public SearchToLuceneVisitor() { + this.queryBuilder = new QueryBuilder(LinkedFilesConstants.LINKED_FILES_ANALYZER); } @Override - public QueryNode visitStart(SearchParser.StartContext ctx) { - QueryNode result = visit(ctx.expression()); - - // If user searches for a single negation, Lucene also (!) interprets it as filter on the entities matched by the other terms - // We need to add a "filter" to match all entities - // See https://github.com/LoayGhreeb/lucene-mwe/issues/1 for more details - if (result instanceof ModifierQueryNode modifierQueryNode) { - if (modifierQueryNode.getModifier() == ModifierQueryNode.Modifier.MOD_NOT) { - return new AndQueryNode(List.of(new FieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), "*", 0, 0), modifierQueryNode)); - } - } - - // User might search for NOT this AND NOT that - we also need to convert properly - if (result instanceof AndQueryNode andQueryNode) { - if (andQueryNode.getChildren().stream().allMatch(child -> child instanceof ModifierQueryNode modifierQueryNode && modifierQueryNode.getModifier() == ModifierQueryNode.Modifier.MOD_NOT)) { - List children = andQueryNode.getChildren().stream() - // prepend "any:* AND" to each child - .map(child -> new AndQueryNode(List.of(new FieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), "*", 0, 0), child))) - .map(child -> (QueryNode) child) - .toList(); - return new AndQueryNode(children); - } - } - - return result; + public Query visitStart(SearchParser.StartContext ctx) { + return visit(ctx.expression()); } @Override - public QueryNode visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { - return new ModifierQueryNode(visit(ctx.expression()), ModifierQueryNode.Modifier.MOD_NOT); + public Query visitParenExpression(SearchParser.ParenExpressionContext ctx) { + return visit(ctx.expression()); } @Override - public QueryNode visitParenExpression(SearchParser.ParenExpressionContext ctx) { - return new GroupQueryNode(visit(ctx.expression())); + public Query visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { + Query innerQuery = visit(ctx.expression()); + if (innerQuery instanceof MatchNoDocsQuery) { + return innerQuery; + } + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(innerQuery, BooleanClause.Occur.MUST_NOT); + return builder.build(); } @Override - public QueryNode visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { - if ("AND".equalsIgnoreCase(ctx.operator.getText())) { - return new AndQueryNode(List.of(visit(ctx.left), visit(ctx.right))); - } else { - return new OrQueryNode(List.of(visit(ctx.left), visit(ctx.right))); + public Query visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { + Query left = visit(ctx.left); + Query right = visit(ctx.right); + + if (left instanceof MatchNoDocsQuery) { + return right; + } + if (right instanceof MatchNoDocsQuery) { + return left; + } + + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + + if (ctx.operator.getType() == SearchParser.AND) { + builder.add(left, BooleanClause.Occur.MUST); + builder.add(right, BooleanClause.Occur.MUST); + } else if (ctx.operator.getType() == SearchParser.OR) { + builder.add(left, BooleanClause.Occur.SHOULD); + builder.add(right, BooleanClause.Occur.SHOULD); } + + return builder.build(); } @Override - public QueryNode visitComparison(SearchParser.ComparisonContext context) { - // The comparison is a leaf node in the tree + public Query visitComparison(SearchParser.ComparisonContext ctx) { + String field = ctx.left != null ? ctx.left.getText().toLowerCase() : null; + String term = ctx.right.getText(); - // remove possible enclosing " symbols - String right = context.right.getText(); - if (right.startsWith("\"") && right.endsWith("\"")) { - right = right.substring(1, right.length() - 1); + if (term.startsWith("\"") && term.endsWith("\"")) { + term = term.substring(1, term.length() - 1); } - Optional fieldDescriptor = Optional.ofNullable(context.left); - int startIndex = context.getStart().getStartIndex(); - int stopIndex = context.getStop().getStopIndex(); - if (fieldDescriptor.isPresent()) { - String field = fieldDescriptor.get().getText(); - - // Direct comparison does not work - // context.CONTAINS() and others are null if absent (thus, we cannot check for getText()) - if (context.CONTAINS() != null || - context.MATCHES() != null || - context.EQUAL() != null || - context.EEQUAL() != null) { // exact match - if (LOGGER.isDebugEnabled() && context.EEQUAL() != null) { - LOGGER.warn("Exact match is currently not supported by Lucene, using contains instead. Term: {}", context.getText()); - } - return getFieldQueryNode(field, right, startIndex, stopIndex, false); - } - - assert (context.NEQUAL() != null); - - // Treating of "wrong" query field != "". This did not work in v5.x, but should work in v6.x - boolean forceRegex; - if (right.isEmpty()) { - forceRegex = true; - right = ".+"; - } else { - forceRegex = false; - } - - return new ModifierQueryNode(getFieldQueryNode(field, right, startIndex, stopIndex, forceRegex), ModifierQueryNode.Modifier.MOD_NOT); + if (field == null || "anyfield".equals(field) || "any".equals(field)) { + return createMultiFieldQuery(term, ctx.operator); + } else if (SEARCH_FIELDS.contains(field)) { + return createFieldQuery(field, term, ctx.operator); } else { - return getFieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), right, startIndex, stopIndex, false); + return new MatchNoDocsQuery(); + } + } + + private Query createMultiFieldQuery(String value, Token operator) { + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + for (String field : SEARCH_FIELDS) { + builder.add(createFieldQuery(field, value, operator), BooleanClause.Occur.SHOULD); } + return builder.build(); } - /** - * A search query can be either a regular expression or a normal query. - * In Lucene, this is represented by a RegexpQueryNode or a FieldQueryNode. - * They are created in this class accordingly. - */ - private QueryNode getFieldQueryNode(String field, String term, int startIndex, int stopIndex, boolean forceRegex) { - field = switch (field) { - case "anyfield" -> SearchFieldConstants.DEFAULT_FIELD.toString(); - case "anykeyword" -> StandardField.KEYWORDS.getName(); - case "key" -> InternalField.KEY_FIELD.getName(); - default -> field; + private Query createFieldQuery(String field, String value, Token operator) { + if (operator == null) { + return createTermOrPhraseQuery(field, value); + } + + return switch (operator.getType()) { + case SearchParser.REQUAL, + SearchParser.CREEQUAL -> + new RegexpQuery(new Term(field, value)); + case SearchParser.NEQUAL, + SearchParser.NCEQUAL, + SearchParser.NEEQUAL, + SearchParser.NCEEQUAL -> + createNegatedQuery(createTermOrPhraseQuery(field, value)); + case SearchParser.NREQUAL, + SearchParser.NCREEQUAL -> + createNegatedQuery(new RegexpQuery(new Term(field, value))); + default -> + createTermOrPhraseQuery(field, value); }; + } + + private Query createNegatedQuery(Query query) { + BooleanQuery.Builder negatedQuery = new BooleanQuery.Builder(); + negatedQuery.add(query, BooleanClause.Occur.MUST_NOT); + return negatedQuery.build(); + } - if (isRegularExpression || forceRegex) { - // Lucene does a sanity check on the positions, thus we provide other fake positions - return new RegexpQueryNode(field, term, 0, term.length()); + private Query createTermOrPhraseQuery(String field, String value) { + if (value.contains("*") || value.contains("?")) { + return new TermQuery(new Term(field, value)); } - return new FieldQueryNode(field, term, startIndex, stopIndex); + return queryBuilder.createPhraseQuery(field, value); } } diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 63592d2361f..906dae1ba7a 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import java.util.Locale; import java.util.Optional; import org.jabref.logic.search.indexing.BibFieldsIndexer; @@ -174,7 +175,7 @@ public String visitComparison(SearchParser.ComparisonContext context) { setFlags(searchFlags, REGULAR_EXPRESSION, true, true); } - cte = getFieldQueryNode(field, right, searchFlags); + cte = getFieldQueryNode(field.toLowerCase(Locale.ROOT), right, searchFlags); } else { // Query without any field name cte = getFieldQueryNode("any", right, EnumSet.of(INEXACT_MATCH, CASE_INSENSITIVE)); diff --git a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java index c38d70a89d4..ec33ed17055 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java @@ -13,7 +13,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; -import org.jabref.model.search.SearchFieldConstants; +import org.jabref.model.search.LinkedFilesConstants; import org.jabref.model.search.SearchFlags; import org.jabref.model.search.query.SearchResult; import org.jabref.model.search.query.SearchResults; @@ -78,16 +78,16 @@ private SearchResults getSearchResults(TopDocs topDocs, StoredFields storedField for (ScoreDoc scoreDoc : topDocs.scoreDocs) { Document document = storedFields.document(scoreDoc.doc); - String fileLink = getFieldContents(document, SearchFieldConstants.PATH); + String fileLink = getFieldContents(document, LinkedFilesConstants.PATH); if (!fileLink.isEmpty()) { List entriesWithFile = linkedFilesMap.get(fileLink); if (entriesWithFile != null && !entriesWithFile.isEmpty()) { SearchResult searchResult = new SearchResult( fileLink, - getFieldContents(document, SearchFieldConstants.CONTENT), - getFieldContents(document, SearchFieldConstants.ANNOTATIONS), - Integer.parseInt(getFieldContents(document, SearchFieldConstants.PAGE_NUMBER)), + getFieldContents(document, LinkedFilesConstants.CONTENT), + getFieldContents(document, LinkedFilesConstants.ANNOTATIONS), + Integer.parseInt(getFieldContents(document, LinkedFilesConstants.PAGE_NUMBER)), highlighter); searchResults.addSearchResult(entriesWithFile, searchResult); } @@ -108,7 +108,7 @@ private Map> getLinkedFilesMap() { return linkedFilesMap; } - private static String getFieldContents(Document document, SearchFieldConstants field) { + private static String getFieldContents(Document document, LinkedFilesConstants field) { return Optional.ofNullable(document.get(field.toString())).orElse(""); } diff --git a/src/main/java/org/jabref/logic/util/Directories.java b/src/main/java/org/jabref/logic/util/Directories.java index 00396975da7..89c4027d6cc 100644 --- a/src/main/java/org/jabref/logic/util/Directories.java +++ b/src/main/java/org/jabref/logic/util/Directories.java @@ -5,7 +5,7 @@ import org.jabref.logic.ai.AiService; import org.jabref.logic.os.OS; -import org.jabref.model.search.SearchFieldConstants; +import org.jabref.model.search.LinkedFilesConstants; import net.harawata.appdirs.AppDirsFactory; @@ -45,7 +45,7 @@ public static Path getBackupDirectory() { public static Path getFulltextIndexBaseDirectory() { return Path.of(AppDirsFactory.getInstance() .getUserDataDir(OS.APP_DIR_APP_NAME, - "lucene" + File.separator + SearchFieldConstants.VERSION, + "lucene" + File.separator + LinkedFilesConstants.VERSION, OS.APP_DIR_APP_AUTHOR)); } diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/LinkedFilesConstants.java similarity index 75% rename from src/main/java/org/jabref/model/search/SearchFieldConstants.java rename to src/main/java/org/jabref/model/search/LinkedFilesConstants.java index c64f88d7884..196146cb071 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/LinkedFilesConstants.java @@ -2,12 +2,10 @@ import java.util.List; -import org.jabref.model.search.analyzer.LatexAwareAnalyzer; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.en.EnglishAnalyzer; -public enum SearchFieldConstants { +public enum LinkedFilesConstants { /** * Version number for the search index. * Increment when: @@ -16,9 +14,6 @@ public enum SearchFieldConstants { * Incrementing triggers reindexing. */ VERSION("1"), - DEFAULT_FIELD("any"), - ENTRY_ID("id"), - ENTRY_TYPE("entrytype"), PATH("path"), CONTENT("content"), ANNOTATIONS("annotations"), @@ -26,11 +21,10 @@ public enum SearchFieldConstants { MODIFIED("modified"); public static final Analyzer LINKED_FILES_ANALYZER = new EnglishAnalyzer(); - public static final Analyzer LATEX_AWARE_ANALYZER = new LatexAwareAnalyzer(); public static final List PDF_FIELDS = List.of(CONTENT.toString(), ANNOTATIONS.toString()); private final String field; - SearchFieldConstants(String field) { + LinkedFilesConstants(String field) { this.field = field; } diff --git a/src/main/java/org/jabref/model/search/analyzer/LatexAwareAnalyzer.java b/src/main/java/org/jabref/model/search/analyzer/LatexAwareAnalyzer.java deleted file mode 100644 index 437575ff810..00000000000 --- a/src/main/java/org/jabref/model/search/analyzer/LatexAwareAnalyzer.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.jabref.model.search.analyzer; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.LowerCaseFilter; -import org.apache.lucene.analysis.StopFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.core.WhitespaceTokenizer; -import org.apache.lucene.analysis.en.EnglishAnalyzer; -import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; - -/** - * Lucene analyzer respecting the special "features" of JabRef. - * Especially, LaTeX-encoded text. - */ -public class LatexAwareAnalyzer extends Analyzer { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new WhitespaceTokenizer(); - TokenStream result = new LatexToUnicodeFoldingFilter(source); - result = new StopFilter(result, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); - result = new ASCIIFoldingFilter(result); - result = new LowerCaseFilter(result); - return new TokenStreamComponents(source, result); - } -} diff --git a/src/main/java/org/jabref/model/search/analyzer/LatexToUnicodeFoldingFilter.java b/src/main/java/org/jabref/model/search/analyzer/LatexToUnicodeFoldingFilter.java deleted file mode 100644 index 9bea4abb467..00000000000 --- a/src/main/java/org/jabref/model/search/analyzer/LatexToUnicodeFoldingFilter.java +++ /dev/null @@ -1,89 +0,0 @@ -package org.jabref.model.search.analyzer; - -import java.io.IOException; -import java.util.Arrays; - -import org.jabref.architecture.AllowedToUseLogic; -import org.jabref.logic.cleanup.Formatter; -import org.jabref.logic.layout.format.LatexToUnicodeFormatter; - -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @implNote Implementation based on {@link org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter} - */ -@AllowedToUseLogic("because it needs access to the LaTeXToUnicodeFormatter") -public final class LatexToUnicodeFoldingFilter extends TokenFilter { - private static final Logger LOGGER = LoggerFactory.getLogger(LatexToUnicodeFoldingFilter.class); - private static final Formatter FORMATTER = new LatexToUnicodeFormatter(); - - private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); - private final PositionIncrementAttribute posIncAttr = - addAttribute(PositionIncrementAttribute.class); - - private State state; - - public LatexToUnicodeFoldingFilter(TokenStream input) { - super(input); - } - - private record FoldingResult(char[] output, int length) { - } - - @Override - public boolean incrementToken() throws IOException { - if (state != null) { - restoreState(state); - posIncAttr.setPositionIncrement(0); - state = null; - return true; - } - if (input.incrementToken()) { - final char[] buffer = termAtt.buffer(); - final int length = termAtt.length(); - FoldingResult foldingResult = foldToUnicode(buffer, length); - termAtt.copyBuffer(foldingResult.output, 0, foldingResult.length); - return true; - } else { - return false; - } - } - - @Override - public void reset() throws IOException { - super.reset(); - state = null; - } - - /** - * @param input The string to fold - * @param length The number of characters in the input string - */ - public FoldingResult foldToUnicode(char[] input, int length) { - FoldingResult result = foldToUnicode(input, 0, length); - if (result.length != length) { - // ASCIIFoldingFilter does "state = captureState();" - // We do not do anything since the index also contains clean LaTeX only. - // If we capture the state, the result is Synonym(LaTeX, Unicode) - } - return result; - } - - /** - * @param input The characters to fold - * @param inputPos Index of the first character to fold - * @param length The number of characters to fold - */ - public static FoldingResult foldToUnicode(char[] input, int inputPos, int length) { - char[] subArray = Arrays.copyOfRange(input, inputPos, inputPos + length); - String s = new String(subArray); - String result = FORMATTER.format(s); - LOGGER.trace("Folding {} to {}", s, result); - return new FoldingResult(result.toCharArray(), result.length()); - } -} diff --git a/src/main/java/org/jabref/model/search/query/SearchQuery.java b/src/main/java/org/jabref/model/search/query/SearchQuery.java index eb349c99c03..3a20912fff4 100644 --- a/src/main/java/org/jabref/model/search/query/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/query/SearchQuery.java @@ -12,6 +12,8 @@ import org.jabref.logic.search.query.SearchQueryConversion; import org.jabref.model.search.SearchFlags; +import org.apache.lucene.search.Query; + public class SearchQuery { /** * The mode of escaping special characters in regular expressions @@ -55,7 +57,7 @@ String format(String regex) { private String parseError; private String sqlQuery; - private String luceneQuery; + private Query luceneQuery; private SearchResults searchResults; public SearchQuery(String searchExpression, EnumSet searchFlags) { @@ -70,7 +72,7 @@ public String getSqlQuery(String table) { return sqlQuery; } - public String getLuceneQuery() { + public Query getLuceneQuery() { if (luceneQuery == null) { luceneQuery = SearchQueryConversion.searchToLucene(searchExpression); } diff --git a/src/main/java/org/jabref/model/search/query/SearchResult.java b/src/main/java/org/jabref/model/search/query/SearchResult.java index dde7c933e26..f8216d02818 100644 --- a/src/main/java/org/jabref/model/search/query/SearchResult.java +++ b/src/main/java/org/jabref/model/search/query/SearchResult.java @@ -4,7 +4,7 @@ import java.util.Arrays; import java.util.List; -import org.jabref.model.search.SearchFieldConstants; +import org.jabref.model.search.LinkedFilesConstants; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.search.highlight.Highlighter; @@ -46,14 +46,14 @@ public SearchResult(String path, String pageContent, String annotation, int page public List getContentResultStringsHtml() { if (contentResultStringsHtml == null) { - return contentResultStringsHtml = getHighlighterFragments(highlighter, SearchFieldConstants.CONTENT, pageContent); + return contentResultStringsHtml = getHighlighterFragments(highlighter, LinkedFilesConstants.CONTENT, pageContent); } return contentResultStringsHtml; } public List getAnnotationsResultStringsHtml() { if (annotationsResultStringsHtml == null) { - annotationsResultStringsHtml = getHighlighterFragments(highlighter, SearchFieldConstants.ANNOTATIONS, annotation); + annotationsResultStringsHtml = getHighlighterFragments(highlighter, LinkedFilesConstants.ANNOTATIONS, annotation); } return annotationsResultStringsHtml; } @@ -70,8 +70,8 @@ public int getPageNumber() { return pageNumber; } - private static List getHighlighterFragments(Highlighter highlighter, SearchFieldConstants field, String content) { - try (TokenStream contentStream = SearchFieldConstants.LINKED_FILES_ANALYZER.tokenStream(field.toString(), content)) { + private static List getHighlighterFragments(Highlighter highlighter, LinkedFilesConstants field, String content) { + try (TokenStream contentStream = LinkedFilesConstants.LINKED_FILES_ANALYZER.tokenStream(field.toString(), content)) { TextFragment[] frags = highlighter.getBestTextFragments(contentStream, content, true, 10); return Arrays.stream(frags).map(TextFragment::toString).toList(); } catch (IOException | InvalidTokenOffsetsException e) { diff --git a/src/test/java/org/jabref/logic/search/query/LuceneQueryParserTest.java b/src/test/java/org/jabref/logic/search/query/LuceneQueryParserTest.java deleted file mode 100644 index bf1429d1c11..00000000000 --- a/src/test/java/org/jabref/logic/search/query/LuceneQueryParserTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.jabref.logic.search.query; - -import java.util.stream.Stream; - -import org.jabref.model.search.SearchFieldConstants; - -import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.queryparser.classic.QueryParser; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class LuceneQueryParserTest { - - public static Stream searchQuires() { - return Stream.of( - // unicode - Arguments.of("any:preissinger", "preißinger"), - Arguments.of("any:jesus", "jesús"), - Arguments.of("any:breitenbucher", "breitenbücher"), - - // latex - Arguments.of("any:preissinger", "\"prei{\\\\ss}inger\""), - Arguments.of("any:jesus", "\"jes{\\\\'{u}}s\""), - Arguments.of("any:breitenbucher", "\"breitenb{\\\\\\\"{u}}cher\""), - - Arguments.of("groups:/exclude", "groups:\\/exclude") - ); - } - - @ParameterizedTest - @MethodSource - void searchQuires(String expected, String query) throws ParseException { - QueryParser parser = new QueryParser(SearchFieldConstants.DEFAULT_FIELD.toString(), SearchFieldConstants.LATEX_AWARE_ANALYZER); - String result = parser.parse(query).toString(); - assertEquals(expected, result); - } -} diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java index c84a196bfdd..964a725ca30 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java @@ -10,64 +10,55 @@ class SearchQueryLuceneConversionTest { - public static Stream transformationNormal() { + public static Stream testSearchConversion() { return Stream.of( - // If "any:" should not be added, see e46e0a23d7b9526bd449cbecfb189ae6dbc40a28 for a fix - Arguments.of("any:chocolate", "chocolate"), + Arguments.of("content:term annotations:term", "term"), + Arguments.of("content:term annotations:term", "any = term"), + Arguments.of("content:term annotations:term", "any CONTAINS term"), + Arguments.of("content:term annotations:term", "any MATCHES term"), + Arguments.of("content:term annotations:term", "any =! term"), + Arguments.of("content:term annotations:term", "any == term"), + Arguments.of("content:term annotations:term", "any ==! term"), - Arguments.of("title:chocolate", "title=chocolate"), - Arguments.of("title:chocolate OR author:smith", "title = chocolate or author = smith"), - Arguments.of("groups:\\/exclude", "groups= /exclude"), - Arguments.of("title:chocolate AND author:smith", "title = \"chocolate\" AND author = \"smith\""), - Arguments.of("title:chocolate AND author:smith", "title contains \"chocolate\" AND author matches \"smith\""), - Arguments.of("( title:chocolate ) OR ( author:smith )", "(title == chocolate) or (author == smith)"), - Arguments.of("( title:chocolate OR author:smith ) AND ( year:2024 )", "(title contains chocolate or author matches smith) AND (year = 2024)"), - Arguments.of("any:video AND year:1932", "video and year == 1932"), - Arguments.of("title:neighbou?r", "title =neighbou?r"), - Arguments.of("abstract:model\\{1,2\\}ing", "abstract = model{1,2}ing"), - Arguments.of("any:* AND -title:chocolate", "title != chocolate"), - Arguments.of("any:* AND -title:chocolate", "not title contains chocolate"), - // https://github.com/JabRef/jabref/issues/11654#issuecomment-2313178736 - Arguments.of("( groups:\\:paywall AND -file:/.+/ ) AND -groups:\\/exclude", "groups=:paywall and file!=\"\" and groups!=/exclude"), + Arguments.of("content:\"two term\" annotations:\"two term\"", "\"two terms\""), + Arguments.of("content:\"two term\" annotations:\"two term\"", "any = \"two terms\""), - Arguments.of("( any:* AND -( groups:alpha ) ) AND ( any:* AND -( groups:beta ) )", "NOT(groups=alpha) AND NOT(groups=beta)"), - // not converted, because not working in JabRef 5.x - // Arguments.of("title:\"image processing\" OR keywords:\"image processing\"", "title|keywords = \"image processing\""), + Arguments.of("content:imag", "content = image"), + Arguments.of("annotations:imag", "annotations = image"), + Arguments.of("content:\"imag process\"", "content = \"image processing\""), + Arguments.of("+content:imag +annotations:process", "content = image AND annotations = processing"), + Arguments.of("+(content:imag annotations:process) +(content:term annotations:term)", "(content = image OR annotations = processing) AND term"), + Arguments.of("(content:on annotations:on) (+(content:two annotations:two) +(content:three annotations:three))", "one OR (two AND three)"), - // not converted, because wrong syntax for JabRef 5.x - // Arguments.of("( author:miller OR title:\"image processing\" OR keywords:\"image processing\" ) AND NOT author:brown AND NOT author:blue", "(author = miller or title|keywords = \"image processing\") and not author = brown and != author = blue"), + Arguments.of("(-content:term) (-annotations:term)", "any != term"), + Arguments.of("(-content:term) (-annotations:term)", "any !== term"), + Arguments.of("(-content:term) (-annotations:term)", "any !=! term"), + Arguments.of("(-content:\"two term\") (-annotations:\"two term\")", "any != \"two terms\""), + Arguments.of("+(-content:imag) +(-annotations:process)", "content != image AND annotations != processing"), - // String with a space - Arguments.of("title:image\\ processing AND author:smith", "title = \"image processing\" AND author= smith"), + Arguments.of("MatchNoDocsQuery(\"\")", "title = image"), + Arguments.of("content:\"imag process\" annotations:\"imag process\"", "\"image processing\" AND author = smith"), + Arguments.of("+(content:imag annotations:imag) +(content:process annotations:process)", "image AND (title = term OR processing)"), + Arguments.of("(content:imag annotations:imag) (content:process annotations:process)", "image OR (title = term OR processing)"), + Arguments.of("MatchNoDocsQuery(\"\")", "title = \"image processing\" AND author = smith"), - // We renamed fields - Arguments.of("any:somecontent", "anyfield = somecontent"), - Arguments.of("keywords:somekeyword", "anykeyword = somekeyword"), - Arguments.of("citationkey:somebibtexkey", "key = somebibtexkey") - ); - } - - @ParameterizedTest - @MethodSource - void transformationNormal(String expected, String query) { - String result = SearchQueryConversion.searchToLucene(query); - assertEquals(expected, result); - } + Arguments.of("content:neighbou?r annotations:neighbou?r", "neighbou?r"), + Arguments.of("content:neighbo* annotations:neighbo*", "neighbo*"), + Arguments.of("MatchNoDocsQuery(\"\")", "title = neighbou?r"), + Arguments.of("MatchNoDocsQuery(\"\")", "(title == chocolate) OR (author == smith)"), - public static Stream transformationRegularExpression() { - return Stream.of( - Arguments.of("any:* AND -groups:/.+/", "groups != .+"), - Arguments.of("( any:* AND -groups:/.+/ ) AND ( any:* AND -readstatus:/.+/ )", "groups != .+ and readstatus != .+"), - Arguments.of("author:/(John|Doe).+(John|Doe)/", "author = \"(John|Doe).+(John|Doe)\""), - Arguments.of("any:/rev*/", "anyfield=rev*"), - Arguments.of("any:/*rev*/", "anyfield=*rev*") + Arguments.of("content:/(John|Doe).+(John|Doe)/ annotations:/(John|Doe).+(John|Doe)/", "any =~ \"(John|Doe).+(John|Doe)\""), + Arguments.of("content:/rev*/ annotations:/rev*/", "anyfield=~ rev*"), + Arguments.of("content:/*rev*/ annotations:/*rev*/", "anyfield=~ *rev*"), + Arguments.of("(-content:/.+/) (-annotations:/.+/)", "any !=~ .+"), + Arguments.of("(-content:/.+/) (-annotations:/.+/)", "groups !=~ .+ AND any !=~ .+") ); } @ParameterizedTest @MethodSource - void transformationRegularExpression(String expected, String query) { - String result = SearchQueryConversion.searchToLucene(query); + void testSearchConversion(String expected, String query) { + String result = SearchQueryConversion.searchToLucene(query).toString(); assertEquals(expected, result); } } From c127aaf4cd0e0908f366cc2410561e25052ea70b Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sat, 5 Oct 2024 04:19:48 +0300 Subject: [PATCH 059/104] Marge linked files and bib fields results --- .../org/jabref/gui/entryeditor/SourceTab.java | 2 +- .../org/jabref/gui/preview/PreviewViewer.java | 4 +- .../jabref/gui/search/GlobalSearchBar.java | 12 -- .../org/jabref/logic/search/IndexManager.java | 8 +- .../search/query/SearchQueryConversion.java | 2 +- .../search/query/SearchToSqlVisitor.java | 5 - .../search/retrieval/BibFieldsSearcher.java | 4 + .../search/retrieval/LinkedFilesSearcher.java | 11 +- .../model/search/query/SearchQuery.java | 112 +++--------------- 9 files changed, 33 insertions(+), 127 deletions(-) diff --git a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java index d7159a87b05..f677e272607 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java @@ -119,7 +119,7 @@ public SourceTab(BibDatabaseContext bibDatabaseContext, this.keyBindingRepository = keyBindingRepository; searchQueryProperty.addListener((observable, oldValue, newValue) -> { - searchHighlightPattern = newValue.flatMap(SearchQuery::getPatternForWords); + // TODO: get search pattern from the search query highlightSearchPattern(); }); } diff --git a/src/main/java/org/jabref/gui/preview/PreviewViewer.java b/src/main/java/org/jabref/gui/preview/PreviewViewer.java index cb7c084dc64..7add9e3aa5d 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewViewer.java +++ b/src/main/java/org/jabref/gui/preview/PreviewViewer.java @@ -126,7 +126,7 @@ function getSelectionHtml() { private final WebView previewView; private final BibDatabaseContext database; private final ChangeListener> listener = (queryObservable, queryOldValue, queryNewValue) -> { - searchHighlightPattern = queryNewValue.flatMap(SearchQuery::getJavaScriptPatternForWords); + // TODO: update search pattern highlightSearchPattern(); }; @@ -161,7 +161,7 @@ public PreviewViewer(BibDatabaseContext database, } if (!registered) { - searchHighlightPattern = searchQueryProperty.get().flatMap(SearchQuery::getJavaScriptPatternForWords); + // TODO: update search pattern searchQueryProperty.addListener(listener); registered = true; } diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index 5bbe833b8c4..eeb7a922402 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -6,8 +6,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; import javax.swing.undo.UndoManager; @@ -351,16 +349,6 @@ public void updateSearchQuery() { stateManager.activeSearchQuery(searchType).set(Optional.of(searchQuery)); } - private boolean validRegex() { - try { - Pattern.compile(searchField.getText()); - } catch (PatternSyntaxException e) { - LOGGER.debug(e.getMessage()); - return false; - } - return true; - } - public void setAutoCompleter(SuggestionProvider searchCompleter) { if (preferences.getAutoCompletePreferences().shouldAutoComplete()) { AutoCompletionTextInputBinding autoComplete = AutoCompletionTextInputBinding.autoComplete(searchField, diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index 7feae979af2..c25229004f3 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -223,13 +223,9 @@ public AutoCloseable blockLinkedFileIndexer() { public SearchResults search(SearchQuery query) { SearchResults searchResults = new SearchResults(); searchResults.mergeSearchResults(bibFieldsSearcher.search(query)); -// if (query.isValid()) { -// query.setSearchResults(linkedFilesSearcher.search(query.getParsedQuery(), query.getSearchFlags())); -// } else { -// query.setSearchResults(new SearchResults()); -// } + if (query.getSearchFlags().contains(SearchFlags.FULLTEXT)) { - // TODO: merge results from lucene and postgres + searchResults.mergeSearchResults(linkedFilesSearcher.search(query)); } query.setSearchResults(searchResults); return searchResults; diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java index 5b699b68026..a142d2ee97d 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java @@ -36,7 +36,7 @@ public static Query searchToLucene(String searchExpression) { return new SearchToLuceneVisitor().visit(context); } - private static SearchParser.StartContext getStartContext(String searchExpression) { + public static SearchParser.StartContext getStartContext(String searchExpression) { SearchLexer lexer = new SearchLexer(CharStreams.fromString(searchExpression)); lexer.removeErrorListeners(); // no infos on file system lexer.addErrorListener(ThrowingErrorListener.INSTANCE); diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 906dae1ba7a..c67ed4ef4ea 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -14,9 +14,6 @@ import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import static org.jabref.model.search.PostgreConstants.ENTRY_ID; import static org.jabref.model.search.PostgreConstants.FIELD_NAME; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_LITERAL; @@ -34,7 +31,6 @@ */ public class SearchToSqlVisitor extends SearchBaseVisitor { - private static final Logger LOGGER = LoggerFactory.getLogger(SearchToSqlVisitor.class); private static final String MAIN_TABLE = "main_table"; private static final String SPLIT_TABLE = "split_table"; private static final String INNER_TABLE = "inner_table"; @@ -64,7 +60,6 @@ public String visitStart(SearchParser.StartContext ctx) { } sql.append("SELECT * FROM ").append(query).append(" GROUP BY ").append(ENTRY_ID); - LOGGER.trace("Converted search query to SQL: {}", sql); return sql.toString(); } diff --git a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java index f090b488e2b..681ee964163 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java @@ -38,6 +38,10 @@ private static SearchQuery createBooleanQueryForEntry(BibEntry entry, SearchQuer public SearchResults search(SearchQuery searchQuery) { String sqlQuery = searchQuery.getSqlQuery(tableName); + if (!searchQuery.isValid()) { + LOGGER.error("Invalid SQL query: {}", sqlQuery); + return new SearchResults(); + } LOGGER.debug("Searching in bib fields with query: {}", sqlQuery); SearchResults searchResults = new SearchResults(); try (PreparedStatement preparedStatement = connection.prepareStatement(sqlQuery)) { diff --git a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java index ec33ed17055..0b1aadecb08 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java @@ -15,6 +15,7 @@ import org.jabref.model.entry.LinkedFile; import org.jabref.model.search.LinkedFilesConstants; import org.jabref.model.search.SearchFlags; +import org.jabref.model.search.query.SearchQuery; import org.jabref.model.search.query.SearchResult; import org.jabref.model.search.query.SearchResults; @@ -44,16 +45,18 @@ public LinkedFilesSearcher(BibDatabaseContext databaseContext, LuceneIndexer lin this.filePreferences = filePreferences; } - public SearchResults search(Query searchQuery, EnumSet searchFlags) { + public SearchResults search(SearchQuery searchQuery) { + Query luceneQuery = searchQuery.getLuceneQuery(); + EnumSet searchFlags = searchQuery.getSearchFlags(); boolean shouldSearchInLinkedFiles = searchFlags.contains(SearchFlags.FULLTEXT) && filePreferences.shouldFulltextIndexLinkedFiles(); - if (!shouldSearchInLinkedFiles) { + if (!shouldSearchInLinkedFiles || !searchQuery.isValid()) { return new SearchResults(); } - LOGGER.debug("Searching in linked files with query: {}", searchQuery); + LOGGER.debug("Searching in linked files with query: {}", luceneQuery); try { IndexSearcher linkedFilesIndexSearcher = acquireIndexSearcher(searcherManager); - SearchResults searchResults = search(linkedFilesIndexSearcher, searchQuery); + SearchResults searchResults = search(linkedFilesIndexSearcher, luceneQuery); releaseIndexSearcher(searcherManager, linkedFilesIndexSearcher); return searchResults; } catch (IOException | IndexSearcher.TooManyClauses e) { diff --git a/src/main/java/org/jabref/model/search/query/SearchQuery.java b/src/main/java/org/jabref/model/search/query/SearchQuery.java index 3a20912fff4..1ce37df125c 100644 --- a/src/main/java/org/jabref/model/search/query/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/query/SearchQuery.java @@ -1,61 +1,20 @@ package org.jabref.model.search.query; -import java.util.Collections; import java.util.EnumSet; -import java.util.List; import java.util.Objects; -import java.util.Optional; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.jabref.logic.search.query.SearchQueryConversion; import org.jabref.model.search.SearchFlags; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.apache.lucene.search.Query; public class SearchQuery { - /** - * The mode of escaping special characters in regular expressions - */ - private enum EscapeMode { - /** - * using \Q and \E marks - */ - JAVA { - @Override - String format(String regex) { - return Pattern.quote(regex); - } - }, - /** - * escaping all javascript regex special characters separately - */ - JAVASCRIPT { - @Override - String format(String regex) { - return JAVASCRIPT_ESCAPED_CHARS_PATTERN.matcher(regex).replaceAll("\\\\$0"); - } - }; - - /** - * Regex pattern for escaping special characters in javascript regular expressions - */ - private static final Pattern JAVASCRIPT_ESCAPED_CHARS_PATTERN = Pattern.compile("[.*+?^${}()|\\[\\]\\\\/]"); - - /** - * Attempt to escape all regex special characters. - * - * @param regex a string containing a regex expression - * @return a regex with all special characters escaped - */ - abstract String format(String regex); - } private final String searchExpression; private final EnumSet searchFlags; - private String parseError; + private boolean isValidExpression; private String sqlQuery; private Query luceneQuery; private SearchResults searchResults; @@ -63,6 +22,12 @@ String format(String regex) { public SearchQuery(String searchExpression, EnumSet searchFlags) { this.searchExpression = Objects.requireNonNull(searchExpression); this.searchFlags = searchFlags; + try { + SearchQueryConversion.getStartContext(searchExpression); + isValidExpression = true; + } catch (ParseCancellationException e) { + isValidExpression = false; + } } public String getSqlQuery(String table) { @@ -91,6 +56,14 @@ public void setSearchResults(SearchResults searchResults) { this.searchResults = searchResults; } + public boolean isValid() { + return isValidExpression; + } + + public EnumSet getSearchFlags() { + return searchFlags; + } + @Override public String toString() { return searchExpression; @@ -112,57 +85,4 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(searchExpression, searchFlags); } - - public boolean isValid() { - return parseError == null; - } - - public EnumSet getSearchFlags() { - return searchFlags; - } - - /** - * Returns a list of words this query searches for. The returned strings can be a regular expression. - */ - public List getSearchWords() { - if (searchFlags.contains(SearchFlags.REGULAR_EXPRESSION)) { - return Collections.singletonList(searchExpression); - } - if (!isValid()) { - return List.of(); - } - return List.of(); - } - - // Returns a regular expression pattern in the form (w1)|(w2)| ... wi are escaped if no regular expression search is enabled - public Optional getPatternForWords() { - return joinWordsToPattern(EscapeMode.JAVA); - } - - // Returns a regular expression pattern in the form (w1)|(w2)| ... wi are escaped for javascript if no regular expression search is enabled - public Optional getJavaScriptPatternForWords() { - return joinWordsToPattern(EscapeMode.JAVASCRIPT); - } - - /** - * Returns a regular expression pattern in the form (w1)|(w2)| ... wi are escaped if no regular expression search is enabled - * - * @param escapeMode the mode of escaping special characters in wi - */ - private Optional joinWordsToPattern(EscapeMode escapeMode) { - List words = getSearchWords(); - - if ((words == null) || words.isEmpty() || words.getFirst().isEmpty()) { - return Optional.empty(); - } - - // compile the words to a regular expression in the form (w1)|(w2)|(w3) - Stream joiner = words.stream(); - if (!searchFlags.contains(SearchFlags.REGULAR_EXPRESSION)) { - // Reformat string when we are looking for a literal match - joiner = joiner.map(escapeMode::format); - } - String searchPattern = joiner.collect(Collectors.joining(")|(", "(", ")")); - return Optional.of(Pattern.compile(searchPattern, Pattern.CASE_INSENSITIVE)); - } } From b2c5428771ddd1e89b4b565d31157d0e9ee88bb0 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sat, 5 Oct 2024 05:11:33 +0300 Subject: [PATCH 060/104] Searching in background task --- .../gui/groups/GroupDialogViewModel.java | 1 - .../jabref/gui/groups/GroupNodeViewModel.java | 18 +++++---------- .../org/jabref/logic/search/IndexManager.java | 22 ++++++++++++++++--- .../search/retrieval/BibFieldsSearcher.java | 4 ++-- .../search/retrieval/LinkedFilesSearcher.java | 9 ++++++-- .../model/search/query/SearchQuery.java | 17 -------------- .../model/search/query/SearchResults.java | 3 ++- 7 files changed, 36 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java index 1e9c9c4eb49..6f3489101cd 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java @@ -334,7 +334,6 @@ public AbstractGroup resultConverter(ButtonType button) { Optional indexManager = stateManager.getIndexManager(currentDatabase); if (indexManager.isPresent()) { SearchGroup searchGroup = (SearchGroup) resultingGroup; - // TODO: search should be done in a background thread searchGroup.setMatchedEntries(indexManager.get().search(searchGroup.getSearchQuery()).getMatchedEntries()); } } else if (typeAutoProperty.getValue()) { diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index b662b1ae115..812674cdd65 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -102,12 +102,9 @@ public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager state databaseContext.getMetaData().groupsBinding().addListener(new WeakInvalidationListener(onInvalidatedGroup)); } else if (groupNode.getGroup() instanceof SearchGroup searchGroup) { stateManager.getIndexManager(databaseContext).ifPresent(indexManager -> { - BackgroundTask.wrap(() -> { - searchGroup.setMatchedEntries(indexManager.search(searchGroup.getSearchQuery()).getMatchedEntries()); - }).onSuccess(success -> { - refreshGroup(); - databaseContext.getMetaData().groupsBinding().invalidate(); - }).executeWith(taskExecutor); + searchGroup.setMatchedEntries(indexManager.search(searchGroup.getSearchQuery()).getMatchedEntries()); + refreshGroup(); + databaseContext.getMetaData().groupsBinding().invalidate(); }); } @@ -541,12 +538,9 @@ class SearchIndexListener { public void listen(IndexStartedEvent event) { if (groupNode.getGroup() instanceof SearchGroup searchGroup) { stateManager.getIndexManager(databaseContext).ifPresent(indexManager -> { - BackgroundTask.wrap(() -> { - searchGroup.setMatchedEntries(indexManager.search(searchGroup.getSearchQuery()).getMatchedEntries()); - }).onSuccess(success -> { - refreshGroup(); - databaseContext.getMetaData().groupsBinding().invalidate(); - }).executeWith(taskExecutor); + searchGroup.setMatchedEntries(indexManager.search(searchGroup.getSearchQuery()).getMatchedEntries()); + refreshGroup(); + databaseContext.getMetaData().groupsBinding().invalidate(); }); } } diff --git a/src/main/java/org/jabref/logic/search/IndexManager.java b/src/main/java/org/jabref/logic/search/IndexManager.java index c25229004f3..7eb1ac31ab9 100644 --- a/src/main/java/org/jabref/logic/search/IndexManager.java +++ b/src/main/java/org/jabref/logic/search/IndexManager.java @@ -1,7 +1,11 @@ package org.jabref.logic.search; import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import javafx.beans.property.BooleanProperty; import javafx.beans.property.SimpleBooleanProperty; @@ -14,6 +18,7 @@ import org.jabref.logic.search.retrieval.BibFieldsSearcher; import org.jabref.logic.search.retrieval.LinkedFilesSearcher; import org.jabref.logic.util.BackgroundTask; +import org.jabref.logic.util.HeadlessExecutorService; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -221,11 +226,22 @@ public AutoCloseable blockLinkedFileIndexer() { } public SearchResults search(SearchQuery query) { - SearchResults searchResults = new SearchResults(); - searchResults.mergeSearchResults(bibFieldsSearcher.search(query)); + List> tasks = new ArrayList<>(); + tasks.add(() -> bibFieldsSearcher.search(query)); if (query.getSearchFlags().contains(SearchFlags.FULLTEXT)) { - searchResults.mergeSearchResults(linkedFilesSearcher.search(query)); + tasks.add(() -> linkedFilesSearcher.search(query)); + } + + List> futures = HeadlessExecutorService.INSTANCE.executeAll(tasks); + + SearchResults searchResults = new SearchResults(); + for (Future future : futures) { + try { + searchResults.mergeSearchResults(future.get()); + } catch (InterruptedException | ExecutionException e) { + LOGGER.error("Error while searching", e); + } } query.setSearchResults(searchResults); return searchResults; diff --git a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java index 681ee964163..adc11398b78 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java @@ -5,6 +5,7 @@ import java.sql.ResultSet; import java.sql.SQLException; +import org.jabref.logic.search.query.SearchQueryConversion; import org.jabref.model.entry.BibEntry; import org.jabref.model.search.query.SearchQuery; import org.jabref.model.search.query.SearchResult; @@ -37,11 +38,10 @@ private static SearchQuery createBooleanQueryForEntry(BibEntry entry, SearchQuer } public SearchResults search(SearchQuery searchQuery) { - String sqlQuery = searchQuery.getSqlQuery(tableName); if (!searchQuery.isValid()) { - LOGGER.error("Invalid SQL query: {}", sqlQuery); return new SearchResults(); } + String sqlQuery = SearchQueryConversion.searchToSql(tableName, searchQuery.getSearchExpression()); LOGGER.debug("Searching in bib fields with query: {}", sqlQuery); SearchResults searchResults = new SearchResults(); try (PreparedStatement preparedStatement = connection.prepareStatement(sqlQuery)) { diff --git a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java index 0b1aadecb08..7cc91e946c7 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java @@ -10,6 +10,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.search.LuceneIndexer; +import org.jabref.logic.search.query.SearchQueryConversion; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; @@ -46,10 +47,14 @@ public LinkedFilesSearcher(BibDatabaseContext databaseContext, LuceneIndexer lin } public SearchResults search(SearchQuery searchQuery) { - Query luceneQuery = searchQuery.getLuceneQuery(); + if (!searchQuery.isValid()) { + return new SearchResults(); + } + + Query luceneQuery = SearchQueryConversion.searchToLucene(searchQuery.getSearchExpression()); EnumSet searchFlags = searchQuery.getSearchFlags(); boolean shouldSearchInLinkedFiles = searchFlags.contains(SearchFlags.FULLTEXT) && filePreferences.shouldFulltextIndexLinkedFiles(); - if (!shouldSearchInLinkedFiles || !searchQuery.isValid()) { + if (!shouldSearchInLinkedFiles) { return new SearchResults(); } diff --git a/src/main/java/org/jabref/model/search/query/SearchQuery.java b/src/main/java/org/jabref/model/search/query/SearchQuery.java index 1ce37df125c..3663fa06a54 100644 --- a/src/main/java/org/jabref/model/search/query/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/query/SearchQuery.java @@ -7,7 +7,6 @@ import org.jabref.model.search.SearchFlags; import org.antlr.v4.runtime.misc.ParseCancellationException; -import org.apache.lucene.search.Query; public class SearchQuery { @@ -15,8 +14,6 @@ public class SearchQuery { private final EnumSet searchFlags; private boolean isValidExpression; - private String sqlQuery; - private Query luceneQuery; private SearchResults searchResults; public SearchQuery(String searchExpression, EnumSet searchFlags) { @@ -30,20 +27,6 @@ public SearchQuery(String searchExpression, EnumSet searchFlags) { } } - public String getSqlQuery(String table) { - if (sqlQuery == null) { - sqlQuery = SearchQueryConversion.searchToSql(table, searchExpression); - } - return sqlQuery; - } - - public Query getLuceneQuery() { - if (luceneQuery == null) { - luceneQuery = SearchQueryConversion.searchToLucene(searchExpression); - } - return luceneQuery; - } - public String getSearchExpression() { return searchExpression; } diff --git a/src/main/java/org/jabref/model/search/query/SearchResults.java b/src/main/java/org/jabref/model/search/query/SearchResults.java index e955a5031c3..1bb164bcfc1 100644 --- a/src/main/java/org/jabref/model/search/query/SearchResults.java +++ b/src/main/java/org/jabref/model/search/query/SearchResults.java @@ -6,12 +6,13 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import org.jabref.model.entry.BibEntry; public class SearchResults { - private final Map> searchResults = new HashMap<>(); + private final Map> searchResults = new ConcurrentHashMap<>(); public void mergeSearchResults(SearchResults additionalResults) { this.searchResults.putAll(additionalResults.searchResults); From a3b83284d448845116f23018dd2d2d525f74b57a Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 6 Oct 2024 03:45:41 +0300 Subject: [PATCH 061/104] Fix search to SQL tests --- .../query/SearchQuerySQLConversionTest.java | 665 ++++++++++++++++-- 1 file changed, 603 insertions(+), 62 deletions(-) diff --git a/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java index 8e3bf5f2a3d..d75a30b5cb5 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java @@ -1,90 +1,631 @@ package org.jabref.logic.search.query; +import java.util.stream.Stream; + import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import static org.junit.jupiter.api.Assertions.assertEquals; class SearchQuerySQLConversionTest { - @ParameterizedTest - @CsvSource({ - "(), alex", - "(), author=alex", - "(), title=dino AND author=alex", - "(), author=computer AND editor=science OR title=math", - "(), (author=computer AND editor=science) OR title=math", - "(), author=computer AND (editor=science OR title=math)", - "(), (title = computer OR comment = science) AND (title = math OR comment = physics)", - "(), (author=computer AND editor=science) OR (title=math AND year=2021)", - // case insensitive contains - "((main_table.field_name = 'title') AND ((main_table.field_value_literal ILIKE '%compute%') OR (main_table.field_value_transformed ILIKE '%compute%'))), title=compute", + public static Stream testSearchConversion() { + return Stream.of( + Arguments.of( + "author CONTAINS smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE '%smith%') OR (main_table.field_value_transformed ILIKE '%smith%')) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), + + Arguments.of( + "author = smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE '%smith%') OR (main_table.field_value_transformed ILIKE '%smith%')) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), + + Arguments.of( + "author =! smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name = 'author') AND ((main_table.field_value_literal LIKE '%smith%') OR (main_table.field_value_transformed LIKE '%smith%')) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), + + Arguments.of( + "author != smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE main_table.entry_id NOT IN ( + SELECT inner_table.entry_id + FROM bib_fields."tableName" AS inner_table + WHERE ( + (inner_table.field_name = 'author') AND ((inner_table.field_value_literal ILIKE '%smith%') OR (inner_table.field_value_transformed ILIKE '%smith%')) + ) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // case sensitive contains - "((main_table.field_name = 'title') AND ((main_table.field_value_literal LIKE '%compute%') OR (main_table.field_value_transformed LIKE '%compute%'))), title=!compute", + Arguments.of( + "author !=! smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE main_table.entry_id NOT IN ( + SELECT inner_table.entry_id + FROM bib_fields."tableName" AS inner_table + WHERE ( + (inner_table.field_name = 'author') AND ((inner_table.field_value_literal LIKE '%smith%') OR (inner_table.field_value_transformed LIKE '%smith%')) + ) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // exact match case insensitive - "((main_table.field_name = 'title') AND ((main_table.field_value_literal ILIKE 'compute') OR (main_table.field_value_transformed ILIKE 'compute'))) OR ((split_table.field_name = 'title') AND ((split_table.field_value_literal ILIKE 'compute') OR (split_table.field_value_transformed ILIKE 'compute'))), title==compute", + Arguments.of( + "author MATCHES smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + LEFT JOIN bib_fields."tableName_split_values" AS split_table + ON (main_table.entry_id = split_table.entry_id AND main_table.field_name = split_table.field_name) + WHERE ( + ((main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE 'smith') OR (main_table.field_value_transformed ILIKE 'smith'))) + OR + ((split_table.field_name = 'author') AND ((split_table.field_value_literal ILIKE 'smith') OR (split_table.field_value_transformed ILIKE 'smith'))) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // exact match case sensitive - "((main_table.field_name = 'title') AND ((main_table.field_value_literal LIKE 'compute') OR (main_table.field_value_transformed LIKE 'compute'))) OR ((split_table.field_name = 'title') AND ((split_table.field_value_literal LIKE 'compute') OR (split_table.field_value_transformed LIKE 'compute'))), title==!compute", + Arguments.of( + "author == smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + LEFT JOIN bib_fields."tableName_split_values" AS split_table + ON (main_table.entry_id = split_table.entry_id AND main_table.field_name = split_table.field_name) + WHERE ( + ((main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE 'smith') OR (main_table.field_value_transformed ILIKE 'smith'))) + OR + ((split_table.field_name = 'author') AND ((split_table.field_value_literal ILIKE 'smith') OR (split_table.field_value_transformed ILIKE 'smith'))) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // Regex search case insensitive - "(main_table.field_value_literal ~* 'Jabref.*Search') OR (main_table.field_value_transformed ~* 'Jabref.*Search'), any=~Jabref.*Search", + Arguments.of( + "author ==! smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + LEFT JOIN bib_fields."tableName_split_values" AS split_table + ON (main_table.entry_id = split_table.entry_id AND main_table.field_name = split_table.field_name) + WHERE ( + ((main_table.field_name = 'author') AND ((main_table.field_value_literal LIKE 'smith') OR (main_table.field_value_transformed LIKE 'smith'))) + OR + ((split_table.field_name = 'author') AND ((split_table.field_value_literal LIKE 'smith') OR (split_table.field_value_transformed LIKE 'smith'))) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // Regex search case sensitive - "(main_table.field_value_literal ~ 'Jabref.*Search') OR (main_table.field_value_transformed ~ 'Jabref.*Search'), any=~!Jabref.*Search", + Arguments.of( + "author !== smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE main_table.entry_id NOT IN ( + SELECT inner_table.entry_id + FROM bib_fields."tableName" AS inner_table + LEFT JOIN bib_fields."tableName_split_values" AS split_table + ON (inner_table.entry_id = split_table.entry_id AND inner_table.field_name = split_table.field_name) + WHERE ( + ((inner_table.field_name = 'author') AND ((inner_table.field_value_literal ILIKE 'smith') OR (inner_table.field_value_transformed ILIKE 'smith'))) + OR + ((split_table.field_name = 'author') AND ((split_table.field_value_literal ILIKE 'smith') OR (split_table.field_value_transformed ILIKE 'smith'))) + ) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // negated case insensitive contains - "((main_table.field_name = 'title') AND ((main_table.field_value_literal NOT ILIKE '%compute%') OR (main_table.field_value_transformed NOT ILIKE '%compute%'))), title!=compute", + Arguments.of( + "author !==! smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE main_table.entry_id NOT IN ( + SELECT inner_table.entry_id + FROM bib_fields."tableName" AS inner_table + LEFT JOIN bib_fields."tableName_split_values" AS split_table + ON (inner_table.entry_id = split_table.entry_id AND inner_table.field_name = split_table.field_name) + WHERE ( + ((inner_table.field_name = 'author') AND ((inner_table.field_value_literal LIKE 'smith') OR (inner_table.field_value_transformed LIKE 'smith'))) + OR + ((split_table.field_name = 'author') AND ((split_table.field_value_literal LIKE 'smith') OR (split_table.field_value_transformed LIKE 'smith'))) + ) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // negated case sensitive contains - "((main_table.field_name = 'title') AND ((main_table.field_value_literal NOT LIKE '%compute%') OR (main_table.field_value_transformed NOT LIKE '%compute%'))), title !=! compute", + Arguments.of( + "author =~ smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name = 'author') AND ((main_table.field_value_literal ~* 'smith') OR (main_table.field_value_transformed ~* 'smith')) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // negated case insensitive exact match - "((main_table.field_name = 'title') AND ((main_table.field_value_literal NOT ILIKE 'compute') OR (main_table.field_value_transformed NOT ILIKE 'compute'))) OR ((split_table.field_name = 'title') AND ((split_table.field_value_literal NOT ILIKE 'compute') OR (split_table.field_value_transformed NOT ILIKE 'compute'))), title !== compute", + Arguments.of( + "author =~! smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name = 'author') AND ((main_table.field_value_literal ~ 'smith') OR (main_table.field_value_transformed ~ 'smith')) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // negated case sensitive exact match - "((main_table.field_name = 'title') AND ((main_table.field_value_literal NOT LIKE 'compute') OR (main_table.field_value_transformed NOT LIKE 'compute'))) OR ((split_table.field_name = 'title') AND ((split_table.field_value_literal NOT LIKE 'compute') OR (split_table.field_value_transformed NOT LIKE 'compute'))), title !==! compute", + Arguments.of( + "author !=~ smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE main_table.entry_id NOT IN ( + SELECT inner_table.entry_id + FROM bib_fields."tableName" AS inner_table + WHERE ( + (inner_table.field_name = 'author') AND ((inner_table.field_value_literal ~* 'smith') OR (inner_table.field_value_transformed ~* 'smith')) + ) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // negated regex search case insensitive - "(main_table.field_value_literal !~* 'Jabref.*Search') OR (main_table.field_value_transformed !~* 'Jabref.*Search'), any!=~Jabref.*Search", + Arguments.of( + "author !=~! smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE main_table.entry_id NOT IN ( + SELECT inner_table.entry_id + FROM bib_fields."tableName" AS inner_table + WHERE ( + (inner_table.field_name = 'author') AND ((inner_table.field_value_literal ~ 'smith') OR (inner_table.field_value_transformed ~ 'smith')) + ) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // negated regex search case sensitive - "(main_table.field_value_literal !~ 'Jabref.*Search') OR (main_table.field_value_transformed !~ 'Jabref.*Search'), any!=~!Jabref.*Search", + Arguments.of( + "smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%smith%') OR (main_table.field_value_transformed ILIKE '%smith%')) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // Default search, query without any field name (case insensitive contains) - "(main_table.field_value_literal ILIKE '%computer%') OR (main_table.field_value_transformed ILIKE '%computer%'), computer", - "(main_table.field_value_literal ILIKE '%computer science%') OR (main_table.field_value_transformed ILIKE '%computer science%'), \"computer science\"", // Phrase search - "(field_value ILIKE '%computer%') OR (field_value ILIKE '%science%'), computer science", // Should be searched as a phrase or as two separate words (OR)? (Throw exception) - "(field_value ILIKE '%!computer%'), !computer", // Is the explanation should be escaped? (Throw exception) - "(main_table.field_value_literal ILIKE '%!computer%') OR (main_table.field_value_transformed ILIKE '%!computer%'), \"!computer\"", + Arguments.of( + "any == smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + LEFT JOIN bib_fields."tableName_split_values" AS split_table + ON (main_table.entry_id = split_table.entry_id AND main_table.field_name = split_table.field_name) + WHERE ( + (main_table.field_name != 'groups') + AND ( + ((main_table.field_value_literal ILIKE 'smith') OR (main_table.field_value_transformed ILIKE 'smith')) + OR + ((split_table.field_value_literal ILIKE 'smith') OR (split_table.field_value_transformed ILIKE 'smith')) + ) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // search in all fields case sensitive contains - "(main_table.field_value_literal LIKE '%computer%') OR (main_table.field_value_transformed LIKE '%computer%'), any=!computer", - "(field_value LIKE '%!computer%'), any=! !computer", // Is the explanation should be escaped? (Throw exception) + Arguments.of( + "anyfield != smith", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE main_table.entry_id NOT IN ( + SELECT inner_table.entry_id + FROM bib_fields."tableName" AS inner_table + WHERE ( + (inner_table.field_name != 'groups') AND ((inner_table.field_value_literal ILIKE '%smith%') OR (inner_table.field_value_transformed ILIKE '%smith%')) + ) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // And - "(main_table.field_value_literal ILIKE '%computer%') OR (main_table.field_value_transformed ILIKE '%computer%') AND (main_table.field_value_literal ILIKE '%science%') OR (main_table.field_value_transformed ILIKE '%science%'), computer AND science", + Arguments.of( + "title = \"computer science\"", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name = 'title') AND ((main_table.field_value_literal ILIKE '%computer science%') OR (main_table.field_value_transformed ILIKE '%computer science%')) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), - // Or - "(main_table.field_value_literal ILIKE '%computer%') OR (main_table.field_value_transformed ILIKE '%computer%') OR (main_table.field_value_literal ILIKE '%science%') OR (main_table.field_value_transformed ILIKE '%science%'), computer OR science", + Arguments.of( + "a OR b AND c", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%a%') OR (main_table.field_value_transformed ILIKE '%a%')) + ) + ) + , + cte1 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%b%') OR (main_table.field_value_transformed ILIKE '%b%')) + ) + ) + , + cte2 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%c%') OR (main_table.field_value_transformed ILIKE '%c%')) + ) + ) + , + cte3 AS ( + SELECT entry_id + FROM cte1 + INTERSECT + SELECT entry_id + FROM cte2 + ) + , + cte4 AS ( + SELECT entry_id + FROM cte0 + UNION + SELECT entry_id + FROM cte3 + ) + SELECT * FROM cte4 GROUP BY entry_id""" + ), - // Grouping - "((main_table.field_value_literal ILIKE '%computer%') OR (main_table.field_value_transformed ILIKE '%computer%') AND (main_table.field_value_literal ILIKE '%science%') OR (main_table.field_value_transformed ILIKE '%science%')) OR (main_table.field_value_literal ILIKE '%math%') OR (main_table.field_value_transformed ILIKE '%math%'), (computer AND science) OR math", - "(main_table.field_value_literal ILIKE '%computer%') OR (main_table.field_value_transformed ILIKE '%computer%') AND ((main_table.field_value_literal ILIKE '%science%') OR (main_table.field_value_transformed ILIKE '%science%') OR (main_table.field_value_literal ILIKE '%math%') OR (main_table.field_value_transformed ILIKE '%math%')), computer AND (science OR math)", - "((main_table.field_value_literal ILIKE '%computer%') OR (main_table.field_value_transformed ILIKE '%computer%') OR (main_table.field_value_literal ILIKE '%science%') OR (main_table.field_value_transformed ILIKE '%science%')) AND ((main_table.field_value_literal ILIKE '%math%') OR (main_table.field_value_transformed ILIKE '%math%') OR (main_table.field_value_literal ILIKE '%physics%') OR (main_table.field_value_transformed ILIKE '%physics%')), (computer OR science) AND (math OR physics)", + Arguments.of( + "a AND b OR c", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%a%') OR (main_table.field_value_transformed ILIKE '%a%')) + ) + ) + , + cte1 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%b%') OR (main_table.field_value_transformed ILIKE '%b%')) + ) + ) + , + cte2 AS ( + SELECT entry_id + FROM cte0 + INTERSECT + SELECT entry_id + FROM cte1 + ) + , + cte3 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%c%') OR (main_table.field_value_transformed ILIKE '%c%')) + ) + ) + , + cte4 AS ( + SELECT entry_id + FROM cte2 + UNION + SELECT entry_id + FROM cte3 + ) + SELECT * FROM cte4 GROUP BY entry_id""" + ), - // Special characters - "(main_table.field_value_literal ILIKE '%{IEEE}%') OR (main_table.field_value_transformed ILIKE '%{IEEE}%'), {IEEE}", - "((main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE '%{IEEE}%') OR (main_table.field_value_transformed ILIKE '%{IEEE}%'))), author={IEEE}", + Arguments.of( + "(a OR b) AND c", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%a%') OR (main_table.field_value_transformed ILIKE '%a%')) + ) + ) + , + cte1 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%b%') OR (main_table.field_value_transformed ILIKE '%b%')) + ) + ) + , + cte2 AS ( + SELECT entry_id + FROM cte0 + UNION + SELECT entry_id + FROM cte1 + ) + , + cte3 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%c%') OR (main_table.field_value_transformed ILIKE '%c%')) + ) + ) + , + cte4 AS ( + SELECT entry_id + FROM cte2 + INTERSECT + SELECT entry_id + FROM cte3 + ) + SELECT * FROM cte4 GROUP BY entry_id""" + ), - // R\"ock - "(field_value ILIKE '%R\\\"ock%'), R\\\"ock", // (Throw exception) - // Breitenb{\"{u}}cher - "(field_value ILIKE '%Breitenb{\\\"{u}}cher%'), Breitenb{\\\"{u}}cher", // (Throw exception) - }) + Arguments.of( + "a OR (b AND c)", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%a%') OR (main_table.field_value_transformed ILIKE '%a%')) + ) + ) + , + cte1 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%b%') OR (main_table.field_value_transformed ILIKE '%b%')) + ) + ) + , + cte2 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%c%') OR (main_table.field_value_transformed ILIKE '%c%')) + ) + ) + , + cte3 AS ( + SELECT entry_id + FROM cte1 + INTERSECT + SELECT entry_id + FROM cte2 + ) + , + cte4 AS ( + SELECT entry_id + FROM cte0 + UNION + SELECT entry_id + FROM cte3 + ) + SELECT * FROM cte4 GROUP BY entry_id""" + ), - void conversion(String expectedWhereClause, String input) { - assertEquals(expectedWhereClause, SearchQueryConversion.searchToSql("tableName", input)); + Arguments.of( + "(a OR b) AND (c OR d)", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%a%') OR (main_table.field_value_transformed ILIKE '%a%')) + ) + ) + , + cte1 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%b%') OR (main_table.field_value_transformed ILIKE '%b%')) + ) + ) + , + cte2 AS ( + SELECT entry_id + FROM cte0 + UNION + SELECT entry_id + FROM cte1 + ) + , + cte3 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%c%') OR (main_table.field_value_transformed ILIKE '%c%')) + ) + ) + , + cte4 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%d%') OR (main_table.field_value_transformed ILIKE '%d%')) + ) + ) + , + cte5 AS ( + SELECT entry_id + FROM cte3 + UNION + SELECT entry_id + FROM cte4 + ) + , + cte6 AS ( + SELECT entry_id + FROM cte2 + INTERSECT + SELECT entry_id + FROM cte5 + ) + SELECT * FROM cte6 GROUP BY entry_id""" + ), + + Arguments.of( + "a AND NOT (b OR c)", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%a%') OR (main_table.field_value_transformed ILIKE '%a%')) + ) + ) + , + cte1 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%b%') OR (main_table.field_value_transformed ILIKE '%b%')) + ) + ) + , + cte2 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%c%') OR (main_table.field_value_transformed ILIKE '%c%')) + ) + ) + , + cte3 AS ( + SELECT entry_id + FROM cte1 + UNION + SELECT entry_id + FROM cte2 + ) + , + cte4 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE main_table.entry_id NOT IN ( + SELECT entry_id + FROM cte3 + ) + ) + , + cte5 AS ( + SELECT entry_id + FROM cte0 + INTERSECT + SELECT entry_id + FROM cte4 + ) + SELECT * FROM cte5 GROUP BY entry_id""" + ) + // Throws exceptions + // computer science, !computer, R\"ock, Breitenb{\"{u}}cher + ); + } + + @ParameterizedTest + @MethodSource + void testSearchConversion(String input, String expected) { + assertEquals(expected, SearchQueryConversion.searchToSql("tableName", input)); } } From b7f6bf357bf9ecf340f18a8c0a90b36e9aaa78d7 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 6 Oct 2024 03:49:31 +0300 Subject: [PATCH 062/104] Localization test --- src/main/resources/l10n/JabRef_en.properties | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 5ca0112a42a..22f0532c1b2 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -491,7 +491,6 @@ Include\ subgroups\:\ When\ selected,\ view\ entries\ contained\ in\ this\ group Independent\ group\:\ When\ selected,\ view\ only\ this\ group's\ entries=Independent group: When selected, view only this group's entries I\ Agree=I Agree -Score=Score Indexing\ bib\ fields\ for\ %0=Indexing bib fields for %0 Indexing\ PDF\ files\ for\ %0=Indexing PDF files for %0 %0\ of\ %1\ entries\ added\ to\ the\ index.=%0 of %1 entries added to the index. @@ -811,7 +810,6 @@ Character\ encoding\ '%0'\ is\ not\ supported.=Character encoding '%0' is not su Filter\ search\ results=Filter search results Filter\ by\ groups=Filter by groups Invert\ groups=Invert groups -Match\ score=Match score Scroll\ to\ previous\ match\ category=Scroll to previous match category Scroll\ to\ next\ match\ category=Scroll to next match category Search=Search From 2c50df82b6c24d1dac297c057e6d94cf71eb0e44 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 6 Oct 2024 03:57:00 +0300 Subject: [PATCH 063/104] Fix DatabaseSearcherTest --- .../java/org/jabref/logic/search/DatabaseSearcherTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index 9e2be7d7996..14db4f7d35b 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -14,6 +14,7 @@ import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.BibEntryPreferences; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.search.SearchFlags; @@ -35,11 +36,17 @@ public class DatabaseSearcherTest { private BibDatabaseContext databaseContext; private final CliPreferences preferences = mock(CliPreferences.class); private final FilePreferences filePreferences = mock(FilePreferences.class); + private final BibEntryPreferences bibEntryPreferences = mock(BibEntryPreferences.class); @TempDir private Path indexDir; @BeforeEach void setUp() { + when(preferences.getBibEntryPreferences()).thenReturn(bibEntryPreferences); + when(preferences.getFilePreferences()).thenReturn(filePreferences); + + when(bibEntryPreferences.getKeywordSeparator()).thenReturn(','); + when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(false); when(filePreferences.fulltextIndexLinkedFilesProperty()).thenReturn(mock(BooleanProperty.class)); databaseContext = spy(new BibDatabaseContext()); From 77bf811fc423a329f1edddd6e852679373866d49 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 6 Oct 2024 04:14:08 +0300 Subject: [PATCH 064/104] Fix DatabaseSearcherWithBibFilesTest --- .../search/indexing/BibFieldsIndexer.java | 12 ++-- .../DatabaseSearcherWithBibFilesTest.java | 62 +++++++++---------- 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index bc39accde09..6312ba54ffb 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -104,14 +104,14 @@ PRIMARY KEY (%s, %s) try { // region btree index on id column connection.createStatement().executeUpdate(""" - CREATE INDEX "%s_%s_index" ON %s ("%s") + CREATE INDEX IF NOT EXISTS "%s_%s_index" ON %s ("%s") """.formatted( mainTable, ENTRY_ID, schemaMainTableReference, ENTRY_ID)); connection.createStatement().executeUpdate(""" - CREATE INDEX "%s_%s_index" ON %s ("%s") + CREATE INDEX IF NOT EXISTS "%s_%s_index" ON %s ("%s") """.formatted( splitValuesTable, ENTRY_ID, schemaSplitValuesTableReference, @@ -120,14 +120,14 @@ PRIMARY KEY (%s, %s) // region btree index on field name column connection.createStatement().executeUpdate(""" - CREATE INDEX "%s_%s_index" ON %s ("%s") + CREATE INDEX IF NOT EXISTS "%s_%s_index" ON %s ("%s") """.formatted( mainTable, FIELD_NAME, schemaMainTableReference, FIELD_NAME)); connection.createStatement().executeUpdate(""" - CREATE INDEX "%s_%s_index" ON %s ("%s") + CREATE INDEX IF NOT EXISTS "%s_%s_index" ON %s ("%s") """.formatted( splitValuesTable, FIELD_NAME, schemaSplitValuesTableReference, @@ -136,7 +136,7 @@ PRIMARY KEY (%s, %s) // trigram index on field value column connection.createStatement().executeUpdate(""" - CREATE INDEX "%s_%s_index" ON %s USING gin ("%s" gin_trgm_ops, "%s" gin_trgm_ops) + CREATE INDEX IF NOT EXISTS "%s_%s_index" ON %s USING gin ("%s" gin_trgm_ops, "%s" gin_trgm_ops) """.formatted( mainTable, FIELD_VALUE_LITERAL, schemaMainTableReference, @@ -144,7 +144,7 @@ PRIMARY KEY (%s, %s) // region btree index on spilt table connection.createStatement().executeUpdate(""" - CREATE INDEX "%s_%s_index" ON %s ("%s", "%s") + CREATE INDEX IF NOT EXISTS "%s_%s_index" ON %s ("%s", "%s") """.formatted( splitValuesTable, FIELD_VALUE_LITERAL, schemaSplitValuesTableReference, diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java index 5d2736796d3..3fe3a3edfdd 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java @@ -18,6 +18,7 @@ import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.BibEntryPreferences; import org.jabref.model.entry.LinkedFile; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; @@ -70,6 +71,7 @@ class DatabaseSearcherWithBibFilesTest { private final CliPreferences preferences = mock(CliPreferences.class); private final FilePreferences filePreferences = mock(FilePreferences.class); + private final BibEntryPreferences bibEntryPreferences = mock(BibEntryPreferences.class); @TempDir private Path indexDir; @@ -81,62 +83,58 @@ private BibDatabaseContext initializeDatabaseFromPath(String testFile) throws Ex private BibDatabaseContext initializeDatabaseFromPath(Path testFile) throws Exception { ParserResult result = new BibtexImporter(mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS), new DummyFileUpdateMonitor()).importDatabase(testFile); BibDatabaseContext databaseContext = spy(result.getDatabaseContext()); - when(databaseContext.getFulltextIndexPath()).thenReturn(indexDir); + when(databaseContext.getFulltextIndexPath()).thenReturn(indexDir); when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(true); when(filePreferences.fulltextIndexLinkedFilesProperty()).thenReturn(new SimpleBooleanProperty(true)); + + when(preferences.getBibEntryPreferences()).thenReturn(bibEntryPreferences); + when(preferences.getFilePreferences()).thenReturn(filePreferences); + + when(bibEntryPreferences.getKeywordSeparator()).thenReturn(','); return databaseContext; } private static Stream searchLibrary() { return Stream.of( // empty library - Arguments.of(List.of(), "empty.bib", "Test", EnumSet.noneOf(SearchFlags.class)), + Arguments.of(List.of(), "empty.bib", "Test", false), // test-library-title-casing - Arguments.of(List.of(), "test-library-title-casing.bib", "NotExisting", EnumSet.noneOf(SearchFlags.class)), - Arguments.of(List.of(TITLE_SENTENCE_CASED, TITLE_MIXED_CASED, TITLE_UPPER_CASED), "test-library-title-casing.bib", "Title", EnumSet.noneOf(SearchFlags.class)), + Arguments.of(List.of(), "test-library-title-casing.bib", "NotExisting", false), + Arguments.of(List.of(TITLE_SENTENCE_CASED, TITLE_MIXED_CASED, TITLE_UPPER_CASED), "test-library-title-casing.bib", "Title", false), - Arguments.of(List.of(), "test-library-title-casing.bib", "title:NotExisting", EnumSet.noneOf(SearchFlags.class)), - Arguments.of(List.of(TITLE_SENTENCE_CASED, TITLE_MIXED_CASED, TITLE_UPPER_CASED), "test-library-title-casing.bib", "title:Title", EnumSet.noneOf(SearchFlags.class)), + Arguments.of(List.of(), "test-library-title-casing.bib", "title = NotExisting", false), + Arguments.of(List.of(TITLE_SENTENCE_CASED, TITLE_MIXED_CASED, TITLE_UPPER_CASED), "test-library-title-casing.bib", "title = Title", false), - // Arguments.of(List.of(), "test-library-title-casing.bib", "title:TiTLE", EnumSet.of(SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "title:Title", EnumSet.of(SearchFlags.CASE_SENSITIVE)), + Arguments.of(List.of(), "test-library-title-casing.bib", "title =! TiTLE", false), + Arguments.of(List.of(TITLE_SENTENCE_CASED), "test-library-title-casing.bib", "title =! Title", false), - // Arguments.of(List.of(), "test-library-title-casing.bib", "TiTLE", EnumSet.of(SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "TiTle", EnumSet.of(SearchFlags.CASE_SENSITIVE)), + Arguments.of(List.of(), "test-library-title-casing.bib", "any =! TiTLE", false), + Arguments.of(List.of(TITLE_MIXED_CASED), "test-library-title-casing.bib", "any =! TiTle", false), - // Arguments.of(List.of(), "test-library-title-casing.bib", "title:NotExisting", EnumSet.of(SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "title:TiTle", EnumSet.of(SearchFlags.CASE_SENSITIVE)), + Arguments.of(List.of(), "test-library-title-casing.bib", "title =! NotExisting", false), + Arguments.of(List.of(TITLE_MIXED_CASED), "test-library-title-casing.bib", "title =! TiTle", false), - Arguments.of(List.of(), "test-library-title-casing.bib", "/[Y]/", EnumSet.noneOf(SearchFlags.class)), + Arguments.of(List.of(), "test-library-title-casing.bib", "any =~ [Y]", false), // test-library-with-attached-files - Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting.", EnumSet.of(SearchFlags.FULLTEXT)), - Arguments.of(List.of(MINIMAL_SENTENCE_CASE, MINIMAL_ALL_UPPER_CASE, MINIMAL_MIXED_CASE), "test-library-with-attached-files.bib", "This is a short sentence, comma included.", EnumSet.of(SearchFlags.FULLTEXT)), - Arguments.of(List.of(MINIMAL_SENTENCE_CASE, MINIMAL_ALL_UPPER_CASE, MINIMAL_MIXED_CASE), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchFlags.FULLTEXT)), - - // TODO: PDF search does not support case sensitive search (yet) - // Arguments.of(List.of(minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "THIS", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(minimalAllUpperCase), "test-library-with-attached-files.bib", "THIS is a short sentence, comma included.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(minimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "THIS IS A SHORT SENTENCE, COMMA INCLUDED.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), - - Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchFlags.FULLTEXT)), - Arguments.of(List.of(MINIMAL_NOTE_SENTENCE_CASE, MINIMAL_NOTE_ALL_UPPER_CASE, MINIMAL_NOTE_MIXED_CASE), "test-library-with-attached-files.bib", "world", EnumSet.of(SearchFlags.FULLTEXT)), - Arguments.of(List.of(MINIMAL_NOTE_SENTENCE_CASE, MINIMAL_NOTE_ALL_UPPER_CASE, MINIMAL_NOTE_MIXED_CASE), "test-library-with-attached-files.bib", "Hello World", EnumSet.of(SearchFlags.FULLTEXT)) - - // TODO: PDF search does not support case sensitive search (yet) - // Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "HELLO WORLD", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)) + Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting.", true), + Arguments.of(List.of(MINIMAL_SENTENCE_CASE, MINIMAL_ALL_UPPER_CASE, MINIMAL_MIXED_CASE), "test-library-with-attached-files.bib", "\"This is a short sentence, comma included.\"", true), + Arguments.of(List.of(MINIMAL_SENTENCE_CASE, MINIMAL_ALL_UPPER_CASE, MINIMAL_MIXED_CASE), "test-library-with-attached-files.bib", "comma", true), + + Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", true), + Arguments.of(List.of(MINIMAL_NOTE_SENTENCE_CASE, MINIMAL_NOTE_ALL_UPPER_CASE, MINIMAL_NOTE_MIXED_CASE), "test-library-with-attached-files.bib", "world", true), + Arguments.of(List.of(MINIMAL_NOTE_SENTENCE_CASE, MINIMAL_NOTE_ALL_UPPER_CASE, MINIMAL_NOTE_MIXED_CASE), "test-library-with-attached-files.bib", "\"Hello World\"", true) ); } @ParameterizedTest @MethodSource - void searchLibrary(List expected, String testFile, String query, EnumSet searchFlags) throws Exception { + void searchLibrary(List expected, String testFile, String query, boolean isFullText) throws Exception { BibDatabaseContext databaseContext = initializeDatabaseFromPath(testFile); - List matches = new DatabaseSearcher(new SearchQuery(query, searchFlags), databaseContext, TASK_EXECUTOR, preferences).getMatches(); + EnumSet flags = isFullText ? EnumSet.of(SearchFlags.FULLTEXT) : EnumSet.noneOf(SearchFlags.class); + List matches = new DatabaseSearcher(new SearchQuery(query, flags), databaseContext, TASK_EXECUTOR, preferences).getMatches(); assertThat(expected, Matchers.containsInAnyOrder(matches.toArray())); } } From b9975fc04041817e1066e092b22df1c473655725 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 6 Oct 2024 04:16:14 +0300 Subject: [PATCH 065/104] Fix exportMatches test --- src/test/java/org/jabref/cli/ArgumentProcessorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/cli/ArgumentProcessorTest.java b/src/test/java/org/jabref/cli/ArgumentProcessorTest.java index 6baa662a9c8..101f08099c9 100644 --- a/src/test/java/org/jabref/cli/ArgumentProcessorTest.java +++ b/src/test/java/org/jabref/cli/ArgumentProcessorTest.java @@ -97,7 +97,7 @@ void exportMatches(@TempDir Path tempDir) throws Exception { Path outputBib = tempDir.resolve("output.bib").toAbsolutePath(); String outputBibFile = outputBib.toAbsolutePath().toString(); - List args = List.of("-n", "--debug", "--exportMatches", "author:Einstein," + outputBibFile, originBibFile); + List args = List.of("-n", "--debug", "--exportMatches", "author = Einstein," + outputBibFile, originBibFile); ArgumentProcessor processor = new ArgumentProcessor( args.toArray(String[]::new), From 9f02d04712c74349510eda029dc9cf0d3a596ccd Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 6 Oct 2024 04:16:59 +0300 Subject: [PATCH 066/104] Update src/main/java/org/jabref/model/entry/BibEntry.java Co-authored-by: Oliver Kopp --- src/main/java/org/jabref/model/entry/BibEntry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/model/entry/BibEntry.java b/src/main/java/org/jabref/model/entry/BibEntry.java index d42c7734cc9..36ff6f61f62 100644 --- a/src/main/java/org/jabref/model/entry/BibEntry.java +++ b/src/main/java/org/jabref/model/entry/BibEntry.java @@ -483,7 +483,7 @@ public Optional getFieldLatexFree(Field field) { } else { Optional fieldValue = getField(field); if (fieldValue.isPresent()) { - // TODO: To we need FieldFactory.isLaTeXField(field) here to filter? + // TODO: Do we need FieldFactory.isLaTeXField(field) here to filter? String latexFreeValue = LatexToUnicodeAdapter.format(fieldValue.get()).intern(); latexFreeFields.put(field, latexFreeValue); return Optional.of(latexFreeValue); From 2751844afba07f5e23c0555e3b94c8e515550978 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 6 Oct 2024 04:20:10 +0300 Subject: [PATCH 067/104] Add SINGLE_ENTRY_LINK to latex field --- src/main/java/org/jabref/model/entry/field/FieldFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/model/entry/field/FieldFactory.java b/src/main/java/org/jabref/model/entry/field/FieldFactory.java index fe76330a35f..7c5b38ad13f 100644 --- a/src/main/java/org/jabref/model/entry/field/FieldFactory.java +++ b/src/main/java/org/jabref/model/entry/field/FieldFactory.java @@ -55,7 +55,7 @@ public static String serializeOrFieldsList(Set fields) { * Checks whether the given field contains LaTeX code or something else */ public static boolean isLatexField(Field field) { - return Collections.disjoint(field.getProperties(), Set.of(FieldProperty.VERBATIM, FieldProperty.MARKDOWN, FieldProperty.NUMERIC, FieldProperty.DATE, FieldProperty.MULTIPLE_ENTRY_LINK)); + return Collections.disjoint(field.getProperties(), Set.of(FieldProperty.VERBATIM, FieldProperty.MARKDOWN, FieldProperty.NUMERIC, FieldProperty.DATE, FieldProperty.SINGLE_ENTRY_LINK, FieldProperty.MULTIPLE_ENTRY_LINK)); } /** From c5bde177b5b16e76cbce40053ed95172fd3037e5 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 6 Oct 2024 04:29:43 +0300 Subject: [PATCH 068/104] Remove changelog entries --- CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5760240dfdd..ba56e74d98f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,6 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Added - We added a "view as BibTeX" option before importing an entry from the citation relation tab. [#11826](https://github.com/JabRef/jabref/issues/11826) -- We added probable search hits instead of exact matches. Sorting by hit score can be done by the new score table column. [#11542](https://github.com/JabRef/jabref/pull/11542) - We added support finding LaTeX-encoded special characters based on plain Unicode and vice versa. [#11542](https://github.com/JabRef/jabref/pull/11542) - When a search hits a file, the file icon of that entry is changed accordingly. [#11542](https://github.com/JabRef/jabref/pull/11542) - We added an AI-based chat for entries with linked PDF files. [#11430](https://github.com/JabRef/jabref/pull/11430) @@ -36,8 +35,6 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Changed -- The search syntax is changed to [Apache Lucene syntax](https://lucene.apache.org/core/9_11_1/queryparser/org/apache/lucene/queryparser/classic/package-summary.html#Overview) (also to be similar to the [online search syntax](https://docs.jabref.org/collect/import-using-online-bibliographic-database#search-syntax)). [#11542](https://github.com/JabRef/jabref/pull/11542/) -- When searching using a regular expression, one needs to enclose the search string in `/`. [#11542](https://github.com/JabRef/jabref/pull/11542/) - A search in "any" fields ignores the [groups](https://docs.jabref.org/finding-sorting-and-cleaning-entries/groups). [#7996](https://github.com/JabRef/jabref/issues/7996) - When a communication error with an [online service](https://docs.jabref.org/collect/import-using-online-bibliographic-database) occurs, JabRef displays the HTTP error. [#11223](https://github.com/JabRef/jabref/issues/11223) - The Pubmed/Medline Plain importer now imports the PMID field as well [#11488](https://github.com/JabRef/jabref/issues/11488) @@ -89,7 +86,6 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Removed -- We removed support for case-sensitive and exact search. [#11542](https://github.com/JabRef/jabref/pull/11542) - We removed the description of search strings. [#11542](https://github.com/JabRef/jabref/pull/11542) - We removed support for importing using the SilverPlatterImporter (`Record INSPEC`). [#11576](https://github.com/JabRef/jabref/pull/11576) From b291b6ccc96163b81e0cb7469ff104387393da9b Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 6 Oct 2024 04:40:24 +0300 Subject: [PATCH 069/104] Remove groups migration from localization --- .../gui/importer/actions/SearchGroupsMigrationAction.java | 4 ++-- src/main/resources/l10n/JabRef_en.properties | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index e1a4cc3daee..c032dccb5ce 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -30,8 +30,8 @@ public boolean isActionNecessary(ParserResult parserResult, DialogService dialog Optional currentVersion = parserResult.getMetaData().getGroupSearchSyntaxVersion(); if (currentVersion.isPresent()) { if (currentVersion.get().equals(VERSION_6_0_ALPHA)) { - dialogService.showErrorDialogAndWait(Localization.lang("Search groups migration of %0", parserResult.getPath().map(Path::toString).orElse("")), - Localization.lang("The search groups syntax has been reverted to the old one. Please use the backup you made before migrating to 6.0-alpha.")); + dialogService.showErrorDialogAndWait("Search groups migration of " + parserResult.getPath().map(Path::toString).orElse(""), + "The search groups syntax has been reverted to the old one. Please use the backup you made before migrating to 6.0-alpha."); } return false; } diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 22f0532c1b2..8ab1d558d69 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -271,7 +271,6 @@ Each\ line\ must\ be\ of\ the\ following\ form\:\ \'tab\:field1;field2;...;field Search\ groups\ migration\ of\ %0=Search groups migration of %0 The\ search\ groups\ syntax\ is\ outdated.\ Do\ you\ want\ to\ migrate\ to\ the\ new\ syntax?=The search groups syntax is outdated. Do you want to migrate to the new syntax? -The\ search\ groups\ syntax\ has\ been\ reverted\ to\ the\ old\ one.\ Please\ use\ the\ backup\ you\ made\ before\ migrating\ to\ 6.0-alpha. = The search groups syntax has been reverted to the old one. Please use the backup you made before migrating to 6.0-alpha. Migrate=Migrate Keep\ as\ is=Keep as is Search\ group\ migration\ failed=Search group migration failed From c0a2593faabb80f04f583ffc68c0026fce770358 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 9 Oct 2024 22:13:38 +0300 Subject: [PATCH 070/104] Extract search terms from query (ignore negated terms) --- .../search/query/SearchQueryConversion.java | 7 ++ .../query/SearchQueryExtractorVisitor.java | 65 +++++++++++++++++++ .../SearchQueryExtractorConversionTest.java | 35 ++++++++++ 3 files changed, 107 insertions(+) create mode 100644 src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java create mode 100644 src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java index a142d2ee97d..33082922ff4 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java @@ -1,6 +1,7 @@ package org.jabref.logic.search.query; import java.util.EnumSet; +import java.util.Set; import org.jabref.model.search.SearchFlags; import org.jabref.model.search.ThrowingErrorListener; @@ -36,6 +37,12 @@ public static Query searchToLucene(String searchExpression) { return new SearchToLuceneVisitor().visit(context); } + public static Set extractSearchTerms(String searchExpression) { + LOGGER.debug("Extracting search terms from search expression: {}", searchExpression); + SearchParser.StartContext context = getStartContext(searchExpression); + return new SearchQueryExtractorVisitor().visit(context); + } + public static SearchParser.StartContext getStartContext(String searchExpression) { SearchLexer lexer = new SearchLexer(CharStreams.fromString(searchExpression)); lexer.removeErrorListeners(); // no infos on file system diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java new file mode 100644 index 00000000000..d97fc741ea2 --- /dev/null +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java @@ -0,0 +1,65 @@ +package org.jabref.logic.search.query; + +import java.util.HashSet; +import java.util.Set; + +import org.jabref.search.SearchBaseVisitor; +import org.jabref.search.SearchParser; + +public class SearchQueryExtractorVisitor extends SearchBaseVisitor> { + + private final Set searchTerms = new HashSet<>(); + private boolean isNegated = false; + + @Override + public Set visitStart(SearchParser.StartContext ctx) { + visit(ctx.expression()); + return searchTerms; + } + + @Override + public Set visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { + isNegated = !isNegated; + visit(ctx.expression()); + isNegated = !isNegated; + return searchTerms; + } + + @Override + public Set visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { + visit(ctx.left); + visit(ctx.right); + return searchTerms; + } + + @Override + public Set visitParenExpression(SearchParser.ParenExpressionContext ctx) { + return visit(ctx.expression()); + } + + @Override + public Set visitAtomExpression(SearchParser.AtomExpressionContext ctx) { + return visit(ctx.comparison()); + } + + @Override + public Set visitComparison(SearchParser.ComparisonContext context) { + if (isNegated || + context.NEQUAL() != null || + context.NCEQUAL() != null || + context.NEEQUAL() != null || + context.NCEEQUAL() != null || + context.NREQUAL() != null || + context.NCREEQUAL() != null) { + return searchTerms; + } + + String right = context.right.getText(); + if (right.startsWith("\"") && right.endsWith("\"")) { + right = right.substring(1, right.length() - 1); + } + searchTerms.add(right); + + return searchTerms; + } +} diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java new file mode 100644 index 00000000000..c3294af22b1 --- /dev/null +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java @@ -0,0 +1,35 @@ +package org.jabref.logic.search.query; + +import java.util.Set; +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class SearchQueryExtractorConversionTest { + public static Stream testSearchConversion() { + return Stream.of( + Arguments.of(Set.of("term"), "term"), + Arguments.of(Set.of("regex.*term"), "regex.*term"), + Arguments.of(Set.of("term"), "any = term"), + Arguments.of(Set.of("term"), "any CONTAINS term"), + Arguments.of(Set.of("a", "b"), "a AND b"), + Arguments.of(Set.of("a", "b", "c"), "a OR b AND c"), + Arguments.of(Set.of("a", "b"), "a OR b AND NOT c"), + Arguments.of(Set.of("a", "b"), "author = a AND title = b"), + Arguments.of(Set.of(), "NOT a"), + Arguments.of(Set.of("a", "b", "c"), "(any = a OR any = b) AND NOT (NOT c AND title = d)"), + Arguments.of(Set.of("b", "c"), "title != a OR b OR c") + ); + } + + @ParameterizedTest + @MethodSource + void testSearchConversion(Set expected, String query) { + Set result = SearchQueryConversion.extractSearchTerms(query); + assertEquals(expected, result); + } +} From bf5de0958261d4c5e429e06bd70bf1bd45c1668e Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 9 Oct 2024 22:38:11 +0300 Subject: [PATCH 071/104] Fix architecture test --- .../actions/SearchGroupsMigrationAction.java | 2 +- .../jabref/gui/search/GlobalSearchBar.java | 3 +- .../search/query/SearchQueryConversion.java | 49 +++++-------------- .../search/retrieval/BibFieldsSearcher.java | 2 +- .../search/retrieval/LinkedFilesSearcher.java | 2 +- .../model/search/query/SearchQuery.java | 30 ++++++++++-- .../logic/search/DatabaseSearcherTest.java | 18 +++---- .../SearchQueryExtractorConversionTest.java | 6 ++- .../query/SearchQueryFlagsConversionTest.java | 15 +++--- .../SearchQueryLuceneConversionTest.java | 6 ++- .../query/SearchQuerySQLConversionTest.java | 6 ++- 11 files changed, 72 insertions(+), 67 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index c032dccb5ce..2055474f9d8 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -68,7 +68,7 @@ public void performAction(ParserResult parserResult, DialogService dialogService private void migrateGroups(GroupTreeNode node, DialogService dialogService) { if (node.getGroup() instanceof SearchGroup searchGroup) { try { - String newSearchExpression = SearchQueryConversion.flagsToSearchExpression(searchGroup.getSearchExpression(), searchGroup.getSearchFlags()); + String newSearchExpression = SearchQueryConversion.flagsToSearchExpression(searchGroup.getSearchQuery()); searchGroup.setSearchExpression(newSearchExpression); } catch (ParseCancellationException e) { Optional newSearchExpression = dialogService.showInputDialogWithDefaultAndWait( diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index eeb7a922402..856868a5254 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -2,7 +2,6 @@ import java.lang.reflect.Field; import java.time.Duration; -import java.util.EnumSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -219,7 +218,7 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, // Async update searchTask.restart(); }, - query -> setSearchTerm(query.orElseGet(() -> new SearchQuery("", EnumSet.noneOf(SearchFlags.class))))); + query -> setSearchTerm(query.orElseGet(() -> new SearchQuery("")))); /* * The listener tracks a change on the focus property value. diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java index 33082922ff4..d0b92b08340 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java @@ -1,16 +1,9 @@ package org.jabref.logic.search.query; -import java.util.EnumSet; import java.util.Set; -import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.ThrowingErrorListener; -import org.jabref.search.SearchLexer; -import org.jabref.search.SearchParser; +import org.jabref.model.search.query.SearchQuery; -import org.antlr.v4.runtime.BailErrorStrategy; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CommonTokenStream; import org.apache.lucene.search.Query; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,39 +11,23 @@ public class SearchQueryConversion { private static final Logger LOGGER = LoggerFactory.getLogger(SearchQueryConversion.class); - public static String searchToSql(String table, String searchExpression) { - LOGGER.debug("Converting search expression to SQL: {}", searchExpression); - SearchParser.StartContext context = getStartContext(searchExpression); - SearchToSqlVisitor searchToSqlVisitor = new SearchToSqlVisitor(table); - return searchToSqlVisitor.visit(context); + public static String searchToSql(String table, SearchQuery searchQuery) { + LOGGER.debug("Converting search expression to SQL: {}", searchQuery.getSearchExpression()); + return new SearchToSqlVisitor(table).visit(searchQuery.getContext()); } - public static String flagsToSearchExpression(String searchExpression, EnumSet searchFlags) { - LOGGER.debug("Converting search flags to search expression: {}, flags {}", searchExpression, searchFlags); - SearchParser.StartContext context = getStartContext(searchExpression); - return new SearchFlagsToExpressionVisitor(searchFlags).visit(context); + public static String flagsToSearchExpression(SearchQuery searchQuery) { + LOGGER.debug("Converting search flags to search expression: {}, flags {}", searchQuery.getSearchExpression(), searchQuery.getSearchFlags()); + return new SearchFlagsToExpressionVisitor(searchQuery.getSearchFlags()).visit(searchQuery.getContext()); } - public static Query searchToLucene(String searchExpression) { - LOGGER.debug("Converting search expression to Lucene: {}", searchExpression); - SearchParser.StartContext context = getStartContext(searchExpression); - return new SearchToLuceneVisitor().visit(context); + public static Query searchToLucene(SearchQuery searchQuery) { + LOGGER.debug("Converting search expression to Lucene: {}", searchQuery.getSearchExpression()); + return new SearchToLuceneVisitor().visit(searchQuery.getContext()); } - public static Set extractSearchTerms(String searchExpression) { - LOGGER.debug("Extracting search terms from search expression: {}", searchExpression); - SearchParser.StartContext context = getStartContext(searchExpression); - return new SearchQueryExtractorVisitor().visit(context); - } - - public static SearchParser.StartContext getStartContext(String searchExpression) { - SearchLexer lexer = new SearchLexer(CharStreams.fromString(searchExpression)); - lexer.removeErrorListeners(); // no infos on file system - lexer.addErrorListener(ThrowingErrorListener.INSTANCE); - SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); - parser.removeErrorListeners(); // no infos on file system - parser.addErrorListener(ThrowingErrorListener.INSTANCE); - parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors - return parser.start(); + public static Set extractSearchTerms(SearchQuery searchQuery) { + LOGGER.debug("Extracting search terms from search expression: {}", searchQuery.getSearchExpression()); + return new SearchQueryExtractorVisitor().visit(searchQuery.getContext()); } } diff --git a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java index adc11398b78..9bb7e50f3f0 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java @@ -41,7 +41,7 @@ public SearchResults search(SearchQuery searchQuery) { if (!searchQuery.isValid()) { return new SearchResults(); } - String sqlQuery = SearchQueryConversion.searchToSql(tableName, searchQuery.getSearchExpression()); + String sqlQuery = SearchQueryConversion.searchToSql(tableName, searchQuery); LOGGER.debug("Searching in bib fields with query: {}", sqlQuery); SearchResults searchResults = new SearchResults(); try (PreparedStatement preparedStatement = connection.prepareStatement(sqlQuery)) { diff --git a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java index 7cc91e946c7..0e88cb47d06 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LinkedFilesSearcher.java @@ -51,7 +51,7 @@ public SearchResults search(SearchQuery searchQuery) { return new SearchResults(); } - Query luceneQuery = SearchQueryConversion.searchToLucene(searchQuery.getSearchExpression()); + Query luceneQuery = SearchQueryConversion.searchToLucene(searchQuery); EnumSet searchFlags = searchQuery.getSearchFlags(); boolean shouldSearchInLinkedFiles = searchFlags.contains(SearchFlags.FULLTEXT) && filePreferences.shouldFulltextIndexLinkedFiles(); if (!shouldSearchInLinkedFiles) { diff --git a/src/main/java/org/jabref/model/search/query/SearchQuery.java b/src/main/java/org/jabref/model/search/query/SearchQuery.java index 3663fa06a54..62c115e8163 100644 --- a/src/main/java/org/jabref/model/search/query/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/query/SearchQuery.java @@ -3,24 +3,33 @@ import java.util.EnumSet; import java.util.Objects; -import org.jabref.logic.search.query.SearchQueryConversion; import org.jabref.model.search.SearchFlags; +import org.jabref.model.search.ThrowingErrorListener; +import org.jabref.search.SearchLexer; +import org.jabref.search.SearchParser; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.misc.ParseCancellationException; public class SearchQuery { private final String searchExpression; private final EnumSet searchFlags; - + private SearchParser.StartContext context; private boolean isValidExpression; private SearchResults searchResults; + public SearchQuery(String searchExpression) { + this(searchExpression, EnumSet.noneOf(SearchFlags.class)); + } + public SearchQuery(String searchExpression, EnumSet searchFlags) { this.searchExpression = Objects.requireNonNull(searchExpression); this.searchFlags = searchFlags; try { - SearchQueryConversion.getStartContext(searchExpression); + this.context = getStartContext(searchExpression); isValidExpression = true; } catch (ParseCancellationException e) { isValidExpression = false; @@ -47,6 +56,10 @@ public EnumSet getSearchFlags() { return searchFlags; } + public SearchParser.StartContext getContext() { + return context; + } + @Override public String toString() { return searchExpression; @@ -68,4 +81,15 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(searchExpression, searchFlags); } + + public static SearchParser.StartContext getStartContext(String searchExpression) { + SearchLexer lexer = new SearchLexer(CharStreams.fromString(searchExpression)); + lexer.removeErrorListeners(); // no infos on file system + lexer.addErrorListener(ThrowingErrorListener.INSTANCE); + SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); + parser.removeErrorListeners(); // no infos on file system + parser.addErrorListener(ThrowingErrorListener.INSTANCE); + parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors + return parser.start(); + } } diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index 14db4f7d35b..0c4b9bba2a1 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.nio.file.Path; -import java.util.EnumSet; import java.util.List; import java.util.stream.Stream; @@ -17,7 +16,6 @@ import org.jabref.model.entry.BibEntryPreferences; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.model.search.SearchFlags; import org.jabref.model.search.query.SearchQuery; import org.junit.jupiter.api.BeforeEach; @@ -73,18 +71,18 @@ private static Stream testDatabaseSearcher() { inCollectionEntry.setField(StandardField.AUTHOR, "tonho"); return Stream.of( - Arguments.of(List.of(), new SearchQuery("whatever", EnumSet.noneOf(SearchFlags.class)), List.of()), - Arguments.of(List.of(), new SearchQuery("whatever", EnumSet.noneOf(SearchFlags.class)), List.of(emptyEntry)), - Arguments.of(List.of(), new SearchQuery("whatever", EnumSet.noneOf(SearchFlags.class)), List.of(emptyEntry, articleEntry, inCollectionEntry)), + Arguments.of(List.of(), new SearchQuery("whatever"), List.of()), + Arguments.of(List.of(), new SearchQuery("whatever"), List.of(emptyEntry)), + Arguments.of(List.of(), new SearchQuery("whatever"), List.of(emptyEntry, articleEntry, inCollectionEntry)), // invalid search syntax - Arguments.of(List.of(), new SearchQuery("author:", EnumSet.noneOf(SearchFlags.class)), List.of(articleEntry)), + Arguments.of(List.of(), new SearchQuery("author:"), List.of(articleEntry)), - Arguments.of(List.of(articleEntry), new SearchQuery("harrer", EnumSet.noneOf(SearchFlags.class)), List.of(articleEntry)), - Arguments.of(List.of(), new SearchQuery("title: harrer", EnumSet.noneOf(SearchFlags.class)), List.of(articleEntry)), + Arguments.of(List.of(articleEntry), new SearchQuery("harrer"), List.of(articleEntry)), + Arguments.of(List.of(), new SearchQuery("title: harrer"), List.of(articleEntry)), - Arguments.of(List.of(inCollectionEntry), new SearchQuery("tonho", EnumSet.noneOf(SearchFlags.class)), List.of(inCollectionEntry)), - Arguments.of(List.of(inCollectionEntry), new SearchQuery("tonho", EnumSet.noneOf(SearchFlags.class)), List.of(articleEntry, inCollectionEntry)) + Arguments.of(List.of(inCollectionEntry), new SearchQuery("tonho"), List.of(inCollectionEntry)), + Arguments.of(List.of(inCollectionEntry), new SearchQuery("tonho"), List.of(articleEntry, inCollectionEntry)) ); } } diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java index c3294af22b1..d9cd260cd9d 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java @@ -3,6 +3,8 @@ import java.util.Set; import java.util.stream.Stream; +import org.jabref.model.search.query.SearchQuery; + import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -28,8 +30,8 @@ public static Stream testSearchConversion() { @ParameterizedTest @MethodSource - void testSearchConversion(Set expected, String query) { - Set result = SearchQueryConversion.extractSearchTerms(query); + void testSearchConversion(Set expected, String searchExpression) { + Set result = SearchQueryConversion.extractSearchTerms(new SearchQuery(searchExpression)); assertEquals(expected, result); } } diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java index f7aba638a8e..311863d40f5 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java @@ -4,6 +4,7 @@ import java.util.stream.Stream; import org.jabref.model.search.SearchFlags; +import org.jabref.model.search.query.SearchQuery; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -72,19 +73,19 @@ private static Stream testSearchConversion() { ).flatMap(stream -> stream); } - private static Stream createTestCases(String query, String noneExpected, String caseSensitiveExpected, String regexExpected, String bothExpected) { + private static Stream createTestCases(String searchExpression, String noneExpected, String caseSensitiveExpected, String regexExpected, String bothExpected) { return Stream.of( - Arguments.of(noneExpected, query, EnumSet.noneOf(SearchFlags.class)), - Arguments.of(caseSensitiveExpected, query, EnumSet.of(SearchFlags.CASE_SENSITIVE)), - Arguments.of(regexExpected, query, EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), - Arguments.of(bothExpected, query, EnumSet.of(SearchFlags.CASE_SENSITIVE, SearchFlags.REGULAR_EXPRESSION)) + Arguments.of(noneExpected, searchExpression, EnumSet.noneOf(SearchFlags.class)), + Arguments.of(caseSensitiveExpected, searchExpression, EnumSet.of(SearchFlags.CASE_SENSITIVE)), + Arguments.of(regexExpected, searchExpression, EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), + Arguments.of(bothExpected, searchExpression, EnumSet.of(SearchFlags.CASE_SENSITIVE, SearchFlags.REGULAR_EXPRESSION)) ); } @ParameterizedTest @MethodSource - void testSearchConversion(String expected, String query, EnumSet flags) { - String result = SearchQueryConversion.flagsToSearchExpression(query, flags); + void testSearchConversion(String expected, String searchExpression, EnumSet flags) { + String result = SearchQueryConversion.flagsToSearchExpression(new SearchQuery(searchExpression, flags)); assertEquals(expected, result); } } diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java index 964a725ca30..abb6e9640ea 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java @@ -2,6 +2,8 @@ import java.util.stream.Stream; +import org.jabref.model.search.query.SearchQuery; + import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -57,8 +59,8 @@ public static Stream testSearchConversion() { @ParameterizedTest @MethodSource - void testSearchConversion(String expected, String query) { - String result = SearchQueryConversion.searchToLucene(query).toString(); + void testSearchConversion(String expected, String searchExpression) { + String result = SearchQueryConversion.searchToLucene(new SearchQuery(searchExpression)).toString(); assertEquals(expected, result); } } diff --git a/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java index d75a30b5cb5..e88cf36dcfb 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java @@ -2,6 +2,8 @@ import java.util.stream.Stream; +import org.jabref.model.search.query.SearchQuery; + import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -625,7 +627,7 @@ cte5 AS ( @ParameterizedTest @MethodSource - void testSearchConversion(String input, String expected) { - assertEquals(expected, SearchQueryConversion.searchToSql("tableName", input)); + void testSearchConversion(String searchExpression, String expected) { + assertEquals(expected, SearchQueryConversion.searchToSql("tableName", new SearchQuery(searchExpression))); } } From 46bb0d3f0c07b5481a26d53bf7883e3e6c521c36 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 10 Oct 2024 00:12:53 +0300 Subject: [PATCH 072/104] Highlight Preview viewer with Postgres regexp_replace --- .../org/jabref/gui/preview/PreviewViewer.java | 106 ++++-------------- .../jabref/gui/search/GlobalSearchBar.java | 14 +-- .../logic/search/retrieval/Highlighter.java | 86 ++++++++++++++ src/main/resources/l10n/JabRef_en.properties | 2 +- 4 files changed, 111 insertions(+), 97 deletions(-) create mode 100644 src/main/java/org/jabref/logic/search/retrieval/Highlighter.java diff --git a/src/main/java/org/jabref/gui/preview/PreviewViewer.java b/src/main/java/org/jabref/gui/preview/PreviewViewer.java index 7add9e3aa5d..7f778060e03 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewViewer.java +++ b/src/main/java/org/jabref/gui/preview/PreviewViewer.java @@ -6,7 +6,6 @@ import java.net.MalformedURLException; import java.util.Objects; import java.util.Optional; -import java.util.regex.Pattern; import javafx.beans.InvalidationListener; import javafx.beans.Observable; @@ -26,6 +25,7 @@ import org.jabref.logic.l10n.Localization; import org.jabref.logic.layout.format.Number; import org.jabref.logic.preview.PreviewLayout; +import org.jabref.logic.search.retrieval.Highlighter; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.TaskExecutor; import org.jabref.logic.util.WebViewStore; @@ -71,69 +71,20 @@ function getSelectionHtml() { } getSelectionHtml(); """; - private static final String JS_HIGHLIGHT_FUNCTION = - """ - - - - - - """; - // This is a string format, and it takes a variable name as an argument to pass to the markInstance.markRegExp() Javascript method. - private static final String JS_MARK_REG_EXP_CALLBACK = - """ - {done: function(){ - markInstance.markRegExp(%s);} - }"""; - - // This is a string format, and it takes a variable name as an argument to pass to the markInstance.unmark() Javascript method. - private static final String JS_UNMARK_WITH_CALLBACK = - """ - var markInstance = new Mark(document.getElementById("content")); - markInstance.unmark(%s);"""; - private static final Pattern UNESCAPED_FORWARD_SLASH = Pattern.compile("(? searchQueryProperty; private final ChangeListener> listener = (queryObservable, queryOldValue, queryNewValue) -> { - // TODO: update search pattern - highlightSearchPattern(); + highlightLayoutText(); }; private Optional entry = Optional.empty(); - private Optional searchHighlightPattern = Optional.empty(); private PreviewLayout layout; private boolean registered; + private String layoutText; /** * @param database Used for resolving strings and pdf directories for links. @@ -148,12 +99,14 @@ public PreviewViewer(BibDatabaseContext database, this.dialogService = dialogService; this.clipBoardManager = Injector.instantiateModelOrService(ClipBoardManager.class); this.taskExecutor = taskExecutor; + this.searchQueryProperty = searchQueryProperty; setFitToHeight(true); setFitToWidth(true); previewView = WebViewStore.get(); setContent(previewView); previewView.setContextMenuEnabled(false); + previewView.getEngine().setJavaScriptEnabled(true); previewView.getEngine().getLoadWorker().stateProperty().addListener((observable, oldValue, newValue) -> { if (newValue != Worker.State.SUCCEEDED) { @@ -161,11 +114,9 @@ public PreviewViewer(BibDatabaseContext database, } if (!registered) { - // TODO: update search pattern searchQueryProperty.addListener(listener); registered = true; } - highlightSearchPattern(); // https://stackoverflow.com/questions/15555510/javafx-stop-opening-url-in-webview-open-in-browser-instead NodeList anchorList = previewView.getEngine().getDocument().getElementsByTagName("a"); @@ -201,30 +152,6 @@ public PreviewViewer(BibDatabaseContext database, this(database, dialogService, preferences, themeManager, taskExecutor, OptionalObjectProperty.empty()); } - private void highlightSearchPattern() { - String callbackForUnmark = ""; - if (searchHighlightPattern.isPresent()) { - String javaScriptRegex = createJavaScriptRegex(searchHighlightPattern.get()); - callbackForUnmark = JS_MARK_REG_EXP_CALLBACK.formatted(javaScriptRegex); - } - String unmarkInstance = JS_UNMARK_WITH_CALLBACK.formatted(callbackForUnmark); - previewView.getEngine().executeScript(unmarkInstance); - } - - /** - * Returns the String representation of a JavaScript regex object. The method does not take into account differences between the regex implementations in Java and JavaScript. - * - * @param regex Java regex to print as a JavaScript regex - * @return JavaScript regex object - */ - private static String createJavaScriptRegex(Pattern regex) { - String pattern = regex.pattern(); - // Create a JavaScript regular expression literal (https://ecma-international.org/ecma-262/10.0/index.html#sec-literals-regular-expression-literals) - // Forward slashes are reserved to delimit the regular expression body. Hence, they must be escaped. - pattern = UNESCAPED_FORWARD_SLASH.matcher(pattern).replaceAll("\\\\/"); - return "/" + pattern + "/gmi"; - } - public void setLayout(PreviewLayout newLayout) { // Change listeners might set the layout to null while the update method is executing, therefore we need to prevent this here if (newLayout == null || newLayout.equals(layout)) { @@ -285,20 +212,29 @@ private void update() { } private void setPreviewText(String text) { - String myText = """ + layoutText = """ - %s

%s
- """.formatted(JS_HIGHLIGHT_FUNCTION, text); - previewView.getEngine().setJavaScriptEnabled(true); - previewView.getEngine().loadContent(myText); - + """.formatted(text); + highlightLayoutText(); this.setHvalue(0); } + private void highlightLayoutText() { + if (layoutText == null) { + return; + } + if (searchQueryProperty.get().isPresent()) { + String highlightedHtml = Highlighter.highlightHtml(layoutText, searchQueryProperty.get().get()); + previewView.getEngine().loadContent(highlightedHtml); + } else { + previewView.getEngine().loadContent(layoutText); + } + } + public void print() { PrinterJob job = PrinterJob.createPrinterJob(); boolean proceed = dialogService.showPrintDialog(job); diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index 856868a5254..74a172c7d79 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -127,17 +127,14 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, currentResults.textProperty().bind(EasyBind.combine( stateManager.activeSearchQuery(searchType), stateManager.searchResultSize(searchType), illegalSearch, (searchQuery, matched, illegal) -> { + searchField.pseudoClassStateChanged(ILLEGAL_SEARCH, illegal); if (illegal) { - searchField.pseudoClassStateChanged(ILLEGAL_SEARCH, true); - return Localization.lang("Search failed: illegal search expression"); + return Localization.lang("Illegal search expression"); } else if (searchQuery.isEmpty()) { - searchField.pseudoClassStateChanged(ILLEGAL_SEARCH, false); return ""; } else if (matched.intValue() == 0) { - searchField.pseudoClassStateChanged(ILLEGAL_SEARCH, false); return Localization.lang("No results found."); } else { - searchField.pseudoClassStateChanged(ILLEGAL_SEARCH, false); return Localization.lang("Found %0 results.", String.valueOf(matched)); } } @@ -339,12 +336,7 @@ public void updateSearchQuery() { } SearchQuery searchQuery = new SearchQuery(this.searchField.getText(), searchPreferences.getSearchFlags()); - if (!searchQuery.isValid()) { - illegalSearch.set(true); - return; - } else { - illegalSearch.set(false); - } + illegalSearch.set(!searchQuery.isValid()); stateManager.activeSearchQuery(searchType).set(Optional.of(searchQuery)); } diff --git a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java new file mode 100644 index 00000000000..86e818a15b1 --- /dev/null +++ b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java @@ -0,0 +1,86 @@ +package org.jabref.logic.search.retrieval; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Set; + +import org.jabref.logic.search.PostgreServer; +import org.jabref.logic.search.query.SearchQueryConversion; +import org.jabref.model.search.query.SearchQuery; + +import com.airhacks.afterburner.injection.Injector; +import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.nodes.Node; +import org.jsoup.nodes.TextNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Highlighter { + private static final Logger LOGGER = LoggerFactory.getLogger(Highlighter.class); + private final static String HIGHLIGHT_QUERY = """ + SELECT + regexp_replace( + '%s', + '(%s)', + '\\1', + 'gi' + ) + """; + private static Connection connection; + + public static String highlightHtml(String htmlText, SearchQuery searchQuery) { + if (!searchQuery.isValid()) { + return htmlText; + } + + LOGGER.debug("Highlighting search terms in text: {}", searchQuery); + Set terms = SearchQueryConversion.extractSearchTerms(searchQuery); + if (terms.isEmpty()) { + return htmlText; + } + + String joinedTerms = String.join("|", terms); + Document document = Jsoup.parse(htmlText); + try { + highlightTextNodes(document.body(), joinedTerms); + String highlightedHtml = document.outerHtml(); + LOGGER.debug("Highlighted HTML: {}", highlightedHtml); + return highlightedHtml; + } catch (InvalidTokenOffsetsException e) { + LOGGER.debug("Error highlighting search terms in HTML", e); + return htmlText; + } + } + + private static void highlightTextNodes(Element element, String searchTerms) throws InvalidTokenOffsetsException { + for (Node node : element.childNodes()) { + if (node instanceof TextNode textNode) { + String highlightedText = highlightNode(textNode.text(), searchTerms); + textNode.text(""); + textNode.after(highlightedText); + } else if (node instanceof Element) { + highlightTextNodes((Element) node, searchTerms); + } + } + } + + public static String highlightNode(String text, String terms) { + if (connection == null) { + connection = Injector.instantiateModelOrService(PostgreServer.class).getConnection(); + } + + try { + String query = String.format(HIGHLIGHT_QUERY, text, terms); + ResultSet resultSet = connection.createStatement().executeQuery(query); + resultSet.next(); + return resultSet.getString(1); + } catch (SQLException e) { + LOGGER.error("Error highlighting search terms in text", e); + return text; + } + } +} diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 8356a708ad8..0568cedbda2 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -827,7 +827,7 @@ Searching\ for\ files=Searching for files Use\ regular\ expression\ search=Use regular expression search search\ expression=search expression Free\ search\ expression=Free search expression -Search\ failed\:\ illegal\ search\ expression=Search failed: illegal search expression +Illegal\ search\ expression=Illegal search expression No\ search\ matches.=No search matches. Web\ search=Web search Search\ results=Search results From d451c2b4126e38d4366310e904289b139bd31dee Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 10 Oct 2024 00:35:49 +0300 Subject: [PATCH 073/104] OpenRewrite --- .../java/org/jabref/logic/search/retrieval/Highlighter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java index 86e818a15b1..5080968cb0d 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java +++ b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java @@ -62,8 +62,8 @@ private static void highlightTextNodes(Element element, String searchTerms) thro String highlightedText = highlightNode(textNode.text(), searchTerms); textNode.text(""); textNode.after(highlightedText); - } else if (node instanceof Element) { - highlightTextNodes((Element) node, searchTerms); + } else if (node instanceof Element element1) { + highlightTextNodes(element1, searchTerms); } } } @@ -74,7 +74,7 @@ public static String highlightNode(String text, String terms) { } try { - String query = String.format(HIGHLIGHT_QUERY, text, terms); + String query = HIGHLIGHT_QUERY.formatted(text, terms); ResultSet resultSet = connection.createStatement().executeQuery(query); resultSet.next(); return resultSet.getString(1); From 887d5e3b858264307b47e6acac2e8061765459dd Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 10 Oct 2024 01:09:28 +0300 Subject: [PATCH 074/104] Remove onRunning --- src/main/java/org/jabref/gui/preview/PreviewViewer.java | 1 - src/main/resources/l10n/JabRef_en.properties | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/preview/PreviewViewer.java b/src/main/java/org/jabref/gui/preview/PreviewViewer.java index 7f778060e03..ff7b11b4054 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewViewer.java +++ b/src/main/java/org/jabref/gui/preview/PreviewViewer.java @@ -194,7 +194,6 @@ private void update() { final BibEntry theEntry = entry.get(); BackgroundTask .wrap(() -> layout.generatePreview(theEntry, database)) - .onRunning(() -> setPreviewText("" + Localization.lang("Processing Citation Style \"%0\"...", layout.getDisplayName()) + "")) .onSuccess(this::setPreviewText) .onFailure(exception -> { LOGGER.error("Error while generating citation style", exception); diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 0568cedbda2..88e83783d06 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -318,7 +318,6 @@ Extract\ References\ (online)=Extract References (online) Processing...=Processing... Processing\ "%0"...=Processing "%0"... -Processing\ Citation\ Style\ "%0"...=Processing Citation Style "%0"... Processing\ PDF(s)=Processing PDF(s) Processing\ file\ %0=Processing file %0 Processing\ a\ large\ number\ of\ files=Processing a large number of files From d26c8b91c775df1490c48e54ede07b9d2e4c2fe3 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 10 Oct 2024 01:38:38 +0300 Subject: [PATCH 075/104] Set search query listener in the constructor --- .../java/org/jabref/gui/preview/PreviewViewer.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/main/java/org/jabref/gui/preview/PreviewViewer.java b/src/main/java/org/jabref/gui/preview/PreviewViewer.java index ff7b11b4054..29d6ed72ea8 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewViewer.java +++ b/src/main/java/org/jabref/gui/preview/PreviewViewer.java @@ -9,7 +9,6 @@ import javafx.beans.InvalidationListener; import javafx.beans.Observable; -import javafx.beans.value.ChangeListener; import javafx.concurrent.Worker; import javafx.print.PrinterJob; import javafx.scene.control.ScrollPane; @@ -77,13 +76,9 @@ function getSelectionHtml() { private final WebView previewView; private final BibDatabaseContext database; private final OptionalObjectProperty searchQueryProperty; - private final ChangeListener> listener = (queryObservable, queryOldValue, queryNewValue) -> { - highlightLayoutText(); - }; private Optional entry = Optional.empty(); private PreviewLayout layout; - private boolean registered; private String layoutText; /** @@ -100,6 +95,7 @@ public PreviewViewer(BibDatabaseContext database, this.clipBoardManager = Injector.instantiateModelOrService(ClipBoardManager.class); this.taskExecutor = taskExecutor; this.searchQueryProperty = searchQueryProperty; + this.searchQueryProperty.addListener((queryObservable, queryOldValue, queryNewValue) -> highlightLayoutText()); setFitToHeight(true); setFitToWidth(true); @@ -113,11 +109,6 @@ public PreviewViewer(BibDatabaseContext database, return; } - if (!registered) { - searchQueryProperty.addListener(listener); - registered = true; - } - // https://stackoverflow.com/questions/15555510/javafx-stop-opening-url-in-webview-open-in-browser-instead NodeList anchorList = previewView.getEngine().getDocument().getElementsByTagName("a"); for (int i = 0; i < anchorList.getLength(); i++) { From eafdf3007e21cd646b4cd2fe882567d9f10215fe Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 10 Oct 2024 01:55:44 +0300 Subject: [PATCH 076/104] Fix preview tab scrolling --- src/main/java/org/jabref/gui/preview/PreviewPanel.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/jabref/gui/preview/PreviewPanel.java b/src/main/java/org/jabref/gui/preview/PreviewPanel.java index 8d3842d2454..23d1510b49c 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewPanel.java +++ b/src/main/java/org/jabref/gui/preview/PreviewPanel.java @@ -14,6 +14,7 @@ import javafx.scene.input.Dragboard; import javafx.scene.input.KeyEvent; import javafx.scene.input.TransferMode; +import javafx.scene.layout.Priority; import javafx.scene.layout.VBox; import org.jabref.gui.DialogService; @@ -105,6 +106,7 @@ public PreviewPanel(BibDatabaseContext database, event.consume(); }); this.getChildren().add(previewView); + VBox.setVgrow(previewView, Priority.ALWAYS); createKeyBindings(); previewView.setLayout(previewPreferences.getSelectedPreviewLayout()); From d17450879edfdec6862809d6cf6b02618dc139ec Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 10 Oct 2024 02:20:26 +0300 Subject: [PATCH 077/104] Use prepared statement to fix escaping --- .../logic/search/retrieval/Highlighter.java | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java index 5080968cb0d..49dba48ec28 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java +++ b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java @@ -1,6 +1,7 @@ package org.jabref.logic.search.retrieval; import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Set; @@ -24,8 +25,8 @@ public class Highlighter { private final static String HIGHLIGHT_QUERY = """ SELECT regexp_replace( - '%s', - '(%s)', + ?, + ?, '\\1', 'gi' ) @@ -68,19 +69,23 @@ private static void highlightTextNodes(Element element, String searchTerms) thro } } - public static String highlightNode(String text, String terms) { + private static String highlightNode(String text, String searchTerms) { if (connection == null) { connection = Injector.instantiateModelOrService(PostgreServer.class).getConnection(); } - try { - String query = HIGHLIGHT_QUERY.formatted(text, terms); - ResultSet resultSet = connection.createStatement().executeQuery(query); - resultSet.next(); - return resultSet.getString(1); + try (PreparedStatement preparedStatement = connection.prepareStatement(HIGHLIGHT_QUERY)) { + preparedStatement.setString(1, text); + preparedStatement.setString(2, '(' + searchTerms + ')'); + + try (ResultSet resultSet = preparedStatement.executeQuery()) { + if (resultSet.next()) { + return resultSet.getString(1); + } + } } catch (SQLException e) { LOGGER.error("Error highlighting search terms in text", e); - return text; } + return text; } } From ef959bc4b6a33532e7557e878d9dd6fbfb30da52 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 10 Oct 2024 07:57:41 +0300 Subject: [PATCH 078/104] Use prepared statement for sql query --- .../search/query/SearchQueryConversion.java | 3 +- .../search/query/SearchToSqlVisitor.java | 266 ++++++++++-------- .../search/retrieval/BibFieldsSearcher.java | 10 +- .../jabref/model/search/query/SqlQuery.java | 27 ++ .../query/SearchQuerySQLConversionTest.java | 133 ++++++--- 5 files changed, 275 insertions(+), 164 deletions(-) create mode 100644 src/main/java/org/jabref/model/search/query/SqlQuery.java diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java index d0b92b08340..7b5cd68fe83 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java @@ -3,6 +3,7 @@ import java.util.Set; import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.search.query.SqlQuery; import org.apache.lucene.search.Query; import org.slf4j.Logger; @@ -11,7 +12,7 @@ public class SearchQueryConversion { private static final Logger LOGGER = LoggerFactory.getLogger(SearchQueryConversion.class); - public static String searchToSql(String table, SearchQuery searchQuery) { + public static SqlQuery searchToSql(String table, SearchQuery searchQuery) { LOGGER.debug("Converting search expression to SQL: {}", searchQuery.getSearchExpression()); return new SearchToSqlVisitor(table).visit(searchQuery.getContext()); } diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index c67ed4ef4ea..7275f1fbe24 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -1,6 +1,7 @@ package org.jabref.logic.search.query; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.Locale; @@ -11,6 +12,7 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.PostgreConstants; import org.jabref.model.search.query.SearchTermFlag; +import org.jabref.model.search.query.SqlQuery; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; @@ -29,7 +31,7 @@ * Converts to a query processable by the scheme created by {@link BibFieldsIndexer}. * Tests are located in {@link org.jabref.logic.search.query.SearchQuerySQLConversionTest}. */ -public class SearchToSqlVisitor extends SearchBaseVisitor { +public class SearchToSqlVisitor extends SearchBaseVisitor { private static final String MAIN_TABLE = "main_table"; private static final String SPLIT_TABLE = "split_table"; @@ -38,8 +40,7 @@ public class SearchToSqlVisitor extends SearchBaseVisitor { private final String mainTableName; private final String splitValuesTableName; - - private final List ctes = new ArrayList<>(); + private final List nodes = new ArrayList<>(); private int cteCounter = 0; public SearchToSqlVisitor(String table) { @@ -48,24 +49,31 @@ public SearchToSqlVisitor(String table) { } @Override - public String visitStart(SearchParser.StartContext ctx) { - String query = visit(ctx.expression()); + public SqlQuery visitStart(SearchParser.StartContext ctx) { + SqlQuery finalNode = visit(ctx.expression()); StringBuilder sql = new StringBuilder("WITH\n"); - ctes.forEach(cte -> sql.append(cte).append(",\n")); + List params = new ArrayList<>(); + + for (SqlQuery node : nodes) { + sql.append(node.cte()).append(",\n"); + params.addAll(node.params()); + } // Remove the last comma and newline - if (!ctes.isEmpty()) { + if (!nodes.isEmpty()) { sql.setLength(sql.length() - 2); } - sql.append("SELECT * FROM ").append(query).append(" GROUP BY ").append(ENTRY_ID); - return sql.toString(); + sql.append("SELECT * FROM ").append(finalNode.cte()).append(" GROUP BY ").append(ENTRY_ID); + params.addAll(finalNode.params()); + + return new SqlQuery(sql.toString(), params); } @Override - public String visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { - String subQuery = visit(ctx.expression()); + public SqlQuery visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { + SqlQuery subNode = visit(ctx.expression()); String cte = """ cte%d AS ( SELECT %s.%s @@ -81,15 +89,17 @@ public String visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { mainTableName, MAIN_TABLE, MAIN_TABLE, ENTRY_ID, ENTRY_ID, - subQuery); - ctes.add(cte); - return "cte" + cteCounter++; + subNode.cte()); + + SqlQuery node = new SqlQuery(cte, subNode.params()); + nodes.add(node); + return new SqlQuery("cte" + cteCounter++); } @Override - public String visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { - String left = visit(ctx.left); - String right = visit(ctx.right); + public SqlQuery visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { + SqlQuery left = visit(ctx.left); + SqlQuery right = visit(ctx.right); String operator = "AND".equalsIgnoreCase(ctx.operator.getText()) ? "INTERSECT" : "UNION"; String cte = """ @@ -103,31 +113,31 @@ public String visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { """.formatted( cteCounter, ENTRY_ID, - left, + left.cte(), operator, ENTRY_ID, - right); - ctes.add(cte); - return "cte" + cteCounter++; + right.cte()); + + List params = new ArrayList<>(left.params()); + params.addAll(right.params()); + + SqlQuery node = new SqlQuery(cte, params); + nodes.add(node); + return new SqlQuery("cte" + cteCounter++); } @Override - public String visitParenExpression(SearchParser.ParenExpressionContext ctx) { + public SqlQuery visitParenExpression(SearchParser.ParenExpressionContext ctx) { return visit(ctx.expression()); } @Override - public String visitAtomExpression(SearchParser.AtomExpressionContext ctx) { + public SqlQuery visitAtomExpression(SearchParser.AtomExpressionContext ctx) { return visit(ctx.comparison()); } @Override - public String visitName(SearchParser.NameContext ctx) { - return ctx.getText(); - } - - @Override - public String visitComparison(SearchParser.ComparisonContext context) { + public SqlQuery visitComparison(SearchParser.ComparisonContext context) { // The comparison is a leaf node in the tree // remove possible enclosing " symbols @@ -137,7 +147,6 @@ public String visitComparison(SearchParser.ComparisonContext context) { } Optional fieldDescriptor = Optional.ofNullable(context.left); - String cte; if (fieldDescriptor.isPresent()) { String field = fieldDescriptor.get().getText(); @@ -170,17 +179,14 @@ public String visitComparison(SearchParser.ComparisonContext context) { setFlags(searchFlags, REGULAR_EXPRESSION, true, true); } - cte = getFieldQueryNode(field.toLowerCase(Locale.ROOT), right, searchFlags); + return getFieldQueryNode(field.toLowerCase(Locale.ROOT), right, searchFlags); } else { // Query without any field name - cte = getFieldQueryNode("any", right, EnumSet.of(INEXACT_MATCH, CASE_INSENSITIVE)); + return getFieldQueryNode("any", right, EnumSet.of(INEXACT_MATCH, CASE_INSENSITIVE)); } - ctes.add(cte); - return "cte" + cteCounter++; } - private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { - String cte; + private SqlQuery getFieldQueryNode(String field, String term, EnumSet searchFlags) { String operator = getOperator(searchFlags); String prefixSuffix = searchFlags.contains(INEXACT_MATCH) ? "%" : ""; @@ -192,66 +198,68 @@ private String getFieldQueryNode(String field, String term, EnumSet params = Collections.nCopies(2, prefixSuffix + term + prefixSuffix); + SqlQuery node = new SqlQuery(cte, params); + nodes.add(node); + return new SqlQuery("cte" + cteCounter++); } - private String buildContainsNegationFieldQuery(String field, String operator, String prefixSuffix, String term) { - return """ + private SqlQuery buildContainsNegationFieldQuery(String field, String operator, String prefixSuffix, String term) { + String cte = """ cte%d AS ( SELECT %s.%s FROM %s AS %s @@ -259,7 +267,7 @@ private String buildContainsNegationFieldQuery(String field, String operator, St SELECT %s.%s FROM %s AS %s WHERE ( - (%s.%s = '%s') AND ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) + (%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?)) ) ) ) @@ -270,27 +278,29 @@ private String buildContainsNegationFieldQuery(String field, String operator, St MAIN_TABLE, ENTRY_ID, INNER_TABLE, ENTRY_ID, mainTableName, INNER_TABLE, - INNER_TABLE, FIELD_NAME, - field, + INNER_TABLE, FIELD_NAME, field, INNER_TABLE, FIELD_VALUE_LITERAL, operator, - prefixSuffix, term, prefixSuffix, INNER_TABLE, FIELD_VALUE_TRANSFORMED, - operator, - prefixSuffix, term, prefixSuffix); + operator); + + List params = Collections.nCopies(2, prefixSuffix + term + prefixSuffix); + SqlQuery node = new SqlQuery(cte, params); + nodes.add(node); + return new SqlQuery("cte" + cteCounter++); } - private String buildExactFieldQuery(String field, String operator, String term) { - return """ + private SqlQuery buildExactFieldQuery(String field, String operator, String term) { + String cte = """ cte%d AS ( SELECT %s.%s FROM %s AS %s LEFT JOIN %s AS %s ON (%s.%s = %s.%s AND %s.%s = %s.%s) WHERE ( - ((%s.%s = '%s') AND ((%s.%s %s '%s') OR (%s.%s %s '%s'))) + ((%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?))) OR - ((%s.%s = '%s') AND ((%s.%s %s '%s') OR (%s.%s %s '%s'))) + ((%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?))) ) ) """.formatted( @@ -301,15 +311,20 @@ private String buildExactFieldQuery(String field, String operator, String term) MAIN_TABLE, ENTRY_ID, SPLIT_TABLE, ENTRY_ID, MAIN_TABLE, FIELD_NAME, SPLIT_TABLE, FIELD_NAME, MAIN_TABLE, FIELD_NAME, field, - MAIN_TABLE, FIELD_VALUE_LITERAL, operator, term, - MAIN_TABLE, FIELD_VALUE_TRANSFORMED, operator, term, + MAIN_TABLE, FIELD_VALUE_LITERAL, operator, + MAIN_TABLE, FIELD_VALUE_TRANSFORMED, operator, SPLIT_TABLE, FIELD_NAME, field, - SPLIT_TABLE, FIELD_VALUE_LITERAL, operator, term, - SPLIT_TABLE, FIELD_VALUE_TRANSFORMED, operator, term); + SPLIT_TABLE, FIELD_VALUE_LITERAL, operator, + SPLIT_TABLE, FIELD_VALUE_TRANSFORMED, operator); + + List params = Collections.nCopies(4, term); + SqlQuery node = new SqlQuery(cte, params); + nodes.add(node); + return new SqlQuery("cte" + cteCounter++); } - private String buildExactNegationFieldQuery(String field, String operator, String term) { - return """ + private SqlQuery buildExactNegationFieldQuery(String field, String operator, String term) { + String cte = """ cte%d AS ( SELECT %s.%s FROM %s AS %s @@ -319,9 +334,9 @@ private String buildExactNegationFieldQuery(String field, String operator, Strin LEFT JOIN %s AS %s ON (%s.%s = %s.%s AND %s.%s = %s.%s) WHERE ( - ((%s.%s = '%s') AND ((%s.%s %s '%s') OR (%s.%s %s '%s'))) + ((%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?))) OR - ((%s.%s = '%s') AND ((%s.%s %s '%s') OR (%s.%s %s '%s'))) + ((%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?))) ) ) ) @@ -336,20 +351,25 @@ private String buildExactNegationFieldQuery(String field, String operator, Strin INNER_TABLE, ENTRY_ID, SPLIT_TABLE, ENTRY_ID, INNER_TABLE, FIELD_NAME, SPLIT_TABLE, FIELD_NAME, INNER_TABLE, FIELD_NAME, field, - INNER_TABLE, FIELD_VALUE_LITERAL, operator, term, - INNER_TABLE, FIELD_VALUE_TRANSFORMED, operator, term, + INNER_TABLE, FIELD_VALUE_LITERAL, operator, + INNER_TABLE, FIELD_VALUE_TRANSFORMED, operator, SPLIT_TABLE, FIELD_NAME, field, - SPLIT_TABLE, FIELD_VALUE_LITERAL, operator, term, - SPLIT_TABLE, FIELD_VALUE_TRANSFORMED, operator, term); + SPLIT_TABLE, FIELD_VALUE_LITERAL, operator, + SPLIT_TABLE, FIELD_VALUE_TRANSFORMED, operator); + + List params = Collections.nCopies(4, term); + SqlQuery node = new SqlQuery(cte, params); + nodes.add(node); + return new SqlQuery("cte" + cteCounter++); } - private String buildContainsAnyFieldQuery(String operator, String prefixSuffix, String term) { - return """ + private SqlQuery buildContainsAnyFieldQuery(String operator, String prefixSuffix, String term) { + String cte = """ cte%d AS ( SELECT %s.%s FROM %s AS %s WHERE ( - (%s.%s != '%s') AND ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) + (%s.%s != '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?)) ) ) """.formatted( @@ -359,14 +379,17 @@ private String buildContainsAnyFieldQuery(String operator, String prefixSuffix, MAIN_TABLE, FIELD_NAME, GROUPS_FIELD, // https://github.com/JabRef/jabref/issues/7996 MAIN_TABLE, FIELD_VALUE_LITERAL, operator, - prefixSuffix, term, prefixSuffix, MAIN_TABLE, FIELD_VALUE_TRANSFORMED, - operator, - prefixSuffix, term, prefixSuffix); + operator); + + List params = Collections.nCopies(2, prefixSuffix + term + prefixSuffix); + SqlQuery node = new SqlQuery(cte, params); + nodes.add(node); + return new SqlQuery("cte" + cteCounter++); } - private String buildExactAnyFieldQuery(String operator, String term) { - return """ + private SqlQuery buildExactAnyFieldQuery(String operator, String term) { + String cte = """ cte%d AS ( SELECT %s.%s FROM %s AS %s @@ -375,9 +398,9 @@ private String buildExactAnyFieldQuery(String operator, String term) { WHERE ( (%s.%s != '%s') AND ( - ((%s.%s %s '%s') OR (%s.%s %s '%s')) + ((%s.%s %s ?) OR (%s.%s %s ?)) OR - ((%s.%s %s '%s') OR (%s.%s %s '%s')) + ((%s.%s %s ?) OR (%s.%s %s ?)) ) ) ) @@ -389,14 +412,19 @@ private String buildExactAnyFieldQuery(String operator, String term) { MAIN_TABLE, ENTRY_ID, SPLIT_TABLE, ENTRY_ID, MAIN_TABLE, FIELD_NAME, SPLIT_TABLE, FIELD_NAME, MAIN_TABLE, FIELD_NAME, GROUPS_FIELD, // https://github.com/JabRef/jabref/issues/7996 - MAIN_TABLE, FIELD_VALUE_LITERAL, operator, term, - MAIN_TABLE, FIELD_VALUE_TRANSFORMED, operator, term, - SPLIT_TABLE, FIELD_VALUE_LITERAL, operator, term, - SPLIT_TABLE, FIELD_VALUE_TRANSFORMED, operator, term); + MAIN_TABLE, FIELD_VALUE_LITERAL, operator, + MAIN_TABLE, FIELD_VALUE_TRANSFORMED, operator, + SPLIT_TABLE, FIELD_VALUE_LITERAL, operator, + SPLIT_TABLE, FIELD_VALUE_TRANSFORMED, operator); + + List params = Collections.nCopies(4, term); + SqlQuery node = new SqlQuery(cte, params); + nodes.add(node); + return new SqlQuery("cte" + cteCounter++); } - private String buildExactNegationAnyFieldQuery(String operator, String term) { - return """ + private SqlQuery buildExactNegationAnyFieldQuery(String operator, String term) { + String cte = """ cte%d AS ( SELECT %s.%s FROM %s AS %s @@ -408,9 +436,9 @@ private String buildExactNegationAnyFieldQuery(String operator, String term) { WHERE ( (%s.%s != '%s') AND ( - ((%s.%s %s '%s') OR (%s.%s %s '%s')) + ((%s.%s %s ?) OR (%s.%s %s ?)) OR - ((%s.%s %s '%s') OR (%s.%s %s '%s')) + ((%s.%s %s ?) OR (%s.%s %s ?)) ) ) ) @@ -426,14 +454,19 @@ private String buildExactNegationAnyFieldQuery(String operator, String term) { INNER_TABLE, FIELD_NAME, SPLIT_TABLE, FIELD_NAME, INNER_TABLE, ENTRY_ID, SPLIT_TABLE, ENTRY_ID, INNER_TABLE, FIELD_NAME, GROUPS_FIELD, // https://github.com/JabRef/jabref/issues/7996 - INNER_TABLE, FIELD_VALUE_LITERAL, operator, term, - INNER_TABLE, FIELD_VALUE_TRANSFORMED, operator, term, - SPLIT_TABLE, FIELD_VALUE_LITERAL, operator, term, - SPLIT_TABLE, FIELD_VALUE_TRANSFORMED, operator, term); + INNER_TABLE, FIELD_VALUE_LITERAL, operator, + INNER_TABLE, FIELD_VALUE_TRANSFORMED, operator, + SPLIT_TABLE, FIELD_VALUE_LITERAL, operator, + SPLIT_TABLE, FIELD_VALUE_TRANSFORMED, operator); + + List params = Collections.nCopies(4, term); + SqlQuery node = new SqlQuery(cte, params); + nodes.add(node); + return new SqlQuery("cte" + cteCounter++); } - private String buildContainsNegationAnyFieldQuery(String operator, String prefixSuffix, String term) { - return """ + private SqlQuery buildContainsNegationAnyFieldQuery(String operator, String prefixSuffix, String term) { + String cte = """ cte%d AS ( SELECT %s.%s FROM %s AS %s @@ -441,7 +474,7 @@ private String buildContainsNegationAnyFieldQuery(String operator, String prefix SELECT %s.%s FROM %s AS %s WHERE ( - (%s.%s != '%s') AND ((%s.%s %s '%s%s%s') OR (%s.%s %s '%s%s%s')) + (%s.%s != '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?)) ) ) ) @@ -455,10 +488,13 @@ private String buildContainsNegationAnyFieldQuery(String operator, String prefix INNER_TABLE, FIELD_NAME, GROUPS_FIELD, // https://github.com/JabRef/jabref/issues/7996 INNER_TABLE, FIELD_VALUE_LITERAL, operator, - prefixSuffix, term, prefixSuffix, INNER_TABLE, FIELD_VALUE_TRANSFORMED, - operator, - prefixSuffix, term, prefixSuffix); + operator); + + List params = Collections.nCopies(2, prefixSuffix + term + prefixSuffix); + SqlQuery node = new SqlQuery(cte, params); + nodes.add(node); + return new SqlQuery("cte" + cteCounter++); } private static void setFlags(EnumSet flags, SearchTermFlag matchType, boolean caseSensitive, boolean negation) { diff --git a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java index 9bb7e50f3f0..0bca7b48594 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/BibFieldsSearcher.java @@ -10,6 +10,7 @@ import org.jabref.model.search.query.SearchQuery; import org.jabref.model.search.query.SearchResult; import org.jabref.model.search.query.SearchResults; +import org.jabref.model.search.query.SqlQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,10 +42,13 @@ public SearchResults search(SearchQuery searchQuery) { if (!searchQuery.isValid()) { return new SearchResults(); } - String sqlQuery = SearchQueryConversion.searchToSql(tableName, searchQuery); - LOGGER.debug("Searching in bib fields with query: {}", sqlQuery); + SqlQuery sqlQuery = SearchQueryConversion.searchToSql(tableName, searchQuery); SearchResults searchResults = new SearchResults(); - try (PreparedStatement preparedStatement = connection.prepareStatement(sqlQuery)) { + try (PreparedStatement preparedStatement = connection.prepareStatement(sqlQuery.cte())) { + for (int i = 0; i < sqlQuery.params().size(); i++) { + preparedStatement.setString(i + 1, sqlQuery.params().get(i)); + } + LOGGER.debug("Executing search query: {}", preparedStatement); ResultSet resultSet = preparedStatement.executeQuery(); while (resultSet.next()) { String entryId = resultSet.getString(1); diff --git a/src/main/java/org/jabref/model/search/query/SqlQuery.java b/src/main/java/org/jabref/model/search/query/SqlQuery.java new file mode 100644 index 00000000000..0991458a3f9 --- /dev/null +++ b/src/main/java/org/jabref/model/search/query/SqlQuery.java @@ -0,0 +1,27 @@ +package org.jabref.model.search.query; + +import java.util.ArrayList; +import java.util.List; + +public class SqlQuery { + private final String cte; + private final List params; + + public SqlQuery(String cte, List params) { + this.cte = cte; + this.params = new ArrayList<>(params); + } + + public SqlQuery(String cte) { + this.cte = cte; + this.params = List.of(); + } + + public String cte() { + return cte; + } + + public List params() { + return params; + } +} diff --git a/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java index e88cf36dcfb..a32eac536bd 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java @@ -1,9 +1,17 @@ package org.jabref.logic.search.query; +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.stream.Stream; import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.search.query.SqlQuery; +import io.zonky.test.db.postgres.embedded.EmbeddedPostgres; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -11,6 +19,18 @@ import static org.junit.jupiter.api.Assertions.assertEquals; class SearchQuerySQLConversionTest { + private static EmbeddedPostgres pg; + + @BeforeAll + public static void setup() throws IOException { + pg = EmbeddedPostgres.builder().start(); + } + + @AfterAll + public static void teardown() throws IOException { + pg.close(); + } + public static Stream testSearchConversion() { return Stream.of( Arguments.of( @@ -21,7 +41,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE '%smith%') OR (main_table.field_value_transformed ILIKE '%smith%')) + (main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE ('%smith%')) OR (main_table.field_value_transformed ILIKE ('%smith%'))) ) ) SELECT * FROM cte0 GROUP BY entry_id""" @@ -35,7 +55,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE '%smith%') OR (main_table.field_value_transformed ILIKE '%smith%')) + (main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE ('%smith%')) OR (main_table.field_value_transformed ILIKE ('%smith%'))) ) ) SELECT * FROM cte0 GROUP BY entry_id""" @@ -49,7 +69,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name = 'author') AND ((main_table.field_value_literal LIKE '%smith%') OR (main_table.field_value_transformed LIKE '%smith%')) + (main_table.field_name = 'author') AND ((main_table.field_value_literal LIKE ('%smith%')) OR (main_table.field_value_transformed LIKE ('%smith%'))) ) ) SELECT * FROM cte0 GROUP BY entry_id""" @@ -66,7 +86,7 @@ WHERE main_table.entry_id NOT IN ( SELECT inner_table.entry_id FROM bib_fields."tableName" AS inner_table WHERE ( - (inner_table.field_name = 'author') AND ((inner_table.field_value_literal ILIKE '%smith%') OR (inner_table.field_value_transformed ILIKE '%smith%')) + (inner_table.field_name = 'author') AND ((inner_table.field_value_literal ILIKE ('%smith%')) OR (inner_table.field_value_transformed ILIKE ('%smith%'))) ) ) ) @@ -84,7 +104,7 @@ WHERE main_table.entry_id NOT IN ( SELECT inner_table.entry_id FROM bib_fields."tableName" AS inner_table WHERE ( - (inner_table.field_name = 'author') AND ((inner_table.field_value_literal LIKE '%smith%') OR (inner_table.field_value_transformed LIKE '%smith%')) + (inner_table.field_name = 'author') AND ((inner_table.field_value_literal LIKE ('%smith%')) OR (inner_table.field_value_transformed LIKE ('%smith%'))) ) ) ) @@ -101,9 +121,9 @@ cte0 AS ( LEFT JOIN bib_fields."tableName_split_values" AS split_table ON (main_table.entry_id = split_table.entry_id AND main_table.field_name = split_table.field_name) WHERE ( - ((main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE 'smith') OR (main_table.field_value_transformed ILIKE 'smith'))) + ((main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE ('smith')) OR (main_table.field_value_transformed ILIKE ('smith')))) OR - ((split_table.field_name = 'author') AND ((split_table.field_value_literal ILIKE 'smith') OR (split_table.field_value_transformed ILIKE 'smith'))) + ((split_table.field_name = 'author') AND ((split_table.field_value_literal ILIKE ('smith')) OR (split_table.field_value_transformed ILIKE ('smith')))) ) ) SELECT * FROM cte0 GROUP BY entry_id""" @@ -119,9 +139,9 @@ cte0 AS ( LEFT JOIN bib_fields."tableName_split_values" AS split_table ON (main_table.entry_id = split_table.entry_id AND main_table.field_name = split_table.field_name) WHERE ( - ((main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE 'smith') OR (main_table.field_value_transformed ILIKE 'smith'))) + ((main_table.field_name = 'author') AND ((main_table.field_value_literal ILIKE ('smith')) OR (main_table.field_value_transformed ILIKE ('smith')))) OR - ((split_table.field_name = 'author') AND ((split_table.field_value_literal ILIKE 'smith') OR (split_table.field_value_transformed ILIKE 'smith'))) + ((split_table.field_name = 'author') AND ((split_table.field_value_literal ILIKE ('smith')) OR (split_table.field_value_transformed ILIKE ('smith')))) ) ) SELECT * FROM cte0 GROUP BY entry_id""" @@ -137,9 +157,9 @@ cte0 AS ( LEFT JOIN bib_fields."tableName_split_values" AS split_table ON (main_table.entry_id = split_table.entry_id AND main_table.field_name = split_table.field_name) WHERE ( - ((main_table.field_name = 'author') AND ((main_table.field_value_literal LIKE 'smith') OR (main_table.field_value_transformed LIKE 'smith'))) + ((main_table.field_name = 'author') AND ((main_table.field_value_literal LIKE ('smith')) OR (main_table.field_value_transformed LIKE ('smith')))) OR - ((split_table.field_name = 'author') AND ((split_table.field_value_literal LIKE 'smith') OR (split_table.field_value_transformed LIKE 'smith'))) + ((split_table.field_name = 'author') AND ((split_table.field_value_literal LIKE ('smith')) OR (split_table.field_value_transformed LIKE ('smith')))) ) ) SELECT * FROM cte0 GROUP BY entry_id""" @@ -158,9 +178,9 @@ WHERE main_table.entry_id NOT IN ( LEFT JOIN bib_fields."tableName_split_values" AS split_table ON (inner_table.entry_id = split_table.entry_id AND inner_table.field_name = split_table.field_name) WHERE ( - ((inner_table.field_name = 'author') AND ((inner_table.field_value_literal ILIKE 'smith') OR (inner_table.field_value_transformed ILIKE 'smith'))) + ((inner_table.field_name = 'author') AND ((inner_table.field_value_literal ILIKE ('smith')) OR (inner_table.field_value_transformed ILIKE ('smith')))) OR - ((split_table.field_name = 'author') AND ((split_table.field_value_literal ILIKE 'smith') OR (split_table.field_value_transformed ILIKE 'smith'))) + ((split_table.field_name = 'author') AND ((split_table.field_value_literal ILIKE ('smith')) OR (split_table.field_value_transformed ILIKE ('smith')))) ) ) ) @@ -180,9 +200,9 @@ WHERE main_table.entry_id NOT IN ( LEFT JOIN bib_fields."tableName_split_values" AS split_table ON (inner_table.entry_id = split_table.entry_id AND inner_table.field_name = split_table.field_name) WHERE ( - ((inner_table.field_name = 'author') AND ((inner_table.field_value_literal LIKE 'smith') OR (inner_table.field_value_transformed LIKE 'smith'))) + ((inner_table.field_name = 'author') AND ((inner_table.field_value_literal LIKE ('smith')) OR (inner_table.field_value_transformed LIKE ('smith')))) OR - ((split_table.field_name = 'author') AND ((split_table.field_value_literal LIKE 'smith') OR (split_table.field_value_transformed LIKE 'smith'))) + ((split_table.field_name = 'author') AND ((split_table.field_value_literal LIKE ('smith')) OR (split_table.field_value_transformed LIKE ('smith')))) ) ) ) @@ -197,7 +217,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name = 'author') AND ((main_table.field_value_literal ~* 'smith') OR (main_table.field_value_transformed ~* 'smith')) + (main_table.field_name = 'author') AND ((main_table.field_value_literal ~* ('smith')) OR (main_table.field_value_transformed ~* ('smith'))) ) ) SELECT * FROM cte0 GROUP BY entry_id""" @@ -211,7 +231,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name = 'author') AND ((main_table.field_value_literal ~ 'smith') OR (main_table.field_value_transformed ~ 'smith')) + (main_table.field_name = 'author') AND ((main_table.field_value_literal ~ ('smith')) OR (main_table.field_value_transformed ~ ('smith'))) ) ) SELECT * FROM cte0 GROUP BY entry_id""" @@ -228,7 +248,7 @@ WHERE main_table.entry_id NOT IN ( SELECT inner_table.entry_id FROM bib_fields."tableName" AS inner_table WHERE ( - (inner_table.field_name = 'author') AND ((inner_table.field_value_literal ~* 'smith') OR (inner_table.field_value_transformed ~* 'smith')) + (inner_table.field_name = 'author') AND ((inner_table.field_value_literal ~* ('smith')) OR (inner_table.field_value_transformed ~* ('smith'))) ) ) ) @@ -246,7 +266,7 @@ WHERE main_table.entry_id NOT IN ( SELECT inner_table.entry_id FROM bib_fields."tableName" AS inner_table WHERE ( - (inner_table.field_name = 'author') AND ((inner_table.field_value_literal ~ 'smith') OR (inner_table.field_value_transformed ~ 'smith')) + (inner_table.field_name = 'author') AND ((inner_table.field_value_literal ~ ('smith')) OR (inner_table.field_value_transformed ~ ('smith'))) ) ) ) @@ -261,7 +281,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%smith%') OR (main_table.field_value_transformed ILIKE '%smith%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%smith%')) OR (main_table.field_value_transformed ILIKE ('%smith%'))) ) ) SELECT * FROM cte0 GROUP BY entry_id""" @@ -279,9 +299,9 @@ cte0 AS ( WHERE ( (main_table.field_name != 'groups') AND ( - ((main_table.field_value_literal ILIKE 'smith') OR (main_table.field_value_transformed ILIKE 'smith')) + ((main_table.field_value_literal ILIKE ('smith')) OR (main_table.field_value_transformed ILIKE ('smith'))) OR - ((split_table.field_value_literal ILIKE 'smith') OR (split_table.field_value_transformed ILIKE 'smith')) + ((split_table.field_value_literal ILIKE ('smith')) OR (split_table.field_value_transformed ILIKE ('smith'))) ) ) ) @@ -299,7 +319,7 @@ WHERE main_table.entry_id NOT IN ( SELECT inner_table.entry_id FROM bib_fields."tableName" AS inner_table WHERE ( - (inner_table.field_name != 'groups') AND ((inner_table.field_value_literal ILIKE '%smith%') OR (inner_table.field_value_transformed ILIKE '%smith%')) + (inner_table.field_name != 'groups') AND ((inner_table.field_value_literal ILIKE ('%smith%')) OR (inner_table.field_value_transformed ILIKE ('%smith%'))) ) ) ) @@ -314,7 +334,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name = 'title') AND ((main_table.field_value_literal ILIKE '%computer science%') OR (main_table.field_value_transformed ILIKE '%computer science%')) + (main_table.field_name = 'title') AND ((main_table.field_value_literal ILIKE ('%computer science%')) OR (main_table.field_value_transformed ILIKE ('%computer science%'))) ) ) SELECT * FROM cte0 GROUP BY entry_id""" @@ -328,7 +348,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%a%') OR (main_table.field_value_transformed ILIKE '%a%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%a%')) OR (main_table.field_value_transformed ILIKE ('%a%'))) ) ) , @@ -336,7 +356,7 @@ cte1 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%b%') OR (main_table.field_value_transformed ILIKE '%b%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%b%')) OR (main_table.field_value_transformed ILIKE ('%b%'))) ) ) , @@ -344,7 +364,7 @@ cte2 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%c%') OR (main_table.field_value_transformed ILIKE '%c%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%c%')) OR (main_table.field_value_transformed ILIKE ('%c%'))) ) ) , @@ -374,7 +394,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%a%') OR (main_table.field_value_transformed ILIKE '%a%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%a%')) OR (main_table.field_value_transformed ILIKE ('%a%'))) ) ) , @@ -382,7 +402,7 @@ cte1 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%b%') OR (main_table.field_value_transformed ILIKE '%b%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%b%')) OR (main_table.field_value_transformed ILIKE ('%b%'))) ) ) , @@ -398,7 +418,7 @@ cte3 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%c%') OR (main_table.field_value_transformed ILIKE '%c%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%c%')) OR (main_table.field_value_transformed ILIKE ('%c%'))) ) ) , @@ -420,7 +440,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%a%') OR (main_table.field_value_transformed ILIKE '%a%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%a%')) OR (main_table.field_value_transformed ILIKE ('%a%'))) ) ) , @@ -428,7 +448,7 @@ cte1 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%b%') OR (main_table.field_value_transformed ILIKE '%b%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%b%')) OR (main_table.field_value_transformed ILIKE ('%b%'))) ) ) , @@ -444,7 +464,7 @@ cte3 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%c%') OR (main_table.field_value_transformed ILIKE '%c%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%c%')) OR (main_table.field_value_transformed ILIKE ('%c%'))) ) ) , @@ -466,7 +486,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%a%') OR (main_table.field_value_transformed ILIKE '%a%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%a%')) OR (main_table.field_value_transformed ILIKE ('%a%'))) ) ) , @@ -474,7 +494,7 @@ cte1 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%b%') OR (main_table.field_value_transformed ILIKE '%b%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%b%')) OR (main_table.field_value_transformed ILIKE ('%b%'))) ) ) , @@ -482,7 +502,7 @@ cte2 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%c%') OR (main_table.field_value_transformed ILIKE '%c%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%c%')) OR (main_table.field_value_transformed ILIKE ('%c%'))) ) ) , @@ -512,7 +532,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%a%') OR (main_table.field_value_transformed ILIKE '%a%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%a%')) OR (main_table.field_value_transformed ILIKE ('%a%'))) ) ) , @@ -520,7 +540,7 @@ cte1 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%b%') OR (main_table.field_value_transformed ILIKE '%b%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%b%')) OR (main_table.field_value_transformed ILIKE ('%b%'))) ) ) , @@ -536,7 +556,7 @@ cte3 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%c%') OR (main_table.field_value_transformed ILIKE '%c%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%c%')) OR (main_table.field_value_transformed ILIKE ('%c%'))) ) ) , @@ -544,7 +564,7 @@ cte4 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%d%') OR (main_table.field_value_transformed ILIKE '%d%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%d%')) OR (main_table.field_value_transformed ILIKE ('%d%'))) ) ) , @@ -574,7 +594,7 @@ cte0 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%a%') OR (main_table.field_value_transformed ILIKE '%a%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%a%')) OR (main_table.field_value_transformed ILIKE ('%a%'))) ) ) , @@ -582,7 +602,7 @@ cte1 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%b%') OR (main_table.field_value_transformed ILIKE '%b%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%b%')) OR (main_table.field_value_transformed ILIKE ('%b%'))) ) ) , @@ -590,7 +610,7 @@ cte2 AS ( SELECT main_table.entry_id FROM bib_fields."tableName" AS main_table WHERE ( - (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE '%c%') OR (main_table.field_value_transformed ILIKE '%c%')) + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%c%')) OR (main_table.field_value_transformed ILIKE ('%c%'))) ) ) , @@ -619,6 +639,20 @@ cte5 AS ( FROM cte4 ) SELECT * FROM cte5 GROUP BY entry_id""" + ), + + Arguments.of( + "a'b", + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%a''b%')) OR (main_table.field_value_transformed ILIKE ('%a''b%'))) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" ) // Throws exceptions // computer science, !computer, R\"ock, Breitenb{\"{u}}cher @@ -627,7 +661,16 @@ cte5 AS ( @ParameterizedTest @MethodSource - void testSearchConversion(String searchExpression, String expected) { - assertEquals(expected, SearchQueryConversion.searchToSql("tableName", new SearchQuery(searchExpression))); + void testSearchConversion(String searchExpression, String expected) throws SQLException { + try (Connection connection = pg.getPostgresDatabase().getConnection()) { + SqlQuery sqlQuery = SearchQueryConversion.searchToSql("tableName", new SearchQuery(searchExpression)); + try (PreparedStatement preparedStatement = connection.prepareStatement(sqlQuery.cte())) { + for (int i = 0; i < sqlQuery.params().size(); i++) { + preparedStatement.setString(i + 1, sqlQuery.params().get(i)); + } + String sql = preparedStatement.toString(); + assertEquals(expected, sql); + } + } } } From 74f98ef896ba865fac24badaab7863548e5d9508 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 11 Oct 2024 18:02:34 +0300 Subject: [PATCH 079/104] Store the start and end positions for every field --- .../org/jabref/gui/entryeditor/SourceTab.java | 10 ++++-- .../jabref/logic/bibtex/BibEntryWriter.java | 31 ++++++++++++++++--- .../org/jabref/logic/exporter/BibWriter.java | 6 ++++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java index f677e272607..fa6fd34d850 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java @@ -142,8 +142,14 @@ private String getSourceString(BibEntry entry, BibDatabaseMode type, FieldPrefer StringWriter writer = new StringWriter(); BibWriter bibWriter = new BibWriter(writer, OS.NEWLINE); FieldWriter fieldWriter = FieldWriter.buildIgnoreHashes(fieldPreferences); - new BibEntryWriter(fieldWriter, entryTypesManager).write(entry, bibWriter, type); - return writer.toString(); + BibEntryWriter bibEntryWriter = new BibEntryWriter(fieldWriter, entryTypesManager); + bibEntryWriter.write(entry, bibWriter, type, true); + String sourceString = writer.toString(); + bibEntryWriter.getFieldPositions().forEach((field, position) -> { + System.out.printf("Field: %s, value: %s\n", field.getName(), sourceString.substring(position.start(), position.end())); + }); + writer.close(); + return sourceString; } /* Work around for different input methods. diff --git a/src/main/java/org/jabref/logic/bibtex/BibEntryWriter.java b/src/main/java/org/jabref/logic/bibtex/BibEntryWriter.java index 2d0f63e51dc..7f742ff0f7c 100644 --- a/src/main/java/org/jabref/logic/bibtex/BibEntryWriter.java +++ b/src/main/java/org/jabref/logic/bibtex/BibEntryWriter.java @@ -4,9 +4,11 @@ import java.io.StringWriter; import java.util.Collection; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.SortedSet; @@ -33,6 +35,7 @@ public class BibEntryWriter { private static final Logger LOGGER = LoggerFactory.getLogger(BibEntryWriter.class); + private final Map fieldPositions = new HashMap<>(); private final BibEntryTypesManager entryTypesManager; private final FieldWriter fieldWriter; @@ -90,10 +93,7 @@ private void writeUserComments(BibEntry entry, BibWriter out) throws IOException */ private void writeRequiredFieldsFirstRemainingFieldsSecond(BibEntry entry, BibWriter out, BibDatabaseMode bibDatabaseMode) throws IOException { - // Write header with type and bibtex-key - TypedBibEntry typedEntry = new TypedBibEntry(entry, bibDatabaseMode); - out.write('@' + typedEntry.getTypeForDisplay() + '{'); - + writeEntryType(entry, out, bibDatabaseMode); writeKeyField(entry, out); Set written = new HashSet<>(); @@ -141,9 +141,20 @@ private void writeRequiredFieldsFirstRemainingFieldsSecond(BibEntry entry, BibWr out.writeLine("}"); } + private void writeEntryType(BibEntry entry, BibWriter out, BibDatabaseMode bibDatabaseMode) throws IOException { + int start = out.getCurrentPosition(); + TypedBibEntry typedEntry = new TypedBibEntry(entry, bibDatabaseMode); + out.write('@' + typedEntry.getTypeForDisplay() + '{'); + int end = out.getCurrentPosition() - 1; // exclude the '{' + fieldPositions.put(InternalField.TYPE_HEADER, new FieldPosition(start, end)); + } + private void writeKeyField(BibEntry entry, BibWriter out) throws IOException { + int start = out.getCurrentPosition(); String keyField = StringUtil.shaveString(entry.getCitationKey().orElse("")); out.writeLine(keyField + ','); + int end = out.getCurrentPosition() - 1; // exclude the ',' + fieldPositions.put(InternalField.KEY_FIELD, new FieldPosition(start, end)); } /** @@ -162,9 +173,12 @@ private void writeField(BibEntry entry, BibWriter out, Field field, int indent) out.write(" "); out.write(getFormattedFieldName(field, indent)); try { + int start = out.getCurrentPosition(); out.write(fieldWriter.write(field, value.get())); + int end = out.getCurrentPosition(); + fieldPositions.put(field, new FieldPosition(start, end)); } catch (InvalidFieldValueException ex) { - LOGGER.warn("Invalid field value {} of field {} of entry {]", value.get(), field, entry.getCitationKey().orElse(""), ex); + LOGGER.warn("Invalid field value {} of field {} of entry {}", value.get(), field, entry.getCitationKey().orElse(""), ex); throw new IOException("Error in field '" + field + " of entry " + entry.getCitationKey().orElse("") + "': " + ex.getMessage(), ex); } out.writeLine(","); @@ -197,4 +211,11 @@ static String getFormattedFieldName(Field field, int indent) { String fieldName = field.getName(); return fieldName.toLowerCase(Locale.ROOT) + StringUtil.repeatSpaces(indent - fieldName.length()) + " = "; } + + public Map getFieldPositions() { + return fieldPositions; + } + + public record FieldPosition(int start, int end) { + } } diff --git a/src/main/java/org/jabref/logic/exporter/BibWriter.java b/src/main/java/org/jabref/logic/exporter/BibWriter.java index f6931653542..86422e9adf4 100644 --- a/src/main/java/org/jabref/logic/exporter/BibWriter.java +++ b/src/main/java/org/jabref/logic/exporter/BibWriter.java @@ -16,6 +16,7 @@ public class BibWriter { private boolean precedingNewLineRequired = false; private boolean somethingWasWritten = false; private boolean lastWriteWasNewline = false; + private int currentPosition = 0; /** * @param newLineSeparator the string used for a line break @@ -35,6 +36,7 @@ public void write(String string) throws IOException { } string = StringUtil.unifyLineBreaks(string, newLineSeparator); writer.write(string); + currentPosition += string.length(); lastWriteWasNewline = string.endsWith(newLineSeparator); somethingWasWritten = true; } @@ -69,4 +71,8 @@ public void finishBlock() throws IOException { this.somethingWasWritten = false; this.precedingNewLineRequired = true; } + + public int getCurrentPosition() { + return currentPosition; + } } From a7ca4d3b7ac1a2793c7a72f89e9428d558dcf3af Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 11 Oct 2024 19:08:07 +0300 Subject: [PATCH 080/104] WIP highlight source tab --- .../org/jabref/gui/entryeditor/SourceTab.java | 85 ++++++++++--------- .../org/jabref/logic/bibtex/FieldWriter.java | 15 ++-- .../jabref/logic/search/PostgreServer.java | 16 ++++ .../logic/search/retrieval/Highlighter.java | 83 +++++++++++++----- .../jabref/model/search/PostgreConstants.java | 45 ++++++++++ 5 files changed, 170 insertions(+), 74 deletions(-) diff --git a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java index fa6fd34d850..8aeafe0111d 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java @@ -3,11 +3,9 @@ import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; +import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import javax.swing.undo.UndoManager; @@ -19,6 +17,7 @@ import javafx.scene.control.Tooltip; import javafx.scene.input.InputMethodRequests; import javafx.scene.input.KeyEvent; +import javafx.util.Pair; import org.jabref.gui.DialogService; import org.jabref.gui.actions.ActionFactory; @@ -43,6 +42,7 @@ import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.l10n.Localization; import org.jabref.logic.os.OS; +import org.jabref.logic.search.retrieval.Highlighter; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; @@ -73,30 +73,11 @@ public class SourceTab extends EntryEditorTab { private final DialogService dialogService; private final BibEntryTypesManager entryTypesManager; private final KeyBindingRepository keyBindingRepository; - private Optional searchHighlightPattern = Optional.empty(); + private final OptionalObjectProperty searchQueryProperty; + private Map fieldPositions; private CodeArea codeArea; private BibEntry previousEntry; - private class EditAction extends SimpleCommand { - - private final StandardActions command; - - public EditAction(StandardActions command) { - this.command = command; - } - - @Override - public void execute() { - switch (command) { - case COPY -> codeArea.copy(); - case CUT -> codeArea.cut(); - case PASTE -> codeArea.paste(); - case SELECT_ALL -> codeArea.selectAll(); - } - codeArea.requestFocus(); - } - } - public SourceTab(BibDatabaseContext bibDatabaseContext, CountingUndoManager undoManager, FieldPreferences fieldPreferences, @@ -117,21 +98,20 @@ public SourceTab(BibDatabaseContext bibDatabaseContext, this.dialogService = dialogService; this.entryTypesManager = entryTypesManager; this.keyBindingRepository = keyBindingRepository; - - searchQueryProperty.addListener((observable, oldValue, newValue) -> { - // TODO: get search pattern from the search query - highlightSearchPattern(); - }); + this.searchQueryProperty = searchQueryProperty; + searchQueryProperty.addListener((observable, oldValue, newValue) -> highlightSearchPattern()); } private void highlightSearchPattern() { if (codeArea != null) { codeArea.setStyleClass(0, codeArea.getLength(), "text"); - if (searchHighlightPattern.isPresent()) { - Matcher matcher = searchHighlightPattern.get().matcher(codeArea.getText()); - while (matcher.find()) { - for (int i = 0; i <= matcher.groupCount(); i++) { - codeArea.setStyleClass(matcher.start(), matcher.end(), "search"); + if (searchQueryProperty.get().isPresent()) { + for (BibEntryWriter.FieldPosition fieldPosition : fieldPositions.values()) { + int start = fieldPosition.start(); + int end = fieldPosition.end(); + List> matchedPositions = Highlighter.getMatchPositions(codeArea.getText(start, end), searchQueryProperty.get().get()); + for (Pair pair : matchedPositions) { + codeArea.setStyleClass(start + pair.getKey(), start + pair.getValue(), "search"); } } } @@ -144,10 +124,8 @@ private String getSourceString(BibEntry entry, BibDatabaseMode type, FieldPrefer FieldWriter fieldWriter = FieldWriter.buildIgnoreHashes(fieldPreferences); BibEntryWriter bibEntryWriter = new BibEntryWriter(fieldWriter, entryTypesManager); bibEntryWriter.write(entry, bibWriter, type, true); + fieldPositions = bibEntryWriter.getFieldPositions(); String sourceString = writer.toString(); - bibEntryWriter.getFieldPositions().forEach((field, position) -> { - System.out.printf("Field: %s, value: %s\n", field.getName(), sourceString.substring(position.start(), position.end())); - }); writer.close(); return sourceString; } @@ -239,7 +217,12 @@ private void updateCodeArea() { codeArea.clear(); try { - codeArea.appendText(getSourceString(currentEntry, mode, fieldPreferences)); + String sourceText = getSourceString(currentEntry, mode, fieldPreferences); + LOGGER.info("sourceText length: {}", sourceText.length()); + LOGGER.info("sourceText: {}", sourceText); + codeArea.appendText(sourceText); + LOGGER.info("codeArea length: {}", codeArea.getLength()); + LOGGER.info("codeArea: {}", codeArea.getText()); codeArea.setEditable(true); highlightSearchPattern(); } catch (IOException ex) { @@ -344,12 +327,30 @@ private void storeSource(BibEntry outOfFocusEntry, String text) { private void listenForSaveKeybinding(KeyEvent event) { keyBindingRepository.mapToKeyBinding(event).ifPresent(binding -> { - switch (binding) { - case SAVE_DATABASE, SAVE_ALL, SAVE_DATABASE_AS -> { - storeSource(currentEntry, codeArea.textProperty().getValue()); - } + case SAVE_DATABASE, SAVE_ALL, SAVE_DATABASE_AS -> + storeSource(currentEntry, codeArea.textProperty().getValue()); } }); } + + private class EditAction extends SimpleCommand { + + private final StandardActions command; + + public EditAction(StandardActions command) { + this.command = command; + } + + @Override + public void execute() { + switch (command) { + case COPY -> codeArea.copy(); + case CUT -> codeArea.cut(); + case PASTE -> codeArea.paste(); + case SELECT_ALL -> codeArea.selectAll(); + } + codeArea.requestFocus(); + } + } } diff --git a/src/main/java/org/jabref/logic/bibtex/FieldWriter.java b/src/main/java/org/jabref/logic/bibtex/FieldWriter.java index 1a6702242eb..b069b307fe5 100644 --- a/src/main/java/org/jabref/logic/bibtex/FieldWriter.java +++ b/src/main/java/org/jabref/logic/bibtex/FieldWriter.java @@ -85,10 +85,10 @@ public String write(Field field, String content) throws InvalidFieldValueExcepti } if (!shouldResolveStrings(field) || field.equals(InternalField.BIBTEX_STRING)) { - return formatWithoutResolvingStrings(content, field); + return formatWithoutResolvingStrings(content); } - return formatAndResolveStrings(content, field); + return formatAndResolveStrings(content); } /** @@ -96,7 +96,7 @@ public String write(Field field, String content) throws InvalidFieldValueExcepti *

* For instance, #jan# - #feb# gets jan #{ - } # feb (see @link{org.jabref.logic.bibtex.LatexFieldFormatterTests#makeHashEnclosedWordsRealStringsInMonthField()}) */ - private String formatAndResolveStrings(String content, Field field) throws InvalidFieldValueException { + private String formatAndResolveStrings(String content) throws InvalidFieldValueException { checkBraces(content); content = content.replace("##", ""); @@ -123,7 +123,7 @@ private String formatAndResolveStrings(String content, Field field) throws Inval if (neverFailOnHashes) { pos1 = content.length(); // just write out the rest of the text, and throw no exception } else { - LOGGER.error("The character {} is not allowed in BibTeX strings unless escaped as in '\\{}'. " + LOGGER.error("The character {} is not allowed in BibTeX strings unless escaped as in '\\\\{}'. " + "In JabRef, use pairs of # characters to indicate a string. " + "Note that the entry causing the problem has been selected. Field value: {}", BIBTEX_STRING_START_END_SYMBOL, @@ -183,12 +183,9 @@ private boolean shouldResolveStrings(Field field) { return false; } - private String formatWithoutResolvingStrings(String content, Field field) throws InvalidFieldValueException { + private String formatWithoutResolvingStrings(String content) throws InvalidFieldValueException { checkBraces(content); - StringBuilder stringBuilder = new StringBuilder(String.valueOf(FIELD_START)); - stringBuilder.append(content); - stringBuilder.append(FIELD_END); - return stringBuilder.toString(); + return FIELD_START + content + FIELD_END; } /** diff --git a/src/main/java/org/jabref/logic/search/PostgreServer.java b/src/main/java/org/jabref/logic/search/PostgreServer.java index 4392cf173f2..13a113c441a 100644 --- a/src/main/java/org/jabref/logic/search/PostgreServer.java +++ b/src/main/java/org/jabref/logic/search/PostgreServer.java @@ -6,6 +6,8 @@ import javax.sql.DataSource; +import org.jabref.model.search.PostgreConstants; + import io.zonky.test.db.postgres.embedded.EmbeddedPostgres; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +36,7 @@ public PostgreServer() { this.dataSource = embeddedPostgres.getPostgresDatabase(); addTrigramExtension(); createScheme(); + addFunctions(); } private void createScheme() { @@ -59,6 +62,19 @@ private void addTrigramExtension() { } } + private void addFunctions() { + try (Connection connection = getConnection()) { + if (connection != null) { + LOGGER.debug("Adding functions to Postgres server"); + for (String function : PostgreConstants.POSTGRES_FUNCTIONS) { + connection.createStatement().execute(function); + } + } + } catch (SQLException e) { + LOGGER.error("Could not add functions to Postgres server", e); + } + } + public Connection getConnection() { if (dataSource != null) { try { diff --git a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java index 49dba48ec28..521175b378c 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java +++ b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java @@ -4,10 +4,17 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; + +import javafx.util.Pair; import org.jabref.logic.search.PostgreServer; import org.jabref.logic.search.query.SearchQueryConversion; +import org.jabref.model.search.PostgreConstants; import org.jabref.model.search.query.SearchQuery; import com.airhacks.afterburner.injection.Injector; @@ -22,41 +29,57 @@ public class Highlighter { private static final Logger LOGGER = LoggerFactory.getLogger(Highlighter.class); - private final static String HIGHLIGHT_QUERY = """ - SELECT - regexp_replace( - ?, - ?, - '\\1', - 'gi' - ) - """; + + /** + * Functions defined in {@link PostgreConstants#POSTGRES_FUNCTIONS} + */ + private static final String REGEXP_MARK = "SELECT regexp_mark(?, ?)"; + private static final String REGEXP_POSITIONS = "SELECT * FROM regexp_positions(?, ?)"; private static Connection connection; public static String highlightHtml(String htmlText, SearchQuery searchQuery) { - if (!searchQuery.isValid()) { - return htmlText; - } - - LOGGER.debug("Highlighting search terms in text: {}", searchQuery); - Set terms = SearchQueryConversion.extractSearchTerms(searchQuery); - if (terms.isEmpty()) { + Optional searchTerms = getSearchTerms(searchQuery); + if (searchTerms.isEmpty()) { return htmlText; } - String joinedTerms = String.join("|", terms); Document document = Jsoup.parse(htmlText); try { - highlightTextNodes(document.body(), joinedTerms); - String highlightedHtml = document.outerHtml(); - LOGGER.debug("Highlighted HTML: {}", highlightedHtml); - return highlightedHtml; + highlightTextNodes(document.body(), searchTerms.get()); + return document.outerHtml(); } catch (InvalidTokenOffsetsException e) { LOGGER.debug("Error highlighting search terms in HTML", e); return htmlText; } } + public static List> getMatchPositions(String text, SearchQuery searchQuery) { + Optional searchTerms = getSearchTerms(searchQuery); + if (searchTerms.isEmpty()) { + return List.of(); + } + + if (connection == null) { + connection = Injector.instantiateModelOrService(PostgreServer.class).getConnection(); + } + + try (PreparedStatement preparedStatement = connection.prepareStatement(REGEXP_POSITIONS)) { + preparedStatement.setString(1, text); + preparedStatement.setString(2, searchTerms.get()); + + try (ResultSet resultSet = preparedStatement.executeQuery()) { + List> positions = new ArrayList<>(); + while (resultSet.next()) { + positions.add(new Pair<>(resultSet.getInt(1), resultSet.getInt(2))); + } + return positions; + } + } catch (SQLException e) { + LOGGER.error("Error getting match positions in text", e); + } + return List.of(); + } + private static void highlightTextNodes(Element element, String searchTerms) throws InvalidTokenOffsetsException { for (Node node : element.childNodes()) { if (node instanceof TextNode textNode) { @@ -74,9 +97,9 @@ private static String highlightNode(String text, String searchTerms) { connection = Injector.instantiateModelOrService(PostgreServer.class).getConnection(); } - try (PreparedStatement preparedStatement = connection.prepareStatement(HIGHLIGHT_QUERY)) { + try (PreparedStatement preparedStatement = connection.prepareStatement(REGEXP_MARK)) { preparedStatement.setString(1, text); - preparedStatement.setString(2, '(' + searchTerms + ')'); + preparedStatement.setString(2, searchTerms); try (ResultSet resultSet = preparedStatement.executeQuery()) { if (resultSet.next()) { @@ -88,4 +111,18 @@ private static String highlightNode(String text, String searchTerms) { } return text; } + + private static Optional getSearchTerms(SearchQuery searchQuery) { + if (!searchQuery.isValid()) { + return Optional.empty(); + } + + Set terms = SearchQueryConversion.extractSearchTerms(searchQuery); + if (terms.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(terms.stream() + .collect(Collectors.joining(")|(", "(", ")"))); + } } diff --git a/src/main/java/org/jabref/model/search/PostgreConstants.java b/src/main/java/org/jabref/model/search/PostgreConstants.java index b5653219b42..5cfc002d5d8 100644 --- a/src/main/java/org/jabref/model/search/PostgreConstants.java +++ b/src/main/java/org/jabref/model/search/PostgreConstants.java @@ -1,5 +1,7 @@ package org.jabref.model.search; +import java.util.List; + public enum PostgreConstants { BIB_FIELDS_SCHEME("bib_fields"), SPLIT_TABLE_SUFFIX("_split_values"), @@ -8,6 +10,49 @@ public enum PostgreConstants { FIELD_VALUE_LITERAL("field_value_literal"), // contains the value as-is FIELD_VALUE_TRANSFORMED("field_value_transformed"); // contains the value transformed for better querying + public static final List POSTGRES_FUNCTIONS = List.of( + // HTML highlighting function + """ + CREATE OR REPLACE FUNCTION regexp_mark(string text, pattern text) + RETURNS text + LANGUAGE plpgsql + AS + $$ + BEGIN + RETURN regexp_replace(string, pattern, '\\1', 'gi'); + END + $$; + """, + + // Extract positions of a pattern in a string + // Source: https://www.postgresql.org/message-id/flat/0aabac3c-9049-4c55-a82d-a70c5ba43d4d%40www.fastmail.com + """ + CREATE OR REPLACE FUNCTION regexp_positions(string text, pattern text, OUT start_pos integer, OUT end_pos integer) + RETURNS SETOF RECORD + LANGUAGE plpgsql + AS + $$ + DECLARE + match text; + remainder text := string; + len integer; + pos integer; + BEGIN + end_pos := 0; + FOR match IN SELECT (regexp_matches(string, pattern, 'gi'))[1] LOOP + len := length(match); + pos := position(match in remainder); + start_pos := pos + end_pos; + end_pos := start_pos + len - 1; + RETURN NEXT; + remainder := right(remainder, 1 - len - pos); + END LOOP; + RETURN; + END + $$; + """ + ); + private final String value; PostgreConstants(String value) { From 5edd1a50a6dd33d8286604d95d98ac8cbbf3ada0 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sat, 12 Oct 2024 01:54:48 +0300 Subject: [PATCH 081/104] Fix source tab highlighting --- .../org/jabref/gui/entryeditor/SourceTab.java | 31 +++++++++---------- .../jabref/logic/bibtex/BibEntryWriter.java | 14 ++++----- .../logic/search/retrieval/Highlighter.java | 20 +++++------- .../java/org/jabref/model/util/Range.java | 4 +++ 4 files changed, 32 insertions(+), 37 deletions(-) create mode 100644 src/main/java/org/jabref/model/util/Range.java diff --git a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java index 8aeafe0111d..5cdc78cf193 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import javax.swing.undo.UndoManager; @@ -17,7 +18,6 @@ import javafx.scene.control.Tooltip; import javafx.scene.input.InputMethodRequests; import javafx.scene.input.KeyEvent; -import javafx.util.Pair; import org.jabref.gui.DialogService; import org.jabref.gui.actions.ActionFactory; @@ -41,7 +41,6 @@ import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.os.OS; import org.jabref.logic.search.retrieval.Highlighter; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; @@ -51,6 +50,7 @@ import org.jabref.model.entry.field.Field; import org.jabref.model.search.query.SearchQuery; import org.jabref.model.util.FileUpdateMonitor; +import org.jabref.model.util.Range; import de.saxsys.mvvmfx.utils.validation.ObservableRuleBasedValidator; import de.saxsys.mvvmfx.utils.validation.ValidationMessage; @@ -74,7 +74,7 @@ public class SourceTab extends EntryEditorTab { private final BibEntryTypesManager entryTypesManager; private final KeyBindingRepository keyBindingRepository; private final OptionalObjectProperty searchQueryProperty; - private Map fieldPositions; + private Map fieldPositions; private CodeArea codeArea; private BibEntry previousEntry; @@ -106,12 +106,16 @@ private void highlightSearchPattern() { if (codeArea != null) { codeArea.setStyleClass(0, codeArea.getLength(), "text"); if (searchQueryProperty.get().isPresent()) { - for (BibEntryWriter.FieldPosition fieldPosition : fieldPositions.values()) { - int start = fieldPosition.start(); - int end = fieldPosition.end(); - List> matchedPositions = Highlighter.getMatchPositions(codeArea.getText(start, end), searchQueryProperty.get().get()); - for (Pair pair : matchedPositions) { - codeArea.setStyleClass(start + pair.getKey(), start + pair.getValue(), "search"); + Optional searchPattern = Highlighter.getSearchTermsPattern(searchQueryProperty.get().get()); + if (searchPattern.isPresent()) { + LOGGER.debug("Highlighting search pattern {}", searchPattern.get()); + for (Range fieldPosition : fieldPositions.values()) { + int start = fieldPosition.start(); + int end = fieldPosition.end(); + List matchedPositions = Highlighter.getMatchPositions(codeArea.getText(start, end), searchPattern.get()); + for (Range range: matchedPositions) { + codeArea.setStyleClass(start + range.start() - 1, start + range.end(), "search"); + } } } } @@ -120,7 +124,7 @@ private void highlightSearchPattern() { private String getSourceString(BibEntry entry, BibDatabaseMode type, FieldPreferences fieldPreferences) throws IOException { StringWriter writer = new StringWriter(); - BibWriter bibWriter = new BibWriter(writer, OS.NEWLINE); + BibWriter bibWriter = new BibWriter(writer, "\n"); FieldWriter fieldWriter = FieldWriter.buildIgnoreHashes(fieldPreferences); BibEntryWriter bibEntryWriter = new BibEntryWriter(fieldWriter, entryTypesManager); bibEntryWriter.write(entry, bibWriter, type, true); @@ -217,12 +221,7 @@ private void updateCodeArea() { codeArea.clear(); try { - String sourceText = getSourceString(currentEntry, mode, fieldPreferences); - LOGGER.info("sourceText length: {}", sourceText.length()); - LOGGER.info("sourceText: {}", sourceText); - codeArea.appendText(sourceText); - LOGGER.info("codeArea length: {}", codeArea.getLength()); - LOGGER.info("codeArea: {}", codeArea.getText()); + codeArea.appendText(getSourceString(currentEntry, mode, fieldPreferences)); codeArea.setEditable(true); highlightSearchPattern(); } catch (IOException ex) { diff --git a/src/main/java/org/jabref/logic/bibtex/BibEntryWriter.java b/src/main/java/org/jabref/logic/bibtex/BibEntryWriter.java index 7f742ff0f7c..449255935eb 100644 --- a/src/main/java/org/jabref/logic/bibtex/BibEntryWriter.java +++ b/src/main/java/org/jabref/logic/bibtex/BibEntryWriter.java @@ -27,6 +27,7 @@ import org.jabref.model.entry.field.InternalField; import org.jabref.model.entry.field.OrFields; import org.jabref.model.strings.StringUtil; +import org.jabref.model.util.Range; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +36,7 @@ public class BibEntryWriter { private static final Logger LOGGER = LoggerFactory.getLogger(BibEntryWriter.class); - private final Map fieldPositions = new HashMap<>(); + private final Map fieldPositions = new HashMap<>(); private final BibEntryTypesManager entryTypesManager; private final FieldWriter fieldWriter; @@ -146,7 +147,7 @@ private void writeEntryType(BibEntry entry, BibWriter out, BibDatabaseMode bibDa TypedBibEntry typedEntry = new TypedBibEntry(entry, bibDatabaseMode); out.write('@' + typedEntry.getTypeForDisplay() + '{'); int end = out.getCurrentPosition() - 1; // exclude the '{' - fieldPositions.put(InternalField.TYPE_HEADER, new FieldPosition(start, end)); + fieldPositions.put(InternalField.TYPE_HEADER, new Range(start, end)); } private void writeKeyField(BibEntry entry, BibWriter out) throws IOException { @@ -154,7 +155,7 @@ private void writeKeyField(BibEntry entry, BibWriter out) throws IOException { String keyField = StringUtil.shaveString(entry.getCitationKey().orElse("")); out.writeLine(keyField + ','); int end = out.getCurrentPosition() - 1; // exclude the ',' - fieldPositions.put(InternalField.KEY_FIELD, new FieldPosition(start, end)); + fieldPositions.put(InternalField.KEY_FIELD, new Range(start, end)); } /** @@ -176,7 +177,7 @@ private void writeField(BibEntry entry, BibWriter out, Field field, int indent) int start = out.getCurrentPosition(); out.write(fieldWriter.write(field, value.get())); int end = out.getCurrentPosition(); - fieldPositions.put(field, new FieldPosition(start, end)); + fieldPositions.put(field, new Range(start, end)); } catch (InvalidFieldValueException ex) { LOGGER.warn("Invalid field value {} of field {} of entry {}", value.get(), field, entry.getCitationKey().orElse(""), ex); throw new IOException("Error in field '" + field + " of entry " + entry.getCitationKey().orElse("") + "': " + ex.getMessage(), ex); @@ -212,10 +213,7 @@ static String getFormattedFieldName(Field field, int indent) { return fieldName.toLowerCase(Locale.ROOT) + StringUtil.repeatSpaces(indent - fieldName.length()) + " = "; } - public Map getFieldPositions() { + public Map getFieldPositions() { return fieldPositions; } - - public record FieldPosition(int start, int end) { - } } diff --git a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java index 521175b378c..efd04a659e1 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java +++ b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java @@ -10,12 +10,11 @@ import java.util.Set; import java.util.stream.Collectors; -import javafx.util.Pair; - import org.jabref.logic.search.PostgreServer; import org.jabref.logic.search.query.SearchQueryConversion; import org.jabref.model.search.PostgreConstants; import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.util.Range; import com.airhacks.afterburner.injection.Injector; import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; @@ -38,7 +37,7 @@ public class Highlighter { private static Connection connection; public static String highlightHtml(String htmlText, SearchQuery searchQuery) { - Optional searchTerms = getSearchTerms(searchQuery); + Optional searchTerms = getSearchTermsPattern(searchQuery); if (searchTerms.isEmpty()) { return htmlText; } @@ -53,24 +52,19 @@ public static String highlightHtml(String htmlText, SearchQuery searchQuery) { } } - public static List> getMatchPositions(String text, SearchQuery searchQuery) { - Optional searchTerms = getSearchTerms(searchQuery); - if (searchTerms.isEmpty()) { - return List.of(); - } - + public static List getMatchPositions(String text, String pattern) { if (connection == null) { connection = Injector.instantiateModelOrService(PostgreServer.class).getConnection(); } try (PreparedStatement preparedStatement = connection.prepareStatement(REGEXP_POSITIONS)) { preparedStatement.setString(1, text); - preparedStatement.setString(2, searchTerms.get()); + preparedStatement.setString(2, pattern); try (ResultSet resultSet = preparedStatement.executeQuery()) { - List> positions = new ArrayList<>(); + List positions = new ArrayList<>(); while (resultSet.next()) { - positions.add(new Pair<>(resultSet.getInt(1), resultSet.getInt(2))); + positions.add(new Range(resultSet.getInt(1), resultSet.getInt(2))); } return positions; } @@ -112,7 +106,7 @@ private static String highlightNode(String text, String searchTerms) { return text; } - private static Optional getSearchTerms(SearchQuery searchQuery) { + public static Optional getSearchTermsPattern(SearchQuery searchQuery) { if (!searchQuery.isValid()) { return Optional.empty(); } diff --git a/src/main/java/org/jabref/model/util/Range.java b/src/main/java/org/jabref/model/util/Range.java new file mode 100644 index 00000000000..36bcc4b7568 --- /dev/null +++ b/src/main/java/org/jabref/model/util/Range.java @@ -0,0 +1,4 @@ +package org.jabref.model.util; + +public record Range(int start, int end) { +} From ef27dc59af6d72aee9ecc975155b455975ec09a9 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sat, 12 Oct 2024 05:46:29 +0300 Subject: [PATCH 082/104] Return regex, case-sensitive flags back to the search bar --- .../org/jabref/gui/frame/JabRefFrame.java | 1 - .../java/org/jabref/gui/icon/IconTheme.java | 1 + .../jabref/gui/search/GlobalSearchBar.java | 47 ++++++++++++++++--- .../preferences/JabRefCliPreferences.java | 6 +++ .../logic/search/SearchPreferences.java | 24 +++++++++- src/main/resources/l10n/JabRef_en.properties | 4 +- 6 files changed, 73 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/jabref/gui/frame/JabRefFrame.java b/src/main/java/org/jabref/gui/frame/JabRefFrame.java index c40a946cffc..2b7ffb38497 100644 --- a/src/main/java/org/jabref/gui/frame/JabRefFrame.java +++ b/src/main/java/org/jabref/gui/frame/JabRefFrame.java @@ -524,7 +524,6 @@ private OpenDatabaseAction getOpenDatabaseAction() { * Refreshes the ui after preferences changes */ public void refresh() { - globalSearchBar.updateHintVisibility(); getLibraryTabs().forEach(LibraryTab::setupMainPanel); getLibraryTabs().forEach(tab -> tab.getMainTable().getTableModel().resetFieldFormatter()); } diff --git a/src/main/java/org/jabref/gui/icon/IconTheme.java b/src/main/java/org/jabref/gui/icon/IconTheme.java index 0cd33366896..aa6cff98462 100644 --- a/src/main/java/org/jabref/gui/icon/IconTheme.java +++ b/src/main/java/org/jabref/gui/icon/IconTheme.java @@ -169,6 +169,7 @@ public enum JabRefIcons implements JabRefIcon { ADD_NOBOX(MaterialDesignP.PLUS), ADD_ARTICLE(MaterialDesignP.PLUS), ADD_ENTRY(MaterialDesignP.PLAYLIST_PLUS), + CASE_SENSITIVE(MaterialDesignA.ALPHABETICAL), EDIT_ENTRY(MaterialDesignT.TOOLTIP_EDIT), EDIT_STRINGS(MaterialDesignT.TOOLTIP_TEXT), FOLDER(MaterialDesignF.FOLDER_OUTLINE), diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index 74a172c7d79..50040d8a31b 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -82,10 +82,12 @@ public class GlobalSearchBar extends HBox { private static final PseudoClass ILLEGAL_SEARCH = PseudoClass.getPseudoClass("illegal-search"); private final CustomTextField searchField; - private final ToggleButton fulltextButton; private final Button openGlobalSearchButton; + private final ToggleButton fulltextButton; private final ToggleButton keepSearchString; private final ToggleButton filterModeButton; + private final ToggleButton regexButton; + private final ToggleButton caseSensitiveButton; private final Tooltip searchFieldTooltip = new Tooltip(); private final StateManager stateManager; private final GuiPreferences preferences; @@ -143,7 +145,7 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, searchField.setTooltip(searchFieldTooltip); searchFieldTooltip.setContentDisplay(ContentDisplay.GRAPHIC_ONLY); searchFieldTooltip.setMaxHeight(10); - updateHintVisibility(); + setSearchBarHint(); searchField.addEventFilter(KeyEvent.KEY_PRESSED, event -> { if (keyBindingRepository.matches(event, KeyBinding.CLEAR_SEARCH)) { @@ -166,6 +168,8 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, searchField.getContextMenu().getItems().add(SearchFieldRightClickMenu.createSearchFromHistorySubMenu(stateManager, searchField)); }); + regexButton = IconTheme.JabRefIcons.REG_EX.asToggleButton(); + caseSensitiveButton = IconTheme.JabRefIcons.CASE_SENSITIVE.asToggleButton(); fulltextButton = IconTheme.JabRefIcons.FULLTEXT.asToggleButton(); openGlobalSearchButton = IconTheme.JabRefIcons.OPEN_GLOBAL_SEARCH.asButton(); keepSearchString = IconTheme.JabRefIcons.KEEP_SEARCH_STRING.asToggleButton(); @@ -174,11 +178,17 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, initSearchModifierButtons(); BooleanBinding focusedOrActive = searchField.focusedProperty() + .or(regexButton.focusedProperty()) + .or(caseSensitiveButton.focusedProperty()) .or(fulltextButton.focusedProperty()) .or(keepSearchString.focusedProperty()) .or(filterModeButton.focusedProperty()) .or(searchField.textProperty().isNotEmpty()); + regexButton.visibleProperty().unbind(); + regexButton.visibleProperty().bind(focusedOrActive); + caseSensitiveButton.visibleProperty().unbind(); + caseSensitiveButton.visibleProperty().bind(focusedOrActive); fulltextButton.visibleProperty().unbind(); fulltextButton.visibleProperty().bind(focusedOrActive); keepSearchString.visibleProperty().unbind(); @@ -188,10 +198,11 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, StackPane modifierButtons; if (searchType == SearchType.NORMAL_SEARCH) { - modifierButtons = new StackPane(new HBox(fulltextButton, keepSearchString, filterModeButton)); + modifierButtons = new StackPane(new HBox(regexButton, caseSensitiveButton, fulltextButton, keepSearchString, filterModeButton)); } else { - modifierButtons = new StackPane(new HBox(fulltextButton)); + modifierButtons = new StackPane(new HBox(regexButton, caseSensitiveButton, fulltextButton)); } + modifierButtons.setAlignment(Pos.CENTER); searchField.setRight(new HBox(searchField.getRight(), modifierButtons)); searchField.getStyleClass().add("global-search-bar"); @@ -263,6 +274,24 @@ private void initSearchModifierButtons() { updateSearchQuery(); }); + regexButton.setSelected(searchPreferences.isRegularExpression()); + regexButton.setTooltip(new Tooltip(Localization.lang("Regular expression\nThis only affects unfielded terms. For using RegEx in a fielded term, use =~ operator"))); + initSearchModifierButton(regexButton); + regexButton.setOnAction(event -> { + searchPreferences.setSearchFlag(SearchFlags.REGULAR_EXPRESSION, regexButton.isSelected()); + searchField.requestFocus(); + updateSearchQuery(); + }); + + caseSensitiveButton.setSelected(searchPreferences.isCaseSensitive()); + caseSensitiveButton.setTooltip(new Tooltip(Localization.lang("Case sensitive\nThis only affects unfielded terms. For using case-sensitive in a fielded term, use =! operator"))); + initSearchModifierButton(caseSensitiveButton); + caseSensitiveButton.setOnAction(event -> { + searchPreferences.setSearchFlag(SearchFlags.CASE_SENSITIVE, caseSensitiveButton.isSelected()); + searchField.requestFocus(); + updateSearchQuery(); + }); + keepSearchString.setSelected(searchPreferences.shouldKeepSearchString()); keepSearchString.setTooltip(new Tooltip(Localization.lang("Keep search string across libraries"))); initSearchModifierButton(keepSearchString); @@ -284,7 +313,11 @@ private void initSearchModifierButtons() { initSearchModifierButton(openGlobalSearchButton); openGlobalSearchButton.setOnAction(evt -> openGlobalSearchDialog()); - searchPreferences.getObservableSearchFlags().addListener((SetChangeListener.Change change) -> fulltextButton.setSelected(searchPreferences.isFulltext())); + searchPreferences.getObservableSearchFlags().addListener((SetChangeListener.Change change) -> { + regexButton.setSelected(searchPreferences.isRegularExpression()); + caseSensitiveButton.setSelected(searchPreferences.isCaseSensitive()); + fulltextButton.setSelected(searchPreferences.isFulltext()); + }); } public void openGlobalSearchDialog() { @@ -367,9 +400,9 @@ private AutoCompletePopup getPopup(AutoCompletionBinding autoCompletio } } - public void updateHintVisibility() { + public void setSearchBarHint() { if (preferences.getWorkspacePreferences().shouldShowAdvancedHints()) { - String genericDescription = Localization.lang("Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor:Smith AND title:electrical"); + String genericDescription = Localization.lang("Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor= Smith AND title= electrical"); List genericDescriptionTexts = TooltipTextUtil.createTextsFromHtml(genericDescription); TextFlow emptyHintTooltip = new TextFlow(); diff --git a/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index 82d250472e3..6da49e680ba 100644 --- a/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -204,6 +204,8 @@ public class JabRefCliPreferences implements CliPreferences { public static final String MAIN_FILE_DIRECTORY = "fileDirectory"; public static final String SEARCH_DISPLAY_MODE = "searchDisplayMode"; + public static final String SEARCH_CASE_SENSITIVE = "caseSensitiveSearch"; + public static final String SEARCH_REG_EXP = "regExpSearch"; public static final String SEARCH_FULLTEXT = "fulltextSearch"; public static final String SEARCH_KEEP_SEARCH_STRING = "keepSearchString"; public static final String SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP = "keepOnTop"; @@ -429,6 +431,8 @@ protected JabRefCliPreferences() { Localization.setLanguage(getLanguage()); defaults.put(SEARCH_DISPLAY_MODE, Boolean.TRUE); + defaults.put(SEARCH_CASE_SENSITIVE, Boolean.FALSE); + defaults.put(SEARCH_REG_EXP, Boolean.FALSE); defaults.put(SEARCH_FULLTEXT, Boolean.FALSE); defaults.put(SEARCH_KEEP_SEARCH_STRING, Boolean.FALSE); defaults.put(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP, Boolean.TRUE); @@ -1877,6 +1881,8 @@ public SearchPreferences getSearchPreferences() { searchPreferences = new SearchPreferences( getBoolean(SEARCH_DISPLAY_MODE) ? SearchDisplayMode.FILTER : SearchDisplayMode.FLOAT, + getBoolean(SEARCH_REG_EXP), + getBoolean(SEARCH_CASE_SENSITIVE), getBoolean(SEARCH_FULLTEXT), getBoolean(SEARCH_KEEP_SEARCH_STRING), getBoolean(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP), diff --git a/src/main/java/org/jabref/logic/search/SearchPreferences.java b/src/main/java/org/jabref/logic/search/SearchPreferences.java index e6ff3d34dc6..e75269f53a4 100644 --- a/src/main/java/org/jabref/logic/search/SearchPreferences.java +++ b/src/main/java/org/jabref/logic/search/SearchPreferences.java @@ -26,8 +26,22 @@ public class SearchPreferences { private final BooleanProperty keepSearchSting; private final ObjectProperty searchDisplayMode; - public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isFulltext, boolean keepSearchString, boolean keepWindowOnTop, double searchWindowHeight, double searchWindowWidth, double searchWindowDividerPosition) { + public SearchPreferences(SearchDisplayMode searchDisplayMode, + boolean isRegularExpression, + boolean isCaseSensitive, + boolean isFulltext, + boolean keepSearchString, + boolean keepWindowOnTop, + double searchWindowHeight, + double searchWindowWidth, + double searchWindowDividerPosition) { this(searchDisplayMode, EnumSet.noneOf(SearchFlags.class), keepSearchString, keepWindowOnTop, searchWindowHeight, searchWindowWidth, searchWindowDividerPosition); + if (isRegularExpression) { + searchFlags.add(SearchFlags.REGULAR_EXPRESSION); + } + if (isCaseSensitive) { + searchFlags.add(SearchFlags.CASE_SENSITIVE); + } if (isFulltext) { searchFlags.add(SearchFlags.FULLTEXT); } @@ -65,6 +79,14 @@ public void setSearchFlag(SearchFlags flag, boolean value) { } } + public boolean isRegularExpression() { + return searchFlags.contains(SearchFlags.REGULAR_EXPRESSION); + } + + public boolean isCaseSensitive() { + return searchFlags.contains(SearchFlags.CASE_SENSITIVE); + } + public boolean isFulltext() { return searchFlags.contains(SearchFlags.FULLTEXT); } diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 88e83783d06..aa6b59936ce 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -815,6 +815,8 @@ Search...=Search... Searching...=Searching... Finished\ Searching=Finished Searching Search\ expression=Search expression +Regular\ expression\nThis\ only\ affects\ unfielded\ terms.\ For\ using\ RegEx\ in\ a\ fielded\ term,\ use\ \=~\ operator=Regular expression\nThis only affects unfielded terms. For using RegEx in a fielded term, use =~ operator +Case\ sensitive\nThis\ only\ affects\ unfielded\ terms.\ For\ using\ case-sensitive\ in\ a\ fielded\ term,\ use\ \=!\ operator=Case sensitive\nThis only affects unfielded terms. For using case-sensitive in a fielded term, use =! operator Fulltext\ search=Fulltext search Enable\ indexing=Enable indexing @@ -841,7 +843,7 @@ Clear\ search=Clear search Search\ document\ identifier\ online=Search document identifier online Search\ for\ unlinked\ local\ files=Search for unlinked local files Search\ full\ text\ documents\ online=Search full text documents online -Hint\:\n\nTo\ search\ all\ fields\ for\ Smith,\ enter\:\nsmith\n\nTo\ search\ the\ field\ author\ for\ Smith\ and\ the\ field\ title\ for\ electrical,\ enter\:\nauthor\:Smith\ AND\ title\:electrical=Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor:Smith AND title:electrical +Hint\:\n\nTo\ search\ all\ fields\ for\ Smith,\ enter\:\nsmith\n\nTo\ search\ the\ field\ author\ for\ Smith\ and\ the\ field\ title\ for\ electrical,\ enter\:\nauthor\=\ Smith\ AND\ title\=\ electrical=Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor= Smith AND title= electrical Search\ term\ is\ empty.=Search term is empty. Invalid\ regular\ expression.=Invalid regular expression. Searching\ for\ a\ keyword=Searching for a keyword From df2e12b6a55bcb360ff09301b99bec2178f5ef02 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sat, 12 Oct 2024 06:17:37 +0300 Subject: [PATCH 083/104] Use for search bar flags for unfielded terms --- .../query/SearchFlagsToExpressionVisitor.java | 19 +- .../search/query/SearchQueryConversion.java | 2 +- .../search/query/SearchToSqlVisitor.java | 34 ++-- .../org/jabref/model/search/SearchFlags.java | 5 +- .../model/search/query/SearchTermFlag.java | 7 - .../query/SearchQuerySQLConversionTest.java | 176 ++++++++++++++++++ 6 files changed, 211 insertions(+), 32 deletions(-) delete mode 100644 src/main/java/org/jabref/model/search/query/SearchTermFlag.java diff --git a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java index 3f10249cc33..cb5d470ceea 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java @@ -4,19 +4,18 @@ import java.util.Optional; import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.query.SearchTermFlag; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.jabref.model.search.query.SearchTermFlag.CASE_INSENSITIVE; -import static org.jabref.model.search.query.SearchTermFlag.CASE_SENSITIVE; -import static org.jabref.model.search.query.SearchTermFlag.EXACT_MATCH; -import static org.jabref.model.search.query.SearchTermFlag.INEXACT_MATCH; -import static org.jabref.model.search.query.SearchTermFlag.NEGATION; -import static org.jabref.model.search.query.SearchTermFlag.REGULAR_EXPRESSION; +import static org.jabref.model.search.SearchFlags.CASE_INSENSITIVE; +import static org.jabref.model.search.SearchFlags.CASE_SENSITIVE; +import static org.jabref.model.search.SearchFlags.EXACT_MATCH; +import static org.jabref.model.search.SearchFlags.INEXACT_MATCH; +import static org.jabref.model.search.SearchFlags.NEGATION; +import static org.jabref.model.search.SearchFlags.REGULAR_EXPRESSION; /** * Tests are located in {@link org.jabref.logic.search.query.SearchQueryFlagsConversionTest}. @@ -58,7 +57,7 @@ public String visitComparison(SearchParser.ComparisonContext context) { String right = context.right.getText(); Optional fieldDescriptor = Optional.ofNullable(context.left); - EnumSet termFlags = EnumSet.noneOf(SearchTermFlag.class); + EnumSet termFlags = EnumSet.noneOf(SearchFlags.class); if (fieldDescriptor.isPresent()) { String field = fieldDescriptor.get().getText(); @@ -89,12 +88,12 @@ public String visitComparison(SearchParser.ComparisonContext context) { } } - private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { + private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { String operator = getOperator(searchFlags); return field + " " + operator + " " + term; } - private static String getOperator(EnumSet searchFlags) { + private static String getOperator(EnumSet searchFlags) { StringBuilder operator = new StringBuilder(); if (searchFlags.contains(NEGATION)) { diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java index 7b5cd68fe83..7a0653543d5 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java @@ -14,7 +14,7 @@ public class SearchQueryConversion { public static SqlQuery searchToSql(String table, SearchQuery searchQuery) { LOGGER.debug("Converting search expression to SQL: {}", searchQuery.getSearchExpression()); - return new SearchToSqlVisitor(table).visit(searchQuery.getContext()); + return new SearchToSqlVisitor(table, searchQuery.getSearchFlags()).visit(searchQuery.getContext()); } public static String flagsToSearchExpression(SearchQuery searchQuery) { diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 7275f1fbe24..3ade5ec3b00 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -11,7 +11,7 @@ import org.jabref.model.entry.field.InternalField; import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.PostgreConstants; -import org.jabref.model.search.query.SearchTermFlag; +import org.jabref.model.search.SearchFlags; import org.jabref.model.search.query.SqlQuery; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; @@ -20,12 +20,12 @@ import static org.jabref.model.search.PostgreConstants.FIELD_NAME; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_LITERAL; import static org.jabref.model.search.PostgreConstants.FIELD_VALUE_TRANSFORMED; -import static org.jabref.model.search.query.SearchTermFlag.CASE_INSENSITIVE; -import static org.jabref.model.search.query.SearchTermFlag.CASE_SENSITIVE; -import static org.jabref.model.search.query.SearchTermFlag.EXACT_MATCH; -import static org.jabref.model.search.query.SearchTermFlag.INEXACT_MATCH; -import static org.jabref.model.search.query.SearchTermFlag.NEGATION; -import static org.jabref.model.search.query.SearchTermFlag.REGULAR_EXPRESSION; +import static org.jabref.model.search.SearchFlags.CASE_INSENSITIVE; +import static org.jabref.model.search.SearchFlags.CASE_SENSITIVE; +import static org.jabref.model.search.SearchFlags.EXACT_MATCH; +import static org.jabref.model.search.SearchFlags.INEXACT_MATCH; +import static org.jabref.model.search.SearchFlags.NEGATION; +import static org.jabref.model.search.SearchFlags.REGULAR_EXPRESSION; /** * Converts to a query processable by the scheme created by {@link BibFieldsIndexer}. @@ -38,12 +38,14 @@ public class SearchToSqlVisitor extends SearchBaseVisitor { private static final String INNER_TABLE = "inner_table"; private static final String GROUPS_FIELD = StandardField.GROUPS.getName(); + private final EnumSet searchBarFlags; private final String mainTableName; private final String splitValuesTableName; private final List nodes = new ArrayList<>(); private int cteCounter = 0; - public SearchToSqlVisitor(String table) { + public SearchToSqlVisitor(String table, EnumSet searchBarFlags) { + this.searchBarFlags = searchBarFlags; this.mainTableName = PostgreConstants.getMainTableSchemaReference(table); this.splitValuesTableName = PostgreConstants.getSplitTableSchemaReference(table); } @@ -147,12 +149,12 @@ public SqlQuery visitComparison(SearchParser.ComparisonContext context) { } Optional fieldDescriptor = Optional.ofNullable(context.left); + EnumSet searchFlags = EnumSet.noneOf(SearchFlags.class); if (fieldDescriptor.isPresent()) { String field = fieldDescriptor.get().getText(); // Direct comparison does not work // context.CONTAINS() and others are null if absent (thus, we cannot check for getText()) - EnumSet searchFlags = EnumSet.noneOf(SearchTermFlag.class); if (context.EQUAL() != null || context.CONTAINS() != null) { setFlags(searchFlags, INEXACT_MATCH, false, false); } else if (context.CEQUAL() != null) { @@ -182,11 +184,17 @@ public SqlQuery visitComparison(SearchParser.ComparisonContext context) { return getFieldQueryNode(field.toLowerCase(Locale.ROOT), right, searchFlags); } else { // Query without any field name - return getFieldQueryNode("any", right, EnumSet.of(INEXACT_MATCH, CASE_INSENSITIVE)); + boolean isCaseSensitive = searchBarFlags.contains(CASE_SENSITIVE); + if (searchBarFlags.contains(REGULAR_EXPRESSION)) { + setFlags(searchFlags, REGULAR_EXPRESSION, isCaseSensitive, false); + } else { + setFlags(searchFlags, INEXACT_MATCH, isCaseSensitive, false); + } + return getFieldQueryNode("any", right, searchFlags); } } - private SqlQuery getFieldQueryNode(String field, String term, EnumSet searchFlags) { + private SqlQuery getFieldQueryNode(String field, String term, EnumSet searchFlags) { String operator = getOperator(searchFlags); String prefixSuffix = searchFlags.contains(INEXACT_MATCH) ? "%" : ""; @@ -497,7 +505,7 @@ private SqlQuery buildContainsNegationAnyFieldQuery(String operator, String pref return new SqlQuery("cte" + cteCounter++); } - private static void setFlags(EnumSet flags, SearchTermFlag matchType, boolean caseSensitive, boolean negation) { + private static void setFlags(EnumSet flags, SearchFlags matchType, boolean caseSensitive, boolean negation) { flags.add(matchType); flags.add(caseSensitive ? CASE_SENSITIVE : CASE_INSENSITIVE); @@ -506,7 +514,7 @@ private static void setFlags(EnumSet flags, SearchTermFlag match } } - private static String getOperator(EnumSet searchFlags) { + private static String getOperator(EnumSet searchFlags) { return searchFlags.contains(REGULAR_EXPRESSION) ? (searchFlags.contains(CASE_SENSITIVE) ? "~" : "~*") : (searchFlags.contains(CASE_SENSITIVE) ? "LIKE" : "ILIKE"); diff --git a/src/main/java/org/jabref/model/search/SearchFlags.java b/src/main/java/org/jabref/model/search/SearchFlags.java index 20dcf22234e..53adf116800 100644 --- a/src/main/java/org/jabref/model/search/SearchFlags.java +++ b/src/main/java/org/jabref/model/search/SearchFlags.java @@ -1,5 +1,8 @@ package org.jabref.model.search; public enum SearchFlags { - REGULAR_EXPRESSION, FULLTEXT, CASE_SENSITIVE + EXACT_MATCH, INEXACT_MATCH, REGULAR_EXPRESSION, // mutually exclusive + CASE_SENSITIVE, CASE_INSENSITIVE, // mutually exclusive + FULLTEXT, + NEGATION } diff --git a/src/main/java/org/jabref/model/search/query/SearchTermFlag.java b/src/main/java/org/jabref/model/search/query/SearchTermFlag.java deleted file mode 100644 index 5707e80a4c2..00000000000 --- a/src/main/java/org/jabref/model/search/query/SearchTermFlag.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.jabref.model.search.query; - -public enum SearchTermFlag { - EXACT_MATCH, INEXACT_MATCH, REGULAR_EXPRESSION, // mutually exclusive - CASE_SENSITIVE, CASE_INSENSITIVE, // mutually exclusive - NEGATION, -} diff --git a/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java index a32eac536bd..9eef9d06b4e 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java @@ -4,8 +4,10 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.EnumSet; import java.util.stream.Stream; +import org.jabref.model.search.SearchFlags; import org.jabref.model.search.query.SearchQuery; import org.jabref.model.search.query.SqlQuery; @@ -16,6 +18,8 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import static org.jabref.model.search.SearchFlags.CASE_SENSITIVE; +import static org.jabref.model.search.SearchFlags.REGULAR_EXPRESSION; import static org.junit.jupiter.api.Assertions.assertEquals; class SearchQuerySQLConversionTest { @@ -673,4 +677,176 @@ void testSearchConversion(String searchExpression, String expected) throws SQLEx } } } + + public static Stream testUnFieldedTermsWithSearchBarFlags() { + return Stream.of( + Arguments.of( + "Test", + EnumSet.noneOf(SearchFlags.class), + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%Test%')) OR (main_table.field_value_transformed ILIKE ('%Test%'))) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), + + Arguments.of( + "Test", + EnumSet.of(CASE_SENSITIVE), + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal LIKE ('%Test%')) OR (main_table.field_value_transformed LIKE ('%Test%'))) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), + + Arguments.of( + "Test", + EnumSet.of(REGULAR_EXPRESSION), + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ~* ('Test')) OR (main_table.field_value_transformed ~* ('Test'))) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), + + Arguments.of( + "Test", + EnumSet.of(CASE_SENSITIVE, REGULAR_EXPRESSION), + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ~ ('Test')) OR (main_table.field_value_transformed ~ ('Test'))) + ) + ) + SELECT * FROM cte0 GROUP BY entry_id""" + ), + + Arguments.of( + "Test AND author =! Smith", + EnumSet.noneOf(SearchFlags.class), + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ILIKE ('%Test%')) OR (main_table.field_value_transformed ILIKE ('%Test%'))) + ) + ) + , + cte1 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name = 'author') AND ((main_table.field_value_literal LIKE ('%Smith%')) OR (main_table.field_value_transformed LIKE ('%Smith%'))) + ) + ) + , + cte2 AS ( + SELECT entry_id + FROM cte0 + INTERSECT + SELECT entry_id + FROM cte1 + ) + SELECT * FROM cte2 GROUP BY entry_id""" + ), + + Arguments.of( + "Test AND author =! Smith", + EnumSet.of(CASE_SENSITIVE), + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal LIKE ('%Test%')) OR (main_table.field_value_transformed LIKE ('%Test%'))) + ) + ) + , + cte1 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name = 'author') AND ((main_table.field_value_literal LIKE ('%Smith%')) OR (main_table.field_value_transformed LIKE ('%Smith%'))) + ) + ) + , + cte2 AS ( + SELECT entry_id + FROM cte0 + INTERSECT + SELECT entry_id + FROM cte1 + ) + SELECT * FROM cte2 GROUP BY entry_id""" + ), + + Arguments.of( + "any =~ Test AND author =! Smith", + EnumSet.of(CASE_SENSITIVE), + """ + WITH + cte0 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name != 'groups') AND ((main_table.field_value_literal ~* ('Test')) OR (main_table.field_value_transformed ~* ('Test'))) + ) + ) + , + cte1 AS ( + SELECT main_table.entry_id + FROM bib_fields."tableName" AS main_table + WHERE ( + (main_table.field_name = 'author') AND ((main_table.field_value_literal LIKE ('%Smith%')) OR (main_table.field_value_transformed LIKE ('%Smith%'))) + ) + ) + , + cte2 AS ( + SELECT entry_id + FROM cte0 + INTERSECT + SELECT entry_id + FROM cte1 + ) + SELECT * FROM cte2 GROUP BY entry_id""" + ) + ); + } + + @ParameterizedTest + @MethodSource + void testUnFieldedTermsWithSearchBarFlags(String searchExpression, EnumSet searchFlags, String expected) throws SQLException { + try (Connection connection = pg.getPostgresDatabase().getConnection()) { + SqlQuery sqlQuery = SearchQueryConversion.searchToSql("tableName", new SearchQuery(searchExpression, searchFlags)); + try (PreparedStatement preparedStatement = connection.prepareStatement(sqlQuery.cte())) { + for (int i = 0; i < sqlQuery.params().size(); i++) { + preparedStatement.setString(i + 1, sqlQuery.params().get(i)); + } + String sql = preparedStatement.toString(); + assertEquals(expected, sql); + } + } + } } From fb0fbb88d5479542e5b559d06674b33fec263b55 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sat, 12 Oct 2024 06:27:53 +0300 Subject: [PATCH 084/104] Skip migrations for unfieleded terms --- .../query/SearchFlagsToExpressionVisitor.java | 11 +++-------- .../query/SearchQueryFlagsConversionTest.java | 16 ++++++++-------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java index cb5d470ceea..4ecdf4f8e2f 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java @@ -57,9 +57,9 @@ public String visitComparison(SearchParser.ComparisonContext context) { String right = context.right.getText(); Optional fieldDescriptor = Optional.ofNullable(context.left); - EnumSet termFlags = EnumSet.noneOf(SearchFlags.class); if (fieldDescriptor.isPresent()) { + EnumSet termFlags = EnumSet.noneOf(SearchFlags.class); String field = fieldDescriptor.get().getText(); termFlags.add(isCaseSensitive ? CASE_SENSITIVE : CASE_INSENSITIVE); @@ -78,13 +78,8 @@ public String visitComparison(SearchParser.ComparisonContext context) { } return getFieldQueryNode(field, right, termFlags); } else { - termFlags.add(isCaseSensitive ? CASE_SENSITIVE : CASE_INSENSITIVE); - if (isRegularExpression) { - termFlags.add(REGULAR_EXPRESSION); - } else { - termFlags.add(INEXACT_MATCH); - } - return getFieldQueryNode("any", right, termFlags); + // Unfielded term, do nothing, the search flags will be used for unfielded search + return right; } } diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java index 311863d40f5..5524ac4aaa7 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java @@ -18,10 +18,10 @@ private static Stream testSearchConversion() { return Stream.of( createTestCases( "Term", - "any = Term", - "any =! Term", - "any =~ Term", - "any =~! Term" + "Term", + "Term", + "Term", + "Term" ), createTestCases( @@ -65,10 +65,10 @@ private static Stream testSearchConversion() { createTestCases( "(title = \"Tem\" AND author != Alex) OR term", - "(title = \"Tem\" AND author != Alex) OR any = term", - "(title =! \"Tem\" AND author !=! Alex) OR any =! term", - "(title =~ \"Tem\" AND author !=~ Alex) OR any =~ term", - "(title =~! \"Tem\" AND author !=~! Alex) OR any =~! term" + "(title = \"Tem\" AND author != Alex) OR term", + "(title =! \"Tem\" AND author !=! Alex) OR term", + "(title =~ \"Tem\" AND author !=~ Alex) OR term", + "(title =~! \"Tem\" AND author !=~! Alex) OR term" ) ).flatMap(stream -> stream); } From fac963f3123fd3f32cbe46a250ef208f59a72587 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sat, 12 Oct 2024 07:47:40 +0300 Subject: [PATCH 085/104] Return regex, case-sensitive CheckBox to search groups dialog --- .../org/jabref/gui/groups/GroupDialog.fxml | 12 +++++++++++ .../jabref/gui/groups/GroupDialogView.java | 7 +++++++ .../gui/groups/GroupDialogViewModel.java | 20 +++++++++++++++---- .../jabref/gui/search/GlobalSearchBar.java | 4 ++-- src/main/resources/l10n/JabRef_en.properties | 4 ++-- 5 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupDialog.fxml b/src/main/java/org/jabref/gui/groups/GroupDialog.fxml index 9ec2a307ad1..e1100317730 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialog.fxml +++ b/src/main/java/org/jabref/gui/groups/GroupDialog.fxml @@ -121,6 +121,18 @@

+ * - STRING_LITERAL: Removes enclosing quotes and unescapes {@code \"} + *

+ * - TERM: Unescapes {@code \=, \!, \~, \(, \)} + */ + private static String unescapeSearchValue(SearchParser.SearchValueContext ctx) { + if (ctx == null) { + return ""; + } + + String term = ctx.getText(); + + if (ctx.getStart().getType() == SearchParser.STRING_LITERAL) { + return term.substring(1, term.length() - 1) + .replace("\\\"", "\""); + } + + if (ctx.getStart().getType() == SearchParser.TERM) { + return term.replaceAll("\\\\([=!~()])", "$1"); + } + + return term; + } } diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryTest.java index db0e19dca03..3639136135a 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQueryTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryTest.java @@ -53,10 +53,9 @@ public void validSearchQuery(String searchExpression) { @ParameterizedTest @CsvSource({ - "!term", // =!~()" should be escaped with a backslash + "!term", // =!~() should be escaped with a backslash "t~erm", "t(erm", - "t\"erm", "term AND", "field CONTAINS NOT value", }) From b4bbe54c2e84df73e79630c60536a5ac46778e54 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 22 Oct 2024 15:39:13 +0300 Subject: [PATCH 092/104] escape SQL wildcard chars --- .../search/query/SearchToSqlVisitor.java | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index e780f513e90..9a96d84ab67 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -169,6 +169,7 @@ public SqlQueryNode visitComparison(SearchParser.ComparisonContext ctx) { // unfielded expression if (ctx.operator() == null) { + // apply search bar flags to unfielded expressions boolean isCaseSensitive = searchBarFlags.contains(CASE_SENSITIVE); if (searchBarFlags.contains(REGULAR_EXPRESSION)) { setFlags(searchFlags, REGULAR_EXPRESSION, isCaseSensitive, false); @@ -212,37 +213,39 @@ public SqlQueryNode visitComparison(SearchParser.ComparisonContext ctx) { } private SqlQueryNode getFieldQueryNode(String field, String term, EnumSet searchFlags) { - String operator = getSqlOperator(searchFlags); + String sqlOperator = getSqlOperator(searchFlags); + term = escapeTermForSql(term, searchFlags); String prefixSuffix = searchFlags.contains(INEXACT_MATCH) ? "%" : ""; // Pseudo-fields field = switch (field) { case "key" -> InternalField.KEY_FIELD.getName(); case "anykeyword" -> StandardField.KEYWORDS.getName(); + case "anyfield" -> "any"; default -> field; }; if (ENTRY_ID.toString().equals(field)) { return buildEntryIdQuery(term); - } else if ("anyfield".equals(field) || "any".equals(field)) { - if (searchFlags.contains(EXACT_MATCH)) { + } else if ("any".equals(field)) { + if (searchFlags.contains(INEXACT_MATCH)) { return searchFlags.contains(NEGATION) - ? buildExactNegationAnyFieldQuery(operator, term) - : buildExactAnyFieldQuery(operator, term); + ? buildContainsNegationAnyFieldQuery(sqlOperator, prefixSuffix, term) + : buildContainsAnyFieldQuery(sqlOperator, prefixSuffix, term); } else { return searchFlags.contains(NEGATION) - ? buildContainsNegationAnyFieldQuery(operator, prefixSuffix, term) - : buildContainsAnyFieldQuery(operator, prefixSuffix, term); + ? buildExactNegationAnyFieldQuery(sqlOperator, term) + : buildExactAnyFieldQuery(sqlOperator, term); } } else { - if (searchFlags.contains(EXACT_MATCH)) { + if (searchFlags.contains(INEXACT_MATCH)) { return searchFlags.contains(NEGATION) - ? buildExactNegationFieldQuery(field, operator, term) - : buildExactFieldQuery(field, operator, term); + ? buildContainsNegationFieldQuery(field, sqlOperator, prefixSuffix, term) + : buildContainsFieldQuery(field, sqlOperator, prefixSuffix, term); } else { return searchFlags.contains(NEGATION) - ? buildContainsNegationFieldQuery(field, operator, prefixSuffix, term) - : buildContainsFieldQuery(field, operator, prefixSuffix, term); + ? buildExactNegationFieldQuery(field, sqlOperator, term) + : buildExactFieldQuery(field, sqlOperator, term); } } } @@ -562,4 +565,17 @@ private static String unescapeSearchValue(SearchParser.SearchValueContext ctx) { return term; } + + /** + * Escapes wildcard characters in the search term for SQL queries. + *

+ * - Escapes {@code \}, {@code _}, and {@code %} for SQL LIKE queries. + */ + private static String escapeTermForSql(String term, EnumSet searchFlags) { + if (searchFlags.contains(REGULAR_EXPRESSION)) { + return term; + } else { + return term.replaceAll("[\\\\_%]", "\\\\$0"); + } + } } From 7c901f9882b6c27467394793672c2dd7b9e8af23 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 22 Oct 2024 15:41:16 +0300 Subject: [PATCH 093/104] Reorder methods --- .../search/query/SearchToSqlVisitor.java | 153 +++++++++--------- 1 file changed, 77 insertions(+), 76 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 9a96d84ab67..1ab094f0a1f 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -158,7 +158,8 @@ public SqlQueryNode visitBinaryExpression(SearchParser.BinaryExpressionContext c return new SqlQueryNode("cte" + cteCounter++); } - @Override public SqlQueryNode visitComparisonExpression(SearchParser.ComparisonExpressionContext ctx) { + @Override + public SqlQueryNode visitComparisonExpression(SearchParser.ComparisonExpressionContext ctx) { return visit(ctx.comparison()); } @@ -263,22 +264,24 @@ private SqlQueryNode buildEntryIdQuery(String entryId) { return new SqlQueryNode("cte" + cteCounter++); } - private SqlQueryNode buildContainsFieldQuery(String field, String operator, String prefixSuffix, String term) { + private SqlQueryNode buildContainsAnyFieldQuery(String operator, String prefixSuffix, String term) { String cte = """ cte%d AS ( SELECT %s.%s FROM %s AS %s WHERE ( - (%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?)) + (%s.%s != '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?)) ) ) """.formatted( cteCounter, MAIN_TABLE, ENTRY_ID, mainTableName, MAIN_TABLE, - MAIN_TABLE, FIELD_NAME, field, - MAIN_TABLE, FIELD_VALUE_LITERAL, operator, - MAIN_TABLE, FIELD_VALUE_TRANSFORMED, operator); + MAIN_TABLE, FIELD_NAME, GROUPS_FIELD, // https://github.com/JabRef/jabref/issues/7996 + MAIN_TABLE, FIELD_VALUE_LITERAL, + operator, + MAIN_TABLE, FIELD_VALUE_TRANSFORMED, + operator); List params = Collections.nCopies(2, prefixSuffix + term + prefixSuffix); SqlQueryNode node = new SqlQueryNode(cte, params); @@ -286,7 +289,7 @@ private SqlQueryNode buildContainsFieldQuery(String field, String operator, Stri return new SqlQueryNode("cte" + cteCounter++); } - private SqlQueryNode buildContainsNegationFieldQuery(String field, String operator, String prefixSuffix, String term) { + private SqlQueryNode buildContainsNegationAnyFieldQuery(String operator, String prefixSuffix, String term) { String cte = """ cte%d AS ( SELECT %s.%s @@ -295,7 +298,7 @@ private SqlQueryNode buildContainsNegationFieldQuery(String field, String operat SELECT %s.%s FROM %s AS %s WHERE ( - (%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?)) + (%s.%s != '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?)) ) ) ) @@ -306,7 +309,7 @@ private SqlQueryNode buildContainsNegationFieldQuery(String field, String operat MAIN_TABLE, ENTRY_ID, INNER_TABLE, ENTRY_ID, mainTableName, INNER_TABLE, - INNER_TABLE, FIELD_NAME, field, + INNER_TABLE, FIELD_NAME, GROUPS_FIELD, // https://github.com/JabRef/jabref/issues/7996 INNER_TABLE, FIELD_VALUE_LITERAL, operator, INNER_TABLE, FIELD_VALUE_TRANSFORMED, @@ -318,7 +321,7 @@ private SqlQueryNode buildContainsNegationFieldQuery(String field, String operat return new SqlQueryNode("cte" + cteCounter++); } - private SqlQueryNode buildExactFieldQuery(String field, String operator, String term) { + private SqlQueryNode buildExactAnyFieldQuery(String operator, String term) { String cte = """ cte%d AS ( SELECT %s.%s @@ -326,9 +329,12 @@ private SqlQueryNode buildExactFieldQuery(String field, String operator, String LEFT JOIN %s AS %s ON (%s.%s = %s.%s AND %s.%s = %s.%s) WHERE ( - ((%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?))) - OR - ((%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?))) + (%s.%s != '%s') + AND ( + ((%s.%s %s ?) OR (%s.%s %s ?)) + OR + ((%s.%s %s ?) OR (%s.%s %s ?)) + ) ) ) """.formatted( @@ -338,10 +344,9 @@ private SqlQueryNode buildExactFieldQuery(String field, String operator, String splitValuesTableName, SPLIT_TABLE, MAIN_TABLE, ENTRY_ID, SPLIT_TABLE, ENTRY_ID, MAIN_TABLE, FIELD_NAME, SPLIT_TABLE, FIELD_NAME, - MAIN_TABLE, FIELD_NAME, field, + MAIN_TABLE, FIELD_NAME, GROUPS_FIELD, // https://github.com/JabRef/jabref/issues/7996 MAIN_TABLE, FIELD_VALUE_LITERAL, operator, MAIN_TABLE, FIELD_VALUE_TRANSFORMED, operator, - SPLIT_TABLE, FIELD_NAME, field, SPLIT_TABLE, FIELD_VALUE_LITERAL, operator, SPLIT_TABLE, FIELD_VALUE_TRANSFORMED, operator); @@ -351,7 +356,7 @@ private SqlQueryNode buildExactFieldQuery(String field, String operator, String return new SqlQueryNode("cte" + cteCounter++); } - private SqlQueryNode buildExactNegationFieldQuery(String field, String operator, String term) { + private SqlQueryNode buildExactNegationAnyFieldQuery(String operator, String term) { String cte = """ cte%d AS ( SELECT %s.%s @@ -362,9 +367,12 @@ private SqlQueryNode buildExactNegationFieldQuery(String field, String operator, LEFT JOIN %s AS %s ON (%s.%s = %s.%s AND %s.%s = %s.%s) WHERE ( - ((%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?))) - OR - ((%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?))) + (%s.%s != '%s') + AND ( + ((%s.%s %s ?) OR (%s.%s %s ?)) + OR + ((%s.%s %s ?) OR (%s.%s %s ?)) + ) ) ) ) @@ -376,12 +384,11 @@ private SqlQueryNode buildExactNegationFieldQuery(String field, String operator, INNER_TABLE, ENTRY_ID, mainTableName, INNER_TABLE, splitValuesTableName, SPLIT_TABLE, - INNER_TABLE, ENTRY_ID, SPLIT_TABLE, ENTRY_ID, INNER_TABLE, FIELD_NAME, SPLIT_TABLE, FIELD_NAME, - INNER_TABLE, FIELD_NAME, field, + INNER_TABLE, ENTRY_ID, SPLIT_TABLE, ENTRY_ID, + INNER_TABLE, FIELD_NAME, GROUPS_FIELD, // https://github.com/JabRef/jabref/issues/7996 INNER_TABLE, FIELD_VALUE_LITERAL, operator, INNER_TABLE, FIELD_VALUE_TRANSFORMED, operator, - SPLIT_TABLE, FIELD_NAME, field, SPLIT_TABLE, FIELD_VALUE_LITERAL, operator, SPLIT_TABLE, FIELD_VALUE_TRANSFORMED, operator); @@ -391,23 +398,53 @@ private SqlQueryNode buildExactNegationFieldQuery(String field, String operator, return new SqlQueryNode("cte" + cteCounter++); } - private SqlQueryNode buildContainsAnyFieldQuery(String operator, String prefixSuffix, String term) { + private SqlQueryNode buildContainsFieldQuery(String field, String operator, String prefixSuffix, String term) { String cte = """ cte%d AS ( SELECT %s.%s FROM %s AS %s WHERE ( - (%s.%s != '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?)) + (%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?)) ) ) """.formatted( cteCounter, MAIN_TABLE, ENTRY_ID, mainTableName, MAIN_TABLE, - MAIN_TABLE, FIELD_NAME, GROUPS_FIELD, // https://github.com/JabRef/jabref/issues/7996 - MAIN_TABLE, FIELD_VALUE_LITERAL, + MAIN_TABLE, FIELD_NAME, field, + MAIN_TABLE, FIELD_VALUE_LITERAL, operator, + MAIN_TABLE, FIELD_VALUE_TRANSFORMED, operator); + + List params = Collections.nCopies(2, prefixSuffix + term + prefixSuffix); + SqlQueryNode node = new SqlQueryNode(cte, params); + nodes.add(node); + return new SqlQueryNode("cte" + cteCounter++); + } + + private SqlQueryNode buildContainsNegationFieldQuery(String field, String operator, String prefixSuffix, String term) { + String cte = """ + cte%d AS ( + SELECT %s.%s + FROM %s AS %s + WHERE %s.%s NOT IN ( + SELECT %s.%s + FROM %s AS %s + WHERE ( + (%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?)) + ) + ) + ) + """.formatted( + cteCounter, + MAIN_TABLE, ENTRY_ID, + mainTableName, MAIN_TABLE, + MAIN_TABLE, ENTRY_ID, + INNER_TABLE, ENTRY_ID, + mainTableName, INNER_TABLE, + INNER_TABLE, FIELD_NAME, field, + INNER_TABLE, FIELD_VALUE_LITERAL, operator, - MAIN_TABLE, FIELD_VALUE_TRANSFORMED, + INNER_TABLE, FIELD_VALUE_TRANSFORMED, operator); List params = Collections.nCopies(2, prefixSuffix + term + prefixSuffix); @@ -416,7 +453,7 @@ private SqlQueryNode buildContainsAnyFieldQuery(String operator, String prefixSu return new SqlQueryNode("cte" + cteCounter++); } - private SqlQueryNode buildExactAnyFieldQuery(String operator, String term) { + private SqlQueryNode buildExactFieldQuery(String field, String operator, String term) { String cte = """ cte%d AS ( SELECT %s.%s @@ -424,12 +461,9 @@ private SqlQueryNode buildExactAnyFieldQuery(String operator, String term) { LEFT JOIN %s AS %s ON (%s.%s = %s.%s AND %s.%s = %s.%s) WHERE ( - (%s.%s != '%s') - AND ( - ((%s.%s %s ?) OR (%s.%s %s ?)) - OR - ((%s.%s %s ?) OR (%s.%s %s ?)) - ) + ((%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?))) + OR + ((%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?))) ) ) """.formatted( @@ -439,9 +473,10 @@ private SqlQueryNode buildExactAnyFieldQuery(String operator, String term) { splitValuesTableName, SPLIT_TABLE, MAIN_TABLE, ENTRY_ID, SPLIT_TABLE, ENTRY_ID, MAIN_TABLE, FIELD_NAME, SPLIT_TABLE, FIELD_NAME, - MAIN_TABLE, FIELD_NAME, GROUPS_FIELD, // https://github.com/JabRef/jabref/issues/7996 + MAIN_TABLE, FIELD_NAME, field, MAIN_TABLE, FIELD_VALUE_LITERAL, operator, MAIN_TABLE, FIELD_VALUE_TRANSFORMED, operator, + SPLIT_TABLE, FIELD_NAME, field, SPLIT_TABLE, FIELD_VALUE_LITERAL, operator, SPLIT_TABLE, FIELD_VALUE_TRANSFORMED, operator); @@ -451,7 +486,7 @@ private SqlQueryNode buildExactAnyFieldQuery(String operator, String term) { return new SqlQueryNode("cte" + cteCounter++); } - private SqlQueryNode buildExactNegationAnyFieldQuery(String operator, String term) { + private SqlQueryNode buildExactNegationFieldQuery(String field, String operator, String term) { String cte = """ cte%d AS ( SELECT %s.%s @@ -462,12 +497,9 @@ private SqlQueryNode buildExactNegationAnyFieldQuery(String operator, String ter LEFT JOIN %s AS %s ON (%s.%s = %s.%s AND %s.%s = %s.%s) WHERE ( - (%s.%s != '%s') - AND ( - ((%s.%s %s ?) OR (%s.%s %s ?)) - OR - ((%s.%s %s ?) OR (%s.%s %s ?)) - ) + ((%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?))) + OR + ((%s.%s = '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?))) ) ) ) @@ -479,11 +511,12 @@ private SqlQueryNode buildExactNegationAnyFieldQuery(String operator, String ter INNER_TABLE, ENTRY_ID, mainTableName, INNER_TABLE, splitValuesTableName, SPLIT_TABLE, - INNER_TABLE, FIELD_NAME, SPLIT_TABLE, FIELD_NAME, INNER_TABLE, ENTRY_ID, SPLIT_TABLE, ENTRY_ID, - INNER_TABLE, FIELD_NAME, GROUPS_FIELD, // https://github.com/JabRef/jabref/issues/7996 + INNER_TABLE, FIELD_NAME, SPLIT_TABLE, FIELD_NAME, + INNER_TABLE, FIELD_NAME, field, INNER_TABLE, FIELD_VALUE_LITERAL, operator, INNER_TABLE, FIELD_VALUE_TRANSFORMED, operator, + SPLIT_TABLE, FIELD_NAME, field, SPLIT_TABLE, FIELD_VALUE_LITERAL, operator, SPLIT_TABLE, FIELD_VALUE_TRANSFORMED, operator); @@ -493,38 +526,6 @@ private SqlQueryNode buildExactNegationAnyFieldQuery(String operator, String ter return new SqlQueryNode("cte" + cteCounter++); } - private SqlQueryNode buildContainsNegationAnyFieldQuery(String operator, String prefixSuffix, String term) { - String cte = """ - cte%d AS ( - SELECT %s.%s - FROM %s AS %s - WHERE %s.%s NOT IN ( - SELECT %s.%s - FROM %s AS %s - WHERE ( - (%s.%s != '%s') AND ((%s.%s %s ?) OR (%s.%s %s ?)) - ) - ) - ) - """.formatted( - cteCounter, - MAIN_TABLE, ENTRY_ID, - mainTableName, MAIN_TABLE, - MAIN_TABLE, ENTRY_ID, - INNER_TABLE, ENTRY_ID, - mainTableName, INNER_TABLE, - INNER_TABLE, FIELD_NAME, GROUPS_FIELD, // https://github.com/JabRef/jabref/issues/7996 - INNER_TABLE, FIELD_VALUE_LITERAL, - operator, - INNER_TABLE, FIELD_VALUE_TRANSFORMED, - operator); - - List params = Collections.nCopies(2, prefixSuffix + term + prefixSuffix); - SqlQueryNode node = new SqlQueryNode(cte, params); - nodes.add(node); - return new SqlQueryNode("cte" + cteCounter++); - } - private static void setFlags(EnumSet flags, SearchFlags matchType, boolean caseSensitive, boolean negation) { flags.add(matchType); From 58e0163d9756d71b8a4a81e6a79e8a959b816c7f Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 22 Oct 2024 15:56:10 +0300 Subject: [PATCH 094/104] Adapt SearchFlagsToExpressionVisitor --- .../query/SearchFlagsToExpressionVisitor.java | 235 +++++++++--------- .../search/query/SearchQueryConversion.java | 2 +- 2 files changed, 124 insertions(+), 113 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java index cf308ebfbf7..0faf6bccf7a 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java @@ -1,112 +1,123 @@ -//package org.jabref.logic.search.query; -// -//import java.util.EnumSet; -//import java.util.Optional; -// -//import org.jabref.model.search.SearchFlags; -//import org.jabref.search.SearchBaseVisitor; -//import org.jabref.search.SearchParser; -// -//import org.slf4j.Logger; -//import org.slf4j.LoggerFactory; -// -//import static org.jabref.model.search.SearchFlags.CASE_INSENSITIVE; -//import static org.jabref.model.search.SearchFlags.CASE_SENSITIVE; -//import static org.jabref.model.search.SearchFlags.EXACT_MATCH; -//import static org.jabref.model.search.SearchFlags.INEXACT_MATCH; -//import static org.jabref.model.search.SearchFlags.NEGATION; -//import static org.jabref.model.search.SearchFlags.REGULAR_EXPRESSION; -// -///** -// * Tests are located in {@link org.jabref.logic.search.query.SearchQueryFlagsConversionTest}. -// */ -//public class SearchFlagsToExpressionVisitor extends SearchBaseVisitor { -// -// private static final Logger LOGGER = LoggerFactory.getLogger(SearchFlagsToExpressionVisitor.class); -// -// private final boolean isCaseSensitive; -// private final boolean isRegularExpression; -// -// public SearchFlagsToExpressionVisitor(EnumSet searchFlags) { -// LOGGER.debug("Converting search flags to search expression: {}", searchFlags); -// this.isCaseSensitive = searchFlags.contains(SearchFlags.CASE_SENSITIVE); -// this.isRegularExpression = searchFlags.contains(SearchFlags.REGULAR_EXPRESSION); -// } -// -// @Override -// public String visitStart(SearchParser.StartContext context) { -// return visit(context.expression()); -// } -// -// @Override -// public String visitParenExpression(SearchParser.ParenExpressionContext ctx) { -// return "(" + visit(ctx.expression()) + ")"; -// } -// -// @Override -// public String visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { -// return "NOT " + visit(ctx.expression()); -// } -// -// @Override -// public String visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { -// return visit(ctx.left) + " " + ctx.operator.getText() + " " + visit(ctx.right); -// } -// -// public String visitComparison(SearchParser.ComparisonContext context) { -// String right = context.right.getText(); -// -// Optional fieldDescriptor = Optional.ofNullable(context.left); -// -// if (fieldDescriptor.isPresent()) { -// EnumSet termFlags = EnumSet.noneOf(SearchFlags.class); -// String field = fieldDescriptor.get().getText(); -// -// termFlags.add(isCaseSensitive ? CASE_SENSITIVE : CASE_INSENSITIVE); -// if (context.NEQUAL() != null) { -// termFlags.add(NEGATION); -// } -// -// if (isRegularExpression) { -// termFlags.add(REGULAR_EXPRESSION); -// } else { -// if (context.EQUAL() != null || context.CONTAINS() != null || context.NEQUAL() != null) { -// termFlags.add(INEXACT_MATCH); -// } else if (context.EEQUAL() != null || context.MATCHES() != null) { -// termFlags.add(EXACT_MATCH); -// } -// } -// return getFieldQueryNode(field, right, termFlags); -// } else { -// // Unfielded term, do nothing, the search flags will be used for unfielded search -// return right; -// } -// } -// -// private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { -// String operator = getOperator(searchFlags); -// return field + " " + operator + " " + term; -// } -// -// private static String getOperator(EnumSet searchFlags) { -// StringBuilder operator = new StringBuilder(); -// -// if (searchFlags.contains(NEGATION)) { -// operator.append("!"); -// } -// -// if (searchFlags.contains(INEXACT_MATCH)) { -// operator.append("="); -// } else if (searchFlags.contains(EXACT_MATCH)) { -// operator.append("=="); -// } else if (searchFlags.contains(REGULAR_EXPRESSION)) { -// operator.append("=~"); -// } -// -// if (searchFlags.contains(CASE_SENSITIVE)) { -// operator.append("!"); -// } -// -// return operator.toString(); -// } -//} +package org.jabref.logic.search.query; + +import java.util.EnumSet; +import java.util.List; + +import org.jabref.model.search.SearchFlags; +import org.jabref.search.SearchBaseVisitor; +import org.jabref.search.SearchParser; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.jabref.model.search.SearchFlags.CASE_INSENSITIVE; +import static org.jabref.model.search.SearchFlags.CASE_SENSITIVE; +import static org.jabref.model.search.SearchFlags.EXACT_MATCH; +import static org.jabref.model.search.SearchFlags.INEXACT_MATCH; +import static org.jabref.model.search.SearchFlags.NEGATION; +import static org.jabref.model.search.SearchFlags.REGULAR_EXPRESSION; + +/** + * Tests are located in {@link org.jabref.logic.search.query.SearchQueryFlagsConversionTest}. + */ +public class SearchFlagsToExpressionVisitor extends SearchBaseVisitor { + + private static final Logger LOGGER = LoggerFactory.getLogger(SearchFlagsToExpressionVisitor.class); + + private final boolean isCaseSensitive; + private final boolean isRegularExpression; + + public SearchFlagsToExpressionVisitor(EnumSet searchFlags) { + LOGGER.debug("Converting search flags to search expression: {}", searchFlags); + this.isCaseSensitive = searchFlags.contains(SearchFlags.CASE_SENSITIVE); + this.isRegularExpression = searchFlags.contains(SearchFlags.REGULAR_EXPRESSION); + } + + @Override + public String visitStart(SearchParser.StartContext ctx) { + return visit(ctx.orExpression()); + } + + @Override + public String visitImplicitOrExpression(SearchParser.ImplicitOrExpressionContext ctx) { + List children = ctx.expression().stream().map(this::visit).toList(); + return String.join(" ", children); + } + + @Override + public String visitParenExpression(SearchParser.ParenExpressionContext ctx) { + return "(" + visit(ctx.orExpression()) + ")"; + } + + @Override + public String visitNegatedExpression(SearchParser.NegatedExpressionContext ctx) { + return "NOT " + visit(ctx.expression()); + } + + @Override + public String visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { + return visit(ctx.left) + " " + ctx.bin_op.getText() + " " + visit(ctx.right); + } + + @Override + public String visitComparisonExpression(SearchParser.ComparisonExpressionContext ctx) { + return visit(ctx.comparison()); + } + + public String visitComparison(SearchParser.ComparisonContext ctx) { + String term = ctx.searchValue().getText(); + + // unfielded expression + if (ctx.operator() == null) { + return term; + } + + // fielded expression + EnumSet searchFlags = EnumSet.noneOf(SearchFlags.class); + String field = ctx.FIELD().getText(); + int operator = ctx.operator().getStart().getType(); + + searchFlags.add(isCaseSensitive ? CASE_SENSITIVE : CASE_INSENSITIVE); + if (operator == SearchParser.NEQUAL) { + searchFlags.add(NEGATION); + } + + if (isRegularExpression) { + searchFlags.add(REGULAR_EXPRESSION); + } else { + if (operator == SearchParser.EQUAL || operator == SearchParser.CONTAINS || operator == SearchParser.NEQUAL) { + searchFlags.add(INEXACT_MATCH); + } else if (operator == SearchParser.EEQUAL || operator == SearchParser.MATCHES) { + searchFlags.add(EXACT_MATCH); + } + } + return getFieldQueryNode(field, term, searchFlags); + } + + private String getFieldQueryNode(String field, String term, EnumSet searchFlags) { + String operator = getOperator(searchFlags); + return field + " " + operator + " " + term; + } + + private static String getOperator(EnumSet searchFlags) { + StringBuilder operator = new StringBuilder(); + + if (searchFlags.contains(NEGATION)) { + operator.append("!"); + } + + if (searchFlags.contains(INEXACT_MATCH)) { + operator.append("="); + } else if (searchFlags.contains(EXACT_MATCH)) { + operator.append("=="); + } else if (searchFlags.contains(REGULAR_EXPRESSION)) { + operator.append("=~"); + } + + if (searchFlags.contains(CASE_SENSITIVE)) { + operator.append("!"); + } + + return operator.toString(); + } +} diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java index b3dce85a188..91b76f48cc3 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java @@ -19,7 +19,7 @@ public static SqlQueryNode searchToSql(String table, SearchQuery searchQuery) { public static String flagsToSearchExpression(SearchQuery searchQuery) { LOGGER.debug("Converting search flags to search expression: {}, flags {}", searchQuery.getSearchExpression(), searchQuery.getSearchFlags()); - return null; + return new SearchFlagsToExpressionVisitor(searchQuery.getSearchFlags()).visit(searchQuery.getContext()); } public static Query searchToLucene(SearchQuery searchQuery) { From 070dcc65428af695a00b33bd370ab0dda8c3be7d Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 22 Oct 2024 16:31:17 +0300 Subject: [PATCH 095/104] Adapt SearchToLuceneVisitor --- .../query/SearchFlagsToExpressionVisitor.java | 5 +- .../search/query/SearchQueryConversion.java | 2 +- .../search/query/SearchToLuceneVisitor.java | 316 ++++++++++-------- .../search/query/SearchToSqlVisitor.java | 7 +- .../logic/search/DatabaseSearcherTest.java | 4 +- .../SearchQueryExtractorConversionTest.java | 4 +- .../query/SearchQueryFlagsConversionTest.java | 4 +- .../query/SearchQuerySQLConversionTest.java | 8 +- 8 files changed, 195 insertions(+), 155 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java index 0faf6bccf7a..aeb863302a1 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java @@ -41,6 +41,9 @@ public String visitStart(SearchParser.StartContext ctx) { @Override public String visitImplicitOrExpression(SearchParser.ImplicitOrExpressionContext ctx) { List children = ctx.expression().stream().map(this::visit).toList(); + if (children.size() == 1) { + return children.getFirst(); + } return String.join(" ", children); } @@ -68,7 +71,7 @@ public String visitComparison(SearchParser.ComparisonContext ctx) { String term = ctx.searchValue().getText(); // unfielded expression - if (ctx.operator() == null) { + if (ctx.FIELD() == null) { return term; } diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java index 91b76f48cc3..e22bfa3739e 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java @@ -24,7 +24,7 @@ public static String flagsToSearchExpression(SearchQuery searchQuery) { public static Query searchToLucene(SearchQuery searchQuery) { LOGGER.debug("Converting search expression to Lucene: {}", searchQuery.getSearchExpression()); - return null; + return new SearchToLuceneVisitor().visit(searchQuery.getContext()); } public static Set extractSearchTerms(SearchQuery searchQuery) { diff --git a/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java index c9b69cf9b56..fee54049c80 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java @@ -1,138 +1,178 @@ -//package org.jabref.logic.search.query; -// -//import java.util.List; -// -//import org.jabref.model.search.LinkedFilesConstants; -//import org.jabref.search.SearchBaseVisitor; -//import org.jabref.search.SearchParser; -// -//import org.antlr.v4.runtime.Token; -//import org.apache.lucene.index.Term; -//import org.apache.lucene.search.BooleanClause; -//import org.apache.lucene.search.BooleanQuery; -//import org.apache.lucene.search.MatchNoDocsQuery; -//import org.apache.lucene.search.Query; -//import org.apache.lucene.search.RegexpQuery; -//import org.apache.lucene.search.TermQuery; -//import org.apache.lucene.util.QueryBuilder; -// -///** -// * Tests are located in {@link org.jabref.logic.search.query.SearchToLuceneVisitor}. -// */ -//public class SearchToLuceneVisitor extends SearchBaseVisitor { -// -// private static final List SEARCH_FIELDS = LinkedFilesConstants.PDF_FIELDS; -// -// private final QueryBuilder queryBuilder; -// -// public SearchToLuceneVisitor() { -// this.queryBuilder = new QueryBuilder(LinkedFilesConstants.LINKED_FILES_ANALYZER); -// } -// -// @Override -// public Query visitStart(SearchParser.StartContext ctx) { -// return visit(ctx.expression()); -// } -// -// @Override -// public Query visitParenExpression(SearchParser.ParenExpressionContext ctx) { -// return visit(ctx.expression()); -// } -// -// @Override -// public Query visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { -// Query innerQuery = visit(ctx.expression()); -// if (innerQuery instanceof MatchNoDocsQuery) { -// return innerQuery; -// } -// BooleanQuery.Builder builder = new BooleanQuery.Builder(); -// builder.add(innerQuery, BooleanClause.Occur.MUST_NOT); -// return builder.build(); -// } -// -// @Override -// public Query visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { -// Query left = visit(ctx.left); -// Query right = visit(ctx.right); -// -// if (left instanceof MatchNoDocsQuery) { -// return right; -// } -// if (right instanceof MatchNoDocsQuery) { -// return left; -// } -// -// BooleanQuery.Builder builder = new BooleanQuery.Builder(); -// -// if (ctx.operator.getType() == SearchParser.AND) { -// builder.add(left, BooleanClause.Occur.MUST); -// builder.add(right, BooleanClause.Occur.MUST); -// } else if (ctx.operator.getType() == SearchParser.OR) { -// builder.add(left, BooleanClause.Occur.SHOULD); -// builder.add(right, BooleanClause.Occur.SHOULD); -// } -// -// return builder.build(); -// } -// -// @Override -// public Query visitComparison(SearchParser.ComparisonContext ctx) { -// String field = ctx.left != null ? ctx.left.getText().toLowerCase() : null; -// String term = ctx.right.getText(); -// -// if (term.startsWith("\"") && term.endsWith("\"")) { -// term = term.substring(1, term.length() - 1); -// } -// -// if (field == null || "anyfield".equals(field) || "any".equals(field)) { -// return createMultiFieldQuery(term, ctx.operator); -// } else if (SEARCH_FIELDS.contains(field)) { -// return createFieldQuery(field, term, ctx.operator); -// } else { -// return new MatchNoDocsQuery(); -// } -// } -// -// private Query createMultiFieldQuery(String value, Token operator) { -// BooleanQuery.Builder builder = new BooleanQuery.Builder(); -// for (String field : SEARCH_FIELDS) { -// builder.add(createFieldQuery(field, value, operator), BooleanClause.Occur.SHOULD); -// } -// return builder.build(); -// } -// -// private Query createFieldQuery(String field, String value, Token operator) { -// if (operator == null) { -// return createTermOrPhraseQuery(field, value); -// } -// -// return switch (operator.getType()) { -// case SearchParser.REQUAL, -// SearchParser.CREEQUAL -> -// new RegexpQuery(new Term(field, value)); -// case SearchParser.NEQUAL, -// SearchParser.NCEQUAL, -// SearchParser.NEEQUAL, -// SearchParser.NCEEQUAL -> -// createNegatedQuery(createTermOrPhraseQuery(field, value)); -// case SearchParser.NREQUAL, -// SearchParser.NCREEQUAL -> -// createNegatedQuery(new RegexpQuery(new Term(field, value))); -// default -> -// createTermOrPhraseQuery(field, value); -// }; -// } -// -// private Query createNegatedQuery(Query query) { -// BooleanQuery.Builder negatedQuery = new BooleanQuery.Builder(); -// negatedQuery.add(query, BooleanClause.Occur.MUST_NOT); -// return negatedQuery.build(); -// } -// -// private Query createTermOrPhraseQuery(String field, String value) { -// if (value.contains("*") || value.contains("?")) { -// return new TermQuery(new Term(field, value)); -// } -// return queryBuilder.createPhraseQuery(field, value); -// } -//} +package org.jabref.logic.search.query; + +import java.util.List; + +import org.jabref.model.search.LinkedFilesConstants; +import org.jabref.search.SearchBaseVisitor; +import org.jabref.search.SearchParser; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.RegexpQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.QueryBuilder; + +/** + * Tests are located in {@link org.jabref.logic.search.query.SearchQueryLuceneConversionTest}. + */ +public class SearchToLuceneVisitor extends SearchBaseVisitor { + + private static final List SEARCH_FIELDS = LinkedFilesConstants.PDF_FIELDS; + + private final QueryBuilder queryBuilder; + + public SearchToLuceneVisitor() { + this.queryBuilder = new QueryBuilder(LinkedFilesConstants.LINKED_FILES_ANALYZER); + } + + @Override + public Query visitStart(SearchParser.StartContext ctx) { + return visit(ctx.orExpression()); + } + + @Override + public Query visitImplicitOrExpression(SearchParser.ImplicitOrExpressionContext ctx) { + List children = ctx.expression().stream().map(this::visit).toList(); + if (children.size() == 1) { + return children.getFirst(); + } + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + for (Query child : children) { + builder.add(child, BooleanClause.Occur.SHOULD); + } + return builder.build(); + } + + @Override + public Query visitParenExpression(SearchParser.ParenExpressionContext ctx) { + return visit(ctx.orExpression()); + } + + @Override + public Query visitNegatedExpression(SearchParser.NegatedExpressionContext ctx) { + Query innerQuery = visit(ctx.expression()); + if (innerQuery instanceof MatchNoDocsQuery) { + return innerQuery; + } + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(innerQuery, BooleanClause.Occur.MUST_NOT); + return builder.build(); + } + + @Override + public Query visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { + Query left = visit(ctx.left); + Query right = visit(ctx.right); + + if (left instanceof MatchNoDocsQuery) { + return right; + } + if (right instanceof MatchNoDocsQuery) { + return left; + } + + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + + if (ctx.bin_op.getType() == SearchParser.AND) { + builder.add(left, BooleanClause.Occur.MUST); + builder.add(right, BooleanClause.Occur.MUST); + } else if (ctx.bin_op.getType() == SearchParser.OR) { + builder.add(left, BooleanClause.Occur.SHOULD); + builder.add(right, BooleanClause.Occur.SHOULD); + } + + return builder.build(); + } + + @Override + public Query visitComparisonExpression(SearchParser.ComparisonExpressionContext ctx) { + return visit(ctx.comparison()); + } + + @Override + public Query visitComparison(SearchParser.ComparisonContext ctx) { + String field = ctx.FIELD() == null ? null : ctx.FIELD().getText(); + String term = unescapeSearchValue(ctx.searchValue()); + + // unfielded expression + if (field == null || "anyfield".equals(field) || "any".equals(field)) { + return createMultiFieldQuery(term, ctx.operator()); + } else if (SEARCH_FIELDS.contains(field)) { + return createFieldQuery(field, term, ctx.operator()); + } else { + return new MatchNoDocsQuery(); + } + } + + private Query createMultiFieldQuery(String value, SearchParser.OperatorContext operator) { + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + for (String field : SEARCH_FIELDS) { + builder.add(createFieldQuery(field, value, operator), BooleanClause.Occur.SHOULD); + } + return builder.build(); + } + + private Query createFieldQuery(String field, String value, SearchParser.OperatorContext operator) { + if (operator == null) { + return createTermOrPhraseQuery(field, value); + } + + return switch (operator.getStart().getType()) { + case SearchParser.REQUAL, + SearchParser.CREEQUAL -> + new RegexpQuery(new Term(field, value)); + case SearchParser.NEQUAL, + SearchParser.NCEQUAL, + SearchParser.NEEQUAL, + SearchParser.NCEEQUAL -> + createNegatedQuery(createTermOrPhraseQuery(field, value)); + case SearchParser.NREQUAL, + SearchParser.NCREEQUAL -> + createNegatedQuery(new RegexpQuery(new Term(field, value))); + default -> + createTermOrPhraseQuery(field, value); + }; + } + + private Query createNegatedQuery(Query query) { + BooleanQuery.Builder negatedQuery = new BooleanQuery.Builder(); + negatedQuery.add(query, BooleanClause.Occur.MUST_NOT); + return negatedQuery.build(); + } + + private Query createTermOrPhraseQuery(String field, String value) { + if (value.contains("*") || value.contains("?")) { + return new TermQuery(new Term(field, value)); + } + return queryBuilder.createPhraseQuery(field, value); + } + + /** + * Unescapes search value based on the Search grammar rules. + *

+ * - STRING_LITERAL: Removes enclosing quotes and unescapes {@code \"} + *

+ * - TERM: Unescapes {@code \=, \!, \~, \(, \)} + */ + private static String unescapeSearchValue(SearchParser.SearchValueContext ctx) { + if (ctx == null) { + return ""; + } + + String term = ctx.getText(); + + if (ctx.getStart().getType() == SearchParser.STRING_LITERAL) { + return term.substring(1, term.length() - 1) + .replace("\\\"", "\""); + } + + if (ctx.getStart().getType() == SearchParser.TERM) { + return term.replaceAll("\\\\([=!~()])", "$1"); + } + + return term; + } +} diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 1ab094f0a1f..78d5cbd83fc 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -75,10 +75,7 @@ public SqlQueryNode visitStart(SearchParser.StartContext ctx) { @Override public SqlQueryNode visitImplicitOrExpression(SearchParser.ImplicitOrExpressionContext ctx) { - List children = new ArrayList<>(); - for (SearchParser.ExpressionContext exprCtx : ctx.expression()) { - children.add(visit(exprCtx)); - } + List children = ctx.expression().stream().map(this::visit).toList(); if (children.size() == 1) { return children.getFirst(); @@ -169,7 +166,7 @@ public SqlQueryNode visitComparison(SearchParser.ComparisonContext ctx) { String term = unescapeSearchValue(ctx.searchValue()); // unfielded expression - if (ctx.operator() == null) { + if (ctx.FIELD() == null) { // apply search bar flags to unfielded expressions boolean isCaseSensitive = searchBarFlags.contains(CASE_SENSITIVE); if (searchBarFlags.contains(REGULAR_EXPRESSION)) { diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index 0c4b9bba2a1..2a6b8ce9ac7 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -53,7 +53,7 @@ void setUp() { @ParameterizedTest @MethodSource - void testDatabaseSearcher(List expectedMatches, SearchQuery query, List entries) throws IOException { + void databaseSearcher(List expectedMatches, SearchQuery query, List entries) throws IOException { for (BibEntry entry : entries) { databaseContext.getDatabase().insertEntry(entry); } @@ -61,7 +61,7 @@ void testDatabaseSearcher(List expectedMatches, SearchQuery query, Lis assertEquals(expectedMatches, matches); } - private static Stream testDatabaseSearcher() { + private static Stream databaseSearcher() { BibEntry emptyEntry = new BibEntry(); BibEntry articleEntry = new BibEntry(StandardEntryType.Article); diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java index d9cd260cd9d..0e8fba2e4ba 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java @@ -12,7 +12,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class SearchQueryExtractorConversionTest { - public static Stream testSearchConversion() { + public static Stream searchConversion() { return Stream.of( Arguments.of(Set.of("term"), "term"), Arguments.of(Set.of("regex.*term"), "regex.*term"), @@ -30,7 +30,7 @@ public static Stream testSearchConversion() { @ParameterizedTest @MethodSource - void testSearchConversion(Set expected, String searchExpression) { + void searchConversion(Set expected, String searchExpression) { Set result = SearchQueryConversion.extractSearchTerms(new SearchQuery(searchExpression)); assertEquals(expected, result); } diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java index 5524ac4aaa7..74281c922ac 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryFlagsConversionTest.java @@ -14,7 +14,7 @@ class SearchQueryFlagsConversionTest { - private static Stream testSearchConversion() { + private static Stream searchConversion() { return Stream.of( createTestCases( "Term", @@ -84,7 +84,7 @@ private static Stream createTestCases(String searchExpression, String @ParameterizedTest @MethodSource - void testSearchConversion(String expected, String searchExpression, EnumSet flags) { + void searchConversion(String expected, String searchExpression, EnumSet flags) { String result = SearchQueryConversion.flagsToSearchExpression(new SearchQuery(searchExpression, flags)); assertEquals(expected, result); } diff --git a/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java index 928eb2ad43a..f4f6c93d96d 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java @@ -35,7 +35,7 @@ public static void teardown() throws IOException { pg.close(); } - public static Stream testSearchConversion() { + public static Stream searchConversion() { return Stream.of( Arguments.of( "author CONTAINS smith", @@ -701,7 +701,7 @@ cte0 AS ( @ParameterizedTest @MethodSource - void testSearchConversion(String searchExpression, String expected) throws SQLException { + void searchConversion(String searchExpression, String expected) throws SQLException { try (Connection connection = pg.getPostgresDatabase().getConnection()) { SqlQueryNode sqlQueryNode = SearchQueryConversion.searchToSql("tableName", new SearchQuery(searchExpression)); try (PreparedStatement preparedStatement = connection.prepareStatement(sqlQueryNode.cte())) { @@ -714,7 +714,7 @@ void testSearchConversion(String searchExpression, String expected) throws SQLEx } } - public static Stream testUnFieldedTermsWithSearchBarFlags() { + public static Stream unFieldedTermsWithSearchBarFlags() { return Stream.of( Arguments.of( "Test", @@ -873,7 +873,7 @@ cte2 AS ( @ParameterizedTest @MethodSource - void testUnFieldedTermsWithSearchBarFlags(String searchExpression, EnumSet searchFlags, String expected) throws SQLException { + void unFieldedTermsWithSearchBarFlags(String searchExpression, EnumSet searchFlags, String expected) throws SQLException { try (Connection connection = pg.getPostgresDatabase().getConnection()) { SqlQueryNode sqlQueryNode = SearchQueryConversion.searchToSql("tableName", new SearchQuery(searchExpression, searchFlags)); try (PreparedStatement preparedStatement = connection.prepareStatement(sqlQueryNode.cte())) { From f4c694a2be3d1c1ebf0f67b6908373a5f25a9287 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 22 Oct 2024 17:12:20 +0300 Subject: [PATCH 096/104] Adapt SearchQueryExtractorVisitor --- .../query/SearchFlagsToExpressionVisitor.java | 1 + .../search/query/SearchQueryConversion.java | 29 +++- .../query/SearchQueryExtractorVisitor.java | 146 ++++++++++-------- .../search/query/SearchToLuceneVisitor.java | 28 +--- .../search/query/SearchToSqlVisitor.java | 28 +--- 5 files changed, 112 insertions(+), 120 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java index aeb863302a1..ae6ca000041 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java @@ -67,6 +67,7 @@ public String visitComparisonExpression(SearchParser.ComparisonExpressionContext return visit(ctx.comparison()); } + @Override public String visitComparison(SearchParser.ComparisonContext ctx) { String term = ctx.searchValue().getText(); diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java index e22bfa3739e..67ca7fdea3f 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java @@ -4,6 +4,7 @@ import org.jabref.model.search.query.SearchQuery; import org.jabref.model.search.query.SqlQueryNode; +import org.jabref.search.SearchParser; import org.apache.lucene.search.Query; import org.slf4j.Logger; @@ -29,6 +30,32 @@ public static Query searchToLucene(SearchQuery searchQuery) { public static Set extractSearchTerms(SearchQuery searchQuery) { LOGGER.debug("Extracting search terms from search expression: {}", searchQuery.getSearchExpression()); - return null; + return new SearchQueryExtractorVisitor().visit(searchQuery.getContext()); + } + + /** + * Unescapes search value based on the Search grammar rules. + *

+ * - STRING_LITERAL: Removes enclosing quotes and unescapes {@code \"} + *

+ * - TERM: Unescapes {@code \=, \!, \~, \(, \)} + */ + public static String unescapeSearchValue(SearchParser.SearchValueContext ctx) { + if (ctx == null) { + return ""; + } + + String term = ctx.getText(); + + if (ctx.getStart().getType() == SearchParser.STRING_LITERAL) { + return term.substring(1, term.length() - 1) + .replace("\\\"", "\""); + } + + if (ctx.getStart().getType() == SearchParser.TERM) { + return term.replaceAll("\\\\([=!~()])", "$1"); + } + + return term; } } diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java index 9418834845f..10ee42dfb11 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java @@ -1,65 +1,81 @@ -//package org.jabref.logic.search.query; -// -//import java.util.HashSet; -//import java.util.Set; -// -//import org.jabref.search.SearchBaseVisitor; -//import org.jabref.search.SearchParser; -// -//public class SearchQueryExtractorVisitor extends SearchBaseVisitor> { -// -// private final Set searchTerms = new HashSet<>(); -// private boolean isNegated = false; -// -// @Override -// public Set visitStart(SearchParser.StartContext ctx) { -// visit(ctx.expression()); -// return searchTerms; -// } -// -// @Override -// public Set visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { -// isNegated = !isNegated; -// visit(ctx.expression()); -// isNegated = !isNegated; -// return searchTerms; -// } -// -// @Override -// public Set visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { -// visit(ctx.left); -// visit(ctx.right); -// return searchTerms; -// } -// -// @Override -// public Set visitParenExpression(SearchParser.ParenExpressionContext ctx) { -// return visit(ctx.expression()); -// } -// -// @Override -// public Set visitAtomExpression(SearchParser.AtomExpressionContext ctx) { -// return visit(ctx.comparison()); -// } -// -// @Override -// public Set visitComparison(SearchParser.ComparisonContext context) { -// if (isNegated || -// context.NEQUAL() != null || -// context.NCEQUAL() != null || -// context.NEEQUAL() != null || -// context.NCEEQUAL() != null || -// context.NREQUAL() != null || -// context.NCREEQUAL() != null) { -// return searchTerms; -// } -// -// String right = context.right.getText(); -// if (right.startsWith("\"") && right.endsWith("\"")) { -// right = right.substring(1, right.length() - 1); -// } -// searchTerms.add(right); -// -// return searchTerms; -// } -//} +package org.jabref.logic.search.query; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.jabref.search.SearchBaseVisitor; +import org.jabref.search.SearchParser; + +/** + * Tests are located in {@link org.jabref.logic.search.query.SearchQueryExtractorConversionTest}. + */ +public class SearchQueryExtractorVisitor extends SearchBaseVisitor> { + + private boolean isNegated = false; + + @Override + public Set visitStart(SearchParser.StartContext ctx) { + return visit(ctx.orExpression()); + } + + @Override + public Set visitImplicitOrExpression(SearchParser.ImplicitOrExpressionContext ctx) { + List> children = ctx.expression().stream().map(this::visit).toList(); + if (children.size() == 1) { + return children.getFirst(); + } else { + Set terms = new HashSet<>(); + for (Set child : children) { + terms.addAll(child); + } + return terms; + } + } + + @Override + public Set visitNegatedExpression(SearchParser.NegatedExpressionContext ctx) { + isNegated = !isNegated; + Set terms = visit(ctx.expression()); + isNegated = !isNegated; + return terms; + } + + @Override + public Set visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { + Set terms = new HashSet<>(); + terms.addAll(visit(ctx.left)); + terms.addAll(visit(ctx.right)); + return terms; + } + + @Override + public Set visitParenExpression(SearchParser.ParenExpressionContext ctx) { + return visit(ctx.orExpression()); + } + + @Override + public Set visitComparisonExpression(SearchParser.ComparisonExpressionContext ctx) { + return visit(ctx.comparison()); + } + + @Override + public Set visitComparison(SearchParser.ComparisonContext ctx) { + if (isNegated) { + return Set.of(); + } + if (ctx.operator() != null) { + int operator = ctx.operator().getStart().getType(); + if (operator == SearchParser.NEQUAL + || operator == SearchParser.NCEQUAL + || operator == SearchParser.NEEQUAL + || operator == SearchParser.NCEEQUAL + || operator == SearchParser.NREQUAL + || operator == SearchParser.NCREEQUAL) { + return Set.of(); + } + } + + return Set.of(SearchQueryConversion.unescapeSearchValue(ctx.searchValue())); + } +} diff --git a/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java index fee54049c80..8cce4fbe8f8 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java @@ -95,7 +95,7 @@ public Query visitComparisonExpression(SearchParser.ComparisonExpressionContext @Override public Query visitComparison(SearchParser.ComparisonContext ctx) { String field = ctx.FIELD() == null ? null : ctx.FIELD().getText(); - String term = unescapeSearchValue(ctx.searchValue()); + String term = SearchQueryConversion.unescapeSearchValue(ctx.searchValue()); // unfielded expression if (field == null || "anyfield".equals(field) || "any".equals(field)) { @@ -149,30 +149,4 @@ private Query createTermOrPhraseQuery(String field, String value) { } return queryBuilder.createPhraseQuery(field, value); } - - /** - * Unescapes search value based on the Search grammar rules. - *

- * - STRING_LITERAL: Removes enclosing quotes and unescapes {@code \"} - *

- * - TERM: Unescapes {@code \=, \!, \~, \(, \)} - */ - private static String unescapeSearchValue(SearchParser.SearchValueContext ctx) { - if (ctx == null) { - return ""; - } - - String term = ctx.getText(); - - if (ctx.getStart().getType() == SearchParser.STRING_LITERAL) { - return term.substring(1, term.length() - 1) - .replace("\\\"", "\""); - } - - if (ctx.getStart().getType() == SearchParser.TERM) { - return term.replaceAll("\\\\([=!~()])", "$1"); - } - - return term; - } } diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 78d5cbd83fc..3f2df97290b 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -163,7 +163,7 @@ public SqlQueryNode visitComparisonExpression(SearchParser.ComparisonExpressionC @Override public SqlQueryNode visitComparison(SearchParser.ComparisonContext ctx) { EnumSet searchFlags = EnumSet.noneOf(SearchFlags.class); - String term = unescapeSearchValue(ctx.searchValue()); + String term = SearchQueryConversion.unescapeSearchValue(ctx.searchValue()); // unfielded expression if (ctx.FIELD() == null) { @@ -538,32 +538,6 @@ private static String getSqlOperator(EnumSet searchFlags) { : (searchFlags.contains(CASE_SENSITIVE) ? "LIKE" : "ILIKE"); } - /** - * Unescapes search value based on the Search grammar rules. - *

- * - STRING_LITERAL: Removes enclosing quotes and unescapes {@code \"} - *

- * - TERM: Unescapes {@code \=, \!, \~, \(, \)} - */ - private static String unescapeSearchValue(SearchParser.SearchValueContext ctx) { - if (ctx == null) { - return ""; - } - - String term = ctx.getText(); - - if (ctx.getStart().getType() == SearchParser.STRING_LITERAL) { - return term.substring(1, term.length() - 1) - .replace("\\\"", "\""); - } - - if (ctx.getStart().getType() == SearchParser.TERM) { - return term.replaceAll("\\\\([=!~()])", "$1"); - } - - return term; - } - /** * Escapes wildcard characters in the search term for SQL queries. *

From e7008fde4aed101df7810a3a474a97d48b37c6a9 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 22 Oct 2024 17:37:35 +0300 Subject: [PATCH 097/104] Fix tests --- .../search/query/SearchToSqlVisitor.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 3f2df97290b..564aa544ce8 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -226,24 +226,24 @@ private SqlQueryNode getFieldQueryNode(String field, String term, EnumSet Date: Tue, 22 Oct 2024 17:54:52 +0300 Subject: [PATCH 098/104] Fix DatabaseSearcherTest --- .../java/org/jabref/logic/search/DatabaseSearcherTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index 2a6b8ce9ac7..a6a57396d76 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -76,10 +76,10 @@ private static Stream databaseSearcher() { Arguments.of(List.of(), new SearchQuery("whatever"), List.of(emptyEntry, articleEntry, inCollectionEntry)), // invalid search syntax - Arguments.of(List.of(), new SearchQuery("author:"), List.of(articleEntry)), + Arguments.of(List.of(), new SearchQuery("author="), List.of(articleEntry)), Arguments.of(List.of(articleEntry), new SearchQuery("harrer"), List.of(articleEntry)), - Arguments.of(List.of(), new SearchQuery("title: harrer"), List.of(articleEntry)), + Arguments.of(List.of(), new SearchQuery("title= harrer"), List.of(articleEntry)), Arguments.of(List.of(inCollectionEntry), new SearchQuery("tonho"), List.of(inCollectionEntry)), Arguments.of(List.of(inCollectionEntry), new SearchQuery("tonho"), List.of(articleEntry, inCollectionEntry)) From e225cdcfcf44d7a9a3b9d1cda2389168f4381ab4 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 22 Oct 2024 22:54:13 +0300 Subject: [PATCH 099/104] Fix search terms patten for highlighting --- .../search/query/SearchQueryConversion.java | 2 +- .../query/SearchQueryExtractorVisitor.java | 21 ++++++++++++++++++- .../search/query/SearchToSqlVisitor.java | 13 ++++++------ .../logic/search/retrieval/Highlighter.java | 20 ++++++++---------- .../jabref/model/search/PostgreConstants.java | 4 ++-- .../SearchQueryExtractorConversionTest.java | 3 ++- 6 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java index 67ca7fdea3f..411f42d3130 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java @@ -30,7 +30,7 @@ public static Query searchToLucene(SearchQuery searchQuery) { public static Set extractSearchTerms(SearchQuery searchQuery) { LOGGER.debug("Extracting search terms from search expression: {}", searchQuery.getSearchExpression()); - return new SearchQueryExtractorVisitor().visit(searchQuery.getContext()); + return new SearchQueryExtractorVisitor(searchQuery.getSearchFlags()).visit(searchQuery.getContext()); } /** diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java index 10ee42dfb11..84a60101b32 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java @@ -1,9 +1,11 @@ package org.jabref.logic.search.query; +import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Set; +import org.jabref.model.search.SearchFlags; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; @@ -12,8 +14,13 @@ */ public class SearchQueryExtractorVisitor extends SearchBaseVisitor> { + private final boolean searchBarRegex; private boolean isNegated = false; + public SearchQueryExtractorVisitor(EnumSet searchFlags) { + searchBarRegex = searchFlags.contains(SearchFlags.REGULAR_EXPRESSION); + } + @Override public Set visitStart(SearchParser.StartContext ctx) { return visit(ctx.orExpression()); @@ -75,7 +82,19 @@ public Set visitComparison(SearchParser.ComparisonContext ctx) { return Set.of(); } } + String term = SearchQueryConversion.unescapeSearchValue(ctx.searchValue()); + + // if not regex, escape the backslashes, because the highlighter uses regex - return Set.of(SearchQueryConversion.unescapeSearchValue(ctx.searchValue())); + // unfielded terms, check the search bar flags + if (ctx.FIELD() == null && !searchBarRegex) { + return Set.of(term.replace("\\", "\\\\")); + } else if (ctx.operator() != null) { + int operator = ctx.operator().getStart().getType(); + if (operator != SearchParser.REQUAL && operator != SearchParser.CREEQUAL) { + return Set.of(term.replace("\\", "\\\\")); + } + } + return Set.of(term); } } diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index 564aa544ce8..ff2575d9c2c 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -212,9 +212,12 @@ public SqlQueryNode visitComparison(SearchParser.ComparisonContext ctx) { private SqlQueryNode getFieldQueryNode(String field, String term, EnumSet searchFlags) { String sqlOperator = getSqlOperator(searchFlags); - term = escapeTermForSql(term, searchFlags); String prefixSuffix = searchFlags.contains(INEXACT_MATCH) ? "%" : ""; + if (!searchFlags.contains(REGULAR_EXPRESSION)) { + term = escapeTermForSql(term); + } + // Pseudo-fields field = switch (field) { case "key" -> InternalField.KEY_FIELD.getName(); @@ -543,11 +546,7 @@ private static String getSqlOperator(EnumSet searchFlags) { *

* - Escapes {@code \}, {@code _}, and {@code %} for SQL LIKE queries. */ - private static String escapeTermForSql(String term, EnumSet searchFlags) { - if (searchFlags.contains(REGULAR_EXPRESSION)) { - return term; - } else { - return term.replaceAll("[\\\\_%]", "\\\\$0"); - } + private static String escapeTermForSql(String term) { + return term.replaceAll("[\\\\_%]", "\\\\$0"); } } diff --git a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java index efd04a659e1..1a52fdff8d6 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java +++ b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import org.jabref.logic.search.PostgreServer; import org.jabref.logic.search.query.SearchQueryConversion; @@ -37,14 +36,14 @@ public class Highlighter { private static Connection connection; public static String highlightHtml(String htmlText, SearchQuery searchQuery) { - Optional searchTerms = getSearchTermsPattern(searchQuery); - if (searchTerms.isEmpty()) { + Optional searchTermsPattern = getSearchTermsPattern(searchQuery); + if (searchTermsPattern.isEmpty()) { return htmlText; } Document document = Jsoup.parse(htmlText); try { - highlightTextNodes(document.body(), searchTerms.get()); + highlightTextNodes(document.body(), searchTermsPattern.get()); return document.outerHtml(); } catch (InvalidTokenOffsetsException e) { LOGGER.debug("Error highlighting search terms in HTML", e); @@ -74,26 +73,26 @@ public static List getMatchPositions(String text, String pattern) { return List.of(); } - private static void highlightTextNodes(Element element, String searchTerms) throws InvalidTokenOffsetsException { + private static void highlightTextNodes(Element element, String searchPattern) throws InvalidTokenOffsetsException { for (Node node : element.childNodes()) { if (node instanceof TextNode textNode) { - String highlightedText = highlightNode(textNode.text(), searchTerms); + String highlightedText = highlightNode(textNode.text(), searchPattern); textNode.text(""); textNode.after(highlightedText); } else if (node instanceof Element element1) { - highlightTextNodes(element1, searchTerms); + highlightTextNodes(element1, searchPattern); } } } - private static String highlightNode(String text, String searchTerms) { + private static String highlightNode(String text, String searchPattern) { if (connection == null) { connection = Injector.instantiateModelOrService(PostgreServer.class).getConnection(); } try (PreparedStatement preparedStatement = connection.prepareStatement(REGEXP_MARK)) { preparedStatement.setString(1, text); - preparedStatement.setString(2, searchTerms); + preparedStatement.setString(2, searchPattern); try (ResultSet resultSet = preparedStatement.executeQuery()) { if (resultSet.next()) { @@ -116,7 +115,6 @@ public static Optional getSearchTermsPattern(SearchQuery searchQuery) { return Optional.empty(); } - return Optional.of(terms.stream() - .collect(Collectors.joining(")|(", "(", ")"))); + return Optional.of(String.join("|", terms)); } } diff --git a/src/main/java/org/jabref/model/search/PostgreConstants.java b/src/main/java/org/jabref/model/search/PostgreConstants.java index 5cfc002d5d8..811d3e7d9af 100644 --- a/src/main/java/org/jabref/model/search/PostgreConstants.java +++ b/src/main/java/org/jabref/model/search/PostgreConstants.java @@ -19,7 +19,7 @@ CREATE OR REPLACE FUNCTION regexp_mark(string text, pattern text) AS $$ BEGIN - RETURN regexp_replace(string, pattern, '\\1', 'gi'); + RETURN regexp_replace(string, format('(%s)', pattern), '\\1', 'gi'); END $$; """, @@ -39,7 +39,7 @@ CREATE OR REPLACE FUNCTION regexp_positions(string text, pattern text, OUT start pos integer; BEGIN end_pos := 0; - FOR match IN SELECT (regexp_matches(string, pattern, 'gi'))[1] LOOP + FOR match IN SELECT (regexp_matches(string, format('(%s)', pattern), 'gi'))[1] LOOP len := length(match); pos := position(match in remainder); start_pos := pos + end_pos; diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java index 0e8fba2e4ba..32af17a08a9 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java @@ -24,7 +24,8 @@ public static Stream searchConversion() { Arguments.of(Set.of("a", "b"), "author = a AND title = b"), Arguments.of(Set.of(), "NOT a"), Arguments.of(Set.of("a", "b", "c"), "(any = a OR any = b) AND NOT (NOT c AND title = d)"), - Arguments.of(Set.of("b", "c"), "title != a OR b OR c") + Arguments.of(Set.of("b", "c"), "title != a OR b OR c"), + Arguments.of(Set.of("a", "b"), "a b") ); } From 77ddecea4ec8058ad898d1d7d2ac3c5f469132c3 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 23 Oct 2024 01:24:23 +0300 Subject: [PATCH 100/104] Highlight source tab field by field according to the search query --- .../org/jabref/gui/entryeditor/SourceTab.java | 51 ++++++++---- .../search/query/SearchQueryConversion.java | 5 +- .../query/SearchQueryExtractorVisitor.java | 79 +++++++++++------- .../logic/search/retrieval/Highlighter.java | 80 ++++++++++++------- .../model/search/query/SearchQueryNode.java | 8 ++ .../SearchQueryExtractorConversionTest.java | 31 +++---- 6 files changed, 165 insertions(+), 89 deletions(-) create mode 100644 src/main/java/org/jabref/model/search/query/SearchQueryNode.java diff --git a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java index 5cdc78cf193..629a991e0e8 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java @@ -63,6 +63,8 @@ public class SourceTab extends EntryEditorTab { private static final Logger LOGGER = LoggerFactory.getLogger(SourceTab.class); + private static final String TEXT_STYLE = "text"; + private static final String SEARCH_STYLE = "search"; private final FieldPreferences fieldPreferences; private final BibDatabaseMode mode; private final UndoManager undoManager; @@ -103,28 +105,45 @@ public SourceTab(BibDatabaseContext bibDatabaseContext, } private void highlightSearchPattern() { - if (codeArea != null) { - codeArea.setStyleClass(0, codeArea.getLength(), "text"); - if (searchQueryProperty.get().isPresent()) { - Optional searchPattern = Highlighter.getSearchTermsPattern(searchQueryProperty.get().get()); - if (searchPattern.isPresent()) { - LOGGER.debug("Highlighting search pattern {}", searchPattern.get()); - for (Range fieldPosition : fieldPositions.values()) { - int start = fieldPosition.start(); - int end = fieldPosition.end(); - List matchedPositions = Highlighter.getMatchPositions(codeArea.getText(start, end), searchPattern.get()); - for (Range range: matchedPositions) { - codeArea.setStyleClass(start + range.start() - 1, start + range.end(), "search"); - } - } - } + if (codeArea == null || searchQueryProperty.get().isEmpty()) { + return; + } + + codeArea.setStyleClass(0, codeArea.getLength(), TEXT_STYLE); + Map, List> searchTermsMap = Highlighter.groupTermsByField(searchQueryProperty.get().get()); + + searchTermsMap.forEach((optionalField, terms) -> { + Optional searchPattern = Highlighter.buildSearchPattern(terms); + if (searchPattern.isEmpty()) { + return; + } + + if (optionalField.isPresent()) { + highlightField(optionalField.get(), searchPattern.get()); + } else { + fieldPositions.keySet().forEach(field -> highlightField(field, searchPattern.get())); } + }); + } + + private void highlightField(Field field, String searchPattern) { + Range fieldPosition = fieldPositions.get(field); + if (fieldPosition == null) { + return; + } + + int start = fieldPosition.start(); + int end = fieldPosition.end(); + List matchedPositions = Highlighter.findMatchPositions(codeArea.getText(start, end), searchPattern); + + for (Range range : matchedPositions) { + codeArea.setStyleClass(start + range.start() - 1, start + range.end(), SEARCH_STYLE); } } private String getSourceString(BibEntry entry, BibDatabaseMode type, FieldPreferences fieldPreferences) throws IOException { StringWriter writer = new StringWriter(); - BibWriter bibWriter = new BibWriter(writer, "\n"); + BibWriter bibWriter = new BibWriter(writer, "\n"); // JavaFX works with LF only FieldWriter fieldWriter = FieldWriter.buildIgnoreHashes(fieldPreferences); BibEntryWriter bibEntryWriter = new BibEntryWriter(fieldWriter, entryTypesManager); bibEntryWriter.write(entry, bibWriter, type, true); diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java index 411f42d3130..9c2b8ea0ff2 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryConversion.java @@ -1,8 +1,9 @@ package org.jabref.logic.search.query; -import java.util.Set; +import java.util.List; import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.search.query.SearchQueryNode; import org.jabref.model.search.query.SqlQueryNode; import org.jabref.search.SearchParser; @@ -28,7 +29,7 @@ public static Query searchToLucene(SearchQuery searchQuery) { return new SearchToLuceneVisitor().visit(searchQuery.getContext()); } - public static Set extractSearchTerms(SearchQuery searchQuery) { + public static List extractSearchTerms(SearchQuery searchQuery) { LOGGER.debug("Extracting search terms from search expression: {}", searchQuery.getSearchExpression()); return new SearchQueryExtractorVisitor(searchQuery.getSearchFlags()).visit(searchQuery.getContext()); } diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java index 84a60101b32..041a88ef7f9 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java @@ -1,18 +1,23 @@ package org.jabref.logic.search.query; +import java.util.ArrayList; import java.util.EnumSet; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import java.util.Locale; +import java.util.Optional; +import org.jabref.model.entry.field.FieldFactory; +import org.jabref.model.entry.field.InternalField; +import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.SearchFlags; +import org.jabref.model.search.query.SearchQueryNode; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; /** * Tests are located in {@link org.jabref.logic.search.query.SearchQueryExtractorConversionTest}. */ -public class SearchQueryExtractorVisitor extends SearchBaseVisitor> { +public class SearchQueryExtractorVisitor extends SearchBaseVisitor> { private final boolean searchBarRegex; private boolean isNegated = false; @@ -22,18 +27,18 @@ public SearchQueryExtractorVisitor(EnumSet searchFlags) { } @Override - public Set visitStart(SearchParser.StartContext ctx) { + public List visitStart(SearchParser.StartContext ctx) { return visit(ctx.orExpression()); } @Override - public Set visitImplicitOrExpression(SearchParser.ImplicitOrExpressionContext ctx) { - List> children = ctx.expression().stream().map(this::visit).toList(); + public List visitImplicitOrExpression(SearchParser.ImplicitOrExpressionContext ctx) { + List> children = ctx.expression().stream().map(this::visit).toList(); if (children.size() == 1) { return children.getFirst(); } else { - Set terms = new HashSet<>(); - for (Set child : children) { + List terms = new ArrayList<>(); + for (List child : children) { terms.addAll(child); } return terms; @@ -41,45 +46,46 @@ public Set visitImplicitOrExpression(SearchParser.ImplicitOrExpressionCo } @Override - public Set visitNegatedExpression(SearchParser.NegatedExpressionContext ctx) { + public List visitNegatedExpression(SearchParser.NegatedExpressionContext ctx) { isNegated = !isNegated; - Set terms = visit(ctx.expression()); + List terms = visit(ctx.expression()); isNegated = !isNegated; return terms; } @Override - public Set visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { - Set terms = new HashSet<>(); + public List visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { + List terms = new ArrayList<>(); terms.addAll(visit(ctx.left)); terms.addAll(visit(ctx.right)); return terms; } @Override - public Set visitParenExpression(SearchParser.ParenExpressionContext ctx) { + public List visitParenExpression(SearchParser.ParenExpressionContext ctx) { return visit(ctx.orExpression()); } @Override - public Set visitComparisonExpression(SearchParser.ComparisonExpressionContext ctx) { + public List visitComparisonExpression(SearchParser.ComparisonExpressionContext ctx) { return visit(ctx.comparison()); } @Override - public Set visitComparison(SearchParser.ComparisonContext ctx) { + public List visitComparison(SearchParser.ComparisonContext ctx) { + // ignore negated comparisons if (isNegated) { - return Set.of(); + return List.of(); } if (ctx.operator() != null) { int operator = ctx.operator().getStart().getType(); if (operator == SearchParser.NEQUAL - || operator == SearchParser.NCEQUAL - || operator == SearchParser.NEEQUAL - || operator == SearchParser.NCEEQUAL - || operator == SearchParser.NREQUAL - || operator == SearchParser.NCREEQUAL) { - return Set.of(); + || operator == SearchParser.NCEQUAL + || operator == SearchParser.NEEQUAL + || operator == SearchParser.NCEEQUAL + || operator == SearchParser.NREQUAL + || operator == SearchParser.NCREEQUAL) { + return List.of(); } } String term = SearchQueryConversion.unescapeSearchValue(ctx.searchValue()); @@ -87,14 +93,33 @@ public Set visitComparison(SearchParser.ComparisonContext ctx) { // if not regex, escape the backslashes, because the highlighter uses regex // unfielded terms, check the search bar flags - if (ctx.FIELD() == null && !searchBarRegex) { - return Set.of(term.replace("\\", "\\\\")); - } else if (ctx.operator() != null) { + if (ctx.FIELD() == null) { + if (!searchBarRegex) { + term = term.replace("\\", "\\\\"); + } + return List.of(new SearchQueryNode(Optional.empty(), term)); + } + + String field = ctx.FIELD().getText().toLowerCase(Locale.ROOT); + + // Pseudo-fields + field = switch (field) { + case "key" -> InternalField.KEY_FIELD.getName(); + case "anykeyword" -> StandardField.KEYWORDS.getName(); + case "anyfield" -> "any"; + default -> field; + }; + + if (ctx.operator() != null) { int operator = ctx.operator().getStart().getType(); if (operator != SearchParser.REQUAL && operator != SearchParser.CREEQUAL) { - return Set.of(term.replace("\\", "\\\\")); + term = term.replace("\\", "\\\\"); } } - return Set.of(term); + + if ("any".equals(field)) { + return List.of(new SearchQueryNode(Optional.empty(), term)); + } + return List.of(new SearchQueryNode(Optional.of(FieldFactory.parseField(field)), term)); } } diff --git a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java index 1a52fdff8d6..092b1f4e1fe 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java +++ b/src/main/java/org/jabref/logic/search/retrieval/Highlighter.java @@ -5,14 +5,17 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; -import java.util.Set; import org.jabref.logic.search.PostgreServer; import org.jabref.logic.search.query.SearchQueryConversion; +import org.jabref.model.entry.field.Field; import org.jabref.model.search.PostgreConstants; import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.search.query.SearchQueryNode; import org.jabref.model.util.Range; import com.airhacks.afterburner.injection.Injector; @@ -36,7 +39,7 @@ public class Highlighter { private static Connection connection; public static String highlightHtml(String htmlText, SearchQuery searchQuery) { - Optional searchTermsPattern = getSearchTermsPattern(searchQuery); + Optional searchTermsPattern = buildSearchPattern(searchQuery); if (searchTermsPattern.isEmpty()) { return htmlText; } @@ -51,28 +54,6 @@ public static String highlightHtml(String htmlText, SearchQuery searchQuery) { } } - public static List getMatchPositions(String text, String pattern) { - if (connection == null) { - connection = Injector.instantiateModelOrService(PostgreServer.class).getConnection(); - } - - try (PreparedStatement preparedStatement = connection.prepareStatement(REGEXP_POSITIONS)) { - preparedStatement.setString(1, text); - preparedStatement.setString(2, pattern); - - try (ResultSet resultSet = preparedStatement.executeQuery()) { - List positions = new ArrayList<>(); - while (resultSet.next()) { - positions.add(new Range(resultSet.getInt(1), resultSet.getInt(2))); - } - return positions; - } - } catch (SQLException e) { - LOGGER.error("Error getting match positions in text", e); - } - return List.of(); - } - private static void highlightTextNodes(Element element, String searchPattern) throws InvalidTokenOffsetsException { for (Node node : element.childNodes()) { if (node instanceof TextNode textNode) { @@ -105,16 +86,57 @@ private static String highlightNode(String text, String searchPattern) { return text; } - public static Optional getSearchTermsPattern(SearchQuery searchQuery) { + public static List findMatchPositions(String text, String pattern) { + if (connection == null) { + connection = Injector.instantiateModelOrService(PostgreServer.class).getConnection(); + } + + try (PreparedStatement preparedStatement = connection.prepareStatement(REGEXP_POSITIONS)) { + preparedStatement.setString(1, text); + preparedStatement.setString(2, pattern); + + try (ResultSet resultSet = preparedStatement.executeQuery()) { + List positions = new ArrayList<>(); + while (resultSet.next()) { + positions.add(new Range(resultSet.getInt(1), resultSet.getInt(2))); + } + return positions; + } + } catch (SQLException e) { + LOGGER.error("Error getting match positions in text", e); + } + return List.of(); + } + + public static Map, List> groupTermsByField(SearchQuery searchQuery) { if (!searchQuery.isValid()) { - return Optional.empty(); + return Map.of(); + } + + List queryNodes = getSearchQueryNodes(searchQuery); + Map, List> searchTermsMap = new HashMap<>(); + for (SearchQueryNode searchTerm : queryNodes) { + searchTermsMap.computeIfAbsent(searchTerm.field(), k -> new ArrayList<>()).add(searchTerm.term()); } + return searchTermsMap; + } - Set terms = SearchQueryConversion.extractSearchTerms(searchQuery); - if (terms.isEmpty()) { + private static Optional buildSearchPattern(SearchQuery searchQuery) { + if (!searchQuery.isValid()) { return Optional.empty(); } - return Optional.of(String.join("|", terms)); + List terms = getSearchQueryNodes(searchQuery).stream() + .map(SearchQueryNode::term) + .toList(); + return buildSearchPattern(terms); + } + + private static List getSearchQueryNodes(SearchQuery searchQuery) { + return searchQuery.isValid() ? SearchQueryConversion.extractSearchTerms(searchQuery) : List.of(); + } + + public static Optional buildSearchPattern(List terms) { + return terms.isEmpty() ? Optional.empty() : Optional.of(String.join("|", terms)); } } diff --git a/src/main/java/org/jabref/model/search/query/SearchQueryNode.java b/src/main/java/org/jabref/model/search/query/SearchQueryNode.java new file mode 100644 index 00000000000..5d283110ea3 --- /dev/null +++ b/src/main/java/org/jabref/model/search/query/SearchQueryNode.java @@ -0,0 +1,8 @@ +package org.jabref.model.search.query; + +import java.util.Optional; + +import org.jabref.model.entry.field.Field; + +public record SearchQueryNode(Optional field, String term) { +} diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java index 32af17a08a9..bdd4340f583 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryExtractorConversionTest.java @@ -1,9 +1,10 @@ package org.jabref.logic.search.query; -import java.util.Set; +import java.util.List; import java.util.stream.Stream; import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.search.query.SearchQueryNode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -14,25 +15,25 @@ public class SearchQueryExtractorConversionTest { public static Stream searchConversion() { return Stream.of( - Arguments.of(Set.of("term"), "term"), - Arguments.of(Set.of("regex.*term"), "regex.*term"), - Arguments.of(Set.of("term"), "any = term"), - Arguments.of(Set.of("term"), "any CONTAINS term"), - Arguments.of(Set.of("a", "b"), "a AND b"), - Arguments.of(Set.of("a", "b", "c"), "a OR b AND c"), - Arguments.of(Set.of("a", "b"), "a OR b AND NOT c"), - Arguments.of(Set.of("a", "b"), "author = a AND title = b"), - Arguments.of(Set.of(), "NOT a"), - Arguments.of(Set.of("a", "b", "c"), "(any = a OR any = b) AND NOT (NOT c AND title = d)"), - Arguments.of(Set.of("b", "c"), "title != a OR b OR c"), - Arguments.of(Set.of("a", "b"), "a b") + Arguments.of(List.of("term"), "term"), + Arguments.of(List.of("regex.*term"), "regex.*term"), + Arguments.of(List.of("term"), "any = term"), + Arguments.of(List.of("term"), "any CONTAINS term"), + Arguments.of(List.of("a", "b"), "a AND b"), + Arguments.of(List.of("a", "b", "c"), "a OR b AND c"), + Arguments.of(List.of("a", "b"), "a OR b AND NOT c"), + Arguments.of(List.of("a", "b"), "author = a AND title = b"), + Arguments.of(List.of(), "NOT a"), + Arguments.of(List.of("a", "b", "c"), "(any = a OR any = b) AND NOT (NOT c AND title = d)"), + Arguments.of(List.of("b", "c"), "title != a OR b OR c"), + Arguments.of(List.of("a", "b"), "a b") ); } @ParameterizedTest @MethodSource - void searchConversion(Set expected, String searchExpression) { - Set result = SearchQueryConversion.extractSearchTerms(new SearchQuery(searchExpression)); + void searchConversion(List expected, String searchExpression) { + List result = SearchQueryConversion.extractSearchTerms(new SearchQuery(searchExpression)).stream().map(SearchQueryNode::term).toList(); assertEquals(expected, result); } } From 0b9eaf5ee7da146a50d1ef944874292455cfe04e Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 23 Oct 2024 01:33:43 +0300 Subject: [PATCH 101/104] Apply suggestions from code review --- .../importer/actions/SearchGroupsMigrationAction.java | 1 + .../java/org/jabref/logic/bibtex/BibEntryWriter.java | 10 ++++++---- .../search/query/SearchQueryLuceneConversionTest.java | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index 2055474f9d8..db0ab10cd65 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -30,6 +30,7 @@ public boolean isActionNecessary(ParserResult parserResult, DialogService dialog Optional currentVersion = parserResult.getMetaData().getGroupSearchSyntaxVersion(); if (currentVersion.isPresent()) { if (currentVersion.get().equals(VERSION_6_0_ALPHA)) { + // TODO: This text will only be shown after releasing 6.0-alpha and then removed dialogService.showErrorDialogAndWait("Search groups migration of " + parserResult.getPath().map(Path::toString).orElse(""), "The search groups syntax has been reverted to the old one. Please use the backup you made before migrating to 6.0-alpha."); } diff --git a/src/main/java/org/jabref/logic/bibtex/BibEntryWriter.java b/src/main/java/org/jabref/logic/bibtex/BibEntryWriter.java index 449255935eb..1b025046813 100644 --- a/src/main/java/org/jabref/logic/bibtex/BibEntryWriter.java +++ b/src/main/java/org/jabref/logic/bibtex/BibEntryWriter.java @@ -145,17 +145,19 @@ private void writeRequiredFieldsFirstRemainingFieldsSecond(BibEntry entry, BibWr private void writeEntryType(BibEntry entry, BibWriter out, BibDatabaseMode bibDatabaseMode) throws IOException { int start = out.getCurrentPosition(); TypedBibEntry typedEntry = new TypedBibEntry(entry, bibDatabaseMode); - out.write('@' + typedEntry.getTypeForDisplay() + '{'); - int end = out.getCurrentPosition() - 1; // exclude the '{' + out.write('@' + typedEntry.getTypeForDisplay()); + int end = out.getCurrentPosition(); fieldPositions.put(InternalField.TYPE_HEADER, new Range(start, end)); + out.write("{"); } private void writeKeyField(BibEntry entry, BibWriter out) throws IOException { int start = out.getCurrentPosition(); String keyField = StringUtil.shaveString(entry.getCitationKey().orElse("")); - out.writeLine(keyField + ','); - int end = out.getCurrentPosition() - 1; // exclude the ',' + out.write(keyField); + int end = out.getCurrentPosition(); fieldPositions.put(InternalField.KEY_FIELD, new Range(start, end)); + out.writeLine(","); } /** diff --git a/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java index abb6e9640ea..992cdd7dca5 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQueryLuceneConversionTest.java @@ -12,7 +12,7 @@ class SearchQueryLuceneConversionTest { - public static Stream testSearchConversion() { + public static Stream searchConversion() { return Stream.of( Arguments.of("content:term annotations:term", "term"), Arguments.of("content:term annotations:term", "any = term"), @@ -59,7 +59,7 @@ public static Stream testSearchConversion() { @ParameterizedTest @MethodSource - void testSearchConversion(String expected, String searchExpression) { + void searchConversion(String expected, String searchExpression) { String result = SearchQueryConversion.searchToLucene(new SearchQuery(searchExpression)).toString(); assertEquals(expected, result); } From 750a3203564aa82b7122bec1276f8cca755c9881 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 23 Oct 2024 11:53:58 +0300 Subject: [PATCH 102/104] Update src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java Co-authored-by: Oliver Kopp --- .../gui/importer/actions/SearchGroupsMigrationAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index db0ab10cd65..c7f85f5bd54 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -32,7 +32,7 @@ public boolean isActionNecessary(ParserResult parserResult, DialogService dialog if (currentVersion.get().equals(VERSION_6_0_ALPHA)) { // TODO: This text will only be shown after releasing 6.0-alpha and then removed dialogService.showErrorDialogAndWait("Search groups migration of " + parserResult.getPath().map(Path::toString).orElse(""), - "The search groups syntax has been reverted to the old one. Please use the backup you made before migrating to 6.0-alpha."); + "The search groups syntax has been reverted to the old one. Please use the backup you made before using 6.0-alpha."); } return false; } From ddd5392c994897b25840acee50da4f05cd572a63 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 23 Oct 2024 12:05:50 +0300 Subject: [PATCH 103/104] Set default operator to AND --- src/main/antlr4/org/jabref/search/Search.g4 | 8 ++++---- .../search/query/SearchFlagsToExpressionVisitor.java | 6 +++--- .../logic/search/query/SearchQueryExtractorVisitor.java | 6 +++--- .../jabref/logic/search/query/SearchToLuceneVisitor.java | 8 ++++---- .../org/jabref/logic/search/query/SearchToSqlVisitor.java | 8 ++++---- .../logic/search/query/SearchQuerySQLConversionTest.java | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/main/antlr4/org/jabref/search/Search.g4 b/src/main/antlr4/org/jabref/search/Search.g4 index e622ec24941..0a8c28c234c 100644 --- a/src/main/antlr4/org/jabref/search/Search.g4 +++ b/src/main/antlr4/org/jabref/search/Search.g4 @@ -41,15 +41,15 @@ TERM: ('\\' [=!~()] | ~[ \t\n\r=!~()])+; // =!~() should be escaped with a ba start : EOF - | orExpression EOF + | andExpression EOF ; -orExpression - : expression+ #implicitOrExpression // example: author = miller OR year = 2010 +andExpression + : expression+ #implicitAndExpression // example: author = miller year = 2010 --> equivalent to: author = miller AND year = 2010 ; expression - : LPAREN orExpression RPAREN #parenExpression // example: (author = miller) + : LPAREN andExpression RPAREN #parenExpression // example: (author = miller) | NOT expression #negatedExpression // example: NOT author = miller | left = expression bin_op = AND right = expression #binaryExpression // example: author = miller AND year = 2010 | left = expression bin_op = OR right = expression #binaryExpression // example: author = miller OR year = 2010 diff --git a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java index ae6ca000041..431cca878f5 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchFlagsToExpressionVisitor.java @@ -35,11 +35,11 @@ public SearchFlagsToExpressionVisitor(EnumSet searchFlags) { @Override public String visitStart(SearchParser.StartContext ctx) { - return visit(ctx.orExpression()); + return visit(ctx.andExpression()); } @Override - public String visitImplicitOrExpression(SearchParser.ImplicitOrExpressionContext ctx) { + public String visitImplicitAndExpression(SearchParser.ImplicitAndExpressionContext ctx) { List children = ctx.expression().stream().map(this::visit).toList(); if (children.size() == 1) { return children.getFirst(); @@ -49,7 +49,7 @@ public String visitImplicitOrExpression(SearchParser.ImplicitOrExpressionContext @Override public String visitParenExpression(SearchParser.ParenExpressionContext ctx) { - return "(" + visit(ctx.orExpression()) + ")"; + return "(" + visit(ctx.andExpression()) + ")"; } @Override diff --git a/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java index 041a88ef7f9..51255fa1442 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchQueryExtractorVisitor.java @@ -28,11 +28,11 @@ public SearchQueryExtractorVisitor(EnumSet searchFlags) { @Override public List visitStart(SearchParser.StartContext ctx) { - return visit(ctx.orExpression()); + return visit(ctx.andExpression()); } @Override - public List visitImplicitOrExpression(SearchParser.ImplicitOrExpressionContext ctx) { + public List visitImplicitAndExpression(SearchParser.ImplicitAndExpressionContext ctx) { List> children = ctx.expression().stream().map(this::visit).toList(); if (children.size() == 1) { return children.getFirst(); @@ -63,7 +63,7 @@ public List visitBinaryExpression(SearchParser.BinaryExpression @Override public List visitParenExpression(SearchParser.ParenExpressionContext ctx) { - return visit(ctx.orExpression()); + return visit(ctx.andExpression()); } @Override diff --git a/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java index 8cce4fbe8f8..4f38a0ed1e2 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToLuceneVisitor.java @@ -30,25 +30,25 @@ public SearchToLuceneVisitor() { @Override public Query visitStart(SearchParser.StartContext ctx) { - return visit(ctx.orExpression()); + return visit(ctx.andExpression()); } @Override - public Query visitImplicitOrExpression(SearchParser.ImplicitOrExpressionContext ctx) { + public Query visitImplicitAndExpression(SearchParser.ImplicitAndExpressionContext ctx) { List children = ctx.expression().stream().map(this::visit).toList(); if (children.size() == 1) { return children.getFirst(); } BooleanQuery.Builder builder = new BooleanQuery.Builder(); for (Query child : children) { - builder.add(child, BooleanClause.Occur.SHOULD); + builder.add(child, BooleanClause.Occur.MUST); } return builder.build(); } @Override public Query visitParenExpression(SearchParser.ParenExpressionContext ctx) { - return visit(ctx.orExpression()); + return visit(ctx.andExpression()); } @Override diff --git a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java index ff2575d9c2c..e204ea9de78 100644 --- a/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java +++ b/src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java @@ -52,7 +52,7 @@ public SearchToSqlVisitor(String table, EnumSet searchBarFlags) { @Override public SqlQueryNode visitStart(SearchParser.StartContext ctx) { - SqlQueryNode finalNode = visit(ctx.orExpression()); + SqlQueryNode finalNode = visit(ctx.andExpression()); StringBuilder sql = new StringBuilder("WITH\n"); List params = new ArrayList<>(); @@ -74,7 +74,7 @@ public SqlQueryNode visitStart(SearchParser.StartContext ctx) { } @Override - public SqlQueryNode visitImplicitOrExpression(SearchParser.ImplicitOrExpressionContext ctx) { + public SqlQueryNode visitImplicitAndExpression(SearchParser.ImplicitAndExpressionContext ctx) { List children = ctx.expression().stream().map(this::visit).toList(); if (children.size() == 1) { @@ -86,7 +86,7 @@ public SqlQueryNode visitImplicitOrExpression(SearchParser.ImplicitOrExpressionC ) """.formatted( cteCounter, - children.stream().map(node -> " SELECT %s FROM %s".formatted(ENTRY_ID, node.cte())).collect(Collectors.joining("\n UNION\n"))); + children.stream().map(node -> " SELECT %s FROM %s".formatted(ENTRY_ID, node.cte())).collect(Collectors.joining("\n INTERSECT\n"))); List params = children.stream().flatMap(node -> node.params().stream()).toList(); SqlQueryNode node = new SqlQueryNode(cte, params); @@ -97,7 +97,7 @@ public SqlQueryNode visitImplicitOrExpression(SearchParser.ImplicitOrExpressionC @Override public SqlQueryNode visitParenExpression(SearchParser.ParenExpressionContext ctx) { - return visit(ctx.orExpression()); + return visit(ctx.andExpression()); } @Override diff --git a/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java index f4f6c93d96d..d207d25e2e4 100644 --- a/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java +++ b/src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java @@ -374,9 +374,9 @@ cte2 AS ( , cte3 AS ( SELECT entry_id FROM cte0 - UNION + INTERSECT SELECT entry_id FROM cte1 - UNION + INTERSECT SELECT entry_id FROM cte2 ) SELECT * FROM cte3 GROUP BY entry_id""" From 3cfeb5dccdf1fe12c536a34f2e7b9086d31dafa8 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 23 Oct 2024 13:50:59 +0300 Subject: [PATCH 104/104] remove debug --- src/main/resources/tinylog.properties | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/resources/tinylog.properties b/src/main/resources/tinylog.properties index 7e9f20f3165..18da98de7c7 100644 --- a/src/main/resources/tinylog.properties +++ b/src/main/resources/tinylog.properties @@ -16,8 +16,6 @@ level@io.zonky.test.db.postgres.embedded = warn #level@org.jabref.gui.JabRefGUI = debug -level@org.jabref.logic.search.query.SearchToSqlVisitor = trace - # AI debugging #level@ai.djl = debug #level@org.jabref.gui.entryeditor.aichattab.AiChat = trace @@ -28,5 +26,3 @@ level@org.jabref.logic.search.query.SearchToSqlVisitor = trace #level@org.jabref.logic.ai.FileEmbeddingsManager = trace #level@org.jabref.logic.ai.impl.embeddings.LowLevelIngestor = trace #level@org.jabref.logic.ai.chat.AiChatLogic = trace -level@org.jabref.logic.search = debug -level@org.jabref.model.search = debug