Searching with Postgres

build.gradle

@@ -355,6 +355,9 @@ dependencies {
     // Even if "compileOnly" is used, IntelliJ always adds to module-info.java. To avoid issues during committing, we use "implementation" instead of "compileOnly"
     implementation 'io.github.adr:e-adr:2.0.0-SNAPSHOT'
 
+    implementation 'io.zonky.test:embedded-postgres:2.0.7'
+    implementation enforcedPlatform('io.zonky.test.postgres:embedded-postgres-binaries-bom:16.4.0')
+
     testImplementation 'io.github.classgraph:classgraph:4.8.175'
     testImplementation 'org.junit.jupiter:junit-jupiter:5.11.0'
     testImplementation 'org.junit.platform:junit-platform-launcher:1.10.3'

external-libraries.md

@@ -356,6 +356,13 @@ URL:     https://github.com/java-diff-utils/java-diff-utils
 License: Apache-2.0
 ```
 
+```yaml
+Id:      io.zonky.test:embedded-postgres
+Project: embedded-postgres
+URL:     https://github.com/zonkyio/embedded-postgres
+License: Apache-2.0
+```
+
 ```yaml
 Id:      jakarta.annotation:jakarata.annotation-api
 Project: Jakarta Annotations
@@ -699,8 +706,12 @@ License: BSD-3-Clause
 3. (on WSL) `sed 's/[^a-z]*//' < build/reports/project/dependencies.txt | sed "s/\(.*\) .*/\1/" | grep -v "\->" | sort | uniq > build/dependencies-for-external-libraries.txt`
 
 ```text
+ai.djl.huggingface:tokenizers:0.29.0
+ai.djl.pytorch:pytorch-engine:0.30.0
+ai.djl.pytorch:pytorch-model-zoo:0.30.0
+ai.djl:api:0.30.0
 at.favre.lib:hkdf:1.1.0
-com.dlsc.gemsfx:gemsfx:2.32.0
+com.dlsc.gemsfx:gemsfx:2.48.0
 com.dlsc.pickerfx:pickerfx:1.3.1
 com.dlsc.unitfx:unitfx:1.0.10
 com.fasterxml.jackson.core:jackson-annotations:2.17.2
@@ -717,29 +728,37 @@ com.github.sialcasa.mvvmFX:mvvmfx-validation:f195849ca9
 com.github.tomtung:latex2unicode_2.13:0.3.2
 com.github.vatbub:mslinks:1.0.6.2
 com.github.weisj:jsvg:1.2.0
-com.google.code.gson:gson:2.10.1
-com.google.errorprone:error_prone_annotations:2.26.1
+com.google.code.gson:gson:2.11.0
+com.google.errorprone:error_prone_annotations:2.27.0
 com.google.guava:failureaccess:1.0.2
 com.google.guava:guava:33.1.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:3.0.0
 com.googlecode.javaewah:JavaEWAH:1.2.3
 com.googlecode.plist:dd-plist:1.28
-com.h2database:h2-mvstore:2.2.224
-com.kohlschutter.junixsocket:junixsocket-common:2.9.1
-com.kohlschutter.junixsocket:junixsocket-core:2.9.1
-com.kohlschutter.junixsocket:junixsocket-mysql:2.9.1
-com.kohlschutter.junixsocket:junixsocket-native-common:2.9.1
-com.konghq:unirest-java-core:4.4.0
-com.konghq:unirest-modules-gson:4.4.0
+com.h2database:h2-mvstore:2.3.232
+com.knuddels:jtokkit:1.1.0
+com.kohlschutter.junixsocket:junixsocket-common:2.10.0
+com.kohlschutter.junixsocket:junixsocket-core:2.10.0
+com.kohlschutter.junixsocket:junixsocket-mysql:2.10.0
+com.kohlschutter.junixsocket:junixsocket-native-common:2.10.0
+com.konghq:unirest-java-core:4.4.4
+com.konghq:unirest-modules-gson:4.4.4
 com.oracle.ojdbc:ojdbc10:19.3.0.0
 com.oracle.ojdbc:ons:19.3.0.0
 com.oracle.ojdbc:osdt_cert:19.3.0.0
 com.oracle.ojdbc:osdt_core:19.3.0.0
 com.oracle.ojdbc:simplefan:19.3.0.0
 com.oracle.ojdbc:ucp:19.3.0.0
+com.squareup.okhttp3:logging-interceptor:4.12.0
+com.squareup.okhttp3:okhttp-sse:4.12.0
+com.squareup.okhttp3:okhttp:4.12.0
+com.squareup.okio:okio-jvm:3.6.0
+com.squareup.okio:okio:3.6.0
+com.squareup.retrofit2:converter-gson:2.9.0
+com.squareup.retrofit2:converter-jackson:2.9.0
+com.squareup.retrofit2:retrofit:2.11.0
 com.sun.istack:istack-commons-runtime:4.1.2
-com.tobiasdiez:easybind:2.2.1-SNAPSHOT
 com.vladsch.flexmark:flexmark-ext-emoji:0.64.8
 com.vladsch.flexmark:flexmark-ext-gfm-strikethrough:0.64.8
 com.vladsch.flexmark:flexmark-ext-ins:0.64.8
@@ -762,12 +781,12 @@ com.vladsch.flexmark:flexmark-util-visitor:0.64.8
 com.vladsch.flexmark:flexmark-util:0.64.8
 com.vladsch.flexmark:flexmark:0.64.8
 commons-beanutils:commons-beanutils:1.9.4
-commons-cli:commons-cli:1.8.0
-commons-codec:commons-codec:1.17.0
+commons-cli:commons-cli:1.9.0
+commons-codec:commons-codec:1.17.1
 commons-collections:commons-collections:3.2.2
 commons-digester:commons-digester:2.1
 commons-io:commons-io:2.16.1
-commons-logging:commons-logging:1.3.2
+commons-logging:commons-logging:1.3.4
 commons-validator:commons-validator:1.8.0
 de.rototor.jeuclid:jeuclid-core:3.1.11
 de.rototor.snuggletex:snuggletex-core:1.3.0
@@ -776,9 +795,20 @@ de.rototor.snuggletex:snuggletex:1.3.0
 de.saxsys:mvvmfx:1.8.0
 de.swiesend:secret-service:1.8.1-jdk17
 de.undercouch:citeproc-java:3.1.0
+dev.ai4j:openai4j:0.20.0
+dev.langchain4j:langchain4j-core:0.34.0
+dev.langchain4j:langchain4j-google-ai-gemini:0.34.0
+dev.langchain4j:langchain4j-hugging-face:0.34.0
+dev.langchain4j:langchain4j-mistral-ai:0.34.0
+dev.langchain4j:langchain4j-open-ai:0.34.0
+dev.langchain4j:langchain4j:0.34.0
 eu.lestard:doc-annotations:0.2
 info.debatty:java-string-similarity:2.0.0
+io.github.adr:e-adr:2.0.0-SNAPSHOT
 io.github.java-diff-utils:java-diff-utils:4.12
+io.github.stefanbratanov:jvm-openai:0.11.0
+io.github.thibaultmeyer:cuid:2.0.3
+io.zonky.test:embedded-postgres:2.0.7
 jakarta.activation:jakarta.activation-api:2.1.3
 jakarta.annotation:jakarta.annotation-api:2.1.1
 jakarta.inject:jakarta.inject-api:2.0.1
@@ -788,30 +818,32 @@ jakarta.xml.bind:jakarta.xml.bind-api:4.0.2
 javax.measure:unit-api:2.2
 net.harawata:appdirs:1.2.2
 net.java.dev.jna:jna-platform:5.13.0
-net.java.dev.jna:jna:5.13.0
+net.java.dev.jna:jna:5.14.0
 net.jcip:jcip-annotations:1.0
 net.jodah:typetools:0.6.1
 net.synedra:validatorfx:0.5.0
 one.jpro.jproutils:tree-showing:0.2.2
-org.antlr:antlr4-runtime:4.13.1
+org.antlr:antlr4-runtime:4.13.2
+org.apache.commons:commons-compress:1.27.1
 org.apache.commons:commons-csv:1.11.0
-org.apache.commons:commons-lang3:3.14.0
+org.apache.commons:commons-lang3:3.17.0
 org.apache.commons:commons-text:1.12.0
 org.apache.httpcomponents.client5:httpclient5:5.3.1
 org.apache.httpcomponents.core5:httpcore5-h2:5.2.4
 org.apache.httpcomponents.core5:httpcore5:5.2.4
-org.apache.logging.log4j:log4j-api:2.23.1
-org.apache.logging.log4j:log4j-to-slf4j:2.23.1
+org.apache.logging.log4j:log4j-api:2.24.0
+org.apache.logging.log4j:log4j-to-slf4j:2.24.0
 org.apache.lucene:lucene-analysis-common:9.11.1
 org.apache.lucene:lucene-core:9.11.1
 org.apache.lucene:lucene-highlighter:9.11.1
 org.apache.lucene:lucene-queries:9.11.1
 org.apache.lucene:lucene-queryparser:9.11.1
 org.apache.lucene:lucene-sandbox:9.11.1
-org.apache.pdfbox:fontbox:3.0.2
-org.apache.pdfbox:pdfbox-io:3.0.2
-org.apache.pdfbox:pdfbox:3.0.2
-org.apache.pdfbox:xmpbox:3.0.2
+org.apache.opennlp:opennlp-tools:1.9.4
+org.apache.pdfbox:fontbox:3.0.3
+org.apache.pdfbox:pdfbox-io:3.0.3
+org.apache.pdfbox:pdfbox:3.0.3
+org.apache.pdfbox:xmpbox:3.0.3
 org.apiguardian:apiguardian-api:1.1.2
 org.bouncycastle:bcprov-jdk18on:1.78.1
 org.checkerframework:checker-qual:3.42.0
@@ -833,18 +865,23 @@ org.glassfish.hk2:osgi-resource-locator:1.0.3
 org.glassfish.jaxb:jaxb-core:4.0.3
 org.glassfish.jaxb:jaxb-runtime:4.0.3
 org.glassfish.jaxb:txw2:4.0.3
-org.glassfish.jersey.containers:jersey-container-grizzly2-http:3.1.7
-org.glassfish.jersey.core:jersey-client:3.1.7
-org.glassfish.jersey.core:jersey-common:3.1.7
-org.glassfish.jersey.core:jersey-server:3.1.7
-org.glassfish.jersey.inject:jersey-hk2:3.1.7
+org.glassfish.jersey.containers:jersey-container-grizzly2-http:3.1.8
+org.glassfish.jersey.core:jersey-client:3.1.8
+org.glassfish.jersey.core:jersey-common:3.1.8
+org.glassfish.jersey.core:jersey-server:3.1.8
+org.glassfish.jersey.inject:jersey-hk2:3.1.8
 org.jabref:afterburner.fx:2.0.0
+org.jabref:easybind:2.2.1-SNAPSHOT
 org.javassist:javassist:3.30.2-GA
 org.jbibtex:jbibtex:1.0.20
+org.jetbrains.kotlin:kotlin-stdlib-common:2.0.20
+org.jetbrains.kotlin:kotlin-stdlib-jdk7:2.0.20
+org.jetbrains.kotlin:kotlin-stdlib-jdk8:2.0.20
+org.jetbrains.kotlin:kotlin-stdlib:2.0.20
 org.jetbrains:annotations:24.0.1
 org.jooq:jool:0.9.15
-org.jsoup:jsoup:1.17.2
-org.jspecify:jspecify:0.3.0
+org.jsoup:jsoup:1.18.1
+org.jspecify:jspecify:1.0.0
 org.kordamp.ikonli:ikonli-bootstrapicons-pack:12.3.1
 org.kordamp.ikonli:ikonli-core:12.3.1
 org.kordamp.ikonli:ikonli-javafx:12.3.1
@@ -854,22 +891,23 @@ org.kordamp.ikonli:ikonli-materialdesign2-pack:12.3.1
 org.libreoffice:libreoffice:24.2.3
 org.libreoffice:unoloader:24.2.3
 org.mariadb.jdbc:mariadb-java-client:2.7.9
-org.openjfx:javafx-base:22.0.1
-org.openjfx:javafx-controls:22.0.1
-org.openjfx:javafx-fxml:22.0.1
-org.openjfx:javafx-graphics:22.0.1
-org.openjfx:javafx-media:22.0.1
-org.openjfx:javafx-swing:22.0.1
-org.openjfx:javafx-web:22.0.1
-org.postgresql:postgresql:42.7.3
+org.openjfx:javafx-base:23
+org.openjfx:javafx-controls:23
+org.openjfx:javafx-fxml:23
+org.openjfx:javafx-graphics:23
+org.openjfx:javafx-media:23
+org.openjfx:javafx-swing:23
+org.openjfx:javafx-web:23
+org.postgresql:postgresql:42.7.4
 org.reactfx:reactfx:2.0-M5
 org.scala-lang:scala-library:2.13.8
-org.slf4j:jul-to-slf4j:2.0.13
-org.slf4j:slf4j-api:2.0.13
+org.slf4j:jul-to-slf4j:2.0.16
+org.slf4j:slf4j-api:2.0.16
 org.tinylog:slf4j-tinylog:2.7.0
 org.tinylog:tinylog-api:2.7.0
 org.tinylog:tinylog-impl:2.7.0
-org.yaml:snakeyaml:2.2
+org.tukaani:xz:1.9
+org.yaml:snakeyaml:2.3
 pt.davidafsilva.apple:jkeychain:1.1.0
 tech.units:indriya:2.2
 tech.uom.lib:uom-lib-common:2.2

src/main/java/module-info.java

@@ -97,6 +97,8 @@
     // endregion
 
     // region: SQL databases
+    requires embedded.postgres;
+    requires org.tukaani.xz;
     requires ojdbc10;
     requires org.postgresql.jdbc;
     requires org.mariadb.jdbc;
@@ -108,6 +110,8 @@
     requires io.github.javadiffutils;
     requires java.string.similarity;
     requires org.apache.commons.cli;
+    requires org.apache.commons.codec;
+    requires org.apache.commons.compress;
     requires org.apache.commons.csv;
     requires org.apache.commons.io;
     requires org.apache.commons.lang3;

src/main/java/org/jabref/Launcher.java

@@ -30,6 +30,7 @@
 import org.jabref.logic.protectedterms.ProtectedTermsLoader;
 import org.jabref.logic.remote.RemotePreferences;
 import org.jabref.logic.remote.client.RemoteClient;
+import org.jabref.logic.search.PostgreServer;
 import org.jabref.logic.util.BuildInfo;
 import org.jabref.logic.util.Directories;
 import org.jabref.logic.util.HeadlessExecutorService;
@@ -92,6 +93,9 @@ public static void main(String[] args) {
                 DirectoryMonitor directoryMonitor = new DefaultDirectoryMonitor();
                 Injector.setModelOrService(DirectoryMonitor.class, directoryMonitor);
 
+                PostgreServer postgreServer = new PostgreServer();
+                Injector.setModelOrService(PostgreServer.class, postgreServer);
+
                 // Process arguments
                 ArgumentProcessor argumentProcessor = new ArgumentProcessor(
                         args,

src/main/java/org/jabref/cli/ArgumentProcessor.java

@@ -467,7 +467,7 @@ private boolean exportMatches(List<ParserResult> loaded) {
 
         List<BibEntry> matches;
         try {
-            // extract current thread task executor from luceneManager
+            // extract current thread task executor from indexManager
             matches = new DatabaseSearcher(query, databaseContext, new CurrentThreadTaskExecutor(), cliPreferences.getFilePreferences()).getMatches();
         } catch (IOException e) {
             LOGGER.error("Error occurred when searching", e);

src/main/java/org/jabref/gui/JabRefGUI.java

@@ -32,6 +32,7 @@
 import org.jabref.logic.net.ProxyRegisterer;
 import org.jabref.logic.remote.RemotePreferences;
 import org.jabref.logic.remote.server.RemoteListenerServerManager;
+import org.jabref.logic.search.PostgreServer;
 import org.jabref.logic.util.BuildInfo;
 import org.jabref.logic.util.FallbackExceptionHandler;
 import org.jabref.logic.util.HeadlessExecutorService;
@@ -398,6 +399,9 @@ public static void shutdownThreadPools() {
         LOGGER.trace("Shutting down directoryMonitor");
         DirectoryMonitor directoryMonitor = Injector.instantiateModelOrService(DirectoryMonitor.class);
         directoryMonitor.shutdown();
+        LOGGER.trace("Shutting down postgreServer");
+        PostgreServer postgreServer = Injector.instantiateModelOrService(PostgreServer.class);
+        postgreServer.shutdown();
         LOGGER.trace("Shutting down HeadlessExecutorService");
         HeadlessExecutorService.INSTANCE.shutdownEverything();
         LOGGER.trace("Finished shutdownThreadPools");

src/main/java/org/jabref/gui/LibraryTab.java

@@ -72,7 +72,7 @@
 import org.jabref.logic.journals.JournalAbbreviationRepository;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.logic.pdf.FileAnnotationCache;
-import org.jabref.logic.search.LuceneManager;
+import org.jabref.logic.search.IndexManager;
 import org.jabref.logic.shared.DatabaseLocation;
 import org.jabref.logic.util.BackgroundTask;
 import org.jabref.logic.util.TaskExecutor;
@@ -92,7 +92,6 @@
 import org.jabref.model.entry.event.FieldChangedEvent;
 import org.jabref.model.entry.field.Field;
 import org.jabref.model.entry.field.FieldFactory;
-import org.jabref.model.entry.field.StandardField;
 import org.jabref.model.groups.GroupTreeNode;
 import org.jabref.model.search.SearchQuery;
 import org.jabref.model.util.DirectoryMonitor;
@@ -167,7 +166,7 @@ private enum PanelMode { MAIN_TABLE, MAIN_TABLE_AND_ENTRY_EDITOR }
     private final DirectoryMonitorManager directoryMonitorManager;
 
     private ImportHandler importHandler;
-    private LuceneManager luceneManager;
+    private IndexManager indexManager;
 
     /**
      * @param isDummyContext Indicates whether the database context is a dummy. A dummy context is used to display a progress indicator while parsing the database.
@@ -210,7 +209,7 @@ private LibraryTab(BibDatabaseContext bibDatabaseContext,
 
     private void initializeComponentsAndListeners(boolean isDummyContext) {
         if (!isDummyContext) {
-            createLuceneManager();
+            createIndexManager();
         }
 
         if (tableModel != null) {
@@ -221,7 +220,7 @@ private void initializeComponentsAndListeners(boolean isDummyContext) {
         bibDatabaseContext.getMetaData().registerListener(this);
 
         this.selectedGroupsProperty = new SimpleListProperty<>(stateManager.getSelectedGroups(bibDatabaseContext));
-        this.tableModel = new MainTableDataModel(getBibDatabaseContext(), preferences, taskExecutor, stateManager, getLuceneManager(), selectedGroupsProperty(), searchQueryProperty(), resultSizeProperty());
+        this.tableModel = new MainTableDataModel(getBibDatabaseContext(), preferences, taskExecutor, getIndexManager(), selectedGroupsProperty(), searchQueryProperty(), resultSizeProperty());
 
         new CitationStyleCache(bibDatabaseContext);
         annotationCache = new FileAnnotationCache(bibDatabaseContext, preferences.getFilePreferences());
@@ -315,17 +314,17 @@ private void onDatabaseLoadingSucceed(ParserResult result) {
         dataLoadingTask = null;
     }
 
-    public void createLuceneManager() {
-        luceneManager = new LuceneManager(bibDatabaseContext, taskExecutor, preferences.getFilePreferences());
-        stateManager.setLuceneManager(bibDatabaseContext, luceneManager);
+    public void createIndexManager() {
+        indexManager = new IndexManager(bibDatabaseContext, taskExecutor, preferences.getFilePreferences());
+        stateManager.setIndexManager(bibDatabaseContext, indexManager);
     }
 
-    public LuceneManager getLuceneManager() {
-        return luceneManager;
+    public IndexManager getIndexManager() {
+        return indexManager;
     }
 
-    public void closeLuceneManger() {
-        luceneManager.close();
+    public void closeIndexManger() {
+        indexManager.close();
     }
 
     private void onDatabaseLoadingFailed(Exception ex) {
@@ -787,11 +786,11 @@ private void onClosed(Event event) {
             LOGGER.error("Problem when closing directory monitor", e);
         }
         try {
-            if (luceneManager != null) {
-                luceneManager.close();
+            if (indexManager != null) {
+                indexManager.close();
             }
         } catch (RuntimeException e) {
-            LOGGER.error("Problem when closing lucene indexer", e);
+            LOGGER.error("Problem when closing index manager", e);
         }
         try {
             AutosaveManager.shutdown(bibDatabaseContext);
@@ -1140,17 +1139,17 @@ private class IndexUpdateListener {
 
         @Subscribe
         public void listen(EntriesAddedEvent addedEntryEvent) {
-            luceneManager.addToIndex(addedEntryEvent.getBibEntries());
+            indexManager.addToIndex(addedEntryEvent.getBibEntries());
         }
 
         @Subscribe
         public void listen(EntriesRemovedEvent removedEntriesEvent) {
-            luceneManager.removeFromIndex(removedEntriesEvent.getBibEntries());
+            indexManager.removeFromIndex(removedEntriesEvent.getBibEntries());
         }
 
         @Subscribe
         public void listen(FieldChangedEvent fieldChangedEvent) {
-            luceneManager.updateEntry(fieldChangedEvent.getBibEntry(), fieldChangedEvent.getOldValue(), fieldChangedEvent.getNewValue(), fieldChangedEvent.getField().equals(StandardField.FILE));
+            indexManager.updateEntry(fieldChangedEvent);
         }
     }