Integrates fetching citation counts online (Google Scholar, Semantic Scholar and OpenCitations), integrates a general structure for fechting reference metadata and integrates a general websocket server for JabRef

CHANGELOG.md

@@ -28,6 +28,8 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 - We reintroduced the index column. [#5844](https://github.com/JabRef/jabref/pull/5844)
 - Filenames of external files can no longer contain curly braces. [#5926](https://github.com/JabRef/jabref/pull/5926)
 - We made the filters more easily accessible in the integrity check dialog. [#5955](https://github.com/JabRef/jabref/pull/5955)
+- We reimplemented and improved the dialog "Customize entry types" [#4719](https://github.com/JabRef/jabref/issues/4719)
+- We integrated fetching and updating citation counts online from Google Scholar, Semantic Scholar and OpenCitations. We integrated a general websocket server which currently allows bidirectional communication with the JabRef-Browser-Extension for fetching citation counts. The latest version of the JabRef-Browser-Extension (JabFox) is needed for that. [#5943](https://github.com/JabRef/jabref/pull/5943)
 - We reimplemented and improved the dialog "Customize entry types". [#4719](https://github.com/JabRef/jabref/issues/4719)
 
 ### Fixed

build.gradle

@@ -191,6 +191,8 @@ dependencies {
         exclude module: 'fastparse_2.12'
     }
 
+    implementation group: 'org.java-websocket', name: 'Java-WebSocket', version: '1.4.0'
+
     implementation group: 'com.microsoft.azure', name: 'applicationinsights-core', version: '2.4.1'
     implementation (group: 'com.microsoft.azure', name: 'applicationinsights-logging-log4j2', version: '2.4.1') {
         exclude module: "log4j-core"

src/main/java/module-info.java

@@ -82,4 +82,5 @@
     requires org.antlr.antlr4.runtime;
     requires flowless;
     requires org.apache.tika.core;
+    requires Java.WebSocket;
 }

src/main/java/org/jabref/Globals.java

@@ -23,6 +23,7 @@
 import org.jabref.model.entry.BibEntryTypesManager;
 import org.jabref.model.util.FileUpdateMonitor;
 import org.jabref.preferences.JabRefPreferences;
+import org.jabref.websocket.JabRefWebsocketServer;
 
 import com.google.common.base.StandardSystemProperty;
 import com.microsoft.applicationinsights.TelemetryClient;
@@ -98,6 +99,8 @@ public static void startBackgroundTasks() throws JabRefException {
         if (Globals.prefs.shouldCollectTelemetry() && !GraphicsEnvironment.isHeadless()) {
             startTelemetryClient();
         }
+
+        JabRefWebsocketServer.getInstance().startServer();
     }
 
     private static void stopTelemetryClient() {
@@ -135,6 +138,7 @@ public static void shutdownThreadPools() {
     public static void stopBackgroundTasks() {
         stopTelemetryClient();
         Unirest.shutDown();
+        JabRefWebsocketServer.getInstance().stopServer();
     }
 
     public static Optional<TelemetryClient> getTelemetryClient() {

src/main/java/org/jabref/gui/JabRefFrame.java

@@ -107,6 +107,7 @@
 import org.jabref.gui.protectedterms.ManageProtectedTermsAction;
 import org.jabref.gui.push.PushToApplicationAction;
 import org.jabref.gui.push.PushToApplicationsManager;
+import org.jabref.gui.referencemetadata.ReferenceMetadataFetcherAction;
 import org.jabref.gui.search.GlobalSearchBar;
 import org.jabref.gui.shared.ConnectToSharedDatabaseCommand;
 import org.jabref.gui.shared.PullChangesFromSharedAction;
@@ -822,6 +823,8 @@ private MenuBar createMenu() {
                 lookupIdentifiers,
                 factory.createMenuItem(StandardActions.DOWNLOAD_FULL_TEXT, new DownloadFullTextAction(dialogService, stateManager, prefs)),
 
+                factory.createMenuItem(StandardActions.FETCH_REFERENCE_METADATA, new ReferenceMetadataFetcherAction(this, prefs, stateManager, undoManager, Globals.TASK_EXECUTOR)),
+
                 new SeparatorMenuItem(),
 
                 factory.createMenuItem(StandardActions.SEND_AS_EMAIL, new SendAsEMailAction(dialogService, stateManager)),

src/main/java/org/jabref/gui/actions/StandardActions.java

@@ -134,6 +134,7 @@ public enum StandardActions implements Action {
     FIND_UNLINKED_FILES(Localization.lang("Search for unlinked local files"), IconTheme.JabRefIcons.SEARCH, KeyBinding.FIND_UNLINKED_FILES),
     AUTO_LINK_FILES(Localization.lang("Automatically set file links"), IconTheme.JabRefIcons.AUTO_FILE_LINK, KeyBinding.AUTOMATICALLY_LINK_FILES),
     LOOKUP_DOC_IDENTIFIER(Localization.lang("Search document identifier online")),
+    FETCH_REFERENCE_METADATA(Localization.lang("Fetch reference metadata online"), IconTheme.JabRefIcons.DOWNLOAD, KeyBinding.FETCH_REFERENCE_METADATA),
     LOOKUP_FULLTEXT(Localization.lang("Search full text documents online"), IconTheme.JabRefIcons.FILE_SEARCH, KeyBinding.DOWNLOAD_FULL_TEXT),
     GENERATE_CITE_KEY(Localization.lang("Generate BibTeX key"), IconTheme.JabRefIcons.MAKE_KEY, KeyBinding.AUTOGENERATE_BIBTEX_KEYS),
     GENERATE_CITE_KEYS(Localization.lang("Generate BibTeX keys"), IconTheme.JabRefIcons.MAKE_KEY, KeyBinding.AUTOGENERATE_BIBTEX_KEYS),

src/main/java/org/jabref/gui/keyboard/KeyBinding.java

@@ -27,6 +27,7 @@ public enum KeyBinding {
     DELETE_ENTRY("Delete entry", Localization.lang("Delete entry"), "DELETE", KeyBindingCategory.BIBTEX),
     DEFAULT_DIALOG_ACTION("Execute default action in dialog", Localization.lang("Execute default action in dialog"), "ctrl+ENTER", KeyBindingCategory.VIEW),
     DOWNLOAD_FULL_TEXT("Download full text documents", Localization.lang("Download full text documents"), "alt+F7", KeyBindingCategory.QUALITY),
+    FETCH_REFERENCE_METADATA("Fetch reference metadata online", Localization.lang("Fetch reference metadata online"), "ctrl+alt+M", KeyBindingCategory.TOOLS),
     EDIT_ENTRY("Edit entry", Localization.lang("Edit entry"), "ctrl+E", KeyBindingCategory.BIBTEX),
     EXPORT("Export", Localization.lang("Export"), "ctrl+alt+e", KeyBindingCategory.FILE),
     EXPORT_SELECTED("Export Selected", Localization.lang("Export selected entries"), "ctrl+shift+e", KeyBindingCategory.FILE),

src/main/java/org/jabref/gui/preferences/TableColumnsTabViewModel.java

@@ -123,6 +123,7 @@ public void setValues() {
                 new MainTableColumnModel(MainTableColumnModel.Type.NORMALFIELD, InternalField.OWNER.getName()),
                 new MainTableColumnModel(MainTableColumnModel.Type.NORMALFIELD, InternalField.GROUPS.getName()),
                 new MainTableColumnModel(MainTableColumnModel.Type.NORMALFIELD, InternalField.KEY_FIELD.getName()),
+                new MainTableColumnModel(MainTableColumnModel.Type.NORMALFIELD, InternalField.CITATION_COUNT.getName()),
                 new MainTableColumnModel(MainTableColumnModel.Type.NORMALFIELD, InternalField.TYPE_HEADER.getName())
         );
 

src/main/java/org/jabref/gui/referencemetadata/ExtendedTask.java

@@ -0,0 +1,31 @@
+package org.jabref.gui.referencemetadata;
+
+import javafx.concurrent.Task;
+
+public abstract class ExtendedTask<V> extends Task<V> {
+
+    @Override
+    public void updateProgress(long workDone, long max) {
+        super.updateProgress(workDone, max);
+    }
+
+    @Override
+    public void updateProgress(double workDone, double max) {
+        super.updateProgress(workDone, max);
+    }
+
+    @Override
+    public void updateMessage(String message) {
+        super.updateMessage(message);
+    }
+
+    @Override
+    public void updateTitle(String title) {
+        super.updateTitle(title);
+    }
+
+    @Override
+    public void updateValue(V value) {
+        super.updateValue(value);
+    }
+}

src/main/java/org/jabref/gui/referencemetadata/IncompleteItem.java

@@ -0,0 +1,39 @@
+package org.jabref.gui.referencemetadata;
+
+import com.google.gson.JsonArray;
+
+public class IncompleteItem {
+    String key;
+    String title;
+    JsonArray authors;
+
+    public IncompleteItem(String key, String title, JsonArray authors) {
+        this.key = key;
+        this.title = title;
+        this.authors = authors;
+    }
+
+    public String getKey() {
+        return key;
+    }
+
+    public void setKey(String key) {
+        this.key = key;
+    }
+
+    public String getTitle() {
+        return title;
+    }
+
+    public void setTitle(String title) {
+        this.title = title;
+    }
+
+    public JsonArray getAuthors() {
+        return authors;
+    }
+
+    public void setAuthors(JsonArray authors) {
+        this.authors = authors;
+    }
+}

src/main/java/org/jabref/gui/referencemetadata/ReferenceMetadataFetcherAction.java

@@ -0,0 +1,126 @@
+package org.jabref.gui.referencemetadata;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import javax.swing.undo.UndoManager;
+
+import javafx.collections.ObservableList;
+
+import org.jabref.gui.DialogService;
+import org.jabref.gui.JabRefFrame;
+import org.jabref.gui.StateManager;
+import org.jabref.gui.actions.SimpleCommand;
+import org.jabref.gui.undo.NamedCompound;
+import org.jabref.gui.util.BindingsHelper;
+import org.jabref.gui.util.TaskExecutor;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.model.database.BibDatabaseContext;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.preferences.JabRefPreferences;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.jabref.gui.actions.ActionHelper.needsDatabase;
+import static org.jabref.gui.actions.ActionHelper.needsEntriesSelected;
+
+/**
+ * This class allows fetching the reference metadata (e.g. citation counts) from the web for the currently selected
+ * entries in a library.
+ */
+public class ReferenceMetadataFetcherAction extends SimpleCommand {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(ReferenceMetadataFetcherAction.class);
+
+    private static boolean USE_REFERENCE_METADATA_FETCHER_GOOGLE_SCHOLAR = true;
+    private static boolean USE_REFERENCE_METADATA_FETCHER_SEMANTIC_SCHOLAR = true;
+    private static boolean USE_REFERENCE_METADATA_FETCHER_OPEN_CITATIONS = true;
+
+    private final DialogService dialogService;
+    private final JabRefPreferences preferences;
+    private final StateManager stateManager;
+    private UndoManager undoManager;
+    private TaskExecutor taskExecutor;
+
+    public ReferenceMetadataFetcherAction(JabRefFrame frame, JabRefPreferences preferences, StateManager stateManager, UndoManager undoManager, TaskExecutor taskExecutor) {
+        this.dialogService = frame.getDialogService();
+        this.preferences = preferences;
+        this.stateManager = stateManager;
+        this.undoManager = undoManager;
+        this.taskExecutor = taskExecutor;
+
+        this.executable.bind(needsDatabase(this.stateManager).and(needsEntriesSelected(stateManager)));
+        this.statusMessage.bind(BindingsHelper.ifThenElse(executable, Localization.lang("This operation fetches " +
+                "reference metadata for the currently selected entries online."), Localization.lang("This " +
+                "operation requires one or more entries to be selected.")));
+    }
+
+    @Override
+    public void execute() {
+        BibDatabaseContext database = stateManager.getActiveDatabase().orElseThrow(() -> new NullPointerException("Database null"));
+        ObservableList<BibEntry> entries = stateManager.getSelectedEntries();
+
+        final NamedCompound nc = new NamedCompound(Localization.lang("Fetch reference metadata"));
+
+        ExtendedTask<List<BibEntry>> fetchReferenceMetadataTask = new ExtendedTask<List<BibEntry>>() {
+
+            @Override
+            protected List<BibEntry> call() {
+
+                boolean processCancelled = false;
+
+                ObservableList<BibEntry> entriesWithIncompleteMetadata = entries;
+
+                // run prioritized metadata fetcher pipeline
+
+                if (!processCancelled && USE_REFERENCE_METADATA_FETCHER_GOOGLE_SCHOLAR && entriesWithIncompleteMetadata.size() > 0) {
+                    LOGGER.info("running " + ReferenceMetadataFetcherGoogleScholar.class.getSimpleName() + "...");
+                    ReferenceMetadataFetcherGoogleScholar referenceMetadataFetcherGoogleScholar = new ReferenceMetadataFetcherGoogleScholar();
+                    processCancelled = referenceMetadataFetcherGoogleScholar.fetchFor(database, entriesWithIncompleteMetadata, dialogService, this);
+                    entriesWithIncompleteMetadata = referenceMetadataFetcherGoogleScholar.getEntriesWithIncompleteMetadata();
+                }
+
+                if (!processCancelled && USE_REFERENCE_METADATA_FETCHER_SEMANTIC_SCHOLAR && entriesWithIncompleteMetadata.size() > 0) {
+                    LOGGER.info("running " + ReferenceMetadataFetcherSemanticScholar.class.getSimpleName() + "...");
+                    ReferenceMetadataFetcherSemanticScholar referenceMetadataFetcherSemanticScholar = new ReferenceMetadataFetcherSemanticScholar();
+                    processCancelled = referenceMetadataFetcherSemanticScholar.fetchFor(database, entriesWithIncompleteMetadata, dialogService, this);
+                    entriesWithIncompleteMetadata = referenceMetadataFetcherSemanticScholar.getEntriesWithIncompleteMetadata();
+                }
+
+                if (!processCancelled && USE_REFERENCE_METADATA_FETCHER_OPEN_CITATIONS && entriesWithIncompleteMetadata.size() > 0) {
+                    LOGGER.info("running " + ReferenceMetadataFetcherOpenCitations.class.getSimpleName() + "...");
+                    ReferenceMetadataFetcherOpenCitations referenceMetadataFetcherOpenCitations = new ReferenceMetadataFetcherOpenCitations();
+                    processCancelled = referenceMetadataFetcherOpenCitations.fetchFor(database, entriesWithIncompleteMetadata, dialogService, this);
+                    entriesWithIncompleteMetadata = referenceMetadataFetcherOpenCitations.getEntriesWithIncompleteMetadata();
+                }
+
+                if (!processCancelled) {
+                    return entries;
+                } else {
+                    return new LinkedList<BibEntry>();
+                }
+            }
+
+            @Override
+            protected void succeeded() {
+                if (!getValue().isEmpty()) {
+                    // reserved for future use
+                    //if (nc.hasEdits()) {
+                    //    nc.end();
+                    //    undoManager.addEdit(nc);
+                    //}
+                    dialogService.notify(Localization.lang("Finished fetching reference metadata."));
+                } else {
+                    dialogService.notify(Localization.lang("Cancelled fetching reference metadata."));
+                }
+            }
+        };
+
+        dialogService.showProgressDialogAndWait(
+                Localization.lang("Fetching reference metadata online"),
+                Localization.lang("Querying reference metadata:"),
+                fetchReferenceMetadataTask);
+        taskExecutor.execute(fetchReferenceMetadataTask);
+    }
+}