Download all favicons (#3169)

* Selecting one or more entries to download icons always forces the download (ie, if a new URL exists the new icon will be downloaded and set)
* Instead of downloading for each entry, the web url's are scraped from the provided entries and only those urls are downloaded. The icon is set for all entries that share a URL. This is useful if a group contains many entries that point to the same url, only 1 download call will occur.
* The icon download dialog displays whether you are doing one entry, many entries, or an entire group. It is also modal so you have to dismiss it to use KeePassXC again.
* Moved DuckDuckGo fallback notice into the download dialog.

src/CMakeLists.txt

@@ -27,6 +27,9 @@ set(keepassx_SOURCES
         core/Alloc.cpp
         core/AutoTypeAssociations.cpp
         core/AutoTypeMatch.cpp
+        core/Base32.cpp
+        core/Bootstrap.cpp
+        core/Clock.cpp
         core/Compare.cpp
         core/Config.cpp
         core/CsvParser.cpp
@@ -39,7 +42,6 @@ set(keepassx_SOURCES
         core/EntrySearcher.cpp
         core/FilePath.cpp
         core/FileWatcher.cpp
-        core/Bootstrap.cpp
         core/Group.cpp
         core/HibpOffline.cpp
         core/InactivityTimer.cpp
@@ -52,10 +54,8 @@ set(keepassx_SOURCES
         core/ScreenLockListenerPrivate.cpp
         core/TimeDelta.cpp
         core/TimeInfo.cpp
-        core/Clock.cpp
         core/Tools.cpp
         core/Translator.cpp
-        core/Base32.cpp
         cli/Utils.cpp
         cli/TextStream.cpp
         crypto/Crypto.cpp
@@ -264,7 +264,12 @@ else()
 endif()
 
 if(WITH_XC_NETWORKING)
-    list(APPEND keepassx_SOURCES updatecheck/UpdateChecker.cpp gui/UpdateCheckDialog.cpp)
+    list(APPEND keepassx_SOURCES
+            core/IconDownloader.cpp
+            core/NetworkManager.cpp
+            gui/UpdateCheckDialog.cpp
+            gui/IconDownloaderDialog.cpp
+            updatecheck/UpdateChecker.cpp)
 endif()
 
 if(WITH_XC_TOUCHID)

src/core/Config.cpp

@@ -194,6 +194,7 @@ void Config::init(const QString& fileName)
     m_defaults.insert("AutoTypeStartDelay", 500);
     m_defaults.insert("UseGroupIconOnEntryCreation", true);
     m_defaults.insert("IgnoreGroupExpansion", true);
+    m_defaults.insert("FaviconDownloadTimeout", 10);
     m_defaults.insert("security/clearclipboard", true);
     m_defaults.insert("security/clearclipboardtimeout", 10);
     m_defaults.insert("security/lockdatabaseidle", false);

src/core/IconDownloader.cpp

@@ -0,0 +1,204 @@
+/*
+ *  Copyright (C) 2019 KeePassXC Team <[email protected]>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 2 or (at your option)
+ *  version 3 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "IconDownloader.h"
+#include "core/Config.h"
+#include "core/NetworkManager.h"
+
+#include <QHostInfo>
+#include <QtNetwork>
+
+#define MAX_REDIRECTS 5
+
+IconDownloader::IconDownloader(QObject* parent)
+    : QObject(parent)
+    , m_reply(nullptr)
+    , m_redirects(0)
+{
+    m_timeout.setSingleShot(true);
+    connect(&m_timeout, SIGNAL(timeout()), SLOT(abortDownload()));
+}
+
+IconDownloader::~IconDownloader()
+{
+    abortDownload();
+}
+
+namespace
+{
+    // Try to get the 2nd level domain of the host part of a QUrl. For example,
+    // "foo.bar.example.com" would become "example.com", and "foo.bar.example.co.uk"
+    // would become "example.co.uk".
+    QString getSecondLevelDomain(const QUrl& url)
+    {
+        QString fqdn = url.host();
+        fqdn.truncate(fqdn.length() - url.topLevelDomain().length());
+        QStringList parts = fqdn.split('.');
+        QString newdom = parts.takeLast() + url.topLevelDomain();
+        return newdom;
+    }
+
+    QUrl convertVariantToUrl(const QVariant& var)
+    {
+        QUrl url;
+        if (var.canConvert<QUrl>()) {
+            url = var.toUrl();
+        }
+        return url;
+    }
+
+    QUrl getRedirectTarget(QNetworkReply* reply)
+    {
+        QVariant var = reply->attribute(QNetworkRequest::RedirectionTargetAttribute);
+        QUrl url = convertVariantToUrl(var);
+        return url;
+    }
+} // namespace
+
+void IconDownloader::setUrl(const QString& entryUrl)
+{
+    m_url = entryUrl;
+    QUrl url(m_url);
+    if (!url.isValid()) {
+        return;
+    }
+
+    m_redirects = 0;
+    m_urlsToTry.clear();
+
+    if (url.scheme().isEmpty()) {
+        url.setUrl(QString("https://%1").arg(url.toString()));
+    }
+
+    QString fullyQualifiedDomain = url.host();
+
+    // Determine if host portion of URL is an IP address by resolving it and
+    // searching for a match with the returned address(es).
+    bool hostIsIp = false;
+    QList<QHostAddress> hostAddressess = QHostInfo::fromName(fullyQualifiedDomain).addresses();
+    for (auto addr : hostAddressess) {
+        if (addr.toString() == fullyQualifiedDomain) {
+            hostIsIp = true;
+        }
+    }
+
+    // Determine the second-level domain, if available
+    QString secondLevelDomain;
+    if (!hostIsIp) {
+        secondLevelDomain = getSecondLevelDomain(m_url);
+    }
+
+    // Start with the "fallback" url (if enabled) to try to get the best favicon
+    if (config()->get("security/IconDownloadFallback", false).toBool()) {
+        QUrl fallbackUrl = QUrl("https://icons.duckduckgo.com");
+        fallbackUrl.setPath("/ip3/" + QUrl::toPercentEncoding(fullyQualifiedDomain) + ".ico");
+        m_urlsToTry.append(fallbackUrl);
+
+        // Also try a direct pull of the second-level domain (if possible)
+        if (!hostIsIp && fullyQualifiedDomain != secondLevelDomain) {
+            fallbackUrl.setPath("/ip3/" + QUrl::toPercentEncoding(secondLevelDomain) + ".ico");
+            m_urlsToTry.append(fallbackUrl);
+        }
+    }
+
+    // Add a direct pull of the website's own favicon.ico file
+    m_urlsToTry.append(QUrl(url.scheme() + "://" + fullyQualifiedDomain + "/favicon.ico"));
+
+    // Also try a direct pull of the second-level domain (if possible)
+    if (!hostIsIp && fullyQualifiedDomain != secondLevelDomain) {
+        m_urlsToTry.append(QUrl(url.scheme() + "://" + secondLevelDomain + "/favicon.ico"));
+    }
+}
+
+void IconDownloader::download()
+{
+    if (!m_timeout.isActive()) {
+        int timeout = config()->get("FaviconDownloadTimeout", 10).toInt();
+        m_timeout.start(timeout * 1000);
+
+        // Use the first URL to start the download process
+        // If a favicon is not found, the next URL will be tried
+        fetchFavicon(m_urlsToTry.takeFirst());
+    }
+}
+
+void IconDownloader::abortDownload()
+{
+    if (m_reply) {
+        m_reply->abort();
+    }
+}
+
+void IconDownloader::fetchFavicon(const QUrl& url)
+{
+    m_bytesReceived.clear();
+    m_fetchUrl = url;
+
+    QNetworkRequest request(url);
+    m_reply = getNetMgr()->get(request);
+
+    connect(m_reply, &QNetworkReply::finished, this, &IconDownloader::fetchFinished);
+    connect(m_reply, &QIODevice::readyRead, this, &IconDownloader::fetchReadyRead);
+}
+
+void IconDownloader::fetchReadyRead()
+{
+    m_bytesReceived += m_reply->readAll();
+}
+
+void IconDownloader::fetchFinished()
+{
+    QImage image;
+    QString url = m_url;
+
+    bool error = (m_reply->error() != QNetworkReply::NoError);
+    QUrl redirectTarget = getRedirectTarget(m_reply);
+
+    m_reply->deleteLater();
+    m_reply = nullptr;
+
+    if (!error) {
+        if (redirectTarget.isValid()) {
+            // Redirected, we need to follow it, or fall through if we have
+            // done too many redirects already.
+            if (m_redirects < MAX_REDIRECTS) {
+                m_redirects++;
+                if (redirectTarget.isRelative()) {
+                    redirectTarget = m_fetchUrl.resolved(redirectTarget);
+                }
+                m_urlsToTry.prepend(redirectTarget);
+            }
+        } else {
+            // No redirect, and we theoretically have some icon data now.
+            image.loadFromData(m_bytesReceived);
+        }
+    }
+
+    if (!image.isNull()) {
+        // Valid icon received
+        m_timeout.stop();
+        emit finished(url, image);
+    } else if (!m_urlsToTry.empty()) {
+        // Try the next url
+        m_redirects = 0;
+        fetchFavicon(m_urlsToTry.takeFirst());
+    } else {
+        // No icon found
+        m_timeout.stop();
+        emit finished(url, image);
+    }
+}

src/core/IconDownloader.h

@@ -0,0 +1,63 @@
+/*
+ *  Copyright (C) 2019 KeePassXC Team <[email protected]>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 2 or (at your option)
+ *  version 3 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef KEEPASSXC_ICONDOWNLOADER_H
+#define KEEPASSXC_ICONDOWNLOADER_H
+
+#include <QImage>
+#include <QObject>
+#include <QTimer>
+#include <QUrl>
+
+#include "core/Global.h"
+
+class QNetworkReply;
+
+class IconDownloader : public QObject
+{
+    Q_OBJECT
+
+public:
+    explicit IconDownloader(QObject* parent = nullptr);
+    ~IconDownloader() override;
+
+    void setUrl(const QString& entryUrl);
+    void download();
+
+signals:
+    void finished(const QString& entryUrl, const QImage& image);
+
+public slots:
+    void abortDownload();
+
+private slots:
+    void fetchFinished();
+    void fetchReadyRead();
+
+private:
+    void fetchFavicon(const QUrl& url);
+
+    QString m_url;
+    QUrl m_fetchUrl;
+    QList<QUrl> m_urlsToTry;
+    QByteArray m_bytesReceived;
+    QNetworkReply* m_reply;
+    QTimer m_timeout;
+    int m_redirects;
+};
+
+#endif // KEEPASSXC_ICONDOWNLOADER_H

src/core/NetworkManager.cpp

@@ -0,0 +1,33 @@
+/*
+ *  Copyright (C) 2019 KeePassXC Team <[email protected]>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 2 or (at your option)
+ *  version 3 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "config-keepassx.h"
+
+#ifdef WITH_XC_NETWORKING
+#include "NetworkManager.h"
+
+#include <QCoreApplication>
+
+QNetworkAccessManager* g_netMgr = nullptr;
+QNetworkAccessManager* getNetMgr()
+{
+    if (!g_netMgr) {
+        g_netMgr = new QNetworkAccessManager(QCoreApplication::instance());
+    }
+    return g_netMgr;
+}
+#endif

src/core/NetworkManager.h

@@ -0,0 +1,34 @@
+/*
+ *  Copyright (C) 2019 KeePassXC Team <[email protected]>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 2 or (at your option)
+ *  version 3 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef KEEPASSXC_NETWORKMANAGER_H
+#define KEEPASSXC_NETWORKMANAGER_H
+
+#include "config-keepassx.h"
+#include <QtGlobal>
+
+#ifdef WITH_XC_NETWORKING
+#include <QNetworkAccessManager>
+#include <QNetworkReply>
+#include <QNetworkRequest>
+
+QNetworkAccessManager* getNetMgr();
+#else
+Q_STATIC_ASSERT_X(false, "Qt Networking used when WITH_XC_NETWORKING is disabled!");
+#endif
+
+#endif // KEEPASSXC_NETWORKMANAGER_H