diff --git a/Telegram/SourceFiles/base/value_ordering.h b/Telegram/SourceFiles/base/value_ordering.h index fbb2ef8377f8eb..941421bfa0ed6e 100644 --- a/Telegram/SourceFiles/base/value_ordering.h +++ b/Telegram/SourceFiles/base/value_ordering.h @@ -7,6 +7,8 @@ For license and copyright information please follow this link: */ #pragma once +#include + namespace base { namespace details { diff --git a/Telegram/SourceFiles/export/data/export_data_types.h b/Telegram/SourceFiles/export/data/export_data_types.h index 8e7a07a54ae42a..5fa7dc8c56b8ad 100644 --- a/Telegram/SourceFiles/export/data/export_data_types.h +++ b/Telegram/SourceFiles/export/data/export_data_types.h @@ -48,6 +48,10 @@ struct UserpicsInfo { struct FileLocation { int dcId = 0; MTPInputFileLocation data; + + explicit operator bool() const { + return dcId != 0; + } }; struct File { diff --git a/Telegram/SourceFiles/export/export_api_wrap.cpp b/Telegram/SourceFiles/export/export_api_wrap.cpp index d96e3eded68a84..bf3fc5ceb3eb52 100644 --- a/Telegram/SourceFiles/export/export_api_wrap.cpp +++ b/Telegram/SourceFiles/export/export_api_wrap.cpp @@ -11,6 +11,7 @@ For license and copyright information please follow this link: #include "export/data/export_data_types.h" #include "export/output/export_output_file.h" #include "mtproto/rpc_sender.h" +#include "base/value_ordering.h" #include "base/bytes.h" #include @@ -25,13 +26,61 @@ constexpr auto kFileNextRequestDelay = TimeMs(20); constexpr auto kChatsSliceLimit = 100; constexpr auto kMessagesSliceLimit = 100; constexpr auto kFileMaxSize = 1500 * 1024 * 1024; +constexpr auto kLocationCacheSize = 100'000; -bool FileIsAvailable(const Data::File &file) { - return file.relativePath.isEmpty() && (file.location.dcId != 0); +struct LocationKey { + uint64 type; + uint64 id; + + inline bool operator<(const LocationKey &other) const { + return std::tie(type, id) < std::tie(other.type, other.id); + } +}; + +std::tuple value_ordering_helper(const LocationKey &value) { + return std::tie( + value.type, + value.id); +} + +LocationKey ComputeLocationKey(const Data::FileLocation &value) { + auto result = LocationKey(); + result.type = value.dcId; + value.data.match([&](const MTPDinputFileLocation &data) { + result.type |= (1ULL << 24); + result.type |= (uint64(uint32(data.vlocal_id.v)) << 32); + result.id = data.vvolume_id.v; + }, [&](const MTPDinputDocumentFileLocation &data) { + result.type |= (2ULL << 24); + result.id = data.vid.v; + }, [&](const MTPDinputSecureFileLocation &data) { + result.type |= (3ULL << 24); + result.id = data.vid.v; + }, [&](const MTPDinputEncryptedFileLocation &data) { + result.type |= (4ULL << 24); + result.id = data.vid.v; + }); + return result; } } // namespace +class ApiWrap::LoadedFileCache { +public: + using Location = Data::FileLocation; + + LoadedFileCache(int limit); + + void save(const Location &location, const QString &relativePath); + base::optional find(const Location &location) const; + +private: + int _limit = 0; + std::map _map; + std::deque _list; + +}; + struct ApiWrap::UserpicsProcess { FnMut start; Fn handleSlice; @@ -94,6 +143,32 @@ struct ApiWrap::DialogsProcess::Single { }; +ApiWrap::LoadedFileCache::LoadedFileCache(int limit) : _limit(limit) { + Expects(limit >= 0); +} + +void ApiWrap::LoadedFileCache::save( + const Location &location, + const QString &relativePath) { + const auto key = ComputeLocationKey(location); + _map[key] = relativePath; + _list.push_back(key); + if (_list.size() > _limit) { + const auto key = _list.front(); + _list.pop_front(); + _map.erase(key); + } +} + +base::optional ApiWrap::LoadedFileCache::find( + const Location &location) const { + const auto key = ComputeLocationKey(location); + if (const auto i = _map.find(key); i != end(_map)) { + return i->second; + } + return base::none; +} + ApiWrap::FileProcess::FileProcess(const QString &path) : file(path) { } @@ -129,7 +204,8 @@ auto ApiWrap::fileRequest(const Data::FileLocation &location, int offset) { } ApiWrap::ApiWrap(Fn)> runner) -: _mtp(std::move(runner)) { +: _mtp(std::move(runner)) +, _fileCache(std::make_unique(kLocationCacheSize)) { } rpl::producer ApiWrap::errors() const { @@ -613,11 +689,11 @@ bool ApiWrap::processFileLoad( if (!file.relativePath.isEmpty()) { return true; - } else if (writePreloadedFile(file)) { - return true; - } else if (!FileIsAvailable(file)) { + } else if (!file.location) { file.skipReason = SkipReason::Unavailable; return true; + } else if (writePreloadedFile(file)) { + return true; } using Type = MediaSettings::Type; @@ -656,11 +732,15 @@ bool ApiWrap::writePreloadedFile(Data::File &file) { using namespace Output; - if (!file.content.isEmpty()) { + if (const auto path = _fileCache->find(file.location)) { + file.relativePath = *path; + return true; + } else if (!file.content.isEmpty()) { const auto process = prepareFileProcess(file); auto &output = _fileProcess->file; if (output.writeBlock(file.content) == File::Result::Success) { file.relativePath = process->relativePath; + _fileCache->save(file.location, file.relativePath); return true; } error(QString("Could not write '%1'.").arg(process->relativePath)); @@ -670,7 +750,7 @@ bool ApiWrap::writePreloadedFile(Data::File &file) { void ApiWrap::loadFile(const Data::File &file, FnMut done) { Expects(_fileProcess == nullptr); - Expects(FileIsAvailable(file)); + Expects(file.location.dcId != 0); _fileProcess = prepareFileProcess(file); _fileProcess->done = std::move(done); @@ -767,7 +847,10 @@ void ApiWrap::filePartDone(int offset, const MTPupload_File &result) { return; } } + auto process = base::take(_fileProcess); + const auto relativePath = process->relativePath; + _fileCache->save(process->location, relativePath); process->done(process->relativePath); } diff --git a/Telegram/SourceFiles/export/export_api_wrap.h b/Telegram/SourceFiles/export/export_api_wrap.h index 20bf636a1dee95..367e8c4c3a5b29 100644 --- a/Telegram/SourceFiles/export/export_api_wrap.h +++ b/Telegram/SourceFiles/export/export_api_wrap.h @@ -60,6 +60,7 @@ class ApiWrap { struct UserpicsProcess; struct FileProcess; struct DialogsProcess; + class LoadedFileCache; void startMainSession(FnMut done); @@ -111,6 +112,7 @@ class ApiWrap { std::unique_ptr _settings; MTPInputUser _user = MTP_inputUserSelf(); + std::unique_ptr _fileCache; std::unique_ptr _userpicsProcess; std::unique_ptr _fileProcess; std::unique_ptr _dialogsProcess; diff --git a/Telegram/SourceFiles/export/export_settings.h b/Telegram/SourceFiles/export/export_settings.h index 13803b16e28355..91dcf0e2049cb2 100644 --- a/Telegram/SourceFiles/export/export_settings.h +++ b/Telegram/SourceFiles/export/export_settings.h @@ -71,7 +71,8 @@ struct Settings { } static inline Types DefaultFullChats() { - return Type::PersonalChats; + return Type::PersonalChats + | Type::BotChats; } }; diff --git a/Telegram/SourceFiles/export/output/export_output_text.cpp b/Telegram/SourceFiles/export/output/export_output_text.cpp index 2bb294866abe77..914b5248433997 100644 --- a/Telegram/SourceFiles/export/output/export_output_text.cpp +++ b/Telegram/SourceFiles/export/output/export_output_text.cpp @@ -178,6 +178,31 @@ QByteArray SerializeMessage( } }; + using SkipReason = Data::File::SkipReason; + const auto pushPath = [&]( + const Data::File &file, + const QByteArray &label) { + Expects(!file.relativePath.isEmpty() + || file.skipReason != SkipReason::None); + + push(label, [&]() -> QByteArray { + switch (file.skipReason) { + case SkipReason::Unavailable: return "(file unavailable)"; + case SkipReason::FileSize: return "(file too large)"; + case SkipReason::FileType: return "(file skipped)"; + case SkipReason::None: return FormatFilePath(file); + } + Unexpected("Skip reason while writing file path."); + }()); + }; + const auto pushPhoto = [&](const Image &image) { + pushPath(image.file, "Photo"); + if (image.width && image.height) { + push("Width", NumberToString(image.width)); + push("Height", NumberToString(image.height)); + } + }; + message.action.content.match([&](const ActionChatCreate &data) { pushActor(); pushAction("Create group"); @@ -190,7 +215,7 @@ QByteArray SerializeMessage( }, [&](const ActionChatEditPhoto &data) { pushActor(); pushAction("Edit group photo"); - push("Photo", FormatFilePath(data.photo.image.file)); + pushPhoto(data.photo.image); }, [&](const ActionChatDeletePhoto &data) { pushActor(); pushAction("Delete group photo"); @@ -306,28 +331,29 @@ QByteArray SerializeMessage( } message.media.content.match([&](const Photo &photo) { + pushPhoto(photo.image); pushTTL(); }, [&](const Document &data) { - const auto pushPath = [&](const QByteArray &label) { - push(label, FormatFilePath(data.file)); + const auto pushMyPath = [&](const QByteArray &label) { + return pushPath(data.file, label); }; if (data.isSticker) { - pushPath("Sticker"); + pushMyPath("Sticker"); push("Emoji", data.stickerEmoji); } else if (data.isVideoMessage) { - pushPath("Video message"); + pushMyPath("Video message"); } else if (data.isVoiceMessage) { - pushPath("Voice message"); + pushMyPath("Voice message"); } else if (data.isAnimated) { - pushPath("Animation"); + pushMyPath("Animation"); } else if (data.isVideoFile) { - pushPath("Video file"); + pushMyPath("Video file"); } else if (data.isAudioFile) { - pushPath("Audio file"); + pushMyPath("Audio file"); push("Performer", data.songPerformer); push("Title", data.songTitle); } else { - pushPath("File"); + pushMyPath("File"); } if (!data.isSticker) { push("Mime type", data.mime);