diff --git a/Telegram/SourceFiles/export/data/export_data_types.cpp b/Telegram/SourceFiles/export/data/export_data_types.cpp index 4bacad3df8569..64f6a886be774 100644 --- a/Telegram/SourceFiles/export/data/export_data_types.cpp +++ b/Telegram/SourceFiles/export/data/export_data_types.cpp @@ -1089,6 +1089,8 @@ void FinalizeDialogsInfo(DialogsInfo &info, const Settings &settings) { Unexpected("Type in ApiWrap::onlyMyMessages."); }(); dialog.onlyMyMessages = ((settings.fullChats & setting) != setting); + + ranges::reverse(dialog.splits); } } diff --git a/Telegram/SourceFiles/export/data/export_data_types.h b/Telegram/SourceFiles/export/data/export_data_types.h index 5029d6eb3f268..707350d06128a 100644 --- a/Telegram/SourceFiles/export/data/export_data_types.h +++ b/Telegram/SourceFiles/export/data/export_data_types.h @@ -446,13 +446,15 @@ struct DialogInfo { TimeId topMessageDate = 0; PeerId peerId = 0; + // User messages splits which contained that dialog. + std::vector splits; + // Filled after the whole dialogs list is accumulated. bool onlyMyMessages = false; QString relativePath; // Filled when requesting dialog messages. - int messagesCount = 0; - + std::vector messagesCountPerSplit; }; struct DialogsInfo { diff --git a/Telegram/SourceFiles/export/export_api_wrap.cpp b/Telegram/SourceFiles/export/export_api_wrap.cpp index c17c900f080ff..efdc0b8b970e1 100644 --- a/Telegram/SourceFiles/export/export_api_wrap.cpp +++ b/Telegram/SourceFiles/export/export_api_wrap.cpp @@ -15,6 +15,7 @@ For license and copyright information please follow this link: #include "base/value_ordering.h" #include "base/bytes.h" +#include #include namespace Export { @@ -107,10 +108,12 @@ struct ApiWrap::StartProcess { enum class Step { UserpicsCount, + SplitRanges, DialogsCount, LeftChannelsCount, }; std::deque steps; + int splitIndex = 0; StartInfo info; }; @@ -160,23 +163,23 @@ struct ApiWrap::FileProgress { int total = 0; }; -struct ApiWrap::LeftChannelsProcess { +struct ApiWrap::ChatsProcess { Fn progress; FnMut done; Data::DialogsInfo info; + int processedCount = 0; + std::map indexByPeer; +}; +struct ApiWrap::LeftChannelsProcess : ChatsProcess { int fullCount = 0; int offset = 0; bool finished = false; }; -struct ApiWrap::DialogsProcess { - Fn progress; - FnMut done; - - Data::DialogsInfo info; - +struct ApiWrap::DialogsProcess : ChatsProcess { + int splitIndexPlusOne = 0; Data::TimeId offsetDate = 0; int32 offsetId = 0; MTPInputPeer offsetPeer = MTP_inputPeerEmpty(); @@ -190,7 +193,8 @@ struct ApiWrap::ChatProcess { Fn handleSlice; FnMut done; - int32 offsetId = 1; + int localSplitIndex = 0; + int32 largestIdPlusOne = 1; Data::ParseMediaContext context; base::optional slice; @@ -234,12 +238,24 @@ auto ApiWrap::mainRequest(Request &&request) { return std::move(_mtp.request(MTPInvokeWithTakeout( MTP_long(*_takeoutId), - request + std::forward(request) )).fail([=](RPCError &&result) { error(std::move(result)); }).toDC(MTP::ShiftDcId(0, MTP::kExportDcShift))); } +template +auto ApiWrap::splitRequest(int index, Request &&request) { + Expects(index < _splits.size()); + + //if (index == _splits.size() - 1) { + // return mainRequest(std::forward(request)); + //} + return mainRequest(MTPInvokeWithMessagesRange( + _splits[index], + std::forward(request))); +} + auto ApiWrap::fileRequest(const Data::FileLocation &location, int offset) { Expects(location.dcId != 0); Expects(_takeoutId.has_value()); @@ -284,6 +300,9 @@ void ApiWrap::startExport( if (_settings->types & Settings::Type::Userpics) { _startProcess->steps.push_back(Step::UserpicsCount); } + if (_settings->types & Settings::Type::NonChannelChatsMask) { + _startProcess->steps.push_back(Step::SplitRanges); + } if (_settings->types & Settings::Type::AnyChatsMask) { _startProcess->steps.push_back(Step::DialogsCount); } @@ -303,14 +322,17 @@ void ApiWrap::sendNextStartRequest() { finishStartProcess(); return; } + using Step = StartProcess::Step; const auto step = steps.front(); steps.pop_front(); switch (step) { - case StartProcess::Step::UserpicsCount: + case Step::UserpicsCount: return requestUserpicsCount(); - case StartProcess::Step::DialogsCount: + case Step::SplitRanges: + return requestSplitRanges(); + case Step::DialogsCount: return requestDialogsCount(); - case StartProcess::Step::LeftChannelsCount: + case Step::LeftChannelsCount: return requestLeftChannelsCount(); } Unexpected("Step in ApiWrap::sendNextStartRequest."); @@ -339,10 +361,20 @@ void ApiWrap::requestUserpicsCount() { }).send(); } +void ApiWrap::requestSplitRanges() { + Expects(_startProcess != nullptr); + + mainRequest(MTPmessages_GetSplitRanges( + )).done([=](const MTPVector &result) { + _splits = result.v; + sendNextStartRequest(); + }).send(); +} + void ApiWrap::requestDialogsCount() { Expects(_startProcess != nullptr); - mainRequest(MTPmessages_GetDialogs( + splitRequest(_startProcess->splitIndex, MTPmessages_GetDialogs( MTP_flags(0), MTP_int(0), // offset_date MTP_int(0), // offset_id @@ -352,14 +384,18 @@ void ApiWrap::requestDialogsCount() { Expects(_settings != nullptr); Expects(_startProcess != nullptr); - _startProcess->info.dialogsCount = result.match( + _startProcess->info.dialogsCount += result.match( [](const MTPDmessages_dialogs &data) { return int(data.vdialogs.v.size()); }, [](const MTPDmessages_dialogsSlice &data) { return data.vcount.v; }); - sendNextStartRequest(); + if (++_startProcess->splitIndex >= _splits.size()) { + sendNextStartRequest(); + } else { + requestDialogsCount(); + } }).send(); } @@ -390,6 +426,8 @@ void ApiWrap::requestLeftChannelsList( FnMut done) { Expects(_leftChannelsProcess != nullptr); + validateSplits(); + _leftChannelsProcess->progress = std::move(progress); _leftChannelsProcess->done = std::move(done); requestLeftChannelsSlice(); @@ -414,13 +452,24 @@ void ApiWrap::requestDialogsList( FnMut done) { Expects(_dialogsProcess == nullptr); + validateSplits(); + _dialogsProcess = std::make_unique(); + _dialogsProcess->splitIndexPlusOne = _splits.size(); _dialogsProcess->progress = std::move(progress); _dialogsProcess->done = std::move(done); requestDialogsSlice(); } +void ApiWrap::validateSplits() { + if (_splits.empty()) { + _splits.push_back(MTP_messageRange( + MTP_int(0), + MTP_int(std::numeric_limits::max()))); + } +} + void ApiWrap::startMainSession(FnMut done) { const auto sizeLimit = _settings->media.sizeLimit; const auto hasFiles = (_settings->media.types != 0) && (sizeLimit > 0); @@ -699,11 +748,41 @@ void ApiWrap::requestMessages( _chatProcess->handleSlice = std::move(slice); _chatProcess->done = std::move(done); - requestMessagesSlice([=](int count) { + requestMessagesCount(0); +} + +void ApiWrap::requestMessagesCount(int localSplitIndex) { + Expects(_chatProcess != nullptr); + Expects(localSplitIndex < _chatProcess->info.splits.size()); + + requestChatMessages( + _chatProcess->info.splits[localSplitIndex], + 0, // offset_id + 0, // add_offset + 1, // limit + [=](const MTPmessages_Messages &result) { Expects(_chatProcess != nullptr); - _chatProcess->info.messagesCount = count; - return _chatProcess->start(_chatProcess->info); + const auto count = result.match( + [](const MTPDmessages_messages &data) { + return data.vmessages.v.size(); + }, [](const MTPDmessages_messagesSlice &data) { + return data.vcount.v; + }, [](const MTPDmessages_channelMessages &data) { + return data.vcount.v; + }, [](const MTPDmessages_messagesNotModified &data) { + return -1; + }); + if (count < 0) { + error("Unexpected messagesNotModified received."); + return; + } + _chatProcess->info.messagesCountPerSplit[localSplitIndex] = count; + if (localSplitIndex + 1 < _chatProcess->info.splits.size()) { + requestMessagesCount(localSplitIndex + 1); + } else if (_chatProcess->start(_chatProcess->info)) { + requestMessagesSlice(); + } }); } @@ -727,14 +806,15 @@ void ApiWrap::cancelExportFast() { void ApiWrap::requestDialogsSlice() { Expects(_dialogsProcess != nullptr); - mainRequest(MTPmessages_GetDialogs( + const auto splitIndex = _dialogsProcess->splitIndexPlusOne - 1; + splitRequest(splitIndex, MTPmessages_GetDialogs( MTP_flags(0), MTP_int(_dialogsProcess->offsetDate), MTP_int(_dialogsProcess->offsetId), _dialogsProcess->offsetPeer, MTP_int(kChatsSliceLimit) )).done([=](const MTPmessages_Dialogs &result) { - const auto finished = result.match( + auto finished = result.match( [](const MTPDmessages_dialogs &data) { return true; }, [](const MTPDmessages_dialogsSlice &data) { @@ -742,30 +822,40 @@ void ApiWrap::requestDialogsSlice() { }); auto info = Data::ParseDialogsInfo(result); - if (finished || info.list.empty()) { - finishDialogsList(); - } else { - const auto &last = info.list.back(); + _dialogsProcess->processedCount += info.list.size(); + const auto last = info.list.empty() + ? Data::DialogInfo() + : info.list.back(); + appendDialogsSlice(std::move(info)); + + if (!_dialogsProcess->progress(_dialogsProcess->processedCount)) { + return; + } + + if (!finished && last.topMessageDate > 0) { _dialogsProcess->offsetId = last.topMessageId; _dialogsProcess->offsetDate = last.topMessageDate; _dialogsProcess->offsetPeer = last.input; - - appendDialogsSlice(std::move(info)); - - const auto count = _dialogsProcess->info.list.size(); - if (!_dialogsProcess->progress(count)) { - return; - } - - requestDialogsSlice(); + } else if (--_dialogsProcess->splitIndexPlusOne > 0) { + _dialogsProcess->offsetId = 0; + _dialogsProcess->offsetDate = 0; + _dialogsProcess->offsetPeer = MTP_inputPeerEmpty(); + } else { + finishDialogsList(); + return; } + requestDialogsSlice(); }).send(); } void ApiWrap::appendDialogsSlice(Data::DialogsInfo &&info) { Expects(_dialogsProcess != nullptr); + Expects(_dialogsProcess->splitIndexPlusOne <= _splits.size()); - appendChatsSlice(_dialogsProcess->info, std::move(info)); + appendChatsSlice( + *_dialogsProcess, + std::move(info), + _dialogsProcess->splitIndexPlusOne - 1); } void ApiWrap::finishDialogsList() { @@ -822,13 +912,18 @@ void ApiWrap::requestLeftChannelsSliceGeneric(FnMut done) { void ApiWrap::appendLeftChannelsSlice(Data::DialogsInfo &&info) { Expects(_leftChannelsProcess != nullptr); + Expects(!_splits.empty()); - appendChatsSlice(_leftChannelsProcess->info, std::move(info)); + appendChatsSlice( + *_leftChannelsProcess, + std::move(info), + _splits.size() - 1); } void ApiWrap::appendChatsSlice( - Data::DialogsInfo &to, - Data::DialogsInfo &&info) { + ChatsProcess &to, + Data::DialogsInfo &&info, + int splitIndex) { Expects(_settings != nullptr); const auto types = _settings->types; @@ -837,41 +932,33 @@ void ApiWrap::appendChatsSlice( ) | ranges::view::filter([&](const Data::DialogInfo &info) { return (types & SettingsFromDialogsType(info.type)) != 0; }); - auto &list = to.list; - if (list.empty()) { - list = filtered | ranges::to_vector; - } else { - list.reserve(list.size() + info.list.size()); - for (auto &info : filtered) { + auto &list = to.info.list; + list.reserve(list.size() + info.list.size()); + for (auto &info : filtered) { + const auto nextIndex = list.size(); + const auto [i, ok] = to.indexByPeer.emplace(info.peerId, nextIndex); + if (ok) { list.push_back(std::move(info)); } + list[i->second].splits.push_back(splitIndex); + list[i->second].messagesCountPerSplit.push_back(0); } } -void ApiWrap::requestMessagesSlice(FnMut start) { +void ApiWrap::requestMessagesSlice() { Expects(_chatProcess != nullptr); - auto handleResult = [=, start = std::move(start)]( - const MTPmessages_Messages &result) mutable { + requestChatMessages( + _chatProcess->info.splits[_chatProcess->localSplitIndex], + _chatProcess->largestIdPlusOne, + -kMessagesSliceLimit, + kMessagesSliceLimit, + [=](const MTPmessages_Messages &result) { Expects(_chatProcess != nullptr); - const auto count = result.match( - [](const MTPDmessages_messages &data) { - return data.vmessages.v.size(); - }, [](const MTPDmessages_messagesSlice &data) { - return data.vcount.v; - }, [](const MTPDmessages_channelMessages &data) { - return data.vcount.v; - }, [](const MTPDmessages_messagesNotModified &data) { - return 0; - }); - result.match([&](const MTPDmessages_messagesNotModified &data) { error("Unexpected messagesNotModified received."); }, [&](const auto &data) { - if (start && !start(count)) { - return; - } if constexpr (MTPDmessages_messages::Is()) { _chatProcess->lastSlice = true; } @@ -882,9 +969,17 @@ void ApiWrap::requestMessagesSlice(FnMut start) { data.vchats, _chatProcess->info.relativePath)); }); - }; + }); +} + +void ApiWrap::requestChatMessages( + int splitIndex, + int offsetId, + int addOffset, + int limit, + FnMut done) { if (_chatProcess->info.onlyMyMessages) { - mainRequest(MTPmessages_Search( + splitRequest(splitIndex, MTPmessages_Search( MTP_flags(MTPmessages_Search::Flag::f_from_id), _chatProcess->info.input, MTP_string(""), // query @@ -892,24 +987,24 @@ void ApiWrap::requestMessagesSlice(FnMut start) { MTP_inputMessagesFilterEmpty(), MTP_int(0), // min_date MTP_int(0), // max_date - MTP_int(_chatProcess->offsetId), - MTP_int(-kMessagesSliceLimit), - MTP_int(kMessagesSliceLimit), + MTP_int(offsetId), + MTP_int(addOffset), + MTP_int(limit), MTP_int(0), // max_id MTP_int(0), // min_id MTP_int(0) // hash - )).done(std::move(handleResult)).send(); + )).done(std::move(done)).send(); } else { - mainRequest(MTPmessages_GetHistory( + splitRequest(splitIndex, MTPmessages_GetHistory( _chatProcess->info.input, - MTP_int(_chatProcess->offsetId), + MTP_int(offsetId), MTP_int(0), // offset_date - MTP_int(-kMessagesSliceLimit), - MTP_int(kMessagesSliceLimit), + MTP_int(addOffset), + MTP_int(limit), MTP_int(0), // max_id MTP_int(0), // min_id MTP_int(0) // hash - )).done(std::move(handleResult)).send(); + )).done(std::move(done)).send(); } } @@ -957,15 +1052,21 @@ void ApiWrap::finishMessagesSlice() { auto slice = *base::take(_chatProcess->slice); if (!slice.list.empty()) { - _chatProcess->offsetId = slice.list.back().id + 1; + _chatProcess->largestIdPlusOne = slice.list.back().id + 1; if (!_chatProcess->handleSlice(std::move(slice))) { return; } } - if (_chatProcess->lastSlice) { - finishMessages(); - } else { + if (_chatProcess->lastSlice + && (++_chatProcess->localSplitIndex + < _chatProcess->info.splits.size())) { + _chatProcess->lastSlice = false; + _chatProcess->largestIdPlusOne = 1; + } + if (!_chatProcess->lastSlice) { requestMessagesSlice(); + } else { + finishMessages(); } } diff --git a/Telegram/SourceFiles/export/export_api_wrap.h b/Telegram/SourceFiles/export/export_api_wrap.h index 4dc7694197c2b..32f5c08331bed 100644 --- a/Telegram/SourceFiles/export/export_api_wrap.h +++ b/Telegram/SourceFiles/export/export_api_wrap.h @@ -93,6 +93,7 @@ class ApiWrap { struct UserpicsProcess; struct FileProcess; struct FileProgress; + struct ChatsProcess; struct LeftChannelsProcess; struct DialogsProcess; struct ChatProcess; @@ -100,6 +101,7 @@ class ApiWrap { void startMainSession(FnMut done); void sendNextStartRequest(); void requestUserpicsCount(); + void requestSplitRanges(); void requestDialogsCount(); void requestLeftChannelsCount(); void finishStartProcess(); @@ -114,6 +116,8 @@ class ApiWrap { void finishUserpicsSlice(); void finishUserpics(); + void validateSplits(); + void requestDialogsSlice(); void appendDialogsSlice(Data::DialogsInfo &&info); void finishDialogsList(); @@ -122,9 +126,19 @@ class ApiWrap { void requestLeftChannelsSlice(); void appendLeftChannelsSlice(Data::DialogsInfo &&info); - void appendChatsSlice(Data::DialogsInfo &to, Data::DialogsInfo &&info); - - void requestMessagesSlice(FnMut start = nullptr); + void appendChatsSlice( + ChatsProcess &to, + Data::DialogsInfo &&info, + int splitIndex); + + void requestMessagesCount(int localSplitIndex); + void requestMessagesSlice(); + void requestChatMessages( + int splitIndex, + int offsetId, + int addOffset, + int limit, + FnMut done); void loadMessagesFiles(Data::MessagesSlice &&slice); void loadNextMessageFile(); bool loadMessageFileProgress(FileProgress value); @@ -150,6 +164,9 @@ class ApiWrap { template [[nodiscard]] auto mainRequest(Request &&request); + template + [[nodiscard]] auto splitRequest(int index, Request &&request); + [[nodiscard]] auto fileRequest( const Data::FileLocation &location, int offset); @@ -173,6 +190,7 @@ class ApiWrap { std::unique_ptr _leftChannelsProcess; std::unique_ptr _dialogsProcess; std::unique_ptr _chatProcess; + QVector _splits; rpl::event_stream _errors; rpl::event_stream _ioErrors; diff --git a/Telegram/SourceFiles/export/export_controller.cpp b/Telegram/SourceFiles/export/export_controller.cpp index 6992692d38469..be1190d7eb4bc 100644 --- a/Telegram/SourceFiles/export/export_controller.cpp +++ b/Telegram/SourceFiles/export/export_controller.cpp @@ -465,7 +465,9 @@ void Controller::exportNextDialog() { return false; } _messagesWritten = 0; - _messagesCount = info.messagesCount; + _messagesCount = ranges::accumulate( + info.messagesCountPerSplit, + 0); setState(stateDialogs(DownloadProgress())); return true; }, [=](DownloadProgress progress) { @@ -509,7 +511,9 @@ void Controller::exportNextLeftChannel() { return false; } _messagesWritten = 0; - _messagesCount = info.messagesCount; + _messagesCount = ranges::accumulate( + info.messagesCountPerSplit, + 0); setState(stateLeftChannels(DownloadProgress())); return true; }, [=](DownloadProgress progress) { diff --git a/Telegram/SourceFiles/export/export_pch.h b/Telegram/SourceFiles/export/export_pch.h index b089b6f1d30d3..8a0835457265b 100644 --- a/Telegram/SourceFiles/export/export_pch.h +++ b/Telegram/SourceFiles/export/export_pch.h @@ -21,6 +21,7 @@ For license and copyright information please follow this link: #include #include +#include #include #include diff --git a/Telegram/SourceFiles/export/export_settings.h b/Telegram/SourceFiles/export/export_settings.h index 618ab4de4cc7c..eb6ea4bee3324 100644 --- a/Telegram/SourceFiles/export/export_settings.h +++ b/Telegram/SourceFiles/export/export_settings.h @@ -39,21 +39,22 @@ struct MediaSettings { struct Settings { enum class Type { - PersonalInfo = 0x001, - Userpics = 0x002, - Contacts = 0x004, - Sessions = 0x008, - PersonalChats = 0x010, - BotChats = 0x020, - PrivateGroups = 0x040, - PublicGroups = 0x080, - PrivateChannels = 0x100, - PublicChannels = 0x200, + PersonalInfo = 0x001, + Userpics = 0x002, + Contacts = 0x004, + Sessions = 0x008, + PersonalChats = 0x010, + BotChats = 0x020, + PrivateGroups = 0x040, + PublicGroups = 0x080, + PrivateChannels = 0x100, + PublicChannels = 0x200, - GroupsMask = PrivateGroups | PublicGroups, - ChannelsMask = PrivateChannels | PublicChannels, - GroupsChannelsMask = GroupsMask | ChannelsMask, - AnyChatsMask = PersonalChats | BotChats | GroupsChannelsMask, + GroupsMask = PrivateGroups | PublicGroups, + ChannelsMask = PrivateChannels | PublicChannels, + GroupsChannelsMask = GroupsMask | ChannelsMask, + NonChannelChatsMask = PersonalChats | BotChats | PrivateGroups, + AnyChatsMask = PersonalChats | BotChats | GroupsChannelsMask, }; using Types = base::flags; friend inline constexpr auto is_flag_type(Type) { return true; }; diff --git a/Telegram/SourceFiles/export/output/export_output_text.cpp b/Telegram/SourceFiles/export/output/export_output_text.cpp index e69d364954afa..4d10b941689a0 100644 --- a/Telegram/SourceFiles/export/output/export_output_text.cpp +++ b/Telegram/SourceFiles/export/output/export_output_text.cpp @@ -490,7 +490,7 @@ Result TextWriter::writeUserpicsSlice(const Data::UserpicsSlice &data) { "Photo", (userpic.image.file.relativePath.isEmpty() ? QByteArray("(file unavailable)") - : userpic.image.file.relativePath.toUtf8()) + : FormatFilePath(userpic.image.file)) }, })); } @@ -792,7 +792,12 @@ Result TextWriter::writeChatEnd() { return _chats->writeBlock(SerializeKeyValue({ { "Name", NameString(_dialog.name, _dialog.type) }, { "Type", TypeString(_dialog.type) }, - { "Messages count", Data::NumberToString(_messagesCount) }, + { + (_dialog.onlyMyMessages + ? "Outgoing messages count" + : "Messages count"), + Data::NumberToString(_messagesCount) + }, { "Content", (_messagesCount > 0