diff --git a/include/config.h b/include/config.h index 3238b4b..b9ff9b5 100644 --- a/include/config.h +++ b/include/config.h @@ -19,6 +19,7 @@ struct DirectoryConfig { bool bSubDirectories; std::string sDirectory; + std::string sWinePrefix; std::string sGameSubdir; std::string sInstallersSubdir; std::string sExtrasSubdir; @@ -102,6 +103,16 @@ class GalaxyConfig return this->token_json; } + std::string getUserId() { + std::unique_lock lock(m); + + if(this->token_json.isMember("user_id")) { + return this->token_json["user_id"].asString(); + } + + return {}; + } + void setJSON(Json::Value json) { std::unique_lock lock(m); @@ -131,16 +142,34 @@ class GalaxyConfig return this->filepath; } + void resetClient() { + std::lock_guard lock(m); + if(token_json.isMember("client_id")) { + token_json["client_id"] = default_client_id; + } + if(token_json.isMember("client_secret")) { + token_json["client_secret"] = default_client_secret; + } + } + std::string getClientId() { - std::unique_lock lock(m); - return this->client_id; + std::lock_guard lock(m); + if(token_json.isMember("client_id")) { + return token_json["client_id"].asString(); + } + + return default_client_id; } std::string getClientSecret() { - std::unique_lock lock(m); - return this->client_secret; + std::lock_guard lock(m); + if(token_json.isMember("client_secret")) { + return token_json["client_secret"].asString(); + } + + return default_client_secret; } std::string getRedirectUri() @@ -154,8 +183,6 @@ class GalaxyConfig GalaxyConfig(const GalaxyConfig& other) { std::lock_guard guard(other.m); - client_id = other.client_id; - client_secret = other.client_secret; redirect_uri = other.redirect_uri; filepath = other.filepath; token_json = other.token_json; @@ -169,8 +196,6 @@ class GalaxyConfig std::unique_lock lock1(m, std::defer_lock); std::unique_lock lock2(other.m, std::defer_lock); std::lock(lock1, lock2); - client_id = other.client_id; - client_secret = other.client_secret; redirect_uri = other.redirect_uri; filepath = other.filepath; token_json = other.token_json; @@ -178,8 +203,9 @@ class GalaxyConfig } protected: private: - std::string client_id = "46899977096215655"; - std::string client_secret = "9d85c43b1482497dbbce61f6e4aa173a433796eeae2ca8c5f6129f2dc4de46d9"; + const std::string default_client_id = "46899977096215655"; + const std::string default_client_secret = "9d85c43b1482497dbbce61f6e4aa173a433796eeae2ca8c5f6129f2dc4de46d9"; + std::string redirect_uri = "https://embed.gog.com/on_login_success?origin=client"; std::string filepath; Json::Value token_json; @@ -281,6 +307,12 @@ class Config Blacklist blacklist; Blacklist ignorelist; Blacklist gamehasdlc; + + // Cloud save options + std::vector cloudWhiteList; + std::vector cloudBlackList; + bool bCloudForce; + std::string sGameHasDLCList; // Integers diff --git a/include/downloader.h b/include/downloader.h index 821f55f..b049019 100644 --- a/include/downloader.h +++ b/include/downloader.h @@ -32,9 +32,11 @@ #include #include #include +#include #include #include +class cloudSaveFile; class Timer { public: @@ -99,6 +101,14 @@ class Downloader void clearUpdateNotifications(); void repair(); void download(); + + void downloadCloudSaves(const std::string& product_id, int build_index = -1); + void downloadCloudSavesById(const std::string& product_id, int build_index = -1); + void uploadCloudSaves(const std::string& product_id, int build_index = -1); + void uploadCloudSavesById(const std::string& product_id, int build_index = -1); + void deleteCloudSaves(const std::string& product_id, int build_index = -1); + void deleteCloudSavesById(const std::string& product_id, int build_index = -1); + void checkOrphans(); void checkStatus(); void updateCache(); @@ -113,9 +123,16 @@ class Downloader void galaxyInstallGame(const std::string& product_id, int build_index = -1, const unsigned int& iGalaxyArch = GlobalConstants::ARCH_X64); void galaxyInstallGameById(const std::string& product_id, int build_index = -1, const unsigned int& iGalaxyArch = GlobalConstants::ARCH_X64); void galaxyShowBuilds(const std::string& product_id, int build_index = -1); + void galaxyShowCloudSaves(const std::string& product_id, int build_index = -1); + void galaxyShowLocalCloudSaves(const std::string& product_id, int build_index = -1); + void galaxyShowLocalCloudSavesById(const std::string& product_id, int build_index = -1); void galaxyShowBuildsById(const std::string& product_id, int build_index = -1); + void galaxyShowCloudSavesById(const std::string& product_id, int build_index = -1); protected: private: + std::map cloudSaveLocations(const std::string& product_id, int build_index); + int cloudSaveListByIdForEach(const std::string& product_id, int build_index, const std::function &f); + CURLcode downloadFile(const std::string& url, const std::string& filepath, const std::string& xml_data = std::string(), const std::string& gamename = std::string()); int repairFile(const std::string& url, const std::string& filepath, const std::string& xml_data = std::string(), const std::string& gamename = std::string()); int getGameDetails(); @@ -133,6 +150,8 @@ class Downloader static std::string getChangelogFromJSON(const Json::Value& json); void saveChangelog(const std::string& changelog, const std::string& filepath); static void processDownloadQueue(Config conf, const unsigned int& tid); + static void processCloudSaveDownloadQueue(Config conf, const unsigned int& tid); + static void processCloudSaveUploadQueue(Config conf, const unsigned int& tid); static int progressCallbackForThread(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow); template void printProgress(const ThreadSafeQueue& download_queue); static void getGameDetailsThread(Config config, const unsigned int& tid); diff --git a/include/galaxyapi.h b/include/galaxyapi.h index 03b6e0b..dd60b56 100644 --- a/include/galaxyapi.h +++ b/include/galaxyapi.h @@ -46,16 +46,20 @@ class galaxyAPI galaxyAPI(CurlConfig& conf); virtual ~galaxyAPI(); int init(); + bool isTokenExpired(); bool refreshLogin(); + bool refreshLogin(const std::string &clientId, const std::string &clientSecret, const std::string &refreshToken, bool newSession); + Json::Value getProductBuilds(const std::string& product_id, const std::string& platform = "windows", const std::string& generation = "2"); Json::Value getManifestV1(const std::string& product_id, const std::string& build_id, const std::string& manifest_id = "repository", const std::string& platform = "windows"); Json::Value getManifestV1(const std::string& manifest_url); Json::Value getManifestV2(std::string manifest_hash, const bool& is_dependency = false); + Json::Value getCloudPathAsJson(const std::string &clientId); Json::Value getSecureLink(const std::string& product_id, const std::string& path); Json::Value getDependencyLink(const std::string& path); - std::string getResponse(const std::string& url); - Json::Value getResponseJson(const std::string& url); + std::string getResponse(const std::string& url, const char *encoding = nullptr); + Json::Value getResponseJson(const std::string& url, const char *encoding = nullptr); std::string hashToGalaxyPath(const std::string& hash); std::vector getDepotItemsVector(const std::string& hash, const bool& is_dependency = false); Json::Value getProductInfo(const std::string& product_id); diff --git a/include/util.h b/include/util.h index b1d9007..4b2a22a 100644 --- a/include/util.h +++ b/include/util.h @@ -66,6 +66,7 @@ namespace Util int createXML(std::string filepath, uintmax_t chunk_size, std::string xml_dir = std::string()); int getGameSpecificConfig(std::string gamename, gameSpecificConfig* conf, std::string directory = std::string()); int replaceString(std::string& str, const std::string& to_replace, const std::string& replace_with); + int replaceAllString(std::string& str, const std::string& to_replace, const std::string& replace_with); void filepathReplaceReservedStrings(std::string& str, const std::string& gamename, const unsigned int& platformId = 0, const std::string& dlcname = ""); void setFilePermissions(const boost::filesystem::path& path, const boost::filesystem::perms& permissions); int getTerminalWidth(); @@ -90,6 +91,7 @@ namespace Util CURLcode CurlHandleGetResponse(CURL* curlhandle, std::string& response, int max_retries = -1); curl_off_t CurlWriteMemoryCallback(char *ptr, curl_off_t size, curl_off_t nmemb, void *userp); curl_off_t CurlWriteChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, void *userp); + curl_off_t CurlReadChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, ChunkMemoryStruct *userp); template std::string formattedString(const std::string& format, Args ... args) { diff --git a/main.cpp b/main.cpp index e7edaba..d54fbf1 100644 --- a/main.cpp +++ b/main.cpp @@ -24,6 +24,18 @@ template void set_vm_value(std::map vFileIdStrings; @@ -213,6 +230,11 @@ int main(int argc, char *argv[]) ("cacert", bpo::value(&Globals::globalConfig.curlConf.sCACertPath)->default_value(""), "Path to CA certificate bundle in PEM format") ("respect-umask", bpo::value(&Globals::globalConfig.bRespectUmask)->zero_tokens()->default_value(false), "Do not adjust permissions of sensitive files") ("user-agent", bpo::value(&Globals::globalConfig.curlConf.sUserAgent)->default_value(DEFAULT_USER_AGENT), "Set user agent") + + ("wine-prefix", bpo::value(&Globals::globalConfig.dirConf.sWinePrefix)->default_value("."), "Set wineprefix directory") + ("cloud-whitelist", bpo::value>(&Globals::globalConfig.cloudWhiteList)->multitoken(), "Include this list of cloud saves, by default all cloud saves are included\n Example: --cloud-whitelist saves/AutoSave-0 saves/AutoSave-1/screenshot.png") + ("cloud-blacklist", bpo::value>(&Globals::globalConfig.cloudBlackList)->multitoken(), "Exclude this list of cloud saves\n Example: --cloud-blacklist saves/AutoSave-0 saves/AutoSave-1/screenshot.png") + ("cloud-force", bpo::value(&Globals::globalConfig.bCloudForce)->zero_tokens()->default_value(false), "Download or Upload cloud saves even if they're up-to-date\nDelete remote cloud saves even if no saves are whitelisted") #ifdef USE_QT_GUI_LOGIN ("enable-login-gui", bpo::value(&Globals::globalConfig.bEnableLoginGUI)->zero_tokens()->default_value(false), "Enable login GUI when encountering reCAPTCHA on login form") #endif @@ -271,6 +293,11 @@ int main(int argc, char *argv[]) options_cli_experimental.add_options() ("galaxy-install", bpo::value(&galaxy_product_id_install)->default_value(""), "Install game using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") ("galaxy-show-builds", bpo::value(&galaxy_product_id_show_builds)->default_value(""), "Show game builds using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") + ("galaxy-download-cloud-saves", bpo::value(&galaxy_product_cloud_saves)->default_value(""), "Download cloud saves using product-id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") + ("galaxy-upload-cloud-saves", bpo::value(&galaxy_upload_product_cloud_saves)->default_value(""), "Upload cloud saves using product-id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") + ("galaxy-show-cloud-saves", bpo::value(&galaxy_product_id_show_cloud_paths)->default_value(""), "Show game cloud-saves using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") + ("galaxy-show-local-cloud-saves", bpo::value(&galaxy_product_id_show_local_cloud_paths)->default_value(""), "Show local cloud-saves using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") + ("galaxy-delete-cloud-saves", bpo::value(&galaxy_product_cloud_saves_delete)->default_value(""), "Delete cloud-saves using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") ("galaxy-platform", bpo::value(&sGalaxyPlatform)->default_value("w"), galaxy_platform_text.c_str()) ("galaxy-language", bpo::value(&sGalaxyLanguage)->default_value("en"), galaxy_language_text.c_str()) ("galaxy-arch", bpo::value(&sGalaxyArch)->default_value("x64"), galaxy_arch_text.c_str()) @@ -578,15 +605,8 @@ int main(int argc, char *argv[]) } // Make sure that directory has trailing slash - if (!Globals::globalConfig.dirConf.sDirectory.empty()) - { - if (Globals::globalConfig.dirConf.sDirectory.at(Globals::globalConfig.dirConf.sDirectory.length()-1)!='/') - Globals::globalConfig.dirConf.sDirectory += "/"; - } - else - { - Globals::globalConfig.dirConf.sDirectory = "./"; // Directory wasn't specified, use current directory - } + ensure_trailing_slash(Globals::globalConfig.dirConf.sDirectory, "./"); + ensure_trailing_slash(Globals::globalConfig.dirConf.sWinePrefix, "./"); // CA certificate bundle if (Globals::globalConfig.curlConf.sCACertPath.empty()) @@ -777,6 +797,39 @@ int main(int argc, char *argv[]) } downloader.galaxyShowBuilds(product_id, build_index); } + else if (!galaxy_product_id_show_cloud_paths.empty()) + { + int build_index = -1; + std::vector tokens = Util::tokenize(galaxy_product_id_show_cloud_paths, "/"); + std::string product_id = tokens[0]; + if (tokens.size() == 2) + { + build_index = std::stoi(tokens[1]); + } + downloader.galaxyShowCloudSaves(product_id, build_index); + } + else if (!galaxy_product_id_show_local_cloud_paths.empty()) + { + int build_index = -1; + std::vector tokens = Util::tokenize(galaxy_product_id_show_local_cloud_paths, "/"); + std::string product_id = tokens[0]; + if (tokens.size() == 2) + { + build_index = std::stoi(tokens[1]); + } + downloader.galaxyShowLocalCloudSaves(product_id, build_index); + } + else if (!galaxy_product_cloud_saves_delete.empty()) + { + int build_index = -1; + std::vector tokens = Util::tokenize(galaxy_product_cloud_saves_delete, "/"); + std::string product_id = tokens[0]; + if (tokens.size() == 2) + { + build_index = std::stoi(tokens[1]); + } + downloader.deleteCloudSaves(product_id, build_index); + } else if (!galaxy_product_id_install.empty()) { int build_index = -1; @@ -788,6 +841,26 @@ int main(int argc, char *argv[]) } downloader.galaxyInstallGame(product_id, build_index, Globals::globalConfig.dlConf.iGalaxyArch); } + else if (!galaxy_product_cloud_saves.empty()) { + int build_index = -1; + std::vector tokens = Util::tokenize(galaxy_product_cloud_saves, "/"); + std::string product_id = tokens[0]; + if (tokens.size() == 2) + { + build_index = std::stoi(tokens[1]); + } + downloader.downloadCloudSaves(product_id, build_index); + } + else if (!galaxy_upload_product_cloud_saves.empty()) { + int build_index = -1; + std::vector tokens = Util::tokenize(galaxy_upload_product_cloud_saves, "/"); + std::string product_id = tokens[0]; + if (tokens.size() == 2) + { + build_index = std::stoi(tokens[1]); + } + downloader.uploadCloudSaves(product_id, build_index); + } else { if (!Globals::globalConfig.bLogin) diff --git a/src/downloader.cpp b/src/downloader.cpp index 159d06d..c2c3a1c 100644 --- a/src/downloader.cpp +++ b/src/downloader.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -39,8 +40,16 @@ namespace bptime = boost::posix_time; +struct cloudSaveFile { + boost::posix_time::ptime lastModified; + unsigned long long fileSize; + std::string path; + std::string location; +}; + std::vector vDownloadInfo; ThreadSafeQueue dlQueue; +ThreadSafeQueue dlCloudSaveQueue; ThreadSafeQueue msgQueue; ThreadSafeQueue createXMLQueue; ThreadSafeQueue gameItemQueue; @@ -50,6 +59,55 @@ ThreadSafeQueue dlQueueGalaxy_MojoSetupHack; std::mutex mtx_create_directories; // Mutex for creating directories in Downloader::processDownloadQueue std::atomic iTotalRemainingBytes(0); +std::string username() { + auto user = std::getenv("USER"); + return user ? user : std::string(); +} + +void dirForEachHelper(const boost::filesystem::path &location, std::function &f) { + boost::filesystem::directory_iterator begin { location }; + boost::filesystem::directory_iterator end; + + for(boost::filesystem::directory_iterator curr_dir { begin }; curr_dir != end; ++curr_dir) { + if(boost::filesystem::is_directory(*curr_dir)) { + + dirForEachHelper(*curr_dir, f); + } + else { + f(curr_dir); + } + } +} + +void dirForEach(const std::string &location, std::function &&f) { + dirForEachHelper(location, f); +} + +bool whitelisted(const std::string &path) { + auto &whitelist = Globals::globalConfig.cloudWhiteList; + auto &blacklist = Globals::globalConfig.cloudBlackList; + + // Check if path is whitelisted + if(!whitelist.empty()) { + return std::any_of(std::begin(whitelist), std::end(whitelist), [&path](const std::string &whitelisted) { + return + path.rfind(whitelisted, 0) == 0 && + (path.size() == whitelisted.size() || path[whitelisted.size()] == '/'); + }); + } + + // Check if blacklisted + if(!blacklist.empty()) { + return !std::any_of(std::begin(blacklist), std::end(blacklist), [&path](const std::string &blacklisted) { + return + path.rfind(blacklisted, 0) == 0 && + (path.size() == blacklisted.size() || path[blacklisted.size()] == '/'); + }); + } + + return true; +} + Downloader::Downloader() { if (Globals::globalConfig.bLogin) @@ -2516,16 +2574,14 @@ void Downloader::showWishlist() return; } -void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) -{ +void Downloader::processCloudSaveUploadQueue(Config conf, const unsigned int& tid) { std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]"; - galaxyAPI* galaxy = new galaxyAPI(Globals::globalConfig.curlConf); + std::unique_ptr galaxy { new galaxyAPI(Globals::globalConfig.curlConf) }; if (!galaxy->init()) { if (!galaxy->refreshLogin()) { - delete galaxy; msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); return; @@ -2533,10 +2589,10 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) } CURL* dlhandle = curl_easy_init(); + Util::CurlHandleSetDefaultOptions(dlhandle, conf.curlConf); curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0); - curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData); - curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData); + curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Util::CurlReadChunkMemoryCallback); curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L); xferInfo xferinfo; @@ -2546,253 +2602,228 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread); curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo); - gameFile gf; - while (dlQueue.try_pop(gf)) - { + cloudSaveFile csf; + + std::string access_token; + if (!Globals::galaxyConf.isExpired()) { + access_token = Globals::galaxyConf.getAccessToken(); + } + + if (access_token.empty()) { + return; + } + + std::string bearer = "Authorization: Bearer " + access_token; + + while(dlCloudSaveQueue.try_pop(csf)) { CURLcode result = CURLE_RECV_ERROR; // assume network error int iRetryCount = 0; - off_t iResumePosition = 0; - vDownloadInfo[tid].setStatus(DLSTATUS_STARTING); + iTotalRemainingBytes.fetch_sub(csf.fileSize); - unsigned long long filesize = 0; - try - { - filesize = std::stoll(gf.size); - } - catch (std::invalid_argument& e) + vDownloadInfo[tid].setFilename(csf.path); + + std::string filecontents; { - filesize = 0; - } - iTotalRemainingBytes.fetch_sub(filesize); + std::ifstream in { csf.location, std::ios_base::in | std::ios_base::binary }; - // Get directory from filepath - boost::filesystem::path filepath = gf.getFilepath(); - filepath = boost::filesystem::absolute(filepath, boost::filesystem::current_path()); - boost::filesystem::path directory = filepath.parent_path(); + in >> filecontents; - // Skip blacklisted files - if (conf.blacklist.isBlacklisted(filepath.string())) - { - msgQueue.push(Message("Blacklisted file: " + filepath.string(), MSGTYPE_INFO, msg_prefix)); - continue; + in.close(); } - std::string filenameXML = filepath.filename().string() + ".xml"; - std::string xml_directory = conf.sXMLDirectory + "/" + gf.gamename; - boost::filesystem::path local_xml_file = xml_directory + "/" + filenameXML; + ChunkMemoryStruct cms { + &filecontents[0], + (curl_off_t)filecontents.size() + }; + + auto md5 = Util::getChunkHash((std::uint8_t*)filecontents.data(), filecontents.size(), RHASH_MD5); - vDownloadInfo[tid].setFilename(filepath.filename().string()); - msgQueue.push(Message("Begin download: " + filepath.filename().string(), MSGTYPE_INFO, msg_prefix)); + auto url = "https://cloudstorage.gog.com/v1/" + Globals::galaxyConf.getUserId() + '/' + Globals::galaxyConf.getClientId() + '/' + csf.path; - // Check that directory exists and create subdirectories - mtx_create_directories.lock(); // Use mutex to avoid possible race conditions - if (boost::filesystem::exists(directory)) + curl_slist *header = nullptr; + header = curl_slist_append(header, bearer.c_str()); + header = curl_slist_append(header, ("X-Object-Meta-LocalLastModified: " + boost::posix_time::to_iso_extended_string(csf.lastModified)).c_str()); + header = curl_slist_append(header, ("Etag: " + md5).c_str()); + header = curl_slist_append(header, "Content-Type: Octet-Stream"); + header = curl_slist_append(header, ("Content-Length: " + std::to_string(filecontents.size())).c_str()); + + curl_easy_setopt(dlhandle, CURLOPT_PUT, 1L); + curl_easy_setopt(dlhandle, CURLOPT_CUSTOMREQUEST, "PUT"); + curl_easy_setopt(dlhandle, CURLOPT_HTTPHEADER, header); + curl_easy_setopt(dlhandle, CURLOPT_READDATA, &cms); + curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str()); + + msgQueue.push(Message("Begin upload: " + csf.path, MSGTYPE_INFO, msg_prefix)); + + bool bShouldRetry = false; + long int response_code = 0; + std::string retry_reason; + do { - if (!boost::filesystem::is_directory(directory)) - { - mtx_create_directories.unlock(); - msgQueue.push(Message(directory.string() + " is not directory, skipping file (" + filepath.filename().string() + ")", MSGTYPE_WARNING, msg_prefix)); - continue; - } - else + if (conf.iWait > 0) + usleep(conf.iWait); // Wait before continuing + + response_code = 0; // Make sure that response code is reset + + if (iRetryCount != 0) { - mtx_create_directories.unlock(); + std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + boost::filesystem::path(csf.location).filename().string(); + if (!retry_reason.empty()) + retry_msg += " (" + retry_reason + ")"; + msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix)); } - } - else - { - if (!boost::filesystem::create_directories(directory)) + retry_reason.clear(); // reset retry reason + + xferinfo.offset = 0; + xferinfo.timer.reset(); + xferinfo.TimeAndSize.clear(); + result = curl_easy_perform(dlhandle); + + switch (result) { - mtx_create_directories.unlock(); - msgQueue.push(Message("Failed to create directory (" + directory.string() + "), skipping file (" + filepath.filename().string() + ")", MSGTYPE_ERROR, msg_prefix)); - continue; + // Retry on these errors + case CURLE_PARTIAL_FILE: + case CURLE_OPERATION_TIMEDOUT: + case CURLE_RECV_ERROR: + case CURLE_SSL_CONNECT_ERROR: + bShouldRetry = true; + break; + // Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "416 Range Not Satisfiable" + case CURLE_HTTP_RETURNED_ERROR: + curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code); + if (response_code == 416 || response_code == 422 || response_code == 400 || response_code == 422) { + msgQueue.push(Message(std::to_string(response_code) + ": " + curl_easy_strerror(result))); + bShouldRetry = false; + } + else + bShouldRetry = true; + break; + default: + bShouldRetry = false; + break; } - else - { - mtx_create_directories.unlock(); + + if (bShouldRetry) { + iRetryCount++; + retry_reason = std::to_string(response_code) + ": " + curl_easy_strerror(result); } - } + } while (bShouldRetry && (iRetryCount <= conf.iRetries)); - bool bSameVersion = true; // assume same version - bool bLocalXMLExists = boost::filesystem::exists(local_xml_file); // This is additional check to see if remote xml should be saved to speed up future version checks + curl_slist_free_all(header); + } - // Refresh Galaxy login if token is expired - if (galaxy->isTokenExpired()) + curl_easy_cleanup(dlhandle); + + vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); + msgQueue.push(Message("Finished all tasks", MSGTYPE_INFO, msg_prefix)); +} + +void Downloader::processCloudSaveDownloadQueue(Config conf, const unsigned int& tid) { + std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]"; + + std::unique_ptr galaxy { new galaxyAPI(Globals::globalConfig.curlConf) }; + if (!galaxy->init()) + { + if (!galaxy->refreshLogin()) { - if (!galaxy->refreshLogin()) - { - msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix)); - vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); - delete galaxy; - return; - } + msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix)); + vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); + return; } + } - // Get downlink JSON from Galaxy API - Json::Value downlinkJson = galaxy->getResponseJson(gf.galaxy_downlink_json_url); + CURL* dlhandle = curl_easy_init(); + + Util::CurlHandleSetDefaultOptions(dlhandle, conf.curlConf); - if (downlinkJson.empty()) + curl_slist *header = nullptr; + + std::string access_token; + if (!Globals::galaxyConf.isExpired()) + access_token = Globals::galaxyConf.getAccessToken(); + if (!access_token.empty()) + { + std::string bearer = "Authorization: Bearer " + access_token; + header = curl_slist_append(header, bearer.c_str()); + } + + curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0); + curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData); + curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData); + curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L); + + xferInfo xferinfo; + xferinfo.tid = tid; + xferinfo.curlhandle = dlhandle; + + curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread); + curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo); + + cloudSaveFile csf; + while(dlCloudSaveQueue.try_pop(csf)) { + CURLcode result = CURLE_RECV_ERROR; // assume network error + int iRetryCount = 0; + off_t iResumePosition = 0; + + bool bResume = false; + + iTotalRemainingBytes.fetch_sub(csf.fileSize); + + // Get directory from filepath + boost::filesystem::path filepath = csf.location + ".~incomplete"; + filepath = boost::filesystem::absolute(filepath, boost::filesystem::current_path()); + boost::filesystem::path directory = filepath.parent_path(); + + vDownloadInfo[tid].setFilename(csf.path); + + bResume = boost::filesystem::exists(filepath); + + msgQueue.push(Message("Begin download: " + csf.path, MSGTYPE_INFO, msg_prefix)); + + // Check that directory exists and create subdirectories + std::unique_lock ul { mtx_create_directories }; // Use mutex to avoid possible race conditions + if (boost::filesystem::exists(directory)) { - msgQueue.push(Message("Empty JSON response, skipping file", MSGTYPE_WARNING, msg_prefix)); - continue; + if (!boost::filesystem::is_directory(directory)) { + msgQueue.push(Message(directory.string() + " is not directory, skipping file (" + filepath.filename().string() + ")", MSGTYPE_WARNING, msg_prefix)); + continue; + } } - - if (!downlinkJson.isMember("downlink")) + else if (!boost::filesystem::create_directories(directory)) { - msgQueue.push(Message("Invalid JSON response, skipping file", MSGTYPE_WARNING, msg_prefix)); + msgQueue.push(Message("Failed to create directory (" + directory.string() + "), skipping file (" + filepath.filename().string() + ")", MSGTYPE_ERROR, msg_prefix)); continue; } - std::string xml; - if (gf.type & (GFTYPE_INSTALLER | GFTYPE_PATCH) && conf.dlConf.bRemoteXML) + auto url = "https://cloudstorage.gog.com/v1/" + Globals::galaxyConf.getUserId() + '/' + Globals::galaxyConf.getClientId() + '/' + csf.path; + curl_easy_setopt(dlhandle, CURLOPT_HTTPHEADER, header); + curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str()); + long int response_code = 0; + bool bShouldRetry = false; + std::string retry_reason; + do { - std::string xml_url; - if (downlinkJson.isMember("checksum")) - if (!downlinkJson["checksum"].empty()) - xml_url = downlinkJson["checksum"].asString(); + if (conf.iWait > 0) + usleep(conf.iWait); // Wait before continuing - // Get XML data - if (conf.dlConf.bRemoteXML && !xml_url.empty()) - xml = galaxy->getResponse(xml_url); + response_code = 0; // Make sure that response code is reset - if (!xml.empty() && !Globals::globalConfig.bSizeOnly) + if (iRetryCount != 0) { - std::string localHash = Util::getLocalFileHash(conf.sXMLDirectory, filepath.string(), gf.gamename); - // Do version check if local hash exists - if (!localHash.empty()) - { - tinyxml2::XMLDocument remote_xml; - remote_xml.Parse(xml.c_str()); - tinyxml2::XMLElement *fileElem = remote_xml.FirstChildElement("file"); - if (fileElem) - { - std::string remoteHash = fileElem->Attribute("md5"); - if (remoteHash != localHash) - bSameVersion = false; - } - } + std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath.filename().string(); + if (!retry_reason.empty()) + retry_msg += " (" + retry_reason + ")"; + msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix)); } - } + retry_reason = ""; // reset retry reason - bool bIsComplete = false; - bool bResume = false; - if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) - { - if (bSameVersion) + FILE* outfile; + // If a file was partially downloaded + if (bResume) { - bResume = true; - - // Check if file is complete so we can skip it instead of resuming - if (!xml.empty()) - { - off_t filesize_xml; - off_t filesize_local = boost::filesystem::file_size(filepath); - - tinyxml2::XMLDocument remote_xml; - remote_xml.Parse(xml.c_str()); - tinyxml2::XMLElement *fileElem = remote_xml.FirstChildElement("file"); - if (fileElem) - { - std::string total_size = fileElem->Attribute("total_size"); - try - { - filesize_xml = std::stoull(total_size); - } - catch (std::invalid_argument& e) - { - filesize_xml = 0; - } - if (filesize_local == filesize_xml) - { - msgQueue.push(Message("Skipping complete file: " + filepath.filename().string(), MSGTYPE_INFO, msg_prefix)); - bIsComplete = true; // Set to true so we can skip after saving xml data - } - } - } - } - else - { - msgQueue.push(Message("Remote file is different, renaming local file", MSGTYPE_INFO, msg_prefix)); - std::string date_old = "." + bptime::to_iso_string(bptime::second_clock::local_time()) + ".old"; - boost::filesystem::path new_name = filepath.string() + date_old; // Rename old file by appending date and ".old" to filename - boost::system::error_code ec; - boost::filesystem::rename(filepath, new_name, ec); // Rename the file - if (ec) - { - msgQueue.push(Message("Failed to rename " + filepath.string() + " to " + new_name.string() + " - Skipping file", MSGTYPE_WARNING, msg_prefix)); - continue; - } - } - } - - // Save remote XML - if (!xml.empty()) - { - if ((bLocalXMLExists && !bSameVersion) || !bLocalXMLExists) - { - // Check that directory exists and create subdirectories - boost::filesystem::path path = xml_directory; - mtx_create_directories.lock(); // Use mutex to avoid race conditions - if (boost::filesystem::exists(path)) - { - if (!boost::filesystem::is_directory(path)) - { - msgQueue.push(Message(path.string() + " is not directory", MSGTYPE_WARNING, msg_prefix)); - } - } - else - { - if (!boost::filesystem::create_directories(path)) - { - msgQueue.push(Message("Failed to create directory: " + path.string(), MSGTYPE_ERROR, msg_prefix)); - } - } - mtx_create_directories.unlock(); - std::ofstream ofs(local_xml_file.string().c_str()); - if (ofs) - { - ofs << xml; - ofs.close(); - } - else - { - msgQueue.push(Message("Can't create " + local_xml_file.string(), MSGTYPE_ERROR, msg_prefix)); - } - } - } - - // File was complete and we have saved xml data so we can skip it - if (bIsComplete) - continue; - - std::string url = downlinkJson["downlink"].asString(); - curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str()); - long int response_code = 0; - bool bShouldRetry = false; - std::string retry_reason; - do - { - if (conf.iWait > 0) - usleep(conf.iWait); // Wait before continuing - - response_code = 0; // Make sure that response code is reset - - if (iRetryCount != 0) - { - std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath.filename().string(); - if (!retry_reason.empty()) - retry_msg += " (" + retry_reason + ")"; - msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix)); - } - retry_reason = ""; // reset retry reason - - FILE* outfile; - // File exists, resume - if (bResume) - { - iResumePosition = boost::filesystem::file_size(filepath); - if ((outfile=fopen(filepath.string().c_str(), "r+"))!=NULL) + iResumePosition = boost::filesystem::file_size(filepath); + if ((outfile=fopen(filepath.string().c_str(), "r+"))!=NULL) { fseek(outfile, 0, SEEK_END); curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, iResumePosition); @@ -2850,8 +2881,9 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) { iRetryCount++; retry_reason = std::string(curl_easy_strerror(result)); - if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) + if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) { bResume = true; + } } } while (bShouldRetry && (iRetryCount <= conf.iRetries)); @@ -2859,6 +2891,7 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) if (result == CURLE_OK || result == CURLE_RANGE_ERROR || (result == CURLE_HTTP_RETURNED_ERROR && response_code == 416)) { // Set timestamp for downloaded file to same value as file on server + // and rename "filename.~incomplete" to "filename" long filetime = -1; CURLcode res = curl_easy_getinfo(dlhandle, CURLINFO_FILETIME, &filetime); if (res == CURLE_OK && filetime >= 0) @@ -2866,11 +2899,12 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) std::time_t timestamp = (std::time_t)filetime; try { - boost::filesystem::last_write_time(filepath, timestamp); + boost::filesystem::rename(filepath, csf.location); + boost::filesystem::last_write_time(csf.location, timestamp); } catch(const boost::filesystem::filesystem_error& e) { - msgQueue.push(Message(e.what(), MSGTYPE_WARNING, msg_prefix)); + msgQueue.push(Message(e.what(), MSGTYPE_ERROR, msg_prefix)); } } @@ -2890,7 +2924,7 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) } dlrate_avg << std::setprecision(2) << std::fixed << progress_info.rate_avg << rate_unit; - msgQueue.push(Message("Download complete: " + filepath.filename().string() + " (@ " + dlrate_avg.str() + ")", MSGTYPE_SUCCESS, msg_prefix)); + msgQueue.push(Message("Download complete: " + csf.path + " (@ " + dlrate_avg.str() + ")", MSGTYPE_SUCCESS, msg_prefix)); } else { @@ -2906,129 +2940,532 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) if ((result != CURLE_PARTIAL_FILE && !bResume && result != CURLE_OPERATION_TIMEDOUT) || boost::filesystem::file_size(filepath) == 0) { if (!boost::filesystem::remove(filepath)) - msgQueue.push(Message("Failed to delete " + filepath.filename().string(), MSGTYPE_ERROR, msg_prefix)); + msgQueue.push(Message("Failed to delete " + filepath.string(), MSGTYPE_ERROR, msg_prefix)); } } } - - // Automatic xml creation - if (conf.dlConf.bAutomaticXMLCreation) - { - if (result == CURLE_OK) - { - if ((gf.type & GFTYPE_EXTRA) || (conf.dlConf.bRemoteXML && !bLocalXMLExists && xml.empty())) - createXMLQueue.push(gf); - } - } } + curl_slist_free_all(header); curl_easy_cleanup(dlhandle); - delete galaxy; vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); msgQueue.push(Message("Finished all tasks", MSGTYPE_INFO, msg_prefix)); - - return; } -int Downloader::progressCallbackForThread(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) +void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) { - // unused so lets prevent warnings and be more pedantic - (void) ulnow; - (void) ultotal; - - xferInfo* xferinfo = static_cast(clientp); + std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]"; - // Update progress info every 100ms - if (xferinfo->timer.getTimeBetweenUpdates()>=100 || dlnow == dltotal) + galaxyAPI* galaxy = new galaxyAPI(Globals::globalConfig.curlConf); + if (!galaxy->init()) { - xferinfo->timer.reset(); - progressInfo info; - info.dlnow = dlnow; - info.dltotal = dltotal; - - // trying to get rate and setting to NaN if it fails - if (CURLE_OK != curl_easy_getinfo(xferinfo->curlhandle, CURLINFO_SPEED_DOWNLOAD, &info.rate_avg)) - info.rate_avg = std::numeric_limits::quiet_NaN(); - - // setting full dlwnow and dltotal - if (xferinfo->offset > 0) + if (!galaxy->refreshLogin()) { - info.dlnow += xferinfo->offset; - info.dltotal += xferinfo->offset; + delete galaxy; + msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix)); + vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); + return; } + } - // 10 second average download speed - // Don't use static value of 10 seconds because update interval depends on when and how often progress callback is called - xferinfo->TimeAndSize.push_back(std::make_pair(time(NULL), static_cast(info.dlnow))); - if (xferinfo->TimeAndSize.size() > 100) // 100 * 100ms = 10s - { - xferinfo->TimeAndSize.pop_front(); - time_t time_first = xferinfo->TimeAndSize.front().first; - uintmax_t size_first = xferinfo->TimeAndSize.front().second; - time_t time_last = xferinfo->TimeAndSize.back().first; - uintmax_t size_last = xferinfo->TimeAndSize.back().second; - info.rate = (size_last - size_first) / static_cast((time_last - time_first)); - } - else - { - info.rate = info.rate_avg; - } + CURL* dlhandle = curl_easy_init(); + Util::CurlHandleSetDefaultOptions(dlhandle, conf.curlConf); + curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0); + curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData); + curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData); + curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L); - vDownloadInfo[xferinfo->tid].setProgressInfo(info); - vDownloadInfo[xferinfo->tid].setStatus(DLSTATUS_RUNNING); - } + xferInfo xferinfo; + xferinfo.tid = tid; + xferinfo.curlhandle = dlhandle; - return 0; -} + curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread); + curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo); -template void Downloader::printProgress(const ThreadSafeQueue& download_queue) -{ - // Print progress information until all threads have finished their tasks - ProgressBar bar(Globals::globalConfig.bUnicode, Globals::globalConfig.bColor); - unsigned int dl_status = DLSTATUS_NOTSTARTED; - while (dl_status != DLSTATUS_FINISHED) + gameFile gf; + while (dlQueue.try_pop(gf)) { - dl_status = DLSTATUS_NOTSTARTED; + CURLcode result = CURLE_RECV_ERROR; // assume network error + int iRetryCount = 0; + off_t iResumePosition = 0; - // Print progress information once per 100ms - std::this_thread::sleep_for(std::chrono::milliseconds(Globals::globalConfig.iProgressInterval)); - std::cout << "\033[J\r" << std::flush; // Clear screen from the current line down to the bottom of the screen + vDownloadInfo[tid].setStatus(DLSTATUS_STARTING); - // Print messages from message queue first - Message msg; - while (msgQueue.try_pop(msg)) + unsigned long long filesize = 0; + try { - std::cout << msg.getFormattedString(Globals::globalConfig.bColor, true) << std::endl; - if (Globals::globalConfig.bReport) - { - this->report_ofs << msg.getTimestampString() << ": " << msg.getMessage() << std::endl; - } + filesize = std::stoll(gf.size); + } + catch (std::invalid_argument& e) + { + filesize = 0; } + iTotalRemainingBytes.fetch_sub(filesize); - int iTermWidth = Util::getTerminalWidth(); - double total_rate = 0; - bptime::time_duration eta_total_seconds; + // Get directory from filepath + boost::filesystem::path filepath = gf.getFilepath(); + filepath = boost::filesystem::absolute(filepath, boost::filesystem::current_path()); + boost::filesystem::path directory = filepath.parent_path(); - // Create progress info text for all download threads - std::vector vProgressText; - for (unsigned int i = 0; i < vDownloadInfo.size(); ++i) + // Skip blacklisted files + if (conf.blacklist.isBlacklisted(filepath.string())) { - std::string progress_text; - int bar_length = 26; - int min_bar_length = 5; + msgQueue.push(Message("Blacklisted file: " + filepath.string(), MSGTYPE_INFO, msg_prefix)); + continue; + } - unsigned int status = vDownloadInfo[i].getStatus(); - dl_status |= status; + std::string filenameXML = filepath.filename().string() + ".xml"; + std::string xml_directory = conf.sXMLDirectory + "/" + gf.gamename; + boost::filesystem::path local_xml_file = xml_directory + "/" + filenameXML; - if (status == DLSTATUS_FINISHED) + vDownloadInfo[tid].setFilename(filepath.filename().string()); + msgQueue.push(Message("Begin download: " + filepath.filename().string(), MSGTYPE_INFO, msg_prefix)); + + // Check that directory exists and create subdirectories + mtx_create_directories.lock(); // Use mutex to avoid possible race conditions + if (boost::filesystem::exists(directory)) + { + if (!boost::filesystem::is_directory(directory)) { - vProgressText.push_back("#" + std::to_string(i) + ": Finished"); + mtx_create_directories.unlock(); + msgQueue.push(Message(directory.string() + " is not directory, skipping file (" + filepath.filename().string() + ")", MSGTYPE_WARNING, msg_prefix)); continue; } - - std::string filename = vDownloadInfo[i].getFilename(); - progressInfo progress_info = vDownloadInfo[i].getProgressInfo(); + else + { + mtx_create_directories.unlock(); + } + } + else + { + if (!boost::filesystem::create_directories(directory)) + { + mtx_create_directories.unlock(); + msgQueue.push(Message("Failed to create directory (" + directory.string() + "), skipping file (" + filepath.filename().string() + ")", MSGTYPE_ERROR, msg_prefix)); + continue; + } + else + { + mtx_create_directories.unlock(); + } + } + + bool bSameVersion = true; // assume same version + bool bLocalXMLExists = boost::filesystem::exists(local_xml_file); // This is additional check to see if remote xml should be saved to speed up future version checks + + // Refresh Galaxy login if token is expired + if (galaxy->isTokenExpired()) + { + if (!galaxy->refreshLogin()) + { + msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix)); + vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); + delete galaxy; + return; + } + } + + // Get downlink JSON from Galaxy API + Json::Value downlinkJson = galaxy->getResponseJson(gf.galaxy_downlink_json_url); + + if (downlinkJson.empty()) + { + msgQueue.push(Message("Empty JSON response, skipping file", MSGTYPE_WARNING, msg_prefix)); + continue; + } + + if (!downlinkJson.isMember("downlink")) + { + msgQueue.push(Message("Invalid JSON response, skipping file", MSGTYPE_WARNING, msg_prefix)); + continue; + } + + std::string xml; + if (gf.type & (GFTYPE_INSTALLER | GFTYPE_PATCH) && conf.dlConf.bRemoteXML) + { + std::string xml_url; + if (downlinkJson.isMember("checksum")) + if (!downlinkJson["checksum"].empty()) + xml_url = downlinkJson["checksum"].asString(); + + // Get XML data + if (conf.dlConf.bRemoteXML && !xml_url.empty()) + xml = galaxy->getResponse(xml_url); + + if (!xml.empty() && !Globals::globalConfig.bSizeOnly) + { + std::string localHash = Util::getLocalFileHash(conf.sXMLDirectory, filepath.string(), gf.gamename); + // Do version check if local hash exists + if (!localHash.empty()) + { + tinyxml2::XMLDocument remote_xml; + remote_xml.Parse(xml.c_str()); + tinyxml2::XMLElement *fileElem = remote_xml.FirstChildElement("file"); + if (fileElem) + { + std::string remoteHash = fileElem->Attribute("md5"); + if (remoteHash != localHash) + bSameVersion = false; + } + } + } + } + + bool bIsComplete = false; + bool bResume = false; + if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) + { + if (bSameVersion) + { + bResume = true; + + // Check if file is complete so we can skip it instead of resuming + if (!xml.empty()) + { + off_t filesize_xml; + off_t filesize_local = boost::filesystem::file_size(filepath); + + tinyxml2::XMLDocument remote_xml; + remote_xml.Parse(xml.c_str()); + tinyxml2::XMLElement *fileElem = remote_xml.FirstChildElement("file"); + if (fileElem) + { + std::string total_size = fileElem->Attribute("total_size"); + try + { + filesize_xml = std::stoull(total_size); + } + catch (std::invalid_argument& e) + { + filesize_xml = 0; + } + if (filesize_local == filesize_xml) + { + msgQueue.push(Message("Skipping complete file: " + filepath.filename().string(), MSGTYPE_INFO, msg_prefix)); + bIsComplete = true; // Set to true so we can skip after saving xml data + } + } + } + } + else + { + msgQueue.push(Message("Remote file is different, renaming local file", MSGTYPE_INFO, msg_prefix)); + std::string date_old = "." + bptime::to_iso_string(bptime::second_clock::local_time()) + ".old"; + boost::filesystem::path new_name = filepath.string() + date_old; // Rename old file by appending date and ".old" to filename + boost::system::error_code ec; + boost::filesystem::rename(filepath, new_name, ec); // Rename the file + if (ec) + { + msgQueue.push(Message("Failed to rename " + filepath.string() + " to " + new_name.string() + " - Skipping file", MSGTYPE_WARNING, msg_prefix)); + continue; + } + } + } + + // Save remote XML + if (!xml.empty()) + { + if ((bLocalXMLExists && !bSameVersion) || !bLocalXMLExists) + { + // Check that directory exists and create subdirectories + boost::filesystem::path path = xml_directory; + mtx_create_directories.lock(); // Use mutex to avoid race conditions + if (boost::filesystem::exists(path)) + { + if (!boost::filesystem::is_directory(path)) + { + msgQueue.push(Message(path.string() + " is not directory", MSGTYPE_WARNING, msg_prefix)); + } + } + else + { + if (!boost::filesystem::create_directories(path)) + { + msgQueue.push(Message("Failed to create directory: " + path.string(), MSGTYPE_ERROR, msg_prefix)); + } + } + mtx_create_directories.unlock(); + std::ofstream ofs(local_xml_file.string().c_str()); + if (ofs) + { + ofs << xml; + ofs.close(); + } + else + { + msgQueue.push(Message("Can't create " + local_xml_file.string(), MSGTYPE_ERROR, msg_prefix)); + } + } + } + + // File was complete and we have saved xml data so we can skip it + if (bIsComplete) + continue; + + std::string url = downlinkJson["downlink"].asString(); + curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str()); + long int response_code = 0; + bool bShouldRetry = false; + std::string retry_reason; + do + { + if (conf.iWait > 0) + usleep(conf.iWait); // Wait before continuing + + response_code = 0; // Make sure that response code is reset + + if (iRetryCount != 0) + { + std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath.filename().string(); + if (!retry_reason.empty()) + retry_msg += " (" + retry_reason + ")"; + msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix)); + } + retry_reason = ""; // reset retry reason + + FILE* outfile; + // File exists, resume + if (bResume) + { + iResumePosition = boost::filesystem::file_size(filepath); + if ((outfile=fopen(filepath.string().c_str(), "r+"))!=NULL) + { + fseek(outfile, 0, SEEK_END); + curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, iResumePosition); + curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, outfile); + } + else + { + msgQueue.push(Message("Failed to open " + filepath.string(), MSGTYPE_ERROR, msg_prefix)); + break; + } + } + else // File doesn't exist, create new file + { + if ((outfile=fopen(filepath.string().c_str(), "w"))!=NULL) + { + curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, 0); // start downloading from the beginning of file + curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, outfile); + } + else + { + msgQueue.push(Message("Failed to create " + filepath.string(), MSGTYPE_ERROR, msg_prefix)); + break; + } + } + + xferinfo.offset = iResumePosition; + xferinfo.timer.reset(); + xferinfo.TimeAndSize.clear(); + result = curl_easy_perform(dlhandle); + fclose(outfile); + + switch (result) + { + // Retry on these errors + case CURLE_PARTIAL_FILE: + case CURLE_OPERATION_TIMEDOUT: + case CURLE_RECV_ERROR: + case CURLE_SSL_CONNECT_ERROR: + bShouldRetry = true; + break; + // Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "416 Range Not Satisfiable" + case CURLE_HTTP_RETURNED_ERROR: + curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code); + if (response_code == 416) + bShouldRetry = false; + else + bShouldRetry = true; + break; + default: + bShouldRetry = false; + break; + } + + if (bShouldRetry) + { + iRetryCount++; + retry_reason = std::string(curl_easy_strerror(result)); + if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) + bResume = true; + } + + } while (bShouldRetry && (iRetryCount <= conf.iRetries)); + + if (result == CURLE_OK || result == CURLE_RANGE_ERROR || (result == CURLE_HTTP_RETURNED_ERROR && response_code == 416)) + { + // Set timestamp for downloaded file to same value as file on server + long filetime = -1; + CURLcode res = curl_easy_getinfo(dlhandle, CURLINFO_FILETIME, &filetime); + if (res == CURLE_OK && filetime >= 0) + { + std::time_t timestamp = (std::time_t)filetime; + try + { + boost::filesystem::last_write_time(filepath, timestamp); + } + catch(const boost::filesystem::filesystem_error& e) + { + msgQueue.push(Message(e.what(), MSGTYPE_WARNING, msg_prefix)); + } + } + + // Average download speed + std::ostringstream dlrate_avg; + std::string rate_unit; + progressInfo progress_info = vDownloadInfo[tid].getProgressInfo(); + if (progress_info.rate_avg > 1048576) // 1 MB + { + progress_info.rate_avg /= 1048576; + rate_unit = "MB/s"; + } + else + { + progress_info.rate_avg /= 1024; + rate_unit = "kB/s"; + } + dlrate_avg << std::setprecision(2) << std::fixed << progress_info.rate_avg << rate_unit; + + msgQueue.push(Message("Download complete: " + filepath.filename().string() + " (@ " + dlrate_avg.str() + ")", MSGTYPE_SUCCESS, msg_prefix)); + } + else + { + std::string msg = "Download complete (" + static_cast(curl_easy_strerror(result)); + if (response_code > 0) + msg += " (" + std::to_string(response_code) + ")"; + msg += "): " + filepath.filename().string(); + msgQueue.push(Message(msg, MSGTYPE_WARNING, msg_prefix)); + + // Delete the file if download failed and was not a resume attempt or the result is zero length file + if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) + { + if ((result != CURLE_PARTIAL_FILE && !bResume && result != CURLE_OPERATION_TIMEDOUT) || boost::filesystem::file_size(filepath) == 0) + { + if (!boost::filesystem::remove(filepath)) + msgQueue.push(Message("Failed to delete " + filepath.filename().string(), MSGTYPE_ERROR, msg_prefix)); + } + } + } + + // Automatic xml creation + if (conf.dlConf.bAutomaticXMLCreation) + { + if (result == CURLE_OK) + { + if ((gf.type & GFTYPE_EXTRA) || (conf.dlConf.bRemoteXML && !bLocalXMLExists && xml.empty())) + createXMLQueue.push(gf); + } + } + } + + curl_easy_cleanup(dlhandle); + delete galaxy; + + vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); + msgQueue.push(Message("Finished all tasks", MSGTYPE_INFO, msg_prefix)); + + return; +} + +int Downloader::progressCallbackForThread(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) +{ + // unused so lets prevent warnings and be more pedantic + (void) ulnow; + (void) ultotal; + + xferInfo* xferinfo = static_cast(clientp); + + // Update progress info every 100ms + if (xferinfo->timer.getTimeBetweenUpdates()>=100 || dlnow == dltotal) + { + xferinfo->timer.reset(); + progressInfo info; + info.dlnow = dlnow; + info.dltotal = dltotal; + + // trying to get rate and setting to NaN if it fails + if (CURLE_OK != curl_easy_getinfo(xferinfo->curlhandle, CURLINFO_SPEED_DOWNLOAD, &info.rate_avg)) + info.rate_avg = std::numeric_limits::quiet_NaN(); + + // setting full dlwnow and dltotal + if (xferinfo->offset > 0) + { + info.dlnow += xferinfo->offset; + info.dltotal += xferinfo->offset; + } + + // 10 second average download speed + // Don't use static value of 10 seconds because update interval depends on when and how often progress callback is called + xferinfo->TimeAndSize.push_back(std::make_pair(time(NULL), static_cast(info.dlnow))); + if (xferinfo->TimeAndSize.size() > 100) // 100 * 100ms = 10s + { + xferinfo->TimeAndSize.pop_front(); + time_t time_first = xferinfo->TimeAndSize.front().first; + uintmax_t size_first = xferinfo->TimeAndSize.front().second; + time_t time_last = xferinfo->TimeAndSize.back().first; + uintmax_t size_last = xferinfo->TimeAndSize.back().second; + info.rate = (size_last - size_first) / static_cast((time_last - time_first)); + } + else + { + info.rate = info.rate_avg; + } + + vDownloadInfo[xferinfo->tid].setProgressInfo(info); + vDownloadInfo[xferinfo->tid].setStatus(DLSTATUS_RUNNING); + } + + return 0; +} + +template void Downloader::printProgress(const ThreadSafeQueue& download_queue) +{ + // Print progress information until all threads have finished their tasks + ProgressBar bar(Globals::globalConfig.bUnicode, Globals::globalConfig.bColor); + unsigned int dl_status = DLSTATUS_NOTSTARTED; + while (dl_status != DLSTATUS_FINISHED) + { + dl_status = DLSTATUS_NOTSTARTED; + + // Print progress information once per 100ms + std::this_thread::sleep_for(std::chrono::milliseconds(Globals::globalConfig.iProgressInterval)); + std::cout << "\033[J\r" << std::flush; // Clear screen from the current line down to the bottom of the screen + + // Print messages from message queue first + Message msg; + while (msgQueue.try_pop(msg)) + { + std::cout << msg.getFormattedString(Globals::globalConfig.bColor, true) << std::endl; + if (Globals::globalConfig.bReport) + { + this->report_ofs << msg.getTimestampString() << ": " << msg.getMessage() << std::endl; + } + } + + int iTermWidth = Util::getTerminalWidth(); + double total_rate = 0; + bptime::time_duration eta_total_seconds; + + // Create progress info text for all download threads + std::vector vProgressText; + for (unsigned int i = 0; i < vDownloadInfo.size(); ++i) + { + std::string progress_text; + int bar_length = 26; + int min_bar_length = 5; + + unsigned int status = vDownloadInfo[i].getStatus(); + dl_status |= status; + + if (status == DLSTATUS_FINISHED) + { + vProgressText.push_back("#" + std::to_string(i) + ": Finished"); + continue; + } + + std::string filename = vDownloadInfo[i].getFilename(); + progressInfo progress_info = vDownloadInfo[i].getProgressInfo(); total_rate += progress_info.rate; bool starting = ((0 == progress_info.dlnow) && (0 == progress_info.dltotal)); @@ -3991,109 +4428,638 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con msgQueue.push(Message("Download complete: " + path.string(), MSGTYPE_SUCCESS, msg_prefix)); } - vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); - delete galaxy; - curl_easy_cleanup(dlhandle); + vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); + delete galaxy; + curl_easy_cleanup(dlhandle); + + return; +} + +void Downloader::galaxyShowBuilds(const std::string& product_id, int build_index) +{ + std::string id; + if(this->galaxySelectProductIdHelper(product_id, id)) + { + if (!id.empty()) + this->galaxyShowBuildsById(id, build_index); + } +} + +void Downloader::galaxyShowBuildsById(const std::string& product_id, int build_index) +{ + std::string sPlatform; + unsigned int iPlatform = Globals::globalConfig.dlConf.iGalaxyPlatform; + if (iPlatform == GlobalConstants::PLATFORM_LINUX) + sPlatform = "linux"; + else if (iPlatform == GlobalConstants::PLATFORM_MAC) + sPlatform = "osx"; + else + sPlatform = "windows"; + + Json::Value json = gogGalaxy->getProductBuilds(product_id, sPlatform); + + // JSON is empty and platform is Linux. Most likely cause is that Galaxy API doesn't have Linux support + if (json.empty() && iPlatform == GlobalConstants::PLATFORM_LINUX) + { + std::cout << "Galaxy API doesn't have Linux support" << std::endl; + + std::cout << "Checking for installers that can be used as repository" << std::endl; + DownloadConfig dlConf = Globals::globalConfig.dlConf; + dlConf.bInstallers = true; + dlConf.bExtras = false; + dlConf.bLanguagePacks = false; + dlConf.bPatches = false; + dlConf.bDLC = true; + dlConf.iInstallerPlatform = dlConf.iGalaxyPlatform; + dlConf.iInstallerLanguage = dlConf.iGalaxyLanguage; + + Json::Value product_info = gogGalaxy->getProductInfo(product_id); + gameDetails game = gogGalaxy->productInfoJsonToGameDetails(product_info, dlConf); + + std::vector vInstallers; + if (!game.installers.empty()) + { + vInstallers.push_back(game.installers[0]); + for (unsigned int i = 0; i < game.dlcs.size(); ++i) + { + if (!game.dlcs[i].installers.empty()) + vInstallers.push_back(game.dlcs[i].installers[0]); + } + } + + if (vInstallers.empty()) + { + std::cout << "No installers found" << std::endl; + } + else + { + std::cout << "Using these installers" << std::endl; + for (unsigned int i = 0; i < vInstallers.size(); ++i) + std::cout << "\t" << vInstallers[i].gamename << "/" << vInstallers[i].id << std::endl; + } + + return; + } + + if (build_index < 0) + { + for (unsigned int i = 0; i < json["items"].size(); ++i) + { + std::cout << i << ": " << "Version " << json["items"][i]["version_name"].asString() << " - " << json["items"][i]["date_published"].asString() << " (Gen " << json["items"][i]["generation"].asInt() << ")" << std::endl; + } + return; + } + + std::string link = json["items"][build_index]["link"].asString(); + + if (json["items"][build_index]["generation"].asInt() == 1) + { + json = gogGalaxy->getManifestV1(link); + } + else if (json["items"][build_index]["generation"].asInt() == 2) + { + std::string buildHash; + buildHash.assign(link.begin()+link.find_last_of("/")+1, link.end()); + json = gogGalaxy->getManifestV2(buildHash); + } + else + { + std::cout << "Only generation 1 and 2 builds are supported currently" << std::endl; + return; + } + + Json::StyledStreamWriter().write(std::cout, json); + + return; +} + +std::string parseLocationHelper(const std::string &location, const std::map &var) { + char search_arg[2] {'?', '>'}; + auto it = std::search(std::begin(location), std::end(location), std::begin(search_arg), std::end(search_arg)); + + if(it == std::end(location)) { + return location; + } + + std::string var_name { std::begin(location) + 2, it }; + auto relative_path = it + 2; + + auto var_value = var.find(var_name); + if(var_value == std::end(var)) { + return location; + } + + std::string parsedLocation; + parsedLocation.insert(std::end(parsedLocation), std::begin(var_value->second), std::end(var_value->second)); + parsedLocation.insert(std::end(parsedLocation), relative_path, std::end(location)); + + return parsedLocation; +} +std::string parseLocation(const std::string &location, const std::map &var) { + auto parsedLocation = parseLocationHelper(location, var); + Util::replaceAllString(parsedLocation, "\\", "/"); + + return parsedLocation; +} + +std::pair getline(std::string::const_iterator begin, std::string::const_iterator end) { + while(begin != end) { + if(*begin == '\r') { + return { begin, begin + 2 }; + } + if(*begin == '\n') { + return { begin, begin + 1 }; + } + + ++begin; + } + + return { end, end }; +} + +void Downloader::uploadCloudSaves(const std::string& product_id, int build_index) +{ + std::string id; + if(this->galaxySelectProductIdHelper(product_id, id)) + { + if (!id.empty()) + this->uploadCloudSavesById(id, build_index); + } +} + +void Downloader::deleteCloudSaves(const std::string& product_id, int build_index) +{ + std::string id; + if(this->galaxySelectProductIdHelper(product_id, id)) + { + if (!id.empty()) + this->deleteCloudSavesById(id, build_index); + } +} - return; +void Downloader::downloadCloudSaves(const std::string& product_id, int build_index) +{ + std::string id; + if(this->galaxySelectProductIdHelper(product_id, id)) + { + if (!id.empty()) + this->downloadCloudSavesById(id, build_index); + } } -void Downloader::galaxyShowBuilds(const std::string& product_id, int build_index) +void Downloader::galaxyShowCloudSaves(const std::string& product_id, int build_index) { std::string id; if(this->galaxySelectProductIdHelper(product_id, id)) { if (!id.empty()) - this->galaxyShowBuildsById(id, build_index); + this->galaxyShowCloudSavesById(id, build_index); } } -void Downloader::galaxyShowBuildsById(const std::string& product_id, int build_index) +void Downloader::galaxyShowLocalCloudSaves(const std::string& product_id, int build_index) { + std::string id; + if(this->galaxySelectProductIdHelper(product_id, id)) + { + if (!id.empty()) + this->galaxyShowLocalCloudSavesById(id, build_index); + } +} + +std::map Downloader::cloudSaveLocations(const std::string& product_id, int build_index) { std::string sPlatform; unsigned int iPlatform = Globals::globalConfig.dlConf.iGalaxyPlatform; - if (iPlatform == GlobalConstants::PLATFORM_LINUX) - sPlatform = "linux"; - else if (iPlatform == GlobalConstants::PLATFORM_MAC) - sPlatform = "osx"; + if (iPlatform == GlobalConstants::PLATFORM_LINUX) { + // Linux is not yet supported for cloud saves + std::cout << "Cloud saves for Linux builds not yet supported" << std::endl; + return {}; + } + else if (iPlatform == GlobalConstants::PLATFORM_MAC) { + std::cout << "Cloud saves for Mac builds not yet supported" << std::endl; + return {}; + } else sPlatform = "windows"; Json::Value json = gogGalaxy->getProductBuilds(product_id, sPlatform); - // JSON is empty and platform is Linux. Most likely cause is that Galaxy API doesn't have Linux support - if (json.empty() && iPlatform == GlobalConstants::PLATFORM_LINUX) + build_index = std::max(0, build_index); + + std::string link = json["items"][build_index]["link"].asString(); + + Json::Value manifest; + if (json["items"][build_index]["generation"].asInt() != 2) { - std::cout << "Galaxy API doesn't have Linux support" << std::endl; + std::cout << "Only generation 2 builds are supported currently" << std::endl; + return {}; + } + std::string buildHash; + buildHash.assign(link.begin()+link.find_last_of("/")+1, link.end()); + manifest = gogGalaxy->getManifestV2(buildHash); - std::cout << "Checking for installers that can be used as repository" << std::endl; - DownloadConfig dlConf = Globals::globalConfig.dlConf; - dlConf.bInstallers = true; - dlConf.bExtras = false; - dlConf.bLanguagePacks = false; - dlConf.bPatches = false; - dlConf.bDLC = true; - dlConf.iInstallerPlatform = dlConf.iGalaxyPlatform; - dlConf.iInstallerLanguage = dlConf.iGalaxyLanguage; + std::string clientId = manifest["clientId"].asString(); + std::string secret = manifest["clientSecret"].asString(); - Json::Value product_info = gogGalaxy->getProductInfo(product_id); - gameDetails game = gogGalaxy->productInfoJsonToGameDetails(product_info, dlConf); + if(!gogGalaxy->refreshLogin(clientId, secret, Globals::galaxyConf.getRefreshToken(), false)) { + std::cout << "Couldn't refresh login" << std::endl; + return {}; + } - std::vector vInstallers; - if (!game.installers.empty()) - { - vInstallers.push_back(game.installers[0]); - for (unsigned int i = 0; i < game.dlcs.size(); ++i) - { - if (!game.dlcs[i].installers.empty()) - vInstallers.push_back(game.dlcs[i].installers[0]); - } + std::string install_directory; + if (Globals::globalConfig.dirConf.bSubDirectories) + { + install_directory = this->getGalaxyInstallDirectory(gogGalaxy, json); + } + + std::string platform; + switch(iPlatform) { + case GlobalConstants::PLATFORM_WINDOWS: + platform = "Windows"; + break; + default: + std::cout << "Only Windows supported for now for cloud support" << std::endl; + return {}; + } + + std::string install_path = Globals::globalConfig.dirConf.sDirectory + install_directory; + std::string document_path = Globals::globalConfig.dirConf.sWinePrefix + "drive_c/users/" + username() + "/Documents/"; + std::string appdata_roaming = Globals::globalConfig.dirConf.sWinePrefix + "drive_c/users/" + username() + "/AppData/Roaming/"; + std::string appdata_local_path = Globals::globalConfig.dirConf.sWinePrefix + "drive_c/users/" + username() + "/AppData/Local/"; + std::string appdata_local_low_path = Globals::globalConfig.dirConf.sWinePrefix + "drive_c/users/" + username() + "/AppData/LocalLow/"; + std::string saved_games = Globals::globalConfig.dirConf.sWinePrefix + "drive_c/users/" + username() + "/Save Games/"; + + auto cloud_saves_json = gogGalaxy->getCloudPathAsJson(manifest["clientId"].asString())["content"][platform]["cloudStorage"]; + auto enabled = cloud_saves_json["enabled"].asBool(); + + if(!enabled) { + return {}; + } + + std::map vars { + { "INSTALL", std::move(install_path) }, + { "DOCUMENTS", std::move(document_path) }, + { "APPLICATION_DATA_ROAMING", std::move(appdata_roaming)}, + { "APPLICATION_DATA_LOCAL", std::move(appdata_local_path) }, + { "APPLICATION_DATA_LOCAL_LOW", std::move(appdata_local_low_path) }, + { "SAVED_GAMES", std::move(saved_games) }, + }; + + std::map name_to_location; + for(auto &cloud_save : cloud_saves_json["locations"]) { + std::string location = parseLocation(cloud_save["location"].asString(), vars); + + name_to_location.insert({cloud_save["name"].asString(), std::move(location)}); + } + + if(name_to_location.empty()) { + std::string location; + switch(iPlatform) { + case GlobalConstants::PLATFORM_WINDOWS: + location = vars["APPLICATION_DATA_LOCAL"] + "/GOG.com/Galaxy/Applications/" + Globals::galaxyConf.getClientId() + "/Storage"; + break; + default: + std::cout << "Only Windows supported for now for cloud support" << std::endl; + return {}; } - if (vInstallers.empty()) - { - std::cout << "No installers found" << std::endl; + name_to_location.insert({"__default", std::move(location)}); + } + + return name_to_location; +} + +int Downloader::cloudSaveListByIdForEach(const std::string& product_id, int build_index, const std::function &f) { + auto name_to_location = this->cloudSaveLocations(product_id, build_index); + if(name_to_location.empty()) { + std::cout << "No cloud save locations found" << std::endl; + return -1; + } + + std::string url = "https://cloudstorage.gog.com/v1/" + Globals::galaxyConf.getUserId() + "/" + Globals::galaxyConf.getClientId(); + auto fileList = gogGalaxy->getResponseJson(url, "application/json"); + + for(auto &fileJson : fileList) { + auto path = fileJson["name"].asString(); + + if(!whitelisted(path)) { + continue; } - else - { - std::cout << "Using these installers" << std::endl; - for (unsigned int i = 0; i < vInstallers.size(); ++i) - std::cout << "\t" << vInstallers[i].gamename << "/" << vInstallers[i].id << std::endl; + + auto pos = path.find_first_of('/'); + + auto location = name_to_location[path.substr(0, pos)] + path.substr(pos); + + auto filesize = fileJson["bytes"].asUInt64(); + + auto last_modified = boost::posix_time::from_iso_extended_string(fileJson["last_modified"].asString()); + + cloudSaveFile csf { + last_modified, + filesize, + std::move(path), + std::move(location) + }; + + f(csf); + } + + return 0; +} + +void Downloader::uploadCloudSavesById(const std::string& product_id, int build_index) +{ + auto name_to_locations = cloudSaveLocations(product_id, build_index); + + if(name_to_locations.empty()) { + std::cout << "Cloud saves not supported for this game" << std::endl; + } + + std::map path_to_cloudSaveFile; + for(auto &name_to_location : name_to_locations) { + auto &name = name_to_location.first; + auto &location = name_to_location.second; + + if(!boost::filesystem::exists(location) || !boost::filesystem::is_directory(location)) { + continue; } + const char endswith[] = ".~incomplete"; + dirForEach(location, [&](boost::filesystem::directory_iterator file) { + auto path = file->path(); + + // If path ends with ".~incomplete", then skip this file + if( + path.size() >= sizeof(endswith) && + strcmp(path.c_str() + (path.size() + 1 - sizeof(endswith)), endswith) == 0 + ) { + return; + } + + auto remote_path = (name / boost::filesystem::relative(*file, location)).string(); + if(!whitelisted(remote_path)) { + return; + } + + + cloudSaveFile csf { + boost::posix_time::from_time_t(boost::filesystem::last_write_time(*file) - 1), + boost::filesystem::file_size(*file), + std::move(remote_path), + file->path().string() + }; + + path_to_cloudSaveFile.insert(std::make_pair(csf.path, std::move(csf))); + }); + } + + if(path_to_cloudSaveFile.empty()) { + std::cout << "No local cloud saves found" << std::endl; + return; } - if (build_index < 0) - { - for (unsigned int i = 0; i < json["items"].size(); ++i) - { - std::cout << i << ": " << "Version " << json["items"][i]["version_name"].asString() << " - " << json["items"][i]["date_published"].asString() << " (Gen " << json["items"][i]["generation"].asInt() << ")" << std::endl; + auto res = this->cloudSaveListByIdForEach(product_id, build_index, [&](cloudSaveFile &csf) { + auto it = path_to_cloudSaveFile.find(csf.path); + + //If remote save is not locally stored, skip + if(it == std::end(path_to_cloudSaveFile)) { + return; + } + + cloudSaveFile local_csf { std::move(it->second) }; + path_to_cloudSaveFile.erase(it); + + if(Globals::globalConfig.bCloudForce || csf.lastModified < local_csf.lastModified) { + iTotalRemainingBytes.fetch_add(local_csf.fileSize); + + dlCloudSaveQueue.push(local_csf); } + }); + + for(auto &path_csf : path_to_cloudSaveFile) { + auto &csf = path_csf.second; + + iTotalRemainingBytes.fetch_add(csf.fileSize); + + dlCloudSaveQueue.push(csf); + } + + if(res || dlCloudSaveQueue.empty()) { return; } - std::string link = json["items"][build_index]["link"].asString(); + // Limit thread count to number of items in upload queue + unsigned int iThreads = std::min(Globals::globalConfig.iThreads, static_cast(dlCloudSaveQueue.size())); - if (json["items"][build_index]["generation"].asInt() == 1) + // Create download threads + std::vector vThreads; + for (unsigned int i = 0; i < iThreads; ++i) { - json = gogGalaxy->getManifestV1(link); + DownloadInfo dlInfo; + dlInfo.setStatus(DLSTATUS_NOTSTARTED); + vDownloadInfo.push_back(dlInfo); + vThreads.push_back(std::thread(Downloader::processCloudSaveUploadQueue, Globals::globalConfig, i)); } - else if (json["items"][build_index]["generation"].asInt() == 2) - { - std::string buildHash; - buildHash.assign(link.begin()+link.find_last_of("/")+1, link.end()); - json = gogGalaxy->getManifestV2(buildHash); + + this->printProgress(dlCloudSaveQueue); + + // Join threads + for (unsigned int i = 0; i < vThreads.size(); ++i) { + vThreads[i].join(); } - else + + vThreads.clear(); + vDownloadInfo.clear(); +} + +void Downloader::deleteCloudSavesById(const std::string& product_id, int build_index) { + if(Globals::globalConfig.cloudWhiteList.empty() && !Globals::globalConfig.bCloudForce) { + std::cout << "No files have been whitelisted, either use \'--cloud-whitelist\' or \'--cloud-force\'" << std::endl; + return; + } + + + curl_slist *header = nullptr; + + std::string access_token; + if (!Globals::galaxyConf.isExpired()) { + access_token = Globals::galaxyConf.getAccessToken(); + } + + if (!access_token.empty()) { + std::string bearer = "Authorization: Bearer " + access_token; + header = curl_slist_append(header, bearer.c_str()); + } + + auto dlhandle = curl_easy_init(); + + curl_easy_setopt(dlhandle, CURLOPT_HTTPHEADER, header); + curl_easy_setopt(dlhandle, CURLOPT_CUSTOMREQUEST, "DELETE"); + + this->cloudSaveListByIdForEach(product_id, build_index, [dlhandle](cloudSaveFile &csf) { + auto url = "https://cloudstorage.gog.com/v1/" + Globals::galaxyConf.getUserId() + '/' + Globals::galaxyConf.getClientId() + '/' + csf.path; + curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str()); + + auto result = curl_easy_perform(dlhandle); + if(result == CURLE_HTTP_RETURNED_ERROR) { + long response_code = 0; + curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code); + + std::cout << response_code << ": " << curl_easy_strerror(result); + } + }); + + curl_slist_free_all(header); + + curl_easy_cleanup(dlhandle); +} + +void Downloader::downloadCloudSavesById(const std::string& product_id, int build_index) { + auto res = this->cloudSaveListByIdForEach(product_id, build_index, [](cloudSaveFile &csf) { + boost::filesystem::path filepath = csf.location; + + if(boost::filesystem::exists(filepath)) { + // last_write_time minus a single second, since time_t is only accurate to the second unlike boost::posix_time::ptime + auto time = boost::posix_time::from_time_t(boost::filesystem::last_write_time(filepath) - 1); + + if(!Globals::globalConfig.bCloudForce && time <= csf.lastModified) { + std::cout << "Already up to date -- skipping: " << csf.path << std::endl; + return; // This file is already completed + } + } + + if(boost::filesystem::is_directory(filepath)) { + std::cout << "is a directory: " << csf.location << std::endl; + return; + } + + iTotalRemainingBytes.fetch_add(csf.fileSize); + + dlCloudSaveQueue.push(std::move(csf)); + }); + + if(res || dlCloudSaveQueue.empty()) { + return; + } + + // Limit thread count to number of items in download queue + unsigned int iThreads = std::min(Globals::globalConfig.iThreads, static_cast(dlCloudSaveQueue.size())); + + // Create download threads + std::vector vThreads; + for (unsigned int i = 0; i < iThreads; ++i) { - std::cout << "Only generation 1 and 2 builds are supported currently" << std::endl; + DownloadInfo dlInfo; + dlInfo.setStatus(DLSTATUS_NOTSTARTED); + vDownloadInfo.push_back(dlInfo); + vThreads.push_back(std::thread(Downloader::processCloudSaveDownloadQueue, Globals::globalConfig, i)); + } + + this->printProgress(dlCloudSaveQueue); + + // Join threads + for (unsigned int i = 0; i < vThreads.size(); ++i) { + vThreads[i].join(); + } + + vThreads.clear(); + vDownloadInfo.clear(); +} + +void Downloader::galaxyShowCloudSavesById(const std::string& product_id, int build_index) +{ + this->cloudSaveListByIdForEach(product_id, build_index, [](cloudSaveFile &csf) { + boost::filesystem::path filepath = csf.location; + filepath = boost::filesystem::absolute(filepath, boost::filesystem::current_path()); + + if(boost::filesystem::exists(filepath)) { + auto size = boost::filesystem::file_size(filepath); + + // last_write_time minus a single second, since time_t is only accurate to the second unlike boost::posix_time::ptime + auto time = boost::filesystem::last_write_time(filepath) - 1; + + if(csf.fileSize < size) { + std::cout << csf.path << " :: not yet completed download" << std::endl; + } + else if(boost::posix_time::from_time_t(time) <= csf.lastModified) { + std::cout << csf.path << " :: Already up to date" << std::endl; + } + else { + std::cout << csf.path << " :: Out of date" << std::endl; + } + } + else { + std::cout << csf.path << " :: Isn't downloaded yet" << std::endl; + } + }); +} + +void Downloader::galaxyShowLocalCloudSavesById(const std::string& product_id, int build_index) { + auto name_to_locations = cloudSaveLocations(product_id, build_index); + + if(name_to_locations.empty()) { + std::cout << "Cloud saves not supported for this game" << std::endl; + } + + std::map path_to_cloudSaveFile; + for(auto &name_to_location : name_to_locations) { + auto &name = name_to_location.first; + auto &location = name_to_location.second; + + if(!boost::filesystem::exists(location) || !boost::filesystem::is_directory(location)) { + continue; + } + + dirForEach(location, [&](boost::filesystem::directory_iterator file) { + auto path = (name / boost::filesystem::relative(*file, location)).string(); + + if(!whitelisted(path)) { + return; + } + + cloudSaveFile csf { + boost::posix_time::from_time_t(boost::filesystem::last_write_time(*file) - 1), + boost::filesystem::file_size(*file), + std::move(path), + file->path().string() + }; + + path_to_cloudSaveFile.insert(std::make_pair(csf.path, std::move(csf))); + }); + } + + if(path_to_cloudSaveFile.empty()) { + std::cout << "No local cloud saves found" << std::endl; + return; } - std::cout << json << std::endl; + this->cloudSaveListByIdForEach(product_id, build_index, [&](cloudSaveFile &csf) { + auto it = path_to_cloudSaveFile.find(csf.path); - return; + //If remote save is not locally stored, skip + if(it == std::end(path_to_cloudSaveFile)) { + return; + } + + cloudSaveFile local_csf { std::move(it->second) }; + path_to_cloudSaveFile.erase(it); + + std::cout << csf.path << ": "; + if(csf.lastModified < local_csf.lastModified) { + std::cout << "remote save out of date: it should be synchronized" << std::endl; + } + else { + std::cout << "up to date" << std::endl; + } + }); + + for(auto &path_csf : path_to_cloudSaveFile) { + auto &csf = path_csf.second; + + std::cout << csf.path << ": there's only a local copy" << std::endl; + } } std::vector Downloader::galaxyGetOrphanedFiles(const std::vector& items, const std::string& install_path) diff --git a/src/galaxyapi.cpp b/src/galaxyapi.cpp index 17595e5..b278411 100644 --- a/src/galaxyapi.cpp +++ b/src/galaxyapi.cpp @@ -53,23 +53,33 @@ int galaxyAPI::init() return res; } -bool galaxyAPI::refreshLogin() +bool galaxyAPI::refreshLogin(const std::string &clientId, const std::string &clientSecret, const std::string &refreshToken, bool newSession) { - std::string refresh_url = "https://auth.gog.com/token?client_id=" + Globals::galaxyConf.getClientId() - + "&client_secret=" + Globals::galaxyConf.getClientSecret() + std::string refresh_url = "https://auth.gog.com/token?client_id=" + clientId + + "&client_secret=" + clientSecret + "&grant_type=refresh_token" - + "&refresh_token=" + Globals::galaxyConf.getRefreshToken(); + + "&refresh_token=" + refreshToken + + (newSession ? "" : "&without_new_session=1"); + // std::cout << refresh_url << std::endl; Json::Value token_json = this->getResponseJson(refresh_url); if (token_json.empty()) return false; + token_json["client_id"] = clientId; + token_json["client_secret"] = clientSecret; + Globals::galaxyConf.setJSON(token_json); return true; } +bool galaxyAPI::refreshLogin() +{ + return refreshLogin(Globals::galaxyConf.getClientId(), Globals::galaxyConf.getClientSecret(), Globals::galaxyConf.getRefreshToken(), true); +} + bool galaxyAPI::isTokenExpired() { bool res = false; @@ -80,7 +90,7 @@ bool galaxyAPI::isTokenExpired() return res; } -std::string galaxyAPI::getResponse(const std::string& url) +std::string galaxyAPI::getResponse(const std::string& url, const char *encoding) { struct curl_slist *header = NULL; @@ -92,13 +102,26 @@ std::string galaxyAPI::getResponse(const std::string& url) std::string bearer = "Authorization: Bearer " + access_token; header = curl_slist_append(header, bearer.c_str()); } + + if(encoding) { + auto accept = "Accept: " + std::string(encoding); + header = curl_slist_append(header, accept.c_str()); + } + curl_easy_setopt(curlhandle, CURLOPT_HTTPHEADER, header); curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str()); curl_easy_setopt(curlhandle, CURLOPT_ACCEPT_ENCODING, ""); int max_retries = std::min(3, Globals::globalConfig.iRetries); std::string response; - Util::CurlHandleGetResponse(curlhandle, response, max_retries); + auto res = Util::CurlHandleGetResponse(curlhandle, response, max_retries); + + if(res) { + long int response_code = 0; + curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code); + + std::cout << "Response code for " << url << " is [" << response_code << ']' << std::endl; + } curl_easy_setopt(curlhandle, CURLOPT_ACCEPT_ENCODING, NULL); curl_easy_setopt(curlhandle, CURLOPT_HTTPHEADER, NULL); @@ -107,9 +130,9 @@ std::string galaxyAPI::getResponse(const std::string& url) return response; } -Json::Value galaxyAPI::getResponseJson(const std::string& url) +Json::Value galaxyAPI::getResponseJson(const std::string& url, const char *encoding) { - std::istringstream response(this->getResponse(url)); + std::istringstream response(this->getResponse(url, encoding)); Json::Value json; if (!response.str().empty()) @@ -149,8 +172,13 @@ Json::Value galaxyAPI::getResponseJson(const std::string& url) catch(const Json::Exception& exc) { // Failed to parse json + + std::cout << "Failed to parse json: " << exc.what(); } } + else { + std::cout << "Failed to parse json: " << exc.what(); + } } } @@ -190,6 +218,12 @@ Json::Value galaxyAPI::getManifestV2(std::string manifest_hash, const bool& is_d return this->getResponseJson(url); } +Json::Value galaxyAPI::getCloudPathAsJson(const std::string &clientId) { + std::string url = "https://remote-config.gog.com/components/galaxy_client/clients/" + clientId + "?component_version=2.0.51"; + + return this->getResponseJson(url); +} + Json::Value galaxyAPI::getSecureLink(const std::string& product_id, const std::string& path) { std::string url = "https://content-system.gog.com/products/" + product_id + "/secure_link?generation=2&path=" + path + "&_version=2"; diff --git a/src/util.cpp b/src/util.cpp index 5fd1c35..948cda6 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -8,6 +8,9 @@ #include #include +#include +#include +#include #include #include #include @@ -391,6 +394,22 @@ int Util::replaceString(std::string& str, const std::string& to_replace, const s return 1; } +int Util::replaceAllString(std::string& str, const std::string& to_replace, const std::string& replace_with) { + size_t pos = str.find(to_replace); + if (pos == std::string::npos) + { + return 0; + } + + do { + str.replace(str.begin()+pos, str.begin()+pos+to_replace.length(), replace_with); + + pos = str.find(to_replace, pos + to_replace.length()); + } while(pos != std::string::npos); + + return 1; +} + void Util::filepathReplaceReservedStrings(std::string& str, const std::string& gamename, const unsigned int& platformId, const std::string& dlcname) { std::string platform; @@ -749,7 +768,6 @@ void Util::CurlHandleSetDefaultOptions(CURL* curlhandle, const CurlConfig& conf) curl_easy_setopt(curlhandle, CURLOPT_USERAGENT, conf.sUserAgent.c_str()); curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); - curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true); curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, conf.iTimeout); curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true); @@ -823,8 +841,9 @@ CURLcode Util::CurlHandleGetResponse(CURL* curlhandle, std::string& response, in // Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "404 Not Found" case CURLE_HTTP_RETURNED_ERROR: curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code); - if (response_code == 404) + if (response_code == 404 || response_code == 403) { bShouldRetry = false; + } else bShouldRetry = true; break; @@ -865,3 +884,14 @@ curl_off_t Util::CurlWriteChunkMemoryCallback(void *contents, curl_off_t size, c return realsize; } + +curl_off_t Util::CurlReadChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, ChunkMemoryStruct *mem) { + curl_off_t realsize = std::min(size * nmemb, mem->size); + + std::copy(mem->memory, mem->memory + realsize, (char*)contents); + + mem->size -= realsize; + mem->memory += realsize; + + return realsize; +} diff --git a/src/website.cpp b/src/website.cpp index 08349b3..75d98a1 100644 --- a/src/website.cpp +++ b/src/website.cpp @@ -264,6 +264,9 @@ std::vector Website::getGames() // Login to GOG website int Website::Login(const std::string& email, const std::string& password) { + // Reset client id and client secret to ensure we can log-in + Globals::galaxyConf.resetClient(); + int res = 0; std::string postdata; std::ostringstream memory;