From f1af33c268d98bbeb133066655c4ae8cfa896549 Mon Sep 17 00:00:00 2001 From: xiaomlove Date: Thu, 7 Nov 2024 03:35:21 +0800 Subject: [PATCH] improve cleanup jobs --- app/Jobs/CalculateUserSeedBonus.php | 28 ++++++++--- app/Jobs/UpdateTorrentSeedersEtc.php | 58 ++++++++++------------ app/Jobs/UpdateUserSeedingLeechingTime.php | 41 +++++++-------- app/Utils/ThirdPartyJob.php | 2 +- 4 files changed, 68 insertions(+), 61 deletions(-) diff --git a/app/Jobs/CalculateUserSeedBonus.php b/app/Jobs/CalculateUserSeedBonus.php index 266f19cf6..91dae9019 100644 --- a/app/Jobs/CalculateUserSeedBonus.php +++ b/app/Jobs/CalculateUserSeedBonus.php @@ -84,6 +84,8 @@ public function handle() $logFile = getLogFile("seed-bonus-points"); do_log("$logPrefix, [GET_UID_REAL], count: " . count($results) . ", logFile: $logFile"); $fd = fopen($logFile, 'a'); + $seedPointsUpdates = $seedPointsPerHourUpdates = $seedBonusUpdates = []; + $logStr = ""; foreach ($results as $userInfo) { $uid = $userInfo['id']; @@ -115,10 +117,13 @@ public function handle() $dividend = 3600 / $autoclean_interval_one; $all_bonus = $all_bonus / $dividend; $seed_points = $seedBonusResult['seed_points'] / $dividend; - $updatedAt = now()->toDateTimeString(); - $sql = "update users set seed_points = ifnull(seed_points, 0) + $seed_points, seed_points_per_hour = {$seedBonusResult['seed_points']}, seedbonus = seedbonus + $all_bonus, seed_points_updated_at = '$updatedAt' where id = $uid limit 1"; - do_log("$bonusLog, query: $sql"); - NexusDB::statement($sql); +// $updatedAt = now()->toDateTimeString(); +// $sql = "update users set seed_points = ifnull(seed_points, 0) + $seed_points, seed_points_per_hour = {$seedBonusResult['seed_points']}, seedbonus = seedbonus + $all_bonus, seed_points_updated_at = '$updatedAt' where id = $uid limit 1"; +// do_log("$bonusLog, query: $sql"); +// NexusDB::statement($sql); + $seedPointsUpdates[] = sprintf("case %d then ifnull(seed_points, 0) + %d", $uid, $seed_points); + $seedPointsPerHourUpdates[] = sprintf("case %d then %d", $uid, $seedBonusResult['seed_points']); + $seedBonusUpdates[] = sprintf("case %d then %d", $uid, $all_bonus); if ($fd) { $log = sprintf( '%s|%s|%s|%s|%s|%s|%s|%s', @@ -126,16 +131,27 @@ public function handle() $userInfo['seed_points'], number_format($seed_points, 1, '.', ''), number_format($userInfo['seed_points'] + $seed_points, 1, '.', ''), $userInfo['seedbonus'], number_format($all_bonus, 1, '.', ''), number_format($userInfo['seedbonus'] + $all_bonus, 1, '.', '') ); - fwrite($fd, $log . PHP_EOL); +// fwrite($fd, $log . PHP_EOL); + $logStr .= $log . PHP_EOL; } else { do_log("logFile: $logFile is not writeable!", 'error'); } } + $nowStr = now()->toDateTimeString(); + $sql = sprintf( + "update users set seed_points = case id %s end, seed_points_per_hour = case id %s end, seedbonus = case id %s end, seed_points_updated_at = '%s' where id in (%s)", + implode(" ", $seedPointsUpdates), implode(" ", $seedPointsPerHourUpdates), implode(" ", $seedBonusUpdates), $nowStr, $idStr + ); + $result = sql_query($sql); if ($delIdRedisKey) { NexusDB::cache_del($this->idRedisKey); } + fwrite($fd, $logStr); $costTime = time() - $beginTimestamp; - do_log("$logPrefix, [DONE], cost time: $costTime seconds"); + do_log(sprintf( + "$logPrefix, [DONE], update user count: %s, result: %s, cost time: %s seconds", + count($seedPointsUpdates), var_export($result, true), $costTime + )); } /** diff --git a/app/Jobs/UpdateTorrentSeedersEtc.php b/app/Jobs/UpdateTorrentSeedersEtc.php index e85a4dd8c..5234bd886 100644 --- a/app/Jobs/UpdateTorrentSeedersEtc.php +++ b/app/Jobs/UpdateTorrentSeedersEtc.php @@ -76,43 +76,39 @@ public function handle() return; } $torrentIdArr = explode(",", $idStr); - foreach ($torrentIdArr as $torrentId) { - if ($torrentId <= 0) { - continue; - } - $peerResult = NexusDB::table('peers') - ->where('torrent', $torrentId) - ->selectRaw("count(*) as count, seeder") - ->groupBy('seeder') - ->get() - ; - $commentResult = NexusDB::table('comments') - ->where('torrent',$torrentId) - ->selectRaw("count(*) as count") - ->first() - ; - $update = [ - 'comments' => $commentResult && $commentResult->count !== null ? $commentResult->count : 0, - 'seeders' => 0, - 'leechers' => 0, - ]; - foreach ($peerResult as $item) { - if ($item->seeder == 'yes') { - $update['seeders'] = $item->count; - } elseif ($item->seeder == 'no') { - $update['leechers'] = $item->count; - } - } - NexusDB::table('torrents')->where('id', $torrentId)->update($update); - do_log("[CLEANUP_CLI_UPDATE_TORRENT_SEEDERS_ETC_HANDLE_TORRENT], [SUCCESS]: $torrentId => " . json_encode($update)); + //批量取，简单化 + $torrents = array(); + $res = sql_query("SELECT torrent, seeder, COUNT(*) AS c FROM peers GROUP BY torrent, seeder where torrent in ($idStr)"); + while ($row = mysql_fetch_assoc($res)) { + if ($row["seeder"] == "yes") + $key = "seeders"; + else + $key = "leechers"; + $torrents[$row["torrent"]][$key] = $row["c"]; } + + $res = sql_query("SELECT torrent, COUNT(*) AS c FROM comments GROUP BY torrent where torrent in ($idStr)"); + while ($row = mysql_fetch_assoc($res)) { + $torrents[$row["torrent"]]["comments"] = $row["c"]; + } + $seedersUpdates = $leechersUpdates = $commentsUpdates = []; + foreach ($torrentIdArr as $id) { + $seedersUpdates[] = sprintf("when %d then %d", $id, $torrents[$id]["seeders"] ?? 0); + $leechersUpdates[] = sprintf("when %d then %d", $id, $torrents[$id]["leechers"] ?? 0); + $commentsUpdates[] = sprintf("when %d then %d", $id, $torrents[$id]["comments"] ?? 0); + } + $sql = sprintf( + "update torrents set seeders = case id %s end, leechers = case id %s end, comments = case id %s end where id in (%s)", + implode(" ", $seedersUpdates), implode(" ", $leechersUpdates), implode(" ", $commentsUpdates), $idStr + ); + $result = sql_query($sql); if ($delIdRedisKey) { NexusDB::cache_del($this->idRedisKey); } $costTime = time() - $beginTimestamp; do_log(sprintf( - "$logPrefix, [DONE], update torrent count: %s, cost time: %s seconds", - count($torrentIdArr), $costTime + "$logPrefix, [DONE], update torrent count: %s, result: %s, cost time: %s seconds", + count($torrentIdArr), var_export($result, true), $costTime )); } diff --git a/app/Jobs/UpdateUserSeedingLeechingTime.php b/app/Jobs/UpdateUserSeedingLeechingTime.php index d12ba4515..7226f16c9 100644 --- a/app/Jobs/UpdateUserSeedingLeechingTime.php +++ b/app/Jobs/UpdateUserSeedingLeechingTime.php @@ -65,7 +65,6 @@ public function handle() $beginTimestamp = time(); $logPrefix = sprintf("[CLEANUP_CLI_UPDATE_SEEDING_LEECHING_TIME_HANDLE_JOB], commonRequestId: %s, beginUid: %s, endUid: %s", $this->requestId, $this->beginUid, $this->endUid); - $count = 0; $idStr = $this->idStr; $delIdRedisKey = false; if (empty($idStr) && !empty($this->idRedisKey)) { @@ -76,33 +75,29 @@ public function handle() do_log("$logPrefix, no idStr or idRedisKey", "error"); return; } - $uidArr = explode(",", $idStr); - foreach ($uidArr as $uid) { - if ($uid <= 0) { - continue; - } - $sumInfo = NexusDB::table('snatched') - ->selectRaw('sum(seedtime) as seedtime_sum, sum(leechtime) as leechtime_sum') - ->where('userid', $uid) - ->first(); - if ($sumInfo && $sumInfo->seedtime_sum !== null) { - $update = [ - 'seedtime' => $sumInfo->seedtime_sum ?? 0, - 'leechtime' => $sumInfo->leechtime_sum ?? 0, - 'seed_time_updated_at' => Carbon::now()->toDateTimeString(), - ]; - NexusDB::table('users') - ->where('id', $uid) - ->update($update); - do_log("[CLEANUP_CLI_UPDATE_SEEDING_LEECHING_TIME_HANDLE_USER], [SUCCESS]: $uid => " . json_encode($update)); - $count++; - } + //批量取，简单化 + $res = sql_query("select userid, sum(seedtime) as seedtime_sum, sum(leechtime) as leechtime_sum from snatched group by userid where userid in ($idStr)"); + $seedtimeUpdates = $leechTimeUpdates = []; + $nowStr = now()->toDateTimeString(); + $count = 0; + while ($row = mysql_fetch_assoc($res)) { + $count++; + $seedtimeUpdates = sprintf("when %d then %d", $row['userid'], $row['seedtime_sum'] ?? 0); + $leechTimeUpdates = sprintf("when %d then %d", $row['userid'], $row['leechtime_sum'] ?? 0); } + $sql = sprintf( + "update users set seedtime = case id %s end, leechtime = case id %s end, seed_time_updated_at = '%s' where id in (%s)", + implode(" ", $seedtimeUpdates), implode(" ", $leechTimeUpdates), $nowStr, $idStr + ); + $result = sql_query($sql); if ($delIdRedisKey) { NexusDB::cache_del($this->idRedisKey); } $costTime = time() - $beginTimestamp; - do_log("$logPrefix, [DONE], user total count: " . count($uidArr) . ", success update count: $count, cost time: $costTime seconds"); + do_log(sprintf( + "$logPrefix, [DONE], update user count: %s, result: %s, cost time: %s seconds", + $count, var_export($result, true), $costTime + )); } /** diff --git a/app/Utils/ThirdPartyJob.php b/app/Utils/ThirdPartyJob.php index 00251ce70..96a8b4a12 100644 --- a/app/Utils/ThirdPartyJob.php +++ b/app/Utils/ThirdPartyJob.php @@ -44,7 +44,7 @@ public function __invoke(): void public static function addBuyTorrent(int $userId, int $torrentId): void { - $key = sprintf("%s:%s_%s_%s", self::$queueKey, convertNamespaceToSnake(__METHOD__), $userId, $torrentId); + $key = sprintf("%s:%s_%s", self::$queueKey, $userId, $torrentId); if (NexusDB::redis()->set($key, now()->toDateTimeString(), ['nx', 'ex' => 3600])) { $value = [ 'name' => self::JOB_BUY_TORRENT,