Skip to content

Commit

Permalink
* no longer re-requesting with param IsFoldCommentReq=1, lumina37/a…
Browse files Browse the repository at this point in the history
…iotieba#123 (comment) @ `ReplyCrawler.GetRequestsForPage()`

@ c#/crawler
$ git submodule update --remote
  • Loading branch information
n0099 committed May 19, 2023
1 parent 1966a74 commit 62f120b
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 17 deletions.
19 changes: 3 additions & 16 deletions c#/crawler/src/Tieba/Crawl/Crawler/ReplyCrawler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public override Exception FillExceptionData(Exception e)
protected override int GetResponseErrorCode(ReplyResponse response) => response.Error.Errorno;
public override TbClient.Page GetResponsePage(ReplyResponse response) => response.Data.Page;

protected override async Task<IEnumerable<Request>> GetRequestsForPage(Page page)
protected override Task<IEnumerable<Request>> GetRequestsForPage(Page page)
{
const string url = "c/f/pb/page?cmd=302001";
const string clientVersion = "12.26.1.0";
Expand All @@ -29,24 +29,11 @@ protected override async Task<IEnumerable<Request>> GetRequestsForPage(Page page
Pn = (int)page,
Rn = 30
};
var response = await Requester.RequestProtoBuf(url, clientVersion,
var response = Requester.RequestProtoBuf(url, clientVersion,
new ReplyRequest {Data = data},
(req, common) => req.Data.Common = common,
() => new ReplyResponse());
var ret = new List<Request>(2) {new(Task.FromResult(response))};
// as of client version 12.12.1.0 (not including), folded replies won't be include in response:
// https://github.com/n0099/TiebaMonitor/commit/b8e7d2645e456271f52457f56500aaedaf28a010#diff-cf67f7f9e82d44aa5be8f85cd24946e5bb7829ca7940c9d056bb1e3849b8f981R32
// so we have to manually requesting the folded replies by appending the returned request tasks
if (response.Data.HasFoldComment != 0)
{
var dataShowOnlyFolded = data.Clone();
dataShowOnlyFolded.IsFoldCommentReq = 1;
ret.Add(new(Requester.RequestProtoBuf(url, clientVersion,
new ReplyRequest {Data = dataShowOnlyFolded},
(req, common) => req.Data.Common = common,
() => new ReplyResponse()), CrawlRequestFlag.ReplyShowOnlyFolded));
}
return ret;
return Task.FromResult(new[] {new Request(response)}.AsEnumerable());
}

public override IList<Reply> GetValidPosts(ReplyResponse response, CrawlRequestFlag flag)
Expand Down

0 comments on commit 62f120b

Please sign in to comment.