qnloft
/
MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: weibo creator duplicate bug

This commit is contained in:
Relakkes 2024-11-13 22:28:54 +08:00
parent f9e7d56d4f
commit c12a63acb2
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
media_platform/weibo/client.py
View File

@ -353,10 +353,7 @@ class WeiboClient:
utils.logger.error( utils.logger.error(
f"[WeiboClient.get_notes_by_creator] The current creator may have been banned by xhs, so they cannot access the data.") f"[WeiboClient.get_notes_by_creator] The current creator may have been banned by xhs, so they cannot access the data.")
break break
notes_has_more = notes_res.get("cardlistInfo", {}).get("total", 0) > crawler_total_count
since_id = notes_res.get("cardlistInfo", {}).get("since_id", "0") since_id = notes_res.get("cardlistInfo", {}).get("since_id", "0")
crawler_total_count += 10
if "cards" not in notes_res: if "cards" not in notes_res:
utils.logger.info( utils.logger.info(
f"[WeiboClient.get_all_notes_by_creator] No 'notes' key found in response: {notes_res}") f"[WeiboClient.get_all_notes_by_creator] No 'notes' key found in response: {notes_res}")
@ -370,5 +367,7 @@ class WeiboClient:
await callback(notes) await callback(notes)
await asyncio.sleep(crawl_interval) await asyncio.sleep(crawl_interval)
result.extend(notes) result.extend(notes)
crawler_total_count += 10
notes_has_more = notes_res.get("cardlistInfo", {}).get("total", 0) > crawler_total_count
return result return result