From c12a63acb2493874ffe6768e857c21f163b01e79 Mon Sep 17 00:00:00 2001 From: Relakkes Date: Wed, 13 Nov 2024 22:28:54 +0800 Subject: [PATCH] fix: weibo creator duplicate bug --- media_platform/weibo/client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/media_platform/weibo/client.py b/media_platform/weibo/client.py index b6c2497..e1deec7 100644 --- a/media_platform/weibo/client.py +++ b/media_platform/weibo/client.py @@ -353,10 +353,7 @@ class WeiboClient: utils.logger.error( f"[WeiboClient.get_notes_by_creator] The current creator may have been banned by xhs, so they cannot access the data.") break - - notes_has_more = notes_res.get("cardlistInfo", {}).get("total", 0) > crawler_total_count since_id = notes_res.get("cardlistInfo", {}).get("since_id", "0") - crawler_total_count += 10 if "cards" not in notes_res: utils.logger.info( f"[WeiboClient.get_all_notes_by_creator] No 'notes' key found in response: {notes_res}") @@ -370,5 +367,7 @@ class WeiboClient: await callback(notes) await asyncio.sleep(crawl_interval) result.extend(notes) + crawler_total_count += 10 + notes_has_more = notes_res.get("cardlistInfo", {}).get("total", 0) > crawler_total_count return result