diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index 406ef22..758ae0f 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -93,7 +93,9 @@ class BilibiliCrawler(AbstractCrawler): :return: """ utils.logger.info("[BilibiliCrawler.search] Begin search bilibli keywords") - bili_limit_count = 20 # bilibili limit page fixed value + bili_limit_count =20 # bilibili limit page fixed value + if config.CRAWLER_MAX_NOTES_COUNT < bili_limit_count: + config.CRAWLER_MAX_NOTES_COUNT = bili_limit_count for keyword in config.KEYWORDS.split(","): utils.logger.info(f"[BilibiliCrawler.search] Current search keyword: {keyword}") page = 1 diff --git a/media_platform/douyin/core.py b/media_platform/douyin/core.py index d184895..50c2a18 100644 --- a/media_platform/douyin/core.py +++ b/media_platform/douyin/core.py @@ -81,10 +81,12 @@ class DouYinCrawler(AbstractCrawler): async def search(self) -> None: utils.logger.info("[DouYinCrawler.search] Begin search douyin keywords") + dy_limit_count = 10 # douyin limit page fixed value + if config.CRAWLER_MAX_NOTES_COUNT < dy_limit_count: + config.CRAWLER_MAX_NOTES_COUNT = dy_limit_count for keyword in config.KEYWORDS.split(","): utils.logger.info(f"[DouYinCrawler.search] Current keyword: {keyword}") aweme_list: List[str] = [] - dy_limit_count = 10 page = 0 while (page + 1) * dy_limit_count <= config.CRAWLER_MAX_NOTES_COUNT: try: diff --git a/media_platform/kuaishou/core.py b/media_platform/kuaishou/core.py index dbc8b56..4eb7796 100644 --- a/media_platform/kuaishou/core.py +++ b/media_platform/kuaishou/core.py @@ -86,6 +86,8 @@ class KuaishouCrawler(AbstractCrawler): async def search(self): utils.logger.info("[KuaishouCrawler.search] Begin search kuaishou keywords") ks_limit_count = 20 # kuaishou limit page fixed value + if config.CRAWLER_MAX_NOTES_COUNT < ks_limit_count: + config.CRAWLER_MAX_NOTES_COUNT = ks_limit_count for keyword in config.KEYWORDS.split(","): utils.logger.info(f"[KuaishouCrawler.search] Current search keyword: {keyword}") page = 1 diff --git a/media_platform/weibo/core.py b/media_platform/weibo/core.py index eca4324..2ec0eae 100644 --- a/media_platform/weibo/core.py +++ b/media_platform/weibo/core.py @@ -104,7 +104,9 @@ class WeiboCrawler(AbstractCrawler): :return: """ utils.logger.info("[WeiboCrawler.search] Begin search weibo keywords") - weibo_limit_count = 10 + weibo_limit_count = 10 # weibo limit page fixed value + if config.CRAWLER_MAX_NOTES_COUNT < weibo_limit_count: + config.CRAWLER_MAX_NOTES_COUNT = weibo_limit_count for keyword in config.KEYWORDS.split(","): utils.logger.info(f"[WeiboCrawler.search] Current search keyword: {keyword}") page = 1 diff --git a/media_platform/xhs/core.py b/media_platform/xhs/core.py index ef1e2ea..5b67d6d 100644 --- a/media_platform/xhs/core.py +++ b/media_platform/xhs/core.py @@ -97,6 +97,8 @@ class XiaoHongShuCrawler(AbstractCrawler): """Search for notes and retrieve their comment information.""" utils.logger.info("[XiaoHongShuCrawler.search] Begin search xiaohongshu keywords") xhs_limit_count = 20 # xhs limit page fixed value + if config.CRAWLER_MAX_NOTES_COUNT < xhs_limit_count: + config.CRAWLER_MAX_NOTES_COUNT = xhs_limit_count for keyword in config.KEYWORDS.split(","): utils.logger.info(f"[XiaoHongShuCrawler.search] Current search keyword: {keyword}") page = 1