From 133f97847755dd3fad984929f17e2ae1bfdfeacf Mon Sep 17 00:00:00 2001
From: leantli <leantli@qq.com>
Date: Wed, 3 Apr 2024 12:18:23 +0800
Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E7=88=AC=E5=8F=96?=
 =?UTF-8?q?=E8=A7=86=E9=A2=91/=E5=B8=96=E5=AD=90=E6=9C=80=E5=A4=A7?=
 =?UTF-8?q?=E6=95=B0=E8=AE=BE=E7=BD=AE=E5=80=BC=E8=BE=83=E4=BD=8E=E5=AF=BC?=
 =?UTF-8?q?=E8=87=B4=E4=B8=8D=E7=88=AC=E5=8F=96=E7=9A=84=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 media_platform/bilibili/core.py | 2 +-
 media_platform/douyin/core.py   | 2 +-
 media_platform/kuaishou/core.py | 2 +-
 media_platform/weibo/core.py    | 2 +-
 media_platform/xhs/core.py      | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py
index 406ef22..5c31b16 100644
--- a/media_platform/bilibili/core.py
+++ b/media_platform/bilibili/core.py
@@ -93,7 +93,7 @@ class BilibiliCrawler(AbstractCrawler):
         :return:
         """
         utils.logger.info("[BilibiliCrawler.search] Begin search bilibli keywords")
-        bili_limit_count = 20  # bilibili limit page fixed value
+        bili_limit_count = min(20, max(1, config.CRAWLER_MAX_NOTES_COUNT))  # bilibili limit page fixed value
         for keyword in config.KEYWORDS.split(","):
             utils.logger.info(f"[BilibiliCrawler.search] Current search keyword: {keyword}")
             page = 1
diff --git a/media_platform/douyin/core.py b/media_platform/douyin/core.py
index d184895..e21a605 100644
--- a/media_platform/douyin/core.py
+++ b/media_platform/douyin/core.py
@@ -84,7 +84,7 @@ class DouYinCrawler(AbstractCrawler):
         for keyword in config.KEYWORDS.split(","):
             utils.logger.info(f"[DouYinCrawler.search] Current keyword: {keyword}")
             aweme_list: List[str] = []
-            dy_limit_count = 10
+            dy_limit_count = min(10, max(1, config.CRAWLER_MAX_NOTES_COUNT))
             page = 0
             while (page + 1) * dy_limit_count <= config.CRAWLER_MAX_NOTES_COUNT:
                 try:
diff --git a/media_platform/kuaishou/core.py b/media_platform/kuaishou/core.py
index dbc8b56..4ec396f 100644
--- a/media_platform/kuaishou/core.py
+++ b/media_platform/kuaishou/core.py
@@ -85,7 +85,7 @@ class KuaishouCrawler(AbstractCrawler):
 
     async def search(self):
         utils.logger.info("[KuaishouCrawler.search] Begin search kuaishou keywords")
-        ks_limit_count = 20  # kuaishou limit page fixed value
+        ks_limit_count = min(20, max(1, config.CRAWLER_MAX_NOTES_COUNT))  # kuaishou limit page fixed value
         for keyword in config.KEYWORDS.split(","):
             utils.logger.info(f"[KuaishouCrawler.search] Current search keyword: {keyword}")
             page = 1
diff --git a/media_platform/weibo/core.py b/media_platform/weibo/core.py
index eca4324..42fb7e5 100644
--- a/media_platform/weibo/core.py
+++ b/media_platform/weibo/core.py
@@ -104,7 +104,7 @@ class WeiboCrawler(AbstractCrawler):
         :return:
         """
         utils.logger.info("[WeiboCrawler.search] Begin search weibo keywords")
-        weibo_limit_count = 10
+        weibo_limit_count = min(10, max(1, config.CRAWLER_MAX_NOTES_COUNT))
         for keyword in config.KEYWORDS.split(","):
             utils.logger.info(f"[WeiboCrawler.search] Current search keyword: {keyword}")
             page = 1
diff --git a/media_platform/xhs/core.py b/media_platform/xhs/core.py
index ef1e2ea..2855af8 100644
--- a/media_platform/xhs/core.py
+++ b/media_platform/xhs/core.py
@@ -96,7 +96,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
     async def search(self) -> None:
         """Search for notes and retrieve their comment information."""
         utils.logger.info("[XiaoHongShuCrawler.search] Begin search xiaohongshu keywords")
-        xhs_limit_count = 20  # xhs limit page fixed value
+        xhs_limit_count = min(20, max(1, config.CRAWLER_MAX_NOTES_COUNT))  # xhs limit page fixed value
         for keyword in config.KEYWORDS.split(","):
             utils.logger.info(f"[XiaoHongShuCrawler.search] Current search keyword: {keyword}")
             page = 1