qnloft
/
MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

When the query has no content, terminate the loop early

This commit is contained in:
AuYeung 2024-07-18 20:44:40 +08:00
parent 548271e537
commit 1fd7827e36
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
media_platform/xhs/core.py
View File

@ -110,6 +110,9 @@ class XiaoHongShuCrawler(AbstractCrawler):
sort=SearchSortType(config.SORT_TYPE) if config.SORT_TYPE != '' else SearchSortType.GENERAL,
)
utils.logger.info(f"[XiaoHongShuCrawler.search] Search notes res:{notes_res}")
if(not notes_res or not notes_res.get('has_more', False)):
utils.logger.info("No more content!")
break
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list = [
self.get_note_detail(post_item.get("id"), semaphore)