qnloft
/
MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 快手支持指定视频列表爬取

This commit is contained in:
Relakkes 2023-11-27 23:07:04 +08:00
parent 62534d7ee2
commit 33721e5fbd
2 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
media_platform/kuaishou/core.py
View File

@ -74,7 +74,7 @@ class KuaishouCrawler(AbstractCrawler):
await self.search()
elif self.crawler_type == "detail":
# Get the information and comments of the specified post
await self.get_specified_notes()
await self.get_specified_videos()
else:
pass
@ -109,8 +109,17 @@ class KuaishouCrawler(AbstractCrawler):
page += 1
await self.batch_get_video_comments(video_id_list)
async def get_specified_notes(self):
pass
async def get_specified_videos(self):
"""Get the information and comments of the specified post"""
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list = [
self.get_video_info_task(video_id=video_id, semaphore=semaphore) for video_id in config.KS_SPECIFIED_ID_LIST
]
video_details = await asyncio.gather(*task_list)
for video_detail in video_details:
if video_detail is not None:
await kuaishou.update_kuaishou_video(video_detail)
await self.batch_get_video_comments(config.KS_SPECIFIED_ID_LIST)
async def get_video_info_task(self, video_id: str, semaphore: asyncio.Semaphore) -> Optional[Dict]:
"""Get video detail task"""
@ -118,7 +127,7 @@ class KuaishouCrawler(AbstractCrawler):
try:
result = await self.ks_client.get_video_info(video_id)
utils.logger.info(f"Get video_id:{video_id} info result: {result} ...")
return result
return result.get("visionVideoDetail")
except DataFetchError as ex:
utils.logger.error(f"Get video detail error: {ex}")
return None
@ -151,15 +160,16 @@ class KuaishouCrawler(AbstractCrawler):
"""
async with semaphore:
try:
utils.logger.info(f"[get_comments] bengin get video_id: {video_id} comments ...")
await self.ks_client.get_video_all_comments(
photo_id=video_id,
crawl_interval=random.random(),
callback=kuaishou.batch_update_ks_video_comments
)
except DataFetchError as ex:
utils.logger.error(f"Get video_id: {video_id} comment error: {ex}")
utils.logger.error(f"[get_comments] get video_id: {video_id} comment error: {ex}")
except Exception as e:
utils.logger.error(f"map by been blocked, err:", e)
utils.logger.error(f"[get_comments] may be been blocked, err:", e)
# use time.sleeep block main coroutine instead of asyncio.sleep and cacel running comment task
# maybe kuaishou block our request, we will take a nap and update the cookie again
current_running_tasks = comment_tasks_var.get()
@ -169,9 +179,6 @@ class KuaishouCrawler(AbstractCrawler):
await self.context_page.goto(f"{self.index_url}?isHome=1")
await self.ks_client.update_cookies(browser_context=self.browser_context)
def create_proxy_info(self) -> Tuple[Optional[str], Optional[Dict], Optional[str]]:
"""Create proxy info for playwright and httpx"""
# phone: 13012345671 ip_proxy: 111.122.xx.xx1:8888

1
models/kuaishou.py
View File

@ -104,6 +104,7 @@ async def update_kuaishou_video(video_item: Dict):
async def batch_update_ks_video_comments(video_id: str, comments: List[Dict]):
utils.logger.info(f"[batch_update_ks_video_comments] video_id:{video_id}, comments:{comments}")
if not comments:
return
for comment_item in comments: