diff --git a/config/base_config.py b/config/base_config.py index 2985d40..96d87b1 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -46,7 +46,7 @@ MAX_CONCURRENCY_NUM = 1 ENABLE_GET_IMAGES = False # 是否开启爬评论模式, 默认不开启爬评论 -ENABLE_GET_COMMENTS = False +ENABLE_GET_COMMENTS = True # 是否开启爬二级评论模式, 默认不开启爬二级评论 # 老版本项目使用了 db, 则需参考 schema/tables.sql line 287 增加表字段 diff --git a/media_platform/tieba/client.py b/media_platform/tieba/client.py index edbbf19..2aa6cde 100644 --- a/media_platform/tieba/client.py +++ b/media_platform/tieba/client.py @@ -9,8 +9,9 @@ from playwright.async_api import BrowserContext from tenacity import (RetryError, retry, stop_after_attempt, wait_fixed) +import config from base.base_crawler import AbstractApiClient -from model.m_baidu_tieba import TiebaNote +from model.m_baidu_tieba import TiebaNote, TiebaComment from proxy.proxy_ip_pool import ProxyIpPool from tools import utils @@ -195,41 +196,38 @@ class BaiduTieBaClient(AbstractApiClient): page_content = await self.get(uri, return_ori_content=True) return self._page_extractor.extract_note_detail(page_content) - async def get_note_all_comments(self, note_id: str, crawl_interval: float = 1.0, - callback: Optional[Callable] = None) -> List[Dict]: + async def get_note_all_comments(self, note_detail: TiebaNote, crawl_interval: float = 1.0, + callback: Optional[Callable] = None) -> List[TiebaComment]: """ 获取指定帖子下的所有一级评论,该方法会一直查找一个帖子下的所有评论信息 Args: - note_id: 帖子ID + note_detail: 帖子详情对象 crawl_interval: 爬取一次笔记的延迟单位(秒) callback: 一次笔记爬取结束后 Returns: """ - uri = f"/p/{note_id}" - result = [] - comments_has_more = True - comments_cursor = 1 - while comments_has_more: - comments_res = await self.get(uri, params={"pn": comments_cursor}) - comments_has_more = comments_res.get("has_more", False) - comments_cursor = comments_res.get("cursor", "") - if "comments" not in comments_res: - utils.logger.info( - f"[XiaoHongShuClient.get_note_all_comments] No 'comments' key found in response: {comments_res}") + uri = f"/p/{note_detail.note_id}" + result: List[TiebaComment] = [] + current_page = 1 + while note_detail.total_replay_page >= current_page: + params = { + "pn": current_page + } + page_content = await self.get(uri, params=params, return_ori_content=True) + comments = self._page_extractor.extract_tieba_note_parment_comments(page_content, note_id=note_detail.note_id) + if not comments: break - comments = comments_res["comments"] if callback: - await callback(note_id, comments) - await asyncio.sleep(crawl_interval) + await callback(note_detail.note_id, comments) result.extend(comments) - sub_comments = await self.get_comments_all_sub_comments(comments, crawl_interval, callback) - result.extend(sub_comments) + await asyncio.sleep(crawl_interval) + current_page += 1 return result async def get_comments_all_sub_comments(self, comments: List[Dict], crawl_interval: float = 1.0, - callback: Optional[Callable] = None) -> List[Dict]: + callback: Optional[Callable] = None) -> List[TiebaComment]: """ 获取指定评论下的所有子评论 Args: @@ -240,12 +238,7 @@ class BaiduTieBaClient(AbstractApiClient): Returns: """ - result = [] - for comment in comments: - sub_comments = comment.get("comments") - if sub_comments: - if callback: - await callback(comment.get("id"), sub_comments) - await asyncio.sleep(crawl_interval) - result.extend(sub_comments) - return result + if not config.ENABLE_GET_SUB_COMMENTS: + return [] + + # todo 未完成子评论的爬取 diff --git a/media_platform/tieba/core.py b/media_platform/tieba/core.py index a03f0ad..2d10a0a 100644 --- a/media_platform/tieba/core.py +++ b/media_platform/tieba/core.py @@ -114,10 +114,12 @@ class TieBaCrawler(AbstractCrawler): self.get_note_detail_async_task(note_id=note_id, semaphore=semaphore) for note_id in note_id_list ] note_details = await asyncio.gather(*task_list) + note_details_model: List[TiebaNote] = [] for note_detail in note_details: if note_detail is not None: + note_details_model.append(note_detail) await tieba_store.update_tieba_note(note_detail) - await self.batch_get_note_comments(config.TIEBA_SPECIFIED_ID_LIST) + await self.batch_get_note_comments(note_details_model) async def get_note_detail_async_task(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[TiebaNote]: """ @@ -146,42 +148,39 @@ class TieBaCrawler(AbstractCrawler): f"[BaiduTieBaCrawler.get_note_detail] have not fund note detail note_id:{note_id}, err: {ex}") return None - async def batch_get_note_comments(self, note_id_list: List[str]): + async def batch_get_note_comments(self, note_detail_list: List[TiebaNote]): """ Batch get note comments Args: - note_id_list: + note_detail_list: Returns: """ if not config.ENABLE_GET_COMMENTS: - utils.logger.info(f"[BaiduTieBaCrawler.batch_get_note_comments] Crawling comment mode is not enabled") return - utils.logger.info( - f"[BaiduTieBaCrawler.batch_get_note_comments] Begin batch get note comments, note list: {note_id_list}") semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) task_list: List[Task] = [] - for note_id in note_id_list: - task = asyncio.create_task(self.get_comments_async_task(note_id, semaphore), name=note_id) + for note_detail in note_detail_list: + task = asyncio.create_task(self.get_comments_async_task(note_detail, semaphore), name=note_detail.note_id) task_list.append(task) await asyncio.gather(*task_list) - async def get_comments_async_task(self, note_id: str, semaphore: asyncio.Semaphore): + async def get_comments_async_task(self, note_detail: TiebaNote, semaphore: asyncio.Semaphore): """ Get comments async task Args: - note_id: + note_detail: semaphore: Returns: """ async with semaphore: - utils.logger.info(f"[BaiduTieBaCrawler.get_comments] Begin get note id comments {note_id}") + utils.logger.info(f"[BaiduTieBaCrawler.get_comments] Begin get note id comments {note_detail.note_id}") await self.tieba_client.get_note_all_comments( - note_id=note_id, + note_detail=note_detail, crawl_interval=random.random(), callback=tieba_store.batch_update_tieba_note_comments ) diff --git a/media_platform/tieba/help.py b/media_platform/tieba/help.py index efba258..2297855 100644 --- a/media_platform/tieba/help.py +++ b/media_platform/tieba/help.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- import re +import json +import html from typing import List, Dict, Tuple from parsel import Selector -from model.m_baidu_tieba import TiebaNote +from model.m_baidu_tieba import TiebaNote, TiebaComment from constant import baidu_tieba as const @@ -40,7 +42,6 @@ class TieBaExtractor: result.append(tieba_note) return result - def extract_note_detail(self, page_content: str) -> TiebaNote: """ 提取贴吧帖子详情 @@ -66,8 +67,10 @@ class TieBaExtractor: title=content_selector.xpath("//title/text()").get(default='').strip(), desc=content_selector.xpath("//meta[@name='description']/@content").get(default='').strip(), note_url=const.TIEBA_URL + f"/p/{note_id}", - user_link=const.TIEBA_URL + first_floor_selector.xpath(".//a[@class='p_author_face ']/@href").get(default='').strip(), - user_nickname=first_floor_selector.xpath(".//a[@class='p_author_name j_user_card']/text()").get(default='').strip(), + user_link=const.TIEBA_URL + first_floor_selector.xpath(".//a[@class='p_author_face ']/@href").get( + default='').strip(), + user_nickname=first_floor_selector.xpath(".//a[@class='p_author_name j_user_card']/text()").get( + default='').strip(), user_avatar=first_floor_selector.xpath(".//a[@class='p_author_face ']/img/@src").get(default='').strip(), tieba_name=content_selector.xpath("//a[@class='card_title_fname']/text()").get(default='').strip(), tieba_link=const.TIEBA_URL + content_selector.xpath("//a[@class='card_title_fname']/@href").get(default=''), @@ -79,33 +82,44 @@ class TieBaExtractor: note.title = note.title.replace(f"【{note.tieba_name}】_百度贴吧", "") return note - @staticmethod - def extract_tieba_note_comments(page_content: str) -> List[Dict]: + def extract_tieba_note_parment_comments(self, page_content: str, note_id: str) -> List[TiebaComment]: """ - 提取贴吧帖子评论 + 提取贴吧帖子一级评论 Args: page_content: + note_id: Returns: """ - xpath_selector = "//div[@id='j_p_postlist']/div[@class='l_post l_post_bright j_l_post clearfix']" + xpath_selector = "//div[@class='l_post l_post_bright j_l_post clearfix ']" comment_list = Selector(text=page_content).xpath(xpath_selector) - result = [] - for comment in comment_list: - comment_id = comment.xpath(".//@data-pid").get(default='').strip() - author = comment.xpath(".//a[@data-field]/text()").get(default='').strip() - author_link = comment.xpath(".//a[@data-field]/@href").get(default='') - content = comment.xpath(".//div[@class='d_post_content j_d_post_content ']/text()").get(default='').strip() - date = comment.xpath(".//span[@class='tail-info']/text()").get(default='').strip() + result: List[TiebaComment] = [] + for comment_selector in comment_list: + comment_field_value: Dict = self.extract_data_field_value(comment_selector) + if not comment_field_value: + continue - result.append({ - "comment_id": comment_id, - "author": author, - "author_link": author_link, - "content": content, - "time": date, - }) + other_info_content = comment_selector.xpath(".//div[@class='post-tail-wrap']").get(default="").strip() + ip_location, publish_time = self.extract_ip_and_pub_time(other_info_content) + tieba_comment = TiebaComment( + comment_id=str(comment_field_value.get("content").get("post_id")), + sub_comment_count=comment_field_value.get("content").get("comment_num"), + content=comment_field_value.get("content").get("content"), + note_url=const.TIEBA_URL + f"/p/{note_id}", + user_link=const.TIEBA_URL + comment_selector.xpath(".//a[@class='p_author_face ']/@href").get(default='').strip(), + user_nickname=comment_selector.xpath(".//a[@class='p_author_name j_user_card']/text()").get( + default='').strip(), + user_avatar=comment_selector.xpath(".//a[@class='p_author_face ']/img/@src").get( + default='').strip(), + tieba_name=comment_selector.xpath("//a[@class='card_title_fname']/text()").get(default='').strip(), + ip_location=ip_location, + publish_time=publish_time, + note_id=note_id, + ) + print(tieba_comment.model_dump()) + result.append(tieba_comment) + return result @staticmethod def extract_ip_and_pub_time(html_content: str) -> Tuple[str, str]: @@ -125,6 +139,31 @@ class TieBaExtractor: pub_time = time_match.group(1) if time_match else "" return ip, pub_time + @staticmethod + def extract_data_field_value(selector: Selector) -> Dict: + """ + 提取data-field的值 + Args: + selector: + + Returns: + + """ + data_field_value = selector.xpath("./@data-field").get(default='').strip() + if not data_field_value or data_field_value == "{}": + return {} + try: + # 先使用 html.unescape 处理转义字符 再json.loads 将 JSON 字符串转换为 Python 字典 + unescaped_json_str = html.unescape(data_field_value) + data_field_dict_value = json.loads(unescaped_json_str) + except Exception as ex: + print(f"extract_data_field_value,错误信息:{ex}, 尝试使用其他方式解析") + data_field_dict_value = {} + return data_field_dict_value + + + + def test_extract_search_note_list(): with open("test_data/search_keyword_notes.html", "r", encoding="utf-8") as f: content = f.read() @@ -140,7 +179,14 @@ def test_extract_note_detail(): result = extractor.extract_note_detail(content) print(result.model_dump()) +def test_extract_tieba_note_parment_comments(): + with open("test_data/note_comments.html", "r", encoding="utf-8") as f: + content = f.read() + extractor = TieBaExtractor() + result = extractor.extract_tieba_note_parment_comments(content, "123456") + print(result) if __name__ == '__main__': - test_extract_search_note_list() - test_extract_note_detail() + # test_extract_search_note_list() + # test_extract_note_detail() + test_extract_tieba_note_parment_comments() diff --git a/media_platform/tieba/test_data/note_comments.html b/media_platform/tieba/test_data/note_comments.html new file mode 100644 index 0000000..5d08b07 --- /dev/null +++ b/media_platform/tieba/test_data/note_comments.html @@ -0,0 +1,874 @@ +