320 lines
11 KiB
Python
320 lines
11 KiB
Python
|
# -*- coding: utf-8 -*-
|
|||
|
|
import asyncio
|
|||
|
|
import json
|
|||
|
|
from typing import Any, Callable, Dict, List, Optional, Union
|
|||
|
|
from urllib.parse import urlencode
|
|||
|
|
|
|||
|
|
import httpx
|
|||
|
|
from playwright.async_api import BrowserContext, Page
|
|||
|
|
from tenacity import retry, stop_after_attempt, wait_fixed
|
|||
|
|
|
|||
|
|
import config
|
|||
|
|
from base.base_crawler import AbstractApiClient
|
|||
|
|
from constant import zhihu as zhihu_constant
|
|||
|
|
from model.m_zhihu import ZhihuComment, ZhihuContent
|
|||
|
|
from tools import utils
|
|||
|
|
|
|||
|
|
from .exception import DataFetchError, ForbiddenError
|
|||
|
|
from .field import SearchSort, SearchTime, SearchType
|
|||
|
|
from .help import ZhiHuJsonExtractor, sign
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ZhiHuClient(AbstractApiClient):
|
|||
|
|
def __init__(
|
|||
|
|
self,
|
|||
|
|
timeout=10,
|
|||
|
|
proxies=None,
|
|||
|
|
*,
|
|||
|
|
headers: Dict[str, str],
|
|||
|
|
playwright_page: Page,
|
|||
|
|
cookie_dict: Dict[str, str],
|
|||
|
|
):
|
|||
|
|
self.proxies = proxies
|
|||
|
|
self.timeout = timeout
|
|||
|
|
self.default_headers = headers
|
|||
|
|
self.cookie_dict = cookie_dict
|
|||
|
|
self._extractor = ZhiHuJsonExtractor()
|
|||
|
|
|
|||
|
|
async def _pre_headers(self, url: str) -> Dict:
|
|||
|
|
"""
|
|||
|
|
请求头参数签名
|
|||
|
|
Args:
|
|||
|
|
url: 请求的URL需要包含请求的参数
|
|||
|
|
Returns:
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
d_c0 = self.cookie_dict.get("d_c0")
|
|||
|
|
if not d_c0:
|
|||
|
|
raise Exception("d_c0 not found in cookies")
|
|||
|
|
sign_res = sign(url, self.default_headers["cookie"])
|
|||
|
|
headers = self.default_headers.copy()
|
|||
|
|
headers['x-zst-81'] = sign_res["x-zst-81"]
|
|||
|
|
headers['x-zse-96'] = sign_res["x-zse-96"]
|
|||
|
|
return headers
|
|||
|
|
|
|||
|
|
@retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
|
|||
|
|
async def request(self, method, url, **kwargs) -> Union[str, Any]:
|
|||
|
|
"""
|
|||
|
|
封装httpx的公共请求方法，对请求响应做一些处理
|
|||
|
|
Args:
|
|||
|
|
method: 请求方法
|
|||
|
|
url: 请求的URL
|
|||
|
|
**kwargs: 其他请求参数，例如请求头、请求体等
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
# return response.text
|
|||
|
|
return_response = kwargs.pop('return_response', False)
|
|||
|
|
|
|||
|
|
async with httpx.AsyncClient(proxies=self.proxies, ) as client:
|
|||
|
|
response = await client.request(
|
|||
|
|
method, url, timeout=self.timeout,
|
|||
|
|
**kwargs
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if response.status_code != 200:
|
|||
|
|
utils.logger.error(f"[ZhiHuClient.request] Requset Url: {url}, Request error: {response.text}")
|
|||
|
|
if response.status_code == 403:
|
|||
|
|
raise ForbiddenError(response.text)
|
|||
|
|
elif response.status_code == 404: # 如果一个content没有评论也是404
|
|||
|
|
return {}
|
|||
|
|
|
|||
|
|
raise DataFetchError(response.text)
|
|||
|
|
|
|||
|
|
if return_response:
|
|||
|
|
return response.text
|
|||
|
|
try:
|
|||
|
|
data: Dict = response.json()
|
|||
|
|
if data.get("error"):
|
|||
|
|
utils.logger.error(f"[ZhiHuClient.request] Request error: {data}")
|
|||
|
|
raise DataFetchError(data.get("error", {}).get("message"))
|
|||
|
|
return data
|
|||
|
|
except json.JSONDecodeError:
|
|||
|
|
utils.logger.error(f"[ZhiHuClient.request] Request error: {response.text}")
|
|||
|
|
raise DataFetchError(response.text)
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def get(self, uri: str, params=None) -> Dict:
|
|||
|
|
"""
|
|||
|
|
GET请求，对请求头签名
|
|||
|
|
Args:
|
|||
|
|
uri: 请求路由
|
|||
|
|
params: 请求参数
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
final_uri = uri
|
|||
|
|
if isinstance(params, dict):
|
|||
|
|
final_uri += '?' + urlencode(params)
|
|||
|
|
headers = await self._pre_headers(final_uri)
|
|||
|
|
return await self.request(method="GET", url=zhihu_constant.ZHIHU_URL + final_uri, headers=headers)
|
|||
|
|
|
|||
|
|
async def pong(self) -> bool:
|
|||
|
|
"""
|
|||
|
|
用于检查登录态是否失效了
|
|||
|
|
Returns:
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
utils.logger.info("[ZhiHuClient.pong] Begin to pong zhihu...")
|
|||
|
|
ping_flag = False
|
|||
|
|
try:
|
|||
|
|
res = await self.get_current_user_info()
|
|||
|
|
if res.get("uid") and res.get("name"):
|
|||
|
|
ping_flag = True
|
|||
|
|
utils.logger.info("[ZhiHuClient.pong] Ping zhihu successfully")
|
|||
|
|
else:
|
|||
|
|
utils.logger.error(f"[ZhiHuClient.pong] Ping zhihu failed, response data: {res}")
|
|||
|
|
except Exception as e:
|
|||
|
|
utils.logger.error(f"[ZhiHuClient.pong] Ping zhihu failed: {e}, and try to login again...")
|
|||
|
|
ping_flag = False
|
|||
|
|
return ping_flag
|
|||
|
|
|
|||
|
|
async def update_cookies(self, browser_context: BrowserContext):
|
|||
|
|
"""
|
|||
|
|
API客户端提供的更新cookies方法，一般情况下登录成功后会调用此方法
|
|||
|
|
Args:
|
|||
|
|
browser_context: 浏览器上下文对象
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
cookie_str, cookie_dict = utils.convert_cookies(await browser_context.cookies())
|
|||
|
|
self.default_headers["cookie"] = cookie_str
|
|||
|
|
self.cookie_dict = cookie_dict
|
|||
|
|
|
|||
|
|
async def get_current_user_info(self) -> Dict:
|
|||
|
|
"""
|
|||
|
|
获取当前登录用户信息
|
|||
|
|
Returns:
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
params = {
|
|||
|
|
"include": "email,is_active,is_bind_phone"
|
|||
|
|
}
|
|||
|
|
return await self.get("/api/v4/me", params)
|
|||
|
|
|
|||
|
|
async def get_note_by_keyword(
|
|||
|
|
self, keyword: str,
|
|||
|
|
page: int = 1,
|
|||
|
|
page_size: int = 20,
|
|||
|
|
sort: SearchSort = SearchSort.DEFAULT,
|
|||
|
|
note_type: SearchType = SearchType.DEFAULT,
|
|||
|
|
search_time: SearchTime = SearchTime.DEFAULT
|
|||
|
|
) -> List[ZhihuContent]:
|
|||
|
|
"""
|
|||
|
|
根据关键词搜索
|
|||
|
|
Args:
|
|||
|
|
keyword: 关键词
|
|||
|
|
page: 第几页
|
|||
|
|
page_size: 分页size
|
|||
|
|
sort: 排序
|
|||
|
|
note_type: 搜索结果类型
|
|||
|
|
search_time: 搜索多久时间的结果
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
uri = "/api/v4/search_v3"
|
|||
|
|
params = {
|
|||
|
|
"gk_version": "gz-gaokao",
|
|||
|
|
"t": "general",
|
|||
|
|
"q": keyword,
|
|||
|
|
"correction": 1,
|
|||
|
|
"offset": (page - 1) * page_size,
|
|||
|
|
"limit": page_size,
|
|||
|
|
"filter_fields": "",
|
|||
|
|
"lc_idx": (page - 1) * page_size,
|
|||
|
|
"show_all_topics": 0,
|
|||
|
|
"search_source": "Filter",
|
|||
|
|
"time_interval": search_time.value,
|
|||
|
|
"sort": sort.value,
|
|||
|
|
"vertical": note_type.value,
|
|||
|
|
}
|
|||
|
|
search_res = await self.get(uri, params)
|
|||
|
|
utils.logger.info(f"[ZhiHuClient.get_note_by_keyword] Search result: {search_res}")
|
|||
|
|
return self._extractor.extract_contents(search_res)
|
|||
|
|
|
|||
|
|
async def get_root_comments(self, content_id: str, content_type: str, offset: str = "", limit: int = 10,
|
|||
|
|
order_by: str = "sort") -> Dict:
|
|||
|
|
"""
|
|||
|
|
获取内容的一级评论
|
|||
|
|
Args:
|
|||
|
|
content_id: 内容ID
|
|||
|
|
content_type: 内容类型(answer, article, zvideo)
|
|||
|
|
offset:
|
|||
|
|
limit:
|
|||
|
|
order_by:
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
uri = f"/api/v4/{content_type}s/{content_id}/root_comments"
|
|||
|
|
params = {
|
|||
|
|
"order": order_by,
|
|||
|
|
"offset": offset,
|
|||
|
|
"limit": limit
|
|||
|
|
}
|
|||
|
|
return await self.get(uri, params)
|
|||
|
|
|
|||
|
|
async def get_child_comments(self, root_comment_id: str, offset: str = "", limit: int = 10,
|
|||
|
|
order_by: str = "sort") -> Dict:
|
|||
|
|
"""
|
|||
|
|
获取一级评论下的子评论
|
|||
|
|
Args:
|
|||
|
|
root_comment_id:
|
|||
|
|
offset:
|
|||
|
|
limit:
|
|||
|
|
order_by:
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
uri = f"/api/v4/comment_v5/comment/{root_comment_id}/child_comment"
|
|||
|
|
params = {
|
|||
|
|
"order": order_by,
|
|||
|
|
"offset": offset,
|
|||
|
|
"limit": limit
|
|||
|
|
}
|
|||
|
|
return await self.get(uri, params)
|
|||
|
|
|
|||
|
|
async def get_note_all_comments(self, content: ZhihuContent, crawl_interval: float = 1.0,
|
|||
|
|
callback: Optional[Callable] = None) -> List[ZhihuComment]:
|
|||
|
|
"""
|
|||
|
|
获取指定帖子下的所有一级评论，该方法会一直查找一个帖子下的所有评论信息
|
|||
|
|
Args:
|
|||
|
|
content: 内容详情对象(问题｜文章｜视频)
|
|||
|
|
crawl_interval: 爬取一次笔记的延迟单位（秒）
|
|||
|
|
callback: 一次笔记爬取结束后
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
result: List[ZhihuComment] = []
|
|||
|
|
is_end: bool = False
|
|||
|
|
offset: str = ""
|
|||
|
|
limit: int = 10
|
|||
|
|
while not is_end:
|
|||
|
|
root_comment_res = await self.get_root_comments(content.content_id, content.content_type, offset, limit)
|
|||
|
|
if not root_comment_res:
|
|||
|
|
break
|
|||
|
|
paging_info = root_comment_res.get("paging", {})
|
|||
|
|
is_end = paging_info.get("is_end")
|
|||
|
|
offset = self._extractor.extract_offset(paging_info)
|
|||
|
|
comments = self._extractor.extract_comments(content, root_comment_res.get("data"))
|
|||
|
|
|
|||
|
|
if not comments:
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
if callback:
|
|||
|
|
await callback(comments)
|
|||
|
|
|
|||
|
|
result.extend(comments)
|
|||
|
|
await self.get_comments_all_sub_comments(content, comments, crawl_interval=crawl_interval, callback=callback)
|
|||
|
|
await asyncio.sleep(crawl_interval)
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
async def get_comments_all_sub_comments(self, content: ZhihuContent, comments: List[ZhihuComment], crawl_interval: float = 1.0,
|
|||
|
|
callback: Optional[Callable] = None) -> List[ZhihuComment]:
|
|||
|
|
"""
|
|||
|
|
获取指定评论下的所有子评论
|
|||
|
|
Args:
|
|||
|
|
content: 内容详情对象(问题｜文章｜视频)
|
|||
|
|
comments: 评论列表
|
|||
|
|
crawl_interval: 爬取一次笔记的延迟单位（秒）
|
|||
|
|
callback: 一次笔记爬取结束后
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
if not config.ENABLE_GET_SUB_COMMENTS:
|
|||
|
|
return []
|
|||
|
|
|
|||
|
|
all_sub_comments: List[ZhihuComment] = []
|
|||
|
|
for parment_comment in comments:
|
|||
|
|
if parment_comment.sub_comment_count == 0:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
is_end: bool = False
|
|||
|
|
offset: str = ""
|
|||
|
|
limit: int = 10
|
|||
|
|
while not is_end:
|
|||
|
|
child_comment_res = await self.get_child_comments(parment_comment.comment_id, offset, limit)
|
|||
|
|
if not child_comment_res:
|
|||
|
|
break
|
|||
|
|
paging_info = child_comment_res.get("paging", {})
|
|||
|
|
is_end = paging_info.get("is_end")
|
|||
|
|
offset = self._extractor.extract_offset(paging_info)
|
|||
|
|
sub_comments = self._extractor.extract_comments(content, child_comment_res.get("data"))
|
|||
|
|
|
|||
|
|
if not sub_comments:
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
if callback:
|
|||
|
|
await callback(sub_comments)
|
|||
|
|
|
|||
|
|
all_sub_comments.extend(sub_comments)
|
|||
|
|
await asyncio.sleep(crawl_interval)
|
|||
|
|
return all_sub_comments
|