feat: B站爬虫搜索关键词实现
This commit is contained in:
Relakkes
parent
5aeee93fc5
commit
a90b411e68
|
|
@ -14,6 +14,7 @@ from tools import utils
|
|||
|
||||
from .help import BilibiliSign
|
||||
from .exception import DataFetchError
|
||||
from .field import OrderType
|
||||
|
||||
|
||||
class BilibiliClient:
|
||||
|
|
@ -53,7 +54,7 @@ class BilibiliClient:
|
|||
:param req_data:
|
||||
:return:
|
||||
"""
|
||||
img_key, sub_key = self.get_wbi_keys()
|
||||
img_key, sub_key = await self.get_wbi_keys()
|
||||
return BilibiliSign(img_key, sub_key).sign(req_data)
|
||||
|
||||
async def get_wbi_keys(self) -> tuple[str, str]:
|
||||
|
|
@ -62,9 +63,10 @@ class BilibiliClient:
|
|||
:return:
|
||||
"""
|
||||
local_storage = await self.playwright_page.evaluate("() => window.localStorage")
|
||||
wbi_img_urls = local_storage.get("wbi_img_urls", "")
|
||||
img_url, sub_url = wbi_img_urls.split("-")
|
||||
if not img_url or not sub_url:
|
||||
wbi_img_urls = local_storage.get("wbi_img_urls", "") or local_storage.get("wbi_img_url") + "-" + local_storage.get("wbi_sub_url")
|
||||
if wbi_img_urls and "-" in wbi_img_urls:
|
||||
img_url, sub_url = wbi_img_urls.split("-")
|
||||
else:
|
||||
resp = await self.request(method="GET", url=self._host + "/x/web-interface/nav")
|
||||
img_url: str = resp['wbi_img']['img_url']
|
||||
sub_url: str = resp['wbi_img']['sub_url']
|
||||
|
|
@ -74,14 +76,14 @@ class BilibiliClient:
|
|||
|
||||
async def get(self, uri: str, params=None) -> Dict:
|
||||
final_uri = uri
|
||||
params = self.pre_request_data(params)
|
||||
params = await self.pre_request_data(params)
|
||||
if isinstance(params, dict):
|
||||
final_uri = (f"{uri}?"
|
||||
f"{urlencode(params)}")
|
||||
return await self.request(method="GET", url=f"{self._host}{final_uri}", headers=self.headers)
|
||||
|
||||
async def post(self, uri: str, data: dict) -> Dict:
|
||||
data = self.pre_request_data(data)
|
||||
data = await self.pre_request_data(data)
|
||||
json_str = json.dumps(data, separators=(',', ':'), ensure_ascii=False)
|
||||
return await self.request(method="POST", url=f"{self._host}{uri}",
|
||||
data=json_str, headers=self.headers)
|
||||
|
|
@ -102,21 +104,30 @@ class BilibiliClient:
|
|||
self.headers["Cookie"] = cookie_str
|
||||
self.cookie_dict = cookie_dict
|
||||
|
||||
async def search_info_by_keyword(self, keyword: str, pcursor: str):
|
||||
async def search_video_by_keyword(self, keyword: str, page: int = 1, page_size: int = 20,
|
||||
order: OrderType = OrderType.DEFAULT):
|
||||
"""
|
||||
KuaiShou web search api
|
||||
:param keyword: search keyword
|
||||
:param pcursor: limite page curson
|
||||
:param keyword: 搜索关键词
|
||||
:param page: 分页参数具体第几页
|
||||
:param page_size: 每一页参数的数量
|
||||
:param order: 搜索结果排序，默认位综合排序
|
||||
:return:
|
||||
"""
|
||||
uri = "/x/web-interface/wbi/search/type"
|
||||
post_data = {
|
||||
"search_type": "video",
|
||||
"keyword": keyword,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
"order": order.value
|
||||
}
|
||||
return await self.post("", post_data)
|
||||
return await self.get(uri, post_data)
|
||||
|
||||
async def get_video_info(self, photo_id: str) -> Dict:
|
||||
async def get_video_info(self, video_id: str) -> Dict:
|
||||
"""
|
||||
Kuaishou web video detail api
|
||||
:param photo_id:
|
||||
:param video_id:
|
||||
:return:
|
||||
"""
|
||||
post_data = {
|
||||
|
|
|
|||
|
|
@ -15,13 +15,14 @@ from playwright.async_api import (BrowserContext, BrowserType, Page,
|
|||
|
||||
import config
|
||||
from base.base_crawler import AbstractCrawler
|
||||
from models import kuaishou
|
||||
from models import bilibili
|
||||
from proxy.proxy_account_pool import AccountPool
|
||||
from tools import utils
|
||||
from var import comment_tasks_var, crawler_type_var
|
||||
|
||||
|
||||
from .client import BilibiliClient
|
||||
from .exception import DataFetchError
|
||||
from .field import OrderType
|
||||
from .login import BilibiliLogin
|
||||
|
||||
|
||||
|
|
@ -45,7 +46,137 @@ class BilibiliCrawler(AbstractCrawler):
|
|||
self.crawler_type = crawler_type
|
||||
|
||||
async def start(self):
|
||||
account_phone, playwright_proxy, httpx_proxy = self.create_proxy_info()
|
||||
async with async_playwright() as playwright:
|
||||
# Launch a browser context.
|
||||
chromium = playwright.chromium
|
||||
self.browser_context = await self.launch_browser(
|
||||
chromium,
|
||||
playwright_proxy,
|
||||
self.user_agent,
|
||||
headless=config.HEADLESS
|
||||
)
|
||||
# stealth.min.js is a js script to prevent the website from detecting the crawler.
|
||||
await self.browser_context.add_init_script(path="libs/stealth.min.js")
|
||||
self.context_page = await self.browser_context.new_page()
|
||||
await self.context_page.goto(self.index_url)
|
||||
|
||||
# Create a client to interact with the xiaohongshu website.
|
||||
self.bili_client = await self.create_bilibili_client(httpx_proxy)
|
||||
if not await self.bili_client.pong():
|
||||
login_obj = BilibiliLogin(
|
||||
login_type=self.login_type,
|
||||
login_phone=account_phone,
|
||||
browser_context=self.browser_context,
|
||||
context_page=self.context_page,
|
||||
cookie_str=config.COOKIES
|
||||
)
|
||||
await login_obj.begin()
|
||||
await self.bili_client.update_cookies(browser_context=self.browser_context)
|
||||
|
||||
crawler_type_var.set(self.crawler_type)
|
||||
if self.crawler_type == "search":
|
||||
# Search for video and retrieve their comment information.
|
||||
await self.search()
|
||||
elif self.crawler_type == "detail":
|
||||
# Get the information and comments of the specified post
|
||||
await self.get_specified_videos()
|
||||
else:
|
||||
pass
|
||||
utils.logger.info("Bilibili Crawler finished ...")
|
||||
pass
|
||||
|
||||
async def search(self):
|
||||
"""
|
||||
search bilibili video with keywords
|
||||
:return:
|
||||
"""
|
||||
utils.logger.info("Begin search bilibli keywords")
|
||||
bili_limit_count = 20 # bilibili limit page fixed value
|
||||
for keyword in config.KEYWORDS.split(","):
|
||||
utils.logger.info(f"Current search keyword: {keyword}")
|
||||
page = 1
|
||||
while page * bili_limit_count <= config.CRAWLER_MAX_NOTES_COUNT:
|
||||
video_id_list: List[str] = []
|
||||
videos_res = await self.bili_client.search_video_by_keyword(
|
||||
keyword=keyword,
|
||||
page=page,
|
||||
page_size=bili_limit_count,
|
||||
order=OrderType.DEFAULT,
|
||||
)
|
||||
video_list: List[Dict] = videos_res.get("result")
|
||||
for video_item in video_list:
|
||||
video_id_list.append(video_item.get("id"))
|
||||
await bilibili.update_bilibili_video(video_item)
|
||||
page += 1
|
||||
|
||||
async def get_specified_videos(self):
|
||||
"""
|
||||
get specified videos info
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
async def create_bilibili_client(self, httpx_proxy: Optional[str]) -> BilibiliClient:
|
||||
"""Create xhs client"""
|
||||
utils.logger.info("Begin create xiaohongshu API client ...")
|
||||
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
|
||||
bilibili_client_obj = BilibiliClient(
|
||||
proxies=httpx_proxy,
|
||||
headers={
|
||||
"User-Agent": self.user_agent,
|
||||
"Cookie": cookie_str,
|
||||
"Origin": "https://www.bilibili.com",
|
||||
"Referer": "https://www.bilibili.com",
|
||||
"Content-Type": "application/json;charset=UTF-8"
|
||||
},
|
||||
playwright_page=self.context_page,
|
||||
cookie_dict=cookie_dict,
|
||||
)
|
||||
return bilibili_client_obj
|
||||
|
||||
def create_proxy_info(self) -> Tuple[Optional[str], Optional[Dict], Optional[str]]:
|
||||
"""Create proxy info for playwright and httpx"""
|
||||
# phone: 13012345671 ip_proxy: 111.122.xx.xx1:8888
|
||||
phone, ip_proxy = self.account_pool.get_account()
|
||||
if not config.ENABLE_IP_PROXY:
|
||||
return phone, None, None
|
||||
utils.logger.info("Begin proxy info for playwright and httpx ...")
|
||||
playwright_proxy = {
|
||||
"server": f"{config.IP_PROXY_PROTOCOL}{ip_proxy}",
|
||||
"username": config.IP_PROXY_USER,
|
||||
"password": config.IP_PROXY_PASSWORD,
|
||||
}
|
||||
httpx_proxy = f"{config.IP_PROXY_PROTOCOL}{config.IP_PROXY_USER}:{config.IP_PROXY_PASSWORD}@{ip_proxy}"
|
||||
return phone, playwright_proxy, httpx_proxy
|
||||
|
||||
async def launch_browser(
|
||||
self,
|
||||
chromium: BrowserType,
|
||||
playwright_proxy: Optional[Dict],
|
||||
user_agent: Optional[str],
|
||||
headless: bool = True
|
||||
) -> BrowserContext:
|
||||
"""Launch browser and create browser context"""
|
||||
utils.logger.info("Begin create browser context ...")
|
||||
if config.SAVE_LOGIN_STATE:
|
||||
# feat issue #14
|
||||
# we will save login state to avoid login every time
|
||||
user_data_dir = os.path.join(os.getcwd(), "browser_data",
|
||||
config.USER_DATA_DIR % self.platform) # type: ignore
|
||||
browser_context = await chromium.launch_persistent_context(
|
||||
user_data_dir=user_data_dir,
|
||||
accept_downloads=True,
|
||||
headless=headless,
|
||||
proxy=playwright_proxy, # type: ignore
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
user_agent=user_agent
|
||||
)
|
||||
return browser_context
|
||||
else:
|
||||
browser = await chromium.launch(headless=headless, proxy=playwright_proxy) # type: ignore
|
||||
browser_context = await browser.new_context(
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
user_agent=user_agent
|
||||
)
|
||||
return browser_context
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# @Author : relakkes@gmail.com
|
||||
# @Time : 2023/12/3 16:20
|
||||
# @Desc :
|
||||
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class OrderType(Enum):
|
||||
# 综合排序
|
||||
DEFAULT = ""
|
||||
|
||||
# 最多点击
|
||||
MOST_CLICK = "click"
|
||||
|
||||
# 最新发布
|
||||
LAST_PUBLISH = "pubdate"
|
||||
|
||||
# 最多弹幕
|
||||
MOST_DANMU = "dm"
|
||||
|
||||
# 最多收藏
|
||||
MOST_MARK = "stow"
|
||||
|
|
@ -52,14 +52,14 @@ class BilibiliSign:
|
|||
salt = self.get_salt()
|
||||
wbi_sign = md5((query + salt).encode()).hexdigest() # 计算 w_rid
|
||||
req_data['w_rid'] = wbi_sign
|
||||
# print(urllib.parse.urlencode(req_data))
|
||||
print(urllib.parse.urlencode(req_data))
|
||||
return req_data
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
_img_key = "7cd084941338484aae1ad9425b84077c"
|
||||
_sub_key = "4932caff0ff746eab6f01bf08b70ac45"
|
||||
_search_url = "category_id=&search_type=video&ad_resource=5654&__refresh__=true&_extra=&context=&page=1&page_size=42&order=click&from_source=&from_spmid=333.337&platform=pc&highlight=1&single_column=0&keyword=python&qv_id=OQ8f2qtgYdBV1UoEnqXUNUl8LEDAdzsD&source_tag=3&gaia_vtoken=&dynamic_offset=0&web_location=1430654"
|
||||
_search_url = "__refresh__=true&_extra=&ad_resource=5654&category_id=&context=&dynamic_offset=0&from_source=&from_spmid=333.337&gaia_vtoken=&highlight=1&keyword=python&order=click&page=1&page_size=20&platform=pc&qv_id=OQ8f2qtgYdBV1UoEnqXUNUl8LEDAdzsD&search_type=video&single_column=0&source_tag=3&web_location=1430654"
|
||||
_req_data = dict()
|
||||
for params in _search_url.split("&"):
|
||||
kvalues = params.split("=")
|
||||
|
|
|
|||
|
|
@ -1,11 +1,38 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# @Author : relakkes@gmail.com
|
||||
# @Time : 2023/12/2 18:44
|
||||
# @Desc :
|
||||
# @Desc : bilibli登录类实现
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
import redis
|
||||
from playwright.async_api import BrowserContext, Page
|
||||
from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt,
|
||||
wait_fixed)
|
||||
|
||||
import config
|
||||
from base.base_crawler import AbstractLogin
|
||||
from tools import utils
|
||||
from base.base_crawler import AbstractLogin
|
||||
|
||||
|
||||
class BilibiliLogin(AbstractLogin):
|
||||
def __init__(self,
|
||||
login_type: str,
|
||||
browser_context: BrowserContext,
|
||||
context_page: Page,
|
||||
login_phone: Optional[str] = "",
|
||||
cookie_str: str = ""
|
||||
):
|
||||
self.login_type = login_type
|
||||
self.browser_context = browser_context
|
||||
self.context_page = context_page
|
||||
self.login_phone = login_phone
|
||||
self.cookie_str = cookie_str
|
||||
|
||||
async def begin(self):
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# @Author : relakkes@gmail.com
|
||||
# @Time : 2023/12/3 16:16
|
||||
# @Desc : B 站的模型类
|
||||
import csv
|
||||
import pathlib
|
||||
from typing import Dict, List
|
||||
|
||||
from tortoise import fields
|
||||
from tortoise.contrib.pydantic import pydantic_model_creator
|
||||
from tortoise.models import Model
|
||||
|
||||
import config
|
||||
from tools import utils
|
||||
from var import crawler_type_var
|
||||
|
||||
|
||||
class BilibiliBaseModel(Model):
|
||||
id = fields.IntField(pk=True, autoincrement=True, description="自增ID")
|
||||
user_id = fields.CharField(null=True, max_length=64, description="用户ID")
|
||||
nickname = fields.CharField(null=True, max_length=64, description="用户昵称")
|
||||
avatar = fields.CharField(null=True, max_length=255, description="用户头像地址")
|
||||
add_ts = fields.BigIntField(description="记录添加时间戳")
|
||||
last_modify_ts = fields.BigIntField(description="记录最后修改时间戳")
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
|
||||
|
||||
class BilibiliVideo(BilibiliBaseModel):
|
||||
video_id = fields.CharField(max_length=64, index=True, description="视频ID")
|
||||
video_type = fields.CharField(max_length=16, description="视频类型")
|
||||
title = fields.CharField(null=True, max_length=500, description="视频标题")
|
||||
desc = fields.TextField(null=True, description="视频描述")
|
||||
create_time = fields.BigIntField(description="视频发布时间戳", index=True)
|
||||
liked_count = fields.CharField(null=True, max_length=16, description="视频点赞数")
|
||||
video_play_count = fields.CharField(null=True, max_length=16, description="视频播放数量")
|
||||
video_danmaku = fields.CharField(null=True, max_length=16, description="视频弹幕数量")
|
||||
video_comment = fields.CharField(null=True, max_length=16, description="视频评论数量")
|
||||
video_url = fields.CharField(null=True, max_length=512, description="视频详情URL")
|
||||
video_cover_url = fields.CharField(null=True, max_length=512, description="视频封面图 URL")
|
||||
|
||||
class Meta:
|
||||
table = "bilibili_video"
|
||||
table_description = "B站视频"
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.video_id} - {self.title}"
|
||||
|
||||
|
||||
async def update_bilibili_video(video_item: Dict):
|
||||
video_id = video_item.get("id")
|
||||
if video_item.get("type") != "video":
|
||||
return
|
||||
local_db_item = {
|
||||
"video_id": video_id,
|
||||
"video_type": str(video_item.get("type")),
|
||||
"title": video_item.get("title", "")[:500],
|
||||
"desc": video_item.get("description", "")[:500],
|
||||
"create_time": video_item.get("pubdate"),
|
||||
"user_id": video_item.get("mid"),
|
||||
"nickname": video_item.get("author"),
|
||||
"avatar": video_item.get("upic", ""),
|
||||
"liked_count": str(video_item.get("like", "")),
|
||||
"video_play_count": str(video_item.get("play", "")),
|
||||
"video_danmaku": str(video_item.get("danmaku", "")),
|
||||
"video_comment": str(video_item.get("review", "")),
|
||||
"last_modify_ts": utils.get_current_timestamp(),
|
||||
"video_url": f"https://www.bilibili.com/video/av{video_id}",
|
||||
"video_cover_url": video_item.get("pic", ""),
|
||||
}
|
||||
print(f"bilibili video id:{video_id}, title:{local_db_item.get('title')}")
|
||||
if config.IS_SAVED_DATABASED:
|
||||
if not await BilibiliVideo.filter(video_id=video_id).exists():
|
||||
local_db_item["add_ts"] = utils.get_current_timestamp()
|
||||
bilibili_video_pydantic = pydantic_model_creator(BilibiliVideo, name='BilibiliVideoCreate', exclude=('id',))
|
||||
bilibili_data = bilibili_video_pydantic(**local_db_item)
|
||||
bilibili_video_pydantic.model_validate(bilibili_data)
|
||||
await BilibiliVideo.create(**bilibili_data.model_dump())
|
||||
else:
|
||||
bilibili_video_pydantic = pydantic_model_creator(BilibiliVideo, name='BilibiliVideoUpdate',
|
||||
exclude=('id', 'add_ts'))
|
||||
bilibili_data = bilibili_video_pydantic(**local_db_item)
|
||||
bilibili_video_pydantic.model_validate(bilibili_data)
|
||||
await BilibiliVideo.filter(video_id=video_id).update(**bilibili_data.model_dump())
|
||||
else:
|
||||
# Below is a simple way to save it in CSV format.
|
||||
pathlib.Path(f"data/bilibili").mkdir(parents=True, exist_ok=True)
|
||||
save_file_name = f"data/bilibili/{crawler_type_var.get()}_videos_{utils.get_current_date()}.csv"
|
||||
with open(save_file_name, mode='a+', encoding="utf-8-sig", newline="") as f:
|
||||
writer = csv.writer(f)
|
||||
if f.tell() == 0:
|
||||
writer.writerow(local_db_item.keys())
|
||||
writer.writerow(local_db_item.values())
|
||||
Loading…
Reference in New Issue