parent
6ae511bb52
commit
a6e877de42
|
@ -54,6 +54,8 @@ class BilibiliClient:
|
||||||
:param req_data:
|
:param req_data:
|
||||||
:return:
|
:return:
|
||||||
"""
|
"""
|
||||||
|
if not req_data:
|
||||||
|
return {}
|
||||||
img_key, sub_key = await self.get_wbi_keys()
|
img_key, sub_key = await self.get_wbi_keys()
|
||||||
return BilibiliSign(img_key, sub_key).sign(req_data)
|
return BilibiliSign(img_key, sub_key).sign(req_data)
|
||||||
|
|
||||||
|
@ -91,10 +93,14 @@ class BilibiliClient:
|
||||||
|
|
||||||
async def pong(self) -> bool:
|
async def pong(self) -> bool:
|
||||||
"""get a note to check if login state is ok"""
|
"""get a note to check if login state is ok"""
|
||||||
utils.logger.info("Begin pong kuaishou...")
|
utils.logger.info("Begin pong bilibili...")
|
||||||
ping_flag = False
|
ping_flag = False
|
||||||
try:
|
try:
|
||||||
pass
|
check_login_uri = "/x/web-interface/nav"
|
||||||
|
response = await self.get(check_login_uri)
|
||||||
|
if response.get("isLogin"):
|
||||||
|
utils.logger.info("use cache login state get web interface successfull!")
|
||||||
|
ping_flag = True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
utils.logger.error(f"Pong kuaishou failed: {e}, and try to login again...")
|
utils.logger.error(f"Pong kuaishou failed: {e}, and try to login again...")
|
||||||
ping_flag = False
|
ping_flag = False
|
||||||
|
|
|
@ -22,7 +22,7 @@ from var import comment_tasks_var, crawler_type_var
|
||||||
|
|
||||||
|
|
||||||
from .client import BilibiliClient
|
from .client import BilibiliClient
|
||||||
from .field import OrderType
|
from .field import SearchOrderType
|
||||||
from .login import BilibiliLogin
|
from .login import BilibiliLogin
|
||||||
|
|
||||||
|
|
||||||
|
@ -102,7 +102,7 @@ class BilibiliCrawler(AbstractCrawler):
|
||||||
keyword=keyword,
|
keyword=keyword,
|
||||||
page=page,
|
page=page,
|
||||||
page_size=bili_limit_count,
|
page_size=bili_limit_count,
|
||||||
order=OrderType.DEFAULT,
|
order=SearchOrderType.DEFAULT,
|
||||||
)
|
)
|
||||||
video_list: List[Dict] = videos_res.get("result")
|
video_list: List[Dict] = videos_res.get("result")
|
||||||
for video_item in video_list:
|
for video_item in video_list:
|
||||||
|
|
|
@ -86,7 +86,7 @@ class DOUYINClient:
|
||||||
return await self.request(method="POST", url=f"{self._host}{uri}", data=data, headers=headers)
|
return await self.request(method="POST", url=f"{self._host}{uri}", data=data, headers=headers)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def ping(browser_context: BrowserContext) -> bool:
|
async def pong(browser_context: BrowserContext) -> bool:
|
||||||
_, cookie_dict = utils.convert_cookies(await browser_context.cookies())
|
_, cookie_dict = utils.convert_cookies(await browser_context.cookies())
|
||||||
# todo send some api to test login status
|
# todo send some api to test login status
|
||||||
return cookie_dict.get("LOGIN_STATUS") == "1"
|
return cookie_dict.get("LOGIN_STATUS") == "1"
|
||||||
|
|
|
@ -54,7 +54,7 @@ class DouYinCrawler(AbstractCrawler):
|
||||||
await self.context_page.goto(self.index_url)
|
await self.context_page.goto(self.index_url)
|
||||||
|
|
||||||
self.dy_client = await self.create_douyin_client(httpx_proxy)
|
self.dy_client = await self.create_douyin_client(httpx_proxy)
|
||||||
if not await self.dy_client.ping(browser_context=self.browser_context):
|
if not await self.dy_client.pong(browser_context=self.browser_context):
|
||||||
login_obj = DouYinLogin(
|
login_obj = DouYinLogin(
|
||||||
login_type=self.login_type,
|
login_type=self.login_type,
|
||||||
login_phone=account_phone,
|
login_phone=account_phone,
|
||||||
|
|
|
@ -55,7 +55,8 @@ class KuaiShouClient:
|
||||||
return await self.request(method="POST", url=f"{self._host}{uri}",
|
return await self.request(method="POST", url=f"{self._host}{uri}",
|
||||||
data=json_str, headers=self.headers)
|
data=json_str, headers=self.headers)
|
||||||
|
|
||||||
async def pong(self) -> bool:
|
@staticmethod
|
||||||
|
async def pong() -> bool:
|
||||||
"""get a note to check if login state is ok"""
|
"""get a note to check if login state is ok"""
|
||||||
utils.logger.info("Begin pong kuaishou...")
|
utils.logger.info("Begin pong kuaishou...")
|
||||||
ping_flag = False
|
ping_flag = False
|
||||||
|
|
|
@ -81,9 +81,9 @@ class XHSClient:
|
||||||
return await self.request(method="POST", url=f"{self._host}{uri}",
|
return await self.request(method="POST", url=f"{self._host}{uri}",
|
||||||
data=json_str, headers=headers)
|
data=json_str, headers=headers)
|
||||||
|
|
||||||
async def ping(self) -> bool:
|
async def pong(self) -> bool:
|
||||||
"""get a note to check if login state is ok"""
|
"""get a note to check if login state is ok"""
|
||||||
utils.logger.info("Begin to ping xhs...")
|
utils.logger.info("Begin to pong xhs...")
|
||||||
ping_flag = False
|
ping_flag = False
|
||||||
try:
|
try:
|
||||||
note_card: Dict = await self.get_note_by_keyword(keyword="小红书")
|
note_card: Dict = await self.get_note_by_keyword(keyword="小红书")
|
||||||
|
|
|
@ -63,7 +63,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||||
|
|
||||||
# Create a client to interact with the xiaohongshu website.
|
# Create a client to interact with the xiaohongshu website.
|
||||||
self.xhs_client = await self.create_xhs_client(httpx_proxy)
|
self.xhs_client = await self.create_xhs_client(httpx_proxy)
|
||||||
if not await self.xhs_client.ping():
|
if not await self.xhs_client.pong():
|
||||||
login_obj = XHSLogin(
|
login_obj = XHSLogin(
|
||||||
login_type=self.login_type,
|
login_type=self.login_type,
|
||||||
login_phone=account_phone,
|
login_phone=account_phone,
|
||||||
|
|
Loading…
Reference in New Issue