From 95ca606938723d7fb36606e087a6ee2aa814490e Mon Sep 17 00:00:00 2001 From: Relakkes Date: Thu, 23 Nov 2023 23:13:54 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=BF=AB=E6=89=8B=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E5=BB=BA=E7=AB=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- media_platform/kuaishou/__init__.py | 1 + media_platform/kuaishou/client.py | 71 ++++++++++++++++++++++++++++ media_platform/kuaishou/core.py | 1 + media_platform/kuaishou/exception.py | 9 ++++ media_platform/kuaishou/field.py | 1 + media_platform/kuaishou/login.py | 20 ++++++++ 6 files changed, 103 insertions(+) create mode 100644 media_platform/kuaishou/__init__.py create mode 100644 media_platform/kuaishou/client.py create mode 100644 media_platform/kuaishou/core.py create mode 100644 media_platform/kuaishou/exception.py create mode 100644 media_platform/kuaishou/field.py create mode 100644 media_platform/kuaishou/login.py diff --git a/media_platform/kuaishou/__init__.py b/media_platform/kuaishou/__init__.py new file mode 100644 index 0000000..7c68785 --- /dev/null +++ b/media_platform/kuaishou/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/media_platform/kuaishou/client.py b/media_platform/kuaishou/client.py new file mode 100644 index 0000000..e89de5b --- /dev/null +++ b/media_platform/kuaishou/client.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +from typing import Dict +import asyncio +import json +from typing import Dict, Optional + +import httpx +from playwright.async_api import BrowserContext, Page + +from tools import utils + +from .exception import DataFetchError, IPBlockError + + + +class KuaishouClient: + def __init__( + self, + timeout=10, + proxies=None, + *, + headers: Dict[str, str], + playwright_page: Page, + cookie_dict: Dict[str, str], + ): + self.proxies = proxies + self.timeout = timeout + self.headers = headers + self._host = "https://edith.xiaohongshu.com" + self.playwright_page = playwright_page + self.cookie_dict = cookie_dict + + async def _pre_headers(self, url: str, data=None): + pass + + async def request(self, method, url, **kwargs) -> Dict: + async with httpx.AsyncClient(proxies=self.proxies) as client: + response = await client.request( + method, url, timeout=self.timeout, + **kwargs + ) + data: Dict = response.json() + if data["success"]: + return data.get("data", data.get("success", {})) + else: + raise DataFetchError(data.get("msg", None)) + + async def get(self, uri: str, params=None) -> Dict: + final_uri = uri + if isinstance(params, dict): + final_uri = (f"{uri}?" + f"{'&'.join([f'{k}={v}' for k, v in params.items()])}") + headers = await self._pre_headers(final_uri) + return await self.request(method="GET", url=f"{self._host}{final_uri}", headers=headers) + + async def post(self, uri: str, data: dict) -> Dict: + headers = await self._pre_headers(uri, data) + json_str = json.dumps(data, separators=(',', ':'), ensure_ascii=False) + return await self.request(method="POST", url=f"{self._host}{uri}", + data=json_str, headers=headers) + + async def ping(self) -> bool: + """get a note to check if login state is ok""" + utils.logger.info("Begin to ping xhs...") + ping_flag = False + try: + pass + except Exception as e: + utils.logger.error(f"Ping xhs failed: {e}, and try to login again...") + ping_flag = False + return ping_flag diff --git a/media_platform/kuaishou/core.py b/media_platform/kuaishou/core.py new file mode 100644 index 0000000..7c68785 --- /dev/null +++ b/media_platform/kuaishou/core.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/media_platform/kuaishou/exception.py b/media_platform/kuaishou/exception.py new file mode 100644 index 0000000..1a8642e --- /dev/null +++ b/media_platform/kuaishou/exception.py @@ -0,0 +1,9 @@ +from httpx import RequestError + + +class DataFetchError(RequestError): + """something error when fetch""" + + +class IPBlockError(RequestError): + """fetch so fast that the server block us ip""" diff --git a/media_platform/kuaishou/field.py b/media_platform/kuaishou/field.py new file mode 100644 index 0000000..7c68785 --- /dev/null +++ b/media_platform/kuaishou/field.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/media_platform/kuaishou/login.py b/media_platform/kuaishou/login.py new file mode 100644 index 0000000..1564422 --- /dev/null +++ b/media_platform/kuaishou/login.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +from base.base_crawler import AbstractLogin + + +class KuaishouLogin(AbstractLogin): + def __init__(self): + pass + + async def begin(self): + pass + + async def login_by_qrcode(self): + pass + + async def login_by_mobile(self): + pass + + async def login_by_cookies(self): + pass