diff --git a/base/base_crawler.py b/base/base_crawler.py index 5a59a3d..24dd83f 100644 --- a/base/base_crawler.py +++ b/base/base_crawler.py @@ -55,3 +55,17 @@ class AbstractStore(ABC): # @abstractmethod async def store_creator(self, creator: Dict): pass + + +class AbstactApiClient(ABC): + @abstractmethod + async def request(self, method, url, **kwargs): + pass + + @abstractmethod + async def update_cookies(self, browser_context: BrowserContext): + pass + + @abstractmethod + async def pong(self): + pass diff --git a/media_platform/bilibili/client.py b/media_platform/bilibili/client.py index 372d26d..f2a33a0 100644 --- a/media_platform/bilibili/client.py +++ b/media_platform/bilibili/client.py @@ -10,6 +10,7 @@ from urllib.parse import urlencode import httpx from playwright.async_api import BrowserContext, Page +from base.base_crawler import AbstactApiClient from tools import utils from .exception import DataFetchError @@ -17,7 +18,7 @@ from .field import CommentOrderType, SearchOrderType from .help import BilibiliSign -class BilibiliClient: +class BilibiliClient(AbstactApiClient): def __init__( self, timeout=10, diff --git a/media_platform/douyin/client.py b/media_platform/douyin/client.py index bca4f95..b7c1da9 100644 --- a/media_platform/douyin/client.py +++ b/media_platform/douyin/client.py @@ -7,6 +7,7 @@ import execjs import httpx from playwright.async_api import BrowserContext, Page +from base.base_crawler import AbstactApiClient from tools import utils from var import request_keyword_var @@ -14,7 +15,7 @@ from .exception import * from .field import * -class DOUYINClient: +class DOUYINClient(AbstactApiClient): def __init__( self, timeout=30, diff --git a/media_platform/douyin/core.py b/media_platform/douyin/core.py index 918386a..d184895 100644 --- a/media_platform/douyin/core.py +++ b/media_platform/douyin/core.py @@ -16,8 +16,8 @@ from var import crawler_type_var from .client import DOUYINClient from .exception import DataFetchError -from .login import DouYinLogin from .field import PublishTimeType +from .login import DouYinLogin class DouYinCrawler(AbstractCrawler): diff --git a/media_platform/kuaishou/client.py b/media_platform/kuaishou/client.py index 889cbb3..c30959f 100644 --- a/media_platform/kuaishou/client.py +++ b/media_platform/kuaishou/client.py @@ -8,13 +8,14 @@ import httpx from playwright.async_api import BrowserContext, Page import config +from base.base_crawler import AbstactApiClient from tools import utils from .exception import DataFetchError from .graphql import KuaiShouGraphQL -class KuaiShouClient: +class KuaiShouClient(AbstactApiClient): def __init__( self, timeout=10, diff --git a/media_platform/xhs/client.py b/media_platform/xhs/client.py index 5247ef7..a2db400 100644 --- a/media_platform/xhs/client.py +++ b/media_platform/xhs/client.py @@ -1,12 +1,13 @@ import asyncio import json import re -from typing import Callable, Dict, List, Optional, Union, Any +from typing import Any, Callable, Dict, List, Optional, Union from urllib.parse import urlencode import httpx from playwright.async_api import BrowserContext, Page +from base.base_crawler import AbstactApiClient from tools import utils from .exception import DataFetchError, IPBlockError @@ -14,7 +15,7 @@ from .field import SearchNoteType, SearchSortType from .help import get_search_id, sign -class XiaoHongShuClient: +class XiaoHongShuClient(AbstactApiClient): def __init__( self, timeout=10,