diff --git a/config/base_config.py b/config/base_config.py index f85a5a7..26761f8 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -34,7 +34,7 @@ CRAWLER_MAX_NOTES_COUNT = 20 MAX_CONCURRENCY_NUM = 4 # 是否开启爬图片模式, 默认不开启爬图片 -ENABLE_GET_IMAGES = True +ENABLE_GET_IMAGES = False # 是否开启爬评论模式, 默认不开启爬评论 ENABLE_GET_COMMENTS = False diff --git a/media_platform/weibo/client.py b/media_platform/weibo/client.py index 89b3ee1..21650e4 100644 --- a/media_platform/weibo/client.py +++ b/media_platform/weibo/client.py @@ -12,7 +12,6 @@ from urllib.parse import urlencode import httpx from playwright.async_api import BrowserContext, Page - from tools import utils from .exception import DataFetchError diff --git a/media_platform/weibo/core.py b/media_platform/weibo/core.py index 1a382ca..4395c8b 100644 --- a/media_platform/weibo/core.py +++ b/media_platform/weibo/core.py @@ -10,11 +10,10 @@ import random from asyncio import Task from typing import Dict, List, Optional, Tuple -from playwright.async_api import (BrowserContext, BrowserType, Page, - async_playwright) - import config from base.base_crawler import AbstractCrawler +from playwright.async_api import (BrowserContext, BrowserType, Page, + async_playwright) from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool from store import weibo as weibo_store from tools import utils diff --git a/media_platform/weibo/login.py b/media_platform/weibo/login.py index dd0d737..c2e99af 100644 --- a/media_platform/weibo/login.py +++ b/media_platform/weibo/login.py @@ -8,11 +8,10 @@ import functools import sys from typing import Optional +from base.base_crawler import AbstractLogin from playwright.async_api import BrowserContext, Page from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt, wait_fixed) - -from base.base_crawler import AbstractLogin from tools import utils diff --git a/store/weibo/__init__.py b/store/weibo/__init__.py index 533296f..3d8e590 100644 --- a/store/weibo/__init__.py +++ b/store/weibo/__init__.py @@ -7,8 +7,8 @@ from typing import List import config -from .weibo_store_impl import * from .weibo_store_image import * +from .weibo_store_impl import * class WeibostoreFactory: diff --git a/store/weibo/weibo_store_image.py b/store/weibo/weibo_store_image.py index 1243b9d..6b89fe0 100644 --- a/store/weibo/weibo_store_image.py +++ b/store/weibo/weibo_store_image.py @@ -3,12 +3,13 @@ # @Time : 2024/4/9 17:35 # @Desc : 微博保存图片类 import pathlib - -from tools import utils -from base.base_crawler import AbstractStoreImage -import aiofiles from typing import Dict +import aiofiles +from base.base_crawler import AbstractStoreImage +from tools import utils + + class WeiboStoreImage(AbstractStoreImage): image_store_path: str = "data/weibo/images" diff --git a/store/weibo/weibo_store_impl.py b/store/weibo/weibo_store_impl.py index e2df846..673cc2a 100644 --- a/store/weibo/weibo_store_impl.py +++ b/store/weibo/weibo_store_impl.py @@ -10,7 +10,6 @@ import pathlib from typing import Dict import aiofiles - from base.base_crawler import AbstractStore from tools import utils from var import crawler_type_var