From 9cd6efb91690d775e33ed5c2295e3dc266d6991c Mon Sep 17 00:00:00 2001 From: Er_Meng <1322567916@qq.com> Date: Wed, 10 Apr 2024 09:54:28 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BD=BF=E7=94=A8isort=E5=AF=B9=E5=BC=95?= =?UTF-8?q?=E7=94=A8=E8=BF=9B=E8=A1=8C=E6=A0=BC=E5=BC=8F=E5=8C=96=E6=8E=92?= =?UTF-8?q?=E5=BA=8F=20=E4=BF=AE=E6=94=B9=E5=BE=AE=E5=8D=9A=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E5=9B=BE=E7=89=87=E9=BB=98=E8=AE=A4=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E5=85=B3=E9=97=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/base_config.py | 2 +- media_platform/weibo/client.py | 1 - media_platform/weibo/core.py | 5 ++--- media_platform/weibo/login.py | 3 +-- store/weibo/__init__.py | 2 +- store/weibo/weibo_store_image.py | 9 +++++---- store/weibo/weibo_store_impl.py | 1 - 7 files changed, 10 insertions(+), 13 deletions(-) diff --git a/config/base_config.py b/config/base_config.py index f85a5a7..26761f8 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -34,7 +34,7 @@ CRAWLER_MAX_NOTES_COUNT = 20 MAX_CONCURRENCY_NUM = 4 # 是否开启爬图片模式, 默认不开启爬图片 -ENABLE_GET_IMAGES = True +ENABLE_GET_IMAGES = False # 是否开启爬评论模式, 默认不开启爬评论 ENABLE_GET_COMMENTS = False diff --git a/media_platform/weibo/client.py b/media_platform/weibo/client.py index 89b3ee1..21650e4 100644 --- a/media_platform/weibo/client.py +++ b/media_platform/weibo/client.py @@ -12,7 +12,6 @@ from urllib.parse import urlencode import httpx from playwright.async_api import BrowserContext, Page - from tools import utils from .exception import DataFetchError diff --git a/media_platform/weibo/core.py b/media_platform/weibo/core.py index 1a382ca..4395c8b 100644 --- a/media_platform/weibo/core.py +++ b/media_platform/weibo/core.py @@ -10,11 +10,10 @@ import random from asyncio import Task from typing import Dict, List, Optional, Tuple -from playwright.async_api import (BrowserContext, BrowserType, Page, - async_playwright) - import config from base.base_crawler import AbstractCrawler +from playwright.async_api import (BrowserContext, BrowserType, Page, + async_playwright) from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool from store import weibo as weibo_store from tools import utils diff --git a/media_platform/weibo/login.py b/media_platform/weibo/login.py index dd0d737..c2e99af 100644 --- a/media_platform/weibo/login.py +++ b/media_platform/weibo/login.py @@ -8,11 +8,10 @@ import functools import sys from typing import Optional +from base.base_crawler import AbstractLogin from playwright.async_api import BrowserContext, Page from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt, wait_fixed) - -from base.base_crawler import AbstractLogin from tools import utils diff --git a/store/weibo/__init__.py b/store/weibo/__init__.py index 533296f..3d8e590 100644 --- a/store/weibo/__init__.py +++ b/store/weibo/__init__.py @@ -7,8 +7,8 @@ from typing import List import config -from .weibo_store_impl import * from .weibo_store_image import * +from .weibo_store_impl import * class WeibostoreFactory: diff --git a/store/weibo/weibo_store_image.py b/store/weibo/weibo_store_image.py index 1243b9d..6b89fe0 100644 --- a/store/weibo/weibo_store_image.py +++ b/store/weibo/weibo_store_image.py @@ -3,12 +3,13 @@ # @Time : 2024/4/9 17:35 # @Desc : 微博保存图片类 import pathlib - -from tools import utils -from base.base_crawler import AbstractStoreImage -import aiofiles from typing import Dict +import aiofiles +from base.base_crawler import AbstractStoreImage +from tools import utils + + class WeiboStoreImage(AbstractStoreImage): image_store_path: str = "data/weibo/images" diff --git a/store/weibo/weibo_store_impl.py b/store/weibo/weibo_store_impl.py index e2df846..673cc2a 100644 --- a/store/weibo/weibo_store_impl.py +++ b/store/weibo/weibo_store_impl.py @@ -10,7 +10,6 @@ import pathlib from typing import Dict import aiofiles - from base.base_crawler import AbstractStore from tools import utils from var import crawler_type_var