From a0e5a29af89de22b8b52effea0378fde9cdb0915 Mon Sep 17 00:00:00 2001 From: Relakkes Yang Date: Mon, 17 Jun 2024 00:25:48 +0800 Subject: [PATCH] fix: weibo bug --- cmd_arg/arg.py | 1 + media_platform/bilibili/login.py | 2 +- media_platform/douyin/client.py | 2 +- media_platform/kuaishou/login.py | 2 +- media_platform/weibo/login.py | 1 + store/bilibili/bilibili_store_impl.py | 4 ++-- store/douyin/douyin_store_impl.py | 6 +++--- store/kuaishou/kuaishou_store_impl.py | 7 ++++--- store/weibo/weibo_store_impl.py | 7 ++++--- store/xhs/xhs_store_impl.py | 7 ++++--- tools/words.py | 10 ++++++---- 11 files changed, 28 insertions(+), 21 deletions(-) diff --git a/cmd_arg/arg.py b/cmd_arg/arg.py index 2d07675..27854f7 100644 --- a/cmd_arg/arg.py +++ b/cmd_arg/arg.py @@ -1,4 +1,5 @@ import argparse + import config from tools.utils import str2bool diff --git a/media_platform/bilibili/login.py b/media_platform/bilibili/login.py index 33c929f..cc7baa2 100644 --- a/media_platform/bilibili/login.py +++ b/media_platform/bilibili/login.py @@ -12,8 +12,8 @@ from playwright.async_api import BrowserContext, Page from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt, wait_fixed) -from base.base_crawler import AbstractLogin import config +from base.base_crawler import AbstractLogin from tools import utils diff --git a/media_platform/douyin/client.py b/media_platform/douyin/client.py index 97722e0..ce04698 100644 --- a/media_platform/douyin/client.py +++ b/media_platform/douyin/client.py @@ -1,8 +1,8 @@ import asyncio import copy +import json import urllib.parse from typing import Any, Callable, Dict, List, Optional -import json import execjs import httpx diff --git a/media_platform/kuaishou/login.py b/media_platform/kuaishou/login.py index be2ce8d..68462ad 100644 --- a/media_platform/kuaishou/login.py +++ b/media_platform/kuaishou/login.py @@ -7,8 +7,8 @@ from playwright.async_api import BrowserContext, Page from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt, wait_fixed) -from base.base_crawler import AbstractLogin import config +from base.base_crawler import AbstractLogin from tools import utils diff --git a/media_platform/weibo/login.py b/media_platform/weibo/login.py index 2c2cf38..9dc1659 100644 --- a/media_platform/weibo/login.py +++ b/media_platform/weibo/login.py @@ -12,6 +12,7 @@ from playwright.async_api import BrowserContext, Page from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt, wait_fixed) +import config from base.base_crawler import AbstractLogin from tools import utils diff --git a/store/bilibili/bilibili_store_impl.py b/store/bilibili/bilibili_store_impl.py index 7b93432..4a07dff 100644 --- a/store/bilibili/bilibili_store_impl.py +++ b/store/bilibili/bilibili_store_impl.py @@ -13,9 +13,9 @@ import aiofiles import config from base.base_crawler import AbstractStore -from tools import utils +from tools import utils, words from var import crawler_type_var -from tools import words + def calculate_number_of_files(file_store_path: str) -> int: """计算数据保存文件的前部分排序数字，支持每次运行代码不写到同一个文件中 diff --git a/store/douyin/douyin_store_impl.py b/store/douyin/douyin_store_impl.py index a4672ee..ed83450 100644 --- a/store/douyin/douyin_store_impl.py +++ b/store/douyin/douyin_store_impl.py @@ -11,10 +11,10 @@ from typing import Dict import aiofiles -from base.base_crawler import AbstractStore -from tools import utils,words -from var import crawler_type_var import config +from base.base_crawler import AbstractStore +from tools import utils, words +from var import crawler_type_var def calculate_number_of_files(file_store_path: str) -> int: diff --git a/store/kuaishou/kuaishou_store_impl.py b/store/kuaishou/kuaishou_store_impl.py index 523f6f0..57a15ad 100644 --- a/store/kuaishou/kuaishou_store_impl.py +++ b/store/kuaishou/kuaishou_store_impl.py @@ -11,10 +11,11 @@ from typing import Dict import aiofiles -from base.base_crawler import AbstractStore -from tools import utils,words -from var import crawler_type_var import config +from base.base_crawler import AbstractStore +from tools import utils, words +from var import crawler_type_var + def calculate_number_of_files(file_store_path: str) -> int: """计算数据保存文件的前部分排序数字，支持每次运行代码不写到同一个文件中 diff --git a/store/weibo/weibo_store_impl.py b/store/weibo/weibo_store_impl.py index fdd21d4..aa5a019 100644 --- a/store/weibo/weibo_store_impl.py +++ b/store/weibo/weibo_store_impl.py @@ -11,10 +11,11 @@ from typing import Dict import aiofiles -from base.base_crawler import AbstractStore -from tools import utils,words -from var import crawler_type_var import config +from base.base_crawler import AbstractStore +from tools import utils, words +from var import crawler_type_var + def calculate_number_of_files(file_store_path: str) -> int: """计算数据保存文件的前部分排序数字，支持每次运行代码不写到同一个文件中 diff --git a/store/xhs/xhs_store_impl.py b/store/xhs/xhs_store_impl.py index 3204d0c..8450de2 100644 --- a/store/xhs/xhs_store_impl.py +++ b/store/xhs/xhs_store_impl.py @@ -11,10 +11,11 @@ from typing import Dict import aiofiles -from base.base_crawler import AbstractStore -from tools import utils,words -from var import crawler_type_var import config +from base.base_crawler import AbstractStore +from tools import utils, words +from var import crawler_type_var + def calculate_number_of_files(file_store_path: str) -> int: """计算数据保存文件的前部分排序数字，支持每次运行代码不写到同一个文件中 diff --git a/tools/words.py b/tools/words.py index b7c2b00..d9f0a8c 100644 --- a/tools/words.py +++ b/tools/words.py @@ -1,10 +1,12 @@ -import aiofiles import asyncio -import jieba -from collections import Counter -from wordcloud import WordCloud import json +from collections import Counter + +import aiofiles +import jieba import matplotlib.pyplot as plt +from wordcloud import WordCloud + import config from tools import utils