fix: weibo bug

This commit is contained in:
Relakkes Yang 2024-06-17 00:25:48 +08:00
parent c3d49c1019
commit a0e5a29af8
11 changed files with 28 additions and 21 deletions

View File

@ -1,4 +1,5 @@
import argparse import argparse
import config import config
from tools.utils import str2bool from tools.utils import str2bool

View File

@ -12,8 +12,8 @@ from playwright.async_api import BrowserContext, Page
from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt, from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt,
wait_fixed) wait_fixed)
from base.base_crawler import AbstractLogin
import config import config
from base.base_crawler import AbstractLogin
from tools import utils from tools import utils

View File

@ -1,8 +1,8 @@
import asyncio import asyncio
import copy import copy
import json
import urllib.parse import urllib.parse
from typing import Any, Callable, Dict, List, Optional from typing import Any, Callable, Dict, List, Optional
import json
import execjs import execjs
import httpx import httpx

View File

@ -7,8 +7,8 @@ from playwright.async_api import BrowserContext, Page
from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt, from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt,
wait_fixed) wait_fixed)
from base.base_crawler import AbstractLogin
import config import config
from base.base_crawler import AbstractLogin
from tools import utils from tools import utils

View File

@ -12,6 +12,7 @@ from playwright.async_api import BrowserContext, Page
from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt, from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt,
wait_fixed) wait_fixed)
import config
from base.base_crawler import AbstractLogin from base.base_crawler import AbstractLogin
from tools import utils from tools import utils

View File

@ -13,9 +13,9 @@ import aiofiles
import config import config
from base.base_crawler import AbstractStore from base.base_crawler import AbstractStore
from tools import utils from tools import utils, words
from var import crawler_type_var from var import crawler_type_var
from tools import words
def calculate_number_of_files(file_store_path: str) -> int: def calculate_number_of_files(file_store_path: str) -> int:
"""计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中 """计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中

View File

@ -11,10 +11,10 @@ from typing import Dict
import aiofiles import aiofiles
from base.base_crawler import AbstractStore
from tools import utils,words
from var import crawler_type_var
import config import config
from base.base_crawler import AbstractStore
from tools import utils, words
from var import crawler_type_var
def calculate_number_of_files(file_store_path: str) -> int: def calculate_number_of_files(file_store_path: str) -> int:

View File

@ -11,10 +11,11 @@ from typing import Dict
import aiofiles import aiofiles
from base.base_crawler import AbstractStore
from tools import utils,words
from var import crawler_type_var
import config import config
from base.base_crawler import AbstractStore
from tools import utils, words
from var import crawler_type_var
def calculate_number_of_files(file_store_path: str) -> int: def calculate_number_of_files(file_store_path: str) -> int:
"""计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中 """计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中

View File

@ -11,10 +11,11 @@ from typing import Dict
import aiofiles import aiofiles
from base.base_crawler import AbstractStore
from tools import utils,words
from var import crawler_type_var
import config import config
from base.base_crawler import AbstractStore
from tools import utils, words
from var import crawler_type_var
def calculate_number_of_files(file_store_path: str) -> int: def calculate_number_of_files(file_store_path: str) -> int:
"""计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中 """计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中

View File

@ -11,10 +11,11 @@ from typing import Dict
import aiofiles import aiofiles
from base.base_crawler import AbstractStore
from tools import utils,words
from var import crawler_type_var
import config import config
from base.base_crawler import AbstractStore
from tools import utils, words
from var import crawler_type_var
def calculate_number_of_files(file_store_path: str) -> int: def calculate_number_of_files(file_store_path: str) -> int:
"""计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中 """计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中

View File

@ -1,10 +1,12 @@
import aiofiles
import asyncio import asyncio
import jieba
from collections import Counter
from wordcloud import WordCloud
import json import json
from collections import Counter
import aiofiles
import jieba
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from wordcloud import WordCloud
import config import config
from tools import utils from tools import utils