fix: weibo bug
This commit is contained in:
parent
c3d49c1019
commit
a0e5a29af8
|
@ -1,4 +1,5 @@
|
||||||
import argparse
|
import argparse
|
||||||
|
|
||||||
import config
|
import config
|
||||||
from tools.utils import str2bool
|
from tools.utils import str2bool
|
||||||
|
|
||||||
|
|
|
@ -12,8 +12,8 @@ from playwright.async_api import BrowserContext, Page
|
||||||
from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt,
|
from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt,
|
||||||
wait_fixed)
|
wait_fixed)
|
||||||
|
|
||||||
from base.base_crawler import AbstractLogin
|
|
||||||
import config
|
import config
|
||||||
|
from base.base_crawler import AbstractLogin
|
||||||
from tools import utils
|
from tools import utils
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
import asyncio
|
import asyncio
|
||||||
import copy
|
import copy
|
||||||
|
import json
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
from typing import Any, Callable, Dict, List, Optional
|
from typing import Any, Callable, Dict, List, Optional
|
||||||
import json
|
|
||||||
|
|
||||||
import execjs
|
import execjs
|
||||||
import httpx
|
import httpx
|
||||||
|
|
|
@ -7,8 +7,8 @@ from playwright.async_api import BrowserContext, Page
|
||||||
from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt,
|
from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt,
|
||||||
wait_fixed)
|
wait_fixed)
|
||||||
|
|
||||||
from base.base_crawler import AbstractLogin
|
|
||||||
import config
|
import config
|
||||||
|
from base.base_crawler import AbstractLogin
|
||||||
from tools import utils
|
from tools import utils
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -12,6 +12,7 @@ from playwright.async_api import BrowserContext, Page
|
||||||
from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt,
|
from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt,
|
||||||
wait_fixed)
|
wait_fixed)
|
||||||
|
|
||||||
|
import config
|
||||||
from base.base_crawler import AbstractLogin
|
from base.base_crawler import AbstractLogin
|
||||||
from tools import utils
|
from tools import utils
|
||||||
|
|
||||||
|
|
|
@ -13,9 +13,9 @@ import aiofiles
|
||||||
|
|
||||||
import config
|
import config
|
||||||
from base.base_crawler import AbstractStore
|
from base.base_crawler import AbstractStore
|
||||||
from tools import utils
|
from tools import utils, words
|
||||||
from var import crawler_type_var
|
from var import crawler_type_var
|
||||||
from tools import words
|
|
||||||
|
|
||||||
def calculate_number_of_files(file_store_path: str) -> int:
|
def calculate_number_of_files(file_store_path: str) -> int:
|
||||||
"""计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中
|
"""计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中
|
||||||
|
|
|
@ -11,10 +11,10 @@ from typing import Dict
|
||||||
|
|
||||||
import aiofiles
|
import aiofiles
|
||||||
|
|
||||||
from base.base_crawler import AbstractStore
|
|
||||||
from tools import utils,words
|
|
||||||
from var import crawler_type_var
|
|
||||||
import config
|
import config
|
||||||
|
from base.base_crawler import AbstractStore
|
||||||
|
from tools import utils, words
|
||||||
|
from var import crawler_type_var
|
||||||
|
|
||||||
|
|
||||||
def calculate_number_of_files(file_store_path: str) -> int:
|
def calculate_number_of_files(file_store_path: str) -> int:
|
||||||
|
|
|
@ -11,10 +11,11 @@ from typing import Dict
|
||||||
|
|
||||||
import aiofiles
|
import aiofiles
|
||||||
|
|
||||||
from base.base_crawler import AbstractStore
|
|
||||||
from tools import utils,words
|
|
||||||
from var import crawler_type_var
|
|
||||||
import config
|
import config
|
||||||
|
from base.base_crawler import AbstractStore
|
||||||
|
from tools import utils, words
|
||||||
|
from var import crawler_type_var
|
||||||
|
|
||||||
|
|
||||||
def calculate_number_of_files(file_store_path: str) -> int:
|
def calculate_number_of_files(file_store_path: str) -> int:
|
||||||
"""计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中
|
"""计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中
|
||||||
|
|
|
@ -11,10 +11,11 @@ from typing import Dict
|
||||||
|
|
||||||
import aiofiles
|
import aiofiles
|
||||||
|
|
||||||
from base.base_crawler import AbstractStore
|
|
||||||
from tools import utils,words
|
|
||||||
from var import crawler_type_var
|
|
||||||
import config
|
import config
|
||||||
|
from base.base_crawler import AbstractStore
|
||||||
|
from tools import utils, words
|
||||||
|
from var import crawler_type_var
|
||||||
|
|
||||||
|
|
||||||
def calculate_number_of_files(file_store_path: str) -> int:
|
def calculate_number_of_files(file_store_path: str) -> int:
|
||||||
"""计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中
|
"""计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中
|
||||||
|
|
|
@ -11,10 +11,11 @@ from typing import Dict
|
||||||
|
|
||||||
import aiofiles
|
import aiofiles
|
||||||
|
|
||||||
from base.base_crawler import AbstractStore
|
|
||||||
from tools import utils,words
|
|
||||||
from var import crawler_type_var
|
|
||||||
import config
|
import config
|
||||||
|
from base.base_crawler import AbstractStore
|
||||||
|
from tools import utils, words
|
||||||
|
from var import crawler_type_var
|
||||||
|
|
||||||
|
|
||||||
def calculate_number_of_files(file_store_path: str) -> int:
|
def calculate_number_of_files(file_store_path: str) -> int:
|
||||||
"""计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中
|
"""计算数据保存文件的前部分排序数字,支持每次运行代码不写到同一个文件中
|
||||||
|
|
|
@ -1,10 +1,12 @@
|
||||||
import aiofiles
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import jieba
|
|
||||||
from collections import Counter
|
|
||||||
from wordcloud import WordCloud
|
|
||||||
import json
|
import json
|
||||||
|
from collections import Counter
|
||||||
|
|
||||||
|
import aiofiles
|
||||||
|
import jieba
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
|
from wordcloud import WordCloud
|
||||||
|
|
||||||
import config
|
import config
|
||||||
from tools import utils
|
from tools import utils
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue