qnloft
/
MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(weibo): weibo data regular

This commit is contained in:
Zzzendrix 2024-05-28 15:11:52 +08:00
parent 478db4cc4b
commit 6409b4fecc
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
store/weibo/__init__.py
View File

@ -4,6 +4,7 @@
# @Desc : # @Desc :
from typing import List from typing import List
import re
import config import config
@ -31,10 +32,12 @@ async def update_weibo_note(note_item: Dict):
mblog: Dict = note_item.get("mblog") mblog: Dict = note_item.get("mblog")
user_info: Dict = mblog.get("user") user_info: Dict = mblog.get("user")
note_id = mblog.get("id") note_id = mblog.get("id")
content_text = mblog.get("text")
clean_text = re.sub(r"<.*?>", "", content_text)
save_content_item = { save_content_item = {
# 微博信息 # 微博信息
"note_id": note_id, "note_id": note_id,
"content": mblog.get("text"), "content": clean_text,
"create_time": utils.rfc2822_to_timestamp(mblog.get("created_at")), "create_time": utils.rfc2822_to_timestamp(mblog.get("created_at")),
"create_date_time": str(utils.rfc2822_to_china_datetime(mblog.get("created_at"))), "create_date_time": str(utils.rfc2822_to_china_datetime(mblog.get("created_at"))),
"liked_count": str(mblog.get("attitudes_count", 0)), "liked_count": str(mblog.get("attitudes_count", 0)),
@ -66,12 +69,14 @@ async def batch_update_weibo_note_comments(note_id: str, comments: List[Dict]):
async def update_weibo_note_comment(note_id: str, comment_item: Dict): async def update_weibo_note_comment(note_id: str, comment_item: Dict):
comment_id = str(comment_item.get("id")) comment_id = str(comment_item.get("id"))
user_info: Dict = comment_item.get("user") user_info: Dict = comment_item.get("user")
content_text = comment_item.get("text")
clean_text = re.sub(r"<.*?>", "", content_text)
save_comment_item = { save_comment_item = {
"comment_id": comment_id, "comment_id": comment_id,
"create_time": utils.rfc2822_to_timestamp(comment_item.get("created_at")), "create_time": utils.rfc2822_to_timestamp(comment_item.get("created_at")),
"create_date_time": str(utils.rfc2822_to_china_datetime(comment_item.get("created_at"))), "create_date_time": str(utils.rfc2822_to_china_datetime(comment_item.get("created_at"))),
"note_id": note_id, "note_id": note_id,
"content": comment_item.get("text"), "content": clean_text,
"sub_comment_count": str(comment_item.get("total_number", 0)), "sub_comment_count": str(comment_item.get("total_number", 0)),
"comment_like_count": str(comment_item.get("like_count", 0)), "comment_like_count": str(comment_item.get("like_count", 0)),
"last_modify_ts": utils.get_current_timestamp(), "last_modify_ts": utils.get_current_timestamp(),