qnloft
/
MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 百度贴吧支持创作者主页帖子爬取

This commit is contained in:
Relakkes Yang 2024-08-24 11:03:23 +08:00
parent 8adb593ba6
commit acb29add28
8 changed files with 210 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/base_config.py
View File

@ -102,7 +102,7 @@ TIEBA_NAME_LIST = [
] ]
TIEBA_CREATOR_URL_LIST = [ TIEBA_CREATOR_URL_LIST = [
"https://tieba.baidu.com/home/main/?id=tb.1.6a328702.02qx9GEBmrwqYDRyOgGKXQ&fr=frs", "https://tieba.baidu.com/home/main/?id=tb.1.7f139e2e.6CyEwxu3VJruH_-QqpCi6g&fr=frs",
# ........................ # ........................
] ]

29
media_platform/tieba/client.py
View File

@ -9,7 +9,7 @@ from tenacity import RetryError, retry, stop_after_attempt, wait_fixed
import config import config
from base.base_crawler import AbstractApiClient from base.base_crawler import AbstractApiClient
from model.m_baidu_tieba import TiebaComment, TiebaNote, TiebaCreator from model.m_baidu_tieba import TiebaComment, TiebaCreator, TiebaNote
from proxy.proxy_ip_pool import ProxyIpPool from proxy.proxy_ip_pool import ProxyIpPool
from tools import utils from tools import utils
@ -317,14 +317,17 @@ class BaiduTieBaClient(AbstractApiClient):
} }
return await self.get(uri, params=params) return await self.get(uri, params=params)
async def get_all_notes_by_creator_user_name(self, user_name: str,crawl_interval: float = 1.0, async def get_all_notes_by_creator_user_name(self,
callback: Optional[Callable] = None) -> List[TiebaNote]: user_name: str, crawl_interval: float = 1.0,
callback: Optional[Callable] = None,
max_note_count: int = 0) -> List[TiebaNote]:
""" """
根据创作者用户名获取创作者所有帖子 根据创作者用户名获取创作者所有帖子
Args: Args:
user_name: user_name: 创作者用户名
crawl_interval: crawl_interval: 爬取一次笔记的延迟单位（秒）
callback: callback: 一次笔记爬取结束后的回调函数，是一个awaitable类型的函数
max_note_count: 帖子最大获取数量，如果为0则获取所有
Returns: Returns:
@ -332,16 +335,17 @@ class BaiduTieBaClient(AbstractApiClient):
result = [] result = []
notes_has_more = 1 notes_has_more = 1
page_number = 1 page_number = 1
while notes_has_more == 1: page_per_count = 20
total_get_count = 0
while notes_has_more == 1 and (max_note_count == 0 or total_get_count < max_note_count):
notes_res = await self.get_notes_by_creator(user_name, page_number) notes_res = await self.get_notes_by_creator(user_name, page_number)
if not notes_res or notes_res.get("no") != 0: if not notes_res or notes_res.get("no") != 0:
utils.logger.error( utils.logger.error(
f"[WeiboClient.get_notes_by_creator] got user_name:{user_name} notes failed, notes_res: {notes_res}") f"[WeiboClient.get_notes_by_creator] got user_name:{user_name} notes failed, notes_res: {notes_res}")
break break
notes_data = notes_res.get("data")
notes_has_more = notes_res.get("has_more") notes_has_more = notes_data.get("has_more")
page_number += 1 notes = notes_data["thread_list"]
notes = notes_res["thread_list"]
utils.logger.info( utils.logger.info(
f"[WeiboClient.get_all_notes_by_creator] got user_name:{user_name} notes len : {len(notes)}") f"[WeiboClient.get_all_notes_by_creator] got user_name:{user_name} notes len : {len(notes)}")
@ -351,5 +355,6 @@ class BaiduTieBaClient(AbstractApiClient):
await callback(notes) await callback(notes)
await asyncio.sleep(crawl_interval) await asyncio.sleep(crawl_interval)
result.extend(notes) result.extend(notes)
page_number += 1
total_get_count += page_per_count
return result return result

22
media_platform/tieba/core.py
View File

@ -9,7 +9,7 @@ from playwright.async_api import (BrowserContext, BrowserType, Page,
import config import config
from base.base_crawler import AbstractCrawler from base.base_crawler import AbstractCrawler
from model.m_baidu_tieba import TiebaNote from model.m_baidu_tieba import TiebaCreator, TiebaNote
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from store import tieba as tieba_store from store import tieba as tieba_store
from tools import utils from tools import utils
@ -226,19 +226,20 @@ class TieBaCrawler(AbstractCrawler):
""" """
utils.logger.info("[WeiboCrawler.get_creators_and_notes] Begin get weibo creators") utils.logger.info("[WeiboCrawler.get_creators_and_notes] Begin get weibo creators")
for creator_url in config.TIEBA_CREATOR_URL_LIST: for creator_url in config.TIEBA_CREATOR_URL_LIST:
createor_info: Dict = await self.tieba_client.get_creator_info_by_url(creator_url=creator_url) creator_info: TiebaCreator = await self.tieba_client.get_creator_info_by_url(creator_url=creator_url)
if createor_info: if creator_info:
utils.logger.info(f"[WeiboCrawler.get_creators_and_notes] creator info: {createor_info}") utils.logger.info(f"[WeiboCrawler.get_creators_and_notes] creator info: {creator_info}")
if not createor_info: if not creator_info:
raise Exception("Get creator info error") raise Exception("Get creator info error")
user_id = createor_info.get("user_id")
await tieba_store.save_creator(user_id, user_info=createor_info) await tieba_store.save_creator(user_info=creator_info)
# Get all note information of the creator # Get all note information of the creator
all_notes_list = await self.tieba_client.get_all_notes_by_creator_user_name( all_notes_list = await self.tieba_client.get_all_notes_by_creator_user_name(
user_name=createor_info.get("user_name"), user_name=creator_info.user_name,
crawl_interval=0, crawl_interval=0,
callback=tieba_store.batch_update_tieba_notes callback=tieba_store.batch_update_tieba_notes,
max_note_count=config.CRAWLER_MAX_NOTES_COUNT
) )
await self.batch_get_note_comments(all_notes_list) await self.batch_get_note_comments(all_notes_list)
@ -247,9 +248,6 @@ class TieBaCrawler(AbstractCrawler):
utils.logger.error( utils.logger.error(
f"[WeiboCrawler.get_creators_and_notes] get creator info error, creator_url:{creator_url}") f"[WeiboCrawler.get_creators_and_notes] get creator info error, creator_url:{creator_url}")
async def launch_browser( async def launch_browser(
self, self,
chromium: BrowserType, chromium: BrowserType,

200
media_platform/tieba/help.py
View File

@ -3,16 +3,16 @@ import html
import json import json
import re import re
from typing import Dict, List, Tuple from typing import Dict, List, Tuple
from urllib.parse import unquote, parse_qs from urllib.parse import parse_qs, unquote
from parsel import Selector from parsel import Selector
from constant import baidu_tieba as const from constant import baidu_tieba as const
from model.m_baidu_tieba import TiebaComment, TiebaNote, TiebaCreator from model.m_baidu_tieba import TiebaComment, TiebaCreator, TiebaNote
from tools import utils from tools import utils
GENDER_MALE = "sex_male" GENDER_MALE = "sex_male"
GENDER_FMALE = "sex_fmale" GENDER_FEMALE = "sex_female"
class TieBaExtractor: class TieBaExtractor:
@ -33,17 +33,19 @@ class TieBaExtractor:
post_list = Selector(text=page_content).xpath(xpath_selector) post_list = Selector(text=page_content).xpath(xpath_selector)
result: List[TiebaNote] = [] result: List[TiebaNote] = []
for post in post_list: for post in post_list:
tieba_note = TiebaNote( tieba_note = TiebaNote(note_id=post.xpath(".//span[@class='p_title']/a/@data-tid").get(default='').strip(),
note_id=post.xpath(".//span[@class='p_title']/a/@data-tid").get(default='').strip(),
title=post.xpath(".//span[@class='p_title']/a/text()").get(default='').strip(), title=post.xpath(".//span[@class='p_title']/a/text()").get(default='').strip(),
desc=post.xpath(".//div[@class='p_content']/text()").get(default='').strip(), desc=post.xpath(".//div[@class='p_content']/text()").get(default='').strip(),
note_url=const.TIEBA_URL + post.xpath(".//span[@class='p_title']/a/@href").get(default=''), note_url=const.TIEBA_URL + post.xpath(".//span[@class='p_title']/a/@href").get(
user_nickname=post.xpath(".//a[starts-with(@href, '/home/main')]/font/text()").get(default='').strip(), default=''),
user_link=const.TIEBA_URL + post.xpath(".//a[starts-with(@href, '/home/main')]/@href").get(default=''), user_nickname=post.xpath(".//a[starts-with(@href, '/home/main')]/font/text()").get(
default='').strip(), user_link=const.TIEBA_URL + post.xpath(
".//a[starts-with(@href, '/home/main')]/@href").get(default=''),
tieba_name=post.xpath(".//a[@class='p_forum']/font/text()").get(default='').strip(), tieba_name=post.xpath(".//a[@class='p_forum']/font/text()").get(default='').strip(),
tieba_link=const.TIEBA_URL + post.xpath(".//a[@class='p_forum']/@href").get(default=''), tieba_link=const.TIEBA_URL + post.xpath(".//a[@class='p_forum']/@href").get(
publish_time=post.xpath(".//font[@class='p_green p_date']/text()").get(default='').strip(), default=''),
) publish_time=post.xpath(".//font[@class='p_green p_date']/text()").get(
default='').strip(), )
result.append(tieba_note) result.append(tieba_note)
return result return result
@ -66,20 +68,19 @@ class TieBaExtractor:
if not post_field_value: if not post_field_value:
continue continue
note_id = str(post_field_value.get("id")) note_id = str(post_field_value.get("id"))
tieba_note = TiebaNote( tieba_note = TiebaNote(note_id=note_id,
note_id=note_id,
title=post_selector.xpath(".//a[@class='j_th_tit ']/text()").get(default='').strip(), title=post_selector.xpath(".//a[@class='j_th_tit ']/text()").get(default='').strip(),
desc=post_selector.xpath(".//div[@class='threadlist_abs threadlist_abs_onlyline ']/text()").get( desc=post_selector.xpath(
default='').strip(), ".//div[@class='threadlist_abs threadlist_abs_onlyline ']/text()").get(
note_url=const.TIEBA_URL + f"/p/{note_id}", default='').strip(), note_url=const.TIEBA_URL + f"/p/{note_id}",
user_link=const.TIEBA_URL + post_selector.xpath( user_link=const.TIEBA_URL + post_selector.xpath(
".//a[@class='frs-author-name j_user_card ']/@href").get(default='').strip(), ".//a[@class='frs-author-name j_user_card ']/@href").get(default='').strip(),
user_nickname=post_field_value.get("authoer_nickname") or post_field_value.get("author_name"), user_nickname=post_field_value.get("authoer_nickname") or post_field_value.get(
tieba_name=content_selector.xpath("//a[@class='card_title_fname']/text()").get(default='').strip(), "author_name"),
tieba_link=const.TIEBA_URL + content_selector.xpath("//a[@class='card_title_fname']/@href").get( tieba_name=content_selector.xpath("//a[@class='card_title_fname']/text()").get(
default=''), default='').strip(), tieba_link=const.TIEBA_URL + content_selector.xpath(
total_replay_num=post_field_value.get("reply_num", 0) "//a[@class='card_title_fname']/@href").get(default=''),
) total_replay_num=post_field_value.get("reply_num", 0))
result.append(tieba_note) result.append(tieba_note)
return result return result
@ -98,28 +99,25 @@ class TieBaExtractor:
note_id = only_view_author_link.split("?")[0].split("/")[-1] note_id = only_view_author_link.split("?")[0].split("/")[-1]
# 帖子回复数、回复页数 # 帖子回复数、回复页数
thread_num_infos = content_selector.xpath( thread_num_infos = content_selector.xpath(
"//div[@id='thread_theme_5']//li[@class='l_reply_num']//span[@class='red']" "//div[@id='thread_theme_5']//li[@class='l_reply_num']//span[@class='red']")
)
# IP地理位置、发表时间 # IP地理位置、发表时间
other_info_content = content_selector.xpath(".//div[@class='post-tail-wrap']").get(default="").strip() other_info_content = content_selector.xpath(".//div[@class='post-tail-wrap']").get(default="").strip()
ip_location, publish_time = self.extract_ip_and_pub_time(other_info_content) ip_location, publish_time = self.extract_ip_and_pub_time(other_info_content)
note = TiebaNote( note = TiebaNote(note_id=note_id, title=content_selector.xpath("//title/text()").get(default='').strip(),
note_id=note_id,
title=content_selector.xpath("//title/text()").get(default='').strip(),
desc=content_selector.xpath("//meta[@name='description']/@content").get(default='').strip(), desc=content_selector.xpath("//meta[@name='description']/@content").get(default='').strip(),
note_url=const.TIEBA_URL + f"/p/{note_id}", note_url=const.TIEBA_URL + f"/p/{note_id}",
user_link=const.TIEBA_URL + first_floor_selector.xpath(".//a[@class='p_author_face ']/@href").get( user_link=const.TIEBA_URL + first_floor_selector.xpath(
".//a[@class='p_author_face ']/@href").get(default='').strip(),
user_nickname=first_floor_selector.xpath(
".//a[@class='p_author_name j_user_card']/text()").get(default='').strip(),
user_avatar=first_floor_selector.xpath(".//a[@class='p_author_face ']/img/@src").get(
default='').strip(), default='').strip(),
user_nickname=first_floor_selector.xpath(".//a[@class='p_author_name j_user_card']/text()").get( tieba_name=content_selector.xpath("//a[@class='card_title_fname']/text()").get(
default='').strip(), default='').strip(), tieba_link=const.TIEBA_URL + content_selector.xpath(
user_avatar=first_floor_selector.xpath(".//a[@class='p_author_face ']/img/@src").get(default='').strip(), "//a[@class='card_title_fname']/@href").get(default=''), ip_location=ip_location,
tieba_name=content_selector.xpath("//a[@class='card_title_fname']/text()").get(default='').strip(),
tieba_link=const.TIEBA_URL + content_selector.xpath("//a[@class='card_title_fname']/@href").get(default=''),
ip_location=ip_location,
publish_time=publish_time, publish_time=publish_time,
total_replay_num=thread_num_infos[0].xpath("./text()").get(default='').strip(), total_replay_num=thread_num_infos[0].xpath("./text()").get(default='').strip(),
total_replay_page=thread_num_infos[1].xpath("./text()").get(default='').strip(), total_replay_page=thread_num_infos[1].xpath("./text()").get(default='').strip(), )
)
note.title = note.title.replace(f"【{note.tieba_name}】_百度贴吧", "") note.title = note.title.replace(f"【{note.tieba_name}】_百度贴吧", "")
return note return note
@ -143,24 +141,20 @@ class TieBaExtractor:
tieba_name = comment_selector.xpath("//a[@class='card_title_fname']/text()").get(default='').strip() tieba_name = comment_selector.xpath("//a[@class='card_title_fname']/text()").get(default='').strip()
other_info_content = comment_selector.xpath(".//div[@class='post-tail-wrap']").get(default="").strip() other_info_content = comment_selector.xpath(".//div[@class='post-tail-wrap']").get(default="").strip()
ip_location, publish_time = self.extract_ip_and_pub_time(other_info_content) ip_location, publish_time = self.extract_ip_and_pub_time(other_info_content)
tieba_comment = TiebaComment( tieba_comment = TiebaComment(comment_id=str(comment_field_value.get("content").get("post_id")),
comment_id=str(comment_field_value.get("content").get("post_id")),
sub_comment_count=comment_field_value.get("content").get("comment_num"), sub_comment_count=comment_field_value.get("content").get("comment_num"),
content=utils.extract_text_from_html(comment_field_value.get("content").get("content")), content=utils.extract_text_from_html(
comment_field_value.get("content").get("content")),
note_url=const.TIEBA_URL + f"/p/{note_id}", note_url=const.TIEBA_URL + f"/p/{note_id}",
user_link=const.TIEBA_URL + comment_selector.xpath(".//a[@class='p_author_face ']/@href").get( user_link=const.TIEBA_URL + comment_selector.xpath(
default='').strip(), ".//a[@class='p_author_face ']/@href").get(default='').strip(),
user_nickname=comment_selector.xpath(".//a[@class='p_author_name j_user_card']/text()").get( user_nickname=comment_selector.xpath(
default='').strip(), ".//a[@class='p_author_name j_user_card']/text()").get(default='').strip(),
user_avatar=comment_selector.xpath(".//a[@class='p_author_face ']/img/@src").get( user_avatar=comment_selector.xpath(
default='').strip(), ".//a[@class='p_author_face ']/img/@src").get(default='').strip(),
tieba_id=str(comment_field_value.get("content").get("forum_id", "")), tieba_id=str(comment_field_value.get("content").get("forum_id", "")),
tieba_name=tieba_name, tieba_name=tieba_name, tieba_link=f"https://tieba.baidu.com/f?kw={tieba_name}",
tieba_link=f"https://tieba.baidu.com/f?kw={tieba_name}", ip_location=ip_location, publish_time=publish_time, note_id=note_id, )
ip_location=ip_location,
publish_time=publish_time,
note_id=note_id,
)
result.append(tieba_comment) result.append(tieba_comment)
return result return result
@ -186,19 +180,15 @@ class TieBaExtractor:
content = utils.extract_text_from_html( content = utils.extract_text_from_html(
comment_ele.xpath(".//span[@class='lzl_content_main']").get(default="")) comment_ele.xpath(".//span[@class='lzl_content_main']").get(default=""))
comment = TiebaComment( comment = TiebaComment(
comment_id=str(comment_value.get("spid")), comment_id=str(comment_value.get("spid")), content=content,
content=content,
user_link=comment_user_a_selector.xpath("./@href").get(default=""), user_link=comment_user_a_selector.xpath("./@href").get(default=""),
user_nickname=comment_value.get("showname"), user_nickname=comment_value.get("showname"),
user_avatar=comment_user_a_selector.xpath("./img/@src").get(default=""), user_avatar=comment_user_a_selector.xpath("./img/@src").get(default=""),
publish_time=comment_ele.xpath(".//span[@class='lzl_time']/text()").get(default="").strip(), publish_time=comment_ele.xpath(".//span[@class='lzl_time']/text()").get(default="").strip(),
parent_comment_id=parent_comment.comment_id, parent_comment_id=parent_comment.comment_id,
note_id=parent_comment.note_id, note_id=parent_comment.note_id, note_url=parent_comment.note_url,
note_url=parent_comment.note_url, tieba_id=parent_comment.tieba_id, tieba_name=parent_comment.tieba_name,
tieba_id=parent_comment.tieba_id, tieba_link=parent_comment.tieba_link)
tieba_name=parent_comment.tieba_name,
tieba_link=parent_comment.tieba_link
)
comments.append(comment) comments.append(comment)
return comments return comments
@ -215,23 +205,26 @@ class TieBaExtractor:
selector = Selector(text=html_content) selector = Selector(text=html_content)
user_link_selector = selector.xpath("//p[@class='space']/a") user_link_selector = selector.xpath("//p[@class='space']/a")
user_link: str = user_link_selector.xpath("./@href").get(default='') user_link: str = user_link_selector.xpath("./@href").get(default='')
user_link_params: Dict = parse_qs(unquote(user_link)) user_link_params: Dict = parse_qs(unquote(user_link.split("?")[-1]))
user_name = user_link_params.get("un")[0] if user_link_params.get("un") else "" user_name = user_link_params.get("un")[0] if user_link_params.get("un") else ""
user_id = user_link_params.get("id")[0] if user_link_params.get("id") else "" user_id = user_link_params.get("id")[0] if user_link_params.get("id") else ""
userinfo_userdata_selector = selector.xpath("//div[@class='userinfo_userdata']") userinfo_userdata_selector = selector.xpath("//div[@class='userinfo_userdata']")
creator = TiebaCreator( follow_fans_selector = selector.xpath("//span[@class='concern_num']")
user_id=user_id, follows, fans = 0, 0
user_name=user_name, if len(follow_fans_selector) == 2:
nickname=selector.xpath(".//a[@class='userinfo_username']/text()").get(default='').strip(), follows, fans = self.extract_follow_and_fans(follow_fans_selector)
avatar=selector.xpath(".//div[@class='userinfo_left_head']//img/@src").get(default='').strip(), user_content = userinfo_userdata_selector.get(default='')
gender=self.extract_gender(userinfo_userdata_selector.get(default='')), return TiebaCreator(user_id=user_id, user_name=user_name,
ip_location=self.extract_ip(userinfo_userdata_selector.get(default='')), nickname=selector.xpath(".//span[@class='userinfo_username ']/text()").get(
follows=0, default='').strip(),
fans=0, avatar=selector.xpath(".//div[@class='userinfo_left_head']//img/@src").get(
follow_tieba_list="", default='').strip(),
registration_duration="", gender=self.extract_gender(user_content),
ip_location=self.extract_ip(user_content),
follows=follows,
fans=fans,
registration_duration=self.extract_registration_duration(user_content)
) )
return creator
def extract_ip_and_pub_time(self, html_content: str) -> Tuple[str, str]: def extract_ip_and_pub_time(self, html_content: str) -> Tuple[str, str]:
""" """
@ -272,7 +265,39 @@ class TieBaExtractor:
Returns: Returns:
""" """
pass if GENDER_MALE in html_content:
return '男'
elif GENDER_FEMALE in html_content:
return '女'
return '未知'
@staticmethod
def extract_follow_and_fans(selectors: List[Selector]) -> Tuple[str, str]:
"""
提取关注数和粉丝数
Args:
selectors:
Returns:
"""
pattern = re.compile(r'<span class="concern_num">\(<a[^>]*>(\d+)</a>\)</span>')
follow_match = pattern.findall(selectors[0].get())
fans_match = pattern.findall(selectors[1].get())
follows = follow_match[0] if follow_match else 0
fans = fans_match[0] if fans_match else 0
return follows, fans
@staticmethod
def extract_registration_duration(html_content: str) -> str:
"""
"<span>吧龄:1.9年</span>"
Returns: 1.9年
"""
pattern = re.compile(r'<span>吧龄:(\S+)</span>')
match = pattern.search(html_content)
return match.group(1) if match else ""
@staticmethod @staticmethod
def extract_data_field_value(selector: Selector) -> Dict: def extract_data_field_value(selector: Selector) -> Dict:
@ -325,19 +350,11 @@ def test_extract_tieba_note_sub_comments():
with open("test_data/note_sub_comments.html", "r", encoding="utf-8") as f: with open("test_data/note_sub_comments.html", "r", encoding="utf-8") as f:
content = f.read() content = f.read()
extractor = TieBaExtractor() extractor = TieBaExtractor()
fake_parment_comment = TiebaComment( fake_parment_comment = TiebaComment(comment_id="123456", content="content", user_link="user_link",
comment_id="123456", user_nickname="user_nickname", user_avatar="user_avatar",
content="content", publish_time="publish_time", parent_comment_id="parent_comment_id",
user_link="user_link", note_id="note_id", note_url="note_url", tieba_id="tieba_id",
user_nickname="user_nickname", tieba_name="tieba_name", )
user_avatar="user_avatar",
publish_time="publish_time",
parent_comment_id="parent_comment_id",
note_id="note_id",
note_url="note_url",
tieba_id="tieba_id",
tieba_name="tieba_name",
)
result = extractor.extract_tieba_note_sub_comments(content, fake_parment_comment) result = extractor.extract_tieba_note_sub_comments(content, fake_parment_comment)
print(result) print(result)
@ -351,8 +368,17 @@ def test_extract_tieba_note_list():
pass pass
def test_extract_creator_info():
with open("test_data/creator_info.html", "r", encoding="utf-8") as f:
content = f.read()
extractor = TieBaExtractor()
result = extractor.extract_creator_info(content)
print(result.model_dump_json())
if __name__ == '__main__': if __name__ == '__main__':
# test_extract_search_note_list() # test_extract_search_note_list()
# test_extract_note_detail() # test_extract_note_detail()
# test_extract_tieba_note_parment_comments() # test_extract_tieba_note_parment_comments()
test_extract_tieba_note_list() # test_extract_tieba_note_list()
test_extract_creator_info()

3
model/m_baidu_tieba.py
View File

@ -45,7 +45,6 @@ class TiebaComment(BaseModel):
tieba_link: str = Field(..., description="贴吧链接") tieba_link: str = Field(..., description="贴吧链接")
class TiebaCreator(BaseModel): class TiebaCreator(BaseModel):
""" """
百度贴吧创作者 百度贴吧创作者
@ -58,6 +57,4 @@ class TiebaCreator(BaseModel):
ip_location: Optional[str] = Field(default="", description="IP地理位置") ip_location: Optional[str] = Field(default="", description="IP地理位置")
follows: int = Field(default=0, description="关注数") follows: int = Field(default=0, description="关注数")
fans: int = Field(default=0, description="粉丝数") fans: int = Field(default=0, description="粉丝数")
follow_tieba_list: str = Field(default="", description="关注的贴吧列表")
registration_duration: str = Field(default="", description="注册时长") registration_duration: str = Field(default="", description="注册时长")

39
schema/tables.sql
View File

@ -400,12 +400,18 @@ CREATE TABLE tieba_comment
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='贴吧评论表'; ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='贴吧评论表';
-- 增加搜索来源关键字字段 -- 增加搜索来源关键字字段
alter table bilibili_video add column `source_keyword` varchar(255) default '' comment '搜索来源关键字'; alter table bilibili_video
alter table douyin_aweme add column `source_keyword` varchar(255) default '' comment '搜索来源关键字'; add column `source_keyword` varchar(255) default '' comment '搜索来源关键字';
alter table kuaishou_video add column `source_keyword` varchar(255) default '' comment '搜索来源关键字'; alter table douyin_aweme
alter table weibo_note add column `source_keyword` varchar(255) default '' comment '搜索来源关键字'; add column `source_keyword` varchar(255) default '' comment '搜索来源关键字';
alter table xhs_note add column `source_keyword` varchar(255) default '' comment '搜索来源关键字'; alter table kuaishou_video
alter table tieba_note add column `source_keyword` varchar(255) default '' comment '搜索来源关键字'; add column `source_keyword` varchar(255) default '' comment '搜索来源关键字';
alter table weibo_note
add column `source_keyword` varchar(255) default '' comment '搜索来源关键字';
alter table xhs_note
add column `source_keyword` varchar(255) default '' comment '搜索来源关键字';
alter table tieba_note
add column `source_keyword` varchar(255) default '' comment '搜索来源关键字';
DROP TABLE IF EXISTS `weibo_creator`; DROP TABLE IF EXISTS `weibo_creator`;
@ -419,7 +425,7 @@ CREATE TABLE `weibo_creator`
`add_ts` bigint NOT NULL COMMENT '记录添加时间戳', `add_ts` bigint NOT NULL COMMENT '记录添加时间戳',
`last_modify_ts` bigint NOT NULL COMMENT '记录最后修改时间戳', `last_modify_ts` bigint NOT NULL COMMENT '记录最后修改时间戳',
`desc` longtext COMMENT '用户描述', `desc` longtext COMMENT '用户描述',
`gender` varchar(1) DEFAULT NULL COMMENT '性别', `gender` varchar(2) DEFAULT NULL COMMENT '性别',
`follows` varchar(16) DEFAULT NULL COMMENT '关注数', `follows` varchar(16) DEFAULT NULL COMMENT '关注数',
`fans` varchar(16) DEFAULT NULL COMMENT '粉丝数', `fans` varchar(16) DEFAULT NULL COMMENT '粉丝数',
`tag_list` longtext COMMENT '标签列表', `tag_list` longtext COMMENT '标签列表',
@ -429,3 +435,22 @@ CREATE TABLE `weibo_creator`
ALTER TABLE `xhs_note_comment` ALTER TABLE `xhs_note_comment`
ADD COLUMN `like_count` VARCHAR(64) DEFAULT NULL COMMENT '评论点赞数量'; ADD COLUMN `like_count` VARCHAR(64) DEFAULT NULL COMMENT '评论点赞数量';
DROP TABLE IF EXISTS `tieba_creator`;
CREATE TABLE `tieba_creator`
(
`id` int NOT NULL AUTO_INCREMENT COMMENT '自增ID',
`user_id` varchar(64) NOT NULL COMMENT '用户ID',
`user_name` varchar(64) NOT NULL COMMENT '用户名',
`nickname` varchar(64) DEFAULT NULL COMMENT '用户昵称',
`avatar` varchar(255) DEFAULT NULL COMMENT '用户头像地址',
`ip_location` varchar(255) DEFAULT NULL COMMENT '评论时的IP地址',
`add_ts` bigint NOT NULL COMMENT '记录添加时间戳',
`last_modify_ts` bigint NOT NULL COMMENT '记录最后修改时间戳',
`gender` varchar(2) DEFAULT NULL COMMENT '性别',
`follows` varchar(16) DEFAULT NULL COMMENT '关注数',
`fans` varchar(16) DEFAULT NULL COMMENT '粉丝数',
`registration_duration` varchar(16) DEFAULT NULL COMMENT '吧龄',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='贴吧创作者';

23
store/tieba/__init__.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from typing import List from typing import List
from model.m_baidu_tieba import TiebaComment, TiebaNote from model.m_baidu_tieba import TiebaComment, TiebaCreator, TiebaNote
from var import source_keyword_var from var import source_keyword_var
from . import tieba_store_impl from . import tieba_store_impl
@ -23,6 +23,7 @@ class TieBaStoreFactory:
"[TieBaStoreFactory.create_store] Invalid save option only supported csv or db or json ...") "[TieBaStoreFactory.create_store] Invalid save option only supported csv or db or json ...")
return store_class() return store_class()
async def batch_update_tieba_notes(note_list: List[TiebaNote]): async def batch_update_tieba_notes(note_list: List[TiebaNote]):
""" """
Batch update tieba notes Batch update tieba notes
@ -37,6 +38,7 @@ async def batch_update_tieba_notes(note_list: List[TiebaNote]):
for note_item in note_list: for note_item in note_list:
await update_tieba_note(note_item) await update_tieba_note(note_item)
async def update_tieba_note(note_item: TiebaNote): async def update_tieba_note(note_item: TiebaNote):
""" """
Add or Update tieba note Add or Update tieba note
@ -54,7 +56,7 @@ async def update_tieba_note(note_item: TiebaNote):
await TieBaStoreFactory.create_store().store_content(save_note_item) await TieBaStoreFactory.create_store().store_content(save_note_item)
async def batch_update_tieba_note_comments(note_id:str, comments: List[TiebaComment]): async def batch_update_tieba_note_comments(note_id: str, comments: List[TiebaComment]):
""" """
Batch update tieba note comments Batch update tieba note comments
Args: Args:
@ -86,27 +88,16 @@ async def update_tieba_note_comment(note_id: str, comment_item: TiebaComment):
await TieBaStoreFactory.create_store().store_comment(save_comment_item) await TieBaStoreFactory.create_store().store_comment(save_comment_item)
async def save_creator(user_id: str, user_info: Dict): async def save_creator(user_info: TiebaCreator):
""" """
Save creator information to local Save creator information to local
Args: Args:
user_id:
user_info: user_info:
Returns: Returns:
""" """
local_db_item = { local_db_item = user_info.model_dump()
'user_id': user_id, local_db_item["last_modify_ts"] = utils.get_current_timestamp()
'nickname': user_info.get('nickname'),
'gender': '女' if user_info.get('gender') == "f" else '男',
'avatar': user_info.get('avatar'),
'ip_location': user_info.get("ip_location", ""),
'follows': user_info.get('follow_count', ''),
'fans': user_info.get('followers_count', ''),
'follow_tieba_list': user_info.get("tieba_list", ''),
'last_modify_ts': utils.get_current_timestamp(),
'registration_duration': user_info.get("registration_duration", ""),
}
utils.logger.info(f"[store.tieba.save_creator] creator:{local_db_item}") utils.logger.info(f"[store.tieba.save_creator] creator:{local_db_item}")
await TieBaStoreFactory.create_store().store_creator(local_db_item) await TieBaStoreFactory.create_store().store_creator(local_db_item)

25
store/tieba/tieba_store_impl.py
View File

@ -24,14 +24,14 @@ def calculate_number_of_files(file_store_path: str) -> int:
if not os.path.exists(file_store_path): if not os.path.exists(file_store_path):
return 1 return 1
try: try:
return max([int(file_name.split("_")[0])for file_name in os.listdir(file_store_path)])+1 return max([int(file_name.split("_")[0]) for file_name in os.listdir(file_store_path)]) + 1
except ValueError: except ValueError:
return 1 return 1
class TieBaCsvStoreImplement(AbstractStore): class TieBaCsvStoreImplement(AbstractStore):
csv_store_path: str = "data/tieba" csv_store_path: str = "data/tieba"
file_count:int=calculate_number_of_files(csv_store_path) file_count: int = calculate_number_of_files(csv_store_path)
def make_save_file_name(self, store_type: str) -> str: def make_save_file_name(self, store_type: str) -> str:
""" """
@ -65,7 +65,7 @@ class TieBaCsvStoreImplement(AbstractStore):
async def store_content(self, content_item: Dict): async def store_content(self, content_item: Dict):
""" """
Xiaohongshu content CSV storage implementation tieba content CSV storage implementation
Args: Args:
content_item: note item dict content_item: note item dict
@ -76,7 +76,7 @@ class TieBaCsvStoreImplement(AbstractStore):
async def store_comment(self, comment_item: Dict): async def store_comment(self, comment_item: Dict):
""" """
Xiaohongshu comment CSV storage implementation tieba comment CSV storage implementation
Args: Args:
comment_item: comment item dict comment_item: comment item dict
@ -87,7 +87,7 @@ class TieBaCsvStoreImplement(AbstractStore):
async def store_creator(self, creator: Dict): async def store_creator(self, creator: Dict):
""" """
Xiaohongshu content CSV storage implementation tieba content CSV storage implementation
Args: Args:
creator: creator dict creator: creator dict
@ -100,7 +100,7 @@ class TieBaCsvStoreImplement(AbstractStore):
class TieBaDbStoreImplement(AbstractStore): class TieBaDbStoreImplement(AbstractStore):
async def store_content(self, content_item: Dict): async def store_content(self, content_item: Dict):
""" """
Xiaohongshu content DB storage implementation tieba content DB storage implementation
Args: Args:
content_item: content item dict content_item: content item dict
@ -120,7 +120,7 @@ class TieBaDbStoreImplement(AbstractStore):
async def store_comment(self, comment_item: Dict): async def store_comment(self, comment_item: Dict):
""" """
Xiaohongshu content DB storage implementation tieba content DB storage implementation
Args: Args:
comment_item: comment item dict comment_item: comment item dict
@ -140,7 +140,7 @@ class TieBaDbStoreImplement(AbstractStore):
async def store_creator(self, creator: Dict): async def store_creator(self, creator: Dict):
""" """
Xiaohongshu content DB storage implementation tieba content DB storage implementation
Args: Args:
creator: creator dict creator: creator dict
@ -163,10 +163,10 @@ class TieBaJsonStoreImplement(AbstractStore):
json_store_path: str = "data/tieba/json" json_store_path: str = "data/tieba/json"
words_store_path: str = "data/tieba/words" words_store_path: str = "data/tieba/words"
lock = asyncio.Lock() lock = asyncio.Lock()
file_count:int=calculate_number_of_files(json_store_path) file_count: int = calculate_number_of_files(json_store_path)
WordCloud = words.AsyncWordCloudGenerator() WordCloud = words.AsyncWordCloudGenerator()
def make_save_file_name(self, store_type: str) -> (str,str): def make_save_file_name(self, store_type: str) -> (str, str):
""" """
make save file name by store type make save file name by store type
Args: Args:
@ -193,7 +193,7 @@ class TieBaJsonStoreImplement(AbstractStore):
""" """
pathlib.Path(self.json_store_path).mkdir(parents=True, exist_ok=True) pathlib.Path(self.json_store_path).mkdir(parents=True, exist_ok=True)
pathlib.Path(self.words_store_path).mkdir(parents=True, exist_ok=True) pathlib.Path(self.words_store_path).mkdir(parents=True, exist_ok=True)
save_file_name,words_file_name_prefix = self.make_save_file_name(store_type=store_type) save_file_name, words_file_name_prefix = self.make_save_file_name(store_type=store_type)
save_data = [] save_data = []
async with self.lock: async with self.lock:
@ -210,6 +210,7 @@ class TieBaJsonStoreImplement(AbstractStore):
await self.WordCloud.generate_word_frequency_and_cloud(save_data, words_file_name_prefix) await self.WordCloud.generate_word_frequency_and_cloud(save_data, words_file_name_prefix)
except: except:
pass pass
async def store_content(self, content_item: Dict): async def store_content(self, content_item: Dict):
""" """
content JSON storage implementation content JSON storage implementation
@ -234,7 +235,7 @@ class TieBaJsonStoreImplement(AbstractStore):
async def store_creator(self, creator: Dict): async def store_creator(self, creator: Dict):
""" """
Xiaohongshu content JSON storage implementation tieba content JSON storage implementation
Args: Args:
creator: creator dict creator: creator dict