qnloft
/
MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 贴吧搜索重构

This commit is contained in:
Relakkes 2024-08-07 01:01:21 +08:00
parent 1b585cb215
commit 3c98808409
17 changed files with 1057 additions and 7707 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
constant/__init__.py Normal file
View File

@ -0,0 +1 @@
# -*- coding: utf-8 -*-

3
constant/baidu_tieba.py Normal file
View File

@ -0,0 +1,3 @@
# -*- coding: utf-8 -*-
TIEBA_URL = 'https://tieba.baidu.com'

2
db.py
View File

@ -85,7 +85,7 @@ async def init_table_schema():
utils.logger.info("[init_table_schema] begin init mysql table schema ...") utils.logger.info("[init_table_schema] begin init mysql table schema ...")
await init_mediacrawler_db() await init_mediacrawler_db()
async_db_obj: AsyncMysqlDB = media_crawler_db_var.get() async_db_obj: AsyncMysqlDB = media_crawler_db_var.get()
async with aiofiles.open("schema/tables.sql", mode="r") as f: async with aiofiles.open("schema/tables.sql", mode="r", encoding="utf-8") as f:
schema_sql = await f.read() schema_sql = await f.read()
await async_db_obj.execute(schema_sql) await async_db_obj.execute(schema_sql)
utils.logger.info("[init_table_schema] mediacrawler table schema init successful") utils.logger.info("[init_table_schema] mediacrawler table schema init successful")

8
media_platform/tieba/client.py
View File

@ -10,6 +10,7 @@ from tenacity import (RetryError, retry, stop_after_attempt,
wait_fixed) wait_fixed)
from base.base_crawler import AbstractApiClient from base.base_crawler import AbstractApiClient
from model.m_baidu_tieba import TiebaNote
from proxy.proxy_ip_pool import ProxyIpPool from proxy.proxy_ip_pool import ProxyIpPool
from tools import utils from tools import utils
@ -98,6 +99,7 @@ class BaiduTieBaClient(AbstractApiClient):
return res return res
utils.logger.error(f"[BaiduTieBaClient.get] 达到了最大重试次数，请尝试更换新的IP代理: {e}") utils.logger.error(f"[BaiduTieBaClient.get] 达到了最大重试次数，请尝试更换新的IP代理: {e}")
raise e
async def post(self, uri: str, data: dict, **kwargs) -> Dict: async def post(self, uri: str, data: dict, **kwargs) -> Dict:
""" """
@ -152,7 +154,7 @@ class BaiduTieBaClient(AbstractApiClient):
sort: SearchSortType = SearchSortType.TIME_DESC, sort: SearchSortType = SearchSortType.TIME_DESC,
note_type: SearchNoteType = SearchNoteType.FIXED_THREAD, note_type: SearchNoteType = SearchNoteType.FIXED_THREAD,
random_sleep: bool = True random_sleep: bool = True
) -> List[Dict]: ) -> List[TiebaNote]:
""" """
根据关键词搜索贴吧帖子 根据关键词搜索贴吧帖子
Args: Args:
@ -180,7 +182,7 @@ class BaiduTieBaClient(AbstractApiClient):
random.randint(1, 5) random.randint(1, 5)
return self._page_extractor.extract_search_note_list(page_content) return self._page_extractor.extract_search_note_list(page_content)
async def get_note_by_id(self, note_id: str) -> Dict: async def get_note_by_id(self, note_id: str) -> TiebaNote:
""" """
根据帖子ID获取帖子详情 根据帖子ID获取帖子详情
Args: Args:
@ -192,8 +194,6 @@ class BaiduTieBaClient(AbstractApiClient):
uri = f"/p/{note_id}" uri = f"/p/{note_id}"
page_content = await self.get(uri, return_ori_content=True) page_content = await self.get(uri, return_ori_content=True)
return self._page_extractor.extract_note_detail(page_content) return self._page_extractor.extract_note_detail(page_content)
# todo impl it
return {}
async def get_note_all_comments(self, note_id: str, crawl_interval: float = 1.0, async def get_note_all_comments(self, note_id: str, crawl_interval: float = 1.0,
callback: Optional[Callable] = None) -> List[Dict]: callback: Optional[Callable] = None) -> List[Dict]:

129
media_platform/tieba/core.py
View File

@ -9,7 +9,8 @@ from playwright.async_api import (BrowserContext, BrowserType, Page,
import config import config
from base.base_crawler import AbstractCrawler from base.base_crawler import AbstractCrawler
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool, ProxyIpPool from model.m_baidu_tieba import TiebaNote
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from store import tieba as tieba_store from store import tieba as tieba_store
from tools import utils from tools import utils
from tools.crawler_util import format_proxy_info from tools.crawler_util import format_proxy_info
@ -66,7 +67,6 @@ class TieBaCrawler(AbstractCrawler):
Returns: Returns:
""" """
utils.logger.info("[BaiduTieBaCrawler.search] Begin search baidu tieba keywords") utils.logger.info("[BaiduTieBaCrawler.search] Begin search baidu tieba keywords")
tieba_limit_count = 10 # tieba limit page fixed value tieba_limit_count = 10 # tieba limit page fixed value
if config.CRAWLER_MAX_NOTES_COUNT < tieba_limit_count: if config.CRAWLER_MAX_NOTES_COUNT < tieba_limit_count:
@ -82,52 +82,36 @@ class TieBaCrawler(AbstractCrawler):
continue continue
try: try:
utils.logger.info(f"[BaiduTieBaCrawler.search] search tieba keyword: {keyword}, page: {page}") utils.logger.info(f"[BaiduTieBaCrawler.search] search tieba keyword: {keyword}, page: {page}")
note_id_list: List[str] = [] notes_list: List[TiebaNote] = await self.tieba_client.get_notes_by_keyword(
notes_list_res = await self.tieba_client.get_notes_by_keyword(
keyword=keyword, keyword=keyword,
page=page, page=page,
page_size=tieba_limit_count, page_size=tieba_limit_count,
sort=SearchSortType.TIME_DESC, sort=SearchSortType.TIME_DESC,
note_type=SearchNoteType.FIXED_THREAD note_type=SearchNoteType.FIXED_THREAD
) )
utils.logger.info(f"[BaiduTieBaCrawler.search] Search notes res:{notes_list_res}") if not notes_list:
if not notes_list_res: utils.logger.info(f"[BaiduTieBaCrawler.search] Search note list is empty")
break break
utils.logger.info(f"[BaiduTieBaCrawler.search] Note List: {notes_list}")
for note_detail in notes_list_res: await self.get_specified_notes(note_id_list=[note_detail.note_id for note_detail in notes_list])
if note_detail:
await tieba_store.update_tieba_note(note_detail)
note_id_list.append(note_detail.get("note_id"))
page += 1 page += 1
utils.logger.info(f"[BaiduTieBaCrawler.search] Note details: {notes_list_res}")
await self.batch_get_note_comments(note_id_list)
except Exception as ex: except Exception as ex:
utils.logger.error(f"[BaiduTieBaCrawler.search] Search note list error, err: {ex}") utils.logger.error(
f"[BaiduTieBaCrawler.search] Search keywords error, current page: {page}, current keyword: {keyword}, err: {ex}")
break break
async def fetch_creator_notes_detail(self, note_list: List[Dict]): async def get_specified_notes(self, note_id_list: List[str] = config.TIEBA_SPECIFIED_ID_LIST):
""" """
Concurrently obtain the specified post list and save the data Get the information and comments of the specified post
Args:
note_id_list:
Returns:
""" """
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list = [ task_list = [
self.get_note_detail( self.get_note_detail_async_task(note_id=note_id, semaphore=semaphore) for note_id in note_id_list
note_id=post_item.get("note_id"),
semaphore=semaphore
)
for post_item in note_list
]
note_details = await asyncio.gather(*task_list)
for note_detail in note_details:
if note_detail:
await tieba_store.update_tieba_note(note_detail)
async def get_specified_notes(self):
"""Get the information and comments of the specified post"""
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list = [
self.get_note_detail(note_id=note_id, semaphore=semaphore) for note_id in config.TIEBA_SPECIFIED_ID_LIST
] ]
note_details = await asyncio.gather(*task_list) note_details = await asyncio.gather(*task_list)
for note_detail in note_details: for note_detail in note_details:
@ -135,11 +119,20 @@ class TieBaCrawler(AbstractCrawler):
await tieba_store.update_tieba_note(note_detail) await tieba_store.update_tieba_note(note_detail)
await self.batch_get_note_comments(config.TIEBA_SPECIFIED_ID_LIST) await self.batch_get_note_comments(config.TIEBA_SPECIFIED_ID_LIST)
async def get_note_detail(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[Dict]: async def get_note_detail_async_task(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[TiebaNote]:
"""Get note detail""" """
Get note detail
Args:
note_id: baidu tieba note id
semaphore: asyncio semaphore
Returns:
"""
async with semaphore: async with semaphore:
try: try:
note_detail: Dict = await self.tieba_client.get_note_by_id(note_id) utils.logger.info(f"[BaiduTieBaCrawler.get_note_detail] Begin get note detail, note_id: {note_id}")
note_detail: TiebaNote = await self.tieba_client.get_note_by_id(note_id)
if not note_detail: if not note_detail:
utils.logger.error( utils.logger.error(
f"[BaiduTieBaCrawler.get_note_detail] Get note detail error, note_id: {note_id}") f"[BaiduTieBaCrawler.get_note_detail] Get note detail error, note_id: {note_id}")
@ -153,23 +146,38 @@ class TieBaCrawler(AbstractCrawler):
f"[BaiduTieBaCrawler.get_note_detail] have not fund note detail note_id:{note_id}, err: {ex}") f"[BaiduTieBaCrawler.get_note_detail] have not fund note detail note_id:{note_id}, err: {ex}")
return None return None
async def batch_get_note_comments(self, note_list: List[str]): async def batch_get_note_comments(self, note_id_list: List[str]):
"""Batch get note comments""" """
Batch get note comments
Args:
note_id_list:
Returns:
"""
if not config.ENABLE_GET_COMMENTS: if not config.ENABLE_GET_COMMENTS:
utils.logger.info(f"[BaiduTieBaCrawler.batch_get_note_comments] Crawling comment mode is not enabled") utils.logger.info(f"[BaiduTieBaCrawler.batch_get_note_comments] Crawling comment mode is not enabled")
return return
utils.logger.info( utils.logger.info(
f"[BaiduTieBaCrawler.batch_get_note_comments] Begin batch get note comments, note list: {note_list}") f"[BaiduTieBaCrawler.batch_get_note_comments] Begin batch get note comments, note list: {note_id_list}")
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list: List[Task] = [] task_list: List[Task] = []
for note_id in note_list: for note_id in note_id_list:
task = asyncio.create_task(self.get_comments(note_id, semaphore), name=note_id) task = asyncio.create_task(self.get_comments_async_task(note_id, semaphore), name=note_id)
task_list.append(task) task_list.append(task)
await asyncio.gather(*task_list) await asyncio.gather(*task_list)
async def get_comments(self, note_id: str, semaphore: asyncio.Semaphore): async def get_comments_async_task(self, note_id: str, semaphore: asyncio.Semaphore):
"""Get note comments with keyword filtering and quantity limitation""" """
Get comments async task
Args:
note_id:
semaphore:
Returns:
"""
async with semaphore: async with semaphore:
utils.logger.info(f"[BaiduTieBaCrawler.get_comments] Begin get note id comments {note_id}") utils.logger.info(f"[BaiduTieBaCrawler.get_comments] Begin get note id comments {note_id}")
await self.tieba_client.get_note_all_comments( await self.tieba_client.get_note_all_comments(
@ -178,23 +186,6 @@ class TieBaCrawler(AbstractCrawler):
callback=tieba_store.batch_update_tieba_note_comments callback=tieba_store.batch_update_tieba_note_comments
) )
async def create_tieba_client(self, ip_pool: ProxyIpPool) -> BaiduTieBaClient:
"""
Create tieba client
Args:
ip_pool:
Returns:
"""
"""Create tieba client"""
utils.logger.info("[BaiduTieBaCrawler.create_tieba_client] Begin create baidutieba API client ...")
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
tieba_client_obj = BaiduTieBaClient(
ip_pool=ip_pool,
)
return tieba_client_obj
async def launch_browser( async def launch_browser(
self, self,
chromium: BrowserType, chromium: BrowserType,
@ -202,7 +193,17 @@ class TieBaCrawler(AbstractCrawler):
user_agent: Optional[str], user_agent: Optional[str],
headless: bool = True headless: bool = True
) -> BrowserContext: ) -> BrowserContext:
"""Launch browser and create browser context""" """
Launch browser and create browser
Args:
chromium:
playwright_proxy:
user_agent:
headless:
Returns:
"""
utils.logger.info("[BaiduTieBaCrawler.launch_browser] Begin create browser context ...") utils.logger.info("[BaiduTieBaCrawler.launch_browser] Begin create browser context ...")
if config.SAVE_LOGIN_STATE: if config.SAVE_LOGIN_STATE:
# feat issue #14 # feat issue #14
@ -227,6 +228,10 @@ class TieBaCrawler(AbstractCrawler):
return browser_context return browser_context
async def close(self): async def close(self):
"""Close browser context""" """
Close browser context
Returns:
"""
await self.browser_context.close() await self.browser_context.close()
utils.logger.info("[BaiduTieBaCrawler.close] Browser context closed ...") utils.logger.info("[BaiduTieBaCrawler.close] Browser context closed ...")

118
media_platform/tieba/help.py
View File

@ -1,18 +1,21 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import re
from typing import List, Dict from typing import List, Dict, Tuple
from parsel import Selector from parsel import Selector
from model.m_baidu_tieba import TiebaNote
from constant import baidu_tieba as const
class TieBaExtractor: class TieBaExtractor:
def __init__(self): def __init__(self):
pass pass
@staticmethod @staticmethod
def extract_search_note_list(page_content: str) -> List[Dict]: def extract_search_note_list(page_content: str) -> List[TiebaNote]:
""" """
提取贴吧帖子列表 提取贴吧帖子列表，这里提取的关键词搜索结果页的数据，还缺少帖子的回复数和回复页等数据
Args: Args:
page_content: 页面内容的HTML字符串 page_content: 页面内容的HTML字符串
@ -21,33 +24,24 @@ class TieBaExtractor:
""" """
xpath_selector = "//div[@class='s_post']" xpath_selector = "//div[@class='s_post']"
post_list = Selector(text=page_content).xpath(xpath_selector) post_list = Selector(text=page_content).xpath(xpath_selector)
result = [] result: List[TiebaNote] = []
for post in post_list: for post in post_list:
post_id = post.xpath(".//span[@class='p_title']/a/@data-tid").get(default='').strip() tieba_note = TiebaNote(
title = post.xpath(".//span[@class='p_title']/a/text()").get(default='').strip() note_id=post.xpath(".//span[@class='p_title']/a/@data-tid").get(default='').strip(),
link = post.xpath(".//span[@class='p_title']/a/@href").get(default='') title=post.xpath(".//span[@class='p_title']/a/text()").get(default='').strip(),
description = post.xpath(".//div[@class='p_content']/text()").get(default='').strip() desc=post.xpath(".//div[@class='p_content']/text()").get(default='').strip(),
forum = post.xpath(".//a[@class='p_forum']/font/text()").get(default='').strip() note_url=const.TIEBA_URL + post.xpath(".//span[@class='p_title']/a/@href").get(default=''),
forum_link = post.xpath(".//a[@class='p_forum']/@href").get(default='') user_nickname=post.xpath(".//a[starts-with(@href, '/home/main')]/font/text()").get(default='').strip(),
author = post.xpath(".//a[starts-with(@href, '/home/main')]/font/text()").get(default='').strip() user_link=const.TIEBA_URL + post.xpath(".//a[starts-with(@href, '/home/main')]/@href").get(default=''),
author_link = post.xpath(".//a[starts-with(@href, '/home/main')]/@href").get(default='') tieba_name=post.xpath(".//a[@class='p_forum']/font/text()").get(default='').strip(),
date = post.xpath(".//font[@class='p_green p_date']/text()").get(default='').strip() tieba_link=const.TIEBA_URL + post.xpath(".//a[@class='p_forum']/@href").get(default=''),
result.append({ publish_time=post.xpath(".//font[@class='p_green p_date']/text()").get(default='').strip(),
"note_id": post_id, )
"title": title, result.append(tieba_note)
"desc": description,
"note_url": link,
"time": date,
"tieba_name": forum,
"tieba_link": forum_link,
"nickname": author,
"nickname_link": author_link,
})
return result return result
@staticmethod
def extract_note_detail(page_content: str) -> Dict: def extract_note_detail(self, page_content: str) -> TiebaNote:
""" """
提取贴吧帖子详情 提取贴吧帖子详情
Args: Args:
@ -57,13 +51,33 @@ class TieBaExtractor:
""" """
content_selector = Selector(text=page_content) content_selector = Selector(text=page_content)
# 查看楼主的链接： only_view_author_link: / p / 9117905169?see_lz = 1 first_floor_selector = content_selector.xpath("//div[@class='p_postlist'][1]")
only_view_author_link = content_selector.xpath("//*[@id='lzonly_cntn']/@href").get(default='').strip() # only_view_author_link = content_selector.xpath("//*[@id='lzonly_cntn']/@href").get(default='').strip()
note_id = only_view_author_link.split("?")[0].split("/")[-1] note_id = only_view_author_link.split("?")[0].split("/")[-1]
title = content_selector.xpath("//*[@id='j_core_title_wrap']/h3").get(default='').strip() # 帖子回复数、回复页数
desc = content_selector.xpath("//meta[@name='description']").get(default='').strip() thread_num_infos = content_selector.xpath(
note_url = f"/p/{note_id}" "//div[@id='thread_theme_5']//li[@class='l_reply_num']//span[@class='red']"
pass )
# IP地理位置、发表时间
other_info_content = content_selector.xpath(".//div[@class='post-tail-wrap']").get(default="").strip()
ip_location, publish_time = self.extract_ip_and_pub_time(other_info_content)
note = TiebaNote(
note_id=note_id,
title=content_selector.xpath("//title/text()").get(default='').strip(),
desc=content_selector.xpath("//meta[@name='description']/@content").get(default='').strip(),
note_url=const.TIEBA_URL + f"/p/{note_id}",
user_link=const.TIEBA_URL + first_floor_selector.xpath(".//a[@class='p_author_face ']/@href").get(default='').strip(),
user_nickname=first_floor_selector.xpath(".//a[@class='p_author_name j_user_card']/text()").get(default='').strip(),
user_avatar=first_floor_selector.xpath(".//a[@class='p_author_face ']/img/@src").get(default='').strip(),
tieba_name=content_selector.xpath("//a[@class='card_title_fname']/text()").get(default='').strip(),
tieba_link=const.TIEBA_URL + content_selector.xpath("//a[@class='card_title_fname']/@href").get(default=''),
ip_location=ip_location,
publish_time=publish_time,
total_replay_num=thread_num_infos[0].xpath("./text()").get(default='').strip(),
total_replay_page=thread_num_infos[1].xpath("./text()").get(default='').strip(),
)
note.title = note.title.replace(f"【{note.tieba_name}】_百度贴吧", "")
return note
@staticmethod @staticmethod
def extract_tieba_note_comments(page_content: str) -> List[Dict]: def extract_tieba_note_comments(page_content: str) -> List[Dict]:
@ -93,12 +107,40 @@ class TieBaExtractor:
"time": date, "time": date,
}) })
@staticmethod
def extract_ip_and_pub_time(html_content: str) -> Tuple[str, str]:
"""
提取IP位置和发布时间
Args:
html_content:
Returns:
if __name__ == '__main__': """
pattern_ip = re.compile(r'IP属地:(\S+)</span>')
pattern_pub_time = re.compile(r'<span class="tail-info">(\d{4}-\d{2}-\d{2} \d{2}:\d{2})</span>')
ip_match = pattern_ip.search(html_content)
time_match = pattern_pub_time.search(html_content)
ip = ip_match.group(1) if ip_match else ""
pub_time = time_match.group(1) if time_match else ""
return ip, pub_time
def test_extract_search_note_list():
with open("test_data/search_keyword_notes.html", "r", encoding="utf-8") as f: with open("test_data/search_keyword_notes.html", "r", encoding="utf-8") as f:
content = f.read() content = f.read()
extractor = TieBaExtractor() extractor = TieBaExtractor()
_result = extractor.extract_search_note_list(content) result = extractor.extract_search_note_list(content)
print(_result) print(result)
print(f"Total: {len(_result)}")
def test_extract_note_detail():
with open("test_data/note_detail.html", "r", encoding="utf-8") as f:
content = f.read()
extractor = TieBaExtractor()
result = extractor.extract_note_detail(content)
print(result.model_dump())
if __name__ == '__main__':
test_extract_search_note_list()
test_extract_note_detail()

839
media_platform/tieba/test_data/note_detail.html Normal file
View File

File diff suppressed because one or more lines are too long

7558
media_platform/tieba/test_data/note_detail_and_comments.html
View File

File diff suppressed because one or more lines are too long

1
model/__init__.py Normal file
View File

@ -0,0 +1 @@
# -*- coding: utf-8 -*-

19
model/m_baidu_tieba.py Normal file
View File

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
from typing import Optional
from pydantic import BaseModel, Field
class TiebaNote(BaseModel):
note_id: str = Field(..., description="帖子ID")
title: str = Field(..., description="帖子标题")
desc: str = Field(default="", description="帖子描述")
note_url: str = Field(..., description="帖子链接")
publish_time: str = Field(default="", description="发布时间")
user_link: str = Field(default="", description="用户主页链接")
user_nickname: str = Field(default="", description="用户昵称")
user_avatar: str = Field(default="", description="用户头像地址")
tieba_name: str = Field(..., description="贴吧名称")
tieba_link: str = Field(..., description="贴吧链接")
total_replay_num: int = Field(default=0, description="回复总数")
total_replay_page: int = Field(default=0, description="回复总页数")
ip_location: Optional[str] = Field(default="", description="IP地理位置")

1
model/m_douyin.py Normal file
View File

@ -0,0 +1 @@
# -*- coding: utf-8 -*-

1
model/m_kuaishou.py Normal file
View File

@ -0,0 +1 @@
# -*- coding: utf-8 -*-

1
model/m_weibo.py Normal file
View File

@ -0,0 +1 @@
# -*- coding: utf-8 -*-

1
model/m_xiaohongshu.py Normal file
View File

@ -0,0 +1 @@
# -*- coding: utf-8 -*-

41
schema/tables.sql
View File

@ -349,29 +349,26 @@ ALTER TABLE `bilibili_video_comment`
ALTER TABLE `weibo_note_comment` ALTER TABLE `weibo_note_comment`
ADD COLUMN `parent_comment_id` VARCHAR(64) DEFAULT NULL COMMENT '父评论ID'; ADD COLUMN `parent_comment_id` VARCHAR(64) DEFAULT NULL COMMENT '父评论ID';
SET
FOREIGN_KEY_CHECKS = 1;
DROP TABLE IF EXISTS `tieba_note`; DROP TABLE IF EXISTS `tieba_note`;
CREATE TABLE `tieba_note` CREATE TABLE tieba_note
( (
`id` int NOT NULL AUTO_INCREMENT COMMENT '自增ID', id BIGINT AUTO_INCREMENT PRIMARY KEY,
`note_id` varchar(64) NOT NULL COMMENT '帖子ID', note_id VARCHAR(644) NOT NULL COMMENT '帖子ID',
`title` varchar(255) DEFAULT NULL COMMENT '笔记标题', title VARCHAR(255) NOT NULL COMMENT '帖子标题',
`desc` longtext COMMENT '笔记描述', `desc` TEXT COMMENT '帖子描述',
`time` varchar NOT NULL COMMENT '笔记发布时间', note_url VARCHAR(255) NOT NULL COMMENT '帖子链接',
`note_url` varchar(255) DEFAULT NULL COMMENT '笔记详情页的URL', publish_time VARCHAR(255) NOT NULL COMMENT '发布时间',
`nickname` varchar(64) DEFAULT NULL COMMENT '用户昵称', user_link VARCHAR(255) NOT NULL COMMENT '用户主页链接',
`nickname_link` varchar(255) DEFAULT NULL COMMENT '用户主页地址', user_nickname VARCHAR(255) NOT NULL COMMENT '用户昵称',
`tieba_name` varchar(255) DEFAULT NULL COMMENT '贴吧名称', user_avatar VARCHAR(255) NOT NULL COMMENT '用户头像地址',
`tieba_link` varchar(255) DEFAULT NULL COMMENT '贴吧链接地址', tieba_name VARCHAR(255) NOT NULL COMMENT '贴吧名称',
`avatar` varchar(255) DEFAULT NULL COMMENT '用户头像地址', tieba_link VARCHAR(255) NOT NULL COMMENT '贴吧链接',
`ip_location` varchar(255) DEFAULT NULL COMMENT '评论时的IP地址', total_replay_num INT DEFAULT 0 COMMENT '帖子回复总数',
`add_ts` bigint NOT NULL COMMENT '记录添加时间戳', total_replay_page INT DEFAULT 0 COMMENT '帖子回复总页数',
`last_modify_ts` bigint NOT NULL COMMENT '记录最后修改时间戳', ip_location VARCHAR(255) DEFAULT '' COMMENT 'IP地理位置',
`comment_count` varchar(16) DEFAULT NULL COMMENT '笔记评论数', add_ts BIGINT NOT NULL COMMENT '添加时间戳',
PRIMARY KEY (`id`), last_modify_ts BIGINT NOT NULL COMMENT '最后修改时间戳',
KEY `idx_tieba_note_id` (`note_id`), KEY `idx_tieba_note_note_id` (`note_id`),
KEY `idx_tieba_note_time` (`time`) KEY `idx_tieba_note_publish_time` (`publish_time`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='贴吧帖子表'; ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='贴吧帖子表';

33
store/tieba/__init__.py
View File

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from typing import List from typing import List
from model.m_baidu_tieba import TiebaNote
from . import tieba_store_impl from . import tieba_store_impl
from .tieba_store_impl import * from .tieba_store_impl import *
@ -21,24 +22,20 @@ class TieBaStoreFactory:
return store_class() return store_class()
async def update_tieba_note(note_item: Dict): async def update_tieba_note(note_item: TiebaNote):
tieba_url = "https://tieba.baidu.com" """
note_id = note_item.get("note_id") Add or Update tieba note
local_db_item = { Args:
"note_id": note_id, note_item:
"title": note_item.get("title") or note_item.get("desc", "")[:255],
"desc": note_item.get("desc", ""), Returns:
"note_url": tieba_url + note_item.get("note_url"),
"time": note_item.get("time"), """
"tieba_name": note_item.get("tieba_name"), save_note_item = note_item.model_dump()
"tieba_link": tieba_url + note_item.get("tieba_link", ""), save_note_item.update({"last_modify_ts": utils.get_current_timestamp()})
"nickname": note_item.get("nickname"), utils.logger.info(f"[store.tieba.update_tieba_note] tieba note: {save_note_item}")
"nickname_link": tieba_url + note_item.get("nickname_link", ""),
"ip_location": note_item.get("ip_location", ""), await TieBaStoreFactory.create_store().store_content(save_note_item)
"last_modify_ts": utils.get_current_timestamp(),
}
utils.logger.info(f"[store.tieba.update_tieba_note] tieba note: {local_db_item}")
await TieBaStoreFactory.create_store().store_content(local_db_item)
async def batch_update_tieba_note_comments(note_id: str, comments: List[Dict]): async def batch_update_tieba_note_comments(note_id: str, comments: List[Dict]):