feat: 贴吧搜索重构
This commit is contained in:
Relakkes
parent
1b585cb215
commit
3c98808409
|
|
@ -0,0 +1 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
TIEBA_URL = 'https://tieba.baidu.com'
|
||||
2
db.py
2
db.py
|
|
@ -85,7 +85,7 @@ async def init_table_schema():
|
|||
utils.logger.info("[init_table_schema] begin init mysql table schema ...")
|
||||
await init_mediacrawler_db()
|
||||
async_db_obj: AsyncMysqlDB = media_crawler_db_var.get()
|
||||
async with aiofiles.open("schema/tables.sql", mode="r") as f:
|
||||
async with aiofiles.open("schema/tables.sql", mode="r", encoding="utf-8") as f:
|
||||
schema_sql = await f.read()
|
||||
await async_db_obj.execute(schema_sql)
|
||||
utils.logger.info("[init_table_schema] mediacrawler table schema init successful")
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from tenacity import (RetryError, retry, stop_after_attempt,
|
|||
wait_fixed)
|
||||
|
||||
from base.base_crawler import AbstractApiClient
|
||||
from model.m_baidu_tieba import TiebaNote
|
||||
from proxy.proxy_ip_pool import ProxyIpPool
|
||||
from tools import utils
|
||||
|
||||
|
|
@ -98,6 +99,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
|||
return res
|
||||
|
||||
utils.logger.error(f"[BaiduTieBaClient.get] 达到了最大重试次数，请尝试更换新的IP代理: {e}")
|
||||
raise e
|
||||
|
||||
async def post(self, uri: str, data: dict, **kwargs) -> Dict:
|
||||
"""
|
||||
|
|
@ -152,7 +154,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
|||
sort: SearchSortType = SearchSortType.TIME_DESC,
|
||||
note_type: SearchNoteType = SearchNoteType.FIXED_THREAD,
|
||||
random_sleep: bool = True
|
||||
) -> List[Dict]:
|
||||
) -> List[TiebaNote]:
|
||||
"""
|
||||
根据关键词搜索贴吧帖子
|
||||
Args:
|
||||
|
|
@ -180,7 +182,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
|||
random.randint(1, 5)
|
||||
return self._page_extractor.extract_search_note_list(page_content)
|
||||
|
||||
async def get_note_by_id(self, note_id: str) -> Dict:
|
||||
async def get_note_by_id(self, note_id: str) -> TiebaNote:
|
||||
"""
|
||||
根据帖子ID获取帖子详情
|
||||
Args:
|
||||
|
|
@ -192,8 +194,6 @@ class BaiduTieBaClient(AbstractApiClient):
|
|||
uri = f"/p/{note_id}"
|
||||
page_content = await self.get(uri, return_ori_content=True)
|
||||
return self._page_extractor.extract_note_detail(page_content)
|
||||
# todo impl it
|
||||
return {}
|
||||
|
||||
async def get_note_all_comments(self, note_id: str, crawl_interval: float = 1.0,
|
||||
callback: Optional[Callable] = None) -> List[Dict]:
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ from playwright.async_api import (BrowserContext, BrowserType, Page,
|
|||
|
||||
import config
|
||||
from base.base_crawler import AbstractCrawler
|
||||
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool, ProxyIpPool
|
||||
from model.m_baidu_tieba import TiebaNote
|
||||
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
|
||||
from store import tieba as tieba_store
|
||||
from tools import utils
|
||||
from tools.crawler_util import format_proxy_info
|
||||
|
|
@ -66,7 +67,6 @@ class TieBaCrawler(AbstractCrawler):
|
|||
Returns:
|
||||
|
||||
"""
|
||||
|
||||
utils.logger.info("[BaiduTieBaCrawler.search] Begin search baidu tieba keywords")
|
||||
tieba_limit_count = 10 # tieba limit page fixed value
|
||||
if config.CRAWLER_MAX_NOTES_COUNT < tieba_limit_count:
|
||||
|
|
@ -82,52 +82,36 @@ class TieBaCrawler(AbstractCrawler):
|
|||
continue
|
||||
try:
|
||||
utils.logger.info(f"[BaiduTieBaCrawler.search] search tieba keyword: {keyword}, page: {page}")
|
||||
note_id_list: List[str] = []
|
||||
notes_list_res = await self.tieba_client.get_notes_by_keyword(
|
||||
notes_list: List[TiebaNote] = await self.tieba_client.get_notes_by_keyword(
|
||||
keyword=keyword,
|
||||
page=page,
|
||||
page_size=tieba_limit_count,
|
||||
sort=SearchSortType.TIME_DESC,
|
||||
note_type=SearchNoteType.FIXED_THREAD
|
||||
)
|
||||
utils.logger.info(f"[BaiduTieBaCrawler.search] Search notes res:{notes_list_res}")
|
||||
if not notes_list_res:
|
||||
if not notes_list:
|
||||
utils.logger.info(f"[BaiduTieBaCrawler.search] Search note list is empty")
|
||||
break
|
||||
|
||||
for note_detail in notes_list_res:
|
||||
if note_detail:
|
||||
await tieba_store.update_tieba_note(note_detail)
|
||||
note_id_list.append(note_detail.get("note_id"))
|
||||
utils.logger.info(f"[BaiduTieBaCrawler.search] Note List: {notes_list}")
|
||||
await self.get_specified_notes(note_id_list=[note_detail.note_id for note_detail in notes_list])
|
||||
page += 1
|
||||
utils.logger.info(f"[BaiduTieBaCrawler.search] Note details: {notes_list_res}")
|
||||
await self.batch_get_note_comments(note_id_list)
|
||||
except Exception as ex:
|
||||
utils.logger.error(f"[BaiduTieBaCrawler.search] Search note list error, err: {ex}")
|
||||
utils.logger.error(
|
||||
f"[BaiduTieBaCrawler.search] Search keywords error, current page: {page}, current keyword: {keyword}, err: {ex}")
|
||||
break
|
||||
|
||||
async def fetch_creator_notes_detail(self, note_list: List[Dict]):
|
||||
async def get_specified_notes(self, note_id_list: List[str] = config.TIEBA_SPECIFIED_ID_LIST):
|
||||
"""
|
||||
Concurrently obtain the specified post list and save the data
|
||||
Get the information and comments of the specified post
|
||||
Args:
|
||||
note_id_list:
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
|
||||
task_list = [
|
||||
self.get_note_detail(
|
||||
note_id=post_item.get("note_id"),
|
||||
semaphore=semaphore
|
||||
)
|
||||
for post_item in note_list
|
||||
]
|
||||
|
||||
note_details = await asyncio.gather(*task_list)
|
||||
for note_detail in note_details:
|
||||
if note_detail:
|
||||
await tieba_store.update_tieba_note(note_detail)
|
||||
|
||||
async def get_specified_notes(self):
|
||||
"""Get the information and comments of the specified post"""
|
||||
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
|
||||
task_list = [
|
||||
self.get_note_detail(note_id=note_id, semaphore=semaphore) for note_id in config.TIEBA_SPECIFIED_ID_LIST
|
||||
self.get_note_detail_async_task(note_id=note_id, semaphore=semaphore) for note_id in note_id_list
|
||||
]
|
||||
note_details = await asyncio.gather(*task_list)
|
||||
for note_detail in note_details:
|
||||
|
|
@ -135,11 +119,20 @@ class TieBaCrawler(AbstractCrawler):
|
|||
await tieba_store.update_tieba_note(note_detail)
|
||||
await self.batch_get_note_comments(config.TIEBA_SPECIFIED_ID_LIST)
|
||||
|
||||
async def get_note_detail(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[Dict]:
|
||||
"""Get note detail"""
|
||||
async def get_note_detail_async_task(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[TiebaNote]:
|
||||
"""
|
||||
Get note detail
|
||||
Args:
|
||||
note_id: baidu tieba note id
|
||||
semaphore: asyncio semaphore
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
async with semaphore:
|
||||
try:
|
||||
note_detail: Dict = await self.tieba_client.get_note_by_id(note_id)
|
||||
utils.logger.info(f"[BaiduTieBaCrawler.get_note_detail] Begin get note detail, note_id: {note_id}")
|
||||
note_detail: TiebaNote = await self.tieba_client.get_note_by_id(note_id)
|
||||
if not note_detail:
|
||||
utils.logger.error(
|
||||
f"[BaiduTieBaCrawler.get_note_detail] Get note detail error, note_id: {note_id}")
|
||||
|
|
@ -153,23 +146,38 @@ class TieBaCrawler(AbstractCrawler):
|
|||
f"[BaiduTieBaCrawler.get_note_detail] have not fund note detail note_id:{note_id}, err: {ex}")
|
||||
return None
|
||||
|
||||
async def batch_get_note_comments(self, note_list: List[str]):
|
||||
"""Batch get note comments"""
|
||||
async def batch_get_note_comments(self, note_id_list: List[str]):
|
||||
"""
|
||||
Batch get note comments
|
||||
Args:
|
||||
note_id_list:
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
if not config.ENABLE_GET_COMMENTS:
|
||||
utils.logger.info(f"[BaiduTieBaCrawler.batch_get_note_comments] Crawling comment mode is not enabled")
|
||||
return
|
||||
|
||||
utils.logger.info(
|
||||
f"[BaiduTieBaCrawler.batch_get_note_comments] Begin batch get note comments, note list: {note_list}")
|
||||
f"[BaiduTieBaCrawler.batch_get_note_comments] Begin batch get note comments, note list: {note_id_list}")
|
||||
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
|
||||
task_list: List[Task] = []
|
||||
for note_id in note_list:
|
||||
task = asyncio.create_task(self.get_comments(note_id, semaphore), name=note_id)
|
||||
for note_id in note_id_list:
|
||||
task = asyncio.create_task(self.get_comments_async_task(note_id, semaphore), name=note_id)
|
||||
task_list.append(task)
|
||||
await asyncio.gather(*task_list)
|
||||
|
||||
async def get_comments(self, note_id: str, semaphore: asyncio.Semaphore):
|
||||
"""Get note comments with keyword filtering and quantity limitation"""
|
||||
async def get_comments_async_task(self, note_id: str, semaphore: asyncio.Semaphore):
|
||||
"""
|
||||
Get comments async task
|
||||
Args:
|
||||
note_id:
|
||||
semaphore:
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
async with semaphore:
|
||||
utils.logger.info(f"[BaiduTieBaCrawler.get_comments] Begin get note id comments {note_id}")
|
||||
await self.tieba_client.get_note_all_comments(
|
||||
|
|
@ -178,23 +186,6 @@ class TieBaCrawler(AbstractCrawler):
|
|||
callback=tieba_store.batch_update_tieba_note_comments
|
||||
)
|
||||
|
||||
async def create_tieba_client(self, ip_pool: ProxyIpPool) -> BaiduTieBaClient:
|
||||
"""
|
||||
Create tieba client
|
||||
Args:
|
||||
ip_pool:
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
"""Create tieba client"""
|
||||
utils.logger.info("[BaiduTieBaCrawler.create_tieba_client] Begin create baidutieba API client ...")
|
||||
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
|
||||
tieba_client_obj = BaiduTieBaClient(
|
||||
ip_pool=ip_pool,
|
||||
)
|
||||
return tieba_client_obj
|
||||
|
||||
async def launch_browser(
|
||||
self,
|
||||
chromium: BrowserType,
|
||||
|
|
@ -202,7 +193,17 @@ class TieBaCrawler(AbstractCrawler):
|
|||
user_agent: Optional[str],
|
||||
headless: bool = True
|
||||
) -> BrowserContext:
|
||||
"""Launch browser and create browser context"""
|
||||
"""
|
||||
Launch browser and create browser
|
||||
Args:
|
||||
chromium:
|
||||
playwright_proxy:
|
||||
user_agent:
|
||||
headless:
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
utils.logger.info("[BaiduTieBaCrawler.launch_browser] Begin create browser context ...")
|
||||
if config.SAVE_LOGIN_STATE:
|
||||
# feat issue #14
|
||||
|
|
@ -227,6 +228,10 @@ class TieBaCrawler(AbstractCrawler):
|
|||
return browser_context
|
||||
|
||||
async def close(self):
|
||||
"""Close browser context"""
|
||||
"""
|
||||
Close browser context
|
||||
Returns:
|
||||
|
||||
"""
|
||||
await self.browser_context.close()
|
||||
utils.logger.info("[BaiduTieBaCrawler.close] Browser context closed ...")
|
||||
|
|
|
|||
|
|
@ -1,18 +1,21 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
from typing import List, Dict
|
||||
import re
|
||||
from typing import List, Dict, Tuple
|
||||
|
||||
from parsel import Selector
|
||||
|
||||
from model.m_baidu_tieba import TiebaNote
|
||||
from constant import baidu_tieba as const
|
||||
|
||||
|
||||
class TieBaExtractor:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def extract_search_note_list(page_content: str) -> List[Dict]:
|
||||
def extract_search_note_list(page_content: str) -> List[TiebaNote]:
|
||||
"""
|
||||
提取贴吧帖子列表
|
||||
提取贴吧帖子列表，这里提取的关键词搜索结果页的数据，还缺少帖子的回复数和回复页等数据
|
||||
Args:
|
||||
page_content: 页面内容的HTML字符串
|
||||
|
||||
|
|
@ -21,33 +24,24 @@ class TieBaExtractor:
|
|||
"""
|
||||
xpath_selector = "//div[@class='s_post']"
|
||||
post_list = Selector(text=page_content).xpath(xpath_selector)
|
||||
result = []
|
||||
result: List[TiebaNote] = []
|
||||
for post in post_list:
|
||||
post_id = post.xpath(".//span[@class='p_title']/a/@data-tid").get(default='').strip()
|
||||
title = post.xpath(".//span[@class='p_title']/a/text()").get(default='').strip()
|
||||
link = post.xpath(".//span[@class='p_title']/a/@href").get(default='')
|
||||
description = post.xpath(".//div[@class='p_content']/text()").get(default='').strip()
|
||||
forum = post.xpath(".//a[@class='p_forum']/font/text()").get(default='').strip()
|
||||
forum_link = post.xpath(".//a[@class='p_forum']/@href").get(default='')
|
||||
author = post.xpath(".//a[starts-with(@href, '/home/main')]/font/text()").get(default='').strip()
|
||||
author_link = post.xpath(".//a[starts-with(@href, '/home/main')]/@href").get(default='')
|
||||
date = post.xpath(".//font[@class='p_green p_date']/text()").get(default='').strip()
|
||||
result.append({
|
||||
"note_id": post_id,
|
||||
"title": title,
|
||||
"desc": description,
|
||||
"note_url": link,
|
||||
"time": date,
|
||||
"tieba_name": forum,
|
||||
"tieba_link": forum_link,
|
||||
"nickname": author,
|
||||
"nickname_link": author_link,
|
||||
})
|
||||
|
||||
tieba_note = TiebaNote(
|
||||
note_id=post.xpath(".//span[@class='p_title']/a/@data-tid").get(default='').strip(),
|
||||
title=post.xpath(".//span[@class='p_title']/a/text()").get(default='').strip(),
|
||||
desc=post.xpath(".//div[@class='p_content']/text()").get(default='').strip(),
|
||||
note_url=const.TIEBA_URL + post.xpath(".//span[@class='p_title']/a/@href").get(default=''),
|
||||
user_nickname=post.xpath(".//a[starts-with(@href, '/home/main')]/font/text()").get(default='').strip(),
|
||||
user_link=const.TIEBA_URL + post.xpath(".//a[starts-with(@href, '/home/main')]/@href").get(default=''),
|
||||
tieba_name=post.xpath(".//a[@class='p_forum']/font/text()").get(default='').strip(),
|
||||
tieba_link=const.TIEBA_URL + post.xpath(".//a[@class='p_forum']/@href").get(default=''),
|
||||
publish_time=post.xpath(".//font[@class='p_green p_date']/text()").get(default='').strip(),
|
||||
)
|
||||
result.append(tieba_note)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def extract_note_detail(page_content: str) -> Dict:
|
||||
|
||||
def extract_note_detail(self, page_content: str) -> TiebaNote:
|
||||
"""
|
||||
提取贴吧帖子详情
|
||||
Args:
|
||||
|
|
@ -57,13 +51,33 @@ class TieBaExtractor:
|
|||
|
||||
"""
|
||||
content_selector = Selector(text=page_content)
|
||||
# 查看楼主的链接： only_view_author_link: / p / 9117905169?see_lz = 1
|
||||
only_view_author_link = content_selector.xpath("//*[@id='lzonly_cntn']/@href").get(default='').strip() #
|
||||
first_floor_selector = content_selector.xpath("//div[@class='p_postlist'][1]")
|
||||
only_view_author_link = content_selector.xpath("//*[@id='lzonly_cntn']/@href").get(default='').strip()
|
||||
note_id = only_view_author_link.split("?")[0].split("/")[-1]
|
||||
title = content_selector.xpath("//*[@id='j_core_title_wrap']/h3").get(default='').strip()
|
||||
desc = content_selector.xpath("//meta[@name='description']").get(default='').strip()
|
||||
note_url = f"/p/{note_id}"
|
||||
pass
|
||||
# 帖子回复数、回复页数
|
||||
thread_num_infos = content_selector.xpath(
|
||||
"//div[@id='thread_theme_5']//li[@class='l_reply_num']//span[@class='red']"
|
||||
)
|
||||
# IP地理位置、发表时间
|
||||
other_info_content = content_selector.xpath(".//div[@class='post-tail-wrap']").get(default="").strip()
|
||||
ip_location, publish_time = self.extract_ip_and_pub_time(other_info_content)
|
||||
note = TiebaNote(
|
||||
note_id=note_id,
|
||||
title=content_selector.xpath("//title/text()").get(default='').strip(),
|
||||
desc=content_selector.xpath("//meta[@name='description']/@content").get(default='').strip(),
|
||||
note_url=const.TIEBA_URL + f"/p/{note_id}",
|
||||
user_link=const.TIEBA_URL + first_floor_selector.xpath(".//a[@class='p_author_face ']/@href").get(default='').strip(),
|
||||
user_nickname=first_floor_selector.xpath(".//a[@class='p_author_name j_user_card']/text()").get(default='').strip(),
|
||||
user_avatar=first_floor_selector.xpath(".//a[@class='p_author_face ']/img/@src").get(default='').strip(),
|
||||
tieba_name=content_selector.xpath("//a[@class='card_title_fname']/text()").get(default='').strip(),
|
||||
tieba_link=const.TIEBA_URL + content_selector.xpath("//a[@class='card_title_fname']/@href").get(default=''),
|
||||
ip_location=ip_location,
|
||||
publish_time=publish_time,
|
||||
total_replay_num=thread_num_infos[0].xpath("./text()").get(default='').strip(),
|
||||
total_replay_page=thread_num_infos[1].xpath("./text()").get(default='').strip(),
|
||||
)
|
||||
note.title = note.title.replace(f"【{note.tieba_name}】_百度贴吧", "")
|
||||
return note
|
||||
|
||||
@staticmethod
|
||||
def extract_tieba_note_comments(page_content: str) -> List[Dict]:
|
||||
|
|
@ -93,12 +107,40 @@ class TieBaExtractor:
|
|||
"time": date,
|
||||
})
|
||||
|
||||
@staticmethod
|
||||
def extract_ip_and_pub_time(html_content: str) -> Tuple[str, str]:
|
||||
"""
|
||||
提取IP位置和发布时间
|
||||
Args:
|
||||
html_content:
|
||||
|
||||
Returns:
|
||||
|
||||
if __name__ == '__main__':
|
||||
"""
|
||||
pattern_ip = re.compile(r'IP属地:(\S+)</span>')
|
||||
pattern_pub_time = re.compile(r'<span class="tail-info">(\d{4}-\d{2}-\d{2} \d{2}:\d{2})</span>')
|
||||
ip_match = pattern_ip.search(html_content)
|
||||
time_match = pattern_pub_time.search(html_content)
|
||||
ip = ip_match.group(1) if ip_match else ""
|
||||
pub_time = time_match.group(1) if time_match else ""
|
||||
return ip, pub_time
|
||||
|
||||
def test_extract_search_note_list():
|
||||
with open("test_data/search_keyword_notes.html", "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
extractor = TieBaExtractor()
|
||||
_result = extractor.extract_search_note_list(content)
|
||||
print(_result)
|
||||
print(f"Total: {len(_result)}")
|
||||
result = extractor.extract_search_note_list(content)
|
||||
print(result)
|
||||
|
||||
|
||||
def test_extract_note_detail():
|
||||
with open("test_data/note_detail.html", "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
extractor = TieBaExtractor()
|
||||
result = extractor.extract_note_detail(content)
|
||||
print(result.model_dump())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_extract_search_note_list()
|
||||
test_extract_note_detail()
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class TiebaNote(BaseModel):
|
||||
note_id: str = Field(..., description="帖子ID")
|
||||
title: str = Field(..., description="帖子标题")
|
||||
desc: str = Field(default="", description="帖子描述")
|
||||
note_url: str = Field(..., description="帖子链接")
|
||||
publish_time: str = Field(default="", description="发布时间")
|
||||
user_link: str = Field(default="", description="用户主页链接")
|
||||
user_nickname: str = Field(default="", description="用户昵称")
|
||||
user_avatar: str = Field(default="", description="用户头像地址")
|
||||
tieba_name: str = Field(..., description="贴吧名称")
|
||||
tieba_link: str = Field(..., description="贴吧链接")
|
||||
total_replay_num: int = Field(default=0, description="回复总数")
|
||||
total_replay_page: int = Field(default=0, description="回复总页数")
|
||||
ip_location: Optional[str] = Field(default="", description="IP地理位置")
|
||||
|
|
@ -0,0 +1 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
|
@ -0,0 +1 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
|
@ -0,0 +1 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
|
@ -0,0 +1 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
|
@ -349,29 +349,26 @@ ALTER TABLE `bilibili_video_comment`
|
|||
ALTER TABLE `weibo_note_comment`
|
||||
ADD COLUMN `parent_comment_id` VARCHAR(64) DEFAULT NULL COMMENT '父评论ID';
|
||||
|
||||
SET
|
||||
FOREIGN_KEY_CHECKS = 1;
|
||||
|
||||
|
||||
DROP TABLE IF EXISTS `tieba_note`;
|
||||
CREATE TABLE `tieba_note`
|
||||
CREATE TABLE tieba_note
|
||||
(
|
||||
`id` int NOT NULL AUTO_INCREMENT COMMENT '自增ID',
|
||||
`note_id` varchar(64) NOT NULL COMMENT '帖子ID',
|
||||
`title` varchar(255) DEFAULT NULL COMMENT '笔记标题',
|
||||
`desc` longtext COMMENT '笔记描述',
|
||||
`time` varchar NOT NULL COMMENT '笔记发布时间',
|
||||
`note_url` varchar(255) DEFAULT NULL COMMENT '笔记详情页的URL',
|
||||
`nickname` varchar(64) DEFAULT NULL COMMENT '用户昵称',
|
||||
`nickname_link` varchar(255) DEFAULT NULL COMMENT '用户主页地址',
|
||||
`tieba_name` varchar(255) DEFAULT NULL COMMENT '贴吧名称',
|
||||
`tieba_link` varchar(255) DEFAULT NULL COMMENT '贴吧链接地址',
|
||||
`avatar` varchar(255) DEFAULT NULL COMMENT '用户头像地址',
|
||||
`ip_location` varchar(255) DEFAULT NULL COMMENT '评论时的IP地址',
|
||||
`add_ts` bigint NOT NULL COMMENT '记录添加时间戳',
|
||||
`last_modify_ts` bigint NOT NULL COMMENT '记录最后修改时间戳',
|
||||
`comment_count` varchar(16) DEFAULT NULL COMMENT '笔记评论数',
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `idx_tieba_note_id` (`note_id`),
|
||||
KEY `idx_tieba_note_time` (`time`)
|
||||
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
note_id VARCHAR(644) NOT NULL COMMENT '帖子ID',
|
||||
title VARCHAR(255) NOT NULL COMMENT '帖子标题',
|
||||
`desc` TEXT COMMENT '帖子描述',
|
||||
note_url VARCHAR(255) NOT NULL COMMENT '帖子链接',
|
||||
publish_time VARCHAR(255) NOT NULL COMMENT '发布时间',
|
||||
user_link VARCHAR(255) NOT NULL COMMENT '用户主页链接',
|
||||
user_nickname VARCHAR(255) NOT NULL COMMENT '用户昵称',
|
||||
user_avatar VARCHAR(255) NOT NULL COMMENT '用户头像地址',
|
||||
tieba_name VARCHAR(255) NOT NULL COMMENT '贴吧名称',
|
||||
tieba_link VARCHAR(255) NOT NULL COMMENT '贴吧链接',
|
||||
total_replay_num INT DEFAULT 0 COMMENT '帖子回复总数',
|
||||
total_replay_page INT DEFAULT 0 COMMENT '帖子回复总页数',
|
||||
ip_location VARCHAR(255) DEFAULT '' COMMENT 'IP地理位置',
|
||||
add_ts BIGINT NOT NULL COMMENT '添加时间戳',
|
||||
last_modify_ts BIGINT NOT NULL COMMENT '最后修改时间戳',
|
||||
KEY `idx_tieba_note_note_id` (`note_id`),
|
||||
KEY `idx_tieba_note_publish_time` (`publish_time`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='贴吧帖子表';
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from typing import List
|
||||
|
||||
from model.m_baidu_tieba import TiebaNote
|
||||
from . import tieba_store_impl
|
||||
from .tieba_store_impl import *
|
||||
|
||||
|
|
@ -21,24 +22,20 @@ class TieBaStoreFactory:
|
|||
return store_class()
|
||||
|
||||
|
||||
async def update_tieba_note(note_item: Dict):
|
||||
tieba_url = "https://tieba.baidu.com"
|
||||
note_id = note_item.get("note_id")
|
||||
local_db_item = {
|
||||
"note_id": note_id,
|
||||
"title": note_item.get("title") or note_item.get("desc", "")[:255],
|
||||
"desc": note_item.get("desc", ""),
|
||||
"note_url": tieba_url + note_item.get("note_url"),
|
||||
"time": note_item.get("time"),
|
||||
"tieba_name": note_item.get("tieba_name"),
|
||||
"tieba_link": tieba_url + note_item.get("tieba_link", ""),
|
||||
"nickname": note_item.get("nickname"),
|
||||
"nickname_link": tieba_url + note_item.get("nickname_link", ""),
|
||||
"ip_location": note_item.get("ip_location", ""),
|
||||
"last_modify_ts": utils.get_current_timestamp(),
|
||||
}
|
||||
utils.logger.info(f"[store.tieba.update_tieba_note] tieba note: {local_db_item}")
|
||||
await TieBaStoreFactory.create_store().store_content(local_db_item)
|
||||
async def update_tieba_note(note_item: TiebaNote):
|
||||
"""
|
||||
Add or Update tieba note
|
||||
Args:
|
||||
note_item:
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
save_note_item = note_item.model_dump()
|
||||
save_note_item.update({"last_modify_ts": utils.get_current_timestamp()})
|
||||
utils.logger.info(f"[store.tieba.update_tieba_note] tieba note: {save_note_item}")
|
||||
|
||||
await TieBaStoreFactory.create_store().store_content(save_note_item)
|
||||
|
||||
|
||||
async def batch_update_tieba_note_comments(note_id: str, comments: List[Dict]):
|
||||
|
|
|
|||
Loading…
Reference in New Issue