feat: 贴吧搜索重构

constant/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

constant/baidu_tieba.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+TIEBA_URL = 'https://tieba.baidu.com'

db.py

@@ -85,7 +85,7 @@ async def init_table_schema():
     utils.logger.info("[init_table_schema] begin init mysql table schema ...")
     await init_mediacrawler_db()
     async_db_obj: AsyncMysqlDB = media_crawler_db_var.get()
-    async with aiofiles.open("schema/tables.sql", mode="r") as f:
+    async with aiofiles.open("schema/tables.sql", mode="r", encoding="utf-8") as f:
         schema_sql = await f.read()
         await async_db_obj.execute(schema_sql)
         utils.logger.info("[init_table_schema] mediacrawler table schema init successful")

media_platform/tieba/client.py

@@ -10,6 +10,7 @@ from tenacity import (RetryError, retry, stop_after_attempt,
                       wait_fixed)
 
 from base.base_crawler import AbstractApiClient
+from model.m_baidu_tieba import TiebaNote
 from proxy.proxy_ip_pool import ProxyIpPool
 from tools import utils
 
@@ -98,6 +99,7 @@ class BaiduTieBaClient(AbstractApiClient):
                 return res
 
             utils.logger.error(f"[BaiduTieBaClient.get] 达到了最大重试次数，请尝试更换新的IP代理: {e}")
+            raise e
 
     async def post(self, uri: str, data: dict, **kwargs) -> Dict:
         """
@@ -152,7 +154,7 @@ class BaiduTieBaClient(AbstractApiClient):
             sort: SearchSortType = SearchSortType.TIME_DESC,
             note_type: SearchNoteType = SearchNoteType.FIXED_THREAD,
             random_sleep: bool = True
-    ) -> List[Dict]:
+    ) -> List[TiebaNote]:
         """
         根据关键词搜索贴吧帖子
         Args:
@@ -180,7 +182,7 @@ class BaiduTieBaClient(AbstractApiClient):
             random.randint(1, 5)
         return self._page_extractor.extract_search_note_list(page_content)
 
-    async def get_note_by_id(self, note_id: str) -> Dict:
+    async def get_note_by_id(self, note_id: str) -> TiebaNote:
         """
         根据帖子ID获取帖子详情
         Args:
@@ -192,8 +194,6 @@ class BaiduTieBaClient(AbstractApiClient):
         uri = f"/p/{note_id}"
         page_content = await self.get(uri, return_ori_content=True)
         return self._page_extractor.extract_note_detail(page_content)
-        # todo impl it
-        return {}
 
     async def get_note_all_comments(self, note_id: str, crawl_interval: float = 1.0,
                                     callback: Optional[Callable] = None) -> List[Dict]:
@@ -229,7 +229,7 @@ class BaiduTieBaClient(AbstractApiClient):
         return result
 
     async def get_comments_all_sub_comments(self, comments: List[Dict], crawl_interval: float = 1.0,
-                                           callback: Optional[Callable] = None) -> List[Dict]:
+                                            callback: Optional[Callable] = None) -> List[Dict]:
         """
         获取指定评论下的所有子评论
         Args:

media_platform/tieba/core.py

@@ -9,7 +9,8 @@ from playwright.async_api import (BrowserContext, BrowserType, Page,
 
 import config
 from base.base_crawler import AbstractCrawler
-from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool, ProxyIpPool
+from model.m_baidu_tieba import TiebaNote
+from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
 from store import tieba as tieba_store
 from tools import utils
 from tools.crawler_util import format_proxy_info
@@ -66,8 +67,7 @@ class TieBaCrawler(AbstractCrawler):
         Returns:
 
         """
-
+        utils.logger.info("[BaiduTieBaCrawler.search] Begin search baidu tieba keywords")
-        utils.logger.info("[BaiduTieBaCrawler.search] Begin search baidutieba keywords")
         tieba_limit_count = 10  # tieba limit page fixed value
         if config.CRAWLER_MAX_NOTES_COUNT < tieba_limit_count:
             config.CRAWLER_MAX_NOTES_COUNT = tieba_limit_count
@@ -82,52 +82,36 @@ class TieBaCrawler(AbstractCrawler):
                     continue
                 try:
                     utils.logger.info(f"[BaiduTieBaCrawler.search] search tieba keyword: {keyword}, page: {page}")
-                    note_id_list: List[str] = []
+                    notes_list: List[TiebaNote] = await self.tieba_client.get_notes_by_keyword(
-                    notes_list_res = await self.tieba_client.get_notes_by_keyword(
                         keyword=keyword,
                         page=page,
                         page_size=tieba_limit_count,
                         sort=SearchSortType.TIME_DESC,
                         note_type=SearchNoteType.FIXED_THREAD
                     )
-                    utils.logger.info(f"[BaiduTieBaCrawler.search] Search notes res:{notes_list_res}")
+                    if not notes_list:
-                    if not notes_list_res:
+                        utils.logger.info(f"[BaiduTieBaCrawler.search] Search note list is empty")
                         break
-
+                    utils.logger.info(f"[BaiduTieBaCrawler.search] Note List: {notes_list}")
-                    for note_detail in notes_list_res:
+                    await self.get_specified_notes(note_id_list=[note_detail.note_id for note_detail in notes_list])
-                        if note_detail:
-                            await tieba_store.update_tieba_note(note_detail)
-                            note_id_list.append(note_detail.get("note_id"))
                     page += 1
-                    utils.logger.info(f"[BaiduTieBaCrawler.search] Note details: {notes_list_res}")
-                    await self.batch_get_note_comments(note_id_list)
                 except Exception as ex:
-                    utils.logger.error(f"[BaiduTieBaCrawler.search] Search note list error, err: {ex}")
+                    utils.logger.error(
+                        f"[BaiduTieBaCrawler.search] Search keywords error, current page: {page}, current keyword: {keyword}, err: {ex}")
                     break
 
-    async def fetch_creator_notes_detail(self, note_list: List[Dict]):
+    async def get_specified_notes(self, note_id_list: List[str] = config.TIEBA_SPECIFIED_ID_LIST):
         """
-        Concurrently obtain the specified post list and save the data
+        Get the information and comments of the specified post
+        Args:
+            note_id_list:
+
+        Returns:
+
         """
         semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
         task_list = [
-            self.get_note_detail(
+            self.get_note_detail_async_task(note_id=note_id, semaphore=semaphore) for note_id in note_id_list
-                note_id=post_item.get("note_id"),
-                semaphore=semaphore
-            )
-            for post_item in note_list
-        ]
-
-        note_details = await asyncio.gather(*task_list)
-        for note_detail in note_details:
-            if note_detail:
-                await tieba_store.update_tieba_note(note_detail)
-
-    async def get_specified_notes(self):
-        """Get the information and comments of the specified post"""
-        semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
-        task_list = [
-            self.get_note_detail(note_id=note_id, semaphore=semaphore) for note_id in config.TIEBA_SPECIFIED_ID_LIST
         ]
         note_details = await asyncio.gather(*task_list)
         for note_detail in note_details:
@@ -135,11 +119,20 @@ class TieBaCrawler(AbstractCrawler):
                 await tieba_store.update_tieba_note(note_detail)
         await self.batch_get_note_comments(config.TIEBA_SPECIFIED_ID_LIST)
 
-    async def get_note_detail(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[Dict]:
+    async def get_note_detail_async_task(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[TiebaNote]:
-        """Get note detail"""
+        """
+        Get note detail
+        Args:
+            note_id: baidu tieba note id
+            semaphore: asyncio semaphore
+
+        Returns:
+
+        """
         async with semaphore:
             try:
-                note_detail: Dict = await self.tieba_client.get_note_by_id(note_id)
+                utils.logger.info(f"[BaiduTieBaCrawler.get_note_detail] Begin get note detail, note_id: {note_id}")
+                note_detail: TiebaNote = await self.tieba_client.get_note_by_id(note_id)
                 if not note_detail:
                     utils.logger.error(
                         f"[BaiduTieBaCrawler.get_note_detail] Get note detail error, note_id: {note_id}")
@@ -153,23 +146,38 @@ class TieBaCrawler(AbstractCrawler):
                     f"[BaiduTieBaCrawler.get_note_detail] have not fund note detail note_id:{note_id}, err: {ex}")
                 return None
 
-    async def batch_get_note_comments(self, note_list: List[str]):
+    async def batch_get_note_comments(self, note_id_list: List[str]):
-        """Batch get note comments"""
+        """
+        Batch get note comments
+        Args:
+            note_id_list:
+
+        Returns:
+
+        """
         if not config.ENABLE_GET_COMMENTS:
             utils.logger.info(f"[BaiduTieBaCrawler.batch_get_note_comments] Crawling comment mode is not enabled")
             return
 
         utils.logger.info(
-            f"[BaiduTieBaCrawler.batch_get_note_comments] Begin batch get note comments, note list: {note_list}")
+            f"[BaiduTieBaCrawler.batch_get_note_comments] Begin batch get note comments, note list: {note_id_list}")
         semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
         task_list: List[Task] = []
-        for note_id in note_list:
+        for note_id in note_id_list:
-            task = asyncio.create_task(self.get_comments(note_id, semaphore), name=note_id)
+            task = asyncio.create_task(self.get_comments_async_task(note_id, semaphore), name=note_id)
             task_list.append(task)
         await asyncio.gather(*task_list)
 
-    async def get_comments(self, note_id: str, semaphore: asyncio.Semaphore):
+    async def get_comments_async_task(self, note_id: str, semaphore: asyncio.Semaphore):
-        """Get note comments with keyword filtering and quantity limitation"""
+        """
+        Get comments async task
+        Args:
+            note_id:
+            semaphore:
+
+        Returns:
+
+        """
         async with semaphore:
             utils.logger.info(f"[BaiduTieBaCrawler.get_comments] Begin get note id comments {note_id}")
             await self.tieba_client.get_note_all_comments(
@@ -178,23 +186,6 @@ class TieBaCrawler(AbstractCrawler):
                 callback=tieba_store.batch_update_tieba_note_comments
             )
 
-    async def create_tieba_client(self, ip_pool: ProxyIpPool) -> BaiduTieBaClient:
-        """
-        Create tieba client
-        Args:
-            ip_pool:
-
-        Returns:
-
-        """
-        """Create tieba client"""
-        utils.logger.info("[BaiduTieBaCrawler.create_tieba_client] Begin create baidutieba API client ...")
-        cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
-        tieba_client_obj = BaiduTieBaClient(
-            ip_pool=ip_pool,
-        )
-        return tieba_client_obj
-
     async def launch_browser(
             self,
             chromium: BrowserType,
@@ -202,7 +193,17 @@ class TieBaCrawler(AbstractCrawler):
             user_agent: Optional[str],
             headless: bool = True
     ) -> BrowserContext:
-        """Launch browser and create browser context"""
+        """
+        Launch browser and create browser
+        Args:
+            chromium:
+            playwright_proxy:
+            user_agent:
+            headless:
+
+        Returns:
+
+        """
         utils.logger.info("[BaiduTieBaCrawler.launch_browser] Begin create browser context ...")
         if config.SAVE_LOGIN_STATE:
             # feat issue #14
@@ -227,6 +228,10 @@ class TieBaCrawler(AbstractCrawler):
             return browser_context
 
     async def close(self):
-        """Close browser context"""
+        """
+        Close browser context
+        Returns:
+
+        """
         await self.browser_context.close()
         utils.logger.info("[BaiduTieBaCrawler.close] Browser context closed ...")

media_platform/tieba/help.py

@@ -1,18 +1,21 @@
 # -*- coding: utf-8 -*-
-
+import re
-from typing import List, Dict
+from typing import List, Dict, Tuple
 
 from parsel import Selector
 
+from model.m_baidu_tieba import TiebaNote
+from constant import baidu_tieba as const
+
 
 class TieBaExtractor:
     def __init__(self):
         pass
 
     @staticmethod
-    def extract_search_note_list(page_content: str) -> List[Dict]:
+    def extract_search_note_list(page_content: str) -> List[TiebaNote]:
         """
-        提取贴吧帖子列表
+        提取贴吧帖子列表，这里提取的关键词搜索结果页的数据，还缺少帖子的回复数和回复页等数据
         Args:
             page_content: 页面内容的HTML字符串
 
@@ -21,33 +24,24 @@ class TieBaExtractor:
         """
         xpath_selector = "//div[@class='s_post']"
         post_list = Selector(text=page_content).xpath(xpath_selector)
-        result = []
+        result: List[TiebaNote] = []
         for post in post_list:
-            post_id = post.xpath(".//span[@class='p_title']/a/@data-tid").get(default='').strip()
+            tieba_note = TiebaNote(
-            title = post.xpath(".//span[@class='p_title']/a/text()").get(default='').strip()
+                note_id=post.xpath(".//span[@class='p_title']/a/@data-tid").get(default='').strip(),
-            link = post.xpath(".//span[@class='p_title']/a/@href").get(default='')
+                title=post.xpath(".//span[@class='p_title']/a/text()").get(default='').strip(),
-            description = post.xpath(".//div[@class='p_content']/text()").get(default='').strip()
+                desc=post.xpath(".//div[@class='p_content']/text()").get(default='').strip(),
-            forum = post.xpath(".//a[@class='p_forum']/font/text()").get(default='').strip()
+                note_url=const.TIEBA_URL + post.xpath(".//span[@class='p_title']/a/@href").get(default=''),
-            forum_link = post.xpath(".//a[@class='p_forum']/@href").get(default='')
+                user_nickname=post.xpath(".//a[starts-with(@href, '/home/main')]/font/text()").get(default='').strip(),
-            author = post.xpath(".//a[starts-with(@href, '/home/main')]/font/text()").get(default='').strip()
+                user_link=const.TIEBA_URL + post.xpath(".//a[starts-with(@href, '/home/main')]/@href").get(default=''),
-            author_link = post.xpath(".//a[starts-with(@href, '/home/main')]/@href").get(default='')
+                tieba_name=post.xpath(".//a[@class='p_forum']/font/text()").get(default='').strip(),
-            date = post.xpath(".//font[@class='p_green p_date']/text()").get(default='').strip()
+                tieba_link=const.TIEBA_URL + post.xpath(".//a[@class='p_forum']/@href").get(default=''),
-            result.append({
+                publish_time=post.xpath(".//font[@class='p_green p_date']/text()").get(default='').strip(),
-                "note_id": post_id,
+            )
-                "title": title,
+            result.append(tieba_note)
-                "desc": description,
-                "note_url": link,
-                "time": date,
-                "tieba_name": forum,
-                "tieba_link": forum_link,
-                "nickname": author,
-                "nickname_link": author_link,
-            })
-
         return result
 
-    @staticmethod
+
-    def extract_note_detail(page_content: str) -> Dict:
+    def extract_note_detail(self, page_content: str) -> TiebaNote:
         """
         提取贴吧帖子详情
         Args:
@@ -57,13 +51,33 @@ class TieBaExtractor:
 
         """
         content_selector = Selector(text=page_content)
-        # 查看楼主的链接： only_view_author_link: / p / 9117905169?see_lz = 1
+        first_floor_selector = content_selector.xpath("//div[@class='p_postlist'][1]")
-        only_view_author_link = content_selector.xpath("//*[@id='lzonly_cntn']/@href").get(default='').strip() #
+        only_view_author_link = content_selector.xpath("//*[@id='lzonly_cntn']/@href").get(default='').strip()
         note_id = only_view_author_link.split("?")[0].split("/")[-1]
-        title = content_selector.xpath("//*[@id='j_core_title_wrap']/h3").get(default='').strip()
+        # 帖子回复数、回复页数
-        desc = content_selector.xpath("//meta[@name='description']").get(default='').strip()
+        thread_num_infos = content_selector.xpath(
-        note_url = f"/p/{note_id}"
+            "//div[@id='thread_theme_5']//li[@class='l_reply_num']//span[@class='red']"
-        pass
+        )
+        # IP地理位置、发表时间
+        other_info_content = content_selector.xpath(".//div[@class='post-tail-wrap']").get(default="").strip()
+        ip_location, publish_time = self.extract_ip_and_pub_time(other_info_content)
+        note = TiebaNote(
+            note_id=note_id,
+            title=content_selector.xpath("//title/text()").get(default='').strip(),
+            desc=content_selector.xpath("//meta[@name='description']/@content").get(default='').strip(),
+            note_url=const.TIEBA_URL + f"/p/{note_id}",
+            user_link=const.TIEBA_URL + first_floor_selector.xpath(".//a[@class='p_author_face ']/@href").get(default='').strip(),
+            user_nickname=first_floor_selector.xpath(".//a[@class='p_author_name j_user_card']/text()").get(default='').strip(),
+            user_avatar=first_floor_selector.xpath(".//a[@class='p_author_face ']/img/@src").get(default='').strip(),
+            tieba_name=content_selector.xpath("//a[@class='card_title_fname']/text()").get(default='').strip(),
+            tieba_link=const.TIEBA_URL + content_selector.xpath("//a[@class='card_title_fname']/@href").get(default=''),
+            ip_location=ip_location,
+            publish_time=publish_time,
+            total_replay_num=thread_num_infos[0].xpath("./text()").get(default='').strip(),
+            total_replay_page=thread_num_infos[1].xpath("./text()").get(default='').strip(),
+        )
+        note.title = note.title.replace(f"【{note.tieba_name}】_百度贴吧", "")
+        return note
 
     @staticmethod
     def extract_tieba_note_comments(page_content: str) -> List[Dict]:
@@ -93,12 +107,40 @@ class TieBaExtractor:
                 "time": date,
             })
 
+    @staticmethod
+    def extract_ip_and_pub_time(html_content: str) -> Tuple[str, str]:
+        """
+        提取IP位置和发布时间
+        Args:
+            html_content:
 
+        Returns:
 
-if __name__ == '__main__':
+        """
+        pattern_ip = re.compile(r'IP属地:(\S+)</span>')
+        pattern_pub_time = re.compile(r'<span class="tail-info">(\d{4}-\d{2}-\d{2} \d{2}:\d{2})</span>')
+        ip_match = pattern_ip.search(html_content)
+        time_match = pattern_pub_time.search(html_content)
+        ip = ip_match.group(1) if ip_match else ""
+        pub_time = time_match.group(1) if time_match else ""
+        return ip, pub_time
+
+def test_extract_search_note_list():
     with open("test_data/search_keyword_notes.html", "r", encoding="utf-8") as f:
         content = f.read()
         extractor = TieBaExtractor()
-        _result = extractor.extract_search_note_list(content)
+        result = extractor.extract_search_note_list(content)
-        print(_result)
+        print(result)
-        print(f"Total: {len(_result)}")
+
+
+def test_extract_note_detail():
+    with open("test_data/note_detail.html", "r", encoding="utf-8") as f:
+        content = f.read()
+        extractor = TieBaExtractor()
+        result = extractor.extract_note_detail(content)
+        print(result.model_dump())
+
+
+if __name__ == '__main__':
+    test_extract_search_note_list()
+    test_extract_note_detail()

model/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

model/m_baidu_tieba.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from typing import Optional
+from pydantic import BaseModel, Field
+
+
+class TiebaNote(BaseModel):
+    note_id: str = Field(..., description="帖子ID")
+    title: str = Field(..., description="帖子标题")
+    desc: str = Field(default="", description="帖子描述")
+    note_url: str = Field(..., description="帖子链接")
+    publish_time: str = Field(default="", description="发布时间")
+    user_link: str = Field(default="", description="用户主页链接")
+    user_nickname: str = Field(default="", description="用户昵称")
+    user_avatar: str = Field(default="", description="用户头像地址")
+    tieba_name: str = Field(..., description="贴吧名称")
+    tieba_link: str = Field(..., description="贴吧链接")
+    total_replay_num: int = Field(default=0, description="回复总数")
+    total_replay_page: int = Field(default=0, description="回复总页数")
+    ip_location: Optional[str] = Field(default="", description="IP地理位置")

model/m_douyin.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

model/m_kuaishou.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

model/m_weibo.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

model/m_xiaohongshu.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

schema/tables.sql

@@ -349,29 +349,26 @@ ALTER TABLE `bilibili_video_comment`
 ALTER TABLE `weibo_note_comment`
     ADD COLUMN `parent_comment_id` VARCHAR(64) DEFAULT NULL COMMENT '父评论ID';
 
-SET
-FOREIGN_KEY_CHECKS = 1;
-
 
 DROP TABLE IF EXISTS `tieba_note`;
-CREATE TABLE `tieba_note`
+CREATE TABLE tieba_note
 (
-    `id`             int         NOT NULL AUTO_INCREMENT COMMENT '自增ID',
+    id                BIGINT AUTO_INCREMENT PRIMARY KEY,
-    `note_id`        varchar(64) NOT NULL COMMENT '帖子ID',
+    note_id           VARCHAR(644) NOT NULL COMMENT '帖子ID',
-    `title`          varchar(255) DEFAULT NULL COMMENT '笔记标题',
+    title             VARCHAR(255) NOT NULL COMMENT '帖子标题',
-    `desc`           longtext COMMENT '笔记描述',
+    `desc`            TEXT COMMENT '帖子描述',
-    `time`           varchar     NOT NULL COMMENT '笔记发布时间',
+    note_url          VARCHAR(255) NOT NULL COMMENT '帖子链接',
-    `note_url`       varchar(255) DEFAULT NULL COMMENT '笔记详情页的URL',
+    publish_time      VARCHAR(255) NOT NULL COMMENT '发布时间',
-    `nickname`       varchar(64)  DEFAULT NULL COMMENT '用户昵称',
+    user_link         VARCHAR(255) NOT NULL COMMENT '用户主页链接',
-    `nickname_link`  varchar(255) DEFAULT NULL COMMENT '用户主页地址',
+    user_nickname     VARCHAR(255) NOT NULL COMMENT '用户昵称',
-    `tieba_name`     varchar(255) DEFAULT NULL COMMENT '贴吧名称',
+    user_avatar       VARCHAR(255) NOT NULL COMMENT '用户头像地址',
-    `tieba_link`     varchar(255) DEFAULT NULL COMMENT '贴吧链接地址',
+    tieba_name        VARCHAR(255) NOT NULL COMMENT '贴吧名称',
-    `avatar`         varchar(255) DEFAULT NULL COMMENT '用户头像地址',
+    tieba_link        VARCHAR(255) NOT NULL COMMENT '贴吧链接',
-    `ip_location`    varchar(255) DEFAULT NULL COMMENT '评论时的IP地址',
+    total_replay_num  INT          DEFAULT 0 COMMENT '帖子回复总数',
-    `add_ts`         bigint      NOT NULL COMMENT '记录添加时间戳',
+    total_replay_page INT          DEFAULT 0 COMMENT '帖子回复总页数',
-    `last_modify_ts` bigint      NOT NULL COMMENT '记录最后修改时间戳',
+    ip_location       VARCHAR(255) DEFAULT '' COMMENT 'IP地理位置',
-    `comment_count`  varchar(16)  DEFAULT NULL COMMENT '笔记评论数',
+    add_ts            BIGINT       NOT NULL COMMENT '添加时间戳',
-    PRIMARY KEY (`id`),
+    last_modify_ts    BIGINT       NOT NULL COMMENT '最后修改时间戳',
-    KEY              `idx_tieba_note_id` (`note_id`),
+    KEY               `idx_tieba_note_note_id` (`note_id`),
-    KEY              `idx_tieba_note_time` (`time`)
+    KEY               `idx_tieba_note_publish_time` (`publish_time`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='贴吧帖子表';

store/tieba/__init__.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from typing import List
 
+from model.m_baidu_tieba import TiebaNote
 from . import tieba_store_impl
 from .tieba_store_impl import *
 
@@ -21,24 +22,20 @@ class TieBaStoreFactory:
         return store_class()
 
 
-async def update_tieba_note(note_item: Dict):
+async def update_tieba_note(note_item: TiebaNote):
-    tieba_url = "https://tieba.baidu.com"
+    """
-    note_id = note_item.get("note_id")
+    Add or Update tieba note
-    local_db_item = {
+    Args:
-        "note_id": note_id,
+        note_item:
-        "title": note_item.get("title") or note_item.get("desc", "")[:255],
+
-        "desc": note_item.get("desc", ""),
+    Returns:
-        "note_url": tieba_url + note_item.get("note_url"),
+
-        "time": note_item.get("time"),
+    """
-        "tieba_name": note_item.get("tieba_name"),
+    save_note_item = note_item.model_dump()
-        "tieba_link": tieba_url + note_item.get("tieba_link", ""),
+    save_note_item.update({"last_modify_ts": utils.get_current_timestamp()})
-        "nickname": note_item.get("nickname"),
+    utils.logger.info(f"[store.tieba.update_tieba_note] tieba note: {save_note_item}")
-        "nickname_link": tieba_url + note_item.get("nickname_link", ""),
+
-        "ip_location": note_item.get("ip_location", ""),
+    await TieBaStoreFactory.create_store().store_content(save_note_item)
-        "last_modify_ts": utils.get_current_timestamp(),
-    }
-    utils.logger.info(f"[store.tieba.update_tieba_note] tieba note: {local_db_item}")
-    await TieBaStoreFactory.create_store().store_content(local_db_item)
 
 
 async def batch_update_tieba_note_comments(note_id: str, comments: List[Dict]):