diff --git a/store/bilibili/__init__.py b/store/bilibili/__init__.py index a600ecb..ed6f5ac 100644 --- a/store/bilibili/__init__.py +++ b/store/bilibili/__init__.py @@ -7,7 +7,6 @@ from typing import List import config -from .bilibili_store_db_types import * from .bilibili_store_impl import * diff --git a/store/bilibili/bilibili_store_db_types.py b/store/bilibili/bilibili_store_db_types.py deleted file mode 100644 index f3bc117..0000000 --- a/store/bilibili/bilibili_store_db_types.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- -# @Author : relakkes@gmail.com -# @Time : 2024/1/14 19:34 -# @Desc : B站存储到DB的模型类集合 - -from tortoise import fields -from tortoise.models import Model - - -class BilibiliBaseModel(Model): - id = fields.IntField(pk=True, autoincrement=True, description="自增ID") - user_id = fields.CharField(null=True, max_length=64, description="用户ID") - nickname = fields.CharField(null=True, max_length=64, description="用户昵称") - avatar = fields.CharField(null=True, max_length=255, description="用户头像地址") - add_ts = fields.BigIntField(description="记录添加时间戳") - last_modify_ts = fields.BigIntField(description="记录最后修改时间戳") - - class Meta: - abstract = True - - -class BilibiliVideo(BilibiliBaseModel): - video_id = fields.CharField(max_length=64, index=True, description="视频ID") - video_type = fields.CharField(max_length=16, description="视频类型") - title = fields.CharField(null=True, max_length=500, description="视频标题") - desc = fields.TextField(null=True, description="视频描述") - create_time = fields.BigIntField(description="视频发布时间戳", index=True) - liked_count = fields.CharField(null=True, max_length=16, description="视频点赞数") - video_play_count = fields.CharField(null=True, max_length=16, description="视频播放数量") - video_danmaku = fields.CharField(null=True, max_length=16, description="视频弹幕数量") - video_comment = fields.CharField(null=True, max_length=16, description="视频评论数量") - video_url = fields.CharField(null=True, max_length=512, description="视频详情URL") - video_cover_url = fields.CharField(null=True, max_length=512, description="视频封面图 URL") - - class Meta: - table = "bilibili_video" - table_description = "B站视频" - - def __str__(self): - return f"{self.video_id} - {self.title}" - - -class BilibiliComment(BilibiliBaseModel): - comment_id = fields.CharField(max_length=64, index=True, description="评论ID") - video_id = fields.CharField(max_length=64, index=True, description="视频ID") - content = fields.TextField(null=True, description="评论内容") - create_time = fields.BigIntField(description="评论时间戳") - sub_comment_count = fields.CharField(max_length=16, description="评论回复数") - - class Meta: - table = "bilibili_video_comment" - table_description = "B 站视频评论" - - def __str__(self): - return f"{self.comment_id} - {self.content}" diff --git a/store/bilibili/bilibili_store_impl.py b/store/bilibili/bilibili_store_impl.py index cdcf948..3f69aa1 100644 --- a/store/bilibili/bilibili_store_impl.py +++ b/store/bilibili/bilibili_store_impl.py @@ -10,7 +10,6 @@ import pathlib from typing import Dict import aiofiles -from tortoise.contrib.pydantic import pydantic_model_creator from base.base_crawler import AbstractStore from tools import utils diff --git a/store/douyin/__init__.py b/store/douyin/__init__.py index db3a147..9f0bfea 100644 --- a/store/douyin/__init__.py +++ b/store/douyin/__init__.py @@ -6,7 +6,6 @@ from typing import List import config -from .douyin_store_db_types import * from .douyin_store_impl import * diff --git a/store/douyin/douyin_store_db_types.py b/store/douyin/douyin_store_db_types.py deleted file mode 100644 index a30fbee..0000000 --- a/store/douyin/douyin_store_db_types.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -# @Author : relakkes@gmail.com -# @Time : 2024/1/14 18:50 -# @Desc : 抖音存储到DB的模型类集合 - - -from tortoise import fields -from tortoise.models import Model - - -class DouyinBaseModel(Model): - id = fields.IntField(pk=True, autoincrement=True, description="自增ID") - user_id = fields.CharField(null=True, max_length=64, description="用户ID") - sec_uid = fields.CharField(null=True, max_length=128, description="用户sec_uid") - short_user_id = fields.CharField(null=True, max_length=64, description="用户短ID") - user_unique_id = fields.CharField(null=True, max_length=64, description="用户唯一ID") - nickname = fields.CharField(null=True, max_length=64, description="用户昵称") - avatar = fields.CharField(null=True, max_length=255, description="用户头像地址") - user_signature = fields.CharField(null=True, max_length=500, description="用户签名") - ip_location = fields.CharField(null=True, max_length=255, description="评论时的IP地址") - add_ts = fields.BigIntField(description="记录添加时间戳") - last_modify_ts = fields.BigIntField(description="记录最后修改时间戳") - - class Meta: - abstract = True - - -class DouyinAweme(DouyinBaseModel): - aweme_id = fields.CharField(max_length=64, index=True, description="视频ID") - aweme_type = fields.CharField(max_length=16, description="视频类型") - title = fields.CharField(null=True, max_length=500, description="视频标题") - desc = fields.TextField(null=True, description="视频描述") - create_time = fields.BigIntField(description="视频发布时间戳", index=True) - liked_count = fields.CharField(null=True, max_length=16, description="视频点赞数") - comment_count = fields.CharField(null=True, max_length=16, description="视频评论数") - share_count = fields.CharField(null=True, max_length=16, description="视频分享数") - collected_count = fields.CharField(null=True, max_length=16, description="视频收藏数") - aweme_url = fields.CharField(null=True, max_length=255, description="视频详情页URL") - - class Meta: - table = "douyin_aweme" - table_description = "抖音视频" - - def __str__(self): - return f"{self.aweme_id} - {self.title}" - - -class DouyinAwemeComment(DouyinBaseModel): - comment_id = fields.CharField(max_length=64, index=True, description="评论ID") - aweme_id = fields.CharField(max_length=64, index=True, description="视频ID") - content = fields.TextField(null=True, description="评论内容") - create_time = fields.BigIntField(description="评论时间戳") - sub_comment_count = fields.CharField(max_length=16, description="评论回复数") - - class Meta: - table = "douyin_aweme_comment" - table_description = "抖音视频评论" - - def __str__(self): - return f"{self.comment_id} - {self.content}" diff --git a/store/douyin/douyin_store_impl.py b/store/douyin/douyin_store_impl.py index a57609b..1977b44 100644 --- a/store/douyin/douyin_store_impl.py +++ b/store/douyin/douyin_store_impl.py @@ -10,7 +10,6 @@ import pathlib from typing import Dict import aiofiles -from tortoise.contrib.pydantic import pydantic_model_creator from base.base_crawler import AbstractStore from tools import utils diff --git a/store/kuaishou/__init__.py b/store/kuaishou/__init__.py index 5fac31f..818c75a 100644 --- a/store/kuaishou/__init__.py +++ b/store/kuaishou/__init__.py @@ -6,7 +6,6 @@ from typing import List import config -from .kuaishou_store_db_types import * from .kuaishou_store_impl import * @@ -48,7 +47,8 @@ async def update_kuaishou_video(video_item: Dict): "video_cover_url": photo_info.get("coverUrl", ""), "video_play_url": photo_info.get("photoUrl", ""), } - utils.logger.info(f"[store.kuaishou.update_kuaishou_video] Kuaishou video id:{video_id}, title:{save_content_item.get('title')}") + utils.logger.info( + f"[store.kuaishou.update_kuaishou_video] Kuaishou video id:{video_id}, title:{save_content_item.get('title')}") await KuaishouStoreFactory.create_store().store_content(content_item=save_content_item) @@ -73,5 +73,6 @@ async def update_ks_video_comment(video_id: str, comment_item: Dict): "sub_comment_count": str(comment_item.get("subCommentCount", 0)), "last_modify_ts": utils.get_current_timestamp(), } - utils.logger.info(f"[store.kuaishou.update_ks_video_comment] Kuaishou video comment: {comment_id}, content: {save_comment_item.get('content')}") - await KuaishouStoreFactory.create_store().store_comment(comment_item=save_comment_item) \ No newline at end of file + utils.logger.info( + f"[store.kuaishou.update_ks_video_comment] Kuaishou video comment: {comment_id}, content: {save_comment_item.get('content')}") + await KuaishouStoreFactory.create_store().store_comment(comment_item=save_comment_item) diff --git a/store/kuaishou/kuaishou_store_db_types.py b/store/kuaishou/kuaishou_store_db_types.py deleted file mode 100644 index 256695e..0000000 --- a/store/kuaishou/kuaishou_store_db_types.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- -# @Author : relakkes@gmail.com -# @Time : 2024/1/14 20:03 -# @Desc : 快手存储到DB的模型类集合 - - -from tortoise import fields -from tortoise.models import Model - - -class KuaishouBaseModel(Model): - id = fields.IntField(pk=True, autoincrement=True, description="自增ID") - user_id = fields.CharField(null=True, max_length=64, description="用户ID") - nickname = fields.CharField(null=True, max_length=64, description="用户昵称") - avatar = fields.CharField(null=True, max_length=255, description="用户头像地址") - add_ts = fields.BigIntField(description="记录添加时间戳") - last_modify_ts = fields.BigIntField(description="记录最后修改时间戳") - - class Meta: - abstract = True - - -class KuaishouVideo(KuaishouBaseModel): - video_id = fields.CharField(max_length=64, index=True, description="视频ID") - video_type = fields.CharField(max_length=16, description="视频类型") - title = fields.CharField(null=True, max_length=500, description="视频标题") - desc = fields.TextField(null=True, description="视频描述") - create_time = fields.BigIntField(description="视频发布时间戳", index=True) - liked_count = fields.CharField(null=True, max_length=16, description="视频点赞数") - viewd_count = fields.CharField(null=True, max_length=16, description="视频浏览数量") - video_url = fields.CharField(null=True, max_length=512, description="视频详情URL") - video_cover_url = fields.CharField(null=True, max_length=512, description="视频封面图 URL") - video_play_url = fields.CharField(null=True, max_length=512, description="视频播放 URL") - - class Meta: - table = "kuaishou_video" - table_description = "快手视频" - - def __str__(self): - return f"{self.video_id} - {self.title}" - - -class KuaishouVideoComment(KuaishouBaseModel): - comment_id = fields.CharField(max_length=64, index=True, description="评论ID") - video_id = fields.CharField(max_length=64, index=True, description="视频ID") - content = fields.TextField(null=True, description="评论内容") - create_time = fields.BigIntField(description="评论时间戳") - sub_comment_count = fields.CharField(max_length=16, description="评论回复数") - - class Meta: - table = "kuaishou_video_comment" - table_description = "快手视频评论" - - def __str__(self): - return f"{self.comment_id} - {self.content}" diff --git a/store/kuaishou/kuaishou_store_impl.py b/store/kuaishou/kuaishou_store_impl.py index 35c7f57..99ae1f4 100644 --- a/store/kuaishou/kuaishou_store_impl.py +++ b/store/kuaishou/kuaishou_store_impl.py @@ -10,7 +10,6 @@ import pathlib from typing import Dict import aiofiles -from tortoise.contrib.pydantic import pydantic_model_creator from base.base_crawler import AbstractStore from tools import utils @@ -83,7 +82,6 @@ class KuaishouDbStoreImplement(AbstractStore): """ - from .kuaishou_store_sql import (add_new_content, query_content_by_content_id, update_content_by_content_id) @@ -116,7 +114,6 @@ class KuaishouDbStoreImplement(AbstractStore): await update_comment_by_comment_id(comment_id, comment_item=comment_item) - class KuaishouJsonStoreImplement(AbstractStore): json_store_path: str = "data/kuaishou" lock = asyncio.Lock() @@ -155,7 +152,6 @@ class KuaishouJsonStoreImplement(AbstractStore): async with aiofiles.open(save_file_name, 'w', encoding='utf-8') as file: await file.write(json.dumps(save_data, ensure_ascii=False)) - async def store_content(self, content_item: Dict): """ content JSON storage implementation diff --git a/store/weibo/__init__.py b/store/weibo/__init__.py index 3424dcb..3a43e52 100644 --- a/store/weibo/__init__.py +++ b/store/weibo/__init__.py @@ -7,7 +7,6 @@ from typing import List import config -from .weibo_store_db_types import * from .weibo_store_impl import * @@ -26,6 +25,7 @@ class WeibostoreFactory: "[WeibotoreFactory.create_store] Invalid save option only supported csv or db or json ...") return store_class() + async def update_weibo_note(note_item: Dict): mblog: Dict = note_item.get("mblog") user_info: Dict = mblog.get("user") @@ -84,5 +84,5 @@ async def update_weibo_note_comment(note_id: str, comment_item: Dict): "avatar": user_info.get("profile_image_url", ""), } utils.logger.info( - f"[store.weibo.update_weibo_note_comment] Weibo note comment: {comment_id}, content: {save_comment_item.get('content','')[:24]} ...") + f"[store.weibo.update_weibo_note_comment] Weibo note comment: {comment_id}, content: {save_comment_item.get('content', '')[:24]} ...") await WeibostoreFactory.create_store().store_comment(comment_item=save_comment_item) diff --git a/store/weibo/weibo_store_db_types.py b/store/weibo/weibo_store_db_types.py deleted file mode 100644 index da672b5..0000000 --- a/store/weibo/weibo_store_db_types.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: utf-8 -*- -# @Author : relakkes@gmail.com -# @Time : 2024/1/14 21:35 -# @Desc : 微博存储到DB的模型类集合 - -from tortoise import fields -from tortoise.models import Model - - -class WeiboBaseModel(Model): - id = fields.IntField(pk=True, autoincrement=True, description="自增ID") - user_id = fields.CharField(null=True, max_length=64, description="用户ID") - nickname = fields.CharField(null=True, max_length=64, description="用户昵称") - avatar = fields.CharField(null=True, max_length=255, description="用户头像地址") - gender = fields.CharField(null=True, max_length=12, description="用户性别") - profile_url = fields.CharField(null=True, max_length=255, description="用户主页地址") - ip_location = fields.CharField(null=True, max_length=32, default="发布微博的地理信息") - add_ts = fields.BigIntField(description="记录添加时间戳") - last_modify_ts = fields.BigIntField(description="记录最后修改时间戳") - - class Meta: - abstract = True - - -class WeiboNote(WeiboBaseModel): - note_id = fields.CharField(max_length=64, index=True, description="帖子ID") - content = fields.TextField(null=True, description="帖子正文内容") - create_time = fields.BigIntField(description="帖子发布时间戳", index=True) - create_date_time = fields.CharField(description="帖子发布日期时间", max_length=32, index=True) - liked_count = fields.CharField(null=True, max_length=16, description="帖子点赞数") - comments_count = fields.CharField(null=True, max_length=16, description="帖子评论数量") - shared_count = fields.CharField(null=True, max_length=16, description="帖子转发数量") - note_url = fields.CharField(null=True, max_length=512, description="帖子详情URL") - - class Meta: - table = "weibo_note" - table_description = "微博帖子" - - def __str__(self): - return f"{self.note_id}" - - -class WeiboComment(WeiboBaseModel): - comment_id = fields.CharField(max_length=64, index=True, description="评论ID") - note_id = fields.CharField(max_length=64, index=True, description="帖子ID") - content = fields.TextField(null=True, description="评论内容") - create_time = fields.BigIntField(description="评论时间戳") - create_date_time = fields.CharField(description="评论日期时间", max_length=32, index=True) - comment_like_count = fields.CharField(max_length=16, description="评论点赞数量") - sub_comment_count = fields.CharField(max_length=16, description="评论回复数") - - class Meta: - table = "weibo_note_comment" - table_description = "微博帖子评论" - - def __str__(self): - return f"{self.comment_id}" diff --git a/store/weibo/weibo_store_impl.py b/store/weibo/weibo_store_impl.py index 05b9e52..e2df846 100644 --- a/store/weibo/weibo_store_impl.py +++ b/store/weibo/weibo_store_impl.py @@ -10,7 +10,6 @@ import pathlib from typing import Dict import aiofiles -from tortoise.contrib.pydantic import pydantic_model_creator from base.base_crawler import AbstractStore from tools import utils diff --git a/store/xhs/__init__.py b/store/xhs/__init__.py index 890ff9d..f10d0ef 100644 --- a/store/xhs/__init__.py +++ b/store/xhs/__init__.py @@ -7,7 +7,6 @@ from typing import List import config from . import xhs_store_impl -from .xhs_store_db_types import * from .xhs_store_impl import * @@ -26,7 +25,6 @@ class XhsStoreFactory: return store_class() - async def update_xhs_note(note_item: Dict): note_id = note_item.get("note_id") user_info = note_item.get("user", {}) @@ -117,7 +115,8 @@ async def save_creator(user_id: str, creator: Dict): 'follows': follows, 'fans': fans, 'interaction': interaction, - 'tag_list': json.dumps({tag.get('tagType'): tag.get('name') for tag in creator.get('tags')}, ensure_ascii=False), + 'tag_list': json.dumps({tag.get('tagType'): tag.get('name') for tag in creator.get('tags')}, + ensure_ascii=False), "last_modify_ts": utils.get_current_timestamp(), } utils.logger.info(f"[store.xhs.save_creator] creator:{local_db_item}") diff --git a/store/xhs/xhs_store_db_types.py b/store/xhs/xhs_store_db_types.py deleted file mode 100644 index ee8f240..0000000 --- a/store/xhs/xhs_store_db_types.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- -# @Author : relakkes@gmail.com -# @Time : 2024/1/14 17:31 -# @Desc : 小红书存储到DB的模型类集合 - -from tortoise import fields -from tortoise.models import Model - - -class XhsBaseModel(Model): - id = fields.IntField(pk=True, autoincrement=True, description="自增ID") - user_id = fields.CharField(max_length=64, description="用户ID") - nickname = fields.CharField(null=True, max_length=64, description="用户昵称") - avatar = fields.CharField(null=True, max_length=255, description="用户头像地址") - ip_location = fields.CharField(null=True, max_length=255, description="评论时的IP地址") - add_ts = fields.BigIntField(description="记录添加时间戳") - last_modify_ts = fields.BigIntField(description="记录最后修改时间戳") - - class Meta: - abstract = True - - -class XHSNote(XhsBaseModel): - note_id = fields.CharField(max_length=64, index=True, description="笔记ID") - type = fields.CharField(null=True, max_length=16, description="笔记类型(normal | video)") - title = fields.CharField(null=True, max_length=255, description="笔记标题") - desc = fields.TextField(null=True, description="笔记描述") - video_url = fields.TextField(null=True, description="视频地址") - time = fields.BigIntField(description="笔记发布时间戳", index=True) - last_update_time = fields.BigIntField(description="笔记最后更新时间戳") - liked_count = fields.CharField(null=True, max_length=16, description="笔记点赞数") - collected_count = fields.CharField(null=True, max_length=16, description="笔记收藏数") - comment_count = fields.CharField(null=True, max_length=16, description="笔记评论数") - share_count = fields.CharField(null=True, max_length=16, description="笔记分享数") - image_list = fields.TextField(null=True, description="笔记封面图片列表") - tag_list = fields.TextField(null=True, description="标签列表") - note_url = fields.CharField(null=True, max_length=255, description="笔记详情页的URL") - - class Meta: - table = "xhs_note" - table_description = "小红书笔记" - - def __str__(self): - return f"{self.note_id} - {self.title}" - - -class XHSNoteComment(XhsBaseModel): - comment_id = fields.CharField(max_length=64, index=True, description="评论ID") - create_time = fields.BigIntField(index=True, description="评论时间戳") - note_id = fields.CharField(max_length=64, description="笔记ID") - content = fields.TextField(description="评论内容") - sub_comment_count = fields.IntField(description="子评论数量") - pictures = fields.CharField(null=True, max_length=512, description="评论的图片集合") - - class Meta: - table = "xhs_note_comment" - table_description = "小红书笔记评论" - - def __str__(self): - return f"{self.comment_id} - {self.content}" - - -class XhsCreator(XhsBaseModel): - desc = fields.TextField(null=True, description="用户描述") - gender = fields.CharField(null=True, max_length=1, description="性别") - follows = fields.CharField(null=True, max_length=16, description="关注数") - fans = fields.CharField(null=True, max_length=16, description="粉丝数") - interaction = fields.CharField(null=True, max_length=16, description="获赞和收藏数") - # follows = fields.IntField(description="关注数") - # fans = fields.IntField(description="粉丝数") - # interaction = fields.IntField(description="获赞和收藏数") - tag_list = fields.TextField(null=True, description="标签列表") # json字符串 - - class Meta: - table = "xhs_creator" - table_description = "小红书博主" diff --git a/store/xhs/xhs_store_impl.py b/store/xhs/xhs_store_impl.py index 80eeede..26fc43e 100644 --- a/store/xhs/xhs_store_impl.py +++ b/store/xhs/xhs_store_impl.py @@ -10,12 +10,10 @@ import pathlib from typing import Dict import aiofiles -from tortoise.contrib.pydantic import pydantic_model_creator from base.base_crawler import AbstractStore -from db import AsyncMysqlDB from tools import utils -from var import crawler_type_var, media_crawler_db_var +from var import crawler_type_var class XhsCsvStoreImplement(AbstractStore):