qnloft
/
MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: 移除orm的所有内容

This commit is contained in:
Relakkes 2024-04-06 23:51:03 +08:00
parent 1627ae4344
commit d392747fe7
15 changed files with 10 additions and 324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
store/bilibili/__init__.py
View File

@ -7,7 +7,6 @@ from typing import List
import config
from .bilibili_store_db_types import *
from .bilibili_store_impl import *

55
store/bilibili/bilibili_store_db_types.py
View File

@ -1,55 +0,0 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2024/1/14 19:34
# @Desc : B站存储到DB的模型类集合
from tortoise import fields
from tortoise.models import Model
class BilibiliBaseModel(Model):
id = fields.IntField(pk=True, autoincrement=True, description="自增ID")
user_id = fields.CharField(null=True, max_length=64, description="用户ID")
nickname = fields.CharField(null=True, max_length=64, description="用户昵称")
avatar = fields.CharField(null=True, max_length=255, description="用户头像地址")
add_ts = fields.BigIntField(description="记录添加时间戳")
last_modify_ts = fields.BigIntField(description="记录最后修改时间戳")
class Meta:
abstract = True
class BilibiliVideo(BilibiliBaseModel):
video_id = fields.CharField(max_length=64, index=True, description="视频ID")
video_type = fields.CharField(max_length=16, description="视频类型")
title = fields.CharField(null=True, max_length=500, description="视频标题")
desc = fields.TextField(null=True, description="视频描述")
create_time = fields.BigIntField(description="视频发布时间戳", index=True)
liked_count = fields.CharField(null=True, max_length=16, description="视频点赞数")
video_play_count = fields.CharField(null=True, max_length=16, description="视频播放数量")
video_danmaku = fields.CharField(null=True, max_length=16, description="视频弹幕数量")
video_comment = fields.CharField(null=True, max_length=16, description="视频评论数量")
video_url = fields.CharField(null=True, max_length=512, description="视频详情URL")
video_cover_url = fields.CharField(null=True, max_length=512, description="视频封面图 URL")
class Meta:
table = "bilibili_video"
table_description = "B站视频"
def __str__(self):
return f"{self.video_id} - {self.title}"
class BilibiliComment(BilibiliBaseModel):
comment_id = fields.CharField(max_length=64, index=True, description="评论ID")
video_id = fields.CharField(max_length=64, index=True, description="视频ID")
content = fields.TextField(null=True, description="评论内容")
create_time = fields.BigIntField(description="评论时间戳")
sub_comment_count = fields.CharField(max_length=16, description="评论回复数")
class Meta:
table = "bilibili_video_comment"
table_description = "B 站视频评论"
def __str__(self):
return f"{self.comment_id} - {self.content}"

1
store/bilibili/bilibili_store_impl.py
View File

@ -10,7 +10,6 @@ import pathlib
from typing import Dict
import aiofiles
from tortoise.contrib.pydantic import pydantic_model_creator
from base.base_crawler import AbstractStore
from tools import utils

1
store/douyin/__init__.py
View File

@ -6,7 +6,6 @@ from typing import List
import config
from .douyin_store_db_types import *
from .douyin_store_impl import *

60
store/douyin/douyin_store_db_types.py
View File

@ -1,60 +0,0 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2024/1/14 18:50
# @Desc : 抖音存储到DB的模型类集合
from tortoise import fields
from tortoise.models import Model
class DouyinBaseModel(Model):
id = fields.IntField(pk=True, autoincrement=True, description="自增ID")
user_id = fields.CharField(null=True, max_length=64, description="用户ID")
sec_uid = fields.CharField(null=True, max_length=128, description="用户sec_uid")
short_user_id = fields.CharField(null=True, max_length=64, description="用户短ID")
user_unique_id = fields.CharField(null=True, max_length=64, description="用户唯一ID")
nickname = fields.CharField(null=True, max_length=64, description="用户昵称")
avatar = fields.CharField(null=True, max_length=255, description="用户头像地址")
user_signature = fields.CharField(null=True, max_length=500, description="用户签名")
ip_location = fields.CharField(null=True, max_length=255, description="评论时的IP地址")
add_ts = fields.BigIntField(description="记录添加时间戳")
last_modify_ts = fields.BigIntField(description="记录最后修改时间戳")
class Meta:
abstract = True
class DouyinAweme(DouyinBaseModel):
aweme_id = fields.CharField(max_length=64, index=True, description="视频ID")
aweme_type = fields.CharField(max_length=16, description="视频类型")
title = fields.CharField(null=True, max_length=500, description="视频标题")
desc = fields.TextField(null=True, description="视频描述")
create_time = fields.BigIntField(description="视频发布时间戳", index=True)
liked_count = fields.CharField(null=True, max_length=16, description="视频点赞数")
comment_count = fields.CharField(null=True, max_length=16, description="视频评论数")
share_count = fields.CharField(null=True, max_length=16, description="视频分享数")
collected_count = fields.CharField(null=True, max_length=16, description="视频收藏数")
aweme_url = fields.CharField(null=True, max_length=255, description="视频详情页URL")
class Meta:
table = "douyin_aweme"
table_description = "抖音视频"
def __str__(self):
return f"{self.aweme_id} - {self.title}"
class DouyinAwemeComment(DouyinBaseModel):
comment_id = fields.CharField(max_length=64, index=True, description="评论ID")
aweme_id = fields.CharField(max_length=64, index=True, description="视频ID")
content = fields.TextField(null=True, description="评论内容")
create_time = fields.BigIntField(description="评论时间戳")
sub_comment_count = fields.CharField(max_length=16, description="评论回复数")
class Meta:
table = "douyin_aweme_comment"
table_description = "抖音视频评论"
def __str__(self):
return f"{self.comment_id} - {self.content}"

1
store/douyin/douyin_store_impl.py
View File

@ -10,7 +10,6 @@ import pathlib
from typing import Dict
import aiofiles
from tortoise.contrib.pydantic import pydantic_model_creator
from base.base_crawler import AbstractStore
from tools import utils

9
store/kuaishou/__init__.py
View File

@ -6,7 +6,6 @@ from typing import List
import config
from .kuaishou_store_db_types import *
from .kuaishou_store_impl import *
@ -48,7 +47,8 @@ async def update_kuaishou_video(video_item: Dict):
"video_cover_url": photo_info.get("coverUrl", ""),
"video_play_url": photo_info.get("photoUrl", ""),
}
utils.logger.info(f"[store.kuaishou.update_kuaishou_video] Kuaishou video id:{video_id}, title:{save_content_item.get('title')}")
utils.logger.info(
f"[store.kuaishou.update_kuaishou_video] Kuaishou video id:{video_id}, title:{save_content_item.get('title')}")
await KuaishouStoreFactory.create_store().store_content(content_item=save_content_item)
@ -73,5 +73,6 @@ async def update_ks_video_comment(video_id: str, comment_item: Dict):
"sub_comment_count": str(comment_item.get("subCommentCount", 0)),
"last_modify_ts": utils.get_current_timestamp(),
}
utils.logger.info(f"[store.kuaishou.update_ks_video_comment] Kuaishou video comment: {comment_id}, content: {save_comment_item.get('content')}")
await KuaishouStoreFactory.create_store().store_comment(comment_item=save_comment_item)
utils.logger.info(
f"[store.kuaishou.update_ks_video_comment] Kuaishou video comment: {comment_id}, content: {save_comment_item.get('content')}")
await KuaishouStoreFactory.create_store().store_comment(comment_item=save_comment_item)

55
store/kuaishou/kuaishou_store_db_types.py
View File

@ -1,55 +0,0 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2024/1/14 20:03
# @Desc : 快手存储到DB的模型类集合
from tortoise import fields
from tortoise.models import Model
class KuaishouBaseModel(Model):
id = fields.IntField(pk=True, autoincrement=True, description="自增ID")
user_id = fields.CharField(null=True, max_length=64, description="用户ID")
nickname = fields.CharField(null=True, max_length=64, description="用户昵称")
avatar = fields.CharField(null=True, max_length=255, description="用户头像地址")
add_ts = fields.BigIntField(description="记录添加时间戳")
last_modify_ts = fields.BigIntField(description="记录最后修改时间戳")
class Meta:
abstract = True
class KuaishouVideo(KuaishouBaseModel):
video_id = fields.CharField(max_length=64, index=True, description="视频ID")
video_type = fields.CharField(max_length=16, description="视频类型")
title = fields.CharField(null=True, max_length=500, description="视频标题")
desc = fields.TextField(null=True, description="视频描述")
create_time = fields.BigIntField(description="视频发布时间戳", index=True)
liked_count = fields.CharField(null=True, max_length=16, description="视频点赞数")
viewd_count = fields.CharField(null=True, max_length=16, description="视频浏览数量")
video_url = fields.CharField(null=True, max_length=512, description="视频详情URL")
video_cover_url = fields.CharField(null=True, max_length=512, description="视频封面图 URL")
video_play_url = fields.CharField(null=True, max_length=512, description="视频播放 URL")
class Meta:
table = "kuaishou_video"
table_description = "快手视频"
def __str__(self):
return f"{self.video_id} - {self.title}"
class KuaishouVideoComment(KuaishouBaseModel):
comment_id = fields.CharField(max_length=64, index=True, description="评论ID")
video_id = fields.CharField(max_length=64, index=True, description="视频ID")
content = fields.TextField(null=True, description="评论内容")
create_time = fields.BigIntField(description="评论时间戳")
sub_comment_count = fields.CharField(max_length=16, description="评论回复数")
class Meta:
table = "kuaishou_video_comment"
table_description = "快手视频评论"
def __str__(self):
return f"{self.comment_id} - {self.content}"

4
store/kuaishou/kuaishou_store_impl.py
View File

@ -10,7 +10,6 @@ import pathlib
from typing import Dict
import aiofiles
from tortoise.contrib.pydantic import pydantic_model_creator
from base.base_crawler import AbstractStore
from tools import utils
@ -83,7 +82,6 @@ class KuaishouDbStoreImplement(AbstractStore):
"""
from .kuaishou_store_sql import (add_new_content,
query_content_by_content_id,
update_content_by_content_id)
@ -116,7 +114,6 @@ class KuaishouDbStoreImplement(AbstractStore):
await update_comment_by_comment_id(comment_id, comment_item=comment_item)
class KuaishouJsonStoreImplement(AbstractStore):
json_store_path: str = "data/kuaishou"
lock = asyncio.Lock()
@ -155,7 +152,6 @@ class KuaishouJsonStoreImplement(AbstractStore):
async with aiofiles.open(save_file_name, 'w', encoding='utf-8') as file:
await file.write(json.dumps(save_data, ensure_ascii=False))
async def store_content(self, content_item: Dict):
"""
content JSON storage implementation

4
store/weibo/__init__.py
View File

@ -7,7 +7,6 @@ from typing import List
import config
from .weibo_store_db_types import *
from .weibo_store_impl import *
@ -26,6 +25,7 @@ class WeibostoreFactory:
"[WeibotoreFactory.create_store] Invalid save option only supported csv or db or json ...")
return store_class()
async def update_weibo_note(note_item: Dict):
mblog: Dict = note_item.get("mblog")
user_info: Dict = mblog.get("user")
@ -84,5 +84,5 @@ async def update_weibo_note_comment(note_id: str, comment_item: Dict):
"avatar": user_info.get("profile_image_url", ""),
}
utils.logger.info(
f"[store.weibo.update_weibo_note_comment] Weibo note comment: {comment_id}, content: {save_comment_item.get('content','')[:24]} ...")
f"[store.weibo.update_weibo_note_comment] Weibo note comment: {comment_id}, content: {save_comment_item.get('content', '')[:24]} ...")
await WeibostoreFactory.create_store().store_comment(comment_item=save_comment_item)

57
store/weibo/weibo_store_db_types.py
View File

@ -1,57 +0,0 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2024/1/14 21:35
# @Desc : 微博存储到DB的模型类集合
from tortoise import fields
from tortoise.models import Model
class WeiboBaseModel(Model):
id = fields.IntField(pk=True, autoincrement=True, description="自增ID")
user_id = fields.CharField(null=True, max_length=64, description="用户ID")
nickname = fields.CharField(null=True, max_length=64, description="用户昵称")
avatar = fields.CharField(null=True, max_length=255, description="用户头像地址")
gender = fields.CharField(null=True, max_length=12, description="用户性别")
profile_url = fields.CharField(null=True, max_length=255, description="用户主页地址")
ip_location = fields.CharField(null=True, max_length=32, default="发布微博的地理信息")
add_ts = fields.BigIntField(description="记录添加时间戳")
last_modify_ts = fields.BigIntField(description="记录最后修改时间戳")
class Meta:
abstract = True
class WeiboNote(WeiboBaseModel):
note_id = fields.CharField(max_length=64, index=True, description="帖子ID")
content = fields.TextField(null=True, description="帖子正文内容")
create_time = fields.BigIntField(description="帖子发布时间戳", index=True)
create_date_time = fields.CharField(description="帖子发布日期时间", max_length=32, index=True)
liked_count = fields.CharField(null=True, max_length=16, description="帖子点赞数")
comments_count = fields.CharField(null=True, max_length=16, description="帖子评论数量")
shared_count = fields.CharField(null=True, max_length=16, description="帖子转发数量")
note_url = fields.CharField(null=True, max_length=512, description="帖子详情URL")
class Meta:
table = "weibo_note"
table_description = "微博帖子"
def __str__(self):
return f"{self.note_id}"
class WeiboComment(WeiboBaseModel):
comment_id = fields.CharField(max_length=64, index=True, description="评论ID")
note_id = fields.CharField(max_length=64, index=True, description="帖子ID")
content = fields.TextField(null=True, description="评论内容")
create_time = fields.BigIntField(description="评论时间戳")
create_date_time = fields.CharField(description="评论日期时间", max_length=32, index=True)
comment_like_count = fields.CharField(max_length=16, description="评论点赞数量")
sub_comment_count = fields.CharField(max_length=16, description="评论回复数")
class Meta:
table = "weibo_note_comment"
table_description = "微博帖子评论"
def __str__(self):
return f"{self.comment_id}"

1
store/weibo/weibo_store_impl.py
View File

@ -10,7 +10,6 @@ import pathlib
from typing import Dict
import aiofiles
from tortoise.contrib.pydantic import pydantic_model_creator
from base.base_crawler import AbstractStore
from tools import utils

5
store/xhs/__init__.py
View File

@ -7,7 +7,6 @@ from typing import List
import config
from . import xhs_store_impl
from .xhs_store_db_types import *
from .xhs_store_impl import *
@ -26,7 +25,6 @@ class XhsStoreFactory:
return store_class()
async def update_xhs_note(note_item: Dict):
note_id = note_item.get("note_id")
user_info = note_item.get("user", {})
@ -117,7 +115,8 @@ async def save_creator(user_id: str, creator: Dict):
'follows': follows,
'fans': fans,
'interaction': interaction,
'tag_list': json.dumps({tag.get('tagType'): tag.get('name') for tag in creator.get('tags')}, ensure_ascii=False),
'tag_list': json.dumps({tag.get('tagType'): tag.get('name') for tag in creator.get('tags')},
ensure_ascii=False),
"last_modify_ts": utils.get_current_timestamp(),
}
utils.logger.info(f"[store.xhs.save_creator] creator:{local_db_item}")

76
store/xhs/xhs_store_db_types.py
View File

@ -1,76 +0,0 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2024/1/14 17:31
# @Desc : 小红书存储到DB的模型类集合
from tortoise import fields
from tortoise.models import Model
class XhsBaseModel(Model):
id = fields.IntField(pk=True, autoincrement=True, description="自增ID")
user_id = fields.CharField(max_length=64, description="用户ID")
nickname = fields.CharField(null=True, max_length=64, description="用户昵称")
avatar = fields.CharField(null=True, max_length=255, description="用户头像地址")
ip_location = fields.CharField(null=True, max_length=255, description="评论时的IP地址")
add_ts = fields.BigIntField(description="记录添加时间戳")
last_modify_ts = fields.BigIntField(description="记录最后修改时间戳")
class Meta:
abstract = True
class XHSNote(XhsBaseModel):
note_id = fields.CharField(max_length=64, index=True, description="笔记ID")
type = fields.CharField(null=True, max_length=16, description="笔记类型(normal | video)")
title = fields.CharField(null=True, max_length=255, description="笔记标题")
desc = fields.TextField(null=True, description="笔记描述")
video_url = fields.TextField(null=True, description="视频地址")
time = fields.BigIntField(description="笔记发布时间戳", index=True)
last_update_time = fields.BigIntField(description="笔记最后更新时间戳")
liked_count = fields.CharField(null=True, max_length=16, description="笔记点赞数")
collected_count = fields.CharField(null=True, max_length=16, description="笔记收藏数")
comment_count = fields.CharField(null=True, max_length=16, description="笔记评论数")
share_count = fields.CharField(null=True, max_length=16, description="笔记分享数")
image_list = fields.TextField(null=True, description="笔记封面图片列表")
tag_list = fields.TextField(null=True, description="标签列表")
note_url = fields.CharField(null=True, max_length=255, description="笔记详情页的URL")
class Meta:
table = "xhs_note"
table_description = "小红书笔记"
def __str__(self):
return f"{self.note_id} - {self.title}"
class XHSNoteComment(XhsBaseModel):
comment_id = fields.CharField(max_length=64, index=True, description="评论ID")
create_time = fields.BigIntField(index=True, description="评论时间戳")
note_id = fields.CharField(max_length=64, description="笔记ID")
content = fields.TextField(description="评论内容")
sub_comment_count = fields.IntField(description="子评论数量")
pictures = fields.CharField(null=True, max_length=512, description="评论的图片集合")
class Meta:
table = "xhs_note_comment"
table_description = "小红书笔记评论"
def __str__(self):
return f"{self.comment_id} - {self.content}"
class XhsCreator(XhsBaseModel):
desc = fields.TextField(null=True, description="用户描述")
gender = fields.CharField(null=True, max_length=1, description="性别")
follows = fields.CharField(null=True, max_length=16, description="关注数")
fans = fields.CharField(null=True, max_length=16, description="粉丝数")
interaction = fields.CharField(null=True, max_length=16, description="获赞和收藏数")
# follows = fields.IntField(description="关注数")
# fans = fields.IntField(description="粉丝数")
# interaction = fields.IntField(description="获赞和收藏数")
tag_list = fields.TextField(null=True, description="标签列表") # json字符串
class Meta:
table = "xhs_creator"
table_description = "小红书博主"

4
store/xhs/xhs_store_impl.py
View File

@ -10,12 +10,10 @@ import pathlib
from typing import Dict
import aiofiles
from tortoise.contrib.pydantic import pydantic_model_creator
from base.base_crawler import AbstractStore
from db import AsyncMysqlDB
from tools import utils
from var import crawler_type_var, media_crawler_db_var
from var import crawler_type_var
class XhsCsvStoreImplement(AbstractStore):