qnloft
/
MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ feat: orm添加Pydantic数据验证

This commit is contained in:
zhoubang 2023-08-13 22:39:50 +08:00
parent 3ae4c5f2f4
commit 1556e1b8cd
3 changed files with 40 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
models/douyin.py
View File

@ -3,6 +3,7 @@ from typing import Dict, List
from tortoise import fields from tortoise import fields
from tortoise.models import Model from tortoise.models import Model
from tortoise.contrib.pydantic import pydantic_model_creator
import config import config
from tools import utils from tools import utils
@ -87,9 +88,15 @@ async def update_douyin_aweme(aweme_item: Dict):
if config.IS_SAVED_DATABASED: if config.IS_SAVED_DATABASED:
if not await DouyinAweme.filter(aweme_id=aweme_id).exists(): if not await DouyinAweme.filter(aweme_id=aweme_id).exists():
local_db_item["add_ts"] = utils.get_current_timestamp() local_db_item["add_ts"] = utils.get_current_timestamp()
await DouyinAweme.create(**local_db_item) douyin_aweme_pydantic = pydantic_model_creator(DouyinAweme,name='DouyinAwemeCreate',exclude=('id',))
douyin_data = douyin_aweme_pydantic(**local_db_item)
douyin_aweme_pydantic.validate(douyin_data)
await DouyinAweme.create(**douyin_data.dict())
else: else:
await DouyinAweme.filter(aweme_id=aweme_id).update(**local_db_item) douyin_aweme_pydantic = pydantic_model_creator(DouyinAweme, name='DouyinAwemeUpdate', exclude=('id','add_ts'))
douyin_data = douyin_aweme_pydantic(**local_db_item)
douyin_aweme_pydantic.validate(douyin_data)
await DouyinAweme.filter(aweme_id=aweme_id).update(**douyin_data.dict())
async def batch_update_dy_aweme_comments(aweme_id: str, comments: List[Dict]): async def batch_update_dy_aweme_comments(aweme_id: str, comments: List[Dict]):
@ -129,6 +136,12 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict):
if config.IS_SAVED_DATABASED: if config.IS_SAVED_DATABASED:
if not await DouyinAwemeComment.filter(comment_id=comment_id).exists(): if not await DouyinAwemeComment.filter(comment_id=comment_id).exists():
local_db_item["add_ts"] = utils.get_current_timestamp() local_db_item["add_ts"] = utils.get_current_timestamp()
await DouyinAwemeComment.create(**local_db_item) comment_pydantic = pydantic_model_creator(DouyinAwemeComment, name='DouyinAwemeCommentCreate', exclude=('id',))
comment_data = comment_pydantic(**local_db_item)
comment_pydantic.validate(comment_data)
await DouyinAwemeComment.create(**comment_data.dict())
else: else:
await DouyinAwemeComment.filter(comment_id=comment_id).update(**local_db_item) comment_pydantic = pydantic_model_creator(DouyinAwemeComment, name='DouyinAwemeCommentUpdate', exclude=('id','add_ts'))
comment_data = comment_pydantic(**local_db_item)
comment_pydantic.validate(comment_data)
await DouyinAwemeComment.filter(comment_id=comment_id).update(**comment_data.dict())

25
models/xiaohongshu.py
View File

@ -1,6 +1,7 @@
from typing import Dict, List from typing import Dict, List
from tortoise import fields from tortoise import fields
from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise.models import Model from tortoise.models import Model
import config import config
@ -84,9 +85,15 @@ async def update_xhs_note(note_item: Dict):
if config.IS_SAVED_DATABASED: if config.IS_SAVED_DATABASED:
if not await XHSNote.filter(note_id=note_id).first(): if not await XHSNote.filter(note_id=note_id).first():
local_db_item["add_ts"] = utils.get_current_timestamp() local_db_item["add_ts"] = utils.get_current_timestamp()
await XHSNote.create(**local_db_item) note_pydantic = pydantic_model_creator(XHSNote, name="XHSPydanticCreate", exclude=('id', ))
note_data = note_pydantic(**local_db_item)
note_pydantic.validate(note_data)
await XHSNote.create(**note_data.dict())
else: else:
await XHSNote.filter(note_id=note_id).update(**local_db_item) note_pydantic = pydantic_model_creator(XHSNote, name="XHSPydanticUpdate", exclude=('id','add_ts'))
note_data = note_pydantic(**local_db_item)
note_pydantic.validate(note_data)
await XHSNote.filter(note_id=note_id).update(**note_data.dict())
async def update_xhs_note_comment(note_id: str, comment_item: Dict): async def update_xhs_note_comment(note_id: str, comment_item: Dict):
@ -108,6 +115,16 @@ async def update_xhs_note_comment(note_id: str, comment_item: Dict):
if config.IS_SAVED_DATABASED: if config.IS_SAVED_DATABASED:
if not await XHSNoteComment.filter(comment_id=comment_id).first(): if not await XHSNoteComment.filter(comment_id=comment_id).first():
local_db_item["add_ts"] = utils.get_current_timestamp() local_db_item["add_ts"] = utils.get_current_timestamp()
await XHSNoteComment.create(**local_db_item) comment_pydantic = pydantic_model_creator(XHSNoteComment, name="CommentPydanticCreate", exclude=('id', ))
comment_data = comment_pydantic(**local_db_item)
comment_pydantic.validate(comment_data)
await XHSNoteComment.create(**comment_data.dict())
else: else:
await XHSNoteComment.filter(comment_id=comment_id).update(**local_db_item) comment_pydantic = pydantic_model_creator(XHSNoteComment, name="CommentPydanticUpdate", exclude=('id','add_ts',))
comment_data = comment_pydantic(**local_db_item)
comment_pydantic.validate(comment_data)
await XHSNoteComment.filter(comment_id=comment_id).update(**comment_data.dict())

1
requirements.txt
View File

@ -9,3 +9,4 @@ tortoise-orm[asyncmy]==0.19.3
aerich==0.7.2 aerich==0.7.2
numpy~=1.24.4 numpy~=1.24.4
redis~=4.6.0 redis~=4.6.0
Pydantic==1.7