qnloft
/
MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat：小红书支持获取评论中的图片链接 #145

This commit is contained in:
Relakkes 2024-03-07 22:30:44 +08:00
parent 861019022a
commit 41fee4ff4f
3 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
media_platform/xhs/client.py
View File

@ -267,7 +267,9 @@ class XHSClient:
uri = "/api/sns/web/v2/comment/page"
params = {
"note_id": note_id,
"cursor": cursor
"cursor": cursor,
"top_comment_id":"",
"image_formats": "jpg,webp,avif"
}
return await self.get(uri, params)

13
store/xhs/__init__.py
View File

@ -37,7 +37,7 @@ async def update_xhs_note(note_item: Dict):
if note_item.get('type') == 'video':
videos = note_item.get('video').get('media').get('stream').get('h264')
if type(videos).__name__ == 'list':
video_url = ','.join([ v.get('master_url') for v in videos])
video_url = ','.join([v.get('master_url') for v in videos])
local_db_item = {
"note_id": note_item.get("note_id"),
@ -56,22 +56,25 @@ async def update_xhs_note(note_item: Dict):
"share_count": interact_info.get("share_count"),
"ip_location": note_item.get("ip_location", ""),
"image_list": ','.join([img.get('url', '') for img in image_list]),
"tag_list": ','.join([tag.get('name', '') for tag in tag_list if tag.get('type')=='topic']),
"tag_list": ','.join([tag.get('name', '') for tag in tag_list if tag.get('type') == 'topic']),
"last_modify_ts": utils.get_current_timestamp(),
"note_url": f"https://www.xiaohongshu.com/explore/{note_id}"
}
utils.logger.info(f"[store.xhs.update_xhs_note] xhs note: {local_db_item}")
await XhsStoreFactory.create_store().store_content(local_db_item)
async def batch_update_xhs_note_comments(note_id: str, comments: List[Dict]):
if not comments:
return
for comment_item in comments:
await update_xhs_note_comment(note_id, comment_item)
async def update_xhs_note_comment(note_id: str, comment_item: Dict):
user_info = comment_item.get("user_info", {})
comment_id = comment_item.get("id")
comment_pictures = [item.get("url_default", "") for item in comment_item.get("pictures", [])]
local_db_item = {
"comment_id": comment_id,
"create_time": comment_item.get("create_time"),
@ -82,11 +85,13 @@ async def update_xhs_note_comment(note_id: str, comment_item: Dict):
"nickname": user_info.get("nickname"),
"avatar": user_info.get("image"),
"sub_comment_count": comment_item.get("sub_comment_count"),
"pictures": ",".join(comment_pictures),
"last_modify_ts": utils.get_current_timestamp(),
}
utils.logger.info(f"[store.xhs.update_xhs_note_comment] xhs note comment:{local_db_item}")
await XhsStoreFactory.create_store().store_comment(local_db_item)
async def save_creator(user_id: str, creator: Dict):
user_info = creator.get('basicInfo', {})
@ -104,14 +109,14 @@ async def save_creator(user_id: str, creator: Dict):
local_db_item = {
'user_id': user_id,
'nickname': user_info.get('nickname'),
'gender': '女' if user_info.get('gender') == 1 else '男' ,
'gender': '女' if user_info.get('gender') == 1 else '男',
'avatar': user_info.get('images'),
'desc': user_info.get('desc'),
'ip_location': user_info.get('ip_location'),
'follows': follows,
'fans': fans,
'interaction': interaction,
'tag_list': json.dumps({tag.get('tagType'):tag.get('name') for tag in creator.get('tags')}),
'tag_list': json.dumps({tag.get('tagType'): tag.get('name') for tag in creator.get('tags')}),
}
utils.logger.info(f"[store.xhs.save_creator] creator:{local_db_item}")
await XhsStoreFactory.create_store().store_creator(local_db_item)

1
store/xhs/xhs_store_db_types.py
View File

@ -50,6 +50,7 @@ class XHSNoteComment(XhsBaseModel):
note_id = fields.CharField(max_length=64, description="笔记ID")
content = fields.TextField(description="评论内容")
sub_comment_count = fields.IntField(description="子评论数量")
pictures = fields.CharField(null=True, max_length=512, description="评论的图片集合")
class Meta:
table = "xhs_note_comment"