qnloft
/
MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 抖音指定创作者done

This commit is contained in:
Relakkes 2024-05-28 01:07:19 +08:00
parent df1e4a7b02
commit 478db4cc4b
8 changed files with 241 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
README.md
View File

@ -18,12 +18,12 @@
> 下面不支持的项目，相关的代码架构已经搭建好，只需要实现对应的方法即可，欢迎大家提交PR
| 平台 | 关键词搜索 | 指定帖子ID爬取 | 二级评论 | 指定创作者主页 | 登录态缓存 | IP代理池 |
|-----|-------|----------|------|---------|-------|-------|
| 小红书 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| 抖音 | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ |
| 快手 | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ |
| B 站 | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ |
| 微博 | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ |
|-----|-------|----------|------|--------|-------|-------|
| 小红书 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| 抖音 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ |
| 快手 | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ |
| B 站 | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ |
| 微博 | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ |
## 使用方法
@ -84,7 +84,7 @@
## 开发者服务
- 付费咨询：提供 200 元/小时的咨询服务，帮你快速解决项目中遇到的问题。
- 知识星球：沉淀高质量常见问题、最佳实践文档，提供付费知识星球服务，主动提问，作者会定期回答问题
- 知识星球：沉淀高质量常见问题、最佳实践文档、多年编程+爬虫经验分享，提供付费知识星球服务，主动提问，作者会定期回答问题
<p><img alt="知识星球" src="static/images/xingqiu.jpeg" style="width: 100%;" ></p>
- 课程服务：
> 如果你想很快入门这个项目，或者想了具体实现原理，我推荐你看看这个课程，从设计出发一步步带你如何使用，门槛大大降低，同时也是对我开源的支持，如果你能支持我的课程，我将会非常开心～<br>

6
config/base_config.py
View File

@ -89,3 +89,9 @@ XHS_CREATOR_ID_LIST = [
"63e36c9a000000002703502b",
# ........................
]
# 指定Dy创作者ID列表(sec_id)
DY_CREATOR_ID_LIST = [
"MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE",
# ........................
]

36
media_platform/douyin/client.py
View File

@ -199,3 +199,39 @@ class DOUYINClient(AbstractApiClient):
continue
# todo fetch sub comments
return result
async def get_user_info(self, sec_user_id: str):
uri = "/aweme/v1/web/user/profile/other/"
params = {
"sec_user_id": sec_user_id,
"publish_video_strategy_type": 2,
"personal_center_strategy": 1,
}
return await self.get(uri, params)
async def get_user_aweme_posts(self, sec_user_id: str, max_cursor: str = "") -> Dict:
uri = "/aweme/v1/web/aweme/post/"
params = {
"sec_user_id": sec_user_id,
"count": 18,
"max_cursor": max_cursor,
"locate_query": "false",
"publish_video_strategy_type": 2
}
return await self.get(uri, params)
async def get_all_user_aweme_posts(self, sec_user_id: str, callback: Optional[Callable] = None):
posts_has_more = 1
max_cursor = ""
result = []
while posts_has_more == 1:
aweme_post_res = await self.get_user_aweme_posts(sec_user_id, max_cursor)
posts_has_more = aweme_post_res.get("has_more", 0)
max_cursor = aweme_post_res.get("max_cursor")
aweme_list = aweme_post_res.get("aweme_list") if aweme_post_res.get("aweme_list") else []
utils.logger.info(
f"[DOUYINClient.get_all_user_aweme_posts] got sec_user_id:{sec_user_id} video len : {len(aweme_list)}")
if callback:
await callback(aweme_list)
result.extend(aweme_list)
return result

55
media_platform/douyin/core.py
View File

@ -27,8 +27,11 @@ class DouYinCrawler(AbstractCrawler):
context_page: Page
dy_client: DOUYINClient
browser_context: BrowserContext
start_page: int
keyword: str
def __init__(self) -> None:
self.start_page = None
self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" # fixed
self.index_url = "https://www.douyin.com"
@ -64,7 +67,7 @@ class DouYinCrawler(AbstractCrawler):
if not await self.dy_client.pong(browser_context=self.browser_context):
login_obj = DouYinLogin(
login_type=self.login_type,
login_phone="", # you phone number
login_phone="", # you phone number
browser_context=self.browser_context,
context_page=self.context_page,
cookie_str=config.COOKIES
@ -78,6 +81,9 @@ class DouYinCrawler(AbstractCrawler):
elif self.crawler_type == "detail":
# Get the information and comments of the specified post
await self.get_specified_awemes()
elif self.crawler_type == "creator":
# Get the information and comments of the specified creator
await self.get_creators_and_videos()
utils.logger.info("[DouYinCrawler.start] Douyin Crawler finished ...")
@ -107,7 +113,8 @@ class DouYinCrawler(AbstractCrawler):
page += 1
if "data" not in posts_res:
utils.logger.error(f"[DouYinCrawler.search] search douyin keyword: {keyword} failed，账号也许被风控了。")
utils.logger.error(
f"[DouYinCrawler.search] search douyin keyword: {keyword} failed，账号也许被风控了。")
break
for post_item in posts_res.get("data"):
@ -142,10 +149,14 @@ class DouYinCrawler(AbstractCrawler):
utils.logger.error(f"[DouYinCrawler.get_aweme_detail] Get aweme detail error: {ex}")
return None
except KeyError as ex:
utils.logger.error(f"[DouYinCrawler.get_aweme_detail] have not fund note detail aweme_id:{aweme_id}, err: {ex}")
utils.logger.error(
f"[DouYinCrawler.get_aweme_detail] have not fund note detail aweme_id:{aweme_id}, err: {ex}")
return None
async def batch_get_note_comments(self, aweme_list: List[str]) -> None:
"""
Batch get note comments
"""
if not config.ENABLE_GET_COMMENTS:
utils.logger.info(f"[DouYinCrawler.batch_get_note_comments] Crawling comment mode is not enabled")
return
@ -156,7 +167,7 @@ class DouYinCrawler(AbstractCrawler):
task = asyncio.create_task(
self.get_comments(aweme_id, semaphore), name=aweme_id)
task_list.append(task)
if len(task_list) > 0 :
if len(task_list) > 0:
await asyncio.wait(task_list)
async def get_comments(self, aweme_id: str, semaphore: asyncio.Semaphore) -> None:
@ -169,10 +180,44 @@ class DouYinCrawler(AbstractCrawler):
callback=douyin_store.batch_update_dy_aweme_comments
)
utils.logger.info(f"[DouYinCrawler.get_comments] aweme_id: {aweme_id} comments have all been obtained and filtered ...")
utils.logger.info(
f"[DouYinCrawler.get_comments] aweme_id: {aweme_id} comments have all been obtained and filtered ...")
except DataFetchError as e:
utils.logger.error(f"[DouYinCrawler.get_comments] aweme_id: {aweme_id} get comments failed, error: {e}")
async def get_creators_and_videos(self) -> None:
"""
Get the information and videos of the specified creator
"""
utils.logger.info("[DouYinCrawler.get_creators_and_videos] Begin get douyin creators")
for user_id in config.DY_CREATOR_ID_LIST:
creator_info: Dict = await self.dy_client.get_user_info(user_id)
if creator_info:
await douyin_store.save_creator(user_id, creator=creator_info)
# Get all video information of the creator
all_video_list = await self.dy_client.get_all_user_aweme_posts(
sec_user_id=user_id,
callback=self.fetch_creator_video_detail
)
video_ids = [video_item.get("aweme_id") for video_item in all_video_list]
await self.batch_get_note_comments(video_ids)
async def fetch_creator_video_detail(self, video_list: List[Dict]):
"""
Concurrently obtain the specified post list and save the data
"""
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list = [
self.get_aweme_detail(post_item.get("aweme_id"), semaphore) for post_item in video_list
]
note_details = await asyncio.gather(*task_list)
for aweme_item in note_details:
if aweme_item is not None:
await douyin_store.update_douyin_aweme(aweme_item)
@staticmethod
def format_proxy_info(ip_proxy_info: IpInfoModel) -> Tuple[Optional[Dict], Optional[Dict]]:
"""format proxy info for playwright and httpx"""

21
schema/tables.sql
View File

@ -103,6 +103,27 @@ CREATE TABLE `douyin_aweme_comment` (
KEY `idx_douyin_awem_aweme_i_c50049` (`aweme_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='抖音视频评论';
-- ----------------------------
-- Table structure for dy_creator
-- ----------------------------
DROP TABLE IF EXISTS `dy_creator`;
CREATE TABLE `dy_creator` (
`id` int NOT NULL AUTO_INCREMENT COMMENT '自增ID',
`user_id` varchar(128) NOT NULL COMMENT '用户ID',
`nickname` varchar(64) DEFAULT NULL COMMENT '用户昵称',
`avatar` varchar(255) DEFAULT NULL COMMENT '用户头像地址',
`ip_location` varchar(255) DEFAULT NULL COMMENT '评论时的IP地址',
`add_ts` bigint NOT NULL COMMENT '记录添加时间戳',
`last_modify_ts` bigint NOT NULL COMMENT '记录最后修改时间戳',
`desc` longtext COMMENT '用户描述',
`gender` varchar(1) DEFAULT NULL COMMENT '性别',
`follows` varchar(16) DEFAULT NULL COMMENT '关注数',
`fans` varchar(16) DEFAULT NULL COMMENT '粉丝数',
`interaction` varchar(16) DEFAULT NULL COMMENT '获赞数',
`videos_count` varchar(16) DEFAULT NULL COMMENT '作品数',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='抖音博主信息';
-- ----------------------------
-- Table structure for kuaishou_video
-- ----------------------------

28
store/douyin/__init__.py
View File

@ -92,3 +92,31 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict):
f"[store.douyin.update_dy_aweme_comment] douyin aweme comment: {comment_id}, content: {save_comment_item.get('content')}")
await DouyinStoreFactory.create_store().store_comment(comment_item=save_comment_item)
async def save_creator(user_id: str, creator: Dict):
user_info = creator.get('user', {})
gender_map = {
0: '未知',
1: '男',
2: '女'
}
avatar_uri = user_info.get('avatar_300x300', {}).get('uri')
local_db_item = {
'user_id': user_id,
'nickname': user_info.get('nickname'),
'gender': gender_map.get(user_info.get('gender'), '未知'),
'avatar': f"https://p3-pc.douyinpic.com/img/{avatar_uri}" + r"~c5_300x300.jpeg?from=2956013662",
'desc': user_info.get('signature'),
'ip_location': user_info.get('ip_location'),
'follows': user_info.get("following_count", 0),
'fans': user_info.get("max_follower_count", 0),
'interaction': user_info.get("total_favorited", 0),
'videos_count': user_info.get("aweme_count", 0),
"last_modify_ts": utils.get_current_timestamp(),
}
utils.logger.info(f"[store.douyin.save_creator] creator:{local_db_item}")
await DouyinStoreFactory.create_store().store_creator(local_db_item)

54
store/douyin/douyin_store_impl.py
View File

@ -26,13 +26,14 @@ def calculate_number_of_files(file_store_path: str) -> int:
if not os.path.exists(file_store_path):
return 1
try:
return max([int(file_name.split("_")[0])for file_name in os.listdir(file_store_path)])+1
return max([int(file_name.split("_")[0]) for file_name in os.listdir(file_store_path)]) + 1
except ValueError:
return 1
class DouyinCsvStoreImplement(AbstractStore):
csv_store_path: str = "data/douyin"
file_count:int=calculate_number_of_files(csv_store_path)
file_count: int = calculate_number_of_files(csv_store_path)
def make_save_file_name(self, store_type: str) -> str:
"""
@ -65,7 +66,7 @@ class DouyinCsvStoreImplement(AbstractStore):
async def store_content(self, content_item: Dict):
"""
Xiaohongshu content CSV storage implementation
Douyin content CSV storage implementation
Args:
content_item: note item dict
@ -76,7 +77,7 @@ class DouyinCsvStoreImplement(AbstractStore):
async def store_comment(self, comment_item: Dict):
"""
Xiaohongshu comment CSV storage implementation
Douyin comment CSV storage implementation
Args:
comment_item: comment item dict
@ -85,6 +86,17 @@ class DouyinCsvStoreImplement(AbstractStore):
"""
await self.save_data_to_csv(save_item=comment_item, store_type="comments")
async def store_creator(self, creator: Dict):
"""
Douyin creator CSV storage implementation
Args:
creator: creator item dict
Returns:
"""
await self.save_data_to_csv(save_item=creator, store_type="creator")
class DouyinDbStoreImplement(AbstractStore):
async def store_content(self, content_item: Dict):
@ -109,7 +121,6 @@ class DouyinDbStoreImplement(AbstractStore):
else:
await update_content_by_content_id(aweme_id, content_item=content_item)
async def store_comment(self, comment_item: Dict):
"""
Douyin content DB storage implementation
@ -130,11 +141,29 @@ class DouyinDbStoreImplement(AbstractStore):
else:
await update_comment_by_comment_id(comment_id, comment_item=comment_item)
async def store_creator(self, creator: Dict):
"""
Douyin content DB storage implementation
Args:
creator: creator dict
Returns:
"""
from .douyin_store_sql import (add_new_creator, query_creator_by_user_id,
update_creator_by_user_id)
user_id = creator.get("user_id")
user_detail: Dict = await query_creator_by_user_id(user_id)
if not user_detail:
creator["add_ts"] = utils.get_current_timestamp()
await add_new_creator(creator)
else:
await update_creator_by_user_id(user_id, creator)
class DouyinJsonStoreImplement(AbstractStore):
json_store_path: str = "data/douyin"
lock = asyncio.Lock()
file_count:int=calculate_number_of_files(json_store_path)
file_count: int = calculate_number_of_files(json_store_path)
def make_save_file_name(self, store_type: str) -> str:
"""
@ -146,7 +175,6 @@ class DouyinJsonStoreImplement(AbstractStore):
"""
return f"{self.json_store_path}/{self.file_count}_{crawler_type_var.get()}_{store_type}_{utils.get_current_date()}.json"
async def save_data_to_json(self, save_item: Dict, store_type: str):
@ -193,3 +221,15 @@ class DouyinJsonStoreImplement(AbstractStore):
"""
await self.save_data_to_json(comment_item, "comments")
async def store_creator(self, creator: Dict):
"""
Douyin creator CSV storage implementation
Args:
creator: creator item dict
Returns:
"""
await self.save_data_to_json(save_item=creator, store_type="creator")

46
store/douyin/douyin_store_sql.py
View File

@ -100,3 +100,49 @@ async def update_comment_by_comment_id(comment_id: str, comment_item: Dict) -> i
async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
effect_row: int = await async_db_conn.update_table("douyin_aweme_comment", comment_item, "comment_id", comment_id)
return effect_row
async def query_creator_by_user_id(user_id: str) -> Dict:
"""
查询一条创作者记录
Args:
user_id:
Returns:
"""
async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
sql: str = f"select * from dy_creator where user_id = '{user_id}'"
rows: List[Dict] = await async_db_conn.query(sql)
if len(rows) > 0:
return rows[0]
return dict()
async def add_new_creator(creator_item: Dict) -> int:
"""
新增一条创作者信息
Args:
creator_item:
Returns:
"""
async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
last_row_id: int = await async_db_conn.item_to_table("dy_creator", creator_item)
return last_row_id
async def update_creator_by_user_id(user_id: str, creator_item: Dict) -> int:
"""
更新一条创作者信息
Args:
user_id:
creator_item:
Returns:
"""
async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
effect_row: int = await async_db_conn.update_table("dy_creator", creator_item, "user_id", user_id)
return effect_row