62 lines
2.0 KiB
Python
62 lines
2.0 KiB
Python
import argparse
|
|
import asyncio
|
|
import sys
|
|
|
|
import config
|
|
import db
|
|
from base.base_crawler import AbstractCrawler
|
|
from media_platform.bilibili import BilibiliCrawler
|
|
from media_platform.douyin import DouYinCrawler
|
|
from media_platform.kuaishou import KuaishouCrawler
|
|
from media_platform.weibo import WeiboCrawler
|
|
from media_platform.xhs import XiaoHongShuCrawler
|
|
|
|
|
|
class CrawlerFactory:
|
|
CRAWLERS = {
|
|
"xhs": XiaoHongShuCrawler,
|
|
"dy": DouYinCrawler,
|
|
"ks": KuaishouCrawler,
|
|
"bili": BilibiliCrawler,
|
|
"wb": WeiboCrawler
|
|
}
|
|
|
|
@staticmethod
|
|
def create_crawler(platform: str) -> AbstractCrawler:
|
|
crawler_class = CrawlerFactory.CRAWLERS.get(platform)
|
|
if not crawler_class:
|
|
raise ValueError("Invalid Media Platform Currently only supported xhs or dy or ks or bili ...")
|
|
return crawler_class()
|
|
|
|
|
|
async def main():
|
|
# define command line params ...
|
|
parser = argparse.ArgumentParser(description='Media crawler program.')
|
|
parser.add_argument('--platform', type=str, help='Media platform select (xhs | dy | ks | bili | wb)',
|
|
choices=["xhs", "dy", "ks", "bili", "wb"], default=config.PLATFORM)
|
|
parser.add_argument('--lt', type=str, help='Login type (qrcode | phone | cookie)',
|
|
choices=["qrcode", "phone", "cookie"], default=config.LOGIN_TYPE)
|
|
parser.add_argument('--type', type=str, help='crawler type (search | detail)',
|
|
choices=["search", "detail"], default=config.CRAWLER_TYPE)
|
|
|
|
# init db
|
|
if config.SAVE_DATA_OPTION == "db":
|
|
await db.init_db()
|
|
|
|
args = parser.parse_args()
|
|
crawler = CrawlerFactory.create_crawler(platform=args.platform)
|
|
crawler.init_config(
|
|
platform=args.platform,
|
|
login_type=args.lt,
|
|
crawler_type=args.type
|
|
)
|
|
await crawler.start()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
# asyncio.run(main())
|
|
asyncio.get_event_loop().run_until_complete(main())
|
|
except KeyboardInterrupt:
|
|
sys.exit()
|