MediaCrawler/main.py

import sys
import asyncio
import argparse

import config
from tools import utils
from base import proxy_account_pool
from media_platform.douyin import DouYinCrawler
from media_platform.xhs import XiaoHongShuCrawler


class CrawlerFactory:
    @staticmethod
    def create_crawler(platform: str):
        if platform == "xhs":
            return XiaoHongShuCrawler()
        elif platform == "dy":
            return DouYinCrawler()
        else:
            raise ValueError("Invalid Media Platform Currently only supported xhs or douyin ...")


async def main():
    utils.init_loging_config()
    # define command line params ...
    parser = argparse.ArgumentParser(description='Media crawler program.')
    parser.add_argument('--platform', type=str, help='Media platform select (xhs|dy)...', default=config.PLATFORM)
    parser.add_argument('--lt', type=str, help='Login type (qrcode | phone | cookie)', default=config.LOGIN_TYPE)

    # init account pool
    account_pool = proxy_account_pool.create_account_pool()

    args = parser.parse_args()
    crawler = CrawlerFactory().create_crawler(platform=args.platform)
    crawler.init_config(
        command_args=args,
        account_pool=account_pool
    )
    await crawler.start()

    """
    # retry when exception ...
    while True:
        try:
            await crawler.start()
        except Exception as e:
            logging.info(f"crawler start error: {e} ...")
            await crawler.close()
            # If you encounter an exception
            # sleep for a period of time before retrying
            # to avoid frequent requests that may result in the account being blocked.
            await asyncio.sleep(config.RETRY_INTERVAL)
    """


if __name__ == '__main__':
    try:
        asyncio.run(main())
    except KeyboardInterrupt:
        sys.exit()