feat: abstract cache class implementation

.gitignore

@@ -165,6 +165,5 @@ cython_debug/
 /temp_image/
 /browser_data/
 /data/
-/cache
 
 */.DS_Store

cache/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+# @Author  : relakkes@gmail.com
+# @Name    : 程序员阿江-Relakkes
+# @Time    : 2024/6/2 11:05
+# @Desc    :

cache/abs_cache.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# @Author  : relakkes@gmail.com
+# @Name    : 程序员阿江-Relakkes
+# @Time    : 2024/6/2 11:06
+# @Desc    : 抽象类
+
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+
+class Cache(ABC):
+
+    @abstractmethod
+    def get(self, key: str) -> Optional[Any]:
+        """
+        从缓存中获取键的值。
+        这是一个抽象方法。子类必须实现这个方法。
+        :param key: 键
+        :return:
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def set(self, key: str, value: Any, expire_time: int) -> None:
+        """
+        将键的值设置到缓存中。
+        这是一个抽象方法。子类必须实现这个方法。
+        :param key: 键
+        :param value: 值
+        :param expire_time: 过期时间
+        :return:
+        """
+        raise NotImplementedError

cache/local_cache.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+# @Author  : relakkes@gmail.com
+# @Name    : 程序员阿江-Relakkes
+# @Time    : 2024/6/2 11:05
+# @Desc    : 本地缓存
+
+import asyncio
+import time
+from typing import Any, Dict, Optional, Tuple
+
+from abs_cache import Cache
+
+
+class ExpiringLocalCache(Cache):
+
+    def __init__(self, cron_interval: int = 10):
+        """
+        初始化本地缓存
+        :param cron_interval: 定时清楚cache的时间间隔
+        :return:
+        """
+        self._cron_interval = cron_interval
+        self._cache_container: Dict[str, Tuple[Any, float]] = {}
+        self._cron_task: Optional[asyncio.Task] = None
+        # 开启定时清理任务
+        self._schedule_clear()
+
+    def __del__(self):
+        """
+        析构函数，清理定时任务
+        :return:
+        """
+        if self._cron_task is not None:
+            self._cron_task.cancel()
+
+    def get(self, key: str) -> Optional[Any]:
+        """
+        从缓存中获取键的值
+        :param key:
+        :return:
+        """
+        value, expire_time = self._cache_container.get(key, (None, 0))
+        if value is None:
+            return None
+
+        # 如果键已过期，则删除键并返回None
+        if expire_time < time.time():
+            del self._cache_container[key]
+            return None
+
+        return value
+
+    def set(self, key: str, value: Any, expire_time: int) -> None:
+        """
+        将键的值设置到缓存中
+        :param key:
+        :param value:
+        :param expire_time:
+        :return:
+        """
+        self._cache_container[key] = (value, time.time() + expire_time)
+
+    def _schedule_clear(self):
+        """
+        开启定时清理任务,
+        :return:
+        """
+
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+        self._cron_task = loop.create_task(self._start_clear_cron())
+
+    def _clear(self):
+        """
+        根据过期时间清理缓存
+        :return:
+        """
+        for key, (value, expire_time) in self._cache_container.items():
+            if expire_time < time.time():
+                del self._cache_container[key]
+
+    async def _start_clear_cron(self):
+        """
+        开启定时清理任务
+        :return:
+        """
+        while True:
+            self._clear()
+            await asyncio.sleep(self._cron_interval)
+
+
+
+if __name__ == '__main__':
+    cache = ExpiringLocalCache(cron_interval=2)
+    cache.set('name', '程序员阿江-Relakkes', 3)
+    print(cache.get('key'))
+    time.sleep(4)
+    print(cache.get('key'))
+    del cache
+    time.sleep(1)
+    print("done")

cache/redis_cache.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# @Author  : relakkes@gmail.com
+# @Name    : 程序员阿江-Relakkes
+# @Time    : 2024/5/29 22:57
+# @Desc    : RedisCache实现
+import os
+import pickle
+import time
+from typing import Any
+
+from abs_cache import Cache
+from redis import Redis
+
+from config import db_config
+
+
+class RedisCache(Cache):
+    def __init__(self) -> None:
+        # 连接redis, 返回redis客户端
+        self._redis_client = self._connet_redis()
+
+    @staticmethod
+    def _connet_redis() -> Redis:
+        """
+        连接redis, 返回redis客户端, 这里按需配置redis连接信息
+        :return:
+        """
+        return Redis(
+            host=db_config.REDIS_DB_HOST,
+            port=db_config.REDIS_DB_PORT,
+            db=db_config.REDIS_DB_NUM,
+            password=db_config.REDIS_DB_PWD,
+        )
+
+    def get(self, key: str) -> Any:
+        """
+        从缓存中获取键的值, 并且反序列化
+        :param key:
+        :return:
+        """
+        value = self._redis_client.get(key)
+        if value is None:
+            return None
+        return pickle.loads(value)
+
+    def set(self, key: str, value: Any, expire_time: int) -> None:
+        """
+        将键的值设置到缓存中, 并且序列化
+        :param key:
+        :param value:
+        :param expire_time:
+        :return:
+        """
+        self._redis_client.set(key, pickle.dumps(value), ex=expire_time)
+
+
+if __name__ == '__main__':
+    redis_cache = RedisCache()
+    # basic usage
+    redis_cache.set("name", "程序员阿江-Relakkes", 1)
+    print(redis_cache.get("name"))  # Relakkes
+    time.sleep(2)
+    print(redis_cache.get("name"))  # None
+
+    # special python type usage
+    # list
+    redis_cache.set("list", [1, 2, 3], 10)
+    _value = redis_cache.get("list")
+    print(_value, f"value type:{type(_value)}")  # [1, 2, 3]

config/db_config.py

@@ -1,11 +1,4 @@
 import os
-import dotenv
-
-dotenv.load_dotenv()
-
-# redis config
-REDIS_DB_HOST = "127.0.0.1"  # your redis host
-REDIS_DB_PWD = os.getenv("REDIS_DB_PWD", "123456")  # your redis password
 
 # mysql config
 RELATION_DB_PWD = os.getenv("RELATION_DB_PWD", "123456")
@@ -14,8 +7,10 @@ RELATION_DB_HOST = os.getenv("RELATION_DB_HOST", "localhost")
 RELATION_DB_PORT = os.getenv("RELATION_DB_PORT", "3306")
 RELATION_DB_NAME = os.getenv("RELATION_DB_NAME", "media_crawler")
 
-
 RELATION_DB_URL = f"mysql://{RELATION_DB_USER}:{RELATION_DB_PWD}@{RELATION_DB_HOST}:{RELATION_DB_PORT}/{RELATION_DB_NAME}"
 
-# sqlite3 config
+# redis config
-# RELATION_DB_URL = f"sqlite://data/media_crawler.sqlite"
+REDIS_DB_HOST = "127.0.0.1"  # your redis host
+REDIS_DB_PWD = os.getenv("REDIS_DB_PWD", "123456")  # your redis password
+REDIS_DB_PORT = os.getenv("REDIS_DB_PORT", 6379)  # your redis port
+REDIS_DB_NUM = os.getenv("REDIS_DB_NUM", 0)  # your redis db num

store/douyin/douyin_store_impl.py

@@ -150,8 +150,9 @@ class DouyinDbStoreImplement(AbstractStore):
         Returns:
 
         """
-        from .douyin_store_sql import (add_new_creator, query_creator_by_user_id,
+        from .douyin_store_sql import (add_new_creator,
-                                    update_creator_by_user_id)
+                                       query_creator_by_user_id,
+                                       update_creator_by_user_id)
         user_id = creator.get("user_id")
         user_detail: Dict = await query_creator_by_user_id(user_id)
         if not user_detail:

store/weibo/__init__.py

@@ -3,8 +3,8 @@
 # @Time    : 2024/1/14 21:34
 # @Desc    :
 
-from typing import List
 import re
+from typing import List
 
 import config