From 6c4116f2406dac6c5ead69bd84c97e81895b075e Mon Sep 17 00:00:00 2001 From: Relakkes Date: Sun, 2 Jun 2024 11:16:18 +0800 Subject: [PATCH] feat: abstract cache class implementation --- .gitignore | 1 - cache/__init__.py | 5 ++ cache/abs_cache.py | 33 ++++++++++ cache/local_cache.py | 105 ++++++++++++++++++++++++++++++ cache/redis_cache.py | 69 ++++++++++++++++++++ config/db_config.py | 15 ++--- store/douyin/douyin_store_impl.py | 5 +- store/weibo/__init__.py | 2 +- 8 files changed, 221 insertions(+), 14 deletions(-) create mode 100644 cache/__init__.py create mode 100644 cache/abs_cache.py create mode 100644 cache/local_cache.py create mode 100644 cache/redis_cache.py diff --git a/.gitignore b/.gitignore index 4b088ab..24210d2 100644 --- a/.gitignore +++ b/.gitignore @@ -165,6 +165,5 @@ cython_debug/ /temp_image/ /browser_data/ /data/ -/cache */.DS_Store \ No newline at end of file diff --git a/cache/__init__.py b/cache/__init__.py new file mode 100644 index 0000000..2e88155 --- /dev/null +++ b/cache/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# @Author : relakkes@gmail.com +# @Name : 程序员阿江-Relakkes +# @Time : 2024/6/2 11:05 +# @Desc : diff --git a/cache/abs_cache.py b/cache/abs_cache.py new file mode 100644 index 0000000..5558d82 --- /dev/null +++ b/cache/abs_cache.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# @Author : relakkes@gmail.com +# @Name : 程序员阿江-Relakkes +# @Time : 2024/6/2 11:06 +# @Desc : 抽象类 + +from abc import ABC, abstractmethod +from typing import Any, Optional + + +class Cache(ABC): + + @abstractmethod + def get(self, key: str) -> Optional[Any]: + """ + 从缓存中获取键的值。 + 这是一个抽象方法。子类必须实现这个方法。 + :param key: 键 + :return: + """ + raise NotImplementedError + + @abstractmethod + def set(self, key: str, value: Any, expire_time: int) -> None: + """ + 将键的值设置到缓存中。 + 这是一个抽象方法。子类必须实现这个方法。 + :param key: 键 + :param value: 值 + :param expire_time: 过期时间 + :return: + """ + raise NotImplementedError diff --git a/cache/local_cache.py b/cache/local_cache.py new file mode 100644 index 0000000..aebf56e --- /dev/null +++ b/cache/local_cache.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# @Author : relakkes@gmail.com +# @Name : 程序员阿江-Relakkes +# @Time : 2024/6/2 11:05 +# @Desc : 本地缓存 + +import asyncio +import time +from typing import Any, Dict, Optional, Tuple + +from abs_cache import Cache + + +class ExpiringLocalCache(Cache): + + def __init__(self, cron_interval: int = 10): + """ + 初始化本地缓存 + :param cron_interval: 定时清楚cache的时间间隔 + :return: + """ + self._cron_interval = cron_interval + self._cache_container: Dict[str, Tuple[Any, float]] = {} + self._cron_task: Optional[asyncio.Task] = None + # 开启定时清理任务 + self._schedule_clear() + + def __del__(self): + """ + 析构函数，清理定时任务 + :return: + """ + if self._cron_task is not None: + self._cron_task.cancel() + + def get(self, key: str) -> Optional[Any]: + """ + 从缓存中获取键的值 + :param key: + :return: + """ + value, expire_time = self._cache_container.get(key, (None, 0)) + if value is None: + return None + + # 如果键已过期，则删除键并返回None + if expire_time < time.time(): + del self._cache_container[key] + return None + + return value + + def set(self, key: str, value: Any, expire_time: int) -> None: + """ + 将键的值设置到缓存中 + :param key: + :param value: + :param expire_time: + :return: + """ + self._cache_container[key] = (value, time.time() + expire_time) + + def _schedule_clear(self): + """ + 开启定时清理任务, + :return: + """ + + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + self._cron_task = loop.create_task(self._start_clear_cron()) + + def _clear(self): + """ + 根据过期时间清理缓存 + :return: + """ + for key, (value, expire_time) in self._cache_container.items(): + if expire_time < time.time(): + del self._cache_container[key] + + async def _start_clear_cron(self): + """ + 开启定时清理任务 + :return: + """ + while True: + self._clear() + await asyncio.sleep(self._cron_interval) + + + +if __name__ == '__main__': + cache = ExpiringLocalCache(cron_interval=2) + cache.set('name', '程序员阿江-Relakkes', 3) + print(cache.get('key')) + time.sleep(4) + print(cache.get('key')) + del cache + time.sleep(1) + print("done") \ No newline at end of file diff --git a/cache/redis_cache.py b/cache/redis_cache.py new file mode 100644 index 0000000..3fcbc40 --- /dev/null +++ b/cache/redis_cache.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# @Author : relakkes@gmail.com +# @Name : 程序员阿江-Relakkes +# @Time : 2024/5/29 22:57 +# @Desc : RedisCache实现 +import os +import pickle +import time +from typing import Any + +from abs_cache import Cache +from redis import Redis + +from config import db_config + + +class RedisCache(Cache): + def __init__(self) -> None: + # 连接redis, 返回redis客户端 + self._redis_client = self._connet_redis() + + @staticmethod + def _connet_redis() -> Redis: + """ + 连接redis, 返回redis客户端, 这里按需配置redis连接信息 + :return: + """ + return Redis( + host=db_config.REDIS_DB_HOST, + port=db_config.REDIS_DB_PORT, + db=db_config.REDIS_DB_NUM, + password=db_config.REDIS_DB_PWD, + ) + + def get(self, key: str) -> Any: + """ + 从缓存中获取键的值, 并且反序列化 + :param key: + :return: + """ + value = self._redis_client.get(key) + if value is None: + return None + return pickle.loads(value) + + def set(self, key: str, value: Any, expire_time: int) -> None: + """ + 将键的值设置到缓存中, 并且序列化 + :param key: + :param value: + :param expire_time: + :return: + """ + self._redis_client.set(key, pickle.dumps(value), ex=expire_time) + + +if __name__ == '__main__': + redis_cache = RedisCache() + # basic usage + redis_cache.set("name", "程序员阿江-Relakkes", 1) + print(redis_cache.get("name")) # Relakkes + time.sleep(2) + print(redis_cache.get("name")) # None + + # special python type usage + # list + redis_cache.set("list", [1, 2, 3], 10) + _value = redis_cache.get("list") + print(_value, f"value type:{type(_value)}") # [1, 2, 3] diff --git a/config/db_config.py b/config/db_config.py index 4a35ea2..1ee1996 100644 --- a/config/db_config.py +++ b/config/db_config.py @@ -1,11 +1,4 @@ import os -import dotenv - -dotenv.load_dotenv() - -# redis config -REDIS_DB_HOST = "127.0.0.1" # your redis host -REDIS_DB_PWD = os.getenv("REDIS_DB_PWD", "123456") # your redis password # mysql config RELATION_DB_PWD = os.getenv("RELATION_DB_PWD", "123456") @@ -14,8 +7,10 @@ RELATION_DB_HOST = os.getenv("RELATION_DB_HOST", "localhost") RELATION_DB_PORT = os.getenv("RELATION_DB_PORT", "3306") RELATION_DB_NAME = os.getenv("RELATION_DB_NAME", "media_crawler") - RELATION_DB_URL = f"mysql://{RELATION_DB_USER}:{RELATION_DB_PWD}@{RELATION_DB_HOST}:{RELATION_DB_PORT}/{RELATION_DB_NAME}" -# sqlite3 config -# RELATION_DB_URL = f"sqlite://data/media_crawler.sqlite" +# redis config +REDIS_DB_HOST = "127.0.0.1" # your redis host +REDIS_DB_PWD = os.getenv("REDIS_DB_PWD", "123456") # your redis password +REDIS_DB_PORT = os.getenv("REDIS_DB_PORT", 6379) # your redis port +REDIS_DB_NUM = os.getenv("REDIS_DB_NUM", 0) # your redis db num diff --git a/store/douyin/douyin_store_impl.py b/store/douyin/douyin_store_impl.py index b87fe6d..6277b4c 100644 --- a/store/douyin/douyin_store_impl.py +++ b/store/douyin/douyin_store_impl.py @@ -150,8 +150,9 @@ class DouyinDbStoreImplement(AbstractStore): Returns: """ - from .douyin_store_sql import (add_new_creator, query_creator_by_user_id, - update_creator_by_user_id) + from .douyin_store_sql import (add_new_creator, + query_creator_by_user_id, + update_creator_by_user_id) user_id = creator.get("user_id") user_detail: Dict = await query_creator_by_user_id(user_id) if not user_detail: diff --git a/store/weibo/__init__.py b/store/weibo/__init__.py index aad007f..933fdd8 100644 --- a/store/weibo/__init__.py +++ b/store/weibo/__init__.py @@ -3,8 +3,8 @@ # @Time : 2024/1/14 21:34 # @Desc : -from typing import List import re +from typing import List import config