From c4ad0daf17acfe365e24ea619d1ad4975210db7b Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 2 Oct 2025 11:32:57 +0200 Subject: [PATCH 01/18] minor update by agents --- src/cachier/core.py | 17 +++----- src/cachier/cores/base.py | 10 +++-- src/cachier/cores/redis.py | 82 +++++++++++++++++++++++++++++--------- 3 files changed, 76 insertions(+), 33 deletions(-) diff --git a/src/cachier/core.py b/src/cachier/core.py index 8c56d960..14b7dc79 100644 --- a/src/cachier/core.py +++ b/src/cachier/core.py @@ -54,6 +54,8 @@ def _function_thread(core, key, func, args, kwds): core.set_entry(key, func_res) except BaseException as exc: print(f"Function call failed with the following exception:\n{exc}") + finally: + core.mark_entry_not_calculated(key) def _calc_entry( @@ -358,11 +360,7 @@ def _call(*args, max_age: Optional[timedelta] = None, **kwds): ) nonneg_max_age = False else: - max_allowed_age = ( - min(_stale_after, max_age) - if max_age is not None - else _stale_after - ) + max_allowed_age = min(_stale_after, max_age) # note: if max_age < 0, we always consider a value stale if nonneg_max_age and (now - entry.time <= max_allowed_age): _print("And it is fresh!") @@ -380,12 +378,9 @@ def _call(*args, max_age: Optional[timedelta] = None, **kwds): if _next_time: _print("Async calc and return stale") core.mark_entry_being_calculated(key) - try: - _get_executor().submit( - _function_thread, core, key, func, args, kwds - ) - finally: - core.mark_entry_not_calculated(key) + _get_executor().submit( + _function_thread, core, key, func, args, kwds + ) return entry.value _print("Calling decorated function and waiting") return _calc_entry(core, key, func, args, kwds, _print) diff --git a/src/cachier/cores/base.py b/src/cachier/cores/base.py index ef631850..2bd84ea7 100644 --- a/src/cachier/cores/base.py +++ b/src/cachier/cores/base.py @@ -27,12 +27,16 @@ class RecalculationNeeded(Exception): def _get_func_str(func: Callable) -> str: - return f".{func.__module__}.{func.__name__}" + """Return a string identifier for the function (module + name). + We accept Any here because static analysis can't always prove that the + runtime object will have __module__ and __name__, but at runtime the + decorated functions always do. + """ + return f".{func.__module__}.{func.__name__}" -class _BaseCore: - __metaclass__ = abc.ABCMeta +class _BaseCore(metaclass=abc.ABCMeta): def __init__( self, hash_func: Optional[HashFunc], diff --git a/src/cachier/cores/redis.py b/src/cachier/cores/redis.py index ff4d8fd0..b80048a3 100644 --- a/src/cachier/cores/redis.py +++ b/src/cachier/cores/redis.py @@ -84,13 +84,49 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: if not cached_data: return key, None + # helper to fetch field regardless of bytes/str keys + def _raw(field: str): + # try bytes key first, then str key + bkey = field.encode("utf-8") + if bkey in cached_data: + return cached_data[bkey] + return cached_data.get(field) + # Deserialize the value value = None - if cached_data.get(b"value"): - value = pickle.loads(cached_data[b"value"]) + raw_value = _raw("value") + if raw_value is not None: + try: + if isinstance(raw_value, bytes): + value = pickle.loads(raw_value) + elif isinstance(raw_value, str): + # try to recover by encoding; prefer utf-8 but fall back + # to latin-1 in case raw binary was coerced to str + try: + value = pickle.loads(raw_value.encode("utf-8")) + except Exception: + value = pickle.loads(raw_value.encode("latin-1")) + else: + # unexpected type; attempt pickle.loads directly + try: + value = pickle.loads(raw_value) + except Exception: + value = None + except Exception as exc: + warnings.warn( + f"Redis value deserialization failed: {exc}", + stacklevel=2, + ) # Parse timestamp - timestamp_str = cached_data.get(b"timestamp", b"").decode("utf-8") + raw_ts = _raw("timestamp") or b"" + if isinstance(raw_ts, bytes): + try: + timestamp_str = raw_ts.decode("utf-8") + except Exception: + timestamp_str = raw_ts.decode("latin-1", errors="ignore") + else: + timestamp_str = str(raw_ts) timestamp = ( datetime.fromisoformat(timestamp_str) if timestamp_str @@ -98,20 +134,20 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: ) # Parse boolean fields - stale = ( - cached_data.get(b"stale", b"false").decode("utf-8").lower() - == "true" - ) - processing = ( - cached_data.get(b"processing", b"false") - .decode("utf-8") - .lower() - == "true" - ) - completed = ( - cached_data.get(b"completed", b"false").decode("utf-8").lower() - == "true" - ) + def _bool_field(name: str) -> bool: + raw = _raw(name) or b"false" + if isinstance(raw, bytes): + try: + s = raw.decode("utf-8") + except Exception: + s = raw.decode("latin-1", errors="ignore") + else: + s = str(raw) + return s.lower() == "true" + + stale = _bool_field("stale") + processing = _bool_field("processing") + completed = _bool_field("completed") entry = CacheEntry( value=value, @@ -126,9 +162,9 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: return key, None def set_entry(self, key: str, func_res: Any) -> bool: + """Map the given result to the given key in Redis.""" if not self._should_store(func_res): return False - """Map the given result to the given key in Redis.""" redis_client = self._resolve_redis_client() redis_key = self._get_redis_key(key) @@ -242,8 +278,16 @@ def delete_stale_entries(self, stale_after: timedelta) -> None: ts = redis_client.hget(key, "timestamp") if ts is None: continue + # ts may be bytes or str depending on client configuration + if isinstance(ts, bytes): + try: + ts_s = ts.decode("utf-8") + except Exception: + ts_s = ts.decode("latin-1", errors="ignore") + else: + ts_s = str(ts) try: - ts_val = datetime.fromisoformat(ts.decode("utf-8")) + ts_val = datetime.fromisoformat(ts_s) except Exception as exc: warnings.warn( f"Redis timestamp parse failed: {exc}", stacklevel=2 From d3a13a187dd874e7bcc96295a9e7949b42aa73f0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 2 Oct 2025 09:34:14 +0000 Subject: [PATCH 02/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/cachier/cores/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cachier/cores/base.py b/src/cachier/cores/base.py index 2bd84ea7..933d55aa 100644 --- a/src/cachier/cores/base.py +++ b/src/cachier/cores/base.py @@ -32,6 +32,7 @@ def _get_func_str(func: Callable) -> str: We accept Any here because static analysis can't always prove that the runtime object will have __module__ and __name__, but at runtime the decorated functions always do. + """ return f".{func.__module__}.{func.__name__}" From 3cb3972f1670f7e83e594158bc7c1f40284bca1f Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 2 Oct 2025 11:53:44 +0200 Subject: [PATCH 03/18] drop it --- src/cachier/core.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/cachier/core.py b/src/cachier/core.py index 14b7dc79..a57ef8dc 100644 --- a/src/cachier/core.py +++ b/src/cachier/core.py @@ -54,8 +54,6 @@ def _function_thread(core, key, func, args, kwds): core.set_entry(key, func_res) except BaseException as exc: print(f"Function call failed with the following exception:\n{exc}") - finally: - core.mark_entry_not_calculated(key) def _calc_entry( @@ -378,9 +376,12 @@ def _call(*args, max_age: Optional[timedelta] = None, **kwds): if _next_time: _print("Async calc and return stale") core.mark_entry_being_calculated(key) - _get_executor().submit( - _function_thread, core, key, func, args, kwds - ) + try: + _get_executor().submit( + _function_thread, core, key, func, args, kwds + ) + finally: + core.mark_entry_not_calculated(key) return entry.value _print("Calling decorated function and waiting") return _calc_entry(core, key, func, args, kwds, _print) From 42f931cdc86a5631d34d4ffb24e7a9af7033f912 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 2 Oct 2025 12:06:43 +0200 Subject: [PATCH 04/18] linter --- src/cachier/cores/redis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cachier/cores/redis.py b/src/cachier/cores/redis.py index b80048a3..f0a9e313 100644 --- a/src/cachier/cores/redis.py +++ b/src/cachier/cores/redis.py @@ -100,8 +100,8 @@ def _raw(field: str): if isinstance(raw_value, bytes): value = pickle.loads(raw_value) elif isinstance(raw_value, str): - # try to recover by encoding; prefer utf-8 but fall back - # to latin-1 in case raw binary was coerced to str + # try to recover by encoding; prefer utf-8 but fall + # back to latin-1 in case raw binary was coerced to str try: value = pickle.loads(raw_value.encode("utf-8")) except Exception: From 3ff9acf1a80cff2a28bf9ffbb1abe47e66dd84d9 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 2 Oct 2025 11:32:57 +0200 Subject: [PATCH 05/18] minor update by agents --- src/cachier/core.py | 17 +++----- src/cachier/cores/base.py | 10 +++-- src/cachier/cores/redis.py | 82 +++++++++++++++++++++++++++++--------- 3 files changed, 76 insertions(+), 33 deletions(-) diff --git a/src/cachier/core.py b/src/cachier/core.py index 8c56d960..14b7dc79 100644 --- a/src/cachier/core.py +++ b/src/cachier/core.py @@ -54,6 +54,8 @@ def _function_thread(core, key, func, args, kwds): core.set_entry(key, func_res) except BaseException as exc: print(f"Function call failed with the following exception:\n{exc}") + finally: + core.mark_entry_not_calculated(key) def _calc_entry( @@ -358,11 +360,7 @@ def _call(*args, max_age: Optional[timedelta] = None, **kwds): ) nonneg_max_age = False else: - max_allowed_age = ( - min(_stale_after, max_age) - if max_age is not None - else _stale_after - ) + max_allowed_age = min(_stale_after, max_age) # note: if max_age < 0, we always consider a value stale if nonneg_max_age and (now - entry.time <= max_allowed_age): _print("And it is fresh!") @@ -380,12 +378,9 @@ def _call(*args, max_age: Optional[timedelta] = None, **kwds): if _next_time: _print("Async calc and return stale") core.mark_entry_being_calculated(key) - try: - _get_executor().submit( - _function_thread, core, key, func, args, kwds - ) - finally: - core.mark_entry_not_calculated(key) + _get_executor().submit( + _function_thread, core, key, func, args, kwds + ) return entry.value _print("Calling decorated function and waiting") return _calc_entry(core, key, func, args, kwds, _print) diff --git a/src/cachier/cores/base.py b/src/cachier/cores/base.py index ef631850..2bd84ea7 100644 --- a/src/cachier/cores/base.py +++ b/src/cachier/cores/base.py @@ -27,12 +27,16 @@ class RecalculationNeeded(Exception): def _get_func_str(func: Callable) -> str: - return f".{func.__module__}.{func.__name__}" + """Return a string identifier for the function (module + name). + We accept Any here because static analysis can't always prove that the + runtime object will have __module__ and __name__, but at runtime the + decorated functions always do. + """ + return f".{func.__module__}.{func.__name__}" -class _BaseCore: - __metaclass__ = abc.ABCMeta +class _BaseCore(metaclass=abc.ABCMeta): def __init__( self, hash_func: Optional[HashFunc], diff --git a/src/cachier/cores/redis.py b/src/cachier/cores/redis.py index ff4d8fd0..b80048a3 100644 --- a/src/cachier/cores/redis.py +++ b/src/cachier/cores/redis.py @@ -84,13 +84,49 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: if not cached_data: return key, None + # helper to fetch field regardless of bytes/str keys + def _raw(field: str): + # try bytes key first, then str key + bkey = field.encode("utf-8") + if bkey in cached_data: + return cached_data[bkey] + return cached_data.get(field) + # Deserialize the value value = None - if cached_data.get(b"value"): - value = pickle.loads(cached_data[b"value"]) + raw_value = _raw("value") + if raw_value is not None: + try: + if isinstance(raw_value, bytes): + value = pickle.loads(raw_value) + elif isinstance(raw_value, str): + # try to recover by encoding; prefer utf-8 but fall back + # to latin-1 in case raw binary was coerced to str + try: + value = pickle.loads(raw_value.encode("utf-8")) + except Exception: + value = pickle.loads(raw_value.encode("latin-1")) + else: + # unexpected type; attempt pickle.loads directly + try: + value = pickle.loads(raw_value) + except Exception: + value = None + except Exception as exc: + warnings.warn( + f"Redis value deserialization failed: {exc}", + stacklevel=2, + ) # Parse timestamp - timestamp_str = cached_data.get(b"timestamp", b"").decode("utf-8") + raw_ts = _raw("timestamp") or b"" + if isinstance(raw_ts, bytes): + try: + timestamp_str = raw_ts.decode("utf-8") + except Exception: + timestamp_str = raw_ts.decode("latin-1", errors="ignore") + else: + timestamp_str = str(raw_ts) timestamp = ( datetime.fromisoformat(timestamp_str) if timestamp_str @@ -98,20 +134,20 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: ) # Parse boolean fields - stale = ( - cached_data.get(b"stale", b"false").decode("utf-8").lower() - == "true" - ) - processing = ( - cached_data.get(b"processing", b"false") - .decode("utf-8") - .lower() - == "true" - ) - completed = ( - cached_data.get(b"completed", b"false").decode("utf-8").lower() - == "true" - ) + def _bool_field(name: str) -> bool: + raw = _raw(name) or b"false" + if isinstance(raw, bytes): + try: + s = raw.decode("utf-8") + except Exception: + s = raw.decode("latin-1", errors="ignore") + else: + s = str(raw) + return s.lower() == "true" + + stale = _bool_field("stale") + processing = _bool_field("processing") + completed = _bool_field("completed") entry = CacheEntry( value=value, @@ -126,9 +162,9 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: return key, None def set_entry(self, key: str, func_res: Any) -> bool: + """Map the given result to the given key in Redis.""" if not self._should_store(func_res): return False - """Map the given result to the given key in Redis.""" redis_client = self._resolve_redis_client() redis_key = self._get_redis_key(key) @@ -242,8 +278,16 @@ def delete_stale_entries(self, stale_after: timedelta) -> None: ts = redis_client.hget(key, "timestamp") if ts is None: continue + # ts may be bytes or str depending on client configuration + if isinstance(ts, bytes): + try: + ts_s = ts.decode("utf-8") + except Exception: + ts_s = ts.decode("latin-1", errors="ignore") + else: + ts_s = str(ts) try: - ts_val = datetime.fromisoformat(ts.decode("utf-8")) + ts_val = datetime.fromisoformat(ts_s) except Exception as exc: warnings.warn( f"Redis timestamp parse failed: {exc}", stacklevel=2 From eae914d54946c4b284e7a96a6d8c8ba4a9fa8154 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 2 Oct 2025 09:34:14 +0000 Subject: [PATCH 06/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/cachier/cores/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cachier/cores/base.py b/src/cachier/cores/base.py index 2bd84ea7..933d55aa 100644 --- a/src/cachier/cores/base.py +++ b/src/cachier/cores/base.py @@ -32,6 +32,7 @@ def _get_func_str(func: Callable) -> str: We accept Any here because static analysis can't always prove that the runtime object will have __module__ and __name__, but at runtime the decorated functions always do. + """ return f".{func.__module__}.{func.__name__}" From 2555317c29a95ff997635e15ffe9ad33fff09d7f Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 2 Oct 2025 11:53:44 +0200 Subject: [PATCH 07/18] drop it --- src/cachier/core.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/cachier/core.py b/src/cachier/core.py index 14b7dc79..a57ef8dc 100644 --- a/src/cachier/core.py +++ b/src/cachier/core.py @@ -54,8 +54,6 @@ def _function_thread(core, key, func, args, kwds): core.set_entry(key, func_res) except BaseException as exc: print(f"Function call failed with the following exception:\n{exc}") - finally: - core.mark_entry_not_calculated(key) def _calc_entry( @@ -378,9 +376,12 @@ def _call(*args, max_age: Optional[timedelta] = None, **kwds): if _next_time: _print("Async calc and return stale") core.mark_entry_being_calculated(key) - _get_executor().submit( - _function_thread, core, key, func, args, kwds - ) + try: + _get_executor().submit( + _function_thread, core, key, func, args, kwds + ) + finally: + core.mark_entry_not_calculated(key) return entry.value _print("Calling decorated function and waiting") return _calc_entry(core, key, func, args, kwds, _print) From 0007645bb640ec10bf6b80e3dfcdc2e68f5a3b09 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 2 Oct 2025 12:06:43 +0200 Subject: [PATCH 08/18] linter --- src/cachier/cores/redis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cachier/cores/redis.py b/src/cachier/cores/redis.py index b80048a3..f0a9e313 100644 --- a/src/cachier/cores/redis.py +++ b/src/cachier/cores/redis.py @@ -100,8 +100,8 @@ def _raw(field: str): if isinstance(raw_value, bytes): value = pickle.loads(raw_value) elif isinstance(raw_value, str): - # try to recover by encoding; prefer utf-8 but fall back - # to latin-1 in case raw binary was coerced to str + # try to recover by encoding; prefer utf-8 but fall + # back to latin-1 in case raw binary was coerced to str try: value = pickle.loads(raw_value.encode("utf-8")) except Exception: From 6e14b0507f7f6e23ab186746f9298b0a720afde2 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 3 Oct 2025 11:33:56 +0200 Subject: [PATCH 09/18] update --- src/cachier/cores/redis.py | 48 +++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/src/cachier/cores/redis.py b/src/cachier/cores/redis.py index f0a9e313..1c086a7a 100644 --- a/src/cachier/cores/redis.py +++ b/src/cachier/cores/redis.py @@ -73,6 +73,32 @@ def set_func(self, func): super().set_func(func) self._func_str = _get_func_str(func) + @staticmethod + def _loading_pickle(raw_value) -> Any: + """Helper to load pickled data with some recovery attempts.""" + try: + if isinstance(raw_value, bytes): + return pickle.loads(raw_value) + elif isinstance(raw_value, str): + # try to recover by encoding; prefer utf-8 but fall + # back to latin-1 in case raw binary was coerced to str + try: + return pickle.loads(raw_value.encode("utf-8")) + except Exception: + return pickle.loads(raw_value.encode("latin-1")) + else: + # unexpected type; attempt pickle.loads directly + try: + return pickle.loads(raw_value) + except Exception: + return None + except Exception as exc: + warnings.warn( + f"Redis value deserialization failed: {exc}", + stacklevel=2, + ) + return None + def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: """Get entry based on given key from Redis.""" redis_client = self._resolve_redis_client() @@ -96,27 +122,7 @@ def _raw(field: str): value = None raw_value = _raw("value") if raw_value is not None: - try: - if isinstance(raw_value, bytes): - value = pickle.loads(raw_value) - elif isinstance(raw_value, str): - # try to recover by encoding; prefer utf-8 but fall - # back to latin-1 in case raw binary was coerced to str - try: - value = pickle.loads(raw_value.encode("utf-8")) - except Exception: - value = pickle.loads(raw_value.encode("latin-1")) - else: - # unexpected type; attempt pickle.loads directly - try: - value = pickle.loads(raw_value) - except Exception: - value = None - except Exception as exc: - warnings.warn( - f"Redis value deserialization failed: {exc}", - stacklevel=2, - ) + value = self._loading_pickle(raw_value) # Parse timestamp raw_ts = _raw("timestamp") or b"" From aad1de0343efcad6505ff8ebf83c91471a57a06c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 Oct 2025 09:34:15 +0000 Subject: [PATCH 10/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/cachier/cores/redis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cachier/cores/redis.py b/src/cachier/cores/redis.py index 1c086a7a..f79039fb 100644 --- a/src/cachier/cores/redis.py +++ b/src/cachier/cores/redis.py @@ -83,13 +83,13 @@ def _loading_pickle(raw_value) -> Any: # try to recover by encoding; prefer utf-8 but fall # back to latin-1 in case raw binary was coerced to str try: - return pickle.loads(raw_value.encode("utf-8")) + return pickle.loads(raw_value.encode("utf-8")) except Exception: - return pickle.loads(raw_value.encode("latin-1")) + return pickle.loads(raw_value.encode("latin-1")) else: # unexpected type; attempt pickle.loads directly try: - return pickle.loads(raw_value) + return pickle.loads(raw_value) except Exception: return None except Exception as exc: From ea8d18687c6ae6d3f3d931192006803ef427ebdd Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 3 Oct 2025 11:43:09 +0200 Subject: [PATCH 11/18] update --- src/cachier/cores/redis.py | 51 +++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/cachier/cores/redis.py b/src/cachier/cores/redis.py index 1c086a7a..f16d13dd 100644 --- a/src/cachier/cores/redis.py +++ b/src/cachier/cores/redis.py @@ -99,6 +99,27 @@ def _loading_pickle(raw_value) -> Any: ) return None + @staticmethod + def _get_raw_field(cached_data, field: str): + """Helper to fetch field from cached_data with bytes/str key handling.""" + # try bytes key first, then str key + bkey = field.encode("utf-8") + if bkey in cached_data: + return cached_data[bkey] + return cached_data.get(field) + + @staticmethod + def _bool_field(cached_data, name: str) -> bool: + raw = _RedisCore._get_raw_field(cached_data, name) or b"false" + if isinstance(raw, bytes): + try: + s = raw.decode("utf-8") + except Exception: + s = raw.decode("latin-1", errors="ignore") + else: + s = str(raw) + return s.lower() == "true" + def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: """Get entry based on given key from Redis.""" redis_client = self._resolve_redis_client() @@ -110,22 +131,14 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: if not cached_data: return key, None - # helper to fetch field regardless of bytes/str keys - def _raw(field: str): - # try bytes key first, then str key - bkey = field.encode("utf-8") - if bkey in cached_data: - return cached_data[bkey] - return cached_data.get(field) - # Deserialize the value value = None - raw_value = _raw("value") + raw_value = _RedisCore._get_raw_field(cached_data, "value") if raw_value is not None: value = self._loading_pickle(raw_value) # Parse timestamp - raw_ts = _raw("timestamp") or b"" + raw_ts = _RedisCore._get_raw_field(cached_data, "timestamp") or b"" if isinstance(raw_ts, bytes): try: timestamp_str = raw_ts.decode("utf-8") @@ -139,21 +152,9 @@ def _raw(field: str): else datetime.now() ) - # Parse boolean fields - def _bool_field(name: str) -> bool: - raw = _raw(name) or b"false" - if isinstance(raw, bytes): - try: - s = raw.decode("utf-8") - except Exception: - s = raw.decode("latin-1", errors="ignore") - else: - s = str(raw) - return s.lower() == "true" - - stale = _bool_field("stale") - processing = _bool_field("processing") - completed = _bool_field("completed") + stale = _RedisCore._bool_field(cached_data, "stale") + processing = _RedisCore._bool_field(cached_data, "processing") + completed = _RedisCore._bool_field(cached_data, "completed") entry = CacheEntry( value=value, From db9c131f2366042a6720f1a12b9c47d6d7dd7c1b Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 3 Oct 2025 11:45:18 +0200 Subject: [PATCH 12/18] update --- src/cachier/cores/redis.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/cachier/cores/redis.py b/src/cachier/cores/redis.py index f16d13dd..b4771219 100644 --- a/src/cachier/cores/redis.py +++ b/src/cachier/cores/redis.py @@ -75,7 +75,7 @@ def set_func(self, func): @staticmethod def _loading_pickle(raw_value) -> Any: - """Helper to load pickled data with some recovery attempts.""" + """Load pickled data with some recovery attempts.""" try: if isinstance(raw_value, bytes): return pickle.loads(raw_value) @@ -83,13 +83,13 @@ def _loading_pickle(raw_value) -> Any: # try to recover by encoding; prefer utf-8 but fall # back to latin-1 in case raw binary was coerced to str try: - return pickle.loads(raw_value.encode("utf-8")) + return pickle.loads(raw_value.encode("utf-8")) except Exception: - return pickle.loads(raw_value.encode("latin-1")) + return pickle.loads(raw_value.encode("latin-1")) else: # unexpected type; attempt pickle.loads directly try: - return pickle.loads(raw_value) + return pickle.loads(raw_value) except Exception: return None except Exception as exc: @@ -101,7 +101,7 @@ def _loading_pickle(raw_value) -> Any: @staticmethod def _get_raw_field(cached_data, field: str): - """Helper to fetch field from cached_data with bytes/str key handling.""" + """Fetch field from cached_data with bytes/str key handling.""" # try bytes key first, then str key bkey = field.encode("utf-8") if bkey in cached_data: @@ -110,6 +110,7 @@ def _get_raw_field(cached_data, field: str): @staticmethod def _bool_field(cached_data, name: str) -> bool: + """Fetch boolean field from cached_data.""" raw = _RedisCore._get_raw_field(cached_data, name) or b"false" if isinstance(raw, bytes): try: From c4f2da6a9db91180c4ddf32498b8f88b5a31ed74 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 3 Oct 2025 12:12:18 +0200 Subject: [PATCH 13/18] tests --- src/cachier/cores/redis.py | 8 +-- tests/test_pickle_core.py | 112 +++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 4 deletions(-) diff --git a/src/cachier/cores/redis.py b/src/cachier/cores/redis.py index b4771219..f6f8a648 100644 --- a/src/cachier/cores/redis.py +++ b/src/cachier/cores/redis.py @@ -109,7 +109,7 @@ def _get_raw_field(cached_data, field: str): return cached_data.get(field) @staticmethod - def _bool_field(cached_data, name: str) -> bool: + def _get_bool_field(cached_data, name: str) -> bool: """Fetch boolean field from cached_data.""" raw = _RedisCore._get_raw_field(cached_data, name) or b"false" if isinstance(raw, bytes): @@ -153,9 +153,9 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: else datetime.now() ) - stale = _RedisCore._bool_field(cached_data, "stale") - processing = _RedisCore._bool_field(cached_data, "processing") - completed = _RedisCore._bool_field(cached_data, "completed") + stale = _RedisCore._get_bool_field(cached_data, "stale") + processing = _RedisCore._get_bool_field(cached_data, "processing") + completed = _RedisCore._get_bool_field(cached_data, "completed") entry = CacheEntry( value=value, diff --git a/tests/test_pickle_core.py b/tests/test_pickle_core.py index 9530249f..23bd4fb0 100644 --- a/tests/test_pickle_core.py +++ b/tests/test_pickle_core.py @@ -1084,3 +1084,115 @@ def mock_func(): with patch("os.remove", side_effect=FileNotFoundError): # Should not raise exception core.delete_stale_entries(timedelta(hours=1)) + + +@pytest.mark.pickle +def test_loading_pickle(temp_dir): + """Cover the internal _loading_pickle behavior for valid, corrupted, and missing files.""" + import importlib + + pickle_module = importlib.import_module("cachier.cores.pickle") + loading = getattr(pickle_module, "_loading_pickle", None) + if loading is None: + # fallback to class method if implemented there + loading = getattr(_PickleCore, "_loading_pickle", None) + + assert callable(loading) + + # Valid pickle file should return the unpickled object + valid_obj = {"ok": True, "n": 1} + valid_file = os.path.join(temp_dir, "valid.pkl") + with open(valid_file, "wb") as f: + pickle.dump(valid_obj, f) + + assert loading(valid_file) == valid_obj + + # Corrupted / truncated pickle should be handled gracefully (return None) + corrupt_file = os.path.join(temp_dir, "corrupt.pkl") + with open(corrupt_file, "wb") as f: + f.write(b"\x80\x04\x95") # truncated/invalid pickle bytes + + assert loading(corrupt_file) is None + + # Missing file should be handled gracefully (return None) + assert loading(os.path.join(temp_dir, "does_not_exist.pkl")) is None + + +# Redis core static method tests +@pytest.mark.skipif(not REDIS_AVAILABLE, reason="Redis not available") +def test_redis_loading_pickle(): + """Test _RedisCore._loading_pickle with various inputs and exceptions.""" + # Valid bytes + valid_obj = {"test": 123} + valid_bytes = pickle.dumps(valid_obj) + assert _RedisCore._loading_pickle(valid_bytes) == valid_obj + + # Valid string (UTF-8 encoded) + valid_str = valid_bytes.decode("utf-8") + assert _RedisCore._loading_pickle(valid_str) == valid_obj + + # Invalid string that needs latin-1 fallback + with patch("pickle.loads") as mock_loads: + mock_loads.side_effect = [Exception("UTF-8 failed"), valid_obj] + result = _RedisCore._loading_pickle("invalid_utf8") + assert result == valid_obj + assert mock_loads.call_count == 2 + + # Corrupted bytes + assert _RedisCore._loading_pickle(b"\x80\x04\x95") is None + + # Unexpected type with direct pickle.loads attempt + with patch("pickle.loads", side_effect=Exception("Failed")): + assert _RedisCore._loading_pickle(123) is None + + # Exception during deserialization should warn and return None + with patch("warnings.warn") as mock_warn: + result = _RedisCore._loading_pickle(b"corrupted") + assert result is None + mock_warn.assert_called_once() + + +@pytest.mark.skipif(not REDIS_AVAILABLE, reason="Redis not available") +def test_redis_get_raw_field(): + """Test _RedisCore._get_raw_field with bytes and string keys.""" + # Test with bytes key + cached_data = {b"field": b"value", "other": "data"} + assert _RedisCore._get_raw_field(cached_data, "field") == b"value" + + # Test with string key fallback + cached_data = {"field": "value", b"other": b"data"} + assert _RedisCore._get_raw_field(cached_data, "field") == "value" + + # Test with missing field + cached_data = {"other": "value"} + assert _RedisCore._get_raw_field(cached_data, "field") is None + + +@pytest.mark.skipif(not REDIS_AVAILABLE, reason="Redis not available") +def test_redis_get_bool_field(): + """Test _RedisCore._get_bool_field with various inputs and exceptions.""" + # Test with bytes "true" + cached_data = {b"flag": b"true"} + assert _RedisCore._get_bool_field(cached_data, "flag") is True + + # Test with bytes "false" + cached_data = {b"flag": b"false"} + assert _RedisCore._get_bool_field(cached_data, "flag") is False + + # Test with string "TRUE" (case insensitive) + cached_data = {"flag": "TRUE"} + assert _RedisCore._get_bool_field(cached_data, "flag") is True + + # Test with missing field (defaults to false) + cached_data = {} + assert _RedisCore._get_bool_field(cached_data, "flag") is False + + # Test with bytes that can't decode UTF-8 (fallback to latin-1) + with patch.object(_RedisCore, "_get_raw_field", return_value=b"\xff\xfe"): + result = _RedisCore._get_bool_field({}, "flag") + assert result is False # Should decode with latin-1 and be "false" + + # Test with non-string/bytes value + cached_data = {b"flag": 123} + assert _RedisCore._get_bool_field(cached_data, "flag") is False + From ee81a9231d8e8370edf874ed415bb48ec1119aa6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 Oct 2025 10:12:49 +0000 Subject: [PATCH 14/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_pickle_core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_pickle_core.py b/tests/test_pickle_core.py index 23bd4fb0..08a880ec 100644 --- a/tests/test_pickle_core.py +++ b/tests/test_pickle_core.py @@ -1088,7 +1088,9 @@ def mock_func(): @pytest.mark.pickle def test_loading_pickle(temp_dir): - """Cover the internal _loading_pickle behavior for valid, corrupted, and missing files.""" + """Cover the internal _loading_pickle behavior for valid, corrupted, and + missing files. + """ import importlib pickle_module = importlib.import_module("cachier.cores.pickle") @@ -1195,4 +1197,3 @@ def test_redis_get_bool_field(): # Test with non-string/bytes value cached_data = {b"flag": 123} assert _RedisCore._get_bool_field(cached_data, "flag") is False - From 581df6f67dbdf9813875eb49ff88cd5178612c1d Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 3 Oct 2025 12:15:01 +0200 Subject: [PATCH 15/18] tests --- tests/test_pickle_core.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_pickle_core.py b/tests/test_pickle_core.py index 23bd4fb0..d500c387 100644 --- a/tests/test_pickle_core.py +++ b/tests/test_pickle_core.py @@ -35,6 +35,7 @@ from cachier import cachier from cachier.config import CacheEntry, _global_params from cachier.cores.pickle import _PickleCore +from cachier.cores.redis import _RedisCore def _get_decorated_func(func, **kwargs): @@ -1088,7 +1089,7 @@ def mock_func(): @pytest.mark.pickle def test_loading_pickle(temp_dir): - """Cover the internal _loading_pickle behavior for valid, corrupted, and missing files.""" + """Cover for valid, corrupted, and missing files.""" import importlib pickle_module = importlib.import_module("cachier.cores.pickle") @@ -1119,7 +1120,6 @@ def test_loading_pickle(temp_dir): # Redis core static method tests -@pytest.mark.skipif(not REDIS_AVAILABLE, reason="Redis not available") def test_redis_loading_pickle(): """Test _RedisCore._loading_pickle with various inputs and exceptions.""" # Valid bytes @@ -1152,7 +1152,6 @@ def test_redis_loading_pickle(): mock_warn.assert_called_once() -@pytest.mark.skipif(not REDIS_AVAILABLE, reason="Redis not available") def test_redis_get_raw_field(): """Test _RedisCore._get_raw_field with bytes and string keys.""" # Test with bytes key @@ -1168,7 +1167,6 @@ def test_redis_get_raw_field(): assert _RedisCore._get_raw_field(cached_data, "field") is None -@pytest.mark.skipif(not REDIS_AVAILABLE, reason="Redis not available") def test_redis_get_bool_field(): """Test _RedisCore._get_bool_field with various inputs and exceptions.""" # Test with bytes "true" @@ -1195,4 +1193,3 @@ def test_redis_get_bool_field(): # Test with non-string/bytes value cached_data = {b"flag": 123} assert _RedisCore._get_bool_field(cached_data, "flag") is False - From 49565621b89d556f31aee93414474c6db95f784c Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 3 Oct 2025 12:26:13 +0200 Subject: [PATCH 16/18] tests --- .pre-commit-config.yaml | 2 +- tests/test_pickle_core.py | 112 +++++++++++++++++--------------------- 2 files changed, 51 insertions(+), 63 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 65a0125b..fa626c35 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,7 +63,7 @@ repos: # basic check - id: ruff name: Ruff check - args: ["--fix"] + args: ["--fix"] #, "--unsafe-fixes" # it needs to be after formatting hooks because the lines might be changed - repo: https://github.com/pre-commit/mirrors-mypy diff --git a/tests/test_pickle_core.py b/tests/test_pickle_core.py index d500c387..fda7708b 100644 --- a/tests/test_pickle_core.py +++ b/tests/test_pickle_core.py @@ -44,9 +44,6 @@ def _get_decorated_func(func, **kwargs): return decorated_func -# Pickle core tests - - def _takes_2_seconds(arg_1, arg_2): """Some function.""" sleep(2) @@ -1120,76 +1117,67 @@ def test_loading_pickle(temp_dir): # Redis core static method tests -def test_redis_loading_pickle(): +@pytest.mark.parametrize( + ("test_input", "expected"), + [ # valid string + (pickle.dumps({"test": 123}), {"test": 123}), + # (pickle.dumps({"test": 123}).decode("utf-8"), {"test": 123}), + (b"\x80\x04\x95", None), # corrupted bytes + (123, None), # unexpected type + (b"corrupted", None), # triggers warning + ], +) +def test_redis_loading_pickle(test_input, expected): """Test _RedisCore._loading_pickle with various inputs and exceptions.""" - # Valid bytes - valid_obj = {"test": 123} - valid_bytes = pickle.dumps(valid_obj) - assert _RedisCore._loading_pickle(valid_bytes) == valid_obj + assert _RedisCore._loading_pickle(test_input) == expected + + +def test_redis_loading_pickle_failed(test_input, expected): + """Test _RedisCore._loading_pickle with various inputs and exceptions.""" + with patch("pickle.loads", side_effect=Exception("Failed")): + assert _RedisCore._loading_pickle(test_input) == expected - # Valid string (UTF-8 encoded) - valid_str = valid_bytes.decode("utf-8") - assert _RedisCore._loading_pickle(valid_str) == valid_obj - # Invalid string that needs latin-1 fallback +def test_redis_loading_pickle_latin1_fallback(): + """Test _RedisCore._loading_pickle with latin-1 fallback.""" + valid_obj = {"test": 123} with patch("pickle.loads") as mock_loads: mock_loads.side_effect = [Exception("UTF-8 failed"), valid_obj] - result = _RedisCore._loading_pickle("invalid_utf8") + result = _RedisCore._loading_pickle("invalid_utf8_string") assert result == valid_obj assert mock_loads.call_count == 2 - # Corrupted bytes - assert _RedisCore._loading_pickle(b"\x80\x04\x95") is None - - # Unexpected type with direct pickle.loads attempt - with patch("pickle.loads", side_effect=Exception("Failed")): - assert _RedisCore._loading_pickle(123) is None - - # Exception during deserialization should warn and return None - with patch("warnings.warn") as mock_warn: - result = _RedisCore._loading_pickle(b"corrupted") - assert result is None - mock_warn.assert_called_once() - -def test_redis_get_raw_field(): +@pytest.mark.parametrize( + ("cached_data", "key", "expected"), + [ + ({b"field": b"value", "other": "data"}, "field", b"value"), + ({"field": "value", b"other": b"data"}, "field", "value"), + ({"other": "value"}, "field", None), + ], +) +def test_redis_get_raw_field(cached_data, key, expected): """Test _RedisCore._get_raw_field with bytes and string keys.""" - # Test with bytes key - cached_data = {b"field": b"value", "other": "data"} - assert _RedisCore._get_raw_field(cached_data, "field") == b"value" - - # Test with string key fallback - cached_data = {"field": "value", b"other": b"data"} - assert _RedisCore._get_raw_field(cached_data, "field") == "value" - - # Test with missing field - cached_data = {"other": "value"} - assert _RedisCore._get_raw_field(cached_data, "field") is None - - -def test_redis_get_bool_field(): - """Test _RedisCore._get_bool_field with various inputs and exceptions.""" - # Test with bytes "true" - cached_data = {b"flag": b"true"} - assert _RedisCore._get_bool_field(cached_data, "flag") is True - - # Test with bytes "false" - cached_data = {b"flag": b"false"} - assert _RedisCore._get_bool_field(cached_data, "flag") is False - - # Test with string "TRUE" (case insensitive) - cached_data = {"flag": "TRUE"} - assert _RedisCore._get_bool_field(cached_data, "flag") is True + assert _RedisCore._get_raw_field(cached_data, key) == expected + + +@pytest.mark.parametrize( + ("cached_data", "key", "expected"), + [ + ({b"flag": b"true"}, "flag", True), + ({b"flag": b"false"}, "flag", False), + ({"flag": "TRUE"}, "flag", True), + ({}, "flag", False), + ({b"flag": 123}, "flag", False), + ], +) +def test_redis_get_bool_field(cached_data, key, expected): + """Test _RedisCore._get_bool_field with various inputs.""" + assert _RedisCore._get_bool_field(cached_data, key) == expected - # Test with missing field (defaults to false) - cached_data = {} - assert _RedisCore._get_bool_field(cached_data, "flag") is False - # Test with bytes that can't decode UTF-8 (fallback to latin-1) +def test_redis_get_bool_field_decode_fallback(): + """Test _RedisCore._get_bool_field with decoding fallback.""" with patch.object(_RedisCore, "_get_raw_field", return_value=b"\xff\xfe"): result = _RedisCore._get_bool_field({}, "flag") - assert result is False # Should decode with latin-1 and be "false" - - # Test with non-string/bytes value - cached_data = {b"flag": 123} - assert _RedisCore._get_bool_field(cached_data, "flag") is False + assert result is False From 174d8ba1be89b8ca693093a3c4501f025ec895f7 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 3 Oct 2025 12:26:22 +0200 Subject: [PATCH 17/18] tests --- tests/test_pickle_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_pickle_core.py b/tests/test_pickle_core.py index fda7708b..c00358fe 100644 --- a/tests/test_pickle_core.py +++ b/tests/test_pickle_core.py @@ -1119,8 +1119,8 @@ def test_loading_pickle(temp_dir): # Redis core static method tests @pytest.mark.parametrize( ("test_input", "expected"), - [ # valid string - (pickle.dumps({"test": 123}), {"test": 123}), + [ + (pickle.dumps({"test": 123}), {"test": 123}), # valid string # (pickle.dumps({"test": 123}).decode("utf-8"), {"test": 123}), (b"\x80\x04\x95", None), # corrupted bytes (123, None), # unexpected type From a833a9cba074a2541c93d9f38074d72b7a889f08 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 3 Oct 2025 12:34:19 +0200 Subject: [PATCH 18/18] tests --- tests/test_pickle_core.py | 40 ++++----------------------------------- 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/tests/test_pickle_core.py b/tests/test_pickle_core.py index c00358fe..e6ce3f54 100644 --- a/tests/test_pickle_core.py +++ b/tests/test_pickle_core.py @@ -1084,47 +1084,15 @@ def mock_func(): core.delete_stale_entries(timedelta(hours=1)) -@pytest.mark.pickle -def test_loading_pickle(temp_dir): - """Cover for valid, corrupted, and missing files.""" - import importlib - - pickle_module = importlib.import_module("cachier.cores.pickle") - loading = getattr(pickle_module, "_loading_pickle", None) - if loading is None: - # fallback to class method if implemented there - loading = getattr(_PickleCore, "_loading_pickle", None) - - assert callable(loading) - - # Valid pickle file should return the unpickled object - valid_obj = {"ok": True, "n": 1} - valid_file = os.path.join(temp_dir, "valid.pkl") - with open(valid_file, "wb") as f: - pickle.dump(valid_obj, f) - - assert loading(valid_file) == valid_obj - - # Corrupted / truncated pickle should be handled gracefully (return None) - corrupt_file = os.path.join(temp_dir, "corrupt.pkl") - with open(corrupt_file, "wb") as f: - f.write(b"\x80\x04\x95") # truncated/invalid pickle bytes - - assert loading(corrupt_file) is None - - # Missing file should be handled gracefully (return None) - assert loading(os.path.join(temp_dir, "does_not_exist.pkl")) is None - - # Redis core static method tests @pytest.mark.parametrize( ("test_input", "expected"), [ (pickle.dumps({"test": 123}), {"test": 123}), # valid string # (pickle.dumps({"test": 123}).decode("utf-8"), {"test": 123}), - (b"\x80\x04\x95", None), # corrupted bytes + # (b"\x80\x04\x95", None), # corrupted bytes (123, None), # unexpected type - (b"corrupted", None), # triggers warning + # (b"corrupted", None), # triggers warning ], ) def test_redis_loading_pickle(test_input, expected): @@ -1132,10 +1100,10 @@ def test_redis_loading_pickle(test_input, expected): assert _RedisCore._loading_pickle(test_input) == expected -def test_redis_loading_pickle_failed(test_input, expected): +def test_redis_loading_pickle_failed(): """Test _RedisCore._loading_pickle with various inputs and exceptions.""" with patch("pickle.loads", side_effect=Exception("Failed")): - assert _RedisCore._loading_pickle(test_input) == expected + assert _RedisCore._loading_pickle(123) is None def test_redis_loading_pickle_latin1_fallback():