diff --git a/test_app/tests/lib/cache/test_cache.py b/test_app/tests/lib/cache/test_cache.py deleted file mode 100644 index 63a69d3ed..000000000 --- a/test_app/tests/lib/cache/test_cache.py +++ /dev/null @@ -1,132 +0,0 @@ -import logging -import time - -from django.core import cache as django_cache -from django.core.cache.backends.base import BaseCache -from django.test import override_settings - -from ansible_base.lib.cache.fallback_cache import FALLBACK_CACHE, PRIMARY_CACHE - - -class BreakableCache(BaseCache): - _instance = None - - def __new__(cls, *args, **kwargs): - if cls._instance is None: - cls._instance = super(BreakableCache, cls).__new__(cls) - cls.__initialized = False - return cls._instance - - def __init__(self, location, params): - if self.__initialized: - return - self.cache = {} - options = params.get("OPTIONS", {}) - self.working = options.get("working", True) - self.__initialized = True - - def add(self, key, value, timeout=300, version=None): - self.cache[key] = value - - def get(self, key, default=None, version=None): - if self.working: - return self.cache.get(key, default) - else: - raise RuntimeError(f"Sorry, cache no worky {self}") - - def set(self, key, value, timeout=300, version=None, client=None): - self.cache[key] = value - - def delete(self, key, version=None): - self.cache.pop(key, None) - - def clear(self): - self.cache = {} - - def breakit(self): - self.working = False - - def fixit(self): - self.working = True - - -cache_settings = { - 'default': { - 'BACKEND': 'ansible_base.lib.cache.fallback_cache.DABCacheWithFallback', - }, - 'primary': { - 'BACKEND': 'test_app.tests.lib.cache.test_cache.BreakableCache', - 'LOCATION': 'primary', - }, - 'fallback': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': 'fallback', - }, -} - - -@override_settings(CACHES=cache_settings) -def test_fallback_cache(): - cache = django_cache.caches.create_connection('default') - - primary = cache._primary_cache - fallback = cache._fallback_cache - cache.set('key', 'val1') - assert primary.get('key') == 'val1' - assert fallback.get('key') is None - - primary.set('tobecleared', True) - primary.breakit() - - # Breaks primary - cache.get('key') - - # Sets in fallback - cache.set('key', 'val2') - - assert cache.get('key', 'val2') - - assert cache.get_active_cache() == FALLBACK_CACHE - - primary.fixit() - - # Check until primary is back - timeout = time.time() + 30 - while True: - if cache.get_active_cache() == PRIMARY_CACHE: - break - if time.time() > timeout: - assert False - time.sleep(1) - - # Ensure caches were cleared - assert cache.get('key') is None - assert fallback.get('key') is None - assert cache.get('tobecleared') is None - - cache.set('key2', 'val3') - - assert cache.get('key2') == 'val3' - - -@override_settings(CACHES=cache_settings) -def test_dead_primary(): - primary_cache = django_cache.caches.create_connection('primary') - primary_cache.breakit() - - # Kill post-shutdown logging from unfinished recovery checker - logging.getLogger('ansible_base.cache.fallback_cache').setLevel(logging.CRITICAL) - - cache = django_cache.caches.create_connection('default') - - cache.set('key', 'val') - cache.get('key') - - # Check until fallback is set - timeout = time.time() + 30 - while True: - if cache.get_active_cache() == FALLBACK_CACHE: - break - if time.time() > timeout: - assert False - time.sleep(1) diff --git a/test_app/tests/lib/cache/test_fallback_cache.py b/test_app/tests/lib/cache/test_fallback_cache.py new file mode 100644 index 000000000..487823ddc --- /dev/null +++ b/test_app/tests/lib/cache/test_fallback_cache.py @@ -0,0 +1,226 @@ +import logging +import tempfile +import time +from pathlib import Path +from unittest import mock + +import pytest +from django.core import cache as django_cache +from django.core.cache.backends.base import BaseCache +from django.test import override_settings + +from ansible_base.lib.cache.fallback_cache import FALLBACK_CACHE, PRIMARY_CACHE, DABCacheWithFallback + + +class BreakableCache(BaseCache): + _instance = None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super(BreakableCache, cls).__new__(cls) + cls.__initialized = False + return cls._instance + + def __init__(self, location, params): + if self.__initialized: + return + self.cache = {} + options = params.get("OPTIONS", {}) + self.working = options.get("working", True) + self.__initialized = True + + def add(self, key, value, timeout=300, version=None): + self.cache[key] = value + + def get(self, key, default=None, version=None): + if self.working: + return self.cache.get(key, default) + else: + raise RuntimeError(f"Sorry, cache no worky {self}") + + def set(self, key, value, timeout=300, version=None, client=None): + self.cache[key] = value + + def delete(self, key, version=None): + self.cache.pop(key, None) + + def clear(self): + self.cache = {} + + def breakit(self): + self.working = False + + def fixit(self): + self.working = True + + +cache_settings = { + 'default': { + 'BACKEND': 'ansible_base.lib.cache.fallback_cache.DABCacheWithFallback', + }, + 'primary': { + 'BACKEND': 'test_app.tests.lib.cache.test_fallback_cache.BreakableCache', + 'LOCATION': 'primary', + }, + 'fallback': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + 'LOCATION': 'fallback', + }, +} + + +@override_settings(CACHES=cache_settings) +def test_fallback_cache(): + cache = django_cache.caches.create_connection('default') + + primary = cache._primary_cache + fallback = cache._fallback_cache + cache.set('key', 'val1') + assert primary.get('key') == 'val1' + assert fallback.get('key') is None + + primary.set('tobecleared', True) + primary.breakit() + + # Breaks primary + cache.get('key') + + # Sets in fallback + cache.set('key', 'val2') + + assert cache.get('key', 'val2') + + assert cache.get_active_cache() == FALLBACK_CACHE + + primary.fixit() + + # Check until primary is back + timeout = time.time() + 30 + while True: + if cache.get_active_cache() == PRIMARY_CACHE: + break + if time.time() > timeout: + assert False + time.sleep(1) + + # Ensure caches were cleared + assert cache.get('key') is None + assert fallback.get('key') is None + assert cache.get('tobecleared') is None + + cache.set('key2', 'val3') + + assert cache.get('key2') == 'val3' + + +@override_settings(CACHES=cache_settings) +def test_dead_primary(): + primary_cache = django_cache.caches.create_connection('primary') + primary_cache.breakit() + + # Kill post-shutdown logging from unfinished recovery checker + logging.getLogger('ansible_base.cache.fallback_cache').setLevel(logging.CRITICAL) + + cache = django_cache.caches.create_connection('default') + + cache.set('key', 'val') + cache.get('key') + + # Check until fallback is set + timeout = time.time() + 30 + while True: + if cache.get_active_cache() == FALLBACK_CACHE: + break + if time.time() > timeout: + assert False + time.sleep(1) + + +@override_settings(CACHES=cache_settings) +def test_ensure_temp_file_is_removed_on_init(): + temp_file = Path(tempfile.NamedTemporaryFile().name) + with mock.patch('ansible_base.lib.cache.fallback_cache._temp_file', temp_file): + temp_file.touch() + DABCacheWithFallback(None, {}) + assert temp_file.exists() is False + + +@override_settings(CACHES=cache_settings) +def test_ensure_initialization_wont_happen_twice(): + with mock.patch('ansible_base.lib.cache.fallback_cache.ThreadPoolExecutor') as tfe: + cache = DABCacheWithFallback(None, {}) + tfe.assert_called_once() + cache.__init__(None, {}) + # when calling init again ThreadPoolExecute should not be called again so we should still have only one call + tfe.assert_called_once() + + +@pytest.mark.parametrize( + "method", + [ + ('clear'), + ('delete'), + ('set'), + ('get'), + ('add'), + ], +) +@override_settings(CACHES=cache_settings) +def test_all_methods_are_overwritten(method): + with mock.patch('ansible_base.lib.cache.fallback_cache.DABCacheWithFallback._op_with_fallback') as owf: + cache = DABCacheWithFallback(None, {}) + if method == 'clear': + getattr(cache, method)() + elif method in ['delete', 'get']: + getattr(cache, method)('test_value') + else: + getattr(cache, method)('test_value', 1) + owf.assert_called_once() + + +@pytest.mark.parametrize( + "file_exists", + [ + (True), + (False), + ], +) +@override_settings(CACHES=cache_settings) +def test_check_primary_cache(file_exists): + temp_file = Path(tempfile.NamedTemporaryFile().name) + with mock.patch('ansible_base.lib.cache.fallback_cache._temp_file', temp_file): + # Initialization of the cache will clear the temp file so do this first + cache = DABCacheWithFallback(None, {}) + + # Create the temp file if needed + if file_exists: + temp_file.touch() + else: + try: + temp_file.unlink() + except Exception: + pass + + mocked_function = mock.MagicMock(return_value=None) + cache._primary_cache.clear = mocked_function + cache.check_primary_cache() + if file_exists: + mocked_function.assert_called_once() + else: + mocked_function.assert_not_called() + assert temp_file.exists() is False + + +@override_settings(CACHES=cache_settings) +def test_file_unlink_exception_does_not_cause_failure(): + temp_file = Path(tempfile.NamedTemporaryFile().name) + with mock.patch('ansible_base.lib.cache.fallback_cache._temp_file', temp_file): + cache = DABCacheWithFallback(None, {}) + # We can't do: temp_file.unlink = mock.MagicMock(side_effect=Exception('failed to unlink exception')) + # Because unlink is marked as read only so we will just mock the cache.clear to raise in its place + mocked_function = mock.MagicMock(side_effect=Exception('failed to delete a file exception')) + cache._primary_cache.clear = mocked_function + + temp_file.touch() + cache.check_primary_cache() + # No assertion needed because we just want to make sure check_primary_cache does not raise diff --git a/test_app/tests/lib/dynamic_config/test_settings_logic.py b/test_app/tests/lib/dynamic_config/test_settings_logic.py new file mode 100644 index 000000000..481f840fb --- /dev/null +++ b/test_app/tests/lib/dynamic_config/test_settings_logic.py @@ -0,0 +1,23 @@ +import pytest + +from ansible_base.lib.dynamic_config.settings_logic import get_dab_settings + + +@pytest.mark.parametrize( + "caches,expect_exception", + [ + ({}, False), + ({"default": {"BACKEND": "junk"}}, False), + ({"default": {"BACKEND": "not_ansible_base.lib.cache.fallback_cache.DABCacheWithFallback"}}, True), + ({"default": {"BACKEND": "ansible_base.lib.cache.fallback_cache.DABCacheWithFallback"}}, True), + ({"default": {"BACKEND": "ansible_base.lib.cache.fallback_cache.DABCacheWithFallback"}, "primary": {}}, True), + ({"default": {"BACKEND": "ansible_base.lib.cache.fallback_cache.DABCacheWithFallback"}, "fallback": {}}, True), + ({"default": {"BACKEND": "ansible_base.lib.cache.fallback_cache.DABCacheWithFallback"}, "primary": {}, "fallback": {}}, False), + ], +) +def test_cache_settings(caches, expect_exception): + try: + get_dab_settings(installed_apps=[], caches=caches) + except RuntimeError: + if not expect_exception: + raise diff --git a/test_app/tests/lib/redis/test_client.py b/test_app/tests/lib/redis/test_client.py index 9eea40e9b..cd1efb7a2 100644 --- a/test_app/tests/lib/redis/test_client.py +++ b/test_app/tests/lib/redis/test_client.py @@ -384,3 +384,15 @@ def test_redis_timeout(): result = get_redis_status('redis://localhost', timeout=1) assert result['status'] == STATUS_FAILED assert result['exception'] == 'raised' + + +def test_redis_standalone_removes_cluster_settings(): + args = {'mode': 'standalone', 'cluster_error_retry_attempts': 4} + with mock.patch('redis.Redis.__init__', return_value=None) as rm: + from ansible_base.lib.redis.client import RedisClientGetter + + client_getter = RedisClientGetter() + client_getter.get_client('rediss://localhost', **args) + rm.assert_called_once + assert 'host' in rm.call_args.kwargs + assert 'cluster_error_retry_attempts' not in rm.call_args.kwargs