From 57789eca1c32ee2308d863c242c73b63f8aa9434 Mon Sep 17 00:00:00 2001 From: "Littles, Raleigh" Date: Thu, 13 Dec 2018 14:47:45 -0800 Subject: [PATCH 1/3] Switched over from using the hand-spun memoization to using the built-in LRU cache --- v1pysdk/v1meta.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/v1pysdk/v1meta.py b/v1pysdk/v1meta.py index 1666815..323632f 100644 --- a/v1pysdk/v1meta.py +++ b/v1pysdk/v1meta.py @@ -12,12 +12,14 @@ from .none_deref import NoneDeref from .string_utils import split_attribute +import functools + class V1Meta(object): def __init__(self, *args, **kw): self.server = V1Server(*args, **kw) self.global_cache = {} self.dirtylist = [] - self._memoized_data = {} + #self._memoized_data = {} def __getattr__(self, attr): "Dynamically build asset type classes when someone tries to get attrs " @@ -35,7 +37,8 @@ def clear_memoized_cache(self): """Clears the memoization cache produced by the @memoized decorator""" self._memoized_data={} - @memoized # from .cache_decorator + #@memoized # from .cache_decorator + @functools.lru_cache(maxsize=512, typed=False) def asset_class(self, asset_type_name): xmldata = self.server.get_meta_xml(asset_type_name) class_members = { @@ -93,7 +96,8 @@ def add_to_dirty_list(self, asset_instance): # are a few triggers to do this, it's best to clear our memoization cache for # query responses as soon as we have something that can get flushed rather than # waiting for it to actually be flushed - self.clear_memoized_cache() + #self.clear_memoized_cache() + self.asset_class.cache_clear() self.dirtylist.append(asset_instance) def commit(self): @@ -101,7 +105,8 @@ def commit(self): # we're flushing changes, make sure our memoization cache is cleared so the updates # are re-queried if self.dirtylist: - self.clear_memoized_cache() + #self.clear_memoized_cache() + self.asset_class.cache_clear() for asset in self.dirtylist: try: asset._v1_commit() From 170084c69e50d16f97d6829894a54e8ea05b187f Mon Sep 17 00:00:00 2001 From: "Littles, Raleigh" Date: Thu, 13 Dec 2018 14:51:56 -0800 Subject: [PATCH 2/3] erased some commented out code relating to the old caching, unit tests still work --- v1pysdk/v1meta.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/v1pysdk/v1meta.py b/v1pysdk/v1meta.py index 323632f..83f16a5 100644 --- a/v1pysdk/v1meta.py +++ b/v1pysdk/v1meta.py @@ -7,7 +7,6 @@ from .client import * from .base_asset import BaseAsset -from .cache_decorator import memoized from .special_class_methods import special_classes from .none_deref import NoneDeref from .string_utils import split_attribute @@ -19,7 +18,6 @@ def __init__(self, *args, **kw): self.server = V1Server(*args, **kw) self.global_cache = {} self.dirtylist = [] - #self._memoized_data = {} def __getattr__(self, attr): "Dynamically build asset type classes when someone tries to get attrs " @@ -30,14 +28,10 @@ def __enter__(self): return self def __exit__(self, *args, **kw): - self.clear_memoized_cache() + self.asset_class.cache_clear() self.commit() - def clear_memoized_cache(self): - """Clears the memoization cache produced by the @memoized decorator""" - self._memoized_data={} - #@memoized # from .cache_decorator @functools.lru_cache(maxsize=512, typed=False) def asset_class(self, asset_type_name): xmldata = self.server.get_meta_xml(asset_type_name) @@ -96,7 +90,6 @@ def add_to_dirty_list(self, asset_instance): # are a few triggers to do this, it's best to clear our memoization cache for # query responses as soon as we have something that can get flushed rather than # waiting for it to actually be flushed - #self.clear_memoized_cache() self.asset_class.cache_clear() self.dirtylist.append(asset_instance) @@ -105,7 +98,6 @@ def commit(self): # we're flushing changes, make sure our memoization cache is cleared so the updates # are re-queried if self.dirtylist: - #self.clear_memoized_cache() self.asset_class.cache_clear() for asset in self.dirtylist: try: From cbb52960f6f07cdbf451c907cd8c068948f5674f Mon Sep 17 00:00:00 2001 From: "Littles, Raleigh" Date: Thu, 20 Dec 2018 09:24:16 -0800 Subject: [PATCH 3/3] Began trying to use a Python2 LRU cache implementation, got the initial caching decorator in place but still need to figure out a nice way to use a different cache_clear() function depending on if its python2 or python3. Also, changed setup.py to allow installation of unittest dependencies as part of the regular install target, so that you don't have to download additional packages when you run unit tests. Also deleted the original handwritten cache-decorator implementation --- setup.py | 6 +++++- tests/creation_tests.py | 2 +- v1pysdk/cache_decorator.py | 25 ------------------------- v1pysdk/v1meta.py | 15 +++++++++++---- 4 files changed, 17 insertions(+), 31 deletions(-) delete mode 100644 v1pysdk/cache_decorator.py diff --git a/setup.py b/setup.py index 26bf8ec..1e88996 100644 --- a/setup.py +++ b/setup.py @@ -5,12 +5,15 @@ from setuptools import setup install_requires = [ - 'future' + 'future', + 'testtools', + 'unittest2' ] if (sys.version_info < (3,0)): # has a different name if supporting Python3 install_requires.append('python-ntlm') + install_requires.append("repoze.lru") else: install_requires.append('python-ntlm3') @@ -59,6 +62,7 @@ tests_require = [ 'testtools', + 'repoze.lru', 'unittest2' # so testtools tests are auto-discovered ], test_suite = "tests", diff --git a/tests/creation_tests.py b/tests/creation_tests.py index e97c17c..0d26551 100644 --- a/tests/creation_tests.py +++ b/tests/creation_tests.py @@ -1,4 +1,4 @@ -from testtools import TestCase +#from testtools import TestCase import v1pysdk from .common_test_server import PublicTestServerConnection diff --git a/v1pysdk/cache_decorator.py b/v1pysdk/cache_decorator.py deleted file mode 100644 index acac901..0000000 --- a/v1pysdk/cache_decorator.py +++ /dev/null @@ -1,25 +0,0 @@ -def key_by_args_and_func_kw(old_f, args, kw, cache_data): - """Function to produce a key hash for the cache_data hash based on the function and the arguments - provided.""" - return repr((old_f, args, kw)) - - -def memoized(old_f): - """Decorator that memoizes calls to old_f into a single instance-specific _memoized_data hash""" - def new_f(self, *args, **kw): - """Function to wrap the decorated method""" - if not self._memoized_data: - self._memoized_data = {} - #generate a hash using the provided hashing function from the closure - new_key = key_by_args_and_func_kw(old_f, args, kw, self._memoized_data) - # check if it's already in the memoized data - if new_key in self._memoized_data: - # return what's already there - return self._memoized_data[new_key] - # not found, call the real function and put the results in the memoized data - new_value = old_f(self, *args, **kw) - self._memoized_data[new_key] = new_value - return new_value - return new_f - -#memoized = cached_by_keyfunc() diff --git a/v1pysdk/v1meta.py b/v1pysdk/v1meta.py index 83f16a5..5536af7 100644 --- a/v1pysdk/v1meta.py +++ b/v1pysdk/v1meta.py @@ -1,5 +1,10 @@ import sys +if (sys.version_info < (3,0)): + from repoze.lru import lru_cache + +else: + from functools import lru_cache try: from xml.etree import ElementTree except ImportError: @@ -11,8 +16,6 @@ from .none_deref import NoneDeref from .string_utils import split_attribute -import functools - class V1Meta(object): def __init__(self, *args, **kw): self.server = V1Server(*args, **kw) @@ -28,11 +31,15 @@ def __enter__(self): return self def __exit__(self, *args, **kw): - self.asset_class.cache_clear() + if (sys.version_info > (3, 0)): + self.asset_class.cache_clear() + else: + self.assert_class.clear() + self.commit() - @functools.lru_cache(maxsize=512, typed=False) + @lru_cache(maxsize=512) def asset_class(self, asset_type_name): xmldata = self.server.get_meta_xml(asset_type_name) class_members = {