From 9d6e066d17a09844f560cedf3fbe459af4cf7cbe Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 25 Jun 2017 11:21:15 +0000 Subject: [PATCH 1/4] Python 3 support done! --- .travis.yml | 23 ++++-- Makefile | 1 + conftest.py | 7 +- pyprometheus/compat.py | 32 ++++++++ pyprometheus/contrib/uwsgi_features.py | 100 +++++++++++++------------ pyprometheus/managers.py | 1 + pyprometheus/metrics.py | 20 ++--- pyprometheus/registry.py | 3 +- pyprometheus/utils/__init__.py | 19 +++-- pyprometheus/utils/exposition.py | 3 +- setup.py | 65 ++++++++-------- tests/test_metrics.py | 11 ++- tests/test_registry.py | 4 +- tests/test_uwsgi_collector.py | 34 +++++---- tox.ini | 13 +++- 15 files changed, 204 insertions(+), 132 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5ac1eea..2285952 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,20 +12,27 @@ script: notifications: email: false +python: + - '2.7' + - '3.5' + - '3.6' + - '3.4' + - '3.3' + matrix: include: - python: "2.7" env: UWSGI="2.0.14" - # - python: "3.3" - # env: UWSGI="2.0.14" + - python: "3.3" + env: UWSGI="2.0.14" - # - python: "3.4" - # env: UWSGI="2.0.14" + - python: "3.4" + env: UWSGI="2.0.14" - # - python: "3.5" - # env: UWSGI="2.0.14" + - python: "3.5" + env: UWSGI="2.0.14" - # - python: "3.6" - # env: UWSGI="2.0.14" + - python: "3.6" + env: UWSGI="2.0.14" diff --git a/Makefile b/Makefile index 78ba566..e766ddb 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ test: clean-containers tox: clean-containers @echo "Tox test application $(version)" + rm -rf ./.tox $(DOCKER_RUN_COMMAND) "tox" @echo "" diff --git a/conftest.py b/conftest.py index c2a3152..fce6834 100644 --- a/conftest.py +++ b/conftest.py @@ -7,10 +7,7 @@ import time from pyprometheus.storage import BaseStorage from pyprometheus.utils import measure_time as measure_time_manager -try: - xrange = xrange -except Exception: - xrange = range +from pyprometheus.compat import PAD_SYMBOL, xrange @pytest.fixture @@ -22,7 +19,7 @@ def project_root(): def run_around_tests(): m = uwsgi.sharedarea_memoryview(0) for x in xrange(len(m)): - m[x] = "\x00" + m[x] = PAD_SYMBOL yield diff --git a/pyprometheus/compat.py b/pyprometheus/compat.py index 5e3d6b2..c409239 100644 --- a/pyprometheus/compat.py +++ b/pyprometheus/compat.py @@ -17,3 +17,35 @@ PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 PY34 = sys.version_info[0:2] >= (3, 4) + + +if PY3: + PAD_SYMBOL = 0 +else: + PAD_SYMBOL = "\x00" + + +try: + xrange = xrange +except Exception: + xrange = range + + +if PY3: + def b(s): + if isinstance(s, bytes): + return s + return s.encode("latin-1") + + def u(s): + return s + +else: + def b(s): + return s + + def u(s): + return s + # if isinstance(s, unicode): + # return s + # return unicode(s.replace(r"\\", r"\\\\"), "unicode_escape") diff --git a/pyprometheus/contrib/uwsgi_features.py b/pyprometheus/contrib/uwsgi_features.py index d336fe3..2783ba2 100644 --- a/pyprometheus/contrib/uwsgi_features.py +++ b/pyprometheus/contrib/uwsgi_features.py @@ -21,7 +21,7 @@ from pyprometheus.const import TYPES from pyprometheus.metrics import Gauge, Counter from pyprometheus.storage import BaseStorage, LocalMemoryStorage - +from pyprometheus import compat try: import uwsgi @@ -45,7 +45,9 @@ class UWSGICollector(object): """ def __init__(self, namespace, labels={}): self._namespace = namespace + self._labels = tuple(sorted(labels.items(), key=lambda x: x[0])) + self._labels_names = tuple(label[0] for label in self._labels) self._collectors = self.declare_metrics() @property @@ -70,29 +72,30 @@ def metric_name(self, name): return ":".join([self._namespace, name]) def declare_metrics(self): + return { - "memory": Gauge(self.metric_name("uwsgi_memory_bytes"), "UWSGI memory usage in bytes", ("type",) + self._labels), - "processes": Gauge(self.metric_name("processes_total"), "Number of UWSGI processes", self._labels), - "worker_status": Gauge(self.metric_name("worker_status_totla"), "Current workers status", self._labels), - "total_requests": Gauge(self.metric_name("requests_total"), "Total processed request", self._labels), - "buffer_size": Gauge(self.metric_name("buffer_size_bytes"), "UWSGI buffer size in bytes", self._labels), - "started_on": Gauge(self.metric_name("started_on"), "UWSGI started on timestamp", self._labels), - "cores": Gauge(self.metric_name("cores"), "system cores", self._labels), - - - "process:respawn_count": Gauge(self.metric_name("process:respawn_count"), "Process respawn count", ("id", ) + self._labels), - "process:last_spawn": Gauge(self.metric_name("process:last_spawn"), "Process last spawn", ("id", ) + self._labels), - "process:signals": Gauge(self.metric_name("process:signals"), "Process signals total", ("id", ) + self._labels), - "process:avg_rt": Gauge(self.metric_name("process:avg_rt"), "Process average response time", ("id", ) + self._labels), - "process:tx": Gauge(self.metric_name("process:tx"), "Process transmitted data", ("id",) + self._labels), - - "process:status": Gauge(self.metric_name("process:status"), "Process status", ("id", "status") + self._labels), - "process:running_time": Gauge(self.metric_name("process:running_time"), "Process running time", ("id", ) + self._labels), - "process:exceptions": Gauge(self.metric_name("process:exceptions"), "Process exceptions", ("id", ) + self._labels), - "process:requests": Gauge(self.metric_name("process:requests"), "Process requests", ("id", ) + self._labels), - "process:delta_requests": Gauge(self.metric_name("process:delta_requests"), "Process delta_requests", ("id", ) + self._labels), - "process:rss": Gauge(self.metric_name("process:rss"), "Process rss memory", ("id", ) + self._labels), - "process:vsz": Gauge(self.metric_name("process:vzs"), "Process vsz address space", ("id", ) + self._labels), + "memory": Gauge(self.metric_name("uwsgi_memory_bytes"), "UWSGI memory usage in bytes", ("type",) + self._labels_names), + "processes": Gauge(self.metric_name("processes_total"), "Number of UWSGI processes", self._labels_names), + "worker_status": Gauge(self.metric_name("worker_status_totla"), "Current workers status", self._labels_names), + "total_requests": Gauge(self.metric_name("requests_total"), "Total processed request", self._labels_names), + "buffer_size": Gauge(self.metric_name("buffer_size_bytes"), "UWSGI buffer size in bytes", self._labels_names), + "started_on": Gauge(self.metric_name("started_on"), "UWSGI started on timestamp", self._labels_names), + "cores": Gauge(self.metric_name("cores"), "system cores", self._labels_names), + + + "process:respawn_count": Gauge(self.metric_name("process:respawn_count"), "Process respawn count", ("id", ) + self._labels_names), + "process:last_spawn": Gauge(self.metric_name("process:last_spawn"), "Process last spawn", ("id", ) + self._labels_names), + "process:signals": Gauge(self.metric_name("process:signals"), "Process signals total", ("id", ) + self._labels_names), + "process:avg_rt": Gauge(self.metric_name("process:avg_rt"), "Process average response time", ("id", ) + self._labels_names), + "process:tx": Gauge(self.metric_name("process:tx"), "Process transmitted data", ("id",) + self._labels_names), + + "process:status": Gauge(self.metric_name("process:status"), "Process status", ("id", "status") + self._labels_names), + "process:running_time": Gauge(self.metric_name("process:running_time"), "Process running time", ("id", ) + self._labels_names), + "process:exceptions": Gauge(self.metric_name("process:exceptions"), "Process exceptions", ("id", ) + self._labels_names), + "process:requests": Gauge(self.metric_name("process:requests"), "Process requests", ("id", ) + self._labels_names), + "process:delta_requests": Gauge(self.metric_name("process:delta_requests"), "Process delta_requests", ("id", ) + self._labels_names), + "process:rss": Gauge(self.metric_name("process:rss"), "Process rss memory", ("id", ) + self._labels_names), + "process:vsz": Gauge(self.metric_name("process:vzs"), "Process vsz address space", ("id", ) + self._labels_names), } def collect(self): @@ -108,7 +111,6 @@ def collect(self): for x in self.get_workers_samples(uwsgi.workers()): yield x - def get_workers_samples(self, workers): """Read worker stats and create samples @@ -122,7 +124,7 @@ def get_workers_samples(self, workers): for worker in workers: labels = self._labels + (("id", worker["id"]),) metric.add_sample(labels, metric.build_sample(labels, - ( (TYPES.GAUGE, metric.name, "", labels, worker[name]), ))) + ( (TYPES.GAUGE, metric.name, "", labels, worker[name]), ))) # noqa yield metric @@ -130,7 +132,7 @@ def get_workers_samples(self, workers): for worker in workers: labels = self._labels + (("id", worker["id"]), ("status", worker["status"])) metric.add_sample(labels, metric.build_sample(labels, - ( (TYPES.GAUGE, metric.name, "", self._labels + (("id", worker["id"]), ("status", worker["status"])), 1), ))) + ( (TYPES.GAUGE, metric.name, "", self._labels + (("id", worker["id"]), ("status", worker["status"])), 1), ))) # noqa yield metric @@ -141,15 +143,15 @@ def get_sample(self, name, value): :param value: """ metric = self._collectors[name] - return metric.build_samples([(self._labels, ( (TYPES.GAUGE, metric.name, "", self._labels, float(value)), ))]) + return metric.build_samples([(self._labels, ( (TYPES.GAUGE, metric.name, "", self._labels, float(value)), ))]) # noqa def get_memory_samples(self): """Get memory usage samples """ metric = self._collectors["memory"] return metric.build_samples( - [(self._labels + (("type", "rss"),), ( (TYPES.GAUGE, metric.name, "", self._labels + (("type", "rss"),), uwsgi.mem()[0]), )), - (self._labels + (("type", "vsz"),), ( (TYPES.GAUGE, metric.name, "", self._labels + (("type", "vsz"),), uwsgi.mem()[1]), ))]) + [(self._labels + (("type", "rss"),), ( (TYPES.GAUGE, metric.name, "", self._labels + (("type", "rss"),), uwsgi.mem()[0]), )), # noqa + (self._labels + (("type", "vsz"),), ( (TYPES.GAUGE, metric.name, "", self._labels + (("type", "vsz"),), uwsgi.mem()[1]), ))]) # noqa class UWSGIStorage(BaseStorage): @@ -199,7 +201,6 @@ def metric_name(self, name): """ return ":".join([self._namespace, name]) - @staticmethod def get_unique_id(): try: @@ -221,7 +222,7 @@ def declare_metrics(self): def collect(self): labels = self._labels + (("sharedarea", self._sharedarea_id), ("id", self.get_unique_id())) metric = self._collectors["memory_sync"] - metric.add_sample(labels, metric.build_sample(labels, ( (TYPES.GAUGE, metric.name, "", labels, self._syncs), ))) + metric.add_sample(labels, metric.build_sample(labels, ( (TYPES.GAUGE, metric.name, "", labels, self._syncs), ))) # noqa yield metric @@ -230,16 +231,15 @@ def collect(self): # yield metric metric = self._collectors["memory_size"] - metric.add_sample(labels, metric.build_sample(labels, ( (TYPES.GAUGE, metric.name, "", labels, self.get_area_size()), ))) + metric.add_sample(labels, metric.build_sample(labels, ( (TYPES.GAUGE, metric.name, "", labels, self.get_area_size()), ))) # noqa yield metric metric = self._collectors["num_keys"] - metric.add_sample(labels, metric.build_sample(labels, ( (TYPES.GAUGE, metric.name, "", labels, len(self._positions)), ))) + metric.add_sample(labels, metric.build_sample(labels, ( (TYPES.GAUGE, metric.name, "", labels, len(self._positions)), ))) # noqa yield metric - @property def m(self): return self._m @@ -279,12 +279,12 @@ def get_area_size_with_lock(self): return self.get_area_size() def get_slice(self, start, size): - return slice(start, start+size) + return slice(start, start + size) def get_area_size(self): """Read area size from uwsgi """ - return struct.unpack(b"i", self.m[self.get_slice(self.AREA_SIZE_POSITION, self.AREA_SIZE_SIZE)])[0] + return struct.unpack(b"i", self.m[self.get_slice(self.AREA_SIZE_POSITION, self.AREA_SIZE_SIZE)].tobytes())[0] def init_area_size(self): return self.update_area_size(self.AREA_SIZE_SIZE) @@ -298,7 +298,6 @@ def update_area_sign(self): self._sign = os.urandom(self.SIGN_SIZE) self.m[self.get_slice(self.SIGN_POSITION, self.SIGN_SIZE)] = self._sign - def get_area_sign(self): """Get current area sign from memory """ @@ -353,21 +352,25 @@ def get_string_padding(self, key): http://stackoverflow.com/questions/11642210/computing-padding-required-for-n-byte-alignment :param key: encoded string """ - #return (4 - (len(key) % 4)) % 4 + return (4 - (len(key) % 4)) % 4 + + # return (8 - (len(key) + 4) % 4) - return (8 - (len(key) + 4) % 8) + def get_string_padded_len(self, key): + padding = self.get_string_padding(key) + return len(key) + padding def get_key_size(self, key): """Calculate how many memory need key :param key: key string """ - return len(self.serialize_key(key)) + self.KEY_SIZE_SIZE + self.KEY_VALUE_SIZE - + return self.get_string_padded_len(self.serialize_key(key)) + self.KEY_SIZE_SIZE + self.KEY_VALUE_SIZE def get_binary_string(self, key, value): - item_template = "=i{0}sd".format(len(key)).encode() + padded_string_len = self.get_string_padded_len(key) + item_template = "=i{0}sd".format(padded_string_len).encode() - return struct.pack(item_template, len(key), key, value) + return struct.pack(item_template, padded_string_len, compat.b(key), value) def init_key(self, key, init_value=0.0): """Initialize memory for key @@ -393,16 +396,15 @@ def read_key_string(self, position, size): :param position: int offset for key string :param size: int key size in bytes to read """ - key_string_bytes = self.m[self.get_slice(position, size)] - return struct.unpack(b"{0}s".format(size), key_string_bytes)[0] - + key_string_bytes = self.m[self.get_slice(position, size)].tobytes() + return struct.unpack(compat.b("{0}s".format(size)), key_string_bytes)[0].strip(compat.b("\x00")) def read_key_value(self, position): """Read float value of position :param position: int offset for key value float """ - key_value_bytes = self.m[self.get_slice(position, self.KEY_VALUE_SIZE)] + key_value_bytes = self.m[self.get_slice(position, self.KEY_VALUE_SIZE)].tobytes() return struct.unpack(b"d", key_value_bytes)[0] def read_key_size(self, position): @@ -410,7 +412,7 @@ def read_key_size(self, position): :param position: int offset for 4-byte key size """ - key_size_bytes = self.m[self.get_slice(position, self.KEY_SIZE_SIZE)] + key_size_bytes = self.m[self.get_slice(position, self.KEY_SIZE_SIZE)].tobytes() return struct.unpack(b"i", key_size_bytes)[0] def write_key_value(self, position, value): @@ -558,7 +560,7 @@ def __len__(self): def clear(self): for x in xrange(self.AREA_SIZE_SIZE + self.AREA_SIZE_SIZE): - self.m[x] = "\x00" + self.m[x] = compat.PAD_SYMBOL self._positions.clear() diff --git a/pyprometheus/managers.py b/pyprometheus/managers.py index ae31347..24667d2 100644 --- a/pyprometheus/managers.py +++ b/pyprometheus/managers.py @@ -16,6 +16,7 @@ default_timer = time.time + class BaseManager(object): def __call__(self, f): @wraps(f) diff --git a/pyprometheus/metrics.py b/pyprometheus/metrics.py index c3584e0..230b33c 100644 --- a/pyprometheus/metrics.py +++ b/pyprometheus/metrics.py @@ -16,6 +16,7 @@ from pyprometheus.values import (MetricValue, GaugeValue, CounterValue, SummaryValue, HistogramValue) +from pyprometheus import compat class BaseMetric(object): @@ -43,7 +44,8 @@ def __init__(self, name, doc, labels=[], registry=None): self._labels_cache = {} def __repr__(self): - return u"<{0}[{1}]: {2} samples>".format(self.__class__.__name__, self._name, len(self._samples)) + return u"<{0}[{1}]: {2} samples>".format( + self.__class__.__name__, self._name, len(self._samples)) def get_proxy(self): if self._labelnames: @@ -94,11 +96,11 @@ def text_export_header(self): # HELP go_gc_duration_seconds A summary of the GC invocation durations. # TYPE go_gc_duration_seconds summary """ - return "\n".join(["# HELP {name} {doc}", - "# TYPE {name} {metric_type}"]).format( - name=escape_str(self.name), - doc=escape_str(self.doc), - metric_type=self.TYPE) + return u"\n".join([u"# HELP {name} {doc}", + u"# TYPE {name} {metric_type}"]).format( + name=escape_str(self.name), + doc=escape_str(self.doc), + metric_type=self.TYPE) def build_samples(self, items): """Build samples from objects @@ -114,16 +116,16 @@ def build_sample(self, label_values, item): """ return self.value_class(self, label_values=label_values, value=item[0][-1]) - def add_sample(self, label_values, value): self._samples[tuple(sorted(label_values, key=lambda x: x[0]))] = value def get_samples(self): """Get samples from storage """ + if compat.PY3: + return list(self._samples.values()) return self._samples.values() - def __getattr__(self, name): if name in self.PARENT_METHODS: return getattr(self.get_proxy(), name) @@ -139,7 +141,7 @@ class Gauge(BaseMetric): value_class = GaugeValue PARENT_METHODS = set(("inc", "dec", "set", "get", "track_inprogress", - "set_to_current_time", "time", "value")) + "set_to_current_time", "time", "value")) class Counter(BaseMetric): diff --git a/pyprometheus/registry.py b/pyprometheus/registry.py index 452cf6e..584ebe9 100644 --- a/pyprometheus/registry.py +++ b/pyprometheus/registry.py @@ -12,7 +12,6 @@ """ - class BaseRegistry(object): """Link with metrics collectors """ @@ -43,7 +42,7 @@ def collect(self): for uid, collector in self.collectors(): # Collectors with collect method already have stats - if hasattr(collector, 'collect'): + if hasattr(collector, "collect"): for item in collector.collect(): yield item else: diff --git a/pyprometheus/utils/__init__.py b/pyprometheus/utils/__init__.py index 1d8d390..5b3fe63 100644 --- a/pyprometheus/utils/__init__.py +++ b/pyprometheus/utils/__init__.py @@ -13,6 +13,7 @@ import sys import time +from pyprometheus import compat def import_storage(path): try: @@ -24,7 +25,7 @@ def import_storage(path): def escape_str(value): - return value.replace("\\", r"\\").replace("\n", r"\n").replace("\"", r"\"") + return compat.u(value).replace("\\", r"\\").replace("\n", r"\n").replace("\"", r"\"") def format_binary(value): @@ -42,7 +43,7 @@ def print_binary(value): class measure_time(object): - def __init__(self,name): + def __init__(self, name): self.name = name self._num_ops = 0 @@ -50,12 +51,20 @@ def __enter__(self): self.start = time.time() return self - def __exit__(self,ty,val,tb): + def __exit__(self, exc_type, exc_value, traceback): end = time.time() print("{0} : {1:.4f} seconds for {2} ops [{3:.4f} / s]".format( - self.name, end-self.start, - self._num_ops, self._num_ops / (end-self.start))) + self.name, end - self.start, + self._num_ops, self._num_ops / (end - self.start))) return False def set_num_ops(self, value): self._num_ops = value + + +def get_string_padding(s, size=4): + return (size - (len(s) % size)) % size + + +def get_padded_string_len(s, size=4): + return len(s) + get_string_padding(s, size) diff --git a/pyprometheus/utils/exposition.py b/pyprometheus/utils/exposition.py index 1f5cd48..2e3556e 100644 --- a/pyprometheus/utils/exposition.py +++ b/pyprometheus/utils/exposition.py @@ -14,6 +14,7 @@ from datetime import datetime from pyprometheus.const import CREDITS +from pyprometheus import compat def registry_to_text(registry): @@ -35,6 +36,6 @@ def write_to_textfile(registry, path): tmp_filename = "{0}.{1}.tmp".format(path, os.getpid()) with open(tmp_filename, "wb") as f: - f.write(registry_to_text(registry)) + f.write(compat.b(registry_to_text(registry))) os.rename(tmp_filename, path) diff --git a/setup.py b/setup.py index ef99cb2..d74fc34 100644 --- a/setup.py +++ b/setup.py @@ -18,23 +18,24 @@ import sys import ast -_version_re = re.compile(r'__version__\s+=\s+(.*)') +_version_re = re.compile(r"__version__\s+=\s+(.*)") -with open('pyprometheus/__init__.py', 'rb') as f: +with open("pyprometheus/__init__.py", "rb") as f: version = str(ast.literal_eval(_version_re.search( - f.read().decode('utf-8')).groups()[0])) + f.read().decode("utf-8")).groups()[0])) install_require = [] -tests_require = [x.strip() for x in open("tests_requirements.txt").readlines() if (x.strip() and not x.strip().startswith('#'))] +tests_require = [x.strip() for x in open("tests_requirements.txt").readlines() if (x.strip() and not x.strip().startswith("#"))] def read_description(): try: - with open("README.rst", 'r') as f: + with open("README.rst", "r") as f: return f.read() except Exception: return __doc__ + class PyTest(TestCommand): def initialize_options(self): @@ -54,45 +55,45 @@ def run_tests(self): setup( - name='pyprometheus', + name="pyprometheus", version=version, - author='Alexandr Lispython', - author_email='lispython@users.noreply.github.com', - url='https://github.com/Lispython/pyprometheus', - description='Prometheus python client and instrumentation library', + author="Alexandr Lispython", + author_email="lispython@users.noreply.github.com", + url="https://github.com/Lispython/pyprometheus", + description="Prometheus python client and instrumentation library", long_description=read_description(), packages=find_packages(exclude=("tests", "tests.*",)), zip_safe=False, extras_require={ - 'tests': tests_require, + "tests": tests_require, }, - license='BSD', + license="BSD", tests_require=tests_require, install_requires=install_require, - cmdclass={'test': PyTest}, + cmdclass={"test": PyTest}, include_package_data=True, entry_points={ - 'console_scripts': [ - 'pyprometheus = pyprometheus.scripts:main', + "console_scripts": [ + "pyprometheus = pyprometheus.scripts:main", ] }, classifiers=[ - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Operating System :: OS Independent', - 'Operating System :: POSIX', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - # 'Programming Language :: Python :: 3', - # 'Programming Language :: Python :: 3.3', - # 'Programming Language :: Python :: 3.4', - # 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python', - 'Programming Language :: Python :: Implementation :: CPython', - - 'Topic :: Software Development', - 'Topic :: System :: Monitoring', - 'Topic :: Software Development :: Libraries', - 'Topic :: System :: Networking :: Monitoring' + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + # "Programming Language :: Python :: 3", + # "Programming Language :: Python :: 3.3", + # "Programming Language :: Python :: 3.4", + # "Programming Language :: Python :: 3.5", + "Programming Language :: Python", + "Programming Language :: Python :: Implementation :: CPython", + + "Topic :: Software Development", + "Topic :: System :: Monitoring", + "Topic :: Software Development :: Libraries", + "Topic :: System :: Networking :: Monitoring" ], ) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index ee2ebfa..4f15f9f 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -7,13 +7,14 @@ from pyprometheus.values import MetricValue from pyprometheus.storage import LocalMemoryStorage from pyprometheus.contrib.uwsgi_features import UWSGIStorage +from pyprometheus import compat @pytest.mark.parametrize("storage_cls", [LocalMemoryStorage, UWSGIStorage]) def test_base_metric(storage_cls): storage = storage_cls() registry = BaseRegistry(storage=storage) - metric_name = "test_base_metric\x00\\" + metric_name = u"test_base_metric\x00\\" metric = BaseMetric(metric_name, "test_base_metric doc \u4500", ("label1", "label2"), registry=registry) assert registry.is_registered(metric) @@ -37,8 +38,12 @@ def test_base_metric(storage_cls): assert labels.get() == 1 - assert metric.text_export_header == "\n".join(["# HELP test_base_metric\x00\\\\ test_base_metric doc \\\\u4500", - "# TYPE test_base_metric\x00\\\\ untyped"]) + if compat.PY3: + assert metric.text_export_header == u"\n".join([u"# HELP test_base_metric\x00\\\\ test_base_metric doc \u4500", + u"# TYPE test_base_metric\x00\\\\ untyped"]) + else: + assert metric.text_export_header == u"\n".join([u"# HELP test_base_metric\x00\\\\ test_base_metric doc \\\\u4500", + u"# TYPE test_base_metric\x00\\\\ untyped"]) assert labels.export_str.split(" ")[:2] == 'test_base_metric\x00\\\\{label1\\\\x\\n\\"="label1_value\\\\x\\n\\"", label2="label2_value\\\\x\\n\\""} 1.0'.split(" ")[:2] # noqa diff --git a/tests/test_registry.py b/tests/test_registry.py index e975f8e..ce14499 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -125,7 +125,7 @@ def test_base_registry(storage_cls, measure_time): else: assert test1.split()[:-1] == test2.split()[:-1] - metrics_count = map(lambda x: x.split(" ")[2], - filter(lambda x: x.startswith("# HELP"), [x for x in registry_to_text(registry).split("\n")])) + metrics_count = list(map(lambda x: x.split(" ")[2], + filter(lambda x: x.startswith("# HELP"), [x for x in registry_to_text(registry).split("\n")]))) assert len(metrics_count) == len(set(metrics_count)) diff --git a/tests/test_uwsgi_collector.py b/tests/test_uwsgi_collector.py index 3cff46a..9b93583 100644 --- a/tests/test_uwsgi_collector.py +++ b/tests/test_uwsgi_collector.py @@ -2,16 +2,17 @@ # -*- coding: utf-8 -*- import os import random +import marshal from multiprocessing import Process import uwsgi from pyprometheus.contrib.uwsgi_features import UWSGICollector, UWSGIStorage, UWSGIFlushStorage from pyprometheus.registry import BaseRegistry from pyprometheus.utils.exposition import registry_to_text -try: - xrange = xrange -except Exception: - xrange = range +from pyprometheus import compat +from pyprometheus.utils import get_padded_string_len + +xrange = compat.xrange def test_uwsgi_collector(): @@ -72,7 +73,7 @@ def test_uwsgi_storage(): assert (storage.get_area_size()) == 14 - assert storage.m[15] == "\x00" + assert storage.m[15] == compat.PAD_SYMBOL with storage.lock(): @@ -95,22 +96,23 @@ def test_uwsgi_storage(): assert area_sign == storage2.get_area_sign() - storage.m[storage.SIGN_POSITION + 2] = os.urandom(1) + storage.m[storage.SIGN_POSITION + 2:storage.SIGN_POSITION + 2 + storage.SIGN_SIZE] = os.urandom(storage.SIGN_SIZE) assert not storage.is_actual s = "keyname" - assert storage.get_string_padding(s) == 5 - assert len(s.encode("utf-8")) + storage.get_string_padding(s) == 12 + assert storage.get_string_padding(s) == 1 + + assert len(s.encode("utf-8")) + storage.get_string_padding(s) == 8 assert storage.validate_actuality() assert storage.is_actual - assert storage.get_key_position("keyname") == ([14, 18, 25, 33], True) + assert storage.get_key_position("keyname") == ([14, 18, 26, 34], True) - assert (storage.get_area_size()) == 33 + assert (storage.get_area_size()) == 34 assert storage.get_key_size("keyname") == 24 @@ -128,22 +130,28 @@ def test_uwsgi_storage(): storage.write_value(DATA[0][0], DATA[0][1]) - assert storage.get_key_size(DATA[0][0]) == 108 + # len of marshaled string depending of python version + assert storage.get_key_size(DATA[0][0]) == get_padded_string_len(marshal.dumps(DATA[0][0])) + storage.KEY_SIZE_SIZE + storage.KEY_VALUE_SIZE - assert storage.get_area_size() == 122 == storage2.get_area_size() + # meta header size + first key size + l = 14 + get_padded_string_len(marshal.dumps(DATA[0][0])) + storage.KEY_SIZE_SIZE + storage.KEY_VALUE_SIZE + assert storage.get_area_size() == l == storage2.get_area_size() assert storage2.get_value(DATA[0][0]) == DATA[0][1] == storage.get_value(DATA[0][0]) + total_size = storage.AREA_SIZE_SIZE + storage.AREA_SIZE_POSITION + storage.SIGN_SIZE + for x in DATA: storage.write_value(x[0], x[1]) assert storage.get_value(x[0]) == x[1] + total_size += get_padded_string_len(marshal.dumps(x[0])) + storage.KEY_SIZE_SIZE + storage.KEY_VALUE_SIZE for x in DATA: assert storage.get_value(x[0]) == x[1] assert len(storage) == len(DATA) == 20 - assert storage.get_area_size() == 2531 + assert storage.get_area_size() == total_size assert not storage2.is_actual assert storage.is_actual diff --git a/tox.ini b/tox.ini index cc73164..d0df920 100644 --- a/tox.ini +++ b/tox.ini @@ -4,12 +4,19 @@ # and then run "tox" from this directory. [tox] -envlist = py27#, py33, py34, py35 +envlist = py27, py35, py36, py34, py33 + [testenv] deps = -rtests_requirements.txt +whitelist_externals = + echo commands = - pyenv local 2.7 3.2 3.3.0 3.4.0 3.5.0 3.6.0 3.7-dev pypy-4.0.1 - #pip install -e .[tests] + # pyenv version + # echo "{envname} - {posargs} - {envpython}" + # pyenv local 2.7 + # pip install -e .[tests] + # ci/test + # uwsgi --pyrun ./ uwsgi --pyrun setup.py --pyargv test --sharedarea=100 --enable-threads \ No newline at end of file From c0343f813b3128d0f2951f3469df629a69de98ed Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 8 Jul 2017 21:16:36 +0000 Subject: [PATCH 2/4] Testing travis. --- .travis.yml | 20 ++++++-------------- ci/setup | 5 +++++ setup.cfg | 4 ++-- tests/test_storage.py | 8 ++++---- tests_requirements.txt | 4 +++- 5 files changed, 20 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2285952..0e51574 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,19 +20,11 @@ python: - '3.3' -matrix: - include: - - python: "2.7" - env: UWSGI="2.0.14" - - - python: "3.3" - env: UWSGI="2.0.14" +env: + - env: UWSGI="2.0.1" + - env: UWSGI="2.0.14" + - env: UWSGI="2.0.15" - - python: "3.4" - env: UWSGI="2.0.14" - - python: "3.5" - env: UWSGI="2.0.14" - - - python: "3.6" - env: UWSGI="2.0.14" +matrix: + include: diff --git a/ci/setup b/ci/setup index 417c291..67f3322 100755 --- a/ci/setup +++ b/ci/setup @@ -2,6 +2,11 @@ pip install "pip>=8.1" +if [ "$UWSGI" = "" ] +then + UWSGI="2.0.15" +fi + pip install "uwsgi==$UWSGI" pip install -U -r tests_requirements.txt diff --git a/setup.cfg b/setup.cfg index 5250f09..65fdff8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,12 +5,12 @@ addopts=-s -p no:doctest --flake8 --cov=./ norecursedirs=pyprometheus build bin dist docs .git flake8-max-line-length = 100 flake8-ignore = - *.py E501 + *.py E501 F821 main/settings/*.py F403 F401 */migrations/* ALL [flake8] -ignore = E501,F403,F401,D100,D101,D102,D103,I004,I001,I003,Q000,D205,D400,D105 +ignore = E501,F403,F401,D100,D101,D102,D103,I004,I001,I003,Q000,D205,D400,D105,F821 max-line-length = 100 exclude = .tox,.git,docs,.ropeproject inline-quotes = double diff --git a/tests/test_storage.py b/tests/test_storage.py index 02dce3a..7c4776d 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -4,10 +4,10 @@ import random import threading -try: - xrange = xrange -except Exception: - xrange = range +from pyprometheus import compat + +xrange = compat.xrange + DATA = ( ((2, "metric_gauge_name", "", (("label1", "value1"), ("label2", "value2"))), 5), diff --git a/tests_requirements.txt b/tests_requirements.txt index 7c11d73..a9f122a 100644 --- a/tests_requirements.txt +++ b/tests_requirements.txt @@ -1,9 +1,11 @@ +setuptools==36.0.1 + flake8==3.2.1 tox==2.3.2 #tox-pyenv==2.3.2 -ipdb +ipdb==0.10.2 uwsgi==2.0.14 pytest==3.0.6 pytest-cov==2.4.0 From e3439370b185e47550bb501c179c63773909af36 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 12 Nov 2017 03:52:20 +0000 Subject: [PATCH 3/4] Fixed uwsgi_features --- pyprometheus/contrib/uwsgi_features.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyprometheus/contrib/uwsgi_features.py b/pyprometheus/contrib/uwsgi_features.py index 2783ba2..bf87dcf 100644 --- a/pyprometheus/contrib/uwsgi_features.py +++ b/pyprometheus/contrib/uwsgi_features.py @@ -177,6 +177,7 @@ def __init__(self, sharedarea_id=SHAREDAREA_ID, namespace="", stats=False, label self._namespace = namespace self._stats = stats self._labels = tuple(sorted(labels.items(), key=lambda x: x[0])) + self._labels_names = tuple(label[0] for label in self._labels) self._syncs = 0 @@ -214,9 +215,9 @@ def get_unique_id(): def declare_metrics(self): return { - "memory_sync": Counter(self.metric_name("memory_read"), "UWSGI shared memory syncs", ("sharedarea", "id") + self._labels), - "memory_size": Gauge(self.metric_name("memory_size"), "UWSGI shared memory size", ("sharedarea", ) + self._labels), - "num_keys": Gauge(self.metric_name("num_keys"), "UWSGI num_keys", ("sharedarea", ) + self._labels) + "memory_sync": Counter(self.metric_name("memory_read"), "UWSGI shared memory syncs", ("sharedarea", "id") + self._labels_names), + "memory_size": Gauge(self.metric_name("memory_size"), "UWSGI shared memory size", ("sharedarea", ) + self._labels_names), + "num_keys": Gauge(self.metric_name("num_keys"), "UWSGI num_keys", ("sharedarea", ) + self._labels_names) } def collect(self): From e5a0191aa06eb8d97d51a6a37b88d2a2c3b062ca Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 3 Feb 2018 17:34:32 +0000 Subject: [PATCH 4/4] Added custom class for UWSGIFlushStorage --- pyprometheus/contrib/uwsgi_features.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyprometheus/contrib/uwsgi_features.py b/pyprometheus/contrib/uwsgi_features.py index bf87dcf..164abdf 100644 --- a/pyprometheus/contrib/uwsgi_features.py +++ b/pyprometheus/contrib/uwsgi_features.py @@ -612,8 +612,9 @@ class UWSGIFlushStorage(LocalMemoryStorage): """ SHAREDAREA_ID = int(os.environ.get("PROMETHEUS_UWSGI_SHAREDAREA", 0)) - def __init__(self, sharedarea_id=UWSGIStorage.SHAREDAREA_ID, namespace="", stats=False, labels={}): - self._uwsgi_storage = UWSGIStorage(sharedarea_id, namespace=namespace, stats=stats, labels=labels) + def __init__(self, sharedarea_id=UWSGIStorage.SHAREDAREA_ID, namespace="", + stats=False, labels={}, storage_class=UWSGIStorage): + self._uwsgi_storage = storage_class(sharedarea_id, namespace=namespace, stats=stats, labels=labels) self._flush = 0 self._get_items = 0 self._clear = 0