another readme update

Author: gistart

.gitignore

@@ -14,6 +14,8 @@ __pycache__
 .vscode*
 .coverage
 coverage.xml
+test/poke.py
+compare/*
 
 !.gitkeep
 !/.gitignore

README.md

@@ -5,11 +5,25 @@
 
 Push metrics from your regular and/or long-running jobs to existing Prometheus/VictoriaMetrics monitoring system.
 
-:no_entry: not tested in real environment yet!
+Currently supports pushes directly to VictoriaMetrics via UDP and HTTP using InfluxDB line protocol as [described here](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html?highlight=telegraf#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
 
-Currently supports pushes directly to VictoriaMetrics via UDP and HTTP using InfluxDB line protocol as [described here](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html?highlight=telegraf#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf), and into StatsD/statsd-exporter in StatsD format via UDP ([not ideal](https://github.com/prometheus/statsd_exporter#with-statsd)).
+For pure Prometheus setups, pushes into StatsD/statsd-exporter in StatsD format via UDP are supported ([see exporter docs](https://github.com/prometheus/statsd_exporter#with-statsd)). Prometheus and StatsD metric types are not fully compatible, so currenly all metrics become StatsD gauges, but `rate`, `increase`, `histogram_quantile` and other PromQL functions produce same results as if types never changed.
 
-## Default labelvalues
+Install it via pip:
+
+```sh
+pip install prometheus-push-client
+```
+
+## Metrics
+
+This library uses `prometheus-client` metric implementation, but adds with some minor tweaks.
+
+### Separate registry
+
+New metric constructors use separate `PUSH_REGISTRY` as a default, not to interfere with other metrics already defined and monitored in existing projects.
+
+### Default labelvalues
 
 With regular prometheus_client, defaults may be defined for either _none_ or _all_ the labels (with `labelvalues`), but that's not enough.
 
@@ -18,6 +32,9 @@ We probably want to define _some_ defaults, like `hostname`, or more importantly
 Following example shows how to use defaults, and how to override them if necessary.
 
 ```python
+import prometheus_push_client as ppc
+
+
 counter1 = ppc.Counter(
     name="c1",
     labelnames=["VictoriaMetrics_AccountID", "host", "event_type"],
@@ -40,13 +57,15 @@ counter1.labels("non-default", "login").inc()
 
 Metrics with no labels are initialized at creation time. This can have unpleasant side-effect: if we initialize lots of metrics not used in currently running job, background clients will have to push their non-changing values in every synchronization session.
 
-To avoid that we'll have to properly isolate each task's metrics, which can be impossible or rather tricky, or we can create metrics with default, non-changing labels (like `hostname`). Such metrics will be initialized on fisrt use (inc), and we'll be pusing only those we actually used!
+To avoid that we'll have to properly isolate each task's metrics, which can be impossible or rather tricky, or we can create metrics with default, non-changing labels (like `hostname`). Such metrics will be initialized on fisrt use (inc), and we'll be pushing only those we actually used.
 
 ## Background clients
 
 Background clients spawn synchronization jobs "in background" (meaning in a thread or asyncio task) to periodically send all metrics from `ppc.PUSH_REGISTRY` to the destination.
 
-:warning: background clients will attempt to stop gracefully, synchronizing registry "one last time" after job exits or crashes. This _may_ mess up sampling aggregation, I'm not sure yet.
+Clients will attempt to stop gracefully, synchronizing registry "one last time" after job exits or crashes. Sometimes this _may_ mess up Grafana sampling, but the worst picture I could atrifically create looks like this:
+
+![graceful push effect](./docs/img/graceful_stop_effect01.png)
 
 Best way to use them is via decorators. These clients are intended to be used with long running, but finite tasks, which could be spawned anywhere, therefor not easily accessible by the scraper. If that's not the case -- just use "passive mode" w/ the scraper instead.
 

docs/img/graceful_stop_effect01.png

@@ -2,6 +2,10 @@
 
 
 class InfluxFormat(BaseFormat):
+    """
+    As descibed at:
+    https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/
+    """
 
     FMT_SAMPLE = "{sample_name}{tag_set} {measurement_name}={value}{timestamp}"
 

prometheus_push_client/formats/influx.py

@@ -1,5 +1,3 @@
-from collections import defaultdict
-
 from prometheus_push_client.formats.base import BaseFormat
 
 
@@ -11,42 +9,19 @@ class StatsdFormat(BaseFormat):
 
     No "realtime" types supported yet.
     """
-    # TODO: support statsd native "sets", "timers" and "histograms" in FG mode
+    # TODO: support statsd native "sets", "timers" and "histograms" in FG mode?
 
     FMT_DATAPOINT = "{measurement}{tag_set}:{value}|{dtype}"  # influx-style tags
 
-    DTYPES = {
-        "gauge": defaultdict(lambda: "g"),
-        "counter": {
-            "total": "c",
-            "created": "g",
-        },
-        "summary": {
-            "sum": "c",
-            "count": "c",
-            "created": "g",
-        },
-        "histogram": {
-            "bucket": "c",
-            "sum": "c",
-            "count": "c",
-            "created": "g",
-        }
-
-        # TODO: info, enum
-    }
-
 
     def format_sample(self, sample, metric):
         # TODO: gauges reset?
-
         measurement_name = sample.name
 
         chunks = measurement_name.rsplit("_", 1)
         suffix = chunks[-1] if len(chunks) > 1 else None
 
-        # dtype = self.DTYPES[metric.type][suffix]
-        dtype = "g"  # TODO: omfg! everything behaves like gauge
+        dtype = "g"  # everything behaves like gauge on statsd-side
         value = sample.value
 
         tag_set = ""

prometheus_push_client/formats/statsd.py

@@ -17,6 +17,7 @@ def stop(self):
         self.transport.close()
 
     def pack_datagrams(self, iterable):
+        # TODO: if first line > mtu?
         datagram = []
         datagram_size = 0
         for line in iterable:

prometheus_push_client/transports/udp.py

@@ -12,7 +12,7 @@
 readme_lines = []
 with open('README.md') as fd:
     readme_lines = filter(None, fd.read().splitlines())
-readme_lines = list(readme_lines)[:5]
+readme_lines = list(readme_lines)[:10]
 readme_lines.append('Read more at [github page](%s).' % github_url)
 readme = '\n\n'.join(readme_lines)
 

setup.py

@@ -74,7 +74,7 @@ def _test():
     with pytest.raises(ZeroDivisionError):
         _test()
 
-    time.sleep(2.0)  # let them sync
+    time.sleep(3.0)  # let them sync
 
     found_after = export(cfg)
     count_after = count_samples(found_after, counter1._name)
@@ -103,7 +103,7 @@ async def _test():
     with pytest.raises(ZeroDivisionError):
         await _test()
 
-    await asyncio.sleep(2.0)  # let them sync
+    await asyncio.sleep(3.0)  # let them sync
 
     found_after = export(cfg)
     count_after = count_samples(found_after, counter1._name)
@@ -131,7 +131,7 @@ def _test():
     with pytest.raises(ZeroDivisionError):
         _test()
 
-    time.sleep(2.0)  # let them sync
+    time.sleep(3.0)  # let them sync
 
     found_after = export(cfg)
     count_after = count_samples(found_after, counter1._name)
@@ -160,7 +160,7 @@ async def _test():
     with pytest.raises(ZeroDivisionError):
         await _test()
 
-    await asyncio.sleep(2.0)  # let them sync
+    await asyncio.sleep(3.0)  # let them sync
 
     found_after = export(cfg)
     count_after = count_samples(found_after, counter1._name)