From db91eff4a69946c840a3210a23c89d0970d53887 Mon Sep 17 00:00:00 2001 From: Robert Swain Date: Mon, 9 May 2016 21:06:00 +0200 Subject: [PATCH 1/5] plugins/prometheus: Add rudimentary support for alertmanager 0.1 JSON payload contains "alerts" key instead of "alert" key. --- plugins/prometheus.py | 16 ++++++++++------ prometheus.json | 5 +++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/plugins/prometheus.py b/plugins/prometheus.py index dec73db..599d5c1 100644 --- a/plugins/prometheus.py +++ b/plugins/prometheus.py @@ -48,12 +48,16 @@ def get_webhook_key(self): def on_receive_webhook(self, url, data, ip, headers): json_data = json.loads(data) log.info("recv %s", json_data) - template = Template(self.store.get("message_template")) - for alert in json_data.get("alert", []): - for room_id in self.rooms.get_room_ids(): - log.debug("queued message for room " + room_id + " at " + str(self.queue_counter) + ": %s", alert) - queue.put((self.queue_counter, room_id, template.render(alert))) - self.queue_counter += 1 + def process_alerts(alerts, template): + for alert in alerts: + for room_id in self.rooms.get_room_ids(): + log.debug("queued message for room " + room_id + " at " + str(self.queue_counter) + ": %s", alert) + queue.put((self.queue_counter, room_id, template.render(alert))) + self.queue_counter += 1 + # try version 1 format + process_alerts(json_data.get("alert", []), Template(self.store.get("v1_message_template")) + # try version 2+ format + process_alerts(json_data.get("alerts", []), Template(self.store.get("v2_message_template")) class MessageConsumer(Thread): diff --git a/prometheus.json b/prometheus.json index 871f1ce..0870768 100644 --- a/prometheus.json +++ b/prometheus.json @@ -1,3 +1,4 @@ { - "message_template": "[ALERT] {{labels.alertname}} : {{summary}}\n{{payload.generatorURL}}\n{{description}}\nValue: {{payload.value}}\n{{payload.alertingRule}}\nActive since: {{payload.activeSince}}" -} \ No newline at end of file + "v1_message_template": "[ALERT] {{labels.alertname}} : {{summary}}\n{{payload.generatorURL}}\n{{description}}\nValue: {{payload.value}}\n{{payload.alertingRule}}\nActive since: {{payload.activeSince}}", + "v2_message_template": "[ALERT] {{labels.alertname}} : {{annotations.summary}}\n{{labels.job}} on {{labels.instance}}\n{{status}}\n{{annotations.description}}\n{{generatorURL}}\nStarted: {{startsAt}} ; Ended: {{endsAt}}" +} From 3053e15b99a83a3d532e8f5e020d4b600b459094 Mon Sep 17 00:00:00 2001 From: Robert Swain Date: Wed, 22 Jun 2016 13:01:46 +0200 Subject: [PATCH 2/5] plugins/prometheus: Add tracking support --- plugins/prometheus.py | 112 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 107 insertions(+), 5 deletions(-) diff --git a/plugins/prometheus.py b/plugins/prometheus.py index 599d5c1..96369ca 100644 --- a/plugins/prometheus.py +++ b/plugins/prometheus.py @@ -21,6 +21,8 @@ class PrometheusPlugin(Plugin): #Webhooks: # /neb/prometheus + + TRACKING = ["track", "tracking"] TYPE_TRACK = "org.matrix.neb.plugin.prometheus.projects.tracking" @@ -35,6 +37,101 @@ def __init__(self, *args, **kwargs): self.consumer.daemon = True self.consumer.start() + + def cmd_show(self, event, action): + """Show information on projects or projects being tracked. + Show which projects are being tracked. 'prometheus show tracking' + Show which proejcts are recognised so they could be tracked. 'prometheus show projects' + """ + if action in self.TRACKING: + return self._get_tracking(event["room_id"]) + elif action == "projects": + projects = self.store.get("known_projects") + return "Available projects: %s" % json.dumps(projects) + else: + return "Invalid arg '%s'.\n %s" % (action, self.cmd_show.__doc__) + + @admin_only + def cmd_track(self, event, *args): + """Track projects. 'prometheus track Foo "bar with spaces"'""" + if len(args) == 0: + return self._get_tracking(event["room_id"]) + + for project in args: + if not project in self.store.get("known_projects"): + return "Unknown project name: %s." % project + + self._send_track_event(event["room_id"], args) + + return "Prometheus notifications for projects %s will be displayed when they fail." % (args) + + @admin_only + def cmd_add(self, event, project): + """Add a project for tracking. 'prometheus add projectName'""" + if project not in self.store.get("known_projects"): + return "Unknown project name: %s." % project + + try: + room_projects = self.rooms.get_content( + event["room_id"], + PrometheusPlugin.TYPE_TRACK)["projects"] + except KeyError: + room_projects = [] + + if project in room_projects: + return "%s is already being tracked." % project + + room_projects.append(project) + self._send_track_event(event["room_id"], room_projects) + + return "Added %s. Prometheus notifications for projects %s will be displayed when they fail." % (project, room_projects) + + @admin_only + def cmd_remove(self, event, project): + """Remove a project from tracking. 'prometheus remove projectName'""" + try: + room_projects = self.rooms.get_content( + event["room_id"], + PrometheusPlugin.TYPE_TRACK)["projects"] + except KeyError: + room_projects = [] + + if project not in room_projects: + return "Cannot remove %s : It isn't being tracked." % project + + room_projects.remove(project) + self._send_track_event(event["room_id"], room_projects) + + return "Removed %s. Prometheus notifications for projects %s will be displayed when they fail." % (project, room_projects) + + @admin_only + def cmd_stop(self, event, action): + """Stop tracking projects. 'prometheus stop tracking'""" + if action in self.TRACKING: + self._send_track_event(event["room_id"], []) + return "Stopped tracking projects." + else: + return "Invalid arg '%s'.\n %s" % (action, self.cmd_stop.__doc__) + + def _get_tracking(self, room_id): + try: + return ("Currently tracking %s" % + json.dumps(self.rooms.get_content( + room_id, PrometheusPlugin.TYPE_TRACK)["projects"] + ) + ) + except KeyError: + return "Not tracking any projects currently." + + def _send_track_event(self, room_id, project_names): + self.matrix.send_state_event( + room_id, + self.TYPE_TRACK, + { + "projects": project_names + } + ) + def on_event(self, event, event_type): self.rooms.update(event) @@ -51,13 +148,18 @@ def on_receive_webhook(self, url, data, ip, headers): def process_alerts(alerts, template): for alert in alerts: for room_id in self.rooms.get_room_ids(): - log.debug("queued message for room " + room_id + " at " + str(self.queue_counter) + ": %s", alert) - queue.put((self.queue_counter, room_id, template.render(alert))) - self.queue_counter += 1 + try: + if (len(self.rooms.get_content( + room_id, PrometheusPlugin.TYPE_TRACK)["projects"])): + log.debug("queued message for room " + room_id + " at " + str(self.queue_counter) + ": %s", alert) + queue.put((self.queue_counter, room_id, template.render(alert))) + self.queue_counter += 1 + except KeyError: + pass # try version 1 format - process_alerts(json_data.get("alert", []), Template(self.store.get("v1_message_template")) + process_alerts(json_data.get("alert", []), Template(self.store.get("v1_message_template"))) # try version 2+ format - process_alerts(json_data.get("alerts", []), Template(self.store.get("v2_message_template")) + process_alerts(json_data.get("alerts", []), Template(self.store.get("v2_message_template"))) class MessageConsumer(Thread): From 26e8008cc3b6e945f84a2290d18c840bc8cc912b Mon Sep 17 00:00:00 2001 From: Robert Swain Date: Wed, 22 Jun 2016 14:34:06 +0200 Subject: [PATCH 3/5] prometheus.json: Fix missing known_projects configuration --- prometheus.json | 1 + 1 file changed, 1 insertion(+) diff --git a/prometheus.json b/prometheus.json index 0870768..853bb98 100644 --- a/prometheus.json +++ b/prometheus.json @@ -1,4 +1,5 @@ { + "known_projects": [], "v1_message_template": "[ALERT] {{labels.alertname}} : {{summary}}\n{{payload.generatorURL}}\n{{description}}\nValue: {{payload.value}}\n{{payload.alertingRule}}\nActive since: {{payload.activeSince}}", "v2_message_template": "[ALERT] {{labels.alertname}} : {{annotations.summary}}\n{{labels.job}} on {{labels.instance}}\n{{status}}\n{{annotations.description}}\n{{generatorURL}}\nStarted: {{startsAt}} ; Ended: {{endsAt}}" } From e02db05ce1099573ea5241c17e7d26fd696a5ff7 Mon Sep 17 00:00:00 2001 From: Robert Swain Date: Wed, 22 Jun 2016 15:21:45 +0200 Subject: [PATCH 4/5] plugins/prometheus: Update documentation --- plugins/prometheus.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/prometheus.py b/plugins/prometheus.py index 96369ca..1c883e8 100644 --- a/plugins/prometheus.py +++ b/plugins/prometheus.py @@ -15,7 +15,13 @@ class PrometheusPlugin(Plugin): - """Plugin for interacting with Prometheus. + """ Plugin for receiving Prometheus alerts. + prometheus show projects : Display which projects this bot recognises. + prometheus show track|tracking : Display which projects this bot is tracking. + prometheus track project1 project2 ... : Track Prometheus notifications for the named projects. + prometheus stop track|tracking : Stop tracking Prometheus notifications. + prometheus add projectName : Start tracking projectName. + prometheus remove projectName : Stop tracking projectName. """ name = "prometheus" From 69bcdc192572d824ca7f9b1f2231cecaeb9e71b0 Mon Sep 17 00:00:00 2001 From: Robert Swain Date: Wed, 22 Jun 2016 15:22:24 +0200 Subject: [PATCH 5/5] prometheus.json: Add default project --- prometheus.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus.json b/prometheus.json index 853bb98..e3f8f3c 100644 --- a/prometheus.json +++ b/prometheus.json @@ -1,5 +1,5 @@ { - "known_projects": [], + "known_projects": ["prometheus"], "v1_message_template": "[ALERT] {{labels.alertname}} : {{summary}}\n{{payload.generatorURL}}\n{{description}}\nValue: {{payload.value}}\n{{payload.alertingRule}}\nActive since: {{payload.activeSince}}", "v2_message_template": "[ALERT] {{labels.alertname}} : {{annotations.summary}}\n{{labels.job}} on {{labels.instance}}\n{{status}}\n{{annotations.description}}\n{{generatorURL}}\nStarted: {{startsAt}} ; Ended: {{endsAt}}" }