show more detail in halist (show which check failed last and what its output was)

hacheck/handlers.py

@@ -13,7 +13,7 @@
 
 log = logging.getLogger('hacheck')
 
-StatusResponse = collections.namedtuple('StatusResponse', ['code', 'remote_ip', 'ts'])
+StatusResponse = collections.namedtuple('StatusResponse', ['code', 'remote_ip', 'ts', 'failed_checker', 'message'])
 
 if hasattr(collections, 'Counter'):
     Counter = collections.Counter  # fast
@@ -87,7 +87,14 @@ def get(self, service_name, port, query):
                 )
                 last_message = message
                 if code > 200:
-                    last_statuses[service_name] = StatusResponse(code, self.request.remote_ip, time.time())
+                    failed_checker_name = str(this_checker.__name__)
+                    if failed_checker_name.startswith('check_'):
+                        failed_checker_name = failed_checker_name[6:]
+                    last_statuses[service_name] = StatusResponse(
+                        code, self.request.remote_ip, time.time(),
+                        failed_checker=failed_checker_name,
+                        message=message,
+                    )
                     if code in tornado.httputil.responses:
                         self.set_status(code)
                     else:
@@ -96,7 +103,10 @@ def get(self, service_name, port, query):
                     self.finish()
                     break
             else:
-                last_statuses[service_name] = StatusResponse(200, self.request.remote_ip, time.time())
+                last_statuses[service_name] = StatusResponse(
+                    200, self.request.remote_ip, time.time(),
+                    failed_checker=None, message=None
+                )
                 self.set_status(200)
                 self.write(last_message)
                 self.finish()

hacheck/haupdown.py

@@ -68,7 +68,7 @@ def main(default_action='list'):
         '--port',
         type=str,
         default=3333,
-        help='Port that the hacheck daemon is running on (default %(default)'
+        help='Port that the hacheck daemon is running on (default %(default)s)'
     )
     opts, args = parser.parse_args()
 

tests/test_application.py

@@ -86,8 +86,8 @@ def test_calls_all_checkers(self):
         rv1.set_result((200, b'OK1'))
         rv2 = tornado.concurrent.Future()
         rv2.set_result((200, b'OK2'))
-        checker1 = mock.Mock(return_value=rv1)
-        checker2 = mock.Mock(return_value=rv2)
+        checker1 = mock.Mock(return_value=rv1, __name__='checker1')
+        checker2 = mock.Mock(return_value=rv2, __name__='checker2')
         with mock.patch.object(handlers.SpoolServiceHandler, 'CHECKERS', [checker1, checker2]):
             response = self.fetch('/spool/foo/1/status')
             self.assertEqual(200, response.code)
@@ -116,8 +116,8 @@ def test_any_failure_fails_all_first(self):
         rv1.set_result((404, b'NOK1'))
         rv2 = tornado.concurrent.Future()
         rv2.set_result((200, b'OK2'))
-        checker1 = mock.Mock(return_value=rv1)
-        checker2 = mock.Mock(return_value=rv2)
+        checker1 = mock.Mock(return_value=rv1, __name__='checker1')
+        checker2 = mock.Mock(return_value=rv2, __name__='checker2')
         with mock.patch.object(handlers.SpoolServiceHandler, 'CHECKERS', [checker1, checker2]):
             response = self.fetch('/spool/foo/2/status')
             self.assertEqual(404, response.code)
@@ -128,8 +128,8 @@ def test_any_failure_fails_all_second(self):
         rv1.set_result((200, b'OK1'))
         rv2 = tornado.concurrent.Future()
         rv2.set_result((404, b'NOK2'))
-        checker1 = mock.Mock(return_value=rv1)
-        checker2 = mock.Mock(return_value=rv2)
+        checker1 = mock.Mock(return_value=rv1, __name__='checker1')
+        checker2 = mock.Mock(return_value=rv2, __name__='checker2')
         with mock.patch.object(handlers.SpoolServiceHandler, 'CHECKERS', [checker1, checker2]):
             response = self.fetch('/spool/foo/2/status')
             self.assertEqual(404, response.code)
@@ -139,14 +139,17 @@ def test_weird_code(self):
         # test that unusual HTTP codes are rewritten to 503s
         rv = tornado.concurrent.Future()
         rv.set_result((6000, 'this code is weird'))
-        checker = mock.Mock(return_value=rv)
+        checker = mock.Mock(return_value=rv, __name__='mock_checker')
         with mock.patch.object(handlers.HTTPServiceHandler, 'CHECKERS', [checker]):
             response = self.fetch('/http/uncached-weird-code/80/status')
             self.assertEqual(503, response.code)
 
     def test_option_parsing(self):
         with nested(
-            mock.patch('sys.argv', ['ignorethis', '-c', self.config_file.name, '--spool-root', 'foo']),
+            mock.patch(
+                'sys.argv',
+                ['ignorethis', '-c', self.config_file.name, '--spool-root', 'foo', '-B', '127.0.0.1']
+            ),
             mock.patch.object(tornado.ioloop.IOLoop, 'instance'),
             mock.patch.object(cache, 'configure'),
             mock.patch.object(main, 'get_app'),
@@ -165,14 +168,33 @@ def test_show_recent(self):
         self.assertEqual(
             b,
             {
-                'seen_services': [['foo', {'code': 200, 'ts': mock.ANY, 'remote_ip': '127.0.0.1'}]],
+                'seen_services': [['foo', {
+                    'code': 200, 'ts': mock.ANY, 'remote_ip': '127.0.0.1', 'failed_checker': None,
+                }]],
                 'threshold_seconds': 600
             })
         response = self.fetch('/recent?threshold=20')
         b = json.loads(response.body.decode('utf-8'))
         self.assertEqual(
             b,
             {
-                'seen_services': [['foo', {'code': 200, 'ts': mock.ANY, 'remote_ip': '127.0.0.1'}]],
+                'seen_services': [[
+                    'foo', {
+                        'code': 200, 'ts': mock.ANY, 'remote_ip': '127.0.0.1', 'failed_checker': None,
+                    }]],
                 'threshold_seconds': 20
             })
+        # now mark it as down
+        with mock.patch.object(spool, 'is_up', return_value=(False, {"service": "any", "reason": ""})):
+            response = self.fetch('/spool/foo/1/status')
+            response = self.fetch('/recent?threshold=20')
+            b = json.loads(response.body.decode('utf-8'))
+            self.assertEqual(
+                b,
+                {
+                    'seen_services': [[
+                        'foo', {
+                            'code': 503, 'ts': mock.ANY, 'remote_ip': '127.0.0.1', 'failed_checker': 'check_spool',
+                        }]],
+                    'threshold_seconds': 20
+                })