diff --git a/Pipfile b/Pipfile index b31ff343..44457097 100644 --- a/Pipfile +++ b/Pipfile @@ -15,6 +15,7 @@ attrs = "*" pyyaml = ">=3.12" flexmock = "*" jsonformatter = "*" +python-logging-loki = "*" [dev-packages] pytest = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 1884df34..b3d6afbc 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "402627e6b2fd06c7425f28a4ae48c0f19900f3d5426b96b9bfa2a3da67e607d3" + "sha256": "1efbd3c8201671fceb21c76605940c6b24d3e23d6b7f3c6a6c2a6c13c3eb6787" }, "pipfile-spec": 6, "requires": { @@ -222,6 +222,14 @@ "markers": "python_version >= '3.4'", "version": "==2.0.1" }, + "python-logging-loki": { + "hashes": [ + "sha256:8a9131db037fbea3d390089c4c32dbe7ed233944905079615a9fb6f669b0f4e6", + "sha256:b83610c8a3adc99fbab072493b91dfb25ced69be4874fefe3ab457b391adbf60" + ], + "index": "pypi", + "version": "==0.3.1" + }, "python-string-utils": { "hashes": [ "sha256:dcf9060b03f07647c0a603408dc8b03f807f3b54a05c6e19eb14460256fac0cb", @@ -280,6 +288,13 @@ ], "version": "==1.3.0" }, + "rfc3339": { + "hashes": [ + "sha256:d53c3b5eefaef892b7240ba2a91fef012e86faa4d0a0ca782359c490e00ad4d0", + "sha256:f44316b21b21db90a625cde04ebb0d46268f153e6093021fa5893e92a96f58a3" + ], + "version": "==6.2" + }, "rfc5424-logging-handler": { "hashes": [ "sha256:9ae14073ef6d76d0c730ad6b6e3aeece841a6d413672d282876c0506dc097257", @@ -514,7 +529,7 @@ "sha256:fda29412a66099af6d6de0baa6bd7c52674de177ec2ad2630ca264142d69c6c7", "sha256:ff1330e8bc996570221b450e2d539134baa9465f5cb98aff0e0f73f34172e0ae" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==5.3.1" }, "dependency-management": { @@ -566,7 +581,7 @@ "sha256:c729845434366216d320e936b8ad6f9d681aab72dc7cbc2d51bedc3582f3ad1e", "sha256:fff4f0c04e1825522ce6949973e83110a6e907750cd92d128b0d14aaaadbffdc" ], - "markers": "python_version >= '3.6' and python_version < '4.0'", + "markers": "python_version >= '3.6' and python_version < '4'", "version": "==5.7.0" }, "lazy-object-proxy": { diff --git a/thoth/common/logging.py b/thoth/common/logging.py index fc3dba6f..838a0631 100644 --- a/thoth/common/logging.py +++ b/thoth/common/logging.py @@ -30,6 +30,8 @@ from typing import Any from jsonformatter import JsonFormatter +import logging_loki +from multiprocessing import Queue from sentry_sdk import init as sentry_sdk_init from sentry_sdk.integrations.logging import ignore_logger import daiquiri @@ -38,6 +40,9 @@ _RSYSLOG_HOST = os.getenv("RSYSLOG_HOST") _RSYSLOG_PORT = os.getenv("RSYSLOG_PORT") +_LOKI_URL = os.getenv("THOTH_LOKI_URL") +_LOKI_USERNAME = os.getenv("THOTH_LOKI_USERNAME") +_LOKI_PASSWORD = os.getenv("THOTH_LOKI_PASSWORD") _DEFAULT_LOGGING_CONF_START = "THOTH_LOG_" _LOGGING_ADJUSTMENT_CONF = "THOTH_ADJUST_LOGGING" _SENTRY_DSN = os.getenv("SENTRY_DSN") @@ -267,7 +272,7 @@ def init_logging( logging.getLogger().setLevel(logging.WARNING) logging.getLogger().propagate = False - root_logger = logging.getLogger("thoth.common") + root_logger = logging.getLogger() environment = os.getenv("SENTRY_ENVIRONMENT", os.getenv("THOTH_DEPLOYMENT_NAME")) # Disable annoying unverified HTTPS request warnings. @@ -364,3 +369,58 @@ def init_logging( ) else: root_logger.info("Logging to rsyslog endpoint is turned off") + + if _LOKI_URL: + root_logger.info("Initializing logging to a Loki instance") + tags = {"application": "thoth"} + thoth_deployment = os.getenv("THOTH_DEPLOYMENT_NAME") + if thoth_deployment: + tags["thoth_deployment"] = thoth_deployment # Note: tags cannot have dashes. + + # loki_handler = Op1stLokiHandler( + loki_handler = Op1stLokiQueueHandler( + Queue(-1), + url=_LOKI_URL, + tags={ + "app": "thoth", + "thoth_deployment": thoth_deployment, + }, + auth=(_LOKI_USERNAME, _LOKI_PASSWORD), + ) + root_logger.addHandler(loki_handler) + + +class Op1stLokiLokiEmitter(logging_loki.emitter.LokiEmitterV1): + """An emitter implementation that is specific for Operate 1st Loki instance.""" + + def __call__(self, record: logging.LogRecord, line: str): + """Send log record to Loki.""" + payload = self.build_payload(record, line) + resp = self.session.post(self.url, json=payload, headers={"X-Scope-OrgID": "opf-thoth"}) + if resp.status_code != self.success_response_code: + raise ValueError("Unexpected Loki API response status code ({0}): {1}".format(resp.status_code, resp.text)) + + +class Op1stLokiQueueHandler(logging.handlers.QueueHandler): + """A custom Loki handler which works with Operate 1st Loki instance.""" + + def __init__(self, queue: Queue, **kwargs): + super().__init__(queue) + self.handler = Op1stLokiHandler(**kwargs) + self.listener = logging.handlers.QueueListener(self.queue, self.handler) + self.listener.start() + + +class Op1stLokiHandler(logging_loki.handlers.LokiHandler): + """A custom Loki handler which works with Operate 1st Loki instance.""" + + def __init__( + self, + url: str, + tags: Optional[Dict[str, str]] = None, + auth: Optional[logging_loki.emitter.BasicAuth] = None, + version: Optional[str] = None, + ) -> None: + """Initialize the Operate 1st Loki handler.""" + logging_loki.LokiHandler.__init__(self, url=url, tags=tags, auth=auth, version=version) + self.emitter = Op1stLokiLokiEmitter(url, tags, auth)