Add basic setup for django-structlog, Celery included

Author: RadoRado

config/django/base.py

@@ -188,7 +188,12 @@
 
 INSTALLED_APPS, MIDDLEWARE = DebugToolbarSetup.do_settings(INSTALLED_APPS, MIDDLEWARE)
 
+from config.settings.loggers.settings import *  # noqa
+from config.settings.loggers.setup import LoggersSetup  # noqa
 
-SHELL_PLUS_IMPORTS = [
-    "from styleguide_example.blog_examples.print_qs_in_shell.utils import print_qs"
-]
+INSTALLED_APPS, MIDDLEWARE = LoggersSetup.setup_settings(INSTALLED_APPS, MIDDLEWARE)
+LoggersSetup.setup_structlog()
+LOGGING = LoggersSetup.setup_logging()
+
+
+SHELL_PLUS_IMPORTS = ["from styleguide_example.blog_examples.print_qs_in_shell.utils import print_qs"]

config/settings/loggers/settings.py

@@ -0,0 +1,4 @@
+import logging
+
+DJANGO_STRUCTLOG_STATUS_4XX_LOG_LEVEL = logging.INFO
+DJANGO_STRUCTLOG_CELERY_ENABLED = True

config/settings/loggers/setup.py

@@ -0,0 +1,114 @@
+import logging
+
+import structlog
+
+
+class IgnoreFilter(logging.Filter):
+    def filter(self, record):
+        return False
+
+
+class LoggersSetup:
+    """
+    We use a class, just for namespacing convenience.
+    """
+
+    @staticmethod
+    def setup_settings(INSTALLED_APPS, MIDDLEWARE, middleware_position=None):
+        INSTALLED_APPS = INSTALLED_APPS + ["django_structlog"]
+
+        django_structlog_middleware = "django_structlog.middlewares.RequestMiddleware"
+
+        if middleware_position is None:
+            MIDDLEWARE = MIDDLEWARE + [django_structlog_middleware]
+        else:
+            # Grab a new copy of the list, since insert mutates the internal structure
+            _middleware = MIDDLEWARE[::]
+            _middleware.insert(middleware_position, django_structlog_middleware)
+
+            MIDDLEWARE = _middleware
+
+        return INSTALLED_APPS, MIDDLEWARE
+
+    @staticmethod
+    def setup_structlog():
+        structlog.configure(
+            processors=[
+                structlog.contextvars.merge_contextvars,
+                structlog.stdlib.filter_by_level,
+                structlog.processors.TimeStamper(fmt="iso", utc=True),
+                structlog.stdlib.add_logger_name,
+                structlog.stdlib.add_log_level,
+                structlog.stdlib.PositionalArgumentsFormatter(),
+                structlog.processors.StackInfoRenderer(),
+                structlog.dev.set_exc_info,
+                structlog.processors.format_exc_info,
+                # structlog.processors.dict_tracebacks,
+                structlog.processors.UnicodeDecoder(),
+                structlog.processors.CallsiteParameterAdder(
+                    {
+                        structlog.processors.CallsiteParameter.FILENAME,
+                        structlog.processors.CallsiteParameter.FUNC_NAME,
+                        structlog.processors.CallsiteParameter.LINENO,
+                    }
+                ),
+                structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+            ],
+            logger_factory=structlog.stdlib.LoggerFactory(),
+            cache_logger_on_first_use=True,
+        )
+
+    @staticmethod
+    def setup_logging():
+        return {
+            "version": 1,
+            "disable_existing_loggers": False,
+            "formatters": {
+                "json_formatter": {
+                    "()": structlog.stdlib.ProcessorFormatter,
+                    "processor": structlog.processors.JSONRenderer(),
+                },
+                "plain_console": {
+                    "()": structlog.stdlib.ProcessorFormatter,
+                    "processor": structlog.dev.ConsoleRenderer(),
+                },
+            },
+            "filters": {
+                "ignore": {
+                    "()": "config.settings.loggers.setup.IgnoreFilter",
+                },
+            },
+            "handlers": {
+                # Important notes regarding handlers.
+                #
+                # 1. Make sure you use handlers adapted for your project.
+                # These handlers configurations are only examples for this library.
+                # See python's logging.handlers: https://docs.python.org/3/library/logging.handlers.html
+                #
+                # 2. You might also want to use different logging configurations depending of the environment.
+                # Different files (local.py, tests.py, production.py, ci.py, etc.) or only conditions.
+                # See https://docs.djangoproject.com/en/dev/topics/settings/#designating-the-settings
+                "console": {
+                    "class": "logging.StreamHandler",
+                    "formatter": "plain_console",
+                }
+            },
+            "loggers": {
+                # We want to get rid of the runserver logs
+                "django.server": {"propagate": False, "handlers": ["console"], "filters": ["ignore"]},
+                # We want to get rid of the logs for 4XX and 5XX
+                "django.request": {"propagate": False, "handlers": ["console"], "filters": ["ignore"]},
+                "django_structlog": {
+                    "handlers": ["console"],
+                    "level": "INFO",
+                },
+                "celery": {
+                    "handlers": ["console"],
+                    "level": "INFO",
+                },
+                "styleguide_example": {
+                    "handlers": ["console"],
+                    "level": "INFO",
+                },
+            },
+        }

requirements/base.txt

@@ -13,6 +13,7 @@ django-filter==25.2
 django-extensions==4.1
 django-cors-headers==4.9.0
 django-storages==1.14.6
+django-structlog[celery]==9.1.1
 
 drf-jwt==1.19.2
 

styleguide_example/errors/apis.py

@@ -1,3 +1,4 @@
+import structlog
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
@@ -8,6 +9,8 @@
 from styleguide_example.errors.services import trigger_errors
 from styleguide_example.users.services import user_create
 
+logger = structlog.get_logger(__name__)
+
 
 class TriggerErrorApi(APIView):
     def get(self, request):
@@ -30,6 +33,12 @@ def get(self, request):
 
 class TriggerUnhandledExceptionApi(APIView):
     def get(self, request):
-        raise Exception("Oops")
+        log = logger.bind()
+
+        try:
+            raise Exception("Oops")
+        except Exception:
+            log.exception("unhandled_exception")
+            raise
 
         return Response()

styleguide_example/tasks/celery.py

@@ -1,13 +1,22 @@
 from __future__ import absolute_import, unicode_literals
 
+import logging
 import os
 
+import structlog
 from celery import Celery
+from celery.signals import setup_logging
+from django.dispatch import receiver
+from django_structlog.celery import signals
+from django_structlog.celery.steps import DjangoStructLogInitStep
+
+from config.settings.loggers.setup import LoggersSetup
 
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.base")
 
 app = Celery("styleguide_example")
+app.steps["worker"].add(DjangoStructLogInitStep)
 
 # Using a string here means the worker doesn't have to serialize
 # the configuration object to child processes.
@@ -17,3 +26,16 @@
 
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks()
+
+
+@setup_logging.connect
+def receiver_setup_logging(loglevel, logfile, format, colorize, **kwargs):  # pragma: no cover
+    logging.config.dictConfig(LoggersSetup.setup_logging())
+    LoggersSetup.setup_structlog()
+
+
+@receiver(signals.bind_extra_task_metadata)
+def receiver_bind_extra_request_metadata(sender, signal, task=None, logger=None, **kwargs):
+    # We want to add the task name to the task_succeeded event
+    if task is not None:
+        structlog.contextvars.bind_contextvars(task=task.name)

styleguide_example/tasks/tasks.py

@@ -1,6 +1,6 @@
 from celery import shared_task
 
 
-@shared_task
+@shared_task(bind=True)
 def debug_task(self):
     print("Request: {0!r}".format(self.request))