Merge pull request #1982 from cmu-delphi/ds/logger2

doc: add logger examples to README and format

Author: dshemetov

_delphi_utils_python/README.md

@@ -17,5 +17,32 @@ Submodules:
 - `validator`: Data sanity checks and anomaly detection.
 
 
-Source code can be found here: 
-[https://github.com/cmu-delphi/covidcast-indicators/](https://github.com/cmu-delphi/covidcast-indicators/)
+Source code can be found here:
+[https://github.com/cmu-delphi/covidcast-indicators/](https://github.com/cmu-delphi/covidcast-indicators/)
+
+## Logger Usage
+
+Single-thread usage.
+
+```py
+from delphi_utils.logger import get_structured_logger
+
+logger = get_structured_logger('my_logger')
+logger.info('Hello, world!')
+```
+
+Multi-thread usage.
+
+```py
+from delphi_utils.logger import get_structured_logger, pool_and_threadedlogger
+
+def f(x, threaded_logger):
+    threaded_logger.info(f'x={x}')
+    return x*x
+
+logger = get_structured_logger('my_logger')
+logger.info('Hello, world!')
+with pool_and_threadedlogger(logger, n_cpu) as (pool, threaded_logger):
+    for i in range(10):
+        pool.apply_async(f, args=(i, threaded_logger))
+```

_delphi_utils_python/delphi_utils/logger.py

@@ -1,5 +1,7 @@
 """Structured logger utility for creating JSON logs.
 
+See the delphi_utils README.md for usage examples.
+
 The Delphi group uses two ~identical versions of this file.
 Try to keep them in sync with edits, for sanity.
   https://github.com/cmu-delphi/covidcast-indicators/blob/main/_delphi_utils_python/delphi_utils/logger.py
@@ -133,19 +135,17 @@ class LoggerThread():
     """
     A construct to use a logger from multiprocessing workers/jobs.
 
-    the bare structlog loggers are thread-safe but not multiprocessing-safe.
-    a `LoggerThread` will spawn a thread that listens to a mp.Queue
-    and logs messages from it with the provided logger,
-    so other processes can send logging messages to it
-    via the logger-like `SubLogger` interface.
-    the SubLogger even logs the pid of the caller.
+    The bare structlog loggers are thread-safe but not multiprocessing-safe. A
+    `LoggerThread` will spawn a thread that listens to a mp.Queue and logs
+    messages from it with the provided logger, so other processes can send
+    logging messages to it via the logger-like `SubLogger` interface. The
+    SubLogger even logs the pid of the caller.
 
-    this is good to use with a set of jobs that are part of a mp.Pool,
-    but isnt recommended for general use
-    because of overhead from threading and multiprocessing,
-    and because it might introduce lag to log messages.
+    This is good to use with a set of jobs that are part of a mp.Pool, but isnt
+    recommended for general use because of overhead from threading and
+    multiprocessing, and because it might introduce lag to log messages.
 
-    somewhat inspired by:
+    Somewhat inspired by:
     docs.python.org/3/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes
     """
 
@@ -236,13 +236,11 @@ def pool_and_threadedlogger(logger, *poolargs):
     """
     Provide (to a context) a multiprocessing Pool and a proxy to the supplied logger.
 
-    Emulates the multiprocessing.Pool() context manager,
-    but also provides (via a LoggerThread) a SubLogger proxy to logger
-    that can be safely used by pool workers.
-    The SubLogger proxy interface supports these methods: debug, info, warning, error,
-    and critical.
-    Also "cleans up" the pool by waiting for workers to complete
-    as it exits the context.
+    Emulates the multiprocessing.Pool() context manager, but also provides (via
+    a LoggerThread) a SubLogger proxy to logger that can be safely used by pool
+    workers. The SubLogger proxy interface supports these methods: debug, info,
+    warning, error, and critical. Also "cleans up" the pool by waiting for
+    workers to complete as it exits the context.
     """
     with multiprocessing.Manager() as manager:
         logger_thread = LoggerThread(logger, manager.Queue())