Switch CDC Covidnet to use structed logger

Author: chinandrew

cdc_covidnet/delphi_cdc_covidnet/covidnet.py

@@ -6,7 +6,7 @@
 """
 
 import json
-import logging
+from logging import Logger
 import os
 from typing import Tuple, List
 from multiprocessing import cpu_count, Pool
@@ -100,7 +100,7 @@ def download_hosp_data(
 
     @staticmethod
     def download_all_hosp_data(
-            mappings_file: str, cache_path: str, parallel: bool = False
+            mappings_file: str, cache_path: str, logger: Logger, parallel: bool = False
         ) -> List[str]:
         """
         Download hospitalization data for all states listed in the mappings JSON file to disk.
@@ -146,7 +146,7 @@ def download_all_hosp_data(
         else:
             for args in state_args:
                 CovidNet.download_hosp_data(*args)
-                logging.debug("Downloading for nid=%s, cid=%s", args[0], args[1])
+                logger.debug("Downloading for nid=%s, cid=%s", args[0], args[1])
 
         return state_files
 

cdc_covidnet/delphi_cdc_covidnet/run.py

@@ -4,12 +4,13 @@
 This module should contain a function called `run_module`, that is executed
 when the module is run with `python -m delphi_cdc_covidnet`.
 """
-import logging
 from datetime import datetime
 from os import remove
 from os.path import join
 from typing import Dict, Any
 
+from delphi_utils import get_structured_logger
+
 from .covidnet import CovidNet
 from .update_sensor import update_sensor
 
@@ -32,7 +33,9 @@ def run_module(params: Dict[str, Dict[str, Any]]):
             - "wip_signal": list of str or bool, to be passed to delphi_utils.add_prefix.
             - "input_cache_dir": str, directory to download source files.
     """
-    logging.basicConfig(level=logging.DEBUG)
+    logger = get_structured_logger(
+        __name__, filename=params["common"].get("log_filename"),
+        log_exceptions=params["common"].get("log_exceptions", True))
 
     start_date = datetime.strptime(params["indicator"]["start_date"], "%Y-%m-%d")
 
@@ -42,23 +45,24 @@ def run_module(params: Dict[str, Dict[str, Any]]):
     else:
         end_date = datetime.strptime(params["indicator"]["end_date"], "%Y-%m-%d")
 
-    logging.info("start date:\t%s", start_date.date())
-    logging.info("end date:\t%s", end_date.date())
+    logger.info("start date:\t%s", start_date.date())
+    logger.info("end date:\t%s", end_date.date())
 
-    logging.info("outpath:\t%s", params["common"]["export_dir"])
-    logging.info("parallel:\t%s", params["indicator"]["parallel"])
+    logger.info("outpath:\t%s", params["common"]["export_dir"])
+    logger.info("parallel:\t%s", params["indicator"]["parallel"])
 
     # Only geo is state, and no weekday adjustment for now
     # COVID-NET data is by weeks anyway, not daily
-    logging.info("starting state, no adj")
+    logger.info("starting state, no adj")
 
     # Download latest COVID-NET files into the cache directory first
     mappings_file = join(params["indicator"]["input_cache_dir"], "init.json")
     CovidNet.download_mappings(outfile=mappings_file)
     _, mmwr_info, _ = CovidNet.read_mappings(mappings_file)
     state_files = CovidNet.download_all_hosp_data(
         mappings_file, params["indicator"]["input_cache_dir"],
-        parallel=params["indicator"]["parallel"])
+        parallel=params["indicator"]["parallel"],
+        logger=logger)
 
     update_sensor(
         state_files,
@@ -73,4 +77,4 @@ def run_module(params: Dict[str, Dict[str, Any]]):
     for state_file in state_files:
         remove(state_file)
 
-    logging.info("finished all")
+    logger.info("finished all")

cdc_covidnet/tests/test_covidnet.py

@@ -1,4 +1,5 @@
 import json
+import logging
 from os.path import join, exists
 from tempfile import TemporaryDirectory
 
@@ -7,6 +8,7 @@
 from delphi_cdc_covidnet.api_config import APIConfig
 from delphi_cdc_covidnet.covidnet import CovidNet
 
+TEST_LOGGER = logging.getLogger()
 
 class TestCovidNet:
 
@@ -65,14 +67,14 @@ def test_hosp_data(self):
 
             # Non-parallel
             state_files = CovidNet.download_all_hosp_data(
-                init_file, temp_dir, parallel=False)
+                init_file, temp_dir, TEST_LOGGER, parallel=False)
             assert len(state_files) == num_states
             for state_file in state_files:
                 assert exists(state_file)
 
             # Parallel
             state_files_par = CovidNet.download_all_hosp_data(
-                init_file, temp_dir, parallel=True)
+                init_file, temp_dir, TEST_LOGGER, parallel=True)
             assert set(state_files) == set(state_files_par)
             assert len(state_files_par) == num_states
             for state_file in state_files_par: