small clarifications in variable names, comments, log messages, and logic in incoming metadata processing

Author: melange396

src/acquisition/covid_hosp/common/utils.py

@@ -192,30 +192,38 @@ def update_dataset(database, network, newer_than=None, older_than=None):
     metadata = network.fetch_metadata(logger=logger)
     datasets = []
     # daily runs specify no bounds; patching runs specify at least one bound
-    patching = any(bound is not None for bound in (newer_than, older_than))
+    is_patch_run = any(bound is not None for bound in (newer_than, older_than))
+    if is_patch_run:
+      logger.warn('runing update_dataset() as a "patch" with some specific date bound[s] specified;'
+                  ' this will include and overwrite any revisions that were already collected.',
+                  newer_than=newer_than, older_than=older_than)
     if older_than is None:
+      # by default, include days "older than tomorrow" which thus includes "today"
       older_than = (datetime.datetime.today().date() + datetime.timedelta(days=1))
     if newer_than is None:
+      # by default, include days "newer than the day before the last update"
+      # which thus includes the day of the last update (in case there are new updates
+      # that day which were published after the one we already ingested)
       with database.connect() as db:
         max_issue = db.get_max_issue(logger=logger)
       newer_than = (max_issue - datetime.timedelta(days=1))
+    logger.info("looking up issues in date range", newer_than=newer_than, older_than=older_than)
     daily_issues = Utils.issues_to_fetch(metadata, newer_than, older_than, logger=logger)
     if not daily_issues:
-      logger.info("no new issues; nothing to do")
+      logger.info("no issues found in date range; nothing to do")
       return False
     for issue, revisions in daily_issues.items():
       issue_int = int(issue.strftime("%Y%m%d"))
       # download dataset(s) and save associated metadata
       dataset_list = []
       all_metadata = []
       for url, index in revisions:
-        if not patching:
-          # for daily runs, we only want new datasets
-          with database.connect() as db:
-            already_in_db = db.contains_revision(url)
-          if already_in_db:
-            logger.info(f"already collected revision: {url}")
-        if patching or not already_in_db:
+        with database.connect() as db:
+          already_in_db = db.contains_revision(url)
+        if already_in_db:
+          logger.info(f"already collected revision: {url}")
+        if is_patch_run or not already_in_db:
+          logger.info(f"including dataset revision: {url}")
           dataset_list.append(network.fetch_dataset(url, logger=logger))
           all_metadata.append((url, metadata.loc[index].reset_index().to_json()))
       if not dataset_list:
@@ -230,8 +238,10 @@ def update_dataset(database, network, newer_than=None, older_than=None):
         dataset,
         all_metadata
       ))
+    tot_revs = sum(len(revisions) for revisions in daily_issues.values())
+    logger.info(f"{len(daily_issues)} issues checked w/ {tot_revs} revisions, resulting in {len(datasets)} datasets.")
     if not datasets:
-      logger.info(f"{len(daily_issues)} issues checked containing {sum(len(revisions) for revisions in daily_issues.values())} revisions; nothing to do")
+      logger.info("nothing to do, exiting")
       return False
     with database.connect() as db:
       for issue_int, dataset, all_metadata in datasets: