Merge pull request #1290 from cmu-delphi/release/indicators_v0.1.21_utils_v0.1.15

Release covidcast-indicators 0.1.21

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.20
+current_version = 0.1.21
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

_delphi_utils_python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.14
+current_version = 0.1.15
 commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False

_delphi_utils_python/delphi_utils/__init__.py

@@ -14,4 +14,4 @@
 from .signal import add_prefix
 from .nancodes import Nans
 
-__version__ = "0.1.14"
+__version__ = "0.1.15"

_delphi_utils_python/delphi_utils/runner.py

@@ -4,7 +4,7 @@
 from typing import Any, Callable, Dict, Optional
 from .archive import ArchiveDiffer, archiver_from_params
 from .logger import get_structured_logger
-from .utils import read_params
+from .utils import read_params, transfer_files
 from .validator.validate import Validator
 from .validator.run import validator_from_params
 
@@ -44,8 +44,11 @@ def run_indicator_pipeline(indicator_fn:  Callable[[Params], None],
         validation_report.log(get_structured_logger(
             name = indicator_fn.__module__,
             filename=params["common"].get("log_filename", None)))
-    if archiver and (not validator or validation_report.success()):
-        archiver.run()
+    if (not validator or validation_report.success()):
+        if archiver:
+            archiver.run()
+        if "delivery" in params:
+            transfer_files()
 
 
 if __name__ == "__main__":

_delphi_utils_python/delphi_utils/utils.py

@@ -1,8 +1,8 @@
 """Read parameter files containing configuration information."""
 # -*- coding: utf-8 -*-
 from json import load,dump
-from os.path import exists
-from shutil import copyfile
+from shutil import copyfile, move
+import os
 import sys
 
 def read_params():
@@ -11,7 +11,7 @@ def read_params():
     If the file does not exist, it copies the file 'params.json.template' to
     'params.json' and then reads the file.
     """
-    if not exists("params.json"):
+    if not os.path.exists("params.json"):
         copyfile("params.json.template", "params.json")
 
     with open("params.json", "r") as json_file:
@@ -87,3 +87,12 @@ def params_run():
         with open("params.json", "w") as f:
             dump(params, f, sort_keys=True, indent=2)
         print(f"Updated {n} items")
+
+def transfer_files():
+    """Transfer files to prepare for acquisition."""
+    params = read_params()
+    export_dir = params["common"].get("export_dir", None)
+    delivery_dir = params["delivery"].get("delivery_dir", None)
+    files_to_export = os.listdir(export_dir)
+    for file_name in files_to_export:
+        move(os.path.join(export_dir, file_name), delivery_dir)

_delphi_utils_python/setup.py

@@ -25,7 +25,7 @@
 
 setup(
     name="delphi_utils",
-    version="0.1.14",
+    version="0.1.15",
     description="Shared Utility Functions for Indicators",
     long_description=long_description,
     long_description_content_type="text/markdown",

ansible/templates/google_symptoms-params-prod.json.j2

@@ -1,6 +1,6 @@
 {
   "common": {
-    "export_dir": "/common/covidcast/receiving/google-symptoms",
+    "export_dir": "./receiving",
     "log_filename": "/var/log/indicators/google_symptoms.log"
   },
   "indicator": {
@@ -41,6 +41,9 @@
         "sum_anosmia_ageusia_smoothed_search",
         "anosmia_smoothed_search"
       ]
+    },
+    "delivery": {
+      "delivery_dir": "/common/covidcast/receiving/google-symptoms"
     }
   }
 }

google_symptoms/params.json.template

@@ -31,6 +31,9 @@
         "sum_anosmia_ageusia_smoothed_search",
         "anosmia_smoothed_search"
       ]
+    },
+    "delivery": {
+      "delivery_dir": "./receiving"
     }
   }
 }