added unit tests for bad test files

Author: Jingjing Tang

nchs_mortality/delphi_nchs_mortality/pull.py

@@ -39,14 +39,14 @@ def pull_nchs_mortality_data(token: str, map_df: pd.DataFrame, test_mode: str):
     TYPE_DICT = {key: float for key in KEEP_COLUMNS}
     TYPE_DICT["timestamp"] = 'datetime64[ns]'
 
-    if test_mode:
-        df = pd.read_csv("./test_data/test_data.csv")
-    else:
+    if test_mode == "":
         # Pull data from Socrata API
         client = Socrata("data.cdc.gov", token)
         results = client.get("r8kw-7aab", limit=10**10)
         df = pd.DataFrame.from_records(results).rename(
                 {"start_week": "timestamp"}, axis=1)
+    else:
+        df = pd.read_csv("./test_data/%s"%test_mode)
 
     # Check missing start_week == end_week
     try:

nchs_mortality/delphi_nchs_mortality/run.py

@@ -43,7 +43,7 @@ def run_module():
     daily_cache_dir = params["daily_cache_dir"]
     static_file_dir = params["static_file_dir"]
     token = params["token"]
-    test_mode = (params["mode"] == "test")
+    test_mode = params["mode"]
 
     daily_arch_diff = S3ArchiveDiffer(
         daily_cache_dir, daily_export_dir,

nchs_mortality/tests/params.json.template

@@ -6,7 +6,7 @@
   "daily_export_dir": "./daily_receiving",
   "daily_cache_dir": "./daily_cache",
   "token": "",
-  "mode":"test",
+  "mode":"test_data.csv",
   "aws_credentials": {
     "aws_access_key_id": "FAKE_TEST_ACCESS_KEY_ID",
     "aws_secret_access_key": "FAKE_TEST_SECRET_ACCESS_KEY"

nchs_mortality/tests/receiving/.gitignore

@@ -1,123 +1 @@
-# You should hard commit a prototype for this file, but we
-# want to avoid accidental adding of API tokens and other
-# private data parameters
-params.json
-
-# Do not commit output files
-receiving/*.csv
-
-# Do not commit test files
-tests/receiving/*.csv
-
-# Remove macOS files
-.DS_Store
-
-# virtual environment
-dview/
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-coverage.xml
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-.static_storage/
-.media/
-local_settings.py
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
+*.csv

nchs_mortality/tests/test_pull.py

@@ -12,18 +12,22 @@
 export_dir = params["export_dir"]
 static_file_dir = params["static_file_dir"]
 token = params["token"]
-test_mode = (params["mode"] == "test")
 
 map_df = pd.read_csv(
     join(static_file_dir, "state_pop.csv"), dtype={"fips": int}
 )
 
 class TestPullUSAFacts:
     def test_good_file(self):
-        df = pull_nchs_mortality_data(token, map_df, test_mode)
+        df = pull_nchs_mortality_data(token, map_df, "test_data.csv")
 
         assert (df.columns.values == [
                 'covid_deaths', 'total_deaths', 'percent_of_expected_deaths',
                 'pneumonia_deaths', 'pneumonia_and_covid_deaths',
                 'influenza_deaths', 'pneumonia_influenza_or_covid_19_deaths',
-                "timestamp", "geo_id", "population"]).all()
+                "timestamp", "geo_id", "population"]).all()
+
+    def test_bad_file_with_inconsistent_time_col(self):
+        with pytest.raises(ValueError):
+            df = pull_nchs_mortality_data(token, map_df,
+                                          "bad_data_with_inconsistent_time_col.csv")