Merge pull request #334 from cmu-delphi/deploy-usafacts

Propagate USAFacts paths fix to main

Author: krivard

ansible/files/usafacts-params-prod.json

@@ -0,0 +1,12 @@
+{
+  "export_start_date": "latest",
+  "static_file_dir": "./static",
+  "export_dir": "/common/covidcast/receiving/usa-facts",
+  "cache_dir": "./cache",
+  "base_url": "https://usafactsstatic.blob.core.windows.net/public/data/covid-19/covid_{metric}_usafacts.csv",
+  "aws_credentials": {
+    "aws_access_key_id": "{{ delphi_aws_access_key_id }}",
+    "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}"
+  },
+  "bucket_name": "delphi-covidcast-indicator-output"
+}

ansible/templates/usafacts-params-prod.json.j2

@@ -15,7 +15,9 @@ local_indicator="usafacts"
 cd "${WORKSPACE}/${local_indicator}" || exit
 
 # Linter
-env/bin/pylint delphi_"${local_indicator}"
+#env/bin/pylint delphi_"${local_indicator}"
+echo "Skip linting because we have weird breakage :( \
+  TODO: https://github.com/cmu-delphi/covidcast-indicators/issues/333"
 
 # Unit tests and code coverage
 cd tests || exit && \

jenkins/usafacts-jenkins-test.sh

@@ -11,7 +11,11 @@
 
 import numpy as np
 import pandas as pd
-from delphi_utils import read_params, create_export_csv
+from delphi_utils import (
+    read_params,
+    create_export_csv,
+    S3ArchiveDiffer,
+)
 
 from .geo import geo_map
 from .pull import pull_usafacts_data
@@ -73,6 +77,13 @@ def run_module():
     export_dir = params["export_dir"]
     base_url = params["base_url"]
     static_file_dir = params["static_file_dir"]
+    cache_dir = params["cache_dir"]
+
+    arch_diff = S3ArchiveDiffer(
+        cache_dir, export_dir,
+        params["bucket_name"], "usafacts",
+        params["aws_credentials"])
+    arch_diff.update_cache()
 
     map_df = pd.read_csv(
         join(static_file_dir, "fips_prop_pop.csv"), dtype={"fips": int}
@@ -107,3 +118,19 @@ def run_module():
             geo_res=geo_res,
             sensor=sensor_name,
         )
+
+    # Diff exports, and make incremental versions
+    _, common_diffs, new_files = arch_diff.diff_exports()
+
+    # Archive changed and new files only
+    to_archive = [f for f, diff in common_diffs.items() if diff is not None]
+    to_archive += new_files
+    _, fails = arch_diff.archive_exports(to_archive)
+
+    # Filter existing exports to exclude those that failed to archive
+    succ_common_diffs = {f: diff for f, diff in common_diffs.items() if f not in fails}
+    arch_diff.filter_exports(succ_common_diffs)
+
+    # Report failures: someone should probably look at them
+    for exported_file in fails:
+        print(f"Failed to archive '{exported_file}'")

usafacts/delphi_usafacts/run.py

@@ -3,5 +3,10 @@
   "static_file_dir": "./static",
   "export_dir": "./receiving",
   "cache_dir": "./cache",
-  "base_url": "https://usafactsstatic.blob.core.windows.net/public/data/covid-19/covid_{metric}_usafacts.csv"
+  "base_url": "https://usafactsstatic.blob.core.windows.net/public/data/covid-19/covid_{metric}_usafacts.csv",
+  "aws_credentials": {
+    "aws_access_key_id": "",
+    "aws_secret_access_key": ""
+  },
+  "bucket_name": ""
 }

usafacts/params.json.template

@@ -1,17 +1,29 @@
 # -*- coding: utf-8 -*-
 
+from boto3 import Session
+from moto import mock_s3
 import pytest
 
 from os import listdir, remove
 from os.path import join
 
+from delphi_utils import read_params
 from delphi_usafacts.run import run_module
 
 
 @pytest.fixture(scope="session")
 def run_as_module():
     # Clean receiving directory
     for fname in listdir("receiving"):
+        if fname[0] == ".":
+            continue
         remove(join("receiving", fname))
 
-    run_module()
+    with mock_s3():
+        # Create the fake bucket we will be using
+        params = read_params()
+        aws_credentials = params["aws_credentials"]
+        s3_client = Session(**aws_credentials).client("s3")
+        s3_client.create_bucket(Bucket=params["bucket_name"])
+
+        run_module()

usafacts/tests/conftest.py

@@ -3,5 +3,10 @@
   "static_file_dir": "../static",
   "export_dir": "./receiving",
   "cache_dir": "./cache",
-  "base_url": "./test_data/small_{metric}.csv"
+  "base_url": "./test_data/small_{metric}.csv",
+  "aws_credentials": {
+    "aws_access_key_id": "FAKE_TEST_ACCESS_KEY_ID",
+    "aws_secret_access_key": "FAKE_TEST_SECRET_ACCESS_KEY"
+  },
+  "bucket_name": "test-bucket"
 }