Skip to content

Commit 8e67b6c

Browse files
aysim319aysim319
committed
modifing test
1 parent 9269621 commit 8e67b6c

File tree

5 files changed

+85
-61
lines changed

5 files changed

+85
-61
lines changed

‎_delphi_utils_python/delphi_utils/validator/datafetcher.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -212,13 +212,14 @@ def fetch_api_reference(data_source, start_date, end_date, geo_type, signal_type
212212
# Something failed in the API and we did not get real metadata
213213
raise RuntimeError("Error when fetching signal data from the API", response["message"])
214214

215-
api_df = pd.DataFrame.from_dict(response["epidata"])
216-
api_df["issue"] = pd.to_datetime(api_df["issue"], format="%Y%m%d")
217-
api_df["time_value"] = pd.to_datetime(api_df["time_value"], format="%Y%m%d")
218-
api_df.drop("direction", axis=1, inplace=True)
219-
api_df["data_source"] = data_source
220-
api_df["signal"] = signal_type
221-
215+
api_df = None
216+
if len(response["epidata"]) > 0:
217+
api_df = pd.DataFrame.from_dict(response["epidata"])
218+
api_df["issue"] = pd.to_datetime(api_df["issue"], format="%Y%m%d")
219+
api_df["time_value"] = pd.to_datetime(api_df["time_value"], format="%Y%m%d")
220+
api_df.drop("direction", axis=1, inplace=True)
221+
api_df["data_source"] = data_source
222+
api_df["signal"] = signal_type
222223

223224
error_context = f"when fetching reference data from {start_date} to {end_date} " +\
224225
f"for data source: {data_source}, signal type: {signal_type}, geo type: {geo_type}"

‎_delphi_utils_python/tests/test_data/sample_epidata_metadata.json‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{"data_source": ["chng", "chng", "chng",
2+
"covid-act-now",
3+
"covid-act-now",
4+
"covid-act-now",
5+
"chng"],
6+
"signal": ["smoothed_outpatient_cli",
7+
"smoothed_outpatient_covid",
8+
"smoothed_outpatient_covid",
9+
"pcr_specimen_positivity_rate",
10+
"pcr_specimen_positivity_rate",
11+
"pcr_specimen_total_tests",
12+
"inactive"],
13+
"geo_type": ["state", "state", "county",
14+
"hrr", "msa", "msa",
15+
"state"],
16+
"min_time": ["20200101", "20200101", "20200101",
17+
"20200101", "20200101", "20200101",
18+
"20200101"],
19+
"max_time": ["20240101", "20240101", "20240101",
20+
"20240101", "20240101", "20240101",
21+
"20240101"],
22+
"last_update": [1711963480, 1711963480, 1711963480,
23+
1711963480, 1711963480, 1711963480,
24+
1711963480],
25+
"time_type": ["day", "day", "day",
26+
"day", "day", "day",
27+
"day"]
28+
}

‎_delphi_utils_python/tests/test_data/sample_epidata_signal_a.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{"geo_value": ["1044"],
2+
"stderr": [null],
3+
"value": [3],
4+
"issue": [20200101],
5+
"lag": [7],
6+
"sample_size": [null],
7+
"time_value": [20200101],
8+
"direction": [null]
9+
}

‎_delphi_utils_python/tests/test_data/sample_epidata_signal_county.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{"geo_value": ["0888"],
2+
"stderr": [2],
3+
"value": [14],
4+
"issue": [20200101],
5+
"lag": [1],
6+
"sample_size": [100],
7+
"time_value": [20200101],
8+
"direction": [null]
9+
}

‎_delphi_utils_python/tests/validator/test_datafetcher.py‎

Lines changed: 31 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
"""Tests for datafetcher.py."""
22

3-
from datetime import date
3+
from datetime import date, datetime
44
import mock
5+
import json
6+
from pathlib import Path
57
import numpy as np
68
import pandas as pd
79
import pytest
@@ -14,6 +16,7 @@
1416
from delphi_utils.validator.errors import ValidationFailure
1517

1618

19+
TEST_DIR = Path(__file__).parent.parent
1720

1821
class TestDataFetcher:
1922
"""Tests for various data fetching utilities."""
@@ -45,6 +48,27 @@ def raise_for_status(self):
4548
{'source': 'covid-act-now', 'db_source': 'covid-act-now'}], 200)
4649
elif "params" in kwargs and kwargs["params"] == {'signal': 'chng:inactive'}:
4750
return MockResponse([{"signals": [{"active": False}]}], 200)
51+
elif args[1] == 'https://api.delphi.cmu.edu/epidata/covidcast_meta/' and \
52+
'delphi_epidata' in kwargs["headers"]["user-agent"]:
53+
with open(f"{TEST_DIR}/test_data/sample_epidata_metadata.json") as f:
54+
epidata = json.load(f)
55+
response = {"epidata": epidata, "result": 1, "message": "success"}
56+
return MockResponse(response, 200)
57+
elif args[0] == 'https://api.delphi.cmu.edu/epidata/covidcast/' and \
58+
'delphi_epidata' in kwargs["headers"]["user-agent"]:
59+
signal_type = args[1].get("signals")
60+
geo_type = args[1].get("geo_type")
61+
if signal_type == "a":
62+
with open(f"{TEST_DIR}/test_data/sample_epidata_signal_a.json") as f:
63+
epidata = json.load(f)
64+
response = {"epidata": epidata, "result": 1, "message": "success"}
65+
return MockResponse(response, 200)
66+
if geo_type == "county":
67+
with open(f"{TEST_DIR}/test_data/sample_epidata_signal_county.json") as f:
68+
epidata = json.load(f)
69+
response = {"epidata": epidata, "result": 1, "message": "success"}
70+
return MockResponse(response, 200)
71+
return MockResponse({"epidata": {}, "result": 1, "message": "success"}, 200)
4872
else:
4973
return MockResponse([{"signals": [{"active": True}]}], 200)
5074

@@ -57,27 +81,9 @@ def test_bad_api_key(self, **kwargs):
5781
get_geo_signal_combos("chng", api_key="")
5882

5983
@mock.patch('requests.get', side_effect=mocked_requests_get)
60-
@mock.patch("delphi_utils.covidcast_wrapper.metadata")
61-
def test_get_geo_signal_combos(self, mock_metadata, mock_get):
84+
def test_get_geo_signal_combos(self, mock_get):
85+
6286
"""Test that the geo signal combos are correctly pulled from the covidcast metadata."""
63-
# Need to use actual data_source and signal names since we reference the API
64-
# We let the chng signal "inactive" be an inactive signal
65-
mock_metadata.return_value = pd.DataFrame({"data_source": ["chng", "chng", "chng",
66-
"covid-act-now",
67-
"covid-act-now",
68-
"covid-act-now",
69-
"chng"],
70-
"signal": ["smoothed_outpatient_cli",
71-
"smoothed_outpatient_covid",
72-
"smoothed_outpatient_covid",
73-
"pcr_specimen_positivity_rate",
74-
"pcr_specimen_positivity_rate",
75-
"pcr_specimen_total_tests",
76-
"inactive"],
77-
"geo_type": ["state", "state", "county",
78-
"hrr", "msa", "msa",
79-
"state"]
80-
})
8187
assert set(get_geo_signal_combos("chng", api_key="")) == set(
8288
[("state", "smoothed_outpatient_cli"),
8389
("state", "smoothed_outpatient_covid"),
@@ -87,49 +93,20 @@ def test_get_geo_signal_combos(self, mock_metadata, mock_get):
8793
("msa", "pcr_specimen_positivity_rate"),
8894
("msa", "pcr_specimen_total_tests")])
8995

90-
@mock.patch("delphi_utils.covidcast_wrapper.signal")
91-
def test_threaded_api_calls(self, mock_signal):
96+
@mock.patch('requests.get', side_effect=mocked_requests_get)
97+
def test_threaded_api_calls(self, mock_get):
9298
"""Test that calls to the covidcast API are made."""
93-
94-
signal_data_1 = pd.DataFrame({"geo_value": ["1044"],
95-
"stderr": [None],
96-
"value": [3],
97-
"issue": [10],
98-
"lag": [7],
99-
"sample_size": [None],
100-
"time_value": [10]
101-
})
102-
signal_data_2 = pd.DataFrame({"geo_value": ["0888"],
103-
"stderr": [2],
104-
"value": [14],
105-
"issue": [10],
106-
"lag": [1],
107-
"sample_size": [100],
108-
"time_value": [8]
109-
})
110-
111-
def mock_signal_return_fn(unused_data_source, signal_type, unused_start_date,
112-
unused_end_date, geo_type):
113-
"""Function to return data when covidcast.signal() is called."""
114-
if signal_type == "a":
115-
return signal_data_1
116-
if geo_type == "county":
117-
return signal_data_2
118-
return None
119-
120-
mock_signal.side_effect = mock_signal_return_fn
121-
12299
processed_signal_data_1 = pd.DataFrame({"geo_id": ["1044"],
123100
"val": [3],
124101
"se": [np.nan],
125102
"sample_size": [np.nan],
126-
"time_value": [10]
103+
"time_value": [datetime.strptime("20200101", "%Y%m%d")],
127104
})
128105
processed_signal_data_2 = pd.DataFrame({"geo_id": ["0888"],
129106
"val": [14],
130107
"se": [2],
131108
"sample_size": [100],
132-
"time_value": [8]
109+
"time_value": [datetime.strptime("20200101", "%Y%m%d")],
133110
})
134111
expected = {
135112
("county", "a"): processed_signal_data_1,

0 commit comments

Comments
 (0)