Merge pull request #275 from JedGrabman/dev-covidcast-efficiency

Batch requests when requesting multiple days of data

Author: capnrefsmmat

.github/workflows/python_ci.yml

@@ -26,8 +26,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements_ci.txt
+        make install-ci
     - name: Lint with pylint and mypy
       run: |
         make lint

Python-packages/covidcast-py/DEVELOP.md

@@ -16,25 +16,21 @@ These are general recommendations for developing. They do not have to be strictl
 but are encouraged. 
 
 __Environment__
-- A virtual environment is recommended, which can be started with the following commands:
-
-    ```sh
-    python3 -m venv env
-    source env/bin/activate
-    ```
-    this will create an `env/` folder containing files required in the environment, which
-    is gitignored. The environment can be deactived by running `deactivate`, and reactived by
-    rerunning `source env/bin/activate`. To create a new environment, you can delete the 
-    `env/` folder and rerun the above commands if you do not require the old one anymore, 
-    or rerun the above command with a new environment name in place of `env`.
+- A virtual environment is recommended to install and develop this package. Run `make install` to generate a 
+virtual environment and install the package in editable mode (code changes will automatically propagate). Note that 
+editable mode may not reveal errors when packaging files, so if the CI (which does not use editable mode) is failing 
+for reasons related to accessing data files, try debugging by installing without editable mode (`make install-ci`)
+- If you want to enter the virtual environment in your shell, you can run source env/bin/activate. Run deactivate 
+to leave the virtual environment.
+- To remove the virtual environment, you can run `make clean` or remove the `env/` folder.
 
 __Style__
-- Run `make lint` from `Python-packages/covidcast-py/` to run the lint commands.
+- Once your environment is set up, run `make lint` from `Python-packages/covidcast-py/` to run the lint commands.
 - `mypy`, `pylint`, and `pydocstyle` are used for linting, with associated configurations for 
 `pylint` in `.pylintrc` and for `mypy` in `mypy.ini`.
 
 __Testing__
-- Run `make test` from `Python-packages/covidcast-py/` to run the test commands.
+- Once your environment is set up, run `make test` from `Python-packages/covidcast-py/` to run the test commands.
 - `pytest` is the framework used in this package.
 - Each function should have corresponding unit tests. 
 - Tests should be deterministic.

Python-packages/covidcast-py/Makefile

@@ -1,18 +1,34 @@
-.PHONY = lint, test, install-requirements, build-and-install
+.PHONY = venv, install-ci, install, build, lint, test, clean
 
-install-requirements:
-	pip install -r requirements_dev.txt
-	pip install -r requirements_ci.txt
 
-build-and-install: install-requirements
+venv:
+	python3 -m venv env
+
+install-ci: venv
+	. env/bin/activate; \
+	python -m pip install --upgrade pip; \
+	pip install -r requirements_dev.txt; \
+	pip install .
+
+install:  venv
+	. env/bin/activate; \
+	python -m pip install --upgrade pip; \
+	pip install -r requirements_dev.txt; \
+	pip install -e .
+
+build:
 	python3 setup.py clean
 	python3 setup.py sdist bdist_wheel
-	pip3 install -e .
 
 lint:
-	pylint covidcast/ --rcfile .pylintrc
-	mypy covidcast --config-file mypy.ini
+	. env/bin/activate; \
+	pylint covidcast/ --rcfile .pylintrc; \
+	mypy covidcast --config-file mypy.ini; \
 	pydocstyle covidcast/
 
 test:
+	. env/bin/activate; \
 	pytest tests/ -W ignore::UserWarning
+
+clean:
+	rm -rf env

Python-packages/covidcast-py/requirements_ci.txt

@@ -1,3 +1,7 @@
-wheel
+mypy
+pylint
+pytest
+pydocstyle
 sphinx
-sphinx-autodoc-typehints
+sphinx-autodoc-typehints
+wheel

Python-packages/covidcast-py/requirements_dev.txt

@@ -31,5 +31,5 @@
         "imageio",
         "tqdm"
     ],
-    package_data={"covidcast": ["shapefiles/*", "geo_mappings/*"]}
+    package_data={"covidcast": ["shapefiles/*/*", "geo_mappings/*"]}
 )